diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b274a54e2c2..f6ec1a24fa7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,9 @@ jobs: runs-on: blacksmith-8vcpu-ubuntu-2404 outputs: rust: ${{ steps.filter.outputs.rust }} - gpui: ${{ steps.filter.outputs.rust == 'true' || steps.filter.outputs.gpui == 'true' }} + # apps/cli is in the root workspace only; the GPUI app is its own + # workspace and never builds it, so CLI-only changes skip the app jobs. + gpui: ${{ steps.filter.outputs.rust-without-cli == 'true' || steps.filter.outputs.gpui == 'true' }} gpui-patches: ${{ steps.filter.outputs.gpui-patches }} ios: ${{ steps.filter.outputs.ios }} steps: @@ -46,8 +48,20 @@ jobs: - 'scripts/setup*' - 'Cargo.toml' - 'Cargo.lock' + rust-without-cli: + - '.cargo/**' + - '.github/actions/install-desktop-deps/**' + - '.github/actions/setup-rust-cache/**' + - '.github/workflows/ci.yml' + - 'crates/**' + - 'vendor/**' + - 'scripts/diagnostic-build.rs' + - 'scripts/setup*' + - 'Cargo.toml' + - 'Cargo.lock' gpui: - '.gitattributes' + - 'scripts/setup.js' - 'apps/desktop-gpui/**' - 'apps/media-server/src/__tests__/fixtures/**' - 'scripts/build-desktop-binaries*' @@ -101,101 +115,24 @@ jobs: bun test apps/render-farm bun run --cwd apps/render-farm typecheck - - name: Test web React compatibility - run: >- - bun run --cwd apps/web test - __tests__/unit/add-videos-dialog-retry.test.ts - __tests__/unit/agent-download-access.test.ts - __tests__/unit/analytics-track-route.test.ts - __tests__/unit/caps-analytics-round-trips.test.ts - __tests__/unit/caps-keyboard.test.ts - __tests__/unit/caption-translation-ownership.test.ts - __tests__/unit/checkout-request-limits.test.ts - __tests__/unit/clock-hours.test.ts - __tests__/unit/comment-composer-composition.test.ts - __tests__/unit/currency-hook.test.ts - __tests__/unit/dashboard-analytics-cap-access.test.ts - __tests__/unit/dashboard-analytics-membership.test.ts - __tests__/unit/dashboard-analytics-space-access.test.ts - __tests__/unit/dashboard-search-item-values.test.ts - __tests__/unit/desktop-subscribe-route.test.ts - __tests__/unit/detect-platform.test.ts - __tests__/unit/embed-page-round-trips.test.ts - __tests__/unit/embed-video-playback-chrome.test.ts - __tests__/unit/first-view-email.test.ts - __tests__/unit/folder-delete.test.ts - __tests__/unit/folder-videos.test.ts - __tests__/unit/generate-ai-duration.test.ts - __tests__/unit/homepage-demo-accessibility.test.ts - __tests__/unit/invite-decline-pro-seat.test.ts - __tests__/unit/invite-decline.test.ts - __tests__/unit/linkified-text.test.ts - __tests__/unit/loom-import-ui.test.ts - __tests__/unit/messenger-message-limits.test.ts - __tests__/unit/messenger-poll-ordering.test.ts - __tests__/unit/media-comment-playback-cleanup.test.ts - __tests__/unit/media-client.test.ts - __tests__/unit/media-format-converter-selection.test.ts - __tests__/unit/media-format-converter.test.ts - __tests__/unit/gif-video-errors.test.ts - __tests__/unit/media-player-store-refs.test.ts - __tests__/unit/mobile-cap-view-notification.test.ts - __tests__/unit/mobile-caps-list.test.ts - __tests__/unit/mobile-loom-import.test.ts - __tests__/unit/mobile-organization-analytics.test.ts - __tests__/unit/notification-read-state.test.ts - __tests__/unit/organization-preferences-switch.test.ts - __tests__/unit/organization-preferences-target.test.ts - __tests__/unit/organization-settings-ownership.test.ts - __tests__/unit/organization-switcher-item-values.test.ts - __tests__/unit/playback-store.test.ts - __tests__/unit/pro-checkout-session.test.ts - __tests__/unit/pro-checkout-webhook.test.ts - __tests__/unit/referral-checkout.test.ts - __tests__/unit/remove-organization-member.test.ts - __tests__/unit/reply-notification.test.ts - __tests__/unit/seo-demo-videos.test.ts - __tests__/unit/share-comments-rendering.test.ts - __tests__/unit/share-page-loading.test.ts - __tests__/unit/share-page-round-trips.test.ts - __tests__/unit/share-transcript-rendering.test.ts - __tests__/unit/share-video-quota-loading.test.ts - __tests__/unit/shared-cap-card-delete.test.ts - __tests__/unit/sdk-recorder-hook.test.ts - __tests__/unit/signed-baa-card.test.ts - __tests__/unit/sonner-theme.test.ts - __tests__/unit/space-folder-page-access.test.ts - __tests__/unit/space-page-round-trips.test.ts - __tests__/unit/speed-controller-metadata.test.ts - __tests__/unit/speed-controller-playback-failure.test.ts - __tests__/unit/speed-controller.test.ts - __tests__/unit/subscribe-checkout.test.ts - __tests__/unit/summary-chapters.test.ts - __tests__/unit/summary-editor.test.ts - __tests__/unit/sso-login-pages.test.ts - __tests__/unit/timeline-paused-window.test.ts - __tests__/unit/toggle-pro-seat.test.ts - __tests__/unit/transcript-arrow-text.test.ts - __tests__/unit/transcript-hours.test.ts - __tests__/unit/transcript-numbers.test.ts - __tests__/unit/translate-transcript-once.test.ts - __tests__/unit/trimming-tool-audio.test.ts - __tests__/unit/update-seat-quantity.test.ts - __tests__/unit/video-ai-route-retry.test.ts - __tests__/unit/video-analytics-access.test.ts - __tests__/unit/video-analytics-bulk.test.ts - __tests__/unit/video-create-atomicity.test.ts - __tests__/unit/video-download-info-published.test.ts - __tests__/unit/video-download-key.test.ts - __tests__/unit/video-download-permissions.test.ts - __tests__/unit/video-download-rpc.test.ts - __tests__/unit/video-settings-action.test.ts - __tests__/unit/viewer-download-action.test.ts - __tests__/unit/voice-note-playback.test.ts - - name: Typecheck run: bun run typecheck + web-tests: + name: Web tests + runs-on: blacksmith-8vcpu-ubuntu-2404 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - uses: ./.github/actions/setup-js + + - run: bun run --cwd apps/web next typegen + + # The whole web unit suite, about a minute on this runner. + - name: Test web + run: bun run --cwd apps/web test __tests__/unit + ios: name: iOS needs: changes @@ -352,6 +289,10 @@ jobs: cargo test --locked -p cap --test cli caps_json_errors_include_the_standard_error_field cargo test --locked -p cap --bin cap caps::tests::cap_operations_validate_wait_timeouts_before_submitting cargo test --locked -p cap --test cli organization_operations_validate_wait_timeouts_before_submitting + cargo test --locked -p cap --bin cap developer_media::tests + cargo test --locked -p cap --test cli developer_ids_that_begin_with_a_dash_are_read_as_values + cargo test --locked -p cap --test cli developer_dry_runs_show_the_target_and_change_nothing + cargo test --locked -p cap --test cli commands_without_their_own_json_errors_still_print_one cargo test --locked -p cap-recording --lib -- --test-threads=1 cargo test --locked -p cap-recording --test x11_shm_capture -- --test-threads=1 cargo test --locked -p scap-targets --lib --test linux_targets @@ -375,6 +316,12 @@ jobs: runner: blacksmith-8vcpu-windows-2025 - target: x86_64-unknown-linux-gnu runner: blacksmith-8vcpu-ubuntu-2404 + # The Intel build is not a merge gate and takes over an hour on a + # GitHub-hosted runner, so it runs on main and by hand, not on PRs. + exclude: + - settings: + target: x86_64-apple-darwin + runner: ${{ github.event_name == 'pull_request' && 'macos-26-intel' || 'none' }} runs-on: ${{ matrix.settings.runner }} env: # Full debuginfo for the opt-level 2 dependency tree is most of the @@ -413,6 +360,18 @@ jobs: env: RUST_TARGET_TRIPLE: ${{ matrix.settings.target }} + # setup.js strips the local symbols from Microsoft's ONNX Runtime dylib + # before Cap bundles it; the C API ort loads through must still resolve + # and run a real inference. + - name: Test the prepared ONNX Runtime + if: ${{ runner.os == 'macOS' }} + shell: bash + run: | + set -euo pipefail + dylib=target/native-deps/onnxruntime/lib/libonnxruntime.dylib + nm -gU "$dylib" | grep -q OrtGetApiBase + cargo test --locked -p cap-camera-effects --lib segmentation::tests::inference_runs_from_a_caller_with_a_tiny_stack -- --ignored --exact + # The test build compiles every line the binary build does (the app has # no cfg(not(test)) code), so a separate binary build only repeated the # app crate's compile and link. Release builds still produce the binary. diff --git a/.github/workflows/docker-build-media-server.yml b/.github/workflows/docker-build-media-server.yml index 2de273fdb9f..d144229cb36 100644 --- a/.github/workflows/docker-build-media-server.yml +++ b/.github/workflows/docker-build-media-server.yml @@ -137,6 +137,41 @@ jobs: src/__tests__/lib/hls-ladder.test.ts \ src/__tests__/lib/hls-ladder-encode.integration.test.ts \ src/__tests__/lib/hls-ladder-route.integration.test.ts + docker run --rm --network none --entrypoint bun "$MEDIA_IMAGE" test \ + src/__tests__/lib/media-address-guard.test.ts \ + src/__tests__/lib/media-relay.integration.test.ts \ + src/__tests__/lib/media-input-policy.test.ts + docker run --rm --network none --entrypoint bun "$MEDIA_IMAGE" test \ + src/__tests__/lib/source-frame-limit.integration.test.ts \ + src/__tests__/lib/media-probe-sample-counts.test.ts + docker run --rm --network none --entrypoint bun "$MEDIA_IMAGE" test \ + src/__tests__/lib/encode-stall.test.ts \ + src/__tests__/lib/media-edit-stderr.test.ts + docker run --rm --network none --entrypoint bun "$MEDIA_IMAGE" test \ + src/__tests__/lib/isolated-worker-pool.test.ts \ + src/__tests__/lib/media-probe-hardening.test.ts \ + src/__tests__/lib/colour-probe-fallback.test.ts \ + src/__tests__/lib/encode-length-limit.test.ts \ + src/__tests__/lib/matroska-length.test.ts + docker run --rm --network none --entrypoint bun "$MEDIA_IMAGE" test \ + src/__tests__/routes/thumbnail-remote.test.ts \ + src/__tests__/routes/health.test.ts \ + src/__tests__/routes/peer-jobs.test.ts + + - name: Verify the server runs without root in the production image + env: + MEDIA_IMAGE: ${{ github.event_name == 'pull_request' && 'cap-media-server:verification' || format('ghcr.io/{0}/cap-media-server@{1}', env.REPOSITORY_OWNER, steps.build.outputs.digest) }} + run: | + test "$(docker run --rm --network none "$MEDIA_IMAGE" id -u)" = 1000 + docker volume create media-gpu-cache + docker run --rm --network none --init \ + -e CUDA_CACHE_PATH=/gpu-cache -v media-gpu-cache:/gpu-cache \ + --device /dev/null:/dev/nvidia0 --entrypoint sh "$MEDIA_IMAGE" -c \ + 'chmod 660 /dev/nvidia0 && chgrp 44 /dev/nvidia0 && exec /app/docker-entrypoint.sh sh -c "test \$(id -u) = 1000 && touch /gpu-cache/probe && mktemp -p /tmp && test -r /dev/nvidia0 && test -w /dev/nvidia0"' + docker run --rm --network none --init "$MEDIA_IMAGE" bun test \ + src/__tests__/lib/process-hardening.test.ts \ + src/__tests__/lib/media-probe-sample-counts.test.ts \ + src/__tests__/lib/media-input-policy.test.ts - name: Verify long recording performance in the production image env: diff --git a/.github/workflows/infra-database.yml b/.github/workflows/infra-database.yml new file mode 100644 index 00000000000..b124d94cb2e --- /dev/null +++ b/.github/workflows/infra-database.yml @@ -0,0 +1,79 @@ +name: Infra Database + +on: + workflow_dispatch: + pull_request: + paths: + - "apps/web/lib/infra-**" + - "apps/web/lib/developer-**" + - "apps/web/__tests__/integration/**" + - "apps/web/actions/developers/**" + - "apps/web/app/(infra)/**" + - "apps/web/app/api/developer/**" + - "apps/web/app/api/cron/developer-storage/**" + - "packages/database/schema.ts" + - "packages/database/migrations/**" + - ".github/workflows/infra-database.yml" + push: + branches: [main] + paths: + - "apps/web/lib/infra-**" + - "apps/web/lib/developer-**" + - "apps/web/__tests__/integration/**" + - "apps/web/actions/developers/**" + - "apps/web/app/(infra)/**" + - "apps/web/app/api/developer/**" + - "apps/web/app/api/cron/developer-storage/**" + - "packages/database/schema.ts" + - "packages/database/migrations/**" + - ".github/workflows/infra-database.yml" + +permissions: + contents: read + +concurrency: + group: infra-database-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + infra-database: + name: Infra billing and uploads against MySQL + runs-on: ubuntu-24.04 + timeout-minutes: 15 + services: + mysql: + image: mysql:8.4 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + MYSQL_ROOT_HOST: "%" + MYSQL_DATABASE: cap_infra_tests + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping --host=127.0.0.1 --user=root" + --health-interval=5s + --health-timeout=3s + --health-retries=30 + env: + DATABASE_URL: mysql://root@127.0.0.1:3306/cap_infra_tests + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-js + - name: Apply migrations + working-directory: packages/database + run: bun run drizzle-kit migrate --config=drizzle.config.ts + # The suites insert billing rows without their parent apps and owners, + # so they run on the migrated schema without foreign keys, the same as + # the local scratch database they were written against. This job does + # not test foreign key behaviour. + - name: Drop foreign keys + run: | + mysql --protocol=TCP -h 127.0.0.1 -u root -N -e \ + "SELECT CONCAT('ALTER TABLE \`', TABLE_NAME, '\` DROP FOREIGN KEY \`', CONSTRAINT_NAME, '\`;') FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA = 'cap_infra_tests'" \ + | mysql --protocol=TCP -h 127.0.0.1 -u root cap_infra_tests + - name: Run Infra database suites + working-directory: apps/web + env: + CAP_INFRA_TEST_DATABASE_URL: mysql://root@127.0.0.1:3306/cap_infra_tests + CAP_INFRA_ROUND_TRIPS_DATABASE_URL: mysql://root@127.0.0.1:3306/cap_infra_tests + run: bun run vitest run __tests__/integration/infra-*-database.test.ts diff --git a/.github/workflows/smg-ci.yml b/.github/workflows/smg-ci.yml index d3fa96b6d15..e5098969abe 100644 --- a/.github/workflows/smg-ci.yml +++ b/.github/workflows/smg-ci.yml @@ -105,7 +105,7 @@ jobs: bun run --cwd apps/web next typegen bun run typecheck - # Same list upstream's CI runs (read from its ci.yml), plus tests this PR touches. + # The tests upstream's CI runs (read from its ci.yml), plus tests this PR touches. - name: Web unit tests env: BASE_REF: ${{ github.base_ref || 'smg' }} diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000000..5bd48a4bd83 --- /dev/null +++ b/.mailmap @@ -0,0 +1 @@ +Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> diff --git a/apps/cli/src/developers.rs b/apps/cli/src/developers.rs index a4c9fb26b82..1b4215945b7 100644 --- a/apps/cli/src/developers.rs +++ b/apps/cli/src/developers.rs @@ -14,10 +14,14 @@ use crate::{ const ABOUT: &str = "\ Cap Infra is Cap's usage-billed video and speech API. A developer app holds its videos, -transcripts, allowed domains, API keys and prepaid credits. +transcripts, its allowed sites, API keys and prepaid credits. + +Only the sites added with `cap developers domains` can embed the app's player. With none +added, any site can. Share links, HLS, MP4 and poster URLs stay open to everyone. To limit +a browser upload to one origin, set corsOrigin when your server creates the upload. Amounts: 100,000 micro-credits are $1. Credit purchases are in cents, from 500 ($5) to -100000 ($1,000). Durations are in seconds. +1000000 ($10,000). Durations are in seconds. A typical job: cap developers list --json @@ -51,7 +55,7 @@ enum DeveloperCommands { Update(DeveloperUpdateArgs), /// Delete an app and revoke its keys Delete(DeveloperDeleteArgs), - /// Add or remove the origins allowed to use an app's public key + /// Add or remove the sites that can embed an app's player Domains(DeveloperDomainsArgs), /// Rotate an app's API keys Keys(DeveloperKeysArgs), @@ -149,9 +153,9 @@ struct DeveloperDomainsArgs { #[derive(Subcommand)] enum DeveloperDomainCommands { - /// Allow an origin, for example https://app.example.com + /// Allow a site to embed the player, for example https://app.example.com. Once one is added, only listed sites can Add(DeveloperDomainAddArgs), - /// Remove an allowed origin by its id or by the origin itself + /// Remove an allowed site by its id or by the origin itself Remove(DeveloperDomainRemoveArgs), } @@ -159,7 +163,7 @@ enum DeveloperDomainCommands { struct DeveloperDomainAddArgs { /// App id from `cap developers list` app: String, - /// Origin to allow, for example https://app.example.com + /// Site to allow, for example https://app.example.com domain: String, /// Confirm without a prompt #[arg(long)] @@ -248,7 +252,7 @@ enum DeveloperCreditCommands { struct DeveloperCreditPurchaseArgs { /// App id from `cap developers list` app: String, - /// Amount in cents, from 500 ($5) to 100000 ($1,000) + /// Amount in cents, from 500 ($5) to 1000000 ($10,000) #[arg(long)] amount_cents: u32, /// Print the checkout link without opening a browser diff --git a/apps/cli/src/mcp.rs b/apps/cli/src/mcp.rs index db9e960d92b..4acc5446f7d 100644 --- a/apps/cli/src/mcp.rs +++ b/apps/cli/src/mcp.rs @@ -556,7 +556,7 @@ struct DeveloperAppDeleteInput { struct DeveloperDomainAddInput { /// App id from developer_apps_list app_id: String, - /// Origin to allow, for example https://app.example.com + /// Site to allow, for example https://app.example.com domain: String, /// true only after the user confirmed the change confirmed: bool, @@ -596,7 +596,7 @@ struct DeveloperAutoTopUpInput { struct DeveloperCreditsCheckoutInput { /// App id from developer_apps_list app_id: String, - /// Amount in cents, from 500 ($5) to 100000 ($1,000) + /// Amount in cents, from 500 ($5) to 1000000 ($10,000) amount_cents: u32, /// true only after the user confirmed the amount confirmed: bool, @@ -3044,7 +3044,7 @@ impl CapMcpServer { #[tool( name = "developer_domain_add", - description = "Add an allowed origin to a developer app after explicit user confirmation", + description = "Allow a site to embed a developer app's player, after explicit user confirmation. With no sites added any site can embed it; once one is added, only listed sites can. Share links, HLS, MP4 and poster URLs stay open to everyone. Only https sites, or localhost for testing, are accepted", annotations( read_only_hint = false, destructive_hint = false, @@ -3078,7 +3078,7 @@ impl CapMcpServer { #[tool( name = "developer_domain_remove", - description = "Remove an allowed origin from a developer app after explicit user confirmation. Call with dry_run=true first to show the user which origin would be removed", + description = "Remove a site from the ones allowed to embed a developer app's player, after explicit user confirmation. Removing the last one lets any site embed it again. Call with dry_run=true first to show the user which site would be removed", annotations( read_only_hint = false, destructive_hint = true, @@ -3177,7 +3177,7 @@ impl CapMcpServer { #[tool( name = "developer_credits_checkout", - description = "Create a developer-credit purchase URL after explicit user confirmation; the user completes payment in a browser and nothing is charged before that. The amount is in cents, from 500 ($5) to 100000 ($1,000)", + description = "Create a developer-credit purchase URL after explicit user confirmation; the user completes payment in a browser and nothing is charged before that. The amount is in cents, from 500 ($5) to 1000000 ($10,000)", annotations( read_only_hint = false, destructive_hint = false, diff --git a/apps/desktop-gpui/kernels/src/screenshot.rs b/apps/desktop-gpui/kernels/src/screenshot.rs index ef353a6625a..94e80a7b0f1 100644 --- a/apps/desktop-gpui/kernels/src/screenshot.rs +++ b/apps/desktop-gpui/kernels/src/screenshot.rs @@ -469,9 +469,11 @@ fn resize_triangle( let vertical_weights = triangle_axis(height, new_height); let horizontal_weights = triangle_axis(width, new_width); let row_bytes = width as usize * 4; - let mut vertical = vec![0f32; row_bytes * new_height as usize]; + let mut vertical = vec![0f32; row_bytes]; + let packed = (horizontal_weights.max_taps <= 3).then(|| pack_tap3(&horizontal_weights)); + let out_row = new_width as usize * 4; + let mut out = vec![0u8; out_row * new_height as usize]; for out_y in 0..new_height as usize { - let dst = &mut vertical[out_y * row_bytes..(out_y + 1) * row_bytes]; let tap_count = vertical_weights.count[out_y] as usize; let weight_at = vertical_weights.offset[out_y]; let src_y0 = vertical_weights.start[out_y]; @@ -479,29 +481,18 @@ fn resize_triangle( for (tap, &weight) in weights.iter().enumerate() { let src_y = src_y0 + tap as u32; let start = (y0 + src_y) as usize * stride + x0 as usize * 4; - accumulate_row(dst, &src[start..start + row_bytes], weight, tap == 0); - } - } - - let out_row = new_width as usize * 4; - let mut out = vec![0u8; out_row * new_height as usize]; - if horizontal_weights.max_taps <= 3 { - let packed = pack_tap3(&horizontal_weights); - for y in 0..new_height as usize { - sample_horizontal_tap3( - &vertical[y * row_bytes..(y + 1) * row_bytes], - &mut out[y * out_row..(y + 1) * out_row], - &packed, - width, + accumulate_row( + &mut vertical, + &src[start..start + row_bytes], + weight, + tap == 0, ); } - } else { - for y in 0..new_height as usize { - sample_horizontal_general( - &vertical[y * row_bytes..(y + 1) * row_bytes], - &mut out[y * out_row..(y + 1) * out_row], - &horizontal_weights, - ); + let dst = &mut out[out_y * out_row..(out_y + 1) * out_row]; + if let Some(packed) = &packed { + sample_horizontal_tap3(&vertical, dst, packed, width); + } else { + sample_horizontal_general(&vertical, dst, &horizontal_weights); } } out diff --git a/apps/media-server/Dockerfile b/apps/media-server/Dockerfile index b86817fc8dc..d50e0d2d6d7 100644 --- a/apps/media-server/Dockerfile +++ b/apps/media-server/Dockerfile @@ -7,7 +7,11 @@ RUN apt-get update \ WORKDIR /app COPY apps/media-server/package.json apps/media-server/bun.lock ./ +COPY apps/media-server/patches ./patches RUN bun install --frozen-lockfile --production +# The build stops here if the parser limits in patches/ were not applied. +RUN grep -q MAX_DECLARED_SAMPLES node_modules/mediabunny/dist/modules/src/isobmff/isobmff-demuxer.js \ + && grep -q PCM_CHUNK_SPAN_LIMIT node_modules/mediabunny/dist/modules/src/isobmff/isobmff-demuxer.js COPY apps/media-server/bunfig.toml ./bunfig.toml COPY apps/media-server/src ./src @@ -18,21 +22,24 @@ RUN bun test src/__tests__/lib/audio-quality-policy.test.ts src/__tests__/lib/au && bun test src/__tests__/lib/audio-levels.integration.test.ts \ && bun test src/__tests__/lib/drive-resumable-upload.test.ts src/__tests__/lib/storage-upload.test.ts src/__tests__/lib/container-memory.test.ts src/__tests__/lib/parallel-download.test.ts \ && bun test src/__tests__/lib/loom-import-limit.test.ts \ - && bun test src/__tests__/lib/media-size.test.ts src/__tests__/lib/media-probe.integration.test.ts \ - && bun test src/__tests__/lib/media-audio.integration.test.ts \ + && bun test src/__tests__/lib/media-size.test.ts src/__tests__/lib/media-probe.integration.test.ts src/__tests__/lib/media-probe-sample-counts.test.ts \ + && bun test src/__tests__/lib/media-audio.integration.test.ts src/__tests__/lib/audio-only-source.integration.test.ts \ && bun test src/__tests__/lib/recording-verification.integration.test.ts src/__tests__/lib/job-manager.test.ts src/__tests__/lib/recording-preverify.test.ts \ && bun test src/__tests__/lib/media-transfer.test.ts src/__tests__/lib/capcodec-encoder.test.ts src/__tests__/lib/nvenc-encoder.test.ts \ && bun test src/__tests__/routes/recording-verification.test.ts \ && bun test src/__tests__/routes/video.test.ts \ && bun test src/__tests__/routes/preverify.test.ts \ && bun test src/__tests__/lib/lifecycle.test.ts src/__tests__/lib/temp-files.test.ts src/__tests__/lib/process-guards.test.ts \ - && bun test src/__tests__/lib/media-input-policy.test.ts \ + && bun test src/__tests__/lib/media-input-policy.test.ts src/__tests__/lib/infra-capacity.test.ts \ && bun test src/__tests__/lib/media-address-guard.test.ts src/__tests__/lib/media-relay.integration.test.ts src/__tests__/lib/source-frame-limit.integration.test.ts src/__tests__/lib/encode-stall.test.ts src/__tests__/lib/auth.test.ts \ && bun test src/__tests__/routes/lifecycle.test.ts \ && bun test src/__tests__/routes/peer-jobs.test.ts \ && bun test src/__tests__/lib/live-transcribe.test.ts src/__tests__/routes/live-transcribe.test.ts \ && bun test src/__tests__/lib/media-routes-real-world.integration.test.ts \ - && bun test src/__tests__/lib/media-video.integration.test.ts + && bun test src/__tests__/lib/infra-poster.integration.test.ts \ + && bun test src/__tests__/lib/media-video.integration.test.ts \ + && bun test src/__tests__/lib/isolated-worker-pool.test.ts src/__tests__/lib/media-probe-hardening.test.ts src/__tests__/lib/process-hardening.test.ts src/__tests__/lib/legacy-codecs.integration.test.ts \ + && bun test src/__tests__/routes/thumbnail-remote.test.ts src/__tests__/routes/health.test.ts RUN MEDIA_SERVER_RECORDING_PERFORMANCE_TESTS=1 bun test \ src/__tests__/lib/recording-verification.integration.test.ts \ @@ -48,4 +55,10 @@ EXPOSE 3456 HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ CMD ["bun", "-e", "fetch(`http://127.0.0.1:${process.env.PORT || 3456}/health`).then(async (response) => process.exit(response.ok || (await response.json().catch(() => ({}))).status === 'draining' ? 0 : 1)).catch(() => process.exit(1))"] +# The container starts as root only to hand its temp and GPU cache volumes +# and the GPU devices to the bun user; the entrypoint then runs the server, +# and every FFmpeg it starts, as that user. See docker-entrypoint.sh. +COPY apps/media-server/docker-entrypoint.sh ./docker-entrypoint.sh +RUN chmod 755 ./docker-entrypoint.sh +ENTRYPOINT ["/app/docker-entrypoint.sh"] CMD ["bun", "run", "src/index.ts"] diff --git a/apps/media-server/Dockerfile.standalone b/apps/media-server/Dockerfile.standalone index 81711675e05..121a5792c52 100644 --- a/apps/media-server/Dockerfile.standalone +++ b/apps/media-server/Dockerfile.standalone @@ -7,11 +7,19 @@ RUN apt-get update \ WORKDIR /app COPY package.json bun.lock ./ +COPY patches ./patches RUN bun install --frozen-lockfile --production +# The build stops here if the parser limits in patches/ were not applied. +RUN grep -q MAX_DECLARED_SAMPLES node_modules/mediabunny/dist/modules/src/isobmff/isobmff-demuxer.js \ + && grep -q PCM_CHUNK_SPAN_LIMIT node_modules/mediabunny/dist/modules/src/isobmff/isobmff-demuxer.js COPY bunfig.toml ./bunfig.toml COPY src ./src COPY capcodec.md ./capcodec.md +# Files whose tables claim far more than they hold must probe at once, or the image is not built. +RUN bun test src/__tests__/lib/media-probe-sample-counts.test.ts +RUN bun test src/__tests__/lib/legacy-codecs.integration.test.ts src/__tests__/lib/audio-only-source.integration.test.ts +RUN bun test src/__tests__/lib/infra-poster.integration.test.ts ENV PORT=3456 # Self-hosted storage usually sits on a private network next to this container. @@ -21,4 +29,10 @@ EXPOSE 3456 HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ CMD ["bun", "-e", "fetch(`http://127.0.0.1:${process.env.PORT || 3456}/health`).then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))"] +# The container starts as root only to hand its temp and GPU cache volumes +# and the GPU devices to the bun user; the entrypoint then runs the server, +# and every FFmpeg it starts, as that user. See docker-entrypoint.sh. +COPY docker-entrypoint.sh ./docker-entrypoint.sh +RUN chmod 755 ./docker-entrypoint.sh +ENTRYPOINT ["/app/docker-entrypoint.sh"] CMD ["bun", "run", "src/index.ts"] diff --git a/apps/media-server/bun.lock b/apps/media-server/bun.lock index 2b31b388a48..976add8790c 100644 --- a/apps/media-server/bun.lock +++ b/apps/media-server/bun.lock @@ -19,6 +19,9 @@ "trustedDependencies": [ "node-av", ], + "patchedDependencies": { + "mediabunny@1.45.2": "patches/mediabunny@1.45.2.patch", + }, "packages": { "@babel/runtime": ["@babel/runtime@7.29.2", "", {}, "sha512-JiDShH45zKHWyGe4ZNVRrCjBz8Nh9TMmZG1kh4QTK8hCBTWBi8Da+i7s1fJw7/lYpM4ccepSNfqzZ/QvABBi5g=="], diff --git a/apps/media-server/docker-entrypoint.sh b/apps/media-server/docker-entrypoint.sh new file mode 100755 index 00000000000..cd95a3ef3e7 --- /dev/null +++ b/apps/media-server/docker-entrypoint.sh @@ -0,0 +1,92 @@ +#!/bin/sh +set -eu + +# Runs the server as an unprivileged user. The container still starts as root, +# because the temp directory is usually a volume the host owns and has to be +# handed to that user first. When the user cannot be given what the server +# needs (its temp directory, or the GPU devices on an encoder host) the server +# keeps the user it started as and says so: a server that cannot write its +# temp files or reach its encoder is worse than one that runs as root. +# MEDIA_SERVER_RUN_AS=root keeps root on purpose. + +run_as="${MEDIA_SERVER_RUN_AS:-bun}" +next=/usr/local/bin/docker-entrypoint.sh +[ -x "$next" ] || next=exec + +start() { + if [ "$next" = exec ]; then exec "$@"; fi + exec "$next" "$@" +} + +if [ "$(id -u)" != "0" ] || [ "$run_as" = "root" ]; then + start "$@" +fi + +if ! id "$run_as" >/dev/null 2>&1; then + echo "[media-server] Running as root: there is no user named $run_as" >&2 + start "$@" +fi +if ! command -v setpriv >/dev/null 2>&1; then + echo "[media-server] Running as root: setpriv is not installed" >&2 + start "$@" +fi + +uid="$(id -u "$run_as")" +gid="$(id -g "$run_as")" +home="$(getent passwd "$run_as" | cut -d: -f6)" +temp_root="${TMPDIR:-/tmp}" + +groups="$gid" +for device in /dev/nvidia* /dev/dri/*; do + [ -c "$device" ] || continue + device_gid="$(stat -c %g "$device")" + # A device anyone may read and write needs no group, and the root group + # is never handed out for one. NVIDIA nodes usually own their devices + # as root:root with mode 666. + if [ "$device_gid" = 0 ]; then + case "$(stat -c %a "$device")" in + *[67]) continue ;; + esac + fi + case ",$groups," in + *",$device_gid,"*) ;; + *) groups="$groups,$device_gid" ;; + esac +done + +as_user() { + setpriv --reuid "$uid" --regid "$gid" --groups "$groups" "$@" +} + +# The GPU driver keeps its compiled kernels in CUDA_CACHE_PATH, usually a +# volume the host owns. +owned_dirs="$temp_root/cap-media-server $temp_root/cap-media-download-cache" +[ -n "${CUDA_CACHE_PATH:-}" ] && owned_dirs="$owned_dirs $CUDA_CACHE_PATH" + +for dir in $owned_dirs; do + mkdir -p "$dir" 2>/dev/null || true + chown -R "$uid:$gid" "$dir" 2>/dev/null || true +done +as_user test -w "$temp_root" || chmod 1777 "$temp_root" 2>/dev/null || true + +problem="" +for dir in "$temp_root" $owned_dirs; do + as_user test -w "$dir" || problem="$run_as cannot write $dir" +done +for device in /dev/nvidia*; do + [ -c "$device" ] || continue + as_user sh -c 'test -r "$1" && test -w "$1"' sh "$device" || + problem="$run_as cannot open $device" +done + +if [ -n "$problem" ]; then + echo "[media-server] Running as root: $problem" >&2 + start "$@" +fi + +export HOME="${home:-/tmp}" +export USER="$run_as" +if [ "$next" = exec ]; then + exec setpriv --reuid "$uid" --regid "$gid" --groups "$groups" --no-new-privs "$@" +fi +exec setpriv --reuid "$uid" --regid "$gid" --groups "$groups" --no-new-privs "$next" "$@" diff --git a/apps/media-server/package.json b/apps/media-server/package.json index 8aaf3128c0b..6fc5cc6c926 100644 --- a/apps/media-server/package.json +++ b/apps/media-server/package.json @@ -21,5 +21,8 @@ }, "trustedDependencies": [ "node-av" - ] + ], + "patchedDependencies": { + "mediabunny@1.45.2": "patches/mediabunny@1.45.2.patch" + } } diff --git a/apps/media-server/patches/mediabunny@1.45.2.patch b/apps/media-server/patches/mediabunny@1.45.2.patch new file mode 100644 index 00000000000..096db603770 --- /dev/null +++ b/apps/media-server/patches/mediabunny@1.45.2.patch @@ -0,0 +1,133 @@ +diff --git a/dist/modules/src/isobmff/isobmff-demuxer.js b/dist/modules/src/isobmff/isobmff-demuxer.js +index b5cce3d36e7385dc2c925ef0743383ca58150df6..bcf68572294591bed068d20e6a525bae8d696a8a 100644 +--- a/dist/modules/src/isobmff/isobmff-demuxer.js ++++ b/dist/modules/src/isobmff/isobmff-demuxer.js +@@ -18,6 +18,8 @@ import { DEFAULT_TRACK_DISPOSITION, RichImageData } from '../metadata.js'; + import { AC3_SAMPLE_RATES } from '../../shared/ac3-misc.js'; + import { Bitstream } from '../../shared/bitstream.js'; + import { Aes128CbcContext } from '../aes.js'; ++// About 18 hours of 60 fps video. Building the presentation order costs memory per sample. ++const MAX_DECLARED_SAMPLES = 4_000_000; + export class IsobmffDemuxer extends Demuxer { + constructor(input) { + super(input); +@@ -218,6 +220,79 @@ export class IsobmffDemuxer extends Demuxer { + this.currentTrack = internalTrack; + this.traverseBox(stblContainerSlice); + this.currentTrack = null; ++ // Counts in a sample table are declared by the file, not stored in it: a few bytes can claim ++ // billions of samples. No count is honored past what the file can hold. With a size entry per ++ // sample, that is the number of entries. With one size shared by all samples, each chunk can hold ++ // only as many samples of that size as fit in the file, and so can the track as a whole. ++ { ++ const sizes = sampleTable.sampleSizes; ++ let maxSamples = 0; ++ let perChunkLimit = 0; ++ if (sizes.length > 1) { ++ maxSamples = sizes.length; ++ perChunkLimit = sizes.length; ++ for (const entry of sampleTable.sampleToChunk) { ++ if (entry.samplesPerChunk > perChunkLimit) { ++ entry.samplesPerChunk = perChunkLimit; ++ } ++ } ++ } ++ else if (sizes.length === 1) { ++ const fileSize = this.reader.fileSize; ++ perChunkLimit = sizes[0] > 0 ++ ? (fileSize !== null ? Math.max(1, Math.floor(fileSize / sizes[0])) : MAX_DECLARED_SAMPLES) ++ : 1; ++ const chunkEntries = sampleTable.sampleToChunk; ++ const chunkCounts = chunkEntries.map((entry, i) => Math.max(0, (chunkEntries[i + 1] ++ ? chunkEntries[i + 1].startChunkIndex ++ : sampleTable.chunkOffsets.length) - entry.startChunkIndex)); ++ for (let i = 0; i < chunkEntries.length; i++) { ++ maxSamples += chunkCounts[i] * Math.min(chunkEntries[i].samplesPerChunk, perChunkLimit); ++ } ++ // Chunks may point at the same bytes, so the sum can pass what the file holds. The total is ++ // held to the samples that fit in the file, or to one per listed chunk if that is more, and ++ // the chunks share that total between them. ++ maxSamples = Math.min(maxSamples, Math.max(perChunkLimit, sampleTable.chunkOffsets.length)); ++ let placed = 0; ++ for (let i = 0; i < chunkEntries.length; i++) { ++ let perChunk = Math.min(chunkEntries[i].samplesPerChunk, perChunkLimit); ++ if (chunkCounts[i] > 0) { ++ perChunk = Math.min(perChunk, Math.floor((maxSamples - placed) / chunkCounts[i])); ++ } ++ chunkEntries[i].samplesPerChunk = Math.max(1, perChunk); ++ placed += chunkCounts[i] * chunkEntries[i].samplesPerChunk; ++ } ++ } ++ { ++ // Where each chunk run starts depends on the counts just held, so it is worked out again ++ // the way the table itself does it. ++ const chunkEntries = sampleTable.sampleToChunk; ++ let startSampleIndex = 0; ++ for (let i = 0; i < chunkEntries.length; i++) { ++ chunkEntries[i].startSampleIndex = startSampleIndex; ++ if (i < chunkEntries.length - 1) { ++ startSampleIndex += (chunkEntries[i + 1].startChunkIndex - chunkEntries[i].startChunkIndex) ++ * chunkEntries[i].samplesPerChunk; ++ } ++ } ++ } ++ let remaining = maxSamples; ++ let timedSamples = 0; ++ for (const entry of sampleTable.sampleTimingEntries) { ++ if (entry.count > remaining) { ++ entry.count = remaining; ++ } ++ remaining -= entry.count; ++ timedSamples += entry.count; ++ } ++ remaining = timedSamples; ++ for (const entry of sampleTable.sampleCompositionTimeOffsets) { ++ if (entry.count > remaining) { ++ entry.count = remaining; ++ } ++ remaining -= entry.count; ++ } ++ } + const isPcmCodec = internalTrack.info?.type === 'audio' + && internalTrack.info.codec + && PCM_AUDIO_CODECS.includes(internalTrack.info.codec); +@@ -236,8 +311,12 @@ export class IsobmffDemuxer extends Demuxer { + for (let i = 0; i < sampleTable.sampleToChunk.length; i++) { + const chunkEntry = sampleTable.sampleToChunk[i]; + const nextEntry = sampleTable.sampleToChunk[i + 1]; +- const chunkCount = (nextEntry ? nextEntry.startChunkIndex : sampleTable.chunkOffsets.length) +- - chunkEntry.startChunkIndex; ++ // A chunk that has no offset in the file cannot hold samples, so a run of chunks ends at the ++ // last chunk the file lists, whatever the next entry claims. Without this, a gap between two ++ // entries makes the loop below run once per claimed chunk, up to four billion times. ++ const PCM_CHUNK_SPAN_LIMIT = sampleTable.chunkOffsets.length; ++ const chunkCount = Math.max(0, Math.min(nextEntry ? nextEntry.startChunkIndex : PCM_CHUNK_SPAN_LIMIT, PCM_CHUNK_SPAN_LIMIT) ++ - chunkEntry.startChunkIndex); + for (let j = 0; j < chunkCount; j++) { + const startSampleIndex = chunkEntry.startSampleIndex + j * chunkEntry.samplesPerChunk; + const endSampleIndex = startSampleIndex + chunkEntry.samplesPerChunk; // Exclusive, outside of chunk +@@ -281,6 +360,13 @@ export class IsobmffDemuxer extends Demuxer { + if (sampleTable.sampleCompositionTimeOffsets.length > 0) { + // If composition time offsets are defined, we must build a list of all presentation timestamps and then + // sort them ++ let timedSamples = 0; ++ for (const entry of sampleTable.sampleTimingEntries) { ++ timedSamples += entry.count; ++ } ++ if (timedSamples > MAX_DECLARED_SAMPLES) { ++ throw new Error('Too many samples in track.'); ++ } + sampleTable.presentationTimestamps = []; + for (const entry of sampleTable.sampleTimingEntries) { + for (let i = 0; i < entry.count; i++) { +@@ -1813,6 +1899,10 @@ export class IsobmffDemuxer extends Demuxer { + }; + this.currentFragment.trackData.set(track.id, trackData); + } ++ if (trackData.samples.length + sampleCount > 2 ** 20) { ++ // Safety measure, as in later versions of this library ++ throw new Error('Too many (> 2^20) samples in fragment.'); ++ } + if (sampleCount === 0) { + // Don't associate the fragment with the track if it has no samples, this simplifies other code + this.currentFragment.implicitBaseDataOffset = trackData.currentOffset; diff --git a/apps/media-server/src/__tests__/fixtures/pool-test-worker.ts b/apps/media-server/src/__tests__/fixtures/pool-test-worker.ts new file mode 100644 index 00000000000..b4ede15c825 --- /dev/null +++ b/apps/media-server/src/__tests__/fixtures/pool-test-worker.ts @@ -0,0 +1,19 @@ +process.on("message", (request: { id: number; op: string; ms?: number }) => { + const reply = (body: object) => process.send?.({ id: request.id, ...body }); + if (request.op === "pid") reply({ ok: true, value: process.pid }); + else if (request.op === "fail") + reply({ ok: false, error: { message: "refused", pid: process.pid } }); + else if (request.op === "crash") process.exit(70); + else if (request.op === "die") process.kill(process.pid, "SIGKILL"); + else if (request.op === "spin") { + const end = Date.now() + (request.ms ?? 60_000); + while (Date.now() < end) {} + reply({ ok: true, value: process.pid }); + } else if (request.op === "slow") + setTimeout( + () => reply({ ok: true, value: process.pid }), + request.ms ?? 1_000, + ); +}); +process.on("disconnect", () => process.exit(0)); +process.send?.({ id: 0, ok: true, value: "ready" }); diff --git a/apps/media-server/src/__tests__/fixtures/test-bframes-64-fragmented.mp4 b/apps/media-server/src/__tests__/fixtures/test-bframes-64-fragmented.mp4 new file mode 100644 index 00000000000..989e2132f79 Binary files /dev/null and b/apps/media-server/src/__tests__/fixtures/test-bframes-64-fragmented.mp4 differ diff --git a/apps/media-server/src/__tests__/fixtures/test-bframes-64-pcm.mov b/apps/media-server/src/__tests__/fixtures/test-bframes-64-pcm.mov new file mode 100644 index 00000000000..39f42d12c91 Binary files /dev/null and b/apps/media-server/src/__tests__/fixtures/test-bframes-64-pcm.mov differ diff --git a/apps/media-server/src/__tests__/fixtures/test-bframes-64.mp4 b/apps/media-server/src/__tests__/fixtures/test-bframes-64.mp4 new file mode 100644 index 00000000000..7fa8b29afee Binary files /dev/null and b/apps/media-server/src/__tests__/fixtures/test-bframes-64.mp4 differ diff --git a/apps/media-server/src/__tests__/index.test.ts b/apps/media-server/src/__tests__/index.test.ts index 12294db6d78..cd83406a215 100644 --- a/apps/media-server/src/__tests__/index.test.ts +++ b/apps/media-server/src/__tests__/index.test.ts @@ -2,8 +2,16 @@ import { describe, expect, test } from "bun:test"; import app from "../app"; describe("GET /", () => { - test("returns server metadata and endpoints", async () => { - const response = await app.fetch(new Request("http://localhost/")); + test("returns server metadata and endpoints to a caller with the secret", async () => { + const saved = process.env.MEDIA_SERVER_WEBHOOK_SECRET; + process.env.MEDIA_SERVER_WEBHOOK_SECRET = "index-secret"; + const response = await app.fetch( + new Request("http://localhost/", { + headers: { "x-media-server-secret": "index-secret" }, + }), + ); + if (saved === undefined) delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; + else process.env.MEDIA_SERVER_WEBHOOK_SECRET = saved; expect(response.status).toBe(200); const data = await response.json(); @@ -26,6 +34,8 @@ describe("GET /", () => { "/video/process/:jobId/cancel", "/video/cleanup", "/video/force-cleanup", + "/live-transcribe/status", + "/live-transcribe/sessions", ], }); }); diff --git a/apps/media-server/src/__tests__/lib/audio-only-source.integration.test.ts b/apps/media-server/src/__tests__/lib/audio-only-source.integration.test.ts new file mode 100644 index 00000000000..cd3f8144266 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/audio-only-source.integration.test.ts @@ -0,0 +1,65 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { checkHasAudioTrack, extractAudio } from "../../lib/media-audio"; + +// Regular Cap transcription checks a source for audio, then extracts it. A +// source with audio and no video (an audio-only recording or upload) must be +// read like any other, and a video with no audio must still say so. +let dir = ""; + +async function ffmpeg(args: string[]): Promise { + const proc = Bun.spawn(["ffmpeg", "-v", "error", "-y", ...args], { + stdout: "ignore", + stderr: "pipe", + }); + const [code, stderr] = await Promise.all([ + proc.exited, + new Response(proc.stderr).text(), + ]); + if (code !== 0) throw new Error(stderr); +} + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), "audio-only-")); + await ffmpeg([ + "-f", + "lavfi", + "-i", + "sine=d=2:r=48000", + "-c:a", + "aac", + join(dir, "audio-only.m4a"), + ]); + await ffmpeg([ + "-f", + "lavfi", + "-i", + "testsrc=d=2:s=128x96:r=10", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + join(dir, "video-only.mp4"), + ]); +}, 60_000); + +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +describe("audio-only sources", () => { + test("an audio-only file has audio", async () => { + expect(await checkHasAudioTrack(join(dir, "audio-only.m4a"))).toBe(true); + }, 30_000); + + test("an audio-only file extracts to MP3", async () => { + const audio = await extractAudio(join(dir, "audio-only.m4a")); + expect(audio.length).toBeGreaterThan(1000); + }, 30_000); + + test("a video with no audio still has no audio", async () => { + expect(await checkHasAudioTrack(join(dir, "video-only.mp4"))).toBe(false); + }, 30_000); +}); diff --git a/apps/media-server/src/__tests__/lib/colour-probe-fallback.test.ts b/apps/media-server/src/__tests__/lib/colour-probe-fallback.test.ts new file mode 100644 index 00000000000..ba31a014ffe --- /dev/null +++ b/apps/media-server/src/__tests__/lib/colour-probe-fallback.test.ts @@ -0,0 +1,101 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { chmod, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { probeVideoFile } from "../../lib/media-probe"; +import { stopProbeWorkers } from "../../lib/media-probe-pool"; +import { colourProbeIsOptional, processVideo } from "../../lib/media-video"; + +let dir = ""; +const savedPath = process.env.PATH; + +async function ffmpeg(args: string[]): Promise { + const proc = Bun.spawn(["ffmpeg", "-v", "error", "-y", ...args], { + stdout: "ignore", + stderr: "pipe", + }); + const [code, stderr] = await Promise.all([ + proc.exited, + new Response(proc.stderr).text(), + ]); + if (code !== 0) throw new Error(stderr); +} + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), "colour-probe-")); + const source = ["-f", "lavfi", "-i", "testsrc=d=0.5:s=128x96:r=10"]; + await ffmpeg([ + ...source, + "-c:v", + "libvpx-vp9", + "-pix_fmt", + "yuv420p", + join(dir, "browser.webm"), + ]); + await ffmpeg([ + ...source, + "-c:v", + "libvpx-vp9", + "-pix_fmt", + "yuv420p10le", + join(dir, "profile2.webm"), + ]); + await ffmpeg([...source, "-c:v", "libx265", join(dir, "phone.mp4")]); + // An ffprobe that cannot answer the colour question and answers the rest. + const real = Bun.which("ffprobe"); + const shim = join(dir, "bin"); + await Bun.write( + join(shim, "ffprobe"), + `#!/bin/sh\ncase "$*" in *color_transfer*) exit 1 ;; esac\nexec "${real}" "$@"\n`, + ); + await chmod(join(shim, "ffprobe"), 0o755); + await writeFile(join(dir, "marker"), ""); + process.env.PATH = `${shim}:${savedPath}`; +}, 60_000); + +afterAll(async () => { + process.env.PATH = savedPath; + stopProbeWorkers(); + await rm(dir, { recursive: true, force: true }); +}); + +describe("colour information that cannot be read", () => { + test("only VP8 and 8-bit VP9 may go without it", () => { + expect(colourProbeIsOptional("vp8", null)).toBe(true); + expect(colourProbeIsOptional("vp9", "yuv420p")).toBe(true); + expect(colourProbeIsOptional("vp9", "yuv420p10le")).toBe(false); + expect(colourProbeIsOptional("vp9", null)).toBe(false); + for (const codec of ["hevc", "av1", "h264", "prores", "unknown"]) + expect(colourProbeIsOptional(codec, "yuv420p")).toBe(false); + }); + + test("a VP9 browser recording is encoded as standard range", async () => { + const path = join(dir, "browser.webm"); + const metadata = await probeVideoFile(path); + expect(metadata.videoCodec).toBe("vp9"); + const output = await processVideo(path, metadata, {}); + try { + expect(Bun.file(output.path).size).toBeGreaterThan(0); + } finally { + await output.cleanup(); + } + }, 60_000); + + test("a 10-bit VP9 source still fails rather than guess standard range", async () => { + const path = join(dir, "profile2.webm"); + const metadata = await probeVideoFile(path); + expect(metadata.videoCodec).toBe("vp9"); + expect(processVideo(path, metadata, {})).rejects.toThrow( + "Could not read the colour information", + ); + }, 60_000); + + test("an HEVC source still fails rather than guess standard range", async () => { + const path = join(dir, "phone.mp4"); + const metadata = await probeVideoFile(path); + expect(metadata.videoCodec).toBe("hevc"); + expect(processVideo(path, metadata, {})).rejects.toThrow( + "Could not read the colour information", + ); + }, 60_000); +}); diff --git a/apps/media-server/src/__tests__/lib/encode-length-limit.test.ts b/apps/media-server/src/__tests__/lib/encode-length-limit.test.ts new file mode 100644 index 00000000000..c0391c15c5d --- /dev/null +++ b/apps/media-server/src/__tests__/lib/encode-length-limit.test.ts @@ -0,0 +1,172 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { VideoMetadata } from "../../lib/job-manager"; +import { encodeLengthLimit } from "../../lib/media-common"; +import { probeVideoFile } from "../../lib/media-probe"; +import { stopProbeWorkers } from "../../lib/media-probe-pool"; +import { processVideo } from "../../lib/media-video"; + +// The real length of every file below, whatever its header says. +const SECONDS = 10; + +let dir = ""; + +async function run(cmd: string[]): Promise { + const proc = Bun.spawn(cmd, { stdout: "pipe", stderr: "pipe" }); + const [code, stdout, stderr] = await Promise.all([ + proc.exited, + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + if (code !== 0) throw new Error(stderr); + return stdout; +} + +async function lengthOf(path: string): Promise { + return Number( + ( + await run([ + "ffprobe", + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "csv=p=0", + path, + ]) + ).trim(), + ); +} + +// The Matroska Duration element (0x4489, an 8-byte float). +function durationElement(data: Buffer): number { + const at = data.indexOf(Buffer.from([0x44, 0x89, 0x88])); + if (at === -1) throw new Error("No Duration element"); + return at; +} + +// What a recorder that never finalises its file leaves: no Duration at all. +// The element becomes a Void element of the same size. +function withoutDuration(bytes: Buffer): Buffer { + const data = Buffer.from(bytes); + const at = durationElement(data); + data[at] = 0xec; + data[at + 1] = 0x89; + data.fill(0, at + 2, at + 11); + return data; +} + +function withDurationMs(bytes: Buffer, milliseconds: number): Buffer { + const data = Buffer.from(bytes); + data.writeDoubleBE(milliseconds, durationElement(data) + 3); + return data; +} + +// Sets the movie header (mvhd) duration of an MP4, leaving the tracks alone. +function withShortMovieHeader(bytes: Buffer, seconds: number): Buffer { + const data = Buffer.from(bytes); + const at = data.indexOf(Buffer.from("mvhd", "latin1")); + if (at === -1 || data[at + 4] !== 0) throw new Error("No version 0 mvhd"); + const body = at + 4; + const timescale = data.readUInt32BE(body + 12); + data.writeUInt32BE(Math.round(seconds * timescale), body + 16); + return data; +} + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), "encode-length-")); + const source = [ + "ffmpeg", + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + `testsrc=d=${SECONDS}:s=128x96:r=10`, + "-f", + "lavfi", + "-i", + `sine=d=${SECONDS}`, + ]; + const webm = join(dir, "recording.webm"); + await run([...source, "-c:v", "libvpx", "-c:a", "libopus", webm]); + const mp4 = join(dir, "recording.mp4"); + await run([...source, "-c:v", "libx264", "-c:a", "aac", mp4]); + const recording = await readFile(webm); + await writeFile(join(dir, "no-duration.webm"), withoutDuration(recording)); + await writeFile( + join(dir, "short-duration.webm"), + withDurationMs(recording, 2_000), + ); + await writeFile( + join(dir, "short-mvhd.mp4"), + withShortMovieHeader(await readFile(mp4), 2), + ); +}, 60_000); + +afterAll(async () => { + stopProbeWorkers(); + await rm(dir, { recursive: true, force: true }); +}); + +describe("the encode length limit never cuts a real file short", () => { + test.each([ + ["no-duration.webm", "a WebM with no Duration element"], + ["short-duration.webm", "a WebM whose Duration says less than it plays"], + ["short-mvhd.mp4", "an MP4 whose movie header says less than its tracks"], + ])( + "%s (%s)", + async (name) => { + const path = join(dir, name); + const metadata = await probeVideoFile(path); + const limit = encodeLengthLimit(metadata); + if (limit !== null) expect(limit).toBeGreaterThanOrEqual(SECONDS); + + const output = await processVideo(path, metadata, {}); + try { + expect(await lengthOf(output.path)).toBeGreaterThanOrEqual( + SECONDS - 0.2, + ); + } finally { + await output.cleanup(); + } + }, + 60_000, + ); + + test("a recovery encode is not held to any length", async () => { + const path = join(dir, "short-duration.webm"); + const metadata: VideoMetadata = { + ...(await probeVideoFile(path)), + // A header that is wrong in the way recovery input often is. + duration: 2, + timestampSpan: 2, + }; + const output = await processVideo( + path, + metadata, + {}, + undefined, + undefined, + { + genPts: true, + }, + ); + try { + expect(await lengthOf(output.path)).toBeGreaterThanOrEqual(SECONDS - 0.2); + } finally { + await output.cleanup(); + } + }, 60_000); + + test("no limit when the probe did not measure the timestamps", () => { + expect(encodeLengthLimit({ duration: 10 })).toBeNull(); + expect(encodeLengthLimit({ duration: 10, timestampSpan: null })).toBeNull(); + expect(encodeLengthLimit({ duration: 2, timestampSpan: 10 })).toBe(12); + expect(encodeLengthLimit({ duration: 10, timestampSpan: 2 })).toBe(12); + }); +}); diff --git a/apps/media-server/src/__tests__/lib/hls-ladder.test.ts b/apps/media-server/src/__tests__/lib/hls-ladder.test.ts index 951bfc42e15..3b611a2e4e8 100644 --- a/apps/media-server/src/__tests__/lib/hls-ladder.test.ts +++ b/apps/media-server/src/__tests__/lib/hls-ladder.test.ts @@ -256,6 +256,13 @@ function valueAfter(args: string[], flag: string) { } describe("buildHlsLadderArgs", () => { + test("holds the output to the length it is given, and only then", () => { + const capped = buildHlsLadderArgs(argsInput({ maxOutputSeconds: 12.5 })); + expect(valueAfter(capped, "-t")).toBe("12.5"); + expect(capped.indexOf("-t")).toBeGreaterThan(capped.lastIndexOf("-i")); + expect(buildHlsLadderArgs(argsInput())).not.toContain("-t"); + }); + test("encodes every rendition on the GPU from one upload", () => { const args = buildHlsLadderArgs( argsInput({ nvenc: { options: NVENC, gpuDecode: false } }), diff --git a/apps/media-server/src/__tests__/lib/infra-capacity.test.ts b/apps/media-server/src/__tests__/lib/infra-capacity.test.ts new file mode 100644 index 00000000000..99cc3e9ceac --- /dev/null +++ b/apps/media-server/src/__tests__/lib/infra-capacity.test.ts @@ -0,0 +1,182 @@ +import { + afterEach, + beforeEach, + describe, + expect, + mock, + spyOn, + test, +} from "bun:test"; +import app from "../../app"; +import * as jobManager from "../../lib/job-manager"; + +const SECRET = "test-secret"; +const created: string[] = []; + +function addActiveJob(heavyLadder = false) { + const jobId = jobManager.generateJobId(); + const job = jobManager.createJob(jobId, `video-${created.length}`, "user"); + if (heavyLadder) job.infraHeavyLadder = true; + created.push(jobId); + return job; +} + +function request(path: string, body: Record) { + return new Request(`http://localhost${path}`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-media-server-secret": SECRET, + }, + body: JSON.stringify(body), + }); +} + +const ladderBody = (extra: Record = {}) => ({ + videoId: "video-ladder", + userId: "app-1", + videoUrl: "https://example.com/video.mp4", + maxTier: "2160p", + uploads: { "master.m3u8": "https://s3.example.com/master" }, + ...extra, +}); + +const processBody = (extra: Record = {}) => ({ + videoId: "video-process", + userId: "app-1", + videoUrl: "https://example.com/video.mp4", + outputPresignedUrl: "https://s3.example.com/output", + ...extra, +}); + +let previousSecret: string | undefined; + +beforeEach(() => { + previousSecret = process.env.MEDIA_SERVER_WEBHOOK_SECRET; + process.env.MEDIA_SERVER_WEBHOOK_SECRET = SECRET; +}); + +afterEach(() => { + mock.restore(); + for (const jobId of created.splice(0)) jobManager.deleteJob(jobId); + if (previousSecret === undefined) + delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; + else process.env.MEDIA_SERVER_WEBHOOK_SECRET = previousSecret; +}); + +describe("capacity for Cap Infra jobs", () => { + const allows = ( + effectiveMax: number, + active: number, + activeHeavyLadders: number, + heavyLadder: boolean, + ) => + jobManager.infraCapacityAllows({ + effectiveMax, + active, + activeHeavyLadders, + heavyLadder, + reservedSlots: 1, + maxHeavyLadders: 1, + }); + + test("leaves one slot that Infra jobs cannot take", () => { + expect(allows(4, 0, 0, false)).toBe(true); + expect(allows(4, 2, 0, false)).toBe(true); + expect(allows(4, 3, 0, false)).toBe(false); + expect(allows(4, 4, 0, false)).toBe(false); + }); + + test("runs one heavy Infra ladder at a time", () => { + expect(allows(4, 0, 0, true)).toBe(true); + expect(allows(4, 1, 1, true)).toBe(false); + expect(allows(4, 1, 1, false)).toBe(true); + }); + + test("a throttled worker with one slot keeps today's behaviour", () => { + expect(allows(1, 0, 0, true)).toBe(true); + expect(allows(1, 1, 0, false)).toBe(false); + expect(allows(0, 0, 0, false)).toBe(false); + }); + + test("a larger reserve shrinks with the worker but never to nothing", () => { + const allowsWithReserve = (effectiveMax: number, active: number) => + jobManager.infraCapacityAllows({ + effectiveMax, + active, + activeHeavyLadders: 0, + heavyLadder: false, + reservedSlots: 2, + maxHeavyLadders: 1, + }); + expect(allowsWithReserve(4, 1)).toBe(true); + expect(allowsWithReserve(4, 2)).toBe(false); + expect(allowsWithReserve(2, 0)).toBe(true); + expect(allowsWithReserve(2, 1)).toBe(false); + expect(allowsWithReserve(1, 0)).toBe(true); + }); + + test("counts only heavy ladders that are still running", () => { + const before = jobManager.getActiveInfraHeavyLadderCount(); + const job = addActiveJob(true); + addActiveJob(false); + expect(jobManager.getActiveInfraHeavyLadderCount()).toBe(before + 1); + jobManager.updateJob(job.jobId, { phase: "complete", progress: 100 }); + expect(jobManager.getActiveInfraHeavyLadderCount()).toBe(before); + }); +}); + +describe("routes", () => { + test("an Infra ladder over its limit gets the busy answer", async () => { + spyOn(jobManager, "canAcceptNewVideoProcess").mockReturnValue(true); + const infra = spyOn( + jobManager, + "canAcceptNewInfraVideoProcess", + ).mockReturnValue(false); + const response = await app.fetch( + request("/video/ladder", ladderBody({ infra: true })), + ); + expect(response.status).toBe(503); + expect((await response.json()).code).toBe("SERVER_BUSY"); + expect(response.headers.get("Retry-After")).not.toBeNull(); + expect(infra).toHaveBeenCalledWith(true); + }); + + test("a 1080p Infra ladder is not counted as heavy", async () => { + spyOn(jobManager, "canAcceptNewVideoProcess").mockReturnValue(true); + const infra = spyOn( + jobManager, + "canAcceptNewInfraVideoProcess", + ).mockReturnValue(false); + await app.fetch( + request("/video/ladder", ladderBody({ infra: true, maxTier: "1080p" })), + ); + expect(infra).toHaveBeenCalledWith(false); + }); + + test("an Infra MP4 over its limit gets the busy answer", async () => { + spyOn(jobManager, "canAcceptNewVideoProcess").mockReturnValue(true); + const infra = spyOn( + jobManager, + "canAcceptNewInfraVideoProcess", + ).mockReturnValue(false); + const response = await app.fetch( + request("/video/process", processBody({ infra: true })), + ); + expect(response.status).toBe(503); + expect(infra).toHaveBeenCalledWith(false); + }); + + test("Cap's own jobs never consult the Infra limits", async () => { + spyOn(jobManager, "canAcceptNewVideoProcess").mockReturnValue(false); + const infra = spyOn( + jobManager, + "canAcceptNewInfraVideoProcess", + ).mockReturnValue(false); + const ladder = await app.fetch(request("/video/ladder", ladderBody())); + const process = await app.fetch(request("/video/process", processBody())); + expect(ladder.status).toBe(503); + expect(process.status).toBe(503); + expect(infra).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/media-server/src/__tests__/lib/infra-poster.integration.test.ts b/apps/media-server/src/__tests__/lib/infra-poster.integration.test.ts new file mode 100644 index 00000000000..e95995731cb --- /dev/null +++ b/apps/media-server/src/__tests__/lib/infra-poster.integration.test.ts @@ -0,0 +1,231 @@ +import { afterAll, beforeAll, describe, expect, spyOn, test } from "bun:test"; +import { execFileSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type appType from "../../app"; +import { + INFRA_POSTER_OPTIONS, + posterThumbnailOptions, +} from "../../lib/infra-poster"; +import type { Job } from "../../lib/job-manager"; +import * as mediaVideo from "../../lib/media-video"; + +const SECRET = "test-secret"; +const HEADERS = { + "Content-Type": "application/json", + "x-media-server-secret": SECRET, +}; +const LADDER_FILES = [ + "master.m3u8", + "audio.m3u8", + "audio.m4s", + ...["2160p", "1440p", "1080p", "720p", "360p"].flatMap((tier) => [ + `${tier}.m3u8`, + `${tier}.m4s`, + ]), +]; + +let app: typeof appType; +let jobManager: typeof import("../../lib/job-manager"); +let server: ReturnType; +let baseUrl = ""; +const workDir = mkdtempSync(join(tmpdir(), "cap-infra-poster-")); +const sources = new Map>(); +const uploads = new Map(); + +function makeSource(name: string, size: string) { + const path = join(workDir, name); + execFileSync("ffmpeg", [ + "-v", + "error", + "-f", + "lavfi", + "-i", + `testsrc2=size=${size}:rate=30:duration=2`, + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-pix_fmt", + "yuv420p", + "-y", + path, + ]); + sources.set(`/${name}`, new Uint8Array(readFileSync(path))); +} + +function imageSize(bytes: Uint8Array | undefined): string { + const path = join(workDir, "poster.jpg"); + writeFileSync(path, bytes ?? new Uint8Array()); + return execFileSync( + "ffprobe", + [ + "-v", + "error", + "-show_entries", + "stream=width,height", + "-of", + "csv=p=0:s=x", + path, + ], + { encoding: "utf8" }, + ).trim(); +} + +async function waitForTerminalJob(jobId: string): Promise { + const startedAt = Date.now(); + while (Date.now() - startedAt < 90_000) { + const job = jobManager.getJob(jobId); + if (!job) throw new Error(`Job ${jobId} disappeared`); + if (["complete", "error", "cancelled"].includes(job.phase)) return job; + await Bun.sleep(100); + } + throw new Error(`Job ${jobId} did not finish`); +} + +async function runJob(path: string, body: Record) { + const thumbnail = spyOn(mediaVideo, "generateThumbnail"); + // Admission reads this machine's free memory and load; the poster size is + // what is under test here. + const capacity = [ + spyOn(jobManager, "canAcceptNewVideoProcess").mockReturnValue(true), + spyOn(jobManager, "canAcceptNewInfraVideoProcess").mockReturnValue(true), + ]; + try { + const response = await app.request(path, { + method: "POST", + headers: HEADERS, + body: JSON.stringify({ + videoId: "poster-video", + userId: "poster-user", + ...body, + }), + }); + expect(response.status).toBe(200); + const { jobId } = (await response.json()) as { jobId: string }; + const job = await waitForTerminalJob(jobId); + jobManager.deleteJob(jobId); + expect(job.error).toBeUndefined(); + expect(job.phase).toBe("complete"); + expect(thumbnail).toHaveBeenCalledTimes(1); + return thumbnail.mock.calls[0]?.[2]; + } finally { + thumbnail.mockRestore(); + for (const spy of capacity) spy.mockRestore(); + } +} + +function processBody(source: string, name: string, extra = {}) { + return { + videoUrl: `${baseUrl}/${source}`, + outputPresignedUrl: `${baseUrl}/uploads/${name}.mp4`, + thumbnailPresignedUrl: `${baseUrl}/uploads/${name}.jpg`, + inputExtension: ".mp4", + ...extra, + }; +} + +function ladderBody(source: string, name: string, extra = {}) { + return { + videoUrl: `${baseUrl}/${source}`, + uploads: Object.fromEntries( + LADDER_FILES.map((file) => [file, `${baseUrl}/uploads/${name}/${file}`]), + ), + thumbnailPresignedUrl: `${baseUrl}/uploads/${name}.jpg`, + ...extra, + }; +} + +beforeAll(async () => { + process.env.MEDIA_SERVER_WEBHOOK_SECRET = SECRET; + process.env.MEDIA_SERVER_MAX_CONCURRENT_VIDEO_PROCESSES = "4"; + app = (await import("../../app")).default; + jobManager = await import("../../lib/job-manager"); + makeSource("wide.mp4", "2560x1440"); + makeSource("tall.mp4", "1080x1920"); + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + const url = new URL(request.url); + if (request.method === "PUT" && url.pathname.startsWith("/uploads/")) { + uploads.set( + url.pathname.slice("/uploads/".length), + new Uint8Array(await request.arrayBuffer()), + ); + return new Response(null, { status: 200, headers: { ETag: '"x"' } }); + } + const source = sources.get(url.pathname); + if (!source) return new Response(null, { status: 404 }); + return new Response(source, { + headers: { + "Content-Type": "video/mp4", + "Content-Length": String(source.byteLength), + }, + }); + }, + }); + baseUrl = `http://127.0.0.1:${server.port}`; +}); + +afterAll(() => { + server?.stop(true); + rmSync(workDir, { recursive: true, force: true }); +}); + +describe("poster size", () => { + test("only an Infra job asks for the larger poster", () => { + expect(posterThumbnailOptions(undefined)).toStrictEqual({}); + expect(posterThumbnailOptions(false)).toStrictEqual({}); + expect(posterThumbnailOptions(true)).toStrictEqual({ + width: 1920, + height: 1920, + }); + }); + + test("a regular Cap process job keeps the default thumbnail", async () => { + const options = await runJob( + "/video/process", + processBody("wide.mp4", "regular-wide"), + ); + expect(options).toStrictEqual({}); + expect(imageSize(uploads.get("regular-wide.jpg"))).toBe("1280x720"); + }, 120_000); + + test("an Infra MP4 job takes its poster up to 1920 wide", async () => { + const options = await runJob( + "/video/process", + processBody("wide.mp4", "infra-wide", { + infra: true, + maxWidth: 2560, + maxHeight: 2560, + }), + ); + expect(options).toStrictEqual(INFRA_POSTER_OPTIONS); + expect(imageSize(uploads.get("infra-wide.jpg"))).toBe("1920x1080"); + }, 120_000); + + test("an Infra ladder takes its poster up to 1920 wide", async () => { + const options = await runJob( + "/video/ladder", + ladderBody("wide.mp4", "ladder-wide", { + infra: true, + maxTier: "1440p", + }), + ); + expect(options).toStrictEqual(INFRA_POSTER_OPTIONS); + expect(imageSize(uploads.get("ladder-wide.jpg"))).toBe("1920x1080"); + }, 120_000); + + test("a portrait Infra ladder keeps its full height and never upscales", async () => { + await runJob( + "/video/ladder", + ladderBody("tall.mp4", "ladder-tall", { + infra: true, + maxTier: "1080p", + }), + ); + expect(imageSize(uploads.get("ladder-tall.jpg"))).toBe("1080x1920"); + }, 120_000); +}); diff --git a/apps/media-server/src/__tests__/lib/isolated-worker-pool.test.ts b/apps/media-server/src/__tests__/lib/isolated-worker-pool.test.ts new file mode 100644 index 00000000000..426a0e934ab --- /dev/null +++ b/apps/media-server/src/__tests__/lib/isolated-worker-pool.test.ts @@ -0,0 +1,130 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { join } from "node:path"; +import { + createWorkerPool, + WorkerLostError, + type WorkerPool, +} from "../../lib/isolated-worker-pool"; + +type Request = { op: string; ms?: number }; + +const pools: WorkerPool[] = []; + +class Refused extends Error { + constructor(readonly pid: number) { + super("refused"); + } +} +class Fault extends Error {} + +function pool(maxWorkers = 2) { + const created = createWorkerPool({ + entry: join(import.meta.dir, "..", "fixtures", "pool-test-worker.ts"), + env: () => ({ PATH: process.env.PATH ?? "" }), + memoryLimitMB: () => 4096, + toError: (failure) => new Refused(failure.pid), + memoryError: () => new Error("memory"), + faultError: () => new Fault("the worker stopped on this request"), + faultExitCode: 70, + maxWorkers: () => maxWorkers, + label: "test worker", + }); + pools.push(created); + return created; +} + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +afterEach(() => { + for (const created of pools.splice(0)) created.stop(); +}); + +describe("isolated worker pool", () => { + test("reuses a worker across requests, including one it answered with a failure", async () => { + const workers = pool(); + const first = (await workers.run({ op: "pid" }, 10_000)) as number; + const refused = await workers + .run({ op: "fail" }, 10_000) + .catch((error: unknown) => error); + expect(refused).toBeInstanceOf(Refused); + expect((refused as Refused).pid).toBe(first); + expect(await workers.run({ op: "pid" }, 10_000)).toBe(first); + expect(workers.stats().recycled).toBe(0); + }, 30_000); + + test("stops a worker that overruns its time and serves the next request on a new one", async () => { + const workers = pool(); + const first = (await workers.run({ op: "pid" }, 10_000)) as number; + const started = performance.now(); + const error = await workers + .run({ op: "spin" }, 300) + .catch((caught: unknown) => caught); + expect(performance.now() - started).toBeLessThan(5_000); + expect((error as Error).message).toContain("timed out"); + await Bun.sleep(100); + expect(isAlive(first)).toBe(false); + const next = (await workers.run({ op: "pid" }, 10_000)) as number; + expect(next).not.toBe(first); + expect(workers.stats().timeoutKills).toBe(1); + }, 30_000); + + test("a worker that crashes fails only its own request and is replaced", async () => { + const workers = pool(); + const first = (await workers.run({ op: "pid" }, 10_000)) as number; + expect( + await workers.run({ op: "crash" }, 10_000).catch((error) => error), + ).toBeInstanceOf(Fault); + expect( + await workers.run({ op: "die" }, 10_000).catch((error) => error), + ).toBeInstanceOf(WorkerLostError); + const next = (await workers.run({ op: "pid" }, 10_000)) as number; + expect(next).not.toBe(first); + expect(isAlive(first)).toBe(false); + }, 30_000); + + test("never runs more workers than its limit, and queued requests wait", async () => { + const workers = pool(2); + const pids = (await Promise.all( + Array.from({ length: 6 }, () => + workers.run({ op: "slow", ms: 300 }, 20_000), + ), + )) as number[]; + expect(new Set(pids).size).toBeLessThanOrEqual(2); + expect(workers.stats().started).toBeLessThanOrEqual(2); + expect(workers.stats().waiting).toBe(0); + }, 30_000); + + test("a queued request that waits past its deadline times out without a worker", async () => { + const workers = pool(1); + const busy = workers.run({ op: "slow", ms: 1_500 }, 20_000); + const queued = await workers + .run({ op: "pid" }, 200) + .catch((error: unknown) => error); + expect((queued as Error).message).toContain("timed out"); + await busy; + expect(workers.stats().waiting).toBe(0); + }, 30_000); + + test("a request cancelled while it waits for a worker is never sent", async () => { + const workers = pool(1); + const first = (await workers.run({ op: "pid" }, 10_000)) as number; + const busy = workers.run({ op: "slow", ms: 500 }, 20_000); + const controller = new AbortController(); + const queued = workers + .run({ op: "crash" }, 20_000, controller.signal) + .catch((error: unknown) => error); + controller.abort(new Error("cancelled")); + expect(((await queued) as Error).message).toBe("cancelled"); + expect(await busy).toBe(first); + // Had the crash been sent, this worker would be gone. + expect(await workers.run({ op: "pid" }, 10_000)).toBe(first); + expect(workers.stats().started).toBe(1); + }, 30_000); +}); diff --git a/apps/media-server/src/__tests__/lib/legacy-codecs.integration.test.ts b/apps/media-server/src/__tests__/lib/legacy-codecs.integration.test.ts new file mode 100644 index 00000000000..87594a3e0df --- /dev/null +++ b/apps/media-server/src/__tests__/lib/legacy-codecs.integration.test.ts @@ -0,0 +1,219 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { probeVideoFile } from "../../lib/media-probe"; +import { stopProbeWorkers } from "../../lib/media-probe-pool"; +import { spawn } from "../../lib/media-spawn"; + +// Older codecs a regular Cap import can carry, in the containers the probe +// reads (Matroska, MP4 and 3GP). Each fixture is made with the local FFmpeg, +// then probed and transcoded to H.264 and AAC through the same hardened spawn +// the server uses, so the codec and decoder lists are what decides. A family +// whose encoder this FFmpeg lacks is skipped here. +type Fixture = { + name: string; + file: string; + video: string[]; + audio: string[] | null; + encoders: string[]; +}; + +const FIXTURES: Fixture[] = [ + { + name: "MPEG-1 video with MPEG audio layer 2", + file: "mpeg1.mkv", + video: ["-c:v", "mpeg1video"], + audio: ["-c:a", "mp2"], + encoders: ["mpeg1video", "mp2"], + }, + { + name: "MPEG-2 video with MPEG audio layer 2", + file: "mpeg2.mkv", + video: ["-c:v", "mpeg2video"], + audio: ["-c:a", "mp2"], + encoders: ["mpeg2video", "mp2"], + }, + { + name: "Windows Media video 7 with Windows Media audio 1", + file: "wmv1.mkv", + video: ["-c:v", "wmv1"], + audio: ["-c:a", "wmav1"], + encoders: ["wmv1", "wmav1"], + }, + { + name: "Windows Media video 8 with Windows Media audio 2", + file: "wmv2.mkv", + video: ["-c:v", "wmv2"], + audio: ["-c:a", "wmav2"], + encoders: ["wmv2", "wmav2"], + }, + { + name: "early MPEG-4 (DivX 3) with Microsoft ADPCM", + file: "msmpeg4v3.mkv", + video: ["-c:v", "msmpeg4"], + audio: ["-c:a", "adpcm_ms"], + encoders: ["msmpeg4", "adpcm_ms"], + }, + { + name: "early MPEG-4 version 2 with IMA ADPCM", + file: "msmpeg4v2.mkv", + video: ["-c:v", "msmpeg4v2"], + audio: ["-c:a", "adpcm_ima_wav"], + encoders: ["msmpeg4v2", "adpcm_ima_wav"], + }, + { + name: "Flash video", + file: "flv1.mkv", + video: ["-c:v", "flv"], + audio: ["-c:a", "aac"], + encoders: ["flv", "aac"], + }, + { + name: "H.263 from a phone", + file: "phone.3gp", + video: ["-c:v", "h263"], + audio: ["-c:a", "aac", "-ar", "8000", "-ac", "1"], + encoders: ["h263", "aac"], + }, + { + name: "H.263 with AMR narrowband audio", + file: "amrnb.3gp", + video: ["-c:v", "h263"], + audio: ["-c:a", "libopencore_amrnb", "-ar", "8000", "-ac", "1"], + encoders: ["h263", "libopencore_amrnb"], + }, + { + name: "H.263 with AMR wideband audio", + file: "amrwb.3gp", + video: ["-c:v", "h263"], + audio: ["-c:a", "libvo_amrwbenc", "-ar", "16000", "-ac", "1"], + encoders: ["h263", "libvo_amrwbenc"], + }, + { + name: "Theora with Vorbis", + file: "theora.mkv", + video: ["-c:v", "libtheora"], + audio: ["-c:a", "libvorbis"], + encoders: ["libtheora", "libvorbis"], + }, + { + name: "H.264 with MPEG audio layer 2", + file: "h264-mp2.mkv", + video: ["-c:v", "libx264"], + audio: ["-c:a", "mp2"], + encoders: ["libx264", "mp2"], + }, + { + name: "H.264 with Windows Media audio 2", + file: "h264-wma.mkv", + video: ["-c:v", "libx264"], + audio: ["-c:a", "wmav2"], + encoders: ["libx264", "wmav2"], + }, +]; + +let dir = ""; +// Read when the file loads: skipIf decides before beforeAll runs. +const encoders = new Set( + Bun.spawnSync(["ffmpeg", "-hide_banner", "-encoders"]) + .stdout.toString() + .split("\n") + .map((line) => line.trim().split(/\s+/)[1] ?? ""), +); + +async function run(cmd: string[]): Promise<{ code: number; stderr: string }> { + const proc = Bun.spawn(cmd, { stdout: "pipe", stderr: "pipe" }); + const [code, stdout, stderr] = await Promise.all([ + proc.exited, + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + return { code, stderr: stderr || stdout }; +} + +const available = (fixture: Fixture) => + fixture.encoders.every((name) => encoders.has(name)); + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), "legacy-codecs-")); + for (const fixture of FIXTURES) { + if (!available(fixture)) continue; + const made = await run([ + "ffmpeg", + "-v", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc=d=1:s=176x144:r=25", + ...(fixture.audio ? ["-f", "lavfi", "-i", "sine=d=1:r=44100"] : []), + ...fixture.video, + ...(fixture.audio ?? []), + join(dir, fixture.file), + ]); + if (made.code !== 0) + throw new Error(`could not make ${fixture.file}: ${made.stderr}`); + } +}, 120_000); + +afterAll(async () => { + stopProbeWorkers(); + await rm(dir, { recursive: true, force: true }); +}); + +describe("older formats a regular import can carry", () => { + for (const fixture of FIXTURES) { + test.skipIf(!available(fixture))( + `${fixture.name} probes and transcodes`, + async () => { + const input = join(dir, fixture.file); + const metadata = await probeVideoFile(input); + expect(metadata.width).toBe(176); + expect(metadata.height).toBe(144); + + const output = join(dir, `${fixture.file}.mp4`); + const transcode = spawn({ + cmd: [ + "ffmpeg", + "-v", + "error", + "-y", + "-i", + input, + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-c:a", + "aac", + output, + ], + stdout: "ignore", + stderr: "pipe", + }); + const [code, stderr] = await Promise.all([ + transcode.exited, + new Response(transcode.stderr).text(), + ]); + expect(stderr).not.toContain("not on whitelist"); + expect(code).toBe(0); + + const streams = await run([ + "ffprobe", + "-v", + "error", + "-show_entries", + "stream=codec_type,codec_name", + "-of", + "csv=p=0", + output, + ]); + expect(streams.stderr).toContain("h264,video"); + if (fixture.audio) expect(streams.stderr).toContain("aac,audio"); + }, + 60_000, + ); + } +}); diff --git a/apps/media-server/src/__tests__/lib/matroska-length.test.ts b/apps/media-server/src/__tests__/lib/matroska-length.test.ts new file mode 100644 index 00000000000..79653aeb12b --- /dev/null +++ b/apps/media-server/src/__tests__/lib/matroska-length.test.ts @@ -0,0 +1,326 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtemp, open, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { matroskaEndFromTail } from "../../lib/matroska-tail"; +import { createMediaInput, encodeLengthLimit } from "../../lib/media-common"; +import { + probeVideo, + UNFINISHED_RECORDING_MESSAGE, +} from "../../lib/media-probe"; +import { stopProbeWorkers } from "../../lib/media-probe-pool"; + +const SPARSE_BYTES = 800 * 1024 * 1024; +const LOOPS = 60; +// What a read of a remote file over a real network does in a second. Reads +// that run from an offset to the end are held to it. +const OPEN_ENDED_BYTES_PER_SECOND = 4 * 1024 * 1024; + +let dir = ""; +let server: ReturnType; +let origin = ""; +const served = new Map(); + +async function run(cmd: string[]): Promise { + const proc = Bun.spawn(cmd, { stdout: "pipe", stderr: "pipe" }); + const [code, stdout, stderr] = await Promise.all([ + proc.exited, + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + if (code !== 0) throw new Error(stderr); + return stdout; +} + +// The time of the last video packet, as FFmpeg reads it. +async function lastPacketTime(path: string): Promise { + const lines = ( + await run([ + "ffprobe", + "-v", + "error", + "-select_streams", + "v:0", + "-show_entries", + "packet=pts_time", + "-of", + "csv=p=0", + path, + ]) + ) + .split("\n") + .map(Number.parseFloat) + .filter(Number.isFinite); + return Math.max(...lines); +} + +async function secondsFromTail(path: string): Promise { + const end = await matroskaEndFromTail(path); + if (typeof end !== "object" || end === null) + throw new Error(`No length from the tail: ${end}`); + return end.seconds; +} + +// The length a probe gave this file before the tail was read: every +// cluster read from the start. +async function fullReadLength(path: string): Promise { + const input = createMediaInput(path); + try { + const [start, end] = await Promise.all([ + input.getFirstTimestamp(), + input.computeDuration(undefined, { skipLiveWait: true }), + ]); + return Number.isFinite(start) && start > 0 ? Math.max(0, end - start) : end; + } finally { + input.dispose(); + } +} + +function slowStream(file: Blob, start: number): ReadableStream { + let offset = start; + const chunk = 256 * 1024; + return new ReadableStream({ + async pull(controller) { + if (offset >= file.size) return controller.close(); + const bytes = new Uint8Array( + await file.slice(offset, offset + chunk).arrayBuffer(), + ); + offset += bytes.length; + await Bun.sleep((bytes.length / OPEN_ENDED_BYTES_PER_SECOND) * 1000); + controller.enqueue(bytes); + }, + }); +} + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), "matroska-length-")); + const video = ["-f", "lavfi", "-i", "testsrc2=d=60:s=160x90:r=30"]; + const audio = ["-f", "lavfi", "-i", "sine=d=60"]; + // FFmpeg writing to a pipe cannot go back to write Cues or a Duration. + await run([ + "sh", + "-c", + `ffmpeg -v error ${[...video, ...audio].join(" ")} -c:v libx264 -preset ultrafast -c:a aac -f matroska - > "${join(dir, "piped.mkv")}"`, + ]); + await run([ + "sh", + "-c", + `ffmpeg -v error ${[...video, ...audio].join(" ")} -c:v libvpx -deadline realtime -cpu-used 8 -c:a libopus -f webm -live 1 - > "${join(dir, "recorder.webm")}"`, + ]); + + // A finished file whose single cluster is larger than any tail read, as + // a high bitrate recording with long clusters has. + await run([ + "sh", + "-c", + `ffmpeg -v error -f lavfi -i testsrc2=d=2:s=1280x720:r=30 -vf noise=alls=30:allf=t -c:v libx264 -preset ultrafast -qp 0 -an -f matroska -cluster_size_limit 1000000000 -cluster_time_limit 60000 - > "${join(dir, "one-cluster.mkv")}"`, + ]); + // A short browser recording: live WebM, no Duration, no Cues. + await run([ + "sh", + "-c", + `ffmpeg -v error -f lavfi -i testsrc2=d=7:s=640x360:r=30 -f lavfi -i sine=d=7 -c:v libvpx -deadline realtime -cpu-used 8 -c:a libopus -f webm -live 1 - > "${join(dir, "browser.webm")}"`, + ]); + + // What production saw: a short recording looped into a long one with + // stream copy into a pipe, so no Cues and no Duration. + await run([ + "ffmpeg", + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=d=6:s=640x360:r=30", + "-f", + "lavfi", + "-i", + "sine=d=6", + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-b:v", + "8M", + "-c:a", + "aac", + "-y", + join(dir, "base.mkv"), + ]); + await run([ + "sh", + "-c", + `ffmpeg -v error -stream_loop ${LOOPS - 1} -i "${join(dir, "base.mkv")}" -c copy -f matroska - > "${join(dir, "looped.mkv")}"`, + ]); + + // The head and tail of the piped file around 800 MB of nothing: a file + // only a bounded read can measure in time. + const pipedBytes = await readFile(join(dir, "piped.mkv")); + const headBytes = pipedBytes.subarray(0, 1024 * 1024); + const tailBytes = pipedBytes.subarray(pipedBytes.length - 1024 * 1024); + const sparse = await open(join(dir, "sparse.mkv"), "w"); + await sparse.write(headBytes, 0, headBytes.length, 0); + await sparse.write( + tailBytes, + 0, + tailBytes.length, + SPARSE_BYTES - tailBytes.length, + ); + await sparse.close(); + + // A finished recording in a file its recorder preallocated: real + // clusters, then zero fill. + await writeFile( + join(dir, "preallocated.mkv"), + Buffer.concat([pipedBytes, Buffer.alloc(8 * 1024 * 1024)]), + ); + + // A recording cut off with no cluster anywhere near its end. + await writeFile( + join(dir, "no-tail.mkv"), + Buffer.concat([headBytes, Buffer.alloc(40 * 1024 * 1024)]), + ); + + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + const requested = new URL(request.url).pathname.slice(1); + // A "-fast" name is the same file from a server with no rate limit. + const fast = requested.includes("-fast."); + const name = requested.replace("-fast.", "."); + const file = Bun.file(join(dir, name)); + const size = file.size; + const range = request.headers.get("range") ?? ""; + let start = 0; + let end = size - 1; + const suffix = range.match(/^bytes=-(\d+)$/); + const span = range.match(/^bytes=(\d+)-(\d*)$/); + if (suffix) start = Math.max(0, size - Number(suffix[1])); + else if (span) { + start = Number(span[1]); + if (span[2]) end = Math.min(Number(span[2]), size - 1); + } + if (start >= size) + return new Response(null, { + status: 416, + headers: { "Content-Range": `bytes */${size}` }, + }); + const length = end - start + 1; + const openEnded = span !== null && !span[2]; + served.set(name, (served.get(name) ?? 0) + (openEnded ? 0 : length)); + return new Response( + openEnded && !fast + ? slowStream(file, start) + : file.slice(start, end + 1), + { + status: range ? 206 : 200, + headers: { + "Accept-Ranges": "bytes", + "Content-Length": String(length), + "Content-Range": `bytes ${start}-${end}/${size}`, + }, + }, + ); + }, + }); + origin = `http://127.0.0.1:${server.port}`; +}, 120_000); + +afterAll(async () => { + stopProbeWorkers(); + await server.stop(true); + await rm(dir, { recursive: true, force: true }); +}); + +describe("the length of a Matroska file with no Duration and no Cues", () => { + test.each(["piped.mkv", "recorder.webm"])( + "%s is read from its tail, locally and over HTTP ranges", + async (name) => { + const expected = await lastPacketTime(join(dir, name)); + expect(expected).toBeGreaterThan(59); + const local = await secondsFromTail(join(dir, name)); + expect(Math.abs((local ?? 0) - expected)).toBeLessThan(1); + served.delete(name); + const remote = await secondsFromTail(`${origin}/${name}`); + expect(Math.abs((remote ?? 0) - expected)).toBeLessThan(1); + }, + 60_000, + ); + + test("a file hundreds of megabytes long is measured from two small reads", async () => { + served.delete("sparse.mkv"); + const started = performance.now(); + const end = await secondsFromTail(`${origin}/sparse.mkv`); + expect(performance.now() - started).toBeLessThan(2_000); + expect( + Math.abs((end ?? 0) - (await lastPacketTime(join(dir, "piped.mkv")))), + ).toBeLessThan(1); + expect(served.get("sparse.mkv") ?? 0).toBeLessThanOrEqual( + 64 * 1024 + 4 * 1024 * 1024, + ); + }, 60_000); + + test("the probe reports it through the probe workers and holds encodes to it", async () => { + expect(Bun.file(join(dir, "looped.mkv")).size).toBeGreaterThan( + 150 * 1024 * 1024, + ); + const started = performance.now(); + // At this rate a full read of the file takes far longer than a probe. + const metadata = await probeVideo(`${origin}/looped.mkv`); + expect(performance.now() - started).toBeLessThan(15_000); + const expected = await lastPacketTime(join(dir, "looped.mkv")); + expect(expected).toBeGreaterThan(LOOPS * 6 - 2); + expect(Math.abs(metadata.duration - expected)).toBeLessThan(1); + expect(Math.abs((metadata.timestampSpan ?? 0) - expected)).toBeLessThan(1); + expect(encodeLengthLimit(metadata)).toBeGreaterThanOrEqual(expected); + }, 60_000); + + test("a finished file whose last cluster is larger than the tail read is measured, not refused", async () => { + const path = join(dir, "one-cluster.mkv"); + expect(Bun.file(path).size).toBeGreaterThan(40 * 1024 * 1024); + expect(await matroskaEndFromTail(path)).toBeNull(); + const expected = await lastPacketTime(path); + for (const source of [path, `${origin}/one-cluster-fast.mkv`]) { + const metadata = await probeVideo(source); + expect(Math.abs(metadata.duration - expected)).toBeLessThan(1); + expect(encodeLengthLimit(metadata)).toBeGreaterThanOrEqual(expected); + } + }, 120_000); + + test("a short browser recording keeps exactly the length it had before", async () => { + const path = join(dir, "browser.webm"); + const before = await fullReadLength(path); + for (const source of [path, `${origin}/browser-fast.webm`]) { + const metadata = await probeVideo(source); + expect(metadata.duration).toBe(before); + expect(metadata.timestampSpan).toBe(before); + } + }, 60_000); + + test("a preallocated recording is measured from its last block before the zero fill", async () => { + const expected = await lastPacketTime(join(dir, "piped.mkv")); + const end = await matroskaEndFromTail(join(dir, "preallocated.mkv")); + expect(end).toMatchObject({ padded: true }); + for (const source of [ + join(dir, "preallocated.mkv"), + `${origin}/preallocated.mkv`, + ]) { + const started = performance.now(); + const metadata = await probeVideo(source); + expect(performance.now() - started).toBeLessThan(10_000); + expect(Math.abs(metadata.duration - expected)).toBeLessThan(1); + expect(encodeLengthLimit(metadata)).toBeGreaterThanOrEqual(expected); + } + }, 60_000); + + test("a file whose tail is all zero bytes is refused as unfinished, not timed out", async () => { + const started = performance.now(); + const error = await probeVideo(`${origin}/no-tail.mkv`).then( + () => null, + (caught: unknown) => caught, + ); + expect((error as Error)?.message).toBe(UNFINISHED_RECORDING_MESSAGE); + expect(performance.now() - started).toBeLessThan(5_000); + }, 60_000); +}); diff --git a/apps/media-server/src/__tests__/lib/media-address-guard.test.ts b/apps/media-server/src/__tests__/lib/media-address-guard.test.ts index d9acff1a011..f7b1f8bf24c 100644 --- a/apps/media-server/src/__tests__/lib/media-address-guard.test.ts +++ b/apps/media-server/src/__tests__/lib/media-address-guard.test.ts @@ -80,6 +80,13 @@ describe("classifyHostLiteral", () => { ["[::ffff:a9fe:a9fe]", "reserved"], ["[64:ff9b::a9fe:a9fe]", "reserved"], ["[2002:a9fe:a9fe::1]", "reserved"], + ["168.63.129.16", "reserved"], + ["168.63.129.17", "public"], + ["[::ffff:168.63.129.16]", "reserved"], + ["[::ffff:0:a9fe:a9fe]", "reserved"], + ["[::ffff:0:10.0.0.1]", "private"], + ["[::ffff:0:127.0.0.1]", "private"], + ["[::ffff:0:8.8.8.8]", "public"], ] as const)("%s is %s", (host, expected) => { expect(classifyHostLiteral(host)).toBe(expected); }); diff --git a/apps/media-server/src/__tests__/lib/media-audio.integration.test.ts b/apps/media-server/src/__tests__/lib/media-audio.integration.test.ts index 88f0163106f..af5fbbac17e 100644 --- a/apps/media-server/src/__tests__/lib/media-audio.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/media-audio.integration.test.ts @@ -187,15 +187,13 @@ describe("mediaAudio integration tests", () => { } }); - test("still rejects audio-only inputs without leaking an operation", async () => { + test("reads audio-only inputs without leaking an operation", async () => { const beforeCount = getActiveProcessCount(); const dirPath = await mkdtemp(join(tmpdir(), "cap-audio-only-")); const outputPath = join(dirPath, "audio.mp3"); try { await Bun.write(outputPath, await extractAudio(TEST_VIDEO_WITH_AUDIO)); - await expect( - checkHasAudioTrack(`file://${outputPath}`), - ).rejects.toThrow("No video stream found"); + expect(await checkHasAudioTrack(`file://${outputPath}`)).toBe(true); await waitForAudioOperations(beforeCount); } finally { await rm(dirPath, { recursive: true, force: true }); diff --git a/apps/media-server/src/__tests__/lib/media-edit-stderr.test.ts b/apps/media-server/src/__tests__/lib/media-edit-stderr.test.ts new file mode 100644 index 00000000000..d81c6f25552 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/media-edit-stderr.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, test } from "bun:test"; +import { + lineReader, + readStreamTail, + STDERR_TAIL_BYTES, +} from "../../lib/media-edit"; + +const encoder = new TextEncoder(); + +function streamOf(chunks: Iterable) { + const iterator = chunks[Symbol.iterator](); + return new ReadableStream({ + pull(controller) { + const next = iterator.next(); + if (next.done) controller.close(); + else controller.enqueue(next.value); + }, + }); +} + +function* lines(count: number) { + for (let index = 0; index < count; index += 1) { + yield encoder.encode(`[h264 @ 0x1] error while decoding MB ${index}\n`); + } +} + +describe("readStreamTail", () => { + test("returns short output whole", async () => { + const text = await readStreamTail( + streamOf([encoder.encode("frame=1\n"), encoder.encode("done\n")]), + ); + expect(text).toBe("frame=1\ndone\n"); + }); + + test("returns an empty string when there is no stream", async () => { + expect(await readStreamTail(null)).toBe(""); + }); + + test("keeps only the end of a long output and reads it all", async () => { + const count = 400_000; + let seen = 0; + const text = await readStreamTail(streamOf(lines(count)), (part) => { + seen += part.length; + }); + expect(encoder.encode(text).length).toBeLessThanOrEqual(STDERR_TAIL_BYTES); + expect(text.length).toBeGreaterThan(STDERR_TAIL_BYTES / 2); + expect(text.endsWith(`error while decoding MB ${count - 1}\n`)).toBe(true); + expect(seen).toBeGreaterThan(count * 30); + }); + + test("keeps the end of a single chunk larger than the limit", async () => { + const text = await readStreamTail( + streamOf([encoder.encode(`${"a".repeat(5000)}END`)]), + undefined, + 100, + ); + expect(text).toBe(`${"a".repeat(97)}END`); + }); + + test("keeps exactly the last bytes across many small chunks", async () => { + const parts = Array.from({ length: 50 }, (_, index) => + encoder.encode(String(index % 10)), + ); + const text = await readStreamTail(streamOf(parts), undefined, 7); + expect(text).toBe("3456789"); + }); + + test("passes every chunk to the listener even past the limit", async () => { + const heard: string[] = []; + await readStreamTail( + streamOf([ + encoder.encode("one "), + encoder.encode("two "), + encoder.encode("three"), + ]), + (part) => heard.push(part), + 4, + ); + expect(heard.join("")).toBe("one two three"); + }); + + test("keeps the last bytes when two chunks together pass the limit", async () => { + const first = encoder.encode("a".repeat(100)); + const second = encoder.encode("b".repeat(99)); + const text = await readStreamTail( + streamOf([first, second]), + undefined, + 100, + ); + expect(text).toBe(`a${"b".repeat(99)}`); + }); +}); + +describe("lineReader", () => { + test("joins lines that arrive in pieces and ends the last one", () => { + const seen: string[] = []; + const lines = lineReader((line) => seen.push(line)); + lines.push("Error opening "); + lines.push("input\nframe=1\r\nlast"); + expect(seen).toEqual(["Error opening input", "frame=1"]); + lines.end(); + expect(seen).toEqual(["Error opening input", "frame=1", "last"]); + }); + + test("an early error line survives far more output than is kept", async () => { + const seen: string[] = []; + const lines = lineReader((line) => { + if (line.includes("Error")) seen.push(line); + }); + function* output() { + yield encoder.encode("Error while opening encoder\n"); + for (let index = 0; index < 20_000; index += 1) { + yield encoder.encode(`frame=${index} fps=30 bitrate=1000kbits/s\n`); + } + } + const text = await readStreamTail(streamOf(output()), lines.push); + lines.end(); + expect(text.includes("Error while opening encoder")).toBe(false); + expect(seen).toEqual(["Error while opening encoder"]); + }); + + test("cuts off a line that never ends", () => { + const seen: string[] = []; + const lines = lineReader((line) => seen.push(line)); + for (let index = 0; index < 100; index += 1) lines.push("x".repeat(1000)); + lines.end(); + expect(seen).toHaveLength(1); + expect(seen[0]?.length).toBeLessThanOrEqual(4096); + }); +}); diff --git a/apps/media-server/src/__tests__/lib/media-input-policy.test.ts b/apps/media-server/src/__tests__/lib/media-input-policy.test.ts index ab3b19f59b5..64042a4ce99 100644 --- a/apps/media-server/src/__tests__/lib/media-input-policy.test.ts +++ b/apps/media-server/src/__tests__/lib/media-input-policy.test.ts @@ -4,9 +4,13 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { createJob, updateJob } from "../../lib/job-manager"; import { + ALLOWED_DEMUXERS, + allowedDemuxersIn, assertMediaFileContent, + DECODER_WHITELIST, getSafeDemuxerWhitelist, INPUT_POLICY_VIOLATION, + isAllowedVideoCodec, isInputPolicyViolationMessage, isTrustedStreamingUrl, looksLikeTextContainer, @@ -59,6 +63,25 @@ describe("demuxer whitelist", () => { ]); }); + test("the allowed list is fixed and a build can only narrow it", () => { + const listing = [ + " D matroska,webm Matroska / WebM", + " D mov,mp4,m4a,3gp,3g2,mj2 QuickTime / MOV", + " D mpegts MPEG-TS", + " D bink Bink", + " D xmv Microsoft XMV", + ].join("\n"); + expect(allowedDemuxersIn(listing)).toEqual([ + "mov,mp4,m4a,3gp,3g2,mj2", + "matroska,webm", + "mpegts", + ]); + expect(allowedDemuxersIn("")).toEqual([...ALLOWED_DEMUXERS]); + const names = new Set(WHITELIST.split(",")); + for (const name of ["bink", "xmv", "nut", "smjpeg"]) + expect(names.has(name)).toBe(false); + }); + test("the installed FFmpeg keeps common containers and excludes playlists", () => { const names = new Set(WHITELIST.split(",")); for (const name of ["mov", "mp4", "matroska", "webm", "mpegts", "mp3"]) @@ -68,6 +91,96 @@ describe("demuxer whitelist", () => { }); }); +describe("decoder whitelist", () => { + test("accepts the codecs Cap and Cap Infra take and nothing else", () => { + for (const codec of [ + "h264", + "hevc", + "vp8", + "vp9", + "av1", + "prores", + "mpeg4", + "mjpeg", + "mpeg1video", + "mpeg2video", + "wmv1", + "wmv2", + "wmv3", + "vc1", + "msmpeg4v1", + "msmpeg4v2", + "msmpeg4v3", + "flv1", + "h263", + "theora", + ]) + expect(isAllowedVideoCodec(codec)).toBe(true); + for (const codec of ["dvvideo", "cinepak", "unknown"]) + expect(isAllowedVideoCodec(codec)).toBe(false); + const decoders = new Set(DECODER_WHITELIST.split(",")); + for (const name of [ + "aac", + "mp3float", + "opus", + "flac", + "dca", + "pcm_s16le", + "mov_text", + "h264_cuvid", + "libdav1d", + "mpeg2video", + "msmpeg4", + "flv", + "mp2", + "wmav2", + "amrnb", + "adpcm_ms", + ]) + expect(decoders.has(name)).toBe(true); + for (const name of ["cinepak", "rawvideo", "dvvideo", "indeo5"]) + expect(decoders.has(name)).toBe(false); + }); + + test("FFmpeg refuses to decode a stream outside the list", async () => { + const dir = await mkdtemp(join(tmpdir(), "codec-whitelist-")); + try { + const path = join(dir, "cinepak.avi"); + const make = spawn({ + cmd: [ + "ffmpeg", + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc=d=0.2:s=64x64", + "-c:v", + "cinepak", + "-y", + path, + ], + stdout: "ignore", + stderr: "pipe", + }); + expect(await make.exited).toBe(0); + const decode = spawn({ + cmd: ["ffmpeg", "-v", "error", "-i", path, "-f", "null", "-"], + stdout: "ignore", + stderr: "pipe", + }); + const [code, stderr] = await Promise.all([ + decode.exited, + new Response(decode.stderr).text(), + ]); + expect(code).not.toBe(0); + expect(stderr).toContain("not on whitelist"); + } finally { + await rm(dir, { recursive: true, force: true }); + } + }); +}); + describe("hardenMediaCommand", () => { test("local, remote and pipe inputs get matching protocol and format whitelists", () => { const cmd = hardenMediaCommand([ @@ -93,6 +206,8 @@ describe("hardenMediaCommand", () => { "file", "-format_whitelist", WHITELIST, + "-codec_whitelist", + DECODER_WHITELIST, ]); expect(inputOptions(cmd, "https://bucket.example/b.webm?sig=1")).toEqual([ "-ss", @@ -101,12 +216,16 @@ describe("hardenMediaCommand", () => { "http,https,tcp,tls", "-format_whitelist", WHITELIST, + "-codec_whitelist", + DECODER_WHITELIST, ]); expect(inputOptions(cmd, "pipe:0")).toEqual([ "-f", "s16le", "-protocol_whitelist", "pipe", + "-codec_whitelist", + DECODER_WHITELIST, ]); expect(cmd.at(-1)).toBe("/tmp/out.mp4"); }); @@ -122,6 +241,8 @@ describe("hardenMediaCommand", () => { "file,concat", "-f", "mov", + "-codec_whitelist", + "h264", "-i", "concat:/tmp/init.mp4|/tmp/seg.m4s", "-f", @@ -131,6 +252,25 @@ describe("hardenMediaCommand", () => { expect(hardenMediaCommand(original)).toEqual(original); }); + test("every other input is held to the allowed decoders", () => { + const cmd = hardenMediaCommand([ + "ffmpeg", + "-f", + "mov", + "-i", + "/tmp/x.mov", + "o.mp4", + ]); + expect(inputOptions(cmd, "/tmp/x.mov")).toEqual([ + "-f", + "mov", + "-protocol_whitelist", + "file", + "-codec_whitelist", + DECODER_WHITELIST, + ]); + }); + test("unknown schemes are limited to local files", () => { const cmd = hardenMediaCommand([ "ffmpeg", @@ -143,6 +283,8 @@ describe("hardenMediaCommand", () => { "file", "-format_whitelist", WHITELIST, + "-codec_whitelist", + DECODER_WHITELIST, ]); }); @@ -157,6 +299,8 @@ describe("hardenMediaCommand", () => { expect(inputOptions(cmd, "/tmp/stream-x/master.m3u8")).toEqual([ "-protocol_whitelist", "file", + "-codec_whitelist", + DECODER_WHITELIST, ]); }); @@ -174,6 +318,8 @@ describe("hardenMediaCommand", () => { "http,https,tcp,tls", "-format_whitelist", WHITELIST, + "-codec_whitelist", + DECODER_WHITELIST, "-v", "error", "https://bucket.example/x.mp4", @@ -184,6 +330,8 @@ describe("hardenMediaCommand", () => { "ffprobe", "-protocol_whitelist", "file", + "-codec_whitelist", + DECODER_WHITELIST, "-f", "mov", "-i", diff --git a/apps/media-server/src/__tests__/lib/media-probe-hardening.test.ts b/apps/media-server/src/__tests__/lib/media-probe-hardening.test.ts new file mode 100644 index 00000000000..b5320047411 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/media-probe-hardening.test.ts @@ -0,0 +1,257 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { chmod, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { maxEncodeSeconds } from "../../lib/media-common"; +import { UNSUPPORTED_FORMAT_MESSAGE } from "../../lib/media-input-policy"; +import { + isUnreadableMediaInput, + probeVideo, + probeVideoFile, + UnreadableMediaInputError, +} from "../../lib/media-probe"; +import { stopProbeWorkers } from "../../lib/media-probe-pool"; + +let dir = ""; +let server: ReturnType; +let origin = ""; + +async function ffmpeg(args: string[]): Promise { + const proc = Bun.spawn(["ffmpeg", "-v", "error", "-y", ...args], { + stdout: "ignore", + stderr: "pipe", + }); + const [code, stderr] = await Promise.all([ + proc.exited, + new Response(proc.stderr).text(), + ]); + if (code !== 0) throw new Error(stderr); +} + +// Rewrites the movie, track and media durations of an MP4 to one second, +// leaving its samples and edit list where they are. +function declareOneSecond(bytes: Uint8Array): Uint8Array { + const data = Buffer.from(bytes); + const walk = (start: number, end: number) => { + let offset = start; + while (offset + 8 <= end) { + let size = data.readUInt32BE(offset); + const type = data.toString("latin1", offset + 4, offset + 8); + let header = 8; + if (size === 1) { + size = Number(data.readBigUInt64BE(offset + 8)); + header = 16; + } + if (size === 0) size = end - offset; + const body = offset + header; + if (["moov", "trak", "mdia", "edts"].includes(type)) { + walk(body, offset + size); + } else if (data[body] === 0) { + if (type === "mvhd" || type === "mdhd") { + data.writeUInt32BE(data.readUInt32BE(body + 12), body + 16); + } else if (type === "tkhd") { + data.writeUInt32BE(1000, body + 20); + } + } + offset += size; + } + }; + walk(0, data.length); + return data; +} + +beforeAll(async () => { + dir = await mkdtemp(join(tmpdir(), "probe-hardening-")); + const source = ["-f", "lavfi", "-i", "testsrc=d=1:s=128x96:r=10"]; + await ffmpeg([...source, "-c:v", "mpeg4", join(dir, "part2.mp4")]); + await ffmpeg([...source, "-c:v", "cinepak", join(dir, "cinepak.mkv")]); + await ffmpeg([ + ...source, + "-f", + "lavfi", + "-i", + "sine=d=1", + "-c:v", + "mpeg2video", + "-c:a", + "mp2", + "-f", + "mpegts", + join(dir, "broadcast.ts"), + ]); + await ffmpeg(["-f", "lavfi", "-i", "sine=d=1", join(dir, "audio-only.m4a")]); + await ffmpeg([...source, "-c:v", "prores_ks", join(dir, "prores.mov")]); + // A one-frame recording written as a live stream: no Duration, and its + // only block is at timestamp zero. + await ffmpeg([ + "-f", + "lavfi", + "-i", + "testsrc=s=128x96:r=10", + "-frames:v", + "1", + "-c:v", + "libvpx", + "-f", + "webm", + "-live", + "1", + join(dir, "single-frame.webm"), + ]); + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + const name = new URL(request.url).pathname.slice(1); + // FFmpeg's own requests to the "flaky" name fail upstream. + if ( + name.startsWith("flaky-") && + request.headers.get("user-agent")?.startsWith("Lavf") + ) + return new Response("Unavailable", { status: 503 }); + const file = Bun.file(join(dir, name.replace(/^flaky-/, ""))); + const size = file.size; + const range = request.headers.get("range")?.match(/^bytes=(\d+)-(\d*)$/); + const start = range ? Number(range[1]) : 0; + const end = range?.[2] ? Math.min(Number(range[2]), size - 1) : size - 1; + if (start >= size) + return new Response(null, { + status: 416, + headers: { "Content-Range": `bytes */${size}` }, + }); + return new Response(file.slice(start, end + 1), { + status: range ? 206 : 200, + headers: { + "Accept-Ranges": "bytes", + "Content-Length": String(end - start + 1), + "Content-Range": `bytes ${start}-${end}/${size}`, + }, + }); + }, + }); + origin = `http://127.0.0.1:${server.port}`; + // Three frames an hour apart. + await ffmpeg([ + "-f", + "lavfi", + "-i", + "testsrc=s=128x96:r=1", + "-vf", + "setpts=N*3600/TB", + "-frames:v", + "3", + "-c:v", + "libx264", + "-fps_mode", + "passthrough", + "-video_track_timescale", + "1000", + join(dir, "spread.mp4"), + ]); + await writeFile( + join(dir, "declared-short.mp4"), + declareOneSecond(await readFile(join(dir, "spread.mp4"))), + ); +}, 60_000); + +afterAll(async () => { + stopProbeWorkers(); + await server.stop(true); + await rm(dir, { recursive: true, force: true }); +}); + +async function probeError(name: string): Promise { + return await probeVideoFile(join(dir, name)).then( + () => null, + (error: unknown) => error, + ); +} + +describe("probe input checks", () => { + test("a track the parser cannot read is unreadable input, not a server error", async () => { + const error = await probeError("part2.mp4"); + expect(error).toBeInstanceOf(UnreadableMediaInputError); + expect(isUnreadableMediaInput(error)).toBe(true); + }, 30_000); + + test("a video codec outside the allowed list is refused", async () => { + const error = await probeError("cinepak.mkv"); + expect(error).toBeInstanceOf(UnreadableMediaInputError); + expect((error as Error).message).toContain("not supported"); + }, 30_000); + + test("a video track the parser cannot read is an unsupported format, not a missing track", async () => { + const error = await probeError("broadcast.ts"); + expect(error).toBeInstanceOf(UnreadableMediaInputError); + expect((error as Error).message).toBe(UNSUPPORTED_FORMAT_MESSAGE); + }, 30_000); + + test("a file with no video at all still says so", async () => { + const error = await probeError("audio-only.m4a"); + expect((error as Error).message).toBe("No video stream found"); + }, 30_000); + + test("an allowed codec the parser does not name is still accepted", async () => { + const metadata = await probeVideoFile(join(dir, "prores.mov")); + expect(metadata.width).toBe(128); + }, 30_000); +}); + +describe("a codec check that cannot run", () => { + test("an allowed codec the parser does not name probes over HTTP too", async () => { + const metadata = await probeVideo(`${origin}/prores.mov`); + expect(metadata.width).toBe(128); + }, 30_000); + + test("a source that fails FFmpeg's reads leaves the probe retryable", async () => { + const error = await probeVideo(`${origin}/flaky-prores.mov`).then( + () => null, + (caught: unknown) => caught, + ); + expect(error).toBeInstanceOf(Error); + expect(isUnreadableMediaInput(error)).toBe(false); + }, 30_000); + + test("leaves an allowed codec retryable instead of refusing it", async () => { + const savedPath = process.env.PATH; + const real = Bun.which("ffprobe"); + const shim = join(dir, "failing-bin"); + // An ffprobe that cannot read the stream list, as when the source + // fails to fetch. + await Bun.write( + join(shim, "ffprobe"), + `#!/bin/sh\ncase "$*" in *codec_type,codec_name*) echo "Server returned 5XX Server Error reply" >&2; exit 1 ;; esac\nexec "${real}" "$@"\n`, + ); + await chmod(join(shim, "ffprobe"), 0o755); + process.env.PATH = `${shim}:${savedPath}`; + try { + const error = await probeError("prores.mov"); + expect(error).toBeInstanceOf(Error); + expect(isUnreadableMediaInput(error)).toBe(false); + } finally { + process.env.PATH = savedPath; + } + }, 30_000); +}); + +describe("probed duration", () => { + test("a one-frame recording with no Duration is measured, not refused", async () => { + const metadata = await probeVideoFile(join(dir, "single-frame.webm")); + expect(metadata.duration).toBeGreaterThan(0); + expect(metadata.duration).toBeLessThan(1); + }, 30_000); + + test("a header that declares less than the timestamps span reports the span", async () => { + const honest = await probeVideoFile(join(dir, "spread.mp4")); + const lying = await probeVideoFile(join(dir, "declared-short.mp4")); + expect(honest.duration).toBeGreaterThanOrEqual(7200); + expect(lying.duration).toBeGreaterThanOrEqual(7200); + }, 30_000); + + test("the encode length limit is the probed length with a margin", () => { + expect(maxEncodeSeconds(10)).toBe(12); + expect(maxEncodeSeconds(600)).toBe(612); + expect(maxEncodeSeconds(0)).toBeNull(); + expect(maxEncodeSeconds(Number.NaN)).toBeNull(); + }); +}); diff --git a/apps/media-server/src/__tests__/lib/media-probe-sample-counts.test.ts b/apps/media-server/src/__tests__/lib/media-probe-sample-counts.test.ts new file mode 100644 index 00000000000..b04111698d3 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/media-probe-sample-counts.test.ts @@ -0,0 +1,217 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { probeTrackSpans, probeVideoFile } from "../../lib/media-probe"; + +const FIXTURES_DIR = join(import.meta.dir, "..", "fixtures"); +const PLAIN = readFileSync(join(FIXTURES_DIR, "test-bframes-64.mp4")); +const FRAGMENTED = readFileSync( + join(FIXTURES_DIR, "test-bframes-64-fragmented.mp4"), +); + +// The limits live in a patch to the parser, which is applied when the media +// server is installed from its own lockfile, as the image is. An install from +// the repository root resolves a different, unpatched copy. +const parserVersion = ( + createRequire(import.meta.url)("mediabunny/package.json") as { + version: string; + } +).version; +const patched = parserVersion === "1.45.2"; + +const dir = mkdtempSync(join(tmpdir(), "sample-counts-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +const MAX = 0xffffffff; +const MB = 1024 * 1024; + +// Overwrites one 32-bit field, counted from the first byte of a box's type. +function withField( + source: Buffer, + box: string, + offset: number, + value: number, + name: string, +) { + const at = source.indexOf(box, 0, "latin1"); + expect(at).toBeGreaterThan(0); + const data = Buffer.from(source); + data.writeUInt32BE(value, at + offset); + const path = join(dir, name); + writeFileSync(path, data); + return path; +} + +async function probeWithin(path: string) { + const rssBefore = process.memoryUsage().rss; + const started = performance.now(); + let metadata: Awaited> | null = null; + let error: unknown = null; + try { + metadata = await probeVideoFile(path); + } catch (caught) { + error = caught; + } + return { + metadata, + error, + ms: performance.now() - started, + grownMb: (process.memoryUsage().rss - rssBefore) / MB, + }; +} + +describe.skipIf(!patched)("sample counts a file only declares", () => { + test("the unchanged files probe normally", async () => { + const plain = await probeVideoFile( + join(FIXTURES_DIR, "test-bframes-64.mp4"), + ); + expect(plain.width).toBe(64); + expect(plain.height).toBe(64); + expect(plain.duration).toBeCloseTo(1.1, 1); + const fragmented = await probeVideoFile( + join(FIXTURES_DIR, "test-bframes-64-fragmented.mp4"), + ); + expect(fragmented.width).toBe(64); + }); + + test("a time table claiming three million samples costs no memory", async () => { + const result = await probeWithin( + withField(PLAIN, "stts", 12, 3_000_000, "stts-3m.mp4"), + ); + expect(result.error).toBeNull(); + expect(result.metadata?.width).toBe(64); + expect(result.metadata?.duration).toBeCloseTo(1.1, 1); + expect(result.grownMb).toBeLessThan(80); + expect(result.ms).toBeLessThan(2000); + }); + + for (const [box, offset, label] of [ + ["stts", 12, "time table"], + ["ctts", 12, "offset table"], + ["stsc", 16, "chunk table"], + ] as const) { + test(`a ${label} claiming the largest possible count is read as the file it is`, async () => { + const result = await probeWithin( + withField(PLAIN, box, offset, MAX, `${box}-max.mp4`), + ); + expect(result.error).toBeNull(); + expect(result.metadata?.width).toBe(64); + expect(result.metadata?.duration).toBeCloseTo(1.1, 1); + expect(result.grownMb).toBeLessThan(80); + expect(result.ms).toBeLessThan(2000); + }); + } + + test("a track with one size for every sample is held to what its chunks can hold", async () => { + const shared = Buffer.from(PLAIN); + shared.writeUInt32BE(100, shared.indexOf("stsz", 0, "latin1") + 8); + const honest = join(dir, "shared-size.mp4"); + writeFileSync(honest, shared); + expect((await probeVideoFile(honest)).duration).toBeCloseTo(1.1, 1); + + const inflated = Buffer.from(shared); + inflated.writeUInt32BE(1, inflated.indexOf("stsz", 0, "latin1") + 8); + for (const [box, offset] of [ + ["stsc", 16], + ["stts", 12], + ["ctts", 12], + ] as const) { + inflated.writeUInt32BE(MAX, inflated.indexOf(box, 0, "latin1") + offset); + } + const path = join(dir, "shared-size-max.mp4"); + writeFileSync(path, inflated); + const result = await probeWithin(path); + expect(result.error).toBeNull(); + expect(result.metadata?.width).toBe(64); + expect(result.grownMb).toBeLessThan(80); + expect(result.ms).toBeLessThan(2000); + }); + + test("chunks of a shared-size track share what the file can hold", async () => { + const source = readFileSync(join(FIXTURES_DIR, "test-bframes-64-pcm.mov")); + const honest = await probeTrackSpans( + join(FIXTURES_DIR, "test-bframes-64-pcm.mov"), + ); + expect(honest.audio).toBeCloseTo(1, 1); + + // The audio track's tables come last in the file. Both chunk runs and + // the time table claim the largest possible count. + const data = Buffer.from(source); + const chunkTable = data.lastIndexOf("stsc", undefined, "latin1"); + data.writeUInt32BE(MAX, chunkTable + 16); + data.writeUInt32BE(MAX, chunkTable + 28); + data.writeUInt32BE(MAX, data.lastIndexOf("stts", undefined, "latin1") + 12); + const path = join(dir, "pcm-max.mov"); + writeFileSync(path, data); + + const started = performance.now(); + const spans = await probeTrackSpans(path); + expect(performance.now() - started).toBeLessThan(2000); + // 16-bit mono at 8 kHz: the whole file could not hold two seconds. + expect(spans.audio).not.toBeNull(); + expect(spans.audio as number).toBeLessThan(2); + expect(spans.video).toBeCloseTo(1, 1); + }); + + // The audio track's chunk table has two runs. Moving the second run's first + // chunk far out claims that many chunks for the first run, while the file + // only lists a handful. + function withChunkGap(gap: number, name: string) { + const data = Buffer.from( + readFileSync(join(FIXTURES_DIR, "test-bframes-64-pcm.mov")), + ); + const chunkTable = data.lastIndexOf("stsc", undefined, "latin1"); + expect(data.readUInt32BE(chunkTable + 8)).toBe(2); + data.writeUInt32BE(1, chunkTable + 12); + data.writeUInt32BE(1, chunkTable + 16); + data.writeUInt32BE(gap + 1, chunkTable + 24); + data.writeUInt32BE(1, chunkTable + 28); + const path = join(dir, name); + writeFileSync(path, data); + return path; + } + + // The parser walks a sound track's chunk runs one chunk at a time without + // yielding, so a timeout cannot stop it. The small gap is probed first: it + // fails in well under a second when the runs are not bounded, which stops + // the test before the largest gap could hang the image build. + test("a sound track's chunk runs end at the last chunk the file lists, even at the largest possible gap", async () => { + const started = performance.now(); + const small = await probeTrackSpans( + withChunkGap(20_000_000, "pcm-chunk-gap-20m.mov"), + ); + expect(performance.now() - started).toBeLessThan(200); + expect(small.video).toBeCloseTo(1, 1); + expect(small.audio as number).toBeLessThan(2); + + const path = withChunkGap(MAX - 1, "pcm-chunk-gap-max.mov"); + const result = await probeWithin(path); + const spans = await probeTrackSpans(path); + expect(result.error).toBeNull(); + expect(result.metadata?.width).toBe(64); + expect(result.metadata?.duration).toBeCloseTo(1.1, 1); + expect(result.grownMb).toBeLessThan(80); + expect(result.ms).toBeLessThan(500); + expect(spans.audio as number).toBeLessThan(2); + }); + + test("a fragment run claiming more samples than a fragment can hold is refused at once", async () => { + const at = FRAGMENTED.indexOf("trun", 0, "latin1"); + expect(at).toBeGreaterThan(0); + const data = Buffer.from(FRAGMENTED); + // Flags: a data offset and nothing per sample, so the count is the only + // thing that says how many samples there are. + data.writeUInt8(0, at + 5); + data.writeUInt8(0, at + 6); + data.writeUInt8(1, at + 7); + data.writeUInt32BE(3_000_000, at + 8); + const path = join(dir, "trun-3m.mp4"); + writeFileSync(path, data); + const result = await probeWithin(path); + expect(result.grownMb).toBeLessThan(80); + expect(result.ms).toBeLessThan(2000); + expect(result.metadata === null || result.metadata.width === 64).toBe(true); + }); +}); diff --git a/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts b/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts index 3fa3a995c8b..a2e18ff9e97 100644 --- a/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts +++ b/apps/media-server/src/__tests__/lib/nvenc-encoder.test.ts @@ -386,6 +386,27 @@ describe("nvenc fallback", () => { }); describe("process video args", () => { + test("a process encode is held to the length it is given, and only then", () => { + const source = { + duration: 10, + width: 1920, + height: 1080, + fps: 30, + videoCodec: "h264", + audioCodec: "aac", + audioChannels: 2, + sampleRate: 48000, + bitrate: 1_000_000, + fileSize: 1_000_000, + }; + const capped = buildProcessVideoArgs( + processArgs(source, { maxOutputSeconds: 12 }), + ); + expect(capped[capped.indexOf("-t") + 1]).toBe("12"); + expect(capped.indexOf("-t")).toBeGreaterThan(capped.lastIndexOf("-i")); + expect(buildProcessVideoArgs(processArgs(source))).not.toContain("-t"); + }); + test("keeps the default libx264 args unchanged", () => { const source = metadata(3024, 1964, "h264"); expect(buildProcessVideoArgs(processArgs(source))).toEqual([ diff --git a/apps/media-server/src/__tests__/lib/process-hardening.test.ts b/apps/media-server/src/__tests__/lib/process-hardening.test.ts new file mode 100644 index 00000000000..59fe44b208c --- /dev/null +++ b/apps/media-server/src/__tests__/lib/process-hardening.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, test } from "bun:test"; +import { readFile } from "node:fs/promises"; +import { hideServerFromChildren } from "../../lib/process-hardening"; + +describe("hideServerFromChildren", () => { + test.skipIf(process.platform !== "linux")( + "a child of the same user cannot read the server's environment", + async () => { + // The environment the process started with, readable by any + // process of the same user until the server hides it. + const before = await readFile(`/proc/${process.pid}/environ`, "utf8"); + expect(before).toContain("PATH="); + + expect(hideServerFromChildren()).toBe(true); + + const child = Bun.spawn( + ["sh", "-c", `cat /proc/${process.pid}/environ`], + { stdout: "pipe", stderr: "pipe", env: { PATH: process.env.PATH } }, + ); + const [code, stdout] = await Promise.all([ + child.exited, + new Response(child.stdout).text(), + ]); + expect(code).not.toBe(0); + expect(stdout).not.toContain("PATH="); + + // The server still reads its own entries and runs children. + const status = await readFile("/proc/self/status", "utf8"); + expect(status).toContain("Name:"); + const echo = Bun.spawn(["sh", "-c", "echo ok"], { stdout: "pipe" }); + expect((await new Response(echo.stdout).text()).trim()).toBe("ok"); + }, + ); + + test.skipIf(process.platform === "linux")( + "does nothing where the platform has no such switch", + () => { + expect(hideServerFromChildren()).toBe(false); + }, + ); +}); diff --git a/apps/media-server/src/__tests__/routes/audio.test.ts b/apps/media-server/src/__tests__/routes/audio.test.ts index de070066fb4..687362e7f55 100644 --- a/apps/media-server/src/__tests__/routes/audio.test.ts +++ b/apps/media-server/src/__tests__/routes/audio.test.ts @@ -153,6 +153,50 @@ describe("POST /audio/extract", () => { expect(data.code).toBe("NO_AUDIO_TRACK"); }); + for (const path of ["/audio/extract", "/audio/check"]) { + test(`${path} answers 422 for a source with no media stream`, async () => { + mock.module("../../lib/media-audio", () => ({ + ...mediaAudio, + checkHasAudioTrack: async () => { + throw new Error("No media stream found"); + }, + })); + const { default: appWithMock } = await import("../../app"); + + const response = await appWithMock.fetch( + audioPostRequest(path, { + videoUrl: "https://example.com/audio-only.m4a", + }), + ); + + expect(response.status).toBe(422); + const data = await response.json(); + expect(data.code).toBe("NO_MEDIA_STREAM"); + expect(data.details).toBe("No media stream found"); + expect(data.error).toBe( + path === "/audio/extract" + ? "Failed to extract audio" + : "Failed to check audio track", + ); + }); + + test(`${path} still answers 500 for other failures`, async () => { + mock.module("../../lib/media-audio", () => ({ + ...mediaAudio, + checkHasAudioTrack: async () => { + throw new Error("FFprobe exited with code 1: broken"); + }, + })); + const { default: appWithMock } = await import("../../app"); + + const response = await appWithMock.fetch( + audioPostRequest(path, { videoUrl: "https://example.com/video.mp4" }), + ); + + expect(response.status).toBe(500); + }); + } + test("returns audio data when extraction succeeds", async () => { const mockAudioData = new Uint8Array([0x00, 0x00, 0x00, 0x1c, 0x66, 0x74]); diff --git a/apps/media-server/src/__tests__/routes/health.test.ts b/apps/media-server/src/__tests__/routes/health.test.ts index 062cd93e868..30732373258 100644 --- a/apps/media-server/src/__tests__/routes/health.test.ts +++ b/apps/media-server/src/__tests__/routes/health.test.ts @@ -1,16 +1,38 @@ -import { describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import app from "../../app"; +const SECRET = "health-secret"; +let saved: string | undefined; + +beforeEach(() => { + saved = process.env.MEDIA_SERVER_WEBHOOK_SECRET; + process.env.MEDIA_SERVER_WEBHOOK_SECRET = SECRET; +}); + +afterEach(() => { + if (saved === undefined) delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; + else process.env.MEDIA_SERVER_WEBHOOK_SECRET = saved; +}); + +function get(path: string, headers?: Record) { + return app.fetch(new Request(`http://localhost${path}`, { headers })); +} + describe("GET /health", () => { - test("returns status ok with media engine info", async () => { - const response = await app.fetch(new Request("http://localhost/health")); + test("returns status ok with media engine info to a caller with the secret", async () => { + const response = await get("/health", { "x-media-server-secret": SECRET }); expect(response.status).toBe(200); const data = await response.json(); - expect(data.status).toBe("ok"); + // "degraded" on a test machine short of memory. + expect(["ok", "degraded"]).toContain(data.status); expect(data.mediaEngine).toBeDefined(); expect(data.mediaEngine.available).toBe(true); - expect(data["ff" + "mpeg"]).toEqual({ available: true }); + expect(typeof data.mediaEngine.version).toBe("string"); + expect(data["ff" + "mpeg"].available).toBe(true); + expect(typeof data["ff" + "mpeg"].version).toBe("string"); + expect(data.videoEncoder).toBeDefined(); + expect(data.runtime).toBeDefined(); expect(typeof data.system.containerMemoryUsageMB).toBe("number"); expect(typeof data.system.containerMemoryLimitMB).toBe("number"); expect(typeof data.system.memoryPressure).toBe("number"); @@ -22,26 +44,34 @@ describe("GET /health", () => { expect(typeof data.system.processUptimeSeconds).toBe("number"); }); - test("names the engine version only to callers with the secret", async () => { - const saved = process.env.MEDIA_SERVER_WEBHOOK_SECRET; - process.env.MEDIA_SERVER_WEBHOOK_SECRET = "health-secret"; - try { - const read = async (headers?: Record) => - ( - await app.fetch(new Request("http://localhost/health", { headers })) - ).json(); - const anonymous = await read(); - expect(anonymous.status).toBe("ok"); - expect(anonymous.mediaEngine.version).toBeUndefined(); - expect(anonymous["ff" + "mpeg"].version).toBeUndefined(); - const wrong = await read({ "x-media-server-secret": "health-secreT" }); - expect(wrong.mediaEngine.version).toBeUndefined(); - const trusted = await read({ "x-media-server-secret": "health-secret" }); - expect(typeof trusted.mediaEngine.version).toBe("string"); - expect(typeof trusted["ff" + "mpeg"].version).toBe("string"); - } finally { - if (saved === undefined) delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; - else process.env.MEDIA_SERVER_WEBHOOK_SECRET = saved; + test("tells a caller without the secret only whether the server is up", async () => { + for (const headers of [ + undefined, + { "x-media-server-secret": "health-secreT" }, + ]) { + const response = await get("/health", headers); + expect(response.status).toBe(200); + const body = await response.json(); + expect(Object.keys(body)).toEqual(["status"]); + expect(["ok", "degraded"]).toContain(body.status); } }); + + test("answers only the status when no secret is configured", async () => { + delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; + const response = await get("/health"); + expect(response.status).toBe(200); + expect(Object.keys(await response.json())).toEqual(["status"]); + }); +}); + +describe("GET /", () => { + test("lists endpoints only to a caller with the secret", async () => { + expect(await (await get("/")).json()).toEqual({ status: "ok" }); + const trusted = await ( + await get("/", { "x-media-server-secret": SECRET }) + ).json(); + expect(trusted.name).toBe("@cap/media-server"); + expect(trusted.endpoints).toContain("/health"); + }); }); diff --git a/apps/media-server/src/__tests__/routes/peer-jobs.test.ts b/apps/media-server/src/__tests__/routes/peer-jobs.test.ts index 546c1b9530e..172a4c93015 100644 --- a/apps/media-server/src/__tests__/routes/peer-jobs.test.ts +++ b/apps/media-server/src/__tests__/routes/peer-jobs.test.ts @@ -102,6 +102,51 @@ describe("peer job lookup", () => { expect((await response.json()).code).toBe("NOT_FOUND"); }); + test("answers a retryable 503 when a peer fails instead of answering", async () => { + respond = () => new Response("overloaded", { status: 500 }); + const response = await app.fetch(statusRequest("job-unknown")); + expect(response.status).toBe(503); + expect(response.headers.get("Retry-After")).toBe("5"); + expect((await response.json()).code).toBe("PEER_LOOKUP_FAILED"); + }); + + test("answers a retryable 503 when a peer cannot be reached", async () => { + const unreachable = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch: () => new Response(null), + }); + const port = unreachable.port; + await unreachable.stop(true); + const peers = process.env.MEDIA_SERVER_PEERS; + process.env.MEDIA_SERVER_PEERS = `${peers},http://127.0.0.1:${port}`; + try { + const response = await app.fetch(statusRequest("job-unreachable")); + expect(response.status).toBe(503); + expect(response.headers.get("Retry-After")).toBe("5"); + } finally { + process.env.MEDIA_SERVER_PEERS = peers; + } + }); + + test("still finds the job when another peer fails", async () => { + respond = () => Response.json({ jobId: "job-found", phase: "processing" }); + const peers = process.env.MEDIA_SERVER_PEERS; + const failing = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch: () => new Response("down", { status: 502 }), + }); + process.env.MEDIA_SERVER_PEERS = `${peers},http://127.0.0.1:${failing.port}`; + try { + const response = await app.fetch(statusRequest("job-found")); + expect(response.status).toBe(200); + } finally { + process.env.MEDIA_SERVER_PEERS = peers; + await failing.stop(true); + } + }); + test("does not forward a lookup that already came from a peer", async () => { respond = () => Response.json({ jobId: "job-loop", phase: "processing" }); const response = await app.fetch( @@ -120,10 +165,13 @@ describe("peer job lookup", () => { expect(peerRequests).toHaveLength(0); }); - test("ignores a peer that rejects the shared secret", async () => { - respond = () => new Response("Unauthorized", { status: 401 }); - const response = await app.fetch(statusRequest("job-rejected")); - expect(response.status).toBe(404); + test("does not call a job missing when a peer rejects the shared secret", async () => { + for (const status of [401, 403]) { + respond = () => new Response("Unauthorized", { status }); + const response = await app.fetch(statusRequest("job-rejected")); + expect(response.status).toBe(503); + expect(response.headers.get("Retry-After")).toBe("5"); + } }); test("waits for an owner that delivers its webhook before confirming a cancel", async () => { diff --git a/apps/media-server/src/__tests__/routes/thumbnail-remote.test.ts b/apps/media-server/src/__tests__/routes/thumbnail-remote.test.ts new file mode 100644 index 00000000000..7ae4810e00f --- /dev/null +++ b/apps/media-server/src/__tests__/routes/thumbnail-remote.test.ts @@ -0,0 +1,166 @@ +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + test, +} from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import app from "../../app"; + +const SECRET = "thumbnail-remote-secret"; +const FIXTURE = readFileSync( + join(import.meta.dir, "..", "fixtures", "test-with-audio.mp4"), +); + +function serveFixture(request: Request, body: Buffer): Response { + const match = request.headers.get("range")?.match(/^bytes=(\d+)-(\d*)$/); + if (!match) { + return new Response(new Uint8Array(body), { + headers: { + "Accept-Ranges": "bytes", + "Content-Length": String(body.length), + "Content-Type": "video/mp4", + }, + }); + } + const start = Number(match[1]); + if (start >= body.length) { + return new Response(null, { + status: 416, + headers: { "Content-Range": `bytes */${body.length}` }, + }); + } + const end = match[2] + ? Math.min(Number(match[2]), body.length - 1) + : body.length - 1; + return new Response(new Uint8Array(body.subarray(start, end + 1)), { + status: 206, + headers: { + "Accept-Ranges": "bytes", + "Content-Range": `bytes ${start}-${end}/${body.length}`, + "Content-Type": "video/mp4", + }, + }); +} + +const previousSecret = process.env.MEDIA_SERVER_WEBHOOK_SECRET; +const previousInternalHosts = process.env.MEDIA_SERVER_INTERNAL_MEDIA_HOSTS; +let server: ReturnType; +let origin = ""; + +beforeAll(() => { + process.env.MEDIA_SERVER_WEBHOOK_SECRET = SECRET; + server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + const { pathname } = new URL(request.url); + if (pathname === "/missing.mp4") + return new Response("Not Found", { status: 404 }); + if (pathname === "/unavailable.mp4") + return new Response("Unavailable", { status: 503 }); + // Serves every read but the one-byte size request. + if ( + pathname === "/size-fails.mp4" && + request.headers.get("range") === "bytes=0-0" + ) + return new Response("Unavailable", { status: 503 }); + if (pathname === "/not-video.mp4") + return serveFixture(request, Buffer.alloc(4096, 0x78)); + return serveFixture(request, FIXTURE); + }, + }); + origin = `http://127.0.0.1:${server.port}`; +}); + +afterEach(() => { + if (previousInternalHosts === undefined) + delete process.env.MEDIA_SERVER_INTERNAL_MEDIA_HOSTS; + else process.env.MEDIA_SERVER_INTERNAL_MEDIA_HOSTS = previousInternalHosts; +}); + +afterAll(async () => { + await server.stop(true); + if (previousSecret === undefined) + delete process.env.MEDIA_SERVER_WEBHOOK_SECRET; + else process.env.MEDIA_SERVER_WEBHOOK_SECRET = previousSecret; +}); + +function thumbnailRequest(videoUrl: string) { + return new Request("http://localhost/video/thumbnail", { + method: "POST", + headers: { + "x-media-server-secret": SECRET, + "Content-Type": "application/json", + }, + body: JSON.stringify({ videoUrl, width: 160, height: 120 }), + }); +} + +describe("POST /video/thumbnail with a remote video", () => { + test("returns a JPEG for a remote video URL", async () => { + const response = await app.fetch( + thumbnailRequest(`${origin}/video.mp4?X-Amz-Signature=abc`), + ); + expect(response.status).toBe(200); + expect(response.headers.get("Content-Type")).toBe("image/jpeg"); + const bytes = new Uint8Array(await response.arrayBuffer()); + expect([bytes[0], bytes[1]]).toEqual([0xff, 0xd8]); + }, 30_000); + + test("answers 422 when the host says the video is not there", async () => { + const response = await app.fetch(thumbnailRequest(`${origin}/missing.mp4`)); + expect(response.status).toBe(422); + expect((await response.json()).code).toBe("INPUT_NOT_ACCESSIBLE"); + }, 30_000); + + test("keeps a 5xx answer when the host fails for now", async () => { + const response = await app.fetch( + thumbnailRequest(`${origin}/unavailable.mp4`), + ); + expect(response.status).toBe(500); + }, 30_000); + + test("answers 422 when the remote file is not a video", async () => { + const response = await app.fetch( + thumbnailRequest(`${origin}/not-video.mp4`), + ); + expect(response.status).toBe(422); + expect((await response.json()).code).toBe("UNREADABLE_INPUT"); + }, 30_000); + + test("answers 422 for an address the server may not fetch", async () => { + process.env.MEDIA_SERVER_INTERNAL_MEDIA_HOSTS = ""; + const response = await app.fetch(thumbnailRequest(`${origin}/video.mp4`)); + expect(response.status).toBe(422); + expect((await response.json()).code).toBe("input-policy-violation"); + }, 30_000); + + test("a size request that fails is not taken for an unreadable file", async () => { + const response = await app.fetch( + new Request("http://localhost/video/probe", { + method: "POST", + headers: { + "x-media-server-secret": SECRET, + "Content-Type": "application/json", + }, + body: JSON.stringify({ videoUrl: `${origin}/size-fails.mp4` }), + }), + ); + expect(response.status).toBe(500); + const healthy = await app.fetch( + new Request("http://localhost/video/probe", { + method: "POST", + headers: { + "x-media-server-secret": SECRET, + "Content-Type": "application/json", + }, + body: JSON.stringify({ videoUrl: `${origin}/video.mp4` }), + }), + ); + expect(healthy.status).toBe(200); + }, 30_000); +}); diff --git a/apps/media-server/src/app.ts b/apps/media-server/src/app.ts index 96e7ca32cba..1c9a98c04f8 100644 --- a/apps/media-server/src/app.ts +++ b/apps/media-server/src/app.ts @@ -1,5 +1,6 @@ import { Hono } from "hono"; import { logger } from "hono/logger"; +import { hasMediaServerSecret } from "./lib/auth"; import { beginRequest } from "./lib/lifecycle"; import audio from "./routes/audio"; import health from "./routes/health"; @@ -25,6 +26,7 @@ app.route("/video", video); app.route("/live-transcribe", liveTranscribe); app.get("/", (c) => { + if (!hasMediaServerSecret(c)) return c.json({ status: "ok" }); return c.json({ name: "@cap/media-server", version: "1.0.0", diff --git a/apps/media-server/src/index.ts b/apps/media-server/src/index.ts index 28d403feef6..feb4847bac1 100644 --- a/apps/media-server/src/index.ts +++ b/apps/media-server/src/index.ts @@ -8,11 +8,13 @@ import { } from "./lib/lifecycle"; import { releaseLiveSessions } from "./lib/live-transcribe"; import { cancelAllMediaOperations } from "./lib/media-operations"; +import { stopProbeWorkers, warmProbeWorkers } from "./lib/media-probe-pool"; import { cleanupMediaTransferCache, sweepMediaTransferCache, } from "./lib/media-transfer"; import { installUnhandledRejectionGuard } from "./lib/process-guards"; +import { hideServerFromChildren } from "./lib/process-hardening"; import { removeOwnTempDir, startTempDirMaintenance } from "./lib/temp-files"; installUnhandledRejectionGuard(); @@ -21,6 +23,13 @@ const port = Number(process.env.PORT) || 3456; console.log(`[media-server] Starting on port ${port}`); +const hidden = hideServerFromChildren(); +console.log( + `[media-server] Running as uid ${process.getuid?.() ?? "unknown"}${hidden ? ", hidden from child processes" : ""}`, +); + +warmProbeWorkers(); + void Promise.allSettled([ startTempDirMaintenance(), sweepMediaTransferCache(), @@ -40,6 +49,7 @@ const drainAndShutdown = createShutdownHandler(async () => { console.log(`[media-server] Aborted ${abortedJobs} active jobs`); } await cancelAllMediaOperations(); + stopProbeWorkers(); await cleanupMediaTransferCache(); await removeOwnTempDir(); process.exit(0); diff --git a/apps/media-server/src/lib/capcodec-encoder.ts b/apps/media-server/src/lib/capcodec-encoder.ts index a64b332d295..ebceda6a4d5 100644 --- a/apps/media-server/src/lib/capcodec-encoder.ts +++ b/apps/media-server/src/lib/capcodec-encoder.ts @@ -1,6 +1,7 @@ import { constants } from "node:fs"; import { access } from "node:fs/promises"; import { file, type Subprocess } from "bun"; +import { outputLengthCapArgs } from "./media-common"; import { spawn } from "./media-spawn"; import { registerSubprocess, terminateProcess } from "./subprocess"; @@ -26,6 +27,7 @@ export interface CapcodecEncodeJob { maxWidth: number; maxHeight: number; extraInputArgs: string[]; + maxOutputSeconds?: number | null; options: CapcodecOptions; totalDurationUs: number; onProgress?: (progress: number, message: string) => void; @@ -176,7 +178,10 @@ function formatCrf(crf: number): string { } export function buildCapcodecDecodeArgs( - job: Pick, + job: Pick< + CapcodecEncodeJob, + "inputPath" | "extraInputArgs" | "maxOutputSeconds" + >, width: number, height: number, fps: string, @@ -198,6 +203,7 @@ export function buildCapcodecDecodeArgs( "cfr", "-r", fps, + ...outputLengthCapArgs(job.maxOutputSeconds), "-f", "rawvideo", "-pix_fmt", diff --git a/apps/media-server/src/lib/hls-ladder-encode.ts b/apps/media-server/src/lib/hls-ladder-encode.ts index dd3885e622d..2905a0c3ea8 100644 --- a/apps/media-server/src/lib/hls-ladder-encode.ts +++ b/apps/media-server/src/lib/hls-ladder-encode.ts @@ -17,6 +17,7 @@ import { import type { VideoMetadata } from "./job-manager"; import { decodeThreadsFromEnv, + encodeLengthLimit, PROCESS_TIMEOUT_MS, type ProgressCallback, } from "./media-common"; @@ -143,6 +144,7 @@ export async function encodeHlsLadder( Math.max(0, metadata.duration - range.startSeconds)) : metadata.duration; const timeoutMs = getProcessTimeoutMs(sliceSeconds, PROCESS_TIMEOUT_MS); + const maxOutputSeconds = encodeLengthLimit(metadata); const totalDurationUs = sliceSeconds * 1_000_000; const argsInput = { inputPath, @@ -154,6 +156,7 @@ export async function encodeHlsLadder( range, hdr, hdrColor, + maxOutputSeconds, }; const encodeVideo = async (): Promise<"nvenc" | "libx264"> => { @@ -214,7 +217,7 @@ export async function encodeHlsLadder( const separateAudio = wantsAudio && range ? runTranscodeCommand( - buildHlsAudioArgs({ inputPath, outputDir: dir }), + buildHlsAudioArgs({ inputPath, outputDir: dir, maxOutputSeconds }), join(dir, audioPlaylistName), metadata.duration * 1_000_000, getProcessTimeoutMs(metadata.duration, PROCESS_TIMEOUT_MS), diff --git a/apps/media-server/src/lib/hls-ladder.ts b/apps/media-server/src/lib/hls-ladder.ts index 857f42ed6a7..5b481666246 100644 --- a/apps/media-server/src/lib/hls-ladder.ts +++ b/apps/media-server/src/lib/hls-ladder.ts @@ -4,6 +4,7 @@ import { hdrToSdrFilter, SDR_COLOR_ARGS, } from "./hdr"; +import { outputLengthCapArgs } from "./media-common"; import { NVENC_HWACCEL_INPUT_ARGS, type NvencOptions } from "./nvenc-encoder"; export const HLS_LADDER_MASTER_PLAYLIST = "master.m3u8"; @@ -214,6 +215,7 @@ export interface HlsLadderArgsInput { range?: HlsLadderRange | null; hdr?: HdrTransfer | null; hdrColor?: HdrColor | null; + maxOutputSeconds?: number | null; } // An HDR source is converted on the CPU, so it is never decoded on the GPU. @@ -409,6 +411,7 @@ export function buildHlsLadderArgs(input: HlsLadderArgsInput): string[] { : videoStreams; args.push( + ...outputLengthCapArgs(input.maxOutputSeconds), ...(input.range ? SLICE_OUTPUT_ARGS : []), "-f", "hls", @@ -541,6 +544,7 @@ export function buildHlsAudioArgs(input: { inputPath: string; outputDir: string; extraInputArgs?: string[]; + maxOutputSeconds?: number | null; }): string[] { return [ "ffmpeg", @@ -557,6 +561,7 @@ export function buildHlsAudioArgs(input: { `${HLS_LADDER_AUDIO_BITRATE_KBPS}k`, "-ac", "2", + ...outputLengthCapArgs(input.maxOutputSeconds), "-hls_segment_options", "movflags=+frag_discont", "-f", diff --git a/apps/media-server/src/lib/infra-poster.ts b/apps/media-server/src/lib/infra-poster.ts new file mode 100644 index 00000000000..dcc21e9d236 --- /dev/null +++ b/apps/media-server/src/lib/infra-poster.ts @@ -0,0 +1,16 @@ +import type { ThumbnailOptions } from "./media-video"; + +export const INFRA_POSTER_MAX_DIMENSION = 1920; + +export const INFRA_POSTER_OPTIONS: ThumbnailOptions = { + width: INFRA_POSTER_MAX_DIMENSION, + height: INFRA_POSTER_MAX_DIMENSION, +}; + +// Cap Infra posters are shown at the player's full size, so they are kept up +// to 1920 on the long side. Every other job keeps the default thumbnail size. +export function posterThumbnailOptions( + infra: boolean | undefined, +): ThumbnailOptions { + return infra === true ? INFRA_POSTER_OPTIONS : {}; +} diff --git a/apps/media-server/src/lib/isolated-worker-pool.ts b/apps/media-server/src/lib/isolated-worker-pool.ts new file mode 100644 index 00000000000..0ef81456e23 --- /dev/null +++ b/apps/media-server/src/lib/isolated-worker-pool.ts @@ -0,0 +1,384 @@ +import { readFileSync } from "node:fs"; +import { spawn as bunSpawn, type Subprocess, spawnSync } from "bun"; + +// Runs requests in child processes that are stopped when one uses too much +// memory or time, so a fault in what they run cannot reach the server. +// Starting a process costs far more than a small request does, so a few are +// kept running and reused, one request at a time each. + +export type WorkerResponse = + | { id: number; ok: true; value: unknown } + | { id: number; ok: false; error: Failure }; + +// The worker stopped without this pool stopping it and without reporting a +// fault of its own: the host killed it, or it was stopped from outside. +export class WorkerLostError extends Error { + constructor() { + super("Worker stopped before it answered"); + } +} + +export class WorkerStartError extends Error {} + +export interface WorkerPoolOptions { + entry: string; + env: () => Record; + memoryLimitMB: () => number; + toError: (failure: Failure) => unknown; + // What a request is answered with when its worker is stopped for using + // too much memory, or exits with faultExitCode. + memoryError: () => unknown; + faultError: () => unknown; + faultExitCode: number; + maxRequestsPerWorker?: number; + // Workers running at once. A request that finds every one busy waits for + // one to come free. + maxWorkers?: () => number; + idleWorkersKept?: number; + idleRetireMs?: number; + startTimeoutMs?: number; + label?: string; +} + +// Reading /proc is a few microseconds; asking ps is a process of its own. +const MEMORY_POLL_MS = process.platform === "linux" ? 25 : 100; + +export function residentMemoryMB(pid: number): number | null { + try { + if (process.platform === "linux") { + const kilobytes = readFileSync(`/proc/${pid}/status`, "utf8").match( + /^VmRSS:\s+(\d+) kB/m, + )?.[1]; + return kilobytes ? Number(kilobytes) / 1024 : null; + } + const result = spawnSync(["ps", "-o", "rss=", "-p", String(pid)], { + stdout: "pipe", + stderr: "ignore", + stdin: "ignore", + }); + const kilobytes = Number(result.stdout.toString().trim()); + return Number.isFinite(kilobytes) && kilobytes > 0 + ? kilobytes / 1024 + : null; + } catch { + return null; + } +} + +interface Pending { + id: number; + resolve: (value: unknown) => void; + reject: (error: unknown) => void; +} + +interface PoolWorker { + proc: Subprocess; + ready: Promise; + busy: boolean; + served: number; + stopped: boolean; + // The id of the last request this worker answered, with a value or with + // a failure it reported itself. + answered: number; + pending?: Pending; + idleTimer?: ReturnType; + stopReason?: unknown; +} + +export interface WorkerPool { + run( + request: Request, + timeoutMs: number, + signal?: AbortSignal, + ): Promise; + warm(): void; + stop(): void; + stats(): { + started: number; + recycled: number; + memoryKills: number; + timeoutKills: number; + running: number; + idle: number; + waiting: number; + }; +} + +export function createWorkerPool( + options: WorkerPoolOptions, +): WorkerPool { + const maxRequestsPerWorker = options.maxRequestsPerWorker ?? 200; + const idleWorkersKept = options.idleWorkersKept ?? 2; + const idleRetireMs = options.idleRetireMs ?? 30_000; + const startTimeoutMs = options.startTimeoutMs ?? 15_000; + const label = options.label ?? "worker"; + const maxWorkers = () => Math.max(1, options.maxWorkers?.() ?? 4); + const workers = new Set(); + const waiting: (() => void)[] = []; + const counters = { + started: 0, + recycled: 0, + memoryKills: 0, + timeoutKills: 0, + }; + let nextRequestId = 1; + + const idleCount = () => { + let idle = 0; + for (const worker of workers) if (!worker.busy) idle++; + return idle; + }; + + function stopWorker(worker: PoolWorker, reason?: unknown): void { + if (worker.stopped) return; + worker.stopped = true; + worker.stopReason = reason; + workers.delete(worker); + clearTimeout(worker.idleTimer); + try { + worker.proc.kill("SIGKILL"); + } catch {} + } + + function startWorker(): PoolWorker { + let markReady: () => void = () => {}; + let failStart: (error: unknown) => void = () => {}; + const ready = new Promise((resolve, reject) => { + markReady = resolve; + failStart = reject; + }); + // Rejected when a worker dies before it is ever used. + ready.catch(() => {}); + const worker: PoolWorker = { + proc: undefined as unknown as Subprocess, + ready, + busy: false, + served: 0, + stopped: false, + answered: 0, + }; + worker.proc = bunSpawn({ + cmd: [process.execPath, options.entry], + env: options.env(), + stdin: "ignore", + stdout: "inherit", + stderr: "inherit", + serialization: "json", + ipc(message: WorkerResponse) { + if (message.id === 0) { + markReady(); + return; + } + const pending = worker.pending; + if (!pending || pending.id !== message.id) return; + worker.pending = undefined; + worker.answered = message.id; + if (message.ok) pending.resolve(message.value); + else pending.reject(options.toError(message.error)); + }, + }); + worker.proc.unref(); + counters.started++; + workers.add(worker); + const startTimer = setTimeout(() => { + failStart(new WorkerStartError(`The ${label} did not start`)); + stopWorker(worker); + }, startTimeoutMs); + const started = () => clearTimeout(startTimer); + void ready.then(started, started); + void worker.proc.exited.then((exitCode) => { + const deliberate = worker.stopped; + worker.stopped = true; + workers.delete(worker); + clearTimeout(worker.idleTimer); + failStart( + new WorkerStartError(`The ${label} exited before it was ready`), + ); + const pending = worker.pending; + worker.pending = undefined; + if (!pending) return; + if (deliberate && worker.stopReason !== undefined) { + pending.reject(worker.stopReason); + } else if (exitCode === options.faultExitCode) { + pending.reject(options.faultError()); + } else { + pending.reject(new WorkerLostError()); + } + }); + return worker; + } + + function wakeWaiter(): void { + waiting.shift()?.(); + } + + function release(worker: PoolWorker, healthy: boolean): void { + worker.busy = false; + if (!healthy || worker.served >= maxRequestsPerWorker) { + counters.recycled++; + stopWorker(worker); + if (idleCount() === 0 && workers.size < maxWorkers()) { + try { + startWorker(); + } catch {} + } + wakeWaiter(); + return; + } + wakeWaiter(); + if (idleCount() > idleWorkersKept) { + worker.idleTimer = setTimeout(() => { + if (!worker.busy) stopWorker(worker); + }, idleRetireMs); + worker.idleTimer.unref?.(); + } + } + + function tryAcquire(): PoolWorker | null { + for (const worker of workers) { + if (worker.busy || worker.stopped) continue; + clearTimeout(worker.idleTimer); + worker.busy = true; + return worker; + } + if (workers.size >= maxWorkers()) return null; + const worker = startWorker(); + worker.busy = true; + return worker; + } + + async function acquire( + deadline: number, + signal?: AbortSignal, + ): Promise { + while (true) { + const worker = tryAcquire(); + if (worker) return worker; + await new Promise((resolve, reject) => { + const wake = () => { + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + resolve(); + }; + const leave = (reason: unknown) => { + const index = waiting.indexOf(wake); + if (index !== -1) waiting.splice(index, 1); + clearTimeout(timer); + signal?.removeEventListener("abort", onAbort); + reject(reason); + }; + const onAbort = () => + leave(signal?.reason ?? new DOMException("Aborted", "AbortError")); + const timer = setTimeout( + () => + leave( + new Error( + `Operation timed out after waiting for a free ${label}`, + ), + ), + Math.max(0, deadline - performance.now()), + ); + waiting.push(wake); + signal?.addEventListener("abort", onAbort, { once: true }); + }); + } + } + + async function run( + request: Request, + timeoutMs: number, + signal?: AbortSignal, + ): Promise { + signal?.throwIfAborted(); + const deadline = performance.now() + timeoutMs; + let worker: PoolWorker; + try { + worker = await acquire(deadline, signal); + } catch (error) { + if (error instanceof Error && error.message.includes("timed out")) + throw error; + if (signal?.aborted) throw error; + throw new WorkerStartError(String(error)); + } + // Cancelled while it waited for a worker: the worker goes back unused. + if (signal?.aborted) { + release(worker, true); + throw signal.reason ?? new DOMException("Aborted", "AbortError"); + } + const requestId = nextRequestId++; + const remainingMs = Math.max(1, deadline - performance.now()); + let timer: ReturnType | undefined; + let memoryWatch: ReturnType | undefined; + let onAbort: (() => void) | undefined; + let healthy = false; + try { + const value = await new Promise((resolve, reject) => { + const stop = (reason: unknown) => { + stopWorker(worker, reason); + reject(reason); + }; + timer = setTimeout(() => { + counters.timeoutKills++; + stop(new Error(`Operation timed out after ${timeoutMs}ms`)); + }, remainingMs); + onAbort = () => + stop(signal?.reason ?? new DOMException("Aborted", "AbortError")); + signal?.addEventListener("abort", onAbort, { once: true }); + const limitMB = options.memoryLimitMB(); + memoryWatch = setInterval(() => { + const usedMB = residentMemoryMB(worker.proc.pid); + if (usedMB === null || usedMB <= limitMB) return; + counters.memoryKills++; + console.warn( + `[${label}] Stopped at ${Math.round(usedMB)} MB, over its ${limitMB} MB limit`, + ); + stop(options.memoryError()); + }, MEMORY_POLL_MS); + worker.ready.then( + () => { + if (worker.stopped) return; + worker.pending = { id: requestId, resolve, reject }; + worker.proc.send({ ...request, id: requestId }); + }, + (error) => { + if (worker.stopReason === undefined) reject(error); + }, + ); + }); + return value; + } finally { + // A worker that answered, even with a failure it reported, is + // reused. One that was stopped, crashed or never answered is not. + healthy = !worker.stopped && worker.answered === requestId; + clearTimeout(timer); + clearInterval(memoryWatch); + if (onAbort) signal?.removeEventListener("abort", onAbort); + worker.served++; + release(worker, healthy); + } + } + + const stop = () => { + for (const worker of [...workers]) stopWorker(worker); + }; + process.on("exit", stop); + + return { + run, + stop, + warm() { + if (workers.size > 0) return; + try { + startWorker(); + } catch (error) { + console.error(`[${label}] Could not start:`, error); + } + }, + stats: () => ({ + ...counters, + running: workers.size, + idle: idleCount(), + waiting: waiting.length, + }), + }; +} diff --git a/apps/media-server/src/lib/job-manager.ts b/apps/media-server/src/lib/job-manager.ts index 1b687173465..771e3dbd661 100644 --- a/apps/media-server/src/lib/job-manager.ts +++ b/apps/media-server/src/lib/job-manager.ts @@ -110,6 +110,10 @@ export interface VideoMetadata { sampleRate: number | null; bitrate: number; fileSize: number; + // How long the file's timestamps run, when the probe measured them. An + // encode is held to the longer of this and duration, and is not held at + // all when it is unknown. + timestampSpan?: number | null; } export interface Job { @@ -156,6 +160,9 @@ export interface Job { webhookUrl?: string; webhookSecret?: string; abortController?: AbortController; + // A ladder for a Cap Infra video at the 1440p or 2160p tier, which keeps the + // GPU encoder busy on its own. + infraHeavyLadder?: boolean; } const jobs = new Map(); @@ -352,6 +359,55 @@ export function hasCriticalMemoryPressure(): boolean { return getSystemResources().memoryPressure >= MEMORY_REJECT_THRESHOLD; } +// Cap Infra jobs share the workers with Cap's own recordings. They may not take +// the last free slots, and only one Infra ladder at 1440p or 2160p runs on a +// worker at a time: one already fills the GPU encoder, and a second one slows +// both of them and every recording that lands beside them. +const INFRA_RESERVED_SLOTS = Math.max( + 0, + Number.parseInt(process.env.MEDIA_SERVER_INFRA_RESERVED_SLOTS ?? "1", 10) || + 0, +); +const MAX_INFRA_HEAVY_LADDERS = Math.max( + 1, + Number.parseInt( + process.env.MEDIA_SERVER_MAX_INFRA_HEAVY_LADDERS ?? "1", + 10, + ) || 1, +); + +export function getActiveInfraHeavyLadderCount(): number { + let count = 0; + for (const job of jobs.values()) { + if (job.infraHeavyLadder && isActivePhase(job.phase)) count++; + } + return count; +} + +export function infraCapacityAllows(input: { + effectiveMax: number; + active: number; + activeHeavyLadders: number; + heavyLadder: boolean; + reservedSlots?: number; + maxHeavyLadders?: number; +}): boolean { + const reservedSlots = input.reservedSlots ?? INFRA_RESERVED_SLOTS; + const maxHeavyLadders = input.maxHeavyLadders ?? MAX_INFRA_HEAVY_LADDERS; + const reserved = Math.min(reservedSlots, Math.max(0, input.effectiveMax - 1)); + if (input.active >= input.effectiveMax - reserved) return false; + return !input.heavyLadder || input.activeHeavyLadders < maxHeavyLadders; +} + +export function canAcceptNewInfraVideoProcess(heavyLadder: boolean): boolean { + return infraCapacityAllows({ + effectiveMax: getSystemResources().effectiveMax, + active: getActiveVideoProcessCount(), + activeHeavyLadders: getActiveInfraHeavyLadderCount(), + heavyLadder, + }); +} + export function canAcceptNewVideoProcess( priority: "interactive" | "recovery" = "interactive", ): boolean { diff --git a/apps/media-server/src/lib/matroska-tail.ts b/apps/media-server/src/lib/matroska-tail.ts new file mode 100644 index 00000000000..a1879366866 --- /dev/null +++ b/apps/media-server/src/lib/matroska-tail.ts @@ -0,0 +1,290 @@ +import { open } from "node:fs/promises"; +import { createGuardedMediaFetch, normalizeLocalPath } from "./media-common"; + +// A Matroska or WebM file that was never finalised (a recorder that stopped, +// FFmpeg writing to a pipe) has no Duration and no Cues, and finding its +// length by reading every cluster from the start takes as long as fetching +// the whole file. Its last cluster is near the end, so the length is read +// from there: one range read of the tail, whatever the file size. + +// A cluster of high bitrate video can be larger than the first read, so a +// tail without a readable cluster is read once more, larger. +const TAIL_READS = [4 * 1024 * 1024, 32 * 1024 * 1024]; +const HEAD_BYTES = 64 * 1024; +const MAX_CLUSTER_CANDIDATES = 64; +const DEFAULT_TIMECODE_SCALE_NS = 1_000_000; +const RANGE_TIMEOUT_MS = 10_000; + +const CLUSTER_ID = [0x1f, 0x43, 0xb6, 0x75]; +const TIMECODE_SCALE_ID = [0x2a, 0xd7, 0xb1]; +const ID_CLUSTER = 0x1f43b675; +const ID_TIMECODE = 0xe7; +const ID_SIMPLE_BLOCK = 0xa3; +const ID_BLOCK_GROUP = 0xa0; +const ID_BLOCK = 0xa1; +const ID_CRC32 = 0xbf; +const ID_VOID = 0xec; + +interface Vint { + value: number; + length: number; + unknown: boolean; +} + +function readVint( + data: Uint8Array, + at: number, + keepMarker: boolean, +): Vint | null { + const first = data[at]; + if (first === undefined || first === 0) return null; + let length = 1; + while (length <= 8 && !(first & (0x80 >> (length - 1)))) length++; + if (length > 8 || at + length > data.length) return null; + let value = keepMarker ? first : first & (0xff >> length); + let allOnes = (first & (0xff >> length)) === 0xff >> length; + for (let i = 1; i < length; i++) { + const byte = data[at + i] as number; + value = value * 256 + byte; + if (byte !== 0xff) allOnes = false; + } + return { value, length, unknown: !keepMarker && allOnes }; +} + +function readUint(data: Uint8Array, at: number, size: number): number | null { + if (size < 1 || size > 8 || at + size > data.length) return null; + let value = 0; + for (let i = 0; i < size; i++) value = value * 256 + (data[at + i] as number); + return value; +} + +function indexesOf(data: Uint8Array, pattern: number[]): number[] { + const found: number[] = []; + for (let i = 0; i + pattern.length <= data.length; i++) { + let match = true; + for (let j = 0; j < pattern.length; j++) { + if (data[i + j] !== pattern[j]) { + match = false; + break; + } + } + if (match) found.push(i); + } + return found; +} + +// The relative timestamp of a Block or SimpleBlock whose body starts at `at`. +function blockTimestamp(data: Uint8Array, at: number): number | null { + const track = readVint(data, at, false); + if (!track) return null; + const offset = at + track.length; + if (offset + 2 > data.length) return null; + const raw = ((data[offset] as number) << 8) | (data[offset + 1] as number); + return raw >= 0x8000 ? raw - 0x10000 : raw; +} + +// The last block timestamp of the cluster starting at `at`, in timecode +// units, or null when this is not a cluster that can be read. +function lastTimestampInCluster(data: Uint8Array, at: number): number | null { + const header = readVint(data, at + CLUSTER_ID.length, false); + if (!header) return null; + const start = at + CLUSTER_ID.length + header.length; + const end = header.unknown + ? data.length + : Math.min(data.length, start + header.value); + let clusterTime: number | null = null; + let last: number | null = null; + let position = start; + while (position < end) { + const id = readVint(data, position, true); + if (!id) break; + const size = readVint(data, position + id.length, false); + if (!size || size.unknown) break; + const body = position + id.length + size.length; + if (id.value === ID_CLUSTER) break; + if (id.value === ID_TIMECODE) { + clusterTime = readUint(data, body, size.value); + if (clusterTime === null) return null; + } else if (clusterTime === null) { + // The cluster timestamp comes first, after at most a checksum or + // padding. + if (id.value !== ID_CRC32 && id.value !== ID_VOID) return null; + } else if (id.value === ID_SIMPLE_BLOCK) { + const relative = blockTimestamp(data, body); + if (relative !== null) last = Math.max(last ?? -Infinity, relative); + } else if (id.value === ID_BLOCK_GROUP) { + let child = body; + const groupEnd = Math.min(data.length, body + size.value); + while (child < groupEnd) { + const childId = readVint(data, child, true); + const childSize = + childId && readVint(data, child + childId.length, false); + if (!childId || !childSize || childSize.unknown) break; + const childBody = child + childId.length + childSize.length; + if (childId.value === ID_BLOCK) { + const relative = blockTimestamp(data, childBody); + if (relative !== null) last = Math.max(last ?? -Infinity, relative); + } + child = childBody + childSize.value; + } + } else if (id.value > 0xffffff) { + // Another top-level element (Cues, Tags) ends the cluster. + break; + } + position = body + size.value; + } + return clusterTime === null || last === null ? null : clusterTime + last; +} + +// The last block timestamp in the tail of a Matroska file, in timecode units. +export function lastBlockTimestamp(tail: Uint8Array): number | null { + const candidates = indexesOf(tail, CLUSTER_ID).reverse(); + for (const at of candidates.slice(0, MAX_CLUSTER_CANDIDATES)) { + const timestamp = lastTimestampInCluster(tail, at); + if (timestamp !== null) return timestamp; + } + return null; +} + +export function timecodeScaleNs(head: Uint8Array): number { + for (const at of indexesOf(head, TIMECODE_SCALE_ID)) { + const size = readVint(head, at + TIMECODE_SCALE_ID.length, false); + if (!size) continue; + const value = readUint( + head, + at + TIMECODE_SCALE_ID.length + size.length, + size.value, + ); + if (value !== null && value >= 1 && value <= 1_000_000_000) return value; + } + return DEFAULT_TIMECODE_SCALE_NS; +} + +async function readRemoteRange( + path: string, + range: string, + limit: number, +): Promise { + const response = await createGuardedMediaFetch(path)(path, { + headers: { Range: range }, + signal: AbortSignal.timeout(RANGE_TIMEOUT_MS), + }); + // A server that ignores the range would send the whole file. + if (response.status !== 206) { + await response.body?.cancel().catch(() => {}); + return null; + } + const reader = response.body?.getReader(); + if (!reader) return null; + const chunks: Uint8Array[] = []; + let length = 0; + try { + while (length < limit) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + length += value.length; + } + } finally { + await reader.cancel().catch(() => {}); + } + const bytes = new Uint8Array(Math.min(length, limit)); + let offset = 0; + for (const chunk of chunks) { + const part = chunk.subarray(0, bytes.length - offset); + bytes.set(part, offset); + offset += part.length; + if (offset >= bytes.length) break; + } + return bytes; +} + +async function readLocalRange( + path: string, + from: "head" | "tail", + bytes: number, +): Promise { + const handle = await open(normalizeLocalPath(path), "r"); + try { + const { size } = await handle.stat(); + const length = Math.min(bytes, size); + const buffer = new Uint8Array(length); + await handle.read(buffer, 0, length, from === "head" ? 0 : size - length); + return buffer; + } finally { + await handle.close(); + } +} + +async function readHead(path: string): Promise { + return /^https?:\/\//i.test(path) + ? await readRemoteRange(path, `bytes=0-${HEAD_BYTES - 1}`, HEAD_BYTES) + : await readLocalRange(path, "head", HEAD_BYTES); +} + +async function readTail( + path: string, + bytes: number, +): Promise { + return /^https?:\/\//i.test(path) + ? await readRemoteRange(path, `bytes=-${bytes}`, bytes) + : await readLocalRange(path, "tail", bytes); +} + +// How much of a tail must be zero bytes for it to count as padding: what a +// recorder that preallocated its file and stopped leaves behind. +const PADDING_BYTES = 64 * 1024; + +// The tail without its trailing zero bytes, and whether those were padding. +function withoutPadding(tail: Uint8Array): { + data: Uint8Array; + padded: boolean; +} { + let end = tail.length; + while (end > 0 && tail[end - 1] === 0) end--; + return { + data: tail.subarray(0, end), + padded: tail.length - end >= PADDING_BYTES, + }; +} + +export type MatroskaTailEnd = + // The time of the last block, and whether zero padding followed it. + | { seconds: number; padded: boolean } + // Zero padding with no cluster before it in the tail. + | "blank" + // No cluster starts in the tail: the file was cut off, or its last + // cluster is larger than the read. + | null + // The source cannot be read by range. + | undefined; + +// Where the last block of a Matroska or WebM file ends, from its first 64 KB +// and its last few megabytes. Trailing zero padding is skipped, so a +// preallocated recording is measured from its last block before it. A zero +// run longer than the largest tail read is not followed further back: how +// far it goes is unknown, so such a file counts as blank. A failed read +// throws. +export async function matroskaEndFromTail( + path: string, +): Promise { + const head = await readHead(path); + if (!head) return undefined; + let padded = false; + for (const bytes of TAIL_READS) { + const tail = await readTail(path, bytes); + if (!tail) return undefined; + const trimmed = withoutPadding(tail); + padded ||= trimmed.padded; + const last = lastBlockTimestamp(trimmed.data); + if (last !== null) { + return { + seconds: (last * timecodeScaleNs(head)) / 1_000_000_000, + padded, + }; + } + // The whole file was read. + if (tail.length < bytes) break; + } + return padded ? "blank" : null; +} diff --git a/apps/media-server/src/lib/media-address-guard.ts b/apps/media-server/src/lib/media-address-guard.ts index bdd326f6c4e..4fecc79bc1f 100644 --- a/apps/media-server/src/lib/media-address-guard.ts +++ b/apps/media-server/src/lib/media-address-guard.ts @@ -11,11 +11,19 @@ const MAX_ADDRESSES_TRIED = 4; // which is where cloud metadata lives, unspecified, multicast) never are. export type HostAddressClass = "public" | "private" | "reserved"; -function classifyIpv4(a: number, b: number, c: number): HostAddressClass { +// 168.63.129.16 sits in public address space but is the fixed address every +// Azure virtual machine reaches its platform services on. +function classifyIpv4( + a: number, + b: number, + c: number, + d: number, +): HostAddressClass { if ( a === 0 || (a === 169 && b === 254) || (a === 192 && b === 0 && c === 0) || + (a === 168 && b === 63 && c === 129 && d === 16) || a >= 224 ) { return "reserved"; @@ -76,7 +84,11 @@ function classifyIpv6(groups: number[]): HostAddressClass { number, ]; const embeddedIpv4 = (high: number, low: number) => - classifyIpv4(high >> 8, high & 0xff, low >> 8); + classifyIpv4(high >> 8, high & 0xff, low >> 8, low & 0xff); + // IPv4-translated addresses (::ffff:0:a.b.c.d) lead to the IPv4 address + // they carry, like the mapped form does. + if (g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0xffff && g5 === 0) + return embeddedIpv4(g6, g7); if (g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0) { if (g5 === 0xffff) return embeddedIpv4(g6, g7); if (g5 === 0) { @@ -118,7 +130,14 @@ function normalizeHost(hostname: string): string { export function classifyHostLiteral(hostname: string): HostAddressClass | null { const host = normalizeHost(hostname); const v4 = host.match(/^(\d+)\.(\d+)\.(\d+)\.(\d+)$/); - if (v4) return classifyIpv4(Number(v4[1]), Number(v4[2]), Number(v4[3])); + if (v4) { + return classifyIpv4( + Number(v4[1]), + Number(v4[2]), + Number(v4[3]), + Number(v4[4]), + ); + } const groups = ipv6Groups(host); return groups ? classifyIpv6(groups) : null; } diff --git a/apps/media-server/src/lib/media-audio.ts b/apps/media-server/src/lib/media-audio.ts index 41383bf42aa..f69f6f35bbb 100644 --- a/apps/media-server/src/lib/media-audio.ts +++ b/apps/media-server/src/lib/media-audio.ts @@ -231,8 +231,10 @@ async function probeAudioTracks( if (exitCode === 0) { const { streams } = AUDIO_PROBE_SCHEMA.parse(JSON.parse(stdoutText)); const trackTypes = streams.map((stream) => stream.codec_type); - if (!trackTypes.includes("video")) { - throw new Error("No video stream found"); + // Only the audio matters here: extraction drops any video (-vn), + // so an audio-only source is read like any other. + if (trackTypes.length === 0) { + throw new Error("No media stream found"); } return trackTypes.includes("audio"); } diff --git a/apps/media-server/src/lib/media-common.ts b/apps/media-server/src/lib/media-common.ts index fad40a51e01..800dd246a70 100644 --- a/apps/media-server/src/lib/media-common.ts +++ b/apps/media-server/src/lib/media-common.ts @@ -38,6 +38,47 @@ export const DOWNLOAD_TIMEOUT_MS = 10 * 60 * 1000; export const UPLOAD_TIMEOUT_MS = 10 * 60 * 1000; const mediaFetch: typeof fetch = globalThis.fetch.bind(globalThis); +const ENCODE_LENGTH_MARGIN_SECONDS = 2; +const ENCODE_LENGTH_MARGIN_RATIO = 0.02; + +// The longest output an encode of a source this long may write: the probed +// length, with room for a last frame the probe and FFmpeg time differently. +// An encode follows the timestamps it finds, so without a limit a file that +// plays for longer than the probe measured would be encoded, and its job +// slot held, for as long as it lasts. Null when the length is not known. +export function maxEncodeSeconds(probedSeconds: number): number | null { + if (!Number.isFinite(probedSeconds) || probedSeconds <= 0) return null; + const margin = Math.max( + ENCODE_LENGTH_MARGIN_SECONDS, + probedSeconds * ENCODE_LENGTH_MARGIN_RATIO, + ); + return Math.ceil((probedSeconds + margin) * 1000) / 1000; +} + +// The output limit for an encode of a probed file: its longest measured +// length with the margin. Null, so nothing is cut, when the probe could not +// measure how long the timestamps run: a header alone can say too little, +// as a browser recording often does. +export function encodeLengthLimit(metadata: { + duration: number; + timestampSpan?: number | null; +}): number | null { + const span = metadata.timestampSpan; + if (typeof span !== "number" || !Number.isFinite(span) || span <= 0) + return null; + return maxEncodeSeconds( + Number.isFinite(metadata.duration) + ? Math.max(span, metadata.duration) + : span, + ); +} + +export function outputLengthCapArgs( + maxOutputSeconds: number | null | undefined, +): string[] { + return maxOutputSeconds ? ["-t", String(maxOutputSeconds)] : []; +} + export function decodeThreadsFromEnv( name: string, fallback: number, @@ -128,10 +169,27 @@ function withInheritedQuery(path: string, query: string): string { function createHttpSource( path: string, onRefused: (reason: string) => void, + onTransportFailure: () => void, ): Source { const root = new URL(path); + const guarded = createGuardedMediaFetch(path); + // A range past the end of the file (416) is the file's own fault: a + // field in it points at bytes it does not have. + const fetchFn = Object.assign( + async (...args: Parameters) => { + try { + const response = await guarded(...args); + if (!response.ok && response.status !== 416) onTransportFailure(); + return response; + } catch (error) { + onTransportFailure(); + throw error; + } + }, + { preconnect: guarded.preconnect }, + ) as typeof fetch; const options = { - fetchFn: createGuardedMediaFetch(path), + fetchFn, getRetryDelay: () => null, }; @@ -188,14 +246,26 @@ export function mediaInputRefusal(input: Input): string | undefined { return refusedInputs.get(input); } +const transportFailedInputs = new WeakSet>(); + +// Whether a remote input failed to fetch something it asked for. An error +// from reading such an input is about the network, not about the file. +export function mediaInputTransportFailed(input: Input): boolean { + return transportFailedInputs.has(input); +} + export function createMediaInput(path: string): Input { registerMediaEngine(); if (isHttpUrl(path)) { const input: Input = withSafeDispose( new Input({ - source: createHttpSource(path, (reason) => { - if (!refusedInputs.has(input)) refusedInputs.set(input, reason); - }), + source: createHttpSource( + path, + (reason) => { + if (!refusedInputs.has(input)) refusedInputs.set(input, reason); + }, + () => transportFailedInputs.add(input), + ), formats: ALL_FORMATS, }), ); diff --git a/apps/media-server/src/lib/media-edit.ts b/apps/media-server/src/lib/media-edit.ts index da0aa8b8054..cfeaa7709a7 100644 --- a/apps/media-server/src/lib/media-edit.ts +++ b/apps/media-server/src/lib/media-edit.ts @@ -1,7 +1,7 @@ import { writeFile } from "node:fs/promises"; import { file } from "bun"; import { type EncodeStallWindows, watchEncodeProgress } from "./encode-stall"; -import { FfmpegExitError, ffmpegErrorLines } from "./ffmpeg-errors"; +import { collectFfmpegErrorLine, FfmpegExitError } from "./ffmpeg-errors"; import type { VideoMetadata } from "./job-manager"; import { redactLogText } from "./log-redaction"; import { spawn } from "./media-spawn"; @@ -249,29 +249,74 @@ async function drainStream(stream: ReadableStream | null) { } } -async function readStream( +export const STDERR_TAIL_BYTES = 64 * 1024; + +// FFmpeg writes progress and warnings to stderr for as long as it runs, and a +// damaged input can make it write a line per packet. Only the end is ever +// used, so the stream is read to its end and at most the last maxBytes are +// kept. +export async function readStreamTail( stream: ReadableStream | null, onText?: (text: string) => void, + maxBytes: number = STDERR_TAIL_BYTES, ) { if (!stream) return ""; const reader = stream.getReader(); const chunks: Uint8Array[] = []; + let heldBytes = 0; const live = new TextDecoder(); try { while (true) { const { done, value } = await reader.read(); if (done) break; - chunks.push(value); onText?.(live.decode(value, { stream: true })); + chunks.push( + value.length > maxBytes ? value.slice(value.length - maxBytes) : value, + ); + heldBytes += Math.min(value.length, maxBytes); + while (heldBytes > maxBytes) { + const oldest = chunks[0]; + if (!oldest) break; + const excess = heldBytes - maxBytes; + if (oldest.length <= excess) { + chunks.shift(); + heldBytes -= oldest.length; + } else { + chunks[0] = oldest.slice(excess); + heldBytes -= excess; + } + } } } finally { reader.releaseLock(); } - const decoder = new TextDecoder(); - return chunks - .map((chunk) => decoder.decode(chunk, { stream: true })) - .join(""); + const held = new Uint8Array(heldBytes); + let offset = 0; + for (const chunk of chunks) { + held.set(chunk, offset); + offset += chunk.length; + } + return new TextDecoder().decode(held); +} + +const MAX_PENDING_LINE_LENGTH = 4096; + +// Hands complete lines to a listener as text arrives in pieces. A line that +// never ends is cut off rather than held. +export function lineReader(onLine: (line: string) => void) { + let pending = ""; + return { + push(text: string) { + const parts = (pending + text).split(/\r\n|\r|\n/); + pending = (parts.pop() ?? "").slice(0, MAX_PENDING_LINE_LENGTH); + for (const line of parts) onLine(line); + }, + end() { + if (pending) onLine(pending); + pending = ""; + }, + }; } async function withTimeout( @@ -335,20 +380,28 @@ export async function runFfmpegCommand( try { await withTimeout( (async () => { + // The first error lines can come long before the exit, with more + // output than is kept after them, so they are picked out as + // they arrive. + const errorLines: string[] = []; + const lines = lineReader((line) => + collectFfmpegErrorLine(errorLines, line), + ); const [stderrText, exitCode] = await Promise.all([ - readStream( - proc.stderr as ReadableStream, - progressWatch.observe, - ), + readStreamTail(proc.stderr as ReadableStream, (text) => { + progressWatch.observe(text); + lines.push(text); + }), drainStream(proc.stdout as ReadableStream).then( () => proc.exited, ), ]); + lines.end(); if (exitCode !== 0) { throw new FfmpegExitError( `FFmpeg exited with code ${exitCode}. Last stderr: ${redactLogText(stderrText.slice(-2000))}`, - ffmpegErrorLines(stderrText), + errorLines, ); } })(), diff --git a/apps/media-server/src/lib/media-input-policy.ts b/apps/media-server/src/lib/media-input-policy.ts index 07334b06fd6..83647b64a55 100644 --- a/apps/media-server/src/lib/media-input-policy.ts +++ b/apps/media-server/src/lib/media-input-policy.ts @@ -48,7 +48,10 @@ export const NESTED_RESOURCE_DEMUXERS = [ const NESTED_RESOURCE_DEMUXER_SET = new Set(NESTED_RESOURCE_DEMUXERS); -const FALLBACK_DEMUXERS = [ +// The containers this server reads. FFmpeg ships hundreds more, and a new +// release can add one that opens other resources, so the list is fixed and a +// build can only narrow it. +export const ALLOWED_DEMUXERS = [ "mov,mp4,m4a,3gp,3g2,mj2", "matroska,webm", "mpegts", @@ -71,7 +74,7 @@ const FALLBACK_DEMUXERS = [ "png_pipe", "jpeg_pipe", "webp_pipe", -]; +] as const; export function parseSafeDemuxers(listing: string): string[] { const names: string[] = []; @@ -88,11 +91,24 @@ export function parseSafeDemuxers(listing: string): string[] { return names; } +// The allowed demuxers the build has. A listing that cannot be read leaves +// the fixed list as it is: FFmpeg ignores a name it does not know. +export function allowedDemuxersIn(listing: string): string[] { + const reported = new Set( + parseSafeDemuxers(listing).flatMap((name) => name.split(",")), + ); + if (reported.size === 0) return [...ALLOWED_DEMUXERS]; + const present = ALLOWED_DEMUXERS.filter((name) => + name.split(",").some((alias) => reported.has(alias)), + ); + return present.length > 0 ? present : [...ALLOWED_DEMUXERS]; +} + let safeDemuxerWhitelist: string | undefined; export function getSafeDemuxerWhitelist(): string { if (safeDemuxerWhitelist !== undefined) return safeDemuxerWhitelist; - let names: string[] = []; + let listing = ""; try { const result = spawnSync(["ffmpeg", "-hide_banner", "-demuxers"], { stdout: "pipe", @@ -100,15 +116,154 @@ export function getSafeDemuxerWhitelist(): string { stdin: "ignore", env: { PATH: process.env.PATH ?? "" }, }); - if (result.exitCode === 0) - names = parseSafeDemuxers(result.stdout.toString()); + if (result.exitCode === 0) listing = result.stdout.toString(); } catch {} - safeDemuxerWhitelist = ( - names.length >= FALLBACK_DEMUXERS.length ? names : FALLBACK_DEMUXERS - ).join(","); + safeDemuxerWhitelist = allowedDemuxersIn(listing).join(","); return safeDemuxerWhitelist; } +// Regular Cap imports still carry these, and FFmpeg's decoders for them are +// mature. Exotic codecs (Cinepak, DV, Indeo, raw video) stay refused. +const LEGACY_VIDEO_CODECS = [ + "mpeg1video", + "mpeg2video", + "wmv1", + "wmv2", + "wmv3", + "vc1", + "msmpeg4v1", + "msmpeg4v2", + "msmpeg4v3", + "flv1", + "h263", + "h263i", + "h263p", + "theora", +]; + +// FFmpeg opens these codecs with decoders of another name, and the decoder +// list is matched by decoder name. +const LEGACY_VIDEO_DECODERS = ["msmpeg4", "flv"]; + +const LEGACY_AUDIO_DECODERS = [ + "mp2", + "mp2float", + "wmav1", + "wmav2", + "amrnb", + "amrwb", + "libopencore_amrnb", + "libopencore_amrwb", + "adpcm_ima_wav", + "adpcm_ms", +]; + +// Video codecs this server accepts in an upload, as ffprobe names them. +export const ALLOWED_VIDEO_CODECS: readonly string[] = [ + "h264", + "hevc", + "vp8", + "vp9", + "av1", + "prores", + "mpeg4", + "mjpeg", + ...LEGACY_VIDEO_CODECS, +]; + +export function isAllowedVideoCodec(codec: string): boolean { + return ALLOWED_VIDEO_CODECS.includes(codec.toLowerCase()); +} + +const PCM_DECODERS = [ + "pcm_alaw", + "pcm_bluray", + "pcm_dvd", + "pcm_f16le", + "pcm_f24le", + "pcm_f32be", + "pcm_f32le", + "pcm_f64be", + "pcm_f64le", + "pcm_mulaw", + "pcm_s16be", + "pcm_s16be_planar", + "pcm_s16le", + "pcm_s16le_planar", + "pcm_s24be", + "pcm_s24le", + "pcm_s24le_planar", + "pcm_s32be", + "pcm_s32le", + "pcm_s32le_planar", + "pcm_s64be", + "pcm_s64le", + "pcm_s8", + "pcm_s8_planar", + "pcm_u16be", + "pcm_u16le", + "pcm_u24be", + "pcm_u24le", + "pcm_u32be", + "pcm_u32le", + "pcm_u8", +]; + +// Every decoder FFmpeg may open for an input, by decoder name, which is what +// FFmpeg matches. The accepted video codecs with the library and GPU +// decoders that read them, the audio a recording or an import carries (DTS +// included, so a file with it keeps its audio), and the subtitles and cover +// art some uploads carry: FFmpeg copies a text subtitle into an MP4 by +// default, and ffprobe stops on any stream whose decoder it may not open. +export const ALLOWED_DECODERS: readonly string[] = [ + ...ALLOWED_VIDEO_CODECS, + "libdav1d", + "libaom-av1", + "libvpx", + "libvpx-vp9", + "h264_cuvid", + "hevc_cuvid", + "vp8_cuvid", + "vp9_cuvid", + "av1_cuvid", + "mjpeg_cuvid", + "mpeg4_cuvid", + "aac", + "aac_fixed", + "aac_latm", + "mp3", + "mp3float", + "opus", + "libopus", + "vorbis", + "libvorbis", + "flac", + "alac", + "ac3", + "ac3_fixed", + "eac3", + "dca", + ...LEGACY_VIDEO_DECODERS, + ...LEGACY_AUDIO_DECODERS, + ...PCM_DECODERS, + "mov_text", + "subrip", + "srt", + "ass", + "ssa", + "webvtt", + "text", + "png", + "bmp", + "gif", + "webp", +]; + +export const DECODER_WHITELIST = ALLOWED_DECODERS.join(","); + +export const UNSUPPORTED_FORMAT_MESSAGE = + "This video format is not supported. Upload H.264, HEVC, VP8, VP9, AV1, ProRes, MPEG-4 or Motion JPEG video."; + const MAX_TRUSTED_PLAYLISTS = 1000; const trustedPlaylists = new Set(); @@ -205,6 +360,11 @@ const NESTED_DEMUXER_REJECTED = new RegExp( `\\[(?:${NESTED_RESOURCE_DEMUXERS.join("|")}) @ [^\\]]*\\] Format not on whitelist`, ); const PROTOCOL_REJECTED = /Protocol '[^']+' not on whitelist/; +const DECODER_REJECTED = /Codec \([^)]*\) not on whitelist/; + +export function isDecoderRejectedMessage(message: string | undefined) { + return Boolean(message) && DECODER_REJECTED.test(message as string); +} export function isInputPolicyViolationMessage(message: string | undefined) { if (!message) return false; diff --git a/apps/media-server/src/lib/media-probe-pool.ts b/apps/media-server/src/lib/media-probe-pool.ts new file mode 100644 index 00000000000..3323b23a974 --- /dev/null +++ b/apps/media-server/src/lib/media-probe-pool.ts @@ -0,0 +1,199 @@ +import { join } from "node:path"; +import { + createWorkerPool, + WorkerLostError, + WorkerStartError, +} from "./isolated-worker-pool"; +import { MediaAddressError } from "./media-address-guard"; +import { withTimeout } from "./media-common"; +import { MediaInputPolicyError } from "./media-input-policy"; +import { getMaxConcurrentProbeOperations } from "./media-operations"; +import { + computeTrackSpans, + MediaInputNotAccessibleError, + measureFastestFrameRate, + PARSER_FAULT_EXIT_CODE, + type ProbedMedia, + probeMedia, + type TrackSpans, + UnreadableMediaInputError, +} from "./media-probe-read"; +import { childProcessEnv } from "./media-spawn"; + +// The parser that reads a file's structure runs in the same language runtime +// as the server, so a file that makes it allocate without bound or loop +// without yielding would take the server and every job on it down. Each read +// therefore runs in a probe worker, a separate process that is stopped when +// it uses too much memory or time. + +export type ProbeWorkerOp = "metadata" | "trackSpans" | "fastestFrameRate"; + +export interface ProbeWorkerRequest { + id: number; + op: ProbeWorkerOp; + path: string; + internalMediaHosts: string | null; +} + +export interface ProbeWorkerFailure { + kind: + | "policy" + | "address" + | "unreadable" + | "not-accessible" + | "unsupported-format" + | "error"; + message: string; + cause?: string; + status?: number | null; +} + +interface ProbeResults { + metadata: ProbedMedia; + trackSpans: TrackSpans; + fastestFrameRate: number | null; +} + +const DEFAULT_MEMORY_LIMIT_MB = 1024; +const MIN_MEMORY_LIMIT_MB = 128; +const UNREADABLE_MESSAGE = "Media input could not be read"; + +export function probeMemoryLimitMB( + env: Record = process.env, +): number { + const configured = Number(env.MEDIA_SERVER_PROBE_MEMORY_LIMIT_MB); + return Number.isFinite(configured) && configured >= MIN_MEMORY_LIMIT_MB + ? configured + : DEFAULT_MEMORY_LIMIT_MB; +} + +export function probeIsolationEnabled( + env: Record = process.env, +): boolean { + return env.MEDIA_SERVER_PROBE_ISOLATION?.trim().toLowerCase() !== "off"; +} + +function failureToError(failure: ProbeWorkerFailure): Error { + if (failure.kind === "policy") { + // The class adds its own prefix to the reason it is given. + const error = new MediaInputPolicyError(""); + error.message = failure.message; + return error; + } + if (failure.kind === "address") return new MediaAddressError(failure.message); + if (failure.kind === "not-accessible") { + return new MediaInputNotAccessibleError(failure.status ?? null); + } + if (failure.kind === "unreadable") { + return new UnreadableMediaInputError( + failure.message, + failure.cause === undefined ? undefined : { cause: failure.cause }, + ); + } + const error = new Error(failure.message); + if (failure.kind === "unsupported-format") { + error.name = "UnsupportedInputFormatError"; + } + return error; +} + +const pool = createWorkerPool< + Omit, + ProbeWorkerFailure +>({ + entry: join(import.meta.dir, "media-probe-worker.ts"), + env: () => childProcessEnv(process.env), + memoryLimitMB: () => probeMemoryLimitMB(), + toError: failureToError, + memoryError: () => + new UnreadableMediaInputError(UNREADABLE_MESSAGE, { + cause: "reading it needs more memory than a probe may use", + }), + faultError: () => + new UnreadableMediaInputError(UNREADABLE_MESSAGE, { + cause: "the parser stopped on this file", + }), + faultExitCode: PARSER_FAULT_EXIT_CODE, + maxWorkers: () => getMaxConcurrentProbeOperations(), + label: "probe worker", +}); + +export const getProbeWorkerStats = pool.stats; +export const stopProbeWorkers = pool.stop; + +// Starts the first worker ahead of the first probe. +export function warmProbeWorkers(): void { + if (probeIsolationEnabled()) pool.warm(); +} + +function inProcess( + op: Op, + path: string, + timeoutMs: number, + signal?: AbortSignal, +): Promise { + const stopped = new AbortController(); + const combined = signal + ? AbortSignal.any([signal, stopped.signal]) + : stopped.signal; + const run = + op === "metadata" + ? probeMedia(path) + : op === "trackSpans" + ? computeTrackSpans(path) + : measureFastestFrameRate(path, combined); + return withTimeout(run as Promise, timeoutMs, () => + stopped.abort(), + ); +} + +function inWorker( + op: Op, + path: string, + timeoutMs: number, + signal?: AbortSignal, +): Promise { + return pool.run( + { + op, + path, + internalMediaHosts: process.env.MEDIA_SERVER_INTERNAL_MEDIA_HOSTS ?? null, + }, + timeoutMs, + signal, + ) as Promise; +} + +let startFailureLogged = false; + +export async function runProbe( + op: Op, + path: string, + timeoutMs: number, + signal?: AbortSignal, +): Promise { + if (!probeIsolationEnabled()) { + return await inProcess(op, path, timeoutMs, signal); + } + try { + try { + return await inWorker(op, path, timeoutMs, signal); + } catch (error) { + // Asked once more on a fresh worker: the first may have been + // lost to something other than this file. + if (!(error instanceof WorkerLostError)) throw error; + return await inWorker(op, path, timeoutMs, signal); + } + } catch (error) { + if (!(error instanceof WorkerStartError)) throw error; + // A server that cannot start a worker still has to answer. + if (!startFailureLogged) { + startFailureLogged = true; + console.error( + "[probe] Probe workers cannot start, reading in the server process:", + error.message, + ); + } + return await inProcess(op, path, timeoutMs, signal); + } +} diff --git a/apps/media-server/src/lib/media-probe-read.ts b/apps/media-server/src/lib/media-probe-read.ts new file mode 100644 index 00000000000..a3e23bf9ba1 --- /dev/null +++ b/apps/media-server/src/lib/media-probe-read.ts @@ -0,0 +1,577 @@ +import { + type AudioCodec, + EncodedPacketSink, + type InputFormat, + type InputTrack, + MATROSKA, + type MediaCodec, + MP4, + QTFF, + type VideoCodec, + WEBM, +} from "mediabunny"; +import type { VideoMetadata } from "./job-manager"; +import { matroskaEndFromTail } from "./matroska-tail"; +import { MediaAddressError } from "./media-address-guard"; +import { + createGuardedMediaFetch, + createMediaInput, + getSourceSize, + isHlsPath, + mediaInputRefusal, + mediaInputTransportFailed, +} from "./media-common"; +import { + exceedsSourceFrameLimit, + looksLikeTextContainer, + MediaInputPolicyError, +} from "./media-input-policy"; + +// What the parser reads out of a file. It runs in a probe worker, never in +// the server process: see media-probe-pool.ts. + +function mapVideoCodec(codec: VideoCodec | MediaCodec | null): string { + if (codec === "avc") return "h264"; + if (codec === "hevc") return "hevc"; + return codec ?? "unknown"; +} + +function mapAudioCodec(codec: AudioCodec | MediaCodec | null): string | null { + if (!codec) return null; + return codec; +} + +function roundFps(fps: number): number { + if (!Number.isFinite(fps) || fps <= 0) return 0; + return Math.round(fps * 100) / 100; +} + +function isHttpUrl(path: string): boolean { + try { + const url = new URL(path); + return url.protocol === "http:" || url.protocol === "https:"; + } catch { + return false; + } +} + +const SNIFF_BYTES = 4096; + +// The input itself cannot be used, so asking again would give the same answer. +export class UnreadableMediaInputError extends Error { + override name = "UnreadableMediaInputError"; +} + +// The remote host did not hand over the file. status is the HTTP status it +// answered with, or null when there was no answer. +export class MediaInputNotAccessibleError extends Error { + override name = "MediaInputNotAccessibleError"; + + constructor(readonly status: number | null = null) { + super("Media input is not accessible"); + } +} + +// Not a policy violation: those count against the uploader, and a camera +// original that is simply too big should not. +export const FRAME_TOO_LARGE_MESSAGE = + "Video frame is larger than 8K, the largest size this server accepts"; + +export function isUnreadableMediaInput(error: unknown): boolean { + return ( + error instanceof UnreadableMediaInputError || + error instanceof MediaInputPolicyError || + (error instanceof Error && error.name === "UnsupportedInputFormatError") + ); +} + +const UNREADABLE_MESSAGE = "Media input could not be read"; + +// What a probe worker exits with when the parser throws outside a request. +export const PARSER_FAULT_EXIT_CODE = 70; + +function isTimeout(error: unknown): boolean { + return ( + error instanceof Error && + (error.name === "TimeoutError" || + error.name === "AbortError" || + error.message.includes("timed out")) + ); +} + +// Errors from reading the source itself rather than parsing it. +const sourceReadErrors = new WeakSet(); + +function asSourceReadError(error: unknown): Error { + const wrapped = error instanceof Error ? error : new Error(String(error)); + sourceReadErrors.add(wrapped); + return wrapped; +} + +// A parser that gives up on a file (an internal assertion, a field out of +// range) says the file cannot be read, and asking again gives the same +// answer. A source that could not be fetched, a refused address and a +// timeout say nothing about the file, so those pass through. +function asUnreadable( + error: unknown, + input: ReturnType, +): unknown { + if ( + (typeof error === "object" && + error !== null && + sourceReadErrors.has(error)) || + isUnreadableMediaInput(error) || + error instanceof MediaAddressError || + isTimeout(error) || + mediaInputTransportFailed(input) + ) { + return error; + } + return new UnreadableMediaInputError(UNREADABLE_MESSAGE, { cause: error }); +} + +// The first bytes of a remote input, an empty array for an empty file, or +// null when it cannot be fetched. +async function readHttpHead(path: string): Promise { + try { + const response = await createGuardedMediaFetch(path)(path, { + headers: { Range: `bytes=0-${SNIFF_BYTES - 1}` }, + signal: AbortSignal.timeout(10_000), + }); + if (response.status === 416) { + await response.body?.cancel(); + return new Uint8Array(0); + } + if (!response.ok) { + await response.body?.cancel(); + throw new MediaInputNotAccessibleError(response.status); + } + const reader = response.body?.getReader(); + if (!reader) return new Uint8Array(0); + const head = new Uint8Array(SNIFF_BYTES); + let length = 0; + while (length < SNIFF_BYTES) { + const { done, value } = await reader.read(); + if (done) break; + const chunk = value.subarray(0, SNIFF_BYTES - length); + head.set(chunk, length); + length += chunk.length; + } + await reader.cancel().catch(() => {}); + return head.subarray(0, length); + } catch (error) { + if ( + error instanceof MediaInputPolicyError || + error instanceof MediaAddressError || + error instanceof MediaInputNotAccessibleError + ) + throw error; + return null; + } +} + +const MAX_PLAYLIST_BYTES = 1_000_000; +const MAX_NESTED_PLAYLISTS = 16; +const PLAYLIST_WALK_TIMEOUT_MS = 15_000; +const OFF_HOST_PLAYLIST = + "a remote playlist may only reference files on its own host"; + +async function readBoundedText(response: Response): Promise { + const reader = response.body?.getReader(); + if (!reader) return ""; + const chunks: Uint8Array[] = []; + let length = 0; + while (length < MAX_PLAYLIST_BYTES) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + length += value.length; + } + await reader.cancel().catch(() => {}); + return new TextDecoder().decode(Buffer.concat(chunks)); +} + +export function playlistReferences(playlist: string): string[] { + const references: string[] = []; + for (const raw of playlist.split(/\r?\n/)) { + const line = raw.trim(); + if (!line) continue; + if (!line.startsWith("#")) { + references.push(line); + continue; + } + for (const match of line.matchAll(/URI="([^"]*)"/g)) { + if (match[1]) references.push(match[1]); + } + } + return references; +} + +// Read before the demuxer sees a remote playlist, so one that names another +// host is refused without any of its entries being requested. Playlists it +// links to on its own host are checked the same way. +export async function assertRemotePlaylistStaysOnHost( + rootUrl: string, + timeoutMs = PLAYLIST_WALK_TIMEOUT_MS, +): Promise { + // One budget for the whole walk, shorter than the probe's own timeout, so + // no request from it is still running once the probe has given up. + const signal = AbortSignal.timeout(timeoutMs); + const root = new URL(rootUrl); + const fetchPlaylist = createGuardedMediaFetch(rootUrl); + const queue = [rootUrl]; + const seen = new Set(); + for (let current = queue.shift(); current; current = queue.shift()) { + if (seen.has(current)) continue; + seen.add(current); + if (seen.size > MAX_NESTED_PLAYLISTS) { + throw new MediaInputPolicyError( + "the playlist links to too many other playlists", + ); + } + let playlist: string | null = null; + try { + const response = await fetchPlaylist(current, { signal }); + if (response.ok) playlist = await readBoundedText(response); + else await response.body?.cancel().catch(() => {}); + } catch (error) { + if (!signal.aborted) throw error; + throw new Error( + `Operation timed out after ${timeoutMs}ms reading the playlist`, + ); + } + if (playlist === null) continue; + for (const reference of playlistReferences(playlist)) { + let target: URL; + try { + target = new URL(reference, current); + } catch { + throw new MediaInputPolicyError(OFF_HOST_PLAYLIST); + } + if (target.origin !== root.origin) { + throw new MediaInputPolicyError(OFF_HOST_PLAYLIST); + } + if (isHlsPath(target.href)) { + if (!target.search) target.search = root.search; + queue.push(target.href); + } + } + } +} + +export interface ProbedMedia { + metadata: VideoMetadata; + // What the container declares and what its timestamps span, when both + // are known. metadata.duration is the declared length when there is one. + declaredDuration: number | null; + measuredDuration: number | null; + // The bitrate the tracks report, 0 when they report none and + // metadata.bitrate was worked out from the file size and duration. + trackBitrate: number; + // Whether the container can hide trimmed media behind an edit list, as + // MP4 and QuickTime can, so a longer span may be real trimming. + mayTrim: boolean; +} + +export async function probeMedia(path: string): Promise { + if (isHttpUrl(path)) { + const head = await readHttpHead(path); + if (!head) throw new MediaInputNotAccessibleError(); + if (head.length === 0) { + throw new UnreadableMediaInputError("Media input is empty"); + } + // A remote file that is really a playlist would have its entries fetched + // by the demuxer. Only a URL that names itself a playlist may be one. + if (!isHlsPath(path) && looksLikeTextContainer(head)) { + throw new MediaInputPolicyError( + "the file is a playlist or resource list, not a media file", + ); + } + if (isHlsPath(path)) await assertRemotePlaylistStaysOnHost(path); + } + + const input = createMediaInput(path); + + try { + return await readMediaMetadata(input, path); + } catch (error) { + const refusal = mediaInputRefusal(input); + if (refusal) throw new MediaInputPolicyError(refusal); + throw asUnreadable(error, input); + } finally { + input.dispose(); + } +} + +// The last timestamp is the length only for a file that starts at zero. A +// transport stream usually does not, so the length is counted from its first +// timestamp. +async function measureLength( + input: ReturnType, +): Promise { + const [start, end] = await Promise.all([ + input.getFirstTimestamp(), + input.computeDuration(undefined, { skipLiveWait: true }), + ]); + return Number.isFinite(start) && start > 0 ? Math.max(0, end - start) : end; +} + +const MEASURE_BUDGET_MS = 3_000; + +// A header can declare a length its timestamps do not have. Measuring costs +// nothing for an indexed file and a read of the tail otherwise; a file that +// would need more than that keeps its declared length, which the encoder is +// held to anyway. +async function measureWithinBudget( + input: ReturnType, + budgetMs: number, +): Promise { + let timer: ReturnType | undefined; + const measured = measureLength(input).catch(() => null); + const budget = new Promise((resolve) => { + timer = setTimeout(() => resolve(null), budgetMs); + }); + try { + const length = await Promise.race([measured, budget]); + return length !== null && Number.isFinite(length) && length > 0 + ? length + : null; + } finally { + clearTimeout(timer); + } +} + +export const NO_VIDEO_STREAM_MESSAGE = "No video stream found"; + +export const UNFINISHED_RECORDING_MESSAGE = + "This recording was not finished properly. Remux or export it, then upload it again."; + +const UNDECLARED_MEASURE_BUDGET_MS = 5_000; + +function isMatroskaFamily(format: InputFormat): boolean { + return format === MATROSKA || format === WEBM; +} + +// The length of a Matroska or WebM file from its last cluster, counted from +// its first timestamp. See matroskaEndFromTail for the other answers; a +// last block at 0 is inconclusive, like a tail with no cluster. +async function matroskaLengthFromTail( + input: ReturnType, + path: string, +): Promise<{ length: number; padded: boolean } | "blank" | null | undefined> { + const [end, start] = await Promise.all([ + matroskaEndFromTail(path).catch((error: unknown) => { + throw asSourceReadError(error); + }), + input.getFirstTimestamp(), + ]); + if (end === undefined || end === null || end === "blank") return end; + const length = Math.max( + 0, + end.seconds - (Number.isFinite(start) && start > 0 ? start : 0), + ); + return Number.isFinite(length) && length > 0 + ? { length, padded: end.padded } + : null; +} + +// The length of a file whose container declares none. Matroska and WebM +// are measured by reading every cluster from the start, which for a large +// unfinalised recording takes as long as fetching the whole file, so the +// last cluster is read from the tail first. The full measure is still used +// when it finishes quickly, so a short recording keeps the length it has +// always had, except after zero padding, which it would read through until +// the deadline. Only a file whose length neither finds is refused as +// unfinished. +async function measureUndeclaredLength( + input: ReturnType, + path: string, +): Promise { + if (!isMatroskaFamily(await input.getFormat())) { + return await measureLength(input); + } + const fromTail = await matroskaLengthFromTail(input, path); + if (fromTail === undefined) return await measureLength(input); + if (fromTail === "blank") { + throw new UnreadableMediaInputError(UNFINISHED_RECORDING_MESSAGE); + } + if (fromTail?.padded) return fromTail.length; + const measured = await measureWithinBudget( + input, + UNDECLARED_MEASURE_BUDGET_MS, + ); + const length = measured ?? fromTail?.length ?? null; + if (length === null || length <= 0) { + throw new UnreadableMediaInputError(UNFINISHED_RECORDING_MESSAGE); + } + return length; +} + +// What the timestamps of a file that declares a length span. A Matroska or +// WebM file that cannot be measured within the budget is measured from its +// tail instead. +async function measureDeclaredLength( + input: ReturnType, + path: string, +): Promise { + const measured = await measureWithinBudget(input, MEASURE_BUDGET_MS); + if (measured !== null || !isMatroskaFamily(await input.getFormat())) { + return measured; + } + const fromTail = await matroskaLengthFromTail(input, path).catch(() => null); + return typeof fromTail === "object" && fromTail !== null + ? fromTail.length + : null; +} + +async function readMediaMetadata( + input: ReturnType, + path: string, +): Promise { + const canRead = await input.canRead(); + if (!canRead) { + throw new UnreadableMediaInputError("Media input could not be read"); + } + + const videoTrack = await input.getPrimaryVideoTrack(); + if (!videoTrack) { + throw new UnreadableMediaInputError(NO_VIDEO_STREAM_MESSAGE); + } + if ( + exceedsSourceFrameLimit( + await videoTrack.getCodedWidth(), + await videoTrack.getCodedHeight(), + ) || + exceedsSourceFrameLimit( + await videoTrack.getDisplayWidth(), + await videoTrack.getDisplayHeight(), + ) + ) { + throw new UnreadableMediaInputError(FRAME_TOO_LARGE_MESSAGE); + } + + const audioTrack = await input.getPrimaryAudioTrack(); + const declaredDuration = await input.getDurationFromMetadata(undefined, { + skipLiveWait: true, + }); + const measuredDuration = + declaredDuration === null ? null : await measureDeclaredLength(input, path); + const duration = + declaredDuration ?? (await measureUndeclaredLength(input, path)); + const packetStats = await videoTrack + .computePacketStats(120, { skipLiveWait: true }) + .catch(() => null); + const videoBitrate = + (await videoTrack.getAverageBitrate()) ?? + (await videoTrack.getBitrate()) ?? + packetStats?.averageBitrate ?? + 0; + const audioBitrate = audioTrack + ? ((await audioTrack.getAverageBitrate()) ?? + (await audioTrack.getBitrate()) ?? + 0) + : 0; + const fileSize = await getSourceSize(path).catch((error: unknown) => { + throw asSourceReadError(error); + }); + const bitrateFromSize = + duration > 0 && fileSize > 0 ? Math.round((fileSize * 8) / duration) : 0; + + const metadata: VideoMetadata = { + duration, + width: await videoTrack.getDisplayWidth(), + height: await videoTrack.getDisplayHeight(), + fps: roundFps(packetStats?.averagePacketRate ?? 0), + videoCodec: mapVideoCodec(await videoTrack.getCodec()), + audioCodec: mapAudioCodec((await audioTrack?.getCodec()) ?? null), + audioChannels: audioTrack ? await audioTrack.getNumberOfChannels() : null, + sampleRate: audioTrack ? await audioTrack.getSampleRate() : null, + bitrate: Math.round(videoBitrate + audioBitrate) || bitrateFromSize, + fileSize, + timestampSpan: declaredDuration === null ? duration : measuredDuration, + }; + const refusal = mediaInputRefusal(input); + if (refusal) throw new MediaInputPolicyError(refusal); + return { + metadata, + declaredDuration, + measuredDuration, + trackBitrate: Math.round(videoBitrate + audioBitrate), + mayTrim: [MP4, QTFF].includes(await input.getFormat()), + }; +} + +export type TrackSpans = { video: number; audio: number | null }; + +export async function computeTrackSpans(path: string): Promise { + const input = createMediaInput(path); + try { + const videoTrack = await input.getPrimaryVideoTrack(); + if (!videoTrack) { + throw new Error("No video stream found"); + } + const audioTrack = await input.getPrimaryAudioTrack(); + const span = async (track: InputTrack) => { + const [start, end] = await Promise.all([ + track.getFirstTimestamp(), + track.computeDuration({ skipLiveWait: true }), + ]); + return end - start; + }; + const [video, audio] = await Promise.all([ + span(videoTrack), + audioTrack ? span(audioTrack) : null, + ]); + return { video, audio }; + } finally { + input.dispose(); + } +} + +const FASTEST_STRETCH_FRAMES = 8; +// Containers whose frame times come from an index or from cluster headers. +// Timing every frame of a transport stream means reading the whole file +// through the demuxer: 3 s and 770 MB of memory for a 1.5 GB file, against +// 11 ms for an MP4 of the same length. +const FRAME_TIMED_FORMATS: readonly InputFormat[] = [MP4, QTFF, MATROSKA, WEBM]; + +// The frame rate of the fastest run of consecutive frames. A run is long enough +// that two frames landing close together do not count, and short enough that +// a burst of motion a quarter of a second long does. +export function fastestFrameRate(timestamps: number[]): number | null { + const times = [...timestamps].sort((a, b) => a - b); + const run = Math.min(FASTEST_STRETCH_FRAMES, times.length - 1); + if (run < 1) return null; + let fastest = 0; + for (let index = run; index < times.length; index++) { + const seconds = (times[index] ?? 0) - (times[index - run] ?? 0); + if (seconds > 0) fastest = Math.max(fastest, run / seconds); + } + const fps = roundFps(fastest); + return fps > 0 ? fps : null; +} + +export async function measureFastestFrameRate( + path: string, + signal?: AbortSignal, +): Promise { + const input = createMediaInput(path); + try { + if (!FRAME_TIMED_FORMATS.includes(await input.getFormat())) return null; + const videoTrack = await input.getPrimaryVideoTrack(); + if (!videoTrack) return null; + const timestamps: number[] = []; + const packets = new EncodedPacketSink(videoTrack).packets( + undefined, + undefined, + { metadataOnly: true, skipLiveWait: true }, + ); + for await (const packet of packets) { + signal?.throwIfAborted(); + timestamps.push(packet.timestamp); + } + return fastestFrameRate(timestamps); + } finally { + input.dispose(); + } +} diff --git a/apps/media-server/src/lib/media-probe-worker.ts b/apps/media-server/src/lib/media-probe-worker.ts new file mode 100644 index 00000000000..52868295964 --- /dev/null +++ b/apps/media-server/src/lib/media-probe-worker.ts @@ -0,0 +1,71 @@ +import type { WorkerResponse } from "./isolated-worker-pool"; +import { MediaAddressError } from "./media-address-guard"; +import { MediaInputPolicyError } from "./media-input-policy"; +import type { + ProbeWorkerFailure, + ProbeWorkerRequest, +} from "./media-probe-pool"; +import { + computeTrackSpans, + MediaInputNotAccessibleError, + measureFastestFrameRate, + PARSER_FAULT_EXIT_CODE, + probeMedia, + UnreadableMediaInputError, +} from "./media-probe-read"; + +function describe(error: unknown): ProbeWorkerFailure { + const message = error instanceof Error ? error.message : String(error); + if (error instanceof MediaInputPolicyError) + return { kind: "policy", message }; + if (error instanceof MediaAddressError) return { kind: "address", message }; + if (error instanceof MediaInputNotAccessibleError) { + return { kind: "not-accessible", message, status: error.status }; + } + if (error instanceof UnreadableMediaInputError) { + const cause = error.cause; + return { + kind: "unreadable", + message, + ...(cause !== undefined && { + cause: cause instanceof Error ? cause.message : String(cause), + }), + }; + } + if (error instanceof Error && error.name === "UnsupportedInputFormatError") { + return { kind: "unsupported-format", message }; + } + return { kind: "error", message }; +} + +async function run(request: ProbeWorkerRequest): Promise { + // The server reads this on every request, so a worker started earlier + // follows it too. + if (request.internalMediaHosts === null) { + delete process.env.MEDIA_SERVER_INTERNAL_MEDIA_HOSTS; + } else { + process.env.MEDIA_SERVER_INTERNAL_MEDIA_HOSTS = request.internalMediaHosts; + } + if (request.op === "metadata") return await probeMedia(request.path); + if (request.op === "trackSpans") return await computeTrackSpans(request.path); + return await measureFastestFrameRate(request.path); +} + +function reply(response: WorkerResponse): void { + process.send?.(response); +} + +process.on("message", (request: ProbeWorkerRequest) => { + run(request).then( + (value) => reply({ id: request.id, ok: true, value }), + (error) => reply({ id: request.id, ok: false, error: describe(error) }), + ); +}); + +// A fault inside the parser that escapes the request (a rejection it left +// behind, a throw from a timer) ends this worker, not the server. +process.on("uncaughtException", () => process.exit(PARSER_FAULT_EXIT_CODE)); +process.on("unhandledRejection", () => {}); +process.on("disconnect", () => process.exit(0)); + +reply({ id: 0, ok: true, value: "ready" }); diff --git a/apps/media-server/src/lib/media-probe.ts b/apps/media-server/src/lib/media-probe.ts index 6d2eb170cd9..14b8857d2cb 100644 --- a/apps/media-server/src/lib/media-probe.ts +++ b/apps/media-server/src/lib/media-probe.ts @@ -1,324 +1,262 @@ -import { - type AudioCodec, - EncodedPacketSink, - type InputFormat, - type InputTrack, - MATROSKA, - type MediaCodec, - MP4, - QTFF, - type VideoCodec, - WEBM, -} from "mediabunny"; import type { VideoMetadata } from "./job-manager"; -import { MediaAddressError } from "./media-address-guard"; -import { - createGuardedMediaFetch, - createMediaInput, - getSourceSize, - isHlsPath, - mediaInputRefusal, - normalizeLocalPath, - withTimeout, -} from "./media-common"; +import { createGuardedMediaFetch, normalizeLocalPath } from "./media-common"; import { - exceedsSourceFrameLimit, - looksLikeTextContainer, - MediaInputPolicyError, + isAllowedVideoCodec, + isDecoderRejectedMessage, + UNSUPPORTED_FORMAT_MESSAGE, } from "./media-input-policy"; import { canAcceptNewProbeOperation, getActiveProbeOperationCount, withMediaOperation, } from "./media-operations"; +import { runProbe } from "./media-probe-pool"; +import { + NO_VIDEO_STREAM_MESSAGE, + type ProbedMedia, + type TrackSpans, + UnreadableMediaInputError, +} from "./media-probe-read"; +import { spawn } from "./media-spawn"; import { materializeMedia } from "./media-transfer"; +import { registerSubprocess, terminateProcess } from "./subprocess"; + +export { + assertRemotePlaylistStaysOnHost, + FRAME_TOO_LARGE_MESSAGE, + fastestFrameRate, + isUnreadableMediaInput, + MediaInputNotAccessibleError, + playlistReferences, + type TrackSpans, + UNFINISHED_RECORDING_MESSAGE, + UnreadableMediaInputError, +} from "./media-probe-read"; const PROBE_TIMEOUT_MS = 30_000; +const FFPROBE_TIMEOUT_MS = 10_000; +const FFPROBE_OUTPUT_LIMIT = 256 * 1024; export { canAcceptNewProbeOperation, getActiveProbeOperationCount }; export const canAcceptNewProbeProcess = canAcceptNewProbeOperation; export const getActiveProbeProcessCount = getActiveProbeOperationCount; -function mapVideoCodec(codec: VideoCodec | MediaCodec | null): string { - if (codec === "avc") return "h264"; - if (codec === "hevc") return "hevc"; - return codec ?? "unknown"; -} - -function mapAudioCodec(codec: AudioCodec | MediaCodec | null): string | null { - if (!codec) return null; - return codec; -} - -function roundFps(fps: number): number { - if (!Number.isFinite(fps) || fps <= 0) return 0; - return Math.round(fps * 100) / 100; -} - -function isHttpUrl(path: string): boolean { - try { - const url = new URL(path); - return url.protocol === "http:" || url.protocol === "https:"; - } catch { - return false; - } -} - -const SNIFF_BYTES = 4096; - -// The input itself cannot be used, so asking again would give the same answer. -export class UnreadableMediaInputError extends Error { - override name = "UnreadableMediaInputError"; -} - -// Not a policy violation: those count against the uploader, and a camera -// original that is simply too big should not. -export const FRAME_TOO_LARGE_MESSAGE = - "Video frame is larger than 8K, the largest size this server accepts"; - -export function isUnreadableMediaInput(error: unknown): boolean { - return ( - error instanceof UnreadableMediaInputError || - error instanceof MediaInputPolicyError || - (error instanceof Error && error.name === "UnsupportedInputFormatError") - ); -} - -// The first bytes of a remote input, an empty array for an empty file, or -// null when it cannot be fetched. -async function readHttpHead(path: string): Promise { +async function readLimited( + stream: ReadableStream, + limit: number, +): Promise { + const reader = stream.getReader(); + const decoder = new TextDecoder(); + let text = ""; try { - const response = await createGuardedMediaFetch(path)(path, { - headers: { Range: `bytes=0-${SNIFF_BYTES - 1}` }, - signal: AbortSignal.timeout(10_000), - }); - if (response.status === 416) { - await response.body?.cancel(); - return new Uint8Array(0); - } - if (!response.ok) { - await response.body?.cancel(); - return null; - } - const reader = response.body?.getReader(); - if (!reader) return new Uint8Array(0); - const head = new Uint8Array(SNIFF_BYTES); - let length = 0; - while (length < SNIFF_BYTES) { + while (text.length < limit) { const { done, value } = await reader.read(); if (done) break; - const chunk = value.subarray(0, SNIFF_BYTES - length); - head.set(chunk, length); - length += chunk.length; + text += decoder.decode(value, { stream: true }); } + } finally { await reader.cancel().catch(() => {}); - return head.subarray(0, length); - } catch (error) { - if ( - error instanceof MediaInputPolicyError || - error instanceof MediaAddressError - ) - throw error; - return null; - } -} - -const MAX_PLAYLIST_BYTES = 1_000_000; -const MAX_NESTED_PLAYLISTS = 16; -const PLAYLIST_WALK_TIMEOUT_MS = 15_000; -const OFF_HOST_PLAYLIST = - "a remote playlist may only reference files on its own host"; - -async function readBoundedText(response: Response): Promise { - const reader = response.body?.getReader(); - if (!reader) return ""; - const chunks: Uint8Array[] = []; - let length = 0; - while (length < MAX_PLAYLIST_BYTES) { - const { done, value } = await reader.read(); - if (done) break; - chunks.push(value); - length += value.length; } - await reader.cancel().catch(() => {}); - return new TextDecoder().decode(Buffer.concat(chunks)); + return text; } -export function playlistReferences(playlist: string): string[] { - const references: string[] = []; - for (const raw of playlist.split(/\r?\n/)) { - const line = raw.trim(); - if (!line) continue; - if (!line.startsWith("#")) { - references.push(line); - continue; - } - for (const match of line.matchAll(/URI="([^"]*)"/g)) { - if (match[1]) references.push(match[1]); +// Null when ffprobe could not be run to an answer in time. +async function runFfprobe( + args: string[], +): Promise<{ exitCode: number; stdout: string; stderr: string } | null> { + const proc = registerSubprocess( + spawn({ cmd: ["ffprobe", ...args], stdout: "pipe", stderr: "pipe" }), + ); + const timer = setTimeout(() => { + void terminateProcess(proc); + }, FFPROBE_TIMEOUT_MS); + try { + const [stdout, stderr] = await Promise.all([ + readLimited( + proc.stdout as ReadableStream, + FFPROBE_OUTPUT_LIMIT, + ), + readLimited(proc.stderr as ReadableStream, 8192), + ]); + if (stdout.length >= FFPROBE_OUTPUT_LIMIT) { + await terminateProcess(proc); + return { exitCode: 0, stdout, stderr }; } + const exitCode = await proc.exited; + return { exitCode, stdout, stderr }; + } catch { + return null; + } finally { + clearTimeout(timer); + await terminateProcess(proc); } - return references; } -// Read before the demuxer sees a remote playlist, so one that names another -// host is refused without any of its entries being requested. Playlists it -// links to on its own host are checked the same way. -export async function assertRemotePlaylistStaysOnHost( - rootUrl: string, - timeoutMs = PLAYLIST_WALK_TIMEOUT_MS, +// The parser names the codecs browsers play. A track in any other codec is +// named by ffprobe, which is held to the decoders this server allows: a file +// whose video codec is not on the list, or cannot be named at all, is +// refused here with a reason, before any job is queued. +async function assertSupportedCodecs( + path: string, + metadata: VideoMetadata, ): Promise { - // One budget for the whole walk, shorter than the probe's own timeout, so - // no request from it is still running once the probe has given up. - const signal = AbortSignal.timeout(timeoutMs); - const root = new URL(rootUrl); - const fetchPlaylist = createGuardedMediaFetch(rootUrl); - const queue = [rootUrl]; - const seen = new Set(); - for (let current = queue.shift(); current; current = queue.shift()) { - if (seen.has(current)) continue; - seen.add(current); - if (seen.size > MAX_NESTED_PLAYLISTS) { - throw new MediaInputPolicyError( - "the playlist links to too many other playlists", - ); - } - let playlist: string | null = null; - try { - const response = await fetchPlaylist(current, { signal }); - if (response.ok) playlist = await readBoundedText(response); - else await response.body?.cancel().catch(() => {}); - } catch (error) { - if (!signal.aborted) throw error; - throw new Error( - `Operation timed out after ${timeoutMs}ms reading the playlist`, - ); - } - if (playlist === null) continue; - for (const reference of playlistReferences(playlist)) { - let target: URL; - try { - target = new URL(reference, current); - } catch { - throw new MediaInputPolicyError(OFF_HOST_PLAYLIST); - } - if (target.origin !== root.origin) { - throw new MediaInputPolicyError(OFF_HOST_PLAYLIST); - } - if (isHlsPath(target.href)) { - if (!target.search) target.search = root.search; - queue.push(target.href); - } - } + const unnamedVideo = metadata.videoCodec === "unknown"; + const unnamedAudio = + metadata.audioCodec === null && metadata.audioChannels !== null; + if (!unnamedVideo && !unnamedAudio) return; + const result = await runFfprobe([ + "-v", + "error", + "-show_entries", + "stream=codec_type,codec_name", + "-of", + "json", + path, + ]); + if (result === null) { + if (!unnamedVideo) return; + throw new Error("Could not read the video codec of the source in time"); } -} - -async function probeMedia(path: string): Promise { - if (isHttpUrl(path)) { - const head = await readHttpHead(path); - if (!head) throw new Error("Media input is not accessible"); - if (head.length === 0) { - throw new UnreadableMediaInputError("Media input is empty"); + if (result.exitCode !== 0) { + if (isDecoderRejectedMessage(result.stderr)) { + throw new UnreadableMediaInputError(UNSUPPORTED_FORMAT_MESSAGE); } - // A remote file that is really a playlist would have its entries fetched - // by the demuxer. Only a URL that names itself a playlist may be one. - if (!isHlsPath(path) && looksLikeTextContainer(head)) { - throw new MediaInputPolicyError( - "the file is a playlist or resource list, not a media file", - ); + // Anything else (a source that failed to fetch, a probe stopped at its + // deadline) says nothing about the format, so it can be asked again. + if (unnamedVideo) { + throw new Error("Could not read the video codec of the source"); } - if (isHlsPath(path)) await assertRemotePlaylistStaysOnHost(path); + return; } - - const input = createMediaInput(path); - + if (!unnamedVideo) return; + let streams: { codec_type?: string; codec_name?: string }[] = []; try { - return await readMediaMetadata(input, path); - } catch (error) { - const refusal = mediaInputRefusal(input); - if (refusal) throw new MediaInputPolicyError(refusal); - throw error; - } finally { - input.dispose(); + streams = + (JSON.parse(result.stdout) as { streams?: typeof streams }).streams ?? []; + } catch {} + const video = streams.find((stream) => stream.codec_type === "video"); + if (!video || !isAllowedVideoCodec(video.codec_name ?? "")) { + throw new UnreadableMediaInputError(UNSUPPORTED_FORMAT_MESSAGE); } } -// The last timestamp is the length only for a file that starts at zero. A -// transport stream usually does not, so the length is counted from its first -// timestamp. -async function measureLength( - input: ReturnType, -): Promise { - const [start, end] = await Promise.all([ - input.getFirstTimestamp(), - input.computeDuration(undefined, { skipLiveWait: true }), +const DURATION_TOLERANCE_SECONDS = 1; +const DURATION_TOLERANCE_RATIO = 0.01; + +export function durationTolerance(declared: number): number { + return Math.max( + DURATION_TOLERANCE_SECONDS, + declared * DURATION_TOLERANCE_RATIO, + ); +} + +const PACKETS_READ_PAST_DECLARED_END = 2000; + +// Whether FFmpeg finds video to decode past the declared end. An edit list +// hides the part of a file that was trimmed away, and FFmpeg honours it, so +// such a file really is as short as it declares. The read starts at the +// keyframe before the declared end and stops after a bounded number of +// packets. Null when unknown. +async function hasPacketsBeyond( + path: string, + declaredEnd: number, +): Promise { + const result = await runFfprobe([ + "-v", + "error", + "-select_streams", + "v:0", + "-read_intervals", + `${declaredEnd}%+#${PACKETS_READ_PAST_DECLARED_END}`, + "-show_entries", + "packet=pts_time", + "-of", + "csv=p=0", + path, ]); - return Number.isFinite(start) && start > 0 ? Math.max(0, end - start) : end; + if (result === null || (result.exitCode !== 0 && result.stdout === "")) { + return null; + } + for (const line of result.stdout.split("\n")) { + const time = Number.parseFloat(line); + if (Number.isFinite(time) && time >= declaredEnd) return true; + } + return false; } -async function readMediaMetadata( - input: ReturnType, +// A header can declare a shorter length than the file plays for. The length +// reported is what the encoder will be held to and what is billed, so when +// the timestamps run past the declared end, and FFmpeg agrees there is video +// there, the measured length is the one reported. +async function withTrueDuration( path: string, + probed: ProbedMedia, ): Promise { - const canRead = await input.canRead(); - if (!canRead) { - throw new UnreadableMediaInputError("Media input could not be read"); + const { metadata, declaredDuration, measuredDuration } = probed; + if (declaredDuration === null || measuredDuration === null) return metadata; + const declaredEnd = declaredDuration + durationTolerance(declaredDuration); + if (measuredDuration <= declaredEnd) return metadata; + if (probed.mayTrim && (await hasPacketsBeyond(path, declaredEnd)) === false) { + return metadata; } + return { + ...metadata, + duration: measuredDuration, + bitrate: + probed.trackBitrate || + (metadata.fileSize > 0 + ? Math.round((metadata.fileSize * 8) / measuredDuration) + : 0), + }; +} - const videoTrack = await input.getPrimaryVideoTrack(); - if (!videoTrack) { - throw new UnreadableMediaInputError("No video stream found"); - } - if ( - exceedsSourceFrameLimit( - await videoTrack.getCodedWidth(), - await videoTrack.getCodedHeight(), - ) || - exceedsSourceFrameLimit( - await videoTrack.getDisplayWidth(), - await videoTrack.getDisplayHeight(), - ) - ) { - throw new UnreadableMediaInputError(FRAME_TOO_LARGE_MESSAGE); +// Whether a file the parser found no video in has a video stream in a codec +// this server does not take (MPEG-2 in a transport stream, for one), as +// opposed to having none. Cover art does not count. +async function hasUnsupportedVideoStream(path: string): Promise { + const result = await runFfprobe([ + "-v", + "error", + "-show_entries", + "stream=codec_type,codec_name:stream_disposition=attached_pic", + "-of", + "json", + path, + ]); + if (result === null) return false; + if (result.exitCode !== 0) return isDecoderRejectedMessage(result.stderr); + try { + const { streams = [] } = JSON.parse(result.stdout) as { + streams?: { + codec_type?: string; + disposition?: { attached_pic?: number }; + }[]; + }; + return streams.some( + (stream) => + stream.codec_type === "video" && stream.disposition?.attached_pic !== 1, + ); + } catch { + return false; } +} - const audioTrack = await input.getPrimaryAudioTrack(); - const duration = - (await input.getDurationFromMetadata(undefined, { - skipLiveWait: true, - })) ?? (await measureLength(input)); - const packetStats = await videoTrack - .computePacketStats(120, { skipLiveWait: true }) - .catch(() => null); - const videoBitrate = - (await videoTrack.getAverageBitrate()) ?? - (await videoTrack.getBitrate()) ?? - packetStats?.averageBitrate ?? - 0; - const audioBitrate = audioTrack - ? ((await audioTrack.getAverageBitrate()) ?? - (await audioTrack.getBitrate()) ?? - 0) - : 0; - const fileSize = await getSourceSize(path); - const bitrateFromSize = - duration > 0 && fileSize > 0 ? Math.round((fileSize * 8) / duration) : 0; - - const metadata: VideoMetadata = { - duration, - width: await videoTrack.getDisplayWidth(), - height: await videoTrack.getDisplayHeight(), - fps: roundFps(packetStats?.averagePacketRate ?? 0), - videoCodec: mapVideoCodec(await videoTrack.getCodec()), - audioCodec: mapAudioCodec((await audioTrack?.getCodec()) ?? null), - audioChannels: audioTrack ? await audioTrack.getNumberOfChannels() : null, - sampleRate: audioTrack ? await audioTrack.getSampleRate() : null, - bitrate: Math.round(videoBitrate + audioBitrate) || bitrateFromSize, - fileSize, - }; - const refusal = mediaInputRefusal(input); - if (refusal) throw new MediaInputPolicyError(refusal); - return metadata; +async function probeMetadata(path: string): Promise { + const probed = await runProbe("metadata", path, PROBE_TIMEOUT_MS).catch( + async (error: unknown) => { + if ( + error instanceof UnreadableMediaInputError && + error.message === NO_VIDEO_STREAM_MESSAGE && + (await hasUnsupportedVideoStream(path)) + ) { + throw new UnreadableMediaInputError(UNSUPPORTED_FORMAT_MESSAGE); + } + throw error; + }, + ); + await assertSupportedCodecs(path, probed.metadata); + return await withTrueDuration(path, probed); } export async function probeVideo(videoUrl: string): Promise { @@ -328,9 +266,7 @@ export async function probeVideo(videoUrl: string): Promise { throw new Error("Server is busy, please try again later"); } - return await withMediaOperation("probe", () => - withTimeout(probeMedia(videoUrl), PROBE_TIMEOUT_MS), - ); + return await withMediaOperation("probe", () => probeMetadata(videoUrl)); } export async function probeVideoFile(filePath: string): Promise { @@ -339,98 +275,20 @@ export async function probeVideoFile(filePath: string): Promise { } return await withMediaOperation("probe", () => - withTimeout(probeMedia(normalizeLocalPath(filePath)), PROBE_TIMEOUT_MS), + probeMetadata(normalizeLocalPath(filePath)), ); } -export type TrackSpans = { video: number; audio: number | null }; - -async function computeTrackSpans(path: string): Promise { - const input = createMediaInput(path); - try { - const videoTrack = await input.getPrimaryVideoTrack(); - if (!videoTrack) { - throw new Error("No video stream found"); - } - const audioTrack = await input.getPrimaryAudioTrack(); - const span = async (track: InputTrack) => { - const [start, end] = await Promise.all([ - track.getFirstTimestamp(), - track.computeDuration({ skipLiveWait: true }), - ]); - return end - start; - }; - const [video, audio] = await Promise.all([ - span(videoTrack), - audioTrack ? span(audioTrack) : null, - ]); - return { video, audio }; - } finally { - input.dispose(); - } -} - export async function probeTrackSpans(filePath: string): Promise { if (!canAcceptNewProbeOperation()) { throw new Error("Server is busy, please try again later"); } return await withMediaOperation("probe", () => - withTimeout( - computeTrackSpans(normalizeLocalPath(filePath)), - PROBE_TIMEOUT_MS, - ), + runProbe("trackSpans", normalizeLocalPath(filePath), PROBE_TIMEOUT_MS), ); } -const FASTEST_STRETCH_FRAMES = 8; -// Containers whose frame times come from an index or from cluster headers. -// Timing every frame of a transport stream means reading the whole file -// through the demuxer: 3 s and 770 MB of memory for a 1.5 GB file, against -// 11 ms for an MP4 of the same length. -const FRAME_TIMED_FORMATS: readonly InputFormat[] = [MP4, QTFF, MATROSKA, WEBM]; - -// The frame rate of the fastest run of consecutive frames. A run is long enough -// that two frames landing close together do not count, and short enough that -// a burst of motion a quarter of a second long does. -export function fastestFrameRate(timestamps: number[]): number | null { - const times = [...timestamps].sort((a, b) => a - b); - const run = Math.min(FASTEST_STRETCH_FRAMES, times.length - 1); - if (run < 1) return null; - let fastest = 0; - for (let index = run; index < times.length; index++) { - const seconds = (times[index] ?? 0) - (times[index - run] ?? 0); - if (seconds > 0) fastest = Math.max(fastest, run / seconds); - } - const fps = roundFps(fastest); - return fps > 0 ? fps : null; -} - -async function measureFastestFrameRate( - path: string, - signal: AbortSignal, -): Promise { - const input = createMediaInput(path); - try { - if (!FRAME_TIMED_FORMATS.includes(await input.getFormat())) return null; - const videoTrack = await input.getPrimaryVideoTrack(); - if (!videoTrack) return null; - const timestamps: number[] = []; - const packets = new EncodedPacketSink(videoTrack).packets( - undefined, - undefined, - { metadataOnly: true, skipLiveWait: true }, - ); - for await (const packet of packets) { - signal.throwIfAborted(); - timestamps.push(packet.timestamp); - } - return fastestFrameRate(timestamps); - } finally { - input.dispose(); - } -} - export class ProbeBusyError extends Error { override name = "ProbeBusyError"; @@ -447,15 +305,12 @@ export async function probeFastestFrameRate( abortSignal?: AbortSignal, ): Promise { if (!canAcceptNewProbeOperation()) throw new ProbeBusyError(); - const stopped = new AbortController(); - const signal = abortSignal - ? AbortSignal.any([abortSignal, stopped.signal]) - : stopped.signal; return await withMediaOperation("probe", () => - withTimeout( - measureFastestFrameRate(normalizeLocalPath(filePath), signal), + runProbe( + "fastestFrameRate", + normalizeLocalPath(filePath), PROBE_TIMEOUT_MS, - () => stopped.abort(), + abortSignal, ), ); } diff --git a/apps/media-server/src/lib/media-spawn.ts b/apps/media-server/src/lib/media-spawn.ts index f4db7fc827d..d7fd96150d8 100644 --- a/apps/media-server/src/lib/media-spawn.ts +++ b/apps/media-server/src/lib/media-spawn.ts @@ -2,6 +2,7 @@ import { spawn as nodeSpawn } from "node:child_process"; import { basename } from "node:path"; import { spawn as bunSpawn } from "bun"; import { + DECODER_WHITELIST, getSafeDemuxerWhitelist, isTrustedLocalPlaylist, MAX_DECODE_PIXELS, @@ -49,6 +50,10 @@ function hardenFfmpegArgs(cmd: readonly string[]): string[] { !isTrustedLocalPlaylist(input) ) group.push("-format_whitelist", getSafeDemuxerWhitelist()); + // An input option: it reaches the decoders of this input and the + // ones the demuxer opens while it reads the stream parameters. + if (!group.includes("-codec_whitelist")) + group.push("-codec_whitelist", DECODER_WHITELIST); } out.push(...group, "-i", input); group = []; @@ -77,6 +82,8 @@ function hardenFfprobeArgs(cmd: readonly string[]): string[] { !args.some(isTrustedLocalPlaylist) ) extra.push("-format_whitelist", getSafeDemuxerWhitelist()); + if (!args.includes("-codec_whitelist")) + extra.push("-codec_whitelist", DECODER_WHITELIST); return [cmd[0] as string, ...extra, ...args]; } diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts index 61fdcdb8ed6..28c8897f779 100644 --- a/apps/media-server/src/lib/media-video.ts +++ b/apps/media-server/src/lib/media-video.ts @@ -24,7 +24,9 @@ import { createGuardedMediaFetch, DOWNLOAD_TIMEOUT_MS, decodeThreadsFromEnv, + encodeLengthLimit, normalizeLocalPath, + outputLengthCapArgs, PROCESS_TIMEOUT_MS, type ProgressCallback, UPLOAD_TIMEOUT_MS, @@ -1772,6 +1774,60 @@ export async function requireVideoSourceTraits( throw new Error("Could not read the colour information of the source video"); } +const EIGHT_BIT_PIXEL_FORMATS = new Set([ + "yuv420p", + "yuvj420p", + "yuv422p", + "yuv444p", + "yuva420p", +]); + +// VP8 is always 8-bit standard range, and 8-bit VP9 is what browser and +// extension recordings are, so such a source whose colour tags cannot be +// read is encoded as standard range instead of failing. VP9 at a higher bit +// depth (profile 2) can carry HDR, so it keeps the requirement. +export function colourProbeIsOptional( + videoCodec: string, + pixelFormat: string | null, +): boolean { + if (videoCodec === "vp8") return true; + return ( + videoCodec === "vp9" && + pixelFormat !== null && + EIGHT_BIT_PIXEL_FORMATS.has(pixelFormat) + ); +} + +async function videoSourceTraitsForEncode( + inputPath: string, + videoCodec: string, + abortSignal?: AbortSignal, +): Promise { + if (videoCodec === "vp8") { + return await probeVideoSourceTraits(inputPath, abortSignal); + } + try { + return await requireVideoSourceTraits(inputPath, abortSignal); + } catch (error) { + abortSignal?.throwIfAborted(); + if ( + videoCodec !== "vp9" || + !(error instanceof Error) || + !error.message.startsWith("Could not read the colour information") || + !colourProbeIsOptional( + videoCodec, + await probeH264PixelFormat(inputPath, abortSignal), + ) + ) { + throw error; + } + console.warn( + "[media-video] Could not read the colour information of an 8-bit VP9 source, encoding it as standard range", + ); + return { rotation: null, hdr: null }; + } +} + export async function processVideo( inputPath: string, metadata: VideoMetadata, @@ -1802,7 +1858,11 @@ export async function processVideo( const traits = remuxOnly || (metadata.videoCodec === "h264" && browserPixelFormat) ? null - : await requireVideoSourceTraits(inputPath, abortSignal); + : await videoSourceTraitsForEncode( + inputPath, + metadata.videoCodec, + abortSignal, + ); abortSignal?.throwIfAborted(); const hdr = traits?.hdr ?? null; // Created only once nothing above can throw, so a job cancelled while its @@ -1838,6 +1898,9 @@ export async function processVideo( opts.timeoutMs, metadata, ); + // A damaged source is read with its timestamps rebuilt, which the probe + // did not see, so a recovery attempt is bounded by its deadline alone. + const maxOutputSeconds = resilientFlags ? null : encodeLengthLimit(metadata); const capcodec = videoTranscode && !normalizeH264Level && hdr === null ? capcodecOptionsForJob(opts) @@ -1854,6 +1917,7 @@ export async function processVideo( extraInputArgs, extraOutputArgs, processTimeoutMs, + maxOutputSeconds, onProgress, abortSignal, ); @@ -1881,6 +1945,7 @@ export async function processVideo( sourceHeight: metadata.height, hdr, hdrColor: traits?.hdrColor, + maxOutputSeconds, // NVENC leaves the CPU free, so decode (and scale) can use every core. decodeThreads: decodeThreadsFromEnv( "MEDIA_SERVER_PROCESS_DECODE_THREADS", @@ -1971,6 +2036,7 @@ export interface ProcessVideoArgsInput { hdr?: HdrTransfer | null; hdrColor?: HdrColor | null; nvenc?: { options: NvencOptions; gpuDecode: boolean } | null; + maxOutputSeconds?: number | null; } export function buildProcessVideoArgs(input: ProcessVideoArgsInput): string[] { @@ -2078,6 +2144,7 @@ export function buildProcessVideoArgs(input: ProcessVideoArgsInput): string[] { } ffmpegArgs.push( + ...outputLengthCapArgs(input.maxOutputSeconds), "-movflags", "+faststart", ...input.extraOutputArgs, @@ -2198,6 +2265,7 @@ async function processVideoWithCapcodec( extraInputArgs: string[], extraOutputArgs: string[], processTimeoutMs: number, + maxOutputSeconds: number | null, onProgress?: ProgressCallback, abortSignal?: AbortSignal, ): Promise { @@ -2215,6 +2283,7 @@ async function processVideoWithCapcodec( maxWidth: opts.maxWidth, maxHeight: opts.maxHeight, extraInputArgs, + maxOutputSeconds, options: capcodec, totalDurationUs: metadata.duration * 1_000_000, onProgress, @@ -2253,6 +2322,7 @@ async function processVideoWithCapcodec( muxArgs.push("-map", "0:v:0", "-c:v", "copy", "-an"); } muxArgs.push( + ...outputLengthCapArgs(maxOutputSeconds), "-movflags", "+faststart", ...extraOutputArgs, diff --git a/apps/media-server/src/lib/peer-jobs.ts b/apps/media-server/src/lib/peer-jobs.ts index 99318e2a13b..a22aa543866 100644 --- a/apps/media-server/src/lib/peer-jobs.ts +++ b/apps/media-server/src/lib/peer-jobs.ts @@ -26,20 +26,29 @@ export function isPeerLookup(request: Request): boolean { return request.headers.get(PEER_LOOKUP_HEADER) === "1"; } -export async function findJobOnPeer( +// What the peers said about a job this instance does not hold. "missing" +// means every peer answered and none has it. "unavailable" means at least +// one peer could not be asked, so the job may still exist. +export type PeerJobLookup = + | { found: Response } + | { found: null; outcome: "missing" | "unavailable" }; + +const NOT_ON_PEER = "Job is not on this peer"; + +export async function lookUpJobOnPeers( path: string, method: "GET" | "POST", -): Promise { +): Promise { const secret = process.env.MEDIA_SERVER_WEBHOOK_SECRET; const peers = mediaServerPeers(); - if (!secret || peers.length === 0) return null; + if (!secret || peers.length === 0) return { found: null, outcome: "missing" }; const controller = new AbortController(); const timeout = setTimeout( () => controller.abort(), method === "POST" ? PEER_CANCEL_TIMEOUT_MS : PEER_STATUS_TIMEOUT_MS, ); try { - return await Promise.any( + const found = await Promise.any( peers.map(async (peer) => { const response = await fetch(`${peer}${path}`, { method, @@ -54,7 +63,14 @@ export async function findJobOnPeer( // Only a peer that owns the job answers 200, or 400 for a finished job's cancel. if (response.status !== 200 && response.status !== 400) { await response.body?.cancel().catch(() => {}); - throw new Error("Job is not on this peer"); + // Only a 404 says the job is not there. A peer that refuses + // the secret (as during a rotation), is starting, overloaded + // or failing may still own it. + throw new Error( + response.status === 404 + ? NOT_ON_PEER + : `Peer answered ${response.status}`, + ); } const body = await response.text(); return new Response(body, { @@ -66,10 +82,22 @@ export async function findJobOnPeer( }); }), ); - } catch { - return null; + return { found }; + } catch (error) { + const failures = error instanceof AggregateError ? error.errors : [error]; + const allAnswered = failures.every( + (failure) => failure instanceof Error && failure.message === NOT_ON_PEER, + ); + return { found: null, outcome: allAnswered ? "missing" : "unavailable" }; } finally { clearTimeout(timeout); controller.abort(); } } + +export async function findJobOnPeer( + path: string, + method: "GET" | "POST", +): Promise { + return (await lookUpJobOnPeers(path, method)).found; +} diff --git a/apps/media-server/src/lib/process-hardening.ts b/apps/media-server/src/lib/process-hardening.ts new file mode 100644 index 00000000000..701a140ad81 --- /dev/null +++ b/apps/media-server/src/lib/process-hardening.ts @@ -0,0 +1,48 @@ +// FFmpeg, ffprobe and the probe workers run as the same user as the server, +// and read files nobody vouches for. A process the kernel marks as not +// dumpable keeps its /proc entries (environ, mem, maps) and ptrace from +// other processes of that user, so a decoder taken over by a hostile file +// cannot read the secrets in the server's environment. Children start +// dumpable again when they exec, so this reaches only the server. + +const PR_GET_DUMPABLE = 3; +const PR_SET_DUMPABLE = 4; + +type Prctl = (option: number, arg: number) => number; + +function loadPrctl(): Prctl | null { + if (process.platform !== "linux") return null; + try { + const { dlopen, FFIType } = require("bun:ffi") as typeof import("bun:ffi"); + for (const name of ["libc.so.6", "libc.so"]) { + try { + const { symbols } = dlopen(name, { + prctl: { + args: [ + FFIType.i32, + FFIType.u64, + FFIType.u64, + FFIType.u64, + FFIType.u64, + ], + returns: FFIType.i32, + }, + }); + return (option, arg) => symbols.prctl(option, arg, 0, 0, 0) as number; + } catch {} + } + } catch {} + return null; +} + +// Whether the server is now hidden from its own children. False where the +// platform has no such switch. +export function hideServerFromChildren(): boolean { + const prctl = loadPrctl(); + if (!prctl) return false; + if (prctl(PR_SET_DUMPABLE, 0) !== 0) { + console.warn("[media-server] Could not mark the server as not dumpable"); + return false; + } + return prctl(PR_GET_DUMPABLE, 0) === 0; +} diff --git a/apps/media-server/src/lib/runtime-report.ts b/apps/media-server/src/lib/runtime-report.ts index bfe0bace96d..b852c19c0cc 100644 --- a/apps/media-server/src/lib/runtime-report.ts +++ b/apps/media-server/src/lib/runtime-report.ts @@ -10,6 +10,7 @@ import { getMaxConcurrentAudioOperations, getMaxConcurrentProbeOperations, } from "./media-operations"; +import { getProbeWorkerStats } from "./media-probe-pool"; import { getMediaTransferCacheUsage } from "./media-transfer"; import { getTempDirUsage } from "./temp-files"; import { @@ -22,6 +23,8 @@ export async function getRuntimeReport() { const jobs = getAllJobs(); return { draining: isDraining(), + runsAsUid: process.getuid?.() ?? null, + probeWorkers: getProbeWorkerStats(), shutdownGraceMs: getShutdownGraceMs(), processUptimeSeconds: Math.round(process.uptime()), activeJobs: jobs.filter( diff --git a/apps/media-server/src/routes/audio.ts b/apps/media-server/src/routes/audio.ts index bd6f7eb669b..1eed21d1c79 100644 --- a/apps/media-server/src/routes/audio.ts +++ b/apps/media-server/src/routes/audio.ts @@ -47,6 +47,12 @@ function isTimeoutError(err: unknown): boolean { return err instanceof Error && err.message.includes("timed out"); } +// A source with no streams at all is a problem with the input, not the +// server, so it is not worth retrying. +function isNoMediaStreamError(err: unknown): boolean { + return err instanceof Error && err.message === "No media stream found"; +} + function drainingResponse(c: Context) { c.header("Retry-After", DRAIN_RETRY_AFTER_SECONDS.toString()); return c.json( @@ -119,6 +125,17 @@ audio.post("/check", async (c) => { ); } + if (isNoMediaStreamError(err)) { + return c.json( + { + error: "Failed to check audio track", + code: "NO_MEDIA_STREAM", + details: (err as Error).message, + }, + 422, + ); + } + return c.json( { error: "Failed to check audio track", @@ -220,6 +237,17 @@ audio.post("/extract", async (c) => { ); } + if (isNoMediaStreamError(err)) { + return c.json( + { + error: "Failed to extract audio", + code: "NO_MEDIA_STREAM", + details: (err as Error).message, + }, + 422, + ); + } + return c.json( { error: "Failed to extract audio", diff --git a/apps/media-server/src/routes/health.ts b/apps/media-server/src/routes/health.ts index 41d367bb208..dafa9193d1c 100644 --- a/apps/media-server/src/routes/health.ts +++ b/apps/media-server/src/routes/health.ts @@ -11,6 +11,21 @@ const health = new Hono(); health.get("/", async (c) => { const { version, ...mediaEngine } = getMediaEngineStatus(); + const resources = getSystemResources(); + const draining = isDraining(); + const status = draining + ? "draining" + : mediaEngine.available && resources.effectiveMax > 0 + ? "ok" + : "degraded"; + + // A caller without the secret (a load balancer) learns only whether this + // instance is up. Engine versions, load, memory and slot figures are for + // callers that hold the secret. + if (!hasMediaServerSecret(c)) { + return c.json({ status }, draining ? 503 : 200); + } + const [, loadAvg5m, loadAvg15m] = os.loadavg(); const totalMemoryBytes = os.totalmem(); const freeMemoryBytes = os.freemem(); @@ -19,23 +34,13 @@ health.get("/", async (c) => { const memoryUsagePercent = Math.round( (1 - freeMemoryBytes / totalMemoryBytes) * 100, ); - const resources = getSystemResources(); - const draining = isDraining(); - const authenticated = hasMediaServerSecret(c); return c.json( { - status: draining - ? "draining" - : mediaEngine.available && resources.effectiveMax > 0 - ? "ok" - : "degraded", - mediaEngine: { ...mediaEngine, ...(authenticated && { version }) }, - ["ff" + "mpeg"]: { - available: mediaEngine.available, - ...(authenticated && { version }), - }, - videoEncoder: getVideoEncoderStatus(process.env, authenticated), + status, + mediaEngine: { ...mediaEngine, version }, + ["ff" + "mpeg"]: { available: mediaEngine.available, version }, + videoEncoder: getVideoEncoderStatus(process.env, true), system: { ...resources, loadAvg5m, @@ -46,7 +51,7 @@ health.get("/", async (c) => { uptimeSeconds: Math.round(os.uptime()), processUptimeSeconds: Math.round(process.uptime()), }, - ...(authenticated && { runtime: await getRuntimeReport() }), + runtime: await getRuntimeReport(), }, draining ? 503 : 200, ); diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts index dbbf40c09d0..e084849a975 100644 --- a/apps/media-server/src/routes/video.ts +++ b/apps/media-server/src/routes/video.ts @@ -11,6 +11,7 @@ import { } from "../lib/auth"; import { HLS_LADDER_TIER_NAMES } from "../lib/hls-ladder"; import { encodeHlsLadder } from "../lib/hls-ladder-encode"; +import { posterThumbnailOptions } from "../lib/infra-poster"; import type { Job, RecordingErrorCode, @@ -19,6 +20,7 @@ import type { import { beginRecordingProcessing, beginRecordingVerification, + canAcceptNewInfraVideoProcess, canAcceptNewVideoProcess, claimRecordingWorker, createJob, @@ -45,6 +47,7 @@ import { trackBackgroundTask, } from "../lib/lifecycle"; import { logReason } from "../lib/log-redaction"; +import { MediaAddressError } from "../lib/media-address-guard"; import { createGuardedMediaFetch, PROCESS_TIMEOUT_MS, @@ -62,6 +65,7 @@ import { canAcceptNewProbeOperation as canAcceptNewProbeProcess, getActiveProbeOperationCount as getActiveProbeProcessCount, isUnreadableMediaInput, + MediaInputNotAccessibleError, probeTrackSpans, probeVideo, probeVideoFile, @@ -95,7 +99,11 @@ import { uploadToS3, } from "../lib/media-video"; import { encodeAttemptsBudgetMs } from "../lib/nvenc-encoder"; -import { findJobOnPeer, isPeerLookup } from "../lib/peer-jobs"; +import { + findJobOnPeer, + isPeerLookup, + lookUpJobOnPeers, +} from "../lib/peer-jobs"; import { MAX_PREVERIFIED_RUN_BYTES, type PreverifiedRunPayload, @@ -142,6 +150,7 @@ const MEDIA_ENGINE_ERROR_CODE = ["FF", "MPEG_ERROR"].join(""); const PROBE_ERROR_CODE = ["FF", "PROBE_ERROR"].join(""); const UNREADABLE_INPUT_CODE = "UNREADABLE_INPUT"; const VIDEO_BUSY_RETRY_AFTER_SECONDS = 15; +const PEER_LOOKUP_RETRY_AFTER_SECONDS = 5; const VIDEO_MEMORY_PRESSURE_ERROR = "SERVER_BUSY: container memory pressure is too high"; const MUX_FAILURE_LOG_LENGTH = 1_000; @@ -222,6 +231,7 @@ const processSchema = z.object({ remuxOnly: z.boolean().optional(), normalizeH264Level: z.boolean().optional(), importedOriginal: z.boolean().optional(), + infra: z.boolean().optional(), }); const ladderSchema = z.object({ @@ -244,6 +254,7 @@ const ladderSchema = z.object({ thumbnailPresignedUrl: z.string().url().optional(), webhookUrl: z.string().url().optional(), webhookSecret: z.string().optional(), + infra: z.boolean().optional(), }); export function isCompleteRecovery( @@ -663,15 +674,15 @@ video.post("/thumbnail", async (c) => { try { const metadata = await probeVideo(result.data.videoUrl); + // Only the options the caller set: an explicit undefined would replace + // the default and reach FFmpeg as NaN. + const { videoUrl, ...requested } = result.data; const thumbnailData = await generateThumbnail( - result.data.videoUrl, + videoUrl, metadata.duration, - { - timestamp: result.data.timestamp, - width: result.data.width, - height: result.data.height, - quality: result.data.quality, - }, + Object.fromEntries( + Object.entries(requested).filter(([, value]) => value !== undefined), + ), ); return new Response(Buffer.from(thumbnailData), { @@ -681,6 +692,43 @@ video.post("/thumbnail", async (c) => { }, }); } catch (err) { + const details = err instanceof Error ? err.message : String(err); + if (isUnreadableMediaInput(err) || err instanceof MediaAddressError) { + console.warn("[video/thumbnail] Unusable input:", details); + return c.json( + { + error: "Media input cannot be used", + code: + err instanceof MediaInputPolicyError || + err instanceof MediaAddressError + ? INPUT_POLICY_VIOLATION + : UNREADABLE_INPUT_CODE, + details, + }, + 422, + ); + } + // The host answered that the file is not there or not ours to read, + // which asking again will not change. + if ( + err instanceof MediaInputNotAccessibleError && + err.status !== null && + err.status >= 400 && + err.status < 500 && + err.status !== 408 && + err.status !== 429 + ) { + console.warn("[video/thumbnail] Input not accessible:", err.status); + return c.json( + { + error: "Media input cannot be used", + code: "INPUT_NOT_ACCESSIBLE", + details: `${details} (HTTP ${err.status})`, + }, + 422, + ); + } + console.error("[video/thumbnail] Error:", err); if (isBusyError(err)) { @@ -914,7 +962,11 @@ video.on("POST", ["/process", "/import"], async (c) => { result.data.priority === "bulk" && capacity.effectiveMaxVideoProcesses > 1 && capacity.activeVideoProcesses >= capacity.effectiveMaxVideoProcesses - 1; - if (bulkCapacityReached || !canAcceptNewVideoProcess()) { + if ( + bulkCapacityReached || + !canAcceptNewVideoProcess() || + (result.data.infra === true && !canAcceptNewInfraVideoProcess(false)) + ) { c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); return c.json(getBusyResponseBody(capacity), 503); } @@ -1004,7 +1056,14 @@ video.post("/ladder", async (c) => { 503, ); } - if (!canAcceptNewVideoProcess()) { + const infraHeavyLadder = + result.data.infra === true && + (result.data.maxTier === "1440p" || result.data.maxTier === "2160p"); + if ( + !canAcceptNewVideoProcess() || + (result.data.infra === true && + !canAcceptNewInfraVideoProcess(infraHeavyLadder)) + ) { c.header("Retry-After", VIDEO_BUSY_RETRY_AFTER_SECONDS.toString()); return c.json(getBusyResponseBody(getVideoCapacitySnapshot()), 503); } @@ -1012,6 +1071,7 @@ video.post("/ladder", async (c) => { const { videoId, userId, webhookUrl, webhookSecret } = result.data; const jobId = generateJobId(); const job = createJob(jobId, videoId, userId, webhookUrl, webhookSecret); + if (infraHeavyLadder) job.infraHeavyLadder = true; void trackBackgroundTask( withMediaTransfers(32 * 1024 ** 3, () => @@ -1546,7 +1606,7 @@ async function encodeLadderAsync( ? generateThumbnail( ladder.poster.path, ladder.poster.durationSeconds, - {}, + posterThumbnailOptions(options.infra), uploadSignal, ) .then((image) => @@ -2019,7 +2079,12 @@ async function processVideoAsync( const previewAbort = new AbortController(); const previewSignal = AbortSignal.any([signal, previewAbort.signal]); const thumbnailRender = thumbnailPresignedUrl - ? generateThumbnail(outputPath, metadata.duration, {}, previewSignal) + ? generateThumbnail( + outputPath, + metadata.duration, + posterThumbnailOptions(options.infra), + previewSignal, + ) : undefined; const previewGifRender = previewGifPresignedUrl ? generatePreviewGif(outputPath, metadata.duration, {}, previewSignal) @@ -2181,11 +2246,23 @@ video.get("/process/:jobId/status", async (c) => { !isPeerLookup(c.req.raw) && !accept?.includes("text/event-stream") ) { - const owner = await findJobOnPeer( + const lookup = await lookUpJobOnPeers( `/video/process/${encodeURIComponent(jobId)}/status`, "GET", ); - if (owner) return owner; + if (lookup.found) return lookup.found; + if (lookup.outcome === "unavailable") { + c.header("Retry-After", PEER_LOOKUP_RETRY_AFTER_SECONDS.toString()); + return c.json( + { + error: "Could not ask every media server about this job", + code: "PEER_LOOKUP_FAILED", + instanceId: getInstanceId(), + pid: process.pid, + }, + 503, + ); + } } return c.json( { diff --git a/apps/web/__tests__/fixtures/developer-captions-golden.vtt b/apps/web/__tests__/fixtures/developer-captions-golden.vtt new file mode 100644 index 00000000000..a857ee0bb27 --- /dev/null +++ b/apps/web/__tests__/fixtures/developer-captions-golden.vtt @@ -0,0 +1,41 @@ +WEBVTT + +00:00:00.240 --> 00:00:01.240 +Welcome back. + +00:00:01.600 --> 00:00:06.320 +Today we are walking through the +quarterly numbers for the whole company, + +00:00:06.400 --> 00:00:13.120 +region by region, and then we will look at +what changed in the product since the last + +00:00:13.200 --> 00:00:15.920 +time we all met in this room. + +00:00:18.400 --> 00:00:21.928 +We shipped 1,250 units in +Q3 2026 for $3.5 million. + +00:00:23.200 --> 00:00:26.400 +And yes I know that is a lot to take +in all at once so bear with me please. + +00:00:26.400 --> 00:00:27.520 +Let me think + +00:00:31.200 --> 00:00:33.120 +about that for a second. + +00:00:34.400 --> 00:00:37.520 +Is 5 < 7 && 9 > 2? + +00:00:37.600 --> 00:00:40.320 +Tom & Jerry said <b>yes</b> --> maybe. + +00:00:41.600 --> 00:00:42.600 +Okay. + +00:00:43.200 --> 00:00:44.800 +Line break inside end. diff --git a/apps/web/__tests__/fixtures/developer-video-takedown.ts b/apps/web/__tests__/fixtures/developer-video-takedown.ts new file mode 100644 index 00000000000..b580466e24b --- /dev/null +++ b/apps/web/__tests__/fixtures/developer-video-takedown.ts @@ -0,0 +1,55 @@ +import type { DeveloperVideoStore } from "./developer-video"; + +// The takedown rewrites metadata in SQL. This applies the same two rewrites +// to the fake's stored row, so a test can take a video down and restore it. +// The SQL itself was run against MySQL 8 when it was written. +export async function withTakedownSql( + store: DeveloperVideoStore, + database: Awaited< + ReturnType + >, +) { + const { MySqlDialect } = await import("drizzle-orm/mysql-core"); + const { SQL } = await import("drizzle-orm"); + const dialect = new MySqlDialect(); + + const evaluate = (value: unknown, current: unknown): unknown => { + if (!(value instanceof SQL)) return value; + const query = dialect.sqlToQuery(value); + if (query.sql.startsWith("JSON_OBJECT(")) { + const [key, at] = query.params as [string, string]; + return { [key]: { at, metadata: current } }; + } + if (query.sql.startsWith("CASE WHEN JSON_TYPE(")) { + const marker = Object.values( + (current ?? {}) as Record, + )[0]; + return marker?.metadata ?? null; + } + throw new Error(`The fake cannot evaluate ${query.sql}`); + }; + + const wrapped = { + ...database, + update: () => ({ + set: (updates: Record) => ({ + where: async () => { + const current = store.video; + const applied = Object.fromEntries( + Object.entries(updates).map(([column, value]) => [ + column, + evaluate( + value, + current?.[column as keyof NonNullable], + ), + ]), + ); + store.updates.push(updates); + if (current) store.video = { ...current, ...applied }; + }, + }), + }), + transaction: async (run: (tx: unknown) => Promise) => run(wrapped), + }; + return wrapped; +} diff --git a/apps/web/__tests__/fixtures/developer-video.ts b/apps/web/__tests__/fixtures/developer-video.ts index eb9a8c898b0..5aa505df2ee 100644 --- a/apps/web/__tests__/fixtures/developer-video.ts +++ b/apps/web/__tests__/fixtures/developer-video.ts @@ -42,6 +42,7 @@ export function developerVideo( storedBytes: null, errorReason: null, processing: processingState(), + storagePaidUntil: null, deletedAt: null, filesDeletedAt: null, createdAt: new Date("2026-11-01T00:00:00Z"), @@ -55,6 +56,7 @@ export interface DeveloperVideoStore { more?: DeveloperVideoFixture[]; appDeleted: boolean; updates: Partial[]; + captionRows?: { videoId: string; language: string | null }[]; } // MySQL compares the externalUserId column with utf8mb4_0900_ai_ci, which @@ -157,8 +159,20 @@ export async function createDeveloperVideoDatabase(store: DeveloperVideoStore) { return chain; }; + const selectDistinct = () => ({ + from: () => ({ + where: async (condition: Condition) => { + const { params } = dialect.sqlToQuery(condition); + return (store.captionRows ?? []).filter((row) => + params.includes(row.videoId), + ); + }, + }), + }); + const database = { select, + selectDistinct, update: () => ({ set: (updates: Partial) => ({ where: async () => { diff --git a/apps/web/__tests__/integration/comment-media-quota-database.test.ts b/apps/web/__tests__/integration/comment-media-quota-database.test.ts new file mode 100644 index 00000000000..db357605314 --- /dev/null +++ b/apps/web/__tests__/integration/comment-media-quota-database.test.ts @@ -0,0 +1,281 @@ +import { randomUUID } from "node:crypto"; +import { + CreateBucketCommand, + HeadObjectCommand, + ListObjectsV2Command, + PutObjectCommand, + S3Client, +} from "@aws-sdk/client-s3"; +import { getSignedUrl } from "@aws-sdk/s3-request-presigner"; +import { drizzle } from "drizzle-orm/mysql2"; +import { Effect, Layer, Option } from "effect"; +import { createPool, type Pool } from "mysql2/promise"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +// Drives the real /initiate handler against a real MySQL database and a real +// S3-compatible bucket. Auth, the video lookup and the plan check are the only +// stubs; the comment throttle, the rate limiter, the quota reservation and the +// bucket listing all run for real. + +const fixture = vi.hoisted(() => ({ + database: undefined as unknown, + s3: undefined as unknown, + bucket: "", + ownerId: "", + videoId: "", +})); + +vi.mock("@cap/database", () => ({ + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@/app/api/utils", () => ({ + withAuth: async ( + c: { + req: { header: (name: string) => string | undefined }; + set: (key: string, value: unknown) => void; + }, + next: () => Promise, + ) => { + c.set("user", { id: c.req.header("x-test-user") }); + await next(); + }, +})); +vi.mock("@cap/web-domain", () => ({ + Comment: { CommentId: { make: (value: string) => value } }, + Policy: { withPublicPolicy: () => (effect: unknown) => effect }, + Video: { + VideoId: { make: (value: string) => value }, + NotFoundError: class {}, + }, +})); +vi.mock("@cap/web-backend", () => { + const client = () => fixture.s3 as S3Client; + return { + createCommentMediaUploadToken: () => "upload-token", + verifyCommentMediaUploadToken: () => null, + makeCurrentUserLayer: () => Layer.empty, + provideOptionalAuth: (effect: unknown) => effect, + VideosRepo: Effect.succeed({ + getById: () => + Effect.succeed( + Option.some([{ id: fixture.videoId, ownerId: fixture.ownerId }]), + ), + }), + VideosPolicy: Effect.succeed({ canView: () => Effect.void }), + Storage: { + getAccessForVideo: () => + Effect.succeed([ + { + provider: "aws", + listObjects: (input: { prefix: string; maxKeys: number }) => + Effect.promise(() => + client().send( + new ListObjectsV2Command({ + Bucket: fixture.bucket, + Prefix: input.prefix, + MaxKeys: input.maxKeys, + }), + ), + ), + createUploadTarget: ( + key: string, + options: { contentType: string }, + ) => + Effect.promise(async () => ({ + url: await getSignedUrl( + client(), + new PutObjectCommand({ + Bucket: fixture.bucket, + Key: key, + ContentType: options.contentType, + }), + { expiresIn: 600 }, + ), + method: "PUT", + })), + }, + ]), + }, + }; +}); +vi.mock("@/lib/media-comments", () => ({ + canUseMediaComments: async () => true, +})); +vi.mock("@/lib/server", () => ({ runPromise: Effect.runPromise })); + +const { app } = await import("@/app/api/upload/[...route]/comment-media"); + +const databaseUrl = process.env.CAP_INFRA_TEST_DATABASE_URL; +const s3Endpoint = process.env.CAP_COMMENT_MEDIA_TEST_S3_ENDPOINT; +const id = () => randomUUID().replaceAll("-", "").slice(0, 15); + +type Initiated = { + key: string; + upload: { url: string }; + thumbnailUpload: { url: string } | null; +}; + +const initiate = (user: string, body: Record = {}) => + app.request("/initiate", { + method: "POST", + headers: { "content-type": "application/json", "x-test-user": user }, + body: JSON.stringify({ + videoId: fixture.videoId, + kind: "video", + contentType: "video/webm", + mode: "single", + withThumbnail: true, + ...body, + }), + }); + +const put = (url: string, contentType: string, bytes: Uint8Array) => + fetch(url, { + method: "PUT", + headers: { "content-type": contentType }, + body: new Blob([bytes as Uint8Array]), + }); + +const storedCount = async () => { + const listed = await (fixture.s3 as S3Client).send( + new ListObjectsV2Command({ + Bucket: fixture.bucket, + Prefix: `${fixture.ownerId}/${fixture.videoId}/comments/`, + }), + ); + return listed.KeyCount ?? 0; +}; + +const freshVideo = () => { + fixture.ownerId = `owner-${id()}`; + fixture.videoId = `video-${id()}`; +}; + +describe.runIf(Boolean(databaseUrl && s3Endpoint))( + "comment media initiate against a real database and bucket", + () => { + let pool: Pool; + + beforeAll(async () => { + if (!databaseUrl || !s3Endpoint) + throw new Error("Missing isolated test database or bucket."); + const url = new URL(databaseUrl); + if ( + url.protocol !== "mysql:" || + !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) || + !/^\/cap_infra_[a-z0-9_]+$/.test(url.pathname) + ) + throw new Error("These tests require a session-scoped local database."); + if (!["127.0.0.1", "localhost"].includes(new URL(s3Endpoint).hostname)) + throw new Error("These tests require a local bucket."); + pool = createPool({ uri: databaseUrl, connectionLimit: 16 }); + fixture.database = drizzle(pool); + fixture.s3 = new S3Client({ + endpoint: s3Endpoint, + region: "us-east-1", + forcePathStyle: true, + credentials: { + accessKeyId: process.env.CAP_COMMENT_MEDIA_TEST_S3_KEY ?? "", + secretAccessKey: process.env.CAP_COMMENT_MEDIA_TEST_S3_SECRET ?? "", + }, + }); + fixture.bucket = `comment-media-test-${id()}`; + await (fixture.s3 as S3Client).send( + new CreateBucketCommand({ Bucket: fixture.bucket }), + ); + }); + + afterAll(async () => { + await pool?.end(); + }); + + it("uploads a normal video, thumbnail and audio comment end to end", async () => { + freshVideo(); + const user = `viewer-${id()}`; + + const video = await initiate(user); + expect(video.status).toBe(200); + const videoBody = (await video.json()) as Initiated; + expect(videoBody.thumbnailUpload).not.toBeNull(); + expect( + (await put(videoBody.upload.url, "video/webm", new Uint8Array(4096))) + .status, + ).toBe(200); + expect( + ( + await put( + videoBody.thumbnailUpload?.url ?? "", + "image/jpeg", + new Uint8Array(512), + ) + ).status, + ).toBe(200); + + const audio = await initiate(user, { + kind: "audio", + contentType: "audio/webm", + withThumbnail: false, + }); + expect(audio.status).toBe(200); + const audioBody = (await audio.json()) as Initiated; + expect(audioBody.thumbnailUpload).toBeNull(); + expect( + (await put(audioBody.upload.url, "audio/webm", new Uint8Array(2048))) + .status, + ).toBe(200); + + const head = await (fixture.s3 as S3Client).send( + new HeadObjectCommand({ Bucket: fixture.bucket, Key: videoBody.key }), + ); + expect(head.ContentLength).toBe(4096); + expect(await storedCount()).toBe(3); + }); + + it("never refuses a viewer well below the quota", async () => { + freshVideo(); + const user = `viewer-${id()}`; + for (let index = 0; index < 30; index++) { + const response = await initiate(user); + expect(response.status).toBe(200); + const body = (await response.json()) as Initiated; + expect( + (await put(body.upload.url, "video/webm", new Uint8Array(64))).status, + ).toBe(200); + } + expect(await storedCount()).toBe(30); + }); + + it("lets exactly one of many concurrent uploads take the last slots", async () => { + freshVideo(); + const prefix = `${fixture.ownerId}/${fixture.videoId}/comments`; + // 398 objects already stored: room for one video plus its thumbnail. + for (let batch = 0; batch < 398; batch += 50) + await Promise.all( + Array.from({ length: Math.min(50, 398 - batch) }, (_, offset) => + (fixture.s3 as S3Client).send( + new PutObjectCommand({ + Bucket: fixture.bucket, + Key: `${prefix}/seed-${batch + offset}/media.webm`, + Body: new Uint8Array(1), + }), + ), + ), + ); + expect(await storedCount()).toBe(398); + + const responses = await Promise.all( + Array.from({ length: 8 }, () => initiate(`viewer-${id()}`)), + ); + const statuses = responses.map((response) => response.status).sort(); + expect(statuses).toEqual([200, 429, 429, 429, 429, 429, 429, 429]); + for (const response of responses) + if (response.status === 429) + expect(await response.json()).toEqual({ + error: "This video has reached its media comment limit", + }); + }); + }, +); diff --git a/apps/web/__tests__/integration/developer-transcripts-database.test.ts b/apps/web/__tests__/integration/developer-transcripts-database.test.ts index e25a3bf5659..6cba759be17 100644 --- a/apps/web/__tests__/integration/developer-transcripts-database.test.ts +++ b/apps/web/__tests__/integration/developer-transcripts-database.test.ts @@ -25,6 +25,8 @@ const fixture = vi.hoisted(() => ({ start: vi.fn(), objectSize: null as number | null, copies: [] as string[], + objects: new Map(), + queries: [] as { sql: string; params: unknown[] }[], })); vi.mock("@cap/database", () => ({ @@ -55,8 +57,17 @@ vi.mock("@/lib/developer-transcript-storage", () => ({ copyDeveloperObject: async (_from: string, to: string) => { fixture.copies.push(to); }, - getDeveloperObjectText: async () => null, - deleteDeveloperObjects: async () => {}, + getDeveloperObjectText: async (key: string) => + fixture.objects.get(key) ?? null, + putDeveloperObject: async (key: string, body: string) => { + fixture.objects.set(key, body); + }, + removeDeveloperObject: async (key: string) => { + fixture.objects.delete(key); + }, + deleteDeveloperObjects: async (keys: string[]) => { + for (const key of keys) fixture.objects.delete(key); + }, })); vi.mock("@/workflows/process-developer-video", () => ({ processDeveloperVideoWorkflow: "developer-video-workflow", @@ -64,7 +75,8 @@ vi.mock("@/workflows/process-developer-video", () => ({ vi.mock("@/lib/media-client", () => ({ isMediaServerConfigured: () => false, })); -vi.mock("@/lib/media-container", () => ({ +vi.mock("@/lib/media-container", async (importOriginal) => ({ + ...(await importOriginal()), urlIsSelfContainedMedia: async () => true, })); vi.mock("@/lib/developer-video-storage", () => ({})); @@ -88,13 +100,21 @@ const { completeDeveloperVideoUpload, createDeveloperVideoUpload } = const { getSpeechUsage } = await import("@/lib/developer-transcript-usage"); const { transcriptUploadExpiry } = await import("@/lib/developer-transcripts"); const { availableBalance } = await import("@/lib/infra-balance"); +const { disputedSince, readStandingAccount } = await import( + "@/lib/infra-standing" +); const { getInfraAppStanding, getInfraUploadAdmission } = await import( "@/lib/infra-standing" ); -const { getAppStorageUsage } = await import("@/lib/infra-storage"); +const { getAppStorage } = await import("@/lib/infra-storage"); const { chargeStorageDay } = await import("@/lib/infra-storage-billing"); +const { developerVideoCaptions, syncVideoCaptions, videoCaptionsKey } = + await import("@/lib/developer-video-captions"); +const { serializeTranscript } = await import("@/lib/developer-transcripts"); const { + reconcileDeletedVideoCaptions, createTranscriptFromUrl, + createTranscriptFromVideo, createTranscriptUpload, deleteTranscript, expireStaleTranscripts, @@ -222,7 +242,7 @@ const chargeStorage = async (appId: string, day: string) => chargeStorageDay({ appId, accountId: await accountOf(appId), - usage: await getAppStorageUsage(appId), + usage: (await getAppStorage(appId)).dailyCharge, dateKey: day, now: new Date(), }); @@ -234,6 +254,7 @@ const videoStarts = () => const STORAGE_DAY_OF_TWO_HOURS = 340; const ADMISSION_FOR_TWO_HOURS = 2_380; +const FIRST_MONTH_FOR_TWO_HOURS = 10_200; const settle = (transcriptId: string, durationSeconds: number) => settleTranscript({ @@ -261,7 +282,13 @@ describe.skipIf(!databaseUrl)("developer transcripts database", () => { ); } pool = createPool({ uri: databaseUrl, connectionLimit: 16 }); - fixture.database = drizzle(pool); + fixture.database = drizzle(pool, { + logger: { + logQuery: (sql, params) => { + fixture.queries.push({ sql, params }); + }, + }, + }); await database().select().from(developerTranscripts).limit(1); }); @@ -275,6 +302,8 @@ describe.skipIf(!databaseUrl)("developer transcripts database", () => { fixture.start.mockResolvedValue(undefined); fixture.objectSize = null; fixture.copies = []; + fixture.objects = new Map(); + fixture.queries = []; }); it("charges a transcript exactly once when settle attempts race", async () => { @@ -881,7 +910,7 @@ describe.skipIf(!databaseUrl)("developer transcripts database", () => { }); it("creates an agent video with a playback id and admits it without charging", async () => { - const appId = await makeApp(ADMISSION_FOR_TWO_HOURS); + const appId = await makeApp(FIRST_MONTH_FOR_TWO_HOURS); const created = await createDeveloperVideoUpload(appId, { fileName: "demo.mp4", contentLength: 5_000_000, @@ -916,12 +945,12 @@ describe.skipIf(!databaseUrl)("developer transcripts database", () => { expect(stored?.s3Key).toBe(fixture.copies[0]); expect(stored?.processing?.encodeId).toBeTruthy(); expect(videoStarts()).toHaveLength(1); - expect(await balanceOf(appId)).toBe(ADMISSION_FOR_TWO_HOURS); + expect(await balanceOf(appId)).toBe(FIRST_MONTH_FOR_TWO_HOURS); expect(await ledgerOf(appId)).toHaveLength(0); }); it("admits a 4K agent upload at the capped price and refuses one the balance cannot store", async () => { - const appId = await makeApp(ADMISSION_FOR_TWO_HOURS - 1); + const appId = await makeApp(FIRST_MONTH_FOR_TWO_HOURS - 1); const created = await createDeveloperVideoUpload(appId, { fileName: "demo.mp4", contentLength: 20_000_000, @@ -948,13 +977,13 @@ describe.skipIf(!databaseUrl)("developer transcripts database", () => { }); expect(videoStarts()).toHaveLength(0); - await setBalance(appId, ADMISSION_FOR_TWO_HOURS); + await setBalance(appId, FIRST_MONTH_FOR_TWO_HOURS); expect(await complete()).toMatchObject({ ok: true, video: { status: "processing", durationSeconds: 7200 }, }); expect(videoStarts()).toHaveLength(1); - expect(await balanceOf(appId)).toBe(ADMISSION_FOR_TWO_HOURS); + expect(await balanceOf(appId)).toBe(FIRST_MONTH_FOR_TWO_HOURS); }); it("records an agent upload once and starts it once when completions race", async () => { @@ -1159,4 +1188,715 @@ describe.skipIf(!databaseUrl)("developer transcripts database", () => { } expect([...seen].sort()).toEqual([40_000, 50_000]); }); + + const readyProcessing = () => ({ + encodeId: "encode-1", + startedAt: 1_000, + probe: { duration: 120, width: 1920, height: 1080, fps: 30 }, + hls: { state: "complete" as const, output: "out-1" }, + mp4: { state: "complete" as const, output: "out-1" }, + attempts: { hls: 1, mp4: 1 }, + unavailableChecks: 0, + poster: null, + bytes: { source: 1_000, hls: 1_000, mp4: 1_000, poster: 0 }, + }); + + const makeReadyVideo = ( + appId: string, + overrides: Partial = {}, + ) => + makeStoredVideo(appId, { + duration: 120, + playbackId: id() + id(), + processing: readyProcessing(), + ...overrides, + }); + + const transcriptRow = async (transcriptId: string) => { + const [row] = await database() + .select() + .from(developerTranscripts) + .where(eq(developerTranscripts.id, transcriptId)); + if (!row) throw new Error("Transcript fixture is missing."); + return row; + }; + + const makeCaptionSource = async ( + appId: string, + videoId: string | null, + text: string, + overrides: Partial = {}, + ) => { + const transcriptId = overrides.id ?? id(); + const resultKey = `developer/${appId}/transcripts/${transcriptId}/result.json`; + fixture.objects.set( + resultKey, + JSON.stringify({ + text, + language: "en", + durationSeconds: 10, + words: text.split(" ").map((word, index) => ({ + text: word, + start: index * 0.5, + end: index * 0.5 + 0.4, + })), + segments: [], + }), + ); + await makeTranscript(appId, { + id: transcriptId, + status: "complete", + sourceType: videoId ? "video" : "upload", + videoId, + language: "en", + text, + wordCount: text.split(" ").length, + resultKey, + completedAt: new Date(), + requestedAt: overrides.createdAt ?? new Date(), + ...overrides, + }); + return transcriptId; + }; + + const explainLastQuery = async (containing: string) => { + const query = fixture.queries.findLast((entry) => + entry.sql.includes(containing), + ); + if (!query || !pool) throw new Error(`No query containing ${containing}`); + const [plan] = await pool.query(`EXPLAIN ${query.sql}`, query.params); + return plan as { key: string | null; rows: number; Extra: string | null }[]; + }; + + it("creates a video transcript once, scoped to its app, and only for a ready video", async () => { + const appId = await makeApp(50_000); + const otherAppId = await makeApp(50_000); + const videoId = await makeReadyVideo(appId); + const otherVideoId = await makeReadyVideo(appId); + const processingVideoId = await makeStoredVideo(appId, { + status: "processing", + }); + const deletedVideoId = await makeReadyVideo(appId, { + deletedAt: new Date(), + }); + const idempotencyKey = `video-key-${id()}`; + + const results = await Promise.all( + Array.from({ length: 8 }, () => + createTranscriptFromVideo(appId, { videoId, idempotencyKey }), + ), + ); + + const ids = new Set( + results.map((result) => (result.ok ? result.transcript.id : "failed")), + ); + expect(ids.size).toBe(1); + expect(ids.has("failed")).toBe(false); + expect(fixture.start).toHaveBeenCalledTimes(1); + const [stored] = await database() + .select() + .from(developerTranscripts) + .where(eq(developerTranscripts.appId, appId)); + expect(stored).toMatchObject({ + sourceType: "video", + videoId, + status: "queued", + s3Key: `developer/${appId}/${videoId}/encode-1/out-1/video.mp4`, + name: "Stored fixture", + }); + if (!stored) throw new Error("Transcript was not stored."); + expect(serializeTranscript(stored)).toMatchObject({ + videoId, + source: { type: "video", url: null }, + }); + + expect( + await createTranscriptFromVideo(appId, { + videoId: otherVideoId, + idempotencyKey, + }), + ).toMatchObject({ + ok: false, + status: 409, + message: "Idempotency key was already used with a different videoId", + }); + expect( + await createTranscriptFromVideo(otherAppId, { videoId }), + ).toMatchObject({ ok: false, status: 404, code: "video_not_found" }); + expect( + await createTranscriptFromVideo(appId, { videoId: deletedVideoId }), + ).toMatchObject({ ok: false, status: 404, code: "video_not_found" }); + expect( + await createTranscriptFromVideo(appId, { videoId: processingVideoId }), + ).toMatchObject({ ok: false, status: 409, code: "video_not_ready" }); + expect(fixture.start).toHaveBeenCalledTimes(1); + }); + + it("keeps videoId null for transcripts that are not from a video", async () => { + const appId = await makeApp(50_000); + const videoId = await makeReadyVideo(appId); + const fromUrl = await createTranscriptFromUrl( + appId, + { url: URL_A }, + guards, + ); + const fromUpload = await createTranscriptUpload(appId, { + fileName: "a.mp3", + contentLength: 1024, + }); + await makeCaptionSource(appId, null, "hello there"); + + for (const created of [fromUrl, fromUpload]) { + if (!created.ok) throw new Error("Transcript was not created."); + expect(created.transcript.videoId).toBeNull(); + expect(serializeTranscript(created.transcript).videoId).toBeNull(); + } + const listed = await listTranscripts(appId, { limit: 10 }); + expect(listed.transcripts).toHaveLength(3); + expect(listed.transcripts.every((row) => row.videoId === null)).toBe(true); + expect(await developerVideoCaptions({ id: videoId, appId })).toEqual([]); + }); + + it("returns the caption files of a video from one indexed query", async () => { + const appId = await makeApp(50_000); + const videoId = await makeReadyVideo(appId); + const filler = Array.from({ length: 3_000 }, (_, index) => ({ + id: id(), + appId, + status: "complete" as const, + sourceType: index % 3 === 0 ? ("upload" as const) : ("video" as const), + videoId: index % 3 === 0 ? null : `filler-${index % 60}`, + language: "en", + wordCount: 4, + })); + for (let offset = 0; offset < filler.length; offset += 500) { + await database() + .insert(developerTranscripts) + .values(filler.slice(offset, offset + 500)); + } + await makeCaptionSource(appId, videoId, "older english words"); + await makeCaptionSource(appId, videoId, "newer english words"); + await makeCaptionSource(appId, videoId, "deutsche worte hier", { + language: "de", + }); + await makeCaptionSource(appId, videoId, "", { + language: "fr", + wordCount: 0, + }); + await makeCaptionSource(appId, videoId, "borrado", { + language: "es", + deletedAt: new Date(), + }); + await pool?.query("ANALYZE TABLE developer_transcripts"); + + fixture.queries = []; + const captions = await developerVideoCaptions({ id: videoId, appId }); + + expect(captions).toEqual([ + { + language: "en", + label: "English", + key: `developer/${appId}/${videoId}/captions/en.vtt`, + }, + { + language: "de", + label: "German", + key: `developer/${appId}/${videoId}/captions/de.vtt`, + }, + ]); + expect(fixture.queries).toHaveLength(1); + const plan = await explainLastQuery("developer_transcripts"); + expect(plan).toHaveLength(1); + expect(plan[0]?.key).toBe("dev_transcripts_app_video_idx"); + expect(Number(plan[0]?.rows)).toBeLessThan(50); + + fixture.queries = []; + await syncVideoCaptions({ appId, videoId, language: "en" }); + const newest = await explainLastQuery("order by"); + expect(newest[0]?.key).toBe("dev_transcripts_app_video_idx"); + expect(newest[0]?.Extra ?? "").not.toContain("filesort"); + }); + + it("writes the newest transcript's captions, replaces them and removes them on delete", async () => { + const appId = await makeApp(50_000); + const videoId = await makeReadyVideo(appId); + const key = videoCaptionsKey({ id: videoId, appId }, "en"); + const older = await makeCaptionSource(appId, videoId, "older words", { + createdAt: new Date("2026-11-12T10:00:00.000Z"), + }); + expect(await syncVideoCaptions({ appId, videoId, language: "en" })).toEqual( + { state: "written", transcriptId: older }, + ); + expect(fixture.objects.get(key)).toContain("older words"); + + const newer = await makeCaptionSource(appId, videoId, "newer words", { + createdAt: new Date("2026-11-12T10:00:05.000Z"), + }); + expect(await syncVideoCaptions({ appId, videoId, language: "en" })).toEqual( + { state: "written", transcriptId: newer }, + ); + expect(fixture.objects.get(key)).toContain("newer words"); + expect(fixture.objects.get(key)).toMatch(/^WEBVTT\n\n00:00:00\.000 --> /); + + expect(await deleteTranscript(appId, newer)).toEqual({ ok: true }); + expect(fixture.objects.get(key)).toContain("older words"); + expect(await deleteTranscript(appId, older)).toEqual({ ok: true }); + expect(fixture.objects.has(key)).toBe(false); + expect(await developerVideoCaptions({ id: videoId, appId })).toEqual([]); + }); + + it("uses the transcript requested last when two were created in the same second", async () => { + const appId = await makeApp(50_000); + const videoId = await makeReadyVideo(appId); + const createdAt = new Date("2026-11-12T10:00:00.000Z"); + const first = await makeCaptionSource(appId, videoId, "first words", { + id: `z${id().slice(1)}`, + createdAt, + requestedAt: new Date("2026-11-12T10:00:00.120Z"), + }); + const second = await makeCaptionSource(appId, videoId, "second words", { + id: `a${id().slice(1)}`, + createdAt, + requestedAt: new Date("2026-11-12T10:00:00.870Z"), + }); + + for (let round = 0; round < 5; round++) { + expect( + await syncVideoCaptions({ appId, videoId, language: "en" }), + ).toEqual({ state: "written", transcriptId: second }); + } + expect((await transcriptRow(second)).requestedAt?.toISOString()).toBe( + "2026-11-12T10:00:00.870Z", + ); + expect(first).not.toBe(second); + }); + + it("orders video transcripts created through the API within one second by when they were requested", async () => { + const appId = await makeApp(50_000); + const videoId = await makeReadyVideo(appId); + const created: string[] = []; + for (let index = 0; index < 3; index++) { + const result = await createTranscriptFromVideo(appId, { videoId }); + if (!result.ok) throw new Error("Video transcript was not created."); + created.push(result.transcript.id); + } + const rows = await Promise.all(created.map(transcriptRow)); + const times = rows.map((row) => row.requestedAt?.getTime() ?? 0); + expect(times.every((time) => time > 0)).toBe(true); + expect([...times].sort((a, b) => a - b)).toEqual(times); + expect(new Set(times).size).toBe(3); + }); + + it("repairs captions left by a failed delete from the hourly cleanup, reading through the status index", async () => { + const appId = await makeApp(50_000); + const videoId = await makeReadyVideo(appId); + const key = videoCaptionsKey({ id: videoId, appId }, "en"); + await makeCaptionSource(appId, videoId, "older words", { + createdAt: new Date("2026-11-12T10:00:00.000Z"), + }); + const newer = await makeCaptionSource(appId, videoId, "newer words", { + createdAt: new Date("2026-11-12T10:00:05.000Z"), + }); + await syncVideoCaptions({ appId, videoId, language: "en" }); + await database() + .update(developerTranscripts) + .set({ deletedAt: new Date() }) + .where(eq(developerTranscripts.id, newer)); + expect(fixture.objects.get(key)).toContain("newer words"); + + fixture.queries = []; + const result = await reconcileDeletedVideoCaptions(); + + expect(result.failed).toBe(0); + expect(result.finished).toBe(true); + expect(result.synced).toBeGreaterThanOrEqual(1); + expect(fixture.objects.get(key)).toContain("older words"); + const plan = await explainLastQuery("`videoId` is not null"); + expect(plan[0]?.key).toBe("dev_transcripts_status_updated_idx"); + }); + + it("charges a transcript deleted while processing exactly once when the delete races the settle", async () => { + for (let round = 0; round < 24; round++) { + const appId = await makeApp(50_000); + const transcriptId = await makeTranscript(appId); + await reserveTranscriptCredits({ transcriptId, durationSeconds: 3600 }); + + const [settled, deleted] = await Promise.all( + round % 2 === 0 + ? [settle(transcriptId, 3600), deleteTranscript(appId, transcriptId)] + : [ + new Promise>>((resolve) => + setTimeout(() => resolve(settle(transcriptId, 3600)), 2), + ), + deleteTranscript(appId, transcriptId), + ], + ); + + expect(settled).toMatchObject({ + state: "complete", + chargedMicroCredits: 10_000, + }); + expect(await balanceOf(appId)).toBe(40_000); + expect(await ledgerOf(appId)).toEqual([ + expect.objectContaining({ + type: "transcription", + amount: -10_000, + referenceId: transcriptId, + }), + ]); + const row = await transcriptRow(transcriptId); + expect(deleted).toEqual({ ok: true }); + expect(row).toMatchObject({ + status: "complete", + billedMicroCredits: 10_000, + reservedMicroCredits: 0, + text: null, + }); + expect(row.deletedAt).toBeInstanceOf(Date); + expect(await getTranscript(appId, transcriptId)).toBeNull(); + expect(await settle(transcriptId, 3600)).toMatchObject({ + state: "complete", + alreadySettled: true, + }); + expect(await balanceOf(appId)).toBe(40_000); + expect(await getAvailableTranscriptBalance(appId)).toBe(40_000); + } + }); + + it("keeps the hold of a transcript deleted while processing, and charges it when it settles", async () => { + const appId = await makeApp(12_000); + const transcriptId = await makeTranscript(appId); + await reserveTranscriptCredits({ transcriptId, durationSeconds: 3600 }); + + expect(await deleteTranscript(appId, transcriptId)).toEqual({ ok: true }); + + expect(await getAvailableTranscriptBalance(appId)).toBe(2_000); + expect(await getSpeechUsage(appId)).toMatchObject({ + balanceMicroCredits: 12_000, + heldMicroCredits: 10_000, + availableMicroCredits: 2_000, + }); + const spend = await database().transaction((tx) => + debitDeveloperUsage(tx, { + appId, + microCredits: 5_000, + type: "video_create", + referenceId: id(), + referenceType: "developer_video", + }), + ); + expect(spend).toEqual({ state: "insufficient" }); + + expect(await settle(transcriptId, 3600)).toMatchObject({ + state: "complete", + chargedMicroCredits: 10_000, + }); + expect(await balanceOf(appId)).toBe(2_000); + expect(await getAvailableTranscriptBalance(appId)).toBe(2_000); + expect((await listTranscripts(appId, { limit: 10 })).transcripts).toEqual( + [], + ); + }); + + it("charges nothing for a transcript deleted before it was processing or after it failed", async () => { + const appId = await makeApp(50_000); + for (const status of ["uploading", "queued", "error"] as const) { + const transcriptId = await makeTranscript(appId, { status }); + await deleteTranscript(appId, transcriptId); + + expect(await settle(transcriptId, 3600)).toEqual({ + state: "skipped", + reason: "deleted", + }); + expect((await transcriptRow(transcriptId)).status).toBe(status); + } + expect(await balanceOf(appId)).toBe(50_000); + expect(await ledgerOf(appId)).toEqual([]); + expect(await getAvailableTranscriptBalance(appId)).toBe(50_000); + }); + + it("admits exactly 20 transcripts when 40 are created at once", async () => { + const appId = await makeApp(500_000); + + const results = await Promise.all( + Array.from({ length: 40 }, (_, index) => + createTranscriptFromUrl( + appId, + { url: `https://media.example.com/clip-${index}.mp3` }, + guards, + ), + ), + ); + + expect(results.filter((result) => result.ok)).toHaveLength(20); + const refused = results.filter((result) => !result.ok); + expect(refused).toHaveLength(20); + for (const result of refused) { + expect(result).toMatchObject({ + status: 429, + code: "too_many_transcripts", + }); + } + const rows = await database() + .select({ status: developerTranscripts.status }) + .from(developerTranscripts) + .where(eq(developerTranscripts.appId, appId)); + expect(rows).toHaveLength(20); + expect(fixture.start).toHaveBeenCalledTimes(20); + + const [first] = results; + if (!first?.ok) { + const admitted = results.find((result) => result.ok); + if (!admitted?.ok) throw new Error("Nothing was admitted."); + await deleteTranscript(appId, admitted.transcript.id); + } else { + await deleteTranscript(appId, first.transcript.id); + } + expect( + (await createTranscriptFromUrl(appId, { url: URL_A }, guards)).ok, + ).toBe(true); + expect( + await createTranscriptFromUrl(appId, { url: URL_A }, guards), + ).toMatchObject({ ok: false, status: 429 }); + }); + + it("starts only as many uploads as there are slots when they are started at once", async () => { + const appId = await makeApp(500_000); + for (let index = 0; index < 18; index++) await makeTranscript(appId); + const uploads: string[] = []; + for (let index = 0; index < 8; index++) { + const created = await createTranscriptUpload(appId, { + fileName: `clip-${index}.mp3`, + contentLength: 1024, + }); + if (!created.ok) throw new Error("Upload transcript was not created."); + uploads.push(created.transcript.id); + } + fixture.objectSize = 1024; + + const results = await Promise.all( + uploads.map((transcriptId) => startTranscriptUpload(appId, transcriptId)), + ); + + expect(results.filter((result) => result.ok)).toHaveLength(2); + expect( + results.filter((result) => !result.ok && result.status === 429), + ).toHaveLength(6); + expect(fixture.start).toHaveBeenCalledTimes(2); + const rows = await database() + .select({ status: developerTranscripts.status }) + .from(developerTranscripts) + .where(eq(developerTranscripts.appId, appId)); + expect(rows.filter((row) => row.status === "queued")).toHaveLength(2); + expect(rows.filter((row) => row.status === "uploading")).toHaveLength(6); + }); + + it("does not let another app's transcripts use this app's slots", async () => { + const busy = await makeApp(500_000); + const quiet = await makeApp(500_000); + for (let index = 0; index < 20; index++) await makeTranscript(busy); + + expect( + await createTranscriptFromUrl(busy, { url: URL_A }, guards), + ).toMatchObject({ ok: false, status: 429 }); + expect( + (await createTranscriptFromUrl(quiet, { url: URL_A }, guards)).ok, + ).toBe(true); + }); + + it("admits every copy of a repeated request or start that races for the last slot", async () => { + for (let round = 0; round < 6; round++) { + const appId = await makeApp(500_000); + for (let index = 0; index < 19; index++) await makeTranscript(appId); + fixture.start.mockClear(); + const key = `race-key-${id()}`; + + const creates = await Promise.all( + Array.from({ length: 4 }, () => + createTranscriptFromUrl( + appId, + { url: URL_A, idempotencyKey: key }, + guards, + ), + ), + ); + + expect(creates.every((result) => result.ok)).toBe(true); + expect( + new Set( + creates.map((result) => (result.ok ? result.transcript.id : "")), + ), + ).toHaveProperty("size", 1); + expect(fixture.start).toHaveBeenCalledTimes(1); + expect( + await createTranscriptFromUrl(appId, { url: URL_A }, guards), + ).toMatchObject({ ok: false, status: 429 }); + } + + for (let round = 0; round < 6; round++) { + const appId = await makeApp(500_000); + for (let index = 0; index < 19; index++) await makeTranscript(appId); + const created = await createTranscriptUpload(appId, { + fileName: "clip.mp3", + contentLength: 1024, + }); + if (!created.ok) throw new Error("Upload transcript was not created."); + fixture.objectSize = 1024; + fixture.start.mockClear(); + + const starts = await Promise.all( + Array.from({ length: 4 }, () => + startTranscriptUpload(appId, created.transcript.id), + ), + ); + + expect(starts.every((result) => result.ok)).toBe(true); + expect(fixture.start).toHaveBeenCalledTimes(1); + } + }); + + it("admits a transcript while another request holds the credit account, and keeps the ceiling exact", async () => { + const appId = await makeApp(500_000); + for (let index = 0; index < 18; index++) await makeTranscript(appId); + if (!pool) throw new Error("No pool"); + const holder = await pool.getConnection(); + try { + await holder.query("START TRANSACTION"); + await holder.query( + "SELECT id FROM developer_credit_accounts WHERE appId = ? FOR UPDATE", + [appId], + ); + const started = Date.now(); + const created = await createTranscriptFromUrl( + appId, + { url: URL_A }, + guards, + ); + const waited = Date.now() - started; + expect(created.ok).toBe(true); + expect(waited).toBeLessThan(1_000); + } finally { + await holder.query("ROLLBACK"); + holder.release(); + } + + const results = await Promise.all( + Array.from({ length: 12 }, (_, index) => + createTranscriptFromUrl( + appId, + { url: `https://media.example.com/race-${index}.mp3` }, + guards, + ), + ), + ); + expect(results.filter((result) => result.ok)).toHaveLength(1); + const rows = await database() + .select({ status: developerTranscripts.status }) + .from(developerTranscripts) + .where(eq(developerTranscripts.appId, appId)); + expect( + rows.filter((row) => ["queued", "processing"].includes(row.status)), + ).toHaveLength(20); + }); + + it("never deadlocks when transcript creates, uploads, video inserts and a storage charge race on one app", async () => { + const today = new Date().toISOString().slice(0, 10); + for (let round = 0; round < 3; round++) { + const appId = await makeApp(5_000_000); + const settled = await Promise.allSettled([ + ...Array.from({ length: 25 }, (_, index) => + createTranscriptFromUrl( + appId, + { url: `https://media.example.com/race-${round}-${index}.mp3` }, + guards, + ), + ), + ...Array.from({ length: 10 }, (_, index) => + createDeveloperVideoUpload(appId, { + fileName: `race-${index}.mp4`, + contentLength: 1_000_000, + }), + ), + ...Array.from({ length: 5 }, () => + database().insert(developerVideos).values({ + id: id(), + appId, + name: "Concurrent insert", + maxResolutionTier: "1080p", + }), + ), + chargeStorage(appId, today), + ]); + + expect( + settled.flatMap((result) => + result.status === "rejected" ? [String(result.reason)] : [], + ), + ).toEqual([]); + const created = settled + .slice(0, 25) + .map((result) => + result.status === "fulfilled" + ? (result.value as { ok: boolean }) + : null, + ); + expect(created.filter((result) => result?.ok)).toHaveLength(20); + const uploads = settled + .slice(25, 35) + .map((result) => + result.status === "fulfilled" + ? (result.value as { ok: boolean }) + : null, + ); + expect(uploads.every((result) => result?.ok)).toBe(true); + const rows = await database() + .select({ status: developerTranscripts.status }) + .from(developerTranscripts) + .where(eq(developerTranscripts.appId, appId)); + expect(rows.filter((row) => row.status === "queued")).toHaveLength(20); + } + }, 120_000); + + it("decides a dispute and a review written in the same second by the larger id, in both reads", async () => { + const second = new Date("2026-11-13T06:00:00.000Z"); + for (const [first, last, disputed] of [ + [ + { id: `z${id().slice(1)}`, type: "manual" }, + { id: `a${id().slice(1)}`, type: "stripe_dispute" }, + false, + ], + [ + { id: `z${id().slice(1)}`, type: "stripe_dispute" }, + { id: `a${id().slice(1)}`, type: "manual" }, + true, + ], + ] as const) { + const appId = await makeApp(50_000); + const accountId = await accountOf(appId); + for (const row of [first, last]) { + await database().insert(developerCreditTransactions).values({ + id: row.id, + accountId, + type: "adjustment", + amountMicroCredits: 0, + balanceAfterMicroCredits: 50_000, + referenceId: row.id, + referenceType: row.type, + createdAt: second, + }); + } + + const expected = disputed ? second.toISOString() : null; + expect( + (await disputedSince(database() as never, accountId))?.toISOString() ?? + null, + ).toBe(expected); + expect( + (await readStandingAccount(appId))?.disputedAt?.toISOString() ?? null, + ).toBe(expected); + expect((await getInfraAppStanding(appId)).state).toBe( + disputed ? "paused" : "active", + ); + } + }); }); diff --git a/apps/web/__tests__/integration/developer-video-staff-delete-database.test.ts b/apps/web/__tests__/integration/developer-video-staff-delete-database.test.ts new file mode 100644 index 00000000000..131b264103e --- /dev/null +++ b/apps/web/__tests__/integration/developer-video-staff-delete-database.test.ts @@ -0,0 +1,107 @@ +import { randomUUID } from "node:crypto"; +import { developerApps, developerVideos } from "@cap/database/schema"; +import { eq, sql } from "drizzle-orm"; +import { drizzle, type MySql2Database } from "drizzle-orm/mysql2"; +import { createPool, type Pool } from "mysql2/promise"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +const fixture = vi.hoisted(() => ({ + database: undefined as unknown, + deleteObjects: vi.fn(async (_prefix: string) => 3), +})); + +vi.mock("@cap/database", () => ({ + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@/lib/developer-video-storage", () => ({ + deleteDeveloperVideoObjects: fixture.deleteObjects, + developerVideoPrefix: (video: { appId: string; id: string }) => + `developer/${video.appId}/${video.id}/`, +})); + +const { purgeDeletedDeveloperVideoNow, purgeDeletedDeveloperVideos } = + await import("@/lib/developer-video-purge"); + +const databaseUrl = process.env.CAP_INFRA_TEST_DATABASE_URL; +const id = () => randomUUID().replaceAll("-", "").slice(0, 15); +const database = () => fixture.database as MySql2Database; + +describe.runIf(Boolean(databaseUrl))( + "staff delete purge against a real database", + () => { + let pool: Pool; + + beforeAll(async () => { + if (!databaseUrl) throw new Error("Missing isolated test database URL."); + const url = new URL(databaseUrl); + if ( + url.protocol !== "mysql:" || + !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) || + !/^\/cap_infra_[a-z0-9_]+$/.test(url.pathname) + ) { + throw new Error("These tests require a session-scoped local database."); + } + pool = createPool({ uri: databaseUrl, connectionLimit: 4 }); + fixture.database = drizzle(pool); + }); + + afterAll(async () => { + await pool?.end(); + }); + + it("purges at once and leaves the late write pass to the cron", async () => { + const appId = id(); + const videoId = id(); + await database() + .insert(developerApps) + .values({ + id: appId, + ownerId: id() as never, + name: "Staff delete fixture", + environment: "development", + }); + await database() + .insert(developerVideos) + .values({ + id: videoId, + appId, + name: "Fixture", + status: "ready", + deletedAt: sql`NOW()` as never, + }); + + expect(await purgeDeletedDeveloperVideoNow({ id: videoId, appId })).toBe( + true, + ); + expect(fixture.deleteObjects.mock.calls).toEqual([ + [`developer/${appId}/${videoId}/`], + ]); + const selected = async () => { + fixture.deleteObjects.mockClear(); + await purgeDeletedDeveloperVideos(10_000); + return fixture.deleteObjects.mock.calls.some( + ([prefix]) => prefix === `developer/${appId}/${videoId}/`, + ); + }; + expect(await selected()).toBe(false); + + await database() + .update(developerVideos) + .set({ + deletedAt: sql`${developerVideos.deletedAt} - INTERVAL 5 HOUR`, + filesDeletedAt: sql`${developerVideos.filesDeletedAt} - INTERVAL 5 HOUR`, + }) + .where(eq(developerVideos.id, videoId)); + expect(await selected()).toBe(true); + expect(await selected()).toBe(false); + + await database() + .delete(developerVideos) + .where(eq(developerVideos.id, videoId)); + await database().delete(developerApps).where(eq(developerApps.id, appId)); + }); + }, +); diff --git a/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts new file mode 100644 index 00000000000..2a2b75aa112 --- /dev/null +++ b/apps/web/__tests__/integration/infra-auto-top-up-database.test.ts @@ -0,0 +1,944 @@ +import { randomUUID } from "node:crypto"; +import { drizzle, type MySql2Database } from "drizzle-orm/mysql2"; +import { createPool, type Pool } from "mysql2/promise"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +type FakeIntent = { + id: string; + status: "succeeded"; + amount: number; + amount_received: number; + currency: string; + metadata: Record; +}; + +const fixture = vi.hoisted(() => ({ + database: undefined as unknown, + keys: new Map< + string, + | { state: "pending"; params: string } + | { state: "done"; params: string; intent: FakeIntent } + >(), + created: [] as FakeIntent[], + createdKeys: [] as string[], + sent: [] as Record[], + duringCreate: null as null | (() => Promise), + loseNextResponse: false, + failNextCall: false, + duringSearch: null as null | (() => Promise), + delayMs: 40, + calledAt: [] as number[], + stored: new Map(), + retrieved: [] as string[], +})); + +vi.mock("@cap/database", () => ({ + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "", NEXT_PUBLIC_WEB_URL: "https://web" }, + serverEnv: () => ({ WEB_URL: "https://web" }), +})); +vi.mock("@/lib/developer-credits-product", () => ({ + developerCreditsProductId: () => "prod_test", +})); +vi.mock("@/lib/infra-billing-emails", () => ({ + sendInfraTopUpFailedEmail: vi.fn(), +})); +vi.mock("@cap/utils", () => ({ + stripe: () => ({ + paymentIntents: { + search: async () => { + const during = fixture.duringSearch; + fixture.duringSearch = null; + await during?.(); + return { data: [] }; + }, + retrieve: async (id: string) => { + fixture.retrieved.push(id); + const intent = fixture.stored.get(id); + if (!intent) throw new Error(`No such payment_intent: ${id}`); + return intent; + }, + create: async ( + params: { amount: number; metadata: Record }, + options: { idempotencyKey: string }, + ) => { + const key = options.idempotencyKey; + if (fixture.failNextCall) { + fixture.failNextCall = false; + throw Object.assign(new Error("socket hang up"), { + type: "StripeConnectionError", + }); + } + const entry = fixture.keys.get(key); + const sent = JSON.stringify(params); + if (entry && entry.params !== sent) { + throw Object.assign(new Error("Keys must match parameters"), { + type: "StripeIdempotencyError", + statusCode: 400, + }); + } + if (entry?.state === "pending") { + throw Object.assign(new Error("Key in use"), { + type: "StripeInvalidRequestError", + code: "idempotency_key_in_use", + statusCode: 409, + }); + } + if (entry?.state === "done") return entry.intent; + fixture.keys.set(key, { state: "pending", params: sent }); + fixture.sent.push(params as unknown as Record); + const during = fixture.duringCreate; + fixture.duringCreate = null; + await during?.(); + fixture.calledAt.push(Date.now()); + await new Promise((resolve) => setTimeout(resolve, fixture.delayMs)); + const intent: FakeIntent = { + id: `pi_${fixture.created.length + 1}`, + status: "succeeded", + amount: params.amount, + amount_received: params.amount, + currency: "usd", + metadata: params.metadata, + }; + fixture.created.push(intent); + fixture.createdKeys.push(key); + fixture.keys.set(key, { state: "done", params: sent, intent }); + if (fixture.loseNextResponse) { + fixture.loseNextResponse = false; + throw Object.assign(new Error("Request aborted due to timeout"), { + type: "StripeConnectionError", + }); + } + return intent; + }, + }, + }), +})); + +const databaseUrl = process.env.CAP_INFRA_TEST_DATABASE_URL; +const id = () => randomUUID().replaceAll("-", "").slice(0, 15); + +describe.runIf(Boolean(databaseUrl))( + "Cap Infra auto top-up against a real database", + () => { + let pool: Pool; + const accounts: string[] = []; + const owners: string[] = []; + + const makeOwner = async () => { + const userId = id(); + owners.push(userId); + await pool.query( + "INSERT INTO users (id, email, activeOrganizationId) VALUES (?, ?, '')", + [userId, `${userId}@example.com`], + ); + return userId; + }; + + const makeAccount = async ( + balanceMicroCredits: number, + { + ownerId, + amountCents = 500, + }: { ownerId?: string; amountCents?: number } = {}, + ) => { + const userId = ownerId ?? (await makeOwner()); + const appId = id(); + const accountId = id(); + accounts.push(accountId); + await pool.query( + "INSERT INTO developer_apps (id, ownerId, name, environment) VALUES (?, ?, 'Top-up test', 'production')", + [appId, userId], + ); + await pool.query( + `INSERT INTO developer_credit_accounts + (id, appId, ownerId, balanceMicroCredits, stripeCustomerId, stripePaymentMethodId, + autoTopUpEnabled, autoTopUpThresholdMicroCredits, autoTopUpAmountCents) + VALUES (?, ?, ?, ?, 'cus_test', 'pm_test', 1, 500000, ?)`, + [accountId, appId, userId, balanceMicroCredits, amountCents], + ); + return { accountId, appId, ownerId: userId }; + }; + + const balanceOf = async (accountId: string) => { + const [rows] = await pool.query( + "SELECT balanceMicroCredits AS balance FROM developer_credit_accounts WHERE id = ?", + [accountId], + ); + return Number((rows as { balance: number }[])[0]?.balance); + }; + + const topUpsOf = async (accountId: string) => { + const [rows] = await pool.query( + "SELECT referenceId FROM developer_credit_transactions WHERE accountId = ? AND referenceType = 'auto_top_up' AND type = 'topup'", + [accountId], + ); + return (rows as { referenceId: string }[]).map((row) => row.referenceId); + }; + + const setBalance = (accountId: string, balanceMicroCredits: number) => + pool.query( + "UPDATE developer_credit_accounts SET balanceMicroCredits = ? WHERE id = ?", + [balanceMicroCredits, accountId], + ); + + beforeAll(() => { + pool = createPool({ uri: databaseUrl, connectionLimit: 12 }); + fixture.database = drizzle(pool, { + mode: "default", + }) as MySql2Database; + }); + + afterAll(async () => { + for (const accountId of accounts) { + await pool.query( + "DELETE FROM developer_credit_transactions WHERE accountId = ?", + [accountId], + ); + await pool.query( + "DELETE a, c FROM developer_credit_accounts c JOIN developer_apps a ON a.id = c.appId WHERE c.id = ?", + [accountId], + ); + } + for (const ownerId of owners) { + await pool.query("DELETE FROM users WHERE id = ?", [ownerId]); + } + await pool.end(); + }); + + it("creates one charge when several triggers race for one account", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { accountId } = await makeAccount(30_000); + const before = fixture.created.length; + + const early = Array.from({ length: 6 }, () => runAutoTopUp(accountId)); + await new Promise((resolve) => setTimeout(resolve, 60)); + const late = Array.from({ length: 4 }, () => runAutoTopUp(accountId)); + const results = await Promise.all([...early, ...late]); + + expect(fixture.created.length - before).toBe(1); + expect(await topUpsOf(accountId)).toHaveLength(1); + expect(await balanceOf(accountId)).toBe(530_000); + expect( + results.filter((result) => result.state === "succeeded").length, + ).toBeGreaterThanOrEqual(1); + for (const result of results) { + expect(["succeeded", "skipped"]).toContain(result.state); + } + }); + + it("creates one charge when the amount changes while triggers race", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { accountId } = await makeAccount(30_000); + const before = fixture.created.length; + + const first = Array.from({ length: 4 }, () => runAutoTopUp(accountId)); + await new Promise((resolve) => setTimeout(resolve, 10)); + await pool.query( + "UPDATE developer_credit_accounts SET autoTopUpAmountCents = 2500 WHERE id = ?", + [accountId], + ); + const changed = Array.from({ length: 4 }, () => runAutoTopUp(accountId)); + await Promise.all([...first, ...changed]); + + expect(fixture.created.length - before).toBe(1); + expect(await topUpsOf(accountId)).toHaveLength(1); + expect(await balanceOf(accountId)).toBe(530_000); + }); + + it("makes no second payment when the amount changes after a lost response", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const { accountId } = await makeAccount(30_000); + const before = fixture.created.length; + + fixture.loseNextResponse = true; + expect(await runAutoTopUp(accountId)).toEqual({ state: "error" }); + expect(await topUpsOf(accountId)).toHaveLength(0); + + await pool.query( + "UPDATE developer_credit_accounts SET autoTopUpAmountCents = 2500 WHERE id = ?", + [accountId], + ); + expect(await runAutoTopUp(accountId)).toMatchObject({ + state: "succeeded", + }); + expect(fixture.created.length - before).toBe(1); + expect(await topUpsOf(accountId)).toHaveLength(1); + + await setBalance(accountId, 10); + expect(await runAutoTopUp(accountId)).toMatchObject({ + state: "succeeded", + }); + expect( + fixture.created.slice(before).map((intent) => intent.amount), + ).toEqual([500, 2500]); + errors.mockRestore(); + }); + + const addRow = ( + accountId: string, + type: string, + referenceType: string, + referenceId: string, + hoursAgo: number, + ) => + pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata, createdAt) + VALUES (?, ?, ?, 0, 30000, ?, ?, '{}', UTC_TIMESTAMP() - INTERVAL ? HOUR)`, + [id(), accountId, type, referenceId, referenceType, hoursAgo], + ); + + it("makes the locked check wait on the same rules the sweep reads", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const declined = await makeAccount(30_000); + await addRow( + declined.accountId, + "topup_failed", + "auto_top_up_failed", + `declined-${declined.accountId}`, + 2, + ); + const young = await makeAccount(30_000); + await addRow( + young.accountId, + "topup_pending", + "auto_top_up_pending", + `pi_young_${young.accountId}`, + 25, + ); + const old = await makeAccount(30_000); + const oldPayment = `pi_old_${old.accountId}`; + await addRow( + old.accountId, + "topup_pending", + "auto_top_up_pending", + oldPayment, + 73, + ); + fixture.stored.set(oldPayment, { + id: oldPayment, + status: "succeeded", + amount: 500, + amount_received: 500, + currency: "usd", + metadata: { type: "developer_auto_top_up", accountId: old.accountId }, + }); + const before = fixture.created.length; + + expect(await runAutoTopUp(declined.accountId)).toEqual({ + state: "skipped", + reason: "recent_decline", + }); + expect(await runAutoTopUp(young.accountId)).toEqual({ + state: "skipped", + reason: "awaiting_payment", + }); + expect(await runAutoTopUp(old.accountId)).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(await topUpsOf(old.accountId)).toEqual([oldPayment]); + expect(fixture.created.length).toBe(before); + }); + + it("lets the sweep reconcile a payment pending for over 72 hours, and only that", async () => { + const { runDueAutoTopUps } = await import("@/lib/infra-auto-top-up"); + + const stale = await makeAccount(30_000); + const stalePayment = `pi_stale_${stale.accountId}`; + await addRow( + stale.accountId, + "topup_pending", + "auto_top_up_pending", + stalePayment, + 73, + ); + fixture.stored.set(stalePayment, { + id: stalePayment, + status: "succeeded", + amount: 500, + amount_received: 500, + currency: "usd", + metadata: { type: "developer_auto_top_up", accountId: stale.accountId }, + }); + const young = await makeAccount(30_000); + await addRow( + young.accountId, + "topup_pending", + "auto_top_up_pending", + `pi_young_${young.accountId}`, + 25, + ); + const declined = await makeAccount(30_000); + await addRow( + declined.accountId, + "topup_failed", + "auto_top_up_failed", + `declined-${declined.accountId}`, + 2, + ); + const free = await makeAccount(30_000); + const before = fixture.created.length; + + await runDueAutoTopUps(new Date()); + + expect(fixture.retrieved).not.toContain(`pi_young_${young.accountId}`); + expect(fixture.retrieved.at(-1)).toBe(stalePayment); + expect(await topUpsOf(stale.accountId)).toEqual([stalePayment]); + expect(await topUpsOf(young.accountId)).toEqual([]); + expect(await topUpsOf(declined.accountId)).toEqual([]); + expect(await topUpsOf(free.accountId)).toHaveLength(1); + const charged = fixture.created + .slice(before) + .map((intent) => intent.metadata.accountId); + expect(charged).toContain(free.accountId); + expect(charged).not.toContain(stale.accountId); + expect(charged).not.toContain(young.accountId); + expect(charged).not.toContain(declined.accountId); + }); + + it("stops an owner's automatic top-ups at $1,000 a day across all their apps", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const first = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const second = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const before = fixture.created.length; + + expect((await runAutoTopUp(first.accountId)).state).toBe("succeeded"); + expect(await runAutoTopUp(second.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + + expect(fixture.created.length - before).toBe(1); + const [[enabled]] = (await pool.query( + "SELECT autoTopUpEnabled AS on_ FROM developer_credit_accounts WHERE id = ?", + [second.accountId], + )) as unknown as [[{ on_: number }]]; + expect(enabled.on_).toBe(1); + expect(warn).toHaveBeenCalledWith( + "[infra-billing] Auto top-up waits for its daily limit", + { accountId: second.accountId, reason: "owner_daily_limit" }, + ); + warn.mockRestore(); + }); + + it("lets two apps of one owner racing near the ceiling make one charge", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const first = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const second = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const before = fixture.created.length; + + const results = await Promise.all([ + runAutoTopUp(first.accountId), + runAutoTopUp(second.accountId), + ]); + + expect(fixture.created.length - before).toBe(1); + expect(results.map((result) => result.state).sort()).toEqual([ + "skipped", + "succeeded", + ]); + expect(results).toContainEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + warn.mockRestore(); + }); + + it("leaves credit bought by hand out of the owner's ceiling", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const ownerId = await makeOwner(); + const bought = await makeAccount(30_000, { ownerId }); + const automatic = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup', 90000000, 90030000, ?, 'stripe_payment_intent', '{}')`, + [id(), bought.accountId, `pi_manual_${bought.accountId}`], + ); + + expect((await runAutoTopUp(automatic.accountId)).state).toBe("succeeded"); + }); + + it("holds the owner lock only while reserving, not through a slow Stripe call", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { lockInfraAccount } = await import("@/lib/infra-balance"); + const ownerId = await makeOwner(); + const slow = await makeAccount(30_000, { ownerId }); + const other = await makeAccount(30_000, { ownerId }); + fixture.delayMs = 2_000; + try { + const startedAt = Date.now(); + const before = fixture.calledAt.length; + const topUp = runAutoTopUp(slow.accountId); + while (fixture.calledAt.length === before) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + const lockHeldMs = (fixture.calledAt.at(-1) ?? 0) - startedAt; + + const waitStarted = Date.now(); + await (fixture.database as MySql2Database).transaction(async (tx) => { + await lockInfraAccount(tx, other.appId); + }); + const otherAppWaitedMs = Date.now() - waitStarted; + + expect((await topUp).state).toBe("succeeded"); + const stripeCallMs = Date.now() - (fixture.calledAt.at(-1) ?? 0); + console.log( + JSON.stringify({ lockHeldMs, otherAppWaitedMs, stripeCallMs }), + ); + expect(stripeCallMs).toBeGreaterThanOrEqual(1_900); + expect(otherAppWaitedMs).toBeLessThan(200); + } finally { + fixture.delayMs = 40; + } + }); + + it("recovers a reservation left by a crash before Stripe was called, with one charge", async () => { + const { autoTopUpIdempotencyKey, runAutoTopUp } = await import( + "@/lib/infra-auto-top-up" + ); + const { accountId } = await makeAccount(30_000); + const key = autoTopUpIdempotencyKey(accountId, { + topUps: 0, + declines: 0, + }); + const [[owner]] = (await pool.query( + "SELECT u.email FROM users u JOIN developer_credit_accounts c ON c.ownerId = u.id WHERE c.id = ?", + [accountId], + )) as unknown as [[{ email: string }]]; + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', ?)`, + [ + id(), + accountId, + key, + JSON.stringify({ + amountCents: 500, + customerId: "cus_test", + paymentMethodId: "pm_test", + receiptEmail: owner.email, + description: "Cap Infra credits for Top-up test, auto top-up", + period: "2026-11-13", + }), + ], + ); + const before = fixture.createdKeys.length; + + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + expect((await runAutoTopUp(accountId)).state).toBe("skipped"); + + expect(fixture.createdKeys.slice(before)).toEqual([key]); + expect(await topUpsOf(accountId)).toHaveLength(1); + const [rows] = await pool.query( + "SELECT COUNT(*) AS n FROM developer_credit_transactions WHERE accountId = ? AND type = 'topup_pending'", + [accountId], + ); + expect(Number((rows as { n: number }[])[0]?.n)).toBe(0); + }); + + it("recovers a reservation whose payment Stripe made before a crash, with one charge", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const { accountId } = await makeAccount(30_000); + const before = fixture.created.length; + + fixture.loseNextResponse = true; + expect(await runAutoTopUp(accountId)).toEqual({ state: "error" }); + expect(await topUpsOf(accountId)).toHaveLength(0); + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + + expect(fixture.created.length - before).toBe(1); + expect(await topUpsOf(accountId)).toHaveLength(1); + expect(await balanceOf(accountId)).toBe(530_000); + errors.mockRestore(); + }); + + it("checks and charges one amount, read under the lock, when the setting changes mid-attempt", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const app = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const other = await makeAccount(30_000, { ownerId }); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup', 50000000, 50030000, ?, 'auto_top_up', '{}')`, + [id(), other.accountId, `pi_today_${other.accountId}`], + ); + const setAmount = (cents: number) => + pool.query( + "UPDATE developer_credit_accounts SET autoTopUpAmountCents = ? WHERE id = ?", + [cents, app.accountId], + ); + + expect(await runAutoTopUp(app.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + + await setAmount(500); + fixture.duringCreate = () => setAmount(60_000).then(() => undefined); + expect((await runAutoTopUp(app.accountId)).state).toBe("succeeded"); + + expect(fixture.sent.at(-1)?.amount).toBe(500); + const [[total]] = (await pool.query( + `SELECT SUM(t.amountMicroCredits) AS micro FROM developer_credit_transactions t + JOIN developer_credit_accounts c ON c.id = t.accountId + WHERE c.ownerId = ? AND t.type = 'topup' AND t.referenceType = 'auto_top_up'`, + [ownerId], + )) as unknown as [[{ micro: number }]]; + expect(Number(total.micro)).toBe(50_500_000); + warn.mockRestore(); + }); + + it("retries a $600 attempt whose Stripe call failed, once, while it still holds back the owner's other app", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const first = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const second = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const before = fixture.created.length; + + fixture.failNextCall = true; + expect(await runAutoTopUp(first.accountId)).toEqual({ state: "error" }); + expect(await runAutoTopUp(second.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + + expect(await runAutoTopUp(first.accountId)).toMatchObject({ + state: "succeeded", + }); + expect(fixture.created.length - before).toBe(1); + expect(fixture.created.at(-1)?.amount).toBe(60_000); + expect(await topUpsOf(first.accountId)).toHaveLength(1); + expect(await runAutoTopUp(second.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + errors.mockRestore(); + warn.mockRestore(); + }); + + it("lets the sweep retry a $600 reservation whose only blocker is itself", async () => { + const { runAutoTopUp, runDueAutoTopUps } = await import( + "@/lib/infra-auto-top-up" + ); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const { accountId } = await makeAccount(30_000, { amountCents: 60_000 }); + const before = fixture.created.length; + + fixture.failNextCall = true; + expect(await runAutoTopUp(accountId)).toEqual({ state: "error" }); + await runDueAutoTopUps(new Date()); + + expect(await topUpsOf(accountId)).toHaveLength(1); + expect( + fixture.created + .slice(before) + .filter((intent) => intent.metadata.accountId === accountId), + ).toHaveLength(1); + errors.mockRestore(); + }); + + it("voids only the stale reservation, never a new one made under the same key", async () => { + const { autoTopUpIdempotencyKey, runAutoTopUp } = await import( + "@/lib/infra-auto-top-up" + ); + const { accountId } = await makeAccount(30_000); + const key = autoTopUpIdempotencyKey(accountId, { + topUps: 0, + declines: 0, + }); + const reserve = (rowId: string, hoursAgo: number) => + pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata, createdAt) + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', '{"amountCents":500}', UTC_TIMESTAMP() - INTERVAL ? HOUR)`, + [rowId, accountId, key, hoursAgo], + ); + const stale = id(); + const fresh = id(); + await reserve(stale, 73); + fixture.duringSearch = () => reserve(fresh, 0).then(() => undefined); + + expect(await runAutoTopUp(accountId)).toEqual({ + state: "skipped", + reason: "reconciled", + }); + + const [rows] = await pool.query( + "SELECT id FROM developer_credit_transactions WHERE accountId = ? AND referenceId = ?", + [accountId, key], + ); + expect((rows as { id: string }[]).map((row) => row.id)).toEqual([fresh]); + }); + + it("lets the sweep void a deleted app's lost $600 attempt, then tops up the owner's other app and never charges the deleted one", async () => { + const { autoTopUpIdempotencyKey, runAutoTopUp, runDueAutoTopUps } = + await import("@/lib/infra-auto-top-up"); + const ownerId = await makeOwner(); + const deleted = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const other = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const key = autoTopUpIdempotencyKey(deleted.accountId, { + topUps: 0, + declines: 0, + }); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata, createdAt) + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', '{"amountCents":60000}', UTC_TIMESTAMP() - INTERVAL 73 HOUR)`, + [id(), deleted.accountId, key], + ); + await pool.query( + "UPDATE developer_apps SET deletedAt = UTC_TIMESTAMP() WHERE id = ?", + [deleted.appId], + ); + const before = fixture.created.length; + + expect(await runAutoTopUp(other.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + + let searched = 0; + fixture.duringSearch = async () => { + searched += 1; + }; + await runDueAutoTopUps(new Date()); + + expect(searched).toBe(1); + const [rows] = await pool.query( + "SELECT id FROM developer_credit_transactions WHERE accountId = ?", + [deleted.accountId], + ); + expect(rows).toEqual([]); + + await runDueAutoTopUps(new Date()); + + const charged = fixture.created.slice(before); + expect(charged.map((intent) => intent.metadata.accountId)).toEqual([ + other.accountId, + ]); + expect(charged[0]?.amount).toBe(60_000); + expect(await topUpsOf(other.accountId)).toHaveLength(1); + + await setBalance(deleted.accountId, 30_000); + expect(await runAutoTopUp(deleted.accountId)).toEqual({ + state: "skipped", + reason: "app_deleted", + }); + expect( + fixture.created.filter( + (intent) => intent.metadata.accountId === deleted.accountId, + ), + ).toEqual([]); + }); + + it("counts an unresolved payment of any age against the owner's ceiling until it resolves", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const first = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const second = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const processing = `pi_processing_${first.accountId}`; + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata, createdAt) + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', '{"amountCents":60000}', UTC_TIMESTAMP() - INTERVAL 25 HOUR)`, + [id(), first.accountId, processing], + ); + const before = fixture.created.length; + + expect(await runAutoTopUp(second.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + expect(fixture.created.length).toBe(before); + + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup_failed', 0, 30000, ?, 'auto_top_up_failed', '{}')`, + [id(), first.accountId, processing], + ); + expect((await runAutoTopUp(second.accountId)).state).toBe("succeeded"); + expect(fixture.created.length - before).toBe(1); + warn.mockRestore(); + }); + + it("keeps counting a charge whose answer was lost against the owner's ceiling", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const first = await makeAccount(30_000, { ownerId, amountCents: 60_000 }); + const second = await makeAccount(30_000, { + ownerId, + amountCents: 60_000, + }); + const before = fixture.created.length; + + fixture.loseNextResponse = true; + expect(await runAutoTopUp(first.accountId)).toEqual({ state: "error" }); + expect(await runAutoTopUp(second.accountId)).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + expect(fixture.created.length - before).toBe(1); + expect(await topUpsOf(second.accountId)).toEqual([]); + errors.mockRestore(); + warn.mockRestore(); + }); + + it("lets the sweep reconcile a 72 hour old payment even when every limit applies", async () => { + const { runDueAutoTopUps } = await import("@/lib/infra-auto-top-up"); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const ownerId = await makeOwner(); + const stale = await makeAccount(30_000, { + ownerId, + amountCents: 100_000, + }); + const other = await makeAccount(30_000, { ownerId }); + const stalePayment = `pi_stale1000_${stale.accountId}`; + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata, createdAt) + VALUES (?, ?, 'topup_pending', 0, 30000, ?, 'auto_top_up_pending', '{"amountCents":100000}', UTC_TIMESTAMP() - INTERVAL 73 HOUR)`, + [id(), stale.accountId, stalePayment], + ); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup', 500000, 530000, ?, 'auto_top_up', '{}')`, + [id(), other.accountId, `pi_today_${other.accountId}`], + ); + fixture.stored.set(stalePayment, { + id: stalePayment, + status: "succeeded", + amount: 100_000, + amount_received: 100_000, + currency: "usd", + metadata: { type: "developer_auto_top_up", accountId: stale.accountId }, + }); + await pool.query( + "UPDATE developer_credit_accounts SET balanceMicroCredits = 99000000, autoTopUpEnabled = 0 WHERE id = ?", + [stale.accountId], + ); + + await runDueAutoTopUps(new Date()); + + expect(await topUpsOf(stale.accountId)).toEqual([stalePayment]); + expect(await balanceOf(stale.accountId)).toBe(99_000_000 + 100_000_000); + warn.mockRestore(); + }); + + it("never shows a pending row in the customer's ledger", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { listAppLedgerPage, readAppLedger } = await import( + "@/lib/infra-ledger-store" + ); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const { accountId, appId } = await makeAccount(30_000); + fixture.loseNextResponse = true; + await runAutoTopUp(accountId); + await pool.query( + `INSERT INTO developer_credit_transactions (id, accountId, type, amountMicroCredits, + balanceAfterMicroCredits, referenceId, referenceType, metadata) + VALUES (?, ?, 'topup_pending', 0, 30000, 'pi_processing', 'auto_top_up_pending', '{}'), + (?, ?, 'storage_daily', -100, 29900, '2026-11-12', 'storage_day', '{}')`, + [id(), accountId, id(), accountId], + ); + + const page = await listAppLedgerPage(appId, { limit: 50 }); + const exported = []; + for await (const rows of readAppLedger(appId)) exported.push(...rows); + + expect(page.rows.map((row) => row.type)).toEqual(["storage_daily"]); + expect(exported.map((row) => row.type)).toEqual(["storage_daily"]); + errors.mockRestore(); + }); + + it("names the app in the charge description", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { accountId } = await makeAccount(30_000); + + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + expect(fixture.sent.at(-1)?.description).toBe( + "Cap Infra credits for Top-up test, auto top-up", + ); + }); + + it("tops up again the same day with a new key once the balance is low again", async () => { + const { runAutoTopUp } = await import("@/lib/infra-auto-top-up"); + const { accountId } = await makeAccount(30_000); + const before = fixture.created.length; + + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + expect((await runAutoTopUp(accountId)).state).toBe("skipped"); + await setBalance(accountId, 10); + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + + const keys = fixture.createdKeys.slice(before); + expect(keys).toHaveLength(2); + expect(new Set(keys).size).toBe(2); + expect(await topUpsOf(accountId)).toHaveLength(2); + }); + + it("stops at the daily limit of automatic top-ups", async () => { + const { AUTO_TOP_UP_DAILY_LIMIT, runAutoTopUp } = await import( + "@/lib/infra-auto-top-up" + ); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const { accountId } = await makeAccount(30_000); + + for (let i = 0; i < AUTO_TOP_UP_DAILY_LIMIT; i++) { + await setBalance(accountId, 10); + expect((await runAutoTopUp(accountId)).state).toBe("succeeded"); + } + await setBalance(accountId, 10); + expect(await runAutoTopUp(accountId)).toEqual({ + state: "skipped", + reason: "daily_limit", + }); + expect(await topUpsOf(accountId)).toHaveLength(AUTO_TOP_UP_DAILY_LIMIT); + warn.mockRestore(); + }); + }, +); diff --git a/apps/web/__tests__/integration/infra-billing-database.test.ts b/apps/web/__tests__/integration/infra-billing-database.test.ts index 22206cddf2d..a7332331629 100644 --- a/apps/web/__tests__/integration/infra-billing-database.test.ts +++ b/apps/web/__tests__/integration/infra-billing-database.test.ts @@ -9,7 +9,16 @@ import { eq, inArray } from "drizzle-orm"; import { drizzle, type MySql2Database } from "drizzle-orm/mysql2"; import { Effect } from "effect"; import { createPool, type Pool } from "mysql2/promise"; -import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + it, + vi, +} from "vitest"; +import { firstMonthStorageMicroCredits } from "@/lib/developer-pricing"; const fixture = vi.hoisted(() => ({ database: undefined as unknown, @@ -121,6 +130,13 @@ describe.runIf(Boolean(databaseUrl))( await database().select().from(developerVideos).limit(1); }); + afterEach(async () => { + const { clearFirstMonthChargesBeganCache } = await import( + "@/lib/infra-first-month-start" + ); + clearFirstMonthChargesBeganCache(); + }); + afterAll(async () => { if (fixture.database && apps.length > 0) { const accounts = await database() @@ -151,12 +167,21 @@ describe.runIf(Boolean(databaseUrl))( fixture.database = undefined; }); - it("lets exactly one of six videos measured at once take the one allowance", async () => { + const firstMonthAt1080p = (durationSeconds: number) => + firstMonthStorageMicroCredits({ durationSeconds, tier: "1080p" }); + + // Every processing video reserves its own first month of storage, so + // the allowance that fits exactly one measured video is that video's + // first month at its measured length plus the others' at the length + // they declared. Without the account lock every video passes. + const measureAtOnce = async (count: number) => { const { prepareDeveloperVideoProcessing } = await import( "@/lib/developer-video-processing" ); - const { appId } = await makeApp(1_500); - const videos = Array.from({ length: 6 }, () => ({ + const { appId } = await makeApp( + firstMonthAt1080p(3600) + (count - 1) * firstMonthAt1080p(60), + ); + const videos = Array.from({ length: count }, () => ({ id: id(), encodeId: id(), })); @@ -206,7 +231,7 @@ describe.runIf(Boolean(databaseUrl))( ); const refused = results.filter((result) => result.status === "rejected"); expect(admitted).toHaveLength(1); - expect(refused).toHaveLength(5); + expect(refused).toHaveLength(count - 1); for (const result of refused) { expect(String((result as PromiseRejectedResult).reason)).toMatch( /Not enough credit.*Add at least \$\d+\.\d\d/, @@ -220,8 +245,16 @@ describe.runIf(Boolean(databaseUrl))( .from(developerVideos) .where(eq(developerVideos.appId, appId)); expect(rows.filter((row) => row.duration === 3600)).toHaveLength(1); - expect(rows.filter((row) => row.duration === 60)).toHaveLength(5); + expect(rows.filter((row) => row.duration === 60)).toHaveLength(count - 1); expect(rows.filter((row) => row.processing?.probe)).toHaveLength(1); + }; + + it("lets exactly one of six videos measured at once take the one allowance", async () => { + await measureAtOnce(6); + }); + + it("lets exactly one of twenty videos measured at once take the one allowance", async () => { + await measureAtOnce(20); }); it("sweeps every unfinished video across pages and removes by the file's age only", async () => { diff --git a/apps/web/__tests__/integration/infra-delivery-billing-database.test.ts b/apps/web/__tests__/integration/infra-delivery-billing-database.test.ts new file mode 100644 index 00000000000..4c480a18799 --- /dev/null +++ b/apps/web/__tests__/integration/infra-delivery-billing-database.test.ts @@ -0,0 +1,1343 @@ +import { randomUUID } from "node:crypto"; +import { + developerApps, + developerCreditAccounts, + developerCreditTransactions, + developerTranscripts, +} from "@cap/database/schema"; +import { and, eq, sql } from "drizzle-orm"; +import { drizzle, type MySql2Database } from "drizzle-orm/mysql2"; +import { Effect } from "effect"; +import { createPool, type Pool } from "mysql2/promise"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +process.env.CAP_INFRA_DELIVERY_BILLING_FROM = "2000-01-01"; + +const fixture = vi.hoisted(() => ({ + database: undefined as unknown, + scheduleAutoTopUp: vi.fn(async (_target: { accountId: string }) => {}), +})); + +vi.mock("@cap/database", () => ({ + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true", NEXT_PUBLIC_WEB_URL: "https://web" }, + serverEnv: () => ({ WEB_URL: "https://web" }), +})); +vi.mock("@cap/web-backend", () => ({ + provideOptionalAuth: (effect: T) => effect, + S3Buckets: {}, +})); +vi.mock("@/lib/server", () => ({ + runPromise: (effect: Effect.Effect) => + Effect.runPromise(effect), +})); +vi.mock("workflow/api", () => ({ start: vi.fn() })); +vi.mock("@cap/database/auth/session", () => ({ getCurrentUser: vi.fn() })); +vi.mock("@cap/utils", () => ({ userIsPro: () => false })); +vi.mock("next/headers", () => ({ + cookies: async () => ({ get: () => null }), + headers: async () => ({ get: () => null }), +})); +vi.mock("next/navigation", () => ({ redirect: vi.fn(), notFound: vi.fn() })); +vi.mock("next/cache", () => ({ revalidatePath: vi.fn() })); +vi.mock("@/lib/infra-access", () => ({ canUseInfra: () => true })); +vi.mock("@/lib/infra-auto-top-up", async (importOriginal) => ({ + ...(await importOriginal()), + scheduleAutoTopUp: fixture.scheduleAutoTopUp, +})); + +const { addCreditsToAccount } = await import("@/lib/developer-credits"); +const { emptyStorageMinutes } = await import("@/lib/developer-pricing"); +const { getInfraAppStanding, getInfraBilling, shortfallOwed } = await import( + "@/lib/infra-standing" +); +const { chargeStorageDay } = await import("@/lib/infra-storage-billing"); +const { + markShortfallsNotified, + markStorageDayNotified, + shortfallsAwaitingNotice, +} = await import("@/lib/infra-storage-minimum"); +const { + chargeDeliveryDay, + deliveryBillingStart, + markDeliveryDayNotified, + settleDeliveryDays, +} = await import("@/lib/infra-delivery-billing"); +await import("@/lib/infra-auto-top-up"); +const { describeLedgerRow } = await import("@/lib/infra-ledger"); + +const databaseUrl = process.env.CAP_INFRA_TEST_DATABASE_URL; +const DAY_MS = 24 * 60 * 60 * 1000; +const id = () => randomUUID().replaceAll("-", "").slice(0, 15); +const database = () => fixture.database as MySql2Database; +// Delivery days here sit a year ahead, so this file's sweeps never reach the +// real-time days the cron route test settles in parallel on the same database. +const NOW = new Date(Date.now() + 400 * DAY_MS); +const dayKey = (daysAgo: number, from = NOW) => + new Date(from.getTime() - daysAgo * DAY_MS).toISOString().slice(0, 10); +const storageDay = (daysAgo: number) => dayKey(daysAgo, new Date()); + +// 3000 s at 1080p and 1234 s at 720p are 4234 delivered seconds, 70.57 +// minutes. At $0.95 per 1,000 minutes and 100,000 micro-credits per dollar +// that is 4234 * 95 / 60 = 6703.83, floored to 6703 micro-credits. +const FIXTURE_TIERS = [ + { tier: "1080p", seconds: 3000, bytes: 900_000_000, requests: 500 }, + { tier: "720p", seconds: 1234, bytes: 200_000_000, requests: 206 }, + { tier: "audio", seconds: 0, bytes: 40_000_000, requests: 706 }, +]; +const FIXTURE_MICRO_CREDITS = 6703; + +const ROLLUP_TABLE = + "CREATE TABLE IF NOT EXISTS `developer_delivery_daily` (`appId` varchar(15) NOT NULL, `day` date NOT NULL, `tier` varchar(8) NOT NULL, `seconds` bigint NOT NULL DEFAULT 0, `bytes` bigint NOT NULL DEFAULT 0, `requests` int NOT NULL DEFAULT 0, `abandonedBytes` bigint NOT NULL DEFAULT 0, `final` boolean NOT NULL DEFAULT false, `finalizedAt` timestamp, `updatedAt` timestamp NOT NULL, CONSTRAINT `developer_delivery_daily_appId_day_tier_pk` PRIMARY KEY(`appId`,`day`,`tier`), CONSTRAINT `developer_delivery_daily_appId_developer_apps_id_fk` FOREIGN KEY (`appId`) REFERENCES `developer_apps`(`id`) ON DELETE no action ON UPDATE no action, INDEX `day_idx` (`day`))"; + +describe.runIf(Boolean(databaseUrl))( + "delivery debit against a real database", + () => { + let pool: Pool; + + const fixtureApps: string[] = []; + const fixtureRowIds: string[] = []; + const makeApp = async (balanceMicroCredits: number) => { + const appId = id(); + fixtureApps.push(appId); + const accountId = id(); + const ownerId = id() as never; + await database().insert(developerApps).values({ + id: appId, + ownerId, + name: "Delivery fixture", + environment: "development", + }); + await database().insert(developerCreditAccounts).values({ + id: accountId, + appId, + ownerId, + balanceMicroCredits, + }); + return { appId, accountId }; + }; + + const writeDay = async ( + appId: string, + day: string, + tiers: { + tier: string; + seconds: number; + bytes?: number; + requests?: number; + }[], + final = true, + ) => { + await pool.query( + "delete from developer_delivery_daily where appId = ? and day = ?", + [appId, day], + ); + for (const row of tiers) { + await pool.query( + "insert into developer_delivery_daily (appId, day, tier, seconds, bytes, requests, updatedAt, final, finalizedAt) values (?, ?, ?, ?, ?, ?, now(), ?, if(?, now(), null))", + [ + appId, + day, + row.tier, + row.seconds, + row.bytes ?? 0, + row.requests ?? 0, + final, + final, + ], + ); + } + }; + + const deliveryRows = (accountId: string) => + database() + .select() + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + eq(developerCreditTransactions.type, "delivery_daily"), + ), + ); + + const balanceOf = async (accountId: string) => { + const [row] = await database() + .select({ balance: developerCreditAccounts.balanceMicroCredits }) + .from(developerCreditAccounts) + .where(eq(developerCreditAccounts.id, accountId)); + return row?.balance; + }; + + const settle = (now = NOW, startDay: string | null = "2000-01-01") => + settleDeliveryDays({ now, startDay }); + + beforeAll(async () => { + if (!databaseUrl) throw new Error("Missing isolated test database URL."); + const url = new URL(databaseUrl); + if ( + url.protocol !== "mysql:" || + !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) || + !/^\/cap_infra_[a-z0-9_]+$/.test(url.pathname) + ) { + throw new Error( + "Delivery tests require a session-scoped local database.", + ); + } + pool = createPool({ uri: databaseUrl, connectionLimit: 8 }); + fixture.database = drizzle(pool); + await pool.query(ROLLUP_TABLE); + }); + + beforeEach(async () => { + fixture.scheduleAutoTopUp.mockClear(); + if (fixtureApps.length > 0) { + await pool.query( + "delete from developer_delivery_daily where appId in (?)", + [fixtureApps], + ); + } + }); + + afterAll(async () => { + if (fixtureRowIds.length > 0) { + await pool?.query( + "delete from developer_credit_transactions where id in (?)", + [fixtureRowIds], + ); + } + if (fixtureApps.length > 0) { + await pool?.query( + "delete from developer_delivery_daily where appId in (?)", + [fixtureApps], + ); + } + await pool?.end(); + }); + + it("debits nothing and writes no row while the flag is off", async () => { + const { appId, accountId } = await makeApp(1_000_000); + await writeDay(appId, dayKey(2), FIXTURE_TIERS); + + for (const value of ["", " ", "true", "2026-13-01", "2026-02-30"]) { + vi.stubEnv("CAP_INFRA_DELIVERY_BILLING_FROM", value); + try { + const startDay = deliveryBillingStart(); + expect(startDay).toBeNull(); + const sweep = await settle(NOW, startDay); + expect(sweep).toMatchObject({ enabled: false, billed: 0 }); + } finally { + vi.unstubAllEnvs(); + } + } + + expect(await deliveryRows(accountId)).toEqual([]); + expect(await balanceOf(accountId)).toBe(1_000_000); + expect(fixture.scheduleAutoTopUp).not.toHaveBeenCalled(); + }); + + it("writes one row per app per final day with the exact micro-credit amount and the per tier minutes", async () => { + const now = NOW; + const first = await makeApp(1_000_000); + const second = await makeApp(1_000_000); + for (const app of [first, second]) { + await writeDay(app.appId, dayKey(2, now), FIXTURE_TIERS); + await writeDay(app.appId, dayKey(3, now), FIXTURE_TIERS); + } + + const sweep = await settle(now); + + expect(sweep).toMatchObject({ + enabled: true, + tableReady: true, + billed: 4, + }); + for (const app of [first, second]) { + const rows = await deliveryRows(app.accountId); + expect(rows.map((row) => row.referenceId).sort()).toEqual([ + dayKey(3, now), + dayKey(2, now), + ]); + for (const row of rows) { + expect(row).toMatchObject({ + amountMicroCredits: -FIXTURE_MICRO_CREDITS, + referenceType: "delivery_day", + }); + expect(row.metadata).toMatchObject({ + deliveryDate: row.referenceId, + owedMicroCredits: FIXTURE_MICRO_CREDITS, + chargedMicroCredits: FIXTURE_MICRO_CREDITS, + shortSince: null, + fellShort: false, + deliveredSeconds: 4234, + minutesByTier: { "1080p": 50, "720p": 20.57 }, + microCreditsPer1000Minutes: 95_000, + }); + expect(describeLedgerRow(row)).toEqual({ + label: "Delivery", + note: "70.57 minutes delivered (1080p 50, 720p 20.57).", + reference: { kind: "Day", value: row.referenceId }, + }); + } + expect(await balanceOf(app.accountId)).toBe( + 1_000_000 - 2 * FIXTURE_MICRO_CREDITS, + ); + } + }); + + it("never settles today, yesterday, a day older than the window or a day before the start day", async () => { + const now = NOW; + const { appId, accountId } = await makeApp(1_000_000); + for (const daysAgo of [0, 1, 4, 9]) { + await writeDay(appId, dayKey(daysAgo, now), FIXTURE_TIERS); + } + await writeDay(appId, dayKey(3, now), FIXTURE_TIERS); + + expect(await settle(now, dayKey(2, now))).toMatchObject({ billed: 0 }); + expect(await deliveryRows(accountId)).toEqual([]); + }); + + it("charges a day once however many times the sweep runs, including runs at the same time", async () => { + const now = NOW; + const { appId, accountId } = await makeApp(1_000_000); + await writeDay(appId, dayKey(2, now), FIXTURE_TIERS); + + const runs = await Promise.all([settle(now), settle(now), settle(now)]); + await settle(now); + await settle(new Date(now.getTime() + DAY_MS)); + + expect(runs.reduce((sum, run) => sum + run.billed, 0)).toBe(1); + expect(await deliveryRows(accountId)).toHaveLength(1); + expect(await balanceOf(accountId)).toBe( + 1_000_000 - FIXTURE_MICRO_CREDITS, + ); + }); + + it("waits until every row of the day is final, charges the final total once, and ignores any later rewrite", async () => { + const now = NOW; + const day = dayKey(2, now); + const { appId, accountId } = await makeApp(1_000_000); + await writeDay(appId, day, [{ tier: "1080p", seconds: 600 }], false); + + expect(await settle(now)).toMatchObject({ billed: 0, waiting: 1 }); + expect(await deliveryRows(accountId)).toEqual([]); + + await writeDay(appId, day, FIXTURE_TIERS); + await pool.query( + "update developer_delivery_daily set final = false, finalizedAt = null where appId = ? and tier = 'audio'", + [appId], + ); + expect(await settle(now)).toMatchObject({ billed: 0, waiting: 1 }); + expect(await deliveryRows(accountId)).toEqual([]); + + await writeDay(appId, day, FIXTURE_TIERS); + expect(await settle(now)).toMatchObject({ billed: 1, waiting: 0 }); + + await writeDay(appId, day, [{ tier: "1080p", seconds: 90_000 }]); + expect(await settle(now)).toMatchObject({ billed: 0 }); + expect(await settle(new Date(now.getTime() + DAY_MS))).toMatchObject({ + billed: 0, + }); + + const rows = await deliveryRows(accountId); + expect(rows).toHaveLength(1); + expect(rows[0]?.amountMicroCredits).toBe(-FIXTURE_MICRO_CREDITS); + expect(await balanceOf(accountId)).toBe( + 1_000_000 - FIXTURE_MICRO_CREDITS, + ); + }); + + it("charges the final total once when a recompute lowers a day or drops an app before it is final", async () => { + const now = NOW; + const day = dayKey(2, now); + const lowered = await makeApp(1_000_000); + const dropped = await makeApp(1_000_000); + await writeDay( + lowered.appId, + day, + [ + { tier: "1080p", seconds: 9_000 }, + { tier: "720p", seconds: 1_234 }, + { tier: "audio", seconds: 0, bytes: 1_000 }, + ], + false, + ); + await writeDay(dropped.appId, day, FIXTURE_TIERS, false); + expect(await settle(now)).toMatchObject({ billed: 0, waiting: 2 }); + + await writeDay(lowered.appId, day, [{ tier: "1080p", seconds: 3_000 }]); + await pool.query( + "delete from developer_delivery_daily where appId = ? and day = ?", + [dropped.appId, day], + ); + expect(await settle(now)).toMatchObject({ billed: 1, waiting: 0 }); + expect(await settle(now)).toMatchObject({ billed: 0 }); + + const [row] = await deliveryRows(lowered.accountId); + expect(row?.amountMicroCredits).toBe(-4_750); + expect(row?.metadata).toMatchObject({ + deliveredSeconds: 3_000, + minutesByTier: { "1080p": 50 }, + }); + expect(await deliveryRows(dropped.accountId)).toEqual([]); + expect(await balanceOf(dropped.accountId)).toBe(1_000_000); + }); + + it("returns the exact rows a real run writes from a dry run, twice the same, and writes nothing", async () => { + const now = NOW; + const rich = await makeApp(1_000_000); + const poor = await makeApp(9_000); + for (const app of [rich, poor]) { + await writeDay(app.appId, dayKey(3, now), FIXTURE_TIERS); + await writeDay(app.appId, dayKey(2, now), FIXTURE_TIERS); + } + const ledgerCount = async () => { + const [rows] = await pool.query( + "select count(*) as count from developer_credit_transactions where accountId in (?, ?)", + [rich.accountId, poor.accountId], + ); + return Number((rows as { count: number }[])[0]?.count); + }; + + const first = await settleDeliveryDays({ + now, + startDay: "2000-01-01", + dryRun: true, + }); + const second = await settleDeliveryDays({ + now, + startDay: "2000-01-01", + dryRun: true, + }); + + expect(await ledgerCount()).toBe(0); + expect(await balanceOf(rich.accountId)).toBe(1_000_000); + expect(await balanceOf(poor.accountId)).toBe(9_000); + expect(fixture.scheduleAutoTopUp).not.toHaveBeenCalled(); + expect(second.planned).toEqual(first.planned); + expect(first).toMatchObject({ dryRun: true, billed: 4, short: 1 }); + const poorPlan = first.planned.filter((row) => row.appId === poor.appId); + expect(poorPlan.map((row) => row.amountMicroCredits)).toEqual([ + -FIXTURE_MICRO_CREDITS, + -(9_000 - FIXTURE_MICRO_CREDITS), + ]); + + await settle(now); + const written = [ + ...(await deliveryRows(rich.accountId)), + ...(await deliveryRows(poor.accountId)), + ].map((row) => ({ + accountId: row.accountId, + referenceId: row.referenceId, + amountMicroCredits: row.amountMicroCredits, + balanceAfterMicroCredits: row.balanceAfterMicroCredits, + metadata: row.metadata, + })); + const byKey = ( + a: { accountId: string; referenceId: string | null }, + b: { accountId: string; referenceId: string | null }, + ) => + `${a.accountId}:${a.referenceId}`.localeCompare( + `${b.accountId}:${b.referenceId}`, + ); + const planned = first.planned + .map(({ appId: _appId, ...row }) => row) + .sort(byKey); + expect(written.sort(byKey)).toEqual(planned); + expect(written).toHaveLength(planned.length); + }); + + it("puts the app in grace, then pauses it and refuses uploads, exactly as a storage day the balance cannot pay", async () => { + const now = NOW; + const delivery = await makeApp(1_000); + const storage = await makeApp(1_000); + await writeDay(delivery.appId, dayKey(2, now), FIXTURE_TIERS); + const fellShort: unknown[] = []; + + await settleDeliveryDays({ + now, + startDay: "2000-01-01", + onFellShort: async (shortfall) => { + fellShort.push(shortfall); + }, + }); + const storageResult = await chargeStorageDay({ + appId: storage.appId, + accountId: storage.accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: FIXTURE_MICRO_CREDITS * 30, + dailyMicroCredits: FIXTURE_MICRO_CREDITS, + }, + dateKey: storageDay(0), + now, + }); + + const [row] = await deliveryRows(delivery.accountId); + expect(row).toMatchObject({ + amountMicroCredits: -1_000, + balanceAfterMicroCredits: 0, + }); + expect(row?.metadata).toMatchObject({ + owedMicroCredits: FIXTURE_MICRO_CREDITS, + chargedMicroCredits: 1_000, + shortSince: now.toISOString(), + fellShort: true, + }); + expect(storageResult).toMatchObject({ + state: "billed", + chargedMicroCredits: 1_000, + shortSince: now, + fellShort: true, + }); + expect(fellShort).toEqual([ + { + accountId: delivery.accountId, + dateKey: dayKey(2, now), + shortSince: now, + }, + ]); + const deliveryStanding = (await getInfraBilling(delivery.appId)).standing; + const storageStanding = (await getInfraBilling(storage.appId)).standing; + expect(deliveryStanding).toEqual({ + state: "grace", + pausesAt: new Date(now.getTime() + 7 * DAY_MS), + }); + expect(deliveryStanding).toEqual(storageStanding); + const pausesAt = new Date(now.getTime() + 7 * DAY_MS); + const deliveryPaused = await getInfraBilling( + delivery.appId, + null, + pausesAt, + ); + const storagePaused = await getInfraBilling( + storage.appId, + null, + pausesAt, + ); + expect(deliveryPaused.standing).toEqual({ state: "paused", pausesAt }); + expect(await getInfraAppStanding(delivery.appId)).toEqual({ + state: "grace", + pausesAt, + }); + expect(deliveryPaused.admission).toMatchObject({ + allowed: false, + reason: "out_of_credit", + balanceMicroCredits: 0, + requiredMicroCredits: FIXTURE_MICRO_CREDITS, + shortfallMicroCredits: FIXTURE_MICRO_CREDITS, + }); + expect(deliveryPaused.standing).toEqual(storagePaused.standing); + expect(deliveryPaused.admission).toEqual(storagePaused.admission); + expect(describeLedgerRow(row as never).note).toBe( + "70.57 minutes delivered (1080p 50, 720p 20.57). The balance paid for part of this day. The rest is not charged later.", + ); + + for (const app of [delivery, storage]) { + await addCreditsToAccount({ + accountId: app.accountId, + amountCents: 7, + referenceId: id(), + referenceType: "stripe_payment_intent", + }); + expect((await getInfraBilling(app.appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + } + }); + + it("settles a day the balance cannot pay at all with one unpaid row that keeps the storage grace start, and never charges it later", async () => { + const now = NOW; + const shortSince = new Date(now.getTime() - 3 * DAY_MS); + const { appId, accountId } = await makeApp(500); + await chargeStorageDay({ + appId, + accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: 30_000, + dailyMicroCredits: 1_000, + }, + dateKey: storageDay(3), + now: shortSince, + }); + await writeDay(appId, dayKey(2, now), FIXTURE_TIERS); + + expect(await settle(now)).toMatchObject({ billed: 1, short: 1 }); + const [unpaid] = await deliveryRows(accountId); + expect(unpaid).toMatchObject({ + amountMicroCredits: 0, + balanceAfterMicroCredits: 0, + }); + expect(unpaid?.metadata).toMatchObject({ + owedMicroCredits: FIXTURE_MICRO_CREDITS, + chargedMicroCredits: 0, + shortSince: shortSince.toISOString(), + fellShort: false, + }); + expect(describeLedgerRow(unpaid as never).label).toBe("Delivery, unpaid"); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "grace", + pausesAt: new Date(shortSince.getTime() + 7 * DAY_MS), + }); + + await database() + .update(developerCreditAccounts) + .set({ balanceMicroCredits: 50_000 }) + .where(eq(developerCreditAccounts.id, accountId)); + expect(await settle(now)).toMatchObject({ billed: 0 }); + expect(await settle(new Date(now.getTime() + 3 * DAY_MS))).toMatchObject({ + billed: 0, + }); + expect(await balanceOf(accountId)).toBe(50_000); + + const later = new Date(now.getTime() + 60_000); + await chargeStorageDay({ + appId, + accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: 3_000, + dailyMicroCredits: 100, + }, + dateKey: storageDay(0), + now: later, + }); + const shortfallRows = await database() + .select({ metadata: developerCreditTransactions.metadata }) + .from(developerCreditTransactions) + .where(eq(developerCreditTransactions.accountId, accountId)); + expect( + shortfallRows + .map( + (row) => + row.metadata as { shortSince?: string; resolvedAt?: string }, + ) + .filter((fields) => fields.shortSince === shortSince.toISOString()) + .map((fields) => fields.resolvedAt), + ).toEqual([later.toISOString(), later.toISOString()]); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + }); + + describe("an unpaid delivery day inside an open shortfall", () => { + const UNPAID_SECONDS = 60; + const SHORT_OWED = 100_000; + const topUp = (accountId: string, amountCents: number) => + addCreditsToAccount({ + accountId, + amountCents, + referenceId: id(), + referenceType: "stripe_payment_intent", + }); + + const expectRecoveryNeedsTheWholeShortfall = async ( + appId: string, + accountId: string, + shortSince: Date, + ) => { + const grace = { + state: "grace", + pausesAt: new Date(shortSince.getTime() + 7 * DAY_MS), + }; + const [unpaid] = (await deliveryRows(accountId)).filter( + (row) => row.referenceId === dayKey(2, NOW), + ); + expect(unpaid).toMatchObject({ amountMicroCredits: 0 }); + expect(unpaid?.metadata).toMatchObject({ + owedMicroCredits: 95, + chargedMicroCredits: 0, + shortSince: shortSince.toISOString(), + fellShort: false, + duringShortfall: true, + }); + expect((await getInfraBilling(appId)).standing).toEqual(grace); + + await topUp(accountId, 1); + expect((await getInfraBilling(appId)).standing).toEqual(grace); + + await topUp(accountId, 100); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + + const balance = await balanceOf(accountId); + expect(await settle(NOW)).toMatchObject({ billed: 0 }); + expect( + await settle(new Date(NOW.getTime() + 3 * DAY_MS)), + ).toMatchObject({ billed: 0 }); + expect(await balanceOf(accountId)).toBe(balance); + }; + + it("keeps the storage shortfall as the amount to recover", async () => { + const { appId, accountId } = await makeApp(0); + const storage = await chargeStorageDay({ + appId, + accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: SHORT_OWED * 30, + dailyMicroCredits: SHORT_OWED, + }, + dateKey: storageDay(0), + now: NOW, + }); + expect(storage).toMatchObject({ state: "billed", fellShort: true }); + await writeDay(appId, dayKey(2, NOW), [ + { tier: "1080p", seconds: UNPAID_SECONDS }, + ]); + await settle(NOW); + + await expectRecoveryNeedsTheWholeShortfall(appId, accountId, NOW); + }); + + it("keeps an earlier delivery shortfall as the amount to recover", async () => { + const { appId, accountId } = await makeApp(0); + await writeDay(appId, dayKey(3, NOW), [ + { tier: "1080p", seconds: Math.ceil((SHORT_OWED * 60) / 95) }, + ]); + await writeDay(appId, dayKey(2, NOW), [ + { tier: "1080p", seconds: UNPAID_SECONDS }, + ]); + await settle(NOW); + + const [first] = (await deliveryRows(accountId)).filter( + (row) => row.referenceId === dayKey(3, NOW), + ); + expect(first?.metadata).toMatchObject({ + owedMicroCredits: SHORT_OWED, + chargedMicroCredits: 0, + fellShort: true, + }); + expect(first?.metadata).not.toHaveProperty("duringShortfall"); + await expectRecoveryNeedsTheWholeShortfall(appId, accountId, NOW); + }); + }); + + describe("a delivery day charged while a shortfall is open", () => { + const SHORT_OWED = 100_000; + const topUp = (accountId: string, amountCents: number) => + addCreditsToAccount({ + accountId, + amountCents, + referenceId: id(), + referenceType: "stripe_payment_intent", + }); + const storageShort = async (appId: string, accountId: string) => + chargeStorageDay({ + appId, + accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: SHORT_OWED * 30, + dailyMicroCredits: SHORT_OWED, + }, + dateKey: storageDay(0), + now: NOW, + }); + const grace = { + state: "grace", + pausesAt: new Date(NOW.getTime() + 7 * DAY_MS), + }; + + it("never clears a storage shortfall by paying a small delivery day in full", async () => { + const { appId, accountId } = await makeApp(0); + expect(await storageShort(appId, accountId)).toMatchObject({ + fellShort: true, + }); + await topUp(accountId, 1); + await writeDay(appId, dayKey(2, NOW), [{ tier: "1080p", seconds: 60 }]); + await settle(NOW); + + const [row] = await deliveryRows(accountId); + expect(row).toMatchObject({ amountMicroCredits: -95 }); + expect(row?.metadata).toMatchObject({ + shortSince: null, + duringShortfall: true, + }); + expect((await getInfraBilling(appId)).standing).toEqual(grace); + expect(await shortfallOwed(database() as never, accountId, NOW)).toBe( + SHORT_OWED, + ); + }); + + it("never lowers what a storage shortfall needs with a delivery day that is itself short", async () => { + const { appId, accountId } = await makeApp(0); + await storageShort(appId, accountId); + await topUp(accountId, 1); + await writeDay(appId, dayKey(2, NOW), FIXTURE_TIERS); + await settle(NOW); + + const [row] = await deliveryRows(accountId); + expect(row).toMatchObject({ amountMicroCredits: -1_000 }); + expect(row?.metadata).toMatchObject({ + owedMicroCredits: FIXTURE_MICRO_CREDITS, + shortSince: NOW.toISOString(), + fellShort: false, + duringShortfall: true, + }); + expect((await getInfraBilling(appId)).standing).toEqual(grace); + expect(await shortfallOwed(database() as never, accountId, NOW)).toBe( + SHORT_OWED, + ); + + await topUp(accountId, 100); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + }); + + it("lets a later storage day paid in full clear an open delivery shortfall, as a later storage day clears a storage shortfall", async () => { + const delivery = await makeApp(0); + await writeDay(delivery.appId, dayKey(2, NOW), [ + { tier: "1080p", seconds: Math.ceil((SHORT_OWED * 60) / 95) }, + ]); + await settle(NOW); + expect((await getInfraBilling(delivery.appId)).standing).toEqual(grace); + await database() + .update(developerCreditTransactions) + .set({ + createdAt: sql`${developerCreditTransactions.createdAt} - INTERVAL 2 SECOND`, + }) + .where(eq(developerCreditTransactions.accountId, delivery.accountId)); + await topUp(delivery.accountId, 1); + const storagePaid = await chargeStorageDay({ + appId: delivery.appId, + accountId: delivery.accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: 3_000, + dailyMicroCredits: 100, + }, + dateKey: storageDay(0), + now: NOW, + }); + + const storageOnly = await makeApp(0); + await chargeStorageDay({ + appId: storageOnly.appId, + accountId: storageOnly.accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: SHORT_OWED * 30, + dailyMicroCredits: SHORT_OWED, + }, + dateKey: storageDay(1), + now: NOW, + }); + await topUp(storageOnly.accountId, 1); + const storageOnlyPaid = await chargeStorageDay({ + appId: storageOnly.appId, + accountId: storageOnly.accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: 3_000, + dailyMicroCredits: 100, + }, + dateKey: storageDay(0), + now: NOW, + }); + + expect(storagePaid).toMatchObject({ + state: "billed", + chargedMicroCredits: 100, + shortSince: null, + }); + expect(storageOnlyPaid).toMatchObject({ + state: "billed", + chargedMicroCredits: 100, + shortSince: null, + }); + const active = { state: "active", pausesAt: null }; + expect((await getInfraBilling(delivery.appId)).standing).toEqual( + active, + ); + expect((await getInfraBilling(storageOnly.appId)).standing).toEqual( + active, + ); + }); + + it("still resolves a delivery-only app active when paid and grace when short", async () => { + const paid = await makeApp(1_000_000); + const short = await makeApp(1_000); + for (const app of [paid, short]) { + await writeDay(app.appId, dayKey(2, NOW), FIXTURE_TIERS); + } + await settle(NOW); + + const [paidRow] = await deliveryRows(paid.accountId); + const [shortRow] = await deliveryRows(short.accountId); + expect(paidRow?.metadata).not.toHaveProperty("duringShortfall"); + expect(shortRow?.metadata).not.toHaveProperty("duringShortfall"); + expect((await getInfraBilling(paid.appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + expect((await getInfraBilling(short.appId)).standing).toEqual(grace); + }); + + it("reads no delivery row for standing while the flag is off", async () => { + const { appId } = await makeApp(1_000); + await writeDay(appId, dayKey(2, NOW), FIXTURE_TIERS); + await settle(NOW); + expect((await getInfraBilling(appId)).standing).toEqual(grace); + + vi.stubEnv("CAP_INFRA_DELIVERY_BILLING_FROM", ""); + try { + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + } finally { + vi.unstubAllEnvs(); + } + }); + }); + + it("charges a day that goes final late, on the first run after it does, and only once", async () => { + const now = NOW; + const day = dayKey(5, now); + const { appId, accountId } = await makeApp(1_000_000); + await writeDay(appId, day, FIXTURE_TIERS, false); + + for (const back of [3, 2, 1]) { + expect( + await settle(new Date(now.getTime() - back * DAY_MS)), + ).toMatchObject({ billed: 0 }); + } + expect(await deliveryRows(accountId)).toEqual([]); + + await writeDay(appId, day, FIXTURE_TIERS); + expect(await settle(now)).toMatchObject({ billed: 1 }); + expect(await settle(new Date(now.getTime() + DAY_MS))).toMatchObject({ + billed: 0, + }); + const rows = await deliveryRows(accountId); + expect( + rows.map((row) => [row.referenceId, row.amountMicroCredits]), + ).toEqual([[day, -FIXTURE_MICRO_CREDITS]]); + }); + + it("counts a day still not final on the last day it can be charged, and leaves it alone after that", async () => { + const now = NOW; + const { appId, accountId } = await makeApp(1_000_000); + await writeDay(appId, dayKey(7, now), FIXTURE_TIERS, false); + await writeDay(appId, dayKey(8, now), FIXTURE_TIERS); + + expect(await settle(now)).toMatchObject({ + billed: 0, + waiting: 1, + notFinalAtWindowEnd: 1, + }); + expect(await deliveryRows(accountId)).toEqual([]); + }); + + it("keeps the later short day as the standing when two days are posted in the same second, whatever their ids", async () => { + const now = NOW; + const { appId, accountId } = await makeApp(9_000); + await writeDay(appId, dayKey(3, now), FIXTURE_TIERS); + await writeDay(appId, dayKey(2, now), FIXTURE_TIERS); + + await settle(now); + + const rows = await deliveryRows(accountId); + const earlier = rows.find((row) => row.referenceId === dayKey(3, now)); + const later = rows.find((row) => row.referenceId === dayKey(2, now)); + expect(earlier?.amountMicroCredits).toBe(-FIXTURE_MICRO_CREDITS); + expect(later?.amountMicroCredits).toBe(-(9_000 - FIXTURE_MICRO_CREDITS)); + const postedAt = new Date(Math.floor(Date.now() / 1000) * 1000); + for (const [row, rowId] of [ + [earlier, "zzzzzzzzzzzzzzz"], + [later, "000000000000000"], + ] as const) { + await database() + .update(developerCreditTransactions) + .set({ id: rowId, createdAt: postedAt }) + .where(eq(developerCreditTransactions.id, row?.id ?? "")); + } + fixtureRowIds.push("zzzzzzzzzzzzzzz", "000000000000000"); + + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "grace", + pausesAt: new Date(now.getTime() + 7 * DAY_MS), + }); + }); + + it("keeps a short delivery day as the standing over a paid storage day posted in the same second", async () => { + const now = NOW; + const { appId, accountId } = await makeApp(1_000); + await chargeStorageDay({ + appId, + accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: 3_000, + dailyMicroCredits: 100, + }, + dateKey: storageDay(0), + now, + }); + await writeDay(appId, dayKey(2, now), FIXTURE_TIERS); + await settle(now); + const postedAt = new Date(Math.floor(Date.now() / 1000) * 1000); + await database() + .update(developerCreditTransactions) + .set({ createdAt: postedAt }) + .where(eq(developerCreditTransactions.accountId, accountId)); + + const [delivery] = await deliveryRows(accountId); + expect(delivery?.metadata).toMatchObject({ + chargedMicroCredits: 900, + shortSince: now.toISOString(), + fellShort: true, + }); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "grace", + pausesAt: new Date(now.getTime() + 7 * DAY_MS), + }); + }); + + it("emails the owner once per shortfall: on the day it starts, never again on later runs, and not for a delivery day that falls short inside a storage shortfall", async () => { + const now = NOW; + const sent: string[] = []; + const email = + (accepted: boolean) => + async ({ + accountId, + dateKey, + }: { + accountId: string; + dateKey: string; + shortSince: Date; + }) => { + sent.push(`${accountId}:${dateKey}`); + if (accepted) await markDeliveryDayNotified(accountId, dateKey, now); + }; + const awaiting = async (accountId: string) => + (await shortfallsAwaitingNotice([accountId], new Date(0))).get( + accountId, + )?.ids ?? []; + + const delivery = await makeApp(1_000); + await writeDay(delivery.appId, dayKey(3, now), FIXTURE_TIERS); + await writeDay(delivery.appId, dayKey(2, now), FIXTURE_TIERS); + await settleDeliveryDays({ + now, + startDay: "2000-01-01", + onFellShort: email(true), + }); + await settleDeliveryDays({ + now: new Date(now.getTime() + DAY_MS), + startDay: "2000-01-01", + onFellShort: email(true), + }); + expect(sent).toEqual([`${delivery.accountId}:${dayKey(3, now)}`]); + expect(await awaiting(delivery.accountId)).toEqual([]); + + const rejected = await makeApp(0); + await writeDay(rejected.appId, dayKey(2, now), FIXTURE_TIERS); + await settleDeliveryDays({ + now, + startDay: "2000-01-01", + onFellShort: email(false), + }); + const [retry] = await awaiting(rejected.accountId); + expect(retry).toBeDefined(); + await markShortfallsNotified([retry as string], now); + expect(await awaiting(rejected.accountId)).toEqual([]); + + const storageFirst = await makeApp(500); + await chargeStorageDay({ + appId: storageFirst.appId, + accountId: storageFirst.accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: 30_000, + dailyMicroCredits: 1_000, + }, + dateKey: storageDay(3), + now: new Date(now.getTime() - DAY_MS), + }); + await markStorageDayNotified(storageFirst.accountId, storageDay(3), now); + await database() + .update(developerCreditAccounts) + .set({ balanceMicroCredits: 300 }) + .where(eq(developerCreditAccounts.id, storageFirst.accountId)); + await writeDay(storageFirst.appId, dayKey(2, now), FIXTURE_TIERS); + sent.length = 0; + await settleDeliveryDays({ + now, + startDay: "2000-01-01", + onFellShort: email(true), + }); + expect(sent).toEqual([]); + expect(await awaiting(storageFirst.accountId)).toEqual([]); + }); + + it("carries an open storage shortfall's start into a delivery day it can only part pay, without a second email", async () => { + const now = NOW; + const shortSince = new Date(now.getTime() - 2 * DAY_MS); + const { appId, accountId } = await makeApp(500); + await chargeStorageDay({ + appId, + accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: 30_000, + dailyMicroCredits: 1_000, + }, + dateKey: storageDay(3), + now: shortSince, + }); + await database() + .update(developerCreditAccounts) + .set({ balanceMicroCredits: 300 }) + .where(eq(developerCreditAccounts.id, accountId)); + await writeDay(appId, dayKey(2, now), FIXTURE_TIERS); + const fellShort: unknown[] = []; + + await settleDeliveryDays({ + now, + startDay: "2000-01-01", + onFellShort: async (shortfall) => { + fellShort.push(shortfall); + }, + }); + + const [row] = await deliveryRows(accountId); + expect(row).toMatchObject({ + amountMicroCredits: -300, + balanceAfterMicroCredits: 0, + }); + expect(row?.metadata).toMatchObject({ + owedMicroCredits: FIXTURE_MICRO_CREDITS, + shortSince: shortSince.toISOString(), + fellShort: false, + }); + expect(fellShort).toEqual([]); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "grace", + pausesAt: new Date(shortSince.getTime() + 7 * DAY_MS), + }); + }); + + it("resolves a delivery shortfall with the rest of its rows once a released hold covers it", async () => { + const now = NOW; + const { appId, accountId } = await makeApp(10_000); + const holdId = id(); + await database().insert(developerTranscripts).values({ + id: holdId, + appId, + status: "processing", + sourceType: "upload", + reservedMicroCredits: 9_000, + }); + await writeDay(appId, dayKey(2, now), FIXTURE_TIERS); + await settle(now); + + const [short] = await deliveryRows(accountId); + expect(short).toMatchObject({ + amountMicroCredits: -1_000, + balanceAfterMicroCredits: 9_000, + }); + expect(short?.metadata).toMatchObject({ shortSince: now.toISOString() }); + expect((await getInfraBilling(appId)).standing.state).toBe("grace"); + + await database() + .update(developerTranscripts) + .set({ status: "complete", reservedMicroCredits: 0 }) + .where(eq(developerTranscripts.id, holdId)); + const later = new Date(now.getTime() + 60_000); + await chargeStorageDay({ + appId, + accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: 3_000, + dailyMicroCredits: 100, + }, + dateKey: storageDay(0), + now: later, + }); + + const [resolved] = await deliveryRows(accountId); + expect(resolved?.metadata).toMatchObject({ + shortSince: now.toISOString(), + resolvedAt: later.toISOString(), + }); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + }); + + it("debits nothing and does not throw while the rollup table, or its final column, does not exist", async () => { + const { appId, accountId } = await makeApp(1_000_000); + await writeDay(appId, dayKey(2), FIXTURE_TIERS); + const emptyName = `${new URL(databaseUrl as string).pathname.slice(1)}_norollup`; + await pool.query(`drop database if exists \`${emptyName}\``); + await pool.query(`create database \`${emptyName}\``); + const emptyUrl = new URL(databaseUrl as string); + emptyUrl.pathname = `/${emptyName}`; + const emptyPool = createPool({ + uri: emptyUrl.toString(), + connectionLimit: 2, + }); + const shared = fixture.database; + fixture.database = drizzle(emptyPool); + try { + expect(await settle()).toMatchObject({ + enabled: true, + tableReady: false, + billed: 0, + failed: 0, + }); + await emptyPool.query( + "create table developer_delivery_daily (appId varchar(15) not null, day date not null, tier varchar(8) not null, seconds bigint not null default 0, updatedAt timestamp not null, primary key (appId, day, tier))", + ); + await emptyPool.query( + "insert into developer_delivery_daily (appId, day, tier, seconds, updatedAt) values (?, ?, '1080p', 3000, now())", + [appId, dayKey(2)], + ); + expect(await settle()).toMatchObject({ + enabled: true, + tableReady: false, + billed: 0, + failed: 0, + }); + } finally { + fixture.database = shared; + await emptyPool.end(); + await pool.query(`drop database if exists \`${emptyName}\``); + } + expect(await deliveryRows(accountId)).toEqual([]); + expect(await balanceOf(accountId)).toBe(1_000_000); + }); + + it("never bills abandoned bytes: a final row with only abandoned bytes bills nothing, and they never change an amount", async () => { + const now = NOW; + const abandonedOnly = await makeApp(1_000_000); + const withAbandoned = await makeApp(1_000_000); + await writeDay(abandonedOnly.appId, dayKey(2, now), [ + { tier: "1080p", seconds: 0, bytes: 9_000_000 }, + ]); + await writeDay(withAbandoned.appId, dayKey(2, now), FIXTURE_TIERS); + await pool.query( + "update developer_delivery_daily set abandonedBytes = 7000000000 where appId in (?, ?)", + [abandonedOnly.appId, withAbandoned.appId], + ); + + await settle(now); + + expect(await deliveryRows(abandonedOnly.accountId)).toEqual([]); + const [row] = await deliveryRows(withAbandoned.accountId); + expect(row?.amountMicroCredits).toBe(-FIXTURE_MICRO_CREDITS); + }); + + it("schedules one auto top-up after an account's charged days, and none when nothing was charged", async () => { + const now = NOW; + const paying = await makeApp(1_000_000); + const empty = await makeApp(0); + for (const app of [paying, empty]) { + await writeDay(app.appId, dayKey(2, now), FIXTURE_TIERS); + await writeDay(app.appId, dayKey(3, now), FIXTURE_TIERS); + } + + await settle(now); + await settle(now); + + expect(fixture.scheduleAutoTopUp.mock.calls).toEqual([ + [{ accountId: paying.accountId }], + ]); + expect( + (await deliveryRows(empty.accountId)).map( + (row) => row.amountMicroCredits, + ), + ).toEqual([0, 0]); + }); + + it("skips a deleted app and leaves a day with no video minutes alone", async () => { + const now = NOW; + const deleted = await makeApp(1_000_000); + const audioOnly = await makeApp(1_000_000); + await database() + .update(developerApps) + .set({ deletedAt: new Date() }) + .where(eq(developerApps.id, deleted.appId)); + await writeDay(deleted.appId, dayKey(2, now), FIXTURE_TIERS); + await writeDay(audioOnly.appId, dayKey(2, now), [ + { tier: "audio", seconds: 0, bytes: 5_000 }, + ]); + + expect(await settle(now)).toMatchObject({ billed: 0 }); + expect(await deliveryRows(deleted.accountId)).toEqual([]); + expect(await deliveryRows(audioOnly.accountId)).toEqual([]); + }); + + it("charges a single day directly with the same rules the sweep uses", async () => { + const { appId, accountId } = await makeApp(10_000); + const result = await chargeDeliveryDay({ + appId, + accountId, + delivered: { + day: "2026-11-10", + seconds: 4234, + secondsByTier: { "1080p": 3000, "720p": 1234 }, + }, + now: new Date(), + }); + expect(result).toMatchObject({ + state: "billed", + owedMicroCredits: FIXTURE_MICRO_CREDITS, + chargedMicroCredits: FIXTURE_MICRO_CREDITS, + balanceMicroCredits: 10_000 - FIXTURE_MICRO_CREDITS, + }); + const [rows] = await pool.query( + "select count(*) as count from developer_credit_transactions where accountId = ?", + [accountId], + ); + expect(Number((rows as { count: number }[])[0]?.count)).toBe(1); + }); + }, +); diff --git a/apps/web/__tests__/integration/infra-delivery-cron-database.test.ts b/apps/web/__tests__/integration/infra-delivery-cron-database.test.ts new file mode 100644 index 00000000000..996493f91ee --- /dev/null +++ b/apps/web/__tests__/integration/infra-delivery-cron-database.test.ts @@ -0,0 +1,199 @@ +import { randomUUID } from "node:crypto"; +import { + developerApps, + developerCreditAccounts, + developerCreditTransactions, +} from "@cap/database/schema"; +import { and, eq, inArray } from "drizzle-orm"; +import { drizzle, type MySql2Database } from "drizzle-orm/mysql2"; +import { createPool, type Pool } from "mysql2/promise"; +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + it, + vi, +} from "vitest"; + +const fixture = vi.hoisted(() => ({ database: undefined as unknown })); + +vi.mock("@cap/database", () => ({ + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true", NEXT_PUBLIC_WEB_URL: "https://web" }, + serverEnv: () => ({ WEB_URL: "https://web" }), +})); +vi.mock("@/lib/infra-pricing", async (importOriginal) => ({ + ...(await importOriginal()), + INFRA_DELIVERY_BILLED_FROM: "2000-01-01", +})); +vi.mock("@/lib/infra-storage", () => ({ + getStorageUsageByApp: async () => new Map(), +})); +vi.mock("@/lib/infra-storage-billing", async (importOriginal) => ({ + ...(await importOriginal()), + chargeStorageDay: async () => ({ state: "skipped" }), + recordStorageDayVisits: async () => {}, +})); +vi.mock("@/lib/infra-storage-minimum", () => ({ + chargeMissedFirstMonths: async () => ({ charged: 0, short: 0, failed: 0 }), + shortfallsAwaitingNotice: async () => new Map(), + openFirstMonthShortfalls: async () => new Set(), + markShortfallsNotified: async () => {}, + markStorageDayNotified: async () => {}, +})); +vi.mock("@/lib/infra-auto-top-up", () => ({ + runDueAutoTopUps: async () => ({ attempted: 0, succeeded: 0, failed: 0 }), + scheduleAutoTopUp: async () => {}, +})); +vi.mock("@/lib/infra-billing-emails", () => ({ + sendInfraOutOfCreditEmail: async () => false, +})); + +const { GET } = await import("@/app/api/cron/developer-storage/route"); + +const databaseUrl = process.env.CAP_INFRA_TEST_DATABASE_URL; +const DAY_MS = 24 * 60 * 60 * 1000; +const CRON_SECRET = "delivery-cron-test-secret"; +const id = () => randomUUID().replaceAll("-", "").slice(0, 15); +const database = () => fixture.database as MySql2Database; +const dayKey = (daysAgo: number) => + new Date(Date.now() - daysAgo * DAY_MS).toISOString().slice(0, 10); + +const ROLLUP_TABLE = + "CREATE TABLE IF NOT EXISTS `developer_delivery_daily` (`appId` varchar(15) NOT NULL, `day` date NOT NULL, `tier` varchar(8) NOT NULL, `seconds` bigint NOT NULL DEFAULT 0, `bytes` bigint NOT NULL DEFAULT 0, `requests` int NOT NULL DEFAULT 0, `abandonedBytes` bigint NOT NULL DEFAULT 0, `final` boolean NOT NULL DEFAULT false, `finalizedAt` timestamp, `updatedAt` timestamp NOT NULL, CONSTRAINT `developer_delivery_daily_appId_day_tier_pk` PRIMARY KEY(`appId`,`day`,`tier`), CONSTRAINT `developer_delivery_daily_appId_developer_apps_id_fk` FOREIGN KEY (`appId`) REFERENCES `developer_apps`(`id`) ON DELETE no action ON UPDATE no action, INDEX `day_idx` (`day`))"; + +describe.runIf(Boolean(databaseUrl))( + "the storage cron's delivery debit against a real database", + () => { + let pool: Pool; + const fixtureApps: string[] = []; + + const makeApp = async () => { + const appId = id(); + const accountId = id(); + const ownerId = id() as never; + fixtureApps.push(appId); + await database().insert(developerApps).values({ + id: appId, + ownerId, + name: "Delivery cron fixture", + environment: "development", + }); + await database().insert(developerCreditAccounts).values({ + id: accountId, + appId, + ownerId, + balanceMicroCredits: 1_000_000, + }); + for (const daysAgo of [5, 4, 3, 2]) { + await pool.query( + "insert into developer_delivery_daily (appId, day, tier, seconds, bytes, requests, updatedAt, final, finalizedAt) values (?, ?, '1080p', 600, 1000, 10, now(), true, now())", + [appId, dayKey(daysAgo)], + ); + } + return { appId, accountId }; + }; + + const deliveryDays = async (accountId: string) => + ( + await database() + .select({ day: developerCreditTransactions.referenceId }) + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + eq(developerCreditTransactions.type, "delivery_daily"), + ), + ) + ) + .map((row) => row.day) + .sort(); + + const runCron = async () => { + const response = await GET( + new Request("https://localhost/api/cron/developer-storage", { + headers: { authorization: `Bearer ${CRON_SECRET}` }, + }), + ); + expect(response.status).toBe(200); + return response.json(); + }; + + beforeAll(async () => { + if (!databaseUrl) throw new Error("Missing isolated test database URL."); + const url = new URL(databaseUrl); + if ( + url.protocol !== "mysql:" || + !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) || + !/^\/cap_infra_[a-z0-9_]+$/.test(url.pathname) + ) { + throw new Error( + "Delivery cron tests require a session-scoped local database.", + ); + } + process.env.CRON_SECRET = CRON_SECRET; + pool = createPool({ uri: databaseUrl, connectionLimit: 8 }); + fixture.database = drizzle(pool); + await pool.query(ROLLUP_TABLE); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + afterAll(async () => { + if (fixtureApps.length > 0) { + await pool?.query( + "delete from developer_delivery_daily where appId in (?)", + [fixtureApps], + ); + } + if (fixtureApps.length > 0) { + await database() + .update(developerApps) + .set({ deletedAt: new Date() }) + .where(inArray(developerApps.id, fixtureApps)); + } + await pool?.end(); + }); + + it("charges nothing with the start day unset", async () => { + vi.stubEnv("CAP_INFRA_DELIVERY_BILLING_FROM", ""); + const { accountId } = await makeApp(); + + const body = await runCron(); + + expect(body.delivery).toMatchObject({ enabled: false, billed: 0 }); + expect(await deliveryDays(accountId)).toEqual([]); + }); + + it("charges final days on and after the start day only, never the days before it", async () => { + const from = dayKey(2); + vi.stubEnv("CAP_INFRA_DELIVERY_BILLING_FROM", from); + const { accountId } = await makeApp(); + + const first = await runCron(); + const second = await runCron(); + + expect(first.delivery).toMatchObject({ enabled: true, tableReady: true }); + expect(second.delivery).toMatchObject({ billed: 0 }); + expect(await deliveryDays(accountId)).toEqual([from]); + }); + + it("charges nothing while every final day is before the start day", async () => { + vi.stubEnv("CAP_INFRA_DELIVERY_BILLING_FROM", dayKey(1)); + const { accountId } = await makeApp(); + + await runCron(); + + expect(await deliveryDays(accountId)).toEqual([]); + }); + }, +); diff --git a/apps/web/__tests__/integration/infra-delivery-database.test.ts b/apps/web/__tests__/integration/infra-delivery-database.test.ts new file mode 100644 index 00000000000..663e58d9af3 --- /dev/null +++ b/apps/web/__tests__/integration/infra-delivery-database.test.ts @@ -0,0 +1,225 @@ +import { drizzle } from "drizzle-orm/mysql2"; +import { createPool, type Pool } from "mysql2/promise"; +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +const fixture = vi.hoisted(() => ({ + database: undefined as unknown, + appId: "", +})); + +vi.mock("@cap/database", () => ({ + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@/app/api/utils", () => ({ + withDeveloperSecretAuth: async ( + c: { set: (key: string, value: unknown) => void }, + next: () => Promise, + ) => { + c.set("developerAppId", fixture.appId); + c.set("developerKeyType", "secret"); + await next(); + }, +})); +vi.mock("@/lib/infra-standing", () => ({ + getInfraBilling: async () => ({ + stored: { videoCount: 0, totalMinutes: 0 }, + storage: { uploadingVideos: 0 }, + }), + infraBillingJson: () => ({ + standing: { state: "active", pausesAt: null }, + upload: { allowed: true }, + }), + infraStorageJson: () => ({ dailyMicroCredits: 0 }), +})); +vi.mock("@/lib/developer-transcript-usage", () => ({ + getSpeechUsage: async () => ({ + balanceMicroCredits: 100_000, + availableMicroCredits: 100_000, + heldMicroCredits: 0, + speech: { + periodStart: "2026-11-01T00:00:00.000Z", + seconds: 0, + microCredits: 0, + transcripts: 0, + }, + }), +})); + +const { app } = await import("@/app/api/developer/v1/[...route]/usage"); +const { deliveryJson, getDeliveredUsage } = await import( + "@/lib/infra-delivery" +); +const { DeliveredMinutes, NO_DELIVERY } = await import( + "@/app/(infra)/dashboard/infra/_components/delivered-minutes" +); +const { formatMinutes } = await import("@/lib/infra-usage"); + +const databaseUrl = process.env.CAP_INFRA_TEST_DATABASE_URL; + +const CREATE_TABLE = `CREATE TABLE \`developer_delivery_daily\` ( + \`appId\` varchar(15) NOT NULL, + \`day\` date NOT NULL, + \`tier\` varchar(8) NOT NULL, + \`seconds\` bigint NOT NULL DEFAULT 0, + \`bytes\` bigint NOT NULL DEFAULT 0, + \`requests\` int NOT NULL DEFAULT 0, + \`abandonedBytes\` bigint NOT NULL DEFAULT 0, + \`final\` boolean NOT NULL DEFAULT false, + \`finalizedAt\` timestamp, + \`updatedAt\` timestamp NOT NULL, + CONSTRAINT \`developer_delivery_daily_appId_day_tier_pk\` PRIMARY KEY(\`appId\`,\`day\`,\`tier\`) +)`; + +const CREATE_DAY_INDEX = + "CREATE INDEX `day_idx` ON `developer_delivery_daily` (`day`)"; + +const CREATE_TABLE_WITHOUT_FINAL = CREATE_TABLE.replace( + /\n\t`final`[^\n]*\n\t`finalizedAt`[^\n]*/, + "", +); + +const usageDelivery = async () => { + const response = await app.request("/"); + expect(response.status).toBe(200); + const { data } = (await response.json()) as { + data: { delivery: unknown }; + }; + return data.delivery; +}; + +const consoleRows = (delivery: ReturnType) => { + const html = renderToStaticMarkup( + createElement(DeliveredMinutes, { delivery, monthName: "November" }), + ); + return [ + ...html.matchAll( + /]*>]*>([^<]+)<\/td>]*>([^<]+)<\/td><\/tr>/g, + ), + ].map((match) => [match[1], match[2]]); +}; + +describe.runIf(Boolean(databaseUrl))( + "delivered minutes against a real database", + () => { + let pool: Pool; + + beforeAll(async () => { + if (!databaseUrl) throw new Error("Missing isolated test database."); + const url = new URL(databaseUrl); + if ( + url.protocol !== "mysql:" || + !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) || + !/^\/cap_infra_[a-z0-9_]+$/.test(url.pathname) + ) + throw new Error("These tests require a session-scoped local database."); + pool = createPool({ uri: databaseUrl, connectionLimit: 4 }); + fixture.database = drizzle(pool); + fixture.appId = "deliveryapp0001"; + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date("2026-11-20T12:00:00Z")); + await pool.query("DROP TABLE IF EXISTS developer_delivery_daily"); + }); + + afterAll(async () => { + vi.useRealTimers(); + await pool?.query("DROP TABLE IF EXISTS developer_delivery_daily"); + await pool?.end(); + }); + + it("reports no delivery while the table does not exist", async () => { + await expect(getDeliveredUsage(fixture.appId)).resolves.toBeNull(); + await expect(usageDelivery()).resolves.toBeNull(); + const html = renderToStaticMarkup( + createElement(DeliveredMinutes, { + delivery: null, + monthName: "November", + }), + ); + expect(html).toContain(NO_DELIVERY); + }); + + it("reports no delivery while the table has no final column yet", async () => { + await pool.query(CREATE_TABLE_WITHOUT_FINAL); + await pool.query( + "INSERT INTO developer_delivery_daily (appId, day, tier, seconds, bytes, requests, updatedAt) VALUES (?, '2026-11-02', '1080p', 600, 1, 1, NOW())", + [fixture.appId], + ); + await expect(getDeliveredUsage(fixture.appId)).resolves.toBeNull(); + await expect(usageDelivery()).resolves.toBeNull(); + await pool.query("DROP TABLE developer_delivery_daily"); + }); + + it("gives the same minutes in the console and in GET /usage", async () => { + await pool.query(CREATE_TABLE); + await pool.query(CREATE_DAY_INDEX); + await pool.query( + `INSERT INTO developer_delivery_daily (appId, day, tier, seconds, bytes, requests, abandonedBytes, final, finalizedAt, updatedAt) VALUES + (?, '2026-11-01', '1080p', 36000, 900000000, 1200, 50000000, true, '2026-11-03 02:05:00', '2026-11-03 02:05:00'), + (?, '2026-11-01', '720p', 7200, 120000000, 300, 0, true, '2026-11-03 02:05:00', '2026-11-03 02:05:00'), + (?, '2026-11-01', 'audio', 0, 5000000, 90, 0, true, '2026-11-03 02:05:00', '2026-11-03 02:05:00'), + (?, '2026-11-15', '1080p', 18000, 450000000, 600, 0, true, '2026-11-17 02:05:00', '2026-11-17 02:05:00'), + (?, '2026-11-19', '360p', 750, 4000000, 40, 0, false, NULL, '2026-11-20 06:00:00'), + (?, '2026-11-19', '720p', 12960, 210000000, 500, 9000000, false, NULL, '2026-11-20 06:00:00'), + (?, '2026-11-19', 'audio', 0, 3000000, 60, 0, false, NULL, '2026-11-20 06:00:00'), + (?, '2026-10-31', '1080p', 99999, 1, 1, 0, true, '2026-11-02 02:05:00', '2026-11-02 02:05:00'), + (?, '2026-12-01', '1080p', 99999, 1, 1, 0, false, NULL, '2026-12-01 06:00:00'), + ('otherapp0000001', '2026-11-18', '1080p', 99999, 1, 1, 0, true, '2026-11-20 02:05:00', '2026-11-20 02:05:00')`, + Array(9).fill(fixture.appId), + ); + + const usage = await getDeliveredUsage(fixture.appId); + expect(usage).toEqual({ + seconds: 74_910, + secondsByTier: { + "360p": 750, + "720p": 20_160, + "1080p": 54_000, + "1440p": 0, + "2160p": 0, + }, + finalThrough: "2026-11-15", + }); + + const api = await usageDelivery(); + expect(api).toEqual({ + minutes: 1248.5, + minutesByTier: { + "360p": 12.5, + "720p": 336, + "1080p": 900, + "1440p": 0, + "2160p": 0, + }, + finalThrough: "2026-11-15", + }); + expect( + Object.keys((api as { minutesByTier: object }).minutesByTier), + ).not.toContain("audio"); + + const shown = consoleRows(deliveryJson(usage)); + const fromApi = api as { + minutes: number; + minutesByTier: Record; + }; + expect(shown).toEqual([ + ["360p", formatMinutes(fromApi.minutesByTier["360p"] ?? 0)], + ["720p", formatMinutes(fromApi.minutesByTier["720p"] ?? 0)], + ["1080p", formatMinutes(fromApi.minutesByTier["1080p"] ?? 0)], + ["Total", formatMinutes(fromApi.minutes)], + ]); + }); + + it("reads on the primary key", async () => { + const [plan] = (await pool.query( + "EXPLAIN SELECT tier, sum(seconds) FROM developer_delivery_daily WHERE appId = ? AND day >= ? AND day < ? AND tier <> 'audio' GROUP BY tier", + [fixture.appId, "2026-11-01", "2026-12-01"], + )) as unknown as [{ key: string | null }[]]; + expect(plan[0]?.key).toBe("PRIMARY"); + }); + }, +); diff --git a/apps/web/__tests__/integration/infra-first-month-auto-top-up-database.test.ts b/apps/web/__tests__/integration/infra-first-month-auto-top-up-database.test.ts new file mode 100644 index 00000000000..2d4e1a04d0c --- /dev/null +++ b/apps/web/__tests__/integration/infra-first-month-auto-top-up-database.test.ts @@ -0,0 +1,231 @@ +import { randomUUID } from "node:crypto"; +import type { DeveloperVideoProcessingState } from "@cap/database/types"; +import { drizzle, type MySql2Database } from "drizzle-orm/mysql2"; +import { createPool, type Pool } from "mysql2/promise"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +const fixture = vi.hoisted(() => ({ + database: undefined as unknown, + created: [] as { id: string; metadata: Record }[], +})); + +vi.mock("@cap/database", () => ({ + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "", NEXT_PUBLIC_WEB_URL: "https://web" }, + serverEnv: () => ({ WEB_URL: "https://web" }), +})); +vi.mock("@/lib/developer-credits-product", () => ({ + developerCreditsProductId: () => "prod_test", +})); +vi.mock("@/lib/infra-billing-emails", () => ({ + sendInfraTopUpFailedEmail: vi.fn(), + sendInfraOutOfCreditEmail: vi.fn(async () => true), +})); +vi.mock("@cap/utils", () => ({ + stripe: () => ({ + paymentIntents: { + search: async () => ({ data: [] }), + retrieve: async (intentId: string) => { + const intent = fixture.created.find((entry) => entry.id === intentId); + if (!intent) throw new Error(`No such payment_intent: ${intentId}`); + return intent; + }, + create: async (params: { + amount: number; + metadata: Record; + }) => { + const intent = { + id: `pi_${randomUUID().slice(0, 8)}`, + status: "succeeded", + amount: params.amount, + amount_received: params.amount, + currency: "usd", + metadata: params.metadata, + }; + fixture.created.push(intent); + return intent; + }, + }, + }), +})); + +const databaseUrl = process.env.CAP_INFRA_TEST_DATABASE_URL; +const id = () => randomUUID().replaceAll("-", "").slice(0, 15); + +const processing = (): DeveloperVideoProcessingState => + ({ + startedAt: Date.now(), + encodeId: "encode", + hls: { state: "complete", output: null }, + mp4: { state: "complete", output: null }, + attempts: { hls: 1, mp4: 1 }, + unavailableChecks: 0, + poster: null, + bytes: { source: 1, hls: 1, mp4: 1 }, + }) as unknown as DeveloperVideoProcessingState; + +describe.runIf(Boolean(databaseUrl))( + "auto top-up after a first month charge against a real database", + () => { + let pool: Pool; + + const makeAccount = async (balanceMicroCredits: number) => { + const ownerId = id(); + const appId = id(); + const accountId = id(); + await pool.query( + "INSERT INTO users (id, email, activeOrganizationId) VALUES (?, ?, '')", + [ownerId, `${ownerId}@example.com`], + ); + await pool.query( + "INSERT INTO developer_apps (id, ownerId, name, environment) VALUES (?, ?, 'Top-up after first month', 'production')", + [appId, ownerId], + ); + await pool.query( + `INSERT INTO developer_credit_accounts + (id, appId, ownerId, balanceMicroCredits, stripeCustomerId, stripePaymentMethodId, + autoTopUpEnabled, autoTopUpThresholdMicroCredits, autoTopUpAmountCents) + VALUES (?, ?, ?, ?, 'cus_test', 'pm_test', 1, 500000, 500)`, + [accountId, appId, ownerId, balanceMicroCredits], + ); + return { appId, accountId }; + }; + + const makeVideo = async (appId: string, durationSeconds: number) => { + const videoId = id(); + const { developerVideos } = await import("@cap/database/schema"); + await (fixture.database as MySql2Database) + .insert(developerVideos) + .values({ + id: videoId, + appId, + name: "Fixture", + status: "ready", + duration: durationSeconds, + width: 3840, + height: 2160, + maxResolutionTier: "2160p", + processing: processing(), + createdAt: new Date("2026-11-12T00:00:00Z"), + updatedAt: new Date("2026-11-12T00:05:00Z"), + }); + return videoId; + }; + + const balanceOf = async (accountId: string) => { + const [rows] = await pool.query( + "SELECT balanceMicroCredits AS balance FROM developer_credit_accounts WHERE id = ?", + [accountId], + ); + return Number((rows as { balance: number }[])[0]?.balance); + }; + + const topUpsOf = async (accountId: string) => { + const [rows] = await pool.query( + "SELECT id FROM developer_credit_transactions WHERE accountId = ? AND referenceType = 'auto_top_up' AND type = 'topup'", + [accountId], + ); + return (rows as { id: string }[]).length; + }; + + const intentsFor = (accountId: string) => + fixture.created.filter( + (intent) => intent.metadata.accountId === accountId, + ).length; + + beforeAll(async () => { + if (!databaseUrl) throw new Error("Missing isolated test database URL."); + const url = new URL(databaseUrl); + if ( + url.protocol !== "mysql:" || + !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) || + !/^\/cap_(infra|speech)_[a-z0-9_]+$/.test(url.pathname) + ) { + throw new Error("These tests require a session-scoped local database."); + } + pool = createPool({ uri: databaseUrl, connectionLimit: 12 }); + fixture.database = drizzle(pool, { mode: "default" }) as MySql2Database; + }); + + afterAll(async () => { + await pool?.end(); + }); + + it("schedules exactly one top-up for a charged month that crosses the threshold, and none for a repeat or a month above it", async () => { + const { chargeDeveloperVideoFirstMonth } = await import( + "@/lib/infra-storage-minimum" + ); + const { appId, accountId } = await makeAccount(510_000); + const videoId = await makeVideo(appId, 14_400); + + const charged = await chargeDeveloperVideoFirstMonth(videoId); + expect(charged).toMatchObject({ + state: "charged", + chargedMicroCredits: 67_200, + }); + expect(intentsFor(accountId)).toBe(1); + expect(await topUpsOf(accountId)).toBe(1); + expect(await balanceOf(accountId)).toBe(942_800); + + expect(await chargeDeveloperVideoFirstMonth(videoId)).toMatchObject({ + state: "skipped", + }); + const above = await makeVideo(appId, 600); + expect(await chargeDeveloperVideoFirstMonth(above)).toMatchObject({ + state: "charged", + chargedMicroCredits: 2_800, + }); + expect(intentsFor(accountId)).toBe(1); + expect(await topUpsOf(accountId)).toBe(1); + }); + + it("does not wait on the top-up inside a caller's transaction that holds the account lock, and tops up once it commits", async () => { + const { chargeDeveloperVideoFirstMonth } = await import( + "@/lib/infra-storage-minimum" + ); + const { lockInfraAccount } = await import("@/lib/infra-balance"); + const { appId, accountId } = await makeAccount(510_000); + const videoId = await makeVideo(appId, 14_400); + const database = fixture.database as MySql2Database; + + const started = Date.now(); + await database.transaction(async (tx) => { + await lockInfraAccount(tx, appId); + expect( + await chargeDeveloperVideoFirstMonth(videoId, { executor: tx }), + ).toMatchObject({ state: "charged", chargedMicroCredits: 67_200 }); + expect(intentsFor(accountId)).toBe(0); + }); + expect(Date.now() - started).toBeLessThan(5_000); + + for (let i = 0; i < 50 && (await topUpsOf(accountId)) === 0; i++) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + expect(intentsFor(accountId)).toBe(1); + expect(await topUpsOf(accountId)).toBe(1); + }); + + it("charges a month with no top-up when the balance pays nothing", async () => { + const { chargeDeveloperVideoFirstMonth } = await import( + "@/lib/infra-storage-minimum" + ); + const { appId, accountId } = await makeAccount(0); + await pool.query( + "UPDATE developer_credit_accounts SET autoTopUpEnabled = 1 WHERE id = ?", + [accountId], + ); + const videoId = await makeVideo(appId, 600); + + expect(await chargeDeveloperVideoFirstMonth(videoId)).toMatchObject({ + state: "charged", + chargedMicroCredits: 0, + }); + expect(intentsFor(accountId)).toBe(0); + }); + }, +); diff --git a/apps/web/__tests__/integration/infra-first-month-database.test.ts b/apps/web/__tests__/integration/infra-first-month-database.test.ts new file mode 100644 index 00000000000..a50cc6df4e5 --- /dev/null +++ b/apps/web/__tests__/integration/infra-first-month-database.test.ts @@ -0,0 +1,2798 @@ +import { randomUUID } from "node:crypto"; +import { + developerApps, + developerCreditAccounts, + developerCreditTransactions, + developerTranscripts, + developerVideos, +} from "@cap/database/schema"; +import type { DeveloperVideoProcessingState } from "@cap/database/types"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { drizzle, type MySql2Database } from "drizzle-orm/mysql2"; +import { Effect } from "effect"; +import { createPool, type Pool } from "mysql2/promise"; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +process.env.CAP_INFRA_DELIVERY_BILLING_FROM = "2000-01-01"; + +const fixture = vi.hoisted(() => ({ database: undefined as unknown })); + +vi.mock("@cap/database", () => ({ + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true", NEXT_PUBLIC_WEB_URL: "https://web" }, + serverEnv: () => ({ WEB_URL: "https://web" }), +})); +vi.mock("@cap/web-backend", () => ({ + provideOptionalAuth: (effect: T) => effect, + S3Buckets: {}, +})); +vi.mock("@/lib/server", () => ({ + runPromise: (effect: Effect.Effect) => + Effect.runPromise(effect), +})); +vi.mock("workflow/api", () => ({ start: vi.fn() })); +vi.mock("@cap/database/auth/session", () => ({ getCurrentUser: vi.fn() })); +vi.mock("@cap/utils", () => ({ userIsPro: () => false })); +vi.mock("next/headers", () => ({ + cookies: async () => ({ get: () => null }), + headers: async () => ({ get: () => null }), +})); +vi.mock("next/navigation", () => ({ redirect: vi.fn(), notFound: vi.fn() })); +vi.mock("next/cache", () => ({ revalidatePath: vi.fn() })); +vi.mock("@/lib/infra-access", () => ({ canUseInfra: () => true })); +vi.mock("@/workflows/process-developer-video", () => ({ + processDeveloperVideoWorkflow: "workflow", +})); + +const { addCreditsToAccount } = await import("@/lib/developer-credits"); +const { emptyStorageMinutes, firstMonthStorageMicroCredits } = await import( + "@/lib/developer-pricing" +); +const { getInfraBilling, getInfraUploadAdmission } = await import( + "@/lib/infra-standing" +); +const { getAppStorage, getStorageUsageByApp } = await import( + "@/lib/infra-storage" +); +const { chargeStorageDay } = await import("@/lib/infra-storage-billing"); +const { restoredSinceCharge } = await import("@/lib/infra-standing"); +const { chargeDeliveryDay } = await import("@/lib/infra-delivery-billing"); +const { deliveryMicroCredits } = await import("@/lib/infra-delivery-rollup"); +const { clearFirstMonthChargesBeganCache, firstMonthChargesBegan } = + await import("@/lib/infra-first-month-start"); +const { + chargeDeveloperVideoFirstMonth, + chargeMissedFirstMonths, + settleFirstMonthAtDelete, + settleFirstMonthAtReady, + shortfallsAwaitingNotice, + markShortfallsNotified, + markStorageDayNotified, + openFirstMonthShortfalls, +} = await import("@/lib/infra-storage-minimum"); +const { recordAdmittedInfraUpload } = await import("@/lib/infra-video-upload"); +const { + beginPart, + createDeveloperVideoProcessingState, + markPartDispatched, + releasePart, + settlePart, +} = await import("@/lib/developer-video-state"); +const { getInfraFirstMonthChargesSince, getInfraStorageChargesSince } = + await import("@/app/(infra)/dashboard/infra/infra-data"); +const { getCurrentUser } = await import("@cap/database/auth/session"); +const { deleteDeveloperApp } = await import("@/actions/developers/delete-app"); + +const databaseUrl = process.env.CAP_INFRA_TEST_DATABASE_URL; +const DAY_MS = 24 * 60 * 60 * 1000; +const id = () => randomUUID().replaceAll("-", "").slice(0, 15); +const database = () => fixture.database as MySql2Database; +const day = (key: string, time = "00:41:00") => new Date(`${key}T${time}.000Z`); + +const processing = (): DeveloperVideoProcessingState => ({ + encodeId: "encode-1", + startedAt: Date.parse("2026-11-12T00:00:00Z"), + probe: { duration: 600, width: 1920, height: 1080, fps: 30 }, + hls: { state: "pending" }, + mp4: { state: "complete", output: "out" }, + attempts: { hls: 0, mp4: 1 }, + unavailableChecks: 0, + poster: null, + bytes: { source: 1, hls: 0, mp4: 1, poster: 0 }, +}); + +describe.runIf(Boolean(databaseUrl))( + "first month of storage against a real database", + () => { + let pool: Pool; + + const makeApp = async (balanceMicroCredits: number) => { + const appId = id(); + const accountId = id(); + const ownerId = id() as never; + await database().insert(developerApps).values({ + id: appId, + ownerId, + name: "First month fixture", + environment: "development", + }); + await database().insert(developerCreditAccounts).values({ + id: accountId, + appId, + ownerId, + balanceMicroCredits, + }); + return { appId, accountId }; + }; + + const makeVideo = async ( + appId: string, + overrides: Partial = {}, + ) => { + const videoId = id(); + await database() + .insert(developerVideos) + .values({ + id: videoId, + appId, + name: "Fixture", + status: "ready", + duration: 600, + width: 1920, + height: 1080, + maxResolutionTier: "1080p", + processing: processing(), + createdAt: new Date("2026-11-12T00:00:00Z"), + updatedAt: new Date("2026-11-12T00:05:00Z"), + ...overrides, + }); + return videoId; + }; + + const balanceOf = async (accountId: string) => { + const [row] = await database() + .select({ balance: developerCreditAccounts.balanceMicroCredits }) + .from(developerCreditAccounts) + .where(eq(developerCreditAccounts.id, accountId)); + return row?.balance; + }; + + const firstMonthRows = (accountId: string) => + database() + .select() + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + eq(developerCreditTransactions.type, "video_create"), + ), + ); + + const videoRow = async (videoId: string) => { + const [row] = await database() + .select() + .from(developerVideos) + .where(eq(developerVideos.id, videoId)); + return row; + }; + + const billDay = (accountId: string, dateKey: string) => + database() + .insert(developerCreditTransactions) + .values({ + id: id(), + accountId, + type: "storage_daily", + amountMicroCredits: 0, + balanceAfterMicroCredits: 0, + referenceId: dateKey, + referenceType: "storage_day", + metadata: { snapshotDate: dateKey, owedMicroCredits: 0 }, + }); + + const dailyMinutes = async (appId: string, at: Date) => + ( + await getStorageUsageByApp([appId], undefined, undefined, { + dailyChargeAt: at, + }) + ).get(appId)?.totalMinutes ?? 0; + + beforeEach(() => { + clearFirstMonthChargesBeganCache(); + }); + + beforeAll(async () => { + if (!databaseUrl) throw new Error("Missing isolated test database URL."); + const url = new URL(databaseUrl); + if ( + url.protocol !== "mysql:" || + !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) || + !/^\/cap_(infra|speech)_[a-z0-9_]+$/.test(url.pathname) + ) { + throw new Error( + "First month tests require a session-scoped local database.", + ); + } + pool = createPool({ uri: databaseUrl, connectionLimit: 12 }); + fixture.database = drizzle(pool); + await database().select().from(developerVideos).limit(1); + }); + + afterAll(async () => { + await pool?.end(); + }); + + it("debits one month at the video's tier and measured length, once, and marks it paid", async () => { + const cases = [ + { seconds: 600, width: 1920, height: 1080, tier: "1080p", owed: 850 }, + { + seconds: 3600, + width: 3840, + height: 2160, + tier: "2160p", + owed: 16_800, + }, + { + seconds: 3600, + width: 3840, + height: 2160, + tier: "1080p", + owed: 5_100, + }, + { seconds: 600, width: 2560, height: 1440, tier: "1440p", owed: 1_450 }, + { seconds: 1, width: 640, height: 360, tier: "1080p", owed: 1 }, + ] as const; + for (const video of cases) { + const { appId, accountId } = await makeApp(1_000_000); + const videoId = await makeVideo(appId, { + duration: video.seconds, + width: video.width, + height: video.height, + maxResolutionTier: video.tier, + }); + const before = await videoRow(videoId); + + const result = await chargeDeveloperVideoFirstMonth(videoId, { + now: new Date("2026-11-12T10:00:00Z"), + }); + + expect(result).toMatchObject({ + state: "charged", + owedMicroCredits: video.owed, + chargedMicroCredits: video.owed, + shortSince: null, + }); + expect(video.owed).toBe( + firstMonthStorageMicroCredits({ + durationSeconds: video.seconds, + tier: result.state === "charged" ? result.tier : "1080p", + }), + ); + expect(await balanceOf(accountId)).toBe(1_000_000 - video.owed); + const rows = await firstMonthRows(accountId); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + amountMicroCredits: -video.owed, + balanceAfterMicroCredits: 1_000_000 - video.owed, + referenceId: videoId, + referenceType: "developer_video", + }); + const after = await videoRow(videoId); + expect(after?.storagePaidUntil).toEqual( + new Date("2026-12-12T00:00:00Z"), + ); + expect(after?.updatedAt).toEqual(before?.updatedAt); + expect(after?.status).toBe("ready"); + + expect( + await chargeDeveloperVideoFirstMonth(videoId, { + now: new Date("2026-11-12T11:00:00Z"), + }), + ).toEqual({ state: "skipped", reason: "already_settled" }); + expect(await firstMonthRows(accountId)).toHaveLength(1); + expect(await balanceOf(accountId)).toBe(1_000_000 - video.owed); + } + }); + + it("charges the month on the duration stored at first ready, not on what the client first claimed or the probe state", async () => { + const { appId, accountId } = await makeApp(1_000_000); + const videoId = await makeVideo(appId, { + status: "processing", + duration: 4 * 60 * 60, + processing: { + ...processing(), + probe: { duration: 4 * 60 * 60, width: 1920, height: 1080, fps: 30 }, + }, + }); + await database() + .update(developerVideos) + .set({ duration: 60 }) + .where(eq(developerVideos.id, videoId)); + await database() + .update(developerVideos) + .set({ status: "ready" }) + .where(eq(developerVideos.id, videoId)); + + await settleFirstMonthAtReady({ id: videoId, deletedAt: null }); + + const [row] = await firstMonthRows(accountId); + expect(row).toMatchObject({ amountMicroCredits: -85 }); + expect(row?.metadata).toMatchObject({ + owedMicroCredits: 85, + durationSeconds: 60, + }); + expect(await balanceOf(accountId)).toBe(1_000_000 - 85); + }); + + it("charges once when the ready transition and the backstop race for the same video", async () => { + const { appId, accountId } = await makeApp(1_000_000); + const videoId = await makeVideo(appId); + + const results = await Promise.all( + Array.from({ length: 8 }, () => + chargeDeveloperVideoFirstMonth(videoId, { + now: new Date("2026-11-12T10:00:00Z"), + }), + ), + ); + + expect(results.filter((r) => r.state === "charged")).toHaveLength(1); + expect( + results.filter( + (r) => r.state === "skipped" && r.reason === "already_settled", + ), + ).toHaveLength(7); + expect(await firstMonthRows(accountId)).toHaveLength(1); + expect(await balanceOf(accountId)).toBe(1_000_000 - 850); + }); + + it("leaves the video out of the daily charge for exactly 30 billed days, then charges it daily with no gap", async () => { + const keys = (from: string, count: number) => + Array.from({ length: count }, (_, n) => + new Date(Date.parse(`${from}T00:00:00Z`) + n * DAY_MS) + .toISOString() + .slice(0, 10), + ); + + const afterTodaysCharge = await makeApp(1_000_000); + await billDay(afterTodaysCharge.accountId, "2026-11-12"); + const lateVideo = await makeVideo(afterTodaysCharge.appId); + await chargeDeveloperVideoFirstMonth(lateVideo, { + now: new Date("2026-11-12T10:00:00Z"), + }); + const late = await Promise.all( + keys("2026-11-13", 31).map((key) => + dailyMinutes(afterTodaysCharge.appId, day(key)), + ), + ); + expect(late.slice(0, 30)).toEqual(Array(30).fill(0)); + expect(late[30]).toBe(10); + expect((await videoRow(lateVideo))?.storagePaidUntil).toEqual( + new Date("2026-12-13T00:00:00Z"), + ); + + const beforeTodaysCharge = await makeApp(1_000_000); + const earlyVideo = await makeVideo(beforeTodaysCharge.appId); + await chargeDeveloperVideoFirstMonth(earlyVideo, { + now: new Date("2026-11-12T00:10:00Z"), + }); + const early = await Promise.all( + keys("2026-11-12", 31).map((key) => + dailyMinutes(beforeTodaysCharge.appId, day(key, "06:41:00")), + ), + ); + expect(early.slice(0, 30)).toEqual(Array(30).fill(0)); + expect(early[30]).toBe(10); + + const charge = async (key: string) => + chargeStorageDay({ + appId: beforeTodaysCharge.appId, + accountId: beforeTodaysCharge.accountId, + usage: + ( + await getStorageUsageByApp( + [beforeTodaysCharge.appId], + undefined, + undefined, + { dailyChargeAt: day(key) }, + ) + ).get(beforeTodaysCharge.appId) ?? + (() => { + throw new Error("no usage"); + })(), + dateKey: key, + now: day(key), + }); + expect(await charge("2026-12-11")).toEqual({ state: "skipped" }); + expect(await charge("2026-12-12")).toMatchObject({ + state: "billed", + chargedMicroCredits: Math.floor(850 / 30), + }); + }); + + it("does not charge a day the first month covers when the video turned ready after the day was measured", async () => { + const { appId, accountId } = await makeApp(1_000_000); + const videoId = await makeVideo(appId); + const at = new Date(); + const measure = async (executor?: never) => + ( + await getStorageUsageByApp([appId], undefined, executor, { + dailyChargeAt: at, + }) + ).get(appId) ?? + (() => { + throw new Error("no usage"); + })(); + const stale = await measure(); + expect(stale.totalMinutes).toBe(10); + + await chargeDeveloperVideoFirstMonth(videoId, { now: at }); + const result = await chargeStorageDay({ + appId, + accountId, + usage: stale, + measure: (tx) => measure(tx as never), + dateKey: at.toISOString().slice(0, 10), + now: at, + }); + + expect(result).toEqual({ state: "skipped" }); + expect(await balanceOf(accountId)).toBe(1_000_000 - 850); + }); + + it("keeps the month paid for a video deleted an hour after it became ready", async () => { + const { appId, accountId } = await makeApp(1_000_000); + const videoId = await makeVideo(appId); + await settleFirstMonthAtReady({ id: videoId, deletedAt: null }); + await database() + .update(developerVideos) + .set({ deletedAt: new Date(Date.now() + 60 * 60 * 1000) }) + .where(eq(developerVideos.id, videoId)); + + expect(await balanceOf(accountId)).toBe(1_000_000 - 850); + expect(await firstMonthRows(accountId)).toHaveLength(1); + expect(await chargeMissedFirstMonths(new Date())).toMatchObject({ + failed: 0, + }); + expect(await firstMonthRows(accountId)).toHaveLength(1); + expect(await balanceOf(accountId)).toBe(1_000_000 - 850); + }); + + it("charges nothing for a video that errored or was deleted before it became ready", async () => { + const { appId, accountId } = await makeApp(1_000_000); + const errored = await makeVideo(appId, { status: "errored" }); + const encoding = await makeVideo(appId, { status: "processing" }); + const deletedFirst = await makeVideo(appId, { + deletedAt: new Date("2026-11-12T09:00:00Z"), + }); + + expect(await chargeDeveloperVideoFirstMonth(errored)).toEqual({ + state: "skipped", + reason: "not_ready", + }); + expect(await chargeDeveloperVideoFirstMonth(encoding)).toEqual({ + state: "skipped", + reason: "not_ready", + }); + await settleFirstMonthAtReady({ + id: deletedFirst, + deletedAt: new Date("2026-11-12T09:00:00Z"), + }); + + const exempt = await videoRow(deletedFirst); + expect(exempt?.storagePaidUntil).not.toBeNull(); + expect(await firstMonthRows(accountId)).toHaveLength(0); + expect(await balanceOf(accountId)).toBe(1_000_000); + + await chargeMissedFirstMonths(new Date()); + expect(await firstMonthRows(accountId)).toHaveLength(0); + expect(await balanceOf(accountId)).toBe(1_000_000); + }); + + it("never charges the minimum for a video that was ready before the feature, or one uploaded before encoding existed", async () => { + await makeVideo((await makeApp(1_000_000)).appId, { + createdAt: new Date("2026-11-12T00:00:00Z"), + storagePaidUntil: new Date("2026-12-12T00:00:00Z"), + }); + const { appId, accountId } = await makeApp(1_000_000); + const old = await makeVideo(appId, { + createdAt: new Date("2026-10-01T00:00:00Z"), + }); + const legacy = await makeVideo(appId, { + status: "uploading", + processing: null, + playbackId: null, + createdAt: new Date("2026-11-20T00:00:00Z"), + }); + + await chargeMissedFirstMonths(new Date("2026-11-21T00:41:00Z")); + await chargeMissedFirstMonths(new Date("2026-11-21T06:41:00Z")); + + expect(await firstMonthRows(accountId)).toHaveLength(0); + expect(await balanceOf(accountId)).toBe(1_000_000); + expect((await videoRow(old))?.storagePaidUntil).toBeNull(); + expect((await videoRow(legacy))?.storagePaidUntil).toBeNull(); + expect(await dailyMinutes(appId, day("2026-11-21"))).toBe(10); + }); + + it("lets the cron charge a live ready video the transition missed exactly once, and exempt a deleted one", async () => { + await makeVideo((await makeApp(1_000_000)).appId, { + storagePaidUntil: new Date("2026-12-12T00:00:00Z"), + }); + const { appId, accountId } = await makeApp(1_000_000); + const missed = await makeVideo(appId, { + createdAt: new Date("2026-11-20T00:00:00Z"), + }); + const missedThenDeleted = await makeVideo(appId, { + createdAt: new Date("2026-11-20T01:00:00Z"), + deletedAt: new Date("2026-11-20T02:00:00Z"), + }); + + const noEncodeState = await makeVideo(appId, { + processing: null, + createdAt: new Date("2026-11-19T00:00:00Z"), + }); + const noAccount = id(); + await database() + .insert(developerApps) + .values({ + id: noAccount, + ownerId: id() as never, + name: "No credit account", + environment: "development", + }); + const unbillable = await makeVideo(noAccount, { + createdAt: new Date("2026-11-19T00:00:00Z"), + }); + + await chargeMissedFirstMonths(new Date("2026-11-20T06:41:00Z"), 2); + await chargeMissedFirstMonths(new Date("2026-11-20T12:41:00Z"), 2); + await chargeMissedFirstMonths(new Date("2026-11-20T18:41:00Z"), 2); + + expect((await videoRow(noEncodeState))?.storagePaidUntil).not.toBeNull(); + expect((await videoRow(unbillable))?.storagePaidUntil).not.toBeNull(); + + const rows = await firstMonthRows(accountId); + expect(rows.map((row) => row.referenceId)).toEqual([missed]); + expect(await balanceOf(accountId)).toBe(1_000_000 - 850); + expect((await videoRow(missed))?.storagePaidUntil).not.toBeNull(); + expect( + (await videoRow(missedThenDeleted))?.storagePaidUntil, + ).not.toBeNull(); + }); + + it("takes what the balance holds when it cannot pay the month, leaves the video ready and the account out of credit until topped up", async () => { + const part = await makeApp(300); + const partVideo = await makeVideo(part.appId); + await settleFirstMonthAtReady({ id: partVideo, deletedAt: null }); + + expect(await balanceOf(part.accountId)).toBe(0); + const [partRow] = await firstMonthRows(part.accountId); + expect(partRow).toMatchObject({ + amountMicroCredits: -300, + balanceAfterMicroCredits: 0, + }); + expect(partRow?.metadata).toMatchObject({ + owedMicroCredits: 850, + chargedMicroCredits: 300, + }); + expect( + (partRow?.metadata as { shortSince: string | null }).shortSince, + ).not.toBeNull(); + const partVideoRow = await videoRow(partVideo); + expect(partVideoRow?.status).toBe("ready"); + expect(partVideoRow?.storagePaidUntil).not.toBeNull(); + + const short = await getInfraBilling(part.appId); + expect(short.standing.state).toBe("grace"); + expect(short.admission).toMatchObject({ + allowed: false, + reason: "out_of_credit", + }); + + const unpaid = await makeApp(0); + await settleFirstMonthAtReady({ + id: await makeVideo(unpaid.appId), + deletedAt: null, + }); + const [unpaidRow] = await firstMonthRows(unpaid.accountId); + expect(unpaidRow).toMatchObject({ + amountMicroCredits: 0, + balanceAfterMicroCredits: 0, + }); + expect((await getInfraBilling(unpaid.appId)).standing.state).toBe( + "grace", + ); + + await addCreditsToAccount({ + accountId: part.accountId, + amountCents: 1, + referenceId: id(), + referenceType: "stripe_payment_intent", + }); + expect(await balanceOf(part.accountId)).toBe(1_000); + expect((await getInfraBilling(part.appId)).standing.state).toBe("active"); + }); + + it("latches a shortfall once when a released hold covers it, so it never comes back", async () => { + const { appId, accountId } = await makeApp(900); + const transcriptId = id(); + await database().insert(developerTranscripts).values({ + id: transcriptId, + appId, + status: "processing", + sourceType: "upload", + reservedMicroCredits: 900, + }); + await settleFirstMonthAtReady({ + id: await makeVideo(appId), + deletedAt: null, + }); + const [month] = await firstMonthRows(accountId); + expect(month).toMatchObject({ amountMicroCredits: 0 }); + expect((await getInfraBilling(appId)).standing.state).toBe("grace"); + + await database() + .update(developerTranscripts) + .set({ status: "complete", reservedMicroCredits: 0 }) + .where(eq(developerTranscripts.id, transcriptId)); + await settleFirstMonthAtReady({ + id: await makeVideo(appId, { duration: 1 }), + deletedAt: null, + }); + const latchedRow = async () => { + const [row] = await database() + .select({ metadata: developerCreditTransactions.metadata }) + .from(developerCreditTransactions) + .where(eq(developerCreditTransactions.id, month?.id ?? "")); + return row?.metadata as { + resolvedAt?: string; + owedMicroCredits?: number; + }; + }; + const first = await latchedRow(); + expect(first.resolvedAt).toEqual(expect.any(String)); + expect(first.owedMicroCredits).toBe(850); + + await settleFirstMonthAtReady({ + id: await makeVideo(appId, { duration: 1 }), + deletedAt: null, + }); + expect((await latchedRow()).resolvedAt).toBe(first.resolvedAt); + expect(await balanceOf(accountId)).toBe(898); + + await database() + .update(developerCreditAccounts) + .set({ balanceMicroCredits: 100 }) + .where(eq(developerCreditAccounts.id, accountId)); + const later = await getInfraBilling( + appId, + undefined, + new Date(Date.now() + 10 * DAY_MS), + ); + expect(later.standing).toEqual({ state: "active", pausesAt: null }); + }); + + it("keeps an unpaid month in grace from its first short date when a later, smaller month is paid in full", async () => { + const { appId, accountId } = await makeApp(900); + const transcriptId = id(); + await database().insert(developerTranscripts).values({ + id: transcriptId, + appId, + status: "processing", + sourceType: "upload", + reservedMicroCredits: 900, + }); + await settleFirstMonthAtReady({ + id: await makeVideo(appId, { + duration: 3_600, + width: 3840, + height: 2160, + maxResolutionTier: "2160p", + }), + deletedAt: null, + }); + const [unpaid] = await firstMonthRows(accountId); + expect(unpaid).toMatchObject({ amountMicroCredits: 0 }); + expect(unpaid?.metadata).toMatchObject({ owedMicroCredits: 16_800 }); + const movedTo = new Date(Math.floor(Date.now() / 1000) * 1000 - 5_000); + await database() + .update(developerCreditTransactions) + .set({ createdAt: movedTo }) + .where(eq(developerCreditTransactions.id, unpaid?.id ?? "")); + const shortSince = new Date( + (unpaid?.metadata as { shortSince: string }).shortSince, + ); + const pausesAt = new Date(shortSince.getTime() + 7 * DAY_MS); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "grace", + pausesAt, + }); + + await database() + .update(developerTranscripts) + .set({ status: "complete", reservedMicroCredits: 0 }) + .where(eq(developerTranscripts.id, transcriptId)); + const paidVideo = await makeVideo(appId); + await settleFirstMonthAtReady({ id: paidVideo, deletedAt: null }); + + const paid = (await firstMonthRows(accountId)).find( + (row) => row.referenceId === paidVideo, + ); + expect(paid).toMatchObject({ + amountMicroCredits: -850, + balanceAfterMicroCredits: 50, + }); + expect(paid?.metadata).toMatchObject({ + owedMicroCredits: 850, + chargedMicroCredits: 850, + outstandingMicroCredits: 16_800, + shortSince: shortSince.toISOString(), + fellShort: false, + }); + expect(paid?.createdAt.getTime()).toBeGreaterThan(movedTo.getTime()); + const billing = await getInfraBilling(appId); + expect(billing.standing).toEqual({ state: "grace", pausesAt }); + expect(billing.admission).toMatchObject({ + allowed: false, + reason: "out_of_credit", + }); + expect(await openFirstMonthShortfalls([accountId])).toEqual( + new Set([accountId]), + ); + expect( + ( + await shortfallsAwaitingNotice( + [accountId], + new Date(Date.now() - DAY_MS), + ) + ).get(accountId), + ).toEqual({ shortSince, ids: [unpaid?.id] }); + + await addCreditsToAccount({ + accountId, + amountCents: 17, + referenceId: id(), + referenceType: "stripe_payment_intent", + }); + expect(await balanceOf(accountId)).toBe(17_050); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + + await database() + .update(developerCreditTransactions) + .set({ createdAt: new Date(movedTo.getTime() + 2_000) }) + .where(eq(developerCreditTransactions.id, paid?.id ?? "")); + const afterVideo = await makeVideo(appId); + await settleFirstMonthAtReady({ id: afterVideo, deletedAt: null }); + const after = (await firstMonthRows(accountId)).find( + (row) => row.referenceId === afterVideo, + ); + expect(after?.metadata).toMatchObject({ shortSince: null }); + expect(after?.metadata).not.toHaveProperty("outstandingMicroCredits"); + expect((await getInfraBilling(appId)).standing.state).toBe("active"); + expect(await openFirstMonthShortfalls([accountId])).toEqual(new Set()); + }); + + it("resolves the daily charge a first month shortfall carried when the shortfall is covered", async () => { + const { appId, accountId } = await makeApp(900); + const transcriptId = id(); + await database().insert(developerTranscripts).values({ + id: transcriptId, + appId, + status: "processing", + sourceType: "upload", + reservedMicroCredits: 900, + }); + const fourK = { + duration: 3_600, + width: 3840, + height: 2160, + maxResolutionTier: "2160p" as const, + }; + await makeVideo(appId, { + ...fourK, + createdAt: new Date("2026-10-01T00:00:00Z"), + }); + const today = new Date().toISOString().slice(0, 10); + const usage = ( + await getStorageUsageByApp([appId], undefined, undefined, { + dailyChargeAt: new Date(`${today}T00:00:00.000Z`), + }) + ).get(appId); + if (!usage) throw new Error("no usage"); + const day = await chargeStorageDay({ + appId, + accountId, + usage, + dateKey: today, + now: new Date(), + }); + expect(day).toMatchObject({ + state: "billed", + chargedMicroCredits: 0, + owedMicroCredits: 560, + }); + const [dailyRow] = await database() + .select() + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + eq(developerCreditTransactions.type, "storage_daily"), + ), + ); + const shortSince = (dailyRow?.metadata as { shortSince: string }) + .shortSince; + expect(shortSince).toEqual(expect.any(String)); + + await settleFirstMonthAtReady({ + id: await makeVideo(appId, fourK), + deletedAt: null, + }); + const [month] = await firstMonthRows(accountId); + expect(month?.metadata).toMatchObject({ + owedMicroCredits: 16_800, + shortSince, + outstandingMicroCredits: 16_800, + }); + + await database() + .update(developerTranscripts) + .set({ status: "complete", reservedMicroCredits: 0 }) + .where(eq(developerTranscripts.id, transcriptId)); + await database() + .update(developerCreditAccounts) + .set({ balanceMicroCredits: 16_800 }) + .where(eq(developerCreditAccounts.id, accountId)); + await settleFirstMonthAtReady({ + id: await makeVideo(appId, fourK), + deletedAt: null, + }); + expect(await balanceOf(accountId)).toBe(0); + + const resolvedAt = async (rowId: string) => { + const [row] = await database() + .select({ metadata: developerCreditTransactions.metadata }) + .from(developerCreditTransactions) + .where(eq(developerCreditTransactions.id, rowId)); + return (row?.metadata as { resolvedAt?: string }).resolvedAt; + }; + expect(await resolvedAt(dailyRow?.id ?? "")).toEqual(expect.any(String)); + expect(await resolvedAt(month?.id ?? "")).toBe( + await resolvedAt(dailyRow?.id ?? ""), + ); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + }); + + it("resolves the first month a daily latch covers, with the daily rows of the same shortfall", async () => { + const { appId, accountId } = await makeApp(900); + const transcriptId = id(); + await database().insert(developerTranscripts).values({ + id: transcriptId, + appId, + status: "processing", + sourceType: "upload", + reservedMicroCredits: 900, + }); + await settleFirstMonthAtReady({ + id: await makeVideo(appId, { duration: 3_600 }), + deletedAt: null, + }); + const [month] = await firstMonthRows(accountId); + const shortSince = (month?.metadata as { shortSince: string }).shortSince; + expect(month?.metadata).toMatchObject({ owedMicroCredits: 5_100 }); + + const dayUsage = (dailyMicroCredits: number) => ({ + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: dailyMicroCredits * 30, + dailyMicroCredits, + }); + const dateKey = (daysAgo: number) => + new Date(Date.now() - daysAgo * DAY_MS).toISOString().slice(0, 10); + await database() + .update(developerTranscripts) + .set({ reservedMicroCredits: 850 }) + .where(eq(developerTranscripts.id, transcriptId)); + expect( + await chargeStorageDay({ + appId, + accountId, + usage: dayUsage(300), + dateKey: dateKey(1), + now: new Date(), + }), + ).toMatchObject({ state: "billed", chargedMicroCredits: 50 }); + const dailyRows = () => + database() + .select() + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + eq(developerCreditTransactions.type, "storage_daily"), + ), + ); + const [carried] = await dailyRows(); + expect(carried?.metadata).toMatchObject({ shortSince }); + + await database() + .update(developerTranscripts) + .set({ status: "complete", reservedMicroCredits: 0 }) + .where(eq(developerTranscripts.id, transcriptId)); + await database() + .update(developerCreditAccounts) + .set({ balanceMicroCredits: 5_100 }) + .where(eq(developerCreditAccounts.id, accountId)); + expect( + await chargeStorageDay({ + appId, + accountId, + usage: dayUsage(28), + dateKey: dateKey(0), + now: new Date(), + }), + ).toMatchObject({ + state: "billed", + chargedMicroCredits: 28, + shortSince: null, + }); + + const resolved = [ + ...(await firstMonthRows(accountId)), + ...(await dailyRows()), + ] + .filter( + (row) => + (row.metadata as { shortSince?: string | null }).shortSince === + shortSince, + ) + .map((row) => (row.metadata as { resolvedAt?: string }).resolvedAt); + expect(resolved).toHaveLength(2); + expect(resolved.every((value) => typeof value === "string")).toBe(true); + await database() + .update(developerCreditAccounts) + .set({ balanceMicroCredits: 100 }) + .where(eq(developerCreditAccounts.id, accountId)); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + }); + + it("keeps standing and resolved shortfalls consistent across 500 seeded charge, delivery, hold and top-up sequences", async () => { + let seed = 0x4145; + const random = () => { + seed = (seed + 0x6d2b79f5) | 0; + let t = Math.imul(seed ^ (seed >>> 15), 1 | seed); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + const pick = (items: readonly T[]) => + items[Math.floor(random() * items.length)] as T; + const between = (low: number, high: number) => + low + Math.floor(random() * (high - low + 1)); + const kinds = [ + "daily_paid", + "daily_short", + "month_paid", + "month_short", + "month_short_pair", + "hold", + "release", + "hold", + "release", + "top_up", + "daily_part_in_month", + "daily_part_in_month", + "top_up_then_short_same_second", + "top_up_then_short_same_second", + "delivery_paid", + "delivery_short", + "delivery_paid", + "delivery_short", + ] as const; + type Period = { since: string; monthMax: number }; + const failures: string[] = []; + const delivered = { billed: 0, short: 0, carried: 0 }; + + for (let sequence = 0; sequence < 500; sequence++) { + const startBalance = pick([0, 500, 5_000, 20_000]); + const { appId, accountId } = await makeApp(startBalance); + let balance = startBalance; + let held = 0; + const holds: { id: string; amount: number }[] = []; + let period: Period | null = null; + let lastDaily: { since: string | null; owed: number } = { + since: null, + owed: 0, + }; + const size = () => + period + ? Math.max( + period.monthMax, + lastDaily.since === period.since ? lastDaily.owed : 0, + ) + : 0; + const close = (latch: boolean) => { + if (!period) return; + covered.add(period.since); + if (latch) latched.add(period.since); + period = null; + lastDaily = { since: null, owed: 0 }; + }; + const covered = new Set(); + const latched = new Set(); + const steps = between(2, 6); + const log: string[] = []; + let dayIndex = 6; + let deliveryIndex = 40; + + const latch = () => { + if (period && balance - held >= size()) close(true); + }; + const topUp = async () => { + const cents = between(1, 200); + const referenceId = id(); + await database() + .update(developerCreditTransactions) + .set({ + createdAt: sql`${developerCreditTransactions.createdAt} - INTERVAL 1 SECOND`, + }) + .where(eq(developerCreditTransactions.accountId, accountId)); + await addCreditsToAccount({ + accountId, + amountCents: cents, + referenceId, + referenceType: "stripe_payment_intent", + }); + const added = Math.floor((cents / 100) * 100_000); + balance += added; + log[log.length - 1] += ` +${added}`; + if (period && balance - held >= size()) close(false); + const [row] = await database() + .select({ createdAt: developerCreditTransactions.createdAt }) + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + eq(developerCreditTransactions.referenceId, referenceId), + ), + ); + return row?.createdAt ?? null; + }; + const rowSince = async (referenceId: string) => { + const [row] = await database() + .select({ metadata: developerCreditTransactions.metadata }) + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + eq(developerCreditTransactions.referenceId, referenceId), + ), + ) + .limit(1); + return String( + (row?.metadata as { shortSince?: string | null } | undefined) + ?.shortSince, + ); + }; + + for (let step = 0; step < steps; step++) { + const kind = pick(kinds); + let available = balance - held; + log.push(`${kind} balance=${balance} held=${held}`); + let stepKind: (typeof kinds)[number] = kind; + if (kind === "daily_part_in_month") { + if (dayIndex < 0) continue; + if (!period || period.monthMax === 0) { + if (available <= 0) continue; + const holdId = id(); + await database().insert(developerTranscripts).values({ + id: holdId, + appId, + status: "processing", + sourceType: "upload", + reservedMicroCredits: available, + }); + held += available; + const minutes = between(1, 240); + const owed = minutes * 85; + latch(); + const videoId = await makeVideo(appId, { + duration: minutes * 60, + }); + await settleFirstMonthAtReady({ id: videoId, deletedAt: null }); + log[log.length - 1] += ` month owed=${owed}`; + period = period + ? { + since: period.since, + monthMax: Math.max(period.monthMax, owed), + } + : { since: await rowSince(videoId), monthMax: owed }; + await database() + .update(developerTranscripts) + .set({ status: "complete", reservedMicroCredits: 0 }) + .where(eq(developerTranscripts.id, holdId)); + held -= available; + } + available = balance - held; + if (available <= 0) continue; + stepKind = "daily_short"; + } + if (kind === "top_up_then_short_same_second") { + if (dayIndex < 0) continue; + const toppedUpAt = await topUp(); + available = balance - held; + if (available > 0) { + const holdId = id(); + await database().insert(developerTranscripts).values({ + id: holdId, + appId, + status: "processing", + sourceType: "upload", + reservedMicroCredits: available, + }); + holds.push({ id: holdId, amount: available }); + held += available; + log[log.length - 1] += ` hold ${available}`; + } + available = balance - held; + stepKind = "daily_short"; + const dateKey = new Date(Date.now() - dayIndex * DAY_MS) + .toISOString() + .slice(0, 10); + const owed = available + between(1, 20_000); + latch(); + const result = await chargeStorageDay({ + appId, + accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: owed * 30, + dailyMicroCredits: owed, + }, + dateKey, + now: new Date(), + }); + dayIndex--; + log[log.length - 1] += + ` owed=${owed} -> ${result.state} same second`; + if (result.state === "billed") { + await database() + .update(developerCreditTransactions) + .set({ createdAt: toppedUpAt ?? new Date() }) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + eq(developerCreditTransactions.referenceId, dateKey), + ), + ); + const charged = Math.min(balance - held, owed); + balance -= charged; + const since: string = + (period as Period | null)?.since ?? (await rowSince(dateKey)); + lastDaily = { since, owed }; + period = period ?? { since, monthMax: 0 }; + } + } else if ( + stepKind === "delivery_paid" || + stepKind === "delivery_short" + ) { + const target = + stepKind === "delivery_paid" + ? between(0, Math.max(0, available)) + : Math.max(0, available) + between(1, 20_000); + const seconds = + stepKind === "delivery_paid" + ? Math.floor((target * 60) / 95) + : Math.ceil((target * 60) / 95); + const owed = deliveryMicroCredits(seconds); + const dateKey = new Date(Date.now() - deliveryIndex * DAY_MS) + .toISOString() + .slice(0, 10); + latch(); + const openShortfall = period !== null; + const result = await chargeDeliveryDay({ + appId, + accountId, + delivered: { + day: dateKey, + seconds, + secondsByTier: { "1080p": seconds }, + }, + now: new Date(), + }); + deliveryIndex--; + await database() + .update(developerCreditTransactions) + .set({ + createdAt: sql`${developerCreditTransactions.createdAt} - INTERVAL 1 SECOND`, + }) + .where(eq(developerCreditTransactions.accountId, accountId)); + log[log.length - 1] += ` delivery owed=${owed} -> ${result.state}`; + if (result.state !== "billed") continue; + delivered.billed++; + const charged = Math.max(0, Math.min(balance - held, owed)); + balance -= charged; + if (charged < owed) delivered.short++; + if (openShortfall) { + if (charged < owed) delivered.carried++; + } else if (charged < owed) { + const since = await rowSince(dateKey); + lastDaily = { since, owed }; + period = { since, monthMax: 0 }; + } else { + lastDaily = { since: null, owed: 0 }; + } + } else if (stepKind === "daily_paid" || stepKind === "daily_short") { + if (dayIndex < 0) continue; + const owed = + stepKind === "daily_paid" + ? between(0, available) + : available + between(1, 20_000); + const dateKey = new Date(Date.now() - dayIndex * DAY_MS) + .toISOString() + .slice(0, 10); + latch(); + const result = await chargeStorageDay({ + appId, + accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: owed * 30, + dailyMicroCredits: owed, + }, + dateKey, + now: new Date(), + }); + dayIndex--; + log[log.length - 1] += ` owed=${owed} -> ${result.state}`; + if (result.state !== "billed") continue; + const charged = Math.min(balance - held, owed); + balance -= charged; + if (charged < owed) { + const since: string = + (period as Period | null)?.since ?? (await rowSince(dateKey)); + lastDaily = { since, owed }; + period = period ?? { since, monthMax: 0 }; + } else { + lastDaily = { since: null, owed: 0 }; + if (period?.monthMax === 0) period = null; + } + } else if ( + kind === "month_paid" || + kind === "month_short" || + kind === "month_short_pair" + ) { + const minutes = Math.min( + 240, + kind === "month_paid" + ? Math.floor(available / 85) + : Math.floor(available / 85) + between(1, 200), + ); + if (kind !== "month_paid" && minutes * 85 <= available) continue; + if (minutes < 1) continue; + if (kind === "month_short_pair" && minutes >= 240) continue; + const sizes = + kind === "month_short_pair" + ? [minutes, between(minutes + 1, 240)] + : [minutes]; + const videoIds: string[] = []; + for (const size of sizes) { + const owed = size * 85; + latch(); + const videoId = await makeVideo(appId, { duration: size * 60 }); + await settleFirstMonthAtReady({ id: videoId, deletedAt: null }); + videoIds.push(videoId); + log[log.length - 1] += ` owed=${owed}`; + const charged = Math.min(balance - held, owed); + balance -= charged; + if (charged < owed) { + period = period + ? { + since: period.since, + monthMax: Math.max(period.monthMax, owed), + } + : { since: await rowSince(videoId), monthMax: owed }; + } + } + if (videoIds.length > 1) { + const [first] = await database() + .select({ createdAt: developerCreditTransactions.createdAt }) + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + eq( + developerCreditTransactions.referenceId, + videoIds[0] ?? "", + ), + ), + ); + await database() + .update(developerCreditTransactions) + .set({ createdAt: first?.createdAt ?? new Date() }) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + inArray(developerCreditTransactions.referenceId, videoIds), + ), + ); + } + } else if (kind === "hold") { + if (available <= 0) continue; + const amount = between(1, available); + const holdId = id(); + await database().insert(developerTranscripts).values({ + id: holdId, + appId, + status: "processing", + sourceType: "upload", + reservedMicroCredits: amount, + }); + holds.push({ id: holdId, amount }); + held += amount; + log[log.length - 1] += ` +${amount}`; + } else if (kind === "release") { + const hold = holds.shift(); + if (!hold) continue; + await database() + .update(developerTranscripts) + .set({ status: "complete", reservedMicroCredits: 0 }) + .where(eq(developerTranscripts.id, hold.id)); + held -= hold.amount; + log[log.length - 1] += ` -${hold.amount}`; + } else { + await topUp(); + } + + const where = `sequence ${sequence} step ${step}: ${log.join(" | ")}`; + if ((await balanceOf(accountId)) !== balance) { + failures.push( + `${where}: balance ${await balanceOf(accountId)} expected ${balance}`, + ); + break; + } + const short = period !== null && balance - held < size(); + const { standing } = await getInfraBilling(appId); + const expected = short + ? { + state: "grace", + pausesAt: new Date( + Date.parse(period?.since ?? "") + 7 * DAY_MS, + ), + } + : { state: "active", pausesAt: null }; + if (JSON.stringify(standing) !== JSON.stringify(expected)) { + failures.push( + `${where}: standing ${JSON.stringify(standing)} expected ${JSON.stringify(expected)} period ${JSON.stringify(period)} size ${size()} lastDaily ${JSON.stringify(lastDaily)}`, + ); + break; + } + if (latched.size > 0) { + const rows = await database() + .select({ metadata: developerCreditTransactions.metadata }) + .from(developerCreditTransactions) + .where(eq(developerCreditTransactions.accountId, accountId)); + const unresolved = rows.filter((row) => { + const fields = row.metadata as { + shortSince?: string | null; + resolvedAt?: string; + } | null; + return ( + fields?.shortSince && + latched.has(fields.shortSince) && + !fields.resolvedAt + ); + }); + if (unresolved.length > 0) { + failures.push( + `${where}: ${unresolved.length} rows of a covered shortfall unresolved`, + ); + break; + } + } + } + } + + expect(failures).toEqual([]); + expect(delivered.billed).toBeGreaterThan(250); + expect(delivered.short).toBeGreaterThan(150); + expect(delivered.carried).toBeGreaterThan(50); + console.info("[seeded] delivery steps", delivered); + }, 600_000); + + const chargeDay = ( + appId: string, + accountId: string, + dailyMicroCredits: number, + daysAgo: number, + ) => + chargeStorageDay({ + appId, + accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: dailyMicroCredits * 30, + dailyMicroCredits, + }, + dateKey: new Date(Date.now() - daysAgo * DAY_MS) + .toISOString() + .slice(0, 10), + now: new Date(), + }); + + const setBalance = (accountId: string, balanceMicroCredits: number) => + database() + .update(developerCreditAccounts) + .set({ balanceMicroCredits }) + .where(eq(developerCreditAccounts.id, accountId)); + + it("forgives an older unpaid day once the latest day is covered, then a first month latches the whole period", async () => { + const { appId, accountId } = await makeApp(0); + expect(await chargeDay(appId, accountId, 5_000, 3)).toMatchObject({ + state: "billed", + chargedMicroCredits: 0, + }); + await setBalance(accountId, 450); + const transcriptId = id(); + await database().insert(developerTranscripts).values({ + id: transcriptId, + appId, + status: "processing", + sourceType: "upload", + reservedMicroCredits: 400, + }); + expect(await chargeDay(appId, accountId, 300, 2)).toMatchObject({ + state: "billed", + chargedMicroCredits: 50, + owedMicroCredits: 300, + }); + await database() + .update(developerTranscripts) + .set({ status: "complete", reservedMicroCredits: 0 }) + .where(eq(developerTranscripts.id, transcriptId)); + expect(await balanceOf(accountId)).toBe(400); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + + await settleFirstMonthAtReady({ + id: await makeVideo(appId, { duration: 106 }), + deletedAt: null, + }); + expect(await balanceOf(accountId)).toBe(250); + const days = await database() + .select({ metadata: developerCreditTransactions.metadata }) + .from(developerCreditTransactions) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + eq(developerCreditTransactions.type, "storage_daily"), + ), + ); + expect(days).toHaveLength(2); + for (const day of days) { + expect((day.metadata as { resolvedAt?: string }).resolvedAt).toEqual( + expect.any(String), + ); + } + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + }); + + it("sizes a run of short days by the latest day, as before first months", async () => { + const { appId, accountId } = await makeApp(0); + await chargeDay(appId, accountId, 1_000, 4); + const [first] = await database() + .select({ metadata: developerCreditTransactions.metadata }) + .from(developerCreditTransactions) + .where(eq(developerCreditTransactions.accountId, accountId)); + const shortSince = (first?.metadata as { shortSince: string }).shortSince; + const grace = { + state: "grace", + pausesAt: new Date(Date.parse(shortSince) + 7 * DAY_MS), + }; + + await setBalance(accountId, 10); + expect(await chargeDay(appId, accountId, 28, 3)).toMatchObject({ + state: "billed", + chargedMicroCredits: 10, + shortSince: new Date(shortSince), + }); + await setBalance(accountId, 20); + expect((await getInfraBilling(appId)).standing).toEqual(grace); + await setBalance(accountId, 100); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + + await setBalance(accountId, 100); + const third = await chargeDay(appId, accountId, 2_000, 2); + expect(third).toMatchObject({ + state: "billed", + chargedMicroCredits: 100, + }); + const thirdSince = + third.state === "billed" ? (third.shortSince ?? null) : null; + expect(thirdSince?.getTime()).toBeGreaterThan(Date.parse(shortSince)); + await setBalance(accountId, 1_500); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "grace", + pausesAt: new Date((thirdSince?.getTime() ?? 0) + 7 * DAY_MS), + }); + await setBalance(accountId, 500); + expect(await chargeDay(appId, accountId, 400, 1)).toMatchObject({ + state: "billed", + chargedMicroCredits: 400, + shortSince: null, + }); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + }); + + it("sizes a first month grace period by a larger daily charge it carries", async () => { + const { appId, accountId } = await makeApp(900); + const transcriptId = id(); + await database().insert(developerTranscripts).values({ + id: transcriptId, + appId, + status: "processing", + sourceType: "upload", + reservedMicroCredits: 900, + }); + await settleFirstMonthAtReady({ + id: await makeVideo(appId), + deletedAt: null, + }); + const [month] = await firstMonthRows(accountId); + const shortSince = (month?.metadata as { shortSince: string }).shortSince; + expect(month).toMatchObject({ amountMicroCredits: 0 }); + expect(month?.metadata).toMatchObject({ owedMicroCredits: 850 }); + + await database() + .update(developerTranscripts) + .set({ reservedMicroCredits: 400 }) + .where(eq(developerTranscripts.id, transcriptId)); + expect( + await chargeStorageDay({ + appId, + accountId, + usage: { + videoCount: 1, + totalMinutes: 1, + minutesByTier: emptyStorageMinutes(), + monthlyMicroCredits: 1_120 * 30, + dailyMicroCredits: 1_120, + }, + dateKey: new Date().toISOString().slice(0, 10), + now: new Date(), + }), + ).toMatchObject({ + state: "billed", + chargedMicroCredits: 500, + owedMicroCredits: 1_120, + }); + + await database() + .update(developerTranscripts) + .set({ status: "complete", reservedMicroCredits: 0 }) + .where(eq(developerTranscripts.id, transcriptId)); + await database() + .update(developerCreditAccounts) + .set({ balanceMicroCredits: 900 }) + .where(eq(developerCreditAccounts.id, accountId)); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "grace", + pausesAt: new Date(Date.parse(shortSince) + 7 * DAY_MS), + }); + + await database() + .update(developerCreditAccounts) + .set({ balanceMicroCredits: 1_120 }) + .where(eq(developerCreditAccounts.id, accountId)); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + }); + + it("reads the largest unpaid month of a shortfall when two fall short in the same second", async () => { + const { appId, accountId } = await makeApp(900); + const transcriptId = id(); + await database().insert(developerTranscripts).values({ + id: transcriptId, + appId, + status: "processing", + sourceType: "upload", + reservedMicroCredits: 900, + }); + const smallVideo = await makeVideo(appId); + await settleFirstMonthAtReady({ id: smallVideo, deletedAt: null }); + const largeVideo = await makeVideo(appId, { + duration: 3_600, + width: 3840, + height: 2160, + maxResolutionTier: "2160p", + }); + await settleFirstMonthAtReady({ id: largeVideo, deletedAt: null }); + const rows = await firstMonthRows(accountId); + const small = rows.find((row) => row.referenceId === smallVideo); + const large = rows.find((row) => row.referenceId === largeVideo); + const shortSince = (small?.metadata as { shortSince: string }).shortSince; + expect(small?.metadata).toMatchObject({ owedMicroCredits: 850 }); + expect(large?.metadata).toMatchObject({ + owedMicroCredits: 16_800, + outstandingMicroCredits: 16_800, + shortSince, + }); + const second = new Date(Math.floor(Date.now() / 1000) * 1000 - 5_000); + const smallId = `0${id().slice(1)}`; + const largeId = `z${id().slice(1)}`; + for (const [from, to] of [ + [small?.id ?? "", smallId], + [large?.id ?? "", largeId], + ] as const) { + await database() + .update(developerCreditTransactions) + .set({ id: to, createdAt: second }) + .where(eq(developerCreditTransactions.id, from)); + } + + await database() + .update(developerTranscripts) + .set({ status: "complete", reservedMicroCredits: 0 }) + .where(eq(developerTranscripts.id, transcriptId)); + const pausesAt = new Date(Date.parse(shortSince) + 7 * DAY_MS); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "grace", + pausesAt, + }); + + await settleFirstMonthAtReady({ + id: await makeVideo(appId, { duration: 60 }), + deletedAt: null, + }); + expect(await balanceOf(accountId)).toBe(815); + const after = new Map( + (await firstMonthRows(accountId)).map((row) => [ + row.id, + (row.metadata as { resolvedAt?: string }).resolvedAt, + ]), + ); + expect(after.get(smallId)).toBeUndefined(); + expect(after.get(largeId)).toBeUndefined(); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "grace", + pausesAt, + }); + + await database() + .update(developerCreditAccounts) + .set({ balanceMicroCredits: 17_000 }) + .where(eq(developerCreditAccounts.id, accountId)); + await settleFirstMonthAtReady({ + id: await makeVideo(appId, { duration: 60 }), + deletedAt: null, + }); + const covered = new Map( + (await firstMonthRows(accountId)).map((row) => [ + row.id, + (row.metadata as { resolvedAt?: string }).resolvedAt, + ]), + ); + expect(covered.get(smallId)).toEqual(expect.any(String)); + expect(covered.get(largeId)).toBe(covered.get(smallId)); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + }); + + it("resolves every copy of a covered first month shortfall together, and no other shortfall", async () => { + const { appId, accountId } = await makeApp(900); + const transcriptId = id(); + await database().insert(developerTranscripts).values({ + id: transcriptId, + appId, + status: "processing", + sourceType: "upload", + reservedMicroCredits: 900, + }); + const fourK = { + duration: 3_600, + width: 3840, + height: 2160, + maxResolutionTier: "2160p" as const, + }; + await settleFirstMonthAtReady({ + id: await makeVideo(appId, fourK), + deletedAt: null, + }); + await settleFirstMonthAtReady({ + id: await makeVideo(appId, fourK), + deletedAt: null, + }); + const copies = await firstMonthRows(accountId); + expect(copies).toHaveLength(2); + const shortSince = (copies[0]?.metadata as { shortSince: string }) + .shortSince; + for (const copy of copies) { + expect(copy.amountMicroCredits).toBe(0); + expect(copy.metadata).toMatchObject({ + owedMicroCredits: 16_800, + shortSince, + }); + } + const second = new Date(Math.floor(Date.now() / 1000) * 1000 - 5_000); + await database() + .update(developerCreditTransactions) + .set({ createdAt: second }) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + eq(developerCreditTransactions.type, "video_create"), + ), + ); + const otherShortSince = new Date( + Date.parse(shortSince) + 60_000, + ).toISOString(); + const otherId = id(); + await database() + .insert(developerCreditTransactions) + .values({ + id: otherId, + accountId, + type: "video_create", + amountMicroCredits: 0, + balanceAfterMicroCredits: 0, + referenceId: id(), + referenceType: "developer_video", + metadata: { + owedMicroCredits: 850, + chargedMicroCredits: 0, + shortSince: otherShortSince, + fellShort: true, + }, + createdAt: new Date(second.getTime() - 60_000), + }); + expect((await getInfraBilling(appId)).standing.state).toBe("grace"); + + await makeVideo(appId, { createdAt: new Date("2026-10-01T00:00:00Z") }); + await database() + .update(developerTranscripts) + .set({ status: "complete", reservedMicroCredits: 0 }) + .where(eq(developerTranscripts.id, transcriptId)); + await database() + .update(developerCreditAccounts) + .set({ balanceMicroCredits: 16_800 }) + .where(eq(developerCreditAccounts.id, accountId)); + const today = new Date().toISOString().slice(0, 10); + const usage = ( + await getStorageUsageByApp([appId], undefined, undefined, { + dailyChargeAt: new Date(`${today}T00:00:00.000Z`), + }) + ).get(appId); + if (!usage) throw new Error("no usage"); + expect( + await chargeStorageDay({ + appId, + accountId, + usage, + dateKey: today, + now: new Date(), + }), + ).toMatchObject({ state: "billed", chargedMicroCredits: 28 }); + expect(await balanceOf(accountId)).toBe(16_772); + + const resolved = new Map( + (await firstMonthRows(accountId)).map((row) => [ + row.id, + (row.metadata as { resolvedAt?: string }).resolvedAt, + ]), + ); + for (const copy of copies) { + expect(resolved.get(copy.id)).toEqual(expect.any(String)); + } + expect(resolved.get(copies[0]?.id ?? "")).toBe( + resolved.get(copies[1]?.id ?? ""), + ); + expect(resolved.get(otherId)).toBeUndefined(); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + + const nextVideo = await makeVideo(appId); + await settleFirstMonthAtReady({ id: nextVideo, deletedAt: null }); + const next = (await firstMonthRows(accountId)).find( + (row) => row.referenceId === nextVideo, + ); + expect(next).toMatchObject({ amountMicroCredits: -850 }); + expect(next?.metadata).toMatchObject({ shortSince: null }); + expect(await balanceOf(accountId)).toBe(15_922); + expect((await getInfraBilling(appId)).standing).toEqual({ + state: "active", + pausesAt: null, + }); + }); + + describe("a video deleted while it encodes", () => { + const encoding = (): DeveloperVideoProcessingState => ({ + ...processing(), + probe: null, + mp4: { + state: "running", + token: "t-mp4", + jobId: "job-1", + startedAt: Date.now(), + }, + attempts: { hls: 0, mp4: 1 }, + }); + const deleteNow = (videoId: string) => + database() + .update(developerVideos) + .set({ deletedAt: new Date() }) + .where(eq(developerVideos.id, videoId)); + + it("is charged its first month once, from its declared length at the tier it was created with, and never again", async () => { + const { appId, accountId } = await makeApp(1_000_000); + const videoId = await makeVideo(appId, { + createdAt: new Date(), + status: "processing", + duration: 3600, + width: null, + height: null, + maxResolutionTier: "2160p", + processing: encoding(), + }); + await deleteNow(videoId); + + const charged = await settleFirstMonthAtDelete(videoId); + expect(charged).toMatchObject({ + state: "charged", + tier: "2160p", + owedMicroCredits: 16_800, + chargedMicroCredits: 16_800, + }); + expect(await settleFirstMonthAtDelete(videoId)).toEqual({ + state: "skipped", + reason: "already_settled", + }); + + await database() + .update(developerVideos) + .set({ status: "ready" }) + .where(eq(developerVideos.id, videoId)); + await settleFirstMonthAtReady({ id: videoId, deletedAt: new Date() }); + await chargeMissedFirstMonths(new Date()); + + const rows = await firstMonthRows(accountId); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + amountMicroCredits: -16_800, + referenceId: videoId, + referenceType: "developer_video", + }); + expect(rows[0]?.metadata).toMatchObject({ deletedWhileEncoding: true }); + expect(await balanceOf(accountId)).toBe(1_000_000 - 16_800); + expect((await videoRow(videoId))?.storagePaidUntil).not.toBeNull(); + }); + + it("is charged when deleted while a failed attempt waits for its retry", async () => { + const { appId, accountId } = await makeApp(1_000_000); + const videoId = await makeVideo(appId, { + createdAt: new Date(), + status: "processing", + duration: 600, + processing: { + ...processing(), + probe: null, + mp4: { state: "pending" }, + hls: { state: "pending" }, + attempts: { hls: 0, mp4: 1 }, + }, + }); + await deleteNow(videoId); + + expect(await settleFirstMonthAtDelete(videoId)).toMatchObject({ + state: "charged", + owedMicroCredits: 850, + }); + expect(await firstMonthRows(accountId)).toHaveLength(1); + }); + + it("is charged once when deleted after a dispatched attempt failed and was reset, even if the retry then found the farm busy", async () => { + const started = createDeveloperVideoProcessingState({ + encodeId: "encode-1", + now: Date.now(), + }); + const running = beginPart(started, "mp4", { + token: "t-1", + now: Date.now(), + }); + const dispatched = + running && + markPartDispatched(running, "mp4", { token: "t-1", jobId: "job-1" }); + const failed = + dispatched && + settlePart(dispatched, "mp4", "t-1", { + type: "failed", + reason: "encoder stopped", + }); + const retried = + failed && beginPart(failed, "mp4", { token: "t-2", now: Date.now() }); + const busy = retried && releasePart(retried, "mp4", "t-2"); + if (!failed || !busy) throw new Error("state did not advance"); + expect(failed.mp4).toEqual({ state: "pending" }); + expect(busy.mp4).toEqual({ state: "pending" }); + + for (const state of [failed, busy]) { + const { appId, accountId } = await makeApp(1_000_000); + const videoId = await makeVideo(appId, { + createdAt: new Date(), + status: "processing", + duration: 600, + processing: state, + }); + await deleteNow(videoId); + + expect(await settleFirstMonthAtDelete(videoId)).toMatchObject({ + state: "charged", + owedMicroCredits: 850, + }); + expect(await settleFirstMonthAtDelete(videoId)).toEqual({ + state: "skipped", + reason: "already_settled", + }); + expect(await firstMonthRows(accountId)).toHaveLength(1); + } + }); + + it("caps the declared length at the 4 hour limit", async () => { + const { appId } = await makeApp(1_000_000); + const videoId = await makeVideo(appId, { + createdAt: new Date(), + status: "processing", + duration: 6 * 60 * 60, + processing: encoding(), + }); + await deleteNow(videoId); + + expect(await settleFirstMonthAtDelete(videoId)).toMatchObject({ + state: "charged", + owedMicroCredits: 240 * 85, + }); + }); + + it("pays nothing when deleted before any encode was handed to the farm, including a slot reserved but not yet sent", async () => { + const { appId, accountId } = await makeApp(1_000_000); + const notClaimed = await makeVideo(appId, { + status: "uploading", + processing: null, + playbackId: null, + }); + const claimedNotSent = await makeVideo(appId, { + createdAt: new Date(), + status: "processing", + processing: { + ...processing(), + mp4: { state: "pending" }, + attempts: { hls: 0, mp4: 0 }, + }, + }); + const reservedNotSent = await makeVideo(appId, { + createdAt: new Date(), + status: "processing", + processing: { + ...processing(), + mp4: { + state: "running", + token: "t-mp4", + jobId: null, + startedAt: Date.now(), + }, + attempts: { hls: 0, mp4: 1 }, + }, + }); + for (const videoId of [notClaimed, claimedNotSent, reservedNotSent]) { + await deleteNow(videoId); + expect(await settleFirstMonthAtDelete(videoId)).toEqual({ + state: "skipped", + reason: "not_dispatched", + }); + expect((await videoRow(videoId))?.storagePaidUntil).toBeNull(); + } + expect(await firstMonthRows(accountId)).toHaveLength(0); + expect(await balanceOf(accountId)).toBe(1_000_000); + }); + + it("pays nothing for an encode that failed on its own, deleted or not", async () => { + const { appId, accountId } = await makeApp(1_000_000); + const failed = await makeVideo(appId, { + status: "errored", + processing: { + ...processing(), + mp4: { state: "failed", reason: "boom" }, + hls: { state: "failed", reason: "boom" }, + attempts: { hls: 2, mp4: 2 }, + }, + }); + await deleteNow(failed); + + expect(await settleFirstMonthAtDelete(failed)).toEqual({ + state: "skipped", + reason: "not_dispatched", + }); + expect(await firstMonthRows(accountId)).toHaveLength(0); + }); + + it("is charged when the whole app is deleted mid-encode, and leaves videos deleted long before alone", async () => { + const { appId, accountId } = await makeApp(1_000_000); + const [owner] = await database() + .select({ ownerId: developerApps.ownerId }) + .from(developerApps) + .where(eq(developerApps.id, appId)); + vi.mocked(getCurrentUser).mockResolvedValue({ + id: owner?.ownerId, + email: "owner@example.com", + } as never); + const encodingNow = await makeVideo(appId, { + createdAt: new Date(), + status: "processing", + duration: 600, + processing: encoding(), + }); + const deletedLongAgo = await makeVideo(appId, { + createdAt: new Date("2026-10-01T00:00:00Z"), + deletedAt: new Date("2026-10-02T00:00:00Z"), + status: "processing", + duration: 600, + processing: encoding(), + }); + const readyLongAgo = await makeVideo(appId, { + createdAt: new Date("2026-10-01T00:00:00Z"), + deletedAt: new Date("2026-10-02T00:00:00Z"), + }); + + await deleteDeveloperApp(appId); + await settleFirstMonthAtDelete(encodingNow); + + const rows = await firstMonthRows(accountId); + expect(rows.map((row) => row.referenceId)).toEqual([encodingNow]); + expect(await balanceOf(accountId)).toBe(1_000_000 - 850); + for (const videoId of [deletedLongAgo, readyLongAgo]) { + expect((await videoRow(videoId))?.storagePaidUntil).toBeNull(); + } + }); + + it("charges nothing for a pre-feature ready video and marks it exempt", async () => { + const { appId, accountId } = await makeApp(1_000_000); + const videoId = await makeVideo(appId, { + createdAt: new Date("2026-10-01T00:00:00Z"), + }); + await deleteNow(videoId); + + expect(await settleFirstMonthAtDelete(videoId)).toMatchObject({ + state: "exempt", + }); + expect((await videoRow(videoId))?.storagePaidUntil).not.toBeNull(); + expect(await firstMonthRows(accountId)).toHaveLength(0); + expect(await balanceOf(accountId)).toBe(1_000_000); + }); + + it("charges nothing for a video mid-encode created before first month charging began", async () => { + const { appId, accountId } = await makeApp(1_000_000); + const videoId = await makeVideo(appId, { + createdAt: new Date("2026-10-01T00:00:00Z"), + status: "processing", + duration: 600, + processing: encoding(), + }); + await deleteNow(videoId); + + expect(await settleFirstMonthAtDelete(videoId)).toMatchObject({ + state: "exempt", + }); + expect(await firstMonthRows(accountId)).toHaveLength(0); + }); + + it("charges nothing more for a post-feature ready video that already paid its month", async () => { + const { appId, accountId } = await makeApp(1_000_000); + const videoId = await makeVideo(appId, { createdAt: new Date() }); + await settleFirstMonthAtReady({ id: videoId, deletedAt: null }); + await deleteNow(videoId); + + expect(await settleFirstMonthAtDelete(videoId)).toEqual({ + state: "skipped", + reason: "already_settled", + }); + expect(await firstMonthRows(accountId)).toHaveLength(1); + expect(await balanceOf(accountId)).toBe(1_000_000 - 850); + }); + + it("records a month the balance cannot pay as owed and leaves the delete in place", async () => { + const { appId, accountId } = await makeApp(100); + const videoId = await makeVideo(appId, { + createdAt: new Date(), + status: "processing", + duration: 600, + processing: encoding(), + }); + await deleteNow(videoId); + + expect(await settleFirstMonthAtDelete(videoId)).toMatchObject({ + state: "charged", + owedMicroCredits: 850, + chargedMicroCredits: 100, + }); + expect(await balanceOf(accountId)).toBe(0); + expect((await videoRow(videoId))?.deletedAt).not.toBeNull(); + expect((await getInfraBilling(appId)).standing.state).toBe("grace"); + }); + }); + + it("finds a new first month shortfall for its email once, and never one that only carried an earlier shortfall", async () => { + const { appId, accountId } = await makeApp(0); + const since = new Date(Date.now() - 60 * 60 * 1000); + await settleFirstMonthAtReady({ + id: await makeVideo(appId), + deletedAt: null, + }); + + const found = await shortfallsAwaitingNotice([accountId], since); + expect([...found.keys()]).toEqual([accountId]); + const [first] = await firstMonthRows(accountId); + expect(first?.metadata).toMatchObject({ fellShort: true }); + + await settleFirstMonthAtReady({ + id: await makeVideo(appId), + deletedAt: null, + }); + const rows = await firstMonthRows(accountId); + expect(rows).toHaveLength(2); + expect( + rows.map((row) => (row.metadata as { fellShort: boolean }).fellShort), + ).toEqual(expect.arrayContaining([true, false])); + expect( + (await shortfallsAwaitingNotice([accountId], since)).get(accountId) + ?.ids, + ).toEqual([first?.id]); + + await markShortfallsNotified(found.get(accountId)?.ids ?? [], new Date()); + await markShortfallsNotified( + found.get(accountId)?.ids ?? [], + new Date(Date.now() + 60_000), + ); + expect(await shortfallsAwaitingNotice([accountId], since)).toEqual( + new Map(), + ); + }); + + it("keeps a daily shortfall whose email was not sent for the next run, and drops it once sent", async () => { + const { appId, accountId } = await makeApp(0); + await makeVideo(appId, { createdAt: new Date("2026-10-01T00:00:00Z") }); + const since = new Date(Date.now() - 60 * 60 * 1000); + const today = new Date().toISOString().slice(0, 10); + const usage = ( + await getStorageUsageByApp([appId], undefined, undefined, { + dailyChargeAt: new Date(`${today}T00:00:00.000Z`), + }) + ).get(appId); + if (!usage) throw new Error("no usage"); + + expect( + await chargeStorageDay({ + appId, + accountId, + usage, + dateKey: today, + now: new Date(), + }), + ).toMatchObject({ state: "billed", fellShort: true }); + + const waiting = await shortfallsAwaitingNotice([accountId], since); + expect(waiting.get(accountId)?.ids).toHaveLength(1); + + await markStorageDayNotified(accountId, today, new Date()); + expect(await shortfallsAwaitingNotice([accountId], since)).toEqual( + new Map(), + ); + }); + + it("never throws from the ready path when the charge fails", async () => { + const broken = { + transaction: async () => { + throw new Error("connection lost"); + }, + } as never; + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + + await expect( + settleFirstMonthAtReady({ id: id(), deletedAt: null }, broken), + ).resolves.toBeUndefined(); + expect(errors).toHaveBeenCalled(); + errors.mockRestore(); + }); + + it("admits one upload on a balance that covers one first month, and refuses the others completing at the same time", async () => { + const { appId } = await makeApp(5_600); + const videos = await Promise.all( + Array.from({ length: 4 }, () => + makeVideo(appId, { + status: "processing", + duration: null, + width: null, + height: null, + maxResolutionTier: "2160p", + }), + ), + ); + + const results = await Promise.all( + videos.map((videoId) => + recordAdmittedInfraUpload( + { + appId, + videoId, + durationSeconds: 1200, + width: 3840, + height: 2160, + maxResolutionTier: "2160p", + }, + { duration: 1200, width: 3840, height: 2160 }, + ), + ), + ); + + const admitted = results.filter( + (result) => "allowed" in result && result.allowed, + ); + const refused = results.filter( + (result) => "allowed" in result && !result.allowed, + ); + expect(admitted).toHaveLength(1); + expect(refused).toHaveLength(3); + for (const refusal of refused) { + expect("message" in refusal && refusal.message).toMatch( + /^Not enough credit /, + ); + expect(refusal).toMatchObject({ + reason: "insufficient_credits", + requiredMicroCredits: 11_200, + shortfallMicroCredits: 5_600, + }); + } + const recorded = await Promise.all(videos.map(videoRow)); + expect(recorded.filter((video) => video?.duration === 1200)).toHaveLength( + 1, + ); + }); + + it("needs the first month of the upload and of every upload in flight", async () => { + const { appId } = await makeApp(0); + const upload = { + durationSeconds: 1200, + width: 3840, + height: 2160, + tier: "2160p" as const, + }; + + expect( + (await getInfraUploadAdmission(appId, upload)).requiredMicroCredits, + ).toBe(5_600); + + for (let n = 0; n < 2; n++) { + await makeVideo(appId, { + status: "processing", + duration: 1200, + width: 3840, + height: 2160, + maxResolutionTier: "2160p", + }); + } + expect( + (await getInfraUploadAdmission(appId, upload)).requiredMicroCredits, + ).toBe(16_800); + + const clips = (await makeApp(0)).appId; + for (let n = 0; n < 3; n++) { + await makeVideo(clips, { status: "processing", duration: 1 }); + } + await makeVideo(clips, { status: "processing", duration: 0.2 }); + const clip = { + durationSeconds: 1, + width: 1920, + height: 1080, + tier: "1080p" as const, + }; + expect((await getAppStorage(clips)).firstMonthMicroCredits).toBe(4); + expect( + (await getInfraUploadAdmission(clips, clip)).requiredMicroCredits, + ).toBe(5); + }); + + describe("a ready video whose first month is not charged yet", () => { + const fourK = { + status: "ready" as const, + duration: 1200, + width: 3840, + height: 2160, + maxResolutionTier: "2160p" as const, + storagePaidUntil: null, + }; + const upload = { + durationSeconds: 1200, + width: 3840, + height: 2160, + tier: "2160p" as const, + }; + let sentinel: string; + + beforeAll(async () => { + const { appId, accountId } = await makeApp(0); + sentinel = await makeVideo(appId, { + storagePaidUntil: new Date("2026-10-31T00:00:00Z"), + }); + await database() + .insert(developerCreditTransactions) + .values({ + id: id(), + accountId, + type: "video_create", + amountMicroCredits: -100, + balanceAfterMicroCredits: 0, + referenceId: sentinel, + referenceType: "developer_video", + metadata: { paidUntil: "2026-10-31T00:00:00.000Z" }, + createdAt: new Date("2026-10-01T12:00:00Z"), + }); + }); + + afterAll(async () => { + await database() + .delete(developerCreditTransactions) + .where(eq(developerCreditTransactions.referenceId, sentinel)); + await database() + .delete(developerVideos) + .where(eq(developerVideos.id, sentinel)); + }); + + it("still reserves its month, so a second upload the balance cannot pay for both is refused", async () => { + const { appId } = await makeApp(7_000); + await makeVideo(appId, fourK); + + const storage = await getAppStorage(appId); + expect(storage.firstMonthMicroCredits).toBe(5_600); + expect(storage.dailyCharge.totalMinutes).toBe(0); + expect(storage.stored.totalMinutes).toBe(20); + expect(await getInfraUploadAdmission(appId, upload)).toMatchObject({ + allowed: false, + reason: "insufficient_credits", + requiredMicroCredits: 11_200, + }); + }); + + it("stops reserving it once the month is charged", async () => { + const { appId, accountId } = await makeApp(7_000); + const videoId = await makeVideo(appId, fourK); + await settleFirstMonthAtReady({ id: videoId, deletedAt: null }); + expect(await balanceOf(accountId)).toBe(1_400); + + const storage = await getAppStorage(appId); + expect(storage.firstMonthMicroCredits).toBe(0); + expect(storage.stored.totalMinutes).toBe(20); + expect( + (await getInfraUploadAdmission(appId, upload)).requiredMicroCredits, + ).toBe(5_600); + }); + + it("leaves a video created before first month charging began to the daily charge", async () => { + const { appId } = await makeApp(7_000); + await makeVideo(appId, { + ...fourK, + createdAt: new Date("2026-09-01T00:00:00Z"), + }); + + const storage = await getAppStorage(appId); + expect(storage.firstMonthMicroCredits).toBe(0); + expect(storage.dailyCharge.totalMinutes).toBe(20); + expect( + (await getInfraUploadAdmission(appId, upload)).requiredMicroCredits, + ).toBe(6_907); + }); + }); + + it("counts first months in the month's storage spending on the console", async () => { + const { appId, accountId } = await makeApp(1_000_000); + const monthStart = new Date(Date.now() - 60 * 60 * 1000); + await settleFirstMonthAtReady({ + id: await makeVideo(appId), + deletedAt: null, + }); + await settleFirstMonthAtReady({ + id: await makeVideo(appId, { + duration: 3600, + width: 3840, + height: 2160, + maxResolutionTier: "2160p", + }), + deletedAt: null, + }); + await billDay(accountId, "2026-11-13"); + await database() + .update(developerCreditTransactions) + .set({ amountMicroCredits: -28 }) + .where( + and( + eq(developerCreditTransactions.accountId, accountId), + eq(developerCreditTransactions.type, "storage_daily"), + ), + ); + + expect(await getInfraStorageChargesSince(accountId, monthStart)).toBe( + 850 + 16_800 + 28, + ); + expect( + await getInfraFirstMonthChargesSince(accountId, monthStart), + ).toEqual({ chargedMicroCredits: 850 + 16_800, videos: 2 }); + }); + + it("reports what is stored apart from what the daily charge covers", async () => { + const { appId } = await makeApp(1_000_000); + await makeVideo(appId, { + storagePaidUntil: new Date(Date.now() + 10 * DAY_MS), + }); + await makeVideo(appId, { + storagePaidUntil: new Date(Date.now() - DAY_MS), + duration: 1200, + }); + await makeVideo(appId, { createdAt: new Date("2026-10-01T00:00:00Z") }); + await makeVideo(appId, { status: "processing", duration: 300 }); + await makeVideo(appId, { status: "uploading", duration: null }); + await makeVideo(appId, { status: "errored" }); + + const storage = await getAppStorage(appId); + + expect(storage.stored).toMatchObject({ + videoCount: 3, + totalMinutes: 40, + dailyMicroCredits: Math.floor((40 * 85) / 30), + }); + expect(storage.dailyCharge).toMatchObject({ + videoCount: 2, + totalMinutes: 30, + monthlyMicroCredits: 30 * 85, + dailyMicroCredits: Math.floor((30 * 85) / 30), + }); + expect(storage.firstMonthMicroCredits).toBe(5 * 85); + expect(storage.uploadingVideos).toBe(1); + + const billing = await getInfraBilling(appId); + expect(billing.stored).toEqual(storage.stored); + expect(billing.admission.requiredMicroCredits).toBe( + Math.ceil(5 * 85 + (30 * 85 * 7) / 30), + ); + }); + + describe("a top-up and a short storage charge", () => { + const SECOND = new Date("2026-11-13T09:00:00.000Z"); + const LATER = new Date("2026-11-13T09:00:01.000Z"); + type LedgerInsert = typeof developerCreditTransactions.$inferInsert; + const row = ( + accountId: string, + type: LedgerInsert["type"], + amountMicroCredits: number, + balanceAfterMicroCredits: number, + metadata: Record, + createdAt = SECOND, + ): LedgerInsert & { id: string } => ({ + id: id(), + accountId, + type, + amountMicroCredits, + balanceAfterMicroCredits, + referenceId: id(), + referenceType: + type === "topup" ? ("stripe_payment_intent" as const) : null, + metadata, + createdAt, + }); + const shortCharge = (accountId: string, balanceAfter: number) => + row(accountId, "storage_daily", 0, balanceAfter, { + owedMicroCredits: 50, + shortSince: SECOND.toISOString(), + }); + const restored = (accountId: string, chargeId: string) => + restoredSinceCharge(database() as never, accountId, { + id: chargeId, + createdAt: SECOND, + owedMicroCredits: 50, + shortSince: SECOND, + }); + + it("never counts a top-up written in the same second as the charge, before or after it", async () => { + const before = await makeApp(0); + const earlier = row(before.accountId, "topup", 100, 100, { + availableAfterMicroCredits: 100, + }); + const chargeAfterTopUp = shortCharge(before.accountId, 100); + const spend = row(before.accountId, "transcription", -100, 0, {}); + const after = await makeApp(0); + const chargeFirst = shortCharge(after.accountId, 0); + const sameSecond = row(after.accountId, "topup", 100, 100, { + availableAfterMicroCredits: 100, + }); + await database() + .insert(developerCreditTransactions) + .values([earlier, chargeAfterTopUp, spend, chargeFirst, sameSecond]); + + expect(await restored(before.accountId, chargeAfterTopUp.id)).toBe( + false, + ); + expect(await restored(after.accountId, chargeFirst.id)).toBe(false); + }); + + it("counts a top-up in a later second that leaves enough available", async () => { + const { accountId } = await makeApp(0); + const charge = shortCharge(accountId, 0); + const topUp = row( + accountId, + "topup", + 100, + 100, + { availableAfterMicroCredits: 100 }, + LATER, + ); + await database() + .insert(developerCreditTransactions) + .values([charge, topUp]); + + expect(await restored(accountId, charge.id)).toBe(true); + }); + + it("does not count a later top-up whose credit is held by transcripts", async () => { + const { accountId } = await makeApp(0); + const charge = shortCharge(accountId, 0); + const topUp = row( + accountId, + "topup", + 100, + 100, + { availableAfterMicroCredits: 40 }, + LATER, + ); + await database() + .insert(developerCreditTransactions) + .values([charge, topUp]); + + expect(await restored(accountId, charge.id)).toBe(false); + }); + + it("compares an older top-up without a recorded available amount on its balance", async () => { + const { accountId } = await makeApp(0); + const charge = shortCharge(accountId, 0); + const topUp = row(accountId, "topup", 100, 100, {}, LATER); + await database() + .insert(developerCreditTransactions) + .values([charge, topUp]); + + expect(await restored(accountId, charge.id)).toBe(true); + }); + }); + + describe("when first month charging began", () => { + const FIRST_CHARGE = new Date("2020-01-10T10:39:54.000Z"); + const before = new Date(FIRST_CHARGE.getTime() - 1000); + const after = new Date(FIRST_CHARGE.getTime() + 1000); + const created: string[] = []; + + const track = async ( + appId: string, + overrides: Partial, + ) => { + const videoId = await makeVideo(appId, overrides); + created.push(videoId); + return videoId; + }; + + const charged = async ( + paidUntil: Date | null, + chargedAt: Date, + { uploadFee = false }: { uploadFee?: boolean } = {}, + ) => { + const { appId, accountId } = await makeApp(0); + const videoId = await track(appId, { + createdAt: new Date(chargedAt.getTime() - 60_000), + storagePaidUntil: paidUntil, + }); + await database() + .insert(developerCreditTransactions) + .values({ + id: id(), + accountId, + type: "video_create", + amountMicroCredits: -850, + balanceAfterMicroCredits: 0, + referenceId: videoId, + referenceType: "developer_video", + metadata: + uploadFee || !paidUntil + ? { durationSeconds: 600 } + : { paidUntil: paidUntil.toISOString() }, + createdAt: chargedAt, + }); + return videoId; + }; + + const encoding = (): DeveloperVideoProcessingState => ({ + ...processing(), + probe: null, + mp4: { + state: "running", + token: "t-mp4", + jobId: "job-1", + startedAt: Date.now(), + }, + attempts: { hls: 0, mp4: 1 }, + }); + + afterEach(async () => { + if (created.length === 0) return; + await database() + .delete(developerCreditTransactions) + .where(inArray(developerCreditTransactions.referenceId, created)); + await database() + .delete(developerVideos) + .where(inArray(developerVideos.id, created)); + created.length = 0; + }); + + it("starts at the exact time of the first real charge, not at a day boundary", async () => { + await charged(new Date("2020-02-09T00:00:00Z"), FIRST_CHARGE); + await charged( + new Date("2020-02-09T00:00:00Z"), + new Date("2020-01-10T15:00:00Z"), + ); + await charged( + new Date("2020-02-10T00:00:00Z"), + new Date("2020-01-11T09:00:00Z"), + ); + + expect(await firstMonthChargesBegan()).toEqual(FIRST_CHARGE); + }); + + it("starts at the same moment when the first charge came after that day was billed", async () => { + await charged(new Date("2020-02-10T00:00:00Z"), FIRST_CHARGE); + await charged( + new Date("2020-02-09T00:00:00Z"), + new Date("2020-01-10T12:00:00Z"), + ); + await charged( + new Date("2020-02-10T00:00:00Z"), + new Date("2020-01-11T03:00:00Z"), + ); + + expect(await firstMonthChargesBegan()).toEqual(FIRST_CHARGE); + }); + + it("is not moved by older charge rows of unsettled videos, upload fee rows of settled videos, or videos settled without a charge", async () => { + await charged(new Date("2020-02-09T00:00:00Z"), FIRST_CHARGE); + await charged(null, new Date("2019-12-01T04:23:00Z")); + await charged( + new Date("2020-02-09T00:00:00Z"), + new Date("2019-12-01T04:58:00Z"), + { uploadFee: true }, + ); + const { appId } = await makeApp(0); + await track(appId, { + createdAt: new Date("2019-12-01T00:00:00Z"), + storagePaidUntil: new Date("2019-12-31T00:00:00Z"), + }); + + expect(await firstMonthChargesBegan()).toEqual(FIRST_CHARGE); + }); + + it("never charges a video created a second before the first charge, and charges one created at or after it exactly once", async () => { + await charged(new Date("2020-02-09T00:00:00Z"), FIRST_CHARGE); + const appBefore = await makeApp(1_000_000); + const videoBefore = await track(appBefore.appId, { + createdAt: before, + }); + const appAt = await makeApp(1_000_000); + const videoAt = await track(appAt.appId, { createdAt: FIRST_CHARGE }); + + expect( + (await getAppStorage(appBefore.appId)).firstMonthMicroCredits, + ).toBe(0); + expect((await getAppStorage(appAt.appId)).firstMonthMicroCredits).toBe( + 850, + ); + + await chargeMissedFirstMonths(new Date(), 1); + await chargeMissedFirstMonths(new Date(), 1); + + expect(await firstMonthRows(appBefore.accountId)).toHaveLength(0); + expect((await videoRow(videoBefore))?.storagePaidUntil).toBeNull(); + expect(await balanceOf(appBefore.accountId)).toBe(1_000_000); + const rows = await firstMonthRows(appAt.accountId); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ referenceId: videoAt }); + expect((await videoRow(videoAt))?.storagePaidUntil).not.toBeNull(); + expect((await getAppStorage(appAt.appId)).firstMonthMicroCredits).toBe( + 0, + ); + }); + + it("never charges a video deleted mid-encode that was created a second before the first charge, and charges one created at or after it once", async () => { + await charged(new Date("2020-02-09T00:00:00Z"), FIRST_CHARGE); + const encodeAt = async (createdAt: Date) => { + const app = await makeApp(1_000_000); + const videoId = await track(app.appId, { + createdAt, + status: "processing", + duration: 600, + processing: encoding(), + }); + await database() + .update(developerVideos) + .set({ deletedAt: new Date() }) + .where(eq(developerVideos.id, videoId)); + return { ...app, videoId }; + }; + const early = await encodeAt(before); + const atStart = await encodeAt(FIRST_CHARGE); + const later = await encodeAt(after); + + expect(await settleFirstMonthAtDelete(early.videoId)).toMatchObject({ + state: "exempt", + }); + expect(await firstMonthRows(early.accountId)).toHaveLength(0); + for (const video of [atStart, later]) { + expect(await settleFirstMonthAtDelete(video.videoId)).toMatchObject({ + state: "charged", + }); + expect(await settleFirstMonthAtDelete(video.videoId)).toEqual({ + state: "skipped", + reason: "already_settled", + }); + expect(await firstMonthRows(video.accountId)).toHaveLength(1); + } + }); + }); + }, +); diff --git a/apps/web/__tests__/integration/infra-upload-round-trips-database.test.ts b/apps/web/__tests__/integration/infra-upload-round-trips-database.test.ts new file mode 100644 index 00000000000..e793c2f797a --- /dev/null +++ b/apps/web/__tests__/integration/infra-upload-round-trips-database.test.ts @@ -0,0 +1,454 @@ +import { randomUUID } from "node:crypto"; +import { + developerApiKeys, + developerApps, + developerCreditAccounts, + developerCreditTransactions, + developerVideos, +} from "@cap/database/schema"; +import type { User } from "@cap/web-domain"; +import { eq } from "drizzle-orm"; +import { drizzle, type MySql2Database } from "drizzle-orm/mysql2"; +import { Effect } from "effect"; +import { createPool, type Pool, type PoolConnection } from "mysql2/promise"; +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + it, + vi, +} from "vitest"; + +// Counts the database statements and storage calls one upload completion and +// one kick start make, against a real MySQL. Storage, the media server and the +// workflow runtime are stand-ins that count their calls. Opt in with +// CAP_INFRA_ROUND_TRIPS_DATABASE_URL; rows are created under fresh ids and +// removed afterwards. + +const fixture = vi.hoisted(() => ({ + database: undefined as unknown, + statements: [] as string[], + timed: [] as { start: number; end: number }[], + statementDelayMs: 0, + storage: [] as string[], + storageDelayMs: 0, + kicks: [] as Promise[], + kickGate: Promise.resolve() as Promise, + releaseKick: () => {}, + objectBytes: 64 * 1024 * 1024, +})); + +const storageCall = async (name: string, value: T) => { + fixture.storage.push(name); + if (fixture.storageDelayMs > 0) + await new Promise((resolve) => setTimeout(resolve, fixture.storageDelayMs)); + return value; +}; + +vi.mock("@cap/database", () => ({ + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true", NEXT_PUBLIC_WEB_URL: "https://web" }, + serverEnv: () => ({ + WEB_URL: "https://web", + NEXTAUTH_SECRET: "round-trips-secret", + }), +})); +vi.mock("@cap/web-backend", () => ({ + provideOptionalAuth: (effect: T) => effect, + S3Buckets: { + getBucketAccess: () => + Effect.succeed([ + { + multipart: { + listParts: () => + Effect.promise(() => + storageCall("ListParts", [ + { PartNumber: 1, ETag: '"e1"', Size: fixture.objectBytes }, + ]), + ), + complete: () => + Effect.promise(() => storageCall("CompleteMultipartUpload", {})), + }, + headObject: () => + Effect.promise(() => + storageCall("HeadObject", { ContentLength: fixture.objectBytes }), + ), + deleteObject: () => + Effect.promise(() => storageCall("DeleteObject", {})), + }, + ]), + }, +})); +vi.mock("@/lib/server", () => ({ + runPromise: (effect: Effect.Effect) => + Effect.runPromise(effect), +})); +vi.mock("@/lib/developer-video-storage", () => ({ + headDeveloperVideoObject: (key: string) => + storageCall(`HEAD ${key.split("/").slice(3).join("/")}`, { + bytes: fixture.objectBytes, + contentType: "video/mp4", + }), + internalDeveloperVideoUrl: async (key: string) => `https://storage/${key}`, + developerVideoUploadUrl: async (key: string) => `https://storage/${key}`, + readDeveloperVideoText: (key: string) => + storageCall(`GET ${key.split("/").slice(3).join("/")}`, null), + deleteDeveloperVideoObjects: async () => 0, + developerVideoPrefix: () => "prefix/", +})); +vi.mock("@/lib/media-container", () => ({ + sniffMediaUrl: () => storageCall("ranged GET (sniff)", "media"), +})); +vi.mock("@/lib/media-client", () => ({ + MEDIA_SERVER_SHUTDOWN_MESSAGE: "shutdown", + tryProbeVideoViaMediaServer: () => + storageCall("media server probe", { + status: "ok", + metadata: { width: 1920, height: 1080, duration: 60, fps: 30 }, + }), + startMediaServerVideoJob: (path: string) => + storageCall(`media server ${path}`, { + status: "started", + jobId: `job-${path}`, + }), + readMediaServerVideoJob: async () => ({ status: "unreachable" }), +})); +vi.mock("next/server", () => ({ + after: (promise: Promise) => { + fixture.kicks.push(promise); + }, +})); +// The kick starts while completion is still running, so it waits here until +// the completion's statements have been counted. +vi.mock("@/lib/developer-video-processing", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + prepareDeveloperVideoProcessing: async ( + ...args: Parameters + ) => { + await fixture.kickGate; + return actual.prepareDeveloperVideoProcessing(...args); + }, + }; +}); +vi.mock("workflow/api", () => ({ start: vi.fn(async () => ({})) })); +vi.mock("@/workflows/process-developer-video", () => ({ + processDeveloperVideoWorkflow: "workflow", +})); + +const databaseUrl = process.env.CAP_INFRA_ROUND_TRIPS_DATABASE_URL; +const id = () => randomUUID().replaceAll("-", "").slice(0, 13); +const database = () => fixture.database as MySql2Database; + +const statementText = (query: unknown) => + (typeof query === "string" + ? query + : String((query as { sql?: string })?.sql ?? query) + ) + .replace(/\s+/g, " ") + .trim(); + +function countStatements(target: T): T { + for (const method of ["query", "execute"] as const) { + const original = target[method].bind(target) as ( + ...args: unknown[] + ) => unknown; + Object.defineProperty(target, method, { + configurable: true, + value: async (...args: unknown[]) => { + fixture.statements.push(statementText(args[0])); + const start = performance.now(); + if (fixture.statementDelayMs > 0) { + await new Promise((resolve) => + setTimeout(resolve, fixture.statementDelayMs), + ); + } + try { + return await original(...args); + } finally { + fixture.timed.push({ start, end: performance.now() }); + } + }, + }); + } + return target; +} + +function summary(statements: readonly string[]) { + let lockedFrom = -1; + let lockHeld = 0; + statements.forEach((statement, index) => { + if ( + /developer_credit_accounts/.test(statement) && + /for update/i.test(statement) + ) { + lockedFrom = index; + } + if (/^commit/i.test(statement) && lockedFrom >= 0) { + lockHeld += index - lockedFrom; + lockedFrom = -1; + } + }); + return { + roundTrips: statements.length, + transactions: statements.filter((s) => /^begin/i.test(s)).length, + statementsUnderAccountLock: lockHeld, + }; +} + +// The longest run of statements where each starts after the previous one +// ended: the round trips a request waits for one after another. +function sequentialDepth(timed: readonly { start: number; end: number }[]) { + let depth = 0; + let lastEnd = Number.NEGATIVE_INFINITY; + for (const { start, end } of [...timed].sort((a, b) => a.end - b.end)) { + if (start >= lastEnd) { + depth++; + lastEnd = end; + } + } + return depth; +} + +const statementList = () => + process.env.CAP_INFRA_ROUND_TRIPS_STATEMENTS + ? { statements: fixture.statements.map((s) => s.slice(0, 110)) } + : {}; + +describe.runIf(Boolean(databaseUrl))( + "Cap Infra upload round trips against a real database", + () => { + let pool: Pool; + const appId = `rt${id()}`; + const ownerId = `rt${id()}` as User.UserId; + const videoIds: string[] = []; + + beforeAll(async () => { + pool = createPool({ uri: databaseUrl, connectionLimit: 4 }); + countStatements(pool); + const getConnection = pool.getConnection.bind(pool); + Object.defineProperty(pool, "getConnection", { + configurable: true, + value: async () => countStatements(await getConnection()), + }); + fixture.database = drizzle(pool); + await database().insert(developerApps).values({ + id: appId, + ownerId, + name: "round trips", + environment: "development", + }); + await database() + .insert(developerCreditAccounts) + .values({ + id: `rt${id()}`, + appId, + ownerId, + balanceMicroCredits: 1_000_000_000, + }); + }); + + afterEach(async () => { + const { clearFirstMonthChargesBeganCache } = await import( + "@/lib/infra-first-month-start" + ); + clearFirstMonthChargesBeganCache(); + }); + + afterAll(async () => { + if (!pool) return; + for (const videoId of videoIds) { + await database() + .delete(developerVideos) + .where(eq(developerVideos.id, videoId)); + } + await database() + .delete(developerApiKeys) + .where(eq(developerApiKeys.appId, appId)); + for (const videoId of videoIds) { + await database() + .delete(developerCreditTransactions) + .where(eq(developerCreditTransactions.referenceId, videoId)); + } + await database() + .delete(developerVideos) + .where(eq(developerVideos.appId, appId)); + await database() + .delete(developerCreditAccounts) + .where(eq(developerCreditAccounts.appId, appId)); + await database().delete(developerApps).where(eq(developerApps.id, appId)); + await pool.end(); + }); + + it("counts one completion, its kick start and the webhook settle", async () => { + const { completeInfraUpload } = await import("@/lib/infra-video-upload"); + const { applyDeveloperVideoProgress } = await import( + "@/lib/developer-video-processing" + ); + const videoId = `rt${id()}`; + videoIds.push(videoId); + await database() + .insert(developerVideos) + .values({ + id: videoId, + appId, + s3Key: `developer/${appId}/${videoId}/video`, + playbackId: `rt${id()}`, + }); + + fixture.kickGate = new Promise((resolve) => { + fixture.releaseKick = resolve; + }); + fixture.statements.length = 0; + fixture.storage.length = 0; + const completed = await completeInfraUpload({ + appId, + videoId, + uploadId: "upload", + parts: [{ partNumber: 1, etag: "e1", size: fixture.objectBytes }], + durationInSecs: 60, + width: 1920, + height: 1080, + }); + expect(completed.status).toBe(200); + const complete = { + ...summary(fixture.statements), + storage: [...fixture.storage], + ...statementList(), + }; + + fixture.statements.length = 0; + fixture.storage.length = 0; + expect(fixture.kicks).toHaveLength(1); + fixture.releaseKick(); + await Promise.all(fixture.kicks.splice(0)); + const kick = { + ...summary(fixture.statements), + storage: [...fixture.storage], + ...statementList(), + }; + + const [row] = await database() + .select({ processing: developerVideos.processing }) + .from(developerVideos) + .where(eq(developerVideos.id, videoId)); + const mp4 = row?.processing?.mp4; + expect(mp4?.state).toBe("running"); + expect(row?.processing?.hls.state).toBe("running"); + + fixture.statements.length = 0; + fixture.storage.length = 0; + fixture.storageDelayMs = 50; + const settleStarted = performance.now(); + const settled = await applyDeveloperVideoProgress( + { videoId, jobId: "job", phase: "complete" }, + "mp4", + mp4?.state === "running" ? mp4.token : null, + ); + const settleMs = Math.round(performance.now() - settleStarted); + fixture.storageDelayMs = 0; + expect(settled.applied).toBe(true); + const settle = { + ...summary(fixture.statements), + storage: [...fixture.storage], + ...statementList(), + wallMsWith50msStorage: settleMs, + }; + + console.log( + JSON.stringify({ complete, kick, settleMp4: settle }, null, 2), + ); + }); + it("creates an upload in 7 statements, 5 of them one after another", async () => { + const { hashKey } = await import("@/lib/developer-key-hash"); + const { POST } = await import("@/app/api/developer/v1/[...route]/route"); + const secret = `csk_rt${id()}`; + await database() + .insert(developerApiKeys) + .values({ + id: `rt${id()}`, + appId, + keyType: "secret", + keyHash: await hashKey(secret), + keyPrefix: secret.slice(0, 12), + encryptedKey: "unused", + }); + const [accountRow] = await database() + .select({ id: developerCreditAccounts.id }) + .from(developerCreditAccounts) + .where(eq(developerCreditAccounts.appId, appId)); + const settledId = `rt${id()}`; + videoIds.push(settledId); + const chargedAt = new Date("2026-11-13T10:39:54Z"); + const paidUntil = new Date("2026-12-13T00:00:00Z"); + await database().insert(developerVideos).values({ + id: settledId, + appId, + status: "ready", + duration: 60, + width: 1920, + height: 1080, + storagePaidUntil: paidUntil, + createdAt: chargedAt, + }); + await database() + .insert(developerCreditTransactions) + .values({ + id: `rt${id()}`, + accountId: accountRow?.id ?? "", + type: "video_create", + amountMicroCredits: -85, + balanceAfterMicroCredits: 1_000_000_000, + referenceId: settledId, + referenceType: "developer_video", + metadata: { paidUntil: paidUntil.toISOString() }, + createdAt: chargedAt, + }); + + const create = async () => { + const response = await POST( + new Request("https://web/api/developer/v1/uploads", { + method: "POST", + headers: { + authorization: `Bearer ${secret}`, + "content-type": "application/json", + }, + body: JSON.stringify({ name: "round trips" }), + }), + ); + expect(response.status).toBe(200); + const { data } = (await response.json()) as { + data: { videoId: string }; + }; + videoIds.push(data.videoId); + }; + // The first create looks up the key's hash key, the first month start + // and the key's last use; none of those repeat. + await create(); + await new Promise((resolve) => setTimeout(resolve, 50)); + + fixture.statements.length = 0; + fixture.timed.length = 0; + fixture.statementDelayMs = 10; + await create(); + fixture.statementDelayMs = 0; + const counted = { + roundTrips: fixture.statements.length, + sequential: sequentialDepth(fixture.timed), + ...statementList(), + }; + console.log(JSON.stringify({ create: counted }, null, 2)); + expect(counted.roundTrips).toBe(7); + expect(counted.sequential).toBe(5); + }); + }, +); diff --git a/apps/web/__tests__/integration/infra-video-import-database.test.ts b/apps/web/__tests__/integration/infra-video-import-database.test.ts new file mode 100644 index 00000000000..961cbf2f2de --- /dev/null +++ b/apps/web/__tests__/integration/infra-video-import-database.test.ts @@ -0,0 +1,854 @@ +import { randomUUID } from "node:crypto"; +import { + developerApps, + developerVideoImports, + developerVideos, +} from "@cap/database/schema"; +import { eq } from "drizzle-orm"; +import { drizzle, type MySql2Database } from "drizzle-orm/mysql2"; +import { createPool, type Pool } from "mysql2/promise"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +const fixture = vi.hoisted(() => ({ + database: undefined as unknown, + admission: { allowed: true } as Record, + createStatus: 200 as 200 | 402, + completeResult: { status: 200, body: { success: true } } as { + status: number; + body: Record; + }, +})); + +const mocks = vi.hoisted(() => ({ + admit: vi.fn(), + createInfraVideo: vi.fn(), + replay: vi.fn(), + initiate: vi.fn(), + presign: vi.fn(), + complete: vi.fn(), + abort: vi.fn(), + announce: vi.fn(), + topUp: vi.fn(), +})); + +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + NODE_ENV: "production", + serverEnv: () => ({}), +})); +vi.mock("@cap/database", () => ({ + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@/lib/developer-video-charge", () => ({ + admitDeveloperVideoUpload: mocks.admit, +})); +vi.mock("@/lib/infra-standing", () => ({ + uploadAdmissionError: ( + admission: Record, + appId: string, + ) => ({ + error: "Insufficient credits", + code: "insufficient_credits", + message: admission.message ?? "Insufficient credits", + hint: `billing for ${appId}`, + }), +})); +vi.mock("@/lib/infra-auto-top-up", () => ({ scheduleAutoTopUp: mocks.topUp })); +vi.mock("@/lib/developer-webhook-dispatch", () => ({ + announceDeveloperVideoEvent: mocks.announce, +})); +vi.mock("@/lib/infra-video-upload", () => ({ + createInfraVideo: mocks.createInfraVideo, + replayKeyedInfraVideo: mocks.replay, + initiateInfraUpload: mocks.initiate, + presignInfraUploadParts: mocks.presign, + completeInfraUpload: mocks.complete, + abortInfraUpload: mocks.abort, +})); + +const { + createInfraVideoImport, + IMPORT_MAX_PER_KEY_PER_HOUR, + IMPORT_STALE_SLOT_MS, + runInfraVideoImport, +} = await import("@/lib/infra-video-import"); + +const databaseUrl = process.env.CAP_INFRA_TEST_DATABASE_URL; +const missingTableUrl = process.env.CAP_INFRA_IMPORT_MISSING_TABLE_DATABASE_URL; +const id = () => randomUUID().replaceAll("-", "").slice(0, 15); +const database = () => fixture.database as MySql2Database; + +const MP4_HEAD = new Uint8Array([ + 0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x69, 0x73, 0x6f, 0x6d, 0x00, + 0x00, 0x02, 0x00, +]); + +const mp4Bytes = (length: number) => { + const bytes = new Uint8Array(length); + bytes.set(MP4_HEAD); + return bytes; +}; + +const serving = (bytes: Uint8Array, declared = true) => ({ + isBlocked: async () => false, + request: vi.fn( + async () => + new Response( + new ReadableStream({ + start(controller) { + for (let offset = 0; offset < bytes.length; offset += 7) { + controller.enqueue(bytes.slice(offset, offset + 7)); + } + controller.close(); + }, + }), + { + status: 200, + headers: { + "content-type": "video/mp4", + ...(declared ? { "content-length": String(bytes.length) } : {}), + }, + }, + ), + ), +}); + +const assertLocal = (value: string | undefined, pattern: RegExp) => { + if (!value) throw new Error("Missing isolated test database URL."); + const url = new URL(value); + if ( + url.protocol !== "mysql:" || + !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) || + !pattern.test(url.pathname) + ) { + throw new Error("These tests require a session-scoped local database."); + } + return value; +}; + +async function seedApp() { + const appId = id(); + await database() + .insert(developerApps) + .values({ + id: appId, + ownerId: id() as never, + name: "Import fixture", + environment: "development", + }); + return appId; +} + +async function importRow(videoId: string) { + const [row] = await database() + .select() + .from(developerVideoImports) + .where(eq(developerVideoImports.videoId, videoId)); + return row ?? null; +} + +async function videoRow(videoId: string) { + const [row] = await database() + .select() + .from(developerVideos) + .where(eq(developerVideos.id, videoId)); + return row ?? null; +} + +const startImport = vi.fn(async (_videoId: string) => {}); + +const request = ( + appId: string, + keyId: string, + overrides: Partial[0]> = {}, +) => ({ + appId, + keyId, + url: "https://media.example.com/video.mp4", + videoId: id(), + ...overrides, +}); + +describe.runIf(Boolean(databaseUrl))( + "video import against a real database", + () => { + let pool: Pool; + + beforeAll(async () => { + const url = assertLocal(databaseUrl, /^\/cap_infra_[a-z0-9_]+$/); + pool = createPool({ uri: url, connectionLimit: 8 }); + fixture.database = drizzle(pool); + }); + + afterAll(async () => { + await pool?.end(); + }); + + beforeEach(() => { + vi.clearAllMocks(); + fixture.admission = { allowed: true }; + fixture.createStatus = 200; + fixture.completeResult = { status: 200, body: { success: true } }; + mocks.admit.mockImplementation(async () => fixture.admission); + mocks.createInfraVideo.mockImplementation( + async (input: { appId: string; videoId: string; name?: string }) => { + if (fixture.createStatus !== 200) { + return { + status: fixture.createStatus, + body: { + code: "insufficient_credits", + error: "Insufficient credits", + }, + }; + } + await database() + .insert(developerVideos) + .values({ + id: input.videoId, + appId: input.appId, + name: input.name ?? "Untitled", + s3Key: `developer/${input.appId}/${input.videoId}/video`, + }) + .onDuplicateKeyUpdate({ set: { name: input.name ?? "Untitled" } }); + return { status: 200, body: { videoId: input.videoId } }; + }, + ); + mocks.replay.mockImplementation(async () => ({ + status: 200, + body: { replayed: true }, + })); + mocks.initiate.mockImplementation(async () => ({ + status: 200, + body: { uploadId: "upload-1" }, + })); + mocks.complete.mockImplementation(async () => fixture.completeResult); + mocks.abort.mockImplementation(async () => ({ + status: 200, + body: { success: true }, + })); + }); + + describe("per-app concurrency", () => { + it("admits two imports at once and refuses a third, even when sent together", async () => { + const appId = await seedApp(); + const results = await Promise.all( + Array.from({ length: 5 }, () => + createInfraVideoImport(request(appId, id()), { + startImport, + guards: serving(mp4Bytes(64)), + }), + ), + ); + const statuses = results.map((result) => result.status).sort(); + expect(statuses).toEqual([202, 202, 429, 429, 429]); + const refused = results.find((result) => result.status === 429); + expect(refused && "body" in refused && refused.body.code).toBe( + "too_many_imports", + ); + expect(refused && "headers" in refused && refused.headers).toEqual({ + "Retry-After": "15", + }); + expect(startImport).toHaveBeenCalledTimes(2); + }); + + it("frees the slot when an import finishes", async () => { + const appId = await seedApp(); + const first = request(appId, id()); + for (const input of [first, request(appId, id())]) { + await createInfraVideoImport(input, { + startImport, + guards: serving(mp4Bytes(64)), + }); + } + await database() + .update(developerVideoImports) + .set({ state: "complete" }) + .where(eq(developerVideoImports.videoId, first.videoId)); + const third = await createInfraVideoImport(request(appId, id()), { + startImport, + guards: serving(mp4Bytes(64)), + }); + expect(third.status).toBe(202); + }); + + it("stops counting a slot that went quiet for longer than the stale window", async () => { + const appId = await seedApp(); + const stale = new Date(Date.now() - IMPORT_STALE_SLOT_MS - 60_000); + for (const videoId of [id(), id()]) { + await database().insert(developerVideoImports).values({ + videoId, + appId, + keyId: id(), + sourceUrl: "https://media.example.com/old.mp4", + startedAt: stale, + createdAt: stale, + }); + } + const fresh = await createInfraVideoImport(request(appId, id()), { + startImport, + guards: serving(mp4Bytes(64)), + }); + expect(fresh.status).toBe(202); + }); + }); + + describe("per-key hourly limit", () => { + it(`refuses the ${IMPORT_MAX_PER_KEY_PER_HOUR + 1}st import in an hour with the wait in Retry-After`, async () => { + const appId = await seedApp(); + const keyId = id(); + const now = Date.now(); + const oldest = new Date(now - 50 * 60_000); + await database() + .insert(developerVideoImports) + .values( + Array.from({ length: IMPORT_MAX_PER_KEY_PER_HOUR }, (_, index) => ({ + videoId: id(), + appId, + keyId, + sourceUrl: "https://media.example.com/a.mp4", + state: "complete" as const, + startedAt: index === 0 ? oldest : new Date(now - 60_000), + createdAt: index === 0 ? oldest : new Date(now - 60_000), + })), + ); + await database() + .insert(developerVideoImports) + .values({ + videoId: id(), + appId, + keyId, + sourceUrl: "https://media.example.com/older.mp4", + state: "complete", + startedAt: new Date(now - 61 * 60_000), + createdAt: new Date(now - 61 * 60_000), + }); + + const refused = await createInfraVideoImport(request(appId, keyId), { + startImport, + guards: serving(mp4Bytes(64)), + }); + expect(refused.status).toBe(429); + if (refused.status !== 429) return; + expect(refused.body.code).toBe("import_rate_limited"); + const wait = Number(refused.headers?.["Retry-After"]); + expect(wait).toBeGreaterThan(9 * 60); + expect(wait).toBeLessThanOrEqual(10 * 60 + 5); + expect(mocks.createInfraVideo).not.toHaveBeenCalled(); + + const otherKey = await createInfraVideoImport(request(appId, id()), { + startImport, + guards: serving(mp4Bytes(64)), + }); + expect(otherKey.status).toBe(202); + }); + + it("lets a key try again once a reservation its request left behind is stale", async () => { + const appId = await seedApp(); + const keyId = id(); + const input = request(appId, keyId, { idempotencyKey: "stuck-key-1" }); + const reserve = async (minutesAgo: number) => { + await database() + .delete(developerVideoImports) + .where(eq(developerVideoImports.videoId, input.videoId)); + const at = new Date(Date.now() - minutesAgo * 60_000); + await database().insert(developerVideoImports).values({ + videoId: input.videoId, + appId, + keyId, + sourceUrl: input.url, + startedAt: at, + createdAt: at, + }); + }; + + await reserve(1); + const recent = await createInfraVideoImport(input, { + startImport, + guards: serving(mp4Bytes(64)), + }); + expect(recent.status).toBe(409); + if (recent.status !== 409) return; + expect(recent.body.code).toBe("import_starting"); + + await reserve(IMPORT_STALE_SLOT_MS / 60_000 + 1); + const reclaimed = await createInfraVideoImport(input, { + startImport, + guards: serving(mp4Bytes(64)), + }); + expect(reclaimed.status).toBe(202); + const row = await importRow(input.videoId); + expect(row?.state).toBe("downloading"); + expect(Date.now() - (row?.createdAt.getTime() ?? 0)).toBeLessThan( + 60_000, + ); + }); + + it("refuses a key reused after a failed attempt, without opening the URL again", async () => { + const appId = await seedApp(); + const keyId = id(); + const dead = { + isBlocked: async () => false, + request: vi.fn(async () => new Response("gone", { status: 404 })), + }; + const input = request(appId, keyId, { idempotencyKey: "retry-key-1" }); + const first = await createInfraVideoImport(input, { + startImport, + guards: dead, + }); + expect(first.status).toBe(422); + for (let attempt = 0; attempt < 3; attempt++) { + const again = await createInfraVideoImport(input, { + startImport, + guards: dead, + }); + expect(again.status).toBe(409); + if (again.status !== 409) return; + expect(again.body).toMatchObject({ + code: "idempotency_conflict", + error: + "This Idempotency-Key belongs to an import that failed (url_unreachable)", + }); + } + expect(dead.request).toHaveBeenCalledTimes(1); + const fresh = await createInfraVideoImport( + request(appId, keyId, { idempotencyKey: "retry-key-2" }), + { startImport, guards: serving(mp4Bytes(64)) }, + ); + expect(fresh.status).toBe(202); + }); + + it("counts a refused preflight against the hour", async () => { + const appId = await seedApp(); + const keyId = id(); + const dead = { + isBlocked: async () => false, + request: async () => new Response("gone", { status: 404 }), + }; + const input = request(appId, keyId); + const refused = await createInfraVideoImport(input, { + startImport, + guards: dead, + }); + expect(refused.status).toBe(422); + if (refused.status !== 422) return; + expect(refused.body).toMatchObject({ + code: "url_unreachable", + error: "The URL answered 404", + }); + expect(Object.keys(refused.body).sort()).toEqual([ + "code", + "docs", + "error", + "hint", + "message", + ]); + expect(await importRow(input.videoId)).toMatchObject({ + state: "failed", + errorCode: "url_unreachable", + keyId, + }); + expect(await videoRow(input.videoId)).toBeNull(); + }); + }); + + describe("credit admission", () => { + it("admits on the declared size before creating anything, and refuses with 402", async () => { + const appId = await seedApp(); + fixture.admission = { allowed: false, message: "Add credit" }; + const input = request(appId, id(), { maxResolutionTier: "2160p" }); + const result = await createInfraVideoImport(input, { + startImport, + guards: serving(mp4Bytes(5_000_000)), + }); + expect(result.status).toBe(402); + expect(result).toMatchObject({ + body: { hint: `billing for ${appId}` }, + }); + expect(mocks.admit).toHaveBeenCalledWith({ + appId, + videoId: input.videoId, + durationSeconds: 2, + width: null, + height: null, + maxResolutionTier: "2160p", + }); + expect(mocks.topUp).toHaveBeenCalledWith({ appId }); + expect(mocks.createInfraVideo).not.toHaveBeenCalled(); + expect(startImport).not.toHaveBeenCalled(); + expect(await importRow(input.videoId)).toMatchObject({ + state: "failed", + errorCode: "insufficient_credits", + }); + expect(await videoRow(input.videoId)).toBeNull(); + }); + + it("releases the slot when the video itself is refused for credit", async () => { + const appId = await seedApp(); + fixture.createStatus = 402; + const input = request(appId, id()); + const result = await createInfraVideoImport(input, { + startImport, + guards: serving(mp4Bytes(64), false), + }); + expect(result.status).toBe(402); + expect(mocks.admit).not.toHaveBeenCalled(); + expect((await importRow(input.videoId))?.state).toBe("failed"); + }); + }); + + describe("idempotency", () => { + it("replays the same key and URL without a second download, and refuses another URL", async () => { + const appId = await seedApp(); + const keyId = id(); + const input = request(appId, keyId, { + idempotencyKey: "same-key", + videoId: id(), + }); + const first = await createInfraVideoImport(input, { + startImport, + guards: serving(mp4Bytes(64)), + }); + const again = await createInfraVideoImport(input, { + startImport, + guards: serving(mp4Bytes(64)), + }); + expect(first).toEqual({ + status: 202, + videoId: input.videoId, + replayed: false, + }); + expect(again).toEqual({ + status: 202, + videoId: input.videoId, + replayed: true, + }); + expect(startImport).toHaveBeenCalledTimes(1); + + fixture.admission = { allowed: false }; + fixture.createStatus = 402; + mocks.replay.mockResolvedValueOnce({ + status: 409, + body: { code: "upload_already_completed" }, + }); + const afterRecorded = await createInfraVideoImport(input, { + startImport, + guards: serving(mp4Bytes(64)), + }); + expect(afterRecorded).toEqual({ + status: 202, + videoId: input.videoId, + replayed: true, + }); + mocks.replay.mockResolvedValueOnce({ + status: 409, + body: { code: "idempotency_conflict" }, + }); + const changedName = await createInfraVideoImport( + { ...input, name: "Another name" }, + { startImport, guards: serving(mp4Bytes(64)) }, + ); + expect(changedName.status).toBe(409); + expect(startImport).toHaveBeenCalledTimes(1); + expect(mocks.createInfraVideo).toHaveBeenCalledTimes(1); + expect(mocks.admit).toHaveBeenCalledTimes(1); + + const other = await createInfraVideoImport( + { ...input, url: "https://media.example.com/other.mp4" }, + { startImport, guards: serving(mp4Bytes(64)) }, + ); + expect(other.status).toBe(409); + }); + }); + + describe("the download step", () => { + async function seedImport(appId: string, startedAt = new Date()) { + const videoId = id(); + await database() + .insert(developerVideos) + .values({ + id: videoId, + appId, + name: "Import", + s3Key: `developer/${appId}/${videoId}/video`, + }); + await database().insert(developerVideoImports).values({ + videoId, + appId, + keyId: id(), + sourceUrl: "https://media.example.com/video.mp4", + startedAt, + createdAt: startedAt, + }); + return videoId; + } + + const recordingWriter = () => { + const parts: { partNumber: number; size: number }[] = []; + return { + parts, + writer: () => ({ + put: async (partNumber: number, bytes: Uint8Array) => { + parts.push({ partNumber, size: bytes.length }); + return `etag-${partNumber}`; + }, + }), + }; + }; + + it("streams the file into parts and completes it like an upload", async () => { + const appId = await seedApp(); + const videoId = await seedImport(appId); + const { parts, writer } = recordingWriter(); + const outcome = await runInfraVideoImport(videoId, { + guards: serving(mp4Bytes(40)), + writer, + partBytes: 16, + }); + expect(outcome).toBe("complete"); + expect(parts).toEqual([ + { partNumber: 1, size: 16 }, + { partNumber: 2, size: 16 }, + { partNumber: 3, size: 8 }, + ]); + expect(mocks.initiate).toHaveBeenCalledWith({ + appId, + videoId, + contentType: "video/mp4", + contentLength: 40, + }); + expect(mocks.complete).toHaveBeenCalledWith({ + appId, + videoId, + uploadId: "upload-1", + parts: [ + { partNumber: 1, etag: "etag-1", size: 16 }, + { partNumber: 2, etag: "etag-2", size: 16 }, + { partNumber: 3, etag: "etag-3", size: 8 }, + ], + durationInSecs: 0, + }); + expect(await importRow(videoId)).toMatchObject({ + state: "complete", + bytes: 40, + }); + expect(mocks.abort).not.toHaveBeenCalled(); + }); + + it("stops a file that grows past the cap mid-stream and fails the video", async () => { + const appId = await seedApp(); + const videoId = await seedImport(appId); + const { writer } = recordingWriter(); + const outcome = await runInfraVideoImport(videoId, { + guards: { ...serving(mp4Bytes(100), false), maxBytes: 60 }, + writer, + partBytes: 16, + }); + expect(outcome).toBe("failed"); + expect(mocks.abort).toHaveBeenCalledWith({ + appId, + videoId, + uploadId: "upload-1", + }); + expect(mocks.complete).not.toHaveBeenCalled(); + expect(await importRow(videoId)).toMatchObject({ + state: "failed", + errorCode: "too_large", + }); + const video = await videoRow(videoId); + expect(video?.status).toBe("errored"); + expect(video?.errorReason).toMatch( + /^Import failed: The file at the URL is larger than/, + ); + expect(mocks.announce).toHaveBeenCalledWith( + { id: videoId, appId }, + { type: "video.errored" }, + ); + }); + + it("asks the balance again as an undeclared file grows, and stops when it runs out", async () => { + const appId = await seedApp(); + const videoId = await seedImport(appId); + let checks = 0; + mocks.admit.mockImplementation(async () => { + checks++; + return checks < 2 + ? { allowed: true } + : { allowed: false, message: "Add credit" }; + }); + const { parts, writer } = recordingWriter(); + const outcome = await runInfraVideoImport(videoId, { + guards: serving(mp4Bytes(200), false), + writer, + partBytes: 16, + checkpointBytes: 64, + }); + expect(outcome).toBe("failed"); + expect(mocks.admit).toHaveBeenCalledTimes(2); + const [first, second] = mocks.admit.mock.calls.map( + ([input]) => input.durationSeconds * 2_500_000, + ); + expect(first).toBeGreaterThanOrEqual(64); + expect(first).toBeLessThan(64 + 16 + 7); + expect(second).toBeGreaterThanOrEqual((first ?? 0) + 64); + expect(parts.length).toBeLessThan(13); + expect(mocks.complete).not.toHaveBeenCalled(); + expect(mocks.abort).toHaveBeenCalled(); + expect(mocks.topUp).toHaveBeenCalledWith({ appId }); + expect(await importRow(videoId)).toMatchObject({ + state: "failed", + errorCode: "insufficient_credits", + }); + expect((await videoRow(videoId))?.errorReason).toBe( + "Import failed: Add credit", + ); + }); + + it("aborts the stored parts when completion fails for storage", async () => { + const appId = await seedApp(); + const videoId = await seedImport(appId); + fixture.completeResult = { + status: 500, + body: { code: "internal_error", error: "Failed to complete upload" }, + }; + const outcome = await runInfraVideoImport(videoId, { + guards: serving(mp4Bytes(40)), + writer: recordingWriter().writer, + partBytes: 16, + }); + expect(outcome).toBe("failed"); + expect(mocks.abort).toHaveBeenCalledWith({ + appId, + videoId, + uploadId: "upload-1", + }); + expect(await importRow(videoId)).toMatchObject({ + state: "failed", + errorCode: "storage_failed", + }); + expect((await videoRow(videoId))?.errorReason).toBe( + "Import failed: The file could not be stored", + ); + }); + + it("fails the video when completion is refused for credit", async () => { + const appId = await seedApp(); + const videoId = await seedImport(appId); + fixture.completeResult = { + status: 402, + body: { code: "insufficient_credits", message: "Add credit" }, + }; + const outcome = await runInfraVideoImport(videoId, { + guards: serving(mp4Bytes(40)), + writer: recordingWriter().writer, + partBytes: 16, + }); + expect(outcome).toBe("failed"); + expect(await importRow(videoId)).toMatchObject({ + state: "failed", + errorCode: "insufficient_credits", + }); + expect((await videoRow(videoId))?.status).toBe("errored"); + expect(mocks.abort).toHaveBeenCalledWith({ + appId, + videoId, + uploadId: "upload-1", + }); + }); + + it("leaves a deleted video alone", async () => { + const appId = await seedApp(); + const videoId = await seedImport(appId); + await database() + .update(developerVideos) + .set({ deletedAt: new Date() }) + .where(eq(developerVideos.id, videoId)); + const guards = serving(mp4Bytes(40)); + const outcome = await runInfraVideoImport(videoId, { guards }); + expect(outcome).toBe("failed"); + expect(guards.request).not.toHaveBeenCalled(); + expect((await importRow(videoId))?.errorCode).toBe("cancelled"); + expect((await videoRow(videoId))?.status).toBe("uploading"); + expect(mocks.announce).not.toHaveBeenCalled(); + }); + + it("does not download again when a re-run comes after the deadline", async () => { + const appId = await seedApp(); + const videoId = await seedImport( + appId, + new Date(Date.now() - 14 * 60_000), + ); + const guards = serving(mp4Bytes(40)); + const outcome = await runInfraVideoImport(videoId, { guards }); + expect(outcome).toBe("failed"); + expect(guards.request).not.toHaveBeenCalled(); + expect((await importRow(videoId))?.errorCode).toBe("timeout"); + expect((await videoRow(videoId))?.status).toBe("errored"); + }); + + it("does nothing for an import that already finished", async () => { + const appId = await seedApp(); + const videoId = await seedImport(appId); + await database() + .update(developerVideoImports) + .set({ state: "complete" }) + .where(eq(developerVideoImports.videoId, videoId)); + const guards = serving(mp4Bytes(40)); + expect(await runInfraVideoImport(videoId, { guards })).toBe("skipped"); + expect(guards.request).not.toHaveBeenCalled(); + }); + }); + }, +); + +describe.runIf(Boolean(missingTableUrl))( + "video import before the table exists", + () => { + let pool: Pool; + + beforeAll(async () => { + const url = assertLocal( + missingTableUrl, + /^\/cap_infra_noimport_[a-z0-9_]+$/, + ); + pool = createPool({ uri: url, connectionLimit: 2 }); + fixture.database = drizzle(pool); + }); + + afterAll(async () => { + await pool?.end(); + }); + + it("answers 503 import_unavailable and creates nothing", async () => { + vi.clearAllMocks(); + const appId = id(); + await database() + .insert(developerApps) + .values({ + id: appId, + ownerId: id() as never, + name: "Import fixture", + environment: "development", + }); + const result = await createInfraVideoImport(request(appId, id()), { + startImport, + guards: serving(mp4Bytes(64)), + }); + expect(result.status).toBe(503); + if (result.status !== 503) return; + expect(result.body.code).toBe("import_unavailable"); + expect(mocks.createInfraVideo).not.toHaveBeenCalled(); + expect(startImport).not.toHaveBeenCalled(); + }); + }, +); diff --git a/apps/web/__tests__/integration/processing-retry-database.test.ts b/apps/web/__tests__/integration/processing-retry-database.test.ts new file mode 100644 index 00000000000..bca8899e786 --- /dev/null +++ b/apps/web/__tests__/integration/processing-retry-database.test.ts @@ -0,0 +1,386 @@ +import { createHash, randomUUID } from "node:crypto"; +import * as Db from "@cap/database/schema"; +import { eq } from "drizzle-orm"; +import { drizzle, type MySql2Database } from "drizzle-orm/mysql2"; +import { createPool, type Pool } from "mysql2/promise"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +// Drives the real agent processCap handler and the real share page retry +// routes against a real migrated MySQL database. Only the session lookup, the +// transcription provider call and the AI workflow start are stubbed, so the +// idempotency store, the rerun charge, the rate limiter and startAiGeneration's +// own state checks all run for real. + +const fixture = vi.hoisted(() => ({ + database: undefined as unknown, + sessionUserId: "", + transcribe: vi.fn(), + startWorkflow: vi.fn(), +})); + +vi.mock("server-only", () => ({})); +vi.mock("@cap/database", async (importOriginal) => ({ + ...(await importOriginal()), + db: () => { + if (!fixture.database) throw new Error("Test database is not connected."); + return fixture.database; + }, +})); +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: async () => { + const [user] = await (fixture.database as MySql2Database) + .select() + .from(Db.users) + .where(eq(Db.users.id, fixture.sessionUserId as never)); + return user ?? null; + }, +})); +vi.mock("@cap/database/auth/request-site", () => ({ + isCrossSiteCookieWrite: () => false, +})); +vi.mock("next/cache", () => ({ revalidatePath: vi.fn() })); +vi.mock("@/lib/transcribe", () => ({ transcribeVideo: fixture.transcribe })); +vi.mock("@/lib/ai/provider", () => ({ isAiConfigured: () => true })); +vi.mock("workflow/api", () => ({ start: fixture.startWorkflow })); + +const databaseUrl = process.env.CAP_INFRA_TEST_DATABASE_URL; +const id = () => randomUUID().replaceAll("-", "").slice(0, 15); +const database = () => fixture.database as MySql2Database; + +type Owner = { userId: string; orgId: string; token: string }; + +const createOwner = async (): Promise => { + const userId = id(); + const orgId = id(); + const token = `cap_cli_${randomUUID().replaceAll("-", "")}${"a".repeat(11)}`; + await database() + .insert(Db.users) + .values({ + id: userId as never, + email: `${userId}@example.com`, + name: "Owner", + activeOrganizationId: orgId as never, + defaultOrgId: orgId as never, + stripeSubscriptionStatus: "active", + }); + await database() + .insert(Db.organizations) + .values({ id: orgId as never, name: "Team", ownerId: userId as never }); + await database() + .insert(Db.organizationMembers) + .values({ + id: id() as never, + userId: userId as never, + organizationId: orgId as never, + role: "owner", + }); + await database() + .insert(Db.agentApiKeys) + .values({ + id: id() as never, + userId: userId as never, + name: "Test agent", + tokenHash: createHash("sha256").update(token).digest("hex"), + scopes: ["caps:read", "caps:process"], + expiresAt: new Date(Date.now() + 60 * 60 * 1000), + } as never); + return { userId, orgId, token }; +}; + +const createVideo = async ( + owner: Owner, + state: { + transcriptionStatus: "COMPLETE" | "ERROR" | null; + aiGenerationStatus?: string; + }, +) => { + const videoId = id(); + await database() + .insert(Db.videos) + .values({ + id: videoId as never, + ownerId: owner.userId as never, + orgId: owner.orgId as never, + name: "Recording", + transcriptionStatus: state.transcriptionStatus, + metadata: state.aiGenerationStatus + ? { aiGenerationStatus: state.aiGenerationStatus } + : {}, + } as never); + return videoId; +}; + +const videoState = async (videoId: string) => { + const [video] = await database() + .select({ + transcriptionStatus: Db.videos.transcriptionStatus, + metadata: Db.videos.metadata, + }) + .from(Db.videos) + .where(eq(Db.videos.id, videoId as never)); + return { + transcriptionStatus: video?.transcriptionStatus ?? null, + aiGenerationStatus: + (video?.metadata as { aiGenerationStatus?: string } | null) + ?.aiGenerationStatus ?? null, + }; +}; + +describe.runIf(Boolean(databaseUrl))( + "processing retries against a real database", + () => { + let pool: Pool; + let v1: typeof import("@/app/api/v1/[...route]/route").POST; + let retryAi: typeof import("@/app/api/videos/[videoId]/retry-ai/route").POST; + let retryTranscription: typeof import("@/app/api/videos/[videoId]/retry-transcription/route").POST; + + beforeAll(async () => { + if (!databaseUrl) throw new Error("Missing isolated test database URL."); + const url = new URL(databaseUrl); + if ( + url.protocol !== "mysql:" || + !["127.0.0.1", "localhost", "[::1]"].includes(url.hostname) || + !/^\/cap_infra_[a-z0-9_]+$/.test(url.pathname) + ) + throw new Error("These tests require a session-scoped local database."); + pool = createPool({ uri: databaseUrl, connectionLimit: 8 }); + fixture.database = drizzle(pool); + Object.assign(process.env, { + NEXT_PUBLIC_WEB_URL: "http://localhost", + WEB_URL: "http://localhost", + DATABASE_URL: databaseUrl, + NEXTAUTH_SECRET: "synthetic-test-secret-that-is-long-enough", + NEXTAUTH_URL: "http://localhost", + CAP_AWS_BUCKET: "synthetic", + CAP_AWS_REGION: "us-east-1", + CAP_AWS_ACCESS_KEY: "synthetic", + CAP_AWS_SECRET_KEY: "synthetic", + NODE_ENV: "test", + }); + ({ POST: v1 } = await import("@/app/api/v1/[...route]/route")); + ({ POST: retryAi } = await import( + "@/app/api/videos/[videoId]/retry-ai/route" + )); + ({ POST: retryTranscription } = await import( + "@/app/api/videos/[videoId]/retry-transcription/route" + )); + }); + + beforeEach(() => { + fixture.transcribe.mockResolvedValue({ success: true, message: "ok" }); + fixture.startWorkflow.mockResolvedValue(undefined); + }); + + afterAll(async () => { + await pool?.end(); + }); + + const processCap = ( + owner: Owner, + videoId: string, + payload: { target: "all" | "transcript" | "ai"; retry?: boolean }, + key = randomUUID(), + ) => + v1( + new Request(`http://localhost/api/v1/caps/${videoId}/process`, { + method: "POST", + headers: { + Authorization: `Bearer ${owner.token}`, + "Content-Type": "application/json", + "Idempotency-Key": key, + "X-Cap-Confirmation": "user", + }, + body: JSON.stringify(payload), + }), + ); + + const sharePageRetry = ( + route: (request: Request, props: never) => Promise, + owner: Owner, + videoId: string, + ) => { + fixture.sessionUserId = owner.userId; + return route( + new Request(`http://localhost/api/videos/${videoId}/retry`, { + method: "POST", + }), + { params: Promise.resolve({ videoId }) } as never, + ); + }; + + it("processes a fresh video through the agent API without charging a retry", async () => { + const owner = await createOwner(); + for (let index = 0; index < 35; index++) { + const videoId = await createVideo(owner, { + transcriptionStatus: "COMPLETE", + }); + fixture.startWorkflow.mockClear(); + const response = await processCap(owner, videoId, { target: "all" }); + expect(response.status, await response.clone().text()).toBe(200); + expect(fixture.startWorkflow).toHaveBeenCalledOnce(); + expect((await videoState(videoId)).aiGenerationStatus).toBe("QUEUED"); + } + + const pending = await createVideo(owner, { transcriptionStatus: null }); + fixture.transcribe.mockClear(); + const transcript = await processCap(owner, pending, { + target: "transcript", + }); + expect(transcript.status).toBe(200); + expect(fixture.transcribe).toHaveBeenCalledOnce(); + }); + + it("retries a failed transcription and a failed AI run once through the agent API", async () => { + const owner = await createOwner(); + const failedTranscript = await createVideo(owner, { + transcriptionStatus: "ERROR", + }); + fixture.transcribe.mockClear(); + const transcript = await processCap(owner, failedTranscript, { + target: "transcript", + retry: true, + }); + expect(transcript.status).toBe(200); + expect(fixture.transcribe).toHaveBeenCalledOnce(); + expect((await videoState(failedTranscript)).transcriptionStatus).toBe( + null, + ); + + const failedAi = await createVideo(owner, { + transcriptionStatus: "COMPLETE", + aiGenerationStatus: "ERROR", + }); + fixture.startWorkflow.mockClear(); + const ai = await processCap(owner, failedAi, { + target: "ai", + retry: true, + }); + expect(ai.status).toBe(200); + expect(fixture.startWorkflow).toHaveBeenCalledOnce(); + expect((await videoState(failedAi)).aiGenerationStatus).toBe("QUEUED"); + }); + + it("retries a failed transcription and a failed AI run once from the share page", async () => { + const owner = await createOwner(); + const failedTranscript = await createVideo(owner, { + transcriptionStatus: "ERROR", + }); + const transcript = await sharePageRetry( + retryTranscription, + owner, + failedTranscript, + ); + expect(transcript.status).toBe(200); + expect((await videoState(failedTranscript)).transcriptionStatus).toBe( + null, + ); + + const failedAi = await createVideo(owner, { + transcriptionStatus: "COMPLETE", + aiGenerationStatus: "ERROR", + }); + fixture.startWorkflow.mockClear(); + const ai = await sharePageRetry(retryAi, owner, failedAi); + expect(ai.status).toBe(200); + expect(fixture.startWorkflow).toHaveBeenCalledOnce(); + expect((await videoState(failedAi)).aiGenerationStatus).toBe("QUEUED"); + }); + + it("only refuses retries past 30 an hour, shared across the share page and the agent API", async () => { + const owner = await createOwner(); + for (let index = 0; index < 15; index++) { + const videoId = await createVideo(owner, { + transcriptionStatus: "ERROR", + }); + expect( + (await sharePageRetry(retryTranscription, owner, videoId)).status, + ).toBe(200); + } + for (let index = 0; index < 15; index++) { + const videoId = await createVideo(owner, { + transcriptionStatus: "ERROR", + }); + expect( + ( + await processCap(owner, videoId, { + target: "transcript", + retry: true, + }) + ).status, + ).toBe(200); + } + + const refusedAgent = await createVideo(owner, { + transcriptionStatus: "ERROR", + }); + const limited = await processCap(owner, refusedAgent, { + target: "transcript", + retry: true, + }); + expect(limited.status).toBe(429); + expect((await videoState(refusedAgent)).transcriptionStatus).toBe( + "ERROR", + ); + + const refusedSharePage = await createVideo(owner, { + transcriptionStatus: "ERROR", + }); + expect( + (await sharePageRetry(retryTranscription, owner, refusedSharePage)) + .status, + ).toBe(429); + expect((await videoState(refusedSharePage)).transcriptionStatus).toBe( + "ERROR", + ); + + const other = await createOwner(); + const otherVideo = await createVideo(other, { + transcriptionStatus: "ERROR", + }); + expect( + (await sharePageRetry(retryTranscription, other, otherVideo)).status, + ).toBe(200); + + const fresh = await createVideo(owner, { + transcriptionStatus: "COMPLETE", + }); + expect((await processCap(owner, fresh, { target: "all" })).status).toBe( + 200, + ); + }); + + it("replays a saved request without charging or starting AI again", async () => { + const owner = await createOwner(); + const failedAi = await createVideo(owner, { + transcriptionStatus: "COMPLETE", + aiGenerationStatus: "ERROR", + }); + const key = randomUUID(); + fixture.startWorkflow.mockClear(); + expect( + (await processCap(owner, failedAi, { target: "ai", retry: true }, key)) + .status, + ).toBe(200); + expect(fixture.startWorkflow).toHaveBeenCalledOnce(); + + await database() + .update(Db.videos) + .set({ metadata: { aiGenerationStatus: "ERROR" } } as never) + .where(eq(Db.videos.id, failedAi as never)); + fixture.startWorkflow.mockClear(); + expect( + (await processCap(owner, failedAi, { target: "ai", retry: true }, key)) + .status, + ).toBe(200); + expect(fixture.startWorkflow).not.toHaveBeenCalled(); + }); + }, +); diff --git a/apps/web/__tests__/unit/agent-developer-billing.test.ts b/apps/web/__tests__/unit/agent-developer-billing.test.ts index 8e4eb09b546..925548096dc 100644 --- a/apps/web/__tests__/unit/agent-developer-billing.test.ts +++ b/apps/web/__tests__/unit/agent-developer-billing.test.ts @@ -9,10 +9,13 @@ vi.mock("@/lib/infra-standing", () => ({ infraBillingJson: (billing: unknown) => ({ json: billing }), })); +import { Agent } from "@cap/web-domain"; +import { Schema } from "effect"; import { BILLING_ACCESS_HINT, developerAppBillingSection, } from "@/lib/agent-developer-billing"; +import { developerOpenApiSpec } from "@/lib/developer-openapi"; const credits = { balanceMicroCredits: 500_000, @@ -59,6 +62,37 @@ describe("billing data in the developer app context", () => { ); }); + it("returns the same storage fields as GET /usage", () => { + const tiers = { "1080p": 87.5, "1440p": 0, "2160p": 0 }; + const restStorage = + developerOpenApiSpec("https://cap.so").components.schemas.Usage.properties + .storage; + const storage = Object.fromEntries( + restStorage.required.map((field: string) => [ + field, + field.endsWith("ByTier") ? tiers : 170, + ]), + ); + const billing = Schema.encodeSync( + Agent.AgentDeveloperAppContextResponse.fields.billing, + )({ + standing: { state: "active", pausesAt: null }, + upload: { + allowed: true, + reason: null, + requiredMicroCredits: 1190, + shortfallMicroCredits: 0, + message: null, + }, + storage: storage as never, + }); + + expect(Object.keys(billing?.storage ?? {}).sort()).toEqual( + [...restStorage.required].sort(), + ); + expect(restStorage.required).toContain("dailyChargeMinutesByTier"); + }); + it("is the only way the app context handler returns credits or billing", () => { const route = readFileSync( join(process.cwd(), "app/api/v1/[...route]/route.ts"), diff --git a/apps/web/__tests__/unit/agent-developer-response-size.test.ts b/apps/web/__tests__/unit/agent-developer-response-size.test.ts index 1156f7c0cf9..3c548515689 100644 --- a/apps/web/__tests__/unit/agent-developer-response-size.test.ts +++ b/apps/web/__tests__/unit/agent-developer-response-size.test.ts @@ -27,6 +27,7 @@ const row = (index: number): AgentDeveloperVideoRow => ({ height: 720, fps: 30, maxResolutionTier: "1080p", + takenDown: false, renditions: [ { name: "720p", diff --git a/apps/web/__tests__/unit/agent-developer-routes.test.ts b/apps/web/__tests__/unit/agent-developer-routes.test.ts index a4c8a957065..fc585aa4ea6 100644 --- a/apps/web/__tests__/unit/agent-developer-routes.test.ts +++ b/apps/web/__tests__/unit/agent-developer-routes.test.ts @@ -1,4 +1,5 @@ import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { creditCheckoutSessionParams } from "@/lib/infra-credit-checkout"; vi.mock("server-only", () => ({})); vi.mock("@/lib/rate-limit", () => ({ @@ -13,12 +14,36 @@ vi.mock("@/lib/rate-limit", () => ({ const state = vi.hoisted(() => ({ scopes: [] as string[], - apps: [] as { id: string }[], + apps: [] as { id: string; appName?: string }[], videos: [] as Record[], transactions: [] as Record[], idempotency: null as Record | null, tables: {} as Record, redrive: vi.fn(async (_videoId: string) => false), + users: [] as Record[], + createCheckoutSession: vi.fn(), + cancelEncodes: vi.fn(async () => undefined), + settleAtDelete: vi.fn(async () => null), + settleDeletedApp: vi.fn(async () => undefined), + videoUpdates: [] as Record[], + appStamps: [] as unknown[], +})); + +vi.mock("@/lib/developer-video-cancel", () => ({ + cancelDeveloperVideoEncodes: state.cancelEncodes, +})); + +vi.mock("@/lib/infra-storage-minimum", async (original) => ({ + ...(await original()), + settleFirstMonthAtDelete: state.settleAtDelete, + settleFirstMonthsForDeletedApp: state.settleDeletedApp, +})); + +vi.mock("@cap/utils", async (original) => ({ + ...(await original()), + stripe: () => ({ + checkout: { sessions: { create: state.createCheckoutSession } }, + }), })); vi.mock("@/lib/developer-video-start", async (original) => ({ @@ -44,6 +69,7 @@ vi.mock("@cap/database", () => { ]; } if (table === tables.developerApps) return state.apps; + if (table === tables.users) return state.users; if (table === tables.developerVideos) return state.videos; if (table === tables.developerCreditTransactions) { return state.transactions; @@ -75,7 +101,18 @@ vi.mock("@cap/database", () => { }; const writer = (table: unknown) => { const chain = Object.assign(Promise.resolve([{ affectedRows: 1 }]), { - set: () => chain, + set: (values: Record) => { + if (table === state.tables.developerVideos) { + state.videoUpdates.push(values); + } + if (table === state.tables.developerApps) { + state.appStamps.push(values.deletedAt); + } + if (table === state.tables.developerApiKeys) { + state.appStamps.push(values.revokedAt); + } + return chain; + }, where: () => chain, onDuplicateKeyUpdate: () => chain, values: (values: Record) => { @@ -133,9 +170,11 @@ describe("developer routes of the agent API", () => { state.tables = { agentApiKeys: schema.agentApiKeys, developerApps: schema.developerApps, + developerApiKeys: schema.developerApiKeys, developerVideos: schema.developerVideos, developerCreditTransactions: schema.developerCreditTransactions, agentApiIdempotency: schema.agentApiIdempotency, + users: schema.users, }; const route = await import("@/app/api/v1/[...route]/route"); handlers = { @@ -152,6 +191,13 @@ describe("developer routes of the agent API", () => { state.videos = []; state.transactions = []; state.idempotency = null; + state.users = []; + state.createCheckoutSession.mockReset(); + state.cancelEncodes.mockClear(); + state.settleAtDelete.mockClear(); + state.settleDeletedApp.mockClear(); + state.videoUpdates = []; + state.appStamps = []; }); const call = async (method: string, path: string, body?: unknown) => { @@ -234,6 +280,183 @@ describe("developer routes of the agent API", () => { expect([...tested].sort()).toEqual([...declared].sort()); }); + it("cancels a deleted video's encode jobs once the delete has committed", async () => { + state.apps = [{ id: APP }]; + state.videos = [{ id: "video0000000001" }]; + + const result = await call( + "DELETE", + `/developer/apps/${APP}/videos/video0000000001`, + ); + + expect(result.status).toBe(200); + expect(state.videoUpdates).toEqual([ + { deletedAt: expect.any(Date), updatedAt: expect.any(Date) }, + ]); + expect(state.cancelEncodes).toHaveBeenCalledExactlyOnceWith({ + appId: APP, + videoId: "video0000000001", + }); + }); + + it("cancels nothing for a video it could not find", async () => { + state.apps = [{ id: APP }]; + + const result = await call( + "DELETE", + `/developer/apps/${APP}/videos/video0000000001`, + ); + + expect(result.status).toBe(404); + expect(state.cancelEncodes).not.toHaveBeenCalled(); + }); + + it("deletes an app's videos with it and cancels their encode jobs", async () => { + state.apps = [{ id: APP }]; + + const result = await call("DELETE", `/developer/apps/${APP}`); + + expect(result.status).toBe(200); + expect(state.videoUpdates).toEqual([ + { deletedAt: expect.any(Date), updatedAt: expect.any(Date) }, + ]); + const deletedAt = state.videoUpdates[0]?.deletedAt; + expect(deletedAt).toBeInstanceOf(Date); + expect((deletedAt as Date).getTime() % 1000).toBe(0); + expect(state.appStamps).toEqual([deletedAt, deletedAt]); + expect(state.cancelEncodes).toHaveBeenCalledExactlyOnceWith({ + appId: APP, + deletedAt, + }); + }); + + it("settles the first month of the videos an app delete stamped, once, after the delete and before the cancel", async () => { + state.apps = [{ id: APP }]; + + const result = await call("DELETE", `/developer/apps/${APP}`); + + expect(result.status).toBe(200); + const deletedAt = state.videoUpdates[0]?.deletedAt; + expect(state.settleDeletedApp).toHaveBeenCalledExactlyOnceWith( + APP, + deletedAt, + ); + const [settled] = state.settleDeletedApp.mock.invocationCallOrder; + const [cancelled] = state.cancelEncodes.mock.invocationCallOrder; + expect(settled).toBeLessThan(cancelled ?? 0); + }); + + it("settles nothing for an app it could not find", async () => { + const result = await call("DELETE", `/developer/apps/${APP}`); + + expect(result.status).toBe(404); + expect(state.settleDeletedApp).not.toHaveBeenCalled(); + }); + + it("settles a deleted video's first month before cancelling its encode", async () => { + state.apps = [{ id: APP }]; + state.videos = [{ id: "video0000000001" }]; + + await call("DELETE", `/developer/apps/${APP}/videos/video0000000001`); + + expect(state.settleAtDelete).toHaveBeenCalledExactlyOnceWith( + "video0000000001", + ); + const [settled] = state.settleAtDelete.mock.invocationCallOrder; + const [cancelled] = state.cancelEncodes.mock.invocationCallOrder; + expect(settled).toBeLessThan(cancelled ?? 0); + }); + + it("opens a credit checkout with the console's receipt, tax and payment tag", async () => { + state.apps = [{ id: "account-1", appName: "Course site" }]; + state.users = [{ stripeCustomerId: "cus_agent" }]; + state.createCheckoutSession.mockResolvedValue({ + url: "https://checkout.stripe.com/agent", + }); + + const response = await call( + "POST", + `/developer/apps/${APP}/credits/checkout`, + { + amountCents: 2500, + }, + ); + + expect(response.status).toBe(200); + expect(response.body.url).toBe("https://checkout.stripe.com/agent"); + expect(state.createCheckoutSession).toHaveBeenCalledTimes(1); + const [params, options] = state.createCheckoutSession.mock.calls[0] ?? []; + expect(params).toEqual({ + ...creditCheckoutSessionParams({ + customerId: "cus_agent", + accountId: "account-1", + appName: "Course site", + amountCents: 2500, + receiptEmail: "owner@example.com", + }), + success_url: "http://localhost/cli/complete?developerCredits=success", + cancel_url: "http://localhost/cli/complete?developerCredits=cancelled", + metadata: { + type: "developer_credits", + appId: APP, + accountId: "account-1", + amountCents: "2500", + userId: "user-1", + }, + }); + expect(params).toMatchObject({ + mode: "payment", + invoice_creation: { + enabled: true, + invoice_data: { description: "Cap Infra credits for Course site" }, + }, + payment_method_types: ["card"], + payment_intent_data: { + description: "Cap Infra credits for Course site", + receipt_email: "owner@example.com", + metadata: { type: "developer_credits", accountId: "account-1" }, + }, + tax_id_collection: { enabled: true }, + billing_address_collection: "auto", + customer_update: { name: "auto", address: "auto" }, + }); + expect(params).not.toHaveProperty("automatic_tax"); + expect(options.idempotencyKey).toMatch(/:checkout$/); + }); + + it("opens a $10,000 credit checkout and refuses one cent more", async () => { + state.apps = [{ id: "account-1" }]; + state.users = [{ stripeCustomerId: "cus_agent" }]; + state.createCheckoutSession.mockResolvedValue({ + url: "https://checkout.stripe.com/agent", + }); + + const refused = await call( + "POST", + `/developer/apps/${APP}/credits/checkout`, + { amountCents: 1_000_001 }, + ); + expect(refused.status).toBe(400); + expect(refused.body.message).toBe( + "Developer credit purchase must be from $5 to $10,000", + ); + expect(state.createCheckoutSession).not.toHaveBeenCalled(); + + const opened = await call( + "POST", + `/developer/apps/${APP}/credits/checkout`, + { amountCents: 1_000_000 }, + ); + expect(opened.status).toBe(200); + const [params] = state.createCheckoutSession.mock.calls[0] ?? []; + expect(params.metadata.amountCents).toBe("1000000"); + expect(params.line_items[0].price_data.unit_amount).toBe(1_000_000); + expect(params.payment_method_types).toEqual(["card"]); + expect(params.payment_method_options).toEqual({ + card: { request_three_d_secure: "any" }, + }); + }); + it("says what to do when the developer scope is missing", async () => { state.scopes = ["caps:read"]; for (const [method, path, scope] of [ @@ -278,6 +501,7 @@ describe("developer routes of the agent API", () => { }, ], transcriptionStatus: null, + takenDown: false, processing: { hls: { state: "complete" }, mp4: { state: "complete" }, @@ -312,6 +536,7 @@ describe("developer routes of the agent API", () => { maxResolutionTier: "1080p", renditions: videoRow().renditions, transcriptionStatus: null, + takenDown: false, hlsUrl: "http://localhost/api/developer/playback/playback-id-000000000001/master.m3u8", mp4Url: diff --git a/apps/web/__tests__/unit/agent-developer-transcript-limit.test.ts b/apps/web/__tests__/unit/agent-developer-transcript-limit.test.ts new file mode 100644 index 00000000000..039da578a4e --- /dev/null +++ b/apps/web/__tests__/unit/agent-developer-transcript-limit.test.ts @@ -0,0 +1,218 @@ +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("server-only", () => ({})); +vi.mock("@/lib/rate-limit", () => ({ + isRateLimited: async () => false, + RATE_LIMIT_IDS: { + AGENT_TOKEN_EXCHANGE: "rl_agent_token_exchange", + AGENT_AUTHORIZATION: "rl_agent_authorization", + AGENT_UNLOCK: "rl_agent_unlock", + AGENT_LOOM_IMPORT: "rl_loom_import_per_user", + }, +})); + +const state = vi.hoisted(() => ({ + scopes: [] as string[], + apps: [] as { id: string }[], + videos: [] as Record[], + transactions: [] as Record[], + idempotency: null as Record | null, + tables: {} as Record, + redrive: vi.fn(async (_videoId: string) => false), + createFromUrl: vi.fn(), + startUpload: vi.fn(), +})); + +vi.mock("@/lib/developer-transcript-store", async (original) => ({ + ...(await original()), + createTranscriptFromUrl: state.createFromUrl, + startTranscriptUpload: state.startUpload, +})); + +vi.mock("@/lib/developer-video-start", async (original) => ({ + ...(await original()), + redriveDeveloperVideoProcessing: state.redrive, +})); + +vi.mock("@cap/database", () => { + const rowsFor = (table: unknown) => { + const { tables } = state; + if (table === tables.agentApiKeys) { + return [ + { + tokenId: "token-1", + scopes: state.scopes, + expiresAt: new Date(Date.now() + 60_000), + revokedAt: null, + lastUsedAt: new Date(), + id: "user-1", + email: "owner@example.com", + activeOrganizationId: "org-1", + }, + ]; + } + if (table === tables.developerApps) return state.apps; + if (table === tables.developerVideos) return state.videos; + if (table === tables.developerCreditTransactions) { + return state.transactions; + } + if (table === tables.agentApiIdempotency) { + return state.idempotency ? [state.idempotency] : []; + } + return []; + }; + const reader = () => { + let table: unknown; + const chain = Object.assign( + Promise.resolve().then(() => rowsFor(table)), + { + from: (next: unknown) => { + table = next; + return chain; + }, + innerJoin: () => chain, + leftJoin: () => chain, + where: () => chain, + orderBy: () => chain, + limit: () => chain, + for: () => chain, + groupBy: () => chain, + }, + ); + return chain; + }; + const writer = (table: unknown) => { + const chain = Object.assign(Promise.resolve([{ affectedRows: 1 }]), { + set: () => chain, + where: () => chain, + onDuplicateKeyUpdate: () => chain, + values: (values: Record) => { + if (table === state.tables.agentApiIdempotency) { + state.idempotency = { ...values, state: "pending", response: null }; + } + return chain; + }, + }); + return chain; + }; + const client = { + select: reader, + insert: writer, + update: writer, + delete: writer, + transaction: async (run: (tx: unknown) => unknown) => run(client), + }; + return { db: () => client }; +}); + +const TOKEN = `cap_cli_${"a".repeat(43)}`; +const APP = "app000000000001"; +const FULL_SCOPES = [ + "caps:read", + "billing:read", + "billing:write", + "developer:read", + "developer:write", + "developer:secrets", +]; + +type Handler = (request: Request) => Promise; + +describe("the transcript ceiling on the agent API", () => { + let handlers: Record; + + beforeAll(async () => { + Object.assign(process.env, { + NEXT_PUBLIC_WEB_URL: "http://localhost", + WEB_URL: "http://localhost", + DATABASE_URL: "mysql://unused:unused@127.0.0.1:3306/unused", + NEXTAUTH_SECRET: "synthetic-test-secret-that-is-long-enough", + NEXTAUTH_URL: "http://localhost", + CAP_AWS_BUCKET: "synthetic", + CAP_AWS_REGION: "us-east-1", + CAP_AWS_ACCESS_KEY: "synthetic", + CAP_AWS_SECRET_KEY: "synthetic", + NODE_ENV: "test", + }); + const schema = await import("@cap/database/schema"); + state.tables = { + agentApiKeys: schema.agentApiKeys, + developerApps: schema.developerApps, + developerVideos: schema.developerVideos, + developerCreditTransactions: schema.developerCreditTransactions, + agentApiIdempotency: schema.agentApiIdempotency, + }; + const route = await import("@/app/api/v1/[...route]/route"); + handlers = { + GET: route.GET, + POST: route.POST, + PATCH: route.PATCH, + DELETE: route.DELETE, + }; + }); + + beforeEach(() => { + state.scopes = FULL_SCOPES; + state.apps = []; + state.videos = []; + state.transactions = []; + state.idempotency = null; + }); + + const call = async (method: string, path: string, body?: unknown) => { + const handler = handlers[method]; + if (!handler) throw new Error(`No handler for ${method}`); + const response = await handler( + new Request(`http://localhost/api/v1${path}`, { + method, + headers: { + Authorization: `Bearer ${TOKEN}`, + "Content-Type": "application/json", + "Idempotency-Key": "idempotency-key-0001", + "X-Cap-Confirmation": "user", + }, + body: method === "GET" ? undefined : JSON.stringify(body ?? {}), + }), + ); + return { status: response.status, body: await response.json() }; + }; + + const AT_CEILING = { + ok: false as const, + status: 429 as const, + code: "too_many_transcripts", + message: "This app already has 20 transcripts queued or processing", + hint: "Wait for one to finish, then send the request again.", + }; + + it.each([ + [ + "creating from a URL", + `/developer/apps/${APP}/transcripts`, + { url: "https://example.com/a.mp3" }, + ], + [ + "starting an upload", + `/developer/apps/${APP}/transcripts/transcript00001/start`, + undefined, + ], + ])( + "answers 429 that says to wait when %s at the ceiling", + async (_label, path, body) => { + state.apps = [{ id: APP }]; + state.createFromUrl.mockResolvedValue(AT_CEILING); + state.startUpload.mockResolvedValue(AT_CEILING); + + const result = await call("POST", path, body); + + expect(result.status).toBe(429); + expect(result.body).toMatchObject({ + code: "RATE_LIMITED", + message: + "This app already has 20 transcripts queued or processing. Wait for one to finish, then retry.", + retryable: true, + retryAfterMs: 15_000, + }); + }, + ); +}); diff --git a/apps/web/__tests__/unit/agent-docs.test.ts b/apps/web/__tests__/unit/agent-docs.test.ts index 38d5f24f4a1..6691ae8b0a4 100644 --- a/apps/web/__tests__/unit/agent-docs.test.ts +++ b/apps/web/__tests__/unit/agent-docs.test.ts @@ -145,7 +145,8 @@ describe("Cap for Agents docs", () => { ); const webhooks = getDocBySlug("api/webhooks")?.content ?? ""; - expect(webhooks).toContain( + expect(webhooks).toContain("POST /api/developer/v1/webhooks"); + expect(webhooks).not.toContain( "does not currently expose a documented public webhook subscription API", ); }); diff --git a/apps/web/__tests__/unit/agent-readiness.test.ts b/apps/web/__tests__/unit/agent-readiness.test.ts index 34aa1ab63bb..b8f3a1d5d61 100644 --- a/apps/web/__tests__/unit/agent-readiness.test.ts +++ b/apps/web/__tests__/unit/agent-readiness.test.ts @@ -313,6 +313,8 @@ describe("developer OpenAPI spec", () => { "error", "code", "message", + "hint", + "docs", "standing", "pausesAt", "balanceMicroCredits", @@ -326,15 +328,21 @@ describe("developer OpenAPI spec", () => { it("matches the 402 schema to the body the upload routes send", async () => { const { uploadAdmissionError } = await import("@/lib/infra-standing"); - const body = uploadAdmissionError({ - allowed: false, - reason: "insufficient_credits", - standing: { state: "grace", pausesAt: new Date("2026-11-19T00:00:00Z") }, - balanceMicroCredits: 10, - requiredMicroCredits: 30, - shortfallMicroCredits: 20, - message: "Add credit.", - }); + const body = uploadAdmissionError( + { + allowed: false, + reason: "insufficient_credits", + standing: { + state: "grace", + pausesAt: new Date("2026-11-19T00:00:00Z"), + }, + balanceMicroCredits: 10, + requiredMicroCredits: 30, + shortfallMicroCredits: 20, + message: "Add credit.", + }, + "app_1", + ); const schema = spec.components.schemas.UploadPaymentRequired; expect(Object.keys(body).sort()).toEqual([...schema.required].sort()); @@ -397,7 +405,7 @@ describe("developer OpenAPI spec", () => { ] as const; const implemented = new Set(); for (const [dir, base] of routeDirs) { - const source = readFileSync(join(apiRoot, dir, "route.ts"), "utf8"); + const source = readFileSync(join(apiRoot, dir, "app.ts"), "utf8"); const modules = new Map( [...source.matchAll(/import \* as (\w+) from "\.\/([^"]+)"/g)].map( ([, name, file]) => [name, file], @@ -442,6 +450,30 @@ describe("proxy content negotiation", () => { }, ); + it.each([ + ["/infra", "text/markdown"], + ["/infra/", "text/markdown, text/html;q=0.5"], + ["/infra.md", "text/html,*/*;q=0.8"], + ])( + "rewrites %s with Accept %s to the Infra Markdown", + async (path, accept) => { + const response = await send(path, accept); + expect(response.headers.get("x-middleware-rewrite")).toBe( + "https://cap.so/infra-markdown", + ); + }, + ); + + it.each([ + ["/infra", "text/html,*/*;q=0.8"], + ["/infra", "text/html, text/markdown;q=0.5"], + ["/infra/pricing", "text/markdown"], + ["/infrastructure", "text/markdown"], + ])("leaves %s with Accept %s alone", async (path, accept) => { + const response = await send(path, accept); + expect(response.headers.get("x-middleware-rewrite")).toBeNull(); + }); + it("serves the homepage as Markdown", async () => { const fetch = vi .spyOn(globalThis, "fetch") diff --git a/apps/web/__tests__/unit/ai-generation-rerun.test.ts b/apps/web/__tests__/unit/ai-generation-rerun.test.ts new file mode 100644 index 00000000000..a10316e86ba --- /dev/null +++ b/apps/web/__tests__/unit/ai-generation-rerun.test.ts @@ -0,0 +1,162 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/database", () => ({ db: vi.fn() })); +vi.mock("workflow/api", () => ({ start: vi.fn() })); +vi.mock("@/lib/ai/provider", () => ({ isAiConfigured: () => true })); +vi.mock("@/workflows/generate-ai", () => ({ generateAiWorkflow: vi.fn() })); + +import { chargesProcessingRerun, isAiGenerationRerun } from "@/lib/generate-ai"; + +const legacySummary = + "The AI was unable to generate a proper summary for this content."; + +describe("AI generation rerun", () => { + it("treats a video that never ran AI generation as a first run", () => { + expect(isAiGenerationRerun({})).toBe(false); + }); + + it("does not count work that is already queued or running", () => { + expect(isAiGenerationRerun({ aiGenerationStatus: "QUEUED" })).toBe(false); + expect(isAiGenerationRerun({ aiGenerationStatus: "PROCESSING" })).toBe( + false, + ); + }); + + it("does not count a finished result, which generation leaves alone", () => { + expect( + isAiGenerationRerun({ + aiGenerationStatus: "COMPLETE", + summary: "A summary", + chapters: [{ title: "Intro", start: 0 }], + }), + ).toBe(false); + }); + + it("counts every state that starts generation again", () => { + expect(isAiGenerationRerun({ aiGenerationStatus: "ERROR" })).toBe(true); + expect(isAiGenerationRerun({ aiGenerationStatus: "SKIPPED" })).toBe(true); + expect( + isAiGenerationRerun({ + aiGenerationStatus: "COMPLETE", + summary: legacySummary, + chapters: [{ title: "Intro", start: 0 }], + }), + ).toBe(true); + expect( + isAiGenerationRerun({ + aiGenerationStatus: "COMPLETE", + summary: "A summary", + }), + ).toBe(true); + }); +}); + +describe("agent processing rerun charge", () => { + const complete = { + aiGenerationStatus: "COMPLETE" as const, + summary: "A summary", + chapters: [{ title: "Intro", start: 0 }], + }; + + it("does not charge first-time processing", () => { + for (const target of ["all", "transcript", "ai"] as const) { + expect( + chargesProcessingRerun({ + target, + retry: undefined, + transcriptionStatus: target === "ai" ? "COMPLETE" : null, + metadata: {}, + }), + ).toBe(false); + } + expect( + chargesProcessingRerun({ + target: "ai", + retry: true, + transcriptionStatus: "COMPLETE", + metadata: complete, + }), + ).toBe(false); + }); + + it("charges a transcript retry after a failure", () => { + for (const target of ["all", "transcript"] as const) { + expect( + chargesProcessingRerun({ + target, + retry: true, + transcriptionStatus: "ERROR", + metadata: {}, + }), + ).toBe(true); + } + expect( + chargesProcessingRerun({ + target: "transcript", + retry: undefined, + transcriptionStatus: "ERROR", + metadata: {}, + }), + ).toBe(false); + }); + + it("charges an AI retry after a failure or skip", () => { + for (const aiGenerationStatus of ["ERROR", "SKIPPED"] as const) { + for (const target of ["all", "ai"] as const) { + expect( + chargesProcessingRerun({ + target, + retry: true, + transcriptionStatus: "COMPLETE", + metadata: { aiGenerationStatus }, + }), + ).toBe(true); + } + } + }); + + it("does not charge a request the mutation refuses as a conflict", () => { + for (const aiGenerationStatus of ["ERROR", "SKIPPED"] as const) { + expect( + chargesProcessingRerun({ + target: "ai", + retry: undefined, + transcriptionStatus: "COMPLETE", + metadata: { aiGenerationStatus }, + }), + ).toBe(false); + } + }); + + it("charges a rerun of an incomplete result even without retry", () => { + expect( + chargesProcessingRerun({ + target: "ai", + retry: undefined, + transcriptionStatus: "COMPLETE", + metadata: { aiGenerationStatus: "COMPLETE", summary: "A summary" }, + }), + ).toBe(true); + }); + + it("does not charge AI work that waits on transcription", () => { + for (const target of ["all", "ai"] as const) { + for (const transcriptionStatus of [ + null, + "PROCESSING", + "ERROR", + "SKIPPED", + "NO_AUDIO", + ]) { + expect( + chargesProcessingRerun({ + target, + retry: target === "ai" ? true : undefined, + transcriptionStatus, + metadata: { aiGenerationStatus: "ERROR" }, + }), + ).toBe(false); + } + } + }); +}); diff --git a/apps/web/__tests__/unit/build-env.test.ts b/apps/web/__tests__/unit/build-env.test.ts index 8efdb5df285..35fd9496808 100644 --- a/apps/web/__tests__/unit/build-env.test.ts +++ b/apps/web/__tests__/unit/build-env.test.ts @@ -10,6 +10,7 @@ const createReference = () => NEXT_PUBLIC_OPENPANEL_API_URL: z.string().optional(), NEXT_PUBLIC_WEB_URL: z.string(), NEXT_PUBLIC_DOCKER_BUILD: z.string().optional(), + NEXT_PUBLIC_INFRA_FILM_URL: z.string().optional(), }, runtimeEnv: { NEXT_PUBLIC_IS_CAP: process.env.NEXT_PUBLIC_IS_CAP, @@ -19,6 +20,7 @@ const createReference = () => NEXT_PUBLIC_WEB_URL: process.env.WEB_URL ?? process.env.NEXT_PUBLIC_WEB_URL, NEXT_PUBLIC_DOCKER_BUILD: process.env.NEXT_PUBLIC_DOCKER_BUILD, + NEXT_PUBLIC_INFRA_FILM_URL: process.env.NEXT_PUBLIC_INFRA_FILM_URL, }, }); @@ -34,6 +36,7 @@ const ENV_KEYS = [ "NEXT_PUBLIC_OPENPANEL_CLIENT_ID", "NEXT_PUBLIC_OPENPANEL_API_URL", "NEXT_PUBLIC_DOCKER_BUILD", + "NEXT_PUBLIC_INFRA_FILM_URL", ] as const; type Scenario = Partial>; @@ -83,6 +86,7 @@ const PROPERTIES: Array = [ "NEXT_PUBLIC_OPENPANEL_API_URL", "NEXT_PUBLIC_WEB_URL", "NEXT_PUBLIC_DOCKER_BUILD", + "NEXT_PUBLIC_INFRA_FILM_URL", "NEXT_PUBLIC_UNKNOWN", "WEB_URL", "DATABASE_URL", diff --git a/apps/web/__tests__/unit/canonical-urls.test.ts b/apps/web/__tests__/unit/canonical-urls.test.ts index ec879541611..90d7efd3000 100644 --- a/apps/web/__tests__/unit/canonical-urls.test.ts +++ b/apps/web/__tests__/unit/canonical-urls.test.ts @@ -141,11 +141,20 @@ describe("Canonical URLs", () => { ); }); - it("dynamic blog [slug] route generates canonical from the post slug", () => { - const content = readPage("(site)/blog/[slug]/page.tsx"); - expect(content).toContain( + it("blog post routes generate canonical from the post slug", () => { + const metadata = readFileSync( + join(process.cwd(), "utils/blog-post-metadata.ts"), + "utf-8", + ); + expect(metadata).toContain( `canonical: \`https://cap.so/blog/\${post.slug}\``, ); + for (const route of [ + "(site)/blog/[slug]/page.tsx", + "(infra-post)/blog/introducing-cap-infra/page.tsx", + ]) { + expect(readPage(route)).toContain("blogPostMetadata("); + } }); it("dynamic docs route generates canonical from the slug, with /docs as the docs home", () => { diff --git a/apps/web/__tests__/unit/cap-analytics-link.test.ts b/apps/web/__tests__/unit/cap-analytics-link.test.ts new file mode 100644 index 00000000000..1bf744216d8 --- /dev/null +++ b/apps/web/__tests__/unit/cap-analytics-link.test.ts @@ -0,0 +1,147 @@ +// @vitest-environment jsdom + +import React, { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +type Children = { children?: React.ReactNode }; + +const mocks = vi.hoisted(() => ({ push: vi.fn() })); + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ push: mocks.push, refresh: vi.fn() }), +})); +vi.mock("@cap/ui", async (importOriginal) => { + const actual = await importOriginal(); + const passThrough = ({ children }: Children) => + React.createElement(React.Fragment, null, children); + return { + ...actual, + DropdownMenu: passThrough, + DropdownMenuTrigger: passThrough, + DropdownMenuContent: passThrough, + DropdownMenuItem: ({ + children, + onClick, + }: Children & { onClick?: (event: unknown) => void }) => + React.createElement( + "button", + { + type: "button", + role: "menuitem", + onClick: (event: React.MouseEvent) => onClick?.(event), + }, + children, + ), + }; +}); +vi.mock("@tanstack/react-query", () => ({ + useMutation: () => ({ mutate: vi.fn(), isPending: false }), + useQueryClient: () => ({ invalidateQueries: vi.fn() }), +})); +vi.mock("@/lib/EffectRuntime", () => ({ + useRpcClient: () => ({}), + useEffectMutation: () => ({ mutateAsync: vi.fn(), isPending: false }), +})); +vi.mock("@/app/(org)/dashboard/Contexts", () => ({ + useDashboardContext: () => ({ + activeOrganization: null, + user: { id: "owner-1" }, + setUpgradeModalOpen: vi.fn(), + }), +})); +vi.mock("@/app/s/[videoId]/_components/ProgressCircle", () => ({ + useUploadProgress: () => null, +})); +vi.mock("@/utils/public-env", () => ({ + usePublicEnv: () => ({ webUrl: "https://cap.test" }), +})); +vi.mock("@/components/FontAwesomeIcon", () => ({ + FontAwesomeIcon: () => null, +})); +vi.mock("@/components/Tooltip", () => ({ + Tooltip: ({ children }: Children) => + React.createElement(React.Fragment, null, children), +})); +vi.mock("@/components/UpgradeModal", () => ({ UpgradeModal: () => null })); +vi.mock("@/components/VideoThumbnail", () => ({ + VideoThumbnail: () => null, +})); +vi.mock("@/app/(org)/dashboard/_components/ConfirmationDialog", () => ({ + ConfirmationDialog: () => null, +})); +vi.mock("@/app/(org)/dashboard/caps/components/MoveItemsDialog", () => ({ + MoveItemsDialog: () => null, +})); +vi.mock("@/app/(org)/dashboard/caps/components/PasswordDialog", () => ({ + PasswordDialog: () => null, +})); +vi.mock("@/app/(org)/dashboard/caps/components/SettingsDialog", () => ({ + SettingsDialog: () => null, +})); +vi.mock("@/app/(org)/dashboard/caps/components/SharingDialog", () => ({ + SharingDialog: () => null, +})); +vi.mock("@/app/(org)/dashboard/caps/components/CapCard/CapCardContent", () => ({ + CapCardContent: () => null, +})); + +import { CapCard } from "@/app/(org)/dashboard/caps/components/CapCard/CapCard"; +import { capAnalyticsHref } from "@/app/(org)/dashboard/caps/components/CapCard/CapCardAnalytics"; + +let root: Root; +let container: HTMLDivElement; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + mocks.push.mockReset(); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); +}); + +const cap = { + id: "abc123", + ownerId: "owner-1", + name: "Demo", + createdAt: new Date("2026-11-01T00:00:00Z"), + totalComments: 0, + totalReactions: 0, + sharedOrganizations: [], + sharedSpaces: [], + ownerName: "Owner", + hasActiveUpload: false, +} as unknown as React.ComponentProps["cap"]; + +describe("cap card analytics", () => { + it("opens the cap's analytics page from the card menu", async () => { + await act(async () => + root.render( + React.createElement(CapCard, { + cap, + analytics: 0, + isLoadingAnalytics: false, + userId: "owner-1", + }), + ), + ); + const item = [ + ...container.querySelectorAll('[role="menuitem"]'), + ].find((element) => element.textContent?.includes("View analytics")); + if (!item) throw new Error("View analytics menu item is missing"); + + await act(async () => item.click()); + + expect(mocks.push).toHaveBeenCalledExactlyOnceWith( + "/dashboard/analytics?capId=abc123", + ); + expect(capAnalyticsHref("abc123")).toBe( + "/dashboard/analytics?capId=abc123", + ); + }); +}); diff --git a/apps/web/__tests__/unit/cap-card-copy-link.test.ts b/apps/web/__tests__/unit/cap-card-copy-link.test.ts new file mode 100644 index 00000000000..b4037a0853d --- /dev/null +++ b/apps/web/__tests__/unit/cap-card-copy-link.test.ts @@ -0,0 +1,189 @@ +// @vitest-environment jsdom + +import React, { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +type Children = { children?: React.ReactNode }; + +const mocks = vi.hoisted(() => ({ + push: vi.fn(), + copy: vi.fn(), + success: vi.fn(), + error: vi.fn(), +})); + +vi.mock("@/lib/video-share-clipboard", () => ({ + copyRichVideoLink: mocks.copy, + videoPreviewImageUrl: () => "https://cap.test/preview.png", +})); +vi.mock("sonner", () => ({ + toast: { success: mocks.success, error: mocks.error, promise: vi.fn() }, +})); + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ push: mocks.push, refresh: vi.fn() }), +})); +vi.mock("@cap/ui", async (importOriginal) => { + const actual = await importOriginal(); + const passThrough = ({ children }: Children) => + React.createElement(React.Fragment, null, children); + return { + ...actual, + DropdownMenu: passThrough, + DropdownMenuTrigger: passThrough, + DropdownMenuContent: passThrough, + DropdownMenuItem: ({ + children, + onClick, + }: Children & { onClick?: (event: unknown) => void }) => + React.createElement( + "button", + { + type: "button", + role: "menuitem", + onClick: (event: React.MouseEvent) => onClick?.(event), + }, + children, + ), + }; +}); +vi.mock("@tanstack/react-query", () => ({ + useMutation: () => ({ mutate: vi.fn(), isPending: false }), + useQueryClient: () => ({ invalidateQueries: vi.fn() }), +})); +vi.mock("@/lib/EffectRuntime", () => ({ + useRpcClient: () => ({}), + useEffectMutation: () => ({ mutateAsync: vi.fn(), isPending: false }), +})); +vi.mock("@/app/(org)/dashboard/Contexts", () => ({ + useDashboardContext: () => ({ + activeOrganization: null, + user: { id: "owner-1" }, + setUpgradeModalOpen: vi.fn(), + }), +})); +vi.mock("@/app/s/[videoId]/_components/ProgressCircle", () => ({ + useUploadProgress: () => null, +})); +vi.mock("@/utils/public-env", () => ({ + usePublicEnv: () => ({ webUrl: "https://cap.test" }), +})); +vi.mock("@/components/FontAwesomeIcon", () => ({ + FontAwesomeIcon: () => null, +})); +vi.mock("@/components/Tooltip", () => ({ + Tooltip: ({ children }: Children) => + React.createElement(React.Fragment, null, children), +})); +vi.mock("@/components/UpgradeModal", () => ({ UpgradeModal: () => null })); +vi.mock("@/components/VideoThumbnail", () => ({ + VideoThumbnail: () => null, +})); +vi.mock("@/app/(org)/dashboard/_components/ConfirmationDialog", () => ({ + ConfirmationDialog: () => null, +})); +vi.mock("@/app/(org)/dashboard/caps/components/MoveItemsDialog", () => ({ + MoveItemsDialog: () => null, +})); +vi.mock("@/app/(org)/dashboard/caps/components/PasswordDialog", () => ({ + PasswordDialog: () => null, +})); +vi.mock("@/app/(org)/dashboard/caps/components/SettingsDialog", () => ({ + SettingsDialog: () => null, +})); +vi.mock("@/app/(org)/dashboard/caps/components/SharingDialog", () => ({ + SharingDialog: () => null, +})); +vi.mock("@/app/(org)/dashboard/caps/components/CapCard/CapCardContent", () => ({ + CapCardContent: () => null, +})); + +import { CapCard } from "@/app/(org)/dashboard/caps/components/CapCard/CapCard"; + +let root: Root; +let container: HTMLDivElement; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + mocks.push.mockReset(); + mocks.copy.mockReset(); + mocks.success.mockReset(); + mocks.error.mockReset(); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); +}); + +const cap = { + id: "abc123", + ownerId: "owner-1", + name: "Demo", + createdAt: new Date("2026-11-01T00:00:00Z"), + totalComments: 0, + totalReactions: 0, + sharedOrganizations: [], + sharedSpaces: [], + ownerName: "Owner", + hasActiveUpload: false, +} as unknown as React.ComponentProps["cap"]; + +const renderCard = () => + act(async () => + root.render( + React.createElement(CapCard, { + cap, + analytics: 0, + isLoadingAnalytics: false, + userId: "owner-1", + }), + ), + ); + +const copyLinkItem = () => { + const item = [ + ...container.querySelectorAll('[role="menuitem"]'), + ].find((element) => element.textContent?.includes("Copy link")); + if (!item) throw new Error("Copy link menu item is missing"); + return item; +}; + +describe("cap card copy link", () => { + it("says the link was copied only once the clipboard accepts it", async () => { + const copy = Promise.withResolvers(); + mocks.copy.mockReturnValue(copy.promise); + await renderCard(); + const checkMark = () => + container.querySelector('path[d="M20 6 9 17l-5-5"]'); + await act(async () => copyLinkItem().click()); + + expect(mocks.copy).toHaveBeenCalledOnce(); + expect(mocks.success).not.toHaveBeenCalled(); + expect(checkMark()).toBeNull(); + + await act(async () => copy.resolve()); + + expect(mocks.success).toHaveBeenCalledExactlyOnceWith( + "Link copied to clipboard", + ); + expect(checkMark()).not.toBeNull(); + expect(mocks.error).not.toHaveBeenCalled(); + }); + + it("says so instead when the clipboard refuses the copy", async () => { + mocks.copy.mockRejectedValue(new Error("Clipboard unavailable")); + vi.spyOn(console, "error").mockImplementation(() => {}); + await renderCard(); + await act(async () => copyLinkItem().click()); + + expect(mocks.success).not.toHaveBeenCalled(); + expect(mocks.error).toHaveBeenCalledExactlyOnceWith( + "Couldn't copy the link. Please try again.", + ); + }); +}); diff --git a/apps/web/__tests__/unit/cap-player-captions.test.ts b/apps/web/__tests__/unit/cap-player-captions.test.ts new file mode 100644 index 00000000000..6713a24d055 --- /dev/null +++ b/apps/web/__tests__/unit/cap-player-captions.test.ts @@ -0,0 +1,301 @@ +// @vitest-environment jsdom + +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + activeCueText, + type CapPlayerCaption, + initialCaptions, + loadCaptionsFile, + toggledCaptions, +} from "@/components/cap-player/captions"; + +vi.mock("@/components/cap-player/source", async (original) => ({ + ...(await original()), + attachSource: () => ({ + stop: () => undefined, + warm: () => undefined, + sharpen: () => undefined, + setLevel: () => undefined, + }), +})); + +const { CapPlayer } = await import("@/components/cap-player/CapPlayer"); + +const english: CapPlayerCaption = { + language: "en", + label: "English", + url: "https://playback.example/p/captions-en.vtt", +}; +const german: CapPlayerCaption = { + language: "de", + label: "German", + url: "https://playback.example/p/captions-de.vtt", +}; +const VTT = "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHello there\n"; + +describe("caption state", () => { + it("starts off unless asked, and on the first language when asked", () => { + expect(initialCaptions([english, german], false)).toBeNull(); + expect(initialCaptions([english, german], true)).toBe("en"); + expect(initialCaptions([], true)).toBeNull(); + }); + + it("turns off when on, and back on in the language picked last", () => { + expect(toggledCaptions("en", "en", [english, german])).toBeNull(); + expect(toggledCaptions(null, "de", [english, german])).toBe("de"); + expect(toggledCaptions(null, null, [english, german])).toBe("en"); + expect(toggledCaptions(null, "fr", [english, german])).toBe("en"); + expect(toggledCaptions(null, null, [])).toBeNull(); + }); + + it("reads the text of every active cue as lines", () => { + const cues = [ + { getCueAsHTML: () => ({ textContent: " Hello &" }) }, + { text: "there" }, + { text: "" }, + ] as unknown as TextTrackCueList; + + expect(activeCueText(cues)).toBe("Hello &\nthere"); + expect(activeCueText(null)).toBe(""); + }); +}); + +describe("loading a caption file", () => { + beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(VTT, { status: 200 })), + ); + URL.createObjectURL = vi.fn(() => "blob:captions"); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("loads each file once and hands it over as a blob URL", async () => { + const cache = new Map>(); + + expect(await loadCaptionsFile(cache, english.url)).toBe("blob:captions"); + expect(await loadCaptionsFile(cache, english.url)).toBe("blob:captions"); + expect(fetch).toHaveBeenCalledTimes(1); + expect(fetch).toHaveBeenCalledWith(english.url, { credentials: "omit" }); + }); + + it("refuses a missing file or one that is not WebVTT, and tries again later", async () => { + const cache = new Map>(); + vi.mocked(fetch).mockResolvedValueOnce( + new Response("Not found", { status: 404 }), + ); + await expect(loadCaptionsFile(cache, english.url)).rejects.toThrow("404"); + expect(cache.size).toBe(0); + + vi.mocked(fetch).mockResolvedValueOnce(new Response("")); + await expect(loadCaptionsFile(cache, english.url)).rejects.toThrow( + "not WebVTT", + ); + + expect(await loadCaptionsFile(cache, english.url)).toBe("blob:captions"); + }); +}); + +describe("the captions button", () => { + let container: HTMLDivElement; + let root: Root; + + const mount = async (props: { + captions?: CapPlayerCaption[]; + showCaptions?: boolean; + controls?: boolean; + }) => { + await act(async () => { + root.render( + createElement(CapPlayer, { + source: { mp4: "https://playback.example/p/video.mp4" }, + title: "Launch demo", + ...props, + }), + ); + }); + }; + const button = () => + container.querySelector( + 'button[aria-label^="Captions"]', + ); + const player = () => container.firstElementChild as HTMLElement; + const press = async (key: string) => { + const event = new KeyboardEvent("keydown", { + key, + bubbles: true, + cancelable: true, + }); + await act(async () => { + player().dispatchEvent(event); + }); + return event; + }; + const click = async (element: Element | null) => { + await act(async () => { + element?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + }; + + beforeEach(() => { + ( + globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean } + ).IS_REACT_ACT_ENVIRONMENT = true; + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(VTT, { status: 200 })), + ); + URL.createObjectURL = vi.fn(() => "blob:captions"); + URL.revokeObjectURL = vi.fn(); + Element.prototype.scrollIntoView = () => undefined; + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); + }); + + afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); + vi.unstubAllGlobals(); + }); + + it("is not there for a video without captions, and c does nothing", async () => { + await mount({}); + + expect(button()).toBeNull(); + expect((await press("c")).defaultPrevented).toBe(false); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("starts off and loads nothing until it is pressed", async () => { + await mount({ captions: [english] }); + + expect(button()?.getAttribute("aria-pressed")).toBe("false"); + expect(fetch).not.toHaveBeenCalled(); + expect(container.querySelectorAll("track[src]")).toHaveLength(0); + + await click(button()); + + expect(button()?.getAttribute("aria-pressed")).toBe("true"); + expect(fetch).toHaveBeenCalledWith(english.url, { credentials: "omit" }); + const track = container.querySelector("track[src]"); + expect(track?.getAttribute("src")).toBe("blob:captions"); + expect(track?.kind).toBe("captions"); + expect(track?.srclang).toBe("en"); + expect(track?.label).toBe("English"); + + await click(button()); + + expect(button()?.getAttribute("aria-pressed")).toBe("false"); + }); + + it("keeps captions on through a refresh that hands over the same list", async () => { + await mount({ captions: [english], showCaptions: true }); + const track = container.querySelector("track[src]"); + + await mount({ captions: [{ ...english }], showCaptions: true }); + + expect(button()?.getAttribute("aria-pressed")).toBe("true"); + expect(fetch).toHaveBeenCalledTimes(1); + expect(container.querySelector("track[src]")).toBe(track); + }); + + it("offers captions that arrive after the player mounted", async () => { + await mount({}); + expect(button()).toBeNull(); + + await mount({ captions: [english] }); + expect(button()?.getAttribute("aria-pressed")).toBe("false"); + + await mount({ captions: [english, german] }); + expect(button()?.getAttribute("aria-label")).toBe("Captions, off"); + }); + + it("turns captions off when a refresh removes the language that was on", async () => { + await mount({ captions: [english, german] }); + await click(button()); + await click( + [...container.querySelectorAll('[role="menuitemradio"]')].find( + (item) => item.textContent === "English", + ) ?? null, + ); + expect(button()?.getAttribute("aria-label")).toBe("Captions, English"); + + await mount({ captions: [german] }); + + expect(button()?.getAttribute("aria-pressed")).toBe("false"); + await press("c"); + expect(button()?.getAttribute("aria-pressed")).toBe("true"); + }); + + it("toggles with the c key", async () => { + await mount({ captions: [english] }); + + const on = await press("c"); + expect(on.defaultPrevented).toBe(true); + expect(button()?.getAttribute("aria-pressed")).toBe("true"); + + await press("C"); + expect(button()?.getAttribute("aria-pressed")).toBe("false"); + }); + + it("starts on when asked", async () => { + await mount({ captions: [english], showCaptions: true }); + + expect(button()?.getAttribute("aria-pressed")).toBe("true"); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("keeps the c key working when the control bar is hidden", async () => { + await mount({ captions: [english], controls: false }); + + await press("c"); + + expect(fetch).toHaveBeenCalledWith(english.url, { credentials: "omit" }); + }); + + it("goes away when the file cannot be loaded", async () => { + vi.mocked(fetch).mockResolvedValue( + new Response("Not found", { status: 404 }), + ); + await mount({ captions: [english], showCaptions: true }); + + expect(button()).toBeNull(); + expect((await press("c")).defaultPrevented).toBe(false); + }); + + it("offers a language menu when there is more than one", async () => { + await mount({ captions: [english, german] }); + + expect(button()?.getAttribute("aria-label")).toBe("Captions, off"); + expect(button()?.getAttribute("aria-haspopup")).toBe("menu"); + + await click(button()); + const items = [...container.querySelectorAll('[role="menuitemradio"]')].map( + (item) => [item.textContent, item.getAttribute("aria-checked")], + ); + expect(items).toEqual([ + ["Off", "true"], + ["English", "false"], + ["German", "false"], + ]); + + await click( + [...container.querySelectorAll('[role="menuitemradio"]')].find( + (item) => item.textContent === "German", + ) ?? null, + ); + expect(button()?.getAttribute("aria-label")).toBe("Captions, German"); + expect(fetch).toHaveBeenCalledWith(german.url, { credentials: "omit" }); + + await press("c"); + expect(button()?.getAttribute("aria-label")).toBe("Captions, off"); + await press("c"); + expect(button()?.getAttribute("aria-label")).toBe("Captions, German"); + }); +}); diff --git a/apps/web/__tests__/unit/cap-player-focus.test.ts b/apps/web/__tests__/unit/cap-player-focus.test.ts new file mode 100644 index 00000000000..e6f44fbbe61 --- /dev/null +++ b/apps/web/__tests__/unit/cap-player-focus.test.ts @@ -0,0 +1,294 @@ +// @vitest-environment jsdom +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { CapPlayer } from "@/components/cap-player/CapPlayer"; + +( + globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean } +).IS_REACT_ACT_ENVIRONMENT = true; + +const SOURCE = { mp4: "https://media.example/video.mp4" }; + +let container: HTMLDivElement; +let root: Root; +let playing = false; + +beforeEach(() => { + playing = false; + vi.spyOn(HTMLMediaElement.prototype, "paused", "get").mockImplementation( + () => !playing, + ); + vi.spyOn(HTMLMediaElement.prototype, "play").mockImplementation(function ( + this: HTMLMediaElement, + ) { + playing = true; + this.dispatchEvent(new Event("play")); + return Promise.resolve(); + }); + vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(function ( + this: HTMLMediaElement, + ) { + playing = false; + this.dispatchEvent(new Event("pause")); + }); + vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation( + () => undefined, + ); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + vi.restoreAllMocks(); +}); + +const render = (props: Partial[0]> = {}) => { + act(() => + root.render( + createElement(CapPlayer, { source: SOURCE, title: "Demo", ...props }), + ), + ); + const player = container.querySelector("[data-idle]"); + const video = container.querySelector("video"); + if (!player || !video) throw new Error("player did not render"); + return { player, video }; +}; + +const press = (target: Element, key: string) => + act(() => { + target.dispatchEvent( + new KeyboardEvent("keydown", { key, bubbles: true, cancelable: true }), + ); + }); + +const click = (target: Element) => + act(() => { + target.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + +describe("cap player keyboard after a mouse play", () => { + it("takes no focus when it loads", () => { + const focus = vi.spyOn(HTMLElement.prototype, "focus"); + const { player } = render(); + expect(document.activeElement).toBe(document.body); + expect(focus).not.toHaveBeenCalled(); + expect(player.tabIndex).toBe(-1); + }); + + it("lets k pause after a click on the big play button", () => { + const { player } = render(); + const start = container.querySelector( + 'button[data-action="toggle"][aria-hidden="true"]', + ); + if (!start) throw new Error("no big play button"); + const focus = vi.spyOn(player, "focus"); + + click(start); + + expect(playing).toBe(true); + expect(focus).toHaveBeenCalledWith({ preventScroll: true }); + expect(document.activeElement).toBe(player); + expect( + container.querySelector( + 'button[data-action="toggle"][aria-hidden="true"]', + ), + ).toBeNull(); + + press(document.activeElement ?? document.body, "k"); + expect(playing).toBe(false); + }); + + it("lets space pause after a click on the picture", () => { + const { player, video } = render(); + click(video); + expect(playing).toBe(true); + expect(document.activeElement).toBe(player); + + press(player, " "); + expect(playing).toBe(false); + }); + + it("leaves focus on a bar control the viewer clicked", () => { + render(); + const barPlay = container.querySelector( + 'button[data-action="toggle"]:not([aria-hidden])', + ); + if (!barPlay) throw new Error("no bar play button"); + act(() => barPlay.focus()); + + click(barPlay); + + expect(document.activeElement).toBe(barPlay); + }); + + it("ignores keys pressed outside the player", () => { + const { video } = render(); + const field = document.createElement("input"); + document.body.append(field); + click(video); + expect(playing).toBe(true); + + act(() => field.focus()); + press(field, "k"); + expect(playing).toBe(true); + press(field, " "); + expect(playing).toBe(true); + press(document.body, "k"); + expect(playing).toBe(true); + field.remove(); + }); + + it("does not take typing from a field inside the player", () => { + const { player, video } = render(); + click(video); + const field = document.createElement("input"); + player.append(field); + + press(field, "k"); + expect(playing).toBe(true); + press(field, " "); + expect(playing).toBe(true); + field.remove(); + }); + + it("ignores keys when the embed turns them off", () => { + const { player, video } = render({ keyboard: false }); + click(video); + press(player, "k"); + expect(playing).toBe(true); + }); +}); + +const box = (left: number, top: number, width: number, height: number) => + ({ + left, + top, + width, + height, + right: left + width, + bottom: top + height, + x: left, + y: top, + toJSON: () => undefined, + }) as DOMRect; + +describe("cap player menus in a short player", () => { + let resized: (() => void) | null = null; + let size = { width: 342, height: 192 }; + + beforeEach(() => { + resized = null; + size = { width: 342, height: 192 }; + HTMLElement.prototype.scrollIntoView = () => undefined; + vi.stubGlobal( + "ResizeObserver", + class { + constructor(callback: () => void) { + resized = callback; + } + observe() {} + disconnect() { + resized = null; + } + }, + ); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + // listHeight is the menu's height as a list above the bar, sheetHeight + // the height of its rows laid out as a sheet. + const openSpeedMenu = (listHeight: number, sheetHeight: number) => { + const { player } = render(); + vi.spyOn(player, "clientHeight", "get").mockImplementation( + () => size.height, + ); + vi.spyOn(player, "getBoundingClientRect").mockImplementation(() => + box(0, 0, size.width, size.height), + ); + const anchor = container.querySelector( + '[data-menu-anchor="speed"]', + ); + if (!anchor) throw new Error("no speed menu"); + vi.spyOn(anchor, "getBoundingClientRect").mockImplementation(() => + box(size.width - 122, size.height - 50, 50, 44), + ); + vi.spyOn(HTMLElement.prototype, "offsetParent", "get").mockReturnValue( + anchor, + ); + vi.spyOn(HTMLElement.prototype, "scrollHeight", "get").mockImplementation( + function (this: HTMLElement) { + return this.dataset.sheet ? sheetHeight : listHeight; + }, + ); + const button = anchor.querySelector("button"); + if (!button) throw new Error("no speed button"); + click(button); + const menu = container.querySelector('[role="menu"]'); + if (!menu) throw new Error("menu did not open"); + return { player, menu }; + }; + + it("opens a sheet as tall as its rows, centred in a 192 px player", () => { + const { player, menu } = openSpeedMenu(274, 142); + expect(player.dataset.menu).toBe("speed"); + expect(menu.dataset.sheet).toBe("true"); + expect(menu.style.left).toBe("-214px"); + expect(menu.style.width).toBe("330px"); + expect(menu.style.height).toBe("142px"); + expect(menu.style.top).toBe("-117px"); + expect(menu.style.maxHeight).toBe("none"); + }); + + it("fills a 240 by 135 player and scrolls from the top when the rows are taller", () => { + size = { width: 240, height: 135 }; + const { menu } = openSpeedMenu(274, 274); + expect(menu.dataset.sheet).toBe("true"); + expect(menu.style.width).toBe("228px"); + expect(menu.style.height).toBe("123px"); + expect(menu.style.top).toBe(`${6 - 85}px`); + expect(menu.querySelector("button")?.textContent).toBe("0.5×"); + }); + + it("keeps the list above the bar when it fits", () => { + size = { width: 1136, height: 600 }; + const { menu } = openSpeedMenu(274, 142); + expect(menu.dataset.sheet).toBeUndefined(); + expect(menu.style.maxHeight).toBe("536px"); + expect(menu.style.left).toBe(""); + }); + + it("places an open menu again when the player changes size", () => { + const { menu } = openSpeedMenu(274, 142); + expect(menu.dataset.sheet).toBe("true"); + expect(resized).not.toBeNull(); + + size = { width: 240, height: 135 }; + act(() => resized?.()); + expect(menu.style.width).toBe("228px"); + expect(menu.style.height).toBe("123px"); + expect(menu.style.left).toBe("-112px"); + + size = { width: 1136, height: 600 }; + act(() => resized?.()); + expect(menu.dataset.sheet).toBeUndefined(); + expect(menu.style.maxHeight).toBe("536px"); + expect(menu.style.left).toBe(""); + expect(menu.style.top).toBe(""); + expect(menu.style.width).toBe(""); + expect(menu.style.height).toBe(""); + }); + + it("stops watching the player when the menu closes", () => { + const { menu } = openSpeedMenu(274, 142); + press(menu.querySelector("button") ?? menu, "Escape"); + expect(container.querySelector('[role="menu"]')).toBeNull(); + expect(resized).toBeNull(); + }); +}); diff --git a/apps/web/__tests__/unit/cap-player-messages.test.ts b/apps/web/__tests__/unit/cap-player-messages.test.ts new file mode 100644 index 00000000000..ed3e6273844 --- /dev/null +++ b/apps/web/__tests__/unit/cap-player-messages.test.ts @@ -0,0 +1,383 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + connectMessages, + createThrottle, + type MessageTarget, + parseCommand, + qualityKey, + TIMEUPDATE_GAP_MS, +} from "@/components/cap-player/messages"; + +class FakeVideo extends EventTarget { + currentTime = 0; + duration = 30; + paused = true; + muted = false; + plays = 0; + refuse = false; + currentSrc = "blob:attached"; + + getAttribute() { + return null; + } + + play() { + this.plays++; + if (this.refuse) { + const error = new Error("refused"); + error.name = "NotAllowedError"; + return Promise.reject(error); + } + this.paused = false; + return Promise.resolve(); + } + + pause() { + this.paused = true; + } +} + +const embedded = ( + options: { seekDuringSwitch?: (time: number) => boolean } = {}, +) => { + const posted: { message: Record; origin: string }[] = []; + const parent = { + postMessage: (message: unknown, origin: string) => { + posted.push({ message: message as Record, origin }); + }, + }; + const view = new EventTarget(); + const target = Object.assign(view, { parent }) as unknown as MessageTarget; + const video = new FakeVideo(); + const setQuality = vi.fn(); + const connection = connectMessages( + target, + video as unknown as HTMLVideoElement, + { setQuality, seekDuringSwitch: options.seekDuringSwitch }, + ); + const command = (data: unknown, source: unknown = parent) => + view.dispatchEvent( + Object.assign(new Event("message"), { data, source }) as MessageEvent, + ); + const events = () => posted.map(({ message }) => message.event); + return { posted, video, setQuality, connection, command, events, parent }; +}; + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +describe("parseCommand", () => { + it.each([ + [{ type: "cap-player", command: "play" }, { command: "play" }], + [{ type: "cap-player", command: "pause" }, { command: "pause" }], + [{ type: "cap-player", command: "mute" }, { command: "mute" }], + [{ type: "cap-player", command: "unmute" }, { command: "unmute" }], + [ + { type: "cap-player", command: "seek", time: 12.5 }, + { command: "seek", time: 12.5 }, + ], + [ + { type: "cap-player", command: "seek", time: 0 }, + { command: "seek", time: 0 }, + ], + [ + { type: "cap-player", command: "setQuality", quality: "720p" }, + { command: "setQuality", quality: "720p" }, + ], + [ + { type: "cap-player", command: "setQuality", quality: 1080 }, + { command: "setQuality", quality: "1080" }, + ], + [ + { type: "cap-player", command: "setQuality", quality: "Auto" }, + { command: "setQuality", quality: "auto" }, + ], + [ + { type: "cap-player", command: "setQuality", quality: "4K" }, + { command: "setQuality", quality: "4k" }, + ], + ])("accepts %j", (data, parsed) => { + expect(parseCommand(data)).toEqual(parsed); + }); + + it.each([ + null, + undefined, + "play", + 42, + [], + ["cap-player", "play"], + { command: "play" }, + { type: "other", command: "play" }, + { type: "cap-player" }, + { type: "cap-player", command: "PLAY" }, + { type: "cap-player", command: "destroy" }, + { type: "cap-player", command: "__proto__" }, + { type: "cap-player", command: "toString" }, + { type: "cap-player", command: { toString: (): string => "play" } }, + { type: "cap-player", command: "seek" }, + { type: "cap-player", command: "seek", time: "12" }, + { type: "cap-player", command: "seek", time: -1 }, + { type: "cap-player", command: "seek", time: Number.NaN }, + { type: "cap-player", command: "seek", time: Number.POSITIVE_INFINITY }, + { type: "cap-player", command: "seek", time: 1e12 }, + { type: "cap-player", command: "setQuality" }, + { type: "cap-player", command: "setQuality", quality: "" }, + { type: "cap-player", command: "setQuality", quality: "best" }, + { type: "cap-player", command: "setQuality", quality: "720p;x" }, + { + type: "cap-player", + command: "setQuality", + quality: "", + }, + { type: "cap-player", command: "setQuality", quality: { label: "720p" } }, + { type: "cap-player", command: "setQuality", quality: 1.5 }, + ])("refuses %j", (data) => { + expect(parseCommand(data)).toBeNull(); + }); +}); + +describe("qualityKey", () => { + it("treats the ways of naming one quality as the same", () => { + expect(qualityKey("1080p")).toBe(qualityKey("1080")); + expect(qualityKey("4K")).toBe(qualityKey("2160p")); + expect(qualityKey("4k")).toBe(qualityKey("2160")); + expect(qualityKey("720p")).not.toBe(qualityKey("1080p")); + }); +}); + +describe("createThrottle", () => { + it("lets at most one event through in each gap", () => { + const pass = createThrottle(TIMEUPDATE_GAP_MS); + const passed = [0, 50, 100, 239, 240, 300, 480, 481, 1000].filter(pass); + + expect(passed).toEqual([0, 240, 480, 1000]); + }); + + it("allows about four a second for events every frame", () => { + const pass = createThrottle(TIMEUPDATE_GAP_MS); + const frames = Array.from({ length: 60 }, (_, index) => index * 16.7); + + expect(frames.filter(pass)).toEqual([0, 250.5, 501, 751.5]); + }); +}); + +describe("connectMessages", () => { + it("says it is ready, to any origin, with the playback state only", () => { + const { posted } = embedded(); + + expect(posted).toEqual([ + { + message: { + type: "cap-player", + event: "ready", + currentTime: 0, + duration: 30, + paused: true, + muted: false, + }, + origin: "*", + }, + ]); + }); + + it("passes on play, pause and ended", () => { + const { video, events } = embedded(); + + video.dispatchEvent(new Event("play")); + video.dispatchEvent(new Event("pause")); + video.dispatchEvent(new Event("ended")); + + expect(events()).toEqual(["ready", "play", "pause", "ended"]); + }); + + it("passes on time updates at most about four times a second", () => { + let now = 0; + vi.stubGlobal("performance", { now: () => now }); + const { video, posted } = embedded(); + + for (let frame = 0; frame < 60; frame++) { + now = frame * 16.7; + video.currentTime = now / 1000; + video.dispatchEvent(new Event("timeupdate")); + } + + const updates = posted.filter( + ({ message }) => message.event === "timeupdate", + ); + expect(updates).toHaveLength(4); + expect(updates[1]?.message.currentTime).toBe(0.251); + }); + + it("reports a duration that is not known yet as zero", () => { + const { video, connection, posted } = embedded(); + video.duration = Number.NaN; + + connection.send("qualitychange", { quality: "720p", auto: true }); + + expect(posted.at(-1)?.message).toMatchObject({ + event: "qualitychange", + duration: 0, + quality: "720p", + auto: true, + }); + }); + + it("follows commands from the embedding page", async () => { + const { video, command, setQuality } = embedded(); + + command({ type: "cap-player", command: "play" }); + await Promise.resolve(); + expect(video.paused).toBe(false); + + command({ type: "cap-player", command: "pause" }); + expect(video.paused).toBe(true); + + command({ type: "cap-player", command: "mute" }); + expect(video.muted).toBe(true); + command({ type: "cap-player", command: "unmute" }); + expect(video.muted).toBe(false); + + command({ type: "cap-player", command: "seek", time: 12 }); + expect(video.currentTime).toBe(12); + command({ type: "cap-player", command: "seek", time: 999 }); + expect(video.currentTime).toBe(30); + + command({ type: "cap-player", command: "setQuality", quality: "720p" }); + expect(setQuality).toHaveBeenCalledWith("720p"); + }); + + it("keeps a seek sent before the duration is known until the metadata arrives", () => { + const { video, command } = embedded(); + video.duration = Number.NaN; + + command({ type: "cap-player", command: "seek", time: 12 }); + expect(video.currentTime).toBe(0); + + video.duration = 30; + video.dispatchEvent(new Event("loadedmetadata")); + expect(video.currentTime).toBe(12); + + video.currentTime = 20; + video.dispatchEvent(new Event("durationchange")); + expect(video.currentTime).toBe(20); + }); + + it("keeps only the last early seek, and drops it once a seek lands", () => { + const { video, command } = embedded(); + video.duration = Number.NaN; + + command({ type: "cap-player", command: "seek", time: 12 }); + command({ type: "cap-player", command: "seek", time: 50 }); + video.duration = 30; + video.dispatchEvent(new Event("durationchange")); + + expect(video.currentTime).toBe(30); + }); + + it("holds a play sent before the source is attached until it is", async () => { + const { video, command, events } = embedded(); + video.currentSrc = ""; + + command({ type: "cap-player", command: "play" }); + await Promise.resolve(); + expect(video.plays).toBe(0); + + video.currentSrc = "blob:attached"; + video.dispatchEvent(new Event("loadedmetadata")); + await Promise.resolve(); + + expect(video.plays).toBe(1); + expect(video.paused).toBe(false); + expect(events()).toEqual(["ready"]); + }); + + it("drops a held play when a pause follows it", async () => { + const { video, command } = embedded(); + video.currentSrc = ""; + + command({ type: "cap-player", command: "play" }); + command({ type: "cap-player", command: "pause" }); + video.currentSrc = "blob:attached"; + video.dispatchEvent(new Event("loadedmetadata")); + await Promise.resolve(); + + expect(video.plays).toBe(0); + }); + + it("hands a seek to a quality switch that is loading", () => { + const seekDuringSwitch = vi.fn(() => true); + const { video, command } = embedded({ seekDuringSwitch }); + video.duration = Number.NaN; + + command({ type: "cap-player", command: "seek", time: 8 }); + video.duration = 30; + video.dispatchEvent(new Event("loadedmetadata")); + + expect(seekDuringSwitch).toHaveBeenCalledWith(8); + expect(video.currentTime).toBe(0); + }); + + it("ignores messages from any window but the embedding page", async () => { + const { video, command } = embedded(); + + command({ type: "cap-player", command: "play" }, null); + command({ type: "cap-player", command: "play" }, {}); + await Promise.resolve(); + + expect(video.plays).toBe(0); + }); + + it("ignores malformed commands", () => { + const { video, command, setQuality, posted } = embedded(); + + command({ type: "cap-player", command: "seek", time: "1e3" }); + command({ type: "cap-player", command: "setQuality", quality: "x" }); + command("play"); + + expect(video.currentTime).toBe(0); + expect(setQuality).not.toHaveBeenCalled(); + expect(posted).toHaveLength(1); + }); + + it("reports a play the browser refuses", async () => { + const { video, command, posted } = embedded(); + video.refuse = true; + + command({ type: "cap-player", command: "play" }); + await vi.waitFor(() => expect(posted).toHaveLength(2)); + + expect(posted[1]?.message).toMatchObject({ + event: "error", + code: "play_refused", + }); + }); + + it("does not report a play that a pause interrupted", async () => { + const { video, command, posted } = embedded(); + video.play = () => { + const error = new Error("interrupted"); + error.name = "AbortError"; + return Promise.reject(error); + }; + + command({ type: "cap-player", command: "play" }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(posted).toHaveLength(1); + }); + + it("stops listening and passing events on once stopped", async () => { + const { video, command, connection, posted } = embedded(); + + connection.stop(); + video.dispatchEvent(new Event("play")); + command({ type: "cap-player", command: "play" }); + await Promise.resolve(); + + expect(posted).toHaveLength(1); + expect(video.plays).toBe(0); + }); +}); diff --git a/apps/web/__tests__/unit/cap-player-quality-label.test.ts b/apps/web/__tests__/unit/cap-player-quality-label.test.ts new file mode 100644 index 00000000000..9faa8522a41 --- /dev/null +++ b/apps/web/__tests__/unit/cap-player-quality-label.test.ts @@ -0,0 +1,153 @@ +// @vitest-environment jsdom +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +( + globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean } +).IS_REACT_ACT_ENVIRONMENT = true; + +const LEVELS = [ + { index: 2, label: "1080p", width: 1920, height: 1080, bitrate: 3_000_000 }, + { index: 1, label: "720p", width: 1280, height: 720, bitrate: 1_500_000 }, + { index: 0, label: "360p", width: 640, height: 360, bitrate: 500_000 }, +]; + +vi.mock("@/components/cap-player/source", async (original) => ({ + ...(await original()), + attachSource: ( + _video: HTMLVideoElement, + _source: unknown, + options: { onLevels?: (levels: typeof LEVELS) => void }, + ) => { + queueMicrotask(() => options.onLevels?.(LEVELS)); + return { + stop: () => undefined, + warm: () => undefined, + setLevel: () => undefined, + seekDuringSwitch: () => false, + heldTime: () => null, + takeSwitchPlay: () => false, + }; + }, +})); + +const { CapPlayer } = await import("@/components/cap-player/CapPlayer"); + +let container: HTMLDivElement; +let root: Root; +let size = { width: 640, height: 360 }; + +beforeEach(() => { + size = { width: 640, height: 360 }; + HTMLElement.prototype.scrollIntoView = () => undefined; + vi.spyOn(HTMLVideoElement.prototype, "videoWidth", "get").mockImplementation( + () => size.width, + ); + vi.spyOn(HTMLVideoElement.prototype, "videoHeight", "get").mockImplementation( + () => size.height, + ); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + vi.restoreAllMocks(); +}); + +const render = async (autoPlay = false) => { + act(() => + root.render( + createElement(CapPlayer, { + source: { hls: "https://media.example/master.m3u8" }, + title: "Demo", + autoPlay, + }), + ), + ); + await act(async () => { + await Promise.resolve(); + }); + const video = container.querySelector("video"); + if (!video) throw new Error("no video"); + act(() => { + video.dispatchEvent(new Event("resize")); + }); + return video; +}; + +const button = () => + container.querySelector( + '[data-menu-anchor="quality"] > button', + ); + +describe("cap player quality label before play", () => { + it("reads Auto, not the rendition held for the first frame", async () => { + await render(); + + expect(button()?.textContent).toBe("Auto"); + expect(button()?.getAttribute("aria-label")).toBe("Quality, auto"); + }); + + it("leaves the Auto row without a rendition hint until playback starts", async () => { + await render(); + const quality = button(); + if (!quality) throw new Error("no quality button"); + act(() => quality.click()); + + const auto = container.querySelector('[role="menuitemradio"]'); + expect(auto?.textContent).toBe("Auto"); + }); + + it("names the rendition on screen once the video plays", async () => { + const video = await render(); + act(() => { + video.dispatchEvent(new Event("play")); + }); + + expect(button()?.textContent).toBe("360p"); + expect(button()?.getAttribute("aria-label")).toBe("Quality, auto, 360p"); + size = { width: 1280, height: 720 }; + act(() => { + video.dispatchEvent(new Event("resize")); + }); + expect(button()?.textContent).toBe("720p"); + }); + + it("names the rendition on screen until a pick reaches it", async () => { + const video = await render(); + act(() => { + video.dispatchEvent(new Event("play")); + }); + const quality = button(); + if (!quality) throw new Error("no quality button"); + act(() => quality.click()); + const item = [ + ...container.querySelectorAll( + '[role="menuitemradio"]', + ), + ].find((entry) => entry.textContent?.startsWith("720p")); + if (!item) throw new Error("no 720p row"); + act(() => item.click()); + + expect(button()?.textContent).toBe("360p"); + expect(button()?.getAttribute("aria-label")).toBe( + "Quality, 720p, switching from 360p", + ); + size = { width: 1280, height: 720 }; + act(() => { + video.dispatchEvent(new Event("resize")); + }); + expect(button()?.textContent).toBe("720p"); + expect(button()?.getAttribute("aria-label")).toBe("Quality, 720p"); + }); + + it("names the rendition from the start when the player autoplays", async () => { + await render(true); + + expect(button()?.textContent).toBe("360p"); + }); +}); diff --git a/apps/web/__tests__/unit/cap-player-replay.test.ts b/apps/web/__tests__/unit/cap-player-replay.test.ts new file mode 100644 index 00000000000..01145eeaa52 --- /dev/null +++ b/apps/web/__tests__/unit/cap-player-replay.test.ts @@ -0,0 +1,189 @@ +// @vitest-environment jsdom +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { CapPlayer } from "@/components/cap-player/CapPlayer"; + +( + globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean } +).IS_REACT_ACT_ENVIRONMENT = true; + +const SOURCE = { mp4: "https://media.example/video.mp4" }; + +let container: HTMLDivElement; +let root: Root; +let playing = false; +let play: ReturnType; + +beforeEach(() => { + playing = false; + vi.spyOn(HTMLMediaElement.prototype, "paused", "get").mockImplementation( + () => !playing, + ); + play = vi.fn(function (this: HTMLMediaElement) { + playing = true; + this.dispatchEvent(new Event("play")); + return Promise.resolve(); + }); + vi.spyOn(HTMLMediaElement.prototype, "play").mockImplementation( + play as unknown as () => Promise, + ); + vi.spyOn(HTMLMediaElement.prototype, "pause").mockImplementation(function ( + this: HTMLMediaElement, + ) { + playing = false; + this.dispatchEvent(new Event("pause")); + }); + vi.spyOn(HTMLMediaElement.prototype, "load").mockImplementation( + () => undefined, + ); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + vi.restoreAllMocks(); +}); + +const render = (props: Partial[0]> = {}) => { + act(() => + root.render( + createElement(CapPlayer, { source: SOURCE, title: "Demo", ...props }), + ), + ); + const player = container.querySelector("[data-idle]"); + const video = container.querySelector("video"); + if (!player || !video) throw new Error("player did not render"); + return { player, video }; +}; + +const fire = (target: EventTarget, name: string) => + act(() => { + target.dispatchEvent(new Event(name)); + }); + +const reachEnd = (video: HTMLVideoElement) => { + act(() => { + void video.play(); + }); + playing = false; + fire(video, "pause"); + fire(video, "ended"); +}; + +const replay = () => + container.querySelector('button[aria-label="Replay"]'); + +describe("cap player replay at the end", () => { + it("offers nothing before the video ends", () => { + const { video } = render(); + expect(replay()).toBeNull(); + act(() => { + void video.play(); + }); + expect(replay()).toBeNull(); + }); + + it("shows a replay button where the big play button sits once the video ends", () => { + const { video } = render(); + reachEnd(video); + const button = replay(); + expect(button).not.toBeNull(); + expect(button?.dataset.action).toBe("toggle"); + expect(button?.getAttribute("aria-hidden")).toBeNull(); + expect(button?.tabIndex).toBe(0); + expect(button?.className).toContain("start"); + }); + + it("comes before the control bar in the tab order", () => { + const { video } = render(); + reachEnd(video); + const focusable = [ + ...container.querySelectorAll("button, [tabindex='0']"), + ].filter((element) => element.tabIndex >= 0); + expect(focusable[0]).toBe(replay()); + }); + + it("plays again from a click or Enter, and nothing else", () => { + const { player, video } = render(); + reachEnd(video); + const button = replay(); + if (!button) throw new Error("no replay button"); + act(() => button.focus()); + play.mockClear(); + const setTime = vi.spyOn(video, "currentTime", "set"); + + act(() => { + button.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + + expect(play).toHaveBeenCalledTimes(1); + expect(setTime).not.toHaveBeenCalled(); + expect(replay()).toBeNull(); + expect(document.activeElement).toBe(player); + }); + + it("keeps the keyboard on the player when k replays from the focused button", () => { + const { player, video } = render(); + reachEnd(video); + const button = replay(); + if (!button) throw new Error("no replay button"); + act(() => button.focus()); + + act(() => { + button.dispatchEvent( + new KeyboardEvent("keydown", { + key: "k", + bubbles: true, + cancelable: true, + }), + ); + }); + + expect(playing).toBe(true); + expect(replay()).toBeNull(); + expect(document.activeElement).toBe(player); + act(() => { + player.dispatchEvent( + new KeyboardEvent("keydown", { + key: "k", + bubbles: true, + cancelable: true, + }), + ); + }); + expect(playing).toBe(false); + }); + + it("goes away when the viewer seeks back from the end", () => { + const { video } = render(); + reachEnd(video); + expect(replay()).not.toBeNull(); + fire(video, "seeking"); + expect(replay()).toBeNull(); + }); + + it("is not shown without controls, where the cover button already plays", () => { + const { video } = render({ controls: false }); + reachEnd(video); + expect(replay()).toBeNull(); + }); + + it("is not shown when the video failed", () => { + const { video } = render(); + reachEnd(video); + fire(video, "error"); + act(() => { + root.render( + createElement(CapPlayer, { + source: { mp4: "" }, + title: "Demo", + }), + ); + }); + expect(replay()).toBeNull(); + }); +}); diff --git a/apps/web/__tests__/unit/cap-player-source.test.ts b/apps/web/__tests__/unit/cap-player-source.test.ts index cc66dee310e..98ff35e4623 100644 --- a/apps/web/__tests__/unit/cap-player-source.test.ts +++ b/apps/web/__tests__/unit/cap-player-source.test.ts @@ -1,11 +1,23 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { + connectMessages, + type MessageTarget, +} from "@/components/cap-player/messages"; +import { + AUTO_LEVEL, attachSource, + isSharper, + listLevels, + parseMasterLevels, pickPlaybackMode, pickStartingLevel, playAllowingMuted, + renditionSource, SECONDS_AHEAD_BEFORE_PLAY, + SECONDS_TO_SHARPEN, + startingEstimate, startWithin, + withShortPlaylists, } from "@/components/cap-player/source"; type Listener = (event: string, data: unknown) => void; @@ -17,20 +29,30 @@ const hlsMock = vi.hoisted(() => { static Events = { MANIFEST_PARSED: "manifestParsed", LEVEL_LOADED: "levelLoaded", + FRAG_BUFFERED: "fragBuffered", ERROR: "error", }; static ErrorTypes = { MEDIA_ERROR: "mediaError" }; static DefaultConfig = { maxMaxBufferLength: 600 }; - config: { maxMaxBufferLength: number }; + config: { + maxMaxBufferLength: number; + abrEwmaDefaultEstimate?: number; + }; startLevel = -1; loadLevel = -1; + currentLevel = -1; + levels: unknown[] = []; + bandwidthEstimate = Number.NaN; startLoads: number[] = []; source = ""; destroyed = false; private listeners = new Map(); - constructor(config: { maxMaxBufferLength: number }) { + constructor(config: { + maxMaxBufferLength: number; + abrEwmaDefaultEstimate?: number; + }) { this.config = { ...config }; FakeHls.instances.push(this); } @@ -51,6 +73,9 @@ const hlsMock = vi.hoisted(() => { } emit(name: string, data: unknown) { + if (name === "manifestParsed") { + this.levels = (data as { levels: unknown[] }).levels; + } for (const listener of this.listeners.get(name) ?? []) { listener(name, data); } @@ -78,54 +103,194 @@ const hlsMock = vi.hoisted(() => { vi.mock("hls.js", () => ({ default: hlsMock.FakeHls })); const LADDER = [ - { bitrate: 450_000, width: 640 }, - { bitrate: 3_400_000, width: 1280 }, - { bitrate: 6_650_000, width: 1920 }, - { bitrate: 11_400_000, width: 2560 }, - { bitrate: 19_400_000, width: 3840 }, + { bitrate: 450_000, width: 640, height: 360 }, + { bitrate: 3_400_000, width: 1280, height: 720 }, + { bitrate: 6_650_000, width: 1920, height: 1080 }, + { bitrate: 11_400_000, width: 2560, height: 1440 }, + { bitrate: 19_400_000, width: 3840, height: 2160 }, ]; +// A constant quality ladder of a simple video: 1080p carries fewer bits than +// 720p, so hls.js, which lists levels by bitrate, puts 1080p before 720p. +const UNEVEN_LADDER = [ + { bitrate: 740_000, width: 640, height: 360, name: "360p" }, + { bitrate: 2_300_000, width: 1920, height: 1080, name: "1080p" }, + { bitrate: 4_800_000, width: 1280, height: 720, name: "720p" }, +]; + +// A player as wide as `width`, judged on width alone. +const wide = (width: number) => ({ width, height: Number.POSITIVE_INFINITY }); + const video = (nativeHls: boolean) => ({ canPlayType: () => (nativeHls ? "maybe" : ""), }) as unknown as HTMLVideoElement; -const browser = (options: { vendor: string; mediaSource: boolean }) => { - vi.stubGlobal("navigator", { vendor: options.vendor }); - vi.stubGlobal("window", options.mediaSource ? { MediaSource: class {} } : {}); +const browser = (options: { + vendor: string; + mediaSource: boolean; + managedOnly?: boolean; + touchPoints?: number; +}) => { + vi.stubGlobal("navigator", { + vendor: options.vendor, + maxTouchPoints: options.touchPoints ?? 0, + }); + vi.stubGlobal("window", { + ...(options.mediaSource ? { MediaSource: class {} } : {}), + ...(options.managedOnly ? { ManagedMediaSource: class {} } : {}), + }); }; +beforeEach(() => { + vi.stubGlobal( + "fetch", + vi.fn(() => Promise.reject(new Error("offline"))), + ); +}); + afterEach(() => { vi.unstubAllGlobals(); }); describe("pickStartingLevel", () => { it("starts at the lowest rendition on a slow connection", () => { - expect(pickStartingLevel(LADDER, 1920, 1_400_000)).toBe(0); + expect(pickStartingLevel(LADDER, wide(1920), 1_400_000)).toBe(0); }); it("stops at the first rendition that covers the player", () => { - expect(pickStartingLevel(LADDER, 1408, 8_000_000)).toBe(2); - expect(pickStartingLevel(LADDER, 600, 50_000_000)).toBe(0); - expect(pickStartingLevel(LADDER, 1280, 50_000_000)).toBe(1); + expect(pickStartingLevel(LADDER, wide(1408), 8_000_000)).toBe(2); + expect(pickStartingLevel(LADDER, wide(600), 50_000_000)).toBe(0); + expect(pickStartingLevel(LADDER, wide(1280), 50_000_000)).toBe(1); + }); + + it("counts a rendition as covering when it is as tall as the player", () => { + expect( + pickStartingLevel(LADDER, { width: 5000, height: 700 }, 50_000_000), + ).toBe(1); }); it("never picks a rendition the connection cannot carry", () => { - expect(pickStartingLevel(LADDER, 3840, 8_000_000)).toBe(2); - expect(pickStartingLevel(LADDER, 3840, 50_000_000)).toBe(4); + expect(pickStartingLevel(LADDER, wide(3840), 8_000_000)).toBe(2); + expect(pickStartingLevel(LADDER, wide(3840), 50_000_000)).toBe(4); + }); + + it("starts a 1136 by 639 player on 720p with the default estimate", () => { + expect( + pickStartingLevel(LADDER, { width: 1136, height: 639 }, 5_000_000), + ).toBe(1); + }); + + it("chooses by resolution when bitrate does not rise with it", () => { + const player = { width: 1136, height: 639 }; + expect(pickStartingLevel(UNEVEN_LADDER, player, 5_000_000)).toBe(2); + expect(pickStartingLevel(UNEVEN_LADDER, player, 3_000_000)).toBe(0); + expect( + pickStartingLevel( + UNEVEN_LADDER, + { width: 1920, height: 1080 }, + 5_000_000, + ), + ).toBe(1); + expect(pickStartingLevel(UNEVEN_LADDER, wide(600), 50_000_000)).toBe(0); }); it("handles a single rendition and an empty ladder", () => { - expect(pickStartingLevel(LADDER.slice(0, 1), 3840, 50_000_000)).toBe(0); - expect(pickStartingLevel([], 3840, 50_000_000)).toBe(0); + expect(pickStartingLevel(LADDER.slice(0, 1), wide(3840), 50_000_000)).toBe( + 0, + ); + expect(pickStartingLevel([], wide(3840), 50_000_000)).toBe(0); + }); +}); + +describe("isSharper", () => { + it("compares resolution, not the order hls.js lists levels in", () => { + expect(isSharper(UNEVEN_LADDER, 1, 2)).toBe(true); + expect(isSharper(UNEVEN_LADDER, 2, 1)).toBe(false); + expect(isSharper(UNEVEN_LADDER, 2, 0)).toBe(true); + expect(isSharper(UNEVEN_LADDER, 1, 1)).toBe(false); + expect(isSharper(UNEVEN_LADDER, 0, -1)).toBe(true); + }); +}); + +describe("listLevels", () => { + it("lists the quality menu by resolution when bitrate does not rise with it", () => { + expect(listLevels(UNEVEN_LADDER).map((level) => level.label)).toEqual([ + "1080p", + "720p", + "360p", + ]); + expect(listLevels(UNEVEN_LADDER).map((level) => level.index)).toEqual([ + 1, 2, 0, + ]); + }); +}); + +describe("startingEstimate", () => { + const hint = (connection?: object) => + vi.stubGlobal("navigator", { vendor: "", connection }); + + it("starts as low as Chrome's reading when the browser gives no hint", () => { + hint(); + expect(startingEstimate()).toBe(1_200_000); + }); + + it("picks the smallest rendition of a 1136 px hero with no hint, so a slow line shows a frame soon", () => { + hint(); + expect( + pickStartingLevel( + LADDER, + { width: 1136, height: 639 }, + startingEstimate(), + ), + ).toBe(0); + }); + + it("takes the browser's reading, never above the default", () => { + hint({ downlink: 1.6, effectiveType: "4g" }); + expect(startingEstimate()).toBe(1_280_000); + hint({ downlink: 0.4 }); + expect(startingEstimate()).toBe(320_000); + hint({ downlink: 10, effectiveType: "4g" }); + expect(startingEstimate()).toBe(5_000_000); + hint({ effectiveType: "3g", downlink: 10 }); + expect(startingEstimate()).toBe(5_000_000); + }); + + it("starts low on a slow line with no reading, and with data saver", () => { + hint({ effectiveType: "slow-2g" }); + expect(startingEstimate()).toBe(500_000); + hint({ effectiveType: "4g" }); + expect(startingEstimate()).toBe(5_000_000); + hint({ saveData: true, downlink: 10 }); + expect(startingEstimate()).toBe(500_000); }); }); describe("pickPlaybackMode", () => { const both = { hls: "/film/master.m3u8", mp4: "/film/video.mp4" }; - it("plays HLS natively on Apple browsers", () => { + it("runs hls.js in Safari on a Mac", () => { browser({ vendor: "Apple Computer, Inc.", mediaSource: true }); + expect(pickPlaybackMode(video(true), both)).toBe("mse"); + }); + + it("plays HLS natively on an iPhone, which has only a managed media source", () => { + browser({ + vendor: "Apple Computer, Inc.", + mediaSource: false, + managedOnly: true, + touchPoints: 5, + }); + expect(pickPlaybackMode(video(true), both)).toBe("native"); + }); + + it("plays HLS natively on an iPad, which has a media source and touch", () => { + browser({ + vendor: "Apple Computer, Inc.", + mediaSource: true, + touchPoints: 5, + }); expect(pickPlaybackMode(video(true), both)).toBe("native"); }); @@ -150,6 +315,7 @@ describe("pickPlaybackMode", () => { }); class FakeVideo extends EventTarget { + playbackRate = 1; preload = "none"; src = ""; paused = true; @@ -157,6 +323,7 @@ class FakeVideo extends EventTarget { currentTime = 0; duration = Number.NaN; clientWidth = 1280; + clientHeight = 720; plays = 0; refusals: string[] = []; @@ -195,18 +362,35 @@ class FakeVideo extends EventTarget { const HLS = "https://media.example/film/master.m3u8"; const MP4 = "https://media.example/film/video.mp4"; +const SHORT_HLS = `${HLS}?segments=short`; +const PLAYED_HLS = `${HLS}?subtitles=none`; +const PLAYED_SHORT_HLS = `${PLAYED_HLS}&segments=short`; const LADDER_DATA = { levels: LADDER }; const page = (options: { vendor: string; mediaSource: boolean; loaded: boolean; + touchPoints?: number; + megabits?: number; + effectiveType?: string; }) => { const view = new EventTarget(); Object.assign(view, { devicePixelRatio: 1 }); if (options.mediaSource) Object.assign(view, { MediaSource: class {} }); vi.stubGlobal("window", view); - vi.stubGlobal("navigator", { vendor: options.vendor }); + vi.stubGlobal("navigator", { + vendor: options.vendor, + maxTouchPoints: options.touchPoints ?? 0, + ...(options.megabits || options.effectiveType + ? { + connection: { + downlink: options.megabits, + effectiveType: options.effectiveType, + }, + } + : {}), + }); const doc = { readyState: options.loaded ? "complete" : "loading" }; vi.stubGlobal("document", doc); return { @@ -217,10 +401,17 @@ const page = (options: { }; }; -const safari = (loaded = false) => +const iphone = (loaded = false) => + page({ + vendor: "Apple Computer, Inc.", + mediaSource: false, + loaded, + touchPoints: 5, + }); +const macSafari = (loaded = false) => page({ vendor: "Apple Computer, Inc.", mediaSource: true, loaded }); const chrome = (loaded = false) => - page({ vendor: "Google Inc.", mediaSource: true, loaded }); + page({ vendor: "Google Inc.", mediaSource: true, loaded, megabits: 10 }); const attach = ( video: FakeVideo, @@ -228,12 +419,14 @@ const attach = ( options: { autoPlay?: boolean; startAt?: number } = {}, ) => { const onFatal = vi.fn(); + const onLevels = vi.fn(); const attached = attachSource(video as unknown as HTMLVideoElement, source, { autoPlay: options.autoPlay ?? false, startAt: options.startAt, onFatal, + onLevels, }); - return { ...attached, onFatal }; + return { ...attached, onFatal, onLevels }; }; const hlsFor = async () => { @@ -313,13 +506,13 @@ describe("playAllowingMuted", () => { }); describe("attachSource before play", () => { - it("loads nothing in Safari until the document has loaded, then only metadata", () => { - const { finishLoading } = safari(); + it("loads nothing on an iPhone until the document has loaded, then only metadata", () => { + const { finishLoading } = iphone(); const video = new FakeVideo(true); attach(video, { hls: HLS, mp4: MP4 }); - expect(video.src).toBe(HLS); + expect(video.src).toBe(PLAYED_HLS); expect(video.preload).toBe("none"); finishLoading(); @@ -329,7 +522,7 @@ describe("attachSource before play", () => { }); it("reads metadata at once when the document has already loaded", () => { - safari(true); + iphone(true); const video = new FakeVideo(true); attach(video, { hls: HLS }); @@ -337,6 +530,16 @@ describe("attachSource before play", () => { expect(video.preload).toBe("metadata"); }); + it("uses the browser's own HLS where there is no media source at all", () => { + page({ vendor: "", mediaSource: false, loaded: true }); + const video = new FakeVideo(true); + + attach(video, { hls: HLS, mp4: MP4 }); + + expect(video.src).toBe(PLAYED_HLS); + expect(video.preload).toBe("metadata"); + }); + it("treats the MP4 the same way", () => { const { finishLoading } = page({ vendor: "", @@ -356,7 +559,7 @@ describe("attachSource before play", () => { }); it("holds autoplay until the document has loaded", () => { - const { finishLoading } = safari(); + const { finishLoading } = iphone(); const video = new FakeVideo(true); attach(video, { hls: HLS }, { autoPlay: true }); @@ -369,7 +572,7 @@ describe("attachSource before play", () => { }); it("leaves a video the viewer already started alone", () => { - const { finishLoading } = safari(); + const { finishLoading } = iphone(); const video = new FakeVideo(true); attach(video, { hls: HLS }); @@ -381,7 +584,7 @@ describe("attachSource before play", () => { }); it("stops waiting for the document when it is detached", () => { - const { finishLoading } = safari(); + const { finishLoading } = iphone(); const video = new FakeVideo(true); const { stop } = attach(video, { hls: HLS }, { autoPlay: true }); @@ -394,7 +597,7 @@ describe("attachSource before play", () => { }); it("falls back to the MP4, still without preloading, when native HLS fails", () => { - const { finishLoading } = safari(); + const { finishLoading } = iphone(); const video = new FakeVideo(true); const { onFatal } = attach(video, { hls: HLS, mp4: MP4 }); @@ -425,8 +628,8 @@ describe("attachSource before play", () => { }); describe("attachSource warm-up on intent", () => { - it("lets Safari load ahead once the document has loaded", () => { - const { finishLoading } = safari(); + it("lets an iPhone load ahead once the document has loaded", () => { + const { finishLoading } = iphone(); const video = new FakeVideo(true); const { warm } = attach(video, { hls: HLS, mp4: MP4 }); @@ -441,7 +644,7 @@ describe("attachSource warm-up on intent", () => { }); it("leaves a playing video alone", () => { - safari(true); + iphone(true); const video = new FakeVideo(true); const { warm } = attach(video, { hls: HLS }); @@ -452,7 +655,7 @@ describe("attachSource warm-up on intent", () => { }); it("never lifts the MP4 above metadata", () => { - safari(true); + iphone(true); const fallback = new FakeVideo(true); const file = new FakeVideo(false); page({ vendor: "", mediaSource: false, loaded: true }); @@ -462,7 +665,7 @@ describe("attachSource warm-up on intent", () => { expect(file.preload).toBe("metadata"); - safari(true); + iphone(true); const native = attach(fallback, { hls: HLS, mp4: MP4 }); fallback.dispatchEvent(new Event("error")); native.warm(); @@ -485,7 +688,7 @@ describe("attachSource warm-up on intent", () => { }); it("does nothing once detached", () => { - safari(true); + iphone(true); const video = new FakeVideo(true); const { stop, warm } = attach(video, { hls: HLS }); @@ -499,7 +702,7 @@ describe("attachSource warm-up on intent", () => { describe("attachSource start time", () => { it("seeks to the start time once the duration is known", () => { - safari(true); + iphone(true); const video = new FakeVideo(true); attach(video, { hls: HLS }, { startAt: 30 }); @@ -509,7 +712,7 @@ describe("attachSource start time", () => { }); it("ignores a start time past the end", () => { - safari(true); + iphone(true); const video = new FakeVideo(true); attach(video, { hls: HLS }, { startAt: 300 }); @@ -519,7 +722,7 @@ describe("attachSource start time", () => { }); it("seeks only once, so a later reload keeps the viewer's place", () => { - safari(true); + iphone(true); const video = new FakeVideo(true); attach(video, { hls: HLS }, { startAt: 30 }); @@ -540,7 +743,7 @@ describe("attachSource with hls.js", () => { const hls = await hlsFor(); expect(video.preload).toBe("auto"); - expect(hls.source).toBe(HLS); + expect(hls.source).toBe(PLAYED_SHORT_HLS); expect(hls.config.maxMaxBufferLength).toBe(SECONDS_AHEAD_BEFORE_PLAY); expect(hls.startLoads).toEqual([]); @@ -707,6 +910,213 @@ describe("attachSource with hls.js", () => { expect(video.currentTime).toBe(0); }); + it("holds Safari on a Mac to one segment like any other hls.js browser", async () => { + macSafari(); + const video = new FakeVideo(true); + + attach(video, { hls: HLS, mp4: MP4 }); + const hls = await hlsFor(); + hls.emit("manifestParsed", LADDER_DATA); + + expect(video.src).toBe(""); + expect(hls.source).toBe(PLAYED_SHORT_HLS); + expect(hls.config).toMatchObject({ preferManagedMediaSource: false }); + expect(hls.config.maxMaxBufferLength).toBe(SECONDS_AHEAD_BEFORE_PLAY); + expect(hls.loadLevel).toBe(hls.startLevel); + expect(hls.startLoads).toEqual([-1]); + }); + + it("starts low when the browser gives no connection estimate, then steps up at the next segment without a flush", async () => { + macSafari(); + const video = new FakeVideo(true); + let seekCount = 0; + Object.defineProperty(video, "currentTime", { + get: () => 0, + set: () => { + seekCount++; + }, + }); + const seeks = () => seekCount; + + attach(video, { hls: HLS }); + const hls = await hlsFor(); + hls.emit("manifestParsed", LADDER_DATA); + + expect(hls.startLevel).toBe(0); + expect(hls.loadLevel).toBe(0); + expect(hls.config.abrEwmaDefaultEstimate).toBe(1_200_000); + + hls.bandwidthEstimate = 1_000_000_000; + hls.emit("fragBuffered", { frag: { type: "main", level: 0, duration: 4 } }); + expect(hls.currentLevel).toBe(-1); + void video.play(); + expect((hls as unknown as { nextLevel?: number }).nextLevel).toBe( + undefined, + ); + video.dispatchEvent(new Event("timeupdate")); + expect(hls.currentLevel).toBe(-1); + expect((hls as unknown as { nextLevel?: number }).nextLevel).toBe(1); + const before = seeks(); + + hls.emit("fragBuffered", { frag: { type: "main", level: 1, duration: 4 } }); + expect(hls.loadLevel).toBe(AUTO_LEVEL); + expect(seeks()).toBe(before); + }); + + it("steps up only once a frame is on screen where the browser can say so", async () => { + macSafari(); + const video = new FakeVideo(true); + let frame: (() => void) | null = null; + Object.assign(video, { + requestVideoFrameCallback: (callback: () => void) => { + frame = callback; + return 1; + }, + }); + + attach(video, { hls: HLS }); + const hls = await hlsFor(); + hls.emit("manifestParsed", LADDER_DATA); + hls.bandwidthEstimate = 1_000_000_000; + hls.emit("fragBuffered", { frag: { type: "main", level: 0, duration: 4 } }); + void video.play(); + video.dispatchEvent(new Event("timeupdate")); + + expect((hls as unknown as { nextLevel?: number }).nextLevel).toBe( + undefined, + ); + (frame as (() => void) | null)?.(); + expect((hls as unknown as { nextLevel?: number }).nextLevel).toBe(1); + expect(hls.currentLevel).toBe(-1); + }); + + it("does not step up after it has stopped", async () => { + macSafari(); + const video = new FakeVideo(true); + + const { stop } = attach(video, { hls: HLS }); + const hls = await hlsFor(); + hls.emit("manifestParsed", LADDER_DATA); + hls.bandwidthEstimate = 1_000_000_000; + hls.emit("fragBuffered", { frag: { type: "main", level: 0, duration: 4 } }); + void video.play(); + stop(); + video.dispatchEvent(new Event("timeupdate")); + + expect(hls.currentLevel).toBe(-1); + expect((hls as unknown as { nextLevel?: number }).nextLevel).toBe( + undefined, + ); + }); + + it("stays low at play when the first segment shows a slow line", async () => { + macSafari(); + const video = new FakeVideo(true); + + attach(video, { hls: HLS }); + const hls = await hlsFor(); + hls.emit("manifestParsed", LADDER_DATA); + hls.bandwidthEstimate = 1_600_000; + hls.emit("fragBuffered", { frag: { type: "main", level: 0, duration: 4 } }); + void video.play(); + + expect(hls.currentLevel).toBe(-1); + }); + + it("hands a fatal hls.js error in Safari to its own HLS player, then to the MP4", async () => { + macSafari(true); + const video = new FakeVideo(true); + + const { onFatal, onLevels, warm } = attach(video, { + hls: HLS, + mp4: MP4, + }); + const hls = await hlsFor(); + hls.emit("manifestParsed", LADDER_DATA); + + expect(onLevels).toHaveBeenCalledTimes(1); + + hls.emit("error", { fatal: true, type: "networkError" }); + + expect(onLevels).toHaveBeenCalledTimes(2); + expect(onLevels).toHaveBeenLastCalledWith([]); + expect(hls.destroyed).toBe(true); + expect(video.src).toBe(PLAYED_HLS); + expect(video.preload).toBe("metadata"); + expect(video.plays).toBe(0); + + warm(); + + expect(video.preload).toBe("auto"); + + video.dispatchEvent(new Event("error")); + + expect(video.src).toBe(MP4); + expect(onFatal).not.toHaveBeenCalled(); + + video.dispatchEvent(new Event("error")); + + expect(onFatal).toHaveBeenCalledTimes(1); + }); + + it("keeps the viewer's place and keeps playing when Safari's own player takes over", async () => { + macSafari(true); + const video = new FakeVideo(true); + + attach(video, { hls: HLS, mp4: MP4 }); + const hls = await hlsFor(); + hls.emit("manifestParsed", LADDER_DATA); + void video.play(); + video.currentTime = 30; + hls.emit("error", { fatal: true, type: "networkError" }); + + expect(video.src).toBe(PLAYED_HLS); + expect(video.currentTime).toBe(30); + expect(video.plays).toBe(2); + }); + + it("carries the start time into Safari's own player when hls.js fails before reaching it", async () => { + macSafari(true); + const video = new FakeVideo(true); + + attach(video, { hls: HLS, mp4: MP4 }, { startAt: 45 }); + const hls = await hlsFor(); + hls.emit("manifestParsed", LADDER_DATA); + hls.emit("levelLoaded", { details: { totalduration: 120 } }); + hls.emit("error", { fatal: true, type: "networkError" }); + video.reach(120); + + expect(video.src).toBe(PLAYED_HLS); + expect(video.currentTime).toBe(45); + }); + + it("starts Safari's own player when hls.js cannot run", async () => { + macSafari(true); + const video = new FakeVideo(true); + const supported = hlsMock.FakeHls.isSupported; + hlsMock.FakeHls.isSupported = () => false; + const before = hlsMock.FakeHls.instances.length; + + attach(video, { hls: HLS, mp4: MP4 }); + await vi.dynamicImportSettled(); + hlsMock.FakeHls.isSupported = supported; + + expect(hlsMock.FakeHls.instances).toHaveLength(before); + expect(video.src).toBe(PLAYED_HLS); + expect(video.preload).toBe("metadata"); + }); + + it("sends a browser that only claims HLS straight to the MP4", async () => { + chrome(true); + const video = new FakeVideo(true); + + attach(video, { hls: HLS, mp4: MP4 }); + const hls = await hlsFor(); + hls.emit("error", { fatal: true, type: "networkError" }); + + expect(video.src).toBe(MP4); + }); + it("releases hls.js when it is detached", async () => { chrome(); const video = new FakeVideo(false); @@ -718,3 +1128,880 @@ describe("attachSource with hls.js", () => { expect(hls.destroyed).toBe(true); }); }); + +describe("withShortPlaylists", () => { + it("asks for the short form and keeps any other query", () => { + vi.stubGlobal("document", { baseURI: "https://cap.so/dev/abc/embed" }); + + expect(withShortPlaylists(HLS)).toBe(SHORT_HLS); + expect(withShortPlaylists(`${HLS}?token=1`)).toBe( + `${HLS}?token=1&segments=short`, + ); + expect(withShortPlaylists(`${HLS}?segments=long`)).toBe(SHORT_HLS); + expect(withShortPlaylists("/api/developer/playback/abc/master.m3u8")).toBe( + "https://cap.so/api/developer/playback/abc/master.m3u8?segments=short", + ); + }); + + it("keeps the plain playlist for Safari's own player and the MP4", () => { + iphone(true); + const native = new FakeVideo(true); + attach(native, { hls: HLS, mp4: MP4 }); + + expect(native.src).toBe(PLAYED_HLS); + + native.dispatchEvent(new Event("error")); + + expect(native.src).toBe(MP4); + }); +}); + +describe("attachSource switching to the rendition that fits", () => { + type Hls = InstanceType; + + // Chrome's reading of about 1.6 Mbps, on a fast line or a slow one, so the + // player starts on the lowest rendition until it has measured the line. + const lowEstimate = () => + page({ + vendor: "Google Inc.", + mediaSource: true, + loaded: true, + megabits: 1.6, + }); + + const countSeeks = (video: FakeVideo) => { + let seeks = 0; + let time = 0; + Object.defineProperty(video, "currentTime", { + get: () => time, + set: (value: number) => { + seeks++; + time = value; + }, + }); + return () => seeks; + }; + + const held = (hls: Hls, level = 0) => + hls.emit("fragBuffered", { frag: { type: "main", level, duration: 4 } }); + + // The bandwidth at which the 1280 wide rendition's 4 second segment + // arrives in exactly the time allowed. + const JUST_FAST_ENOUGH = (LADDER[1]?.bitrate ?? 0) * (4 / SECONDS_TO_SHARPEN); + + const started = async ( + video: FakeVideo, + options: { startAt?: number } = {}, + ) => { + const attached = attach(video, { hls: HLS }, options); + const hls = await hlsFor(); + hls.emit("manifestParsed", LADDER_DATA); + return { ...attached, hls }; + }; + + it("fetches nothing more before play and switches as playback starts", async () => { + lowEstimate(); + const video = new FakeVideo(false); + const seeks = countSeeks(video); + const { hls } = await started(video); + + expect(hls.loadLevel).toBe(0); + + hls.bandwidthEstimate = JUST_FAST_ENOUGH; + held(hls); + + expect(hls.currentLevel).toBe(-1); + expect(hls.loadLevel).toBe(0); + expect(hls.config.maxMaxBufferLength).toBe(SECONDS_AHEAD_BEFORE_PLAY); + + void video.play(); + + expect(hls.currentLevel).toBe(1); + expect(seeks()).toBe(0); + + hls.loadLevel = 1; + held(hls, 1); + + expect(seeks()).toBe(1); + expect(video.currentTime).toBe(0); + expect(hls.loadLevel).toBe(-1); + + held(hls, 1); + + expect(seeks()).toBe(1); + }); + + it("switches as soon as the first segment arrives when already playing", async () => { + lowEstimate(); + const video = new FakeVideo(false); + const { hls } = await started(video); + + hls.bandwidthEstimate = 1_000_000_000; + void video.play(); + held(hls); + + expect(hls.currentLevel).toBe(1); + }); + + it("never switches above the rendition that fits the player", async () => { + lowEstimate(); + const video = new FakeVideo(false); + const { hls } = await started(video); + + hls.bandwidthEstimate = 1_000_000_000; + held(hls); + void video.play(); + + expect(hls.currentLevel).toBe(1); + }); + + it("keeps the first segment when the line could not fetch a sharper one in time", async () => { + lowEstimate(); + const video = new FakeVideo(false); + const { hls } = await started(video); + + hls.bandwidthEstimate = JUST_FAST_ENOUGH - 1; + held(hls); + void video.play(); + + expect(hls.currentLevel).toBe(-1); + }); + + it("keeps a first segment that already fits", async () => { + chrome(true); + const video = new FakeVideo(false); + const { hls } = await started(video); + + expect(hls.loadLevel).toBe(1); + + hls.bandwidthEstimate = 1_000_000_000; + held(hls, 1); + void video.play(); + + expect(hls.currentLevel).toBe(-1); + }); + + it("only measures video segments", async () => { + lowEstimate(); + const video = new FakeVideo(false); + const { hls } = await started(video); + + hls.bandwidthEstimate = 1_000_000_000; + hls.emit("fragBuffered", { + frag: { type: "audio", level: 0, duration: 4 }, + }); + void video.play(); + + expect(hls.currentLevel).toBe(-1); + }); + + it("measures the line once", async () => { + lowEstimate(); + const video = new FakeVideo(false); + const { hls } = await started(video); + + hls.bandwidthEstimate = JUST_FAST_ENOUGH - 1; + held(hls); + hls.bandwidthEstimate = 1_000_000_000; + held(hls); + void video.play(); + + expect(hls.currentLevel).toBe(-1); + }); + + it("leaves a quality the viewer picked alone", async () => { + lowEstimate(); + const video = new FakeVideo(false); + const { hls, setLevel } = await started(video); + + hls.bandwidthEstimate = 1_000_000_000; + held(hls); + setLevel(3); + void video.play(); + + expect(hls.currentLevel).toBe(-1); + }); + + it("switches once hls.js has reached a start time", async () => { + lowEstimate(); + const video = new FakeVideo(false); + const { hls } = await started(video, { startAt: 45 }); + + hls.bandwidthEstimate = 1_000_000_000; + held(hls); + void video.play(); + + expect(hls.currentLevel).toBe(-1); + + video.dispatchEvent(new Event("seeked")); + + expect(hls.currentLevel).toBe(1); + }); + + it("drops a waiting switch when the viewer picks a quality", async () => { + lowEstimate(); + const video = new FakeVideo(false); + const { hls, setLevel } = await started(video, { startAt: 45 }); + + hls.bandwidthEstimate = 1_000_000_000; + held(hls); + void video.play(); + setLevel(3); + video.dispatchEvent(new Event("seeked")); + + expect(hls.currentLevel).toBe(3); + }); + + it("switches at once to a pick sharper than the picture, and redraws when it lands", async () => { + lowEstimate(); + const video = new FakeVideo(false); + const seeks = countSeeks(video); + const { hls, setLevel } = await started(video); + held(hls); + void video.play(); + hls.currentLevel = 0; + hls.bandwidthEstimate = (LADDER[2]?.bitrate ?? 0) * 4; + const before = seeks(); + + setLevel(2); + + expect(hls.currentLevel).toBe(2); + expect((hls as unknown as { nextLevel?: number }).nextLevel).toBe( + undefined, + ); + held(hls, 2); + expect(seeks()).toBe(before + 1); + }); + + it("changes to a sharper pick at the next segment when the line could not refill the buffer quickly", async () => { + lowEstimate(); + const video = new FakeVideo(false); + const { hls, setLevel } = await started(video); + held(hls); + void video.play(); + hls.currentLevel = 0; + hls.bandwidthEstimate = (LADDER[2]?.bitrate ?? 0) * 4 - 1; + + setLevel(2); + + expect(hls.currentLevel).toBe(0); + expect((hls as unknown as { nextLevel?: number }).nextLevel).toBe(2); + }); + + it("changes to a lower pick or to Auto at the next segment", async () => { + lowEstimate(); + const video = new FakeVideo(false); + const { hls, setLevel } = await started(video); + held(hls); + void video.play(); + hls.currentLevel = 2; + hls.bandwidthEstimate = 1_000_000_000; + + setLevel(0); + expect(hls.currentLevel).toBe(2); + expect((hls as unknown as { nextLevel?: number }).nextLevel).toBe(0); + + setLevel(AUTO_LEVEL); + expect(hls.currentLevel).toBe(2); + expect((hls as unknown as { nextLevel?: number }).nextLevel).toBe( + AUTO_LEVEL, + ); + }); + + it("keeps a pick made before play from loading anything until play", async () => { + lowEstimate(); + const video = new FakeVideo(false); + const { hls, setLevel } = await started(video); + held(hls); + + setLevel(2); + expect(hls.currentLevel).toBe(-1); + expect((hls as unknown as { nextLevel?: number }).nextLevel).toBe( + undefined, + ); + void video.play(); + expect((hls as unknown as { nextLevel?: number }).nextLevel).toBe(2); + }); + + it("switches by resolution when bitrate does not rise with it", async () => { + lowEstimate(); + const video = new FakeVideo(false); + attach(video, { hls: HLS }); + const hls = await hlsFor(); + hls.emit("manifestParsed", { levels: UNEVEN_LADDER }); + + expect(hls.loadLevel).toBe(0); + + hls.bandwidthEstimate = 1_000_000_000; + held(hls); + void video.play(); + + expect(hls.currentLevel).toBe(2); + }); +}); + +const MASTER = [ + "#EXTM3U", + "#EXT-X-VERSION:7", + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="group_aud",NAME="audio",DEFAULT=YES,URI="audio.m3u8"', + '#EXT-X-STREAM-INF:BANDWIDTH=2501452,RESOLUTION=1280x720,CODECS="avc1.64001f,mp4a.40.2",AUDIO="group_aud"', + "720p.m3u8", + '#EXT-X-STREAM-INF:BANDWIDTH=4573338,RESOLUTION=1920x1080,CODECS="avc1.640028,mp4a.40.2",AUDIO="group_aud"', + "1080p.m3u8", + '#EXT-X-STREAM-INF:BANDWIDTH=739748,RESOLUTION=640x360,CODECS="avc1.64001e,mp4a.40.2",AUDIO="group_aud"', + "360p.m3u8", + "", +].join("\n"); + +describe("native HLS renditions", () => { + it("reads Cap's renditions from a master playlist", () => { + expect(parseMasterLevels(MASTER)).toEqual([ + { width: 1280, height: 720, bitrate: 2501452, uri: "720p.m3u8" }, + { width: 1920, height: 1080, bitrate: 4573338, uri: "1080p.m3u8" }, + { width: 640, height: 360, bitrate: 739748, uri: "360p.m3u8" }, + ]); + expect( + listLevels(parseMasterLevels(MASTER)).map((level) => level.label), + ).toEqual(["1080p", "720p", "360p"]); + }); + + it("orders the menu by height when bandwidth does not rise with it", () => { + iphone(true); + const uneven = [ + "#EXTM3U", + '#EXT-X-STREAM-INF:BANDWIDTH=1600000,RESOLUTION=3840x2160,AUDIO="aud"', + "2160p.m3u8", + '#EXT-X-STREAM-INF:BANDWIDTH=800000,RESOLUTION=1920x1080,AUDIO="aud"', + "1080p.m3u8", + '#EXT-X-STREAM-INF:BANDWIDTH=1900000,RESOLUTION=1280x720,AUDIO="aud"', + "720p.m3u8", + '#EXT-X-STREAM-INF:BANDWIDTH=900000,RESOLUTION=640x360,AUDIO="aud"', + "360p.m3u8", + ].join("\n"); + const levels = parseMasterLevels(uneven); + const menu = listLevels(levels); + + expect(menu.map((level) => level.label)).toEqual([ + "4K", + "1080p", + "720p", + "360p", + ]); + const picked = menu.find((level) => level.label === "1080p"); + expect(renditionSource(HLS, levels[picked?.index ?? -1]?.uri)).toBe( + `${HLS}?rendition=1080p`, + ); + }); + + it("lists native renditions in the same order and labels as hls.js does", () => { + iphone(true); + const uneven = [ + "#EXTM3U", + '#EXT-X-STREAM-INF:BANDWIDTH=1900000,RESOLUTION=1280x720,AUDIO="aud"', + "720p.m3u8", + '#EXT-X-STREAM-INF:BANDWIDTH=800000,RESOLUTION=1920x1080,AUDIO="aud"', + "1080p.m3u8", + '#EXT-X-STREAM-INF:BANDWIDTH=900000,RESOLUTION=640x360,AUDIO="aud"', + "360p.m3u8", + '#EXT-X-STREAM-INF:BANDWIDTH=1600000,RESOLUTION=3840x2160,AUDIO="aud"', + "2160p.m3u8", + ].join("\n"); + const native = listLevels(parseMasterLevels(uneven)); + const hlsJs = listLevels( + [...parseMasterLevels(uneven)] + .sort((a, b) => a.bitrate - b.bitrate) + .map(({ width, height, bitrate, uri }) => ({ + width, + height, + bitrate, + name: uri.replace(".m3u8", ""), + })), + ); + + expect(native.map((level) => level.label)).toEqual([ + "4K", + "1080p", + "720p", + "360p", + ]); + expect(native.map((level) => level.label)).toEqual( + hlsJs.map((level) => level.label), + ); + }); + + it("skips variants that are not Cap renditions", () => { + const odd = [ + "#EXTM3U", + "#EXT-X-STREAM-INF:BANDWIDTH=1,RESOLUTION=1280x720", + "https://elsewhere.example/720p.m3u8", + "#EXT-X-STREAM-INF:BANDWIDTH=1,RESOLUTION=1280x720", + "../720p.m3u8", + "#EXT-X-STREAM-INF:BANDWIDTH=1,RESOLUTION=1280x720", + "#EXT-X-ENDLIST", + ].join("\n"); + + expect(parseMasterLevels(odd)).toEqual([]); + expect(parseMasterLevels("")).toEqual([]); + }); + + it("points a picked rendition at a master that lists only it", () => { + iphone(true); + expect(renditionSource(HLS, "720p.m3u8")).toBe(`${HLS}?rendition=720p`); + expect(renditionSource(`${HLS}?max=1080`, "720p.m3u8")).toBe( + `${HLS}?max=1080&rendition=720p`, + ); + expect(renditionSource(HLS, "720p.m3u8?segments=short")).toBe( + `${HLS}?rendition=720p`, + ); + expect(renditionSource(HLS, "audio.m3u8")).toBeNull(); + expect(renditionSource(HLS, undefined)).toBeNull(); + }); + + const nativePlayer = async () => { + iphone(true); + const fetchMaster = vi.fn(async (_url: string) => new Response(MASTER)); + vi.stubGlobal("fetch", fetchMaster); + const video = new FakeVideo(true); + const attached = attach(video, { hls: HLS, mp4: MP4 }); + expect(fetchMaster).not.toHaveBeenCalled(); + video.reach(30); + await vi.waitFor(() => expect(attached.onLevels).toHaveBeenCalledTimes(2)); + const levels = attached.onLevels.mock.calls[1]?.[0] as ReturnType< + typeof listLevels + >; + const index = (label: string) => { + const level = levels.find((entry) => entry.label === label); + if (!level) throw new Error(`no ${label}`); + return level.index; + }; + return { video, fetchMaster, index, ...attached }; + }; + + it("lists the renditions once the metadata is in, from the same master", async () => { + const { fetchMaster, onLevels } = await nativePlayer(); + + expect(fetchMaster).toHaveBeenCalledTimes(1); + expect(fetchMaster.mock.calls[0]?.[0]).toBe(PLAYED_HLS); + expect( + (onLevels.mock.calls[1]?.[0] as { label: string }[]).map( + (level) => level.label, + ), + ).toEqual(["1080p", "720p", "360p"]); + }); + + it("switches to a picked rendition and keeps the time and play state", async () => { + const { video, setLevel, index } = await nativePlayer(); + await video.play(); + video.currentTime = 12.5; + + setLevel(index("720p")); + + expect(video.src).toBe(`${PLAYED_HLS}&rendition=720p`); + expect(video.plays).toBe(2); + video.currentTime = 0; + video.reach(30); + expect(video.currentTime).toBe(12.5); + }); + + it("keeps a paused video paused when it switches", async () => { + const { video, setLevel, index } = await nativePlayer(); + video.currentTime = 4; + + setLevel(index("360p")); + + expect(video.src).toBe(`${PLAYED_HLS}&rendition=360p`); + expect(video.plays).toBe(0); + }); + + it("goes back to the full master for Auto, and does nothing for the same pick", async () => { + const { video, setLevel, index } = await nativePlayer(); + + setLevel(AUTO_LEVEL); + expect(video.src).toBe(PLAYED_HLS); + + setLevel(index("1080p")); + expect(video.src).toBe(`${PLAYED_HLS}&rendition=1080p`); + video.src = "changed-elsewhere"; + setLevel(index("1080p")); + expect(video.src).toBe("changed-elsewhere"); + + setLevel(AUTO_LEVEL); + expect(video.src).toBe(PLAYED_HLS); + }); + + it("does not carry a switch's time over to a source attached after it stops", async () => { + const { video, setLevel, index, stop } = await nativePlayer(); + video.currentTime = 9; + + setLevel(index("360p")); + stop(); + video.currentTime = 0; + video.reach(30); + + expect(video.currentTime).toBe(0); + }); + + it("keeps the place across picks made before the first switch has loaded", async () => { + const { video, setLevel, index } = await nativePlayer(); + await video.play(); + video.currentTime = 12.5; + + setLevel(index("360p")); + video.currentTime = 0; + setLevel(index("1080p")); + + expect(video.src).toBe(`${PLAYED_HLS}&rendition=1080p`); + expect(video.plays).toBe(3); + video.reach(30); + expect(video.currentTime).toBe(12.5); + + video.currentTime = 20; + setLevel(AUTO_LEVEL); + video.currentTime = 0; + video.reach(30); + expect(video.currentTime).toBe(20); + }); + + it("keeps a pause made while a switch loads through the next pick", async () => { + const { video, setLevel, index } = await nativePlayer(); + await video.play(); + video.currentTime = 12.5; + + setLevel(index("360p")); + expect(video.plays).toBe(2); + video.currentTime = 0; + video.paused = true; + setLevel(index("1080p")); + + expect(video.plays).toBe(2); + video.reach(30); + expect(video.currentTime).toBe(12.5); + expect(video.paused).toBe(true); + }); + + it("resumes a switch at a seek asked for while it loads", async () => { + const { video, setLevel, index, seekDuringSwitch } = await nativePlayer(); + await video.play(); + video.currentTime = 12.5; + + expect(seekDuringSwitch(3)).toBe(false); + setLevel(index("360p")); + expect(seekDuringSwitch(18)).toBe(true); + video.currentTime = 0; + video.reach(30); + + expect(video.currentTime).toBe(18); + expect(seekDuringSwitch(5)).toBe(false); + }); + + it("keeps a switch's place, and a seek asked for during it, when it falls back to the MP4", async () => { + const { video, setLevel, index, seekDuringSwitch } = await nativePlayer(); + await video.play(); + video.currentTime = 12.5; + + setLevel(index("360p")); + expect(seekDuringSwitch(18)).toBe(true); + video.currentTime = 0; + video.dispatchEvent(new Event("error")); + + expect(video.src).toBe(MP4); + expect(video.currentTime).toBe(18); + expect(seekDuringSwitch(5)).toBe(false); + }); + + it("keeps the playback speed across a quality switch", async () => { + const { video, setLevel, index } = await nativePlayer(); + video.playbackRate = 1.5; + const reset = Object.getOwnPropertyDescriptor(video, "src"); + let src = video.src; + Object.defineProperty(video, "src", { + get: () => src, + set: (value: string) => { + src = value; + video.playbackRate = 1; + }, + configurable: true, + }); + + setLevel(index("360p")); + expect(video.playbackRate).toBe(1.5); + video.playbackRate = 1; + video.reach(30); + expect(video.playbackRate).toBe(1.5); + if (reset) Object.defineProperty(video, "src", reset); + }); + + it("ignores a pick it has no rendition for", async () => { + const { video, setLevel } = await nativePlayer(); + + setLevel(42); + + expect(video.src).toBe(PLAYED_HLS); + }); + + it("offers no menu when the master cannot be read", async () => { + iphone(true); + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("nope", { status: 404 })), + ); + const video = new FakeVideo(true); + const { onLevels } = attach(video, { hls: HLS }); + video.reach(30); + await vi.waitFor(() => expect(onLevels).toHaveBeenCalledTimes(2)); + + expect(onLevels.mock.calls[1]?.[0]).toEqual([]); + }); + + it("stops listing once detached", async () => { + iphone(true); + let release: (response: Response) => void = () => undefined; + vi.stubGlobal( + "fetch", + vi.fn( + () => + new Promise((resolve) => { + release = resolve; + }), + ), + ); + const video = new FakeVideo(true); + const { onLevels, stop } = attach(video, { hls: HLS }); + video.reach(30); + stop(); + release(new Response(MASTER)); + await new Promise((resolve) => setTimeout(resolve, 0)); + + expect(onLevels).toHaveBeenCalledTimes(1); + }); +}); + +// Safari's own HLS player, as measured on iPhone: after a new source is set +// the element reads time zero and no duration until its metadata is in, and a +// seek reads zero until it lands. The events sent to the page must not. +describe("events sent from Safari's own HLS player", () => { + const iosPlayer = async (startAt?: number) => { + iphone(true); + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(MASTER)), + ); + const video = new FakeVideo(true); + Object.assign(video, { currentSrc: "", getAttribute: () => null }); + const attached = attach(video, { hls: HLS, mp4: MP4 }, { startAt }); + const posted: Record[] = []; + const target = Object.assign(new EventTarget(), { + parent: { + postMessage: (message: unknown) => + posted.push(message as Record), + }, + }) as unknown as MessageTarget; + const connection = connectMessages( + target, + video as unknown as HTMLVideoElement, + { + setQuality: () => undefined, + seekDuringSwitch: attached.seekDuringSwitch, + heldTime: attached.heldTime, + takeSwitchPlay: attached.takeSwitchPlay, + }, + ); + const sequence = () => + posted + .filter((message) => message.event !== "timeupdate") + .map( + (message) => + `${message.event} ${message.currentTime} ${message.duration}${message.paused ? " paused" : ""}`, + ); + const swapSource = () => { + video.currentTime = 0; + video.duration = Number.NaN; + }; + const land = () => video.dispatchEvent(new Event("seeked")); + return { + video, + connection, + posted, + sequence, + swapSource, + land, + ...attached, + }; + }; + + const loaded = async (player: Awaited>) => { + player.video.reach(30); + await vi.waitFor(() => expect(player.onLevels).toHaveBeenCalledTimes(2)); + const levels = player.onLevels.mock.calls[1]?.[0] as ReturnType< + typeof listLevels + >; + return (label: string) => { + const level = levels.find((entry) => entry.label === label); + if (!level) throw new Error(`no ${label}`); + return level.index; + }; + }; + + it("keeps the time and sends no play when a rendition is picked while playing", async () => { + const player = await iosPlayer(); + const index = await loaded(player); + const { video } = player; + await video.play(); + video.currentTime = 6.3; + + player.setLevel(index("1080p")); + player.swapSource(); + player.connection.send("qualitychange", { quality: "1080p", auto: false }); + video.dispatchEvent(new Event("timeupdate")); + video.reach(30); + expect(video.currentTime).toBe(6.3); + player.land(); + video.currentTime = 7.5; + video.paused = true; + video.dispatchEvent(new Event("pause")); + + expect(player.sequence()).toEqual([ + "ready 0 0 paused", + "play 0 30", + "qualitychange 6.3 30", + "pause 7.5 30 paused", + ]); + expect( + player.posted + .filter((message) => message.event === "timeupdate") + .map((message) => message.currentTime), + ).toEqual([6.3]); + }); + + it("keeps the time of a paused switch, then sends the viewer's play from it", async () => { + const player = await iosPlayer(); + const index = await loaded(player); + const { video } = player; + video.currentTime = 7.5; + + player.setLevel(index("360p")); + player.swapSource(); + player.connection.send("qualitychange", { quality: "360p", auto: false }); + video.reach(30); + player.land(); + await video.play(); + + expect(player.sequence()).toEqual([ + "ready 0 0 paused", + "qualitychange 7.5 30 paused", + "play 7.5 30", + ]); + }); + + it("sends a play the viewer makes after pausing during a switch", async () => { + const player = await iosPlayer(); + const index = await loaded(player); + const { video } = player; + await video.play(); + video.currentTime = 4; + + player.setLevel(index("720p")); + player.swapSource(); + video.paused = true; + video.dispatchEvent(new Event("pause")); + await video.play(); + + expect(player.sequence()).toEqual([ + "ready 0 0 paused", + "play 0 30", + "pause 4 30 paused", + "play 4 30", + ]); + }); + + it("does not hold back a later play when the switch's own play never came", async () => { + const player = await iosPlayer(); + const index = await loaded(player); + const { video } = player; + await video.play(); + video.currentTime = 4; + video.refusals.push("AbortError"); + + player.setLevel(index("720p")); + player.swapSource(); + video.paused = true; + video.reach(30); + player.land(); + await video.play(); + + expect(player.sequence()).toEqual([ + "ready 0 0 paused", + "play 0 30", + "play 4 30", + ]); + }); + + it("reports the start time from the first play when t is set", async () => { + const player = await iosPlayer(10); + const { video } = player; + + await video.play(); + video.dispatchEvent(new Event("timeupdate")); + video.reach(30); + expect(video.currentTime).toBe(10); + player.land(); + video.currentTime = 10.4; + video.paused = true; + video.dispatchEvent(new Event("pause")); + + expect(player.sequence()).toEqual([ + "ready 10 0 paused", + "play 10 0", + "pause 10.4 30 paused", + ]); + }); + + it("reports the real time once a start past the end is dropped", async () => { + const player = await iosPlayer(300); + const { video } = player; + + video.reach(30); + await video.play(); + + expect(player.sequence()).toEqual(["ready 300 0 paused", "play 0 30"]); + }); + + it("lets a seek made while a switch loads set the time the events carry", async () => { + const player = await iosPlayer(); + const index = await loaded(player); + const { video } = player; + await video.play(); + video.currentTime = 6; + + player.setLevel(index("720p")); + player.swapSource(); + expect(player.seekDuringSwitch(20)).toBe(true); + player.connection.send("qualitychange", { quality: "720p", auto: false }); + video.reach(30); + player.land(); + + expect(player.sequence()).toEqual([ + "ready 0 0 paused", + "play 0 30", + "qualitychange 20 30", + ]); + expect(video.currentTime).toBe(20); + }); + + it("leaves the events of hls.js and of an unswitched native player as they were", async () => { + const player = await iosPlayer(); + await loaded(player); + const { video } = player; + await video.play(); + video.currentTime = 3; + video.dispatchEvent(new Event("timeupdate")); + + expect(player.heldTime()).toBeNull(); + expect(player.takeSwitchPlay()).toBe(false); + expect(player.sequence()).toEqual(["ready 0 0 paused", "play 0 30"]); + expect(player.posted.at(-1)).toMatchObject({ + event: "timeupdate", + currentTime: 3, + }); + }); +}); diff --git a/apps/web/__tests__/unit/cloudfront-signed-url.test.ts b/apps/web/__tests__/unit/cloudfront-signed-url.test.ts index 809bb2095a6..0a1cc0c0bd0 100644 --- a/apps/web/__tests__/unit/cloudfront-signed-url.test.ts +++ b/apps/web/__tests__/unit/cloudfront-signed-url.test.ts @@ -2,8 +2,10 @@ import { generateKeyPairSync } from "node:crypto"; import { describe, expect, it } from "vitest"; import { + cloudFrontExpiresAt, createCloudFrontSignedUrl, getCloudFrontObjectUrl, + isCloudFrontSignedUrl, } from "../../../../packages/web-backend/src/S3Buckets/cloudfront-signed-url"; const { privateKey } = generateKeyPairSync("rsa", { @@ -14,13 +16,13 @@ const { privateKey } = generateKeyPairSync("rsa", { const bucketUrl = "https://cdn.example.com"; -const sign = (key: string) => +const sign = (key: string, expiresAt = 1_900_000_000) => createCloudFrontSignedUrl({ bucketUrl, key, keyPairId: "K2JCJMDEHXQW5F", privateKey, - expiresAt: 1_900_000_000, + expiresAt, }); const decodePolicy = (signedUrl: string) => { @@ -82,4 +84,76 @@ describe("createCloudFrontSignedUrl", () => { ); expect(baseOf(signedUrl)).toBe(resourceOf(signedUrl)); }); + + it("gives the same URL for the same key and expiry, and another for a later expiry", () => { + const signedUrl = sign("user/video/result.mp4"); + + expect(sign("user/video/result.mp4")).toBe(signedUrl); + expect(sign("user/video/result.mp4", 1_900_000_001)).not.toBe(signedUrl); + expect(sign("user/video/other.mp4")).not.toBe(signedUrl); + }); +}); + +describe("isCloudFrontSignedUrl", () => { + it("recognises what the signer returns and nothing else", () => { + expect(isCloudFrontSignedUrl(sign("user/video/result.mp4"))).toBe(true); + expect( + isCloudFrontSignedUrl( + "https://bucket.s3.us-east-1.amazonaws.com/user/video/result.mp4?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Expires=64800&X-Amz-Signature=abc", + ), + ).toBe(false); + expect(isCloudFrontSignedUrl("https://cdn.example.com/a.mp4")).toBe(false); + expect(isCloudFrontSignedUrl("not a url")).toBe(false); + }); +}); + +describe("cloudFrontExpiresAt", () => { + const now = 1_794_500_000_999; + + it("counts from now when no signing date is given", () => { + expect(cloudFrontExpiresAt(undefined, now)).toBe(1_794_503_600); + expect(cloudFrontExpiresAt({}, now)).toBe(1_794_503_600); + expect(cloudFrontExpiresAt({ expiresIn: 43_200 }, now)).toBe( + Math.floor((now + 43_200 * 1000) / 1000), + ); + }); + + it("counts from the signing date when one is given", () => { + const signingDate = new Date("2026-11-12T12:00:00.000Z"); + const expected = signingDate.getTime() / 1000 + 600; + + expect(cloudFrontExpiresAt({ signingDate, expiresIn: 600 }, now)).toBe( + expected, + ); + expect( + cloudFrontExpiresAt( + { signingDate: signingDate.toISOString(), expiresIn: 600 }, + now, + ), + ).toBe(expected); + expect( + cloudFrontExpiresAt( + { signingDate: signingDate.getTime() / 1000, expiresIn: 600 }, + now, + ), + ).toBe(expected); + expect( + cloudFrontExpiresAt( + { signingDate: String(signingDate.getTime() / 1000), expiresIn: 600 }, + now, + ), + ).toBe(expected); + }); + + it("counts from now when the signing date cannot be read", () => { + expect( + cloudFrontExpiresAt({ signingDate: "not a date", expiresIn: 600 }, now), + ).toBe(Math.floor((now + 600_000) / 1000)); + expect( + cloudFrontExpiresAt( + { signingDate: new Date(Number.NaN), expiresIn: 600 }, + now, + ), + ).toBe(Math.floor((now + 600_000) / 1000)); + }); }); diff --git a/apps/web/__tests__/unit/comment-composer-composition.test.ts b/apps/web/__tests__/unit/comment-composer-composition.test.ts index 0635fc009d7..c90630fe24b 100644 --- a/apps/web/__tests__/unit/comment-composer-composition.test.ts +++ b/apps/web/__tests__/unit/comment-composer-composition.test.ts @@ -411,3 +411,77 @@ it("leaves a newer timeline draft alone when an earlier comment fails", async () expect(container.querySelector("input")).toBe(newer); expect(newer.value).toBe("Newer draft"); }); + +it.each(["success", "failure"])( + "posts a toolbar comment once while the first save is still in flight, then allows a new one after %s", + async (outcome) => { + const respond = vi.mocked(newComment).getMockImplementation(); + if (!respond) throw new Error("Missing comment response"); + const save = Promise.withResolvers(); + vi.mocked(newComment).mockImplementationOnce(async (data) => { + await save.promise; + if (outcome === "failure") throw new Error("Offline"); + return respond(data); + }); + vi.spyOn(console, "error").mockImplementation(() => {}); + await act(async () => { + root.render( + createElement(Toolbar, { data: video, disableReactions: true }), + ); + }); + const open = container.querySelector("button"); + if (!open) throw new Error("Missing comment button"); + await act(async () => open.click()); + const input = container.querySelector("input"); + if (!input) throw new Error("Missing comment field"); + await change(input, "Only once"); + const pressEnter = () => + act(async () => { + input.dispatchEvent( + new KeyboardEvent("keydown", { + key: "Enter", + bubbles: true, + cancelable: true, + }), + ); + }); + await pressEnter(); + await pressEnter(); + const submitButton = Array.from(container.querySelectorAll("button")).find( + (button) => button.textContent === "Comment", + ); + if (!submitButton) throw new Error("Missing submit button"); + expect(submitButton.disabled).toBe(true); + await act(async () => submitButton.click()); + expect(newComment).toHaveBeenCalledOnce(); + + await act(async () => save.resolve()); + + if (outcome === "failure") { + expect(input.value).toBe("Only once"); + await pressEnter(); + expect(newComment).toHaveBeenCalledTimes(2); + } else { + expect(container.querySelector("input")).toBeNull(); + } + }, +); + +it("says so when a toolbar reaction fails to post", async () => { + vi.mocked(newComment).mockRejectedValueOnce(new Error("Offline")); + vi.spyOn(console, "error").mockImplementation(() => {}); + showError.mockReset(); + await act(async () => { + root.render(createElement(Toolbar, { data: video })); + }); + const reaction = container.querySelector( + 'button[aria-label="up"]', + ); + if (!reaction) throw new Error("Missing reaction button"); + await act(async () => reaction.click()); + + expect(newComment).toHaveBeenCalledOnce(); + expect(showError).toHaveBeenCalledExactlyOnceWith( + "Couldn't add your reaction. Please try again.", + ); +}); diff --git a/apps/web/__tests__/unit/comment-media-initiate-limits.test.ts b/apps/web/__tests__/unit/comment-media-initiate-limits.test.ts new file mode 100644 index 00000000000..445b309129c --- /dev/null +++ b/apps/web/__tests__/unit/comment-media-initiate-limits.test.ts @@ -0,0 +1,166 @@ +import { Effect, Layer, Option } from "effect"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + reserve: vi.fn(), + rateLimited: vi.fn(), + listObjects: vi.fn(), + createUploadTarget: vi.fn(), +})); + +const video = { id: "video-1", ownerId: "owner-1" }; + +vi.mock("@/app/api/utils", () => ({ + withAuth: async ( + c: { set: (key: string, value: unknown) => void }, + next: () => Promise, + ) => { + c.set("user", { id: "viewer-1" }); + await next(); + }, +})); +vi.mock("@cap/database", () => ({ + db: () => ({ + select: () => ({ + from: () => ({ where: () => ({ limit: async () => [] }) }), + }), + }), +})); +vi.mock("@cap/database/helpers", () => ({ nanoId: () => "comment-1" })); +vi.mock("@cap/database/schema", () => ({ comments: {} })); +vi.mock("drizzle-orm", () => ({ + and: () => undefined, + eq: () => undefined, + gt: () => undefined, + inArray: () => undefined, +})); +vi.mock("@cap/web-domain", () => ({ + Comment: { CommentId: { make: (value: string) => value } }, + Policy: { withPublicPolicy: () => (effect: unknown) => effect }, + Video: { + VideoId: { make: (value: string) => value }, + NotFoundError: class {}, + }, +})); +vi.mock("@cap/web-backend", () => ({ + createCommentMediaUploadToken: () => "upload-token", + verifyCommentMediaUploadToken: () => null, + makeCurrentUserLayer: () => Layer.empty, + provideOptionalAuth: (effect: unknown) => effect, + VideosRepo: Effect.succeed({ + getById: () => Effect.succeed(Option.some([video])), + }), + VideosPolicy: Effect.succeed({ canView: () => Effect.void }), + Storage: { + getAccessForVideo: () => + Effect.succeed([ + { + provider: "aws", + listObjects: mocks.listObjects, + createUploadTarget: mocks.createUploadTarget, + }, + ]), + }, +})); +vi.mock("@/lib/media-comments", () => ({ + canUseMediaComments: async () => true, +})); +vi.mock("@/lib/comment-media-quota", () => ({ + MAX_MEDIA_OBJECTS_PER_VIDEO: 400, + reserveCommentMediaObjects: mocks.reserve, +})); +vi.mock("@/lib/rate-limit", () => ({ + RATE_LIMIT_IDS: { COMMENT_MEDIA_UPLOAD: "rl_comment_media_upload" }, + isRateLimited: mocks.rateLimited, +})); +vi.mock("@/lib/server", () => ({ runPromise: Effect.runPromise })); + +import { app } from "@/app/api/upload/[...route]/comment-media"; + +const initiate = (body: Record = {}) => + app.request("/initiate", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + videoId: "video-1", + kind: "video", + contentType: "video/webm", + mode: "single", + withThumbnail: true, + ...body, + }), + }); + +beforeEach(() => { + vi.clearAllMocks(); + mocks.rateLimited.mockResolvedValue(false); + mocks.reserve.mockResolvedValue(true); + mocks.listObjects.mockReturnValue(Effect.succeed({ Contents: [{}, {}, {}] })); + mocks.createUploadTarget.mockImplementation((key: string) => + Effect.succeed({ url: `https://upload.test/${key}` }), + ); +}); + +describe("comment media initiate limits", () => { + it("issues upload targets for a normal media comment", async () => { + const response = await initiate(); + + expect(response.status).toBe(200); + const body = (await response.json()) as { upload: { url: string } }; + expect(body.upload.url).toContain("comments/comment-1/media.webm"); + expect(mocks.reserve).toHaveBeenCalledExactlyOnceWith({ + videoId: "video-1", + objects: 2, + storedObjects: 3, + }); + expect(mocks.rateLimited).toHaveBeenCalledWith( + "rl_comment_media_upload", + expect.objectContaining({ key: "comment-media-upload:viewer-1" }), + ); + }); + + it("reserves one object for audio", async () => { + const response = await initiate({ + kind: "audio", + contentType: "audio/webm", + withThumbnail: false, + }); + + expect(response.status).toBe(200); + expect(mocks.reserve).toHaveBeenCalledWith( + expect.objectContaining({ objects: 1 }), + ); + }); + + it("refuses with a clear error once the video is full", async () => { + mocks.reserve.mockResolvedValue(false); + + const response = await initiate(); + + expect(response.status).toBe(429); + expect(await response.json()).toEqual({ + error: "This video has reached its media comment limit", + }); + expect(mocks.createUploadTarget).not.toHaveBeenCalled(); + }); + + it("refuses a viewer who starts too many uploads", async () => { + mocks.rateLimited.mockResolvedValue(true); + + const response = await initiate(); + + expect(response.status).toBe(429); + expect(await response.json()).toEqual({ + error: "Too many media comments, slow down", + }); + expect(mocks.reserve).not.toHaveBeenCalled(); + expect(mocks.createUploadTarget).not.toHaveBeenCalled(); + }); + + it("does not reserve space for an unsupported content type", async () => { + const response = await initiate({ contentType: "video/x-msvideo" }); + + expect(response.status).toBe(415); + expect(mocks.reserve).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/__tests__/unit/comment-media-quota.test.ts b/apps/web/__tests__/unit/comment-media-quota.test.ts new file mode 100644 index 00000000000..5174b94eac5 --- /dev/null +++ b/apps/web/__tests__/unit/comment-media-quota.test.ts @@ -0,0 +1,186 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + COMMENT_MEDIA_RESERVATION_WINDOW_MS, + MAX_MEDIA_OBJECTS_PER_VIDEO, + reserveCommentMediaObjects, +} from "@/lib/comment-media-quota"; + +type Row = { + id: string; + limitBytes: number; + reservedBytes: number; + expiresAt: Date; +}; + +const state = vi.hoisted(() => ({ + rows: new Map(), + locks: new Map>(), + lockRows: true, +})); + +const pause = () => new Promise((resolve) => setTimeout(resolve, 0)); + +vi.mock("@cap/database/schema", () => ({ + mediaProcessingBudgets: { id: "id", reservedBytes: "reservedBytes" }, +})); +vi.mock("drizzle-orm", () => ({ + asc: (value: unknown) => value, + inArray: (_column: unknown, ids: string[]) => ids, + sql: (_strings: TemplateStringsArray, ...values: unknown[]) => values, +})); +vi.mock("@cap/database", () => ({ + db: () => ({ + transaction: async (run: (tx: unknown) => Promise) => { + const releases: (() => void)[] = []; + const acquire = async (id: string) => { + const previous = state.locks.get(id) ?? Promise.resolve(); + let release = () => {}; + const held = new Promise((resolve) => { + release = resolve; + }); + state.locks.set( + id, + previous.then(() => held), + ); + await previous; + releases.push(release); + }; + try { + return await run({ + insert: () => ({ + values: (rows: Omit[]) => ({ + onDuplicateKeyUpdate: async () => { + await pause(); + for (const row of rows) + if (!state.rows.has(row.id)) + state.rows.set(row.id, { ...row, reservedBytes: 0 }); + }, + }), + }), + select: () => ({ + from: () => ({ + where: (ids: string[]) => ({ + orderBy: () => ({ + for: async () => { + if (state.lockRows) for (const id of ids) await acquire(id); + await pause(); + return ids + .map((id) => state.rows.get(id)) + .filter(Boolean) + .map((row) => ({ ...(row as Row) })); + }, + }), + }), + }), + }), + update: () => ({ + set: (values: { reservedBytes: [unknown, number] }) => ({ + where: async (ids: string[]) => { + await pause(); + for (const id of ids) { + const row = state.rows.get(id); + if (row) row.reservedBytes += values.reservedBytes[1]; + } + }, + }), + }), + }); + } finally { + for (const release of releases) release(); + } + }, + }), +})); + +const now = new Date(COMMENT_MEDIA_RESERVATION_WINDOW_MS * 1000 + 60_000); + +const reserve = ( + input: Partial[0]> = {}, +) => + reserveCommentMediaObjects({ + videoId: "video-1", + objects: 1, + storedObjects: 0, + now, + ...input, + }); + +beforeEach(() => { + state.rows.clear(); + state.locks.clear(); + state.lockRows = true; +}); + +describe("comment media object quota", () => { + it("lets a normal upload through on a video with room", async () => { + await expect(reserve()).resolves.toBe(true); + await expect(reserve({ objects: 2, storedObjects: 3 })).resolves.toBe(true); + }); + + it("refuses an upload that would pass the per-video cap", async () => { + await expect( + reserve({ storedObjects: MAX_MEDIA_OBJECTS_PER_VIDEO }), + ).resolves.toBe(false); + await expect( + reserve({ objects: 2, storedObjects: MAX_MEDIA_OBJECTS_PER_VIDEO - 1 }), + ).resolves.toBe(false); + await expect( + reserve({ storedObjects: MAX_MEDIA_OBJECTS_PER_VIDEO - 1 }), + ).resolves.toBe(true); + await expect( + reserve({ storedObjects: MAX_MEDIA_OBJECTS_PER_VIDEO - 1 }), + ).resolves.toBe(false); + }); + + it("never issues more than the cap to concurrent initiates", async () => { + const results = await Promise.all( + Array.from({ length: 500 }, () => reserve()), + ); + expect(results.filter(Boolean)).toHaveLength(MAX_MEDIA_OBJECTS_PER_VIDEO); + }); + + it("counts thumbnails against the cap under concurrency", async () => { + const results = await Promise.all( + Array.from({ length: 300 }, () => reserve({ objects: 2 })), + ); + expect(results.filter(Boolean)).toHaveLength( + MAX_MEDIA_OBJECTS_PER_VIDEO / 2, + ); + }); + + it("needs the row lock to hold the cap", async () => { + state.lockRows = false; + const results = await Promise.all( + Array.from({ length: 500 }, () => reserve()), + ); + expect(results.filter(Boolean).length).toBeGreaterThan( + MAX_MEDIA_OBJECTS_PER_VIDEO, + ); + }); + + it("keeps counting a reservation past its token across window boundaries", async () => { + const W = COMMENT_MEDIA_RESERVATION_WINDOW_MS; + const lastMomentOfWindow = new Date(2 * W - 1); + await Promise.all( + Array.from({ length: MAX_MEDIA_OBJECTS_PER_VIDEO }, () => + reserve({ now: lastMomentOfWindow }), + ), + ); + + for (const at of [ + lastMomentOfWindow.getTime() + 1, + lastMomentOfWindow.getTime() + W + 60_000, + 3 * W + W - 1, + ]) + await expect(reserve({ now: new Date(at) })).resolves.toBe(false); + + await expect(reserve({ now: new Date(4 * W) })).resolves.toBe(true); + }); + + it("keeps each video's reservations separate", async () => { + await Promise.all( + Array.from({ length: MAX_MEDIA_OBJECTS_PER_VIDEO }, () => reserve()), + ); + await expect(reserve({ videoId: "video-2" })).resolves.toBe(true); + }); +}); diff --git a/apps/web/__tests__/unit/dashboard-mobile-tab-infra.test.ts b/apps/web/__tests__/unit/dashboard-mobile-tab-infra.test.ts new file mode 100644 index 00000000000..47e835291a9 --- /dev/null +++ b/apps/web/__tests__/unit/dashboard-mobile-tab-infra.test.ts @@ -0,0 +1,88 @@ +// @vitest-environment jsdom + +import React, { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const context = vi.hoisted(() => ({ + user: { id: "owner", email: "richie@cap.so" }, + ownerId: "owner", +})); + +vi.mock("@cap/env", () => ({ buildEnv: { NEXT_PUBLIC_IS_CAP: "true" } })); +vi.mock("@uidotdev/usehooks", () => ({ + useClickAway: () => ({ current: null }), +})); +vi.mock("next/navigation", () => ({ + useRouter: () => ({ push: vi.fn() }), +})); +vi.mock("@/app/(org)/dashboard/_components/Navbar/server", () => ({ + updateActiveOrganization: vi.fn(), +})); +vi.mock("@/components/SignedImageUrl", () => ({ + SignedImageUrl: () => null, +})); +vi.mock("@/app/(org)/dashboard/Contexts", () => ({ + useDashboardContext: () => ({ + user: context.user, + activeOrganization: { + organization: { id: "org", name: "Acme", ownerId: context.ownerId }, + members: [{ userId: context.user.id, role: "owner" }], + }, + organizationData: [], + }), +})); + +import MobileTab from "@/app/(org)/dashboard/_components/MobileTab"; + +let root: Root; +let container: HTMLDivElement; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + context.user = { id: "owner", email: "richie@cap.so" }; + context.ownerId = "owner"; +}); + +const hrefs = () => { + act(() => root.render(React.createElement(MobileTab))); + return Array.from(container.querySelectorAll("a")).map((link) => + link.getAttribute("href"), + ); +}; + +describe("dashboard mobile tabs", () => { + it("shows Infra to an org owner who can use Infra, as on desktop", () => { + expect(hrefs()).toContain("/dashboard/infra"); + expect( + container + .querySelector('a[href="/dashboard/infra"]') + ?.getAttribute("aria-label"), + ).toBe("Infra"); + }); + + it("shows Infra to any org owner, not only a preview list", () => { + context.user = { id: "owner", email: "someone@example.com" }; + + expect(hrefs()).toEqual([ + "/dashboard/spaces/browse", + "/dashboard/caps", + "/dashboard/settings/organization", + "/dashboard/infra", + ]); + }); + + it("does not show Infra outside the org the user owns", () => { + context.ownerId = "someone-else"; + + expect(hrefs()).not.toContain("/dashboard/infra"); + }); +}); diff --git a/apps/web/__tests__/unit/deliver-developer-webhook-workflow.test.ts b/apps/web/__tests__/unit/deliver-developer-webhook-workflow.test.ts new file mode 100644 index 00000000000..ef2ea05ab6c --- /dev/null +++ b/apps/web/__tests__/unit/deliver-developer-webhook-workflow.test.ts @@ -0,0 +1,95 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + sleep: vi.fn(), + attempt: vi.fn(), +})); + +vi.mock("workflow", () => ({ sleep: mocks.sleep })); +vi.mock("@/lib/developer-webhook-delivery", () => ({ + deliverDeveloperWebhookAttempt: mocks.attempt, +})); + +const { deliverDeveloperWebhookWorkflow } = await import( + "@/workflows/deliver-developer-webhook" +); + +const job = { + endpointId: "wh-1", + eventId: "evt_1", + eventType: "video.ready", + body: "{}", +}; + +beforeEach(() => { + vi.clearAllMocks(); + mocks.sleep.mockResolvedValue(undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); +}); + +describe("deliver developer webhook workflow", () => { + it("stops after the first attempt when it is delivered", async () => { + mocks.attempt.mockResolvedValue("delivered"); + expect(await deliverDeveloperWebhookWorkflow(job)).toEqual({ + outcome: "delivered", + attempts: 1, + }); + expect(mocks.attempt).toHaveBeenCalledWith(job); + expect(mocks.sleep).not.toHaveBeenCalled(); + }); + + it("retries a failing endpoint on the schedule and then delivers", async () => { + mocks.attempt + .mockResolvedValueOnce("retry") + .mockResolvedValueOnce("retry") + .mockResolvedValueOnce("delivered"); + expect(await deliverDeveloperWebhookWorkflow(job)).toEqual({ + outcome: "delivered", + attempts: 3, + }); + expect(mocks.sleep.mock.calls.map(([wait]) => wait)).toEqual([ + "60s", + "300s", + ]); + }); + + it("gives up after eight attempts spread over about a day", async () => { + mocks.attempt.mockResolvedValue("retry"); + expect(await deliverDeveloperWebhookWorkflow(job)).toEqual({ + outcome: "gave_up", + attempts: 8, + }); + expect(mocks.attempt).toHaveBeenCalledTimes(8); + expect(mocks.sleep.mock.calls.map(([wait]) => wait)).toEqual([ + "60s", + "300s", + "1800s", + "7200s", + "18000s", + "28800s", + "28800s", + ]); + }); + + it("stops retrying once the endpoint is deleted or disabled", async () => { + mocks.attempt + .mockResolvedValueOnce("retry") + .mockResolvedValueOnce("dropped"); + expect(await deliverDeveloperWebhookWorkflow(job)).toEqual({ + outcome: "dropped", + attempts: 2, + }); + expect(mocks.sleep).toHaveBeenCalledTimes(1); + }); + + it("keeps an attempt that could not run on the same schedule", async () => { + mocks.attempt + .mockRejectedValueOnce(new Error("database is away")) + .mockResolvedValueOnce("delivered"); + expect(await deliverDeveloperWebhookWorkflow(job)).toEqual({ + outcome: "delivered", + attempts: 2, + }); + expect(mocks.sleep.mock.calls.map(([wait]) => wait)).toEqual(["60s"]); + }); +}); diff --git a/apps/web/__tests__/unit/developer-actions-preview-gate.test.ts b/apps/web/__tests__/unit/developer-actions-preview-gate.test.ts index ded21e31beb..6beec7dc63c 100644 --- a/apps/web/__tests__/unit/developer-actions-preview-gate.test.ts +++ b/apps/web/__tests__/unit/developer-actions-preview-gate.test.ts @@ -101,23 +101,54 @@ const actions: [string, () => Promise][] = [ "video-1", ), ], + [ + "add a webhook endpoint", + async () => + (await import("@/actions/developers/webhooks")).createInfraWebhook( + "app-1", + { url: "https://example.com/hook", events: ["video.ready"] }, + ), + ], + [ + "turn a webhook endpoint off", + async () => + (await import("@/actions/developers/webhooks")).setInfraWebhookEnabled( + "app-1", + "hook-1", + false, + ), + ], + [ + "rotate a webhook secret", + async () => + (await import("@/actions/developers/webhooks")).rotateInfraWebhookSecret( + "app-1", + "hook-1", + ), + ], + [ + "send a webhook test", + async () => + (await import("@/actions/developers/webhooks")).sendInfraWebhookTest( + "app-1", + "hook-1", + ), + ], + [ + "delete a webhook endpoint", + async () => + (await import("@/actions/developers/webhooks")).deleteInfraWebhook( + "app-1", + "hook-1", + ), + ], ]; -describe("Cap Infra console actions and the preview list", () => { +describe("Cap Infra console actions and sign-in", () => { beforeEach(() => { mocks.touched = []; }); - it.each(actions)( - "does not let an account outside the preview %s", - async (_name, run) => { - mocks.user = { id: "user-1", email: "someone@example.com" }; - - await expect(run()).rejects.toThrow("Unauthorized"); - expect(mocks.touched).toEqual([]); - }, - ); - it.each(actions)( "does not let a signed-out visitor %s", async (_name, run) => { @@ -129,9 +160,9 @@ describe("Cap Infra console actions and the preview list", () => { ); it.each(actions)( - "lets an account on the preview list through to %s", + "lets any signed-in account through to %s", async (_name, run) => { - mocks.user = { id: "user-1", email: "richie@cap.so" }; + mocks.user = { id: "user-1", email: "someone@example.com" }; const outcome = await run().then( () => "returned", diff --git a/apps/web/__tests__/unit/developer-actions.test.ts b/apps/web/__tests__/unit/developer-actions.test.ts index 7ba5f291b7e..342f37745b5 100644 --- a/apps/web/__tests__/unit/developer-actions.test.ts +++ b/apps/web/__tests__/unit/developer-actions.test.ts @@ -4,6 +4,16 @@ vi.mock("@/lib/infra-access", () => ({ canUseInfra: () => true, })); +const firstMonth = vi.hoisted(() => ({ + atDelete: vi.fn(), + forDeletedApp: vi.fn(), +})); + +vi.mock("@/lib/infra-storage-minimum", () => ({ + settleFirstMonthAtDelete: firstMonth.atDelete, + settleFirstMonthsForDeletedApp: firstMonth.forDeletedApp, +})); + vi.mock("@cap/database", () => { const mockDb = { select: vi.fn(() => mockDb), @@ -40,6 +50,11 @@ vi.mock("@cap/database/crypto", () => ({ encrypt: vi.fn(async (value: string) => `encrypted:${value}`), })); +vi.mock("@/lib/infra-balance", async (original) => ({ + ...(await original()), + heldTranscriptMicroCredits: async () => 0, +})); + vi.mock("@cap/database/schema", () => ({ developerApps: { id: "id", @@ -78,6 +93,13 @@ vi.mock("@cap/database/schema", () => ({ developerCreditTransactions: {}, })); +vi.mock("@/lib/developer-app-limits", async (importOriginal) => ({ + ...(await importOriginal()), + developerAppCreationLimited: async () => false, + developerKeyRotationLimited: async () => false, + reserveDeveloperAppSlot: async () => true, +})); + vi.mock("@/lib/developer-key-hash", () => ({ hashKey: vi.fn(() => Promise.resolve("mocked-hash-value")), })); @@ -86,6 +108,15 @@ vi.mock("next/cache", () => ({ revalidatePath: vi.fn(), })); +const purge = vi.hoisted(() => vi.fn(async () => undefined)); +vi.mock("@vercel/functions", () => ({ + dangerouslyDeleteByTag: purge, +})); + +vi.mock("@/lib/developer-video-cancel", () => ({ + cancelDeveloperVideoEncodes: vi.fn(async () => undefined), +})); + vi.mock("drizzle-orm", () => ({ and: vi.fn((...args: unknown[]) => args), eq: vi.fn((field: unknown, value: unknown) => ({ field, value })), @@ -95,7 +126,11 @@ vi.mock("drizzle-orm", () => ({ import * as capDatabase from "@cap/database"; import { getCurrentUser } from "@cap/database/auth/session"; +import { cancelDeveloperVideoEncodes } from "@/lib/developer-video-cancel"; +const mockCancelEncodes = cancelDeveloperVideoEncodes as ReturnType< + typeof vi.fn +>; const mockGetCurrentUser = getCurrentUser as ReturnType; type MockFn = ReturnType; @@ -289,6 +324,71 @@ describe("deleteDeveloperApp", () => { const result = await deleteDeveloperApp("app-456"); expect(result).toEqual({ success: true }); }); + + it("settles the first month of the videos this delete stamped, after the delete", async () => { + mockGetCurrentUser.mockResolvedValue(mockUser); + mockDb.limit.mockResolvedValueOnce([mockApp]); + await deleteDeveloperApp("app-456"); + const stamps = mockDb.set.mock.calls + .map(([values]) => (values as { deletedAt?: Date }).deletedAt) + .filter(Boolean); + expect(stamps).toHaveLength(2); + expect(stamps[0]?.getMilliseconds()).toBe(0); + expect(firstMonth.forDeletedApp).toHaveBeenCalledWith("app-456", stamps[0]); + expect(stamps[1]).toBe(stamps[0]); + }); + + it("cancels the app's encode jobs once the delete has committed", async () => { + mockGetCurrentUser.mockResolvedValue(mockUser); + mockDb.limit.mockResolvedValueOnce([mockApp]); + let committed = false; + mockDb.transaction.mockImplementationOnce(async (fn) => { + await fn(mockDb); + committed = true; + }); + mockCancelEncodes.mockImplementationOnce(async () => { + expect(committed).toBe(true); + }); + + await deleteDeveloperApp("app-456"); + + const stamps = mockDb.set.mock.calls.map( + ([values]) => values.deletedAt ?? values.revokedAt, + ); + const deletedAt = stamps[0]; + expect(deletedAt).toBeInstanceOf(Date); + expect(deletedAt.getTime() % 1000).toBe(0); + expect(stamps).toEqual([deletedAt, deletedAt, deletedAt]); + expect(mockCancelEncodes).toHaveBeenCalledExactlyOnceWith({ + appId: "app-456", + deletedAt, + }); + }); + + it("cancels nothing for an app it could not find", async () => { + mockGetCurrentUser.mockResolvedValue(mockUser); + mockDb.limit.mockResolvedValue([]); + await expect(deleteDeveloperApp("app-456")).rejects.toThrow(); + expect(mockCancelEncodes).not.toHaveBeenCalled(); + expect(purge).not.toHaveBeenCalled(); + }); + + it("drops the cached playback of every video in the app once the delete has committed", async () => { + mockGetCurrentUser.mockResolvedValue(mockUser); + mockDb.limit.mockResolvedValueOnce([mockApp]); + let committed = false; + mockDb.transaction.mockImplementationOnce(async (fn) => { + await fn(mockDb); + committed = true; + }); + purge.mockImplementationOnce(async () => { + expect(committed).toBe(true); + }); + + await deleteDeveloperApp("app-456"); + + expect(purge.mock.calls).toEqual([["infra-app-app-456"]]); + }); }); describe("updateDeveloperApp", () => { @@ -449,6 +549,18 @@ describe("addDeveloperDomain", () => { expect(result).toEqual({ success: true, domain: "http://localhost:3000" }); }); + it("refuses a plain http site, which cannot embed the player", async () => { + mockGetCurrentUser.mockResolvedValue(mockUser); + mockDb.limit.mockResolvedValueOnce([mockApp]); + const result = await addDeveloperDomain("app-456", "http://myshop.com"); + expect(result).toEqual({ + success: false, + message: + "Only https sites can embed your player. Enter one like https://myshop.com.", + }); + expect(mockDb.insert).not.toHaveBeenCalled(); + }); + it("normalizes to lowercase", async () => { mockGetCurrentUser.mockResolvedValue(mockUser); mockDb.limit.mockResolvedValueOnce([mockApp]); @@ -608,10 +720,16 @@ describe("regenerateDeveloperKeys", () => { ); }); + const rotate = async () => { + const result = await regenerateDeveloperKeys("app-456"); + if ("success" in result) throw new Error(result.message); + return result; + }; + it("creates new public and secret keys", async () => { mockGetCurrentUser.mockResolvedValue(mockUser); mockDb.limit.mockResolvedValueOnce([mockApp]); - const result = await regenerateDeveloperKeys("app-456"); + const result = await rotate(); expect(result.publicKey).toMatch(/^cpk_/); expect(result.secretKey).toMatch(/^csk_/); @@ -620,7 +738,7 @@ describe("regenerateDeveloperKeys", () => { it("returns new publicKey and secretKey", async () => { mockGetCurrentUser.mockResolvedValue(mockUser); mockDb.limit.mockResolvedValueOnce([mockApp]); - const result = await regenerateDeveloperKeys("app-456"); + const result = await rotate(); expect(result).toHaveProperty("publicKey"); expect(result).toHaveProperty("secretKey"); @@ -752,21 +870,82 @@ describe("deleteDeveloperVideo", () => { ); }); - it("soft deletes video", async () => { + it("soft deletes video and settles its first month after the delete", async () => { mockGetCurrentUser.mockResolvedValue(mockUser); mockDb.limit.mockResolvedValueOnce([mockApp]); + mockDb.where + .mockReturnValueOnce(mockDb) + .mockResolvedValueOnce([{ affectedRows: 1 }] as never); await deleteDeveloperVideo("app-456", "video-001"); expect(mockDb.update).toHaveBeenCalled(); expect(mockDb.set).toHaveBeenCalledWith( expect.objectContaining({ deletedAt: expect.any(Date) }), ); + expect(firstMonth.atDelete).toHaveBeenCalledWith("video-001"); + }); + + it("does not settle anything when the video was already deleted or is another app's", async () => { + mockGetCurrentUser.mockResolvedValue(mockUser); + mockDb.limit.mockResolvedValueOnce([mockApp]); + mockDb.where + .mockReturnValueOnce(mockDb) + .mockResolvedValueOnce([{ affectedRows: 0 }] as never); + await deleteDeveloperVideo("app-456", "video-001"); + + expect(firstMonth.atDelete).not.toHaveBeenCalled(); + expect(purge).not.toHaveBeenCalled(); + }); + + it("drops the video's cached playback after the delete is written", async () => { + mockGetCurrentUser.mockResolvedValue(mockUser); + mockDb.limit.mockResolvedValueOnce([mockApp]); + mockDb.where + .mockReturnValueOnce(mockDb) + .mockResolvedValueOnce([{ affectedRows: 1 }] as never); + + await deleteDeveloperVideo("app-456", "video-001"); + + expect(purge.mock.calls).toEqual([["infra-video-video-001"]]); + const [deleteWrite] = mockDb.set.mock.invocationCallOrder; + const [purged] = purge.mock.invocationCallOrder; + expect(purged).toBeGreaterThan(deleteWrite ?? Infinity); }); it("returns success true", async () => { mockGetCurrentUser.mockResolvedValue(mockUser); mockDb.limit.mockResolvedValueOnce([mockApp]); + mockDb.where + .mockReturnValueOnce(mockDb) + .mockResolvedValueOnce([{ affectedRows: 1 }] as never); const result = await deleteDeveloperVideo("app-456", "video-001"); expect(result).toEqual({ success: true }); }); + + it("cancels the video's encode jobs after the delete is written", async () => { + mockGetCurrentUser.mockResolvedValue(mockUser); + mockDb.limit.mockResolvedValueOnce([mockApp]); + mockDb.where + .mockReturnValueOnce(mockDb) + .mockResolvedValueOnce([{ affectedRows: 1 }] as never); + + await deleteDeveloperVideo("app-456", "video-001"); + + expect(mockCancelEncodes).toHaveBeenCalledExactlyOnceWith({ + appId: "app-456", + videoId: "video-001", + }); + const [deleteWrite] = mockDb.set.mock.invocationCallOrder; + const [cancel] = mockCancelEncodes.mock.invocationCallOrder; + expect(cancel).toBeGreaterThan(deleteWrite ?? Infinity); + }); + + it("cancels nothing when the caller does not own the app", async () => { + mockGetCurrentUser.mockResolvedValue(mockUser); + mockDb.limit.mockResolvedValue([]); + await expect(deleteDeveloperVideo("app-456", "video-001")).rejects.toThrow( + "App not found", + ); + expect(mockCancelEncodes).not.toHaveBeenCalled(); + }); }); diff --git a/apps/web/__tests__/unit/developer-api-cold-path.test.ts b/apps/web/__tests__/unit/developer-api-cold-path.test.ts new file mode 100644 index 00000000000..96e717c8b13 --- /dev/null +++ b/apps/web/__tests__/unit/developer-api-cold-path.test.ts @@ -0,0 +1,85 @@ +import { fileURLToPath } from "node:url"; +import { build, type Metafile } from "esbuild"; +import { describe, expect, it } from "vitest"; + +// A cold start of a developer API function evaluates every module the route +// imports statically. Sign-in (next-auth, SSO) and email rendering are only +// needed by cookie-authenticated requests and by billing emails, so they must +// stay behind dynamic imports and out of these routes' startup graph. +const webRoot = fileURLToPath(new URL("../../", import.meta.url)); + +const ROUTES = [ + "app/api/developer/v1/[...route]/route.ts", + "app/api/developer/mcp/route.ts", +]; + +const NOT_AT_STARTUP = [ + "packages/database/auth/auth-options", + "packages/database/auth/session", + "packages/database/emails/", + "node_modules/next-auth/", + "node_modules/nodemailer/", + "node_modules/@workos-inc/", + "node_modules/@react-email/", + "node_modules/resend/", +]; + +async function staticGraph(route: string) { + const result = await build({ + entryPoints: [route], + absWorkingDir: webRoot, + bundle: true, + write: false, + metafile: true, + platform: "node", + format: "esm", + logLevel: "silent", + tsconfig: "tsconfig.json", + conditions: ["react-server", "node"], + external: ["next", "next/*", "@vercel/*"], + loader: { + ".node": "empty", + ".wasm": "empty", + ".svg": "empty", + ".png": "empty", + ".css": "empty", + }, + }); + const inputs: Metafile["inputs"] = result.metafile.inputs; + const entry = Object.keys(inputs).find((path) => path.endsWith(route)); + if (!entry) throw new Error(`entry not found for ${route}`); + const parent = new Map([[entry, null]]); + const queue = [entry]; + while (queue.length > 0) { + const current = queue.shift() as string; + for (const imported of inputs[current]?.imports ?? []) { + if (imported.kind === "dynamic-import") continue; + if (parent.has(imported.path) || !inputs[imported.path]) continue; + parent.set(imported.path, current); + queue.push(imported.path); + } + } + return parent; +} + +function chainTo(parent: Map, path: string) { + const chain: string[] = []; + for (let at: string | null | undefined = path; at; at = parent.get(at)) { + chain.unshift(at.replace(/^.*node_modules\//, "")); + } + return chain.join(" -> "); +} + +describe("developer API cold path", () => { + it.each(ROUTES)( + "%s does not load sign-in or email code at startup", + async (route) => { + const parent = await staticGraph(route); + const offenders = [...parent.keys()] + .filter((path) => NOT_AT_STARTUP.some((part) => path.includes(part))) + .map((path) => chainTo(parent, path)); + expect(offenders.slice(0, 3)).toEqual([]); + }, + 60_000, + ); +}); diff --git a/apps/web/__tests__/unit/developer-api-contract.test.ts b/apps/web/__tests__/unit/developer-api-contract.test.ts index 4a2c65a0400..b7a99507889 100644 --- a/apps/web/__tests__/unit/developer-api-contract.test.ts +++ b/apps/web/__tests__/unit/developer-api-contract.test.ts @@ -4,9 +4,14 @@ import { MySqlDialect } from "drizzle-orm/mysql-core"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ - keyRows: [] as { appId: string; environment?: string }[], + keyRows: [] as { + appId: string; + keyId?: string; + ownerId?: string; + environment?: string; + }[], + hashKey: vi.fn(), accounts: [] as { balanceMicroCredits: number }[], - videoStats: [] as Record[], storedVideos: [] as Record[], selected: [] as Record[], createVideo: vi.fn(), @@ -20,25 +25,26 @@ vi.mock("@vercel/firewall", () => ({ checkRateLimit: mocks.firewall })); vi.mock("@cap/database", async () => { const schema = await import("@cap/database/schema"); - const rowsFor = (table: unknown, fields?: Record) => { + const rowsFor = (table: unknown) => { if (table === schema.developerApiKeys) return mocks.keyRows; if (table === schema.developerCreditAccounts) return mocks.accounts; - if (table === schema.developerVideos) { - return fields && "createdVideos" in fields - ? mocks.videoStats - : mocks.storedVideos; - } + if (table === schema.developerVideos) return mocks.storedVideos; return []; }; const select = (fields?: Record) => ({ from: (table: unknown) => { if (fields) mocks.selected.push(fields); const where = () => - Object.assign(Promise.resolve(rowsFor(table, fields)), { - limit: async () => rowsFor(table, fields), - groupBy: async () => rowsFor(table, fields), + Object.assign(Promise.resolve(rowsFor(table)), { + limit: async () => rowsFor(table), + groupBy: async () => rowsFor(table), + orderBy: () => ({ limit: async () => rowsFor(table) }), }); - const query = { innerJoin: () => query, where }; + const query = { + innerJoin: () => query, + leftJoin: () => query, + where, + }; return query; }, }); @@ -59,9 +65,7 @@ vi.mock("@cap/env", () => ({ NEXTAUTH_SECRET: "unit-test-signing-secret", }), })); -vi.mock("@/lib/developer-key-hash", () => ({ - hashKey: async (key: string) => `hash:${key}`, -})); +vi.mock("@/lib/developer-key-hash", () => ({ hashKey: mocks.hashKey })); vi.mock("@/lib/developer-video-start", () => ({ redriveDeveloperVideoProcessing: vi.fn(), })); @@ -97,8 +101,13 @@ vi.mock("@/lib/infra-video-upload", () => ({ abortInfraUpload: vi.fn(), })); +import { Hono } from "hono"; import * as sdk from "@/app/api/developer/sdk/v1/[...route]/route"; import * as rest from "@/app/api/developer/v1/[...route]/route"; +import { + developerOwnerRequestLimited, + withDeveloperSecretAuth, +} from "@/app/api/utils"; import { developerOpenApiSpec } from "@/lib/developer-openapi"; import { MICRO_CREDITS_PER_DOLLAR, @@ -108,18 +117,27 @@ import { } from "@/lib/developer-pricing"; import { DEVELOPER_KEY_REQUESTS_PER_MINUTE, + DEVELOPER_OWNER_REQUESTS_PER_MINUTE, DEVELOPER_UPLOAD_TOKEN_REQUESTS_PER_MINUTE, + DEVELOPER_AUTH_FAILURES_PER_MINUTE as FAILURES, } from "@/lib/developer-rate-limits"; import { createDeveloperUploadToken } from "@/lib/developer-upload-token"; import { + INFRA_DELIVERY_DOLLARS_PER_1000_MINUTES, INFRA_RESOLUTION_TIERS, INFRA_STORAGE_DOLLARS_PER_1000_MINUTES_PER_MONTH, } from "@/lib/infra-pricing"; +import { getAppStorage } from "@/lib/infra-storage"; + +const KEY_LIMIT = DEVELOPER_KEY_REQUESTS_PER_MINUTE; +const OWNER_LIMIT = DEVELOPER_OWNER_REQUESTS_PER_MINUTE; +const keyLeft = (sent: number) => String(KEY_LIMIT - sent); type Method = "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "OPTIONS"; const ORIGIN = "https://cap.so"; let keySerial = 0; +let ownerSerial = 0; const freshKey = (prefix: "csk" | "cpk" | "cut") => prefix === "cut" ? createDeveloperUploadToken({ @@ -127,6 +145,7 @@ const freshKey = (prefix: "csk" | "cpk" | "cut") => videoId: `video-${++keySerial}`, expiresAt: Date.now() + 60_000, corsOrigin: null, + keyId: "key-contract", }) : `${prefix}_key_${++keySerial}`; @@ -159,9 +178,15 @@ const rateLimit = (response: Response) => ({ beforeEach(() => { vi.clearAllMocks(); - mocks.keyRows = [{ appId: "app-1", environment: "development" }]; + mocks.hashKey.mockImplementation(async (key: string) => `hash:${key}`); + mocks.keyRows = [ + { + appId: "app-1", + ownerId: `owner-${++ownerSerial}`, + environment: "development", + }, + ]; mocks.accounts = [{ balanceMicroCredits: 4_532_000 }]; - mocks.videoStats = [{ createdVideos: 5, uploadedVideos: 4 }]; mocks.storedVideos = [ { appId: "app-1", @@ -171,6 +196,9 @@ beforeEach(() => { maxResolutionTier: "1080p", seconds: 750, count: 3, + rows: 3, + dailySeconds: 750, + dailyCount: 3, }, { appId: "app-1", @@ -180,10 +208,28 @@ beforeEach(() => { maxResolutionTier: "1080p", seconds: 345.21, count: 1, + rows: 1, + dailySeconds: 345.21, + dailyCount: 1, + }, + { + appId: "app-1", + width: null, + height: null, + status: "uploading", + maxResolutionTier: "1080p", + seconds: 0, + count: 0, + rows: 1, + dailySeconds: 0, + dailyCount: 0, }, ]; mocks.selected = []; - mocks.billing.mockResolvedValue({ standing: { state: "active" } }); + mocks.billing.mockImplementation(async (appId: string) => { + const storage = await getAppStorage(appId); + return { standing: { state: "active" }, storage, stored: storage.stored }; + }); mocks.createVideo.mockResolvedValue({ status: 200, body: { videoId: "video-1" }, @@ -207,22 +253,25 @@ describe("rate limit headers", () => { const first = await getUsage(key); expect(first.status).toBe(200); expect(rateLimit(first)).toEqual({ - limit: "60", - remaining: "59", + limit: String(KEY_LIMIT), + remaining: keyLeft(1), reset: "60", retryAfter: null, }); vi.advanceTimersByTime(20_500); const second = await getUsage(key); - expect(rateLimit(second)).toMatchObject({ remaining: "58", reset: "40" }); + expect(rateLimit(second)).toMatchObject({ + remaining: keyLeft(2), + reset: "40", + }); - for (let request = 3; request <= 60; request++) await getUsage(key); + for (let request = 3; request <= KEY_LIMIT; request++) await getUsage(key); vi.advanceTimersByTime(4_500); const limited = await getUsage(key); expect(limited.status).toBe(429); expect(rateLimit(limited)).toEqual({ - limit: "60", + limit: String(KEY_LIMIT), remaining: "0", reset: "35", retryAfter: "35", @@ -236,7 +285,7 @@ describe("rate limit headers", () => { const afterWait = await getUsage(key); expect(afterWait.status).toBe(200); expect(rateLimit(afterWait)).toMatchObject({ - remaining: "59", + remaining: keyLeft(1), reset: "60", }); }); @@ -245,7 +294,7 @@ describe("rate limit headers", () => { vi.useFakeTimers(); vi.setSystemTime(new Date("2026-11-12T10:00:00Z")); const key = freshKey("csk"); - for (let request = 1; request <= 60; request++) await getUsage(key); + for (let request = 1; request <= KEY_LIMIT; request++) await getUsage(key); for (const [elapsed, expected] of [ [0, "60"], @@ -273,17 +322,18 @@ describe("rate limit headers", () => { }); const other = await getUsage(freshKey("csk")); - expect(rateLimit(upper).remaining).toBe("59"); - expect(rateLimit(lower).remaining).toBe("58"); - expect(rateLimit(spaced).remaining).toBe("57"); - expect(rateLimit(other).remaining).toBe("59"); + expect(rateLimit(upper).remaining).toBe(keyLeft(1)); + expect(rateLimit(lower).remaining).toBe(keyLeft(2)); + expect(rateLimit(spaced).remaining).toBe(keyLeft(3)); + expect(rateLimit(other).remaining).toBe(keyLeft(1)); }); - it("carries the headers on errors too", async () => { + it("carries the headers on errors too, and spends none of a key's budget on a key that was not accepted", async () => { const key = freshKey("csk"); + const accepted = mocks.keyRows; mocks.keyRows = []; const unauthorized = await getUsage(key); - mocks.keyRows = [{ appId: "app-1" }]; + mocks.keyRows = accepted; const notFound = await call(rest, "GET", "/api/developer/v1/nope", { key }); const wrongMethod = await call(rest, "POST", "/api/developer/v1/usage", { key, @@ -298,10 +348,10 @@ describe("rate limit headers", () => { rateLimit(response).remaining, ]), ).toEqual([ - [401, "59"], - [404, "58"], - [405, "57"], - [500, "56"], + [401, null], + [404, keyLeft(1)], + [405, keyLeft(2)], + [500, keyLeft(3)], ]); }); @@ -335,6 +385,7 @@ describe("rate limit headers", () => { "X-RateLimit-Limit", "X-RateLimit-Remaining", "X-RateLimit-Reset", + "X-Request-Id", ]); }); }); @@ -364,7 +415,7 @@ describe("rate limit shared across servers", () => { expect(limited.status).toBe(429); expect(rateLimit(limited)).toEqual({ - limit: "60", + limit: String(KEY_LIMIT), remaining: "0", reset: "60", retryAfter: "60", @@ -372,7 +423,7 @@ describe("rate limit shared across servers", () => { expect(await limited.json()).toMatchObject({ error: "Rate limit exceeded", code: "rate_limited", - hint: "Wait for the Retry-After seconds, then retry. Each key allows 60 requests per minute.", + hint: `Wait for the Retry-After seconds, then retry. Each key allows ${KEY_LIMIT} requests per minute.`, }); expect(mocks.billing).not.toHaveBeenCalled(); }); @@ -404,13 +455,13 @@ describe("rate limit shared across servers", () => { it("lets the shared rule overrule a count this server kept alone", async () => { const key = freshKey("csk"); - for (let request = 1; request <= 60; request++) await getUsage(key); + for (let request = 1; request <= KEY_LIMIT; request++) await getUsage(key); const allowed = await getUsage(key); expect(allowed.status).toBe(200); expect(rateLimit(allowed)).toMatchObject({ - limit: "60", + limit: String(KEY_LIMIT), remaining: "0", retryAfter: null, }); @@ -425,7 +476,9 @@ describe("rate limit shared across servers", () => { await getUsage(freshKey("csk")); await initiate(token); - const [first, again, other, upload] = firewallCalls(); + const [first, again, other, upload] = firewallCalls().filter( + (asked) => asked.ruleId !== "rl_developer_api_owner", + ); expect([first, again, other, upload].map((asked) => asked?.ruleId)).toEqual( [ "rl_developer_api_key", @@ -461,12 +514,12 @@ describe("rate limit shared across servers", () => { const first = await getUsage(key); expect(rateLimit(first)).toEqual({ - limit: "60", - remaining: "59", + limit: String(KEY_LIMIT), + remaining: keyLeft(1), reset: "60", retryAfter: null, }); - for (let request = 2; request <= 60; request++) { + for (let request = 2; request <= KEY_LIMIT; request++) { expect((await getUsage(key)).status).toBe(200); } vi.advanceTimersByTime(25_000); @@ -474,7 +527,7 @@ describe("rate limit shared across servers", () => { expect(limited.status).toBe(429); expect(rateLimit(limited)).toEqual({ - limit: "60", + limit: String(KEY_LIMIT), remaining: "0", reset: "35", retryAfter: "35", @@ -482,6 +535,545 @@ describe("rate limit shared across servers", () => { }); }); +describe("one account's keys share a budget", () => { + const ownedBy = (ownerId: string) => { + mocks.keyRows = [{ appId: "app-1", ownerId }]; + }; + + it("allows the account budget across every key of an account, then answers 429", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-11-12T10:00:00Z")); + ownedBy("owner-shared"); + // Every key stays under its own limit, so only the account budget can + // refuse the request after them. + const perKey = Math.min(KEY_LIMIT - 1, OWNER_LIMIT); + const keys = Array.from({ length: Math.ceil(OWNER_LIMIT / perKey) }, () => + freshKey("csk"), + ); + expect(keys.length).toBeGreaterThan(1); + + for (const [index, key] of keys.entries()) { + const sends = Math.min(perKey, OWNER_LIMIT - index * perKey); + for (let request = 1; request <= sends; request++) { + expect((await getUsage(key)).status).toBe(200); + } + } + vi.advanceTimersByTime(20_000); + const limited = await getUsage(freshKey("csk")); + + expect(limited.status).toBe(429); + expect(rateLimit(limited)).toEqual({ + limit: String(KEY_LIMIT), + remaining: "0", + reset: "40", + retryAfter: "40", + }); + expect(await limited.json()).toMatchObject({ + code: "rate_limited", + hint: `Wait for the Retry-After seconds, then retry. Each account, across all of its keys, allows ${OWNER_LIMIT} requests per minute.`, + }); + expect(mocks.billing).toHaveBeenCalledTimes(OWNER_LIMIT); + + ownedBy("owner-other"); + expect((await getUsage(freshKey("csk"))).status).toBe(200); + + ownedBy("owner-shared"); + vi.advanceTimersByTime(40_001); + expect((await getUsage(keys[0] as string)).status).toBe(200); + }); + + it("takes the account from the key lookup, with no query of its own", async () => { + const response = await call(rest, "GET", "/api/developer/v1/nope", { + key: freshKey("csk"), + }); + + expect(response.status).toBe(404); + expect(mocks.selected).toHaveLength(1); + expect(Object.keys(mocks.selected[0] ?? {}).sort()).toEqual([ + "appId", + "keyId", + "ownerId", + ]); + expect(mocks.hashKey).toHaveBeenCalledTimes(1); + }); + + it("asks the shared rule about the account by its id", async () => { + vi.stubEnv("NODE_ENV", "production"); + ownedBy("owner-asked"); + + await getUsage(freshKey("csk")); + + const asked = mocks.firewall.mock.calls.map(([ruleId, options]) => [ + ruleId, + options.rateLimitKey, + ]); + expect(asked).toContainEqual(["rl_developer_api_owner", "owner-asked"]); + expect(asked).toHaveLength(2); + }); + + it("rejects when the shared rule for the account says so", async () => { + vi.stubEnv("NODE_ENV", "production"); + ownedBy("owner-limited"); + mocks.firewall.mockImplementation(async (ruleId: string) => ({ + rateLimited: ruleId === "rl_developer_api_owner", + })); + + const limited = await getUsage(freshKey("csk")); + + expect(limited.status).toBe(429); + expect(rateLimit(limited)).toMatchObject({ + limit: String(KEY_LIMIT), + remaining: "0", + retryAfter: "60", + }); + expect(mocks.billing).not.toHaveBeenCalled(); + }); +}); + +describe("the console's webhook actions share the account budget", () => { + it("counts console requests and REST requests in one account budget", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-11-13T10:00:00Z")); + mocks.keyRows = [{ appId: "app-1", ownerId: "owner-console" }]; + const fromConsole = (owner = "owner-console") => + developerOwnerRequestLimited(owner, new Headers()); + + for (let request = 1; request < OWNER_LIMIT; request++) { + expect(await fromConsole()).toBe(false); + } + expect((await getUsage(freshKey("csk"))).status).toBe(200); + expect(await fromConsole()).toBe(true); + expect((await getUsage(freshKey("csk"))).status).toBe(429); + expect(await fromConsole("owner-other")).toBe(false); + + vi.advanceTimersByTime(60_001); + expect(await fromConsole()).toBe(false); + }); + + it("asks the shared rule about the account by its id", async () => { + vi.stubEnv("NODE_ENV", "production"); + + await developerOwnerRequestLimited("owner-console-asked", new Headers()); + + expect( + mocks.firewall.mock.calls.map(([ruleId, options]) => [ + ruleId, + options.rateLimitKey, + ]), + ).toEqual([["rl_developer_api_owner", "owner-console-asked"]]); + }); + + it("is refused when the shared rule for the account says so", async () => { + vi.stubEnv("NODE_ENV", "production"); + mocks.firewall.mockResolvedValue({ rateLimited: true }); + + expect( + await developerOwnerRequestLimited("owner-console-shared", new Headers()), + ).toBe(true); + }); +}); + +describe("secret key check behind other middleware", () => { + const guarded = (preset: Record) => + new Hono<{ Variables: Record }>() + .use(async (c, next) => { + for (const [name, value] of Object.entries(preset)) c.set(name, value); + await next(); + }) + .use(withDeveloperSecretAuth) + .get("/", (c) => + c.json({ + appId: c.get("developerAppId"), + keyId: c.get("developerKeyId") ?? null, + }), + ); + const send = ( + preset: Record, + headers: Record = {}, + ) => guarded(preset).request("https://cap.so/", { headers }); + + it("still asks for a secret key when an app was set by something other than a secret key", async () => { + const response = await send({ developerAppId: "app-from-token" }); + + expect(response.status).toBe(401); + expect(await response.json()).toMatchObject({ + code: "invalid_secret_key", + }); + expect(mocks.hashKey).not.toHaveBeenCalled(); + }); + + it("takes the app and key from the secret key, not from what was set before", async () => { + mocks.keyRows = [{ appId: "app-1", keyId: "key-1", ownerId: "owner-1" }]; + + const response = await send( + { developerAppId: "app-from-token" }, + { Authorization: `Bearer ${freshKey("csk")}` }, + ); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ appId: "app-1", keyId: "key-1" }); + expect(mocks.hashKey).toHaveBeenCalledTimes(1); + }); + + it("does not look the key up again once the rate limiter accepted it", async () => { + const response = await send({ + developerAppId: "app-1", + developerKeyId: "key-1", + developerKeyType: "secret", + }); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ appId: "app-1", keyId: "key-1" }); + expect(mocks.hashKey).not.toHaveBeenCalled(); + }); +}); + +describe("credentials that are not accepted", () => { + const from = (address: string) => ({ "x-forwarded-for": address }); + + it.each([ + ["no credential", undefined, 401, "invalid_secret_key"], + [ + "a credential with no prefix", + "Bearer not-a-key", + 401, + "invalid_secret_key", + ], + ["a public key", "Bearer cpk_public_1", 403, "public_key_not_allowed"], + [ + "an oversized key", + `Bearer csk_${"a".repeat(200)}`, + 401, + "invalid_secret_key", + ], + [ + "a key with characters no key has", + "Bearer csk_abc!def", + 401, + "invalid_secret_key", + ], + ] as const)( + "refuses %s without hashing, looking up or counting anything", + async (_label, authorization, status, code) => { + vi.stubEnv("NODE_ENV", "production"); + + const response = await call(rest, "GET", "/api/developer/v1/usage", { + headers: authorization ? { Authorization: authorization } : {}, + }); + + expect(response.status).toBe(status); + expect(await response.json()).toMatchObject({ code }); + expect(rateLimit(response)).toEqual({ + limit: null, + remaining: null, + reset: null, + retryAfter: null, + }); + expect(mocks.hashKey).not.toHaveBeenCalled(); + expect(mocks.selected).toHaveLength(0); + expect(mocks.firewall).not.toHaveBeenCalled(); + }, + ); + + it("answers an unknown key 401 on any path, and stops looking keys up for an address after many different ones in a minute", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-11-12T10:00:00Z")); + mocks.keyRows = []; + const address = from("203.0.113.9"); + + for (let attempt = 1; attempt <= FAILURES; attempt++) { + const refused = await call(rest, "GET", "/api/developer/v1/nope", { + key: freshKey("csk"), + headers: address, + }); + expect(refused.status).toBe(401); + } + expect(mocks.hashKey).toHaveBeenCalledTimes(FAILURES); + + const limited = await getUsage(freshKey("csk"), address); + expect(limited.status).toBe(429); + expect(limited.headers.get("Retry-After")).toBe("60"); + expect(await limited.json()).toMatchObject({ + code: "rate_limited", + error: "Too many requests with a key or token that was not accepted", + }); + expect(mocks.hashKey).toHaveBeenCalledTimes(FAILURES + 1); + + mocks.hashKey.mockClear(); + mocks.selected = []; + for (let attempt = 1; attempt <= 50; attempt++) { + expect((await getUsage(freshKey("csk"), address)).status).toBe(429); + } + expect(mocks.hashKey).not.toHaveBeenCalled(); + expect(mocks.selected).toHaveLength(0); + + const elsewhere = await getUsage(freshKey("csk"), from("198.51.100.4")); + expect(elsewhere.status).toBe(401); + + vi.advanceTimersByTime(60_001); + expect((await getUsage(freshKey("csk"), address)).status).toBe(401); + }); + + it("keeps a key it accepted in the last minute working from an address that is being refused", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-11-12T11:00:00Z")); + const address = from("203.0.113.77"); + const accepted = mocks.keyRows; + const customer = freshKey("csk"); + expect((await getUsage(customer, address)).status).toBe(200); + + mocks.keyRows = []; + for (let attempt = 1; attempt <= FAILURES + 1; attempt++) { + await getUsage(freshKey("csk"), address); + } + expect((await getUsage(freshKey("csk"), address)).status).toBe(429); + + mocks.keyRows = accepted; + expect((await getUsage(customer, address)).status).toBe(200); + }); + + it("counts that minute from the key's last accepted request, not from when its count began", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-11-12T11:30:00Z")); + const address = from("203.0.113.78"); + const accepted = mocks.keyRows; + const customer = freshKey("csk"); + expect((await getUsage(customer, address)).status).toBe(200); + vi.advanceTimersByTime(59_000); + expect((await getUsage(customer, address)).status).toBe(200); + + mocks.keyRows = []; + for (let attempt = 1; attempt <= FAILURES + 1; attempt++) { + await getUsage(freshKey("csk"), address); + } + expect((await getUsage(freshKey("csk"), address)).status).toBe(429); + + mocks.keyRows = accepted; + vi.advanceTimersByTime(2_000); + expect((await getUsage(customer, address)).status).toBe(200); + + vi.advanceTimersByTime(61_000); + mocks.keyRows = []; + for (let attempt = 1; attempt <= FAILURES + 1; attempt++) { + await getUsage(freshKey("csk"), address); + } + mocks.keyRows = accepted; + expect((await getUsage(customer, address)).status).toBe(429); + }); + + it("keeps a valid key working from an address that sent refused keys, on an instance that never saw it before", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-11-12T13:00:00Z")); + const address = from("203.0.113.90"); + const accepted = mocks.keyRows; + mocks.keyRows = []; + for (let attempt = 1; attempt <= 30; attempt++) { + expect((await getUsage(freshKey("csk"), address)).status).toBe(401); + } + + mocks.keyRows = accepted; + expect((await getUsage(freshKey("csk"), address)).status).toBe(200); + }); + + it("refuses a key it already refused at once, without a lookup or a count, so a stale worker cannot block its address", async () => { + vi.stubEnv("NODE_ENV", "production"); + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-11-12T13:10:00Z")); + const address = { "x-real-ip": "203.0.113.91" }; + const accepted = mocks.keyRows; + mocks.keyRows = []; + const rotated = freshKey("csk"); + + for (let attempt = 1; attempt <= FAILURES * 2; attempt++) { + expect((await getUsage(rotated, address)).status).toBe(401); + } + expect(mocks.hashKey).toHaveBeenCalledTimes(1); + expect( + mocks.firewall.mock.calls.filter( + ([ruleId]) => ruleId === "rl_developer_auth_failure", + ), + ).toHaveLength(1); + + mocks.keyRows = accepted; + expect((await getUsage(freshKey("csk"), address)).status).toBe(200); + + mocks.keyRows = []; + vi.advanceTimersByTime(60_001); + mocks.hashKey.mockClear(); + expect((await getUsage(rotated, address)).status).toBe(401); + expect(mocks.hashKey).toHaveBeenCalledTimes(1); + }); + + it("counts a burst of parallel retries with one refused key once, so a valid key from that address still passes", async () => { + vi.stubEnv("NODE_ENV", "production"); + const address = { "x-real-ip": "203.0.113.93" }; + const accepted = mocks.keyRows; + mocks.keyRows = []; + const rotated = freshKey("csk"); + let release: () => void = () => undefined; + const gate = new Promise((resolve) => { + release = resolve; + }); + mocks.hashKey.mockImplementation(async (key: string) => { + await gate; + return `hash:${key}`; + }); + + const burst = Array.from({ length: 50 }, () => getUsage(rotated, address)); + await new Promise((resolve) => setTimeout(resolve, 50)); + release(); + const statuses = (await Promise.all(burst)).map( + (response) => response.status, + ); + + expect(statuses).toEqual(Array(50).fill(401)); + expect(mocks.hashKey).toHaveBeenCalledTimes(1); + expect( + mocks.firewall.mock.calls.filter( + ([ruleId]) => ruleId === "rl_developer_auth_failure", + ), + ).toHaveLength(1); + + mocks.hashKey.mockImplementation(async (key: string) => `hash:${key}`); + mocks.keyRows = accepted; + expect((await getUsage(freshKey("csk"), address)).status).toBe(200); + }); + + it("bounds the lookups an address sending different refused keys can cause to the per-address ceiling", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-11-12T13:20:00Z")); + mocks.keyRows = []; + const address = from("203.0.113.92"); + + for (let attempt = 1; attempt <= FAILURES * 3; attempt++) { + await getUsage(freshKey("csk"), address); + } + + expect(mocks.hashKey).toHaveBeenCalledTimes(FAILURES + 1); + }); + + it("counts by the address the platform reports, not one the caller writes", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-11-12T12:00:00Z")); + vi.stubEnv("VERCEL", "1"); + mocks.keyRows = []; + const statuses: number[] = []; + + for (let attempt = 1; attempt <= FAILURES + 2; attempt++) { + const response = await getUsage(freshKey("csk"), { + "x-real-ip": "203.0.113.200", + "x-forwarded-for": `10.0.0.${attempt}, 203.0.113.200`, + "x-client-ip": `10.9.9.${attempt}`, + }); + statuses.push(response.status); + } + + expect(statuses.slice(0, FAILURES)).toEqual(Array(FAILURES).fill(401)); + expect(statuses.slice(FAILURES)).toEqual([429, 429]); + }); + + it("asks the shared rule about the address, sending it no credential", async () => { + vi.stubEnv("NODE_ENV", "production"); + mocks.keyRows = []; + const key = freshKey("csk"); + + const refused = await getUsage(key, { "x-real-ip": "203.0.113.50" }); + + expect(refused.status).toBe(401); + expect(mocks.firewall).toHaveBeenCalledTimes(1); + const [ruleId, options] = mocks.firewall.mock.calls[0] ?? []; + expect(ruleId).toBe("rl_developer_auth_failure"); + expect(options.rateLimitKey).toBeUndefined(); + expect(options.request.headers.get("x-real-ip")).toBe("203.0.113.50"); + expect(JSON.stringify([...options.request.headers])).not.toContain(key); + }); + + it("answers 429 as soon as the shared rule says the address has sent too many", async () => { + vi.stubEnv("NODE_ENV", "production"); + mocks.keyRows = []; + mocks.firewall.mockResolvedValue({ rateLimited: true }); + const address = { "x-real-ip": "203.0.113.60" }; + + expect((await getUsage(freshKey("csk"), address)).status).toBe(429); + mocks.hashKey.mockClear(); + mocks.firewall.mockClear(); + expect((await getUsage(freshKey("csk"), address)).status).toBe(429); + expect(mocks.hashKey).not.toHaveBeenCalled(); + expect(mocks.firewall).not.toHaveBeenCalled(); + }); + + it("does not count an upload token that is malformed, expired or simply absent", async () => { + vi.stubEnv("NODE_ENV", "production"); + const send = (authorization?: string) => + call(sdk, "POST", "/api/developer/sdk/v1/upload/multipart/initiate", { + headers: { + "Content-Type": "application/json", + ...(authorization ? { Authorization: authorization } : {}), + }, + body: "{}", + }); + const expired = createDeveloperUploadToken({ + appId: "app-1", + videoId: "video-expired", + expiresAt: Date.now() - 1, + corsOrigin: null, + }); + + const answers = [ + await send(), + await send("Bearer csk_secret_on_the_wrong_route"), + await send("Bearer cpk_public_1"), + await send(`Bearer ${expired}`), + ]; + + expect(answers.map((response) => response.status)).toEqual([ + 401, 401, 403, 401, + ]); + expect(mocks.firewall).not.toHaveBeenCalled(); + for (const response of answers) { + expect(rateLimit(response).limit).toBeNull(); + } + }); + + it("counts a forged upload token against the address and still words the refusal as before", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-11-12T13:00:00Z")); + const forged = () => + call(sdk, "POST", "/api/developer/sdk/v1/upload/multipart/initiate", { + key: `${freshKey("cut")}x`, + headers: { + "Content-Type": "application/json", + "x-forwarded-for": "203.0.113.120", + }, + body: "{}", + }); + + const first = await forged(); + expect(first.status).toBe(401); + expect(await first.json()).toMatchObject({ code: "invalid_upload_token" }); + for (let attempt = 2; attempt <= FAILURES; attempt++) await forged(); + + const limited = await forged(); + expect(limited.status).toBe(429); + expect(limited.headers.get("Access-Control-Allow-Origin")).toBe("*"); + + const genuine = await call( + sdk, + "POST", + "/api/developer/sdk/v1/upload/multipart/initiate", + { + key: freshKey("cut"), + headers: { + "Content-Type": "application/json", + "x-forwarded-for": "203.0.113.120", + }, + body: "{}", + }, + ); + expect(genuine.status).toBe(200); + }); +}); + describe("upload tokens have their own budget", () => { const initiate = (key: string) => call(sdk, "POST", "/api/developer/sdk/v1/upload/multipart/initiate", { @@ -517,8 +1109,8 @@ describe("upload tokens have their own budget", () => { expect(rateLimit(await initiate(freshKey("cut"))).remaining).toBe("1199"); }); - it("keeps the secret key at 60 a minute", async () => { - expect(rateLimit(await getUsage(freshKey("csk"))).limit).toBe("60"); + it("keeps the secret key at 300 a minute", async () => { + expect(rateLimit(await getUsage(freshKey("csk"))).limit).toBe("300"); }); it("documents the limits the limiter applies", async () => { @@ -542,6 +1134,12 @@ describe("upload tokens have their own budget", () => { expect(spec.info.description).toContain( `each upload token to ${token} requests per minute`, ); + expect(docs).toContain( + `All the keys of one account together allow ${DEVELOPER_OWNER_REQUESTS_PER_MINUTE.toLocaleString("en-US")} requests per minute`, + ); + expect(spec.info.description).toContain( + `all the keys of one account together to ${DEVELOPER_OWNER_REQUESTS_PER_MINUTE}`, + ); }); }); @@ -574,7 +1172,7 @@ describe("browser access", () => { await from("POST", "/api/developer/v1/usage"), ]; const sentWithKey = answers.length - 1; - for (let request = sentWithKey; request < 60; request++) { + for (let request = sentWithKey; request < KEY_LIMIT; request++) { await from("GET", "/api/developer/v1/usage"); } answers.push(await from("GET", "/api/developer/v1/usage")); @@ -626,11 +1224,33 @@ describe("browser access", () => { "access-control-allow-methods": "GET,POST,OPTIONS", "access-control-allow-headers": "Content-Type,Authorization", "access-control-expose-headers": - "Retry-After,X-RateLimit-Limit,X-RateLimit-Remaining,X-RateLimit-Reset", + "Retry-After,X-RateLimit-Limit,X-RateLimit-Remaining,X-RateLimit-Reset,X-Request-Id", + "access-control-max-age": "7200", + "cache-control": "private, no-store", vary: "Access-Control-Request-Headers", }); }); + it.each([ + "/api/developer/sdk/v1/upload/multipart/initiate", + "/api/developer/sdk/v1/upload/multipart/presign-part", + "/api/developer/sdk/v1/upload/multipart/complete", + "/api/developer/sdk/v1/upload/multipart/abort", + ])( + "lets a browser cache the preflight for %s for two hours", + async (path) => { + const preflight = await call(sdk, "OPTIONS", path, { + headers: { + ...PREFLIGHT_HEADERS, + "Access-Control-Request-Method": "POST", + }, + }); + + expect(preflight.status).toBe(204); + expect(preflight.headers.get("Access-Control-Max-Age")).toBe("7200"); + }, + ); + it("still lets a page read an upload route's answer", async () => { const response = await call( sdk, @@ -648,11 +1268,53 @@ describe("browser access", () => { }); }); +describe("the credential sent to the REST API", () => { + it("says the key is missing when no Authorization header is sent", async () => { + const response = await getUsage(""); + + expect(response.status).toBe(401); + expect(await response.json()).toMatchObject({ + code: "invalid_secret_key", + error: "Missing secret key", + hint: expect.stringContaining("No Authorization header was sent"), + }); + }); + + it("refuses a public key with its own code", async () => { + const response = await getUsage(freshKey("cpk")); + + expect(response.status).toBe(403); + expect(await response.json()).toMatchObject({ + code: "public_key_not_allowed", + error: "Public keys cannot call the REST API", + hint: expect.stringContaining("csk_"), + }); + expect(mocks.billing).not.toHaveBeenCalled(); + }); + + it("keeps the answers for any other credential and for a wrong secret key", async () => { + const other = await getUsage("not_a_key"); + mocks.keyRows = []; + const wrong = await getUsage(freshKey("csk")); + + expect(other.status).toBe(401); + expect(await other.json()).toMatchObject({ + code: "invalid_secret_key", + error: "Invalid secret key", + }); + expect(wrong.status).toBe(401); + expect(await wrong.json()).toMatchObject({ + code: "invalid_secret_key", + error: "Invalid or revoked secret key", + }); + }); +}); + describe("wrong method and unknown path", () => { it.each([ [rest, "PATCH", "/api/developer/v1/videos/video-1", "csk", "GET, DELETE"], [rest, "PUT", "/api/developer/v1/videos/video-1", "csk", "GET, DELETE"], - [rest, "POST", "/api/developer/v1/videos", "csk", "GET"], + [rest, "PUT", "/api/developer/v1/videos", "csk", "POST, GET"], [rest, "DELETE", "/api/developer/v1/videos/video-1/status", "csk", "GET"], [rest, "POST", "/api/developer/v1/usage", "csk", "GET"], [rest, "DELETE", "/api/developer/v1/transcripts", "csk", "POST, GET"], @@ -681,6 +1343,47 @@ describe("wrong method and unknown path", () => { }, ); + it("lists the 405 on every operation of a path that answers it, with the methods the path takes", async () => { + const spec = developerOpenApiSpec("https://cap.so") as unknown as { + paths: Record< + string, + Record }> + >; + }; + const methods = ["PUT", "PATCH", "DELETE", "POST", "GET"] as const; + const paths = Object.entries(spec.paths).filter(([path]) => + path.startsWith("/api/developer/"), + ); + expect(paths.length).toBeGreaterThan(15); + + for (const [path, item] of paths) { + const listed = Object.keys(item).map((method) => method.toUpperCase()); + for (const method of listed) { + expect( + item[method.toLowerCase()]?.responses?.["405"], + `${method} ${path}`, + ).toEqual({ $ref: "#/components/responses/MethodNotAllowed" }); + } + + const sdkPath = path.startsWith("/api/developer/sdk/"); + const concrete = path.replace("{id}", "video-1"); + const unlisted = methods.find((method) => !listed.includes(method)); + if (!unlisted) throw new Error(`${path} takes every method`); + const response = await call(sdkPath ? sdk : rest, unlisted, concrete, { + key: freshKey(sdkPath ? "cut" : "csk"), + }); + + expect(response.status, `${unlisted} ${path}`).toBe(405); + expect( + response.headers.get("Allow")?.split(", "), + `${unlisted} ${path}`, + ).toEqual(expect.arrayContaining(listed)); + expect(await response.json()).toMatchObject({ + code: "method_not_allowed", + }); + } + }); + it.each([ [rest, "GET", "/api/developer/v1/nope", "csk"], [rest, "POST", "/api/developer/v1/videos/a/b/c", "csk"], @@ -854,20 +1557,18 @@ describe("usage", () => { }, }); - const stats = mocks.selected.find((fields) => "uploadedVideos" in fields); + const aggregate = mocks.selected.filter((fields) => "rows" in fields); + expect(aggregate).toHaveLength(1); const dialect = new MySqlDialect(); const sqlOf = (name: string) => dialect.sqlToQuery( - stats?.[name] as Parameters[0], + aggregate[0]?.[name] as Parameters[0], ).sql; - expect(sqlOf("uploadedVideos")).toBe( - "count(`developer_videos`.`duration`)", - ); - expect(sqlOf("createdVideos")).toBe("count(*)"); + expect(sqlOf("count")).toBe("COUNT(`developer_videos`.`duration`)"); + expect(sqlOf("rows")).toBe("COUNT(*)"); }); it("reports zero for an app with no videos", async () => { - mocks.videoStats = []; mocks.storedVideos = []; mocks.accounts = []; @@ -885,6 +1586,7 @@ describe("usage", () => { microCreditsPerDollar: MICRO_CREDITS_PER_DOLLAR, storageDollarsPer1000MinutesPerMonth: INFRA_STORAGE_DOLLARS_PER_1000_MINUTES_PER_MONTH, + deliveryDollarsPer1000Minutes: INFRA_DELIVERY_DOLLARS_PER_1000_MINUTES, speechDollarsPerHour: SPEECH_DOLLARS_PER_HOUR, uploadAdmissionDays: UPLOAD_ADMISSION_DAYS, }); @@ -898,7 +1600,13 @@ describe("usage", () => { }); it("returns the fields the OpenAPI description requires", async () => { - mocks.billing.mockResolvedValue({ standing: {}, upload: {}, storage: {} }); + const storage = await getAppStorage("app-1"); + mocks.billing.mockResolvedValue({ + standing: {}, + upload: {}, + storage, + stored: storage.stored, + }); const schema = developerOpenApiSpec(ORIGIN).components.schemas.Usage; const data = await usage(); @@ -909,3 +1617,172 @@ describe("usage", () => { ); }); }); + +describe("Idempotency-Key on create upload", () => { + const create = (headers: Record) => + call(rest, "POST", "/api/developer/v1/uploads", { + key: freshKey("csk"), + headers: { "Content-Type": "application/json", ...headers }, + body: JSON.stringify({ name: "Demo" }), + }); + + it("passes a valid key to the store", async () => { + const response = await create({ "Idempotency-Key": "retry-upload-0001" }); + + expect(response.status).toBe(200); + expect(mocks.createVideo).toHaveBeenCalledExactlyOnceWith({ + appId: "app-1", + name: "Demo", + idempotencyKey: "retry-upload-0001", + }); + }); + + it.each([ + ["too short", "short"], + ["a colon", "retry:upload:0001"], + ["a space", "retry upload 0001"], + ["too long", "k".repeat(129)], + ])("refuses a key with %s before creating anything", async (_label, key) => { + const response = await create({ "Idempotency-Key": key }); + + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ + code: "invalid_idempotency_key", + error: "Idempotency-Key is not valid", + }); + expect(mocks.createVideo).not.toHaveBeenCalled(); + }); +}); + +describe("response headers on the developer APIs", () => { + const PRIVATE = "private, no-store"; + + it("marks every secret-key API answer private, errors and limits included", async () => { + const key = freshKey("csk"); + const answers = [ + await call(rest, "GET", "/api/developer/v1/usage", { key }), + await call(rest, "GET", "/api/developer/v1/usage"), + await call(rest, "GET", "/api/developer/v1/nope", { key }), + await call(rest, "POST", "/api/developer/v1/usage", { key }), + ]; + for ( + let request = 3; + request < DEVELOPER_KEY_REQUESTS_PER_MINUTE; + request++ + ) { + await call(rest, "GET", "/api/developer/v1/usage", { key }); + } + answers.push(await call(rest, "GET", "/api/developer/v1/usage", { key })); + + expect(answers.map((response) => response.status)).toEqual([ + 200, 401, 404, 405, 429, + ]); + for (const response of answers) { + expect(response.headers.get("Cache-Control")).toBe(PRIVATE); + } + }); + + it("marks the upload token API answers private too", async () => { + const answers = [ + await call( + sdk, + "POST", + "/api/developer/sdk/v1/upload/multipart/initiate", + { + key: freshKey("cut"), + headers: { "Content-Type": "application/json" }, + body: "{}", + }, + ), + await call( + sdk, + "POST", + "/api/developer/sdk/v1/upload/multipart/initiate", + { + headers: { "Content-Type": "application/json" }, + body: "{}", + }, + ), + ]; + + expect(answers.map((response) => response.status)).toEqual([200, 401]); + for (const response of answers) { + expect(response.headers.get("Cache-Control")).toBe(PRIVATE); + } + }); + + it("echoes the platform request id and lets a page read it", async () => { + const requestId = "fra1::iad1::abcde-1763031000000-0123456789ab"; + const rested = await call(rest, "GET", "/api/developer/v1/usage", { + key: freshKey("csk"), + headers: { "x-vercel-id": requestId }, + }); + const sdked = await call( + sdk, + "POST", + "/api/developer/sdk/v1/upload/multipart/initiate", + { + key: freshKey("cut"), + headers: { + Origin: "https://app.example", + "Content-Type": "application/json", + "x-vercel-id": requestId, + }, + body: "{}", + }, + ); + + expect(rested.headers.get("X-Request-Id")).toBe(requestId); + expect(sdked.headers.get("X-Request-Id")).toBe(requestId); + expect( + sdked.headers + .get("Access-Control-Expose-Headers") + ?.split(",") + .map((name) => name.trim()), + ).toContain("X-Request-Id"); + const withoutId = await call(rest, "GET", "/api/developer/v1/usage", { + key: freshKey("csk"), + }); + expect(withoutId.headers.get("X-Request-Id")).toBeNull(); + }); +}); + +describe("OpenAPI description", () => { + it("names the product Cap Infra API", () => { + const { info } = developerOpenApiSpec(ORIGIN); + expect(info.title).toBe("Cap Infra API"); + expect(info.description).toMatch(/^The Cap Infra API does two things/); + expect(JSON.stringify(info)).not.toContain("Developer API"); + }); +}); + +describe("video captions in OpenAPI", () => { + it("documents captions on the video as language and url, outside what webhooks must carry", () => { + const spec = developerOpenApiSpec(ORIGIN) as unknown as { + components: { + schemas: { + DeveloperVideo: { + required: string[]; + properties: { + captions: { + type: string; + items: { required: string[]; properties: object }; + }; + }; + }; + }; + }; + }; + const video = spec.components.schemas.DeveloperVideo; + expect(video.properties.captions.type).toBe("array"); + expect(video.properties.captions.items.required).toEqual([ + "language", + "url", + ]); + expect(Object.keys(video.properties.captions.items.properties)).toEqual([ + "language", + "url", + ]); + expect(video.required).not.toContain("captions"); + }); +}); diff --git a/apps/web/__tests__/unit/developer-app-limits.test.ts b/apps/web/__tests__/unit/developer-app-limits.test.ts new file mode 100644 index 00000000000..8653e31a5cf --- /dev/null +++ b/apps/web/__tests__/unit/developer-app-limits.test.ts @@ -0,0 +1,797 @@ +import * as Db from "@cap/database/schema"; +import type { SQL } from "drizzle-orm"; +import { MySqlDialect, type MySqlTable } from "drizzle-orm/mysql-core"; +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; + +const OWNER = "user_one"; +const LIMIT = 25; +const LIMIT_SENTENCE = + "You can have up to 25 apps. Delete one you no longer use, or contact us."; + +const mocks = vi.hoisted(() => ({ + db: vi.fn(), + limited: false, + limitChecks: [] as Array<{ ruleId: string; key?: string }>, +})); + +vi.mock("server-only", () => ({})); +vi.mock("next/cache", () => ({ revalidatePath: vi.fn() })); +vi.mock("@cap/database", async (importOriginal) => ({ + ...(await importOriginal()), + db: mocks.db, +})); +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: async () => ({ id: "user_one", email: "owner@example.com" }), +})); +vi.mock("@cap/database/crypto", () => ({ + encrypt: async (value: string) => `sealed:${value}`, + decrypt: async (value: string) => value.replace(/^sealed:/, ""), +})); +vi.mock("@/lib/infra-access", () => ({ canUseInfra: () => true })); +vi.mock("@/lib/rate-limit", async (importOriginal) => ({ + ...(await importOriginal()), + isRateLimited: async (ruleId: string, opts?: { key?: string }) => { + mocks.limitChecks.push({ ruleId, key: opts?.key }); + return mocks.limited; + }, +})); + +type Row = Record; + +const dialect = new MySqlDialect(); + +function matches(condition: SQL | undefined, row: Row) { + if (!condition) return true; + const query = dialect.sqlToQuery(condition); + let parameter = 0; + return query.sql + .replaceAll("(", "") + .replaceAll(")", "") + .split(" and ") + .every((clause) => { + const equals = /^`[^`]+`\.`([^`]+)` = \?$/.exec(clause); + if (equals?.[1]) return row[equals[1]] === query.params[parameter++]; + const empty = /^`[^`]+`\.`([^`]+)` is null$/.exec(clause); + if (empty?.[1]) return row[empty[1]] == null; + const among = /^`[^`]+`\.`([^`]+)` in ([?, ]+)$/.exec(clause); + if (among?.[1]) { + const size = among[2]?.split(",").length ?? 0; + const values = query.params.slice(parameter, parameter + size); + parameter += size; + return values.includes(row[among[1]]); + } + throw new Error(`Unexpected predicate: ${clause}`); + }); +} + +function makeDatabase() { + const apps: Row[] = []; + const keys: Row[] = []; + const accounts: Row[] = []; + const idempotency: Row[] = []; + const ownerLocks = new Map>(); + let writes = 0; + + const rowsOf = (table: MySqlTable) => { + if (table === Db.developerApps) return apps; + if (table === Db.developerApiKeys) return keys; + if (table === Db.developerCreditAccounts) return accounts; + if (table === Db.agentApiIdempotency) return idempotency; + throw new Error("Unexpected table"); + }; + + const makeClient = (held: Array<() => void> | null) => { + const lockOwner = async (ownerId: string) => { + if (!held) return; + const previous = ownerLocks.get(ownerId) ?? Promise.resolve(); + let release: () => void = () => {}; + ownerLocks.set( + ownerId, + new Promise((resolve) => { + release = resolve; + }), + ); + held.push(release); + await previous; + }; + + return { + select: (fields?: Record) => { + let table: MySqlTable; + let condition: SQL | undefined; + const counting = fields !== undefined && "total" in fields; + const read = (): Row[] => { + if (table === Db.agentApiKeys) { + return [ + { + id: OWNER, + email: "owner@example.com", + activeOrganizationId: "org_one", + tokenId: "token_one", + scopes: ["caps:read", "developer:write", "developer:secrets"], + expiresAt: new Date(Date.now() + 60_000), + revokedAt: null, + lastUsedAt: new Date(), + }, + ]; + } + if (table === Db.users) return [{ id: OWNER }]; + if (table === Db.developerApiKeys) { + const ids = dialect.sqlToQuery(condition as SQL).params; + return keys.filter((key) => ids.includes(key.id)); + } + const found = rowsOf(table).filter((row) => matches(condition, row)); + return counting ? [{ total: found.length }] : found; + }; + const settle = () => + Object.assign(Promise.resolve().then(read), { + limit: () => settle(), + for: async () => { + if (table === Db.users) await lockOwner(OWNER); + return read(); + }, + }); + const query = { + from: (value: MySqlTable) => { + table = value; + return query; + }, + innerJoin: () => query, + where: (value: SQL | undefined) => { + condition = value; + return settle(); + }, + }; + return query; + }, + insert: (table: MySqlTable) => ({ + values: (value: Row | Row[]) => { + const write = async () => { + writes += 1; + const target = rowsOf(table); + if (table === Db.developerApps) { + await new Promise((resolve) => setTimeout(resolve, 20)); + } + for (const row of Array.isArray(value) ? value : [value]) { + if ( + table === Db.agentApiIdempotency && + idempotency.some( + (existing) => + existing.userId === row.userId && + existing.operation === row.operation && + existing.keyHash === row.keyHash, + ) + ) { + continue; + } + target.push( + table === Db.agentApiIdempotency + ? { state: "pending", ...row } + : { deletedAt: null, ...row }, + ); + } + }; + return Object.assign(Promise.resolve().then(write), { + onDuplicateKeyUpdate: () => Promise.resolve(), + }); + }, + }), + update: (table: MySqlTable) => ({ + set: (patch: Row) => ({ + where: async (condition: SQL | undefined) => { + writes += 1; + const rows = rowsOf(table).filter((row) => matches(condition, row)); + for (const row of rows) { + for (const [field, value] of Object.entries(patch)) { + if (value !== undefined) row[field] = value; + } + } + return [{ affectedRows: rows.length }]; + }, + }), + }), + delete: (table: MySqlTable) => ({ + where: async (condition: SQL | undefined) => { + const rows = rowsOf(table); + for (const row of rows.filter((value) => matches(condition, value))) { + rows.splice(rows.indexOf(row), 1); + } + return [{ affectedRows: 1 }]; + }, + }), + }; + }; + + const client = { + ...makeClient(null), + transaction: async (run: (tx: unknown) => Promise) => { + const held: Array<() => void> = []; + try { + return await run(makeClient(held)); + } finally { + for (const release of held) release(); + } + }, + }; + + return { + client, + apps, + keys, + accounts, + writes: () => writes, + seed(live: number, deleted = 0) { + for (let index = 0; index < live + deleted; index += 1) { + apps.push({ + id: `seeded_${index}`, + ownerId: OWNER, + name: `Seeded ${index}`, + environment: "development", + deletedAt: index < live ? null : new Date(), + }); + } + }, + liveApps: () => apps.filter((app) => app.deletedAt === null), + deleteOne() { + const app = apps.find((candidate) => candidate.deletedAt === null); + if (app) app.deletedAt = new Date(); + }, + }; +} + +describe("limits on creating developer apps", () => { + let POST: typeof import("@/app/api/v1/[...route]/route").POST; + let PATCH: typeof import("@/app/api/v1/[...route]/route").PATCH; + let createDeveloperApp: typeof import("@/actions/developers/create-app").createDeveloperApp; + let updateDeveloperApp: typeof import("@/actions/developers/update-app").updateDeveloperApp; + let regenerateDeveloperKeys: typeof import("@/actions/developers/regenerate-keys").regenerateDeveloperKeys; + let database: ReturnType; + let requests = 0; + + beforeAll(async () => { + Object.assign(process.env, { + NEXT_PUBLIC_WEB_URL: "http://localhost", + WEB_URL: "http://localhost", + DATABASE_URL: "mysql://unused:unused@127.0.0.1:3306/unused", + NEXTAUTH_SECRET: "synthetic-test-secret-that-is-long-enough", + NEXTAUTH_URL: "http://localhost", + CAP_AWS_BUCKET: "synthetic", + CAP_AWS_REGION: "us-east-1", + CAP_AWS_ACCESS_KEY: "synthetic", + CAP_AWS_SECRET_KEY: "synthetic", + NODE_ENV: "test", + }); + ({ POST, PATCH } = await import("@/app/api/v1/[...route]/route")); + ({ createDeveloperApp } = await import("@/actions/developers/create-app")); + ({ updateDeveloperApp } = await import("@/actions/developers/update-app")); + ({ regenerateDeveloperKeys } = await import( + "@/actions/developers/regenerate-keys" + )); + }); + + beforeEach(() => { + database = makeDatabase(); + mocks.db.mockReturnValue(database.client); + mocks.limited = false; + mocks.limitChecks = []; + }); + + const agentRequest = ( + method: "POST" | "PATCH", + path: string, + body: Record, + ) => + new Request(`http://localhost/api/v1${path}`, { + method, + headers: { + Authorization: `Bearer cap_cli_${"a".repeat(43)}`, + "Content-Type": "application/json", + "Idempotency-Key": `app-limit-key-${requests++}`, + "X-Cap-Confirmation": "user", + }, + body: JSON.stringify(body), + }); + + const agentCreate = (name = "Agent app") => + POST( + agentRequest("POST", "/developer/apps", { + name, + environment: "development", + }), + ); + + const agentRename = (appId: string, name: string) => + PATCH(agentRequest("PATCH", `/developer/apps/${appId}`, { name })); + + const consoleCreate = (name = "Console app") => + createDeveloperApp({ name, environment: "development" }); + + const agentRotate = (appId: string) => + POST(agentRequest("POST", `/developer/apps/${appId}/keys/rotate`, {})); + + const agentUpdate = (appId: string, body: Record) => + PATCH(agentRequest("PATCH", `/developer/apps/${appId}`, body)); + + describe("the cap of 25 live apps", () => { + it("lets the console create the 25th app and refuses the 26th", async () => { + database.seed(LIMIT - 1); + + expect(await consoleCreate()).toMatchObject({ + secretKey: expect.stringMatching(/^csk_/), + }); + const keysBefore = database.keys.length; + expect(await consoleCreate()).toEqual({ + success: false, + message: LIMIT_SENTENCE, + }); + + expect(database.liveApps()).toHaveLength(LIMIT); + expect(database.keys).toHaveLength(keysBefore); + expect(database.accounts).toHaveLength(1); + }); + + it("lets the agent API create the 25th app and refuses the 26th", async () => { + database.seed(LIMIT - 1); + + const allowed = await agentCreate(); + expect(allowed.status).toBe(200); + expect(await allowed.json()).toMatchObject({ + secretKey: expect.stringMatching(/^csk_/), + }); + + const refused = await agentCreate(); + expect(refused.status).toBe(409); + expect(await refused.json()).toMatchObject({ + code: "CONFLICT", + message: LIMIT_SENTENCE, + retryable: false, + }); + expect(database.liveApps()).toHaveLength(LIMIT); + expect(database.accounts).toHaveLength(1); + }); + + it("leaves 25 when two console creates race at 24", async () => { + database.seed(LIMIT - 1); + + const results = await Promise.all([consoleCreate(), consoleCreate()]); + + expect(results.filter((result) => "appId" in result)).toHaveLength(1); + expect(results.filter((result) => "success" in result)).toEqual([ + { success: false, message: LIMIT_SENTENCE }, + ]); + expect(database.liveApps()).toHaveLength(LIMIT); + }); + + it("leaves 25 when two agent creates race at 24", async () => { + database.seed(LIMIT - 1); + + const responses = await Promise.all([agentCreate(), agentCreate()]); + + expect(responses.map((response) => response.status).sort()).toEqual([ + 200, 409, + ]); + expect(database.liveApps()).toHaveLength(LIMIT); + }); + + it("leaves 25 when the console and the agent API race at 24", async () => { + database.seed(LIMIT - 1); + + const [fromConsole, fromAgent] = await Promise.all([ + consoleCreate(), + agentCreate(), + ]); + + expect( + Number("appId" in fromConsole) + Number(fromAgent.status === 200), + ).toBe(1); + expect(database.liveApps()).toHaveLength(LIMIT); + }); + + it("does not count a deleted app", async () => { + database.seed(LIMIT - 1, 10); + + expect(await consoleCreate()).toHaveProperty("appId"); + expect(database.liveApps()).toHaveLength(LIMIT); + + database.deleteOne(); + expect((await agentCreate()).status).toBe(200); + expect(database.liveApps()).toHaveLength(LIMIT); + expect(database.apps).toHaveLength(LIMIT + 11); + }); + + it("lets a refused agent create succeed with the same key once an app is deleted", async () => { + database.seed(LIMIT); + const request = agentRequest("POST", "/developer/apps", { + name: "Retried", + environment: "production", + }); + + expect((await POST(request.clone())).status).toBe(409); + database.deleteOne(); + expect((await POST(request.clone())).status).toBe(200); + expect(database.liveApps()).toHaveLength(LIMIT); + }); + }); + + describe("the rate limit on creation", () => { + it("is checked per user under its own rule on both paths", async () => { + await consoleCreate(); + await agentCreate(); + + expect(mocks.limitChecks).toEqual([ + { + ruleId: "rl_developer_app_create", + key: "developer-app-create:user_one", + }, + { + ruleId: "rl_developer_app_create", + key: "developer-app-create:user_one", + }, + ]); + }); + + it("refuses a console create with no database write", async () => { + mocks.limited = true; + + expect(await consoleCreate()).toEqual({ + success: false, + message: "You are creating apps too quickly. Try again in an hour.", + }); + expect(database.writes()).toBe(0); + expect(database.apps).toHaveLength(0); + }); + + it("refuses an agent create with no database write", async () => { + mocks.limited = true; + + const response = await agentCreate(); + + expect(response.status).toBe(429); + expect(await response.json()).toMatchObject({ code: "RATE_LIMITED" }); + expect(database.writes()).toBe(0); + expect(database.apps).toHaveLength(0); + }); + + it("allows ten creations an hour for one user when no shared rule answers", async () => { + const { isRateLimited, RATE_LIMIT_IDS } = + await vi.importActual( + "@/lib/rate-limit", + ); + const attempt = (user: string) => + isRateLimited(RATE_LIMIT_IDS.DEVELOPER_APP_CREATE, { + key: `developer-app-create:${user}`, + headers: new Headers(), + }); + const outcomes = []; + for (let index = 0; index < 11; index += 1) { + outcomes.push(await attempt("user_window")); + } + + expect(RATE_LIMIT_IDS.DEVELOPER_APP_CREATE).toBe( + "rl_developer_app_create", + ); + expect(outcomes).toEqual([...Array(10).fill(false), true]); + expect(await attempt("user_other")).toBe(false); + }); + }); + + describe("the name rule", () => { + const longest = "a".repeat(100); + const tooLong = "a".repeat(101); + const lengthSentence = "An app name can be up to 100 characters."; + + it("refuses a blank or over-long name on a console create", async () => { + expect(await consoleCreate(" ")).toEqual({ + success: false, + message: "Give the app a name.", + }); + expect(await consoleCreate(tooLong)).toEqual({ + success: false, + message: lengthSentence, + }); + expect(database.writes()).toBe(0); + expect(mocks.limitChecks).toEqual([]); + }); + + it("refuses a blank or over-long name on an agent create", async () => { + const blank = await agentCreate(" "); + expect(blank.status).toBe(400); + expect(await blank.json()).toMatchObject({ + code: "INVALID_REQUEST", + message: "Give the app a name.", + }); + + const long = await agentCreate(tooLong); + expect(long.status).toBe(400); + expect(await long.json()).toMatchObject({ + code: "INVALID_REQUEST", + message: lengthSentence, + }); + expect(database.writes()).toBe(0); + expect(mocks.limitChecks).toEqual([]); + }); + + it("stores a name of exactly 100 characters, trimmed, on both paths", async () => { + expect(await consoleCreate(` ${longest} `)).toHaveProperty("appId"); + expect((await agentCreate(` ${longest} `)).status).toBe(200); + + expect(database.apps.map((app) => app.name)).toEqual([longest, longest]); + }); + + it("refuses a blank or over-long name on a console rename", async () => { + database.seed(1); + + expect( + await updateDeveloperApp({ appId: "seeded_0", name: " " }), + ).toEqual({ success: false, message: "Give the app a name." }); + expect( + await updateDeveloperApp({ appId: "seeded_0", name: tooLong }), + ).toEqual({ success: false, message: lengthSentence }); + expect(database.apps[0]?.name).toBe("Seeded 0"); + + expect( + await updateDeveloperApp({ appId: "seeded_0", name: ` ${longest} ` }), + ).toEqual({ success: true }); + expect(database.apps[0]?.name).toBe(longest); + }); + + it("refuses a blank or over-long name on an agent rename", async () => { + database.seed(1); + + const blank = await agentRename("seeded_0", " "); + expect(blank.status).toBe(400); + expect(await blank.json()).toMatchObject({ + code: "INVALID_REQUEST", + message: "Give the app a name.", + }); + + const long = await agentRename("seeded_0", tooLong); + expect(long.status).toBe(400); + expect(await long.json()).toMatchObject({ + code: "INVALID_REQUEST", + message: lengthSentence, + }); + expect(database.apps[0]?.name).toBe("Seeded 0"); + + expect((await agentRename("seeded_0", ` ${longest} `)).status).toBe(200); + expect(database.apps[0]?.name).toBe(longest); + }); + }); + describe("a repeated agent request with the same idempotency key", () => { + const repeatable = (path: string, body: Record) => { + const request = agentRequest("POST", path, body); + return () => POST(request.clone()); + }; + + it("returns the saved app keys without spending the creation allowance", async () => { + const create = repeatable("/developer/apps", { + name: "Replayed", + environment: "development", + }); + + const first = await create(); + expect(first.status).toBe(200); + const saved = await first.json(); + expect(mocks.limitChecks).toHaveLength(1); + + mocks.limited = true; + for (let attempt = 0; attempt < 12; attempt += 1) { + const replay = await create(); + expect(replay.status).toBe(200); + expect(await replay.json()).toMatchObject({ + appId: saved.appId, + secretKey: saved.secretKey, + }); + } + + expect(mocks.limitChecks).toHaveLength(1); + expect(database.liveApps()).toHaveLength(1); + expect((await agentCreate("Another")).status).toBe(429); + expect(mocks.limitChecks).toHaveLength(2); + }); + + it("returns the saved rotated keys without spending the rotation allowance", async () => { + database.seed(1); + const rotate = repeatable("/developer/apps/seeded_0/keys/rotate", {}); + + const first = await rotate(); + expect(first.status).toBe(200); + const saved = await first.json(); + expect(mocks.limitChecks).toHaveLength(1); + + mocks.limited = true; + for (let attempt = 0; attempt < 12; attempt += 1) { + const replay = await rotate(); + expect(replay.status).toBe(200); + expect(await replay.json()).toMatchObject({ + secretKey: saved.secretKey, + }); + } + + expect(mocks.limitChecks).toHaveLength(1); + expect(database.keys).toHaveLength(2); + expect((await agentRotate("seeded_0")).status).toBe(429); + }); + + it("charges a request that was limited again when it is retried and runs", async () => { + const create = repeatable("/developer/apps", { + name: "Retried", + environment: "development", + }); + + mocks.limited = true; + expect((await create()).status).toBe(429); + expect(database.writes()).toBe(0); + + mocks.limited = false; + expect((await create()).status).toBe(200); + expect(mocks.limitChecks).toHaveLength(2); + expect(database.liveApps()).toHaveLength(1); + }); + }); + + describe("the rate limit on rotating keys", () => { + const rotationSentence = + "You are rotating keys too quickly. Try again in an hour."; + + it("is checked per user under its own rule on both paths", async () => { + database.seed(1); + + expect(await regenerateDeveloperKeys("seeded_0")).toMatchObject({ + secretKey: expect.stringMatching(/^csk_/), + }); + expect((await agentRotate("seeded_0")).status).toBe(200); + + expect(mocks.limitChecks).toEqual([ + { + ruleId: "rl_developer_key_rotate", + key: "developer-key-rotate:user_one", + }, + { + ruleId: "rl_developer_key_rotate", + key: "developer-key-rotate:user_one", + }, + ]); + expect(database.keys).toHaveLength(4); + expect(database.keys.filter((key) => key.revokedAt)).toHaveLength(2); + }); + + it("refuses a console rotation with no database write", async () => { + database.seed(1); + mocks.limited = true; + + expect(await regenerateDeveloperKeys("seeded_0")).toEqual({ + success: false, + message: rotationSentence, + }); + expect(database.writes()).toBe(0); + expect(database.keys).toHaveLength(0); + }); + + it("refuses an agent rotation with no database write", async () => { + database.seed(1); + mocks.limited = true; + + const response = await agentRotate("seeded_0"); + + expect(response.status).toBe(429); + expect(await response.json()).toMatchObject({ code: "RATE_LIMITED" }); + expect(database.writes()).toBe(0); + expect(database.keys).toHaveLength(0); + }); + + it("allows ten rotations an hour for one user when no shared rule answers", async () => { + const { isRateLimited, RATE_LIMIT_IDS } = + await vi.importActual( + "@/lib/rate-limit", + ); + const outcomes = []; + for (let index = 0; index < 11; index += 1) { + outcomes.push( + await isRateLimited(RATE_LIMIT_IDS.DEVELOPER_KEY_ROTATE, { + key: "developer-key-rotate:user_window", + headers: new Headers(), + }), + ); + } + + expect(outcomes).toEqual([...Array(10).fill(false), true]); + }); + }); + + describe("the environment and logo rules", () => { + const environmentSentence = "Choose development or production."; + const logoSentence = + "A logo must be an https URL of up to 1,024 characters."; + const longLogo = `https://cdn.example.com/${"a".repeat(1024)}`; + + it("refuses an unknown environment on a console create", async () => { + expect( + await createDeveloperApp({ + name: "Console app", + environment: "staging" as "production", + }), + ).toEqual({ success: false, message: environmentSentence }); + expect(database.writes()).toBe(0); + }); + + it("refuses an unknown environment on a console update", async () => { + database.seed(1); + + expect( + await updateDeveloperApp({ + appId: "seeded_0", + environment: "staging" as "production", + }), + ).toEqual({ success: false, message: environmentSentence }); + expect(database.apps[0]?.environment).toBe("development"); + + expect( + await updateDeveloperApp({ + appId: "seeded_0", + environment: "production", + }), + ).toEqual({ success: true }); + expect(database.apps[0]?.environment).toBe("production"); + }); + + it("refuses an unknown environment on an agent update", async () => { + database.seed(1); + + const response = await agentUpdate("seeded_0", { + environment: "staging", + }); + + expect(response.status).toBe(400); + expect(database.apps[0]?.environment).toBe("development"); + }); + + it.each([ + ["a script URL", "javascript:alert(1)"], + ["plain http", "http://cdn.example.com/logo.png"], + ["a bare word", "logo"], + ["an over-long URL", longLogo], + ["an empty string", " "], + ])("refuses %s as a logo on both paths", async (_label, logoUrl) => { + database.seed(1); + + expect(await updateDeveloperApp({ appId: "seeded_0", logoUrl })).toEqual({ + success: false, + message: logoSentence, + }); + + const response = await agentUpdate("seeded_0", { logoUrl }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ + code: "INVALID_REQUEST", + message: logoSentence, + }); + expect(database.apps[0]?.logoUrl).toBeUndefined(); + }); + + it("stores an https logo, trimmed, and clears it with null on both paths", async () => { + database.seed(2); + const logoUrl = "https://cdn.example.com/logo.png"; + + expect( + await updateDeveloperApp({ + appId: "seeded_0", + logoUrl: ` ${logoUrl} `, + }), + ).toEqual({ success: true }); + expect( + (await agentUpdate("seeded_1", { logoUrl: ` ${logoUrl} ` })).status, + ).toBe(200); + expect(database.apps.map((app) => app.logoUrl)).toEqual([ + logoUrl, + logoUrl, + ]); + + await updateDeveloperApp({ appId: "seeded_0", logoUrl: null }); + expect((await agentUpdate("seeded_1", { logoUrl: null })).status).toBe( + 200, + ); + expect(database.apps.map((app) => app.logoUrl)).toEqual([null, null]); + }); + }); +}); diff --git a/apps/web/__tests__/unit/developer-billing-docs.test.ts b/apps/web/__tests__/unit/developer-billing-docs.test.ts index 6967402fd33..6861ec54460 100644 --- a/apps/web/__tests__/unit/developer-billing-docs.test.ts +++ b/apps/web/__tests__/unit/developer-billing-docs.test.ts @@ -24,9 +24,11 @@ vi.mock("@/lib/developer-credits-product", () => ({ developerCreditsProductId: () => "prod_test", })); +import { developerOpenApiSpec } from "@/lib/developer-openapi"; import { dailyStorageMicroCredits, emptyStorageMinutes, + firstMonthStorageMicroCredits, MICRO_CREDITS_PER_DOLLAR, monthlyStorageMicroCredits, SPEECH_DOLLARS_PER_HOUR, @@ -43,13 +45,18 @@ import { } from "@/lib/developer-transcript-hold"; import { TRANSCRIPT_MIN_START_BALANCE_MICRO_CREDITS } from "@/lib/developer-transcripts"; import { + AUTO_TOP_UP_DAILY_LIMIT, AUTO_TOP_UP_MAX_AMOUNT_CENTS, AUTO_TOP_UP_MAX_THRESHOLD_MICRO_CREDITS, AUTO_TOP_UP_MIN_AMOUNT_CENTS, AUTO_TOP_UP_MIN_THRESHOLD_MICRO_CREDITS, + AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS, } from "@/lib/infra-auto-top-up"; import { LEDGER_PAGE_SIZE } from "@/lib/infra-ledger"; import { + INFRA_DELIVERY_BILLED_FROM_TEXT, + INFRA_DELIVERY_DOLLARS_PER_1000_MINUTES, + INFRA_DELIVERY_PRICE_TEXT, INFRA_RESOLUTION_TIERS, INFRA_STORAGE_DOLLARS_PER_1000_MINUTES_PER_MONTH, INFRA_TIER_MAX_PIXELS, @@ -57,6 +64,8 @@ import { } from "@/lib/infra-pricing"; import { PRO_MONTHLY_CREDIT_DOLLARS } from "@/lib/infra-pro-credit"; import { + DELIVERY_DAY_REFERENCE_TYPE, + DELIVERY_DAY_TRANSACTION_TYPE, INFRA_GRACE_DAYS, uploadAdmission, uploadAdmissionError, @@ -74,6 +83,21 @@ import { docsConfig } from "../../app/(docs)/docs/docs-config"; const billing = getDocBySlug("api/billing")?.content ?? ""; const rest = getDocBySlug("api/rest-api")?.content ?? ""; +const getStarted = getDocBySlug("api/get-started")?.content ?? ""; + +const POLICY_SENTENCES = [ + "Stripe emails a receipt for every payment, a purchase or an auto top-up, to the email on your Cap account.", + "A purchase also comes with a paid invoice, which shows any tax ID you add at checkout.", + "Stripe's checkout has no US tax ID type, so a US business adds its EIN with \"Add your EIN\" on the console's Billing page before it pays, and the EIN is then on the invoice.", + "Credit does not expire.", + "If a payment is refunded or disputed, the credit it bought is taken back, and a partial refund takes back the same share of it.", +]; + +const DELIVERY_SENTENCE = + "**Delivery.** `delivery` is the minutes of video delivered to your viewers in the same calendar month as `speech`, in total and for each rendition tier, and it is `null` until delivery has been measured for the app. Delivered minutes are the media seconds in the video bytes we actually send, counting part of a segment in proportion; transfers the viewer abandons part-way aren't counted. For HLS it counts the exact seconds of the segments served, from each segment's EXTINF duration and EXT-X-BYTERANGE range. For MP4 downloads it counts the bytes served divided by the file's average bytes per second. Playlists, posters, `HEAD` requests, responses other than `200` or `206` and empty responses are never counted. A day's figures become final once all of its delivery logs are in, normally in the first update after 02:00 UTC two days later, and `finalThrough` is the latest final day in the period, so today and yesterday may still change."; + +const DELIVERY_EXAMPLE = + /"delivery": \{[\s\S]*?"finalThrough": "\d{4}-\d{2}-\d{2}"\n {4}\},/; const wholeDollars = (cents: number) => `$${(cents / 100).toLocaleString("en-US")}`; @@ -128,9 +152,11 @@ describe("the Billing page", () => { compileMDX({ source: rest, components: {} }), ).resolves.toBeDefined(); + const slugsOf = (source: string) => + extractHeadings(source.replace(/^####\s/gm, "### ")).map((h) => h.slug); const anchors = new Map([ - ["/docs/api/rest-api", extractHeadings(rest).map((h) => h.slug)], - ["/docs/api/billing", extractHeadings(billing).map((h) => h.slug)], + ["/docs/api/rest-api", slugsOf(rest)], + ["/docs/api/billing", slugsOf(billing)], ]); const links = [ ...billing.matchAll(/\]\((\/docs\/api\/rest-api)#([\w-]+)\)/g), @@ -152,13 +178,16 @@ describe("the Billing page", () => { for (const tier of INFRA_RESOLUTION_TIERS) { const price = `$${INFRA_STORAGE_DOLLARS_PER_1000_MINUTES_PER_MONTH[tier].toFixed(2)}`; expect(billing).toContain( - `| ${names[tier]} | ${price} per 1,000 minutes per month | Once a day |`, + `| ${names[tier]} | ${price} per 1,000 minutes per month | First month when the video is ready, then once a day after it |`, ); expect(rest).toContain(`| ${price} |`); } expect(billing).toContain( `| Speech to text | $${SPEECH_DOLLARS_PER_HOUR.toFixed(2)} per audio hour | When a transcript completes |`, ); + expect(billing).toContain( + `| Delivery | ${INFRA_DELIVERY_PRICE_TEXT} | Once a day, when the day is final |`, + ); expect(billing).toContain( `which is ${SPEECH_MICRO_CREDITS_PER_HOUR.toLocaleString("en-US")} micro-credits`, ); @@ -206,6 +235,90 @@ describe("the Billing page", () => { expect(rest).toContain(`covers ${UPLOAD_ADMISSION_DAYS} days of storage`); }); + it("works every first month example out from the pricing functions", () => { + const tiers: Record = { + "1080p": "1080p", + "1440p": "1440p", + "4K": "2160p", + }; + const rows = tableRows(billing, "### The first month"); + expect(rows).toHaveLength(3); + for (const [video, firstMonth, perDay] of rows) { + const match = /^([\d,]+) minutes at (\S+)$/.exec(video ?? ""); + expect(match, video).not.toBeNull(); + const tier = tiers[match?.[2] ?? ""]; + expect(tier, video).toBeDefined(); + if (!tier) continue; + const minutes = Number(match?.[1]?.replaceAll(",", "")); + expect(firstMonth).toBe( + formatCharge( + firstMonthStorageMicroCredits({ + durationSeconds: minutes * 60, + tier, + }), + ), + ); + expect(perDay).toBe( + formatCharge( + dailyStorageMicroCredits({ + ...emptyStorageMinutes(), + [tier]: minutes, + }), + ), + ); + } + const twentyMinutes4K = firstMonthStorageMicroCredits({ + durationSeconds: 1200, + tier: "2160p", + }); + expect(billing).toContain( + `With ${formatNeed(uploadAdmissionMicroCredits(0, twentyMinutes4K))} of credit and nothing else in flight, one 20 minute video billed at 4K is accepted. Three of them in flight need ${formatNeed(uploadAdmissionMicroCredits(0, 3 * twentyMinutes4K))}.`, + ); + expect(billing).toContain( + `The month is ${STORAGE_BILLING_DAYS_PER_MONTH} UTC days`, + ); + }); + + it("states the first month rule and its example the same way in the billing and get started docs", () => { + const rule = + "Each video is charged one month of storage when it first becomes ready, then daily once that month is over. Uploading and encoding are free."; + const tenMinutes = { durationSeconds: 600, tier: "1080p" as const }; + const example = `A 10 minute 1080p video costs ${formatCharge(firstMonthStorageMicroCredits(tenMinutes))} when it becomes ready, then ${formatCharge(dailyStorageMicroCredits({ ...emptyStorageMinutes(), "1080p": 10 }))} a day after its first month.`; + + for (const doc of [billing, getStarted]) { + expect(doc).toContain(rule); + expect(doc).toContain(example); + } + expect(billing).toContain( + "Videos that were ready before November 13, 2026 are not charged a first month.", + ); + expect(billing).not.toMatch(/\b(before|from) today\b/i); + }); + + it("says in the billing and REST docs that a video deleted while it is processed pays its first month", () => { + const deleted = + "A video deleted while it is being processed is charged its first month too"; + expect(billing).toContain(deleted); + expect(rest).toContain(deleted); + expect(rest).not.toMatch(/\b(before|from) today\b/i); + }); + + it("says grace can follow any storage charge, first month or daily", () => { + const outOfCredit = + "| `402` | `out_of_credit` | The app is in `grace` or `paused` because a storage or delivery charge could not be paid. The message says how much to add |"; + const graceStanding = + "grace: a storage or delivery charge was more than the balance, uploads are blocked"; + const specText = JSON.stringify(developerOpenApiSpec("https://cap.so")); + + expect(rest).toContain(outOfCredit); + expect(specText).toContain(graceStanding); + for (const text of [rest, billing, specText]) { + expect(text).not.toMatch( + /daily storage charge (could not be paid|was more than)/, + ); + } + }); + it("works every speech example out from the pricing functions", () => { const rows = tableRows(billing, "## Speech to text"); expect(rows).toHaveLength(5); @@ -257,6 +370,11 @@ describe("the Billing page", () => { expect(billing).toContain( `One purchase is from ${wholeDollars(TOP_UP_MIN_CENTS)} to ${wholeDollars(TOP_UP_MAX_CENTS)}`, ); + expect( + fs.readFileSync(path.join(process.cwd(), "public/llms-full.txt"), "utf8"), + ).toContain( + `top-ups from ${wholeDollars(TOP_UP_MIN_CENTS)} to ${wholeDollars(TOP_UP_MAX_CENTS)}, codes`, + ); expect(billing).toContain( `Cap Pro includes $${PRO_MONTHLY_CREDIT_DOLLARS} of credit every month`, ); @@ -269,6 +387,12 @@ describe("the Billing page", () => { expect(billing).toContain( `${INFRA_GRACE_DAYS} days after the app first fell short`, ); + expect(billing).toContain( + "- **When.** As soon as the available balance falls below your threshold.", + ); + expect(billing).toContain( + `up to ${AUTO_TOP_UP_DAILY_LIMIT} top-ups in any 24 hours, and at most ${wholeDollars(AUTO_TOP_UP_OWNER_DAILY_LIMIT_CENTS)} a day across all your apps.`, + ); expect(billing).toContain(`${LEDGER_PAGE_SIZE} to a page`); expect(billing).toContain( `reads ${formatCredits(2_999_901)} and never $30.00`, @@ -288,7 +412,17 @@ describe("the Billing page", () => { expect(billing.split(times)).toHaveLength(3); }); + it("states receipts, expiry and refunds only in the agreed sentences", () => { + for (const sentence of POLICY_SENTENCES) { + expect(billing.split(sentence), sentence).toHaveLength(2); + } + }); + it("leaves the open policy questions and uncharged prices out", () => { + const unagreed = POLICY_SENTENCES.reduce( + (doc, sentence) => doc.replace(sentence, ""), + billing, + ); for (const word of [ "refund", "refunds", @@ -303,13 +437,74 @@ describe("the Billing page", () => { "invoices", "receipt", "receipts", - "delivery", "watched", ]) { - expect(billing, word).not.toMatch(new RegExp(`\\b${word}\\b`, "i")); + expect(unagreed, word).not.toMatch(new RegExp(`\\b${word}\\b`, "i")); } expect(billing).not.toContain(String.fromCodePoint(0x2014)); - expect(rest).not.toMatch(/\bdelivery\b|\bwatched\b/i); + expect(rest).not.toMatch(/\bwatched\b/i); + expect(rest.split(DELIVERY_SENTENCE)).toHaveLength(2); + expect(DELIVERY_SENTENCE).not.toMatch(/bill|price|charge|cost|credit|\$/i); + expect(rest).toMatch(DELIVERY_EXAMPLE); + }); + + it("documents delivery billing and its ledger rows the way the code writes them", () => { + const definition = + "A delivered minute is a minute of video your viewers' players download. Delivered minutes are the media seconds in the video bytes we actually send, counting part of a segment in proportion; transfers the viewer abandons part-way aren't counted. For HLS it counts the exact seconds of the segments served, from each segment's EXTINF duration and EXT-X-BYTERANGE range. For MP4 downloads it counts the bytes served divided by the file's average bytes per second. Playlists, posters, `HEAD` requests, responses other than `200` or `206` and empty responses are never counted."; + const spec = developerOpenApiSpec("https://cap.so") as unknown as { + components: { + schemas: { + Transaction: { + properties: { + type: { description: string }; + referenceType: { enum: (string | null)[] }; + }; + }; + Usage: { properties: { rates: { properties: object } } }; + }; + }; + }; + const transaction = spec.components.schemas.Transaction.properties; + + for (const doc of [billing, getStarted]) { + expect(doc).toContain(definition); + } + for (const doc of [billing, rest, getStarted]) { + expect(doc).not.toMatch(/\b(before|from) today\b/i); + } + for (const sentence of definition.split(/(?<=\.) (?=[A-Z])/).slice(1)) { + expect(rest.split(sentence), sentence).toHaveLength(2); + expect(billing.split(sentence), sentence).toHaveLength(2); + } + expect(billing).toContain( + "Delivery follows the same balance, shortfall and grace rules as storage.", + ); + expect(billing).toContain(INFRA_DELIVERY_BILLED_FROM_TEXT); + expect(billing).toContain( + "Each day is charged once from the app's credit balance, when its figures become final.", + ); + expect(billing).toContain( + "Three things are charged: stored video, delivered video and speech to text. Uploading and encoding are free.", + ); + expect(billing).toContain("or one day of delivery,"); + expect(billing).not.toMatch(/abandonedBytes/i); + expect(billing).toContain("| Delivery | The UTC day it charges |"); + expect(rest).toContain( + `| \`${DELIVERY_DAY_TRANSACTION_TYPE}\` | Delivery | \`${DELIVERY_DAY_REFERENCE_TYPE}\` | The UTC day, as \`YYYY-MM-DD\` |`, + ); + expect(transaction.type.description).toContain( + `${DELIVERY_DAY_TRANSACTION_TYPE} (one row per app per delivered day)`, + ); + expect(transaction.referenceType.enum).toContain( + DELIVERY_DAY_REFERENCE_TYPE, + ); + expect( + Object.keys(spec.components.schemas.Usage.properties.rates.properties), + ).toContain("deliveryDollarsPer1000Minutes"); + expect(rest).not.toContain("The delivery price is not in `rates`."); + expect(billing).toContain( + 'labelled "Delivery", with the UTC day as its reference.', + ); }); }); @@ -339,17 +534,18 @@ describe("examples in the REST API reference", () => { balanceMicroCredits: body.balanceMicroCredits, standing: { state: "active", pausesAt: null }, lastOwedMicroCredits: 0, - usage: { - monthlyMicroCredits: - (body.requiredMicroCredits * STORAGE_BILLING_DAYS_PER_MONTH) / - UPLOAD_ADMISSION_DAYS, + storage: { + dailyCharge: { monthlyMicroCredits: 0 }, + firstMonthMicroCredits: body.requiredMicroCredits, }, }), + "app_789", ); expect(body).toEqual(refusal); + expect(body.message).toMatch(/^Not enough credit /); expect(body.message).toContain( - `needs ${formatNeed(body.requiredMicroCredits)} of credit`, + `needs ${formatNeed(body.requiredMicroCredits)}, enough for the first month of storage`, ); expect(body.message).toContain( `The balance is ${formatCredits(body.balanceMicroCredits)}, so add at least ${formatNeed(body.shortfallMicroCredits)}.`, @@ -361,13 +557,20 @@ describe("examples in the REST API reference", () => { it("keeps the usage example consistent with the pricing functions", () => { const { data } = jsonAfter(rest, "### Get Usage"); - const monthly = monthlyStorageMicroCredits(data.storage.minutesByTier); + const daily = data.storage.dailyChargeMinutesByTier; + const monthly = monthlyStorageMicroCredits(daily); expect(data.balanceDollars).toBe(creditDollars(data.balanceMicroCredits)); expect(data.storage.monthlyMicroCredits).toBe(Math.round(monthly)); + expect(data.storage.storedMonthlyMicroCredits).toBe( + Math.round(monthlyStorageMicroCredits(data.storage.minutesByTier)), + ); expect(data.storage.dailyMicroCredits).toBe( - dailyStorageMicroCredits(data.storage.minutesByTier), + dailyStorageMicroCredits(daily), ); + for (const tier of INFRA_RESOLUTION_TIERS) { + expect(daily[tier]).toBeLessThanOrEqual(data.storage.minutesByTier[tier]); + } expect(data.upload.requiredMicroCredits).toBe( uploadAdmissionMicroCredits(monthly), ); @@ -379,8 +582,29 @@ describe("examples in the REST API reference", () => { expect(data.rates.storageDollarsPer1000MinutesPerMonth).toEqual( INFRA_STORAGE_DOLLARS_PER_1000_MINUTES_PER_MONTH, ); + expect(data.rates.deliveryDollarsPer1000Minutes).toBe( + INFRA_DELIVERY_DOLLARS_PER_1000_MINUTES, + ); expect(data.rates.speechDollarsPerHour).toBe(SPEECH_DOLLARS_PER_HOUR); expect(data.rates.uploadAdmissionDays).toBe(UPLOAD_ADMISSION_DAYS); + expect(Object.keys(data.delivery)).toEqual([ + "minutes", + "minutesByTier", + "finalThrough", + ]); + expect(data.delivery.finalThrough).toMatch(/^\d{4}-\d{2}-\d{2}$/); + expect( + Object.values( + data.delivery.minutesByTier as Record, + ).reduce((sum, minutes) => sum + minutes, 0), + ).toBe(data.delivery.minutes); + expect(Object.keys(data.delivery.minutesByTier)).toEqual([ + "360p", + "720p", + "1080p", + "1440p", + "2160p", + ]); expect(rest).toContain( `2,999,901 micro-credits is \`"${creditDollars(2_999_901)}"\``, ); diff --git a/apps/web/__tests__/unit/developer-credit-idempotency.test.ts b/apps/web/__tests__/unit/developer-credit-idempotency.test.ts index 8f8fad99f0f..4d5d07381c4 100644 --- a/apps/web/__tests__/unit/developer-credit-idempotency.test.ts +++ b/apps/web/__tests__/unit/developer-credit-idempotency.test.ts @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const state = vi.hoisted(() => ({ balance: 0, + heldMicroCredits: 0, transactions: [] as Record[], lock: Promise.resolve(), })); @@ -65,6 +66,10 @@ vi.mock("@cap/database", () => ({ })); vi.mock("@cap/database/helpers", () => ({ nanoId: () => "txn" })); +vi.mock("@/lib/infra-balance", () => ({ + AVAILABLE_AFTER_FIELD: "availableAfterMicroCredits", + heldTranscriptMicroCredits: async () => state.heldMicroCredits, +})); vi.mock("@cap/database/schema", () => ({ developerCreditAccounts: { id: "id", balanceMicroCredits: "balance" }, developerCreditTransactions: { @@ -93,10 +98,24 @@ const purchase = { describe("developer credit purchase idempotency", () => { beforeEach(() => { state.balance = 0; + state.heldMicroCredits = 0; state.transactions = []; state.lock = Promise.resolve(); }); + it("records the credit available after the purchase, net of held transcript credit", async () => { + state.balance = 400_000; + state.heldMicroCredits = 150_000; + + await addCreditsToAccount(purchase); + + expect(state.transactions).toHaveLength(1); + expect(state.transactions[0]?.metadata).toEqual({ + amountCents: 2500, + availableAfterMicroCredits: 400_000 + 2_500_000 - 150_000, + }); + }); + it("grants a concurrently delivered purchase only once", async () => { await Promise.all([ addCreditsToAccount(purchase), diff --git a/apps/web/__tests__/unit/developer-credits-checkout.test.ts b/apps/web/__tests__/unit/developer-credits-checkout.test.ts index 7ce5c072376..95a58b2b1e4 100644 --- a/apps/web/__tests__/unit/developer-credits-checkout.test.ts +++ b/apps/web/__tests__/unit/developer-credits-checkout.test.ts @@ -84,6 +84,18 @@ vi.mock("@cap/utils", () => ({ stripe: () => mockStripe, })); +const mockCheckoutStripe = { + customers: { + listTaxIds: vi.fn(), + createTaxId: vi.fn(), + deleteTaxId: vi.fn(), + }, +}; + +vi.mock("@/lib/checkout/stripe", () => ({ + checkoutStripe: () => mockCheckoutStripe, +})); + vi.mock("drizzle-orm", () => ({ and: vi.fn((...args: unknown[]) => args), eq: vi.fn((a: unknown, b: unknown) => ({ eq: [a, b] })), @@ -152,6 +164,48 @@ describe("POST /api/developer/credits/checkout", () => { expect(res.status).toBe(400); }); + it("refuses an amount outside $5 to $10,000 with the console's sentence", async () => { + mockGetCurrentUser.mockResolvedValue(mockUser); + for (const amountCents of [499, 1_000_001]) { + const res = await POST(makeRequest({ appId: "app-456", amountCents })); + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: "Enter an amount from $5 to $10,000.", + }); + } + expect(mockStripe.checkout.sessions.create).not.toHaveBeenCalled(); + }); + + it("opens a $10,000 purchase", async () => { + mockGetCurrentUser.mockResolvedValue(mockUser); + mockDb.limit + .mockResolvedValueOnce([mockApp]) + .mockResolvedValueOnce([ + { ...mockAccount, stripeCustomerId: "cus_acct" }, + ]); + mockStripe.checkout.sessions.create.mockResolvedValue({ + url: "https://checkout.stripe.com/test", + }); + + const res = await POST( + makeRequest({ appId: "app-456", amountCents: 1_000_000 }), + ); + + expect(res.status).toBe(200); + expect(mockStripe.checkout.sessions.create).toHaveBeenCalledWith( + expect.objectContaining({ + line_items: [ + expect.objectContaining({ + price_data: expect.objectContaining({ unit_amount: 1_000_000 }), + }), + ], + metadata: expect.objectContaining({ amountCents: "1000000" }), + payment_method_types: ["card"], + payment_method_options: { card: { request_three_d_secure: "any" } }, + }), + ); + }); + it("returns 400 when amountCents is not an integer", async () => { mockGetCurrentUser.mockResolvedValue(mockUser); const res = await POST( @@ -168,27 +222,17 @@ describe("POST /api/developer/credits/checkout", () => { expect(res.status).toBe(400); }); - it("refuses a purchase from an account outside the preview list", async () => { + it("lets any signed-in account reach the purchase on Cap's own deployment", async () => { const { buildEnv } = await import("@cap/env"); const env = buildEnv as { NEXT_PUBLIC_IS_CAP?: string }; env.NEXT_PUBLIC_IS_CAP = "true"; try { mockGetCurrentUser.mockResolvedValue(mockUser); - const res = await POST( - makeRequest({ appId: "app-456", amountCents: 1000 }), - ); - expect(res.status).toBe(403); - expect(mockStripe.checkout.sessions.create).not.toHaveBeenCalled(); - - mockGetCurrentUser.mockResolvedValue({ - ...mockUser, - email: "richie@cap.so", - }); mockDb.limit.mockResolvedValueOnce([]); - const allowed = await POST( + const res = await POST( makeRequest({ appId: "app-456", amountCents: 1000 }), ); - expect(allowed.status).toBe(404); + expect(res.status).toBe(404); } finally { env.NEXT_PUBLIC_IS_CAP = ""; } @@ -344,6 +388,42 @@ describe("POST /api/developer/credits/checkout", () => { expect(cancelUrl.search).toBe(""); }); + it("emails a paid invoice to the account email and collects tax details like Cap Pro", async () => { + mockGetCurrentUser.mockResolvedValue({ + ...mockUser, + stripeCustomerId: "cus_exist", + }); + mockDb.limit + .mockResolvedValueOnce([mockApp]) + .mockResolvedValueOnce([ + { ...mockAccount, stripeCustomerId: "cus_exist" }, + ]); + mockStripe.checkout.sessions.create.mockResolvedValue({ + url: "https://checkout.stripe.com/test", + }); + + await POST(makeRequest({ appId: "app-456", amountCents: 2500 })); + + const [params] = mockStripe.checkout.sessions.create.mock.calls[0] ?? []; + expect(params).toMatchObject({ + customer: "cus_exist", + mode: "payment", + invoice_creation: { + enabled: true, + invoice_data: { description: "Cap Infra credits for Test App" }, + }, + payment_intent_data: { + description: "Cap Infra credits for Test App", + receipt_email: "test@example.com", + metadata: { type: "developer_credits", accountId: "account-001" }, + }, + tax_id_collection: { enabled: true }, + billing_address_collection: "auto", + customer_update: { name: "auto", address: "auto" }, + }); + expect(params).not.toHaveProperty("automatic_tax"); + }); + it("returns checkout URL on success", async () => { mockGetCurrentUser.mockResolvedValue({ ...mockUser, @@ -680,3 +760,226 @@ describe("POST /api/developer/credits/checkout", () => { expect(call.line_items[0].quantity).toBe(1); }); }); + +describe("credit checkout for a named app and a US business", () => { + const fundedAccount = { ...mockAccount, stripeCustomerId: "cus_exist" }; + let POST: typeof import("@/app/api/developer/credits/checkout/route").POST; + let buyer = 0; + + beforeEach(async () => { + vi.clearAllMocks(); + resetMockDb(); + POST = (await import("@/app/api/developer/credits/checkout/route")).POST; + buyer += 1; + mockGetCurrentUser.mockResolvedValue({ + ...mockUser, + id: `user-business-${buyer}`, + stripeCustomerId: "cus_exist", + }); + mockStripe.checkout.sessions.create.mockResolvedValue({ + url: "https://checkout.stripe.com/test", + }); + mockCheckoutStripe.customers.listTaxIds.mockResolvedValue({ data: [] }); + mockCheckoutStripe.customers.createTaxId.mockResolvedValue({}); + mockCheckoutStripe.customers.deleteTaxId.mockResolvedValue({}); + }); + + const lastParams = () => + mockStripe.checkout.sessions.create.mock.calls[0]?.[0] as Record< + string, + unknown + >; + + it("names the app on the Stripe page, the invoice and the payment, and offers card only", async () => { + mockDb.limit + .mockResolvedValueOnce([{ ...mockApp, name: " Course site " }]) + .mockResolvedValueOnce([fundedAccount]); + + const res = await POST( + makeRequest({ appId: "app-456", amountCents: 1000 }), + ); + + expect(res.status).toBe(200); + expect(lastParams()).toMatchObject({ + payment_method_types: ["card"], + custom_text: { + submit: { + message: + "Cap Infra credits for Course site. Credit stays with this app and cannot be moved to another app.", + }, + }, + invoice_creation: { + invoice_data: { description: "Cap Infra credits for Course site" }, + }, + payment_intent_data: { + description: "Cap Infra credits for Course site", + }, + }); + expect(lastParams()).not.toHaveProperty("payment_method_configuration"); + }); + + it("saves a US EIN on the customer before the session is created", async () => { + mockDb.limit + .mockResolvedValueOnce([mockApp]) + .mockResolvedValueOnce([fundedAccount]); + + const res = await POST( + makeRequest({ appId: "app-456", amountCents: 1000, usEin: "123456789" }), + ); + + expect(res.status).toBe(200); + expect(mockCheckoutStripe.customers.createTaxId).toHaveBeenCalledWith( + "cus_exist", + { type: "us_ein", value: "12-3456789" }, + ); + expect( + mockCheckoutStripe.customers.createTaxId.mock.invocationCallOrder[0], + ).toBeLessThan( + mockStripe.checkout.sessions.create.mock.invocationCallOrder[0] ?? 0, + ); + expect(mockCheckoutStripe.customers.deleteTaxId).not.toHaveBeenCalled(); + }); + + it("keeps an EIN the customer already has and adds nothing twice", async () => { + mockDb.limit + .mockResolvedValueOnce([mockApp]) + .mockResolvedValueOnce([fundedAccount]); + mockCheckoutStripe.customers.listTaxIds.mockResolvedValue({ + data: [{ id: "txi_1", type: "us_ein", value: "12-3456789" }], + }); + + await POST( + makeRequest({ appId: "app-456", amountCents: 1000, usEin: "12-3456789" }), + ); + + expect(mockCheckoutStripe.customers.createTaxId).not.toHaveBeenCalled(); + expect(mockCheckoutStripe.customers.deleteTaxId).not.toHaveBeenCalled(); + expect(mockStripe.checkout.sessions.create).toHaveBeenCalledTimes(1); + }); + + it("replaces a different saved EIN and leaves other tax IDs alone", async () => { + mockDb.limit + .mockResolvedValueOnce([mockApp]) + .mockResolvedValueOnce([fundedAccount]); + mockCheckoutStripe.customers.listTaxIds.mockResolvedValue({ + data: [ + { id: "txi_vat", type: "eu_vat", value: "DE123456789" }, + { id: "txi_old", type: "us_ein", value: "98-7654321" }, + ], + }); + + await POST( + makeRequest({ appId: "app-456", amountCents: 1000, usEin: "12-3456789" }), + ); + + expect(mockCheckoutStripe.customers.deleteTaxId).toHaveBeenCalledTimes(1); + expect(mockCheckoutStripe.customers.deleteTaxId).toHaveBeenCalledWith( + "cus_exist", + "txi_old", + ); + expect(mockCheckoutStripe.customers.createTaxId).toHaveBeenCalledWith( + "cus_exist", + { type: "us_ein", value: "12-3456789" }, + ); + }); + + it("keeps the saved EIN when Stripe will not take its replacement", async () => { + mockDb.limit + .mockResolvedValueOnce([mockApp]) + .mockResolvedValueOnce([fundedAccount]); + mockCheckoutStripe.customers.listTaxIds.mockResolvedValue({ + data: [{ id: "txi_old", type: "us_ein", value: "98-7654321" }], + }); + mockCheckoutStripe.customers.createTaxId.mockRejectedValue( + new Error("tax id rejected"), + ); + + const res = await POST( + makeRequest({ appId: "app-456", amountCents: 1000, usEin: "12-3456789" }), + ); + + expect(res.status).toBe(500); + expect(mockCheckoutStripe.customers.deleteTaxId).not.toHaveBeenCalled(); + expect(mockStripe.checkout.sessions.create).not.toHaveBeenCalled(); + }); + + it("adds the new EIN before removing the old one", async () => { + mockDb.limit + .mockResolvedValueOnce([mockApp]) + .mockResolvedValueOnce([fundedAccount]); + mockCheckoutStripe.customers.listTaxIds + .mockResolvedValueOnce({ + data: [{ id: "txi_old", type: "us_ein", value: "98-7654321" }], + }) + .mockResolvedValueOnce({ + data: [{ id: "txi_old", type: "us_ein", value: "98-7654321" }], + }) + .mockResolvedValue({ + data: [ + { id: "txi_old", type: "us_ein", value: "98-7654321" }, + { id: "txi_new", type: "us_ein", value: "12-3456789" }, + ], + }); + + await POST( + makeRequest({ appId: "app-456", amountCents: 1000, usEin: "12-3456789" }), + ); + + expect( + mockCheckoutStripe.customers.createTaxId.mock.invocationCallOrder[0], + ).toBeLessThan( + mockCheckoutStripe.customers.deleteTaxId.mock.invocationCallOrder[0] ?? 0, + ); + expect(mockCheckoutStripe.customers.deleteTaxId).toHaveBeenCalledWith( + "cus_exist", + "txi_old", + ); + expect(mockCheckoutStripe.customers.createTaxId).toHaveBeenCalledTimes(1); + }); + + it("touches no tax IDs when the EIN is left blank", async () => { + mockDb.limit + .mockResolvedValueOnce([mockApp]) + .mockResolvedValueOnce([fundedAccount]); + + const res = await POST( + makeRequest({ appId: "app-456", amountCents: 1000, usEin: " " }), + ); + + expect(res.status).toBe(200); + expect(mockCheckoutStripe.customers.listTaxIds).not.toHaveBeenCalled(); + expect(mockCheckoutStripe.customers.createTaxId).not.toHaveBeenCalled(); + }); + + it("refuses an EIN that is not 9 digits before opening a checkout", async () => { + const res = await POST( + makeRequest({ appId: "app-456", amountCents: 1000, usEin: "12-345" }), + ); + + expect(res.status).toBe(400); + expect(await res.json()).toEqual({ + error: "Enter the EIN as 9 digits, for example 12-3456789", + }); + expect(mockStripe.checkout.sessions.create).not.toHaveBeenCalled(); + expect(mockCheckoutStripe.customers.listTaxIds).not.toHaveBeenCalled(); + }); + + it("opens no checkout when Stripe will not save the EIN", async () => { + mockDb.limit + .mockResolvedValueOnce([mockApp]) + .mockResolvedValueOnce([fundedAccount]); + mockCheckoutStripe.customers.createTaxId.mockRejectedValue( + new Error("tax id rejected"), + ); + + const res = await POST( + makeRequest({ appId: "app-456", amountCents: 1000, usEin: "123456789" }), + ); + + expect(res.status).toBe(500); + expect(await res.json()).toEqual({ + error: "Could not save the EIN. Try again or leave it blank", + }); + expect(mockStripe.checkout.sessions.create).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/__tests__/unit/developer-credits-webhook.test.ts b/apps/web/__tests__/unit/developer-credits-webhook.test.ts index c10af8f9a1c..5a87b6ee3e8 100644 --- a/apps/web/__tests__/unit/developer-credits-webhook.test.ts +++ b/apps/web/__tests__/unit/developer-credits-webhook.test.ts @@ -64,6 +64,19 @@ vi.mock("@/lib/infra-auto-top-up", () => ({ mockSettleAutoTopUp(...args), })); +const mockReverseRefund = vi.fn(); +const mockReverseDispute = vi.fn(); +vi.mock("@/lib/infra-credit-reversal", () => ({ + reverseRefundedCharge: (...args: unknown[]) => mockReverseRefund(...args), + reverseDisputedPayment: (...args: unknown[]) => mockReverseDispute(...args), + awaitsCreditGrant: (results: unknown) => + [results] + .flat() + .some( + (result) => (result as { state?: string })?.state === "grant_pending", + ), +})); + vi.mock("@cap/web-domain", () => ({ Organisation: { OrganisationId: { make: (v: string) => v } }, User: { UserId: { make: (v: string) => v } }, @@ -491,6 +504,115 @@ describe("Stripe webhook — developer credits", () => { expect(mockAddCredits).not.toHaveBeenCalled(); }); + it("takes credit back when a charge is refunded", async () => { + const charge = { id: "ch_1", payment_intent: "pi_test_abc", amount: 2500 }; + mockStripe.webhooks.constructEvent.mockReturnValue({ + type: "charge.refunded", + data: { object: charge }, + }); + + const res = await POST(makeWebhookRequest()); + + expect(res.status).toBe(200); + expect(mockReverseRefund).toHaveBeenCalledTimes(1); + expect(mockReverseRefund).toHaveBeenCalledWith(charge); + expect(mockReverseDispute).not.toHaveBeenCalled(); + expect(mockAddCredits).not.toHaveBeenCalled(); + expect(mockStripe.customers.retrieve).not.toHaveBeenCalled(); + }); + + it("takes credit back when a payment is disputed", async () => { + const dispute = { id: "dp_1", payment_intent: "pi_test_abc", amount: 2500 }; + mockStripe.webhooks.constructEvent.mockReturnValue({ + type: "charge.dispute.created", + data: { object: dispute }, + }); + + const res = await POST(makeWebhookRequest()); + + expect(res.status).toBe(200); + expect(mockReverseDispute).toHaveBeenCalledTimes(1); + expect(mockReverseDispute).toHaveBeenCalledWith(dispute); + expect(mockReverseRefund).not.toHaveBeenCalled(); + expect(mockStripe.customers.retrieve).not.toHaveBeenCalled(); + }); + + it.each([ + ["charge.refunded", mockReverseRefund], + ["charge.dispute.created", mockReverseDispute], + ])( + "asks Stripe to retry %s when the ledger write fails", + async (type, reverse) => { + reverse.mockRejectedValueOnce(new Error("database unavailable")); + mockStripe.webhooks.constructEvent.mockReturnValue({ + type, + data: { object: { id: "obj_1", payment_intent: "pi_test_abc" } }, + }); + + const res = await POST(makeWebhookRequest()); + + expect(res.status).toBe(400); + }, + ); + + it.each([ + ["charge.refunded", mockReverseRefund, [{ state: "grant_pending" }]], + ["charge.dispute.created", mockReverseDispute, { state: "grant_pending" }], + ])( + "does not acknowledge %s that arrives before its grant", + async (type, reverse, pending) => { + reverse.mockResolvedValueOnce(pending); + mockStripe.webhooks.constructEvent.mockReturnValue({ + type, + data: { object: { id: "obj_1", payment_intent: "pi_test_abc" } }, + }); + + const res = await POST(makeWebhookRequest()); + + expect(res.status).toBe(503); + expect(mockAddCredits).not.toHaveBeenCalled(); + }, + ); + + it.each([ + [ + "charge.refunded", + mockReverseRefund, + [{ state: "ignored", reason: "not_a_credit_payment" }], + ], + [ + "charge.dispute.created", + mockReverseDispute, + { state: "ignored", reason: "not_a_credit_payment" }, + ], + ["charge.refunded", mockReverseRefund, [{ state: "duplicate" }]], + ])( + "acknowledges %s for an unrelated or already handled charge", + async (type, reverse, outcome) => { + reverse.mockResolvedValueOnce(outcome); + mockStripe.webhooks.constructEvent.mockReturnValue({ + type, + data: { object: { id: "obj_1", payment_intent: "pi_subscription" } }, + }); + + const res = await POST(makeWebhookRequest()); + + expect(res.status).toBe(200); + }, + ); + + it("leaves other dispute and refund events to no handler", async () => { + mockStripe.webhooks.constructEvent.mockReturnValue({ + type: "charge.dispute.closed", + data: { object: { id: "dp_1" } }, + }); + + const res = await POST(makeWebhookRequest()); + + expect(res.status).toBe(400); + expect(mockReverseDispute).not.toHaveBeenCalled(); + }); + it("settles an auto top-up payment that failed", async () => { const paymentIntent = { id: "pi_auto", diff --git a/apps/web/__tests__/unit/developer-cron-storage.test.ts b/apps/web/__tests__/unit/developer-cron-storage.test.ts index ad67787a9fb..a9d73b5c1f7 100644 --- a/apps/web/__tests__/unit/developer-cron-storage.test.ts +++ b/apps/web/__tests__/unit/developer-cron-storage.test.ts @@ -2,49 +2,115 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ selects: [] as Record[][], + page: [] as Record[], + accountsTable: { id: "id", appId: "appId" }, + appsTable: { id: "id", deletedAt: "deletedAt" }, + ledgerTable: { + accountId: "accountId", + referenceId: "referenceId", + referenceType: "referenceType", + }, + snapshotsTable: { appId: "appId", snapshotDate: "snapshotDate" }, + yesterdayBilled: null as Record[] | null, + markers: [] as unknown[], getStorageUsageByApp: vi.fn(), + chargeMissedFirstMonths: vi.fn(), + shortfallsAwaitingNotice: vi.fn(), + openFirstMonthShortfalls: vi.fn(), + markShortfallsNotified: vi.fn(), + markStorageDayNotified: vi.fn(), chargeStorageDay: vi.fn(), latestStorageCharge: vi.fn(), + latestStorageCharges: vi.fn(), + restoredSinceCharge: vi.fn(), runDueAutoTopUps: vi.fn(), sendInfraOutOfCreditEmail: vi.fn(), + deliveryBillingStart: vi.fn(), + settleDeliveryDays: vi.fn(), + markDeliveryDayNotified: vi.fn(), })); vi.mock("@cap/database", () => ({ db: () => { + let table: unknown; + const dateOf = (condition: unknown) => { + const match = JSON.stringify(condition).match( + /\{"eq":\["referenceId","(\d{4}-\d{2}-\d{2})"\]\}/, + ); + return match?.[1] ?? null; + }; + const read = (condition: unknown) => { + if (table === mocks.appsTable) { + return mocks.page.map((account) => ({ id: account.appId })); + } + if (table === mocks.accountsTable) { + mocks.page = mocks.selects[0] ?? []; + return mocks.page; + } + if (table === mocks.ledgerTable) { + if (dateOf(condition) === new Date().toISOString().slice(0, 10)) { + return mocks.selects[1] ?? []; + } + return ( + mocks.yesterdayBilled ?? + mocks.page.map((account) => ({ accountId: account.id })) + ); + } + return []; + }; const chain = { select: () => chain, - from: () => chain, - innerJoin: () => chain, - where: async () => mocks.selects.shift() ?? [], + from: (next: unknown) => { + table = next; + return chain; + }, + where: (condition: unknown) => { + const rows = Promise.resolve(read(condition)); + return Object.assign(rows, { + orderBy: () => ({ limit: () => rows }), + }); + }, }; return chain; }, })); vi.mock("@cap/database/schema", () => ({ - developerApps: { id: "id", deletedAt: "deletedAt" }, - developerCreditAccounts: { id: "id", appId: "appId" }, - developerCreditTransactions: { - accountId: "accountId", - referenceId: "referenceId", - referenceType: "referenceType", - }, + developerApps: mocks.appsTable, + developerCreditAccounts: mocks.accountsTable, + developerCreditTransactions: mocks.ledgerTable, + developerDailyStorageSnapshots: mocks.snapshotsTable, })); vi.mock("drizzle-orm", () => ({ and: (...args: unknown[]) => args, + asc: (a: unknown) => ({ asc: a }), + gt: (a: unknown, b: unknown) => ({ gt: [a, b] }), eq: (a: unknown, b: unknown) => ({ eq: [a, b] }), inArray: (a: unknown, b: unknown) => ({ inArray: [a, b] }), isNull: (a: unknown) => ({ isNull: a }), + isNotNull: (a: unknown) => ({ isNotNull: a }), })); vi.mock("@/lib/infra-storage", () => ({ getStorageUsageByApp: mocks.getStorageUsageByApp, })); +vi.mock("@/lib/infra-storage-minimum", () => ({ + chargeMissedFirstMonths: mocks.chargeMissedFirstMonths, + shortfallsAwaitingNotice: mocks.shortfallsAwaitingNotice, + openFirstMonthShortfalls: mocks.openFirstMonthShortfalls, + markShortfallsNotified: mocks.markShortfallsNotified, + markStorageDayNotified: mocks.markStorageDayNotified, +})); vi.mock("@/lib/infra-storage-billing", () => ({ chargeStorageDay: mocks.chargeStorageDay, + recordStorageDayVisits: async ({ appIds }: { appIds: string[] }) => { + mocks.markers.push(...appIds); + }, })); vi.mock("@/lib/infra-standing", () => ({ INFRA_GRACE_DAYS: 7, STORAGE_DAY_REFERENCE_TYPE: "storage_day", latestStorageCharge: mocks.latestStorageCharge, + latestStorageCharges: mocks.latestStorageCharges, + restoredSinceCharge: mocks.restoredSinceCharge, })); vi.mock("@/lib/infra-auto-top-up", () => ({ runDueAutoTopUps: mocks.runDueAutoTopUps, @@ -52,11 +118,17 @@ vi.mock("@/lib/infra-auto-top-up", () => ({ vi.mock("@/lib/infra-billing-emails", () => ({ sendInfraOutOfCreditEmail: mocks.sendInfraOutOfCreditEmail, })); +vi.mock("@/lib/infra-delivery-billing", () => ({ + deliveryBillingStart: mocks.deliveryBillingStart, + settleDeliveryDays: mocks.settleDeliveryDays, + markDeliveryDayNotified: mocks.markDeliveryDayNotified, +})); import { GET } from "@/app/api/cron/developer-storage/route"; const CRON_SECRET = "test-cron-secret"; const NO_TOP_UPS = { attempted: 0, succeeded: 0, failed: 0 }; +const NO_FIRST_MONTHS = { charged: 0, short: 0, failed: 0 }; const usage = (dailyMicroCredits: number) => ({ videoCount: dailyMicroCredits > 0 ? 1 : 0, @@ -90,18 +162,41 @@ const authorized = () => run(`Bearer ${CRON_SECRET}`); beforeEach(() => { process.env.CRON_SECRET = CRON_SECRET; mocks.selects = []; + mocks.page = []; + mocks.yesterdayBilled = null; + mocks.markers = []; for (const mock of [ mocks.getStorageUsageByApp, mocks.chargeStorageDay, - mocks.latestStorageCharge, + mocks.latestStorageCharges, mocks.runDueAutoTopUps, mocks.sendInfraOutOfCreditEmail, ]) { mock.mockReset(); } + mocks.chargeMissedFirstMonths.mockReset(); + mocks.chargeMissedFirstMonths.mockResolvedValue(NO_FIRST_MONTHS); + mocks.shortfallsAwaitingNotice.mockReset(); + mocks.shortfallsAwaitingNotice.mockResolvedValue(new Map()); + mocks.markShortfallsNotified.mockReset(); + mocks.markStorageDayNotified.mockReset(); + mocks.openFirstMonthShortfalls.mockReset(); + mocks.openFirstMonthShortfalls.mockResolvedValue(new Set()); + mocks.restoredSinceCharge.mockReset(); + mocks.restoredSinceCharge.mockResolvedValue(false); mocks.runDueAutoTopUps.mockResolvedValue(NO_TOP_UPS); - mocks.latestStorageCharge.mockResolvedValue(null); + mocks.latestStorageCharges.mockResolvedValue(new Map()); mocks.chargeStorageDay.mockResolvedValue(billed()); + mocks.deliveryBillingStart.mockReset(); + mocks.deliveryBillingStart.mockReturnValue(null); + mocks.settleDeliveryDays.mockReset(); + mocks.settleDeliveryDays.mockImplementation( + async ({ startDay }: { startDay: string | null }) => ({ + enabled: !!startDay, + billed: 0, + }), + ); + mocks.markDeliveryDayNotified.mockReset(); }); describe("developer-storage cron job", () => { @@ -147,7 +242,10 @@ describe("developer-storage cron job", () => { autoTopUps: NO_TOP_UPS, }); expect(body.date).toMatch(/^\d{4}-\d{2}-\d{2}$/); - expect(mocks.getStorageUsageByApp).toHaveBeenCalledWith(["app-1", "app-2"]); + expect(mocks.getStorageUsageByApp.mock.calls[0]?.[0]).toEqual([ + "app-1", + "app-2", + ]); expect(mocks.chargeStorageDay).toHaveBeenCalledTimes(2); expect(mocks.chargeStorageDay).toHaveBeenCalledWith( expect.objectContaining({ @@ -157,6 +255,17 @@ describe("developer-storage cron job", () => { dateKey: body.date, }), ); + const measured = new Map([["app-2", usage(56)]]); + mocks.getStorageUsageByApp.mockResolvedValueOnce(measured); + const tx = { lock: "held" }; + const [{ measure }] = mocks.chargeStorageDay.mock.calls[1] ?? [{}]; + expect(await measure(tx)).toEqual(usage(56)); + expect(mocks.getStorageUsageByApp).toHaveBeenLastCalledWith( + ["app-2"], + undefined, + tx, + { dailyChargeAt: expect.any(Date) }, + ); }); it("skips accounts already billed today", async () => { @@ -174,7 +283,7 @@ describe("developer-storage cron job", () => { const { body } = await authorized(); expect(body.appsProcessed).toBe(1); - expect(mocks.getStorageUsageByApp).toHaveBeenCalledWith(["app-2"]); + expect(mocks.getStorageUsageByApp.mock.calls[0]?.[0]).toEqual(["app-2"]); expect(mocks.chargeStorageDay).toHaveBeenCalledTimes(1); expect(mocks.chargeStorageDay.mock.calls[0]?.[0]).toMatchObject({ accountId: "account-2", @@ -191,6 +300,34 @@ describe("developer-storage cron job", () => { expect(body.appsProcessed).toBe(0); expect(mocks.chargeStorageDay).not.toHaveBeenCalled(); + expect(mocks.markers).toEqual(["app-1"]); + }); + + it("looks up the last charge of every idle account in one read", async () => { + mocks.selects = [ + [ + { id: "account-1", appId: "app-1" }, + { id: "account-2", appId: "app-2" }, + { id: "account-3", appId: "app-3" }, + ], + [], + ]; + mocks.getStorageUsageByApp.mockResolvedValue( + new Map([ + ["app-1", usage(0)], + ["app-2", usage(28)], + ["app-3", usage(0)], + ]), + ); + + const { body } = await authorized(); + + expect(body).toMatchObject({ complete: true, appsProcessed: 1 }); + expect(mocks.latestStorageCharges).toHaveBeenCalledTimes(1); + expect(mocks.latestStorageCharges).toHaveBeenCalledWith(expect.anything(), [ + "account-1", + "account-3", + ]); }); it("still settles an app that stores nothing now but is short from before", async () => { @@ -198,14 +335,24 @@ describe("developer-storage cron job", () => { mocks.getStorageUsageByApp.mockResolvedValue( new Map([["app-1", usage(0)]]), ); - mocks.latestStorageCharge.mockResolvedValue({ - createdAt: new Date(), - owedMicroCredits: 28, - shortSince: new Date(), - }); + mocks.latestStorageCharges.mockResolvedValue( + new Map([ + [ + "account-1", + { + createdAt: new Date(), + owedMicroCredits: 28, + shortSince: new Date(), + }, + ], + ]), + ); await authorized(); + expect(mocks.latestStorageCharges).toHaveBeenCalledWith(expect.anything(), [ + "account-1", + ]); expect(mocks.chargeStorageDay).toHaveBeenCalledTimes(1); }); @@ -309,6 +456,245 @@ describe("developer-storage cron job", () => { expect(body.autoTopUps).toEqual({ attempted: 3, succeeded: 2, failed: 1 }); }); + it("charges missed first months after the first top-ups and before the day is measured", async () => { + const order: string[] = []; + mocks.selects = [[{ id: "account-1", appId: "app-1" }], []]; + mocks.runDueAutoTopUps.mockImplementation(async () => { + order.push("top-up"); + return NO_TOP_UPS; + }); + mocks.chargeMissedFirstMonths.mockImplementation(async () => { + order.push("first months"); + return { charged: 2, short: 1, failed: 0 }; + }); + mocks.getStorageUsageByApp.mockImplementation(async () => { + order.push("measure"); + return new Map([["app-1", usage(28)]]); + }); + + const { body } = await authorized(); + + expect(order).toEqual(["top-up", "first months", "measure", "top-up"]); + expect(body.firstMonths).toEqual({ charged: 2, short: 1, failed: 0 }); + }); + + it("opens the day's charge for a short first month even when nothing is charged daily, so a recovery can be recorded", async () => { + mocks.selects = [[{ id: "account-1", appId: "app-1" }], []]; + mocks.getStorageUsageByApp.mockResolvedValue( + new Map([["app-1", usage(0)]]), + ); + mocks.openFirstMonthShortfalls.mockResolvedValue(new Set(["account-1"])); + + await authorized(); + + expect(mocks.openFirstMonthShortfalls).toHaveBeenCalledWith(["account-1"]); + expect(mocks.chargeStorageDay).toHaveBeenCalledTimes(1); + expect(mocks.chargeStorageDay.mock.calls[0]?.[0]).toMatchObject({ + accountId: "account-1", + measure: expect.any(Function), + }); + }); + + it("emails a new first month shortfall once, like a daily one, with the pause date seven days after it began", async () => { + const shortSince = new Date(Date.now() - 2 * 24 * 60 * 60 * 1000); + const accounts = () => [ + [ + { id: "account-1", appId: "app-1" }, + { id: "account-2", appId: "app-2" }, + { id: "account-3", appId: "app-3" }, + ], + [], + ]; + mocks.selects = accounts(); + mocks.getStorageUsageByApp.mockResolvedValue( + new Map([ + ["app-1", usage(0)], + ["app-2", usage(0)], + ["app-3", usage(0)], + ]), + ); + mocks.shortfallsAwaitingNotice.mockResolvedValueOnce( + new Map([ + ["account-2", { shortSince, ids: ["row-2"] }], + ["account-3", { shortSince, ids: ["row-3"] }], + ]), + ); + const short = { + createdAt: shortSince, + owedMicroCredits: 850, + shortSince, + firstMonth: true, + }; + mocks.latestStorageCharge.mockImplementation( + async (_db: unknown, accountId: string) => + accountId === "account-1" ? null : short, + ); + mocks.restoredSinceCharge.mockImplementation( + async (_db: unknown, accountId: string) => accountId === "account-3", + ); + mocks.sendInfraOutOfCreditEmail.mockResolvedValue(true); + + const before = Date.now(); + await authorized(); + + const [accountIds, since] = + mocks.shortfallsAwaitingNotice.mock.calls[0] ?? []; + expect(accountIds).toEqual(["account-1", "account-2", "account-3"]); + expect(before - (since as Date).getTime()).toBeGreaterThanOrEqual( + 7 * 24 * 60 * 60 * 1000 - 1000, + ); + expect(mocks.sendInfraOutOfCreditEmail).toHaveBeenCalledTimes(1); + expect(mocks.sendInfraOutOfCreditEmail).toHaveBeenCalledWith({ + accountId: "account-2", + shortSince, + pausesAt: new Date(shortSince.getTime() + 7 * 24 * 60 * 60 * 1000), + }); + expect(mocks.markShortfallsNotified.mock.calls.map(([ids]) => ids)).toEqual( + [["row-2"], ["row-3"]], + ); + + mocks.selects = accounts(); + await authorized(); + expect(mocks.sendInfraOutOfCreditEmail).toHaveBeenCalledTimes(1); + }); + + it("stamps a daily shortfall's row once its email was accepted, and leaves a rejected one for the next run", async () => { + const shortSince = new Date("2026-11-11T06:41:00Z"); + const run = async (sent: boolean) => { + mocks.selects = [[{ id: "account-1", appId: "app-1" }], []]; + mocks.getStorageUsageByApp.mockResolvedValue( + new Map([["app-1", usage(28)]]), + ); + mocks.chargeStorageDay.mockResolvedValueOnce( + billed({ chargedMicroCredits: 5, shortSince, fellShort: true }), + ); + mocks.sendInfraOutOfCreditEmail.mockResolvedValueOnce(sent); + return authorized(); + }; + + const { body } = await run(false); + expect(mocks.sendInfraOutOfCreditEmail).toHaveBeenCalledTimes(1); + expect(mocks.markStorageDayNotified).not.toHaveBeenCalled(); + + await run(true); + expect(mocks.markStorageDayNotified).toHaveBeenCalledWith( + "account-1", + body.date, + expect.any(Date), + ); + }); + + it("keeps a first month shortfall to email again when the email could not be sent", async () => { + const shortSince = new Date(); + mocks.selects = [[{ id: "account-1", appId: "app-1" }], []]; + mocks.getStorageUsageByApp.mockResolvedValue( + new Map([["app-1", usage(0)]]), + ); + mocks.shortfallsAwaitingNotice.mockResolvedValueOnce( + new Map([["account-1", { shortSince, ids: ["row-1"] }]]), + ); + mocks.latestStorageCharge.mockResolvedValue({ + createdAt: shortSince, + owedMicroCredits: 850, + shortSince, + firstMonth: true, + }); + mocks.restoredSinceCharge.mockResolvedValue(false); + mocks.sendInfraOutOfCreditEmail.mockResolvedValue(false); + + await authorized(); + + expect(mocks.sendInfraOutOfCreditEmail).toHaveBeenCalledTimes(1); + expect(mocks.markShortfallsNotified).not.toHaveBeenCalled(); + }); + + it("settles delivery days after the storage charges with the start day from the env, and none while it is unset", async () => { + mocks.selects = [[{ id: "account-1", appId: "app-1" }], []]; + mocks.getStorageUsageByApp.mockResolvedValue( + new Map([["app-1", usage(28)]]), + ); + const order: string[] = []; + mocks.chargeStorageDay.mockImplementation(async () => { + order.push("storage"); + return billed(); + }); + mocks.settleDeliveryDays.mockImplementation( + async ({ startDay }: { startDay: string | null }) => { + order.push(`delivery ${startDay}`); + return { enabled: !!startDay, billed: 0 }; + }, + ); + + const off = await authorized(); + expect(off.body.delivery).toMatchObject({ enabled: false }); + + mocks.deliveryBillingStart.mockReturnValue("2026-11-15"); + mocks.selects = [[{ id: "account-1", appId: "app-1" }], []]; + const on = await authorized(); + + expect(on.body.delivery).toMatchObject({ enabled: true }); + expect(order.filter((step) => step.startsWith("delivery"))).toEqual([ + "delivery null", + "delivery 2026-11-15", + ]); + expect(order.lastIndexOf("storage")).toBeLessThan( + order.indexOf("delivery 2026-11-15"), + ); + }); + + it("emails a delivery day that first falls short, and stamps its row only once the email is accepted", async () => { + mocks.deliveryBillingStart.mockReturnValue("2026-11-15"); + const shortSince = new Date("2026-11-17T06:41:00Z"); + mocks.settleDeliveryDays.mockImplementation( + async ({ + onFellShort, + }: { + onFellShort: (shortfall: { + accountId: string; + dateKey: string; + shortSince: Date; + }) => Promise; + }) => { + await onFellShort({ + accountId: "account-1", + dateKey: "2026-11-15", + shortSince, + }); + await onFellShort({ + accountId: "account-2", + dateKey: "2026-11-15", + shortSince, + }); + return { billed: 2, short: 2 }; + }, + ); + mocks.sendInfraOutOfCreditEmail + .mockResolvedValueOnce(true) + .mockResolvedValueOnce(false); + + await authorized(); + + expect(mocks.sendInfraOutOfCreditEmail.mock.calls).toEqual([ + [ + { + accountId: "account-1", + shortSince, + pausesAt: new Date(shortSince.getTime() + 7 * 24 * 60 * 60 * 1000), + }, + ], + [ + { + accountId: "account-2", + shortSince, + pausesAt: new Date(shortSince.getTime() + 7 * 24 * 60 * 60 * 1000), + }, + ], + ]); + expect(mocks.markDeliveryDayNotified.mock.calls).toEqual([ + ["account-1", "2026-11-15", expect.any(Date)], + ]); + }); + it("answers with zero apps when there are none", async () => { const { body } = await authorized(); diff --git a/apps/web/__tests__/unit/developer-get-started-docs.test.ts b/apps/web/__tests__/unit/developer-get-started-docs.test.ts index a06082e7119..07119def467 100644 --- a/apps/web/__tests__/unit/developer-get-started-docs.test.ts +++ b/apps/web/__tests__/unit/developer-get-started-docs.test.ts @@ -70,10 +70,12 @@ describe("the Get Started guide", () => { } }); - it("keeps real credentials, the CLI and the em dash out", () => { + it("keeps real credentials, the CLI and the em dash out, and points agents at the hosted MCP server", () => { expect(content).not.toContain("—"); expect(content).not.toMatch(/\bcap (guide|record|upload|caps|mcp)\b/); - expect(content).not.toMatch(/\bMCP\b/); + expect(content.match(/https:\/\/[^\s"`]*\/mcp\b/g) ?? []).toEqual( + Array(4).fill("https://cap.so/api/developer/mcp"), + ); const keys = content.match(/\bc(?:sk|pk|ut)_[A-Za-z0-9._-]+/g) ?? []; expect(keys.every((key) => key === "csk_your_secret_key")).toBe(true); }); diff --git a/apps/web/__tests__/unit/developer-ids.test.ts b/apps/web/__tests__/unit/developer-ids.test.ts index 956ee2ea720..86cd43a0697 100644 --- a/apps/web/__tests__/unit/developer-ids.test.ts +++ b/apps/web/__tests__/unit/developer-ids.test.ts @@ -8,7 +8,12 @@ vi.mock("@cap/env", () => ({ serverEnv: () => ({ WEB_URL: "https://cap.test" }), })); -import { developerIdFromKey, randomDeveloperId } from "@/lib/developer-ids"; +import { nanoId, nanoIdLength } from "@cap/database/helpers"; +import { + developerIdFromKey, + legacyDeveloperVideoIdForIdempotencyKey, + randomDeveloperId, +} from "@/lib/developer-ids"; import { transcriptIdForIdempotencyKey } from "@/lib/developer-transcript-store"; import { createDeveloperEncodeId, @@ -27,7 +32,7 @@ const keys = Array.from({ length: 4_000 }, (_, index) => `key-${index}`); describe("developer ids", () => { it.each([ ["developer-transcript", transcriptIdForIdempotencyKey], - ["developer-video", developerVideoIdForIdempotencyKey], + ["developer-video", legacyDeveloperVideoIdForIdempotencyKey], ] as const)("%s ids never begin with a dash", (kind, idFor) => { const dashed = keys.filter((key) => legacyId(kind, "app-1", key).startsWith("-"), @@ -43,6 +48,49 @@ describe("developer ids", () => { } }); + it("gives a keyed video an id of the same alphabet and length as any other video", () => { + const characters = (ids: string[]) => new Set(ids.join("")).size; + const random = Array.from({ length: keys.length }, () => nanoId()); + const keyed = keys.map((key) => + developerVideoIdForIdempotencyKey("app-1", key), + ); + const randomAlphabet = new Set(random.join("")); + + for (const id of keyed) { + expect(id).toHaveLength(nanoIdLength); + for (const character of id) { + expect(randomAlphabet.has(character)).toBe(true); + } + } + expect(characters(keyed)).toBe(32); + expect(characters(random)).toBe(32); + expect(new Set(keyed).size).toBe(keys.length); + }); + + it("gives a key the same video id every time, separately per app", () => { + for (const key of keys.slice(0, 200)) { + const id = developerVideoIdForIdempotencyKey("app-1", key); + expect(developerVideoIdForIdempotencyKey("app-1", key)).toBe(id); + expect(developerVideoIdForIdempotencyKey("app-2", key)).not.toBe(id); + expect(legacyDeveloperVideoIdForIdempotencyKey("app-1", key)).not.toBe( + id, + ); + } + expect( + developerVideoIdForIdempotencyKey("app-1", "retry-upload-0001"), + ).toBe("kktezv96c0155px"); + }); + + it("keeps the old id for a key whose video was created before", () => { + for (const key of keys) { + const legacy = legacyId("developer-video", "app-1", key); + if (legacy.startsWith("-")) continue; + expect(legacyDeveloperVideoIdForIdempotencyKey("app-1", key)).toBe( + legacy, + ); + } + }); + it("keeps the id of every key that never produced a leading dash", () => { for (const key of keys) { const legacy = legacyId("developer-transcript", "app-1", key); diff --git a/apps/web/__tests__/unit/developer-sdk-upload-billing.test.ts b/apps/web/__tests__/unit/developer-sdk-upload-billing.test.ts index 9243e844102..5621856ae7e 100644 --- a/apps/web/__tests__/unit/developer-sdk-upload-billing.test.ts +++ b/apps/web/__tests__/unit/developer-sdk-upload-billing.test.ts @@ -19,16 +19,28 @@ const mocks = vi.hoisted(() => ({ listParts: vi.fn(), headObject: vi.fn(), deleteObject: vi.fn(), + listUploads: vi.fn(), unrecorded: [] as { id: string; appId: string; s3Key: string | null }[], deletedAt: null as Date | null, maxTier: "1080p" as string, status: "uploading" as string, + claimedAt: new Date(0), creditTransactions: [] as Record[], videoUpdates: [] as Record[], admission: vi.fn(), claim: vi.fn(), release: vi.fn(), startProcessing: vi.fn(), + cancelEncodes: vi.fn(), + scheduleAutoTopUp: vi.fn(), +})); + +vi.mock("@/lib/developer-video-cancel", () => ({ + cancelDeveloperVideoEncodes: mocks.cancelEncodes, +})); + +vi.mock("@/lib/infra-auto-top-up", () => ({ + scheduleAutoTopUp: mocks.scheduleAutoTopUp, })); vi.mock("@/lib/developer-video-start", () => ({ @@ -89,6 +101,9 @@ vi.mock("@cap/database", async () => { }, ]; } + if (keys === "status,updatedAt") { + return [{ status: mocks.status, updatedAt: mocks.claimedAt }]; + } if (keys === "deletedAt,duration") { return [ { @@ -204,6 +219,7 @@ vi.mock("@cap/web-backend", async () => { multipart: { complete: mocks.complete, listParts: mocks.listParts, + listUploads: mocks.listUploads, }, }, ]), @@ -280,6 +296,7 @@ const refused = { describe("developer SDK multipart completion", () => { beforeEach(() => { + mocks.scheduleAutoTopUp.mockReset(); mocks.balance = 10_000_000; mocks.accountExists = true; mocks.storedParts = [{ PartNumber: 1, ETag: '"etag-1"', Size: 1_000 }]; @@ -333,6 +350,13 @@ describe("developer SDK multipart completion", () => { mocks.unrecorded = []; }); + it("asks for no top-up when an upload is admitted", async () => { + mocks.admission.mockResolvedValue(admitted); + + expect((await complete({ durationInSecs: 60 })).status).toBe(200); + expect(mocks.scheduleAutoTopUp).not.toHaveBeenCalled(); + }); + it("charges nothing when an upload completes", async () => { mocks.storedParts = [{ PartNumber: 1, ETag: '"etag-1"', Size: 10_000_000 }]; @@ -437,6 +461,7 @@ describe("developer SDK multipart completion", () => { expect(mocks.startProcessing).not.toHaveBeenCalled(); expect(mocks.videoUpdates).toEqual([]); expect(mocks.storedDuration).toBe(null); + expect(mocks.scheduleAutoTopUp).toHaveBeenCalledWith({ appId: "app-1" }); }); it("keeps the file when the upload is refused before storage assembles it", async () => { @@ -448,6 +473,7 @@ describe("developer SDK multipart completion", () => { await expect(response.json()).resolves.not.toHaveProperty("fileRemoved"); expect(mocks.complete).not.toHaveBeenCalled(); expect(mocks.deleteObject).not.toHaveBeenCalled(); + expect(mocks.scheduleAutoTopUp).toHaveBeenCalledWith({ appId: "app-1" }); }); it("still answers 402 when the refused file cannot be removed", async () => { @@ -612,6 +638,8 @@ describe("developer SDK multipart completion", () => { error: "Insufficient credits", code: "insufficient_credits", message: refused.message, + hint: "Add credits to this app at https://cap.so/dashboard/infra/app-1/billing, then retry.", + docs: "https://cap.so/docs/api/rest-api", standing: "active", pausesAt: null, balanceMicroCredits: 100, @@ -703,11 +731,14 @@ describe("developer SDK multipart completion", () => { it("answers success when another request recorded the video before storage completion", async () => { mocks.recordedReads = [60]; + mocks.admission.mockResolvedValue({ + allowed: false, + reason: "insufficient_balance", + }); const response = await complete({ durationInSecs: 60 }); expect(response.status).toBe(200); - expect(mocks.admission).not.toHaveBeenCalled(); expect(mocks.complete).not.toHaveBeenCalled(); expect(mocks.videoUpdates).toEqual([]); }); @@ -758,6 +789,8 @@ describe("removing files that were assembled but never recorded", () => { const old = () => Effect.succeed({ LastModified: hoursAgo(25) }); beforeEach(() => { + mocks.status = "uploading"; + mocks.claimedAt = hoursAgo(25); mocks.storedDuration = null; mocks.deletedAt = null; mocks.events = []; @@ -768,6 +801,8 @@ describe("removing files that were assembled but never recorded", () => { return Effect.succeed({}); }); mocks.headObject.mockReset(); + mocks.listUploads.mockReset(); + mocks.listUploads.mockImplementation(() => Effect.succeed({ Uploads: [] })); mocks.admission.mockReset(); mocks.admission.mockResolvedValue(admitted); mocks.claim.mockReset(); @@ -781,6 +816,10 @@ describe("removing files that were assembled but never recorded", () => { Effect.fail(new Error("no such upload")), ); mocks.complete.mockReset(); + mocks.cancelEncodes.mockReset(); + mocks.cancelEncodes.mockImplementation(async () => { + mocks.events.push("cancel"); + }); }); it("retires the video under the account lock, then deletes the file", async () => { @@ -788,7 +827,11 @@ describe("removing files that were assembled but never recorded", () => { mocks.headObject.mockImplementation(old); expect(await removeUnrecordedInfraFiles(now)).toBe(1); - expect(mocks.events).toEqual(["lock", "retire", "delete"]); + expect(mocks.events).toEqual(["lock", "retire", "cancel", "delete"]); + expect(mocks.cancelEncodes).toHaveBeenCalledExactlyOnceWith({ + appId: "app-1", + videoId: "video-1", + }); expect(mocks.videoUpdates).toEqual([{ deletedAt: now }]); expect(mocks.deleteObject).toHaveBeenCalledWith( "developer/app-1/old/video", @@ -806,6 +849,7 @@ describe("removing files that were assembled but never recorded", () => { expect(mocks.deleteObject).not.toHaveBeenCalled(); expect(mocks.deletedAt).toBe(null); expect(mocks.videoUpdates).toEqual([]); + expect(mocks.cancelEncodes).not.toHaveBeenCalled(); }); it("refuses a completion that reaches the lock after the sweep retired the video", async () => { @@ -902,6 +946,159 @@ describe("removing files that were assembled but never recorded", () => { expect(mocks.videoUpdates).toEqual([]); }); + const missing = () => + Effect.fail({ _tag: "S3Error", cause: { name: "NotFound" } }); + + it("retires a day-old video that never received a file", async () => { + mocks.unrecorded = [row("empty")]; + mocks.headObject.mockImplementation(missing); + + expect(await removeUnrecordedInfraFiles(now)).toBe(1); + + expect(mocks.events).toEqual(["lock", "lock", "retire", "cancel"]); + expect(mocks.videoUpdates).toEqual([{ deletedAt: now }]); + expect(mocks.deleteObject).not.toHaveBeenCalled(); + expect(mocks.listUploads).toHaveBeenCalledTimes(2); + expect(mocks.listUploads).toHaveBeenCalledWith({ + prefix: "developer/app-1/empty/video", + }); + }); + + it("does not retire a video whose completion assembled the file before the sweep reached the lock", async () => { + mocks.unrecorded = [row("finishing")]; + mocks.headObject + .mockImplementationOnce(missing) + .mockImplementation(() => Effect.succeed({ LastModified: now })); + + expect(await removeUnrecordedInfraFiles(now)).toBe(0); + expect(mocks.videoUpdates).toEqual([]); + expect(mocks.deletedAt).toBe(null); + expect(mocks.deleteObject).not.toHaveBeenCalled(); + }); + + it("retires a video with no file whose completion died holding its claim", async () => { + mocks.unrecorded = [row("abandoned")]; + mocks.status = "processing"; + mocks.claimedAt = hoursAgo(1); + mocks.headObject.mockImplementation(missing); + + expect(await removeUnrecordedInfraFiles(now)).toBe(1); + expect(mocks.videoUpdates).toEqual([{ deletedAt: now }]); + }); + + it("leaves a video with no file alone while a completion's claim is recent", async () => { + mocks.unrecorded = [row("claimed")]; + mocks.status = "processing"; + mocks.claimedAt = new Date(now.getTime() - 60_000); + mocks.headObject.mockImplementation(missing); + + expect(await removeUnrecordedInfraFiles(now)).toBe(0); + expect(mocks.videoUpdates).toEqual([]); + expect(mocks.deletedAt).toBe(null); + }); + + it("does not retire a video whose upload is assembled between storage's file read and its upload listing", async () => { + mocks.unrecorded = [row("assembling")]; + let assembled = false; + let listings = 0; + mocks.headObject.mockImplementation(() => + assembled ? Effect.succeed({ LastModified: now }) : missing(), + ); + mocks.listUploads.mockImplementation(() => { + listings++; + if (listings === 2) assembled = true; + return Effect.succeed({ Uploads: [] }); + }); + + expect(await removeUnrecordedInfraFiles(now)).toBe(0); + expect(mocks.videoUpdates).toEqual([]); + expect(mocks.deletedAt).toBe(null); + expect(mocks.deleteObject).not.toHaveBeenCalled(); + }); + + it("does not retire a video with no file when storage shows a file or an open upload once the row is locked", async () => { + mocks.unrecorded = [row("late")]; + mocks.headObject + .mockImplementationOnce(missing) + .mockImplementation(() => Effect.succeed({ LastModified: hoursAgo(0) })); + + expect(await removeUnrecordedInfraFiles(now)).toBe(0); + + mocks.headObject.mockReset(); + mocks.headObject.mockImplementation(missing); + mocks.listUploads + .mockImplementationOnce(() => Effect.succeed({ Uploads: [] })) + .mockImplementation(() => + Effect.succeed({ + Uploads: [ + { + Key: "developer/app-1/late/video", + UploadId: "upload-2", + Initiated: now, + }, + ], + }), + ); + + expect(await removeUnrecordedInfraFiles(now)).toBe(0); + expect(mocks.videoUpdates).toEqual([]); + expect(mocks.deletedAt).toBe(null); + }); + + it("leaves a video with no file alone while an upload for it is still open", async () => { + mocks.unrecorded = [row("sending")]; + mocks.headObject.mockImplementation(missing); + mocks.listUploads.mockImplementation(() => + Effect.succeed({ + Uploads: [ + { + Key: "developer/app-1/sending/video", + UploadId: "upload-1", + Initiated: hoursAgo(1), + }, + ], + }), + ); + + expect(await removeUnrecordedInfraFiles(now)).toBe(0); + expect(mocks.videoUpdates).toEqual([]); + }); + + it("leaves a video with no file alone when storage cannot say what is there", async () => { + mocks.unrecorded = [row("unsure")]; + mocks.headObject.mockImplementation(() => + Effect.fail({ _tag: "S3Error", cause: { name: "SlowDown" } }), + ); + + expect(await removeUnrecordedInfraFiles(now)).toBe(0); + + mocks.headObject.mockImplementation(missing); + mocks.listUploads.mockImplementation(() => + Effect.fail({ _tag: "S3Error", cause: { name: "InternalError" } }), + ); + + expect(await removeUnrecordedInfraFiles(now)).toBe(0); + + mocks.listUploads.mockImplementation(() => + Effect.succeed({ Uploads: [], IsTruncated: true }), + ); + + expect(await removeUnrecordedInfraFiles(now)).toBe(0); + expect(mocks.videoUpdates).toEqual([]); + }); + + it("does not retire a video with no file when a completion recorded it first", async () => { + mocks.unrecorded = [row("raced")]; + mocks.headObject.mockImplementation(() => { + mocks.storedDuration = 60; + return missing(); + }); + + expect(await removeUnrecordedInfraFiles(now)).toBe(0); + expect(mocks.videoUpdates).toEqual([]); + expect(mocks.deletedAt).toBe(null); + }); + it("counts only the files it removed", async () => { mocks.unrecorded = [row("a"), row("b")]; mocks.headObject.mockImplementation(old); diff --git a/apps/web/__tests__/unit/developer-sdk-upload.test.ts b/apps/web/__tests__/unit/developer-sdk-upload.test.ts index 7b075eb140a..7e43f2ba295 100644 --- a/apps/web/__tests__/unit/developer-sdk-upload.test.ts +++ b/apps/web/__tests__/unit/developer-sdk-upload.test.ts @@ -474,8 +474,10 @@ describe("developer SDK multipart upload", () => { const response = await post("/abort", { uploadId: "upload-1" }); expect(response.status).toBe(409); + expect(response.headers.get("Retry-After")).toBeNull(); await expect(response.json()).resolves.toMatchObject({ code: "completion_in_progress", + hint: "Do not repeat the abort. Wait until GET /api/developer/v1/videos/{id} no longer shows the video as processing, then delete the video if it is not wanted.", }); expect(mocks.abort).not.toHaveBeenCalled(); }); @@ -677,17 +679,19 @@ describe("client mistakes on complete and abort", () => { }, ); - it("answers 404 when aborting an upload that does not exist", async () => { + it("answers a repeated abort with success, whether storage accepts it again or no longer knows the upload", async () => { mocks.video = uploaded; - mocks.abort.mockImplementation(fails("NoSuchUpload", 404)); - const response = await post("/abort", { uploadId: "not-an-upload" }); + const first = await post("/abort", { uploadId: "upload-1" }); + const repeated = await post("/abort", { uploadId: "upload-1" }); + mocks.abort.mockImplementation(fails("NoSuchUpload", 404)); + const gone = await post("/abort", { uploadId: "upload-1" }); - expect(await refusal(response)).toMatchObject({ - status: 404, - code: "upload_not_found", - hint: expect.stringContaining("Nothing was aborted"), - }); + for (const response of [first, repeated, gone]) { + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ success: true }); + } + expect(mocks.abort).toHaveBeenCalledTimes(3); expect(console.error).not.toHaveBeenCalled(); }); diff --git a/apps/web/__tests__/unit/developer-transactions-api.test.ts b/apps/web/__tests__/unit/developer-transactions-api.test.ts index a4e29961c0d..95ef9c4d0ea 100644 --- a/apps/web/__tests__/unit/developer-transactions-api.test.ts +++ b/apps/web/__tests__/unit/developer-transactions-api.test.ts @@ -55,14 +55,22 @@ vi.mock("@cap/database", async () => { b.createdAt.getTime() - a.createdAt.getTime() || (a.id < b.id ? 1 : -1), ); - if (cursor.length > 0) { - const time = asTime(cursor[0]); + const at = (index: number) => asTime(cursor[index]); + if (cursor.length === 3) { const id = String(cursor.at(-1)); rows = rows.filter( (row) => - row.createdAt.getTime() < time || - (row.createdAt.getTime() === time && row.id < id), + row.createdAt.getTime() < at(0) || + (row.createdAt.getTime() === at(0) && row.id < id), ); + } else if (rendered.sql.includes(">= ?")) { + rows = rows.filter( + (row) => + row.createdAt.getTime() >= at(0) && + row.createdAt.getTime() <= at(1), + ); + } else if (cursor.length === 1) { + rows = rows.filter((row) => row.createdAt.getTime() === at(0)); } if (fields && "total" in fields) return [{ total: rows.length }]; return rows.slice(offset, offset + limit); @@ -129,6 +137,7 @@ vi.mock("@/lib/infra-video-upload", () => ({ import * as rest from "@/app/api/developer/v1/[...route]/route"; import { developerOpenApiSpec } from "@/lib/developer-openapi"; +import { DEVELOPER_KEY_REQUESTS_PER_MINUTE } from "@/lib/developer-rate-limits"; import { readAppLedger } from "@/lib/infra-ledger-store"; let serial = 0; @@ -186,10 +195,14 @@ describe("GET /transactions", () => { const publicKey = await get("/transactions", "cpk_public_key"); const unknown = await get("/transactions", "csk_not_a_key"); - for (const response of [missing, publicKey, unknown]) { - expect(response.status).toBe(401); + for (const [response, status, code] of [ + [missing, 401, "invalid_secret_key"], + [publicKey, 403, "public_key_not_allowed"], + [unknown, 401, "invalid_secret_key"], + ] as const) { + expect(response.status).toBe(status); expect(await response.json()).toMatchObject({ - code: "invalid_secret_key", + code, hint: expect.any(String), docs: expect.any(String), }); @@ -351,9 +364,15 @@ describe("GET /transactions", () => { const first = await get("/transactions", key); const usageAfter = await get("/usage/nope", key); - expect(first.headers.get("X-RateLimit-Limit")).toBe("60"); - expect(first.headers.get("X-RateLimit-Remaining")).toBe("59"); - expect(usageAfter.headers.get("X-RateLimit-Remaining")).toBe("58"); + expect(first.headers.get("X-RateLimit-Limit")).toBe( + String(DEVELOPER_KEY_REQUESTS_PER_MINUTE), + ); + expect(first.headers.get("X-RateLimit-Remaining")).toBe( + String(DEVELOPER_KEY_REQUESTS_PER_MINUTE - 1), + ); + expect(usageAfter.headers.get("X-RateLimit-Remaining")).toBe( + String(DEVELOPER_KEY_REQUESTS_PER_MINUTE - 2), + ); }); it("answers 405 for a method the path does not take", async () => { diff --git a/apps/web/__tests__/unit/developer-transactions-export.test.ts b/apps/web/__tests__/unit/developer-transactions-export.test.ts index 227f957c10d..1ad4b34d0a0 100644 --- a/apps/web/__tests__/unit/developer-transactions-export.test.ts +++ b/apps/web/__tests__/unit/developer-transactions-export.test.ts @@ -149,9 +149,9 @@ describe("exporting the ledger as CSV", () => { const lines = (await response.text()).trimEnd().split("\r\n"); expect(lines).toEqual([ "id,time_utc,type,note,reference_type,reference_id,amount_usd,amount_micro_credits,balance_usd,balance_micro_credits", - "a3,2026-11-12T14:05:09Z,Transcription,,developer_transcript,transcript-a3,-0.00033,-33,24.99934,2499934", - "a2,2026-11-12T14:05:09Z,Transcription,,developer_transcript,transcript-a2,-0.00033,-33,24.99967,2499967", - "a1,2026-11-12T14:05:09Z,Top-up,,stripe_payment_intent,pi_3TestPayment,25.00,2500000,25.00,2500000", + "a3,2026-11-12T14:05:09Z,Transcription,,Transcript,transcript-a3,-0.00033,-33,24.99934,2499934", + "a2,2026-11-12T14:05:09Z,Transcription,,Transcript,transcript-a2,-0.00033,-33,24.99967,2499967", + "a1,2026-11-12T14:05:09Z,Top-up,,Payment,pi_3TestPayment,25.00,2500000,25.00,2500000", ]); expect(mocks.read).toEqual(["app-a"]); }); diff --git a/apps/web/__tests__/unit/developer-transcribe-download.test.ts b/apps/web/__tests__/unit/developer-transcribe-download.test.ts index 09b2fc91634..d155ed5dee7 100644 --- a/apps/web/__tests__/unit/developer-transcribe-download.test.ts +++ b/apps/web/__tests__/unit/developer-transcribe-download.test.ts @@ -80,6 +80,25 @@ vi.mock("@/lib/developer-transcripts", async (importOriginal) => ({ TRANSCRIPT_URL_DOWNLOAD_TIMEOUT_MS: 2_000, })); +// requestPublicOnly refuses a loopback address, by name or as a literal, so +// the local server is reached through a name this lookup pins to it. +vi.mock("@/lib/s3-endpoint-policy", async (importOriginal) => { + const actual = + await importOriginal(); + const publicOnlyLookup: typeof actual.publicOnlyLookup = ( + _hostname, + options, + callback, + ) => { + if (typeof options === "object" && options.all) { + callback(null, [{ address: "127.0.0.1", family: 4 }]); + return; + } + callback(null, "127.0.0.1", 4); + }; + return { ...actual, publicOnlyLookup }; +}); + vi.mock("@/lib/public-media-fetch", async (importOriginal) => { const actual = await importOriginal(); @@ -108,7 +127,8 @@ vi.mock("@/lib/public-media-fetch", async (importOriginal) => { }; }); -vi.mock("@/lib/media-container", () => ({ +vi.mock("@/lib/media-container", async (importOriginal) => ({ + ...(await importOriginal()), urlIsSelfContainedMedia: async () => true, })); @@ -199,6 +219,7 @@ import { import { developerTranscribeWorkflow } from "@/workflows/developer-transcribe"; const SOURCE_KEY = "developer/_uploads/app-1/t-tr-1"; +const MEDIA_HEAD = new TextEncoder().encode("ID3"); const SIGNED_URL = "https://media.example.com/clip.mp3?sig=secret&expires=1"; type Origin = { @@ -216,6 +237,7 @@ const serve = async (options: { everyMs: number; contentLength: boolean; stopAfterChunks?: number; + head?: Uint8Array; }) => { const state: Origin = { server: createServer((_request, response) => { @@ -238,12 +260,14 @@ const serve = async (options: { sent += next; chunk += 1; state.served = sent; + const body = Buffer.alloc(next, 7); + if (chunk === 1) Buffer.from(options.head ?? MEDIA_HEAD).copy(body); if (sent >= options.bytes) { state.finishedAt = Date.now(); - response.end(Buffer.alloc(next, 7)); + response.end(body); return; } - response.write(Buffer.alloc(next, 7)); + response.write(body); setTimeout(tick, options.everyMs); }; response.on("close", () => { @@ -258,7 +282,7 @@ const serve = async (options: { await new Promise((resolve) => state.server.listen(0, "127.0.0.1", resolve), ); - context.origin = `http://127.0.0.1:${(state.server.address() as AddressInfo).port}`; + context.origin = `http://media.test:${(state.server.address() as AddressInfo).port}`; origin = state; return state; }; @@ -415,4 +439,111 @@ describe("downloading a URL source from a slow server", () => { }); expect(balance()).toBe(100_000); }); + + describe("a URL that is not an audio or video file", () => { + const notMedia = (head: Uint8Array) => + serve({ + bytes: 50 * 1024 * 1024, + chunks: 50 * 16, + everyMs: 1, + contentLength: true, + head, + }); + + const WEB_PAGE = "The URL returned a web page, not a media file"; + const NOT_MEDIA = "The file is not a supported audio or video format"; + const bytes = (text: string) => new TextEncoder().encode(text); + + it.each([ + ["an HTML page", bytes(""), WEB_PAGE], + [ + "an HTML page after a BOM and blank lines", + new Uint8Array([0xef, 0xbb, 0xbf, ...bytes("\r\n\t ")]), + WEB_PAGE, + ], + ["an HTML page with no doctype", bytes("\n"), WEB_PAGE], + ["an XML document", bytes(''), NOT_MEDIA], + ["a playlist", bytes("#EXTM3U\n#EXT-X-VERSION:3\n"), NOT_MEDIA], + ["random bytes", new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]), NOT_MEDIA], + ])( + "is refused from its first bytes when it is %s, with nothing stored", + async (_label, head, message) => { + const server = await notMedia(head); + + const result = await run(); + + expect(result).toEqual({ success: false, message }); + expect(transcript()).toMatchObject({ + status: "error", + errorCode: "unsupported_format", + errorMessage: message, + billedMicroCredits: 0, + reservedMicroCredits: 0, + }); + expect(ledger()).toHaveLength(0); + expect(context.streamedPuts).toBe(0); + expect(context.filePuts).toEqual([]); + expect(context.objects.has(SOURCE_KEY)).toBe(false); + expect(context.extract).not.toHaveBeenCalled(); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(server.closedEarly).toBe(true); + expect(server.served).toBeLessThan(2 * 1024 * 1024); + }, + ); + + const storedAtExtract = () => { + const seen: { bytes: Uint8Array | null } = { bytes: null }; + const answer = context.extract.getMockImplementation(); + context.extract.mockImplementationOnce(async (...args: unknown[]) => { + seen.bytes = context.objects.get(SOURCE_KEY) ?? null; + return answer?.(...args); + }); + return seen; + }; + + it("still stores every byte of a media file, its first bytes included", async () => { + const server = await serve({ + bytes: 40_000, + chunks: 7, + everyMs: 5, + contentLength: false, + }); + const seen = storedAtExtract(); + + await run(); + + const stored = seen.bytes; + expect(stored?.length).toBe(40_000); + expect( + Buffer.from(stored ?? []) + .subarray(0, 3) + .toString(), + ).toBe("ID3"); + expect(stored?.[3]).toBe(7); + expect(stored?.[39_999]).toBe(7); + expect(server.closedEarly).toBe(false); + expect(transcript()?.status).toBe("complete"); + }); + + it("still stores a media file shorter than the bytes it looks at", async () => { + await serve({ bytes: 1_000, chunks: 1, everyMs: 1, contentLength: true }); + const seen = storedAtExtract(); + + await run(); + + expect(seen.bytes?.length).toBe(1_000); + expect(transcript()?.status).toBe("complete"); + }); + + it("still says an empty file could not be downloaded", async () => { + await serve({ bytes: 0, chunks: 1, everyMs: 1, contentLength: false }); + + await run(); + + expect(transcript()).toMatchObject({ + status: "error", + errorCode: "source_unreachable", + }); + }); + }); }); diff --git a/apps/web/__tests__/unit/developer-transcribe-fast-path.test.ts b/apps/web/__tests__/unit/developer-transcribe-fast-path.test.ts new file mode 100644 index 00000000000..31916ff087f --- /dev/null +++ b/apps/web/__tests__/unit/developer-transcribe-fast-path.test.ts @@ -0,0 +1,674 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const context = vi.hoisted(() => ({ + fake: null as null | { + db: unknown; + rows: (table: object) => Record[]; + seed: ( + table: object, + row: Record, + ) => Record; + reset: () => void; + failNextCommit: () => void; + }, + objects: new Map(), + ledgerIds: 0, + transcribe: vi.fn(), + measured: null as number | null, + writes: new Map(), +})); + +vi.mock("@cap/database", async () => { + const schema = await import("@cap/database/schema"); + const { createFakeDatabase } = await import("./developer-transcript-fake-db"); + const fake = createFakeDatabase( + new Map Record>([ + [ + schema.developerTranscripts, + () => ({ + status: "queued", + sourceType: "upload", + sourceUrl: null, + requestedLanguage: null, + durationSeconds: null, + text: null, + resultKey: null, + reservedMicroCredits: 0, + billedMicroCredits: 0, + errorCode: null, + errorMessage: null, + deletedAt: null, + }), + ], + ]), + ); + context.fake = fake; + return { db: () => fake.db }; +}); + +vi.mock("@cap/database/helpers", () => ({ + nanoId: () => { + context.ledgerIds += 1; + return `ledger-${context.ledgerIds}`; + }, + nanoIdLength: 15, +})); + +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ + MEDIA_SERVER_URL: "http://media.test", + MEDIA_SERVER_WEBHOOK_SECRET: "secret", + }), + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + NODE_ENV: "test", +})); + +vi.mock("@/lib/mp3-duration", async (importOriginal) => { + const actual = await importOriginal(); + return { + measureMp3DurationSeconds: (audio: Uint8Array) => + context.measured ?? actual.measureMp3DurationSeconds(audio), + }; +}); + +vi.mock("@/lib/developer-transcript-storage", () => ({ + developerObjectSize: async (key: string) => + context.objects.get(key)?.length ?? null, + developerObjectUrl: async (key: string) => `http://storage.test/${key}`, + getDeveloperObjectText: async (key: string) => { + const bytes = context.objects.get(key); + return bytes ? new TextDecoder().decode(bytes) : null; + }, + putDeveloperObject: async (key: string, body: string | Uint8Array) => { + context.writes.set(key, (context.writes.get(key) ?? 0) + 1); + context.objects.set( + key, + typeof body === "string" ? new TextEncoder().encode(body) : body, + ); + }, + putDeveloperObjectFile: async () => {}, + putDeveloperObjectStream: async () => {}, + deleteDeveloperObjects: async (keys: string[]) => { + for (const key of keys) context.objects.delete(key); + }, +})); + +vi.mock("@/lib/developer-transcript-speech", () => ({ + transcribeSpeech: context.transcribe, + parseTranscriptLanguage: () => "auto", +})); + +import { + developerCreditAccounts, + developerCreditTransactions, + developerTranscripts, +} from "@cap/database/schema"; +import { transcriptionMicroCredits } from "@/lib/developer-pricing"; +import { + developerTranscribeWorkflow, + FAST_PATH_BUDGET_MS, + FAST_PATH_MAX_AUDIO_SECONDS, + runTranscriptFastPath, +} from "@/workflows/developer-transcribe"; + +const SOURCE_KEY = "developer/_uploads/app-1/t-tr-1"; +const AUDIO_KEY = "developer/app-1/transcripts/tr-1/audio.mp3"; +const RESULT_KEY = "developer/app-1/transcripts/tr-1/result.json"; +const FRAME_SECONDS = 1152 / 44_100; + +function mp3Frames(count: number, bitrateIndex = 9) { + const bitrate = [0, 32, 40, 48, 56, 64, 80, 96, 112, 128][bitrateIndex] ?? 0; + const length = Math.floor((144_000 * bitrate) / 44_100); + const out = new Uint8Array(length * count); + for (let index = 0; index < count; index++) { + out.set([0xff, 0xfb, bitrateIndex << 4, 0x00], index * length); + } + return out; +} + +const wavHeader = () => { + const bytes = new Uint8Array(64); + bytes.set(new TextEncoder().encode("RIFF"), 0); + bytes.set(new TextEncoder().encode("WAVE"), 8); + return bytes; +}; + +type Media = { + probe: () => Response; + convert: () => Response; + check: () => Response; + convertCalls: number; + convertBytesPulled: number; + convertCancelled: boolean; +}; + +const media: Media = { + probe: () => new Response("{}", { status: 500 }), + convert: () => new Response(null, { status: 500 }), + check: () => Response.json({ hasAudio: true }), + convertCalls: 0, + convertBytesPulled: 0, + convertCancelled: false, +}; + +const streamOf = (chunk: Uint8Array, chunks: number) => { + let sent = 0; + return new ReadableStream({ + pull(controller) { + if (sent >= chunks) { + controller.close(); + return; + } + sent += 1; + media.convertBytesPulled += chunk.length; + controller.enqueue(chunk); + }, + cancel() { + media.convertCancelled = true; + }, + }); +}; + +const fake = () => { + if (!context.fake) throw new Error("Fake database is not ready"); + return context.fake; +}; +const transcript = () => fake().rows(developerTranscripts)[0]; +const balance = () => + fake().rows(developerCreditAccounts)[0]?.balanceMicroCredits; +const ledger = () => fake().rows(developerCreditTransactions); +const run = () => + developerTranscribeWorkflow({ transcriptId: "tr-1", runId: "run-1" }); + +beforeEach(() => { + fake().reset(); + context.objects = new Map([[SOURCE_KEY, wavHeader()]]); + context.ledgerIds = 0; + context.measured = null; + context.writes = new Map(); + context.transcribe.mockReset(); + context.transcribe.mockResolvedValue({ + state: "ok", + text: "hello world", + words: [ + { text: "hello", start: 0, end: 400, confidence: 0.9 }, + { text: "world", start: 450, end: 900, confidence: 0.9 }, + ], + language: "en", + durationSeconds: 30, + }); + media.probe = () => new Response("{}", { status: 500 }); + media.convert = () => new Response(null, { status: 500 }); + media.check = () => Response.json({ hasAudio: true }); + media.convertCalls = 0; + media.convertBytesPulled = 0; + media.convertCancelled = false; + fake().seed(developerCreditAccounts, { + id: "account-1", + appId: "app-1", + balanceMicroCredits: 100_000, + }); + fake().seed(developerTranscripts, { + id: "tr-1", + appId: "app-1", + runId: "run-1", + }); + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = String(input); + if (url.startsWith("http://storage.test/")) { + const bytes = context.objects.get( + url.slice("http://storage.test/".length), + ); + return bytes + ? new Response(bytes.slice(), { status: 200 }) + : new Response(null, { status: 404 }); + } + if (url === "http://media.test/video/probe") return media.probe(); + if (url === "http://media.test/audio/check") return media.check(); + if (url === "http://media.test/audio/convert") { + media.convertCalls += 1; + return media.convert(); + } + throw new Error(`Unexpected fetch ${url}`); + }); +}); + +afterEach(() => vi.unstubAllGlobals()); + +const thirtySeconds = () => { + const frames = Math.round(30 / FRAME_SECONDS); + media.convert = () => + new Response(streamOf(mp3Frames(frames), 1), { status: 200 }); + return transcriptionMicroCredits(frames * FRAME_SECONDS); +}; + +const fast = () => runTranscriptFastPath("tr-1", "run-1"); + +const failSettleOnce = () => { + const answer = context.transcribe.getMockImplementation(); + context.transcribe.mockImplementationOnce(async (...args: unknown[]) => { + fake().failNextCommit(); + return answer ? answer(...args) : undefined; + }); +}; + +const longSource = () => { + const bytes = new Uint8Array(1024 * 1024); + bytes.set(wavHeader()); + context.objects.set(SOURCE_KEY, bytes); + const account = fake().rows(developerCreditAccounts)[0]; + if (account) account.balanceMicroCredits = 1_000_000; +}; + +describe("the fast path", () => { + it("completes a short file in one step: claimed, held, extracted, transcribed and charged", async () => { + const cost = thirtySeconds(); + + const result = await fast(); + + expect(result).toMatchObject({ resume: "done", settlement: "complete" }); + expect(transcript()).toMatchObject({ + status: "complete", + text: "hello world", + billedMicroCredits: cost, + reservedMicroCredits: 0, + wordCount: 2, + }); + expect(balance()).toBe(100_000 - cost); + expect(ledger()).toHaveLength(1); + expect(context.transcribe).toHaveBeenCalledTimes(1); + expect(media.convertCalls).toBe(1); + }); + + it("hands the audio to speech recognition without reading it back from storage", async () => { + thirtySeconds(); + const reads: string[] = []; + const realFetch = globalThis.fetch; + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + reads.push(String(input)); + return realFetch(input); + }); + + await fast(); + + expect(reads.filter((url) => url.endsWith(AUDIO_KEY))).toEqual([]); + expect(context.objects.has(AUDIO_KEY)).toBe(true); + }); + + it("leaves nothing for the rest of the workflow to redo", async () => { + const cost = thirtySeconds(); + + const result = await run(); + + expect(result).toEqual({ success: true, message: "Transcript completed" }); + expect(context.transcribe).toHaveBeenCalledTimes(1); + expect(media.convertCalls).toBe(1); + expect(context.writes.get(RESULT_KEY)).toBe(1); + expect(context.writes.get(AUDIO_KEY)).toBe(1); + expect(ledger()).toHaveLength(1); + expect(balance()).toBe(100_000 - cost); + expect([...context.objects.keys()]).toEqual([RESULT_KEY]); + }); + + it("does not charge, transcribe or write twice when the whole step runs again", async () => { + const cost = thirtySeconds(); + + const first = await fast(); + const second = await fast(); + + expect(first).toMatchObject({ resume: "done", settlement: "complete" }); + expect(second).toEqual({ resume: "finalize", settlement: "complete" }); + expect(context.transcribe).toHaveBeenCalledTimes(1); + expect(context.writes.get(RESULT_KEY)).toBe(1); + expect(ledger()).toHaveLength(1); + expect(balance()).toBe(100_000 - cost); + }); + + it("still cleans up and writes captions when it runs again after its settle was saved", async () => { + const cost = thirtySeconds(); + expect(await fast()).toMatchObject({ resume: "done" }); + context.objects.set(SOURCE_KEY, wavHeader()); + + const result = await run(); + + expect(result).toEqual({ success: true, message: "Transcript completed" }); + expect([...context.objects.keys()]).toEqual([RESULT_KEY]); + expect(context.transcribe).toHaveBeenCalledTimes(1); + expect(ledger()).toHaveLength(1); + expect(balance()).toBe(100_000 - cost); + }); + + it("cleans up when it runs again after its own settle ended the transcript for lack of credit", async () => { + thirtySeconds(); + const settled = context.transcribe.getMockImplementation(); + context.transcribe.mockImplementationOnce(async (...args: unknown[]) => { + const account = fake().rows(developerCreditAccounts)[0]; + if (account) account.balanceMicroCredits = 0; + return settled ? settled(...args) : undefined; + }); + expect(await fast()).toMatchObject({ + resume: "done", + settlement: "insufficient_credits", + }); + context.objects.set(SOURCE_KEY, wavHeader()); + + const result = await run(); + + expect(result).toEqual({ + success: false, + message: "Transcript was not completed: skipped", + }); + expect([...context.objects.keys()]).toEqual([]); + expect(ledger()).toHaveLength(0); + }); + + it("leaves a transcript another run owns alone even when it is finished", async () => { + thirtySeconds(); + await fast(); + const row = transcript(); + if (row) row.runId = "run-2"; + + expect(await fast()).toEqual({ resume: "unclaimed" }); + expect(context.objects.has(RESULT_KEY)).toBe(true); + }); + + const stopAfterExtraction = () => { + let calls = 0; + return vi + .spyOn(performance, "now") + .mockImplementation(() => (calls++ < 2 ? 0 : FAST_PATH_BUDGET_MS + 1)); + }; + + const deleteWhileProcessing = () => { + const row = transcript(); + if (!row) throw new Error("no transcript"); + row.deletedAt = new Date("2026-11-13T05:00:00.000Z"); + row.text = null; + }; + + it("charges a transcript deleted after its speech finished when the whole step runs again", async () => { + const cost = thirtySeconds(); + failSettleOnce(); + expect(await fast()).toMatchObject({ resume: "settle" }); + expect(transcript()?.status).toBe("processing"); + deleteWhileProcessing(); + + const retry = await fast(); + + expect(retry).toMatchObject({ resume: "done", settlement: "complete" }); + expect(context.transcribe).toHaveBeenCalledTimes(1); + expect(ledger()).toHaveLength(1); + expect(balance()).toBe(100_000 - cost); + expect(transcript()).toMatchObject({ + status: "complete", + billedMicroCredits: cost, + reservedMicroCredits: 0, + text: null, + }); + expect(transcript()?.deletedAt).toBeInstanceOf(Date); + + expect((await run()).success).toBe(true); + expect([...context.objects.keys()]).toEqual([]); + expect(ledger()).toHaveLength(1); + }); + + it("does not ask for the source hold again when it runs again after extraction", async () => { + const cost = thirtySeconds(); + const clock = stopAfterExtraction(); + expect(await fast()).toEqual({ resume: "transcribe" }); + clock.mockRestore(); + expect(Number(transcript()?.reservedMicroCredits)).toBe(cost); + fake().seed(developerTranscripts, { + id: "tr-other", + appId: "app-1", + status: "processing", + reservedMicroCredits: 99_500, + }); + + const retry = await fast(); + + expect(retry).toMatchObject({ resume: "done", settlement: "complete" }); + expect(media.convertCalls).toBe(1); + expect(context.transcribe).toHaveBeenCalledTimes(1); + expect(ledger()).toHaveLength(1); + expect(balance()).toBe(100_000 - cost); + }); + + it("finishes through the workflow without a new hold when it runs again after extraction", async () => { + const cost = thirtySeconds(); + const clock = stopAfterExtraction(); + await fast(); + clock.mockRestore(); + fake().seed(developerTranscripts, { + id: "tr-other", + appId: "app-1", + status: "processing", + reservedMicroCredits: 99_500, + }); + + const result = await run(); + + expect(result).toEqual({ success: true, message: "Transcript completed" }); + expect(context.transcribe).toHaveBeenCalledTimes(1); + expect(ledger()).toHaveLength(1); + expect(balance()).toBe(100_000 - cost); + }); + + it("ends a transcript deleted before its speech without a charge when the step runs again", async () => { + thirtySeconds(); + const clock = stopAfterExtraction(); + await fast(); + clock.mockRestore(); + deleteWhileProcessing(); + + const result = await run(); + + expect(result.success).toBe(false); + expect(context.transcribe).not.toHaveBeenCalled(); + expect(transcript()).toMatchObject({ + status: "error", + billedMicroCredits: 0, + reservedMicroCredits: 0, + }); + expect(ledger()).toHaveLength(0); + expect([...context.objects.keys()]).toEqual([]); + }); + + it("leaves a run that was never claimed alone", async () => { + thirtySeconds(); + const row = transcript(); + if (row) row.status = "processing"; + if (row) row.runId = "run-other"; + + expect(await fast()).toEqual({ resume: "unclaimed" }); + expect(media.convertCalls).toBe(0); + expect(context.transcribe).not.toHaveBeenCalled(); + }); + + it("does not transcribe or charge twice when it runs again after the settle failed", async () => { + const cost = thirtySeconds(); + failSettleOnce(); + + const first = await fast(); + expect(first).toMatchObject({ resume: "settle" }); + expect(transcript()?.status).toBe("processing"); + expect(ledger()).toHaveLength(0); + + const second = await fast(); + + expect(second).toMatchObject({ resume: "done", settlement: "complete" }); + expect(context.transcribe).toHaveBeenCalledTimes(1); + expect(context.writes.get(RESULT_KEY)).toBe(1); + expect(ledger()).toHaveLength(1); + expect(balance()).toBe(100_000 - cost); + }); + + it("is finished step by step when its settle fails, with one charge", async () => { + const cost = thirtySeconds(); + failSettleOnce(); + + const result = await run(); + + expect(result.success).toBe(true); + expect(context.transcribe).toHaveBeenCalledTimes(1); + expect(ledger()).toHaveLength(1); + expect(balance()).toBe(100_000 - cost); + }); + + it("hands a long recording to its own transcription step", async () => { + thirtySeconds(); + context.measured = FAST_PATH_MAX_AUDIO_SECONDS + 1; + longSource(); + + const result = await fast(); + + expect(result).toEqual({ resume: "transcribe" }); + expect(context.transcribe).not.toHaveBeenCalled(); + expect(transcript()).toMatchObject({ + status: "processing", + durationSeconds: FAST_PATH_MAX_AUDIO_SECONDS + 1, + }); + + const finished = await run(); + expect(finished).toEqual({ + success: true, + message: "Transcript completed", + }); + expect(media.convertCalls).toBe(1); + expect(context.transcribe).toHaveBeenCalledTimes(1); + expect(ledger()).toHaveLength(1); + }); + + it("finishes a long recording through the separate step with the same charge", async () => { + thirtySeconds(); + context.measured = FAST_PATH_MAX_AUDIO_SECONDS + 1; + longSource(); + + const result = await run(); + + expect(result.success).toBe(true); + expect(context.transcribe).toHaveBeenCalledTimes(1); + expect(transcript()).toMatchObject({ + status: "complete", + billedMicroCredits: transcriptionMicroCredits( + FAST_PATH_MAX_AUDIO_SECONDS + 1, + ), + }); + expect(ledger()).toHaveLength(1); + }); + + it("hands extraction to its own step when getting the file took most of the budget", async () => { + thirtySeconds(); + let calls = 0; + const clock = vi + .spyOn(performance, "now") + .mockImplementation(() => (calls++ === 0 ? 0 : FAST_PATH_BUDGET_MS + 1)); + + const result = await fast(); + clock.mockRestore(); + + expect(result).toEqual({ resume: "extract", audio: null }); + expect(media.convertCalls).toBe(0); + expect(Number(transcript()?.reservedMicroCredits)).toBeGreaterThan(0); + + expect((await run()).success).toBe(true); + expect(media.convertCalls).toBe(1); + expect(context.transcribe).toHaveBeenCalledTimes(1); + expect(ledger()).toHaveLength(1); + }); + + it("hands transcription to its own step when extraction took most of the budget", async () => { + thirtySeconds(); + let calls = 0; + const clock = vi + .spyOn(performance, "now") + .mockImplementation(() => (calls++ < 2 ? 0 : FAST_PATH_BUDGET_MS + 1)); + + const result = await fast(); + clock.mockRestore(); + + expect(result).toEqual({ resume: "transcribe" }); + expect(media.convertCalls).toBe(1); + expect(context.transcribe).not.toHaveBeenCalled(); + }); + + it("hands over at the hold when another transcript holds the credit", async () => { + thirtySeconds(); + fake().seed(developerTranscripts, { + id: "tr-other", + appId: "app-1", + status: "processing", + reservedMicroCredits: 99_500, + }); + + const result = await fast(); + + expect(result).toMatchObject({ + resume: "prepare", + source: { ok: false, wait: true }, + }); + expect(media.convertCalls).toBe(0); + expect(context.transcribe).not.toHaveBeenCalled(); + }); + + it("hands over to the hold raise when the audio is longer than its file suggested", async () => { + const frames = Math.round(30 / FRAME_SECONDS); + media.probe = () => + Response.json({ + metadata: { duration: 3 * 3600, width: 1, height: 1 }, + }); + media.convert = () => + new Response(streamOf(mp3Frames(frames), 1), { status: 200 }); + + const result = await fast(); + + expect(result).toMatchObject({ + resume: "extract", + audio: { ok: false, raise: true, reportedSeconds: 3 * 3600 }, + }); + expect(context.transcribe).not.toHaveBeenCalled(); + }); + + it("keeps the failure code and message of a file it cannot use", async () => { + context.objects.set( + SOURCE_KEY, + new TextEncoder().encode("#EXTM3U\nhttp://127.0.0.1/secret.ts\n"), + ); + + const result = await run(); + + expect(result).toEqual({ + success: false, + message: "The file is not a supported audio or video format", + }); + expect(transcript()).toMatchObject({ + status: "error", + errorCode: "unsupported_format", + errorMessage: "The file is not a supported audio or video format", + billedMicroCredits: 0, + }); + }); + + it("keeps the failure code when speech recognition keeps failing", async () => { + thirtySeconds(); + context.transcribe.mockRejectedValue(new Error("speech is down")); + + await expect(run()).rejects.toThrow("speech is down"); + + expect(transcript()).toMatchObject({ + status: "error", + errorCode: "transcription_failed", + errorMessage: "The transcript could not be created", + billedMicroCredits: 0, + }); + expect(ledger()).toHaveLength(0); + }); + + it("does nothing for a run that no longer owns the transcript", async () => { + const row = transcript(); + if (row) row.runId = "run-2"; + + expect(await fast()).toEqual({ resume: "unclaimed" }); + expect(media.convertCalls).toBe(0); + }); +}); diff --git a/apps/web/__tests__/unit/developer-transcribe-no-speech.test.ts b/apps/web/__tests__/unit/developer-transcribe-no-speech.test.ts index 7b4205a58fc..6802a2cff5a 100644 --- a/apps/web/__tests__/unit/developer-transcribe-no-speech.test.ts +++ b/apps/web/__tests__/unit/developer-transcribe-no-speech.test.ts @@ -338,7 +338,7 @@ describe("audio with no speech", () => { resultKey: RESULT_KEY, }); - expect(again.message).toBe("Transcript is not waiting to run"); + expect(again.message).toBe("Transcript completed"); expect(resettled).toEqual({ state: "complete", chargedMicroCredits: 0, diff --git a/apps/web/__tests__/unit/developer-transcribe-video.test.ts b/apps/web/__tests__/unit/developer-transcribe-video.test.ts new file mode 100644 index 00000000000..b65db219cad --- /dev/null +++ b/apps/web/__tests__/unit/developer-transcribe-video.test.ts @@ -0,0 +1,387 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const context = vi.hoisted(() => ({ + fake: null as null | { + db: unknown; + rows: (table: object) => Record[]; + seed: ( + table: object, + row: Record, + ) => Record; + reset: () => void; + }, + objects: new Map(), + ledgerIds: 0, + transcribe: vi.fn(), + fetched: [] as string[], + convertBodies: [] as string[], +})); + +vi.mock("@cap/database", async () => { + const schema = await import("@cap/database/schema"); + const { createFakeDatabase } = await import("./developer-transcript-fake-db"); + const fake = createFakeDatabase( + new Map Record>([ + [ + schema.developerTranscripts, + () => ({ + status: "queued", + sourceType: "video", + sourceUrl: null, + videoId: null, + s3Key: null, + requestedLanguage: null, + durationSeconds: null, + language: null, + text: null, + wordCount: null, + resultKey: null, + reservedMicroCredits: 0, + billedMicroCredits: 0, + errorCode: null, + errorMessage: null, + deletedAt: null, + }), + ], + [ + schema.developerVideos, + () => ({ + name: "Untitled", + duration: 30, + status: "ready", + processing: null, + deletedAt: null, + filesDeletedAt: null, + }), + ], + ]), + ); + context.fake = fake; + return { db: () => fake.db }; +}); + +vi.mock("@cap/database/helpers", () => ({ + nanoId: () => { + context.ledgerIds += 1; + return `ledger-${context.ledgerIds}`; + }, + nanoIdLength: 15, +})); + +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ + MEDIA_SERVER_URL: "http://media.test", + MEDIA_SERVER_WEBHOOK_SECRET: "secret", + }), + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + NODE_ENV: "test", +})); + +vi.mock("@/lib/developer-transcript-storage", () => ({ + developerObjectSize: async (key: string) => + context.objects.get(key)?.length ?? null, + developerObjectUrl: async (key: string) => `http://storage.test/${key}`, + getDeveloperObjectText: async (key: string) => { + const bytes = context.objects.get(key); + return bytes ? new TextDecoder().decode(bytes) : null; + }, + putDeveloperObject: async (key: string, body: string | Uint8Array) => { + context.objects.set( + key, + typeof body === "string" ? new TextEncoder().encode(body) : body, + ); + }, + putDeveloperObjectFile: async () => { + throw new Error("A video transcript must not copy its source"); + }, + putDeveloperObjectStream: async () => { + throw new Error("A video transcript must not copy its source"); + }, + removeDeveloperObject: async (key: string) => { + context.objects.delete(key); + }, + deleteDeveloperObjects: async (keys: string[]) => { + for (const key of keys) context.objects.delete(key); + }, +})); + +vi.mock("@/lib/developer-transcript-speech", () => ({ + transcribeSpeech: context.transcribe, + parseTranscriptLanguage: () => "auto", +})); + +import { + developerCreditAccounts, + developerCreditTransactions, + developerTranscripts, + developerVideos, +} from "@cap/database/schema"; +import { transcriptionMicroCredits } from "@/lib/developer-pricing"; +import { developerVideoCaptions } from "@/lib/developer-video-captions"; +import { developerTranscribeWorkflow } from "@/workflows/developer-transcribe"; +import { processingState } from "../fixtures/developer-video"; + +const MP4_KEY = "developer/app-1/video-1/encode-1/out-1/video.mp4"; +const UPLOAD_KEY = "developer/_uploads/app-1/t-tr-1"; +const RESULT_KEY = "developer/app-1/transcripts/tr-1/result.json"; +const CAPTIONS_KEY = "developer/app-1/video-1/captions/en.vtt"; +const FRAME_SECONDS = 1152 / 44_100; + +function mp3Frames(count: number) { + const length = Math.floor((144_000 * 128) / 44_100); + const out = new Uint8Array(length * count); + for (let index = 0; index < count; index++) { + out.set([0xff, 0xfb, 9 << 4, 0x00], index * length); + } + return out; +} + +const mp4Bytes = (size = 4_000_000) => { + const bytes = new Uint8Array(size); + bytes.set(new TextEncoder().encode("ftypisom"), 4); + return bytes; +}; + +const fake = () => { + if (!context.fake) throw new Error("Fake database is not ready"); + return context.fake; +}; +const transcript = () => fake().rows(developerTranscripts)[0]; +const video = () => fake().rows(developerVideos)[0]; +const balance = () => + fake().rows(developerCreditAccounts)[0]?.balanceMicroCredits; +const ledger = () => fake().rows(developerCreditTransactions); +const run = () => + developerTranscribeWorkflow({ transcriptId: "tr-1", runId: "run-1" }); + +const speech = (text: string) => ({ + state: "ok", + text, + words: text.split(" ").map((word, index) => ({ + text: word, + start: index * 500, + end: index * 500 + 400, + confidence: 0.9, + })), + language: "en", + durationSeconds: 30, +}); + +beforeEach(() => { + fake().reset(); + context.objects = new Map([[MP4_KEY, mp4Bytes()]]); + context.ledgerIds = 0; + context.fetched = []; + context.convertBodies = []; + context.transcribe.mockReset(); + context.transcribe.mockResolvedValue(speech("hello from the video")); + fake().seed(developerCreditAccounts, { + id: "account-1", + appId: "app-1", + balanceMicroCredits: 100_000, + }); + fake().seed(developerVideos, { + id: "video-1", + appId: "app-1", + playbackId: "playback-id-0123456789ab", + processing: processingState({ + hls: { state: "complete", output: "out-1" }, + mp4: { state: "complete", output: "out-1" }, + }), + }); + fake().seed(developerTranscripts, { + id: "tr-1", + appId: "app-1", + runId: "run-1", + videoId: "video-1", + s3Key: MP4_KEY, + }); + const audio = mp3Frames(Math.round(30 / FRAME_SECONDS)); + vi.stubGlobal( + "fetch", + async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + context.fetched.push(url); + if (url.startsWith("http://storage.test/")) { + const bytes = context.objects.get( + url.slice("http://storage.test/".length), + ); + return bytes + ? new Response(bytes.slice(0, 1_000_000), { status: 200 }) + : new Response(null, { status: 404 }); + } + if (url === "http://media.test/video/probe") { + return new Response("{}", { status: 500 }); + } + if (url === "http://media.test/audio/check") { + return Response.json({ hasAudio: true }); + } + if (url === "http://media.test/audio/convert") { + context.convertBodies.push(String(init?.body ?? "")); + return new Response(audio.slice(), { status: 200 }); + } + throw new Error(`Unexpected fetch ${url}`); + }, + ); +}); + +afterEach(() => vi.unstubAllGlobals()); + +describe("transcribing a stored video", () => { + it("reads the stored MP4, charges the transcript price and writes the caption file", async () => { + const result = await run(); + + const measured = Math.round(30 / FRAME_SECONDS) * FRAME_SECONDS; + const cost = transcriptionMicroCredits(measured); + expect(result.success).toBe(true); + expect(transcript()).toMatchObject({ + status: "complete", + text: "hello from the video", + wordCount: 4, + language: "en", + billedMicroCredits: cost, + reservedMicroCredits: 0, + sourceBytes: 4_000_000, + }); + expect(balance()).toBe(100_000 - cost); + expect(ledger()).toHaveLength(1); + expect(ledger()[0]).toMatchObject({ type: "transcription" }); + + expect(context.convertBodies).toHaveLength(1); + expect(context.convertBodies[0]).toContain( + `http://storage.test/${MP4_KEY}`, + ); + expect( + context.fetched.every( + (url) => + url.startsWith("http://storage.test/") || + url.startsWith("http://media.test/"), + ), + ).toBe(true); + expect(context.objects.has(UPLOAD_KEY)).toBe(false); + + expect([...context.objects.keys()].sort()).toEqual( + [MP4_KEY, RESULT_KEY, CAPTIONS_KEY].sort(), + ); + const captions = new TextDecoder().decode( + context.objects.get(CAPTIONS_KEY), + ); + expect(captions).toBe( + "WEBVTT\n\n00:00:00.000 --> 00:00:01.900\nhello from the video\n", + ); + expect( + await developerVideoCaptions({ id: "video-1", appId: "app-1" }), + ).toEqual([{ language: "en", label: "English", key: CAPTIONS_KEY }]); + }); + + it("holds for the video's known length, not for its file size", async () => { + let held = 0; + context.transcribe.mockImplementation(async () => { + held = Number(transcript()?.reservedMicroCredits); + return speech("hello"); + }); + const row = video(); + if (row) row.duration = 3600; + context.objects.set(MP4_KEY, mp4Bytes(8_000_000)); + + await run(); + + expect(transcript()?.status).toBe("complete"); + expect(held).toBeGreaterThan(0); + expect(held).toBeLessThanOrEqual(transcriptionMicroCredits(3602)); + }); + + it("writes no caption file when the video has no speech and still bills it", async () => { + context.transcribe.mockResolvedValue({ state: "no_speech" }); + + const result = await run(); + + expect(result.success).toBe(true); + expect(transcript()).toMatchObject({ + status: "complete", + text: "", + wordCount: 0, + }); + expect(Number(transcript()?.billedMicroCredits)).toBeGreaterThan(0); + expect(context.objects.has(CAPTIONS_KEY)).toBe(false); + expect( + await developerVideoCaptions({ id: "video-1", appId: "app-1" }), + ).toEqual([]); + }); + + it("fails without a charge when the video was deleted before the transcript ran", async () => { + const row = video(); + if (row) row.deletedAt = new Date("2026-11-12T09:00:00.000Z"); + + const result = await run(); + + expect(result.success).toBe(false); + expect(transcript()).toMatchObject({ + status: "error", + errorCode: "source_missing", + errorMessage: "The video was deleted before it could be transcribed", + billedMicroCredits: 0, + reservedMicroCredits: 0, + }); + expect(ledger()).toHaveLength(0); + expect(context.transcribe).not.toHaveBeenCalled(); + expect(context.objects.has(MP4_KEY)).toBe(true); + expect(context.objects.has(CAPTIONS_KEY)).toBe(false); + }); + + it("fails without a charge when the stored file is gone", async () => { + context.objects.delete(MP4_KEY); + + await run(); + + expect(transcript()).toMatchObject({ + status: "error", + errorCode: "source_missing", + billedMicroCredits: 0, + }); + expect(context.convertBodies).toHaveLength(0); + }); + + it("never deletes the video's file when the transcript fails", async () => { + context.transcribe.mockRejectedValue(new Error("speech is down")); + + await expect(run()).rejects.toThrow("speech is down"); + + expect(transcript()).toMatchObject({ + status: "error", + errorCode: "transcription_failed", + }); + expect(context.objects.has(MP4_KEY)).toBe(true); + expect(context.objects.has(CAPTIONS_KEY)).toBe(false); + }); + + it("fails with insufficient credits when the hold cannot be covered, and writes nothing", async () => { + const account = fake().rows(developerCreditAccounts)[0]; + if (account) account.balanceMicroCredits = 0; + + await run(); + + expect(transcript()).toMatchObject({ + status: "error", + errorCode: "insufficient_credits", + }); + expect(context.convertBodies).toHaveLength(0); + expect(context.objects.has(CAPTIONS_KEY)).toBe(false); + }); + + it("leaves a transcript that is not from a video without a caption file", async () => { + const row = transcript(); + if (row) { + row.sourceType = "upload"; + row.videoId = null; + } + context.objects.set(UPLOAD_KEY, mp4Bytes(64_000)); + + await run(); + + expect(transcript()?.status).toBe("complete"); + expect( + [...context.objects.keys()].some((key) => key.includes("/captions/")), + ).toBe(false); + }); +}); diff --git a/apps/web/__tests__/unit/developer-transcribe-workflow.test.ts b/apps/web/__tests__/unit/developer-transcribe-workflow.test.ts index 0ec8870f810..7a08db28fbc 100644 --- a/apps/web/__tests__/unit/developer-transcribe-workflow.test.ts +++ b/apps/web/__tests__/unit/developer-transcribe-workflow.test.ts @@ -266,7 +266,8 @@ describe("developer transcribe workflow", () => { expect([...context.objects.keys()]).toEqual([RESULT_KEY]); const again = await run(); - expect(again.message).toBe("Transcript is not waiting to run"); + expect(again.message).toBe("Transcript completed"); + expect([...context.objects.keys()]).toEqual([RESULT_KEY]); expect(ledger()).toHaveLength(1); expect(context.transcribe).toHaveBeenCalledTimes(1); }); @@ -370,4 +371,201 @@ describe("developer transcribe workflow", () => { expect(context.objects.has(SOURCE_KEY)).toBe(true); expect(context.objects.has(AUDIO_KEY)).toBe(false); }); + + describe("a transcript deleted while it runs", () => { + const deletedAt = new Date("2026-11-12T10:00:00.000Z"); + const thirtySeconds = () => { + const frames = Math.round(30 / FRAME_SECONDS); + media.convert = () => + new Response(streamOf(mp3Frames(frames), 1), { status: 200 }); + return transcriptionMicroCredits(frames * FRAME_SECONDS); + }; + const deleteNow = () => { + const row = transcript(); + if (row) { + row.deletedAt = deletedAt; + row.text = null; + } + for (const key of [SOURCE_KEY, AUDIO_KEY, RESULT_KEY]) { + context.objects.delete(key); + } + }; + const speech = { + state: "ok", + text: "hello world", + words: [ + { text: "hello", start: 0, end: 400, confidence: 0.9 }, + { text: "world", start: 450, end: 900, confidence: 0.9 }, + ], + language: "en", + durationSeconds: 30, + }; + + it("is still charged for its measured length when deleted during the speech call", async () => { + const cost = thirtySeconds(); + context.transcribe.mockImplementation(async () => { + deleteNow(); + return speech; + }); + + const result = await run(); + + expect(result.success).toBe(true); + expect(context.transcribe).toHaveBeenCalledTimes(1); + expect(transcript()).toMatchObject({ + status: "complete", + deletedAt, + billedMicroCredits: cost, + reservedMicroCredits: 0, + text: null, + resultKey: null, + }); + expect(balance()).toBe(100_000 - cost); + expect(ledger()).toHaveLength(1); + expect(ledger()[0]).toMatchObject({ + type: "transcription", + amountMicroCredits: -cost, + referenceId: "tr-1", + }); + expect([...context.objects.keys()]).toEqual([]); + }); + + it("keeps its credit held from the delete until it is charged", async () => { + const cost = thirtySeconds(); + let heldAfterDelete = -1; + context.transcribe.mockImplementation(async () => { + deleteNow(); + heldAfterDelete = Number(transcript()?.reservedMicroCredits); + return speech; + }); + + await run(); + + expect(heldAfterDelete).toBe(cost); + expect(transcript()?.reservedMicroCredits).toBe(0); + }); + + it("costs nothing when deleted after the audio was extracted and held for but before it was sent", async () => { + thirtySeconds(); + const storage = await import("@/lib/developer-transcript-storage"); + let reserved = 0; + const read = vi + .spyOn(storage, "getDeveloperObjectText") + .mockImplementation(async (key: string) => { + const bytes = context.objects.get(key); + if (key === RESULT_KEY && !bytes) { + reserved = Number(transcript()?.reservedMicroCredits); + deleteNow(); + } + return bytes ? new TextDecoder().decode(bytes) : null; + }); + + const result = await run(); + read.mockRestore(); + + expect(reserved).toBeGreaterThan(0); + + expect(result.success).toBe(false); + expect(context.transcribe).not.toHaveBeenCalled(); + expect(transcript()).toMatchObject({ + status: "error", + deletedAt, + billedMicroCredits: 0, + reservedMicroCredits: 0, + }); + expect(balance()).toBe(100_000); + expect(ledger()).toHaveLength(0); + expect([...context.objects.keys()]).toEqual([]); + }); + + it("costs nothing when deleted while the audio is being extracted", async () => { + const frames = Math.round(30 / FRAME_SECONDS); + media.convert = () => { + deleteNow(); + return new Response(streamOf(mp3Frames(frames), 1), { status: 200 }); + }; + + const result = await run(); + + expect(result.success).toBe(false); + expect(context.transcribe).not.toHaveBeenCalled(); + expect(transcript()).toMatchObject({ + status: "error", + billedMicroCredits: 0, + reservedMicroCredits: 0, + }); + expect(balance()).toBe(100_000); + expect(ledger()).toHaveLength(0); + }); + + it("costs nothing when deleted while queued", async () => { + thirtySeconds(); + deleteNow(); + + const result = await run(); + + expect(result.message).toBe("Transcript is not waiting to run"); + expect(media.convertCalls).toBe(0); + expect(context.transcribe).not.toHaveBeenCalled(); + expect(transcript()).toMatchObject({ + status: "queued", + billedMicroCredits: 0, + }); + expect(ledger()).toHaveLength(0); + }); + + it("costs nothing when the speech call fails after the delete", async () => { + thirtySeconds(); + context.transcribe.mockImplementation(async () => { + deleteNow(); + throw new Error("speech is down"); + }); + + const result = await run(); + + expect(result.success).toBe(false); + expect(context.transcribe).toHaveBeenCalledTimes(1); + expect(transcript()).toMatchObject({ + status: "error", + errorCode: "source_missing", + billedMicroCredits: 0, + reservedMicroCredits: 0, + }); + expect(balance()).toBe(100_000); + expect(ledger()).toHaveLength(0); + }); + + it("is charged when deleted between the speech call and the settle", async () => { + const cost = thirtySeconds(); + context.transcribe.mockResolvedValue(speech); + let resultReads = 0; + const storage = await import("@/lib/developer-transcript-storage"); + const read = vi + .spyOn(storage, "getDeveloperObjectText") + .mockImplementation(async (key: string) => { + const bytes = context.objects.get(key); + if (key === RESULT_KEY && bytes) { + resultReads += 1; + const text = new TextDecoder().decode(bytes); + deleteNow(); + return text; + } + return bytes ? new TextDecoder().decode(bytes) : null; + }); + + const result = await run(); + read.mockRestore(); + + expect(resultReads).toBe(1); + expect(result.success).toBe(true); + expect(transcript()).toMatchObject({ + status: "complete", + deletedAt, + billedMicroCredits: cost, + text: null, + }); + expect(ledger()).toHaveLength(1); + expect([...context.objects.keys()]).toEqual([]); + }); + }); }); diff --git a/apps/web/__tests__/unit/developer-transcript-concurrency.test.ts b/apps/web/__tests__/unit/developer-transcript-concurrency.test.ts new file mode 100644 index 00000000000..41c8ac33d33 --- /dev/null +++ b/apps/web/__tests__/unit/developer-transcript-concurrency.test.ts @@ -0,0 +1,412 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const context = vi.hoisted(() => ({ + fake: null as null | { + db: unknown; + rows: (table: object) => Record[]; + seed: ( + table: object, + row: Record, + ) => Record; + reset: () => void; + }, + ids: 0, + speechConfigured: true, + hosted: true, + start: vi.fn(), + objectSize: null as number | null, + deleted: [] as string[], + presigned: [] as { key: string; contentLength: number }[], + failSigning: false, + onDelete: null as null | (() => Promise), +})); + +vi.mock("@cap/database", async () => { + const schema = await import("@cap/database/schema"); + const { createFakeDatabase } = await import("./developer-transcript-fake-db"); + const fake = createFakeDatabase( + new Map Record>([ + [ + schema.developerTranscripts, + () => ({ + externalUserId: null, + name: "Untitled", + sourceUrl: null, + resultKey: null, + sourceBytes: null, + durationSeconds: null, + requestedLanguage: null, + language: null, + text: null, + wordCount: null, + reservedMicroCredits: 0, + billedMicroCredits: 0, + errorCode: null, + errorMessage: null, + metadata: null, + runId: null, + queuedAt: null, + startedAt: null, + completedAt: null, + deletedAt: null, + }), + ], + ]), + ); + context.fake = fake; + return { db: () => fake.db }; +}); + +vi.mock("@cap/database/helpers", () => ({ + nanoId: () => { + context.ids += 1; + return `generated-${context.ids}`; + }, + nanoIdLength: 15, +})); + +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + NODE_ENV: "production", +})); + +vi.mock("@/lib/infra-access", () => ({ + canUseInfra: (email: string | null | undefined) => + !context.hosted || email === "richie@cap.so", +})); + +vi.mock("workflow/api", () => ({ start: context.start })); + +vi.mock("@/workflows/developer-transcribe", () => ({ + developerTranscribeWorkflow: "developer-transcribe-workflow", +})); + +vi.mock("@/lib/developer-transcript-speech", () => ({ + isSpeechConfigured: () => context.speechConfigured, + parseTranscriptLanguage: (value: string | null | undefined) => { + if (!value || value === "auto") return "auto"; + return ["en", "de"].includes(value) ? value : null; + }, +})); + +vi.mock("@/lib/developer-transcript-storage", () => ({ + presignDeveloperUpload: async (key: string, contentLength: number) => { + if (context.failSigning) throw new Error("signing unavailable"); + context.presigned.push({ key, contentLength }); + return `https://storage.example/${key}?signed=${context.presigned.length}`; + }, + developerObjectSize: async () => context.objectSize, + getDeveloperObjectText: async () => null, + deleteDeveloperObjects: async (keys: string[]) => { + context.deleted.push(...keys); + const hook = context.onDelete; + context.onDelete = null; + await hook?.(); + }, +})); + +import { + developerApps, + developerCreditAccounts, + developerTranscripts, + users, +} from "@cap/database/schema"; +import { + createTranscriptFromUrl, + createTranscriptUpload, + deleteTranscript, + startTranscriptUpload, +} from "@/lib/developer-transcript-store"; +import { TRANSCRIPT_MAX_CONCURRENT } from "@/lib/developer-transcripts"; +import { isBlockedS3Endpoint } from "@/lib/s3-endpoint-policy"; + +const URL_A = "https://media.example.com/standup.mp3"; + +const dns = + (table: Record) => + (url: string): Promise => + isBlockedS3Endpoint(url, async (hostname) => table[hostname] ?? []); + +const guards = { + isBlocked: dns({ + "media.example.com": ["93.184.216.34"], + "rebind.example.com": ["10.0.0.9"], + }), +}; + +const fake = () => { + if (!context.fake) throw new Error("Fake database is not ready"); + return context.fake; +}; + +const transcripts = () => fake().rows(developerTranscripts); +const setBalance = (balanceMicroCredits: number) => { + const account = fake().rows(developerCreditAccounts)[0]; + if (account) account.balanceMicroCredits = balanceMicroCredits; +}; + +const fromUrl = (url = URL_A, idempotencyKey?: string, appId = "app-1") => + createTranscriptFromUrl(appId, { url, idempotencyKey }, guards); + +beforeEach(() => { + fake().reset(); + context.ids = 0; + context.speechConfigured = true; + context.hosted = true; + context.start.mockReset(); + context.start.mockResolvedValue(undefined); + context.objectSize = null; + context.deleted = []; + context.presigned = []; + context.failSigning = false; + context.onDelete = null; + fake().seed(developerCreditAccounts, { + id: "account-1", + appId: "app-1", + balanceMicroCredits: 100_000, + }); + fake().seed(users, { id: "owner-1", email: "richie@cap.so" }); + fake().seed(users, { id: "owner-9", email: "someone@example.com" }); + for (const id of ["app-1", "app-2", "app-without-credits"]) { + fake().seed(developerApps, { id, ownerId: "owner-1", deletedAt: null }); + } + fake().seed(developerApps, { + id: "app-outside-preview", + ownerId: "owner-9", + deletedAt: null, + }); + fake().seed(developerCreditAccounts, { + id: "account-9", + appId: "app-outside-preview", + balanceMicroCredits: 100_000, + }); +}); + +const CEILING = { + ok: false, + status: 429, + code: "too_many_transcripts", + message: "This app already has 20 transcripts queued or processing", +}; + +const seedRunning = ( + count: number, + row: Record = {}, + prefix = "running", +) => { + for (let index = 0; index < count; index++) { + fake().seed(developerTranscripts, { + id: `${prefix}-${index}`, + appId: "app-1", + status: "processing", + sourceType: "upload", + ...row, + }); + } +}; + +const waitingUpload = async () => { + const created = await createTranscriptUpload("app-1", { + fileName: "a.mp3", + contentLength: 1024, + }); + if (!created.ok) throw new Error("create failed"); + context.objectSize = 1024; + return created.transcript.id; +}; + +describe("the ceiling on transcripts running at once", () => { + it("is 20", () => { + expect(TRANSCRIPT_MAX_CONCURRENT).toBe(20); + }); + + it("refuses a URL transcript once 20 are queued or processing, without storing or starting it", async () => { + seedRunning(12); + seedRunning(8, { status: "queued" }, "queued"); + + const result = await fromUrl(); + + expect(result).toMatchObject(CEILING); + expect(transcripts()).toHaveLength(20); + expect(context.start).not.toHaveBeenCalled(); + }); + + it("admits the twentieth and refuses the twenty-first", async () => { + seedRunning(19); + + expect((await fromUrl()).ok).toBe(true); + expect(await fromUrl(URL_A)).toMatchObject(CEILING); + expect(context.start).toHaveBeenCalledTimes(1); + }); + + it("admits exactly 20 of 30 created at the same moment", async () => { + const results = await Promise.all( + Array.from({ length: 30 }, (_, index) => + fromUrl(`https://media.example.com/clip-${index}.mp3`), + ), + ); + + expect(results.filter((result) => result.ok)).toHaveLength(20); + expect( + results.filter((result) => !result.ok && result.status === 429), + ).toHaveLength(10); + expect(transcripts()).toHaveLength(20); + expect(context.start).toHaveBeenCalledTimes(20); + }); + + it("does not count finished, failed, waiting or other apps' transcripts", async () => { + seedRunning(30, { status: "complete" }, "complete"); + seedRunning(30, { status: "error" }, "error"); + seedRunning(30, { status: "uploading" }, "uploading"); + seedRunning(30, { appId: "app-2" }, "other"); + + expect((await fromUrl()).ok).toBe(true); + }); + + it("frees a slot when a queued transcript is deleted, and keeps it for one deleted while processing", async () => { + seedRunning(20, { status: "queued" }, "queued"); + expect(await fromUrl()).toMatchObject(CEILING); + + await deleteTranscript("app-1", "queued-0"); + expect((await fromUrl()).ok).toBe(true); + + fake().reset(); + fake().seed(developerCreditAccounts, { + id: "account-1", + appId: "app-1", + balanceMicroCredits: 100_000, + }); + fake().seed(users, { id: "owner-1", email: "richie@cap.so" }); + fake().seed(developerApps, { + id: "app-1", + ownerId: "owner-1", + deletedAt: null, + }); + seedRunning(20); + await deleteTranscript("app-1", "running-0"); + expect(await fromUrl()).toMatchObject(CEILING); + }); + + it("frees a slot when a transcript finishes or fails", async () => { + seedRunning(20); + expect(await fromUrl()).toMatchObject(CEILING); + + const first = transcripts().find((row) => row.id === "running-0"); + if (first) first.status = "complete"; + expect((await fromUrl()).ok).toBe(true); + + const second = transcripts().find((row) => row.id === "running-1"); + if (second) second.status = "error"; + expect((await fromUrl(URL_A)).ok).toBe(true); + }); + + it("still returns the same transcript for a repeated request at the ceiling", async () => { + seedRunning(19); + const first = await fromUrl(URL_A, "retry-key-1"); + if (!first.ok) throw new Error("create failed"); + + const again = await fromUrl(URL_A, "retry-key-1"); + + expect(again).toMatchObject({ + ok: true, + transcript: { id: first.transcript.id }, + }); + expect(context.start).toHaveBeenCalledTimes(1); + }); + + it("lets a file be created at the ceiling but refuses to start it", async () => { + seedRunning(20); + + const id = await waitingUpload(); + const refused = await startTranscriptUpload("app-1", id); + + expect(refused).toMatchObject(CEILING); + expect(transcripts().find((row) => row.id === id)).toMatchObject({ + status: "uploading", + runId: null, + startedAt: null, + }); + expect(context.start).not.toHaveBeenCalled(); + + const running = transcripts().find((row) => row.id === "running-0"); + if (running) running.status = "complete"; + expect((await startTranscriptUpload("app-1", id)).ok).toBe(true); + expect(context.start).toHaveBeenCalledTimes(1); + }); + + it("starts exactly as many uploads as there are slots when they are started together", async () => { + seedRunning(17); + const ids = []; + for (let index = 0; index < 6; index++) ids.push(await waitingUpload()); + + const results = await Promise.all( + ids.map((id) => startTranscriptUpload("app-1", id)), + ); + + expect(results.filter((result) => result.ok)).toHaveLength(3); + expect( + results.filter((result) => !result.ok && result.status === 429), + ).toHaveLength(3); + expect(context.start).toHaveBeenCalledTimes(3); + }); + + it("returns a transcript that already started unchanged at the ceiling", async () => { + seedRunning(19); + const id = await waitingUpload(); + expect((await startTranscriptUpload("app-1", id)).ok).toBe(true); + + expect(await startTranscriptUpload("app-1", id)).toMatchObject({ + ok: true, + transcript: { id, status: "queued" }, + }); + }); + + it("answers the credit refusal before the ceiling", async () => { + seedRunning(20); + setBalance(0); + + expect(await fromUrl()).toMatchObject({ ok: false, status: 402 }); + }); + + it("admits both copies of a repeated request that race each other at the last slot", async () => { + seedRunning(19); + + const [first, second] = await Promise.all([ + fromUrl(URL_A, "race-key-01"), + fromUrl(URL_A, "race-key-01"), + ]); + + expect(first.ok && second.ok).toBe(true); + if (!first.ok || !second.ok) throw new Error("create failed"); + expect(second.transcript.id).toBe(first.transcript.id); + expect(transcripts()).toHaveLength(20); + expect(context.start).toHaveBeenCalledTimes(1); + }); + + it("still refuses a different request that races for the last slot", async () => { + seedRunning(19); + + const results = await Promise.all([ + fromUrl(URL_A, "race-key-01"), + fromUrl(URL_A, "race-key-02"), + ]); + + expect(results.filter((result) => result.ok)).toHaveLength(1); + expect(results.filter((result) => !result.ok)).toEqual([ + expect.objectContaining(CEILING), + ]); + }); + + it("answers both copies of a start sent twice at the last slot, and starts it once", async () => { + seedRunning(19); + const id = await waitingUpload(); + + const results = await Promise.all([ + startTranscriptUpload("app-1", id), + startTranscriptUpload("app-1", id), + ]); + + expect(results.every((result) => result.ok)).toBe(true); + expect(context.start).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/web/__tests__/unit/developer-transcript-console-ceiling.test.ts b/apps/web/__tests__/unit/developer-transcript-console-ceiling.test.ts new file mode 100644 index 00000000000..7bfd42763e7 --- /dev/null +++ b/apps/web/__tests__/unit/developer-transcript-console-ceiling.test.ts @@ -0,0 +1,51 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + start: vi.fn(), + create: vi.fn(), +})); + +vi.mock("@cap/database", () => { + const chain = Object.assign(Promise.resolve([{ id: "app-1" }]), { + from: () => chain, + where: () => chain, + limit: () => chain, + }); + return { db: () => ({ select: () => chain }) }; +}); +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: async () => ({ id: "user-1", email: "owner@example.com" }), +})); +vi.mock("@/lib/infra-access", () => ({ canUseInfra: () => true })); +vi.mock("@/lib/developer-transcript-store", () => ({ + startTranscriptUpload: mocks.start, + createTranscriptUpload: mocks.create, + deleteTranscript: vi.fn(), + getTranscript: vi.fn(), + listConsoleTranscripts: vi.fn(), +})); + +import { startAudioTranscript } from "@/actions/developers/transcripts"; + +beforeEach(() => { + mocks.start.mockReset(); +}); + +describe("the transcript ceiling in the console", () => { + it("passes the refusal and its wording to the page when a start is refused", async () => { + mocks.start.mockResolvedValue({ + ok: false, + status: 429, + code: "too_many_transcripts", + message: "This app already has 20 transcripts queued or processing", + hint: "Wait for one to finish, then send the request again.", + }); + + expect(await startAudioTranscript("app-1", "tr-1")).toEqual({ + ok: false, + code: "too_many_transcripts", + message: "This app already has 20 transcripts queued or processing", + }); + expect(mocks.start).toHaveBeenCalledWith("app-1", "tr-1"); + }); +}); diff --git a/apps/web/__tests__/unit/developer-transcript-docs.test.ts b/apps/web/__tests__/unit/developer-transcript-docs.test.ts index 56a92b29034..76e5a4b357d 100644 --- a/apps/web/__tests__/unit/developer-transcript-docs.test.ts +++ b/apps/web/__tests__/unit/developer-transcript-docs.test.ts @@ -7,6 +7,10 @@ import { TRANSCRIPT_FAILURE_CODES, TRANSCRIPT_REQUEST_ERRORS, } from "@/lib/developer-transcript-errors"; +import { + TRANSCRIPT_BUSY_RETRY_SECONDS, + TRANSCRIPT_MAX_CONCURRENT, +} from "@/lib/developer-transcripts"; const read = (path: string) => readFileSync(join(process.cwd(), path), "utf8"); @@ -26,6 +30,7 @@ const REQUEST_SOURCES = [ "lib/developer-transcript-store.ts", "lib/developer-transcript-billing.ts", "app/api/developer/v1/[...route]/transcripts.ts", + "lib/developer-ids.ts", ]; const codesIn = (source: string) => { @@ -133,6 +138,108 @@ describe("the language parameter", () => { }); }); +describe("transcribing a stored video", () => { + const request = spec.components.schemas.CreateTranscriptRequest as { + oneOf?: { required: string[] }[]; + required?: string[]; + properties: Record; + }; + const transcript = spec.components.schemas.Transcript as { + properties: { + videoId?: { type: string[] }; + source: { properties: { type: { enum: string[] } } }; + }; + }; + + it("accepts videoId or url on Create Transcript in the OpenAPI document", () => { + expect(Object.keys(request.properties)).toEqual( + expect.arrayContaining(["url", "videoId"]), + ); + expect(request.required).toBeUndefined(); + expect(request.oneOf).toEqual([ + { required: ["url"] }, + { required: ["videoId"] }, + ]); + }); + + it("describes the videoId field and the video source type on the transcript", () => { + expect(transcript.properties.videoId?.type).toEqual(["string", "null"]); + expect(transcript.properties.source.properties.type.enum).toEqual([ + "upload", + "url", + "video", + ]); + }); + + it("documents videoId, what it costs and that captions come from it", () => { + expect(transcriptDocs).toContain( + "| `videoId` | string | One of `url` or `videoId` |", + ); + expect(transcriptDocs).toContain( + '"source": { "type": "video", "url": null }', + ); + expect(transcriptDocs).toContain('"videoId": "abc123def456"'); + expect(transcriptDocs).toContain( + "It costs the transcript price only, $0.10 per audio hour.", + ); + expect(transcriptDocs).toContain( + "When a transcript of a video completes with speech, Cap writes a WebVTT captions file for the video in that language, from the newest completed transcript of the video in that language.", + ); + expect(transcriptDocs).toContain("`videoId`, `contentLength`, `name`"); + }); + + it("says only what ships: captions in the hosted player and no captions URL", () => { + for (const text of [transcriptDocs, specText]) { + expect(text).not.toMatch(/\.vtt|captions\//i); + } + expect(transcriptDocs).toContain( + "The hosted player and the embed then show a captions button, and the `c` key turns captions on and off. Add `captions=1` to `embedUrl` to start with them on.", + ); + expect(specText).toContain("controls=0 and captions=1"); + }); +}); + +describe("limits and deletes that cost money", () => { + it("states the ceiling on running transcripts and how long to wait, from the code", () => { + expect(TRANSCRIPT_MAX_CONCURRENT).toBe(20); + expect(transcriptDocs).toContain( + `An app can have ${TRANSCRIPT_MAX_CONCURRENT} transcripts \`queued\` or \`processing\` at once.`, + ); + expect(transcriptDocs).toContain( + `a \`Retry-After\` header of ${TRANSCRIPT_BUSY_RETRY_SECONDS} seconds`, + ); + expect(transcriptDocs).toContain("| `429` | `too_many_transcripts` |"); + for (const operation of ["createTranscript", "startTranscript"]) { + const described = Object.values(spec.paths) + .flatMap((path) => Object.values(path as Record)) + .find( + (entry) => + (entry as { operationId?: string }).operationId === operation, + ) as { description: string }; + expect(described.description).toContain( + `${TRANSCRIPT_MAX_CONCURRENT} transcripts queued or processing at once`, + ); + } + }); + + it("says a transcript deleted after its audio was sent is still charged", () => { + expect(transcriptDocs).not.toContain( + "A transcript deleted before it completes is not charged", + ); + expect(specText).not.toContain( + "A transcript deleted before it completes is not charged", + ); + for (const text of [transcriptDocs, specText]) { + expect(text).toContain( + "A transcript deleted before its audio is sent to speech recognition is not charged.", + ); + expect(text).toContain( + "One deleted after that is still charged for the length of its audio when it finishes", + ); + } + }); +}); + describe("transcript docs keep up with the API", () => { it("states the characters an Idempotency-Key may contain", () => { expect(transcriptDocs).toContain( @@ -146,7 +253,8 @@ describe("transcript docs keep up with the API", () => { expect(JSON.stringify(schemas.TranscriptDetail)).not.toContain( "confidence", ); - expect(transcriptDocs).toContain("steps of 0.08 seconds"); + expect(transcriptDocs).not.toContain("steps of 0.08 seconds"); + expect(JSON.stringify(schemas.TranscriptDetail)).not.toContain("0.08"); }); it("states the download limit and the upload expiry", () => { diff --git a/apps/web/__tests__/unit/developer-transcript-fake-db.ts b/apps/web/__tests__/unit/developer-transcript-fake-db.ts index b08bda693f5..6abcc458282 100644 --- a/apps/web/__tests__/unit/developer-transcript-fake-db.ts +++ b/apps/web/__tests__/unit/developer-transcript-fake-db.ts @@ -7,6 +7,17 @@ type Table = object; const dialect = new MySqlDialect(); +const metadataOf = (value: unknown): Row => { + if (typeof value === "string") { + try { + return metadataOf(JSON.parse(value)); + } catch { + return {}; + } + } + return typeof value === "object" && value !== null ? (value as Row) : {}; +}; + const comparable = (value: unknown) => value instanceof Date ? value.toISOString().slice(0, -1).replace("T", " ") @@ -36,7 +47,7 @@ const compare = (left: unknown, operator: string, right: unknown) => { } }; -function compile(condition: SQL | undefined): Predicate { +function compile(condition: SQL | undefined, outer: Row = {}): Predicate { if (!condition) return () => true; const query = dialect.sqlToQuery(condition); const tokens = @@ -82,8 +93,73 @@ function compile(condition: SQL | undefined): Predicate { return (row) => row[column] === value; }; + const jsonField = () => { + expect("json_extract"); + expect("("); + const column = columnName(); + expect(","); + const field = take(); + expect(")"); + return (row: Row) => metadataOf(row[column])[field]; + }; + + const jsonComparison = (): Predicate => { + if (peek() === "json_unquote") { + take(); + expect("("); + const field = jsonField(); + expect(")"); + expect("="); + const value = param(); + return (row) => { + const found = field(row); + return found !== null && found !== undefined && String(found) === value; + }; + } + const field = jsonField(); + expect("is"); + const negated = peek() === "not"; + if (negated) take(); + expect("null"); + return (row) => + (field(row) === null || field(row) === undefined) !== negated; + }; + + const coalescedJsonComparison = (): Predicate => { + expect("coalesce"); + expect("("); + expect("cast"); + expect("("); + expect("json_extract"); + expect("("); + const column = columnName(); + expect(","); + const field = String(param()).replace(/^\$\./, ""); + expect(")"); + expect("as"); + expect("signed"); + expect(")"); + expect(","); + const fallback = columnName(); + expect(")"); + const operator = take(); + const value = param(); + return (row) => { + const found = metadataOf(row[column])[field]; + return compare( + found === null || found === undefined ? row[fallback] : Number(found), + operator, + value, + ); + }; + }; + const comparison = (): Predicate => { + if (peek() === "coalesce") return coalescedJsonComparison(); if (peek() === "cast") return binaryComparison(); + if (peek() === "json_unquote" || peek() === "json_extract") { + return jsonComparison(); + } const column = columnName(); const operator = take().toLowerCase(); if (operator === "is") { @@ -105,6 +181,10 @@ function compile(condition: SQL | undefined): Predicate { expect(")"); return (row) => values.some((value) => compare(row[column], "=", value)); } + if (peek()?.startsWith("`")) { + const reference = columnName(); + return (row) => compare(row[column], operator, outer[reference]); + } const value = param(); return (row) => compare(row[column], operator, value); }; @@ -159,7 +239,8 @@ const columnName = (value: unknown) => : null; const isAggregate = (value: unknown) => - isSql(value) && /SUM\(|count\(\*\)/i.test(dialect.sqlToQuery(value).sql); + isSql(value) && + /SUM\(|count\(\*\)|MAX\(GREATEST\(/i.test(dialect.sqlToQuery(value).sql); function rowExpression(row: Row, field: SQL) { const text = dialect.sqlToQuery(field).sql; @@ -190,6 +271,17 @@ function project(rows: Row[], fields: Record | undefined) { ); } else if (/count\(\*\)/i.test(text)) { result[alias] = rows.length; + } else if (/MAX\(GREATEST\(/i.test(text)) { + const values = rows.map((row) => { + const metadata = metadataOf(row.metadata); + const owed = Number(metadata.owedMicroCredits) || 0; + const charged = Number(metadata.chargedMicroCredits) || 0; + return Math.max( + charged < owed ? owed : 0, + Number(metadata.outstandingMicroCredits) || 0, + ); + }); + result[alias] = values.length > 0 ? Math.max(...values) : null; } else { throw new Error(`Unsupported aggregate ${text}`); } @@ -228,69 +320,107 @@ export function createFakeDatabase(defaults: Map Row>) { return new Date(clock); }; - const select = (fields?: Record) => ({ + const select = (fields?: Record, distinct = false) => ({ from: (table: Table) => { const state = { condition: undefined as SQL | undefined, ordered: false, + orderColumn: "createdAt", + ascending: false, limit: undefined as number | undefined, offset: 0, }; - const run = async () => { - let rows = rowsOf(table).filter(compile(state.condition)); + const run = async (outer: Row = {}) => { + let rows = rowsOf(table).filter(compile(state.condition, outer)); if (state.ordered) { + const time = (row: Row) => { + const value = row[state.orderColumn]; + return value instanceof Date ? value.getTime() : 0; + }; + const direction = state.ascending ? -1 : 1; rows = [...rows].sort((a, b) => { - const byDate = - (b.createdAt as Date).getTime() - (a.createdAt as Date).getTime(); - return byDate !== 0 - ? byDate - : String(b.id).localeCompare(String(a.id)); + const byDate = time(b) - time(a); + return ( + direction * + (byDate !== 0 ? byDate : String(b.id).localeCompare(String(a.id))) + ); }); } - const scalars: Row = {}; - for (const [alias, field] of Object.entries(fields ?? {})) { - const inner = isSql(field) - ? field.queryChunks.find(isQuery) - : undefined; - if (!inner) continue; - const [first] = await inner; - scalars[alias] = Object.values(first ?? {})[0] ?? null; - } + const subqueries = Object.entries(fields ?? {}).flatMap( + ([alias, field]) => { + const inner = isSql(field) + ? field.queryChunks.find(isQuery) + : undefined; + return inner ? [[alias, inner] as const] : []; + }, + ); + const scalarAliases = new Set(subqueries.map(([alias]) => alias)); const selected = fields ? Object.fromEntries( - Object.entries(fields).filter(([alias]) => !(alias in scalars)), + Object.entries(fields).filter( + ([alias]) => !scalarAliases.has(alias), + ), ) : fields; - const projected = project(rows, selected).map((row) => ({ - ...row, - ...scalars, - })); + const all: Row[] = []; + const base = project(rows, selected); + for (const [index, row] of base.entries()) { + const scalars: Row = {}; + for (const [alias, inner] of subqueries) { + const correlated = inner as PromiseLike & { + runFor?: (outer: Row) => Promise; + }; + const [first] = correlated.runFor + ? await correlated.runFor(rows[index] ?? {}) + : await inner; + scalars[alias] = Object.values(first ?? {})[0] ?? null; + } + all.push({ ...row, ...scalars }); + } + const seen = new Set(); + const projected = distinct + ? all.filter((row) => { + const key = JSON.stringify(row); + if (seen.has(key)) return false; + seen.add(key); + return true; + }) + : all; return projected.slice( state.offset, state.limit === undefined ? undefined : state.offset + state.limit, ); }; - const chain = Object.assign(Promise.resolve().then(run), { - where(condition: SQL | undefined) { - state.condition = condition; - return chain; - }, - orderBy() { - state.ordered = true; - return chain; - }, - limit(count: number) { - state.limit = count; - return chain; - }, - offset(count: number) { - state.offset = count; - return chain; - }, - for() { - return chain; + const chain = Object.assign( + Promise.resolve().then(() => run()), + { + runFor: (outer: Row) => run(outer), + where(condition: SQL | undefined) { + state.condition = condition; + return chain; + }, + orderBy(first?: SQL) { + state.ordered = true; + const named = first + ? dialect.sqlToQuery(first).sql.match(/`(\w+)`(?: (desc|asc))?$/i) + : null; + state.orderColumn = named?.[1] ?? "createdAt"; + state.ascending = named?.[2]?.toLowerCase() === "asc"; + return chain; + }, + limit(count: number) { + state.limit = count; + return chain; + }, + offset(count: number) { + state.offset = count; + return chain; + }, + for() { + return chain; + }, }, - }); + ); return chain; }, }); @@ -359,7 +489,13 @@ export function createFakeDatabase(defaults: Map Row>) { }, }); - const client = { select, insert, update, delete: remove }; + const client = { + select: (fields?: Record) => select(fields), + selectDistinct: (fields: Record) => select(fields, true), + insert, + update, + delete: remove, + }; const transaction = (callback: (tx: typeof client) => Promise) => { const run = async () => { diff --git a/apps/web/__tests__/unit/developer-transcript-math.test.ts b/apps/web/__tests__/unit/developer-transcript-math.test.ts index 29550eb6064..ce55d250c7b 100644 --- a/apps/web/__tests__/unit/developer-transcript-math.test.ts +++ b/apps/web/__tests__/unit/developer-transcript-math.test.ts @@ -35,6 +35,8 @@ const row = ( status: "complete", sourceType: "upload", sourceUrl: null, + videoId: null, + requestedAt: null, s3Key: "developer/app-1/transcripts/tr-1/source", resultKey: "developer/app-1/transcripts/tr-1/result.json", sourceBytes: 1_000, @@ -264,6 +266,7 @@ describe("transcript serialization", () => { "text", "updatedAt", "userId", + "videoId", "wordCount", ]); expect(Object.keys(agent).sort()).toEqual([ @@ -282,6 +285,7 @@ describe("transcript serialization", () => { "sourceUrl", "status", "updatedAt", + "videoId", "wordCount", ]); for (const serialized of [JSON.stringify(rest), JSON.stringify(agent)]) { diff --git a/apps/web/__tests__/unit/developer-transcript-settle.test.ts b/apps/web/__tests__/unit/developer-transcript-settle.test.ts index dc1c69fc27f..5fad3070f58 100644 --- a/apps/web/__tests__/unit/developer-transcript-settle.test.ts +++ b/apps/web/__tests__/unit/developer-transcript-settle.test.ts @@ -12,6 +12,11 @@ const context = vi.hoisted(() => ({ reset: () => void; }, ledgerIds: 0, + scheduleAutoTopUp: vi.fn(), +})); + +vi.mock("@/lib/infra-auto-top-up", () => ({ + scheduleAutoTopUp: context.scheduleAutoTopUp, })); vi.mock("@cap/database", async () => { @@ -99,6 +104,7 @@ const settle = (durationSeconds = ONE_HOUR, transcriptId = "tr-1") => beforeEach(() => { fake().reset(); context.ledgerIds = 0; + context.scheduleAutoTopUp.mockReset(); fake().seed(developerCreditAccounts, { id: "account-1", appId: "app-1", @@ -259,15 +265,134 @@ describe("transcript settlement", () => { expect(transcript()?.status).toBe("error"); }); - it("does not charge for a transcript deleted while it ran", async () => { + it("charges a transcript deleted while it was processing and keeps it hidden with no text", async () => { + const deletedAt = new Date("2026-11-12T10:00:00.000Z"); const current = transcript(); - if (current) current.deletedAt = new Date(); + if (current) current.deletedAt = deletedAt; + const result = await settle(); - expect(result).toEqual({ state: "skipped", reason: "deleted" }); - expect(account()?.balanceMicroCredits).toBe(50_000); + + expect(result).toEqual({ + state: "complete", + chargedMicroCredits: 10_000, + alreadySettled: false, + }); + expect(account()?.balanceMicroCredits).toBe(40_000); + expect(ledger()).toHaveLength(1); + expect(ledger()[0]).toMatchObject({ + type: "transcription", + amountMicroCredits: -10_000, + referenceId: "tr-1", + metadata: { durationSeconds: ONE_HOUR, billedSeconds: ONE_HOUR }, + }); + expect(transcript()).toMatchObject({ + status: "complete", + deletedAt, + billedMicroCredits: 10_000, + reservedMicroCredits: 0, + durationSeconds: ONE_HOUR, + text: null, + language: null, + wordCount: null, + resultKey: null, + }); + }); + + it("charges a deleted transcript once when the settle step is retried", async () => { + const current = transcript(); + if (current) current.deletedAt = new Date(); + + await settle(); + const retry = await settle(); + + expect(retry).toEqual({ + state: "complete", + chargedMicroCredits: 0, + alreadySettled: true, + }); + expect(account()?.balanceMicroCredits).toBe(40_000); + expect(ledger()).toHaveLength(1); + }); + + it("never charges a deleted transcript more than its hold", async () => { + const current = transcript(); + if (current) { + current.deletedAt = new Date(); + current.reservedMicroCredits = 5_000; + } + + expect(await settle()).toMatchObject({ chargedMicroCredits: 5_000 }); + expect(account()?.balanceMicroCredits).toBe(45_000); + }); + + it("uses the length measured before the delete when the step reports none", async () => { + const current = transcript(); + if (current) { + current.deletedAt = new Date(); + current.durationSeconds = 1800; + current.reservedMicroCredits = 5_000; + } + + expect(await settle(Number.NaN)).toMatchObject({ + chargedMicroCredits: 5_000, + }); + expect(ledger()[0]).toMatchObject({ + metadata: { durationSeconds: 1800, billedSeconds: 1800 }, + }); + expect(transcript()?.durationSeconds).toBe(1800); + }); + + it("charges what the hold covered when a deleted transcript can no longer be measured", async () => { + const current = transcript(); + if (current) { + current.deletedAt = new Date(); + current.durationSeconds = null; + current.reservedMicroCredits = 7_000; + } + + expect(await settle(0)).toMatchObject({ + state: "complete", + chargedMicroCredits: 7_000, + }); + expect(account()?.balanceMicroCredits).toBe(43_000); + expect(ledger()[0]).toMatchObject({ + amountMicroCredits: -7_000, + metadata: { heldMicroCredits: 7_000 }, + }); + expect(transcript()).toMatchObject({ + status: "complete", + billedMicroCredits: 7_000, + durationSeconds: null, + }); + }); + + it("fails a deleted transcript without charging when the balance ran out", async () => { + const current = transcript(); + if (current) current.deletedAt = new Date(); + const credit = account(); + if (credit) credit.balanceMicroCredits = 100; + + expect(await settle()).toEqual({ state: "insufficient_credits" }); expect(ledger()).toHaveLength(0); + expect(transcript()).toMatchObject({ status: "error", text: null }); }); + it.each(["uploading", "queued", "error"])( + "does not charge a deleted transcript that is %s", + async (status) => { + const current = transcript(); + if (current) { + current.deletedAt = new Date(); + current.status = status; + } + + expect(await settle()).toEqual({ state: "skipped", reason: "deleted" }); + expect(account()?.balanceMicroCredits).toBe(50_000); + expect(ledger()).toHaveLength(0); + expect(transcript()?.status).toBe(status); + }, + ); + it("does not charge for a transcript that already failed or timed out", async () => { const current = transcript(); if (current) current.status = "error"; @@ -284,6 +409,48 @@ describe("transcript settlement", () => { }); }); +describe("auto top-up after a transcript charge or hold", () => { + const scheduled = () => context.scheduleAutoTopUp.mock.calls; + + it("asks for a top-up once after a settle that charged, not on a retry", async () => { + await settle(); + await settle(); + + expect(scheduled()).toEqual([[{ appId: "app-1" }]]); + }); + + it("asks for a top-up when the balance could not cover the transcript", async () => { + const current = account(); + if (current) current.balanceMicroCredits = 9_999; + + await settle(); + + expect(scheduled()).toEqual([[{ appId: "app-1" }]]); + }); + + it("asks for a top-up after a hold or a reservation, and not for skipped work", async () => { + addTranscript("tr-2"); + await holdTranscriptCredits({ transcriptId: "tr-1", microCredits: 1_000 }); + await reserveTranscriptCredits({ + transcriptId: "tr-2", + durationSeconds: ONE_HOUR, + }); + expect(scheduled()).toEqual([[{ appId: "app-1" }], [{ appId: "app-1" }]]); + + addTranscript("tr-failed", { status: "error" }); + await holdTranscriptCredits({ + transcriptId: "tr-failed", + microCredits: 1_000, + }); + await reserveTranscriptCredits({ + transcriptId: "tr-failed", + durationSeconds: ONE_HOUR, + }); + await settle(ONE_HOUR, "tr-failed"); + expect(scheduled()).toHaveLength(2); + }); +}); + describe("transcript holds", () => { const reserve = (transcriptId: string, durationSeconds: number) => reserveTranscriptCredits({ transcriptId, durationSeconds }); @@ -511,9 +678,15 @@ describe("holds and other usage", () => { expect(account()?.balanceMicroCredits).toBe(45_000); }); - it("ignores holds on deleted transcripts and other apps", async () => { + it("ignores holds on other apps and on deleted transcripts that are not processing", async () => { setBalance(10_000); - addTranscript("tr-deleted", { + addTranscript("tr-deleted-failed", { + status: "error", + reservedMicroCredits: 9_000, + deletedAt: new Date(), + }); + addTranscript("tr-deleted-queued", { + status: "queued", reservedMicroCredits: 9_000, deletedAt: new Date(), }); @@ -524,6 +697,26 @@ describe("holds and other usage", () => { }); expect((await otherUsage(10_000, "video-1")).state).toBe("debited"); }); + + it("keeps the hold of a transcript deleted while processing until it settles", async () => { + setBalance(10_000); + addTranscript("tr-deleted", { + reservedMicroCredits: 9_000, + durationSeconds: 3240, + deletedAt: new Date(), + }); + + expect(await getAvailableTranscriptBalance("app-1")).toBe(1_000); + expect(await otherUsage(5_000, "video-1")).toEqual({ + state: "insufficient", + }); + expect(await settle(3240, "tr-deleted")).toMatchObject({ + state: "complete", + chargedMicroCredits: 9_000, + }); + expect(account()?.balanceMicroCredits).toBe(1_000); + expect(await getAvailableTranscriptBalance("app-1")).toBe(1_000); + }); }); describe("the hold taken before any work", () => { diff --git a/apps/web/__tests__/unit/developer-transcript-store.test.ts b/apps/web/__tests__/unit/developer-transcript-store.test.ts index d18dda13969..cf7b2d18a3f 100644 --- a/apps/web/__tests__/unit/developer-transcript-store.test.ts +++ b/apps/web/__tests__/unit/developer-transcript-store.test.ts @@ -19,6 +19,11 @@ const context = vi.hoisted(() => ({ presigned: [] as { key: string; contentLength: number }[], failSigning: false, onDelete: null as null | (() => Promise), + scheduleAutoTopUp: vi.fn(), +})); + +vi.mock("@/lib/infra-auto-top-up", () => ({ + scheduleAutoTopUp: context.scheduleAutoTopUp, })); vi.mock("@cap/database", async () => { @@ -163,6 +168,7 @@ const fromUrl = (url = URL_A, idempotencyKey?: string, appId = "app-1") => beforeEach(() => { fake().reset(); context.ids = 0; + context.scheduleAutoTopUp.mockReset(); context.speechConfigured = true; context.hosted = true; context.start.mockReset(); @@ -305,6 +311,19 @@ describe("creating a transcript from a URL", () => { }); expect(transcripts()).toHaveLength(0); expect(context.start).not.toHaveBeenCalled(); + expect(context.scheduleAutoTopUp).toHaveBeenCalledWith({ + appId: "app-1", + }); + }); + + it("links the app's own Billing page when it refuses for credit", async () => { + setBalance(0); + const result = await fromUrl(); + + expect(result).toMatchObject({ ok: false, status: 402 }); + expect(result.ok === false && result.hint).toBe( + "Add credits to this app at https://cap.so/dashboard/infra/app-1/billing, then retry.", + ); }); it("says how much to add below the minimum balance", async () => { @@ -953,12 +972,17 @@ describe("starting an uploaded transcript", () => { const id = await created(); context.objectSize = 1024; setBalance(0); + context.scheduleAutoTopUp.mockReset(); expect(await startTranscriptUpload("app-1", id)).toMatchObject({ ok: false, status: 402, + hint: "Add credits to this app at https://cap.so/dashboard/infra/app-1/billing, then retry.", }); expect(context.start).not.toHaveBeenCalled(); + expect(context.scheduleAutoTopUp).toHaveBeenCalledWith({ + appId: "app-1", + }); }); it("can be retried after the job could not be started", async () => { @@ -1139,10 +1163,49 @@ describe("deleting", () => { "developer/app-1/transcripts/tr-1/result.json", ]); expect(await getTranscript("app-1", "tr-1")).toBeNull(); - expect(await deleteTranscript("app-1", "tr-1")).toMatchObject({ - ok: false, - status: 404, + }); + + it("answers a repeat delete of a deleted transcript with success and clears its files again", async () => { + fake().seed(developerTranscripts, { + id: "tr-1", + appId: "app-1", + status: "complete", + sourceType: "upload", + text: "secret words", + }); + expect(await deleteTranscript("app-1", "tr-1")).toEqual({ ok: true }); + const deletedAt = transcripts()[0]?.deletedAt; + context.deleted = []; + + expect(await deleteTranscript("app-1", "tr-1")).toEqual({ ok: true }); + + expect(transcripts()[0]?.deletedAt).toEqual(deletedAt); + expect(context.deleted).toEqual([ + "developer/_uploads/app-1/t-tr-1", + "developer/app-1/transcripts/tr-1/audio.mp3", + "developer/app-1/transcripts/tr-1/result.json", + ]); + }); + + it("still answers 404 for a transcript that never existed, or another app's deleted one", async () => { + fake().seed(developerTranscripts, { + id: "tr-2", + appId: "app-2", + status: "complete", + sourceType: "upload", + text: "other app", }); + expect(await deleteTranscript("app-2", "tr-2")).toEqual({ ok: true }); + context.deleted = []; + + for (const id of ["tr-2", "never-existed"]) { + expect(await deleteTranscript("app-1", id)).toMatchObject({ + ok: false, + status: 404, + code: "transcript_not_found", + }); + } + expect(context.deleted).toEqual([]); }); it("does not delete another app's transcript", async () => { @@ -1459,3 +1522,32 @@ describe("input validation", () => { expect(isTranscriptIdempotencyKey(undefined)).toBe(false); }); }); + +describe("deleting a transcript that is still running", () => { + it("leaves its files for the run to settle and clean up, and removes them otherwise", async () => { + for (const [status, kept] of [ + ["processing", true], + ["queued", false], + ["complete", false], + ["error", false], + ] as const) { + context.deleted = []; + const id = `tr-${status}`; + fake().seed(developerTranscripts, { + id, + appId: "app-1", + status, + sourceType: "upload", + }); + + expect(await deleteTranscript("app-1", id)).toEqual({ ok: true }); + + expect(context.deleted.length > 0).toBe(!kept); + expect( + fake() + .rows(developerTranscripts) + .find((row) => row.id === id)?.deletedAt, + ).toBeInstanceOf(Date); + } + }); +}); diff --git a/apps/web/__tests__/unit/developer-transcript-vtt.test.ts b/apps/web/__tests__/unit/developer-transcript-vtt.test.ts new file mode 100644 index 00000000000..8b4dd055e81 --- /dev/null +++ b/apps/web/__tests__/unit/developer-transcript-vtt.test.ts @@ -0,0 +1,220 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { + buildCaptionCues, + buildWebVtt, + CAPTION_LINE_CHARACTERS, + CAPTION_MAX_CUE_MS, + CAPTION_READING_MS_PER_CHARACTER, + type CaptionWord, + escapeVttText, + formatVttTimestamp, +} from "@/lib/developer-transcript-vtt"; + +const GOLDEN = join(__dirname, "../fixtures/developer-captions-golden.vtt"); + +const say = ( + startSeconds: number, + secondsPerWord: number, + text: string, +): CaptionWord[] => + text.split(" ").map((word, index) => { + const start = startSeconds + index * secondsPerWord; + return { + text: word, + start: Math.round(start * 1000) / 1000, + end: Math.round((start + secondsPerWord * 0.8) * 1000) / 1000, + }; + }); + +const FIXED_WORDS: CaptionWord[] = [ + ...say(0.24, 0.32, "Welcome back."), + ...say( + 1.6, + 0.4, + "Today we are walking through the quarterly numbers for the whole company, region by region, and then we will look at what changed in the product since the last time we all met in this room.", + ), + ...say(18.4, 0.36, "We shipped 1,250 units in Q3 2026 for $3.5 million."), + ...say( + 23.2, + 0.12, + "And yes I know that is a lot to take in all at once so bear with me please.", + ), + ...say(26.4, 0.4, "Let me think"), + ...say(31.2, 0.4, "about that for a second."), + ...say( + 34.4, + 0.4, + "Is 5 < 7 && 9 > 2? Tom & Jerry said yes --> maybe.", + ), + ...say(41.6, 0.4, "Okay."), + { text: "Line\nbreak inside", start: 43.2, end: 44 }, + { text: "", start: 44.2, end: 44.4 }, + { text: "end.", start: 44.4, end: 44.8 }, +]; + +type ParsedCue = { start: number; end: number; lines: string[] }; + +const parseTimestamp = (value: string) => { + const match = /^(\d{2,}):(\d{2}):(\d{2})\.(\d{3})$/.exec(value); + if (!match) throw new Error(`Bad timestamp ${value}`); + const [, hours, minutes, seconds, millis] = match; + return ( + Number(hours) * 3_600_000 + + Number(minutes) * 60_000 + + Number(seconds) * 1000 + + Number(millis) + ); +}; + +function parseVtt(text: string): ParsedCue[] { + const [header, ...blocks] = text.trimEnd().split("\n\n"); + if (header !== "WEBVTT") throw new Error("Missing WEBVTT header"); + return blocks.map((block) => { + const [timing, ...lines] = block.split("\n"); + const [start, arrow, end, ...rest] = (timing ?? "").split(" "); + if (arrow !== "-->" || rest.length > 0 || !start || !end) { + throw new Error(`Bad timing line ${timing}`); + } + if (lines.length === 0 || lines.some((line) => line.trim() === "")) { + throw new Error("Cue has an empty line"); + } + return { start: parseTimestamp(start), end: parseTimestamp(end), lines }; + }); +} + +describe("WebVTT captions from word timings", () => { + it("matches the golden file for fixed word timings", () => { + expect(buildWebVtt(FIXED_WORDS, 46)).toBe(readFileSync(GOLDEN, "utf8")); + }); + + it("parses, and its cues never overlap or go backwards", () => { + const cues = parseVtt(buildWebVtt(FIXED_WORDS, 46) ?? ""); + expect(cues.length).toBeGreaterThan(8); + let previousEnd = 0; + for (const cue of cues) { + expect(cue.start).toBeGreaterThanOrEqual(previousEnd); + expect(cue.end).toBeGreaterThan(cue.start); + previousEnd = cue.end; + } + }); + + it("keeps every cue to two lines of about 42 characters and 7 seconds", () => { + const cues = buildCaptionCues(FIXED_WORDS, 46); + for (const cue of cues) { + expect(cue.lines.length).toBeLessThanOrEqual(2); + for (const line of cue.lines) { + expect(line.length).toBeLessThanOrEqual(CAPTION_LINE_CHARACTERS); + } + expect(cue.end - cue.start).toBeLessThanOrEqual(CAPTION_MAX_CUE_MS); + } + }); + + it("keeps every word, in order", () => { + const cues = buildCaptionCues(FIXED_WORDS, 46); + expect(cues.flatMap((cue) => cue.lines).join(" ")).toBe( + FIXED_WORDS.map((word) => word.text.replace(/\s+/g, " ").trim()) + .filter(Boolean) + .join(" "), + ); + }); + + it("ends a cue at the end of a sentence", () => { + const cues = buildCaptionCues(FIXED_WORDS, 46); + expect(cues[0]?.lines).toEqual(["Welcome back."]); + expect(cues.at(-1)?.lines.join(" ")).toBe("Line break inside end."); + }); + + it("starts a new cue after a pause, even inside a sentence", () => { + const cues = buildCaptionCues(FIXED_WORDS, 46); + const before = cues.find((cue) => cue.lines.join(" ") === "Let me think"); + const after = cues.find( + (cue) => cue.lines.join(" ") === "about that for a second.", + ); + expect(before).toBeDefined(); + expect(after?.start).toBe(31_200); + expect(before?.end).toBeLessThanOrEqual(28_520); + }); + + it("escapes the characters WebVTT reserves and never emits an arrow in text", () => { + const text = buildWebVtt(FIXED_WORDS, 46) ?? ""; + expect(text).toContain("Is 5 < 7 && 9 > 2?"); + expect(text).toContain("<b>yes</b> --> maybe."); + expect(text).toContain("Tom & Jerry"); + const cueText = parseVtt(text).flatMap((cue) => cue.lines); + expect(cueText.some((line) => line.includes("-->"))).toBe(false); + expect(cueText.some((line) => /[<>]/.test(line))).toBe(false); + expect(escapeVttText("a & b < c > d")).toBe("a & b < c > d"); + }); + + it("holds quick speech on screen long enough to read, up to the next cue", () => { + const cues = buildCaptionCues(FIXED_WORDS, 46); + const quick = cues.filter( + (cue) => cue.start >= 23_200 && cue.start < 26_400, + ); + const text = quick.flatMap((cue) => cue.lines).join(" "); + expect(text).toBe( + "And yes I know that is a lot to take in all at once so bear with me please.", + ); + expect(quick).toHaveLength(1); + const spokenEnd = 25_456; + expect(quick[0]?.end).toBeGreaterThan(spokenEnd); + expect(quick[0]?.end).toBe(26_400); + expect((quick[0]?.end ?? 0) - (quick[0]?.start ?? 0)).toBeLessThan( + text.length * CAPTION_READING_MS_PER_CHARACTER, + ); + }); + + it("does not hold normal speech past its last word", () => { + const cues = buildCaptionCues(FIXED_WORDS, 46); + const shipped = cues.find((cue) => cue.lines[0]?.startsWith("We shipped")); + expect(shipped?.end).toBe(21_928); + }); + + it("writes one cue for a single word and holds it for a second", () => { + expect(buildWebVtt([{ text: "Hello", start: 2, end: 2.24 }], 10)).toBe( + "WEBVTT\n\n00:00:02.000 --> 00:00:03.000\nHello\n", + ); + }); + + it("does not hold a single word past the end of the media", () => { + expect(buildWebVtt([{ text: "Bye", start: 9.6, end: 9.84 }], 10)).toBe( + "WEBVTT\n\n00:00:09.600 --> 00:00:10.000\nBye\n", + ); + }); + + it("writes nothing when there are no words", () => { + expect(buildWebVtt([], 30)).toBeNull(); + expect(buildWebVtt(null, 30)).toBeNull(); + expect(buildWebVtt([{ text: " ", start: 1, end: 2 }], 30)).toBeNull(); + }); + + it("never lets a cue start before the one before it ended when timings overlap or go backwards", () => { + const cues = buildCaptionCues([ + { text: "First.", start: 5, end: 9 }, + { text: "Second.", start: 4, end: 6 }, + { text: "Third.", start: 6, end: 6.5 }, + { text: "Fourth.", start: Number.NaN, end: -3 }, + ]); + let previousEnd = 0; + for (const cue of cues) { + expect(cue.start).toBeGreaterThanOrEqual(previousEnd); + expect(cue.end).toBeGreaterThan(cue.start); + previousEnd = cue.end; + } + expect(cues).toHaveLength(4); + }); + + it("caps one very long word at 7 seconds and keeps it on its own line", () => { + const long = "a".repeat(60); + const cues = buildCaptionCues([{ text: long, start: 0, end: 30 }]); + expect(cues).toEqual([{ start: 0, end: 7_000, lines: [long] }]); + }); + + it("formats hours past the first", () => { + expect(formatVttTimestamp(0)).toBe("00:00:00.000"); + expect(formatVttTimestamp(3_723_045)).toBe("01:02:03.045"); + expect(formatVttTimestamp(14_399_999)).toBe("03:59:59.999"); + }); +}); diff --git a/apps/web/__tests__/unit/developer-transcripts-api.test.ts b/apps/web/__tests__/unit/developer-transcripts-api.test.ts index 5600e4da9b0..33981d2a0b5 100644 --- a/apps/web/__tests__/unit/developer-transcripts-api.test.ts +++ b/apps/web/__tests__/unit/developer-transcripts-api.test.ts @@ -4,6 +4,7 @@ const mocks = vi.hoisted(() => ({ keyRows: [] as { appId: string }[], store: { createTranscriptFromUrl: vi.fn(), + createTranscriptFromVideo: vi.fn(), createTranscriptUpload: vi.fn(), startTranscriptUpload: vi.fn(), getTranscript: vi.fn(), @@ -62,6 +63,7 @@ const transcript = (overrides: Record = {}) => ({ status: "queued", sourceType: "url", sourceUrl: "https://media.example.com/a.mp3", + videoId: null, s3Key: "developer/app-1/transcripts/tr-1/source", resultKey: null, sourceBytes: null, @@ -99,29 +101,49 @@ describe("developer transcripts API", () => { describe("authentication", () => { it.each([ - ["no key", {}], - ["a public key", { Authorization: "Bearer cpk_public_key" }], - ["an agent token", { Authorization: "Bearer cap_cli_token" }], - ["a malformed header", { Authorization: "csk_valid_key" }], - ])("rejects %s before touching transcripts", async (_label, headers) => { - for (const [method, path] of [ - ["GET", "/"], - ["GET", "/tr-1"], - ["DELETE", "/tr-1"], - ["POST", "/tr-1/start"], - ] as const) { - const response = await app.request(path, { method, headers }); - expect(response.status).toBe(401); - expect(await response.json()).toMatchObject({ - code: "invalid_secret_key", - }); - } - const created = await post("/", { url: "https://a.example/a" }, headers); - expect(created.status).toBe(401); - for (const mock of Object.values(mocks.store)) { - expect(mock).not.toHaveBeenCalled(); - } - }); + ["no key", {}, 401, "invalid_secret_key"], + [ + "a public key", + { Authorization: "Bearer cpk_public_key" }, + 403, + "public_key_not_allowed", + ], + [ + "an agent token", + { Authorization: "Bearer cap_cli_token" }, + 401, + "invalid_secret_key", + ], + [ + "a malformed header", + { Authorization: "csk_valid_key" }, + 401, + "invalid_secret_key", + ], + ])( + "rejects %s before touching transcripts", + async (_label, headers, status, code) => { + for (const [method, path] of [ + ["GET", "/"], + ["GET", "/tr-1"], + ["DELETE", "/tr-1"], + ["POST", "/tr-1/start"], + ] as const) { + const response = await app.request(path, { method, headers }); + expect(response.status).toBe(status); + expect(await response.json()).toMatchObject({ code }); + } + const created = await post( + "/", + { url: "https://a.example/a" }, + headers, + ); + expect(created.status).toBe(status); + for (const mock of Object.values(mocks.store)) { + expect(mock).not.toHaveBeenCalled(); + } + }, + ); it("rejects a revoked or unknown secret key", async () => { mocks.keyRows = []; @@ -317,7 +339,7 @@ describe("developer transcripts API", () => { expect(body).toMatchObject({ code: "invalid_request", error: `Invalid request: Unrecognized key: "${field}"`, - hint: "This request accepts url, name, language, userId, metadata. Field names are case-sensitive.", + hint: "This request accepts url, videoId, name, language, userId, metadata. Field names are case-sensitive.", }); expect(body.issues).toEqual([ expect.objectContaining({ code: "unrecognized_keys", keys: [field] }), @@ -349,6 +371,143 @@ describe("developer transcripts API", () => { }, ); + it("passes a video id, options and idempotency key to the store", async () => { + mocks.store.createTranscriptFromVideo.mockResolvedValue({ + ok: true, + transcript: transcript({ + sourceType: "video", + sourceUrl: null, + videoId: "video-1", + s3Key: "developer/app-1/video-1/encode-1/out-1/video.mp4", + }), + }); + const response = await post( + "/", + { videoId: "video-1", language: "en", metadata: { a: 1 } }, + { ...SECRET, "Idempotency-Key": "retry-key-1" }, + ); + expect(response.status).toBe(200); + expect(mocks.store.createTranscriptFromVideo).toHaveBeenCalledWith( + "app-1", + { + videoId: "video-1", + language: "en", + metadata: { a: 1 }, + idempotencyKey: "retry-key-1", + }, + ); + expect(mocks.store.createTranscriptFromUrl).not.toHaveBeenCalled(); + const body = await response.json(); + expect(body.data).toMatchObject({ + id: "tr-1", + videoId: "video-1", + source: { type: "video", url: null }, + }); + expect(JSON.stringify(body)).not.toContain("video.mp4"); + }); + + it("returns a null video id for a transcript that is not from a video", async () => { + mocks.store.createTranscriptFromUrl.mockResolvedValue({ + ok: true, + transcript: transcript(), + }); + const response = await post( + "/", + { url: "https://media.example.com/a.mp3" }, + SECRET, + ); + expect((await response.json()).data).toMatchObject({ + videoId: null, + source: { type: "url" }, + }); + }); + + it.each([ + ["both url and videoId", { url: "https://a.example/a", videoId: "v1" }], + ["neither url nor videoId", { name: "Standup" }], + ["an empty body", {}], + ])("refuses %s", async (_label, body) => { + const response = await post("/", body, SECRET); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ + code: "invalid_request", + error: "Invalid request: Send exactly one of url or videoId", + }); + expect(mocks.store.createTranscriptFromUrl).not.toHaveBeenCalled(); + expect(mocks.store.createTranscriptFromVideo).not.toHaveBeenCalled(); + }); + + it.each([ + [{ videoId: "" }], + [{ videoId: 42 }], + [{ videoId: "v".repeat(65) }], + [{ videoId: "video-1", video_id: "video-1" }], + ])("rejects the invalid video body %j", async (body) => { + const response = await post("/", body, SECRET); + expect(response.status).toBe(400); + expect(mocks.store.createTranscriptFromVideo).not.toHaveBeenCalled(); + }); + + it.each([ + [404, "video_not_found"], + [409, "video_not_ready"], + ] as const)("answers %i %s from the store", async (status, code) => { + mocks.store.createTranscriptFromVideo.mockResolvedValue({ + ok: false, + status, + code, + message: "Video problem", + hint: "Fix it", + }); + const response = await post("/", { videoId: "video-1" }, SECRET); + expect(response.status).toBe(status); + expect(await response.json()).toMatchObject({ code }); + }); + + it.each([ + ["creating from a URL", "/", { url: "https://media.example.com/a.mp3" }], + ["starting an upload", "/tr-1/start", {}], + ])( + "answers 429 with Retry-After when %s at the ceiling", + async (_label, path, body) => { + const atCeiling = { + ok: false, + status: 429, + code: "too_many_transcripts", + message: "This app already has 20 transcripts queued or processing", + hint: "Wait for one to finish, then send the request again.", + }; + mocks.store.createTranscriptFromUrl.mockResolvedValue(atCeiling); + mocks.store.startTranscriptUpload.mockResolvedValue(atCeiling); + + const response = await post(path, body, SECRET); + + expect(response.status).toBe(429); + expect(response.headers.get("Retry-After")).toBe("15"); + expect(await response.json()).toMatchObject({ + error: "This app already has 20 transcripts queued or processing", + code: "too_many_transcripts", + hint: "Wait for one to finish, then send the request again.", + }); + }, + ); + + it("sets no Retry-After on other refusals", async () => { + mocks.store.createTranscriptFromUrl.mockResolvedValue({ + ok: false, + status: 402, + code: "insufficient_credits", + message: "Insufficient credits", + hint: "Add credits", + }); + const response = await post( + "/", + { url: "https://media.example.com/a.mp3" }, + SECRET, + ); + expect(response.headers.get("Retry-After")).toBeNull(); + }); + it("rejects a malformed idempotency key", async () => { const response = await post( "/", diff --git a/apps/web/__tests__/unit/developer-upload-harness.ts b/apps/web/__tests__/unit/developer-upload-harness.ts index a2e9a22f105..ad5f4e3f76c 100644 --- a/apps/web/__tests__/unit/developer-upload-harness.ts +++ b/apps/web/__tests__/unit/developer-upload-harness.ts @@ -8,9 +8,12 @@ export type StoredVideo = { status: string; }; +export type StoredKey = { appId: string; revokedAt: string | null }; + export const state = { query: vi.fn(), videos: new Map(), + keys: new Map(), apps: new Set(), environment: "production", nextVideo: 0, @@ -39,22 +42,61 @@ const admitted = { const KEY_TYPES: Record = { cpk: "public", csk: "secret" }; +export const keyIdOf = (key: string) => key.replace(/^c[ps]k_/, "key_"); + +function storedKey(key: string, appId: string) { + const id = keyIdOf(key); + if (!state.keys.has(id)) state.keys.set(id, { appId, revokedAt: null }); + return state.keys.get(id) ?? null; +} + +export function revokeKey(key: string) { + const stored = state.keys.get(keyIdOf(key)); + if (stored) stored.revokedAt = "2026-11-12 10:00:01"; +} + +export const removeKey = (key: string) => state.keys.set(keyIdOf(key), null); + +const revokedTime = (revokedAt: string) => + Date.parse(`${revokedAt.replace(" ", "T")}Z`); + async function answer(query: string, params: unknown[]) { + if ( + query.includes("from `developer_api_keys`") && + query.includes("`developer_api_keys`.`revokedAt` > ?") + ) { + const [appId, , since] = params; + const sinceTime = new Date(since as Date | string).getTime(); + const revoked = [...state.keys.entries()].find(([, key]) => { + if (!key || key.appId !== appId || key.revokedAt === null) return false; + return revokedTime(key.revokedAt) > sinceTime; + }); + return { rows: revoked ? [[revoked[0]]] : [] }; + } if (query.includes("from `developer_api_keys`")) { - const [prefix = "", appId = ""] = String(params[0]) - .replace("hash:", "") - .split("_"); + const key = String(params[0]).replace("hash:", ""); + const [prefix = "", appId = ""] = key.split("_"); if (KEY_TYPES[prefix] !== params[1] || !state.apps.has(appId)) { return { rows: [] }; } - return { rows: [[appId, state.environment]] }; + const stored = storedKey(key, appId); + if (!stored || stored.revokedAt) return { rows: [] }; + return { rows: [[appId, keyIdOf(key), `owner-${appId}`]] }; } if (query.includes("from `developer_videos` inner join `developer_apps`")) { - const video = state.videos.get(String(params[0])); - if (!video || video.appId !== params[1] || !state.apps.has(video.appId)) { + const [keyId, , videoId, appId] = params; + const video = state.videos.get(String(videoId)); + if (!video || video.appId !== appId || !state.apps.has(video.appId)) { return { rows: [] }; } - return { rows: [[video.duration, video.deletedAt]] }; + const key = state.keys.get(String(keyId)); + return { + rows: [ + key && key.appId === video.appId + ? [video.duration, video.deletedAt, keyId, key.revokedAt] + : [video.duration, video.deletedAt, null, null], + ], + }; } if (query.includes("from `developer_app_domains`")) { return { rows: [["domain-1"]] }; @@ -110,6 +152,7 @@ export function resetUploadHarness() { state.listParts.mockReset(); state.presign.mockReset(); state.videos.clear(); + state.keys.clear(); state.apps = new Set(["app1", "app2"]); state.environment = "production"; state.nextVideo = 0; diff --git a/apps/web/__tests__/unit/developer-upload-token-flow.test.ts b/apps/web/__tests__/unit/developer-upload-token-flow.test.ts index 593d91aaa65..acd9fc0ea10 100644 --- a/apps/web/__tests__/unit/developer-upload-token-flow.test.ts +++ b/apps/web/__tests__/unit/developer-upload-token-flow.test.ts @@ -42,12 +42,24 @@ vi.mock("@/lib/developer-transcript-store", () => ({ const sdk = await import("@/app/api/developer/sdk/v1/[...route]/route"); const rest = await import("@/app/api/developer/v1/[...route]/route"); +const { DEVELOPER_KEY_REQUESTS_PER_MINUTE: KEY_LIMIT } = await import( + "@/lib/developer-rate-limits" +); const { createDeveloperUploadToken } = await import( "@/lib/developer-upload-token" ); -const { state, post, secretKey, videoWrites, storageCalls, UPLOAD_ROUTES } = - harness; +const { + state, + post, + secretKey, + keyIdOf, + revokeKey, + removeKey, + videoWrites, + storageCalls, + UPLOAD_ROUTES, +} = harness; const NOW = Date.parse("2026-11-12T10:00:00Z"); const PAGE = "https://app.example.com"; @@ -75,6 +87,19 @@ async function created(body: Record = {}, appId = "app1") { return (await response.json()).data as Created; } +async function createdWith(key: string, body: Record = {}) { + const response = await post(rest, "/api/developer/v1/uploads", key, body); + expect(response.status).toBe(200); + return (await response.json()).data as Created; +} + +const payloadOf = (token: string) => + JSON.parse( + Buffer.from(token.slice(4).split(".")[0] ?? "", "base64url").toString( + "utf8", + ), + ) as Record; + const upload = ( route: string, credential: string | null, @@ -252,21 +277,24 @@ describe("creating an upload with the secret key", () => { }); it.each([ - ["a public key", "cpk_app1_page"], - ["an upload token", "cut_abc.def"], - ["no key", null], - ])("refuses %s and inserts nothing", async (_label, credential) => { - const response = await post( - rest, - "/api/developer/v1/uploads", - credential, - {}, - ); - - expect(response.status).toBe(401); - expect(await response.json()).toMatchObject({ code: "invalid_secret_key" }); - expect(inserts()).toHaveLength(0); - }); + ["a public key", "cpk_app1_page", 403, "public_key_not_allowed"], + ["an upload token", "cut_abc.def", 401, "invalid_secret_key"], + ["no key", null, 401, "invalid_secret_key"], + ])( + "refuses %s and inserts nothing", + async (_label, credential, status, code) => { + const response = await post( + rest, + "/api/developer/v1/uploads", + credential, + {}, + ); + + expect(response.status).toBe(status); + expect(await response.json()).toMatchObject({ code }); + expect(inserts()).toHaveLength(0); + }, + ); it.each([ [60, 60], @@ -329,14 +357,16 @@ describe("creating an upload with the secret key", () => { const send = () => post(rest, "/api/developer/v1/uploads", key, {}); const first = await send(); - expect(first.headers.get("X-RateLimit-Limit")).toBe("60"); - expect(first.headers.get("X-RateLimit-Remaining")).toBe("59"); - for (let request = 2; request <= 60; request++) await send(); + expect(first.headers.get("X-RateLimit-Limit")).toBe(String(KEY_LIMIT)); + expect(first.headers.get("X-RateLimit-Remaining")).toBe( + String(KEY_LIMIT - 1), + ); + for (let request = 2; request <= KEY_LIMIT; request++) await send(); const limited = await send(); expect(limited.status).toBe(429); expect(await limited.json()).toMatchObject({ code: "rate_limited" }); - expect(inserts()).toHaveLength(60); + expect(inserts()).toHaveLength(KEY_LIMIT); }); }); @@ -379,6 +409,99 @@ describe("uploading with the token", () => { expect(inserts()).toHaveLength(1); }); + it.each(["video/avi", "video/x-msvideo", "Video/AVI; codecs=xvid"])( + "refuses %s at initiate and names what is supported", + async (contentType) => { + const { token } = await created(); + + const response = await upload("initiate", token, { contentType }); + + expect(response.status).toBe(400); + const body = await response.json(); + expect(body).toMatchObject({ code: "unsupported_content_type" }); + expect(body.hint).toContain( + "MP4, MOV, MKV and WebM files with H.264, HEVC, VP8, VP9, AV1, ProRes, MPEG-4 or Motion JPEG video", + ); + expect(body.hint).toContain( + "video/mp4, video/webm, video/quicktime, video/x-matroska, application/octet-stream", + ); + expect(state.create).not.toHaveBeenCalled(); + }, + ); + + it.each([ + "text/plain", + "text/csv; charset=utf-8", + "image/png", + "audio/mpeg", + "application/json", + "application/pdf", + ])( + "refuses %s at initiate with the reason the farm gives", + async (contentType) => { + const { token } = await created(); + + const response = await upload("initiate", token, { contentType }); + + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ + code: "unsupported_content_type", + message: + "This file could not be read as a video. Upload MP4, MOV, MKV and WebM files with H.264, HEVC, VP8, VP9, AV1, ProRes, MPEG-4 or Motion JPEG video.", + }); + expect(state.create).not.toHaveBeenCalled(); + }, + ); + + it.each([ + ["video/mp4", "video/mp4"], + ["video/quicktime", "video/quicktime"], + ["video/x-matroska", "video/x-matroska"], + ["video/webm", "video/webm"], + ])("still accepts %s at initiate", async (contentType, stored) => { + const { token } = await created(); + + const response = await upload("initiate", token, { contentType }); + + expect(response.status).toBe(200); + expect(state.create).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ ContentType: stored }), + ); + }); + + it("still accepts application/octet-stream at initiate, as browsers send for MKV and MOV", async () => { + const { token } = await created(); + + const response = await upload("initiate", token, { + contentType: "application/octet-stream", + }); + + expect(response.status).toBe(200); + expect(state.create).toHaveBeenCalledWith( + expect.any(String), + expect.objectContaining({ ContentType: "application/octet-stream" }), + ); + }); + + it("still accepts an empty content type at initiate", async () => { + const { token } = await created(); + + const response = await upload("initiate", token, { contentType: "" }); + + expect(response.status).toBe(200); + expect(state.create).toHaveBeenCalledTimes(1); + }); + + it("still accepts an initiate with no content type", async () => { + const { token } = await created(); + + const response = await upload("initiate", token, {}); + + expect(response.status).toBe(200); + expect(state.create).toHaveBeenCalledTimes(1); + }); + it("accepts the token's own videoId in the body", async () => { const { token, videoId } = await created(); @@ -419,17 +542,40 @@ describe("uploading with the token", () => { await refusedEverywhere(token, 404, "video_not_found"); }); - it("looks the video up by its id and app, in an app that is not deleted", async () => { - const { token, videoId } = await created(); + it("looks the video up by its id and app, in an app that is not deleted, with the key that made the token", async () => { + const key = secretKey("app1"); + const { token, videoId } = await createdWith(key); state.query.mockClear(); await upload("initiate", token); const [query, params] = state.query.mock.calls[0] ?? []; expect(String(query)).toBe( - "select `developer_videos`.`duration`, `developer_videos`.`deletedAt` from `developer_videos` inner join `developer_apps` on (`developer_videos`.`appId` = `developer_apps`.`id` and `developer_apps`.`deletedAt` is null) where (`developer_videos`.`id` = ? and `developer_videos`.`appId` = ?) limit ?", + "select `developer_videos`.`duration`, `developer_videos`.`deletedAt`, `developer_api_keys`.`id`, `developer_api_keys`.`revokedAt` from `developer_videos` inner join `developer_apps` on (`developer_videos`.`appId` = `developer_apps`.`id` and `developer_apps`.`deletedAt` is null) left join `developer_api_keys` on (`developer_api_keys`.`id` = ? and `developer_api_keys`.`appId` = `developer_videos`.`appId` and `developer_api_keys`.`keyType` = ?) where (`developer_videos`.`id` = ? and `developer_videos`.`appId` = ?) limit ?", ); - expect(params).toEqual([videoId, "app1", 1]); + expect(params).toEqual([keyIdOf(key), "secret", videoId, "app1", 1]); + }); + + it("checks the token, its video, app and key with one query on every route", async () => { + const { token } = await created(); + const queries = () => + state.query.mock.calls.map(([query]) => String(query)); + + for (const [route, body] of UPLOAD_ROUTES) { + state.query.mockClear(); + + await upload(route, token, body); + + const tokenChecks = queries().filter((query) => + query.includes("from `developer_videos` inner join `developer_apps`"), + ); + const keyReads = queries().filter((query) => + query.includes("from `developer_api_keys`"), + ); + expect([route, tokenChecks.length]).toEqual([route, 1]); + expect([route, keyReads.length]).toEqual([route, 0]); + expect(queries()[0]).toBe(tokenChecks[0]); + } }); it("stops working once the app is deleted", async () => { @@ -749,15 +895,15 @@ describe("renewing the token of an open upload", () => { }); it.each([ - ["a public key", "cpk_app1_page"], - ["no key", null], - ])("refuses %s", async (_label, credential) => { + ["a public key", "cpk_app1_page", 403, "public_key_not_allowed"], + ["no key", null, 401, "invalid_secret_key"], + ])("refuses %s", async (_label, credential, status, code) => { const first = await created(); const response = await renew(first.videoId, {}, credential); - expect(response.status).toBe(401); - expect(await response.json()).toMatchObject({ code: "invalid_secret_key" }); + expect(response.status).toBe(status); + expect(await response.json()).toMatchObject({ code }); }); it("cannot be renewed with the upload token itself", async () => { @@ -800,6 +946,287 @@ describe("renewing the token of an open upload", () => { }); }); +describe("a token after its secret key is rotated", () => { + const ROTATED = { + code: "invalid_upload_token", + error: "Upload token is not valid", + hint: expect.stringContaining("was rotated or revoked"), + }; + + it("names the key that made it", async () => { + const key = secretKey("app1"); + + const { token } = await createdWith(key); + + expect(payloadOf(token)).toMatchObject({ key: keyIdOf(key) }); + }); + + it("works while the key is live", async () => { + const { token } = await createdWith(secretKey("app1")); + + expect((await upload("initiate", token)).status).toBe(200); + expect( + ( + await upload("presign-part", token, { + uploadId: "upload-1", + partNumber: 1, + }) + ).status, + ).toBe(200); + expect((await upload("complete", token, completeBody)).status).toBe(200); + }); + + it("is refused on every route once the key is revoked", async () => { + const key = secretKey("app1"); + const { token } = await createdWith(key); + revokeKey(key); + + await refusedEverywhere(token, 401, "invalid_upload_token"); + const response = await upload("initiate", token); + expect(await response.json()).toMatchObject({ + ...ROTATED, + hint: expect.stringMatching( + /was rotated or revoked.*POST \/api\/developer\/v1\/uploads\/\{videoId\}\/token/, + ), + }); + }); + + it("is refused once the key no longer exists", async () => { + const key = secretKey("app1"); + const { token } = await createdWith(key); + removeKey(key); + + await refusedEverywhere(token, 401, "invalid_upload_token"); + }); + + it("stops an upload that was already open", async () => { + const key = secretKey("app1"); + const { token } = await createdWith(key); + expect((await upload("initiate", token)).status).toBe(200); + revokeKey(key); + const storageBefore = storageCalls(); + + const response = await upload("complete", token, completeBody); + + expect(response.status).toBe(401); + expect(await response.json()).toMatchObject(ROTATED); + expect(storageCalls()).toBe(storageBefore); + expect(durationWrites()).toHaveLength(0); + }); + + it("keeps working when another key of the app is revoked", async () => { + const other = secretKey("app1"); + await createdWith(other); + const { token } = await createdWith(secretKey("app1")); + revokeKey(other); + + expect((await upload("initiate", token)).status).toBe(200); + }); + + it("is refused when the key it names belongs to another app", async () => { + const foreign = secretKey("app2"); + await createdWith(foreign); + const { videoId } = await created(); + const crossed = createDeveloperUploadToken({ + appId: "app1", + videoId, + expiresAt: NOW + 60_000, + corsOrigin: null, + keyId: keyIdOf(foreign), + }); + + await refusedEverywhere(crossed, 401, "invalid_upload_token"); + }); + + it("answers a deleted video before it looks at the key", async () => { + const key = secretKey("app1"); + const { token, videoId } = await createdWith(key); + revokeKey(key); + const video = state.videos.get(videoId); + if (video) video.deletedAt = "2026-11-12 10:00:01"; + + await refusedEverywhere(token, 404, "video_not_found"); + }); + + it("still accepts a token made before tokens named their key while no key of the app was revoked, until it expires", async () => { + const { videoId } = await createdWith(secretKey("app1")); + expect(videoId).toBe("video-1"); + const issuedBeforeKeyIds = + "cut_eyJ2IjoxLCJhcHAiOiJhcHAxIiwidmlkZW8iOiJ2aWRlby0xIiwiZXhwIjoxNzk0NDgxMjAwMDAwLCJvcmlnaW4iOm51bGx9.fMI6nUNp1BF0VWSS3A5xA1jeK3ghEPavCqVE4uW0xNI"; + expect(payloadOf(issuedBeforeKeyIds)).not.toHaveProperty("key"); + + const initiated = await upload("initiate", issuedBeforeKeyIds); + const completed = await upload( + "complete", + issuedBeforeKeyIds, + completeBody, + ); + + expect(initiated.status).toBe(200); + expect(completed.status).toBe(200); + expect(durationWrites()).toHaveLength(1); + vi.setSystemTime(Date.parse("2026-11-12T11:00:00Z")); + expect((await upload("initiate", issuedBeforeKeyIds)).status).toBe(401); + }); + + it("refuses a token made before tokens named their key once a secret key of the app was revoked in its lifetime", async () => { + const key = secretKey("app1"); + await createdWith(key); + const issuedBeforeKeyIds = + "cut_eyJ2IjoxLCJhcHAiOiJhcHAxIiwidmlkZW8iOiJ2aWRlby0xIiwiZXhwIjoxNzk0NDgxMjAwMDAwLCJvcmlnaW4iOm51bGx9.fMI6nUNp1BF0VWSS3A5xA1jeK3ghEPavCqVE4uW0xNI"; + revokeKey(key); + const storageBefore = storageCalls(); + + await refusedEverywhere(issuedBeforeKeyIds, 401, "invalid_upload_token"); + const response = await upload("initiate", issuedBeforeKeyIds); + expect(await response.json()).toMatchObject(ROTATED); + expect(storageCalls()).toBe(storageBefore); + expect(durationWrites()).toHaveLength(0); + }); + + it("gives a renewed token the key that renewed it", async () => { + const first = secretKey("app1"); + const second = secretKey("app1"); + const original = await createdWith(first); + + const response = await renew(original.videoId, {}, second); + expect(response.status).toBe(200); + const next = (await response.json()).data as Created; + + expect(payloadOf(original.token)).toMatchObject({ key: keyIdOf(first) }); + expect(payloadOf(next.token)).toMatchObject({ key: keyIdOf(second) }); + + revokeKey(first); + expect((await upload("initiate", original.token)).status).toBe(401); + expect((await upload("initiate", next.token)).status).toBe(200); + + revokeKey(second); + const refused = await upload("initiate", next.token); + expect(refused.status).toBe(401); + expect(await refused.json()).toMatchObject(ROTATED); + }); + + it("cannot be renewed with the revoked key", async () => { + const key = secretKey("app1"); + const { videoId } = await createdWith(key); + revokeKey(key); + + const response = await renew(videoId, {}, key); + + expect(response.status).toBe(401); + expect(await response.json()).toMatchObject({ code: "invalid_secret_key" }); + }); +}); + +describe("the part numbers a token can sign", () => { + const PART = 5 * 1024 * 1024; + const presign = (token: string, body: Record) => + upload("presign-part", token, { uploadId: "upload-1", ...body }); + + it.each([ + ["a cap of one byte", 1, 1], + ["a cap under one part", 1, PART - 1], + ["a cap of exactly one part", 1, PART], + ["a cap one byte over one part", 2, PART + 1], + ["a cap of exactly ten parts", 10, 10 * PART], + ["a cap of 250 MB", 48, 250_000_000], + ["the largest cap", 6867, 36_000_000_000], + ])("with %s, signs up to part %i", async (_label, highest, maxBytes) => { + const { token } = await created({ maxBytes }); + + const allowed = await presign(token, { partNumber: highest }); + const refused = await presign(token, { partNumber: highest + 1 }); + + expect(allowed.status).toBe(200); + expect(state.presign).toHaveBeenCalledExactlyOnceWith( + expect.any(String), + "upload-1", + highest, + ); + expect(refused.status).toBe(400); + expect(await refused.json()).toMatchObject({ + code: "part_number_too_large", + error: `Part number ${highest + 1} is over ${highest}, the highest part number this upload can use`, + }); + expect(state.presign).toHaveBeenCalledTimes(1); + }); + + it("names the cap and how to raise it", async () => { + const { token } = await created({ maxBytes: 250_000_000 }); + + const response = await presign(token, { partNumber: 49 }); + + expect(await response.json()).toMatchObject({ + code: "part_number_too_large", + error: + "Part number 49 is over 48, the highest part number this upload can use", + hint: "Every part except the last is at least 5 MB, so a file within the 250000000 byte limit set for this upload never needs a part number above 48. No part in this request was signed. Number the parts from 1 with no gaps and ask again, or get a token with a larger maxBytes from your server.", + }); + }); + + it("holds a token with no cap to what the upload limit allows", async () => { + const { token } = await created(); + + const allowed = await presign(token, { partNumber: 6867 }); + const refused = await presign(token, { partNumber: 6868 }); + + expect(allowed.status).toBe(200); + expect(refused.status).toBe(400); + expect(await refused.json()).toMatchObject({ + code: "part_number_too_large", + error: + "Part number 6868 is over 6867, the highest part number this upload can use", + hint: "Every part except the last is at least 5 MB, so a file within the 36 GB upload limit never needs a part number above 6867. No part in this request was signed. Number the parts from 1 with no gaps and ask again.", + }); + expect(state.presign).toHaveBeenCalledTimes(1); + }); + + it("refuses a whole batch when one number in it is over", async () => { + const { token } = await created({ maxBytes: 3 * PART }); + + const response = await presign(token, { partNumbers: [1, 2, 3, 4] }); + + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ + code: "part_number_too_large", + error: + "Part number 4 is over 3, the highest part number this upload can use", + }); + expect(state.presign).not.toHaveBeenCalled(); + + const within = await presign(token, { partNumbers: [1, 2, 3] }); + expect(within.status).toBe(200); + expect(state.presign).toHaveBeenCalledTimes(3); + }); + + it("refuses an over number wherever it sits in the batch", async () => { + const { token } = await created({ maxBytes: 3 * PART }); + + const response = await presign(token, { partNumbers: [9, 1, 2] }); + + expect(response.status).toBe(400); + expect(state.presign).not.toHaveBeenCalled(); + }); + + it("applies the cap of a renewed token", async () => { + const first = await created({ maxBytes: PART }); + const next = await renewed(first.videoId, { maxBytes: 4 * PART }); + + expect((await presign(first.token, { partNumber: 2 })).status).toBe(400); + expect((await presign(next.token, { partNumber: 4 })).status).toBe(200); + expect((await presign(next.token, { partNumber: 5 })).status).toBe(400); + }); + + it("still answers a number past 10000 as a malformed request", async () => { + const { token } = await created(); + + const response = await presign(token, { partNumber: 10_001 }); + + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: "invalid_request" }); + }); +}); + describe("signing several parts in one request", () => { const numbered = (count: number) => Array.from({ length: count }, (_, index) => index + 1); diff --git a/apps/web/__tests__/unit/developer-upload-token.test.ts b/apps/web/__tests__/unit/developer-upload-token.test.ts index f7f896e2796..b5263736600 100644 --- a/apps/web/__tests__/unit/developer-upload-token.test.ts +++ b/apps/web/__tests__/unit/developer-upload-token.test.ts @@ -235,6 +235,85 @@ describe("upload token size cap", () => { ); }); +describe("the key an upload token was made with", () => { + const issuedBeforeKeyIds = createDeveloperUploadToken(fields); + + it("carries the id of the key", () => { + const token = createDeveloperUploadToken({ ...fields, keyId: "key-1" }); + + expect(token).not.toBe(issuedBeforeKeyIds); + expect(verifyDeveloperUploadToken(token, NOW)).toEqual({ + ok: true, + token: { ...fields, keyId: "key-1" }, + }); + }); + + it("carries the key next to a cap", () => { + const token = createDeveloperUploadToken({ + ...fields, + maxBytes: 250_000_000, + keyId: "key-1", + }); + + expect(verifyDeveloperUploadToken(token, NOW)).toEqual({ + ok: true, + token: { ...fields, maxBytes: 250_000_000, keyId: "key-1" }, + }); + }); + + it("still verifies a token made without a key and reports none", () => { + const check = verifyDeveloperUploadToken(issuedBeforeKeyIds, NOW); + + expect(check).toEqual({ ok: true, token: fields }); + expect("keyId" in (check as { token: object }).token).toBe(false); + }); + + it.each([ + ["another key", { key: "key-2" }], + ["no key", {}], + ])( + "refuses a payload rewritten to name %s under the old signature", + (_label, change) => { + const token = createDeveloperUploadToken({ ...fields, keyId: "key-1" }); + const payload = encode({ + v: 1, + app: fields.appId, + video: fields.videoId, + exp: fields.expiresAt, + origin: fields.corsOrigin, + ...change, + }); + + expect( + verifyDeveloperUploadToken( + `cut_${payload}.${parts(token).signature}`, + NOW, + ), + ).toEqual({ ok: false, reason: "invalid" }); + }, + ); + + it.each(["", "k".repeat(256)])( + "does not issue a token with the key id %j", + (keyId) => { + expect(() => createDeveloperUploadToken({ ...fields, keyId })).toThrow(); + }, + ); + + it("answers a token of a rotated key as an invalid token that names the rotation", () => { + expect(uploadTokenRefusal("rotated")).toMatchObject({ + code: "invalid_upload_token", + error: "Upload token is not valid", + hint: expect.stringMatching( + /rotated or revoked.*POST \/api\/developer\/v1\/uploads\/\{videoId\}\/token.*current secret key/, + ), + }); + expect(Object.keys(uploadTokenRefusal("rotated")).sort()).toEqual( + Object.keys(uploadTokenRefusal("invalid")).sort(), + ); + }); +}); + describe("an expired upload token", () => { it("points at the route that renews it", () => { expect(uploadTokenRefusal("expired")).toMatchObject({ diff --git a/apps/web/__tests__/unit/developer-usage-counts.test.ts b/apps/web/__tests__/unit/developer-usage-counts.test.ts index 1b3373f3953..6431b11d59a 100644 --- a/apps/web/__tests__/unit/developer-usage-counts.test.ts +++ b/apps/web/__tests__/unit/developer-usage-counts.test.ts @@ -16,46 +16,33 @@ const mocks = vi.hoisted(() => ({ vi.mock("@cap/database", async () => { const schema = await import("@cap/database/schema"); - const rowsFor = (table: unknown, fields?: Record) => { + const rowsFor = (table: unknown) => { if (table === schema.developerApiKeys) return [{ appId: "app-1" }]; if (table === schema.developerCreditAccounts) { return [{ balanceMicroCredits: mocks.balanceMicroCredits }]; } if (table !== schema.developerVideos) return []; - if (fields && "createdVideos" in fields) { - return [ - { - createdVideos: mocks.videos.length, - uploadedVideos: mocks.videos.filter( - (video) => video.duration !== null, - ).length, - totalDurationMinutes: - mocks.videos.reduce( - (sum, video) => sum + (video.duration ?? 0), - 0, - ) / 60, - }, - ]; - } - return mocks.videos - .filter((video) => video.duration !== null) - .map((video) => ({ - appId: "app-1", - width: video.width ?? 1920, - height: video.height ?? 1080, - status: video.status, - maxResolutionTier: video.maxResolutionTier ?? "1080p", - seconds: video.duration, - count: 1, - })); + return mocks.videos.map((video) => ({ + appId: "app-1", + width: video.width ?? 1920, + height: video.height ?? 1080, + status: video.status, + maxResolutionTier: video.maxResolutionTier ?? "1080p", + seconds: video.duration ?? 0, + count: video.duration === null ? 0 : 1, + rows: 1, + dailySeconds: video.duration ?? 0, + dailyCount: video.duration === null ? 0 : 1, + })); }; - const select = (fields?: Record) => ({ + const select = () => ({ from: (table: unknown) => { const where = () => { - const rows = Promise.resolve(rowsFor(table, fields)); + const rows = Promise.resolve(rowsFor(table)); const result = Object.assign(rows, { limit: () => result, groupBy: () => result, + orderBy: () => result, }); return result; }; @@ -86,7 +73,7 @@ vi.mock("@/lib/infra-standing", async (original) => ({ })); import { app } from "@/app/api/developer/v1/[...route]/usage"; -import { storageUsage } from "@/lib/developer-pricing"; +import { getAppStorage } from "@/lib/infra-storage"; const usage = async () => { const response = await app.request("/", { @@ -99,16 +86,20 @@ const usage = async () => { beforeEach(() => { mocks.balanceMicroCredits = 2_999_901; mocks.videos = []; - mocks.billing.mockResolvedValue({ - standing: { state: "active", pausesAt: null }, - usage: storageUsage([]), - admission: { - allowed: true, - reason: null, - requiredMicroCredits: 1, - shortfallMicroCredits: 0, - message: null, - }, + mocks.billing.mockImplementation(async (appId: string) => { + const storage = await getAppStorage(appId); + return { + standing: { state: "active", pausesAt: null }, + storage, + stored: storage.stored, + admission: { + allowed: true, + reason: null, + requiredMicroCredits: 1, + shortfallMicroCredits: 0, + message: null, + }, + }; }); }); diff --git a/apps/web/__tests__/unit/developer-usage-route.test.ts b/apps/web/__tests__/unit/developer-usage-route.test.ts index 7054c4816ff..552e110696d 100644 --- a/apps/web/__tests__/unit/developer-usage-route.test.ts +++ b/apps/web/__tests__/unit/developer-usage-route.test.ts @@ -3,6 +3,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ rows: [] as Record[], speech: vi.fn(), + delivered: vi.fn(), })); vi.mock("@cap/database", () => { @@ -33,19 +34,23 @@ vi.mock("@/lib/developer-key-hash", () => ({ hashKey: async (key: string) => `hash:${key}`, })); vi.mock("@/lib/infra-standing", () => ({ - getInfraBilling: async () => ({}), + getInfraBilling: async () => ({ + stored: { videoCount: 2, totalMinutes: 12.5, monthlyMicroCredits: 1062.5 }, + storage: { uploadingVideos: 1 }, + }), infraBillingJson: () => ({ standing: { state: "active", pausesAt: null }, upload: { allowed: true }, }), infraStorageJson: () => ({ dailyMicroCredits: 0 }), })); -vi.mock("@/lib/infra-storage", () => ({ - getAppStorageUsage: async () => ({ videoCount: 2, totalMinutes: 12.5 }), -})); vi.mock("@/lib/developer-transcript-usage", () => ({ getSpeechUsage: mocks.speech, })); +vi.mock("@/lib/infra-delivery", async (importOriginal) => ({ + ...(await importOriginal()), + getDeliveredUsage: mocks.delivered, +})); import { app } from "@/app/api/developer/v1/[...route]/usage"; @@ -60,6 +65,8 @@ beforeEach(() => { }, ]; mocks.speech.mockReset(); + mocks.delivered.mockReset(); + mocks.delivered.mockResolvedValue(null); mocks.speech.mockResolvedValue({ balanceMicroCredits: 949_286, availableMicroCredits: 909_286, @@ -97,6 +104,29 @@ describe("GET /usage", () => { expect(mocks.speech).toHaveBeenCalledWith("app-1"); }); + it("says which app the key belongs to, so a credit link can name it", async () => { + const response = await app.request("/", { + headers: { Authorization: "Bearer csk_valid_key" }, + }); + const { data } = (await response.json()) as { + data: Record; + }; + expect(data.appId).toBe("app-1"); + }); + + it("adds the monthly cost of every stored video to the storage figures", async () => { + const response = await app.request("/", { + headers: { Authorization: "Bearer csk_valid_key" }, + }); + const { data } = (await response.json()) as { + data: Record; + }; + expect(data.storage).toEqual({ + dailyMicroCredits: 0, + storedMonthlyMicroCredits: 1063, + }); + }); + it("keeps every field it returned before", async () => { const response = await app.request("/", { headers: { Authorization: "Bearer csk_valid_key" }, @@ -117,4 +147,91 @@ describe("GET /usage", () => { ); expect(data.balanceDollars).toBe("9.49286"); }); + + it("reports delivery as null until it has been measured", async () => { + const response = await app.request("/", { + headers: { Authorization: "Bearer csk_valid_key" }, + }); + const { data } = (await response.json()) as { + data: Record; + }; + + expect(data).toHaveProperty("delivery", null); + expect(mocks.delivered).toHaveBeenCalledWith("app-1"); + }); + + it("reports delivered minutes in total and by rendition tier, with the last final day", async () => { + mocks.delivered.mockResolvedValue({ + seconds: 74_910, + secondsByTier: { + "360p": 750, + "720p": 20_160, + "1080p": 54_000, + "1440p": 0, + "2160p": 0, + }, + finalThrough: "2026-11-18", + }); + + const response = await app.request("/", { + headers: { Authorization: "Bearer csk_valid_key" }, + }); + const { data } = (await response.json()) as { + data: Record; + }; + + expect(data.delivery).toEqual({ + minutes: 1248.5, + minutesByTier: { + "360p": 12.5, + "720p": 336, + "1080p": 900, + "1440p": 0, + "2160p": 0, + }, + finalThrough: "2026-11-18", + }); + }); + + it("keeps answering with storage, speech and billing when the delivery read fails", async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + mocks.delivered.mockRejectedValue( + Object.assign(new Error("Failed query"), { + cause: Object.assign(new Error("Lock wait timeout exceeded"), { + code: "ER_LOCK_WAIT_TIMEOUT", + errno: 1205, + }), + }), + ); + + const response = await app.request("/", { + headers: { Authorization: "Bearer csk_valid_key" }, + }); + + expect(response.status).toBe(200); + const { data } = (await response.json()) as { + data: Record; + }; + expect(data.delivery).toBeNull(); + expect(data).toMatchObject({ + balanceMicroCredits: 949_286, + totalVideos: 2, + uploadingVideos: 1, + totalDurationMinutes: 12.5, + standing: { state: "active", pausesAt: null }, + upload: { allowed: true }, + storage: { dailyMicroCredits: 0 }, + speech: { + periodStart: "2026-11-01T00:00:00.000Z", + seconds: 18_252.4, + microCredits: 50_714, + transcripts: 104, + }, + }); + expect(errors).toHaveBeenCalledWith( + "[developer-usage] Delivery usage unavailable", + expect.any(Error), + ); + errors.mockRestore(); + }); }); diff --git a/apps/web/__tests__/unit/developer-usage-speech.test.ts b/apps/web/__tests__/unit/developer-usage-speech.test.ts index f79c5187aee..e146b1a498a 100644 --- a/apps/web/__tests__/unit/developer-usage-speech.test.ts +++ b/apps/web/__tests__/unit/developer-usage-speech.test.ts @@ -148,7 +148,7 @@ describe("speech usage for an app", () => { }); }); - it("reports credit held by running transcripts and what is left available", async () => { + it("reports credit held by running transcripts, deleted ones included, and what is left available", async () => { seed("running-a", { status: "processing", reservedMicroCredits: 40_000, @@ -169,9 +169,9 @@ describe("speech usage for an app", () => { const usage = await getSpeechUsage("app-1", NOW); - expect(usage.heldMicroCredits).toBe(45_000); + expect(usage.heldMicroCredits).toBe(85_000); expect(usage.balanceMicroCredits).toBe(949_286); - expect(usage.availableMicroCredits).toBe(949_286 - 45_000); + expect(usage.availableMicroCredits).toBe(949_286 - 85_000); expect(usage.speech.microCredits).toBe(0); }); diff --git a/apps/web/__tests__/unit/developer-video-api.test.ts b/apps/web/__tests__/unit/developer-video-api.test.ts index ed081dba51a..03d1c5a0245 100644 --- a/apps/web/__tests__/unit/developer-video-api.test.ts +++ b/apps/web/__tests__/unit/developer-video-api.test.ts @@ -1,5 +1,6 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; +import { DEVELOPER_VIDEO_TAKEDOWN_KEY } from "@cap/database/types"; import { MySqlDialect } from "drizzle-orm/mysql-core"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { @@ -13,6 +14,16 @@ const mocks = vi.hoisted(() => ({ appId: "app-1", playbackUrl: undefined as string | undefined, redrive: vi.fn(), + settleAtDelete: vi.fn(), + cancelJob: vi.fn(), + createImport: vi.fn(), + importView: vi.fn(), + startWorkflow: vi.fn(), + purge: vi.fn(), +})); + +vi.mock("@vercel/functions", () => ({ + dangerouslyDeleteByTag: mocks.purge, })); vi.mock("@cap/database", async () => { @@ -31,11 +42,29 @@ vi.mock("@cap/env", () => ({ vi.mock("@/lib/developer-video-start", () => ({ redriveDeveloperVideoProcessing: mocks.redrive, })); +vi.mock("@/lib/infra-storage-minimum", () => ({ + settleFirstMonthAtDelete: mocks.settleAtDelete, +})); + +vi.mock("@/lib/infra-video-import", () => ({ + createInfraVideoImport: mocks.createImport, + developerVideoImportView: mocks.importView, + newImportVideoId: () => "new-video-1", +})); +vi.mock("@/workflows/import-developer-video", () => ({ + importDeveloperVideoWorkflow: "import-workflow", +})); +vi.mock("workflow/api", () => ({ start: mocks.startWorkflow })); + +vi.mock("@/lib/media-client", () => ({ + cancelMediaServerVideoJob: mocks.cancelJob, +})); vi.mock("@/app/api/utils", async () => { const { createMiddleware } = await import("hono/factory"); return { withDeveloperSecretAuth: createMiddleware(async (c, next) => { c.set("developerAppId", mocks.appId); + c.set("developerKeyId", "key-1"); c.set("developerKeyType", "secret"); await next(); }), @@ -43,6 +72,9 @@ vi.mock("@/app/api/utils", async () => { }); const { app } = await import("@/app/api/developer/v1/[...route]/videos"); +const { markDeveloperVideoDeleted } = await import( + "@/lib/developer-video-delete" +); const BASE = "https://web.example/api/developer/playback/playback-id-0123456789ab"; @@ -107,9 +139,11 @@ beforeEach(() => { mocks.store.video = ready(); mocks.store.more = []; mocks.store.updates = []; + mocks.store.captionRows = []; mocks.appId = "app-1"; mocks.playbackUrl = undefined; mocks.redrive.mockResolvedValue(false); + mocks.cancelJob.mockResolvedValue("cancelled"); }); describe("developer video API", () => { @@ -123,6 +157,7 @@ describe("developer video API", () => { externalUserId: null, name: "Untitled", duration: 12, + durationSeconds: 12, width: 1920, height: 1080, fps: 30, @@ -146,11 +181,194 @@ describe("developer video API", () => { deletedAt: null, createdAt: "2026-11-01T00:00:00.000Z", updatedAt: "2026-11-01T00:00:00.000Z", + takenDown: false, hlsUrl: `${BASE}/master.m3u8`, mp4Url: `${BASE}/video.mp4`, posterUrl: `${BASE}/poster.jpg`, + outputs: { hls: "ready", mp4: "ready" }, shareUrl: PAGE, embedUrl: `${PAGE}/embed`, + captions: [], + }); + }); + + it("lists the captions made from the video's transcripts on get and list", async () => { + mocks.store.captionRows = [ + { videoId: "video-1", language: "fr" }, + { videoId: "video-1", language: "en" }, + { videoId: "video-1", language: null }, + { videoId: "video-other", language: "de" }, + ]; + const captions = [ + { language: "en", url: `${BASE}/captions-en.vtt` }, + { language: "fr", url: `${BASE}/captions-fr.vtt` }, + { language: "und", url: `${BASE}/captions-und.vtt` }, + ]; + + expect((await get("/video-1")).body.data.captions).toEqual(captions); + expect((await get("/")).body.data[0].captions).toEqual(captions); + }); + + it("lists no captions for a video that was taken down", async () => { + mocks.store.captionRows = [{ videoId: "video-1", language: "en" }]; + const video = ready(); + video.metadata = { + [DEVELOPER_VIDEO_TAKEDOWN_KEY]: { at: "2026-11-01", metadata: null }, + }; + mocks.store.video = video; + + const { body } = await get("/video-1"); + + expect(body.data.takenDown).toBe(true); + expect(body.data.captions).toEqual([]); + }); + + describe("output states", () => { + const outputsOf = async (path: string) => + (await get(path)).body.data.outputs; + const withParts = ( + status: "uploading" | "processing" | "ready" | "errored", + hls: Record, + mp4: Record, + ) => { + const base = ready(); + mocks.store.video = { + ...base, + status, + processing: processingState({ + ...base.processing, + hls, + mp4, + } as never), + }; + }; + + it("says an MP4 that failed for good will not come on a ready video", async () => { + withParts( + "ready", + { state: "complete", output: "hls-out" }, + { state: "failed", reason: "encoder refused" }, + ); + for (const path of ["/video-1", "/video-1/status"]) { + const { body } = await get(path); + expect(body.data.hlsUrl).toBe(`${BASE}/master.m3u8`); + expect(body.data.mp4Url).toBeNull(); + expect(body.data.outputs).toEqual({ hls: "ready", mp4: "failed" }); + } + }); + + it("says an MP4 still encoding or waiting for a retry is still coming", async () => { + withParts( + "ready", + { state: "complete", output: "hls-out" }, + { state: "running", token: "t", jobId: null, startedAt: 1 }, + ); + expect(await outputsOf("/video-1")).toEqual({ + hls: "ready", + mp4: "processing", + }); + withParts( + "ready", + { state: "complete", output: "hls-out" }, + { state: "unavailable", reason: "busy", retryAt: 2 }, + ); + expect(await outputsOf("/video-1")).toEqual({ + hls: "ready", + mp4: "processing", + }); + }); + + it("never says ready while the URL is still null", async () => { + withParts( + "processing", + { state: "complete", output: "hls-out" }, + { state: "pending" }, + ); + const { body } = await get("/video-1"); + expect(body.data.hlsUrl).toBeNull(); + expect(body.data.outputs).toEqual({ + hls: "processing", + mp4: "processing", + }); + }); + + it("says nothing is coming for an errored video", async () => { + withParts("errored", { state: "pending" }, { state: "pending" }); + expect(await outputsOf("/video-1/status")).toEqual({ + hls: "failed", + mp4: "failed", + }); + }); + + it("reads a video still uploading as processing", async () => { + mocks.store.video = { ...ready(), status: "uploading", processing: null }; + expect(await outputsOf("/video-1")).toEqual({ + hls: "processing", + mp4: "processing", + }); + }); + + it("reports no duration for a video that failed before it was measured", async () => { + const base = ready(); + mocks.store.video = { + ...base, + status: "errored", + errorReason: "This file could not be read as a video.", + duration: 0.02, + processing: processingState({ + ...base.processing, + probe: null, + } as never), + }; + for (const path of ["/video-1", "/video-1/status"]) { + expect((await get(path)).body.data.duration).toBeNull(); + } + }); + + it("reports no duration when a length measured at upload was never confirmed by processing", async () => { + mocks.store.video = { + ...ready(), + status: "errored", + errorReason: "Processing took too long. Upload the video again.", + duration: 60, + processing: null, + }; + for (const path of ["/video-1", "/video-1/status"]) { + expect((await get(path)).body.data.duration).toBeNull(); + } + }); + + it("reports no duration when processing read the file but not its length", async () => { + const base = ready(); + mocks.store.video = { + ...base, + status: "errored", + duration: 45, + processing: processingState({ + ...base.processing, + probe: { duration: null, width: 1920, height: 1080, fps: 30 }, + } as never), + }; + for (const path of ["/video-1", "/video-1/status"]) { + expect((await get(path)).body.data.duration).toBeNull(); + } + }); + + it("keeps the measured duration of a video that failed after it was measured", async () => { + mocks.store.video = { ...ready(), status: "errored", duration: 12 }; + for (const path of ["/video-1", "/video-1/status"]) { + expect((await get(path)).body.data.duration).toBe(12); + } + }); + + it("says none for a video uploaded before encoding existed", async () => { + mocks.store.video = { + ...ready(), + status: "ready", + playbackId: null, + processing: null, + }; + expect(await outputsOf("/video-1")).toEqual({ hls: "none", mp4: "none" }); }); }); @@ -175,17 +393,66 @@ describe("developer video API", () => { } }); + it.each([ + ["a measured video", { duration: 95.25 }], + ["a video with no length yet", { duration: null }], + ])( + "gives durationSeconds the value of duration for %s, on get, list and status", + async (_name, row) => { + mocks.store.video = { ...ready(), ...row }; + + const got = (await get("/video-1")).body.data; + const listed = (await get("/")).body.data[0]; + const status = (await get("/video-1/status")).body.data; + for (const video of [got, listed, status]) { + expect(video.durationSeconds).toBe(row.duration); + expect(video.duration).toBe(row.duration); + } + }, + ); + + it("marks duration deprecated in favour of durationSeconds in the OpenAPI description only", async () => { + const { developerOpenApiSpec } = await import("@/lib/developer-openapi"); + const schemas = developerOpenApiSpec("https://cap.so").components.schemas; + + for (const schema of [schemas.DeveloperVideo, schemas.VideoStatus]) { + expect(schema.properties.duration.description).toMatch( + /^Deprecated: use durationSeconds/, + ); + expect(schema.properties.duration).not.toHaveProperty("deprecated"); + expect(schema.properties.durationSeconds.type).toEqual([ + "number", + "null", + ]); + } + }); + it("returns exactly the fields the OpenAPI description lists", async () => { const { developerOpenApiSpec } = await import("@/lib/developer-openapi"); const schemas = developerOpenApiSpec("https://cap.so").components.schemas; + const always = (properties: object) => + Object.keys(properties) + .filter((name) => name !== "import") + .sort(); + const video = await get("/video-1"); expect(Object.keys(video.body.data).sort()).toEqual( - Object.keys(schemas.DeveloperVideo.properties).sort(), + always(schemas.DeveloperVideo.properties), ); const status = await get("/video-1/status"); expect(Object.keys(status.body.data).sort()).toEqual( - Object.keys(schemas.VideoStatus.properties).sort(), + always(schemas.VideoStatus.properties), + ); + + mocks.importView.mockResolvedValueOnce({ + sourceUrl: "https://media.example.com/a.mp4", + state: "complete", + errorCode: null, + }); + const imported = await get("/video-1"); + expect(Object.keys(imported.body.data).sort()).toEqual( + Object.keys(schemas.DeveloperVideo.properties).sort(), ); }); @@ -194,7 +461,9 @@ describe("developer video API", () => { const status = Object.keys((await get("/video-1/status")).body.data).sort(); const reference = [ ...docsSection("## Video Object Reference").matchAll(/^\| `(\w+)` \|/gm), - ].map((row) => row[1]); + ] + .map((row) => row[1]) + .filter((name) => name !== "import"); expect(reference.sort()).toEqual(video); expect(Object.keys(docsExampleResponse("### Get Video")).sort()).toEqual( @@ -627,6 +896,159 @@ describe("developer video API", () => { expect(response.status).toBe(200); expect(await response.json()).toEqual({ success: true }); expect(mocks.store.updates).toEqual([{ deletedAt: expect.any(Date) }]); + expect(mocks.settleAtDelete).toHaveBeenCalledTimes(1); + expect(mocks.settleAtDelete).toHaveBeenCalledWith("video-1"); + }); + + it("drops the video's cached playback once it is marked deleted", async () => { + mocks.purge.mockReset(); + mocks.purge.mockImplementation(async () => { + expect(mocks.store.video?.deletedAt).toBeInstanceOf(Date); + }); + + expect((await remove()).status).toBe(200); + expect(mocks.purge.mock.calls).toEqual([["infra-video-video-1"]]); + }); + + it("still deletes the video when the cache purge fails", async () => { + mocks.purge.mockReset(); + mocks.purge.mockRejectedValue(new Error("429")); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + const response = await remove(); + + expect(response.status).toBe(200); + expect(mocks.store.video?.deletedAt).toBeInstanceOf(Date); + expect(mocks.settleAtDelete).toHaveBeenCalledWith("video-1"); + mocks.purge.mockReset(); + }); + + it("cancels the encode jobs of a video deleted mid-encode", async () => { + mocks.store.video = developerVideo({ + processing: processingState({ + hls: { + state: "running", + token: "t-hls", + jobId: "job-hls", + startedAt: 2_000, + }, + mp4: { + state: "running", + token: "t-mp4", + jobId: "job-mp4", + startedAt: 2_000, + }, + }), + }); + mocks.cancelJob.mockImplementation(async () => { + expect(mocks.store.video?.deletedAt).toBeInstanceOf(Date); + return "cancelled"; + }); + + const response = await remove(); + + expect(response.status).toBe(200); + expect(mocks.cancelJob.mock.calls.flat().sort()).toEqual([ + "job-hls", + "job-mp4", + ]); + + await remove(); + expect(mocks.cancelJob).toHaveBeenCalledTimes(2); + }); + + it("cancels only the part that is still running and has a job", async () => { + mocks.store.video = developerVideo({ + processing: processingState({ + hls: { + state: "running", + token: "t-hls", + jobId: null, + startedAt: 2_000, + }, + mp4: { + state: "running", + token: "t-mp4", + jobId: "job-mp4", + startedAt: 2_000, + }, + }), + }); + + await remove(); + + expect(mocks.cancelJob.mock.calls).toEqual([["job-mp4"]]); + }); + + it("answers as before and settles and cancels once, also on a repeat delete", async () => { + const now = new Date("2026-11-13T12:00:00.000Z"); + mocks.store.video = developerVideo({ + processing: processingState({ + hls: { + state: "running", + token: "t-hls", + jobId: "job-hls", + startedAt: 2_000, + }, + }), + }); + + const first = await remove(); + const repeated = await remove(); + + expect([first.status, repeated.status]).toEqual([200, 200]); + expect(await first.text()).toBe('{"success":true}'); + expect(await repeated.text()).toBe('{"success":true}'); + expect(mocks.store.video?.deletedAt).toBeInstanceOf(Date); + expect(mocks.store.updates).toEqual([{ deletedAt: expect.any(Date) }]); + expect(mocks.settleAtDelete.mock.calls).toEqual([["video-1"]]); + expect(mocks.cancelJob.mock.calls).toEqual([["job-hls"]]); + + mocks.store.updates = []; + mocks.store.video = { ...mocks.store.video, deletedAt: null } as never; + await markDeveloperVideoDeleted( + { appId: "app-1", videoId: "video-1" }, + now, + ); + expect(mocks.store.updates).toEqual([{ deletedAt: now }]); + expect(mocks.settleAtDelete.mock.calls).toEqual([ + ["video-1"], + ["video-1"], + ]); + expect(mocks.cancelJob.mock.calls).toEqual([["job-hls"], ["job-hls"]]); + }); + + it("cancels nothing for a video that finished encoding", async () => { + const response = await remove(); + + expect(response.status).toBe(200); + expect(mocks.cancelJob).not.toHaveBeenCalled(); + }); + + it("still deletes when the media server cannot cancel the job", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + mocks.store.video = developerVideo({ + processing: processingState({ + mp4: { + state: "running", + token: "t-mp4", + jobId: "job-mp4", + startedAt: 2_000, + }, + }), + }); + mocks.cancelJob.mockResolvedValue("failed"); + + const response = await remove(); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ success: true }); + expect(mocks.store.video?.deletedAt).toBeInstanceOf(Date); + expect(warn).toHaveBeenCalledWith( + "[developer-video] Could not cancel encode job", + { videoId: "video-1", jobId: "job-mp4" }, + ); + warn.mockRestore(); }); it("answers a repeated delete with success and leaves the row alone", async () => { @@ -639,6 +1061,7 @@ describe("developer video API", () => { expect(await second.json()).toEqual({ success: true }); expect(mocks.store.updates).toHaveLength(1); expect(mocks.store.video?.deletedAt).toBe(deletedAt); + expect(mocks.settleAtDelete).toHaveBeenCalledTimes(1); }); it("still answers 404 for a video of another app, deleted or not", async () => { @@ -703,6 +1126,7 @@ describe("developer video API", () => { id: "video-1", externalUserId: null, duration: 12, + durationSeconds: 12, width: 1920, height: 1080, transcriptionStatus: null, @@ -711,9 +1135,11 @@ describe("developer video API", () => { errorReason: null, playbackId: "playback-id-0123456789ab", renditions: ready().renditions, + takenDown: false, hlsUrl: `${BASE}/master.m3u8`, mp4Url: `${BASE}/video.mp4`, posterUrl: `${BASE}/poster.jpg`, + outputs: { hls: "ready", mp4: "ready" }, }); }); @@ -865,3 +1291,165 @@ describe("developer video API", () => { expect((await get("/video-1/status")).status).toBe(404); }); }); + +describe("import from a URL", () => { + const post = (body: unknown, headers: Record = {}) => + app.request("/", { + method: "POST", + headers: { "content-type": "application/json", ...headers }, + body: JSON.stringify(body), + }); + + beforeEach(() => { + mocks.store.video = developerVideo({ status: "uploading" }); + mocks.createImport.mockResolvedValue({ + status: 202, + videoId: "video-1", + replayed: false, + }); + mocks.importView.mockResolvedValue({ + sourceUrl: "https://media.example.com/a.mp4", + state: "downloading", + errorCode: null, + }); + }); + + it("starts an import with the key, a new video id and the fields sent", async () => { + const response = await post({ + url: "https://media.example.com/a.mp4", + name: "Launch film", + userId: "user-7", + maxResolutionTier: "1440p", + metadata: { campaign: "spring" }, + }); + expect(response.status).toBe(200); + const [input, deps] = mocks.createImport.mock.calls[0] ?? []; + expect(input).toEqual({ + url: "https://media.example.com/a.mp4", + name: "Launch film", + userId: "user-7", + maxResolutionTier: "1440p", + metadata: { campaign: "spring" }, + appId: "app-1", + keyId: "key-1", + idempotencyKey: undefined, + videoId: "new-video-1", + }); + const body = await response.json(); + expect(body.data).toMatchObject({ + id: "video-1", + status: "uploading", + import: { + sourceUrl: "https://media.example.com/a.mp4", + state: "downloading", + errorCode: null, + }, + }); + + await deps.startImport("video-1"); + expect(mocks.startWorkflow).toHaveBeenCalledWith("import-workflow", [ + { videoId: "video-1" }, + ]); + }); + + it("derives the video id from an Idempotency-Key", async () => { + const { developerVideoIdForIdempotencyKey } = await import( + "@/lib/developer-ids" + ); + await post( + { url: "https://media.example.com/a.mp4" }, + { "idempotency-key": "import-key-1" }, + ); + expect(mocks.createImport.mock.calls[0]?.[0]).toMatchObject({ + idempotencyKey: "import-key-1", + videoId: developerVideoIdForIdempotencyKey("app-1", "import-key-1"), + }); + }); + + it("keeps the id of a video imported under the key before keyed ids changed", async () => { + const { legacyDeveloperVideoIdForIdempotencyKey } = await import( + "@/lib/developer-ids" + ); + const legacy = legacyDeveloperVideoIdForIdempotencyKey( + "app-1", + "import-key-1", + ); + mocks.store.video = { ...ready(), id: legacy }; + + await post( + { url: "https://media.example.com/a.mp4" }, + { "idempotency-key": "import-key-1" }, + ); + expect(mocks.createImport.mock.calls[0]?.[0]).toMatchObject({ + idempotencyKey: "import-key-1", + videoId: legacy, + }); + }); + + it.each([ + ["no url", {}], + ["an empty url", { url: "" }], + ["an unknown field", { url: "https://media.example.com/a.mp4", file: "x" }], + [ + "a misspelt field", + { url: "https://media.example.com/a.mp4", userid: "u" }, + ], + ["a url that is not a string", { url: 42 }], + [ + "a url over the length limit", + { url: `https://a.example/${"a".repeat(2_050)}` }, + ], + [ + "upload-only options", + { + url: "https://media.example.com/a.mp4", + corsOrigin: "https://a.example", + }, + ], + ])("refuses %s with 400 before doing anything", async (_label, body) => { + const response = await post(body); + expect(response.status).toBe(400); + expect((await response.json()).code).toBe("invalid_request"); + expect(mocks.createImport).not.toHaveBeenCalled(); + }); + + it("refuses a malformed Idempotency-Key", async () => { + const response = await post( + { url: "https://media.example.com/a.mp4" }, + { "idempotency-key": "no spaces allowed" }, + ); + expect(response.status).toBe(400); + expect(mocks.createImport).not.toHaveBeenCalled(); + }); + + it("shows the import on Get Video and Get Video Status, and nothing on other videos", async () => { + for (const path of ["/video-1", "/video-1/status"]) { + const imported = await get(path); + expect(imported.body.data.import).toEqual({ + sourceUrl: "https://media.example.com/a.mp4", + state: "downloading", + errorCode: null, + }); + } + expect(mocks.importView).toHaveBeenCalledWith("video-1"); + + mocks.importView.mockResolvedValue(null); + for (const path of ["/video-1", "/video-1/status"]) { + expect(Object.keys((await get(path)).body.data)).not.toContain("import"); + } + const listed = await get("/"); + expect(Object.keys(listed.body.data[0])).not.toContain("import"); + }); + + it("passes refusals through with their status and Retry-After", async () => { + mocks.createImport.mockResolvedValueOnce({ + status: 429, + body: { code: "too_many_imports", error: "busy" }, + headers: { "Retry-After": "15" }, + }); + const response = await post({ url: "https://media.example.com/a.mp4" }); + expect(response.status).toBe(429); + expect(response.headers.get("Retry-After")).toBe("15"); + expect((await response.json()).code).toBe("too_many_imports"); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-cache.test.ts b/apps/web/__tests__/unit/developer-video-cache.test.ts new file mode 100644 index 00000000000..1dbe33d4141 --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-cache.test.ts @@ -0,0 +1,39 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const purge = vi.hoisted(() => vi.fn()); +vi.mock("@vercel/functions", () => ({ dangerouslyDeleteByTag: purge })); + +const cache = await import("@/lib/developer-video-cache"); + +beforeEach(() => { + purge.mockReset(); + purge.mockResolvedValue(undefined); +}); + +describe("developer playback cache", () => { + it("tags an answer with its video and its app", () => { + expect( + cache.developerPlaybackCacheTags({ id: "video-1", appId: "app-1" }), + ).toBe("infra-video-video-1,infra-app-app-1"); + }); + + it("deletes a video's or an app's cached answers at once rather than marking them stale", async () => { + await cache.purgeDeveloperPlaybackCache({ videoId: "video-1" }); + await cache.purgeDeveloperPlaybackCache({ appId: "app-1" }); + + expect(purge.mock.calls).toEqual([ + ["infra-video-video-1"], + ["infra-app-app-1"], + ]); + }); + + it("logs a failed purge instead of throwing it", async () => { + purge.mockRejectedValue(new Error("429")); + const log = vi.spyOn(console, "error").mockImplementation(() => undefined); + + await expect( + cache.purgeDeveloperPlaybackCache({ videoId: "video-1" }), + ).resolves.toBeUndefined(); + expect(log).toHaveBeenCalledOnce(); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-cancel.test.ts b/apps/web/__tests__/unit/developer-video-cancel.test.ts new file mode 100644 index 00000000000..37720c55e78 --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-cancel.test.ts @@ -0,0 +1,187 @@ +import { MySqlDialect } from "drizzle-orm/mysql-core"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { processingState } from "../fixtures/developer-video"; + +const mocks = vi.hoisted(() => ({ + cancelJob: vi.fn(), + rows: [] as { id: string; processing: unknown }[], + where: [] as unknown[], + readFails: false, +})); + +vi.mock("@/lib/media-client", () => ({ + cancelMediaServerVideoJob: mocks.cancelJob, +})); +vi.mock("@cap/database", () => ({ + db: () => ({ + select: () => ({ + from: () => ({ + where: async (condition: unknown) => { + mocks.where.push(condition); + if (mocks.readFails) throw new Error("database unavailable"); + return mocks.rows; + }, + }), + }), + }), +})); + +const { + cancelDeveloperVideoEncodes, + cancelDeveloperVideoJobs, + runningDeveloperVideoJobIds, +} = await import("@/lib/developer-video-cancel"); + +const running = (jobId: string | null) => ({ + state: "running" as const, + token: `t-${jobId}`, + jobId, + startedAt: 2_000, +}); + +const whereSql = () => { + const [condition] = mocks.where; + return new MySqlDialect().sqlToQuery( + condition as Parameters[0], + ); +}; + +let warn: ReturnType; + +beforeEach(() => { + vi.clearAllMocks(); + mocks.rows = []; + mocks.where = []; + mocks.readFails = false; + mocks.cancelJob.mockResolvedValue("cancelled"); + warn = vi.spyOn(console, "warn").mockImplementation(() => {}); +}); + +describe("runningDeveloperVideoJobIds", () => { + it("returns the job of every running part", () => { + expect( + runningDeveloperVideoJobIds( + processingState({ hls: running("job-hls"), mp4: running("job-mp4") }), + ).sort(), + ).toEqual(["job-hls", "job-mp4"]); + }); + + it("skips parts with no job id and parts that are not running", () => { + expect( + runningDeveloperVideoJobIds( + processingState({ + hls: running(null), + mp4: { state: "complete", output: "mp4-out" }, + }), + ), + ).toEqual([]); + expect( + runningDeveloperVideoJobIds( + processingState({ + hls: { state: "failed", reason: "no" }, + mp4: { state: "pending" }, + }), + ), + ).toEqual([]); + expect(runningDeveloperVideoJobIds(null)).toEqual([]); + }); +}); + +describe("cancelDeveloperVideoJobs", () => { + it("cancels every job at once", async () => { + const pending: (() => void)[] = []; + mocks.cancelJob.mockImplementation( + () => + new Promise((resolve) => { + pending.push(() => resolve("cancelled")); + }), + ); + + const done = cancelDeveloperVideoJobs("video-1", ["job-1", "job-2"]); + await vi.waitFor(() => expect(mocks.cancelJob).toHaveBeenCalledTimes(2)); + for (const resolve of pending) resolve(); + await done; + }); + + it("warns only when a cancel could not be delivered", async () => { + mocks.cancelJob.mockImplementation(async (jobId: string) => + jobId === "job-1" ? "failed" : jobId === "job-2" ? "finished" : "missing", + ); + + await cancelDeveloperVideoJobs("video-1", ["job-1", "job-2", "job-3"]); + + expect(warn).toHaveBeenCalledExactlyOnceWith( + "[developer-video] Could not cancel encode job", + { videoId: "video-1", jobId: "job-1" }, + ); + }); + + it("does nothing without a job id", async () => { + await cancelDeveloperVideoJobs("video-1", []); + + expect(mocks.cancelJob).not.toHaveBeenCalled(); + }); +}); + +describe("cancelDeveloperVideoEncodes", () => { + it("reads one deleted video of the app and cancels its running jobs", async () => { + mocks.rows = [ + { + id: "video-1", + processing: processingState({ + hls: running("job-hls"), + mp4: running(null), + }), + }, + ]; + + await cancelDeveloperVideoEncodes({ appId: "app-1", videoId: "video-1" }); + + const query = whereSql(); + expect(query.sql).toContain("`developer_videos`.`appId` = ?"); + expect(query.sql).toContain("`developer_videos`.`id` = ?"); + expect(query.sql).toContain("`developer_videos`.`deletedAt` is not null"); + expect(query.params).toEqual(["app-1", "video-1"]); + expect(mocks.cancelJob.mock.calls).toEqual([["job-hls"]]); + }); + + it("reads only the videos the app delete just removed that still have a running part", async () => { + mocks.rows = [ + { id: "video-1", processing: processingState({ hls: running("job-1") }) }, + { id: "video-2", processing: processingState({ mp4: running("job-2") }) }, + ]; + + await cancelDeveloperVideoEncodes({ + appId: "app-1", + deletedAt: new Date("2026-11-13T06:00:00.750Z"), + }); + + const query = whereSql(); + expect(query.sql).toContain( + "JSON_UNQUOTE(JSON_EXTRACT(`developer_videos`.`processing`, '$.hls.state')) = 'running'", + ); + expect(query.sql).toContain( + "JSON_UNQUOTE(JSON_EXTRACT(`developer_videos`.`processing`, '$.mp4.state')) = 'running'", + ); + expect(query.sql).toContain("`developer_videos`.`deletedAt` is not null"); + expect(query.sql).toContain("`developer_videos`.`deletedAt` >= ?"); + expect(query.params).toEqual(["app-1", "2026-11-13 06:00:00.000"]); + expect(mocks.cancelJob.mock.calls.flat().sort()).toEqual([ + "job-1", + "job-2", + ]); + }); + + it("never fails the delete when the jobs cannot be read", async () => { + mocks.readFails = true; + + await expect( + cancelDeveloperVideoEncodes({ appId: "app-1", videoId: "video-1" }), + ).resolves.toBeUndefined(); + expect(mocks.cancelJob).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + "[developer-video] Could not cancel encode jobs", + expect.objectContaining({ appId: "app-1", videoId: "video-1" }), + ); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-captions-purge.test.ts b/apps/web/__tests__/unit/developer-video-captions-purge.test.ts new file mode 100644 index 00000000000..3798ec540d2 --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-captions-purge.test.ts @@ -0,0 +1,79 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const store = vi.hoisted(() => ({ keys: new Set() })); + +vi.mock("@cap/web-backend/src/S3Buckets/index", async () => { + const { Effect, Layer, Context, Option } = await import("effect"); + class S3Buckets extends Context.Tag("S3Buckets")() { + static Default = Layer.succeed(S3Buckets, {}); + static getBucketAccess = () => + Effect.succeed([ + { + listObjects: ({ prefix }: { prefix: string }) => + Effect.succeed({ + Contents: [...store.keys] + .filter((key) => key.startsWith(prefix)) + .map((Key) => ({ Key })), + }), + deleteObjects: (objects: { Key: string }[]) => { + for (const object of objects) store.keys.delete(object.Key); + return Effect.succeed(Option.some({})); + }, + }, + ]); + } + return { S3Buckets }; +}); +vi.mock("@cap/database", () => ({ db: () => ({}) })); + +const { deleteDeveloperVideoObjects, developerVideoPrefix } = await import( + "@/lib/developer-video-storage" +); +const { videoCaptionsKey } = await import("@/lib/developer-video-captions"); + +const VIDEO = { id: "video-1", appId: "app-1" }; +const OTHER = { id: "video-2", appId: "app-1" }; +const ENCODE = "developer/app-1/video-1/encode-1/"; + +beforeEach(() => { + store.keys = new Set([ + "developer/app-1/video-1/video", + `${ENCODE}out-1/video.mp4`, + `${ENCODE}out-1/hls/master.m3u8`, + `${ENCODE}out-0/hls/master.m3u8`, + videoCaptionsKey(VIDEO, "en"), + videoCaptionsKey(VIDEO, "de"), + videoCaptionsKey(OTHER, "en"), + "developer/app-1/transcripts/tr-1/result.json", + ]); +}); + +describe("caption files and the video file purge", () => { + it("keeps caption files inside the folder the purge deletes", () => { + expect(videoCaptionsKey(VIDEO, "en")).toBe( + `${developerVideoPrefix(VIDEO)}captions/en.vtt`, + ); + }); + + it("removes a deleted video's caption files with the rest of its folder", async () => { + const removed = await deleteDeveloperVideoObjects( + developerVideoPrefix(VIDEO), + ); + + expect(removed).toBe(6); + expect([...store.keys].sort()).toEqual([ + "developer/app-1/transcripts/tr-1/result.json", + videoCaptionsKey(OTHER, "en"), + ]); + }); + + it("leaves caption files alone when an encode's unused outputs are cleaned up", async () => { + await deleteDeveloperVideoObjects(ENCODE, (key) => + key.startsWith(`${ENCODE}out-1/`), + ); + + expect(store.keys.has(`${ENCODE}out-0/hls/master.m3u8`)).toBe(false); + expect(store.keys.has(videoCaptionsKey(VIDEO, "en"))).toBe(true); + expect(store.keys.has(videoCaptionsKey(VIDEO, "de"))).toBe(true); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-captions.test.ts b/apps/web/__tests__/unit/developer-video-captions.test.ts new file mode 100644 index 00000000000..bea66e653cf --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-captions.test.ts @@ -0,0 +1,1501 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const context = vi.hoisted(() => ({ + fake: null as null | { + db: unknown; + rows: (table: object) => Record[]; + seed: ( + table: object, + row: Record, + ) => Record; + reset: () => void; + }, + ids: 0, + start: vi.fn(), + objects: new Map(), + puts: [] as { key: string; contentType: string }[], + afterPut: null as null | (() => void), + afterDelete: null as null | (() => void), + failPuts: 0, + stuckKeys: new Set(), +})); + +vi.mock("@cap/database", async () => { + const schema = await import("@cap/database/schema"); + const { createFakeDatabase } = await import("./developer-transcript-fake-db"); + const fake = createFakeDatabase( + new Map Record>([ + [ + schema.developerTranscripts, + () => ({ + externalUserId: null, + name: "Untitled", + sourceUrl: null, + videoId: null, + resultKey: null, + sourceBytes: null, + durationSeconds: null, + requestedLanguage: null, + language: null, + text: null, + wordCount: null, + reservedMicroCredits: 0, + billedMicroCredits: 0, + errorCode: null, + errorMessage: null, + metadata: null, + runId: null, + queuedAt: null, + startedAt: null, + completedAt: null, + deletedAt: null, + }), + ], + [ + schema.developerVideos, + () => ({ + name: "Untitled", + duration: 12, + status: "ready", + processing: null, + deletedAt: null, + filesDeletedAt: null, + }), + ], + ]), + ); + context.fake = fake; + return { db: () => fake.db }; +}); + +vi.mock("@cap/database/helpers", () => ({ + nanoId: () => { + context.ids += 1; + return `generated-${context.ids}`; + }, + nanoIdLength: 15, +})); + +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + NODE_ENV: "production", + serverEnv: () => ({ WEB_URL: "https://web.example" }), +})); + +vi.mock("@/lib/infra-access", () => ({ canUseInfra: () => true })); + +vi.mock("workflow/api", () => ({ start: context.start })); + +vi.mock("@/workflows/developer-transcribe", () => ({ + developerTranscribeWorkflow: "developer-transcribe-workflow", +})); + +vi.mock("@/lib/developer-transcript-speech", () => ({ + isSpeechConfigured: () => true, + parseTranscriptLanguage: (value: string | null | undefined) => { + if (!value || value === "auto") return "auto"; + return ["en", "de"].includes(value) ? value : null; + }, +})); + +vi.mock("@/lib/developer-transcript-storage", () => ({ + presignDeveloperUpload: async () => "https://storage.example/upload", + developerObjectSize: async (key: string) => + context.objects.get(key)?.length ?? null, + getDeveloperObjectText: async (key: string) => + context.objects.get(key) ?? null, + putDeveloperObject: async ( + key: string, + body: string, + contentType: string, + ) => { + if (context.failPuts > 0) { + context.failPuts -= 1; + throw new Error("storage is unavailable"); + } + context.objects.set(key, body); + context.puts.push({ key, contentType }); + const hook = context.afterPut; + context.afterPut = null; + hook?.(); + }, + deleteDeveloperObjects: async (keys: string[]) => { + for (const key of keys) { + if (!context.stuckKeys.has(key)) context.objects.delete(key); + } + }, + removeDeveloperObject: async (key: string) => { + if (context.stuckKeys.has(key)) + throw new Error("storage refused the delete"); + context.objects.delete(key); + const hook = context.afterDelete; + context.afterDelete = null; + hook?.(); + }, +})); + +import { + developerApps, + developerCreditAccounts, + developerTranscripts, + developerVideos, + users, +} from "@cap/database/schema"; +import { + createTranscriptFromUrl, + createTranscriptFromVideo, + deleteTranscript, + deleteTranscriptObjects, + listTranscripts, + reconcileDeletedVideoCaptions, +} from "@/lib/developer-transcript-store"; +import { + serializeTranscript, + toAgentDeveloperTranscript, +} from "@/lib/developer-transcripts"; +import { + captionLanguage, + captionLanguageLabel, + captionsFileLanguage, + captionsFileName, + developerVideoCaptions, + developerVideoCaptionTracks, + developerVideosCaptionLinks, + findVideoForTranscript, + syncVideoCaptions, + videoCaptionsKey, +} from "@/lib/developer-video-captions"; +import { processingState } from "../fixtures/developer-video"; + +const VIDEO = { id: "video-1", appId: "app-1" }; +const MP4_KEY = "developer/app-1/video-1/encode-1/out-1/video.mp4"; +const EN_KEY = "developer/app-1/video-1/captions/en.vtt"; +const DE_KEY = "developer/app-1/video-1/captions/de.vtt"; + +const fake = () => { + if (!context.fake) throw new Error("Fake database is not ready"); + return context.fake; +}; + +const transcripts = () => fake().rows(developerTranscripts); + +const readyProcessing = () => + processingState({ + hls: { state: "complete", output: "out-1" }, + mp4: { state: "complete", output: "out-1" }, + }); + +const seedVideo = (row: Record = {}) => + fake().seed(developerVideos, { + id: "video-1", + appId: "app-1", + name: "Launch walkthrough", + playbackId: "playback-id-0123456789ab", + processing: readyProcessing(), + ...row, + }); + +const resultFor = (text: string) => + JSON.stringify({ + text, + language: "en", + durationSeconds: 10, + words: text.split(" ").map((word, index) => ({ + text: word, + start: index * 0.5, + end: index * 0.5 + 0.4, + })), + segments: [], + }); + +let requests = 0; + +const seedCompleted = ( + id: string, + text: string, + row: Record = {}, +) => { + requests += 1; + const resultKey = `developer/app-1/transcripts/${id}/result.json`; + context.objects.set(resultKey, resultFor(text)); + return fake().seed(developerTranscripts, { + id, + appId: "app-1", + status: "complete", + sourceType: "video", + videoId: "video-1", + language: "en", + text, + wordCount: text.split(" ").length, + resultKey, + requestedAt: new Date(Date.parse("2026-11-12T10:00:00.000Z") + requests), + completedAt: new Date("2026-11-12T10:00:00.000Z"), + ...row, + }); +}; + +beforeEach(() => { + fake().reset(); + context.ids = 0; + context.start.mockReset(); + context.start.mockResolvedValue(undefined); + context.objects = new Map(); + context.puts = []; + context.afterPut = null; + context.afterDelete = null; + context.failPuts = 0; + context.stuckKeys = new Set(); + fake().seed(users, { id: "owner-1", email: "owner@example.com" }); + for (const id of ["app-1", "app-2"]) { + fake().seed(developerApps, { id, ownerId: "owner-1", deletedAt: null }); + fake().seed(developerCreditAccounts, { + id: `account-${id}`, + appId: id, + balanceMicroCredits: 100_000, + }); + } +}); + +describe("creating a transcript from a stored video", () => { + it("queues a transcript that reads the video's stored MP4 and starts it once", async () => { + seedVideo(); + + const result = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + }); + + expect(result.ok).toBe(true); + expect(transcripts()).toHaveLength(1); + expect(transcripts()[0]).toMatchObject({ + id: "generated-1", + appId: "app-1", + status: "queued", + sourceType: "video", + videoId: "video-1", + sourceUrl: null, + s3Key: MP4_KEY, + name: "Launch walkthrough", + }); + expect(context.start).toHaveBeenCalledTimes(1); + if (!result.ok) throw new Error("create failed"); + expect(serializeTranscript(result.transcript)).toMatchObject({ + source: { type: "video", url: null }, + videoId: "video-1", + }); + expect(toAgentDeveloperTranscript(result.transcript)).toMatchObject({ + sourceType: "video", + videoId: "video-1", + }); + }); + + it("keeps the same options as a URL transcript", async () => { + seedVideo(); + + const result = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + name: "Captions run", + language: "de", + userId: "user-7", + metadata: { ticket: 42 }, + }); + + expect(result.ok).toBe(true); + expect(transcripts()[0]).toMatchObject({ + name: "Captions run", + requestedLanguage: "de", + externalUserId: "user-7", + metadata: { ticket: 42 }, + }); + expect( + await createTranscriptFromVideo("app-1", { + videoId: "video-1", + language: "xx", + }), + ).toMatchObject({ ok: false, status: 400, code: "invalid_language" }); + }); + + it.each([ + ["uploading", null, "The video is not ready yet"], + ["processing", processingState(), "The video is not ready yet"], + [ + "errored", + processingState({ mp4: { state: "failed", reason: "bad file" } }), + "The video failed to process and has no file to transcribe", + ], + [ + "ready", + processingState({ + hls: { state: "complete", output: "out-1" }, + mp4: { state: "failed", reason: "bad file" }, + }), + "The video has no stored file to transcribe", + ], + ])( + "answers 409 for a video that is %s without storing or starting anything", + async (status, processing, message) => { + seedVideo({ status, processing }); + + const result = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + }); + + expect(result).toMatchObject({ + ok: false, + status: 409, + code: "video_not_ready", + message, + }); + expect(transcripts()).toHaveLength(0); + expect(context.start).not.toHaveBeenCalled(); + }, + ); + + it("answers 404 for another app's video", async () => { + seedVideo({ appId: "app-2" }); + + expect( + await createTranscriptFromVideo("app-1", { videoId: "video-1" }), + ).toMatchObject({ ok: false, status: 404, code: "video_not_found" }); + expect(transcripts()).toHaveLength(0); + expect(context.start).not.toHaveBeenCalled(); + }); + + it("answers 404 for a deleted video and for one that never existed", async () => { + seedVideo({ deletedAt: new Date("2026-11-12T09:00:00.000Z") }); + + for (const videoId of ["video-1", "video-unknown"]) { + expect( + await createTranscriptFromVideo("app-1", { videoId }), + ).toMatchObject({ ok: false, status: 404, code: "video_not_found" }); + } + expect(transcripts()).toHaveLength(0); + }); + + it("answers 409 for a video whose files were removed", async () => { + seedVideo({ filesDeletedAt: new Date("2026-11-12T09:00:00.000Z") }); + + expect( + await createTranscriptFromVideo("app-1", { videoId: "video-1" }), + ).toMatchObject({ ok: false, status: 409, code: "video_not_ready" }); + }); + + it("is refused at the ceiling of 20 running transcripts, and a repeat of an admitted one is not", async () => { + seedVideo(); + for (let index = 0; index < 19; index++) { + fake().seed(developerTranscripts, { + id: `running-${index}`, + appId: "app-1", + status: "processing", + sourceType: "upload", + }); + } + + const admitted = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: "video-key-01", + }); + const refused = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + }); + const repeated = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: "video-key-01", + }); + + expect(admitted.ok).toBe(true); + expect(refused).toMatchObject({ + ok: false, + status: 429, + code: "too_many_transcripts", + }); + expect(repeated.ok).toBe(true); + expect(context.start).toHaveBeenCalledTimes(1); + }); + + it("refuses an empty or oversized video id", async () => { + for (const videoId of ["", " ", "v".repeat(65)]) { + expect( + await createTranscriptFromVideo("app-1", { videoId }), + ).toMatchObject({ ok: false, status: 400, code: "invalid_video_id" }); + } + }); + + it("answers 402 below the minimum balance", async () => { + seedVideo(); + const account = fake().rows(developerCreditAccounts)[0]; + if (account) account.balanceMicroCredits = 999; + + expect( + await createTranscriptFromVideo("app-1", { videoId: "video-1" }), + ).toMatchObject({ ok: false, status: 402, code: "insufficient_credits" }); + expect(transcripts()).toHaveLength(0); + }); + + it("lists a video transcript with its video id", async () => { + seedVideo(); + await createTranscriptFromVideo("app-1", { videoId: "video-1" }); + + const listed = await listTranscripts("app-1", { limit: 10 }); + + expect(listed.transcripts.map((row) => serializeTranscript(row))).toEqual([ + expect.objectContaining({ + videoId: "video-1", + source: { type: "video", url: null }, + }), + ]); + }); +}); + +describe("repeating a video transcript with an Idempotency-Key", () => { + const KEY = "captions-key-1"; + + it("returns the same transcript and starts it once", async () => { + seedVideo(); + const first = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }); + const second = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }); + + expect(first.ok && second.ok).toBe(true); + if (!first.ok || !second.ok) throw new Error("create failed"); + expect(second.transcript.id).toBe(first.transcript.id); + expect(transcripts()).toHaveLength(1); + expect(context.start).toHaveBeenCalledTimes(1); + }); + + it("refuses the key for a different video and names the field", async () => { + seedVideo(); + seedVideo({ id: "video-2" }); + await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }); + + expect( + await createTranscriptFromVideo("app-1", { + videoId: "video-2", + idempotencyKey: KEY, + }), + ).toMatchObject({ + ok: false, + status: 409, + code: "idempotency_conflict", + message: "Idempotency key was already used with a different videoId", + }); + expect(transcripts()).toHaveLength(1); + }); + + it.each([ + ["name", { name: "Other" }], + ["language", { language: "de" }], + ["userId", { userId: "user-9" }], + ["metadata", { metadata: { a: 1 } }], + ])("refuses the key with a different %s", async (field, change) => { + seedVideo(); + await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }); + + expect( + await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + ...change, + }), + ).toMatchObject({ + ok: false, + status: 409, + message: `Idempotency key was already used with a different ${field}`, + }); + }); + + it("refuses a key that was used for a URL transcript, and the reverse", async () => { + seedVideo(); + const guards = { isBlocked: async () => false }; + await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }); + await createTranscriptFromUrl( + "app-1", + { url: "https://media.example.com/a.mp3", idempotencyKey: "url-key-01" }, + guards, + ); + + expect( + await createTranscriptFromUrl( + "app-1", + { url: "https://media.example.com/a.mp3", idempotencyKey: KEY }, + guards, + ), + ).toMatchObject({ ok: false, status: 409, code: "idempotency_conflict" }); + expect( + await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: "url-key-01", + }), + ).toMatchObject({ ok: false, status: 409, code: "idempotency_conflict" }); + expect(transcripts()).toHaveLength(2); + }); + + it("still returns the transcript after the video was renamed or deleted", async () => { + const video = seedVideo(); + const first = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }); + video.name = "Renamed"; + video.deletedAt = new Date("2026-11-12T09:00:00.000Z"); + + const second = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }); + + if (!first.ok || !second.ok) throw new Error("create failed"); + expect(second.transcript.id).toBe(first.transcript.id); + }); +}); + +describe("the user of a transcript made from a video", () => { + const KEY = "user-key-01"; + + it("takes the video's userId when the request sends none", async () => { + seedVideo({ externalUserId: "user-7" }); + + const result = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + }); + + if (!result.ok) throw new Error("create failed"); + expect(transcripts()[0]?.externalUserId).toBe("user-7"); + expect(serializeTranscript(result.transcript)).toMatchObject({ + userId: "user-7", + }); + expect( + (await listTranscripts("app-1", { limit: 10, userId: "user-7" })) + .transcripts, + ).toHaveLength(1); + }); + + it("keeps a userId the request sends over the video's", async () => { + seedVideo({ externalUserId: "user-7" }); + + await createTranscriptFromVideo("app-1", { + videoId: "video-1", + userId: "user-9", + }); + + expect(transcripts()[0]?.externalUserId).toBe("user-9"); + }); + + it("stores no user for a video without one", async () => { + seedVideo({ externalUserId: null }); + + await createTranscriptFromVideo("app-1", { videoId: "video-1" }); + + expect(transcripts()[0]?.externalUserId).toBeNull(); + }); + + it("replays a retry that leaves userId out, or sends the video's", async () => { + seedVideo({ externalUserId: "user-7" }); + const first = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }); + const omitted = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }); + const explicit = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + userId: "user-7", + }); + + if (!first.ok || !omitted.ok || !explicit.ok) { + throw new Error("create failed"); + } + expect(omitted.transcript.id).toBe(first.transcript.id); + expect(explicit.transcript.id).toBe(first.transcript.id); + expect(transcripts()).toHaveLength(1); + expect(context.start).toHaveBeenCalledTimes(1); + }); + + it("replays a retry without userId of a request that sent the video's userId", async () => { + seedVideo({ externalUserId: "user-7" }); + const first = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + userId: "user-7", + }); + const retry = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }); + + if (!first.ok || !retry.ok) throw new Error("create failed"); + expect(retry.transcript.id).toBe(first.transcript.id); + }); + + it("still replays a retry without userId after the video was deleted", async () => { + const video = seedVideo({ externalUserId: "user-7" }); + const first = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }); + video.deletedAt = new Date("2026-11-12T09:00:00.000Z"); + + const retry = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }); + + if (!first.ok || !retry.ok) throw new Error("create failed"); + expect(retry.transcript.id).toBe(first.transcript.id); + }); + + it("replays a retry without userId of a transcript stored before userIds were inherited", async () => { + seedVideo({ externalUserId: "user-7" }); + const first = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }); + const row = transcripts()[0]; + if (row) row.externalUserId = null; + + const retry = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }); + + if (!first.ok || !retry.ok) throw new Error("create failed"); + expect(retry.transcript.id).toBe(first.transcript.id); + }); + + it("refuses a retry without userId when the first request sent a different one", async () => { + seedVideo({ externalUserId: "user-7" }); + await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + userId: "user-9", + }); + + expect( + await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }), + ).toMatchObject({ + ok: false, + status: 409, + message: "Idempotency key was already used with a different userId", + }); + }); + + it("refuses a retry with another userId than the inherited one", async () => { + seedVideo({ externalUserId: "user-7" }); + await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + }); + + expect( + await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: KEY, + userId: "user-9", + }), + ).toMatchObject({ + ok: false, + status: 409, + message: "Idempotency key was already used with a different userId", + }); + }); +}); + +describe("finding the stored file of a video", () => { + it("returns the MP4 of the finished encode and the known duration", async () => { + seedVideo({ duration: 95.5 }); + + expect(await findVideoForTranscript("app-1", "video-1")).toEqual({ + state: "ok", + name: "Launch walkthrough", + externalUserId: null, + sourceKey: MP4_KEY, + durationSeconds: 95.5, + }); + }); + + it("reports no duration when the video has none", async () => { + seedVideo({ duration: null }); + + expect(await findVideoForTranscript("app-1", "video-1")).toMatchObject({ + state: "ok", + durationSeconds: null, + }); + }); +}); + +describe("caption files for the playback side", () => { + it("returns nothing for a video with no transcripts", async () => { + expect(await developerVideoCaptions(VIDEO)).toEqual([]); + }); + + it("returns one caption file for one completed transcript", async () => { + seedCompleted("tr-1", "hello there"); + + expect(await developerVideoCaptions(VIDEO)).toEqual([ + { language: "en", label: "English", key: EN_KEY }, + ]); + }); + + it("returns one file per language, ordered by label", async () => { + seedCompleted("tr-1", "hello there"); + seedCompleted("tr-2", "hallo zusammen", { language: "de" }); + + expect(await developerVideoCaptions(VIDEO)).toEqual([ + { language: "en", label: "English", key: EN_KEY }, + { language: "de", label: "German", key: DE_KEY }, + ]); + }); + + it("returns one entry for a language however many transcripts it has", async () => { + seedCompleted("tr-old", "older words"); + seedCompleted("tr-new", "newer words"); + seedCompleted("tr-newest", "newest words"); + + expect(await developerVideoCaptions(VIDEO)).toEqual([ + { language: "en", label: "English", key: EN_KEY }, + ]); + }); + + it("leaves out transcripts that are not finished, failed, deleted, silent or for another video or app", async () => { + seedCompleted("tr-queued", "a b", { status: "queued", language: "de" }); + seedCompleted("tr-failed", "a b", { status: "error", language: "fr" }); + seedCompleted("tr-deleted", "a b", { + language: "es", + deletedAt: new Date("2026-11-12T11:00:00.000Z"), + }); + seedCompleted("tr-silent", "", { language: "it", wordCount: 0 }); + seedCompleted("tr-other-video", "a b", { + language: "nl", + videoId: "video-2", + }); + seedCompleted("tr-other-app", "a b", { language: "pl", appId: "app-2" }); + seedCompleted("tr-not-video", "a b", { language: "ja", videoId: null }); + + expect(await developerVideoCaptions(VIDEO)).toEqual([]); + }); + + it("labels a language it has no name for and one that was not reported", async () => { + expect(captionLanguageLabel("sv")).toBe("Swedish"); + expect(captionLanguageLabel("und")).toBe("Captions"); + expect(captionLanguage(null)).toBe("und"); + expect(captionLanguage("../../etc")).toBe("und"); + expect(captionLanguage("EN")).toBe("en"); + expect(videoCaptionsKey(VIDEO, "../x")).toBe( + "developer/app-1/video-1/captions/und.vtt", + ); + seedCompleted("tr-1", "hello there", { language: null }); + expect(await developerVideoCaptions(VIDEO)).toEqual([ + { + language: "und", + label: "Captions", + key: "developer/app-1/video-1/captions/und.vtt", + }, + ]); + }); +}); + +describe("listing a caption track only once its file is in storage", () => { + const PLAYABLE = { ...VIDEO, playbackId: "playback-1" }; + const fresh = (id: string, row: Record = {}) => + seedCompleted(id, "fresh words here", { + completedAt: new Date(), + updatedAt: new Date("2026-11-13T10:00:00.000Z"), + ...row, + }); + const listedEverywhere = async () => ({ + player: (await developerVideoCaptionTracks(PLAYABLE)).map( + (track) => track.language, + ), + api: ( + (await developerVideosCaptionLinks("app-1", [PLAYABLE])).get("video-1") ?? + [] + ).map((link) => link.language), + console: (await developerVideoCaptions(VIDEO)).map( + (caption) => caption.language, + ), + }); + + it("leaves out a just-completed transcript until its file is written, then lists it", async () => { + seedVideo(); + fresh("t-fresh"); + + expect(await listedEverywhere()).toEqual({ + player: [], + api: [], + console: [], + }); + + await syncVideoCaptions({ + appId: "app-1", + videoId: "video-1", + language: "en", + }); + + expect(context.objects.has(EN_KEY)).toBe(true); + expect(await listedEverywhere()).toEqual({ + player: ["en"], + api: ["en"], + console: ["en"], + }); + }); + + it("records the write only after storage took the file", async () => { + seedVideo(); + fresh("t-fresh"); + let recordedAtPut: unknown = "not called"; + context.afterPut = () => { + recordedAtPut = transcripts()[0]?.metadata ?? null; + }; + + await syncVideoCaptions({ + appId: "app-1", + videoId: "video-1", + language: "en", + }); + + expect(recordedAtPut).toBeNull(); + expect(transcripts()[0]?.metadata).toMatchObject({ + __capCaptionsWritten: { metadataWasNull: true }, + }); + }); + + it("keeps a track out when the upload fails", async () => { + seedVideo(); + fresh("t-fresh"); + context.failPuts = 1; + + await expect( + syncVideoCaptions({ appId: "app-1", videoId: "video-1", language: "en" }), + ).rejects.toThrow("storage is unavailable"); + + expect(context.objects.has(EN_KEY)).toBe(false); + expect(await listedEverywhere()).toEqual({ + player: [], + api: [], + console: [], + }); + }); + + it("still lists a transcript that completed before writes were recorded", async () => { + seedVideo(); + seedCompleted("t-old", "older words here", { + completedAt: new Date(Date.now() - 11 * 60 * 1000), + }); + + expect((await listedEverywhere()).api).toEqual(["en"]); + }); + + it("hides the record from the client and keeps metadata and updatedAt as they were", async () => { + seedVideo(); + const withNone = fresh("t-none", { metadata: null }); + const withSome = fresh("t-some", { + language: "de", + metadata: { ticket: 42 }, + }); + const before = { + none: serializeTranscript(withNone as never), + some: serializeTranscript(withSome as never), + }; + + for (const language of ["en", "de"]) { + await syncVideoCaptions({ appId: "app-1", videoId: "video-1", language }); + } + + const rows = transcripts(); + const none = rows.find((row) => row.id === "t-none"); + const some = rows.find((row) => row.id === "t-some"); + expect(none?.metadata).toMatchObject({ + __capCaptionsWritten: { metadataWasNull: true }, + }); + expect(some?.metadata).toMatchObject({ + ticket: 42, + __capCaptionsWritten: { metadataWasNull: false }, + }); + expect(serializeTranscript(none as never)).toEqual(before.none); + expect(serializeTranscript(some as never)).toEqual(before.some); + }); +}); + +describe("Cap's caption record in a transcript's metadata", () => { + it("cannot be set by a client", async () => { + const created = await createTranscriptFromUrl( + "app-1", + { + url: "https://media.example.com/a.mp3", + metadata: { ticket: 1, __capCaptionsWritten: { at: "x" } }, + }, + { isBlocked: async () => false }, + ); + + expect(created.ok).toBe(true); + expect(transcripts()[0]?.metadata).toEqual({ ticket: 1 }); + }); + + it("does not turn a repeat of the same request into a conflict", async () => { + seedVideo(); + const first = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: "captions-meta-01", + metadata: { ticket: 7 }, + }); + if (!first.ok) throw new Error("create failed"); + const row = transcripts()[0]; + if (!row) throw new Error("missing row"); + row.metadata = { + ticket: 7, + __capCaptionsWritten: { + at: "2026-11-13T10:00:00.000Z", + metadataWasNull: false, + }, + }; + + const repeat = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + idempotencyKey: "captions-meta-01", + metadata: { ticket: 7 }, + }); + + expect(repeat.ok).toBe(true); + }); +}); + +describe("caption tracks for the hosted player", () => { + const PLAYBACK = "https://web.example/api/developer/playback/playback-1"; + const PLAYABLE = { ...VIDEO, playbackId: "playback-1" }; + + it("lists nothing for a video without captions", async () => { + expect(await developerVideoCaptionTracks(PLAYABLE)).toEqual([]); + }); + + it("points each language at its file on the playback route", async () => { + seedCompleted("tr-1", "hello there"); + seedCompleted("tr-2", "hallo zusammen", { language: "de" }); + seedCompleted("tr-3", "olá a todos", { language: "pt_BR" }); + + const tracks = await developerVideoCaptionTracks(PLAYABLE); + + expect(tracks).toHaveLength(3); + expect(tracks).toContainEqual({ + language: "en", + label: "English", + url: `${PLAYBACK}/captions-en.vtt`, + }); + expect(tracks).toContainEqual({ + language: "de", + label: "German", + url: `${PLAYBACK}/captions-de.vtt`, + }); + expect(tracks.find((track) => track.language === "pt-br")?.url).toBe( + `${PLAYBACK}/captions-pt-br.vtt`, + ); + }); + + it("never hands out a storage key or a storage URL", async () => { + seedCompleted("tr-1", "hello there"); + + const [track] = await developerVideoCaptionTracks(PLAYABLE); + + expect(track).not.toHaveProperty("key"); + expect(track?.url.startsWith(`${PLAYBACK}/`)).toBe(true); + expect(track?.url).not.toContain("developer/app-1"); + }); + + it("lists nothing for a video with no playback id", async () => { + seedCompleted("tr-1", "hello there"); + + expect( + await developerVideoCaptionTracks({ ...VIDEO, playbackId: null }), + ).toEqual([]); + }); + + it("leaves the video playing without captions when the lookup fails", async () => { + const error = vi + .spyOn(console, "error") + .mockImplementation(() => undefined); + const broken = { ...PLAYABLE }; + Object.defineProperty(broken, "appId", { + get: () => { + throw new Error("database down"); + }, + }); + + expect(await developerVideoCaptionTracks(broken)).toEqual([]); + expect(error).toHaveBeenCalled(); + error.mockRestore(); + }); + + it("names a file by language and reads the language back", () => { + expect(captionsFileName("en")).toBe("captions-en.vtt"); + expect(captionsFileName("PT_br")).toBe("captions-pt-br.vtt"); + expect(captionsFileName("../../x")).toBe("captions-und.vtt"); + expect(captionsFileLanguage("captions-en.vtt")).toBe("en"); + expect(captionsFileLanguage("captions-pt-br.vtt")).toBe("pt-br"); + expect(captionsFileLanguage("captions-und.vtt")).toBe("und"); + for (const file of [ + "captions-EN.vtt", + "captions-.vtt", + "en.vtt", + "captions-en.m3u8", + "master.m3u8", + "video.mp4", + ]) { + expect(captionsFileLanguage(file)).toBeNull(); + } + }); +}); + +describe("the caption file of a video", () => { + const sync = (language: string | null = "en") => + syncVideoCaptions({ appId: "app-1", videoId: "video-1", language }); + + it("is written as WebVTT from the transcript's word timings", async () => { + seedVideo(); + seedCompleted("tr-1", "hello there"); + + expect(await sync()).toEqual({ state: "written", transcriptId: "tr-1" }); + expect(context.objects.get(EN_KEY)).toBe( + "WEBVTT\n\n00:00:00.000 --> 00:00:01.000\nhello there\n", + ); + expect(context.puts).toEqual([ + { key: EN_KEY, contentType: "text/vtt; charset=utf-8" }, + ]); + }); + + it("is replaced by a newer transcript for the same video and language", async () => { + seedVideo(); + seedCompleted("tr-old", "older words"); + await sync(); + expect(context.objects.get(EN_KEY)).toContain("older words"); + + seedCompleted("tr-new", "newer words"); + expect(await sync()).toEqual({ state: "written", transcriptId: "tr-new" }); + expect(context.objects.get(EN_KEY)).toContain("newer words"); + expect(context.objects.get(EN_KEY)).not.toContain("older words"); + }); + + it("is not replaced by an older transcript that finishes late", async () => { + seedVideo(); + const old = seedCompleted("tr-old", "older words", { + status: "processing", + }); + seedCompleted("tr-new", "newer words"); + await sync(); + old.status = "complete"; + + expect(await sync()).toEqual({ state: "written", transcriptId: "tr-new" }); + expect(context.objects.get(EN_KEY)).toContain("newer words"); + }); + + it("treats a stored code with a region as one language in the list and the file", async () => { + seedVideo(); + seedCompleted("tr-old", "older words", { language: "en_us" }); + seedCompleted("tr-new", "newer words", { language: "en-US" }); + const key = "developer/app-1/video-1/captions/en-us.vtt"; + + expect(await sync("en_us")).toEqual({ + state: "written", + transcriptId: "tr-new", + }); + expect(context.objects.get(key)).toContain("newer words"); + expect(await developerVideoCaptions(VIDEO)).toEqual([ + { language: "en-us", label: "American English", key }, + ]); + + await deleteTranscript("app-1", "tr-new"); + expect(context.objects.get(key)).toContain("older words"); + await deleteTranscript("app-1", "tr-old"); + expect(context.objects.has(key)).toBe(false); + }); + + it("writes a file for a transcript whose language was not reported", async () => { + seedVideo(); + seedCompleted("tr-1", "hello there", { language: null }); + seedCompleted("tr-en", "english words"); + + expect(await sync(null)).toEqual({ + state: "written", + transcriptId: "tr-1", + }); + expect( + context.objects.get("developer/app-1/video-1/captions/und.vtt"), + ).toContain("hello there"); + expect(context.objects.has(EN_KEY)).toBe(false); + }); + + it("leaves another language's file alone", async () => { + seedVideo(); + seedCompleted("tr-en", "hello there"); + seedCompleted("tr-de", "hallo zusammen", { language: "de" }); + await sync("en"); + await sync("de"); + + expect(context.objects.get(EN_KEY)).toContain("hello there"); + expect(context.objects.get(DE_KEY)).toContain("hallo zusammen"); + }); + + it("ends with the newest transcript when another one completes during the write", async () => { + seedVideo(); + seedCompleted("tr-old", "older words"); + context.afterPut = () => { + seedCompleted("tr-new", "newer words"); + }; + + expect(await sync()).toEqual({ state: "written", transcriptId: "tr-new" }); + expect(context.objects.get(EN_KEY)).toContain("newer words"); + }); + + it("is not written for a transcript with no speech", async () => { + seedVideo(); + seedCompleted("tr-1", "", { wordCount: 0 }); + + expect(await sync()).toEqual({ state: "removed" }); + expect(context.objects.has(EN_KEY)).toBe(false); + expect(context.puts).toHaveLength(0); + }); + + it("is not written once the video is deleted", async () => { + seedVideo({ deletedAt: new Date("2026-11-12T09:00:00.000Z") }); + seedCompleted("tr-1", "hello there"); + + expect(await sync()).toEqual({ state: "removed" }); + expect(context.objects.has(EN_KEY)).toBe(false); + }); + + it("is removed when its transcript is deleted", async () => { + seedVideo(); + seedCompleted("tr-1", "hello there"); + await sync(); + expect(context.objects.has(EN_KEY)).toBe(true); + + expect(await deleteTranscript("app-1", "tr-1")).toEqual({ ok: true }); + + expect(context.objects.has(EN_KEY)).toBe(false); + expect(await developerVideoCaptions(VIDEO)).toEqual([]); + }); + + it("falls back to the transcript before it when the newest one is deleted", async () => { + seedVideo(); + seedCompleted("tr-old", "older words"); + seedCompleted("tr-new", "newer words"); + await sync(); + + await deleteTranscript("app-1", "tr-new"); + + expect(context.objects.get(EN_KEY)).toContain("older words"); + expect(await developerVideoCaptions(VIDEO)).toEqual([ + { language: "en", label: "English", key: EN_KEY }, + ]); + }); + + it("keeps the file when an older transcript of the same language is deleted", async () => { + seedVideo(); + seedCompleted("tr-old", "older words"); + seedCompleted("tr-new", "newer words"); + await sync(); + + await deleteTranscript("app-1", "tr-old"); + + expect(context.objects.get(EN_KEY)).toContain("newer words"); + }); + + it("never deletes the video's own file when a video transcript is deleted", async () => { + seedVideo(); + context.objects.set(MP4_KEY, "mp4-bytes"); + const created = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + }); + if (!created.ok) throw new Error("create failed"); + + await deleteTranscript("app-1", created.transcript.id); + + expect(context.objects.get(MP4_KEY)).toBe("mp4-bytes"); + }); + + it("keeps a file another transcript wrote while the last one was being removed", async () => { + seedVideo(); + seedCompleted("tr-old", "older words"); + await sync(); + const old = transcripts().find((row) => row.id === "tr-old"); + if (old) old.deletedAt = new Date("2026-11-12T11:00:00.000Z"); + context.afterDelete = () => { + seedCompleted("tr-new", "newer words"); + context.objects.set(EN_KEY, "WEBVTT\n\nwritten by the other run\n"); + }; + + expect(await sync()).toEqual({ state: "written", transcriptId: "tr-new" }); + expect(context.objects.get(EN_KEY)).toContain("newer words"); + }); + + it("fails when the file cannot be removed, so deleted words are never left behind silently", async () => { + seedVideo(); + seedCompleted("tr-1", "hello there"); + await sync(); + context.stuckKeys.add(EN_KEY); + + await expect(deleteTranscript("app-1", "tr-1")).rejects.toThrow( + "storage refused the delete", + ); + context.stuckKeys.delete(EN_KEY); + expect(await deleteTranscript("app-1", "tr-1")).toEqual({ ok: true }); + expect(context.objects.has(EN_KEY)).toBe(false); + }); + + it("finishes the caption cleanup when a failed delete is sent again", async () => { + seedVideo(); + seedCompleted("tr-old", "older words"); + seedCompleted("tr-new", "newer words"); + await sync(); + context.failPuts = 1; + + await expect(deleteTranscript("app-1", "tr-new")).rejects.toThrow( + "storage is unavailable", + ); + expect(context.objects.get(EN_KEY)).toContain("newer words"); + + expect(await deleteTranscript("app-1", "tr-new")).toEqual({ ok: true }); + expect(context.objects.get(EN_KEY)).toContain("older words"); + expect(context.objects.get(EN_KEY)).not.toContain("newer words"); + }); + + it("answers not found without touching captions for a transcript that never existed", async () => { + seedVideo(); + seedCompleted("tr-1", "hello there"); + await sync(); + context.puts = []; + + expect(await deleteTranscript("app-1", "tr-unknown")).toMatchObject({ + ok: false, + status: 404, + }); + expect(context.puts).toHaveLength(0); + expect(context.objects.get(EN_KEY)).toContain("hello there"); + }); + + it("updates the captions when a transcript is deleted outside the store, as the agent API does", async () => { + seedVideo(); + seedCompleted("tr-old", "older words"); + const newest = seedCompleted("tr-new", "newer words"); + await sync(); + + newest.deletedAt = new Date("2026-11-12T11:00:00.000Z"); + newest.text = null; + await deleteTranscriptObjects("app-1", "tr-new"); + + expect(context.objects.get(EN_KEY)).toContain("older words"); + + const old = transcripts().find((row) => row.id === "tr-old"); + if (old) old.deletedAt = new Date("2026-11-12T11:05:00.000Z"); + await deleteTranscriptObjects("app-1", "tr-old"); + + expect(context.objects.has(EN_KEY)).toBe(false); + }); + + it("removes the captions of a transcript that completed between the delete's read and its update", async () => { + seedVideo(); + const row = seedCompleted("tr-1", "hello there", { + status: "processing", + language: null, + wordCount: null, + text: null, + }); + const database = fake().db as { + update: (table: object) => unknown; + }; + const update = database.update; + let completed = false; + database.update = (table: object) => { + if (!completed) { + completed = true; + row.status = "complete"; + row.language = "en"; + row.wordCount = 2; + context.objects.set(EN_KEY, "WEBVTT\n\nwritten at completion\n"); + } + return update(table); + }; + + try { + expect(await deleteTranscript("app-1", "tr-1")).toEqual({ ok: true }); + } finally { + database.update = update; + } + + expect(completed).toBe(true); + expect(context.objects.has(EN_KEY)).toBe(false); + expect(await developerVideoCaptions(VIDEO)).toEqual([]); + }); + + it("is repaired by the hourly cleanup when a delete failed and was never sent again", async () => { + seedVideo(); + seedCompleted("tr-old", "older words"); + seedCompleted("tr-new", "newer words"); + await sync(); + context.failPuts = 1; + await expect(deleteTranscript("app-1", "tr-new")).rejects.toThrow(); + expect(context.objects.get(EN_KEY)).toContain("newer words"); + + const deleted = transcripts().find((row) => row.id === "tr-new"); + expect( + await reconcileDeletedVideoCaptions(deleted?.updatedAt as Date), + ).toEqual({ synced: 1, failed: 0, finished: true }); + expect(context.objects.get(EN_KEY)).toContain("older words"); + }); + + it("only revisits video transcripts deleted in the last three hours", async () => { + seedVideo(); + const now = new Date("2026-11-12T12:00:00.000Z"); + seedCompleted("tr-recent", "recent words", { + deletedAt: new Date("2026-11-12T11:00:00.000Z"), + updatedAt: new Date("2026-11-12T11:00:00.000Z"), + }); + seedCompleted("tr-old", "old words", { + language: "de", + deletedAt: new Date("2026-11-12T08:00:00.000Z"), + updatedAt: new Date("2026-11-12T08:00:00.000Z"), + }); + seedCompleted("tr-live", "live words", { + language: "fr", + updatedAt: new Date("2026-11-12T11:30:00.000Z"), + }); + seedCompleted("tr-not-video", "other words", { + language: "es", + videoId: null, + deletedAt: new Date("2026-11-12T11:00:00.000Z"), + updatedAt: new Date("2026-11-12T11:00:00.000Z"), + }); + context.objects.set(EN_KEY, "WEBVTT\n\nstale\n"); + context.objects.set(DE_KEY, "WEBVTT\n\nstale\n"); + + expect(await reconcileDeletedVideoCaptions(now)).toEqual({ + synced: 1, + failed: 0, + finished: true, + }); + expect(context.objects.has(EN_KEY)).toBe(false); + expect(context.objects.has(DE_KEY)).toBe(true); + }); + + it("keeps going when one video cannot be repaired", async () => { + seedVideo(); + seedVideo({ id: "video-2" }); + seedCompleted("tr-1", "hello there", { deletedAt: new Date() }); + seedCompleted("tr-2", "other video", { + videoId: "video-2", + deletedAt: new Date(), + }); + context.stuckKeys.add(EN_KEY); + const latest = transcripts().find((row) => row.id === "tr-2"); + + expect( + await reconcileDeletedVideoCaptions(latest?.updatedAt as Date), + ).toEqual({ synced: 1, failed: 1, finished: true }); + }); + + it("uses the transcript requested last, even when both were created in the same second", async () => { + seedVideo(); + const second = new Date("2026-11-12T10:00:00.000Z"); + seedCompleted("tr-z-first", "first words", { + createdAt: second, + requestedAt: new Date("2026-11-12T10:00:00.100Z"), + }); + seedCompleted("tr-a-second", "second words", { + createdAt: second, + requestedAt: new Date("2026-11-12T10:00:00.900Z"), + }); + + expect(await sync()).toEqual({ + state: "written", + transcriptId: "tr-a-second", + }); + }); + + it("records when a video transcript was requested, to the millisecond", async () => { + seedVideo(); + const before = Date.now(); + const created = await createTranscriptFromVideo("app-1", { + videoId: "video-1", + }); + if (!created.ok) throw new Error("create failed"); + const requestedAt = transcripts().find( + (row) => row.id === created.transcript.id, + )?.requestedAt as Date; + + expect(requestedAt).toBeInstanceOf(Date); + expect(requestedAt.getTime()).toBeGreaterThanOrEqual(before); + expect(requestedAt.getTime()).toBeLessThanOrEqual(Date.now()); + }); + + it("walks past the first page and repairs every group deleted in the window", async () => { + const base = Date.parse("2026-11-12T11:00:00.000Z"); + for (let index = 0; index < 450; index++) { + seedVideo({ id: `video-${index}` }); + const resultKey = `developer/app-1/transcripts/tr-${index}/result.json`; + context.objects.set(resultKey, resultFor("gone words")); + fake().seed(developerTranscripts, { + id: `tr-${index}`, + appId: "app-1", + status: "complete", + sourceType: "video", + videoId: `video-${index}`, + language: "en", + wordCount: 2, + resultKey, + deletedAt: new Date(base), + updatedAt: new Date(base + Math.floor(index / 3) * 1000), + }); + context.objects.set( + `developer/app-1/video-${index}/captions/en.vtt`, + "WEBVTT\n\nstale\n", + ); + } + + expect( + await reconcileDeletedVideoCaptions(new Date(base + 600_000)), + ).toEqual({ synced: 450, failed: 0, finished: true }); + expect( + [...context.objects.keys()].filter((key) => key.endsWith(".vtt")), + ).toEqual([]); + }); + + it("repairs each video and language once per run however many of its transcripts were deleted", async () => { + seedVideo(); + for (const id of ["tr-1", "tr-2", "tr-3"]) { + seedCompleted(id, "gone words", { deletedAt: new Date() }); + } + context.puts = []; + const latest = transcripts().find((row) => row.id === "tr-3"); + + expect( + await reconcileDeletedVideoCaptions(latest?.updatedAt as Date), + ).toEqual({ synced: 1, failed: 0, finished: true }); + }); + + it("stops at its time budget and says it did not finish", async () => { + seedVideo(); + seedCompleted("tr-1", "gone words", { deletedAt: new Date() }); + const latest = transcripts().find((row) => row.id === "tr-1"); + + expect( + await reconcileDeletedVideoCaptions(latest?.updatedAt as Date, 0), + ).toEqual({ synced: 0, failed: 0, finished: false }); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-dispatch-parts.test.ts b/apps/web/__tests__/unit/developer-video-dispatch-parts.test.ts new file mode 100644 index 00000000000..ffb36bc1bcb --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-dispatch-parts.test.ts @@ -0,0 +1,381 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + type DeveloperVideoStore, + developerVideo, + processingState, +} from "../fixtures/developer-video"; + +const mocks = vi.hoisted(() => ({ + store: { video: null, appDeleted: false, updates: [] } as DeveloperVideoStore, + head: vi.fn(), + readText: vi.fn(), + probe: vi.fn(), + startJob: vi.fn(), + cancelJob: vi.fn(), + charge: vi.fn(), + sniff: vi.fn(), +})); + +vi.mock("@cap/database", async () => { + const { createDeveloperVideoDatabase } = await import( + "../fixtures/developer-video" + ); + const database = await createDeveloperVideoDatabase(mocks.store); + return { db: () => database }; +}); +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ + WEB_URL: "https://web.example", + MEDIA_SERVER_WEBHOOK_URL: undefined, + }), +})); +vi.mock("@/lib/developer-video-storage", () => ({ + headDeveloperVideoObject: mocks.head, + readDeveloperVideoText: mocks.readText, + deleteDeveloperVideoObjects: vi.fn(), + developerVideoPrefix: (video: { appId: string; id: string }) => + `developer/${video.appId}/${video.id}/`, + internalDeveloperVideoUrl: async (key: string) => `https://storage/${key}`, + developerVideoUploadUrl: async (key: string, contentType: string) => + `https://storage/put/${key}?type=${contentType}`, +})); +vi.mock("@/lib/developer-video-charge", () => ({ + DEVELOPER_VIDEO_MAX_DURATION_SECONDS: 4 * 60 * 60, + chargeDeveloperVideoUpload: mocks.charge, +})); +vi.mock("@/lib/media-client", () => ({ + MEDIA_SERVER_SHUTDOWN_MESSAGE: "Server shutting down", + tryProbeVideoViaMediaServer: mocks.probe, + startMediaServerVideoJob: mocks.startJob, + cancelMediaServerVideoJob: mocks.cancelJob, + readMediaServerVideoJob: vi.fn(), +})); +vi.mock("@/lib/media-container", () => ({ + sniffMediaUrl: mocks.sniff, +})); + +const { + applyDeveloperVideoProgress, + dispatchDeveloperVideoParts, + prepareDeveloperVideoProcessing, +} = await import("@/lib/developer-video-processing"); + +const SOURCE = "developer/app-1/video-1/video"; + +function video() { + if (!mocks.store.video) throw new Error("no video"); + return mocks.store.video; +} + +const sourceHeads = () => + mocks.head.mock.calls.filter(([key]) => key === SOURCE).length; + +beforeEach(() => { + vi.clearAllMocks(); + mocks.store.video = developerVideo(); + mocks.store.appDeleted = false; + mocks.store.updates = []; + mocks.head.mockResolvedValue({ + bytes: 1_000, + contentType: "video/quicktime", + }); + mocks.sniff.mockResolvedValue("media"); + mocks.cancelJob.mockResolvedValue("cancelled"); + mocks.charge.mockResolvedValue("charged"); + mocks.startJob.mockImplementation(async (path: string) => ({ + status: "started", + jobId: path === "/video/ladder" ? "job-hls" : "job-mp4", + })); +}); + +describe("dispatching every due part at once", () => { + it("begins both parts in one write and sends both with the source type it was given", async () => { + const results = await dispatchDeveloperVideoParts("video-1", "encode-1", { + contentType: "video/quicktime", + }); + + expect(results.map((result) => result.status)).toEqual([ + "fulfilled", + "fulfilled", + ]); + expect(mocks.store.updates[0]?.processing).toMatchObject({ + hls: { state: "running", jobId: null }, + mp4: { state: "running", jobId: null }, + attempts: { hls: 1, mp4: 1 }, + }); + expect(sourceHeads()).toBe(0); + expect(mocks.startJob).toHaveBeenCalledTimes(2); + for (const [, body] of mocks.startJob.mock.calls) { + expect(body).toMatchObject({ inputExtension: ".mov", infra: true }); + } + expect(video().processing).toMatchObject({ + hls: { state: "running", jobId: "job-hls" }, + mp4: { state: "running", jobId: "job-mp4" }, + }); + const tokens = mocks.startJob.mock.calls.map( + ([, body]) => new URL(body.webhookUrl).searchParams.get("token") ?? "", + ); + expect(new Set(tokens).size).toBe(2); + }); + + it("reads the source type once for both parts when it was not given", async () => { + await dispatchDeveloperVideoParts("video-1", "encode-1"); + + expect(sourceHeads()).toBe(1); + expect(mocks.startJob).toHaveBeenCalledTimes(2); + for (const [, body] of mocks.startJob.mock.calls) { + expect(body).toMatchObject({ inputExtension: ".mov" }); + } + }); + + it("only begins and sends the parts that are due", async () => { + mocks.store.video = developerVideo({ + processing: processingState({ + mp4: { + state: "running", + token: "other", + jobId: "job-1", + startedAt: Date.now(), + }, + }), + }); + + await dispatchDeveloperVideoParts("video-1", "encode-1"); + + expect(mocks.store.updates[0]?.processing).toMatchObject({ + hls: { state: "running", jobId: null }, + mp4: { state: "running", token: "other", jobId: "job-1" }, + }); + expect(mocks.startJob.mock.calls.map(([path]) => path)).toEqual([ + "/video/ladder", + ]); + expect(sourceHeads()).toBe(1); + }); + + it("writes and sends nothing when no part is due", async () => { + const now = Date.now(); + mocks.store.video = developerVideo({ + processing: processingState({ + hls: { state: "running", token: "a", jobId: "j1", startedAt: now }, + mp4: { state: "running", token: "b", jobId: "j2", startedAt: now }, + }), + }); + + expect(await dispatchDeveloperVideoParts("video-1", "encode-1")).toEqual( + [], + ); + expect(mocks.store.updates).toEqual([]); + expect(mocks.startJob).not.toHaveBeenCalled(); + expect(mocks.head).not.toHaveBeenCalled(); + }); + + it("still sends one part when the other fails to start", async () => { + mocks.startJob.mockImplementation(async (path: string) => { + if (path === "/video/process") throw new Error("connection reset"); + return { status: "started", jobId: "job-hls" }; + }); + + const results = await dispatchDeveloperVideoParts("video-1", "encode-1", { + contentType: "video/mp4", + }); + + expect(results.map((result) => result.status).sort()).toEqual([ + "fulfilled", + "rejected", + ]); + expect(video().processing).toMatchObject({ + hls: { state: "running", jobId: "job-hls" }, + mp4: { state: "running", jobId: null }, + }); + }); + + it("leaves a stale encode alone", async () => { + expect(await dispatchDeveloperVideoParts("video-1", "encode-2")).toEqual( + [], + ); + expect(mocks.store.updates).toEqual([]); + expect(mocks.startJob).not.toHaveBeenCalled(); + }); +}); + +describe("source facts from preparation", () => { + it("reports the source it read so the parts need not read it again", async () => { + mocks.store.video = developerVideo({ + processing: processingState({ probe: null }), + }); + mocks.probe.mockResolvedValue({ + status: "ok", + metadata: { duration: 12, width: 1920, height: 1080, fps: 30 }, + }); + const seen: { source?: { contentType: string | null } | null } = {}; + + expect( + await prepareDeveloperVideoProcessing("video-1", "encode-1", seen), + ).toBe("ready"); + + expect(seen.source).toEqual({ + bytes: 1_000, + contentType: "video/quicktime", + }); + expect(sourceHeads()).toBe(1); + }); + + it("reports nothing when the probe was already done", async () => { + const seen: { source?: { contentType: string | null } | null } = {}; + + expect( + await prepareDeveloperVideoProcessing("video-1", "encode-1", seen), + ).toBe("ready"); + + expect(seen.source).toBeUndefined(); + expect(mocks.head).not.toHaveBeenCalled(); + }); +}); + +describe("settling a finished part", () => { + const running = (token: string) => ({ + state: "running" as const, + token, + jobId: "job-1", + startedAt: 2_000, + }); + + it("asks for the poster and the MP4 at the same time", async () => { + mocks.store.video = developerVideo({ + processing: processingState({ mp4: running("t1") }), + }); + const pending: Array<() => void> = []; + mocks.head.mockImplementation( + (key: string) => + new Promise((resolve) => { + pending.push(() => + resolve({ + bytes: key.endsWith(".mp4") ? 2_000 : 50, + contentType: null, + }), + ); + }), + ); + + const settling = applyDeveloperVideoProgress( + { videoId: "video-1", phase: "complete" }, + "mp4", + "t1", + ); + await vi.waitFor(() => expect(pending).toHaveLength(2)); + for (const resolve of pending) resolve(); + + expect(await settling).toEqual({ success: true, applied: true }); + expect(video().processing?.mp4).toMatchObject({ state: "complete" }); + expect(video().processing?.bytes).toMatchObject({ mp4: 2_000, poster: 50 }); + }); + + it("asks for the poster and the master playlist at the same time", async () => { + mocks.store.video = developerVideo({ + processing: processingState({ hls: running("t2") }), + }); + const pending: Array<() => void> = []; + mocks.head.mockImplementation( + () => + new Promise((resolve) => { + pending.push(() => resolve({ bytes: 50, contentType: null })); + }), + ); + mocks.readText.mockImplementation( + () => + new Promise((resolve) => { + pending.push(() => + resolve( + [ + "#EXTM3U", + "#EXT-X-STREAM-INF:BANDWIDTH=4000000,RESOLUTION=1920x1080", + "1080p.m3u8", + ].join("\n"), + ), + ); + }), + ); + + const settling = applyDeveloperVideoProgress( + { + videoId: "video-1", + phase: "complete", + ladder: { + encoder: "libx264", + renditions: [ + { + name: "1080p", + width: 1920, + height: 1080, + bandwidth: 4_000_000, + bytes: 5_000, + }, + ], + audio: { bytes: 300 }, + }, + }, + "hls", + "t2", + ); + await vi.waitFor(() => expect(pending).toHaveLength(2)); + for (const resolve of pending) resolve(); + + expect(await settling).toEqual({ success: true, applied: true }); + expect(video().processing?.hls).toMatchObject({ state: "complete" }); + }); +}); + +describe("a video deleted while its parts are dispatched", () => { + const deleteDuringStart = (result: (path: string) => unknown) => + mocks.startJob.mockImplementation(async (path: string) => { + mocks.store.video = { ...video(), deletedAt: new Date() }; + return result(path); + }); + + it("cancels the job of each part that started", async () => { + deleteDuringStart((path) => ({ + status: "started", + jobId: path === "/video/ladder" ? "job-hls" : "job-mp4", + })); + + const results = await dispatchDeveloperVideoParts("video-1", "encode-1", { + contentType: "video/mp4", + }); + + expect(results).toEqual([ + { status: "fulfilled", value: "dispatched" }, + { status: "fulfilled", value: "dispatched" }, + ]); + expect(mocks.cancelJob).toHaveBeenCalledTimes(2); + expect(mocks.cancelJob.mock.calls.map(([jobId]) => jobId).sort()).toEqual([ + "job-hls", + "job-mp4", + ]); + expect(video().processing).toMatchObject({ + hls: { jobId: null }, + mp4: { jobId: null }, + }); + }); + + it("cancels only the part whose start was confirmed", async () => { + deleteDuringStart((path) => + path === "/video/ladder" + ? { status: "started", jobId: "job-hls" } + : { status: "unconfirmed" }, + ); + + await dispatchDeveloperVideoParts("video-1", "encode-1", { + contentType: "video/mp4", + }); + + expect(mocks.cancelJob).toHaveBeenCalledExactlyOnceWith("job-hls"); + }); + + it("cancels nothing when the video is still there", async () => { + await dispatchDeveloperVideoParts("video-1", "encode-1", { + contentType: "video/mp4", + }); + + expect(mocks.cancelJob).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-embed-error.test.tsx b/apps/web/__tests__/unit/developer-video-embed-error.test.tsx new file mode 100644 index 00000000000..63b846fc593 --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-embed-error.test.tsx @@ -0,0 +1,75 @@ +// @vitest-environment jsdom +import { act, createElement } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { EmbedError } from "@/app/dev/_components/EmbedError"; + +const realParent = window.parent; + +const mount = (code: "unavailable" | "paused") => { + const container = document.createElement("div"); + const root = createRoot(container); + act(() => root.render(createElement(EmbedError, { code }))); + return { container, unmount: () => act(() => root.unmount()) }; +}; + +afterEach(() => { + Object.defineProperty(window, "parent", { + value: realParent, + configurable: true, + }); +}); + +describe("EmbedError", () => { + it("tells the embedding page there is nothing to play, with no video details", () => { + const postMessage = vi.fn(); + Object.defineProperty(window, "parent", { + value: { postMessage }, + configurable: true, + }); + + const { container, unmount } = mount("unavailable"); + + expect(container.innerHTML).toBe(""); + expect(postMessage).toHaveBeenCalledTimes(1); + expect(postMessage).toHaveBeenCalledWith( + { + type: "cap-player", + event: "error", + currentTime: 0, + duration: 0, + paused: true, + muted: false, + code: "unavailable", + }, + "*", + ); + unmount(); + }); + + it("says so for a paused app", () => { + const postMessage = vi.fn(); + Object.defineProperty(window, "parent", { + value: { postMessage }, + configurable: true, + }); + + const { unmount } = mount("paused"); + + expect(postMessage.mock.calls[0]?.[0]).toMatchObject({ + event: "error", + code: "paused", + }); + unmount(); + }); + + it("posts nothing when the page is not embedded", () => { + const spy = vi.spyOn(window, "postMessage"); + + const { unmount } = mount("unavailable"); + + expect(spy).not.toHaveBeenCalled(); + unmount(); + spy.mockRestore(); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-embed-not-found.test.ts b/apps/web/__tests__/unit/developer-video-embed-not-found.test.ts new file mode 100644 index 00000000000..d9756f458ab --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-embed-not-found.test.ts @@ -0,0 +1,96 @@ +// @vitest-environment jsdom +import { act, createElement, type ReactElement } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { EmbedError, EmbedNotFound } from "@/app/dev/_components/EmbedError"; + +( + globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean } +).IS_REACT_ACT_ENVIRONMENT = true; + +const realParent = window.parent; + +const embedded = () => { + const postMessage = vi.fn(); + Object.defineProperty(window, "parent", { + value: { postMessage }, + configurable: true, + }); + return postMessage; +}; + +const mountAt = ( + path: string, + element: ReactElement = createElement(EmbedNotFound), +) => { + window.history.replaceState(null, "", path); + const container = document.createElement("div"); + const root = createRoot(container); + act(() => root.render(element)); + return { container, unmount: () => act(() => root.unmount()) }; +}; + +const UNAVAILABLE = { + type: "cap-player", + event: "error", + currentTime: 0, + duration: 0, + paused: true, + muted: false, + code: "unavailable", +}; + +afterEach(() => { + Object.defineProperty(window, "parent", { + value: realParent, + configurable: true, + }); + window.history.replaceState(null, "", "/"); +}); + +describe("EmbedNotFound", () => { + it.each(["/dev/Zk3vQ9s2Lm8Rt1Xy7Bc4Df6G/embed", "/dev/unknown-id/embed/"])( + "reports a deleted or unknown video at %s to the embedding page", + (path) => { + const postMessage = embedded(); + + const { container, unmount } = mountAt(path); + + expect(container.innerHTML).toBe(""); + expect(postMessage).toHaveBeenCalledTimes(1); + const [message, origin] = postMessage.mock.calls[0] ?? []; + expect(message).toStrictEqual(UNAVAILABLE); + expect(origin).toBe("*"); + expect(JSON.stringify(message)).not.toMatch(/Zk3vQ9|unknown-id|http|\//); + unmount(); + }, + ); + + it("sends the same message as a video that failed to process", () => { + const postMessage = embedded(); + + mountAt("/dev/abc/embed").unmount(); + mountAt( + "/dev/abc/embed", + createElement(EmbedError, { code: "unavailable" }), + ).unmount(); + + expect(postMessage.mock.calls[0]).toStrictEqual(postMessage.mock.calls[1]); + }); + + it.each(["/dev/abc", "/dev/abc/embed/extra", "/s/abc/embed"])( + "stays silent on %s, which is not an embed", + (path) => { + const postMessage = embedded(); + mountAt(path).unmount(); + expect(postMessage).not.toHaveBeenCalled(); + }, + ); + + it("posts nothing when the page is not embedded", () => { + const spy = vi.spyOn(window, "postMessage"); + mountAt("/dev/abc/embed").unmount(); + expect(spy).not.toHaveBeenCalled(); + spy.mockRestore(); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-oembed.test.ts b/apps/web/__tests__/unit/developer-video-oembed.test.ts new file mode 100644 index 00000000000..7a3a8c03dd6 --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-oembed.test.ts @@ -0,0 +1,254 @@ +import { DEVELOPER_VIDEO_TAKEDOWN_KEY } from "@cap/database/types"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + type DeveloperVideoStore, + developerVideo, +} from "../fixtures/developer-video"; + +const mocks = vi.hoisted(() => ({ + store: { video: null, appDeleted: false, updates: [] } as DeveloperVideoStore, + origins: [] as string[], +})); + +vi.mock("@/lib/developer-embed-origins", () => ({ + findEmbedOrigins: async () => mocks.origins, +})); + +vi.mock("@cap/database", async () => { + const { createDeveloperVideoDatabase } = await import( + "../fixtures/developer-video" + ); + const database = await createDeveloperVideoDatabase(mocks.store); + return { db: () => database }; +}); +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ WEB_URL: "https://web.example" }), +})); +vi.mock("@/lib/server", async () => { + const { HttpApiBuilder, HttpServer } = await import("@effect/platform"); + const { Layer } = await import("effect"); + return { + apiToHandler: (api: unknown) => { + const { handler } = HttpApiBuilder.toWebHandler( + Layer.mergeAll( + api as Parameters[0], + HttpServer.layerContext, + ), + ); + return (request: Request) => handler(request); + }, + }; +}); + +const { GET } = await import("@/app/api/developer/oembed/route"); +const { developerVideoOEmbed, developerVideoPlaybackIdFromUrl } = await import( + "@/lib/developer-video-oembed" +); + +const PLAYBACK_ID = "playback-id-0123456789ab"; +const SHARE = `https://web.example/dev/${PLAYBACK_ID}`; + +const request = (query: Record) => { + const url = new URL("https://web.example/api/developer/oembed"); + for (const [key, value] of Object.entries(query)) { + url.searchParams.set(key, value); + } + return new Request(url); +}; + +beforeEach(() => { + mocks.store.video = developerVideo({ + name: 'Launch "demo" ', + status: "ready", + width: 1920, + height: 1080, + }); + mocks.store.appDeleted = false; + mocks.origins = []; +}); + +describe("which links the developer oEmbed endpoint answers for", () => { + it.each([ + [SHARE, PLAYBACK_ID], + [`${SHARE}/embed`, PLAYBACK_ID], + [`${SHARE}/`, PLAYBACK_ID], + [`${SHARE}?t=30`, PLAYBACK_ID], + [`https://www.web.example/dev/${PLAYBACK_ID}`, PLAYBACK_ID], + [`https://WEB.example/dev/${PLAYBACK_ID}`, PLAYBACK_ID], + ])("reads %s", (url, playbackId) => { + expect(developerVideoPlaybackIdFromUrl(url, "https://web.example")).toBe( + playbackId, + ); + }); + + it.each([ + `http://web.example/dev/${PLAYBACK_ID}`, + `https://other.example/dev/${PLAYBACK_ID}`, + `https://web.example.evil/dev/${PLAYBACK_ID}`, + `https://web.example/s/${PLAYBACK_ID}`, + `https://web.example/dev/${PLAYBACK_ID}/embed/x`, + "https://web.example/dev/short", + "https://web.example/dev/bad%20id%20here%20xxxxxx", + "https://web.example/dev/", + "not a url", + "", + ])("refuses %j", (url) => { + expect( + developerVideoPlaybackIdFromUrl(url, "https://web.example"), + ).toBeNull(); + }); +}); + +describe("the developer oEmbed answer", () => { + it("is a video with the embed's iframe at the video's shape", () => { + const answer = developerVideoOEmbed({ + name: "Demo", + width: 1920, + height: 1080, + embedUrl: `${SHARE}/embed`, + webUrl: "https://web.example/", + maxWidth: null, + maxHeight: null, + }); + expect(answer).toEqual({ + version: "1.0", + type: "video", + provider_name: "Cap", + provider_url: "https://web.example", + title: "Demo", + html: ``, + width: 1280, + height: 720, + cache_age: 60, + }); + }); + + it.each([ + [{ maxWidth: "640", maxHeight: null }, 640, 360], + [{ maxWidth: null, maxHeight: "180" }, 320, 180], + [{ maxWidth: "9999", maxHeight: "9999" }, 1280, 720], + [{ maxWidth: "0", maxHeight: "abc" }, 1280, 720], + [{ maxWidth: "-5", maxHeight: "1e3" }, 1280, 720], + ])("fits %j", (max, width, height) => { + const answer = developerVideoOEmbed({ + name: "Demo", + width: 1920, + height: 1080, + embedUrl: `${SHARE}/embed`, + webUrl: "https://web.example", + ...max, + }); + expect([answer.width, answer.height]).toEqual([width, height]); + }); + + it("keeps a portrait or small video's shape and never enlarges it", () => { + const tall = developerVideoOEmbed({ + name: "Demo", + width: 1080, + height: 1920, + embedUrl: `${SHARE}/embed`, + webUrl: "https://web.example", + maxWidth: null, + maxHeight: null, + }); + expect([tall.width, tall.height]).toEqual([405, 720]); + const small = developerVideoOEmbed({ + name: "Demo", + width: 640, + height: 360, + embedUrl: `${SHARE}/embed`, + webUrl: "https://web.example", + maxWidth: null, + maxHeight: null, + }); + expect([small.width, small.height]).toEqual([640, 360]); + }); +}); + +describe("GET /api/developer/oembed", () => { + it("answers a share link with JSON a shared cache may keep for a minute", async () => { + const response = await GET(request({ url: SHARE, format: "json" })); + expect(response.status).toBe(200); + expect(response.headers.get("Content-Type")).toBe( + "application/json; charset=utf-8", + ); + expect(response.headers.get("Cache-Control")).toBe( + "public, max-age=60, s-maxage=60", + ); + expect(response.headers.get("Access-Control-Allow-Origin")).toBe("*"); + const body = await response.json(); + expect(body).toMatchObject({ + type: "video", + version: "1.0", + title: 'Launch "demo" ', + width: 1280, + height: 720, + }); + expect(body.html).toBe( + ``, + ); + expect(JSON.stringify(body)).not.toMatch(/m3u8|mp4|playback\/|app-1/); + }); + + it("answers an embed link the same way, and without a format", async () => { + const share = await (await GET(request({ url: SHARE }))).json(); + const embed = await (await GET(request({ url: `${SHARE}/embed` }))).json(); + expect(embed).toEqual(share); + }); + + it("answers a video still processing, whose embed says so", async () => { + mocks.store.video = developerVideo({ status: "processing" }); + expect((await GET(request({ url: SHARE }))).status).toBe(200); + }); + + it("uses 16 by 9 for a video with no measured size", async () => { + mocks.store.video = developerVideo({ width: null, height: null }); + const body = await (await GET(request({ url: SHARE }))).json(); + expect([body.width, body.height]).toEqual([1280, 720]); + }); + + it.each([ + ["a deleted video", () => developerVideo({ deletedAt: new Date() })], + ["a video in a deleted app", () => developerVideo()], + [ + "a taken down video", + () => + developerVideo({ + metadata: { + [DEVELOPER_VIDEO_TAKEDOWN_KEY]: { + at: "2026-11-13T00:00:00.000Z", + metadata: null, + }, + }, + }), + ], + ["an unknown playback id", () => null], + ])("answers 404 for %s and lets nothing cache it", async (label, make) => { + mocks.store.video = make(); + mocks.store.appDeleted = label === "a video in a deleted app"; + const response = await GET(request({ url: SHARE })); + expect(response.status).toBe(404); + expect(response.headers.get("Cache-Control")).toBe("no-store"); + expect(await response.text()).not.toContain("Launch"); + }); + + it("answers 404 for a video its owner limited to their own sites", async () => { + mocks.origins = ["https://myshop.com"]; + const response = await GET(request({ url: SHARE })); + expect(response.status).toBe(404); + expect(response.headers.get("Cache-Control")).toBe("no-store"); + }); + + it("answers 404 without a query for a link that is not a hosted video page", async () => { + const response = await GET( + request({ url: `https://web.example/s/${PLAYBACK_ID}` }), + ); + expect(response.status).toBe(404); + }); + + it("refuses formats other than JSON", async () => { + const response = await GET(request({ url: SHARE, format: "xml" })); + expect(response.status).toBe(501); + expect(response.headers.get("Cache-Control")).toBe("no-store"); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-page.test.ts b/apps/web/__tests__/unit/developer-video-page.test.ts index e3797926c09..fbe244d791b 100644 --- a/apps/web/__tests__/unit/developer-video-page.test.ts +++ b/apps/web/__tests__/unit/developer-video-page.test.ts @@ -16,6 +16,9 @@ vi.mock("next/navigation", () => ({ }, useRouter: () => ({ refresh: () => undefined }), })); +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ WEB_URL: "https://web.example" }), +})); vi.mock("@/lib/developer-video-player", () => ({ loadDeveloperVideoPlayer: mocks.load, })); @@ -34,6 +37,13 @@ const playing: DeveloperVideoPlayer = { kind: "playing", source: { hls: `${BASE}/master.m3u8`, mp4: `${BASE}/video.mp4` }, poster: `${BASE}/poster.jpg`, + captions: null, +}; +const captioned: DeveloperVideoPlayer = { + ...playing, + captions: [ + { language: "en", label: "English", url: `${BASE}/captions-en.vtt` }, + ], }; const processing: DeveloperVideoPlayer = { ...frame, @@ -42,6 +52,11 @@ const processing: DeveloperVideoPlayer = { }; const failed: DeveloperVideoPlayer = { ...frame, kind: "failed" }; const paused: DeveloperVideoPlayer = { ...frame, kind: "paused" }; +const takenDown: DeveloperVideoPlayer = { ...frame, kind: "taken_down" }; +const REPORT_HREF = `mailto:hello@cap.so?subject=Report%20video%20${PLAYBACK_ID}`; + +const links = (html: string) => + [...html.matchAll(/]*>[^<]*<\/a>/g)].map((match) => match[0]); const render = ( player: DeveloperVideoPlayer, @@ -49,7 +64,12 @@ const render = ( playback?: DeveloperVideoPlayback, ) => renderToStaticMarkup( - createElement(DeveloperVideoView, { player, variant, playback }), + createElement(DeveloperVideoView, { + playbackId: PLAYBACK_ID, + player, + variant, + playback, + }), ); const params = (query: DeveloperVideoQuery = {}, playbackId = PLAYBACK_ID) => ({ @@ -110,17 +130,20 @@ describe("developer video page states", () => { it("says when the upload has not finished", () => { const html = render({ ...processing, uploading: true }); - expect(html).toContain("Waiting for the upload"); + expect(html).toContain("This video is being prepared"); + expect(html).not.toContain("upload"); expect(html).not.toContain(" { + it("tells a viewer of a failed video a neutral heading and line and nothing else", () => { const html = render(failed); const text = html .replace(/", + '#fff" onload="alert(1)', + "3b82f6;background:url(https://evil.example/x)", + ])("keeps the hostile colour %j out of the page", (color) => { + const html = embed({ color }); + + expect(html).toBe(embed({})); + expect(html).not.toContain("evil.example"); + expect(html).not.toContain("alert(1)"); + }); + + it("rounds the embed's corners on a clear page", () => { + const html = embed({ radius: "12" }); + + expect(html).toContain("border-radius:12px"); + expect(html).toContain("html,body{background:transparent}"); + expect(embed({})).toContain("html,body{background:#000}"); + expect(embed({})).not.toContain("border-radius:"); + expect(embed({ radius: "12px;}" })).toBe(embed({})); + }); + + it.each([ + ["failed", failed], + ["paused", paused], + ])( + "keeps the %s embed square and opaque whatever the radius", + (_, player) => { + const html = render( + player, + "embed", + developerVideoPlayback({ radius: "12" }, "embed"), + ); + + expect(html).not.toContain("border-radius"); + expect(html).not.toContain("background:transparent"); + }, + ); + + it("leaves the poster out when asked", () => { + expect(embed({ poster: "0" })).not.toContain("poster="); + expect(embed({ poster: "1" })).toContain(`poster="${BASE}/poster.jpg"`); + }); + + it("hides the buttons the embed switches off", () => { + expect(embed({})).toContain('aria-label="Full screen"'); + const html = embed({ fullscreen: "0", speed_menu: "0" }); + + expect(html).not.toContain('aria-label="Full screen"'); + expect(html).not.toContain("Playback speed"); + expect(html).toContain('aria-label="Play"'); + }); + it("keeps the embed-only switches off the share page", () => { - const query = { autoplay: "1", muted: "1", loop: "1", controls: "0" }; + const query = { + autoplay: "1", + muted: "1", + loop: "1", + controls: "0", + color: "f00", + radius: "12", + poster: "0", + fullscreen: "0", + quality: "360", + }; expect( render(playing, "share", developerVideoPlayback(query, "share")), @@ -257,6 +537,72 @@ describe("developer video page routes", () => { }, ); + it("unfurls the share page as the MP4 with an oEmbed link to the embed", async () => { + const metadata = await sharePage.generateMetadata(params()); + const share = `https://web.example/dev/${PLAYBACK_ID}`; + + expect(metadata.openGraph).toEqual({ + title: "Launch demo", + type: "video.other", + url: share, + images: [`${BASE}/poster.jpg`], + videos: [ + { + url: `${BASE}/video.mp4`, + secureUrl: `${BASE}/video.mp4`, + type: "video/mp4", + width: 1920, + height: 1080, + }, + ], + }); + expect(metadata.alternates).toEqual({ + types: { + "application/json+oembed": `https://web.example/api/developer/oembed?url=${encodeURIComponent(share)}&format=json`, + }, + }); + }); + + it("leaves the embed document without video tags or an oEmbed link", async () => { + const metadata = await embedPage.generateMetadata(params()); + + expect(metadata.openGraph).not.toHaveProperty("videos"); + expect(metadata).not.toHaveProperty("alternates"); + }); + + it("names no video file for a video that cannot play yet, and keeps the oEmbed link", async () => { + for (const player of [processing, failed, paused]) { + mocks.load.mockResolvedValue(player); + const metadata = await sharePage.generateMetadata(params()); + expect(metadata.openGraph).not.toHaveProperty("videos"); + expect(JSON.stringify(metadata)).not.toMatch(/mp4|m3u8/); + expect(metadata.alternates?.types).toHaveProperty( + "application/json+oembed", + ); + } + }); + + it("gives a taken down or unknown video no video tags or oEmbed link", async () => { + for (const player of [takenDown, null]) { + mocks.load.mockResolvedValue(player); + const metadata = await sharePage.generateMetadata(params()); + expect(metadata.openGraph).not.toHaveProperty("videos"); + expect(metadata).not.toHaveProperty("alternates"); + } + }); + + it("leaves secure_url out for a video file served over plain HTTP", async () => { + mocks.load.mockResolvedValue({ + ...playing, + source: { mp4: "http://localhost:3000/video.mp4" }, + }); + const metadata = await sharePage.generateMetadata(params()); + expect( + (metadata.openGraph as { videos: { secureUrl?: string }[] }).videos[0] + ?.secureUrl, + ).toBeUndefined(); + }); + it("previews with no image when there is nothing to play", async () => { mocks.load.mockResolvedValue(paused); @@ -322,7 +668,7 @@ describe("developer video page routes", () => { }), ); - expect(html).toContain("This video is not available."); + expect(html).toContain("This video isn't available"); }); it("hands the query of each route to the player", async () => { diff --git a/apps/web/__tests__/unit/developer-video-params.test.ts b/apps/web/__tests__/unit/developer-video-params.test.ts index fae3544fdc9..f6241995d26 100644 --- a/apps/web/__tests__/unit/developer-video-params.test.ts +++ b/apps/web/__tests__/unit/developer-video-params.test.ts @@ -1,12 +1,26 @@ import { describe, expect, it } from "vitest"; -import { developerVideoPlayback } from "@/lib/developer-video-params"; +import { + cappedPlaylistUrl, + developerVideoPlayback, +} from "@/lib/developer-video-params"; const DEFAULTS = { autoPlay: false, muted: false, loop: false, controls: true, + captions: false, startAt: 0, + accentColor: null, + radius: 0, + maxQuality: null, + poster: true, + keyboard: true, + events: true, + qualityMenu: true, + speedMenu: true, + pip: true, + fullscreen: true, }; describe("developerVideoPlayback", () => { @@ -17,14 +31,23 @@ describe("developerVideoPlayback", () => { it("reads every embed parameter", () => { expect( developerVideoPlayback( - { autoplay: "1", muted: "1", loop: "1", controls: "0", t: "90" }, + { + autoplay: "1", + muted: "1", + loop: "1", + controls: "0", + captions: "1", + t: "90", + }, "embed", ), ).toEqual({ + ...DEFAULTS, autoPlay: true, muted: true, loop: true, controls: false, + captions: true, startAt: 90, }); }); @@ -43,6 +66,15 @@ describe("developerVideoPlayback", () => { ["0", 0], ["007", 7], ["999999", 999999], + ["90s", 90], + ["1m30s", 90], + ["1m", 60], + ["1h2m3s", 3723], + ["1h", 3600], + ["2h5s", 7205], + ["1M30S", 90], + ["1m2.5s", 62.5], + ["120m", 7200], ])("starts at %s seconds", (t, startAt) => { expect(developerVideoPlayback({ t }, "embed").startAt).toBe(startAt); }); @@ -51,9 +83,18 @@ describe("developerVideoPlayback", () => { "-5", "1e3", "abc", - "1m30s", - "90s", " 90", + "1m30", + "30s1m", + "1h1h", + "m", + "s", + "hms", + "1.5m", + "1h-2m", + "1m 30s", + "999h", + "300h", "", "NaN", "Infinity", @@ -69,13 +110,41 @@ describe("developerVideoPlayback", () => { (value) => { expect( developerVideoPlayback( - { autoplay: value, muted: value, loop: value, controls: value }, + { + autoplay: value, + muted: value, + loop: value, + controls: value, + captions: value, + }, "embed", ), ).toEqual(DEFAULTS); }, ); + it("keeps captions off unless the embed asks for them", () => { + expect(developerVideoPlayback({}, "embed").captions).toBe(false); + expect(developerVideoPlayback({ captions: "0" }, "embed").captions).toBe( + false, + ); + expect( + developerVideoPlayback({ captions: "false" }, "embed").captions, + ).toBe(false); + expect(developerVideoPlayback({ captions: "1" }, "embed").captions).toBe( + true, + ); + expect(developerVideoPlayback({ captions: "true" }, "embed").captions).toBe( + true, + ); + expect( + developerVideoPlayback({ captions: ["1", "0"] }, "embed").captions, + ).toBe(true); + expect(developerVideoPlayback({ captions: "en" }, "embed").captions).toBe( + false, + ); + }); + it("ignores parameters it does not know", () => { expect( developerVideoPlayback( @@ -97,9 +166,176 @@ describe("developerVideoPlayback", () => { it("takes only the start time on the share page", () => { expect( developerVideoPlayback( - { autoplay: "1", muted: "1", loop: "1", controls: "0", t: "42" }, + { + autoplay: "1", + muted: "1", + loop: "1", + controls: "0", + captions: "1", + t: "42", + }, "share", ), - ).toEqual({ ...DEFAULTS, startAt: 42 }); + ).toEqual({ ...DEFAULTS, startAt: 42, events: false }); + }); + + it("reads every look and feel parameter", () => { + expect( + developerVideoPlayback( + { + color: "3B82F6", + radius: "12", + quality: "720", + poster: "0", + keyboard: "0", + quality_menu: "0", + speed_menu: "0", + pip: "0", + fullscreen: "0", + }, + "embed", + ), + ).toEqual({ + ...DEFAULTS, + accentColor: "#3b82f6", + radius: 12, + maxQuality: 720, + poster: false, + keyboard: false, + qualityMenu: false, + speedMenu: false, + pip: false, + fullscreen: false, + }); + }); + + it.each([ + ["3b82f6", "#3b82f6"], + ["#3b82f6", "#3b82f6"], + ["F80", "#ff8800"], + ["#abc", "#aabbcc"], + ])("takes the accent colour %j", (color, accentColor) => { + expect(developerVideoPlayback({ color }, "embed").accentColor).toBe( + accentColor, + ); + }); + + it.each([ + "red", + "#12345", + "1234567", + "#3b82f6;background:url(x)", + "3b82f6", + "var(--x)", + "#3b82f6 ", + "%233b82f6", + "", + "#", + "rgb(0,0,0)", + "expression(alert(1))", + ])("ignores the hostile or malformed colour %j", (color) => { + expect(developerVideoPlayback({ color }, "embed").accentColor).toBeNull(); + }); + + it.each([ + ["0", 0], + ["8", 8], + ["23", 23], + ["24", 24], + ["25", 24], + ["99", 24], + ["100", 24], + ["999999999", 24], + ["-4", 0], + ["12px", 0], + ["1.5", 0], + ["1e9", 0], + ["abc", 0], + ["012", 12], + ["1".padEnd(400, "0"), 24], + ["", 0], + ["12;}", 0], + ])("reads the corner radius %j as %d px", (radius, expected) => { + expect(developerVideoPlayback({ radius }, "embed").radius).toBe(expected); + }); + + it.each([ + ["720", 720], + ["720p", 720], + ["1080P", 1080], + ["2160", 2160], + ["360", 360], + ["144", 144], + ["145", 145], + ["4319", 4319], + ["4320", 4320], + ["143", 144], + ["100", 144], + ["99", 144], + ["1", 144], + ["4321", 4320], + ["9999", 4320], + ["99999", 4320], + ["0720", 720], + ["007", 144], + ["1".padEnd(400, "0"), 4320], + ["0", null], + ["000", null], + ["abc", null], + ["720px", null], + ["auto", null], + ["-720", null], + ["7.5", null], + ["1e9", null], + ["720&max=1", null], + ["", null], + ])("reads the quality cap %j as %s", (quality, expected) => { + expect(developerVideoPlayback({ quality }, "embed").maxQuality).toBe( + expected, + ); + }); + + it("hides only the menus and buttons that are switched off", () => { + expect( + developerVideoPlayback( + { quality_menu: "0", speed_menu: "1", pip: "yes", fullscreen: "false" }, + "embed", + ), + ).toMatchObject({ + qualityMenu: false, + speedMenu: true, + pip: true, + fullscreen: false, + }); + }); + + it("keeps the look and feel parameters to the embed", () => { + expect( + developerVideoPlayback( + { color: "f00", radius: "8", quality: "720", poster: "0", pip: "0" }, + "share", + ), + ).toMatchObject({ + accentColor: null, + radius: 0, + maxQuality: null, + poster: true, + pip: true, + }); + }); +}); + +describe("cappedPlaylistUrl", () => { + const MASTER = "https://web.example/api/developer/playback/abc/master.m3u8"; + + it("leaves the playlist alone without a cap", () => { + expect(cappedPlaylistUrl(MASTER, null)).toBe(MASTER); + }); + + it("asks the playback route for renditions up to the cap", () => { + expect(cappedPlaylistUrl(MASTER, 720)).toBe(`${MASTER}?max=720`); + expect(cappedPlaylistUrl(`${MASTER}?max=1080`, 720)).toBe( + `${MASTER}?max=720`, + ); }); }); diff --git a/apps/web/__tests__/unit/developer-video-playback.test.ts b/apps/web/__tests__/unit/developer-video-playback.test.ts index f11603ed1bc..d061e611bfc 100644 --- a/apps/web/__tests__/unit/developer-video-playback.test.ts +++ b/apps/web/__tests__/unit/developer-video-playback.test.ts @@ -1,5 +1,10 @@ +import { generateKeyPairSync } from "node:crypto"; import type { NextRequest } from "next/server"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + cloudFrontExpiresAt, + createCloudFrontSignedUrl, +} from "../../../../packages/web-backend/src/S3Buckets/cloudfront-signed-url"; import { type DeveloperVideoStore, developerVideo, @@ -10,7 +15,9 @@ const mocks = vi.hoisted(() => ({ store: { video: null, appDeleted: false, updates: [] } as DeveloperVideoStore, readText: vi.fn(), sign: vi.fn(), + answersAnyOrigin: vi.fn(), standing: vi.fn(), + captions: vi.fn(), })); vi.mock("@cap/database", async () => { @@ -26,7 +33,12 @@ vi.mock("@cap/env", () => ({ vi.mock("@/lib/infra-standing", () => ({ getInfraAppStanding: mocks.standing, })); +vi.mock("@/lib/developer-video-captions", async (importOriginal) => ({ + ...(await importOriginal()), + developerVideoCaptions: mocks.captions, +})); vi.mock("@/lib/developer-video-storage", () => ({ + developerVideoUrlAnswersAnyOrigin: mocks.answersAnyOrigin, readDeveloperVideoText: mocks.readText, signedDeveloperVideoUrl: mocks.sign, })); @@ -34,6 +46,12 @@ vi.mock("@/lib/developer-video-storage", () => ({ const { GET, HEAD, OPTIONS } = await import( "@/app/api/developer/playback/[playbackId]/[file]/route" ); +const { signedUrlValidity } = await import("@/lib/developer-video-playback"); + +const HOUR = 60 * 60 * 1000; +const WINDOW_START = Date.UTC(2026, 10, 12, 12); +const WINDOW_END = WINDOW_START + 6 * HOUR; +const EXPIRES = (WINDOW_END + 12 * HOUR) / 1000; const PLAYBACK_ID = "playback-id-0123456789ab"; const HLS_PREFIX = "developer/app-1/video-1/encode-1/hls-out/hls/"; @@ -55,6 +73,24 @@ const rendition = [ "#EXT-X-ENDLIST", ].join("\n"); +const storedRendition = (segments: number) => + [ + "#EXTM3U", + "#EXT-X-VERSION:7", + "#EXT-X-TARGETDURATION:4", + "#EXT-X-MEDIA-SEQUENCE:0", + "#EXT-X-PLAYLIST-TYPE:VOD", + "#EXT-X-INDEPENDENT-SEGMENTS", + '#EXT-X-MAP:URI="1080p.m4s",BYTERANGE="862@0"', + ...Array.from({ length: segments }, (_, index) => [ + "#EXTINF:4.004000,", + `#EXT-X-BYTERANGE:350000@${862 + index * 350_000}`, + "1080p.m4s", + ]).flat(), + "#EXT-X-ENDLIST", + "", + ].join("\n"); + const ready = (overrides = {}) => developerVideo({ status: "ready", @@ -77,6 +113,11 @@ function play(file: string, playbackId = PLAYBACK_ID, from = request) { }); } +const asking = (query: string) => + new Request(`${request.url}${query}`) as NextRequest; +const short = asking("?segments=short"); +const playShort = (file: string) => play(file, PLAYBACK_ID, short); + const accepting = (encodings: string) => new Request(request.url, { headers: { "accept-encoding": encodings }, @@ -97,14 +138,22 @@ const ladder = (...variants: string[][]) => const order = (playlist: string) => playlist.split("\n").filter((line) => line.endsWith(".m3u8")); +afterEach(() => { + vi.useRealTimers(); +}); + beforeEach(() => { vi.clearAllMocks(); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(WINDOW_START + 2 * HOUR); mocks.store.video = ready(); mocks.store.appDeleted = false; mocks.standing.mockResolvedValue({ state: "active", pausesAt: null }); + mocks.answersAnyOrigin.mockReturnValue(true); + mocks.captions.mockResolvedValue([]); mocks.sign.mockImplementation( - async (key: string, expiresIn: number) => - `https://cdn.example/${key}?expires=${expiresIn}&sig=a,b`, + async (key: string, validity: { signingDate: Date; expiresIn: number }) => + `https://cdn.example/${key}?expires=${cloudFrontExpiresAt(validity, Date.now())}&sig=a,b`, ); mocks.readText.mockImplementation(async (key: string) => key === `${HLS_PREFIX}master.m3u8` @@ -116,6 +165,60 @@ beforeEach(() => { vi.spyOn(console, "error").mockImplementation(() => undefined); }); +describe("signed URL validity", () => { + const { privateKey } = generateKeyPairSync("rsa", { + modulusLength: 2048, + privateKeyEncoding: { type: "pkcs1", format: "pem" }, + publicKeyEncoding: { type: "spki", format: "pem" }, + }); + const signAt = (now: number, key = "developer/app/video/hls/1080p.m4s") => + createCloudFrontSignedUrl({ + bucketUrl: "https://cdn.example", + key, + keyPairId: "K2JCJMDEHXQW5F", + privateKey, + expiresAt: cloudFrontExpiresAt(signedUrlValidity(now), now), + }); + + it("signs one file to one URL for the whole window", () => { + const url = signAt(WINDOW_START); + + expect(signAt(WINDOW_START + 86_000)).toBe(url); + expect(signAt(WINDOW_START + 3 * HOUR + 1_234)).toBe(url); + expect(signAt(WINDOW_END - 1)).toBe(url); + expect(signAt(WINDOW_START, "developer/app/video/hls/audio.m4s")).not.toBe( + url, + ); + }); + + it("signs a new URL in the next window", () => { + expect(signAt(WINDOW_END)).not.toBe(signAt(WINDOW_END - 1)); + expect(signAt(WINDOW_START - 1)).not.toBe(signAt(WINDOW_START)); + }); + + it("keeps every URL valid for at least 12 hours and at most 18", () => { + for ( + let now = WINDOW_START - HOUR; + now <= WINDOW_END + HOUR; + now += 61_987 + ) { + const left = + cloudFrontExpiresAt(signedUrlValidity(now), now) * 1000 - now; + + expect(left).toBeGreaterThanOrEqual(12 * HOUR); + expect(left).toBeLessThanOrEqual(18 * HOUR); + } + for (const now of [WINDOW_START, WINDOW_END - 1, WINDOW_END]) { + expect( + cloudFrontExpiresAt(signedUrlValidity(now), now) * 1000 - now, + ).toBeGreaterThanOrEqual(12 * HOUR); + } + expect( + cloudFrontExpiresAt(signedUrlValidity(WINDOW_END - 1), WINDOW_END - 1), + ).toBe(EXPIRES); + }); +}); + describe("developer video playback route", () => { it("serves the master playlist to any origin with a short public cache", async () => { const response = await play("master.m3u8"); @@ -125,6 +228,7 @@ describe("developer video playback route", () => { expect(response.headers.get("content-type")).toBe( "application/vnd.apple.mpegurl", ); + expect(response.headers.get("vary")).toBeNull(); expect(response.headers.get("access-control-allow-origin")).toBe("*"); expect(response.headers.get("cache-control")).toBe( "public, max-age=60, s-maxage=60", @@ -228,7 +332,7 @@ describe("developer video playback route", () => { expect(await (await play("master.m3u8")).text()).toBe(stored); }); - it("sends a long playlist as plain text whatever the player accepts", async () => { + it("sends a rendition playlist uncompressed as text/plain whatever the player accepts, for the platform to compress", async () => { const long = [ "#EXTM3U", '#EXT-X-MAP:URI="1080p.m4s",BYTERANGE="800@0"', @@ -251,8 +355,13 @@ describe("developer video playback route", () => { expect(response.headers.get("content-encoding")).toBeNull(); expect(response.headers.get("content-type")).toBe( - "application/vnd.apple.mpegurl", + "text/plain; charset=utf-8", + ); + expect(response.headers.get("vary")).toBeNull(); + expect(response.headers.get("cache-control")).toBe( + "public, max-age=60, s-maxage=60", ); + expect(response.headers.get("access-control-allow-origin")).toBe("*"); expect(body.startsWith("#EXTM3U\n")).toBe(true); expect(body.endsWith("#EXT-X-ENDLIST")).toBe(true); expect(body.split("\n")).toHaveLength(long.split("\n").length); @@ -261,7 +370,7 @@ describe("developer video playback route", () => { it("signs every media reference in a rendition playlist", async () => { const response = await play("1080p.m3u8"); - const signed = `https://cdn.example/${HLS_PREFIX}1080p.m4s?expires=43200&sig=a,b`; + const signed = `https://cdn.example/${HLS_PREFIX}1080p.m4s?expires=${EXPIRES}&sig=a,b`; expect(response.status).toBe(200); expect((await response.text()).split("\n")).toEqual([ @@ -278,11 +387,286 @@ describe("developer video playback route", () => { expect(mocks.sign).toHaveBeenCalledTimes(1); expect(mocks.sign).toHaveBeenCalledWith( `${HLS_PREFIX}1080p.m4s`, - 43200, + { signingDate: new Date(WINDOW_START), expiresIn: 18 * 60 * 60 }, request, ); }); + it("keeps every segment of a 317 segment playlist signed unless the short form is asked for", async () => { + mocks.readText.mockResolvedValue(storedRendition(317)); + const signed = `https://cdn.example/${HLS_PREFIX}1080p.m4s?expires=${EXPIRES}&sig=a,b`; + + const lines = (await (await play("1080p.m3u8")).text()).split("\n"); + + expect(lines.filter((line) => line === signed)).toHaveLength(317); + expect(lines).not.toContain("1080p.m4s"); + expect(mocks.answersAnyOrigin).not.toHaveBeenCalled(); + }); + + it.each([ + "?segments=SHORT", + "?segments=short&segments=short", + "?segments=short&segments=long", + "?segments=", + "?segments", + "?segments=1", + "?segments=short%20", + "?Segments=short", + "?short", + ])("serves the fully signed form for %s", async (query) => { + mocks.readText.mockResolvedValue(storedRendition(40)); + + const body = await ( + await play("1080p.m3u8", PLAYBACK_ID, asking(query)) + ).text(); + + expect(body.split("\n")).not.toContain("1080p.m4s"); + }); + + it("keeps every segment signed in a short form playlist of up to 15 segments", async () => { + mocks.readText.mockResolvedValue(storedRendition(15)); + const signed = `https://cdn.example/${HLS_PREFIX}1080p.m4s?expires=${EXPIRES}&sig=a,b`; + + const lines = (await (await playShort("1080p.m3u8")).text()).split("\n"); + + expect(lines.filter((line) => line === signed)).toHaveLength(15); + expect(lines).not.toContain("1080p.m4s"); + }); + + it("signs the init section and first segment of a longer short form playlist and lists the rest by file name", async () => { + mocks.readText.mockResolvedValue(storedRendition(16)); + const signed = `https://cdn.example/${HLS_PREFIX}1080p.m4s?expires=${EXPIRES}&sig=a,b`; + + const response = await playShort("1080p.m3u8"); + + expect(response.status).toBe(200); + expect(await response.text()).toBe( + [ + "#EXTM3U", + "#EXT-X-VERSION:7", + "#EXT-X-TARGETDURATION:4", + "#EXT-X-MEDIA-SEQUENCE:0", + "#EXT-X-PLAYLIST-TYPE:VOD", + "#EXT-X-INDEPENDENT-SEGMENTS", + `#EXT-X-MAP:URI="${signed}",BYTERANGE="862@0"`, + "#EXTINF:4.004000,", + "#EXT-X-BYTERANGE:350000@862", + signed, + ...Array.from({ length: 15 }, (_, index) => [ + "#EXTINF:4.004000,", + `#EXT-X-BYTERANGE:350000@${862 + (index + 1) * 350_000}`, + "1080p.m4s", + ]).flat(), + "#EXT-X-ENDLIST", + "", + ].join("\n"), + ); + expect(mocks.sign).toHaveBeenCalledTimes(1); + }); + + it("keeps every segment of a short form playlist signed where media cannot be reached through a redirect", async () => { + mocks.readText.mockResolvedValue(storedRendition(40)); + mocks.answersAnyOrigin.mockReturnValue(false); + const signed = `https://cdn.example/${HLS_PREFIX}1080p.m4s?expires=${EXPIRES}&sig=a,b`; + + const lines = (await (await playShort("1080p.m3u8")).text()).split("\n"); + + expect(lines.filter((line) => line === signed)).toHaveLength(40); + expect(lines).not.toContain("1080p.m4s"); + expect(mocks.answersAnyOrigin).toHaveBeenCalledWith(signed); + }); + + it("serves a 21 minute short form playlist in about 21 KB instead of 250 KB", async () => { + const signature = "s".repeat(344); + const policy = "p".repeat(280); + const signed = `https://cdn.example/${HLS_PREFIX}1080p.m4s?Policy=${policy}&Key-Pair-Id=K2JCJMDEHXQW5F&Signature=${signature}`; + mocks.sign.mockResolvedValue(signed); + const stored = storedRendition(317); + mocks.readText.mockResolvedValue(stored); + + const body = await (await playShort("1080p.m3u8")).text(); + + expect((await (await play("1080p.m3u8")).text()).length).toBeGreaterThan( + 250_000, + ); + expect(body.length).toBe(stored.length + 2 * (signed.length - 9)); + expect(body.length).toBeLessThan(22_000); + expect(body.split(signed)).toHaveLength(3); + expect( + body.split("\n").filter((line) => line === "1080p.m4s"), + ).toHaveLength(316); + }); + + it.each([ + { name: "another folder", line: "../../other/video.m4s" }, + { name: "another host", line: "https://elsewhere.example/1080p.m4s" }, + { name: "another playlist", line: "720p.m3u8" }, + ])( + "refuses a stored playlist whose last segment points at $name in either form", + async ({ line }) => { + const stored = storedRendition(40).split("\n"); + stored[stored.lastIndexOf("1080p.m4s")] = line; + mocks.readText.mockResolvedValue(stored.join("\n")); + + for (const response of [ + await play("1080p.m3u8"), + await playShort("1080p.m3u8"), + ]) { + expect(response.status).toBe(502); + expect(response.headers.get("cache-control")).toBe("no-store"); + } + }, + ); + + it("carries the short form from the master playlist to every playlist it lists", async () => { + const stored = ladder( + variant("2160p", 3_374_149, "3024x1964"), + variant("1080p", 1_706_075, "1662x1080"), + variant("360p", 426_279, "554x360"), + ); + mocks.readText.mockResolvedValue(stored); + + const plain = await (await play("master.m3u8")).text(); + const body = await (await playShort("master.m3u8")).text(); + + expect(order(plain)).toEqual(["1080p.m3u8", "2160p.m3u8", "360p.m3u8"]); + expect(body).toBe( + plain + .replace('URI="audio.m3u8"', 'URI="audio.m3u8?segments=short"') + .replace(/^([0-9a-z]+\.m3u8)$/gm, "$1?segments=short"), + ); + expect(body.split("\n").filter((line) => line.includes(".m3u8"))).toEqual([ + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud",NAME="audio",URI="audio.m3u8?segments=short"', + "1080p.m3u8?segments=short", + "2160p.m3u8?segments=short", + "360p.m3u8?segments=short", + ]); + }); + + it("leaves the master playlist alone for a value it does not know", async () => { + const plain = await (await play("master.m3u8")).text(); + + for (const query of ["?segments=long", "?segments=short&segments=short"]) { + expect( + await (await play("master.m3u8", PLAYBACK_ID, asking(query))).text(), + ).toBe(plain); + } + }); + + describe("choosing renditions", () => { + const stored = ladder( + variant("1080p", 4_573_338, "1920x1080"), + variant("720p", 2_501_452, "1280x720"), + variant("360p", 739_748, "640x360"), + ); + const served = async (query: string) => + (await play("master.m3u8", PLAYBACK_ID, asking(query))).text(); + + beforeEach(() => { + mocks.readText.mockResolvedValue(stored); + }); + + it("serves a master that lists only the picked rendition, with its audio", async () => { + const body = await served("?rendition=1080p"); + + expect(order(body)).toEqual(["1080p.m3u8"]); + expect(body).toContain( + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud",NAME="audio",URI="audio.m3u8"', + ); + expect(body).toContain("RESOLUTION=1920x1080"); + }); + + it("keeps renditions no taller than the cap", async () => { + expect(order(await served("?max=720"))).toEqual([ + "720p.m3u8", + "360p.m3u8", + ]); + expect(order(await served("?max=1080"))).toEqual([ + "720p.m3u8", + "1080p.m3u8", + "360p.m3u8", + ]); + }); + + it("keeps the smallest rendition when the cap is below every one", async () => { + expect(order(await served("?max=240"))).toEqual(["360p.m3u8"]); + }); + + it("carries a cap and a pick into the short form", async () => { + expect( + (await served("?max=720&segments=short")) + .split("\n") + .filter((line) => line.endsWith(".m3u8?segments=short")), + ).toEqual(["720p.m3u8?segments=short", "360p.m3u8?segments=short"]); + expect(order(await served("?rendition=360p&max=720"))).toEqual([ + "360p.m3u8", + ]); + }); + + describe("on a constant quality ladder, where bandwidth does not rise with height", () => { + const uneven = ladder( + variant("2160p", 1_600_000, "3840x2160"), + variant("1080p", 800_000, "1920x1080"), + variant("720p", 1_900_000, "1280x720"), + variant("360p", 900_000, "640x360"), + ); + const names = (body: string) => order(body).sort(); + + beforeEach(() => { + mocks.readText.mockResolvedValue(uneven); + }); + + it("caps by height, not bandwidth", async () => { + expect(names(await served("?max=1080"))).toEqual([ + "1080p.m3u8", + "360p.m3u8", + "720p.m3u8", + ]); + expect(names(await served("?max=720"))).toEqual([ + "360p.m3u8", + "720p.m3u8", + ]); + }); + + it("falls back to the shortest rendition, not the cheapest", async () => { + expect(order(await served("?max=240"))).toEqual(["360p.m3u8"]); + }); + + it("picks a rendition by its name whatever its bandwidth", async () => { + expect(order(await served("?rendition=1080p"))).toEqual(["1080p.m3u8"]); + expect(names(await served("?rendition=2160p&max=1080"))).toEqual( + names(await served("?max=1080")), + ); + }); + }); + + it.each([ + "?rendition=480p", + "?rendition=audio", + "?rendition=../1080p", + "?rendition=1080p.m3u8", + "?rendition=1080p&rendition=720p", + "?max=720p", + "?max=-1", + "?max=abc", + "?max=720&max=360", + "?rendition=%0A%23EXT-X-ENDLIST", + ])("leaves the master as it serves it for %s", async (query) => { + expect(await served(query)).toBe(await served("")); + }); + }); + + it("redirects a segment listed by file name to the URL the playlist signs", async () => { + mocks.readText.mockResolvedValue(storedRendition(16)); + + const playlist = (await (await playShort("1080p.m3u8")).text()).split("\n"); + const listed = playlist[playlist.lastIndexOf("1080p.m4s")] ?? ""; + const redirect = await play(listed); + + expect(redirect.status).toBe(302); + expect(playlist).toContain(redirect.headers.get("location")); + }); + it.each([ { file: "1080p.m4s", key: `${HLS_PREFIX}1080p.m4s` }, { @@ -298,12 +682,32 @@ describe("developer video playback route", () => { expect(response.status).toBe(302); expect(response.headers.get("location")).toBe( - `https://cdn.example/${key}?expires=43200&sig=a,b`, + `https://cdn.example/${key}?expires=${EXPIRES}&sig=a,b`, ); expect(response.headers.get("access-control-allow-origin")).toBe("*"); expect(response.headers.get("cache-control")).toContain("max-age=60"); }); + it.each(["1080p.m3u8", "1080p.m4s", "video.mp4", "poster.jpg"])( + "gives %s the same signed URLs all through a window and new ones in the next", + async (file) => { + const answer = async (at: number) => { + vi.setSystemTime(at); + const response = await play(file); + return response.headers.get("location") ?? (await response.text()); + }; + + const first = await answer(WINDOW_START); + expect(first).toContain(`expires=${EXPIRES}`); + expect(await answer(WINDOW_START + 86_000)).toBe(first); + expect(await answer(WINDOW_END - 1)).toBe(first); + + const next = await answer(WINDOW_END); + expect(next).not.toBe(first); + expect(next).toContain(`expires=${EXPIRES + 6 * 60 * 60}`); + }, + ); + it.each([ { name: "an unknown playback id", playbackId: "playback-id-0123456789zz" }, { name: "the video id", playbackId: "video-1" }, @@ -440,6 +844,392 @@ describe("developer video playback route", () => { expect(response.headers.get("cache-control")).toBe("no-store"); }); + describe("captions", () => { + const CAPTIONS_KEY = "developer/app-1/video-1/captions/en.vtt"; + const vtt = "WEBVTT\n\n00:00:00.000 --> 00:00:02.000\nHello there\n"; + + beforeEach(() => { + mocks.readText.mockImplementation(async (key: string) => + key === CAPTIONS_KEY ? vtt : null, + ); + }); + + it("serves the WebVTT file from storage to any origin with the playlists' cache", async () => { + const response = await play("captions-en.vtt"); + + expect(response.status).toBe(200); + expect(await response.text()).toBe(vtt); + expect(response.headers.get("content-type")).toBe( + "text/vtt; charset=utf-8", + ); + expect(response.headers.get("cache-control")).toBe( + "public, max-age=60, s-maxage=60", + ); + expect(response.headers.get("access-control-allow-origin")).toBe("*"); + expect(response.headers.get("location")).toBeNull(); + expect(mocks.readText).toHaveBeenCalledWith(CAPTIONS_KEY); + }); + + it("signs no storage URL for captions", async () => { + await play("captions-en.vtt"); + + expect(mocks.sign).not.toHaveBeenCalled(); + }); + + it("reads the file of a language with a region and of an unreported language", async () => { + await play("captions-pt-br.vtt"); + await play("captions-und.vtt"); + + expect(mocks.readText.mock.calls.map(([key]) => key)).toEqual([ + "developer/app-1/video-1/captions/pt-br.vtt", + "developer/app-1/video-1/captions/und.vtt", + ]); + }); + + it("answers 404 without caching when the video has no captions in that language", async () => { + const response = await play("captions-fr.vtt"); + + expect(response.status).toBe(404); + expect(response.headers.get("cache-control")).toBe("no-store"); + }); + + it("answers 404 for captions of a deleted video", async () => { + mocks.store.video = null; + + expect((await play("captions-en.vtt")).status).toBe(404); + expect(mocks.readText).not.toHaveBeenCalled(); + }); + + it("answers 410 for captions of a video that was taken down", async () => { + mocks.store.video = ready({ + metadata: { + __capTakedown: { at: "2026-11-13T07:00:00.000Z", metadata: null }, + }, + }); + + const response = await play("captions-en.vtt"); + + expect(response.status).toBe(410); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(mocks.readText).not.toHaveBeenCalled(); + }); + + it("refuses captions of a paused app", async () => { + mocks.standing.mockResolvedValue({ state: "paused", pausesAt: null }); + + expect((await play("captions-en.vtt")).status).toBe(402); + expect(mocks.readText).not.toHaveBeenCalled(); + }); + + it.each([ + "captions-EN.vtt", + "captions-.vtt", + "captions-en", + "captions-e.vtt", + "captions-en-.vtt", + "captions-en/../x.vtt", + "captions-en.vtt.bak", + "en.vtt", + "captions-english-language.vtt", + ])("answers 404 for the captions file name %s", async (file) => { + expect((await play(file)).status).toBe(404); + expect(mocks.readText).not.toHaveBeenCalled(); + }); + }); + + describe("cache tags", () => { + const TAGS = "infra-video-video-1,infra-app-app-1"; + + it("tags every answer the CDN may keep with the video and its app", async () => { + mocks.captions.mockResolvedValue([ + { language: "en", label: "English", key: "k" }, + ]); + mocks.readText.mockImplementation(async (key: string) => + key === `${HLS_PREFIX}master.m3u8` + ? master + : key === `${HLS_PREFIX}1080p.m3u8` + ? rendition + : key.endsWith("/captions/en.vtt") + ? "WEBVTT\n" + : null, + ); + + for (const file of [ + "master.m3u8", + "1080p.m3u8", + "1080p.m4s", + "video.mp4", + "poster.jpg", + "captions-en.vtt", + "captions-en.m3u8", + ]) { + const response = await play(file); + expect([200, 302]).toContain(response.status); + expect(response.headers.get("cache-control")).toBe( + "public, max-age=60, s-maxage=60", + ); + expect(response.headers.get("vercel-cache-tag")).toBe(TAGS); + } + + mocks.standing.mockResolvedValue({ state: "paused", pausesAt: null }); + const paused = await play("master.m3u8"); + expect(paused.status).toBe(402); + expect(paused.headers.get("vercel-cache-tag")).toBe(TAGS); + }); + + it("tags nothing the CDN does not keep", async () => { + mocks.readText.mockResolvedValue(null); + const missing = await play("720p.m3u8"); + expect(missing.status).toBe(404); + expect(missing.headers.get("vercel-cache-tag")).toBeNull(); + + mocks.store.video = ready({ + metadata: { + __capTakedown: { at: "2026-11-13T07:00:00.000Z", metadata: null }, + }, + }); + const gone = await play("master.m3u8"); + expect(gone.status).toBe(410); + expect(gone.headers.get("vercel-cache-tag")).toBeNull(); + }); + }); + + describe("subtitles in the master playlist", () => { + const english = { + language: "en", + label: "English", + key: "developer/app-1/video-1/captions/en.vtt", + }; + const SUBTITLES_LINE = + '#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="subs",NAME="English",LANGUAGE="en",DEFAULT=NO,AUTOSELECT=YES,FORCED=NO,URI="captions-en.m3u8"'; + const stored = ladder( + variant("1080p", 4_573_338, "1920x1080"), + variant("720p", 2_501_452, "1280x720"), + ); + const withoutSubtitles = (body: string) => + body + .split("\n") + .filter((line) => !line.includes("TYPE=SUBTITLES")) + .map((line) => line.replace(',SUBTITLES="subs"', "")) + .join("\n"); + + beforeEach(() => { + mocks.readText.mockResolvedValue(stored); + }); + + it("serves the master of a video with no captions byte for byte as before", async () => { + const plain = await (await play("master.m3u8")).text(); + const shortForm = await (await playShort("master.m3u8")).text(); + const picked = await ( + await play("master.m3u8", PLAYBACK_ID, asking("?rendition=720p")) + ).text(); + + mocks.captions.mockResolvedValue([english]); + const opted = asking("?subtitles=none"); + expect(await (await play("master.m3u8", PLAYBACK_ID, opted)).text()).toBe( + plain, + ); + + expect(plain).not.toContain("SUBTITLES"); + expect(shortForm).not.toContain("SUBTITLES"); + expect(picked).not.toContain("SUBTITLES"); + }); + + it("lists each captions language as a WebVTT subtitle rendition every variant points at", async () => { + const plain = await (await play("master.m3u8")).text(); + mocks.captions.mockResolvedValue([english]); + + const response = await play("master.m3u8"); + const body = await response.text(); + const lines = body.split("\n"); + + expect(response.headers.get("content-type")).toBe( + "application/vnd.apple.mpegurl", + ); + expect(lines.filter((line) => line.includes("TYPE=SUBTITLES"))).toEqual([ + SUBTITLES_LINE, + ]); + expect(lines.indexOf(SUBTITLES_LINE)).toBe( + lines.findIndex((line) => line.includes("TYPE=AUDIO")) + 1, + ); + const variants = lines.filter((line) => + line.startsWith("#EXT-X-STREAM-INF:"), + ); + expect(variants).toHaveLength(2); + for (const line of variants) { + expect(line.endsWith(',AUDIO="aud",SUBTITLES="subs"')).toBe(true); + } + expect(withoutSubtitles(body)).toBe(plain); + }); + + it("keeps the subtitles in a short form master without asking for their short form", async () => { + mocks.captions.mockResolvedValue([english]); + + const lines = (await (await playShort("master.m3u8")).text()).split("\n"); + + expect(lines).toContain(SUBTITLES_LINE); + expect( + lines.filter((line) => line.endsWith(".m3u8?segments=short")), + ).toEqual(["720p.m3u8?segments=short", "1080p.m3u8?segments=short"]); + }); + + it("keeps the subtitles in a master narrowed to one rendition", async () => { + mocks.captions.mockResolvedValue([english]); + + const body = await ( + await play("master.m3u8", PLAYBACK_ID, asking("?rendition=1080p")) + ).text(); + + expect(order(body)).toEqual(["1080p.m3u8"]); + expect(body).toContain(SUBTITLES_LINE); + expect(body).toContain(',AUDIO="aud",SUBTITLES="subs"'); + }); + + it("names every language once, and leaves out the language of captions with none reported", async () => { + mocks.captions.mockResolvedValue([ + { language: "pt-br", label: "Portuguese", key: "a" }, + { language: "pt", label: "Portuguese", key: "b" }, + { language: "und", label: "Captions", key: "c" }, + ]); + + const lines = (await (await play("master.m3u8")).text()) + .split("\n") + .filter((line) => line.includes("TYPE=SUBTITLES")); + + expect(lines).toEqual([ + '#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="subs",NAME="Portuguese",LANGUAGE="pt-br",DEFAULT=NO,AUTOSELECT=YES,FORCED=NO,URI="captions-pt-br.m3u8"', + '#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="subs",NAME="Portuguese (pt)",LANGUAGE="pt",DEFAULT=NO,AUTOSELECT=YES,FORCED=NO,URI="captions-pt.m3u8"', + '#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="subs",NAME="Captions",DEFAULT=NO,AUTOSELECT=YES,FORCED=NO,URI="captions-und.m3u8"', + ]); + }); + + it("does not look captions up for Cap's own player", async () => { + await play("master.m3u8", PLAYBACK_ID, asking("?subtitles=none")); + await play( + "master.m3u8", + PLAYBACK_ID, + asking("?subtitles=none&segments=short"), + ); + + expect(mocks.captions).not.toHaveBeenCalled(); + }); + + it("serves the master without subtitles when the captions lookup fails", async () => { + const plain = await (await play("master.m3u8")).text(); + mocks.captions.mockRejectedValue(new Error("database down")); + + const response = await play("master.m3u8"); + + expect(response.status).toBe(200); + expect(await response.text()).toBe(plain); + }); + + it("lists no subtitles for a video of unknown length", async () => { + mocks.store.video = ready({ duration: null }); + mocks.captions.mockResolvedValue([english]); + const video = mocks.store.video; + if (video?.processing) { + video.processing = { + ...video.processing, + probe: { ...video.processing.probe, duration: null }, + } as typeof video.processing; + } + + const body = await (await play("master.m3u8")).text(); + + expect(body).not.toContain("SUBTITLES"); + expect(mocks.captions).not.toHaveBeenCalled(); + }); + + it("leaves a master that already has subtitles as it is", async () => { + const own = [ + "#EXTM3U", + '#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID="text",NAME="Own",URI="own.m3u8"', + '#EXT-X-STREAM-INF:BANDWIDTH=900000,RESOLUTION=640x360,SUBTITLES="text"', + "360p.m3u8", + ].join("\n"); + mocks.readText.mockResolvedValue(own); + mocks.captions.mockResolvedValue([english]); + + expect(await (await play("master.m3u8")).text()).toBe(own); + }); + }); + + describe("subtitle playlists", () => { + const english = { language: "en", label: "English", key: "k" }; + + beforeEach(() => { + mocks.captions.mockResolvedValue([english]); + }); + + it("wraps the captions file in a one segment playlist as long as the video", async () => { + mocks.store.video = ready({ duration: 154.32 }); + + const response = await play("captions-en.m3u8"); + + expect(response.status).toBe(200); + expect(await response.text()).toBe( + [ + "#EXTM3U", + "#EXT-X-VERSION:3", + "#EXT-X-TARGETDURATION:155", + "#EXT-X-MEDIA-SEQUENCE:0", + "#EXT-X-PLAYLIST-TYPE:VOD", + "#EXTINF:154.320000,", + "captions-en.vtt", + "#EXT-X-ENDLIST", + "", + ].join("\n"), + ); + expect(response.headers.get("content-type")).toBe( + "application/vnd.apple.mpegurl", + ); + expect(response.headers.get("cache-control")).toBe( + "public, max-age=60, s-maxage=60", + ); + expect(response.headers.get("access-control-allow-origin")).toBe("*"); + expect(mocks.readText).not.toHaveBeenCalled(); + expect(mocks.sign).not.toHaveBeenCalled(); + }); + + it("answers 404 without caching for a language the video has no captions in", async () => { + const response = await play("captions-fr.m3u8"); + + expect(response.status).toBe(404); + expect(response.headers.get("cache-control")).toBe("no-store"); + }); + + it("answers 404 when the captions cannot be listed", async () => { + mocks.captions.mockRejectedValue(new Error("database down")); + + expect((await play("captions-en.m3u8")).status).toBe(404); + }); + + it("refuses the subtitle playlist of a paused app or a video that was taken down", async () => { + mocks.standing.mockResolvedValue({ state: "paused", pausesAt: null }); + expect((await play("captions-en.m3u8")).status).toBe(402); + + mocks.standing.mockResolvedValue({ state: "active", pausesAt: null }); + mocks.store.video = ready({ + metadata: { + __capTakedown: { at: "2026-11-13T07:00:00.000Z", metadata: null }, + }, + }); + expect((await play("captions-en.m3u8")).status).toBe(410); + expect(mocks.captions).not.toHaveBeenCalled(); + }); + + it.each([ + "captions-EN.m3u8", + "captions-.m3u8", + "captions-e.m3u8", + "captions-en.m3u8.bak", + ])("answers 404 for the subtitle playlist name %s", async (file) => { + expect((await play(file)).status).toBe(404); + expect(mocks.captions).not.toHaveBeenCalled(); + }); + }); + it("answers preflight and HEAD requests", async () => { const preflight = OPTIONS(); diff --git a/apps/web/__tests__/unit/developer-video-player.test.ts b/apps/web/__tests__/unit/developer-video-player.test.ts index 3d5c9707468..81d3a2f99dc 100644 --- a/apps/web/__tests__/unit/developer-video-player.test.ts +++ b/apps/web/__tests__/unit/developer-video-player.test.ts @@ -9,6 +9,7 @@ const mocks = vi.hoisted(() => ({ store: { video: null, appDeleted: false, updates: [] } as DeveloperVideoStore, standing: vi.fn(), redrive: vi.fn(), + captions: vi.fn(), })); vi.mock("@cap/database", async () => { @@ -28,6 +29,9 @@ vi.mock("@/lib/infra-standing", () => ({ vi.mock("@/lib/developer-video-start", () => ({ redriveDeveloperVideoProcessing: mocks.redrive, })); +vi.mock("@/lib/developer-video-captions", () => ({ + developerVideoCaptionTracks: mocks.captions, +})); const { loadDeveloperVideoPlayer } = await import( "@/lib/developer-video-player" @@ -54,6 +58,7 @@ beforeEach(() => { mocks.store.appDeleted = false; mocks.standing.mockResolvedValue({ state: "active", pausesAt: null }); mocks.redrive.mockResolvedValue(false); + mocks.captions.mockResolvedValue([]); }); const MINUTE = 60_000; @@ -65,6 +70,56 @@ const uploadedLongAgo = () => }); describe("developer video player data", () => { + it.each([ + ["a ready video", () => ready()], + ["a paused app's video", () => ready()], + ["a video still uploading", uploadedLongAgo], + ])( + "shows %s that Cap took down as unavailable, without redriving it", + async (name, video) => { + if (name === "a paused app's video") { + mocks.standing.mockResolvedValue({ state: "paused", pausesAt: null }); + } + mocks.store.video = { + ...video(), + metadata: { + __capTakedown: { at: "2026-11-13T12:00:00.000Z", metadata: null }, + }, + }; + + expect(await loadDeveloperVideoPlayer(PLAYBACK_ID)).toEqual({ + kind: "taken_down", + name: "", + width: 16, + height: 9, + }); + expect(mocks.standing).not.toHaveBeenCalled(); + expect(mocks.redrive).not.toHaveBeenCalled(); + }, + ); + + it("shows a video taken down while its stalled processing was retried as unavailable", async () => { + mocks.store.video = uploadedLongAgo(); + mocks.redrive.mockImplementation(async () => { + if (!mocks.store.video) throw new Error("missing video"); + mocks.store.video = { + ...mocks.store.video, + metadata: { + __capTakedown: { at: "2026-11-13T12:00:00.000Z", metadata: null }, + }, + }; + return true; + }); + + expect(await loadDeveloperVideoPlayer(PLAYBACK_ID)).toEqual({ + kind: "taken_down", + name: "", + width: 16, + height: 9, + }); + expect(mocks.redrive).toHaveBeenCalledTimes(1); + }); + it("plays the adaptive stream with the MP4 behind it and the poster", async () => { expect(await loadDeveloperVideoPlayer(PLAYBACK_ID)).toEqual({ kind: "playing", @@ -73,10 +128,40 @@ describe("developer video player data", () => { height: 1080, source: { hls: `${BASE}/master.m3u8`, mp4: `${BASE}/video.mp4` }, poster: `${BASE}/poster.jpg`, + captions: null, }); expect(mocks.standing).toHaveBeenCalledWith("app-1"); }); + it("lists the captions of a video that has them", async () => { + const english = { + language: "en", + label: "English", + url: `${BASE}/captions-en.vtt`, + }; + mocks.captions.mockResolvedValue([english]); + + const player = await loadDeveloperVideoPlayer(PLAYBACK_ID); + + expect(player).toMatchObject({ kind: "playing", captions: [english] }); + expect(mocks.captions).toHaveBeenCalledWith( + expect.objectContaining({ + id: "video-1", + appId: "app-1", + playbackId: PLAYBACK_ID, + }), + ); + }); + + it("looks up no captions for a video that cannot play yet", async () => { + mocks.store.video = ready({ status: "processing", processing: null }); + + expect(await loadDeveloperVideoPlayer(PLAYBACK_ID)).toMatchObject({ + kind: "processing", + }); + expect(mocks.captions).not.toHaveBeenCalled(); + }); + it("plays the MP4 while the adaptive stream is still encoding", async () => { mocks.store.video = ready({ processing: processingState({ diff --git a/apps/web/__tests__/unit/developer-video-processing.test.ts b/apps/web/__tests__/unit/developer-video-processing.test.ts index 2eb6962bc22..c89b15c3c53 100644 --- a/apps/web/__tests__/unit/developer-video-processing.test.ts +++ b/apps/web/__tests__/unit/developer-video-processing.test.ts @@ -12,7 +12,9 @@ const mocks = vi.hoisted(() => ({ probe: vi.fn(), startJob: vi.fn(), readJob: vi.fn(), + cancelJob: vi.fn(), charge: vi.fn(), + firstMonth: vi.fn(), deleteObjects: vi.fn(), sniff: vi.fn(), })); @@ -40,6 +42,9 @@ vi.mock("@/lib/developer-video-storage", () => ({ developerVideoUploadUrl: async (key: string, contentType: string) => `https://storage/put/${key}?type=${contentType}`, })); +vi.mock("@/lib/infra-storage-minimum", () => ({ + settleFirstMonthAtReady: mocks.firstMonth, +})); vi.mock("@/lib/developer-video-charge", () => ({ DEVELOPER_VIDEO_MAX_DURATION_SECONDS: 4 * 60 * 60, chargeDeveloperVideoUpload: mocks.charge, @@ -49,6 +54,7 @@ vi.mock("@/lib/media-client", () => ({ tryProbeVideoViaMediaServer: mocks.probe, startMediaServerVideoJob: mocks.startJob, readMediaServerVideoJob: mocks.readJob, + cancelMediaServerVideoJob: mocks.cancelJob, })); vi.mock("@/lib/media-container", () => ({ sniffMediaUrl: mocks.sniff, @@ -131,6 +137,7 @@ beforeEach(() => { mocks.charge.mockResolvedValue("charged"); mocks.deleteObjects.mockResolvedValue(0); mocks.sniff.mockResolvedValue("media"); + mocks.cancelJob.mockResolvedValue("cancelled"); vi.spyOn(console, "error").mockImplementation(() => undefined); }); @@ -198,6 +205,64 @@ describe("developer video webhook", () => { expect(video().storedBytes).toBe(1_000 + 2_000 + 50); }); + it("charges the first month once, after the video turns ready, and not when its second part lands", async () => { + mocks.store.video = developerVideo({ + processing: processingState({ + hls: runningPart("t-hls"), + mp4: runningPart("t-mp4"), + }), + }); + mocks.firstMonth.mockImplementation(async () => { + expect(video().status).toBe("ready"); + }); + + await applyDeveloperVideoProgress( + { videoId: "video-1", phase: "complete" }, + "mp4", + "t-mp4", + ); + + expect(mocks.firstMonth).toHaveBeenCalledTimes(1); + expect(mocks.firstMonth.mock.calls[0]?.[0]).toMatchObject({ + id: "video-1", + status: "ready", + deletedAt: null, + }); + + await applyDeveloperVideoProgress( + { videoId: "video-1", phase: "complete", ladder }, + "hls", + "t-hls", + ); + + expect(video().processing?.hls.state).toBe("complete"); + expect(mocks.firstMonth).toHaveBeenCalledTimes(1); + }); + + it("does not charge the first month for a part that leaves the video encoding or failed", async () => { + mocks.store.video = developerVideo({ + processing: processingState({ + hls: runningPart("t-hls"), + mp4: runningPart("t-mp4"), + attempts: { hls: 2, mp4: 2 }, + }), + }); + + await applyDeveloperVideoProgress( + { videoId: "video-1", phase: "error", error: "boom" }, + "mp4", + "t-mp4", + ); + await applyDeveloperVideoProgress( + { videoId: "video-1", phase: "error", error: "boom" }, + "hls", + "t-hls", + ); + + expect(video().status).toBe("errored"); + expect(mocks.firstMonth).not.toHaveBeenCalled(); + }); + it("marks the video ready with its renditions once both parts are stored", async () => { mocks.store.video = developerVideo({ duration: 999, @@ -1168,6 +1233,7 @@ describe("developer video dispatch", () => { maxTier: "1080p", includeAudio: true, inputExtension: ".mov", + infra: true, }); expect(Object.keys(body.uploads)).toEqual([ "master.m3u8", @@ -1197,6 +1263,47 @@ describe("developer video dispatch", () => { ); }); + it("marks every job request it can send as Cap Infra work", async () => { + const sizes = [ + { width: 640, height: 360 }, + { width: 1920, height: 1080 }, + { width: 2560, height: 1440 }, + { width: 3840, height: 2160 }, + { width: 1080, height: 1920 }, + ]; + const contentTypes = ["video/quicktime", "video/mp4", undefined]; + const sent = new Set(); + for (const tier of ["1080p", "1440p", "2160p"] as const) { + for (const size of sizes) { + for (const contentType of contentTypes) { + for (const part of ["hls", "mp4"] as const) { + mocks.startJob.mockReset(); + mocks.startJob.mockResolvedValue({ + status: "started", + jobId: "job-infra", + }); + mocks.head.mockResolvedValue({ bytes: 1_000, contentType }); + mocks.store.video = developerVideo({ + maxResolutionTier: tier, + processing: processingState({ + probe: { duration: 12, ...size, fps: 30 }, + }), + }); + await dispatchDeveloperVideoPart("video-1", "encode-1", part); + expect(mocks.startJob).toHaveBeenCalledTimes(1); + const [path, body] = mocks.startJob.mock.calls[0] ?? []; + sent.add(path); + expect({ path, infra: body?.infra }).toEqual({ + path, + infra: true, + }); + } + } + } + } + expect([...sent].sort()).toEqual(["/video/ladder", "/video/process"]); + }); + it("caps the MP4 job at the video's tier", async () => { mocks.startJob.mockResolvedValue({ status: "started", jobId: "job-9" }); mocks.store.video = developerVideo({ @@ -1209,7 +1316,11 @@ describe("developer video dispatch", () => { const [path, body] = mocks.startJob.mock.calls[0] ?? []; expect(path).toBe("/video/process"); - expect(body).toMatchObject({ maxWidth: 1920, maxHeight: 1920 }); + expect(body).toMatchObject({ + maxWidth: 1920, + maxHeight: 1920, + infra: true, + }); const mp4 = video().processing?.mp4; const token = mp4?.state === "running" ? mp4.token : ""; expect(body.outputPresignedUrl).toContain(`${ENCODE}/${token}/video.mp4`); @@ -1240,6 +1351,16 @@ describe("developer video dispatch", () => { expect(video().processing?.attempts.mp4).toBe(0); }); + it("records when the media server last answered busy", async () => { + mocks.startJob.mockResolvedValue({ status: "busy" }); + const before = Date.now(); + + await dispatchDeveloperVideoPart("video-1", "encode-1", "mp4"); + + expect(video().processing?.busyAt).toBeGreaterThanOrEqual(before); + expect(video().processing?.busyAt).toBeLessThanOrEqual(Date.now()); + }); + it("keeps the part for a later try when the media server cannot be reached", async () => { mocks.startJob.mockResolvedValue({ status: "unreachable" }); @@ -1311,6 +1432,20 @@ describe("developer video dispatch", () => { expect(video().processing?.probing).toBeUndefined(); }); + it("counts a probe the media server gave up on toward the limit instead of retrying it forever", async () => { + mocks.probe.mockResolvedValue({ status: "timed_out" }); + mocks.store.video = developerVideo({ + processing: processingState({ probe: null }), + }); + + expect(await prepareDeveloperVideoProcessing("video-1", "encode-1")).toBe( + "unclear", + ); + expect(video().status).toBe("processing"); + expect(video().processing?.probe).toBeNull(); + expect(video().processing?.probing).toBeUndefined(); + }); + it("probes a file once when two callers prepare it at the same time", async () => { let finish: (value: unknown) => void = () => undefined; mocks.probe.mockReturnValue( @@ -1406,7 +1541,7 @@ describe("developer video dispatch", () => { await expect(refusal).rejects.toBeInstanceOf(DeveloperVideoRejected); await expect(refusal).rejects.toThrow( - "This file could not be read as a video. Upload a video file such as MP4, MOV, WebM or MKV.", + "This file could not be read as a video. Upload MP4, MOV, MKV and WebM files with H.264, HEVC, VP8, VP9, AV1, ProRes, MPEG-4 or Motion JPEG video.", ); expect(video().processing?.probing).toBeUndefined(); }); @@ -1420,11 +1555,11 @@ describe("developer video dispatch", () => { ["Media input is empty", "The uploaded file is empty. Upload it again."], [ "Media input could not be read", - "This file could not be read as a video. Upload a video file such as MP4, MOV, WebM or MKV.", + "This file could not be read as a video. Upload MP4, MOV, MKV and WebM files with H.264, HEVC, VP8, VP9, AV1, ProRes, MPEG-4 or Motion JPEG video.", ], [ "Input has an unsupported or unrecognizable format.", - "This file could not be read as a video. Upload a video file such as MP4, MOV, WebM or MKV.", + "This file could not be read as a video. Upload MP4, MOV, MKV and WebM files with H.264, HEVC, VP8, VP9, AV1, ProRes, MPEG-4 or Motion JPEG video.", ], [ "Video frame is larger than 8K, the largest size this server accepts", @@ -1508,7 +1643,7 @@ describe("developer video dispatch", () => { [ "a file that is not media", "unknown", - "This file could not be read as a video. Upload a video file such as MP4, MOV, WebM or MKV.", + "This file could not be read as a video. Upload MP4, MOV, MKV and WebM files with H.264, HEVC, VP8, VP9, AV1, ProRes, MPEG-4 or Motion JPEG video.", ], ])( "refuses %s without asking the media server", @@ -1869,3 +2004,192 @@ describe("superseded output sweep", () => { expect(mocks.deleteObjects).not.toHaveBeenCalled(); }); }); + +describe("a deleted developer video", () => { + const deletedVideo = ( + overrides: Parameters[0] = {}, + ) => + developerVideo({ + deletedAt: new Date("2026-11-02T00:00:00Z"), + processing: processingState({ + hls: runningPart("t-hls"), + mp4: { ...runningPart("t-mp4"), jobId: "job-2" }, + ...overrides, + }), + }); + + it.each([ + { + name: "the cancel the delete asked for", + payload: { phase: "cancelled", message: "Processing cancelled by user" }, + }, + { + name: "a shutdown that would otherwise send the part again", + payload: { phase: "cancelled", message: "Server shutting down" }, + }, + { + name: "a failure that would otherwise spend an attempt", + payload: { phase: "error", error: "Encoding failed" }, + }, + { + name: "a completion that would otherwise charge for a measured length", + payload: { + phase: "complete", + metadata: { duration: 30 }, + }, + probe: { duration: null, width: 1920, height: 1080, fps: 30 }, + }, + ])("ignores $name", async ({ payload, probe }) => { + mocks.store.video = deletedVideo(probe ? { probe } : {}); + const before = structuredClone(video()); + + const result = await applyDeveloperVideoProgress( + { videoId: "video-1", jobId: "job-2", ...payload }, + "mp4", + "t-mp4", + ); + + expect(result).toEqual({ success: true, applied: false }); + expect(mocks.store.updates).toHaveLength(0); + expect(video()).toEqual(before); + expect(mocks.charge).not.toHaveBeenCalled(); + expect(mocks.startJob).not.toHaveBeenCalled(); + expect(mocks.head).not.toHaveBeenCalled(); + expect(mocks.cancelJob).not.toHaveBeenCalled(); + expect(await readDeveloperVideoParts("video-1", "encode-1")).toBeNull(); + }); + + it("cancels a job that reports progress after the delete", async () => { + mocks.store.video = deletedVideo({ + hls: { state: "running", token: "t-hls", jobId: null, startedAt: 2_000 }, + }); + + const result = await applyDeveloperVideoProgress( + { videoId: "video-1", jobId: "job-7", phase: "downloading" }, + "hls", + "t-hls", + ); + + expect(result).toEqual({ success: true, applied: false }); + expect(mocks.cancelJob).toHaveBeenCalledExactlyOnceWith("job-7"); + expect(mocks.store.updates).toHaveLength(0); + }); + + // The callback reads the video while it is live, and `change` commits just + // before the callback's own write takes its row lock. + const progressBeforeConfirm = async (change: () => void) => { + mocks.store.video = developerVideo({ + processing: processingState({ + hls: { + state: "running", + token: "t-hls", + jobId: null, + startedAt: 2_000, + }, + }), + }); + const { db } = await import("@cap/database"); + const database = db(); + const transaction = database.transaction.bind(database); + const spy = vi.spyOn(database, "transaction").mockImplementationOnce((( + run: Parameters[0], + ) => { + change(); + return transaction(run); + }) as typeof transaction); + try { + return await applyDeveloperVideoProgress( + { videoId: "video-1", jobId: "job-7", phase: "downloading" }, + "hls", + "t-hls", + ); + } finally { + spy.mockRestore(); + } + }; + + it("cancels a job whose first progress lost its job id write to the delete", async () => { + const result = await progressBeforeConfirm(() => { + mocks.store.video = { ...video(), deletedAt: new Date() }; + }); + + expect(result).toEqual({ success: true, applied: false }); + expect(mocks.cancelJob).toHaveBeenCalledExactlyOnceWith("job-7"); + expect(mocks.store.updates).toHaveLength(0); + }); + + it("does not cancel a job whose first progress lost its write on a live video", async () => { + await progressBeforeConfirm(() => { + const current = video(); + mocks.store.video = { + ...current, + processing: current.processing && { + ...current.processing, + hls: runningPart("newer"), + }, + }; + }); + + expect(mocks.cancelJob).not.toHaveBeenCalled(); + expect(mocks.store.updates).toHaveLength(0); + }); + + it("does not cancel on progress from another attempt or without a job id", async () => { + mocks.store.video = deletedVideo(); + + await applyDeveloperVideoProgress( + { videoId: "video-1", jobId: "job-old", phase: "processing" }, + "hls", + "stale", + ); + await applyDeveloperVideoProgress( + { videoId: "video-1", phase: "processing" }, + "hls", + "t-hls", + ); + + expect(mocks.cancelJob).not.toHaveBeenCalled(); + expect(mocks.store.updates).toHaveLength(0); + }); + + it("cancels a job whose start returned after the video was deleted", async () => { + mocks.startJob.mockImplementation(async () => { + mocks.store.video = { ...video(), deletedAt: new Date() }; + return { status: "started", jobId: "job-9" }; + }); + + expect(await dispatchDeveloperVideoPart("video-1", "encode-1", "hls")).toBe( + "dispatched", + ); + + expect(mocks.cancelJob).toHaveBeenCalledExactlyOnceWith("job-9"); + expect(video().processing?.hls).toMatchObject({ jobId: null }); + }); + + it("does not cancel a started job that was recorded", async () => { + mocks.startJob.mockResolvedValue({ status: "started", jobId: "job-9" }); + + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + + expect(video().processing?.hls).toMatchObject({ jobId: "job-9" }); + expect(mocks.cancelJob).not.toHaveBeenCalled(); + }); + + it("does not cancel a started job that lost to another attempt on a live video", async () => { + mocks.startJob.mockImplementation(async () => { + const current = video(); + mocks.store.video = { + ...current, + processing: current.processing && { + ...current.processing, + hls: runningPart("newer"), + }, + }; + return { status: "started", jobId: "job-9" }; + }); + + await dispatchDeveloperVideoPart("video-1", "encode-1", "hls"); + + expect(mocks.cancelJob).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-staff-delete.test.ts b/apps/web/__tests__/unit/developer-video-staff-delete.test.ts new file mode 100644 index 00000000000..b8d0d2df5a2 --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-staff-delete.test.ts @@ -0,0 +1,327 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + type DeveloperVideoStore, + developerVideo, + processingState, +} from "../fixtures/developer-video"; + +const mocks = vi.hoisted(() => ({ + store: { video: null, appDeleted: false, updates: [] } as DeveloperVideoStore, + user: null as { id: string; email: string } | null, + settleAtDelete: vi.fn(), + cancelEncodes: vi.fn(), + deleteObjects: vi.fn(), +})); + +vi.mock("@cap/database", async () => { + const { createDeveloperVideoDatabase } = await import( + "../fixtures/developer-video" + ); + const { withTakedownSql } = await import( + "../fixtures/developer-video-takedown" + ); + const database = await withTakedownSql( + mocks.store, + await createDeveloperVideoDatabase(mocks.store), + ); + return { db: () => database }; +}); +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: async () => mocks.user, +})); +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ WEB_URL: "https://web.example" }), +})); +vi.mock("@/lib/infra-storage-minimum", () => ({ + settleFirstMonthAtDelete: mocks.settleAtDelete, +})); +vi.mock("@/lib/developer-video-cancel", () => ({ + cancelDeveloperVideoEncodes: mocks.cancelEncodes, +})); +vi.mock("@/lib/developer-video-storage", () => ({ + deleteDeveloperVideoObjects: mocks.deleteObjects, + developerVideoPrefix: (video: { appId: string; id: string }) => + `developer/${video.appId}/${video.id}/`, +})); + +const { POST } = await import("@/app/api/admin/developer-video-takedown/route"); +const purge = await import("@/lib/developer-video-purge"); +const takedown = await import("@/lib/developer-video-takedown"); +const script = await import("../../../../scripts/infra-video-takedown"); + +const PLAYBACK_ID = "playback-id-0123456789ab"; +const STAFF = { id: "staff-user", email: "richie@cap.so" }; + +const ready = () => + developerVideo({ + status: "ready", + processing: processingState({ + hls: { state: "complete", output: "hls-out" }, + mp4: { state: "complete", output: "mp4-out" }, + poster: { output: "mp4-out" }, + }), + }); + +const admin = (body: unknown, headers: Record = {}) => + POST( + new Request("https://web.example/api/admin/developer-video-takedown", { + method: "POST", + headers: { "content-type": "application/json", ...headers }, + body: JSON.stringify(body), + }), + ); + +beforeEach(() => { + mocks.store.video = ready(); + mocks.store.more = []; + mocks.store.updates = []; + mocks.store.appDeleted = false; + mocks.user = STAFF; + mocks.settleAtDelete.mockReset(); + mocks.cancelEncodes.mockReset(); + mocks.deleteObjects.mockReset(); + mocks.deleteObjects.mockResolvedValue(6); + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(console, "info").mockImplementation(() => undefined); +}); + +describe("staff delete", () => { + it("takes the video down, deletes it as its owner would and purges its files at once", async () => { + const response = await admin({ video: PLAYBACK_ID, delete: true }); + + expect(response.status).toBe(200); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect((await response.json()).data).toEqual({ + videoId: "video-1", + appId: "app-1", + playbackId: PLAYBACK_ID, + alreadyDeleted: false, + filesPurged: true, + }); + expect(mocks.store.updates.map((update) => Object.keys(update))).toEqual([ + ["metadata"], + ["deletedAt"], + ["filesDeletedAt"], + ]); + const video = mocks.store.video; + expect( + takedown.developerVideoIsTakenDown({ metadata: video?.metadata }), + ).toBe(true); + expect(video?.deletedAt).toBeInstanceOf(Date); + expect(video?.filesDeletedAt).toBeInstanceOf(Date); + expect(mocks.settleAtDelete.mock.calls).toEqual([["video-1"]]); + expect(mocks.cancelEncodes.mock.calls).toEqual([ + [{ appId: "app-1", videoId: "video-1" }], + ]); + expect(mocks.deleteObjects.mock.calls).toEqual([ + ["developer/app-1/video-1/"], + ]); + expect(console.info).toHaveBeenCalledWith( + "[developer-video] Deleted by staff", + { + by: "staff-user", + videoId: "video-1", + alreadyDeleted: false, + filesPurged: true, + }, + ); + }); + + it("settles and cancels only once when asked again, and purges again", async () => { + await admin({ video: PLAYBACK_ID, delete: true }); + const again = await admin({ video: PLAYBACK_ID, delete: true }); + + expect((await again.json()).data).toMatchObject({ + alreadyDeleted: true, + filesPurged: true, + }); + expect(mocks.settleAtDelete).toHaveBeenCalledTimes(1); + expect(mocks.cancelEncodes).toHaveBeenCalledTimes(1); + expect(mocks.deleteObjects.mock.calls).toEqual([ + ["developer/app-1/video-1/"], + ["developer/app-1/video-1/"], + ]); + }); + + it("purges a video its owner already deleted without billing it again", async () => { + mocks.store.video = { ...ready(), deletedAt: new Date() }; + + const response = await admin({ video: "video-1", delete: true }); + + expect((await response.json()).data).toMatchObject({ + alreadyDeleted: true, + filesPurged: true, + }); + expect(mocks.settleAtDelete).not.toHaveBeenCalled(); + expect(mocks.cancelEncodes).not.toHaveBeenCalled(); + expect(mocks.store.updates.map((update) => Object.keys(update))).toEqual([ + ["filesDeletedAt"], + ]); + }); + + it.each([ + ["a video id", "video-1"], + ["the hosted page", `https://cap.so/dev/${PLAYBACK_ID}`], + ])("finds the video from %s", async (_, input) => { + const response = await admin({ video: input, delete: true }); + + expect((await response.json()).data.videoId).toBe("video-1"); + }); + + it.each([ + ["no video", null], + ["input that is not an id", "not an id!"], + ])("answers 404 for %s and deletes nothing", async (_, input) => { + if (input === null) mocks.store.video = null; + + const response = await admin({ + video: input ?? PLAYBACK_ID, + delete: true, + }); + + expect(response.status).toBe(404); + expect(mocks.store.updates).toEqual([]); + expect(mocks.deleteObjects).not.toHaveBeenCalled(); + }); + + it("keeps the delete and leaves the files to the purge job when storage fails", async () => { + mocks.deleteObjects.mockRejectedValueOnce(new Error("storage down")); + + const response = await admin({ video: PLAYBACK_ID, delete: true }); + + expect(response.status).toBe(200); + expect((await response.json()).data.filesPurged).toBe(false); + expect(mocks.store.video?.deletedAt).toBeInstanceOf(Date); + expect(mocks.store.video?.filesDeletedAt).toBeNull(); + }); + + it("clears an earlier purge mark when the purge fails, so the cron retries soon", async () => { + mocks.store.video = { + ...ready(), + deletedAt: new Date(), + filesDeletedAt: new Date(), + }; + mocks.deleteObjects.mockRejectedValueOnce(new Error("storage down")); + + const response = await admin({ video: PLAYBACK_ID, delete: true }); + + expect((await response.json()).data.filesPurged).toBe(false); + expect(mocks.store.video?.filesDeletedAt).toBeNull(); + expect(mocks.store.video?.deletedAt).toBeInstanceOf(Date); + }); + + it.each([ + ["signed out", null, {}], + [ + "a signed-in account that is not staff", + { id: "user-2", email: "someone@example.com" }, + {}, + ], + [ + "an Infra secret key sent as Bearer", + null, + { authorization: "Bearer cap_sk_live_0123456789abcdef" }, + ], + ])("refuses %s and touches nothing", async (_, user, headers) => { + mocks.user = user; + + const response = await admin({ video: PLAYBACK_ID, delete: true }, headers); + + expect(response.status).toBe(403); + expect(mocks.store.updates).toEqual([]); + expect(mocks.settleAtDelete).not.toHaveBeenCalled(); + expect(mocks.deleteObjects).not.toHaveBeenCalled(); + }); + + it.each([ + ["JSON from another site", { origin: "https://evil.example" }], + ["a form post", { "content-type": "application/x-www-form-urlencoded" }], + ])("refuses %s before reading the session", async (_, headers) => { + const response = await admin({ video: PLAYBACK_ID, delete: true }, headers); + + expect(response.status).toBe(400); + expect(mocks.store.updates).toEqual([]); + expect(mocks.deleteObjects).not.toHaveBeenCalled(); + }); + + it("leaves the takedown contract as it was when delete is not true", async () => { + const response = await admin({ video: PLAYBACK_ID, delete: "yes" }); + + expect(response.status).toBe(400); + expect(mocks.store.updates).toEqual([]); + }); +}); + +describe("the immediate purge", () => { + it.each([ + ["an empty video id", { appId: "app-1", id: "" }], + ["an empty app id", { appId: "", id: "video-1" }], + ["a video id that climbs out", { appId: "app-1", id: ".." }], + ["a video id with a slash", { appId: "app-1", id: "video-1/hls" }], + ["an app id with a slash", { appId: "app-1/video-1", id: "x" }], + ])("refuses %s and deletes nothing", async (_, row) => { + expect(() => purge.developerVideoPurgePrefix(row)).toThrow( + /Refusing to purge/, + ); + await expect(purge.purgeDeletedDeveloperVideoNow(row)).rejects.toThrow( + /Refusing to purge/, + ); + expect(mocks.deleteObjects).not.toHaveBeenCalled(); + expect(mocks.store.updates).toEqual([]); + }); + + it("deletes only that video's folder", async () => { + expect( + await purge.purgeDeletedDeveloperVideoNow({ + appId: "app-1", + id: "video-1", + }), + ).toBe(true); + expect(mocks.deleteObjects.mock.calls).toEqual([ + ["developer/app-1/video-1/"], + ]); + }); +}); + +describe("the staff script", () => { + const run = async (argv: string[]) => { + const lines: string[] = []; + const calls: { url: string; body: unknown }[] = []; + const code = await script.runTakedown({ + argv, + env: { CAP_SESSION_TOKEN: "session" }, + fetchImpl: async (url: string, init: RequestInit) => { + const body = JSON.parse(String(init.body)); + calls.push({ url, body }); + return admin(body); + }, + log: (line: string) => lines.push(line), + fail: (line: string) => lines.push(`error: ${line}`), + }); + return { code, lines, calls }; + }; + + it("deletes through the staff route", async () => { + const deleted = await run(["delete", PLAYBACK_ID]); + + expect(deleted.code).toBe(0); + expect(deleted.calls).toEqual([ + { + url: "https://cap.so/api/admin/developer-video-takedown", + body: { video: PLAYBACK_ID, delete: true }, + }, + ]); + expect(deleted.lines).toEqual([ + `video-1 (playback ${PLAYBACK_ID}, app app-1) is deleted, and its files are removed.`, + ]); + }); + + it("fails when the files could not be removed yet", async () => { + mocks.deleteObjects.mockRejectedValueOnce(new Error("storage down")); + + const deleted = await run(["delete", PLAYBACK_ID]); + + expect(deleted.code).toBe(1); + expect(deleted.lines[0]).toMatch(/purge job will retry/); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-start.test.ts b/apps/web/__tests__/unit/developer-video-start.test.ts index f1be74c4870..2c7152d9205 100644 --- a/apps/web/__tests__/unit/developer-video-start.test.ts +++ b/apps/web/__tests__/unit/developer-video-start.test.ts @@ -6,7 +6,6 @@ const mocks = vi.hoisted(() => ({ claim: vi.fn(), release: vi.fn(), prepare: vi.fn(), - readParts: vi.fn(), dispatch: vi.fn(), fail: vi.fn(), order: [] as string[], @@ -23,8 +22,7 @@ vi.mock("@/lib/developer-video-processing", () => ({ claimDeveloperVideoProcessing: mocks.claim, releaseDeveloperVideoClaim: mocks.release, prepareDeveloperVideoProcessing: mocks.prepare, - readDeveloperVideoParts: mocks.readParts, - dispatchDeveloperVideoPart: mocks.dispatch, + dispatchDeveloperVideoParts: mocks.dispatch, failDeveloperVideoProcessing: mocks.fail, isDeveloperVideoRefusal: (error: unknown) => error instanceof Refused, })); @@ -53,10 +51,9 @@ beforeEach(() => { return "ready"; }); mocks.fail.mockResolvedValue(undefined); - mocks.readParts.mockResolvedValue({ hls: "dispatch", mp4: "dispatch" }); - mocks.dispatch.mockImplementation(async (_video, _encode, part) => { - mocks.order.push(`dispatch:${part}`); - return "dispatched"; + mocks.dispatch.mockImplementation(async () => { + mocks.order.push("dispatch"); + return []; }); vi.spyOn(console, "error").mockImplementation(() => undefined); }); @@ -70,12 +67,7 @@ describe("developer video start", () => { expect(mocks.start).toHaveBeenCalledWith("workflow", [ { videoId: "video-1", encodeId: "encode-1" }, ]); - expect(mocks.order[0]).toBe("workflow"); - expect(mocks.order[1]).toBe("prepare"); - expect(mocks.order.slice(2).sort()).toEqual([ - "dispatch:hls", - "dispatch:mp4", - ]); + expect(mocks.order).toEqual(["workflow", "prepare", "dispatch"]); expect(mocks.after).toHaveBeenCalledTimes(1); }); @@ -85,34 +77,49 @@ describe("developer video start", () => { }); expect(await startDeveloperVideoProcessing("video-1", claim)).toBe(true); - await vi.waitFor(() => expect(mocks.dispatch).toHaveBeenCalledTimes(2)); + await vi.waitFor(() => expect(mocks.dispatch).toHaveBeenCalledTimes(1)); }); - it("only sends the outputs that still need a job", async () => { - mocks.readParts.mockResolvedValue({ hls: "dispatch", mp4: "done" }); + it("hands the source the probe read to the dispatch so it is not read again", async () => { + const source = { bytes: 1_000, contentType: "video/quicktime" }; + mocks.prepare.mockImplementation( + async (_video: string, _encode: string, seen: { source?: unknown }) => { + seen.source = source; + return "ready"; + }, + ); + + await kickStartDeveloperVideo("video-1", "encode-1"); + + expect(mocks.dispatch).toHaveBeenCalledWith("video-1", "encode-1", source); + }); + it("leaves the source unknown when the probe was done before", async () => { await kickStartDeveloperVideo("video-1", "encode-1"); - expect(mocks.dispatch.mock.calls).toEqual([["video-1", "encode-1", "hls"]]); + expect(mocks.dispatch).toHaveBeenCalledWith( + "video-1", + "encode-1", + undefined, + ); }); - it("waits for both outputs to be sent even when one of them fails", async () => { + it("waits for the outputs to be sent before the first pass ends", async () => { let release: () => void = () => undefined; const slow = new Promise((resolve) => { release = resolve; }); - mocks.dispatch.mockImplementation(async (_video, _encode, part) => { - if (part === "mp4") throw new Error("storage unavailable"); + mocks.dispatch.mockImplementation(async () => { await slow; - mocks.order.push("dispatch:hls"); - return "dispatched"; + mocks.order.push("dispatch"); + return []; }); let settled = false; const kick = kickStartDeveloperVideo("video-1", "encode-1").then(() => { settled = true; }); - await vi.waitFor(() => expect(mocks.dispatch).toHaveBeenCalledTimes(2)); + await vi.waitFor(() => expect(mocks.dispatch).toHaveBeenCalledTimes(1)); await Promise.resolve(); expect(settled).toBe(false); @@ -120,7 +127,7 @@ describe("developer video start", () => { release(); await kick; - expect(mocks.order).toContain("dispatch:hls"); + expect(mocks.order).toContain("dispatch"); }); it.each(["retry", "stale", "probing"])( @@ -130,7 +137,6 @@ describe("developer video start", () => { await kickStartDeveloperVideo("video-1", "encode-1"); - expect(mocks.readParts).not.toHaveBeenCalled(); expect(mocks.dispatch).not.toHaveBeenCalled(); }, ); @@ -195,12 +201,13 @@ describe("developer video start", () => { action: "retry-hls", encodeId: "encode-9", }); - mocks.readParts.mockResolvedValue({ hls: "dispatch", mp4: "done" }); expect(await redriveDeveloperVideoProcessing("video-1")).toBe(true); await kicked(); - expect(mocks.prepare).toHaveBeenCalledWith("video-1", "encode-9"); - expect(mocks.dispatch.mock.calls).toEqual([["video-1", "encode-9", "hls"]]); + expect(mocks.prepare).toHaveBeenCalledWith("video-1", "encode-9", {}); + expect(mocks.dispatch.mock.calls).toEqual([ + ["video-1", "encode-9", undefined], + ]); }); }); diff --git a/apps/web/__tests__/unit/developer-video-state.test.ts b/apps/web/__tests__/unit/developer-video-state.test.ts index 31c48ab8601..4a4341b99f5 100644 --- a/apps/web/__tests__/unit/developer-video-state.test.ts +++ b/apps/web/__tests__/unit/developer-video-state.test.ts @@ -532,6 +532,20 @@ describe("developer video redrive", () => { expect(developerVideoRedrive(startedAt(7), now)).toBe("start"); }); + it("times a video that waited for a free worker from its last busy answer", () => { + const waited = (busyHoursAgo: number) => + developerVideo({ + processing: processingState({ + startedAt: now - 7 * 3_600_000, + busyAt: now - busyHoursAgo * 3_600_000, + }), + }); + + expect(developerVideoRedrive(waited(1), now)).toBeNull(); + expect(developerVideoRedrive(waited(5.9), now)).toBeNull(); + expect(developerVideoRedrive(waited(6), now)).toBe("start"); + }); + it("retries adaptive streaming only once its retry time has passed", () => { const parked = (retryAt: number) => developerVideo({ diff --git a/apps/web/__tests__/unit/developer-video-storage.test.ts b/apps/web/__tests__/unit/developer-video-storage.test.ts index 30189603ea2..a771396ea35 100644 --- a/apps/web/__tests__/unit/developer-video-storage.test.ts +++ b/apps/web/__tests__/unit/developer-video-storage.test.ts @@ -4,14 +4,20 @@ vi.mock("@cap/web-backend/src/S3Buckets/index", async () => { const { Effect, Layer, Option } = await import("effect"); const HttpServerRequest = await import("@effect/platform/HttpServerRequest"); const access = { - getSignedObjectUrl: (key: string, args: { expiresIn: number }) => + getSignedObjectUrl: ( + key: string, + args: { signingDate: Date; expiresIn: number }, + ) => Effect.serviceOption(HttpServerRequest.HttpServerRequest).pipe( Effect.map( (request) => - `signed:${key}:${args.expiresIn}:${Option.match(request, { - onNone: () => "no-request", - onSome: (value) => value.originalUrl, - })}`, + `signed:${key}:${args.signingDate.toISOString()}+${args.expiresIn}:${Option.match( + request, + { + onNone: () => "no-request", + onSome: (value) => value.originalUrl, + }, + )}`, ), ), }; @@ -23,9 +29,11 @@ vi.mock("@cap/web-backend/src/S3Buckets/index", async () => { }; }); -const { requestAsSeenByViewer, signedDeveloperVideoUrl } = await import( - "@/lib/developer-video-storage" -); +const { + developerVideoUrlAnswersAnyOrigin, + requestAsSeenByViewer, + signedDeveloperVideoUrl, +} = await import("@/lib/developer-video-storage"); const playback = (headers: Record) => new Request("http://localhost:3000/api/developer/playback/id/video.mp4", { @@ -36,12 +44,12 @@ describe("developer video storage signing", () => { it("signs with the address the viewer used, so a local storage endpoint can follow it", async () => { const url = await signedDeveloperVideoUrl( "developer/app/video/file.m4s", - 600, + { signingDate: new Date("2026-11-12T12:00:00.000Z"), expiresIn: 600 }, playback({ host: "192.168.1.20:3000" }), ); expect(url).toBe( - "signed:developer/app/video/file.m4s:600:http://192.168.1.20:3000/api/developer/playback/id/video.mp4", + "signed:developer/app/video/file.m4s:2026-11-12T12:00:00.000Z+600:http://192.168.1.20:3000/api/developer/playback/id/video.mp4", ); }); @@ -64,4 +72,17 @@ describe("developer video storage signing", () => { bare.url, ); }); + + it("lets a player reach media through a redirect only when the CDN serves it", () => { + expect( + developerVideoUrlAnswersAnyOrigin( + "https://cdn.example/developer/app/video/file.m4s?Policy=p&Key-Pair-Id=K1&Signature=s", + ), + ).toBe(true); + expect( + developerVideoUrlAnswersAnyOrigin( + "http://localhost:9000/capso/developer/app/video/file.m4s?X-Amz-Signature=s", + ), + ).toBe(false); + }); }); diff --git a/apps/web/__tests__/unit/developer-video-takedown-playback.test.ts b/apps/web/__tests__/unit/developer-video-takedown-playback.test.ts new file mode 100644 index 00000000000..0c091fa6aa2 --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-takedown-playback.test.ts @@ -0,0 +1,258 @@ +import type { NextRequest } from "next/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + type DeveloperVideoStore, + developerVideo, + processingState, +} from "../fixtures/developer-video"; + +const mocks = vi.hoisted(() => ({ + store: { video: null, appDeleted: false, updates: [] } as DeveloperVideoStore, + readText: vi.fn(), + sign: vi.fn(), + answersAnyOrigin: vi.fn(), + standing: vi.fn(), +})); + +vi.mock("@cap/database", async () => { + const { createDeveloperVideoDatabase } = await import( + "../fixtures/developer-video" + ); + const { withTakedownSql } = await import( + "../fixtures/developer-video-takedown" + ); + const database = await withTakedownSql( + mocks.store, + await createDeveloperVideoDatabase(mocks.store), + ); + return { db: () => database }; +}); +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ WEB_URL: "https://web.example" }), +})); +vi.mock("@/lib/infra-standing", () => ({ + getInfraAppStanding: mocks.standing, +})); +vi.mock("@/lib/developer-video-storage", () => ({ + developerVideoUrlAnswersAnyOrigin: mocks.answersAnyOrigin, + readDeveloperVideoText: mocks.readText, + signedDeveloperVideoUrl: mocks.sign, +})); + +const { GET, HEAD, OPTIONS } = await import( + "@/app/api/developer/playback/[playbackId]/[file]/route" +); +const { setDeveloperVideoTakedown } = await import( + "@/lib/developer-video-takedown-store" +); + +const PLAYBACK_ID = "playback-id-0123456789ab"; +const HLS_PREFIX = "developer/app-1/video-1/encode-1/hls-out/hls/"; +const master = [ + "#EXTM3U", + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud",NAME="audio",URI="audio.m3u8"', + "#EXT-X-STREAM-INF:BANDWIDTH=4000000", + "1080p.m3u8", +].join("\n"); +const rendition = [ + "#EXTM3U", + '#EXT-X-MAP:URI="1080p.m4s",BYTERANGE="800@0"', + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:1000@800", + "1080p.m4s", + "#EXT-X-ENDLIST", +].join("\n"); + +const ready = (overrides = {}) => + developerVideo({ + status: "ready", + processing: processingState({ + hls: { state: "complete", output: "hls-out" }, + mp4: { state: "complete", output: "mp4-out" }, + poster: { output: "mp4-out" }, + bytes: { source: 1_000, hls: 5_000, mp4: 2_000, poster: 50 }, + }), + ...overrides, + }); + +const FILES = [ + "master.m3u8", + "master.m3u8?segments=short", + "1080p.m3u8", + "1080p.m3u8?segments=short", + "audio.m3u8", + "1080p.m4s", + "video.mp4", + "poster.jpg", + "missing.m3u8", + "anything.txt", +]; + +function call(handler: typeof GET, target: string) { + const [file, query] = target.split("?") as [string, string?]; + const request = new Request( + `https://web.example/api/developer/playback/${PLAYBACK_ID}/${file}${query ? `?${query}` : ""}`, + { method: handler === HEAD ? "HEAD" : "GET" }, + ) as NextRequest; + return handler(request, { + params: Promise.resolve({ playbackId: PLAYBACK_ID, file }), + }); +} + +async function snapshot(response: Response) { + return { + status: response.status, + headers: [...response.headers.entries()].sort(), + body: await response.text(), + }; +} + +async function answers(handler: typeof GET = GET) { + const result: Record>> = {}; + for (const file of FILES) + result[file] = await snapshot(await call(handler, file)); + return result; +} + +afterEach(() => { + vi.useRealTimers(); +}); + +beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(Date.UTC(2026, 10, 13, 12)); + mocks.store.video = ready(); + mocks.store.updates = []; + mocks.store.appDeleted = false; + mocks.standing.mockResolvedValue({ state: "active", pausesAt: null }); + mocks.answersAnyOrigin.mockReturnValue(true); + mocks.sign.mockImplementation( + async (key: string) => `https://cdn.example/${key}?sig=a`, + ); + mocks.readText.mockImplementation(async (key: string) => + key === `${HLS_PREFIX}master.m3u8` + ? master + : key.endsWith(".m3u8") && key !== `${HLS_PREFIX}missing.m3u8` + ? rendition + : null, + ); + vi.spyOn(console, "error").mockImplementation(() => undefined); +}); + +describe("a taken down video on the playback route", () => { + it.each(FILES)( + "answers %s with a plain 410 that is never cached", + async (file) => { + await setDeveloperVideoTakedown(PLAYBACK_ID, true); + vi.clearAllMocks(); + + const response = await call(GET, file); + + expect(response.status).toBe(410); + expect(await response.text()).toBe("This video is not available."); + expect(response.headers.get("cache-control")).toBe("no-store"); + expect(response.headers.get("content-type")).toBe( + "text/plain; charset=utf-8", + ); + expect(response.headers.get("access-control-allow-origin")).toBe("*"); + expect(response.headers.get("location")).toBeNull(); + expect(mocks.sign).not.toHaveBeenCalled(); + expect(mocks.readText).not.toHaveBeenCalled(); + expect(mocks.standing).not.toHaveBeenCalled(); + }, + ); + + it("answers HEAD with the same 410", async () => { + await setDeveloperVideoTakedown(PLAYBACK_ID, true); + + for (const file of FILES) { + const response = await call(HEAD, file); + expect(response.status).toBe(410); + expect(response.headers.get("cache-control")).toBe("no-store"); + } + }); + + it("answers 410 before the out of credit pause, and for a video still processing", async () => { + mocks.standing.mockResolvedValue({ state: "paused", pausesAt: null }); + mocks.store.video = ready({ status: "processing" }); + await setDeveloperVideoTakedown(PLAYBACK_ID, true); + + expect((await call(GET, "master.m3u8")).status).toBe(410); + mocks.store.video = { ...mocks.store.video, processing: null }; + expect((await call(GET, "video.mp4")).status).toBe(410); + }); + + it("leaves the preflight answer alone", async () => { + const before = await snapshot(OPTIONS()); + await setDeveloperVideoTakedown(PLAYBACK_ID, true); + + expect(await snapshot(OPTIONS())).toEqual(before); + }); + + it("still answers 404 for a deleted video", async () => { + await setDeveloperVideoTakedown(PLAYBACK_ID, true); + mocks.store.video = { + ...(mocks.store.video ?? ready()), + deletedAt: new Date(), + }; + + expect((await call(GET, "master.m3u8")).status).toBe(404); + }); +}); + +describe("restoring a taken down video", () => { + it.each([ + { name: "no metadata", metadata: null }, + { + name: "the developer's metadata", + metadata: { orderId: 42, tags: ["a"] }, + }, + ])( + "answers every file exactly as before, with $name", + async ({ metadata }) => { + mocks.store.video = ready({ metadata }); + const before = await answers(); + const beforeHead = await answers(HEAD); + expect(before["master.m3u8"]?.status).toBe(200); + expect(before["1080p.m4s"]?.status).toBe(302); + + await setDeveloperVideoTakedown(PLAYBACK_ID, true); + const down = await answers(); + for (const file of FILES) expect(down[file]?.status).toBe(410); + + await setDeveloperVideoTakedown(PLAYBACK_ID, false); + + expect(await answers()).toEqual(before); + expect(await answers(HEAD)).toEqual(beforeHead); + expect(mocks.store.video?.metadata).toEqual(metadata); + }, + ); +}); + +describe("videos that were never taken down", () => { + it("answer as they did before this change", async () => { + const response = await call(GET, "master.m3u8"); + + expect(response.status).toBe(200); + expect(response.headers.get("cache-control")).toBe( + "public, max-age=60, s-maxage=60", + ); + const media = await call(GET, "1080p.m4s"); + expect(media.status).toBe(302); + expect(media.headers.get("location")).toBe( + `https://cdn.example/${HLS_PREFIX}1080p.m4s?sig=a`, + ); + expect(media.headers.get("cache-control")).toBe( + "public, max-age=60, s-maxage=60", + ); + }); + + it("keep developer metadata that only looks like a marker playing", async () => { + mocks.store.video = ready({ + metadata: { __capTakedown: "not a marker" }, + }); + + expect((await call(GET, "master.m3u8")).status).toBe(200); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-takedown.test.ts b/apps/web/__tests__/unit/developer-video-takedown.test.ts new file mode 100644 index 00000000000..e4c3a6546ea --- /dev/null +++ b/apps/web/__tests__/unit/developer-video-takedown.test.ts @@ -0,0 +1,514 @@ +import { MySqlDialect } from "drizzle-orm/mysql-core"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + type DeveloperVideoStore, + developerVideo, + processingState, +} from "../fixtures/developer-video"; + +const mocks = vi.hoisted(() => ({ + store: { video: null, appDeleted: false, updates: [] } as DeveloperVideoStore, + user: null as { id: string; email: string } | null, + purge: vi.fn(), +})); + +vi.mock("@vercel/functions", () => ({ + dangerouslyDeleteByTag: mocks.purge, +})); + +vi.mock("@cap/database", async () => { + const { createDeveloperVideoDatabase } = await import( + "../fixtures/developer-video" + ); + const { withTakedownSql } = await import( + "../fixtures/developer-video-takedown" + ); + const database = await withTakedownSql( + mocks.store, + await createDeveloperVideoDatabase(mocks.store), + ); + return { db: () => database }; +}); +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: async () => mocks.user, +})); +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ WEB_URL: "https://web.example" }), +})); + +const takedown = await import("@/lib/developer-video-takedown"); +const store = await import("@/lib/developer-video-takedown-store"); +const sqlHelpers = await import("@cap/database/developer-video-takedown"); +const api = await import("@/lib/developer-video-api"); +const view = await import("@/lib/infra-video-view"); +const { POST } = await import("@/app/api/admin/developer-video-takedown/route"); +const script = await import("../../../../scripts/infra-video-takedown"); + +const PLAYBACK_ID = "playback-id-0123456789ab"; +const NOW = new Date("2026-11-13T12:00:00.000Z"); +const STAFF = { id: "staff-user", email: "richie@cap.so" }; + +const ready = (overrides: Parameters[0] = {}) => + developerVideo({ + status: "ready", + processing: processingState({ + hls: { state: "complete", output: "hls-out" }, + mp4: { state: "complete", output: "mp4-out" }, + poster: { output: "mp4-out" }, + }), + ...overrides, + }); + +const admin = (body: unknown, headers: Record = {}) => + POST( + new Request("https://web.example/api/admin/developer-video-takedown", { + method: "POST", + headers: { "content-type": "application/json", ...headers }, + body: typeof body === "string" ? body : JSON.stringify(body), + }), + ); + +beforeEach(() => { + mocks.store.video = ready(); + mocks.store.more = []; + mocks.store.updates = []; + mocks.store.appDeleted = false; + mocks.user = STAFF; + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(console, "info").mockImplementation(() => undefined); +}); + +describe("takedown marker", () => { + it("keeps the developer's metadata whole and gives it back on restore", async () => { + const metadata = { orderId: 42, nested: { list: [1, "two"] } }; + mocks.store.video = ready({ metadata }); + + await store.setDeveloperVideoTakedown(PLAYBACK_ID, true, NOW); + const stored = mocks.store.video?.metadata; + + expect(stored).toEqual({ + __capTakedown: { at: NOW.toISOString(), metadata }, + }); + expect(takedown.developerVideoIsTakenDown({ metadata: stored })).toBe(true); + expect(takedown.developerVideoTakenDownAt(stored)).toBe(NOW.toISOString()); + expect(takedown.developerVideoPublicMetadata(stored)).toEqual(metadata); + + await store.setDeveloperVideoTakedown(PLAYBACK_ID, false, NOW); + expect(mocks.store.video?.metadata).toEqual(metadata); + }); + + it.each([ + null, + {}, + { __capTakedown: "a developer value" }, + { __capTakedown: { at: 5, metadata: null } }, + { __capTakedown: { at: "2026-11-13" } }, + [1, 2], + "text", + ])("does not read %j as taken down", (metadata) => { + expect(takedown.developerVideoIsTakenDown({ metadata })).toBe(false); + expect(takedown.developerVideoPublicMetadata(metadata)).toEqual(metadata); + }); + + it("drops the reserved key from metadata a developer sends, and leaves other metadata as sent", () => { + const own = { a: 1 }; + expect(takedown.withoutReservedMetadata(own)).toBe(own); + expect(takedown.withoutReservedMetadata(undefined)).toBeUndefined(); + expect( + takedown.withoutReservedMetadata({ a: 1, __capTakedown: { at: "x" } }), + ).toEqual({ a: 1 }); + }); + + it("rewrites metadata in SQL so large numbers in it are never parsed and written back", () => { + const dialect = new MySqlDialect(); + const down = dialect.sqlToQuery(sqlHelpers.takenDownMetadataSql(NOW)); + const restore = dialect.sqlToQuery(sqlHelpers.restoredMetadataSql()); + const listed = dialect.sqlToQuery(sqlHelpers.developerVideoTakenDownSql()); + + expect(down.sql).toBe( + "JSON_OBJECT(?, JSON_OBJECT('at', ?, 'metadata', `developer_videos`.`metadata`))", + ); + expect(down.params).toEqual(["__capTakedown", NOW.toISOString()]); + expect(restore.sql).toBe( + "CASE WHEN JSON_TYPE(JSON_EXTRACT(`developer_videos`.`metadata`, ?)) = 'NULL' THEN NULL ELSE JSON_EXTRACT(`developer_videos`.`metadata`, ?) END", + ); + expect(restore.params).toEqual([ + '$."__capTakedown".metadata', + '$."__capTakedown".metadata', + ]); + expect(listed.sql).toMatch(/^COALESCE\(\(JSON_TYPE\(.*\), FALSE\)$/); + expect(listed.params).toEqual([ + '$."__capTakedown".at', + '$."__capTakedown".metadata', + ]); + }); +}); + +describe("the takedown switch", () => { + it.each([ + ["a playback id", PLAYBACK_ID], + ["a video id", "video-1"], + ["the hosted page", `https://cap.so/dev/${PLAYBACK_ID}`], + ["the embed", `https://cap.so/dev/${PLAYBACK_ID}/embed?autoplay=1`], + [ + "a playback file", + `https://cap.so/api/developer/playback/${PLAYBACK_ID}/master.m3u8`, + ], + ["a playback host", `https://play.example/${PLAYBACK_ID}/video.mp4`], + ])("finds the video from %s", async (_, input) => { + const result = await store.setDeveloperVideoTakedown(input, true, NOW); + + expect(result).toEqual({ + videoId: "video-1", + appId: "app-1", + playbackId: PLAYBACK_ID, + takenDown: true, + takenDownAt: NOW.toISOString(), + changed: true, + }); + }); + + it("only writes metadata, so status, duration and every billed column stay as they were", async () => { + const before = { ...mocks.store.video }; + + await store.setDeveloperVideoTakedown(PLAYBACK_ID, true, NOW); + await store.setDeveloperVideoTakedown(PLAYBACK_ID, false, NOW); + + expect(mocks.store.updates.map((update) => Object.keys(update))).toEqual([ + ["metadata"], + ["metadata"], + ]); + expect(mocks.store.video).toEqual(before); + }); + + it("does nothing when the video is already in the asked state", async () => { + await store.setDeveloperVideoTakedown(PLAYBACK_ID, true, NOW); + const again = await store.setDeveloperVideoTakedown( + PLAYBACK_ID, + true, + new Date(NOW.getTime() + 60_000), + ); + await store.setDeveloperVideoTakedown(PLAYBACK_ID, false, NOW); + const restoredAgain = await store.setDeveloperVideoTakedown( + PLAYBACK_ID, + false, + NOW, + ); + + expect(again).toMatchObject({ + changed: false, + takenDownAt: NOW.toISOString(), + }); + expect(restoredAgain).toMatchObject({ changed: false, takenDown: false }); + expect(mocks.store.updates).toHaveLength(2); + }); + + it("drops the video's cached playback on takedown and on restore, and only when it changed", async () => { + mocks.purge.mockClear(); + + await store.setDeveloperVideoTakedown(PLAYBACK_ID, true, NOW); + await store.setDeveloperVideoTakedown(PLAYBACK_ID, true, NOW); + await store.setDeveloperVideoTakedown(PLAYBACK_ID, false, NOW); + await store.setDeveloperVideoTakedown(PLAYBACK_ID, false, NOW); + + expect(mocks.purge.mock.calls).toEqual([ + ["infra-video-video-1"], + ["infra-video-video-1"], + ]); + }); + + it("still takes a video down when the cache purge fails", async () => { + mocks.purge.mockRejectedValueOnce(new Error("429")); + vi.spyOn(console, "error").mockImplementation(() => undefined); + + expect( + await store.setDeveloperVideoTakedown(PLAYBACK_ID, true, NOW), + ).toMatchObject({ takenDown: true, changed: true }); + expect(mocks.store.updates).toHaveLength(1); + }); + + it("finds no deleted video and nothing for input that is not an id", async () => { + expect( + await store.setDeveloperVideoTakedown("not an id!", true), + ).toBeNull(); + mocks.store.video = ready({ deletedAt: NOW }); + expect(await store.setDeveloperVideoTakedown(PLAYBACK_ID, true)).toBeNull(); + expect(mocks.store.updates).toEqual([]); + }); +}); + +describe("the staff route", () => { + it("takes a video down and restores it for staff", async () => { + const down = await admin({ video: PLAYBACK_ID, takenDown: true }); + expect(down.status).toBe(200); + expect(down.headers.get("cache-control")).toBe("no-store"); + expect((await down.json()).data).toMatchObject({ + videoId: "video-1", + takenDown: true, + changed: true, + }); + expect( + takedown.developerVideoIsTakenDown({ + metadata: mocks.store.video?.metadata, + }), + ).toBe(true); + + const restore = await admin({ video: PLAYBACK_ID, takenDown: false }); + expect((await restore.json()).data).toMatchObject({ + takenDown: false, + changed: true, + }); + expect(mocks.store.video?.metadata).toBeNull(); + }); + + it.each([ + ["signed out", null], + ["any other account", { id: "user-2", email: "someone@example.com" }], + ])("refuses %s and changes nothing", async (_, user) => { + mocks.user = user; + + const response = await admin({ video: PLAYBACK_ID, takenDown: true }); + + expect(response.status).toBe(403); + expect(mocks.store.updates).toEqual([]); + }); + + it.each([ + ["a form post", { "content-type": "application/x-www-form-urlencoded" }], + ["a text post", { "content-type": "text/plain" }], + [ + "JSON from another site", + { "content-type": "application/json", origin: "https://evil.example" }, + ], + ])("refuses %s before reading the session", async (_, headers) => { + const response = await admin( + { video: PLAYBACK_ID, takenDown: true }, + headers, + ); + + expect(response.status).toBe(400); + expect(mocks.store.updates).toEqual([]); + }); + + it("accepts JSON from its own origin", async () => { + const response = await admin( + { video: PLAYBACK_ID, takenDown: true }, + { origin: "https://web.example" }, + ); + + expect(response.status).toBe(200); + }); + + it.each([ + ["not JSON", "{"], + ["no video", { takenDown: true }], + ["a string flag", { video: PLAYBACK_ID, takenDown: "true" }], + ])("answers 400 for %s", async (_, body) => { + expect((await admin(body)).status).toBe(400); + }); + + it("answers 404 when no video matches", async () => { + mocks.store.video = null; + + expect((await admin({ video: PLAYBACK_ID, takenDown: true })).status).toBe( + 404, + ); + }); +}); + +describe("the staff script", () => { + const run = async (argv: string[], env: Record) => { + const lines: string[] = []; + const calls: { url: string; init: RequestInit }[] = []; + const code = await script.runTakedown({ + argv, + env, + fetchImpl: async (url: string, init: RequestInit) => { + calls.push({ url, init }); + return admin(JSON.parse(String(init.body))); + }, + log: (line: string) => lines.push(line), + fail: (line: string) => lines.push(`error: ${line}`), + }); + return { code, lines, calls }; + }; + + it("takes a video down and restores it through the staff route", async () => { + const down = await run(["down", PLAYBACK_ID], { + CAP_SESSION_TOKEN: "session", + CAP_URL: "https://web.example/", + }); + + expect(down.code).toBe(0); + expect(down.calls[0]?.url).toBe( + "https://web.example/api/admin/developer-video-takedown", + ); + expect(down.calls[0]?.init.headers).toEqual({ + "Content-Type": "application/json", + Cookie: "next-auth.session-token=session", + }); + expect(down.lines).toEqual([ + `video-1 (playback ${PLAYBACK_ID}, app app-1) is taken down.`, + ]); + + const restore = await run(["restore", PLAYBACK_ID], { + CAP_SESSION_TOKEN: "session", + }); + expect(restore.calls[0]?.url).toBe( + "https://cap.so/api/admin/developer-video-takedown", + ); + expect(restore.lines).toEqual([ + `video-1 (playback ${PLAYBACK_ID}, app app-1) is playing normally.`, + ]); + expect(mocks.store.video?.metadata).toBeNull(); + }); + + it("explains its usage and needs a session", async () => { + expect((await run(["remove", PLAYBACK_ID], {})).code).toBe(2); + expect((await run(["down"], {})).code).toBe(2); + const missing = await run(["down", PLAYBACK_ID], {}); + expect(missing.code).toBe(2); + expect(missing.calls).toEqual([]); + }); + + it("fails when the route refuses", async () => { + mocks.user = null; + + const refused = await run(["down", PLAYBACK_ID], { + CAP_SESSION_TOKEN: "session", + }); + + expect(refused.code).toBe(1); + expect(refused.lines).toEqual(["error: 403: Not allowed."]); + }); +}); + +describe("the takedown docs", () => { + it("promise 410 only for playback files, not for the page or embed", async () => { + const { readFileSync } = await import("node:fs"); + const { join } = await import("node:path"); + const { developerOpenApiSpec } = await import("@/lib/developer-openapi"); + const docs = readFileSync( + join(process.cwd(), "content/docs/api/rest-api.mdx"), + "utf8", + ); + const described = [ + ...docs.split("\n").filter((line) => line.includes("`takenDown`")), + String( + developerOpenApiSpec("https://cap.so").components.schemas.DeveloperVideo + .properties.takenDown.description, + ), + ]; + + expect(described.length).toBeGreaterThan(2); + for (const text of described) { + expect(text).not.toMatch(/page[^.]*410|embed[^.]*410/i); + } + }); +}); + +describe("the developer API", () => { + const withEnv = () => ready({ metadata: { orderId: 7 } }); + + it("reports a taken down video honestly and leaves every other field as it was", async () => { + mocks.store.video = withEnv(); + const before = api.serializeDeveloperVideo(mocks.store.video); + const beforeStatus = api.serializeDeveloperVideoStatus(mocks.store.video); + const agentRow = (row: NonNullable) => ({ + ...row, + takenDown: takedown.developerVideoIsTakenDown(row), + }); + const beforeAgent = api.toAgentDeveloperVideo(agentRow(mocks.store.video)); + + await store.setDeveloperVideoTakedown(PLAYBACK_ID, true, NOW); + const video = mocks.store.video; + if (!video) throw new Error("missing video"); + + const nulls = { hlsUrl: null, mp4Url: null, posterUrl: null }; + expect(api.serializeDeveloperVideo(video)).toEqual({ + ...before, + ...nulls, + takenDown: true, + }); + expect(api.serializeDeveloperVideoStatus(video)).toEqual({ + ...beforeStatus, + ...nulls, + takenDown: true, + }); + expect(api.toAgentDeveloperVideo(agentRow(video))).toEqual({ + ...beforeAgent, + ...nulls, + takenDown: true, + }); + expect(api.serializeDeveloperVideo(video)).toMatchObject({ + status: "ready", + metadata: { orderId: 7 }, + shareUrl: `https://web.example/dev/${PLAYBACK_ID}`, + }); + }); + + it("gives the agent API a boolean even when the database answers NULL", () => { + const row = { ...withEnv(), takenDown: null as unknown as boolean }; + + expect(api.toAgentDeveloperVideo(row).takenDown).toBe(false); + }); + + it("adds takenDown false to a regular video and changes nothing else", () => { + const video = withEnv(); + const serialized = api.serializeDeveloperVideo(video); + + expect(serialized.takenDown).toBe(false); + expect(serialized.metadata).toEqual({ orderId: 7 }); + expect(serialized.hlsUrl).toBe( + `https://web.example/api/developer/playback/${PLAYBACK_ID}/master.m3u8`, + ); + expect(api.serializeDeveloperVideoStatus(video)).toMatchObject({ + ready: true, + status: "ready", + takenDown: false, + }); + }); +}); + +describe("the owner's console", () => { + const row = ( + takenDown: boolean, + status: "ready" | "processing" = "ready", + ) => { + const video = ready({ status }); + return { ...video, legacy: false, takenDown }; + }; + + it("shows a taken down video as taken down, not as failed", () => { + expect(view.infraVideoState(row(true))).toBe("taken_down"); + expect(view.infraVideoState(row(true, "processing"))).toBe("taken_down"); + expect(view.INFRA_VIDEO_DISPLAY_LABELS.taken_down).toBe("Taken down"); + expect(view.infraVideoDisplayState("taken_down", null)).toBe("taken_down"); + expect( + view.infraVideoIsChanging( + { state: "taken_down", createdAt: NOW }, + NOW.getTime(), + ), + ).toBe(false); + }); + + it("offers no page, embed code or failure for a taken down video", () => { + const detail = view.toInfraVideoDetail( + row(true), + { hlsUrl: null, mp4Url: null, posterUrl: null }, + { + shareUrl: "https://web.example/dev/x", + embedUrl: "https://web.example/dev/x/embed", + }, + ); + + expect(detail).toMatchObject({ + state: "taken_down", + failure: null, + shareUrl: null, + embedCode: null, + stillEncoding: false, + }); + expect(view.infraVideoState(row(false))).toBe("ready"); + }); +}); diff --git a/apps/web/__tests__/unit/developer-video-upload.test.ts b/apps/web/__tests__/unit/developer-video-upload.test.ts index 251fbecce06..6cd92b5012e 100644 --- a/apps/web/__tests__/unit/developer-video-upload.test.ts +++ b/apps/web/__tests__/unit/developer-video-upload.test.ts @@ -96,37 +96,48 @@ vi.mock("@/workflows/process-developer-video", () => ({ vi.mock("@/lib/infra-storage", async () => { const schema = await import("@cap/database/schema"); - const { billableVideo, storageUsage } = await import( - "@/lib/developer-pricing" - ); + const { billableVideo, firstMonthStorageMicroCredits, storageUsage } = + await import("@/lib/developer-pricing"); return { - getAppStorageUsage: async ( + getAppStorage: async ( appId: string, newVideo?: Parameters[0][number]["video"], _executor?: unknown, - options: { includeProcessing?: boolean; excludeVideoId?: string } = {}, + options: { excludeVideoId?: string } = {}, ) => { - const entries = (context.fake?.rows(schema.developerVideos) ?? []) - .filter( - (row) => - row.appId === appId && - !row.deletedAt && - row.duration !== null && - row.id !== options.excludeVideoId, - ) - .map((row) => ({ - video: billableVideo({ - duration: row.duration as number, - width: row.width as number | null, - height: row.height as number | null, - status: row.status as string, - maxResolutionTier: row.maxResolutionTier as string | null, - includeProcessing: options.includeProcessing, - }), - count: 1, - })); - if (newVideo) entries.push({ video: newVideo, count: 1 }); - return storageUsage(entries); + const rows = (context.fake?.rows(schema.developerVideos) ?? []).filter( + (row) => + row.appId === appId && + !row.deletedAt && + row.duration !== null && + row.id !== options.excludeVideoId, + ); + const entriesIn = (status: string) => + rows + .filter((row) => row.status === status) + .map((row) => ({ + video: billableVideo({ + duration: row.duration as number, + width: row.width as number | null, + height: row.height as number | null, + maxResolutionTier: row.maxResolutionTier as string | null, + }), + count: 1, + })); + const ready = entriesIn("ready"); + const due = entriesIn("processing"); + if (newVideo) due.push({ video: newVideo, count: 1 }); + return { + stored: storageUsage(ready), + dailyCharge: storageUsage(ready), + firstMonthMicroCredits: due.reduce( + (sum, entry) => + sum + + (entry.video ? firstMonthStorageMicroCredits(entry.video) : 0), + 0, + ), + uploadingVideos: 0, + }; }, }; }); @@ -175,6 +186,7 @@ import { developerTranscripts, developerVideos, } from "@cap/database/schema"; +import { legacyDeveloperVideoIdForIdempotencyKey } from "@/lib/developer-ids"; import { completeDeveloperVideoUpload, createDeveloperVideoUpload, @@ -204,8 +216,8 @@ const storedCopies = () => [...context.objects.keys()].filter((key) => key.startsWith(STORED_PREFIX)); const START = Date.parse("2026-11-11T10:05:00.000Z"); const FOUR_K = { duration: 7200, width: 3840, height: 2160, fps: 30 }; -const CAPPED_4K_ADMISSION = 2_380; -const UNCAPPED_4K_ADMISSION = 7_840; +const CAPPED_4K_ADMISSION = 10_200; +const UNCAPPED_4K_ADMISSION = 33_600; const create = (overrides: Record = {}) => createDeveloperVideoUpload("app-1", { @@ -367,7 +379,7 @@ describe("developer video upload target", () => { ok: false, status: 402, code: "insufficient_credits", - message: "This app has no credit. Add credits to start uploading.", + message: "Not enough credit to upload. Add credits to start uploading.", shortfallMicroCredits: 1, }); expect(videos()).toHaveLength(0); @@ -388,7 +400,7 @@ describe("developer video upload target", () => { status: 402, code: "insufficient_credits", message: - "Uploading needs $0.02380 of credit, enough for 7 days of storage for everything this app stores. The balance is $0.01, so add at least $0.01380.", + "Not enough credit to upload. Uploading needs $0.02380, enough for 7 days of storage for the videos past their first month. The balance is $0.01, so add at least $0.01380.", shortfallMicroCredits: 1_380, }); }); @@ -427,6 +439,30 @@ describe("developer video upload target", () => { }); }); +describe("a video created under an Idempotency-Key before ids changed", () => { + it("is returned again for the same key, and a new key gets a regular id", async () => { + const key = "older-key-00000001"; + const legacy = legacyDeveloperVideoIdForIdempotencyKey("app-1", key); + fake().seed(developerVideos, { + id: legacy, + appId: "app-1", + name: "demo.mov", + s3Key: `developer/app-1/${legacy}/video`, + playbackId: "playback-legacy-0001", + duration: null, + deletedAt: null, + }); + + const repeated = await create({ idempotencyKey: key }); + const fresh = await create({ idempotencyKey: "newer-key-00000001" }); + if (!repeated.ok || !fresh.ok) throw new Error("create failed"); + + expect(repeated.video.id).toBe(legacy); + expect(fresh.video.id).toMatch(/^[0-9a-hjkmnp-tv-z]{15}$/); + expect(videos()).toHaveLength(2); + }); +}); + describe("completing a developer video upload", () => { it("records a private copy of the upload, starts processing once and charges nothing", async () => { const id = await createAndUpload(); @@ -704,7 +740,7 @@ describe("admission when the upload is recorded", () => { status: 402, code: "insufficient_credits", message: - "Uploading needs $0.02380 of credit, enough for 7 days of storage for everything this app stores. The balance is $0.01, so add at least $0.01380.", + "Not enough credit to upload. Uploading needs $0.10200, enough for the first month of storage for each video being uploaded or encoded. The balance is $0.01, so add at least $0.09200.", shortfallMicroCredits: CAPPED_4K_ADMISSION - 1_000, }); expect(videos()[0]).toMatchObject({ diff --git a/apps/web/__tests__/unit/developer-webhook-actions.test.ts b/apps/web/__tests__/unit/developer-webhook-actions.test.ts new file mode 100644 index 00000000000..d89f9cff51b --- /dev/null +++ b/apps/web/__tests__/unit/developer-webhook-actions.test.ts @@ -0,0 +1,407 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const context = vi.hoisted(() => ({ + fake: null as null | { + db: unknown; + rows: (table: object) => Record[]; + seed: (table: object, row: Record) => unknown; + reset: () => void; + }, + user: null as null | { id: string; email: string }, + limited: false, + budgetOwners: [] as string[], + resolved: ["93.184.216.34"] as string[], + request: null as + | null + | ((url: string, init: RequestInit) => Promise), +})); + +vi.mock("@cap/database", async () => { + const schema = await import("@cap/database/schema"); + const { createWebhookFakeDatabase, webhookEndpointDefaults } = await import( + "./developer-webhook-fake-db" + ); + const fake = createWebhookFakeDatabase( + new Map([[schema.developerWebhookEndpoints, webhookEndpointDefaults]]), + ); + context.fake = fake; + return { db: () => fake.db }; +}); +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: async () => context.user, +})); +vi.mock("@/lib/infra-access", () => ({ canUseInfra: () => true })); +vi.mock("@cap/env", () => ({ + NODE_ENV: "production", + buildEnv: { + NEXT_PUBLIC_IS_CAP: "true", + NEXT_PUBLIC_WEB_URL: "https://cap.so", + }, + serverEnv: () => ({ DATABASE_ENCRYPTION_KEY: "ab".repeat(32) }), +})); +vi.mock("next/cache", () => ({ revalidatePath: vi.fn() })); +vi.mock("next/headers", () => ({ headers: async () => new Headers() })); +vi.mock("@/app/api/utils", async () => { + const { createMiddleware } = await import("hono/factory"); + return { + developerOwnerRequestLimited: async (ownerId: string) => { + context.budgetOwners.push(ownerId); + return context.limited; + }, + withDeveloperSecretAuth: createMiddleware(async (c, next) => { + c.set("developerAppId", "app-a"); + await next(); + }), + }; +}); +vi.mock("@/lib/s3-endpoint-policy", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + isBlockedS3Endpoint: (url: string) => + actual.isBlockedS3Endpoint(url, async () => context.resolved), + }; +}); +vi.mock("@/lib/storage-fetch", () => ({ + requestPublicOnly: (url: string, init: RequestInit) => { + if (!context.request) throw new Error("no request expected"); + return context.request(url, init); + }, +})); + +const { developerApps, developerWebhookEndpoints } = await import( + "@cap/database/schema" +); +const { decrypt } = await import("@cap/database/crypto"); +const actions = await import("@/actions/developers/webhooks"); +const { verifyWebhookSignature } = await import( + "@/lib/developer-webhook-signature" +); +const { app: restApp } = await import( + "@/app/api/developer/v1/[...route]/webhooks" +); + +const OWNER_A = { id: "user-a", email: "a@example.com" }; +const OWNER_B = { id: "user-b", email: "b@example.com" }; +const URL_A = "https://hooks.example.com/cap"; + +const endpoints = () => context.fake?.rows(developerWebhookEndpoints) ?? []; + +const created = async (appId = "app-a", url = URL_A) => { + const result = await actions.createInfraWebhook(appId, { + url, + events: ["video.ready", "video.errored"], + }); + if (!result.success) throw new Error(result.message); + return result; +}; + +const restCreate = async (body: Record) => { + const response = await restApp.request("/", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + return { + status: response.status, + body: (await response.json()) as { + code: string; + message: string; + hint: string; + }, + }; +}; + +beforeEach(() => { + context.fake?.reset(); + context.fake?.seed(developerApps, { + id: "app-a", + ownerId: OWNER_A.id, + deletedAt: null, + }); + context.fake?.seed(developerApps, { + id: "app-b", + ownerId: OWNER_B.id, + deletedAt: null, + }); + context.fake?.seed(developerApps, { + id: "app-gone", + ownerId: OWNER_A.id, + deletedAt: new Date("2026-11-01T00:00:00Z"), + }); + context.user = OWNER_A; + context.limited = false; + context.budgetOwners = []; + context.resolved = ["93.184.216.34"]; + context.request = null; +}); + +describe("console webhook actions: scoping", () => { + it("answers 404 for another account's app and writes nothing", async () => { + context.user = OWNER_B; + const result = await actions.createInfraWebhook("app-a", { + url: URL_A, + events: ["video.ready"], + }); + expect(result).toMatchObject({ + success: false, + status: 404, + code: "app_not_found", + }); + expect(endpoints()).toHaveLength(0); + }); + + it("answers 404 for every action on another account's endpoint", async () => { + const { endpoint } = await created(); + const before = structuredClone(endpoints()); + context.user = OWNER_B; + + for (const appId of ["app-a", "app-b"]) { + const results = [ + await actions.setInfraWebhookEnabled(appId, endpoint.id, false), + await actions.rotateInfraWebhookSecret(appId, endpoint.id), + await actions.sendInfraWebhookTest(appId, endpoint.id), + await actions.deleteInfraWebhook(appId, endpoint.id), + ]; + for (const result of results) { + expect(result).toMatchObject({ + success: false, + status: 404, + code: appId === "app-a" ? "app_not_found" : "webhook_not_found", + }); + } + } + expect(endpoints()).toEqual(before); + }); + + it("answers 404 for a deleted app", async () => { + const result = await actions.createInfraWebhook("app-gone", { + url: URL_A, + events: ["video.ready"], + }); + expect(result).toMatchObject({ status: 404, code: "app_not_found" }); + expect(endpoints()).toHaveLength(0); + }); + + it("refuses a signed-out visitor before reading anything", async () => { + context.user = null; + await expect( + actions.createInfraWebhook("app-a", { url: URL_A, events: [] }), + ).rejects.toThrow("Unauthorized"); + }); +}); + +describe("console webhook actions: the per-app cap", () => { + it("allows five endpoints and refuses the sixth like the API", async () => { + for (let index = 0; index < 5; index += 1) { + await created("app-a", `https://hooks.example.com/cap/${index}`); + } + const sixth = await actions.createInfraWebhook("app-a", { + url: "https://hooks.example.com/cap/6", + events: ["video.ready"], + }); + expect(sixth).toMatchObject({ + success: false, + status: 409, + code: "webhook_limit_reached", + message: "An app can have 5 webhook endpoints", + }); + expect(endpoints()).toHaveLength(5); + + const rest = await restCreate({ url: "https://hooks.example.com/cap/7" }); + expect(rest.status).toBe(409); + expect(rest.body.message).toBe((sixth as { message: string }).message); + }); + + it("counts each app on its own and frees a slot on delete", async () => { + const ids: string[] = []; + for (let index = 0; index < 5; index += 1) { + ids.push( + (await created("app-a", `https://hooks.example.com/a/${index}`)) + .endpoint.id, + ); + } + context.user = OWNER_B; + await created("app-b", "https://hooks.example.com/b"); + context.user = OWNER_A; + + const removed = await actions.deleteInfraWebhook("app-a", ids[0] ?? ""); + expect(removed).toEqual({ success: true }); + await created("app-a", "https://hooks.example.com/a/again"); + }); +}); + +describe("console webhook actions: validation", () => { + const cases: [string, { url: string; events: string[] }][] = [ + [ + "plain http", + { url: "http://hooks.example.com/cap", events: ["video.ready"] }, + ], + [ + "a port", + { url: "https://hooks.example.com:8443/cap", events: ["video.ready"] }, + ], + [ + "credentials", + { url: "https://u:p@hooks.example.com/cap", events: ["video.ready"] }, + ], + ["not a URL", { url: "hooks example", events: ["video.ready"] }], + ["no events", { url: URL_A, events: [] }], + ["an unknown event", { url: URL_A, events: ["video.deleted"] }], + [ + "a repeated event", + { url: URL_A, events: ["video.ready", "video.ready"] }, + ], + ]; + + it.each(cases)("refuses %s with the API's message", async (_name, input) => { + const result = await actions.createInfraWebhook("app-a", input); + expect(result).toMatchObject({ success: false, status: 400 }); + if (result.success) throw new Error("expected a refusal"); + + const rest = await restCreate(input); + expect(rest.status).toBe(400); + if (rest.body.code === "invalid_webhook_url") { + expect(result).toMatchObject({ + code: "invalid_webhook_url", + message: rest.body.message, + hint: rest.body.hint, + }); + } else { + expect(rest.body.message).toMatch( + new RegExp(`: ${escape(result.message)}$`), + ); + } + expect(endpoints()).toHaveLength(0); + }); + + it("refuses a host that resolves to a private address", async () => { + context.resolved = ["10.0.0.5"]; + const result = await actions.createInfraWebhook("app-a", { + url: URL_A, + events: ["video.ready"], + }); + expect(result).toMatchObject({ + status: 400, + code: "invalid_webhook_url", + message: "The URL must point to a public address", + }); + }); + + it("refuses a non-boolean enabled value", async () => { + const { endpoint } = await created(); + const result = await actions.setInfraWebhookEnabled( + "app-a", + endpoint.id, + "yes" as unknown as boolean, + ); + expect(result).toMatchObject({ status: 400, code: "invalid_request" }); + expect(endpoints()[0]?.enabled).toBe(true); + }); +}); + +describe("console webhook actions: secrets and the rest", () => { + it("returns the signing secret once and never in an endpoint", async () => { + const result = await created(); + expect(result.secret).toMatch(/^cws_[0-9a-z]{30}$/); + expect(Object.keys(result.endpoint)).not.toContain("encryptedSecret"); + expect(JSON.stringify(result.endpoint)).not.toContain(result.secret); + const [row] = endpoints(); + expect(String(row?.encryptedSecret)).not.toContain(result.secret); + expect(await decrypt(String(row?.encryptedSecret))).toBe(result.secret); + + const rotated = await actions.rotateInfraWebhookSecret( + "app-a", + result.endpoint.id, + ); + if (!rotated.success) throw new Error(rotated.message); + expect(rotated.secret).not.toBe(result.secret); + expect(JSON.stringify(rotated.endpoint)).not.toContain(rotated.secret); + expect(await decrypt(String(endpoints()[0]?.encryptedSecret))).toBe( + rotated.secret, + ); + }); + + it("turns an endpoint off and on, clearing a failure stop", async () => { + const { endpoint } = await created(); + const row = endpoints()[0]; + if (!row) throw new Error("no row"); + Object.assign(row, { + enabled: false, + disabledReason: "delivery_failures", + consecutiveFailures: 20, + }); + + const on = await actions.setInfraWebhookEnabled("app-a", endpoint.id, true); + expect(on).toMatchObject({ + success: true, + endpoint: { enabled: true, disabledReason: null, consecutiveFailures: 0 }, + }); + const off = await actions.setInfraWebhookEnabled( + "app-a", + endpoint.id, + false, + ); + expect(off).toMatchObject({ success: true, endpoint: { enabled: false } }); + }); + + it("sends a signed test event and records it", async () => { + const { endpoint, secret } = await created(); + const seen: { url: string; signature: string | null; body: string }[] = []; + context.request = async (url, init) => { + seen.push({ + url, + signature: new Headers(init.headers).get("Cap-Signature"), + body: String(init.body), + }); + return new Response(null, { status: 204 }); + }; + + const result = await actions.sendInfraWebhookTest("app-a", endpoint.id); + expect(result).toMatchObject({ + success: true, + test: { type: "webhook.test", delivered: true, responseStatus: 204 }, + }); + expect(seen).toHaveLength(1); + expect(seen[0]?.url).toBe(URL_A); + expect( + verifyWebhookSignature({ + secret, + body: seen[0]?.body ?? "", + header: seen[0]?.signature ?? null, + }), + ).toBe(true); + expect(seen[0]?.body).not.toContain(secret); + expect(endpoints()[0]).toMatchObject({ + lastDeliveryStatus: "succeeded", + lastResponseStatus: 204, + }); + }); + + it("deletes an endpoint so it is gone from every other action", async () => { + const { endpoint } = await created(); + expect(await actions.deleteInfraWebhook("app-a", endpoint.id)).toEqual({ + success: true, + }); + expect(endpoints()[0]?.deletedAt).toBeInstanceOf(Date); + expect( + await actions.rotateInfraWebhookSecret("app-a", endpoint.id), + ).toMatchObject({ status: 404, code: "webhook_not_found" }); + }); + + it("stops at the shared rate limit before writing", async () => { + context.limited = true; + const result = await actions.createInfraWebhook("app-a", { + url: URL_A, + events: ["video.ready"], + }); + expect(result).toMatchObject({ status: 429, code: "rate_limited" }); + expect(endpoints()).toHaveLength(0); + expect(context.budgetOwners).toEqual([OWNER_A.id]); + }); +}); + +function escape(value: string) { + return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} diff --git a/apps/web/__tests__/unit/developer-webhook-delivery.test.ts b/apps/web/__tests__/unit/developer-webhook-delivery.test.ts new file mode 100644 index 00000000000..a5d00f767da --- /dev/null +++ b/apps/web/__tests__/unit/developer-webhook-delivery.test.ts @@ -0,0 +1,595 @@ +import { createServer, type IncomingMessage, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +const context = vi.hoisted(() => ({ + fake: null as null | { + db: unknown; + rows: (table: object) => Record[]; + seed: ( + table: object, + row: Record, + ) => Record; + reset: () => void; + }, + appDeleted: false, +})); + +vi.mock("@cap/database", async () => { + const schema = await import("@cap/database/schema"); + const { createWebhookFakeDatabase, webhookEndpointDefaults } = await import( + "./developer-webhook-fake-db" + ); + const fake = createWebhookFakeDatabase( + new Map([[schema.developerWebhookEndpoints, webhookEndpointDefaults]]), + { joined: () => !context.appDeleted }, + ); + context.fake = fake; + return { db: () => fake.db }; +}); +vi.mock("@cap/database/crypto", () => ({ + encrypt: async (text: string) => `sealed:${text}`, + decrypt: async (text: string) => text.replace(/^sealed:/, ""), +})); +vi.mock("@cap/env", () => ({ + NODE_ENV: "test", + buildEnv: {}, + serverEnv: () => ({}), +})); +// The production transport refuses loopback, so the local server is reached +// through a name this lookup pins to it. +vi.mock("@/lib/s3-endpoint-policy", async (importOriginal) => { + const actual = + await importOriginal(); + const publicOnlyLookup: typeof actual.publicOnlyLookup = ( + _hostname, + options, + callback, + ) => { + if (typeof options === "object" && options.all) { + callback(null, [{ address: "127.0.0.1", family: 4 }]); + return; + } + callback(null, "127.0.0.1", 4); + }; + return { ...actual, publicOnlyLookup }; +}); + +const { developerWebhookEndpoints } = await import("@cap/database/schema"); +const { + deliverDeveloperWebhookAttempt, + recordWebhookDelivery, + sendDeveloperWebhook, +} = await import("@/lib/developer-webhook-delivery"); +const { + WEBHOOK_DISABLE_AFTER_FAILURES, + WEBHOOK_DISABLE_AFTER_MS, + WEBHOOK_DISABLED_BY_FAILURES, +} = await import("@/lib/developer-webhook-events"); +const { verifyWebhookSignature } = await import( + "@/lib/developer-webhook-signature" +); +const { requestPublicOnly } = await import("@/lib/storage-fetch"); + +const SECRET = "cws_0123456789abcdefghjkmnpqrstvwx"; + +function fake() { + if (!context.fake) throw new Error("no database"); + return context.fake; +} + +function seedEndpoint(overrides: Record = {}) { + return fake().seed(developerWebhookEndpoints, { + id: "wh-1", + appId: "app-1", + url: "https://hooks.example.com/cap", + secretPrefix: SECRET.slice(0, 12), + encryptedSecret: `sealed:${SECRET}`, + ...overrides, + }); +} + +const endpoint = () => { + const [row] = fake().rows(developerWebhookEndpoints); + if (!row) throw new Error("no endpoint"); + return row; +}; + +const job = { + endpointId: "wh-1", + eventId: "evt_1", + eventType: "video.ready", + body: JSON.stringify({ id: "evt_1", type: "video.ready", data: {} }), +}; + +const answers = (status: number) => ({ + isBlocked: async () => false, + request: vi.fn(async () => new Response("ignored", { status })), +}); + +beforeEach(() => { + fake().reset(); + context.appDeleted = false; +}); + +describe("a customer endpoint over real HTTP", () => { + let server: Server; + let origin: string; + const received: { + headers: IncomingMessage["headers"]; + body: string; + method?: string; + }[] = []; + const held: IncomingMessage[] = []; + const closed: string[] = []; + + beforeAll(async () => { + server = createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => { + received.push({ + headers: req.headers, + method: req.method, + body: Buffer.concat(chunks).toString("utf8"), + }); + if (req.url === "/hang") { + held.push(req); + return; + } + if (req.url === "/big") { + res.writeHead(200); + res.write("x".repeat(64 * 1024)); + held.push(req); + return; + } + if (req.url === "/reset-content-flood") { + res.writeHead(205, { "transfer-encoding": "chunked" }); + const flood = setInterval(() => res.write("x".repeat(1024)), 5); + req.socket.once("close", () => { + clearInterval(flood); + closed.push(req.url ?? ""); + }); + return; + } + if (req.url === "/reset-content-stall") { + res.writeHead(205, { "transfer-encoding": "chunked" }); + res.flushHeaders(); + req.socket.once("close", () => closed.push(req.url ?? "")); + return; + } + if (req.url === "/moved") { + res.writeHead(307, { location: "/ok" }); + res.end(); + return; + } + res.writeHead(req.url === "/down" ? 503 : 200); + res.end("thanks"); + }); + }); + await new Promise((resolve) => + server.listen(0, "127.0.0.1", resolve), + ); + origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + }); + + afterAll(async () => { + for (const req of held) req.socket.destroy(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(resolve)); + }); + + beforeEach(() => { + received.length = 0; + }); + + const send = (path: string, timeoutMs?: number) => + sendDeveloperWebhook( + { + url: `${origin}${path}`, + secret: SECRET, + eventId: job.eventId, + eventType: job.eventType, + body: job.body, + }, + { allowPrivate: true, timeoutMs }, + ); + + it("posts the exact body with a signature the receiver can verify", async () => { + expect(await send("/ok")).toMatchObject({ ok: true, responseStatus: 200 }); + const [request] = received; + expect(request?.method).toBe("POST"); + expect(request?.body).toBe(job.body); + expect(request?.headers["content-type"]).toBe("application/json"); + expect(request?.headers["cap-event-id"]).toBe("evt_1"); + expect(request?.headers["cap-event-type"]).toBe("video.ready"); + expect( + verifyWebhookSignature({ + secret: SECRET, + body: request?.body ?? "", + header: String(request?.headers["cap-signature"]), + }), + ).toBe(true); + }); + + it("reports a server error as a failed delivery", async () => { + expect(await send("/down")).toMatchObject({ + ok: false, + errorCode: "http_error", + responseStatus: 503, + }); + }); + + it("does not follow a redirect", async () => { + expect(await send("/moved")).toMatchObject({ + ok: false, + errorCode: "redirect", + responseStatus: 307, + }); + expect(received).toHaveLength(1); + }); + + it("gives up on an endpoint that never answers", async () => { + const started = Date.now(); + expect(await send("/hang", 150)).toMatchObject({ + ok: false, + errorCode: "timeout", + responseStatus: null, + }); + expect(Date.now() - started).toBeLessThan(2_000); + }); + + it("does not wait for or read a response body that never ends", async () => { + const started = Date.now(); + expect(await send("/big", 5_000)).toMatchObject({ + ok: true, + responseStatus: 200, + }); + expect(Date.now() - started).toBeLessThan(2_000); + }); + + describe("through the transport a production build uses", () => { + const sendPinned = (path: string, timeoutMs?: number) => + sendDeveloperWebhook( + { + url: `${origin.replace("127.0.0.1", "hooks.test")}${path}`, + secret: SECRET, + eventId: job.eventId, + eventType: job.eventType, + body: job.body, + }, + { allowPrivate: true, request: requestPublicOnly, timeoutMs }, + ); + + it("posts the exact body with its length and a verifiable signature", async () => { + expect(await sendPinned("/ok")).toMatchObject({ + ok: true, + responseStatus: 200, + }); + const [request] = received; + expect(request?.method).toBe("POST"); + expect(request?.body).toBe(job.body); + expect(request?.headers["content-length"]).toBe( + String(Buffer.byteLength(job.body)), + ); + expect(request?.headers.host).toContain("hooks.test"); + expect( + verifyWebhookSignature({ + secret: SECRET, + body: request?.body ?? "", + header: String(request?.headers["cap-signature"]), + }), + ).toBe(true); + }); + + it("reports a server error and does not follow a redirect", async () => { + expect(await sendPinned("/down")).toMatchObject({ + ok: false, + errorCode: "http_error", + responseStatus: 503, + }); + expect(await sendPinned("/moved")).toMatchObject({ + ok: false, + errorCode: "redirect", + responseStatus: 307, + }); + expect(received).toHaveLength(2); + }); + + it("gives up on an endpoint that never answers", async () => { + const started = Date.now(); + expect(await sendPinned("/hang", 150)).toMatchObject({ + ok: false, + errorCode: "timeout", + }); + expect(Date.now() - started).toBeLessThan(2_000); + }); + + it.each(["/reset-content-flood", "/reset-content-stall"])( + "closes the connection after a no-content answer from %s", + async (path) => { + expect(await sendPinned(path, 5_000)).toMatchObject({ + ok: true, + responseStatus: 205, + }); + await vi.waitFor(() => expect(closed).toContain(path), { + timeout: 2_000, + }); + }, + ); + + it("does not wait for or read a response body that never ends", async () => { + const started = Date.now(); + expect(await sendPinned("/big", 5_000)).toMatchObject({ + ok: true, + responseStatus: 200, + }); + expect(Date.now() - started).toBeLessThan(2_000); + }); + }); + + it("reports a closed port as a failed connection", async () => { + const result = await sendDeveloperWebhook( + { ...job, url: "http://127.0.0.1:1/cap", secret: SECRET }, + { allowPrivate: true }, + ); + expect(result).toMatchObject({ + ok: false, + errorCode: "connection_failed", + }); + }); +}); + +describe("the delivery deadline and the signature time", () => { + const event = { + url: "https://hooks.example.com/cap", + secret: SECRET, + eventId: job.eventId, + eventType: job.eventType, + body: job.body, + }; + + it("counts a host lookup that never finishes against the deadline", async () => { + const request = vi.fn(); + const started = Date.now(); + const result = await sendDeveloperWebhook(event, { + allowPrivate: false, + isBlocked: () => new Promise(() => {}), + request, + timeoutMs: 100, + }); + expect(result).toMatchObject({ ok: false, errorCode: "timeout" }); + expect(Date.now() - started).toBeLessThan(1_500); + expect(request).not.toHaveBeenCalled(); + }); + + it("signs with the time the request is sent, not the time the lookup began", async () => { + let clock = 1_794_500_000_000; + const seen: RequestInit[] = []; + await sendDeveloperWebhook(event, { + allowPrivate: false, + now: () => clock, + isBlocked: async () => { + clock += 7_000; + return false; + }, + request: async (_url, init) => { + seen.push(init); + return new Response(null, { status: 204 }); + }, + }); + const header = new Headers(seen[0]?.headers).get("cap-signature"); + expect(header).toMatch(/^t=1794500007,/); + expect( + verifyWebhookSignature({ + secret: SECRET, + body: job.body, + header, + now: clock, + }), + ).toBe(true); + }); +}); + +describe("one delivery attempt", () => { + it("delivers, records the result and clears the failure streak", async () => { + seedEndpoint({ + consecutiveFailures: 4, + failingSince: new Date("2026-11-10T00:00:00Z"), + }); + const guards = answers(200); + + expect(await deliverDeveloperWebhookAttempt(job, guards)).toBe("delivered"); + + expect(guards.request).toHaveBeenCalledTimes(1); + expect(endpoint()).toMatchObject({ + lastDeliveryStatus: "succeeded", + lastResponseStatus: 200, + lastErrorCode: null, + consecutiveFailures: 0, + failingSince: null, + enabled: true, + }); + expect(endpoint().lastDeliveryAt).toBeInstanceOf(Date); + }); + + it("asks for a retry and counts the failure", async () => { + seedEndpoint(); + expect(await deliverDeveloperWebhookAttempt(job, answers(500))).toBe( + "retry", + ); + expect(endpoint()).toMatchObject({ + lastDeliveryStatus: "failed", + lastResponseStatus: 500, + lastErrorCode: "http_error", + consecutiveFailures: 1, + enabled: true, + }); + expect(endpoint().failingSince).toBeInstanceOf(Date); + }); + + it("keeps the start of a failure streak across later failures", async () => { + const failingSince = new Date("2026-11-10T00:00:00Z"); + seedEndpoint({ consecutiveFailures: 2, failingSince }); + await deliverDeveloperWebhookAttempt(job, answers(500)); + expect(endpoint()).toMatchObject({ consecutiveFailures: 3, failingSince }); + }); + + it("leaves the endpoint's updatedAt alone", async () => { + const seeded = seedEndpoint(); + const before = seeded.updatedAt; + await deliverDeveloperWebhookAttempt(job, answers(500)); + await deliverDeveloperWebhookAttempt(job, answers(200)); + expect(endpoint().updatedAt).toBe(before); + }); + + it.each([ + ["was deleted", { deletedAt: new Date() }], + ["was disabled", { enabled: false }], + ["no longer wants the event", { events: ["video.errored"] }], + ])("drops the delivery when the endpoint %s", async (_label, overrides) => { + seedEndpoint(overrides); + const guards = answers(200); + expect(await deliverDeveloperWebhookAttempt(job, guards)).toBe("dropped"); + expect(guards.request).not.toHaveBeenCalled(); + expect(endpoint().lastDeliveryAt).toBeNull(); + }); + + it("drops the delivery when the endpoint is gone or its app was deleted", async () => { + const guards = answers(200); + expect(await deliverDeveloperWebhookAttempt(job, guards)).toBe("dropped"); + seedEndpoint(); + context.appDeleted = true; + expect(await deliverDeveloperWebhookAttempt(job, guards)).toBe("dropped"); + expect(guards.request).not.toHaveBeenCalled(); + }); +}); + +describe("disabling an endpoint that keeps failing", () => { + const failed = { + ok: false as const, + responseStatus: 500, + errorCode: "http_error" as const, + durationMs: 5, + }; + const now = new Date("2026-11-20T12:00:00Z"); + const longAgo = new Date(now.getTime() - WEBHOOK_DISABLE_AFTER_MS - 1_000); + + it("disables after many failures over several days", async () => { + seedEndpoint({ + consecutiveFailures: WEBHOOK_DISABLE_AFTER_FAILURES - 1, + failingSince: longAgo, + }); + await recordWebhookDelivery("wh-1", failed, { + countsTowardStreak: true, + now, + }); + expect(endpoint()).toMatchObject({ + enabled: false, + disabledReason: WEBHOOK_DISABLED_BY_FAILURES, + consecutiveFailures: WEBHOOK_DISABLE_AFTER_FAILURES, + }); + }); + + it("keeps an endpoint with many failures in a short time", async () => { + seedEndpoint({ + consecutiveFailures: 500, + failingSince: new Date(now.getTime() - 60 * 60 * 1000), + }); + await recordWebhookDelivery("wh-1", failed, { + countsTowardStreak: true, + now, + }); + expect(endpoint()).toMatchObject({ enabled: true, disabledReason: null }); + }); + + it("keeps an endpoint with a few failures over a long time", async () => { + seedEndpoint({ consecutiveFailures: 3, failingSince: longAgo }); + await recordWebhookDelivery("wh-1", failed, { + countsTowardStreak: true, + now, + }); + expect(endpoint()).toMatchObject({ enabled: true, consecutiveFailures: 4 }); + }); + + it("keeps the count and the streak start together when a success and a failure land at once", async () => { + const succeeded = { ok: true as const, responseStatus: 200, durationMs: 5 }; + for (let round = 0; round < 20; round++) { + fake().reset(); + seedEndpoint({ consecutiveFailures: 3, failingSince: longAgo }); + const pair = [ + recordWebhookDelivery("wh-1", failed, { + countsTowardStreak: true, + now, + }), + recordWebhookDelivery("wh-1", succeeded, { + countsTowardStreak: true, + now, + }), + ]; + await Promise.all(round % 2 === 0 ? pair : pair.reverse()); + const row = endpoint(); + expect(row.consecutiveFailures === 0).toBe(row.failingSince === null); + if (row.failingSince !== null) { + expect([now, longAgo]).toContain(row.failingSince); + expect(row.consecutiveFailures).toBeGreaterThan(0); + } + expect(row.enabled).toBe(true); + } + }); + + it("starts a new streak from the failure that began it", async () => { + seedEndpoint(); + const first = new Date(now.getTime() - WEBHOOK_DISABLE_AFTER_MS - 60_000); + await recordWebhookDelivery("wh-1", failed, { + countsTowardStreak: true, + now: first, + }); + await recordWebhookDelivery( + "wh-1", + { ok: true, responseStatus: 200, durationMs: 5 }, + { countsTowardStreak: true, now: first }, + ); + for ( + let attempt = 0; + attempt < WEBHOOK_DISABLE_AFTER_FAILURES + 5; + attempt++ + ) { + await recordWebhookDelivery("wh-1", failed, { + countsTowardStreak: true, + now, + }); + } + expect(endpoint()).toMatchObject({ + enabled: true, + disabledReason: null, + failingSince: now, + consecutiveFailures: WEBHOOK_DISABLE_AFTER_FAILURES + 5, + }); + }); + + it("never counts a test event towards the streak", async () => { + seedEndpoint({ + consecutiveFailures: WEBHOOK_DISABLE_AFTER_FAILURES + 5, + failingSince: longAgo, + }); + await recordWebhookDelivery("wh-1", failed, { + countsTowardStreak: false, + now, + }); + expect(endpoint()).toMatchObject({ + enabled: true, + consecutiveFailures: WEBHOOK_DISABLE_AFTER_FAILURES + 5, + lastDeliveryStatus: "failed", + lastErrorCode: "http_error", + }); + }); +}); diff --git a/apps/web/__tests__/unit/developer-webhook-dispatch.test.ts b/apps/web/__tests__/unit/developer-webhook-dispatch.test.ts new file mode 100644 index 00000000000..4146dba39db --- /dev/null +++ b/apps/web/__tests__/unit/developer-webhook-dispatch.test.ts @@ -0,0 +1,809 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + type DeveloperVideoStore, + developerVideo, + processingState, +} from "../fixtures/developer-video"; + +type Endpoint = { + id: string; + appId: string; + events: string[]; + enabled: boolean; + deletedAt: Date | null; +}; + +const mocks = vi.hoisted(() => ({ + store: { video: null, appDeleted: false, updates: [] } as DeveloperVideoStore, + endpoints: [] as Endpoint[], + endpointQueries: 0, + failEndpointQuery: false, + hangEndpointQuery: false, + failTransactions: 0, + start: vi.fn(), + attempt: vi.fn(), + sleep: vi.fn(), + head: vi.fn(), + readText: vi.fn(), +})); + +vi.mock("@cap/database", async () => { + const { createDeveloperVideoDatabase } = await import( + "../fixtures/developer-video" + ); + const schema = await import("@cap/database/schema"); + const videos = await createDeveloperVideoDatabase(mocks.store); + const endpointRows = async () => { + mocks.endpointQueries += 1; + if (mocks.hangEndpointQuery) await new Promise(() => {}); + if (mocks.failEndpointQuery) throw new Error("database is away"); + const appId = mocks.store.video?.appId; + return mocks.endpoints.filter( + (endpoint) => + endpoint.appId === appId && endpoint.enabled && !endpoint.deletedAt, + ); + }; + // The real row lock makes two writers to one video take turns. The video + // fake has no lock, so transactions are queued here to stand in for it. + let turn: Promise = Promise.resolve(); + const database = { + ...videos, + select: (fields?: Record) => ({ + from: (table: object) => + table === schema.developerWebhookEndpoints + ? { where: endpointRows } + : videos.select(fields).from(), + }), + transaction: (run: (tx: unknown) => Promise) => { + const next = turn.then(() => { + if (mocks.failTransactions > 0) { + mocks.failTransactions -= 1; + throw new Error("connection lost"); + } + return run(database); + }); + turn = next.catch(() => undefined); + return next; + }, + }; + return { db: () => database }; +}); +vi.mock("@cap/env", () => ({ + NODE_ENV: "test", + serverEnv: () => ({ + WEB_URL: "https://web.example", + INFRA_PLAYBACK_URL: "https://play.example", + MEDIA_SERVER_WEBHOOK_URL: undefined, + }), +})); +vi.mock("workflow/api", () => ({ start: mocks.start })); +vi.mock("@/workflows/deliver-developer-webhook", () => ({ + deliverDeveloperWebhookWorkflow: "deliver-developer-webhook", +})); +vi.mock("workflow", () => ({ sleep: mocks.sleep })); +vi.mock("@/lib/developer-webhook-delivery", () => ({ + deliverDeveloperWebhookAttempt: mocks.attempt, +})); +vi.mock("@/lib/developer-video-storage", () => ({ + headDeveloperVideoObject: mocks.head, + readDeveloperVideoText: mocks.readText, + deleteDeveloperVideoObjects: vi.fn(), + developerVideoPrefix: (video: { appId: string; id: string }) => + `developer/${video.appId}/${video.id}/`, + internalDeveloperVideoUrl: async (key: string) => `https://storage/${key}`, + developerVideoUploadUrl: async (key: string) => `https://storage/put/${key}`, +})); +vi.mock("@/lib/developer-video-charge", () => ({ + DEVELOPER_VIDEO_MAX_DURATION_SECONDS: 4 * 60 * 60, + chargeDeveloperVideoUpload: vi.fn(async () => "charged"), +})); +vi.mock("@/lib/media-client", () => ({ + MEDIA_SERVER_SHUTDOWN_MESSAGE: "Server shutting down", + tryProbeVideoViaMediaServer: vi.fn(), + startMediaServerVideoJob: vi.fn(), + readMediaServerVideoJob: vi.fn(), +})); +vi.mock("@/lib/media-container", () => ({ sniffMediaUrl: vi.fn() })); + +const { + applyDeveloperVideoProgress, + claimDeveloperVideoProcessing, + failDeveloperVideoProcessing, + readDeveloperVideoParts, +} = await import("@/lib/developer-video-processing"); +const { developerVideoEventStep, READY_EVENT_MAX_WAIT_MS } = await import( + "@/lib/developer-webhook-events" +); +const { deliverDeveloperWebhookWorkflow } = await vi.importActual< + typeof import("@/workflows/deliver-developer-webhook") +>("@/workflows/deliver-developer-webhook"); +const { enqueueDeveloperVideoEvent } = await import( + "@/lib/developer-webhook-dispatch" +); +const { developerVideoEventId } = await import( + "@/lib/developer-webhook-signature" +); +const { serializeDeveloperVideo } = await import("@/lib/developer-video-api"); + +const running = (token: string) => ({ + state: "running" as const, + token, + jobId: `job-${token}`, + startedAt: 2_000, +}); + +const ladder = { + encoder: "libx264", + renditions: [ + { + name: "1080p", + width: 1920, + height: 1080, + bandwidth: 4_000_000, + averageBandwidth: 3_000_000, + codecs: "avc1.640028,mp4a.40.2", + bytes: 5_000, + }, + ], + audio: { bytes: 300 }, +}; +const master = [ + "#EXTM3U", + "#EXT-X-STREAM-INF:BANDWIDTH=4000000,RESOLUTION=1920x1080", + "1080p.m3u8", +].join("\n"); + +const mp4Done = () => + applyDeveloperVideoProgress( + { videoId: "video-1", phase: "complete" }, + "mp4", + "m1", + ); +const hlsDone = () => + applyDeveloperVideoProgress( + { videoId: "video-1", phase: "complete", ladder }, + "hls", + "h1", + ); + +const settled = () => new Promise((resolve) => setTimeout(resolve, 0)); + +function readyData() { + const job = jobs().find((queued) => queued.eventType === "video.ready"); + return JSON.parse(job?.body ?? "{}").data; +} + +function video() { + if (!mocks.store.video) throw new Error("no video"); + return mocks.store.video; +} + +const jobs = () => + mocks.start.mock.calls.map( + ([, [job]]) => + job as { + endpointId: string; + eventId: string; + eventType: string; + body: string; + }, + ); + +afterEach(() => { + vi.useRealTimers(); +}); + +beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.clearAllMocks(); + mocks.store.video = developerVideo({ + processing: processingState({ mp4: running("m1"), hls: running("h1") }), + }); + mocks.store.updates = []; + mocks.endpoints = [ + { + id: "wh-1", + appId: "app-1", + events: ["video.ready", "video.errored"], + enabled: true, + deletedAt: null, + }, + ]; + mocks.endpointQueries = 0; + mocks.failEndpointQuery = false; + mocks.hangEndpointQuery = false; + mocks.failTransactions = 0; + mocks.start.mockResolvedValue({ runId: "run-1" }); + mocks.sleep.mockResolvedValue(undefined); + mocks.head.mockResolvedValue({ bytes: 2_000, contentType: "video/mp4" }); + mocks.readText.mockResolvedValue(master); + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(console, "warn").mockImplementation(() => undefined); +}); + +describe("video.ready", () => { + it("queues one delivery carrying the video as GET /videos/:id returns it", async () => { + await mp4Done(); + await hlsDone(); + await settled(); + + expect(video().status).toBe("ready"); + expect(mocks.start).toHaveBeenCalledTimes(1); + const [job] = jobs(); + expect(job).toMatchObject({ + endpointId: "wh-1", + eventType: "video.ready", + eventId: developerVideoEventId("video-1", "video.ready"), + }); + const event = JSON.parse(job?.body ?? "{}"); + expect(event).toMatchObject({ + id: job?.eventId, + type: "video.ready", + }); + expect(Number.isNaN(Date.parse(event.createdAt))).toBe(false); + expect(event.data).toEqual( + JSON.parse(JSON.stringify(serializeDeveloperVideo(video()))), + ); + expect(event.data).toMatchObject({ id: "video-1", status: "ready" }); + expect(event.data.mp4Url).toContain("https://play.example/"); + expect(event.data.hlsUrl).toContain("https://play.example/"); + expect(job?.body).not.toContain("s3Key"); + expect(job?.body).not.toContain("developer/app-1"); + }); + + it("waits for HLS when the MP4 makes the video ready first", async () => { + await mp4Done(); + await settled(); + expect(video().status).toBe("ready"); + expect(video().processing?.readyEventDueAt).toEqual(expect.any(Number)); + expect(mocks.start).not.toHaveBeenCalled(); + + await hlsDone(); + await settled(); + expect(mocks.start).toHaveBeenCalledTimes(1); + expect(readyData()).toMatchObject({ + hlsUrl: expect.stringContaining("/master.m3u8"), + mp4Url: expect.stringContaining("https://play.example/"), + }); + expect(video().processing?.readyEventDueAt).toBeUndefined(); + }); + + it("waits for the MP4 when HLS makes the video ready first", async () => { + await hlsDone(); + await settled(); + expect(video().status).toBe("ready"); + expect(mocks.start).not.toHaveBeenCalled(); + + await mp4Done(); + await settled(); + expect(mocks.start).toHaveBeenCalledTimes(1); + expect(readyData()).toMatchObject({ + hlsUrl: expect.stringContaining("/master.m3u8"), + mp4Url: expect.stringContaining("https://play.example/"), + }); + }); + + it("queues one event carrying both URLs when both outputs finish at once", async () => { + await Promise.all([mp4Done(), hlsDone()]); + await settled(); + expect(video().status).toBe("ready"); + expect(mocks.start).toHaveBeenCalledTimes(1); + expect(jobs()[0]?.eventType).toBe("video.ready"); + expect(readyData().hlsUrl).toContain("/master.m3u8"); + expect(readyData().mp4Url).toContain("https://play.example/"); + }); + + it("queues one event when both outputs settle in one update", async () => { + mocks.store.video = developerVideo({ + processing: processingState({ + mp4: { state: "complete", output: "m0" }, + hls: running("h1"), + }), + }); + await hlsDone(); + await settled(); + expect(video().status).toBe("ready"); + expect(mocks.start).toHaveBeenCalledTimes(1); + expect(video().processing?.readyEventDueAt).toBeUndefined(); + }); + + it("queues nothing more when the encoder's callbacks arrive again", async () => { + await mp4Done(); + await mp4Done(); + await hlsDone(); + await hlsDone(); + await mp4Done(); + await settled(); + expect(mocks.start).toHaveBeenCalledTimes(1); + }); + + it("is sent with hlsUrl null when HLS fails", async () => { + mocks.store.video = developerVideo({ + processing: processingState({ + mp4: running("m1"), + hls: running("h1"), + attempts: { hls: 2, mp4: 1 }, + }), + }); + await mp4Done(); + await settled(); + expect(mocks.start).not.toHaveBeenCalled(); + + await applyDeveloperVideoProgress( + { videoId: "video-1", phase: "error", error: "boom" }, + "hls", + "h1", + ); + await settled(); + expect(video().processing?.hls.state).toBe("failed"); + expect(jobs().map((job) => job.eventType)).toEqual(["video.ready"]); + expect(readyData()).toMatchObject({ + hlsUrl: null, + mp4Url: expect.stringContaining("https://play.example/"), + }); + }); + + it("is sent once, with hlsUrl null, when HLS never settles", async () => { + const realNow = Date.now(); + vi.setSystemTime(realNow); + await mp4Done(); + await settled(); + expect(mocks.start).not.toHaveBeenCalled(); + + vi.setSystemTime(realNow + READY_EVENT_MAX_WAIT_MS - 1_000); + await readDeveloperVideoParts("video-1", "encode-1"); + await settled(); + expect(mocks.start).not.toHaveBeenCalled(); + + vi.setSystemTime(realNow + READY_EVENT_MAX_WAIT_MS); + const parts = await readDeveloperVideoParts("video-1", "encode-1"); + await settled(); + expect(parts?.hls).toBe("check"); + expect(video().processing?.hls.state).toBe("running"); + expect(jobs().map((job) => job.eventType)).toEqual(["video.ready"]); + expect(readyData()).toMatchObject({ + hlsUrl: null, + mp4Url: expect.stringContaining("https://play.example/"), + }); + + await readDeveloperVideoParts("video-1", "encode-1"); + await hlsDone(); + await settled(); + expect(video().processing?.hls.state).toBe("complete"); + expect(mocks.start).toHaveBeenCalledTimes(1); + }); + + it("is sent with hlsUrl null when processing times out on a stuck HLS job", async () => { + await mp4Done(); + await settled(); + await failDeveloperVideoProcessing("video-1", "encode-1", "Too long"); + await settled(); + expect(video().status).toBe("ready"); + expect(video().processing?.hls.state).toBe("failed"); + expect(jobs().map((job) => job.eventType)).toEqual(["video.ready"]); + expect(readyData().hlsUrl).toBeNull(); + }); + + it("is sent on the next poll when the last output was settled without sending", async () => { + mocks.store.video = developerVideo({ + status: "ready", + processing: processingState({ + mp4: { state: "complete", output: "m1" }, + hls: { state: "complete", output: "h1" }, + readyEventDueAt: Date.now() - 5_000, + }), + }); + const parts = await readDeveloperVideoParts("video-1", "encode-1"); + await readDeveloperVideoParts("video-1", "encode-1"); + await settled(); + expect(parts).toEqual({ hls: "done", mp4: "done" }); + expect(jobs().map((job) => job.eventType)).toEqual(["video.ready"]); + expect(readyData().hlsUrl).toContain("/master.m3u8"); + expect(video().processing?.readyEventDueAt).toBeUndefined(); + }); + + it("keeps the workflow polling when sending a due event fails", async () => { + mocks.store.video = developerVideo({ + status: "ready", + processing: processingState({ + mp4: { state: "complete", output: "m1" }, + hls: { state: "complete", output: "h1" }, + readyEventDueAt: Date.now() - 5_000, + }), + }); + mocks.failTransactions = 1; + expect(await readDeveloperVideoParts("video-1", "encode-1")).toEqual({ + hls: "wait", + mp4: "wait", + }); + await settled(); + expect(mocks.start).not.toHaveBeenCalled(); + expect(video().processing?.readyEventDueAt).toEqual(expect.any(Number)); + + expect(await readDeveloperVideoParts("video-1", "encode-1")).toEqual({ + hls: "done", + mp4: "done", + }); + await settled(); + expect(jobs().map((job) => job.eventType)).toEqual(["video.ready"]); + }); + + it("sends no second event for a video that was ready before this change", async () => { + mocks.store.video = developerVideo({ + status: "ready", + processing: processingState({ + mp4: { state: "complete", output: "m0" }, + hls: running("h1"), + }), + }); + await hlsDone(); + await settled(); + expect(video().processing?.hls.state).toBe("complete"); + + vi.setSystemTime(Date.now() + 2 * READY_EVENT_MAX_WAIT_MS); + await readDeveloperVideoParts("video-1", "encode-1"); + await settled(); + expect(mocks.endpointQueries).toBe(0); + expect(mocks.start).not.toHaveBeenCalled(); + }); + + it("sends no event for a video that was ready before webhooks existed", async () => { + mocks.store.video = developerVideo({ + status: "ready", + createdAt: new Date("2026-10-01T00:00:00Z"), + updatedAt: new Date("2026-10-01T00:00:00Z"), + processing: processingState({ + mp4: { state: "complete", output: "m0" }, + hls: { state: "unavailable", reason: "not yet", retryAt: 0 }, + }), + }); + expect(await claimDeveloperVideoProcessing("video-1")).toMatchObject({ + action: "retry-hls", + }); + expect(video().processing?.hls.state).toBe("pending"); + mocks.store.video = { + ...video(), + processing: processingState({ + ...video().processing, + hls: running("h1"), + }), + }; + await hlsDone(); + await settled(); + expect(video().processing?.hls.state).toBe("complete"); + expect(mocks.endpointQueries).toBe(0); + expect(mocks.start).not.toHaveBeenCalled(); + }); + + it("re-sends the same event id and body on every retry", async () => { + await mp4Done(); + await hlsDone(); + await settled(); + const [job] = jobs(); + if (!job) throw new Error("no event was queued"); + mocks.attempt + .mockResolvedValueOnce("retry") + .mockResolvedValueOnce("retry") + .mockResolvedValueOnce("delivered"); + + expect(await deliverDeveloperWebhookWorkflow(job)).toEqual({ + outcome: "delivered", + attempts: 3, + }); + const sent = mocks.attempt.mock.calls.map(([attempt]) => attempt); + expect(sent).toEqual([job, job, job]); + expect(JSON.parse(sent[2]?.body ?? "{}")).toMatchObject({ + id: developerVideoEventId("video-1", "video.ready"), + data: { hlsUrl: expect.stringContaining("/master.m3u8") }, + }); + }); + + it("queues one delivery for each endpoint that wants the event", async () => { + mocks.endpoints.push( + { + id: "wh-2", + appId: "app-1", + events: ["video.ready"], + enabled: true, + deletedAt: null, + }, + { + id: "wh-errors-only", + appId: "app-1", + events: ["video.errored"], + enabled: true, + deletedAt: null, + }, + { + id: "wh-off", + appId: "app-1", + events: ["video.ready"], + enabled: false, + deletedAt: null, + }, + { + id: "wh-other-app", + appId: "app-2", + events: ["video.ready"], + enabled: true, + deletedAt: null, + }, + ); + await Promise.all([mp4Done(), hlsDone()]); + await settled(); + expect(jobs().map((job) => job.endpointId)).toEqual(["wh-1", "wh-2"]); + expect(new Set(jobs().map((job) => job.eventId)).size).toBe(1); + expect(new Set(jobs().map((job) => job.body)).size).toBe(1); + }); + + it("reads nothing more and queues nothing for an app with no endpoints", async () => { + mocks.endpoints = []; + await mp4Done(); + await hlsDone(); + await settled(); + expect(video().status).toBe("ready"); + expect(mocks.endpointQueries).toBe(1); + expect(mocks.start).not.toHaveBeenCalled(); + }); + + it("asks about endpoints only when an event is due", async () => { + await applyDeveloperVideoProgress( + { videoId: "video-1", phase: "processing", jobId: "job-9" }, + "mp4", + "m1", + ); + await mp4Done(); + await settled(); + expect(mocks.endpointQueries).toBe(0); + }); +}); + +describe("video.errored", () => { + it("queues one event when processing fails, and none when failed again", async () => { + await failDeveloperVideoProcessing("video-1", "encode-1", "Too long"); + await failDeveloperVideoProcessing("video-1", "encode-1", "Too long"); + await settled(); + + expect(video().status).toBe("errored"); + expect(mocks.start).toHaveBeenCalledTimes(1); + const [job] = jobs(); + expect(job).toMatchObject({ + eventType: "video.errored", + eventId: developerVideoEventId("video-1", "video.errored"), + }); + expect(JSON.parse(job?.body ?? "{}").data).toMatchObject({ + status: "errored", + errorReason: "Too long", + }); + }); + + it("queues nothing for an output that fails after the video is ready", async () => { + await mp4Done(); + await hlsDone(); + await settled(); + await applyDeveloperVideoProgress( + { videoId: "video-1", phase: "error", error: "boom" }, + "hls", + "h1", + ); + await applyDeveloperVideoProgress( + { videoId: "video-1", phase: "error", error: "boom" }, + "hls", + "h1", + ); + await settled(); + expect(video().status).toBe("ready"); + expect(jobs().map((job) => job.eventType)).toEqual(["video.ready"]); + }); +}); + +describe("queueing an event", () => { + it("sends nothing for a status that was never committed", async () => { + const queued = await enqueueDeveloperVideoEvent( + { id: "video-1", appId: "app-1" }, + "video.ready", + ); + expect(queued).toBe(0); + expect(mocks.start).not.toHaveBeenCalled(); + }); + + it("sends nothing for a deleted video", async () => { + mocks.store.video = developerVideo({ + status: "ready", + deletedAt: new Date(), + }); + expect( + await enqueueDeveloperVideoEvent( + { id: "video-1", appId: "app-1" }, + "video.ready", + ), + ).toBe(0); + }); + + it("uses the same event id and data when it is called again", async () => { + mocks.store.video = developerVideo({ status: "ready" }); + const target = { id: "video-1", appId: "app-1" }; + await enqueueDeveloperVideoEvent(target, "video.ready"); + await enqueueDeveloperVideoEvent(target, "video.ready"); + const [first, second] = jobs(); + expect(first?.eventId).toBe(second?.eventId); + expect(JSON.parse(first?.body ?? "{}").data).toEqual( + JSON.parse(second?.body ?? "{}").data, + ); + }); +}); + +describe("webhooks never hold up a video", () => { + it("marks the video ready while the customer's endpoint is still hanging", async () => { + const hanging = new Promise(() => {}); + mocks.start.mockReturnValue(hanging); + + vi.useRealTimers(); + const started = performance.now(); + expect(await mp4Done()).toEqual({ success: true, applied: true }); + expect(await hlsDone()).toEqual({ success: true, applied: true }); + await settled(); + + expect(mocks.start).toHaveBeenCalledTimes(1); + expect(video().status).toBe("ready"); + expect(video().processing?.hls.state).toBe("complete"); + expect(performance.now() - started).toBeLessThan(1_000); + }); + + it("records the outputs when queueing the delivery fails", async () => { + mocks.start.mockRejectedValue(new Error("queue is down")); + expect(await mp4Done()).toEqual({ success: true, applied: true }); + expect(await hlsDone()).toEqual({ success: true, applied: true }); + await settled(); + expect(mocks.start).toHaveBeenCalledTimes(1); + expect(video().status).toBe("ready"); + expect(video().processing?.hls.state).toBe("complete"); + }); + + it("records the outputs when the endpoint lookup fails", async () => { + mocks.failEndpointQuery = true; + expect(await mp4Done()).toEqual({ success: true, applied: true }); + expect(await hlsDone()).toEqual({ success: true, applied: true }); + await settled(); + expect(mocks.endpointQueries).toBe(1); + expect(video().status).toBe("ready"); + expect(video().processing?.hls.state).toBe("complete"); + expect(mocks.start).not.toHaveBeenCalled(); + }); + + it("records a failure while the endpoint lookup is still hanging", async () => { + mocks.hangEndpointQuery = true; + await failDeveloperVideoProcessing("video-1", "encode-1", "Too long"); + expect(video().status).toBe("errored"); + expect(mocks.endpointQueries).toBe(1); + expect(mocks.start).not.toHaveBeenCalled(); + }); +}); + +describe("deciding the event inside the status write", () => { + const complete = { state: "complete" as const, output: "o" }; + const now = 10_000_000; + + it("sends video.ready at once when every output has settled", () => { + const processing = processingState({ + mp4: complete, + hls: { state: "unavailable", reason: "no ladder", retryAt: now + 1 }, + }); + expect( + developerVideoEventStep( + { status: "processing", processing }, + { status: "ready", processing }, + now, + ), + ).toEqual({ type: "video.ready" }); + }); + + it("marks the video and waits while an output is still encoding", () => { + const processing = processingState({ mp4: complete, hls: running("h") }); + expect( + developerVideoEventStep( + { status: "processing", processing }, + { status: "ready", processing }, + now, + ), + ).toEqual({ + type: null, + processing: { ...processing, readyEventDueAt: now }, + }); + }); + + it("sends it once the waiting output settles, and clears the mark", () => { + const waiting = processingState({ + mp4: complete, + hls: running("h"), + readyEventDueAt: now, + }); + const settledState = { ...waiting, hls: complete }; + const step = developerVideoEventStep( + { status: "ready", processing: waiting }, + { status: "ready", processing: settledState }, + now + 5_000, + ); + expect(step.type).toBe("video.ready"); + expect(step.processing).not.toHaveProperty("readyEventDueAt"); + expect( + developerVideoEventStep( + { status: "ready", processing: step.processing ?? null }, + { status: "ready", processing: step.processing ?? null }, + now + 6_000, + ), + ).toEqual({ type: null }); + }); + + it("sends it once the wait runs out, even with an output still encoding", () => { + const waiting = processingState({ + mp4: complete, + hls: running("h"), + readyEventDueAt: now, + }); + const video = { status: "ready", processing: waiting }; + expect( + developerVideoEventStep(video, video, now + READY_EVENT_MAX_WAIT_MS - 1), + ).toEqual({ type: null }); + expect( + developerVideoEventStep(video, video, now + READY_EVENT_MAX_WAIT_MS).type, + ).toBe("video.ready"); + }); + + it("sends nothing for a ready video without the mark", () => { + const before = processingState({ mp4: complete, hls: running("h") }); + const after = { ...before, hls: complete }; + expect( + developerVideoEventStep( + { status: "ready", processing: before }, + { status: "ready", processing: after }, + now + 10 * READY_EVENT_MAX_WAIT_MS, + ), + ).toEqual({ type: null }); + expect( + developerVideoEventStep( + { status: "ready", processing: null }, + { status: "ready", processing: null }, + now, + ), + ).toEqual({ type: null }); + }); + + it("sends video.errored on the change to errored, as before", () => { + const processing = processingState(); + expect( + developerVideoEventStep( + { status: "processing", processing }, + { status: "errored", processing }, + now, + ), + ).toEqual({ type: "video.errored" }); + expect( + developerVideoEventStep( + { status: "errored", processing }, + { status: "errored", processing }, + now, + ), + ).toEqual({ type: null }); + }); +}); + +describe("a rolled back announcement", () => { + it("is not sent while the committed row still carries the mark", async () => { + mocks.store.video = developerVideo({ + status: "ready", + processing: processingState({ + mp4: { state: "complete", output: "m0" }, + hls: { state: "complete", output: "h0" }, + readyEventDueAt: 1, + }), + }); + expect( + await enqueueDeveloperVideoEvent( + { id: "video-1", appId: "app-1" }, + "video.ready", + ), + ).toBe(0); + expect(mocks.start).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/__tests__/unit/developer-webhook-docs.test.ts b/apps/web/__tests__/unit/developer-webhook-docs.test.ts new file mode 100644 index 00000000000..ce7c9a606ca --- /dev/null +++ b/apps/web/__tests__/unit/developer-webhook-docs.test.ts @@ -0,0 +1,324 @@ +import { execFileSync, spawn } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { compileMDX } from "next-mdx-remote/rsc"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/env", () => ({ + NODE_ENV: "production", + buildEnv: { NEXT_PUBLIC_WEB_URL: "https://cap.so" }, + serverEnv: () => ({}), +})); +vi.mock("@cap/database", () => ({ db: () => ({}) })); + +const { app } = await import("@/app/api/developer/v1/[...route]/webhooks"); +const { developerOpenApiSpec } = await import("@/lib/developer-openapi"); +const { + MAX_WEBHOOK_ENDPOINTS_PER_APP, + READY_EVENT_MAX_WAIT_MS, + WEBHOOK_DISABLE_AFTER_FAILURES, + WEBHOOK_DISABLE_AFTER_MS, + WEBHOOK_EVENT_TYPES, + WEBHOOK_MAX_ATTEMPTS, + WEBHOOK_RETRY_DELAYS_SECONDS, + WEBHOOK_TEST_EVENT_TYPE, + WEBHOOK_TIMEOUT_MS, +} = await import("@/lib/developer-webhook-events"); +const { signWebhookBody, WEBHOOK_SIGNATURE_TOLERANCE_SECONDS } = await import( + "@/lib/developer-webhook-signature" +); +const { extractHeadings } = await import("@/utils/docs"); + +const read = (name: string) => + readFileSync(join(process.cwd(), "content/docs/api", name), "utf8"); +const guide = read("webhooks.mdx"); +const rest = read("rest-api.mdx"); +const spec = developerOpenApiSpec("https://cap.so"); + +const gap = (seconds: number) => + seconds % 3_600 === 0 + ? `${seconds / 3_600} hour${seconds === 3_600 ? "" : "s"}` + : `${seconds / 60} minute${seconds === 60 ? "" : "s"}`; + +describe("the webhooks guide", () => { + it("compiles as MDX and links only to headings that exist", async () => { + await expect( + compileMDX({ source: guide, components: {} }), + ).resolves.toBeDefined(); + const anchors = new Map([ + ["/docs/api/rest-api", extractHeadings(rest).map((h) => h.slug)], + ["", extractHeadings(guide).map((h) => h.slug)], + ]); + const links = [ + ...guide.matchAll(/\]\((\/docs\/api\/rest-api)?#([\w-]+)\)/g), + ]; + expect(links.length).toBeGreaterThan(4); + for (const [, page = "", anchor] of links) { + expect(anchors.get(page), `${page}#${anchor}`).toContain(anchor); + } + expect(rest).toContain("](/docs/api/webhooks)"); + expect(extractHeadings(rest).map((h) => h.slug)).toContain("webhooks"); + }); + + it("names every event type, header and limit the code uses", () => { + for (const type of [...WEBHOOK_EVENT_TYPES, WEBHOOK_TEST_EVENT_TYPE]) { + expect(guide).toContain(`\`${type}\``); + } + for (const name of ["Cap-Signature", "Cap-Event-Id", "Cap-Event-Type"]) { + expect(guide).toContain(name); + } + expect(guide).toContain( + `An app can have ${MAX_WEBHOOK_ENDPOINTS_PER_APP} endpoints`, + ); + expect(rest).toContain( + `An app can have ${MAX_WEBHOOK_ENDPOINTS_PER_APP} endpoints`, + ); + expect(guide).toContain(`within ${WEBHOOK_TIMEOUT_MS / 1000} seconds`); + expect(WEBHOOK_SIGNATURE_TOLERANCE_SECONDS).toBe(300); + expect(guide).toContain("more than five minutes"); + expect(guide).toContain("> 300"); + }); + + it("states the retry schedule the workflow follows", () => { + expect(guide).toContain( + `That is ${WEBHOOK_MAX_ATTEMPTS} attempts over about a day`, + ); + const rows = [...guide.matchAll(/^\| (\d) \| (.+) \|$/gm)].map( + ([, attempt, sent]) => [Number(attempt), sent] as const, + ); + expect(rows.map(([attempt]) => attempt)).toEqual( + Array.from({ length: WEBHOOK_MAX_ATTEMPTS }, (_, index) => index + 1), + ); + expect(rows[0]?.[1]).toBe("Straight away"); + WEBHOOK_RETRY_DELAYS_SECONDS.forEach((seconds, index) => { + expect(rows[index + 1]?.[1]).toContain(gap(seconds)); + }); + expect(guide).toContain( + `fails ${WEBHOOK_DISABLE_AFTER_FAILURES} attempts in a row over five days`, + ); + expect(WEBHOOK_DISABLE_AFTER_MS).toBe(5 * 24 * 60 * 60 * 1000); + }); + + it("gives a Node example that accepts Cap's signature and rejects a forged one", async () => { + const example = guide.match(/```js\n([\s\S]*?verifyCapWebhook[\s\S]*?)```/); + expect(example).not.toBeNull(); + const folder = mkdtempSync(join(tmpdir(), "cap-webhook-doc-")); + try { + writeFileSync(join(folder, "verify.mjs"), example?.[1] ?? ""); + const secret = "cws_0123456789abcdefghjkmnpqrstvwx"; + const body = JSON.stringify({ id: "evt_1", type: "video.ready" }); + const now = Math.floor(Date.now() / 1000); + const header = signWebhookBody(secret, body, now); + const cases = [ + [body, header, secret], + [`${body} `, header, secret], + [body, header, `${secret}x`], + [body, null, secret], + [body, "t=1,v1=zz", secret], + [body, signWebhookBody(secret, body, now - 301), secret], + ]; + writeFileSync(join(folder, "cases.json"), JSON.stringify(cases)); + writeFileSync( + join(folder, "run.mjs"), + [ + 'import { readFileSync } from "node:fs";', + 'import { verifyCapWebhook } from "./verify.mjs";', + 'const cases = JSON.parse(readFileSync(new URL("./cases.json", import.meta.url), "utf8"));', + "console.log(JSON.stringify(cases.map(([body, header, secret]) => verifyCapWebhook(body, header ?? undefined, secret))));", + ].join("\n"), + ); + const output = execFileSync(process.execPath, [join(folder, "run.mjs")], { + encoding: "utf8", + }); + expect(JSON.parse(output)).toEqual([ + true, + false, + false, + false, + false, + false, + ]); + } finally { + rmSync(folder, { recursive: true, force: true }); + } + }); + + it("gives a receiver that answers Cap's request and refuses a forged one", async () => { + const jsBlocks = [...guide.matchAll(/```js\n([\s\S]*?)```/g)].map( + ([, code]) => code ?? "", + ); + const verify = jsBlocks.find((code) => code.includes("timingSafeEqual")); + const receiver = jsBlocks.find((code) => code.includes("createServer")); + expect(receiver).toContain('from "./verify-cap-webhook.mjs"'); + const folder = mkdtempSync(join(tmpdir(), "cap-webhook-receiver-")); + const port = await new Promise((resolve) => { + const probe = createServer().listen(0, () => { + const address = probe.address(); + probe.close(() => + resolve(typeof address === "object" && address ? address.port : 0), + ); + }); + }); + const secret = "cws_0123456789abcdefghjkmnpqrstvwx"; + writeFileSync(join(folder, "verify-cap-webhook.mjs"), verify ?? ""); + writeFileSync(join(folder, "server.mjs"), receiver ?? ""); + const child = spawn(process.execPath, [join(folder, "server.mjs")], { + env: { ...process.env, PORT: String(port), CAP_WEBHOOK_SECRET: secret }, + }); + let output = ""; + child.stdout.on("data", (chunk) => { + output += chunk; + }); + try { + const url = `http://127.0.0.1:${port}/webhooks/cap`; + const body = JSON.stringify({ + id: "evt_1", + type: "video.ready", + data: { id: "video-1", hlsUrl: "https://play/master.m3u8" }, + }); + const post = async (signature: string) => { + for (let tries = 0; ; tries++) { + try { + return await fetch(url, { + method: "POST", + headers: { "Cap-Signature": signature }, + body, + }); + } catch (error) { + if (tries > 50) throw error; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + } + }; + const oversized = await post("").then(() => + fetch(url, { method: "POST", body: "x".repeat(2_000_000) }).then( + (response) => response.status, + () => "reset", + ), + ); + expect([413, "reset"]).toContain(oversized); + const now = Math.floor(Date.now() / 1000); + expect( + (await post(signWebhookBody(`${secret}x`, body, now))).status, + ).toBe(400); + expect((await post(signWebhookBody(secret, body, now))).status).toBe(200); + expect((await post(signWebhookBody(secret, body, now))).status).toBe(200); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(output.trim().split("\n")).toEqual([ + "ready video-1 https://play/master.m3u8", + ]); + } finally { + child.kill(); + rmSync(folder, { recursive: true, force: true }); + } + }); + + it("says when video.ready is sent, matching the code", () => { + expect(guide).not.toContain("No second event is sent"); + expect(guide).toContain( + "`video.ready` arrives once every playback URL is ready, so `hlsUrl` is set unless HLS failed", + ); + expect(READY_EVENT_MAX_WAIT_MS).toBe(30 * 60 * 1000); + expect(guide).toContain("still encoding 30 minutes after"); + expect( + (spec.webhooks as Record)[ + "video.ready" + ]?.post.description, + ).toContain("30 minutes after"); + }); + + it("indents code like the rest of the docs", () => { + const blocks = [...guide.matchAll(/```\w*\n([\s\S]*?)```/g)]; + for (const [, code] of blocks) expect(code).not.toMatch(/^\t/m); + }); + + it("teaches the REST API only", () => { + expect(guide).not.toMatch(/\bcap (developers|caps|webhooks)\b/); + expect(guide).not.toMatch(/\bMCP\b/); + expect(guide).not.toContain("—"); + }); +}); + +describe("webhooks in the OpenAPI description", () => { + const paths = spec.paths as Record>; + + it("describes every webhook route the API serves, and no others", () => { + const served = [ + ...new Set( + app.routes + .filter((route) => route.method !== "ALL") + .map( + (route) => + `${route.method} /api/developer/v1/webhooks${ + route.path === "/" ? "" : route.path.replace(/:(\w+)/g, "{$1}") + }`, + ), + ), + ].sort(); + const described = Object.entries(paths) + .filter(([path]) => path.startsWith("/api/developer/v1/webhooks")) + .flatMap(([path, item]) => + Object.keys(item).map((method) => `${method.toUpperCase()} ${path}`), + ) + .sort(); + expect(described).toEqual(served); + expect(described).toHaveLength(7); + }); + + it("returns the signing secret from create and rotate only", () => { + const schemas = spec.components.schemas as Record< + string, + { properties?: object } + >; + expect( + Object.keys(schemas.WebhookEndpoint?.properties ?? {}), + ).not.toContain("secret"); + const text = (operation: unknown) => JSON.stringify(operation); + for (const operation of [ + paths["/api/developer/v1/webhooks"]?.post, + paths["/api/developer/v1/webhooks/{id}/rotate-secret"]?.post, + ]) { + expect(text(operation)).toContain("WebhookEndpointWithSecret"); + } + for (const operation of [ + paths["/api/developer/v1/webhooks"]?.get, + paths["/api/developer/v1/webhooks/{id}"]?.get, + paths["/api/developer/v1/webhooks/{id}"]?.patch, + paths["/api/developer/v1/webhooks/{id}/test"]?.post, + ]) { + expect(text(operation)).not.toContain("WebhookEndpointWithSecret"); + } + }); + + it("promises retries for video events only", () => { + const outbound = spec.webhooks as Record< + string, + { post: { description: string } } + >; + for (const type of WEBHOOK_EVENT_TYPES) { + expect(outbound[type]?.post.description).toContain( + `up to ${WEBHOOK_MAX_ATTEMPTS} attempts`, + ); + expect(outbound[type]?.post.description).toContain("enabled endpoint"); + } + const test = outbound[WEBHOOK_TEST_EVENT_TYPE]?.post.description ?? ""; + expect(test).toContain("never retried"); + expect(test).toContain("whether or not that endpoint is enabled"); + expect(test).not.toContain("attempts"); + }); + + it("describes the request Cap sends for each event", () => { + const outbound = spec.webhooks as Record; + expect(Object.keys(outbound).sort()).toEqual( + [...WEBHOOK_EVENT_TYPES, WEBHOOK_TEST_EVENT_TYPE].sort(), + ); + for (const delivery of Object.values(outbound)) { + const text = JSON.stringify(delivery.post); + expect(text).toContain("Cap-Signature"); + expect(text).toContain("Cap-Event-Id"); + expect(text).toContain("Cap-Event-Type"); + } + }); +}); diff --git a/apps/web/__tests__/unit/developer-webhook-fake-db.ts b/apps/web/__tests__/unit/developer-webhook-fake-db.ts new file mode 100644 index 00000000000..f0999bf7afa --- /dev/null +++ b/apps/web/__tests__/unit/developer-webhook-fake-db.ts @@ -0,0 +1,100 @@ +import type { SQL } from "drizzle-orm"; +import { MySqlDialect } from "drizzle-orm/mysql-core"; +import { createFakeDatabase } from "./developer-transcript-fake-db"; + +type Row = Record; + +const dialect = new MySqlDialect(); + +const isSql = (value: unknown): value is SQL => + typeof value === "object" && value !== null && "queryChunks" in value; + +const keepsColumn = (value: unknown) => + isSql(value) && + /^`[^`]+`(\.`[^`]+`)?$/.test(dialect.sqlToQuery(value).sql.trim()); + +export const webhookEndpointDefaults = (): Row => ({ + events: ["video.ready", "video.errored"], + enabled: true, + disabledReason: null, + lastDeliveryAt: null, + lastDeliveryStatus: null, + lastResponseStatus: null, + lastErrorCode: null, + consecutiveFailures: 0, + failingSince: null, + deletedAt: null, +}); + +// Adds the two things the endpoint queries use that the shared fake does not +// read: a join to the app table, and an update that leaves updatedAt alone. +export function createWebhookFakeDatabase( + defaults: Map Row>, + options: { joined?: (row: Row) => boolean } = {}, +) { + const fake = createFakeDatabase(defaults); + const base = fake.db; + + const select = (fields?: Record) => ({ + from: (table: object) => { + const state = { joined: false }; + const chain = base.select(fields).from(table); + const rows = async () => { + const found = await chain; + return state.joined && options.joined + ? found.filter(options.joined) + : found; + }; + const wrapped = Object.assign(Promise.resolve().then(rows), { + innerJoin() { + state.joined = true; + return wrapped; + }, + where(condition: SQL | undefined) { + chain.where(condition); + return wrapped; + }, + orderBy() { + chain.orderBy(); + return wrapped; + }, + limit(count: number) { + chain.limit(count); + return wrapped; + }, + offset(count: number) { + chain.offset(count); + return wrapped; + }, + for() { + return wrapped; + }, + }); + return wrapped; + }, + }); + + const update = (table: object) => ({ + set: (values: Row) => + base + .update(table) + .set( + Object.fromEntries( + Object.entries(values).map(([key, value]) => [ + key, + keepsColumn(value) ? undefined : value, + ]), + ), + ), + }); + + const client = { ...base, select, update }; + return { + ...fake, + db: { + ...client, + transaction: (run: (tx: typeof client) => Promise) => + base.transaction(() => run(client)), + }, + }; +} diff --git a/apps/web/__tests__/unit/developer-webhook-guard.test.ts b/apps/web/__tests__/unit/developer-webhook-guard.test.ts new file mode 100644 index 00000000000..1173c7f4377 --- /dev/null +++ b/apps/web/__tests__/unit/developer-webhook-guard.test.ts @@ -0,0 +1,291 @@ +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/env", () => ({ + NODE_ENV: "production", + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + serverEnv: () => ({}), +})); +vi.mock("@cap/database", () => ({ db: () => ({}) })); + +const { checkWebhookUrl, sendDeveloperWebhook, webhookUrlProblem } = + await import("@/lib/developer-webhook-delivery"); +const { isBlockedS3Endpoint, isPrivateAddress } = await import( + "@/lib/s3-endpoint-policy" +); +const { requestPublicOnly } = await import("@/lib/storage-fetch"); +const { verifyWebhookSignature } = await import( + "@/lib/developer-webhook-signature" +); + +const resolvesTo = + (...addresses: string[]) => + (url: string) => + isBlockedS3Endpoint(url, async () => addresses); + +const event = { + secret: "cws_0123456789abcdefghjkmnpqrstvwx", + eventId: "evt_1", + eventType: "video.ready", + body: JSON.stringify({ id: "evt_1", type: "video.ready" }), +}; + +describe("webhook URL policy in a production build", () => { + it("is strict unless a caller opts in", () => { + expect(webhookUrlProblem("http://127.0.0.1/cap")).toBe("https_required"); + expect(webhookUrlProblem("http://127.0.0.1/cap", true)).toBeNull(); + }); + + it("accepts a public https URL", () => { + expect(webhookUrlProblem("https://hooks.example.com/cap?x=1")).toBeNull(); + }); + + it.each([ + ["plain http", "http://hooks.example.com/cap", "https_required"], + ["another scheme", "ftp://hooks.example.com/cap", "https_required"], + ["a file URL", "file:///etc/passwd", "https_required"], + ["no scheme", "hooks.example.com/cap", "invalid_url"], + ["an empty value", "", "invalid_url"], + [ + "a URL over the length cap", + `https://hooks.example.com/${"a".repeat(2_048)}`, + "invalid_url", + ], + [ + "a username and password", + "https://user:pass@hooks.example.com/cap", + "credentials_in_url", + ], + ["another port", "https://hooks.example.com:8443/cap", "unsupported_port"], + ["localhost", "https://localhost/cap", "private_address"], + ["a localhost subdomain", "https://app.localhost/cap", "private_address"], + ["an internal name", "https://db.internal/cap", "private_address"], + ["a .local name", "https://printer.local/cap", "private_address"], + ["loopback v4", "https://127.0.0.1/cap", "private_address"], + ["private 10/8", "https://10.1.2.3/cap", "private_address"], + ["private 172.16/12", "https://172.20.0.1/cap", "private_address"], + ["private 192.168/16", "https://192.168.1.10/cap", "private_address"], + ["carrier NAT", "https://100.64.0.1/cap", "private_address"], + ["the metadata address", "https://169.254.169.254/cap", "private_address"], + ["the unspecified address", "https://0.0.0.0/cap", "private_address"], + ["a documentation range", "https://203.0.113.9/cap", "private_address"], + ["multicast", "https://224.0.0.1/cap", "private_address"], + ["a decimal loopback", "https://2130706433/cap", "private_address"], + ["a hex loopback", "https://0x7f.1/cap", "private_address"], + ["an octal loopback", "https://017700000001/cap", "private_address"], + ["loopback v6", "https://[::1]/cap", "private_address"], + ["unique local v6", "https://[fd00:ec2::254]/cap", "private_address"], + ["link-local v6", "https://[fe80::1]/cap", "private_address"], + ["mapped v4 loopback", "https://[::ffff:127.0.0.1]/cap", "private_address"], + ["mapped v4 private", "https://[::ffff:a00:1]/cap", "private_address"], + [ + "mapped v4 metadata", + "https://[::ffff:169.254.169.254]/cap", + "private_address", + ], + [ + "NAT64 of a private v4", + "https://[64:ff9b::a00:1]/cap", + "private_address", + ], + ["6to4 of loopback", "https://[2002:7f00:1::]/cap", "private_address"], + ["Teredo", "https://[2001:0:4136:e378::1]/cap", "private_address"], + ])("refuses %s", (_label, url, problem) => { + expect(webhookUrlProblem(url)).toBe(problem); + }); + + it("measures the URL as it is stored, after encoding", () => { + const accented = `https://hooks.example.com/${"é".repeat(400)}`; + expect(accented.length).toBeLessThan(2_048); + expect(new URL(accented).href.length).toBeGreaterThan(2_048); + expect(webhookUrlProblem(accented)).toBe("invalid_url"); + expect(webhookUrlProblem(accented, true)).toBe("invalid_url"); + expect( + webhookUrlProblem(`https://hooks.example.com/${"é".repeat(300)}`), + ).toBeNull(); + }); + + it("refuses a name that resolves to a private address", async () => { + for (const address of ["10.0.0.5", "127.0.0.1", "::1", "::ffff:10.0.0.5"]) { + expect( + await checkWebhookUrl("https://hooks.example.com/cap", { + isBlocked: resolvesTo(address), + }), + ).toBe("private_address"); + } + }); + + it("refuses a name with one private address among public ones", async () => { + expect( + await checkWebhookUrl("https://hooks.example.com/cap", { + isBlocked: resolvesTo("93.184.216.34", "192.168.0.7"), + }), + ).toBe("private_address"); + }); + + it("refuses a name that does not resolve", async () => { + expect( + await checkWebhookUrl("https://hooks.example.com/cap", { + isBlocked: resolvesTo(), + }), + ).toBe("private_address"); + }); + + it("accepts a name that resolves to public addresses only", async () => { + expect( + await checkWebhookUrl("https://hooks.example.com/cap", { + isBlocked: resolvesTo("93.184.216.34", "2606:2800:220:1::1"), + }), + ).toBeNull(); + }); + + it("covers the reserved ranges added for webhooks", () => { + for (const address of [ + "192.0.2.1", + "198.51.100.1", + "203.0.113.1", + "192.88.99.1", + "2001:db8::1", + "100::1", + "64:ff9b:1::1", + "2002:a00:1::1", + "2001:0:1::1", + ]) { + expect(isPrivateAddress(address)).toBe(true); + } + for (const address of ["93.184.216.34", "8.8.8.8", "2606:4700::1111"]) { + expect(isPrivateAddress(address)).toBe(false); + } + }); +}); + +describe("sending a webhook in a production build", () => { + let server: Server; + let port: number; + let hits = 0; + + beforeAll(async () => { + server = createServer((_req, res) => { + hits += 1; + res.end("internal"); + }); + await new Promise((resolve) => + server.listen(0, "127.0.0.1", resolve), + ); + port = (server.address() as AddressInfo).port; + }); + + afterAll(async () => { + await new Promise((resolve) => server.close(resolve)); + }); + + it("does not connect to a private address written as a literal", async () => { + await expect( + requestPublicOnly(`http://127.0.0.1:${port}/`, { method: "POST" }), + ).rejects.toThrow("Refused to connect to private address"); + await expect( + requestPublicOnly(`http://[::ffff:127.0.0.1]:${port}/`), + ).rejects.toThrow("Refused to connect to private address"); + expect(hits).toBe(0); + }); + + it("does not connect when the name resolves to a private address at connect time", async () => { + const request = vi.fn((url: string, init: RequestInit) => + requestPublicOnly(url.replace("hooks.example.com", "localhost"), init), + ); + const result = await sendDeveloperWebhook( + { ...event, url: `https://hooks.example.com/cap` }, + { isBlocked: async () => false, request }, + ); + expect(request).toHaveBeenCalledTimes(1); + expect(result).toMatchObject({ ok: false, errorCode: "private_address" }); + expect(hits).toBe(0); + }); + + it("sends nothing when the URL no longer passes the policy", async () => { + const request = vi.fn(); + for (const url of [ + "http://hooks.example.com/cap", + "https://127.0.0.1/cap", + "https://user:pass@hooks.example.com/cap", + ]) { + const result = await sendDeveloperWebhook({ ...event, url }, { request }); + expect(result.ok).toBe(false); + } + const rebound = await sendDeveloperWebhook( + { ...event, url: "https://hooks.example.com/cap" }, + { request, isBlocked: resolvesTo("10.0.0.5") }, + ); + expect(rebound).toMatchObject({ ok: false, errorCode: "private_address" }); + expect(request).not.toHaveBeenCalled(); + }); + + it("treats a redirect as a failed delivery and never follows it", async () => { + const request = vi.fn( + async () => + new Response("moved", { + status: 302, + headers: { location: "http://169.254.169.254/latest/meta-data/" }, + }), + ); + const result = await sendDeveloperWebhook( + { ...event, url: "https://hooks.example.com/cap" }, + { isBlocked: async () => false, request }, + ); + expect(result).toMatchObject({ + ok: false, + errorCode: "redirect", + responseStatus: 302, + }); + expect(request).toHaveBeenCalledTimes(1); + }); + + it("signs the request and names the event in headers", async () => { + const seen: { url: string; init: RequestInit }[] = []; + const result = await sendDeveloperWebhook( + { ...event, url: "https://hooks.example.com/cap" }, + { + isBlocked: async () => false, + request: async (url, init) => { + seen.push({ url, init }); + return new Response(null, { status: 204 }); + }, + }, + ); + expect(result).toMatchObject({ ok: true, responseStatus: 204 }); + const [sent] = seen; + const headers = new Headers(sent?.init.headers); + expect(sent?.init.method).toBe("POST"); + expect(sent?.init.body).toBe(event.body); + expect(headers.get("content-type")).toBe("application/json"); + expect(headers.get("cap-event-id")).toBe("evt_1"); + expect(headers.get("cap-event-type")).toBe("video.ready"); + expect( + verifyWebhookSignature({ + secret: event.secret, + body: event.body, + header: headers.get("cap-signature"), + }), + ).toBe(true); + }); + + it.each([ + [200, true], + [201, true], + [299, true], + [400, false], + [404, false], + [500, false], + ])("counts %i as delivered: %s", async (status, delivered) => { + const result = await sendDeveloperWebhook( + { ...event, url: "https://hooks.example.com/cap" }, + { + isBlocked: async () => false, + request: async () => new Response("x", { status }), + }, + ); + expect(result.ok).toBe(delivered); + expect(result.responseStatus).toBe(status); + }); +}); diff --git a/apps/web/__tests__/unit/developer-webhook-private-targets.test.ts b/apps/web/__tests__/unit/developer-webhook-private-targets.test.ts new file mode 100644 index 00000000000..889eb66396d --- /dev/null +++ b/apps/web/__tests__/unit/developer-webhook-private-targets.test.ts @@ -0,0 +1,102 @@ +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + it, + vi, +} from "vitest"; + +// Whatever NODE_ENV says, the code a route or the delivery workflow runs +// (no guards passed) never reaches plain http or a private address. +const env = vi.hoisted(() => ({ + NODE_ENV: "development" as string | undefined, + buildEnv: { NEXT_PUBLIC_IS_CAP: "false" }, + serverEnv: () => ({}), +})); +vi.mock("@cap/env", () => env); +vi.mock("@cap/database", () => ({ db: () => ({}) })); + +const { checkWebhookUrl, sendDeveloperWebhook, webhookUrlProblem } = + await import("@/lib/developer-webhook-delivery"); + +const event = { + secret: "cws_0123456789abcdefghjkmnpqrstvwx", + eventId: "evt_1", + eventType: "video.ready", + body: "{}", +}; + +let server: Server; +let port: number; +let hits = 0; + +beforeAll(async () => { + server = createServer((_req, res) => { + hits += 1; + res.end("ok"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + port = (server.address() as AddressInfo).port; +}); + +afterAll(async () => { + await new Promise((resolve) => server.close(resolve)); +}); + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe.each([ + "development", + "test", + "staging", + "preview", + "", + undefined, + "production", +])("with NODE_ENV %s", (nodeEnv) => { + it("refuses private and plain http targets on create and on delivery", async () => { + env.NODE_ENV = nodeEnv; + vi.stubEnv("NODE_ENV", nodeEnv); + hits = 0; + + expect(webhookUrlProblem(`http://127.0.0.1:${port}/`)).toBe( + "https_required", + ); + expect(await checkWebhookUrl("https://127.0.0.1/cap")).toBe( + "private_address", + ); + expect(await checkWebhookUrl("https://localhost/cap")).toBe( + "private_address", + ); + expect(await checkWebhookUrl("https://169.254.169.254/cap")).toBe( + "private_address", + ); + + for (const url of [ + `http://127.0.0.1:${port}/`, + `http://localhost:${port}/`, + "https://127.0.0.1/cap", + "https://[::1]/cap", + ]) { + const result = await sendDeveloperWebhook({ ...event, url }); + expect(result.ok).toBe(false); + } + expect(hits).toBe(0); + }); +}); + +it("reaches a local server only when a test passes allowPrivate", async () => { + hits = 0; + const result = await sendDeveloperWebhook( + { ...event, url: `http://127.0.0.1:${port}/` }, + { allowPrivate: true }, + ); + expect(result.ok).toBe(true); + expect(hits).toBe(1); +}); diff --git a/apps/web/__tests__/unit/developer-webhook-rebinding.test.ts b/apps/web/__tests__/unit/developer-webhook-rebinding.test.ts new file mode 100644 index 00000000000..66384ee1dbd --- /dev/null +++ b/apps/web/__tests__/unit/developer-webhook-rebinding.test.ts @@ -0,0 +1,229 @@ +import type { LookupAddress } from "node:dns"; +import { createServer, request as httpRequest, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +// The name below answers with a public address the first time it is asked +// (the check) and a private one after that (the connection). The guard must +// dial only an address it has checked. +const dns = vi.hoisted(() => ({ + checkAnswers: [] as string[], + connectAnswers: [] as string[], + checkCalls: 0, + connectCalls: 0, +})); + +vi.mock("node:dns/promises", async (importOriginal) => { + const actual = await importOriginal(); + const lookup = async (hostname: string) => { + if (hostname !== "rebind.example") return actual.lookup(hostname); + dns.checkCalls += 1; + return dns.checkAnswers.map((address) => ({ + address, + family: address.includes(":") ? 6 : 4, + })); + }; + return { ...actual, default: { ...actual, lookup }, lookup }; +}); + +vi.mock("node:dns", async (importOriginal) => { + const actual = await importOriginal(); + const lookup = ( + hostname: string, + options: { all?: boolean } | number | undefined, + callback: ( + error: NodeJS.ErrnoException | null, + address: string | LookupAddress[], + family?: number, + ) => void, + ) => { + if (hostname !== "rebind.example") { + return (actual.lookup as (...args: unknown[]) => void)( + hostname, + options, + callback, + ); + } + dns.connectCalls += 1; + const answers = dns.connectAnswers.map((address) => ({ + address, + family: address.includes(":") ? 6 : 4, + })); + const all = typeof options === "object" && options?.all; + queueMicrotask(() => { + if (all) callback(null, answers); + else callback(null, answers[0]?.address ?? "", answers[0]?.family); + }); + }; + return { ...actual, default: { ...actual, lookup }, lookup }; +}); + +vi.mock("@cap/env", () => ({ + NODE_ENV: "production", + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + serverEnv: () => ({}), +})); +vi.mock("@cap/database", () => ({ db: () => ({}) })); + +const { sendDeveloperWebhook } = await import( + "@/lib/developer-webhook-delivery" +); +const { publicOnlyLookup } = await import("@/lib/s3-endpoint-policy"); +const { requestPublicOnly } = await import("@/lib/storage-fetch"); +const nodeDns = await import("node:dns"); + +const event = { + secret: "cws_0123456789abcdefghjkmnpqrstvwx", + eventId: "evt_1", + eventType: "video.ready", + body: JSON.stringify({ id: "evt_1", type: "video.ready" }), +}; + +let server: Server; +let port: number; +let hits = 0; + +beforeAll(async () => { + server = createServer((_req, res) => { + hits += 1; + res.end("internal"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + port = (server.address() as AddressInfo).port; +}); + +afterAll(async () => { + await new Promise((resolve) => server.close(resolve)); +}); + +beforeEach(() => { + hits = 0; + dns.checkCalls = 0; + dns.connectCalls = 0; + dns.checkAnswers = ["93.184.216.34"]; + dns.connectAnswers = ["127.0.0.1"]; +}); + +describe("a name that changes to a private address between check and connect", () => { + it("passes the check, then is refused at connect and nothing is sent", async () => { + const result = await sendDeveloperWebhook({ + ...event, + url: "https://rebind.example/cap", + }); + expect(dns.checkCalls).toBe(1); + expect(dns.connectCalls).toBe(1); + expect(result).toMatchObject({ ok: false, errorCode: "private_address" }); + expect(hits).toBe(0); + }); + + it.each([ + ["the metadata address", "169.254.169.254"], + ["a unique local v6 address", "fd00:ec2::254"], + ["a mapped v4 loopback", "::ffff:127.0.0.1"], + ["one private address among public ones", "93.184.216.34,10.0.0.5"], + ])("refuses %s at connect", async (_label, answers) => { + dns.connectAnswers = answers.split(","); + const result = await sendDeveloperWebhook({ + ...event, + url: "https://rebind.example/cap", + }); + expect(result).toMatchObject({ ok: false, errorCode: "private_address" }); + }); + + it("dials only the address the connection's own lookup checked", async () => { + // Without the guard, the rebound answer is what gets dialled: this + // plain request with the same name reaches the local server. + const unguarded = await new Promise((resolve, reject) => { + const req = httpRequest( + { + host: "rebind.example", + port, + path: "/", + lookup: nodeDns.lookup, + }, + (res) => { + res.resume(); + resolve(res.statusCode ?? 0); + }, + ); + req.on("error", reject); + req.end(); + }); + expect(unguarded).toBe(200); + expect(hits).toBe(1); + + await expect( + requestPublicOnly(`http://rebind.example:${port}/`, { + method: "POST", + body: "{}", + }), + ).rejects.toThrow("Refused to connect to private address 127.0.0.1"); + expect(hits).toBe(1); + }); + + it.each([ + ["a 6to4 AAAA", ["93.184.216.34", "2002:a00:1::1"]], + ["a Teredo AAAA", ["2001:0:4136:e378::1", "93.184.216.34"]], + ["a documentation address", ["203.0.113.9", "2606:4700::1111"]], + ])( + "hands the connection only the public address next to %s", + async (_label, answers) => { + dns.connectAnswers = answers; + const result = await new Promise<{ + error: NodeJS.ErrnoException | null; + address: unknown; + }>((resolve) => + publicOnlyLookup("rebind.example", { all: true }, (error, address) => + resolve({ error, address }), + ), + ); + expect(result.error).toBeNull(); + const dialled = (result.address as { address: string }[]).map( + (entry) => entry.address, + ); + expect(dialled).toEqual( + answers.filter((address) => + ["93.184.216.34", "2606:4700::1111"].includes(address), + ), + ); + }, + ); + + it("refuses a name whose only addresses are reserved", async () => { + dns.connectAnswers = ["2002:a00:1::1", "192.0.2.1"]; + const error = await new Promise((resolve) => + publicOnlyLookup("rebind.example", { all: true }, (lookupError) => + resolve(lookupError), + ), + ); + expect(error?.code).toBe("ECONNREFUSED"); + }); + + it("still refuses a public address next to a NAT64 form of a private one", async () => { + dns.connectAnswers = ["93.184.216.34", "64:ff9b::a00:1"]; + const error = await new Promise((resolve) => + publicOnlyLookup("rebind.example", { all: true }, (lookupError) => + resolve(lookupError), + ), + ); + expect(error?.code).toBe("ECONNREFUSED"); + }); + + it("checks every address the lookup returns, not only the first", async () => { + dns.connectAnswers = ["93.184.216.34", "127.0.0.1"]; + const error = await new Promise((resolve) => + publicOnlyLookup("rebind.example", { all: true }, (lookupError) => + resolve(lookupError), + ), + ); + expect(error?.code).toBe("ECONNREFUSED"); + }); +}); diff --git a/apps/web/__tests__/unit/developer-webhook-signature.test.ts b/apps/web/__tests__/unit/developer-webhook-signature.test.ts new file mode 100644 index 00000000000..627c157b131 --- /dev/null +++ b/apps/web/__tests__/unit/developer-webhook-signature.test.ts @@ -0,0 +1,122 @@ +import { createHmac } from "node:crypto"; +import { describe, expect, it } from "vitest"; +import { + WEBHOOK_MAX_ATTEMPTS, + WEBHOOK_RETRY_DELAYS_SECONDS, + webhookRetryDelaySeconds, +} from "@/lib/developer-webhook-events"; +import { + createWebhookSecret, + developerVideoEventId, + signWebhookBody, + verifyWebhookSignature, +} from "@/lib/developer-webhook-signature"; + +const secret = "cws_0123456789abcdefghjkmnpqrstvwx"; +const body = JSON.stringify({ id: "evt_1", type: "video.ready" }); +const at = 1_794_500_000; + +describe("webhook signature", () => { + it("signs the timestamp and the raw body with the endpoint secret", () => { + const expected = createHmac("sha256", secret) + .update(`${at}.${body}`) + .digest("hex"); + expect(signWebhookBody(secret, body, at)).toBe(`t=${at},v1=${expected}`); + }); + + it("accepts its own signature inside the five minute window", () => { + const header = signWebhookBody(secret, body, at); + for (const drift of [0, 299, -299]) { + expect( + verifyWebhookSignature({ + secret, + body, + header, + now: (at + drift) * 1000, + }), + ).toBe(true); + } + }); + + it("rejects a timestamp older than five minutes", () => { + const header = signWebhookBody(secret, body, at); + expect( + verifyWebhookSignature({ secret, body, header, now: (at + 301) * 1000 }), + ).toBe(false); + }); + + it("rejects a changed body, another secret and a moved timestamp", () => { + const header = signWebhookBody(secret, body, at); + const now = at * 1000; + expect( + verifyWebhookSignature({ secret, body: `${body} `, header, now }), + ).toBe(false); + expect( + verifyWebhookSignature({ secret: `${secret}x`, body, header, now }), + ).toBe(false); + expect( + verifyWebhookSignature({ + secret, + body, + header: header.replace(`t=${at}`, `t=${at + 1}`), + now, + }), + ).toBe(false); + }); + + it.each([ + null, + "", + "v1=abc", + `t=${at}`, + `t=${at},v1=zz`, + `t=soon,v1=${"a".repeat(64)}`, + ])("rejects the malformed header %j", (header) => { + expect( + verifyWebhookSignature({ secret, body, header, now: at * 1000 }), + ).toBe(false); + }); + + it("makes secrets with a clear prefix and enough randomness", () => { + const first = createWebhookSecret(); + expect(first).toMatch(/^cws_[0-9a-z]{30}$/); + expect(createWebhookSecret()).not.toBe(first); + }); +}); + +describe("webhook event ids", () => { + it("names the same event for the same video and type every time", () => { + const id = developerVideoEventId("video-1", "video.ready"); + expect(id).toMatch(/^evt_[0-9a-f]{24}$/); + expect(developerVideoEventId("video-1", "video.ready")).toBe(id); + }); + + it("names a different event for another type or another video", () => { + const id = developerVideoEventId("video-1", "video.ready"); + expect(developerVideoEventId("video-1", "video.errored")).not.toBe(id); + expect(developerVideoEventId("video-2", "video.ready")).not.toBe(id); + }); +}); + +describe("webhook retry schedule", () => { + it("makes eight attempts with growing gaps over about a day", () => { + expect(WEBHOOK_MAX_ATTEMPTS).toBe(8); + expect([...WEBHOOK_RETRY_DELAYS_SECONDS]).toEqual([ + 60, 300, 1_800, 7_200, 18_000, 28_800, 28_800, + ]); + const total = WEBHOOK_RETRY_DELAYS_SECONDS.reduce((a, b) => a + b, 0); + expect(total).toBeGreaterThan(22 * 3_600); + expect(total).toBeLessThan(26 * 3_600); + for (let i = 1; i < WEBHOOK_RETRY_DELAYS_SECONDS.length; i++) { + expect(WEBHOOK_RETRY_DELAYS_SECONDS[i]).toBeGreaterThanOrEqual( + WEBHOOK_RETRY_DELAYS_SECONDS[i - 1] ?? 0, + ); + } + }); + + it("has a wait after every attempt but the last", () => { + expect(webhookRetryDelaySeconds(1)).toBe(60); + expect(webhookRetryDelaySeconds(7)).toBe(28_800); + expect(webhookRetryDelaySeconds(8)).toBeNull(); + }); +}); diff --git a/apps/web/__tests__/unit/developer-webhooks-api.test.ts b/apps/web/__tests__/unit/developer-webhooks-api.test.ts new file mode 100644 index 00000000000..bae4eac7ffd --- /dev/null +++ b/apps/web/__tests__/unit/developer-webhooks-api.test.ts @@ -0,0 +1,538 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const context = vi.hoisted(() => ({ + fake: null as null | { + db: unknown; + rows: (table: object) => Record[]; + reset: () => void; + }, + resolved: ["93.184.216.34"] as string[], + request: null as + | null + | ((url: string, init: RequestInit) => Promise), +})); + +vi.mock("@cap/database", async () => { + const schema = await import("@cap/database/schema"); + const { createWebhookFakeDatabase, webhookEndpointDefaults } = await import( + "./developer-webhook-fake-db" + ); + const fake = createWebhookFakeDatabase( + new Map([[schema.developerWebhookEndpoints, webhookEndpointDefaults]]), + ); + context.fake = fake; + return { db: () => fake.db }; +}); +vi.mock("@cap/env", () => ({ + NODE_ENV: "production", + buildEnv: { + NEXT_PUBLIC_IS_CAP: "true", + NEXT_PUBLIC_WEB_URL: "https://cap.so", + }, + serverEnv: () => ({ DATABASE_ENCRYPTION_KEY: "ab".repeat(32) }), +})); +vi.mock("@/app/api/utils", async () => { + const { createMiddleware } = await import("hono/factory"); + const apps: Record = { + csk_app_a: "app-a", + csk_app_b: "app-b", + }; + return { + withDeveloperSecretAuth: createMiddleware(async (c, next) => { + const key = c.req.header("authorization")?.split(" ")[1] ?? ""; + const appId = apps[key]; + if (!appId) return c.json({ code: "invalid_secret_key" }, 401); + c.set("developerAppId", appId); + await next(); + }), + }; +}); +vi.mock("@/lib/s3-endpoint-policy", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + isBlockedS3Endpoint: (url: string) => + actual.isBlockedS3Endpoint(url, async () => context.resolved), + }; +}); +vi.mock("@/lib/storage-fetch", () => ({ + requestPublicOnly: (url: string, init: RequestInit) => { + if (!context.request) throw new Error("no request expected"); + return context.request(url, init); + }, +})); + +const { developerWebhookEndpoints } = await import("@cap/database/schema"); +const { decrypt } = await import("@cap/database/crypto"); +const { app } = await import("@/app/api/developer/v1/[...route]/webhooks"); +const { verifyWebhookSignature } = await import( + "@/lib/developer-webhook-signature" +); + +const A = { Authorization: "Bearer csk_app_a" }; +const B = { Authorization: "Bearer csk_app_b" }; + +const send = ( + method: string, + path: string, + headers: Record, + body?: unknown, +) => + app.request(path, { + method, + headers: { + ...headers, + ...(body === undefined ? {} : { "Content-Type": "application/json" }), + }, + body: body === undefined ? undefined : JSON.stringify(body), + }); + +async function create( + headers: Record = A, + body: Record = { url: "https://hooks.example.com/cap" }, +) { + const response = await send("POST", "/", headers, body); + return { + status: response.status, + body: (await response.json()) as { + data: Record & { id: string; secret: string }; + code?: string; + message?: string; + hint?: string; + docs?: string; + }, + }; +} + +const rows = () => context.fake?.rows(developerWebhookEndpoints) ?? []; + +beforeEach(() => { + context.fake?.reset(); + context.resolved = ["93.184.216.34"]; + context.request = null; +}); + +describe("creating a webhook endpoint", () => { + it("returns the signing secret once and stores it encrypted", async () => { + const created = await create(); + expect(created.status).toBe(200); + const { id, secret } = created.body.data; + expect(secret).toMatch(/^cws_[0-9a-z]{30}$/); + expect(created.body.data).toMatchObject({ + url: "https://hooks.example.com/cap", + events: ["video.ready", "video.errored"], + enabled: true, + disabledReason: null, + secretPrefix: secret.slice(0, 12), + lastDeliveryAt: null, + consecutiveFailures: 0, + }); + + const [row] = rows(); + expect(row?.encryptedSecret).not.toContain(secret); + expect(await decrypt(String(row?.encryptedSecret))).toBe(secret); + + for (const response of [ + await send("GET", `/${id}`, A), + await send("GET", "/", A), + await send("PATCH", `/${id}`, A, { enabled: false }), + await send("POST", `/${id}/test`, A), + ]) { + const text = await response.text(); + expect(text).not.toContain(secret); + expect(text).not.toContain("encryptedSecret"); + } + }); + + it("takes the events and enabled flag it is given", async () => { + const created = await create(A, { + url: "https://hooks.example.com/cap", + events: ["video.errored"], + enabled: false, + }); + expect(created.body.data).toMatchObject({ + events: ["video.errored"], + enabled: false, + }); + }); + + it.each([ + ["an unknown field", { url: "https://hooks.example.com/cap", name: "x" }], + ["a missing url", {}], + ["an unknown event", { url: "https://a.example/x", events: ["video.up"] }], + ["no events", { url: "https://a.example/x", events: [] }], + [ + "an event listed twice", + { url: "https://a.example/x", events: ["video.ready", "video.ready"] }, + ], + [ + "a secret of the caller's choosing", + { url: "https://a.example/x", secret: "s" }, + ], + ])("refuses %s", async (_label, body) => { + const created = await create(A, body); + expect(created.status).toBe(400); + expect(created.body.code).toBe("invalid_request"); + expect(rows()).toHaveLength(0); + }); + + it("explains an unknown or missing event in plain words", async () => { + const unknown = await create(A, { + url: "https://a.example/x", + events: ["video.ready", "video.updated"], + }); + expect(unknown.body.message).toBe( + 'Invalid request: events.1: Unknown event "video.updated". Use video.ready or video.errored.', + ); + const none = await create(A, { url: "https://a.example/x", events: [] }); + expect(none.body.message).toBe( + "Invalid request: events: events must list at least one of: video.ready, video.errored", + ); + expect(unknown.body.docs).toBe("https://cap.so/docs/api/rest-api"); + + const objectEvent = await create(A, { + url: "https://a.example/x", + events: [{ toString: null }], + }); + expect(objectEvent.status).toBe(400); + expect(objectEvent.body.message).toBe( + "Invalid request: events.0: Unknown event. Use video.ready or video.errored.", + ); + expect(rows()).toHaveLength(0); + }); + + it("links webhook errors to the webhooks guide", async () => { + const missing = await send("GET", "/wh_missing", A); + expect(await missing.json()).toMatchObject({ + code: "webhook_not_found", + docs: "https://cap.so/docs/api/webhooks", + }); + const refused = await create(A, { url: "http://hooks.example.com/cap" }); + expect(refused.body).toMatchObject({ + code: "invalid_webhook_url", + docs: "https://cap.so/docs/api/webhooks", + }); + }); + + it.each([ + ["http", "http://hooks.example.com/cap"], + ["a private address", "https://10.0.0.8/cap"], + ["the metadata address", "https://169.254.169.254/latest"], + ["a mapped loopback address", "https://[::ffff:127.0.0.1]/cap"], + ["credentials", "https://user:pass@hooks.example.com/cap"], + ["another port", "https://hooks.example.com:8080/cap"], + ])("refuses a URL with %s", async (_label, url) => { + const created = await create(A, { url }); + expect(created.status).toBe(400); + expect(created.body.code).toBe("invalid_webhook_url"); + expect(rows()).toHaveLength(0); + }); + + it("refuses a URL that is too long once encoded, on create and on update", async () => { + const accented = `https://hooks.example.com/${"é".repeat(400)}`; + const created = await create(A, { url: accented }); + expect(created.status).toBe(400); + expect(created.body.code).toBe("invalid_webhook_url"); + expect(rows()).toHaveLength(0); + + const { id } = (await create(A)).body.data; + const updated = await send("PATCH", `/${id}`, A, { url: accented }); + expect(updated.status).toBe(400); + expect(rows()[0]?.url).toBe("https://hooks.example.com/cap"); + }); + + it("refuses a name that resolves to a private address", async () => { + context.resolved = ["192.168.1.20"]; + const created = await create(); + expect(created.status).toBe(400); + expect(created.body.code).toBe("invalid_webhook_url"); + expect(rows()).toHaveLength(0); + }); + + it("stops at five endpoints per app, and counts each app on its own", async () => { + for (let i = 0; i < 5; i++) { + expect((await create(A)).status).toBe(200); + } + const sixth = await create(A); + expect(sixth.status).toBe(409); + expect(sixth.body.code).toBe("webhook_limit_reached"); + expect(rows()).toHaveLength(5); + + expect((await create(B)).status).toBe(200); + }); + + it("does not let simultaneous requests pass the limit", async () => { + const results = await Promise.all( + Array.from({ length: 8 }, () => create(A)), + ); + expect(results.filter((result) => result.status === 200)).toHaveLength(5); + expect(rows()).toHaveLength(5); + }); + + it("frees a place when an endpoint is deleted", async () => { + const ids: string[] = []; + for (let i = 0; i < 5; i++) ids.push((await create(A)).body.data.id); + expect((await send("DELETE", `/${ids[0]}`, A)).status).toBe(200); + expect((await create(A)).status).toBe(200); + }); +}); + +describe("reading and changing webhook endpoints", () => { + it("lists an app's endpoints newest first with pagination", async () => { + const first = (await create(A)).body.data.id; + const second = (await create(A)).body.data.id; + const list = await (await send("GET", "/?limit=1", A)).json(); + expect(list.data.map((row: { id: string }) => row.id)).toEqual([second]); + expect(list.pagination).toEqual({ limit: 1, offset: 0, hasMore: true }); + const next = await (await send("GET", "/?limit=1&offset=1", A)).json(); + expect(next.data.map((row: { id: string }) => row.id)).toEqual([first]); + expect(next.pagination.hasMore).toBe(false); + }); + + it("refuses an unknown query parameter on the list", async () => { + const response = await send("GET", "/?userId=u1", A); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: "unknown_parameter" }); + }); + + it("updates url, events and enabled", async () => { + const { id } = (await create(A)).body.data; + const response = await send("PATCH", `/${id}`, A, { + url: "https://other.example.com/hook", + events: ["video.ready"], + enabled: false, + }); + expect(response.status).toBe(200); + expect((await response.json()).data).toMatchObject({ + url: "https://other.example.com/hook", + events: ["video.ready"], + enabled: false, + }); + }); + + it("refuses an update with nothing in it, an unknown field or a bad URL", async () => { + const { id } = (await create(A)).body.data; + expect((await send("PATCH", `/${id}`, A, {})).status).toBe(400); + expect((await send("PATCH", `/${id}`, A, { secret: "x" })).status).toBe( + 400, + ); + const bad = await send("PATCH", `/${id}`, A, { + url: "https://127.0.0.1/x", + }); + expect(bad.status).toBe(400); + expect(await bad.json()).toMatchObject({ code: "invalid_webhook_url" }); + expect(rows()[0]?.url).toBe("https://hooks.example.com/cap"); + }); + + it("turns a disabled endpoint back on with a clean failure streak", async () => { + const { id } = (await create(A)).body.data; + Object.assign(rows()[0] ?? {}, { + enabled: false, + disabledReason: "delivery_failures", + consecutiveFailures: 31, + failingSince: new Date("2026-11-01T00:00:00Z"), + }); + const response = await send("PATCH", `/${id}`, A, { enabled: true }); + expect((await response.json()).data).toMatchObject({ + enabled: true, + disabledReason: null, + consecutiveFailures: 0, + }); + expect(rows()[0]?.failingSince).toBeNull(); + }); + + it("deletes an endpoint, and a second delete still succeeds", async () => { + const { id } = (await create(A)).body.data; + expect(await (await send("DELETE", `/${id}`, A)).json()).toEqual({ + success: true, + }); + expect((await send("GET", `/${id}`, A)).status).toBe(404); + expect((await send("DELETE", `/${id}`, A)).status).toBe(200); + expect((await send("GET", "/", A)).status).toBe(200); + expect((await (await send("GET", "/", A)).json()).data).toEqual([]); + }); +}); + +describe("one app cannot reach another app's endpoints", () => { + it("answers not found for every call made with another app's key", async () => { + const { id } = (await create(A)).body.data; + context.request = async () => new Response(null, { status: 200 }); + + const attempts = [ + await send("GET", `/${id}`, B), + await send("PATCH", `/${id}`, B, { enabled: false }), + await send("PATCH", `/${id}`, B, { url: "https://evil.example.com/x" }), + await send("POST", `/${id}/rotate-secret`, B), + await send("DELETE", `/${id}`, B), + await send("POST", `/${id}/test`, B), + ]; + for (const response of attempts) { + expect(response.status).toBe(404); + expect(await response.json()).toMatchObject({ + code: "webhook_not_found", + }); + } + + expect((await (await send("GET", "/", B)).json()).data).toEqual([]); + expect(rows()[0]).toMatchObject({ + id, + appId: "app-a", + url: "https://hooks.example.com/cap", + enabled: true, + deletedAt: null, + lastDeliveryAt: null, + }); + expect((await send("GET", `/${id}`, A)).status).toBe(200); + }); + + it("refuses a call with no valid key", async () => { + const { id } = (await create(A)).body.data; + for (const [method, path] of [ + ["GET", "/"], + ["GET", `/${id}`], + ["PATCH", `/${id}`], + ["DELETE", `/${id}`], + ["POST", `/${id}/rotate-secret`], + ["POST", `/${id}/test`], + ["POST", "/"], + ] as const) { + expect((await send(method, path, {})).status).toBe(401); + } + }); +}); + +describe("rotating the signing secret", () => { + it("returns a new secret once, stores it encrypted and signs with it from then on", async () => { + const { id, secret: oldSecret } = (await create(A)).body.data; + const response = await send("POST", `/${id}/rotate-secret`, A); + expect(response.status).toBe(200); + const { data } = await response.json(); + expect(data.secret).toMatch(/^cws_[0-9a-z]{30}$/); + expect(data.secret).not.toBe(oldSecret); + expect(data).toMatchObject({ id, secretPrefix: data.secret.slice(0, 12) }); + + const [row] = rows(); + expect(await decrypt(String(row?.encryptedSecret))).toBe(data.secret); + for (const later of [ + await send("GET", `/${id}`, A), + await send("GET", "/", A), + ]) { + const text = await later.text(); + expect(text).not.toContain(data.secret); + expect(text).not.toContain(oldSecret); + } + + const seen: RequestInit[] = []; + context.request = async (_url, init) => { + seen.push(init); + return new Response(null, { status: 204 }); + }; + await send("POST", `/${id}/test`, A); + const header = new Headers(seen[0]?.headers).get("cap-signature"); + const body = String(seen[0]?.body); + expect(verifyWebhookSignature({ secret: data.secret, body, header })).toBe( + true, + ); + expect(verifyWebhookSignature({ secret: oldSecret, body, header })).toBe( + false, + ); + }); + + it("keeps the endpoint's settings and delivery state", async () => { + const { id } = (await create(A)).body.data; + await send("PATCH", `/${id}`, A, { + url: "https://hooks.example.com/other", + events: ["video.errored"], + enabled: false, + }); + const before = rows()[0]; + await send("POST", `/${id}/rotate-secret`, A); + expect(rows()[0]).toMatchObject({ + url: before?.url, + events: before?.events, + enabled: false, + consecutiveFailures: before?.consecutiveFailures, + }); + }); + + it("answers not found for a deleted endpoint", async () => { + const { id } = (await create(A)).body.data; + await send("DELETE", `/${id}`, A); + const encrypted = rows()[0]?.encryptedSecret; + const response = await send("POST", `/${id}/rotate-secret`, A); + expect(response.status).toBe(404); + expect(rows()[0]?.encryptedSecret).toBe(encrypted); + }); +}); + +describe("sending a test event", () => { + it("sends one signed webhook.test event and reports what came back", async () => { + const { id, secret } = (await create(A)).body.data; + const seen: { url: string; init: RequestInit }[] = []; + context.request = async (url, init) => { + seen.push({ url, init }); + return new Response("never shown to the caller", { status: 200 }); + }; + + const response = await send("POST", `/${id}/test`, A); + expect(response.status).toBe(200); + const text = await response.text(); + expect(text).not.toContain("never shown"); + const { data } = JSON.parse(text); + expect(data).toMatchObject({ + type: "webhook.test", + delivered: true, + responseStatus: 200, + errorCode: null, + }); + expect(data.eventId).toMatch(/^evt_test_/); + + expect(seen).toHaveLength(1); + const headers = new Headers(seen[0]?.init.headers); + const body = String(seen[0]?.init.body); + expect(JSON.parse(body)).toMatchObject({ + id: data.eventId, + type: "webhook.test", + data: { webhookId: id }, + }); + expect(headers.get("cap-event-type")).toBe("webhook.test"); + expect( + verifyWebhookSignature({ + secret, + body, + header: headers.get("cap-signature"), + }), + ).toBe(true); + expect(rows()[0]).toMatchObject({ + lastDeliveryStatus: "succeeded", + lastResponseStatus: 200, + }); + }); + + it("reports a failing endpoint without counting it towards disabling", async () => { + const { id } = (await create(A)).body.data; + context.request = async () => new Response("boom", { status: 500 }); + const { data } = await (await send("POST", `/${id}/test`, A)).json(); + expect(data).toMatchObject({ + delivered: false, + responseStatus: 500, + errorCode: "http_error", + }); + expect(rows()[0]).toMatchObject({ + consecutiveFailures: 0, + failingSince: null, + lastDeliveryStatus: "failed", + }); + }); + + it("sends nothing when the URL now resolves to a private address", async () => { + const { id } = (await create(A)).body.data; + context.resolved = ["10.0.0.9"]; + const { data } = await (await send("POST", `/${id}/test`, A)).json(); + expect(data).toMatchObject({ + delivered: false, + errorCode: "private_address", + }); + }); +}); diff --git a/apps/web/__tests__/unit/developer-webhooks-auth.test.ts b/apps/web/__tests__/unit/developer-webhooks-auth.test.ts new file mode 100644 index 00000000000..4b0c3d87895 --- /dev/null +++ b/apps/web/__tests__/unit/developer-webhooks-auth.test.ts @@ -0,0 +1,175 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + queries: 0, + keyRows: [] as { appId: string }[], + store: { + createWebhookEndpoint: vi.fn(), + deleteWebhookEndpoint: vi.fn(), + getWebhookEndpoint: vi.fn(), + listWebhookEndpoints: vi.fn(), + rotateWebhookEndpointSecret: vi.fn(), + sendWebhookTestEvent: vi.fn(), + updateWebhookEndpoint: vi.fn(), + }, +})); + +vi.mock("@cap/database", () => { + const chain = { + select: () => { + mocks.queries += 1; + return chain; + }, + from: () => chain, + innerJoin: () => chain, + leftJoin: () => chain, + where: () => chain, + limit: async () => mocks.keyRows, + update: () => { + mocks.queries += 1; + return chain; + }, + set: () => chain, + catch: () => chain, + }; + return { db: () => chain }; +}); +vi.mock("@cap/database/auth/session", () => ({ getCurrentUser: vi.fn() })); +vi.mock("@cap/database/auth/domain-utils", () => ({ + isBlockedAccountEmail: () => false, +})); +vi.mock("@cap/database/auth/request-site", () => ({ + isCrossSiteCookieWrite: () => false, +})); +vi.mock("@cap/env", () => ({ + NODE_ENV: "production", + buildEnv: { NEXT_PUBLIC_WEB_URL: "https://cap.so" }, +})); +vi.mock("@/lib/developer-key-hash", () => ({ + hashKey: async (key: string) => `hash:${key}`, +})); +vi.mock("@/lib/developer-webhooks", () => ({ + ...mocks.store, + serializeWebhookEndpoint: (endpoint: unknown) => endpoint, + serializeWebhookTestResult: (eventId: string) => ({ eventId }), +})); + +const { app } = await import("@/app/api/developer/v1/[...route]/webhooks"); + +const ROUTES = [ + ["GET", "/"], + ["POST", "/"], + ["GET", "/wh-1"], + ["PATCH", "/wh-1"], + ["DELETE", "/wh-1"], + ["POST", "/wh-1/rotate-secret"], + ["POST", "/wh-1/test"], +] as const; + +const call = (method: string, path: string, headers: Record) => + app.request(path, { + method, + headers: { "Content-Type": "application/json", ...headers }, + body: + method === "POST" || method === "PATCH" + ? JSON.stringify({ url: "https://hooks.example.com/cap" }) + : undefined, + }); + +beforeEach(() => { + mocks.queries = 0; + mocks.keyRows = [{ appId: "app-1" }]; + for (const mock of Object.values(mocks.store)) mock.mockReset(); +}); + +describe("webhook routes and a bad key", () => { + it.each([ + ["no key", {}, 401, "invalid_secret_key"], + [ + "a public key", + { Authorization: "Bearer cpk_public_key" }, + 403, + "public_key_not_allowed", + ], + [ + "an upload token", + { Authorization: "Bearer cut_upload_token" }, + 401, + "invalid_secret_key", + ], + [ + "an agent token", + { Authorization: "Bearer cap_cli_token" }, + 401, + "invalid_secret_key", + ], + [ + "a signing secret", + { Authorization: "Bearer cws_signing_secret" }, + 401, + "invalid_secret_key", + ], + [ + "a malformed header", + { Authorization: "csk_valid_key" }, + 401, + "invalid_secret_key", + ], + ])( + "refuses %s without a single database query", + async (_label, headers, status, code) => { + for (const [method, path] of ROUTES) { + const response = await call(method, path, headers); + expect(response.status).toBe(status); + expect(await response.json()).toMatchObject({ code }); + } + expect(mocks.queries).toBe(0); + for (const mock of Object.values(mocks.store)) { + expect(mock).not.toHaveBeenCalled(); + } + }, + ); + + it("refuses an unknown or revoked secret key before reaching any endpoint", async () => { + mocks.keyRows = []; + for (const [method, path] of ROUTES) { + const response = await call(method, path, { + Authorization: "Bearer csk_revoked", + }); + expect(response.status).toBe(401); + } + expect(mocks.queries).toBe(ROUTES.length); + for (const mock of Object.values(mocks.store)) { + expect(mock).not.toHaveBeenCalled(); + } + }); + + it("passes the app that owns the key to every call", async () => { + mocks.keyRows = [{ appId: "app-from-key" }]; + mocks.store.listWebhookEndpoints.mockResolvedValue({ + endpoints: [], + hasMore: false, + }); + mocks.store.getWebhookEndpoint.mockResolvedValue(null); + const missing = { + ok: false, + status: 404, + code: "webhook_not_found", + message: "Webhook endpoint not found", + hint: "", + }; + mocks.store.createWebhookEndpoint.mockResolvedValue(missing); + mocks.store.updateWebhookEndpoint.mockResolvedValue(missing); + mocks.store.deleteWebhookEndpoint.mockResolvedValue(missing); + mocks.store.rotateWebhookEndpointSecret.mockResolvedValue(missing); + mocks.store.sendWebhookTestEvent.mockResolvedValue(missing); + + for (const [method, path] of ROUTES) { + await call(method, path, { Authorization: "Bearer csk_valid_key" }); + } + for (const mock of Object.values(mocks.store)) { + expect(mock).toHaveBeenCalledTimes(1); + expect(mock.mock.calls[0]?.[0]).toBe("app-from-key"); + } + }); +}); diff --git a/apps/web/__tests__/unit/docs-markdown.test.ts b/apps/web/__tests__/unit/docs-markdown.test.ts new file mode 100644 index 00000000000..b278ee5ad8d --- /dev/null +++ b/apps/web/__tests__/unit/docs-markdown.test.ts @@ -0,0 +1,156 @@ +import { NextRequest } from "next/server"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/database", () => ({ + db: () => ({ + select: () => ({ from: () => ({ where: async () => [] }) }), + }), +})); +vi.mock("@cap/database/schema", () => ({ organizations: {} })); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + serverEnv: () => ({ WEB_URL: "https://cap.so" }), +})); + +import { GET } from "@/app/docs-markdown/[...slug]/route"; +import { CAP_AGENT_PROMPT } from "@/data/agent-prompt"; +import { DOCS_MARKDOWN_ROUTE, docsMarkdownSlug } from "@/lib/agent-markdown"; +import { docToMarkdown } from "@/lib/docs-markdown"; +import { getAllDocs, getDocBySlug } from "@/utils/docs"; +import { proxy } from "../../proxy"; + +const getMarkdown = (slug: string, method = "GET") => + GET(new Request(`https://cap.so/docs/${slug}.md`, { method }), { + params: Promise.resolve({ slug: slug.split("/") }), + }); + +const proseLines = (markdown: string) => { + const lines: string[] = []; + let fence: string | null = null; + for (const line of markdown.split("\n")) { + const marker = /^\s*(`{3,}|~{3,})/.exec(line)?.[1]; + if (fence) { + if (marker && marker[0] === fence[0] && marker.length >= fence.length) + fence = null; + } else if (marker) fence = marker; + else lines.push(line); + } + return lines; +}; + +describe("docsMarkdownSlug", () => { + it.each([ + ["/docs/api/rest-api.md", "text/html", "api/rest-api"], + ["/docs/api/rest-api.mdx", null, "api/rest-api"], + ["/docs/api/rest-api", "text/markdown", "api/rest-api"], + ["/docs/api/rest-api/", "text/markdown", "api/rest-api"], + ["/docs", "text/markdown", "introduction"], + ["/docs.md", null, "introduction"], + ["/docs/api/rest-api", "text/html,*/*;q=0.8", null], + ["/docs/api/rest-api", null, null], + ["/pricing.md", null, null], + ["/docsearch", "text/markdown", null], + ])("%s with Accept %s gives %s", (path, accept, slug) => { + expect(docsMarkdownSlug(path, accept)).toBe(slug); + }); +}); + +describe("docToMarkdown", () => { + const docs = getAllDocs(); + + it.each(docs.map((doc) => [doc.slug, doc] as const))( + "turns %s into plain Markdown", + (_slug, doc) => { + const markdown = docToMarkdown(doc, "https://cap.so"); + expect(markdown.startsWith(`# ${doc.metadata.title}\n`)).toBe(true); + expect(markdown).not.toMatch(/^---\n/); + for (const line of proseLines(markdown)) { + expect(line).not.toMatch(/^\s*<\/?[A-Z]/); + expect(line).not.toMatch(/\]\(\/[^/]/); + } + }, + ); + + it("keeps every code block exactly as written", () => { + const doc = getDocBySlug("api/get-started"); + if (!doc) throw new Error("missing get-started doc"); + const markdown = docToMarkdown(doc, "https://cap.so"); + const blocks = doc.content.match(/```[\s\S]*?\n```/g) ?? []; + expect(blocks.length).toBeGreaterThan(5); + for (const block of blocks) expect(markdown).toContain(block); + }); + + it("shows the copyable prompt and warnings as text", () => { + const agents = getDocBySlug("agents"); + const selfHosting = getDocBySlug("self-hosting"); + if (!agents || !selfHosting) throw new Error("missing docs"); + expect(docToMarkdown(agents, "https://cap.so")).toContain(CAP_AGENT_PROMPT); + const hosting = docToMarkdown(selfHosting, "https://cap.so"); + expect(hosting).toContain( + "> **Security Notice for Production Deployments**", + ); + expect(hosting).toContain( + "[Watch the video](https://cap.so/embed/k310v2vf41d6ffp)", + ); + }); +}); + +describe("docs Markdown route", () => { + it("serves a docs page as Markdown", async () => { + const response = await getMarkdown("api/rest-api"); + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toBe( + "text/markdown; charset=utf-8", + ); + expect(response.headers.get("vary")).toBe("Accept"); + const body = await response.text(); + expect(body.startsWith("# REST API\n")).toBe(true); + expect(body).toContain("every 3 seconds"); + expect(body).not.toContain(" { + const response = await getMarkdown("api/nope"); + expect(response.status).toBe(404); + expect(response.headers.get("content-type")).toBe( + "text/markdown; charset=utf-8", + ); + expect(await response.text()).toContain("/docs/api/nope"); + }); + + it("refuses a path outside the docs folder", async () => { + const response = await getMarkdown("../../package"); + expect(response.status).toBe(404); + }); +}); + +describe("proxy routes docs Markdown requests", () => { + const send = (path: string, accept: string, method = "GET") => + proxy( + new NextRequest(`https://cap.so${path}`, { + method, + headers: { accept }, + }), + ); + const rewriteOf = (response: Response) => + response.headers.get("x-middleware-rewrite"); + + it.each([ + ["/docs/api/get-started", "text/markdown"], + ["/docs/api/get-started.md", "text/html,*/*;q=0.8"], + ])("rewrites %s (Accept %s)", async (path, accept) => { + const response = await send(path, accept); + expect(rewriteOf(response)).toBe( + `https://cap.so${DOCS_MARKDOWN_ROUTE}/api/get-started`, + ); + }); + + it("leaves the HTML page alone for a browser", async () => { + const response = await send( + "/docs/api/get-started", + "text/html,application/xhtml+xml,*/*;q=0.8", + ); + expect(rewriteOf(response)).toBeNull(); + expect(response.headers.get("x-middleware-next")).toBe("1"); + }); +}); diff --git a/apps/web/__tests__/unit/faststart-remux.test.ts b/apps/web/__tests__/unit/faststart-remux.test.ts new file mode 100644 index 00000000000..d9506842885 --- /dev/null +++ b/apps/web/__tests__/unit/faststart-remux.test.ts @@ -0,0 +1,509 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ after: vi.fn() })); + +vi.mock("next/server", () => ({ after: mocks.after })); + +const { FASTSTART_REMUX_RETRY, startFaststartRemux } = await import( + "@/lib/faststart-remux" +); + +const request = { + videoId: "video-1", + userId: "user-1", + videoUrl: "https://storage.example/get/raw", + outputPresignedUrl: "https://storage.example/put/raw", + previewGifPresignedUrl: "https://storage.example/put/gif", +}; + +const answer = (status: number, body = "", headers?: HeadersInit) => + new Response(body, { status, headers }); + +const unreachable = (code: string) => + new TypeError("fetch failed", { + cause: Object.assign(new Error(code), { code }), + }); + +describe("faststart remux dispatch", () => { + const fetchMock = vi.fn(); + const settled = vi.fn(); + let deferred: Promise[] = []; + const start = (overrides: { webhookSecret?: string } = {}) => + startFaststartRemux({ + mediaServerUrl: "https://media.example", + webhookSecret: + "webhookSecret" in overrides ? overrides.webhookSecret : "secret", + request, + onRetriesSettled: settled, + retryDelaysMs: [1, 1, 1], + defer: (work) => { + deferred.push(work); + }, + }); + + beforeEach(() => { + vi.clearAllMocks(); + deferred = []; + settled.mockResolvedValue(undefined); + vi.stubGlobal("fetch", fetchMock); + vi.spyOn(console, "error").mockImplementation(() => undefined); + vi.spyOn(console, "warn").mockImplementation(() => undefined); + }); + + afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + }); + + it("sends one request, the same one as before, when the media server accepts it", async () => { + fetchMock.mockResolvedValue(answer(200, "{}")); + + expect(await start()).toBe("queued"); + + expect(fetchMock.mock.calls).toEqual([ + [ + "https://media.example/video/process", + { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-media-server-secret": "secret", + }, + body: JSON.stringify({ + videoId: "video-1", + userId: "user-1", + videoUrl: "https://storage.example/get/raw", + outputPresignedUrl: "https://storage.example/put/raw", + previewGifPresignedUrl: "https://storage.example/put/gif", + remuxOnly: true, + }), + }, + ], + ]); + expect(deferred).toHaveLength(0); + expect(mocks.after).not.toHaveBeenCalled(); + expect(settled).not.toHaveBeenCalled(); + expect(console.error).not.toHaveBeenCalled(); + }); + + it("leaves the secret header out when no secret is configured, as before", async () => { + fetchMock.mockResolvedValue(answer(200, "{}")); + + await start({ webhookSecret: undefined }); + + expect(fetchMock.mock.calls[0]?.[1].headers).toEqual({ + "Content-Type": "application/json", + }); + }); + + it("answers straight away when the media server is busy and queues the remux once it has room", async () => { + fetchMock + .mockResolvedValueOnce(answer(503, "SERVER_BUSY")) + .mockResolvedValueOnce(answer(503, "SERVER_BUSY")) + .mockResolvedValue(answer(200, "{}")); + + expect(await start()).toBe("retrying"); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(deferred).toHaveLength(1); + + await deferred[0]; + + expect(fetchMock).toHaveBeenCalledTimes(3); + expect(settled.mock.calls).toEqual([[true]]); + expect(fetchMock.mock.calls[2]?.[0]).toBe( + "https://media.example/video/process", + ); + expect(JSON.parse(fetchMock.mock.calls[2]?.[1].body)).toMatchObject({ + videoId: "video-1", + remuxOnly: true, + }); + }); + + it("retries when the media server cannot be reached", async () => { + fetchMock + .mockRejectedValueOnce(unreachable("ECONNREFUSED")) + .mockResolvedValue(answer(200, "{}")); + + expect(await start()).toBe("retrying"); + await deferred[0]; + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(settled.mock.calls).toEqual([[true]]); + }); + + it("gives up after three more tries and says so, so transcription can go ahead without the remux", async () => { + fetchMock.mockImplementation(async () => answer(503, "SERVER_BUSY")); + + expect(await start()).toBe("retrying"); + await deferred[0]; + + expect(fetchMock).toHaveBeenCalledTimes(4); + expect(settled.mock.calls).toEqual([[false]]); + const retryLines = vi + .mocked(console.warn) + .mock.calls.filter(([line]) => line === "[faststart-remux] retrying"); + expect(retryLines.map(([, detail]) => detail)).toEqual([ + expect.objectContaining({ videoId: "video-1", attempt: 2 }), + expect.objectContaining({ videoId: "video-1", attempt: 3 }), + expect.objectContaining({ videoId: "video-1", attempt: 4 }), + ]); + const gaveUp = vi + .mocked(console.error) + .mock.calls.filter(([line]) => line === "[faststart-remux] gave up"); + expect(gaveUp).toEqual([ + [ + "[faststart-remux] gave up", + expect.objectContaining({ videoId: "video-1", reason: "status 503" }), + ], + ]); + }); + + it("retries a rate limited answer", async () => { + fetchMock + .mockResolvedValueOnce(answer(429)) + .mockResolvedValue(answer(200, "{}")); + + expect(await start()).toBe("retrying"); + await deferred[0]; + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(settled.mock.calls).toEqual([[true]]); + }); + + it("does not retry an internal error, which would be the same again", async () => { + fetchMock.mockResolvedValue(answer(500, "boom")); + + expect(await start()).toBe("failed"); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(deferred).toHaveLength(0); + expect(console.warn).not.toHaveBeenCalled(); + }); + + it.each([ + ["a gateway 502", () => answer(502)], + ["a gateway 504", () => answer(504)], + ])( + "does not retry %s, since the job may have been taken", + async (_, reply) => { + fetchMock.mockImplementation(async () => reply()); + + expect(await start()).toBe("failed"); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(deferred).toHaveLength(0); + }, + ); + + it("retries when every address refused the connection", async () => { + fetchMock + .mockRejectedValueOnce( + new TypeError("fetch failed", { + cause: new AggregateError([ + Object.assign(new Error("v6"), { code: "ECONNREFUSED" }), + Object.assign(new Error("v4"), { code: "EHOSTUNREACH" }), + ]), + }), + ) + .mockResolvedValue(answer(200, "{}")); + + expect(await start()).toBe("retrying"); + await deferred[0]; + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(settled.mock.calls).toEqual([[true]]); + }); + + it("does not retry a connection reset, since the job may have been taken", async () => { + fetchMock + .mockRejectedValueOnce(unreachable("ECONNRESET")) + .mockResolvedValue(answer(200, "{}")); + + expect(await start()).toBe("failed"); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(deferred).toHaveLength(0); + }); + + it("stops after a retry whose connection was reset", async () => { + fetchMock + .mockResolvedValueOnce(answer(503)) + .mockRejectedValueOnce(unreachable("ECONNRESET")) + .mockResolvedValue(answer(200, "{}")); + + expect(await start()).toBe("retrying"); + await deferred[0]; + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(settled.mock.calls).toEqual([[false]]); + expect(console.error).toHaveBeenCalledWith( + "[faststart-remux] gave up", + expect.objectContaining({ reason: "ECONNRESET" }), + ); + }); + + it("waits at least as long as the media server asks before retrying", async () => { + fetchMock + .mockResolvedValueOnce(answer(503, "SERVER_BUSY", { "Retry-After": "1" })) + .mockResolvedValue(answer(200, "{}")); + + const startedAt = Date.now(); + expect(await start()).toBe("retrying"); + await deferred[0]; + + expect(Date.now() - startedAt).toBeGreaterThanOrEqual(1000); + expect(fetchMock).toHaveBeenCalledTimes(2); + const [, detail] = + vi + .mocked(console.warn) + .mock.calls.find(([line]) => line === "[faststart-remux] retrying") ?? + []; + expect(detail.waitMs).toBeGreaterThanOrEqual(1000); + expect(detail.waitMs).toBeLessThanOrEqual(1250); + }); + + it("never sends again after a retry that timed out, since it may have been taken", async () => { + fetchMock + .mockResolvedValueOnce(answer(503, "SERVER_BUSY")) + .mockRejectedValueOnce( + new DOMException("The operation timed out.", "TimeoutError"), + ) + .mockResolvedValue(answer(200, "{}")); + + expect(await start()).toBe("retrying"); + await deferred[0]; + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(settled.mock.calls).toEqual([[false]]); + expect(console.error).toHaveBeenCalledWith( + "[faststart-remux] gave up", + expect.objectContaining({ reason: "TimeoutError" }), + ); + }); + + it("gives each retry its own timeout and the first attempt none, as before", async () => { + fetchMock + .mockResolvedValueOnce(answer(503)) + .mockResolvedValue(answer(200, "{}")); + + await start(); + await deferred[0]; + + expect(fetchMock.mock.calls[0]?.[1].signal).toBeUndefined(); + expect(fetchMock.mock.calls[1]?.[1].signal).toBeInstanceOf(AbortSignal); + }); + + it("does not retry a request the media server refused", async () => { + fetchMock.mockResolvedValue(answer(400, "bad request")); + + expect(await start()).toBe("failed"); + + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(deferred).toHaveLength(0); + expect(settled).not.toHaveBeenCalled(); + expect(console.error).toHaveBeenCalledWith( + "Failed to queue faststart remux:", + expect.objectContaining({ + message: "Media server remux failed: 400 bad request", + }), + ); + }); + + it("stops retrying when a later answer is a refusal", async () => { + fetchMock + .mockResolvedValueOnce(answer(503)) + .mockResolvedValue(answer(422, "unreadable")); + + expect(await start()).toBe("retrying"); + await deferred[0]; + + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(settled.mock.calls).toEqual([[false]]); + }); + + it("behaves as it did before when nothing can run after the response", async () => { + fetchMock.mockResolvedValue(answer(503, "SERVER_BUSY")); + mocks.after.mockImplementation(() => { + throw new Error("after() called outside a request scope"); + }); + + expect( + await startFaststartRemux({ + mediaServerUrl: "https://media.example", + webhookSecret: "secret", + request, + onRetriesSettled: settled, + }), + ).toBe("failed"); + + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(settled).not.toHaveBeenCalled(); + }); + + it("hands the retries to the platform so they outlive the response", async () => { + fetchMock + .mockResolvedValueOnce(answer(503)) + .mockResolvedValue(answer(200, "{}")); + + expect( + await startFaststartRemux({ + mediaServerUrl: "https://media.example", + webhookSecret: "secret", + request, + onRetriesSettled: settled, + retryDelaysMs: [1], + }), + ).toBe("retrying"); + + expect(mocks.after).toHaveBeenCalledTimes(1); + await mocks.after.mock.calls[0]?.[0]; + expect(settled.mock.calls).toEqual([[true]]); + }); + + it("does not let a failure while settling escape into the platform", async () => { + fetchMock + .mockResolvedValueOnce(answer(503)) + .mockResolvedValue(answer(200, "{}")); + settled.mockRejectedValue(new Error("database unavailable")); + + expect(await start()).toBe("retrying"); + + await expect(deferred[0]).resolves.toBeUndefined(); + }); + describe("retry budget", () => { + const startWithDefaults = () => + startFaststartRemux({ + mediaServerUrl: "https://media.example", + webhookSecret: "secret", + request, + onRetriesSettled: settled, + defer: (work) => { + deferred.push(work); + }, + }); + + const hangUntilAborted = (_url: string, init: RequestInit) => + new Promise((_, reject) => { + init.signal?.addEventListener("abort", () => + reject(init.signal?.reason), + ); + }); + + const busyFor = (seconds: string) => () => + Promise.resolve(answer(503, "SERVER_BUSY", { "Retry-After": seconds })); + + const retryWaits = () => + vi + .mocked(console.warn) + .mock.calls.filter(([line]) => line === "[faststart-remux] retrying") + .map(([, detail]) => detail.waitMs as number); + + let settledAt: number | null; + + beforeEach(() => { + vi.useFakeTimers(); + settledAt = null; + settled.mockImplementation(async () => { + settledAt = Date.now(); + }); + }); + + it("uses a 10 s attempt timeout, a 20 s Retry-After cap and a 60 s budget", () => { + expect(FASTSTART_REMUX_RETRY).toMatchObject({ + attemptTimeoutMs: 10_000, + maxRetryAfterMs: 20_000, + budgetMs: 60_000, + }); + }); + + it("times a retry out at 10 s and does not send it again", async () => { + vi.spyOn(Math, "random").mockReturnValue(0.5); + fetchMock + .mockResolvedValueOnce(answer(503, "SERVER_BUSY")) + .mockImplementation(hangUntilAborted); + + expect(await startWithDefaults()).toBe("retrying"); + await vi.advanceTimersByTimeAsync(2_000); + expect(fetchMock).toHaveBeenCalledTimes(2); + const signal: AbortSignal = fetchMock.mock.calls[1]?.[1].signal; + + await vi.advanceTimersByTimeAsync(9_999); + expect(signal.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(signal.aborted).toBe(true); + expect(signal.reason.name).toBe("TimeoutError"); + + await deferred[0]; + expect(fetchMock).toHaveBeenCalledTimes(2); + expect(settled.mock.calls).toEqual([[false]]); + }); + + it("caps a large Retry-After at 20 s", async () => { + vi.spyOn(Math, "random").mockReturnValue(0); + fetchMock + .mockImplementationOnce(busyFor("3600")) + .mockResolvedValue(answer(200, "{}")); + + expect(await startWithDefaults()).toBe("retrying"); + await vi.advanceTimersByTimeAsync(19_999); + expect(fetchMock).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + expect(fetchMock).toHaveBeenCalledTimes(2); + + await deferred[0]; + expect(retryWaits()).toEqual([20_000]); + expect(settled.mock.calls).toEqual([[true]]); + }); + + it.each([ + ["the least jitter", 0], + ["the most jitter", 0.999_999], + ])( + "gives up inside the budget with a Retry-After of 3600 and %s, so transcription is queued", + async (_, random) => { + vi.spyOn(Math, "random").mockReturnValue(random); + fetchMock.mockImplementation(busyFor("3600")); + const firstFailedAt = Date.now(); + + expect(await startWithDefaults()).toBe("retrying"); + await vi.advanceTimersByTimeAsync(FASTSTART_REMUX_RETRY.budgetMs); + await deferred[0]; + + expect(settled.mock.calls).toEqual([[false]]); + expect(settledAt).not.toBeNull(); + expect((settledAt ?? 0) - firstFailedAt).toBeLessThanOrEqual( + FASTSTART_REMUX_RETRY.budgetMs, + ); + for (const wait of retryWaits()) { + expect(wait).toBeGreaterThanOrEqual(20_000); + expect(wait).toBeLessThanOrEqual(25_000); + } + expect(console.error).toHaveBeenCalledWith( + "[faststart-remux] gave up", + expect.objectContaining({ outOfTime: true, reason: "status 503" }), + ); + }, + ); + + it("stays inside the budget when the last allowed retry hangs until its timeout", async () => { + vi.spyOn(Math, "random").mockReturnValue(0.999_999); + fetchMock + .mockImplementationOnce(busyFor("20")) + .mockImplementationOnce(busyFor("20")) + .mockImplementation(hangUntilAborted); + const firstFailedAt = Date.now(); + + expect(await startWithDefaults()).toBe("retrying"); + await vi.advanceTimersByTimeAsync(FASTSTART_REMUX_RETRY.budgetMs); + await deferred[0]; + + expect(fetchMock).toHaveBeenCalledTimes(3); + expect(settled.mock.calls).toEqual([[false]]); + expect((settledAt ?? 0) - firstFailedAt).toBeLessThanOrEqual( + FASTSTART_REMUX_RETRY.budgetMs, + ); + }); + }); +}); diff --git a/apps/web/__tests__/unit/folders-dropdown-permissions.test.ts b/apps/web/__tests__/unit/folders-dropdown-permissions.test.ts new file mode 100644 index 00000000000..a72d58d9a11 --- /dev/null +++ b/apps/web/__tests__/unit/folders-dropdown-permissions.test.ts @@ -0,0 +1,104 @@ +// @vitest-environment jsdom + +import React, { act, createRef } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +type Children = { children?: React.ReactNode }; + +vi.mock("@cap/ui", () => ({ + DropdownMenu: ({ children }: Children) => + React.createElement("div", null, children), + DropdownMenuTrigger: ({ children }: Children) => + React.createElement("div", null, children), + DropdownMenuContent: ({ children }: Children) => + React.createElement("div", { role: "menu" }, children), + DropdownMenuItem: ({ children }: Children) => + React.createElement("div", { role: "menuitem" }, children), +})); +vi.mock("@/components/FontAwesomeIcon", () => ({ + FontAwesomeIcon: () => null, +})); + +import { FoldersDropdown } from "@/app/(org)/dashboard/caps/components/FoldersDropdown"; + +let root: Root; +let container: HTMLDivElement; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); +}); + +const baseProps = { + id: "folder-1", + setIsRenaming: () => {}, + setConfirmDeleteFolderOpen: () => {}, + nameRef: createRef(), + onPublicToggle: () => {}, + onCopyPublicLink: () => {}, + onMove: () => {}, +}; + +const renderMenu = async (props: { + canEdit: boolean; + canMove: boolean; + public: boolean; +}) => { + await act(async () => + root.render( + React.createElement(FoldersDropdown, { ...baseProps, ...props }), + ), + ); + return { + trigger: container.querySelector("button"), + items: [...container.querySelectorAll('[role="menuitem"]')].map((item) => + item.textContent?.trim(), + ), + }; +}; + +describe("folder card menu", () => { + it("offers every action to someone who can edit the folder", async () => { + const { items } = await renderMenu({ + canEdit: true, + canMove: true, + public: true, + }); + + expect(items).toEqual([ + "Rename", + "Make private", + "Copy public link", + "Move", + "Delete", + ]); + }); + + it("only offers the public link to a member who cannot edit a public folder", async () => { + const { items } = await renderMenu({ + canEdit: false, + canMove: false, + public: true, + }); + + expect(items).toEqual(["Copy public link"]); + }); + + it("shows no menu to a member who cannot edit a private folder", async () => { + const { trigger, items } = await renderMenu({ + canEdit: false, + canMove: false, + public: false, + }); + + expect(trigger).toBeNull(); + expect(items).toEqual([]); + }); +}); diff --git a/apps/web/__tests__/unit/homepage-seo.test.ts b/apps/web/__tests__/unit/homepage-seo.test.ts index 82e6735a817..72a69c9238e 100644 --- a/apps/web/__tests__/unit/homepage-seo.test.ts +++ b/apps/web/__tests__/unit/homepage-seo.test.ts @@ -14,6 +14,7 @@ import { } from "@/components/pages/HomeTwo/metadata"; import { HomeTwoSchema } from "@/components/pages/HomeTwo/Schema"; import { homepageSchema, homepageSeo } from "@/components/pages/HomeTwo/seo"; +import { homepageCopy } from "@/data/homepage-copy"; import { PRICING } from "@/data/pricing"; describe("homepage SEO", () => { @@ -51,10 +52,22 @@ describe("homepage SEO", () => { const serialized = JSON.stringify(homepageSchema); expect(serialized).not.toContain("aggregateRating"); expect(serialized).not.toContain("reviewRating"); - expect(serialized).not.toContain("FAQPage"); expect(serialized).not.toContain("priceValidUntil"); }); + it("marks up only the FAQ the homepage shows", () => { + const faq = homepageSchema["@graph"].find( + (entity) => entity["@type"] === "FAQPage", + ); + expect(faq?.mainEntity).toEqual( + homepageCopy.faq.items.map((item) => ({ + "@type": "Question", + name: item.question, + acceptedAnswer: { "@type": "Answer", text: item.answer }, + })), + ); + }); + it("uses the displayed plan prices and supported desktop platforms", () => { const software = homepageSchema["@graph"].find( (entity) => entity["@type"] === "SoftwareApplication", diff --git a/apps/web/__tests__/unit/infra-access.test.ts b/apps/web/__tests__/unit/infra-access.test.ts index 2c07bbc57dd..3e0604a331e 100644 --- a/apps/web/__tests__/unit/infra-access.test.ts +++ b/apps/web/__tests__/unit/infra-access.test.ts @@ -1,35 +1,17 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; - -const env = vi.hoisted(() => ({ - NEXT_PUBLIC_IS_CAP: "true" as string | undefined, -})); - -vi.mock("@cap/env", () => ({ buildEnv: env })); +import { describe, expect, it } from "vitest"; import { canUseInfra } from "@/lib/infra-access"; -describe("Cap Infra preview access", () => { - beforeEach(() => { - env.NEXT_PUBLIC_IS_CAP = "true"; - }); - - it("lets a preview account in, whatever the letter case", () => { +describe("Cap Infra access", () => { + it("is open to every signed-in account", () => { expect(canUseInfra("richie@cap.so")).toBe(true); - expect(canUseInfra("Richie@Cap.so")).toBe(true); - }); - - it("keeps everyone else out on Cap's own deployment", () => { - expect(canUseInfra("someone@example.com")).toBe(false); - expect(canUseInfra("richie@cap.so.example.com")).toBe(false); - expect(canUseInfra("")).toBe(false); - expect(canUseInfra(null)).toBe(false); - expect(canUseInfra(undefined)).toBe(false); + expect(canUseInfra("someone@example.com")).toBe(true); + expect(canUseInfra("Someone@Example.com")).toBe(true); }); - it("stays open on self-hosted deployments", () => { - for (const value of ["false", "", undefined]) { - env.NEXT_PUBLIC_IS_CAP = value; - expect(canUseInfra("someone@example.com")).toBe(true); - } + it("reports open access when asked without an account", () => { + expect(canUseInfra(null)).toBe(true); + expect(canUseInfra(undefined)).toBe(true); + expect(canUseInfra("")).toBe(true); }); }); diff --git a/apps/web/__tests__/unit/infra-agent-prompt.test.ts b/apps/web/__tests__/unit/infra-agent-prompt.test.ts index f542379374d..713d19baf87 100644 --- a/apps/web/__tests__/unit/infra-agent-prompt.test.ts +++ b/apps/web/__tests__/unit/infra-agent-prompt.test.ts @@ -1,8 +1,12 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { CAP_INFRA_AGENT_PROMPT } from "@/data/infra-agent-prompt"; +import { + CAP_INFRA_AGENT_PROMPT, + CAP_INFRA_MCP_URL, +} from "@/data/infra-agent-prompt"; import { developerOpenApiSpec } from "@/lib/developer-openapi"; +import { DEVELOPER_KEY_REQUESTS_PER_MINUTE } from "@/lib/developer-rate-limits"; const METHODS = ["get", "post", "put", "patch", "delete"] as const; @@ -30,7 +34,9 @@ describe("Cap Infra agent prompt", () => { ); } const pathMentions = - CAP_INFRA_AGENT_PROMPT.match(/\/api\/developer\//g) ?? []; + CAP_INFRA_AGENT_PROMPT.replaceAll(CAP_INFRA_MCP_URL, "").match( + /\/api\/developer\//g, + ) ?? []; expect(pathMentions).toHaveLength(calls.length); }); @@ -55,22 +61,34 @@ describe("Cap Infra agent prompt", () => { ]) { expect(named(CAP_INFRA_AGENT_PROMPT)).toContain(call); } - expect(CAP_INFRA_AGENT_PROMPT).toContain("every 2 seconds"); + expect(CAP_INFRA_AGENT_PROMPT).toContain( + "/videos/{id}/status every 3 seconds", + ); + expect(CAP_INFRA_AGENT_PROMPT).toContain( + "/transcripts/{id} every 5 seconds", + ); expect(CAP_INFRA_AGENT_PROMPT).toContain("Retry-After"); - expect(CAP_INFRA_AGENT_PROMPT).toContain("60 requests a minute"); + expect(CAP_INFRA_AGENT_PROMPT).toContain( + `${DEVELOPER_KEY_REQUESTS_PER_MINUTE} requests a minute`, + ); }); - it("links only to the API reference", () => { + it("links only to the API reference and the hosted MCP server", () => { const links = CAP_INFRA_AGENT_PROMPT.match(/https?:\/\/[^\s)]+/g) ?? []; expect([...new Set(links)].sort()).toEqual([ + "https://cap.so/api/developer/mcp", "https://cap.so/docs/api/rest-api", "https://cap.so/openapi.json", ]); }); - it("never asks for the Cap CLI, an MCP server or a pasted key", () => { + it("never asks for the Cap CLI, a local MCP server or a pasted key", () => { expect(CAP_INFRA_AGENT_PROMPT).not.toMatch(/\bcap [a-z]/); - expect(CAP_INFRA_AGENT_PROMPT).not.toMatch(/\bCLI\b|\bMCP\b|curl|install/i); + expect(CAP_INFRA_AGENT_PROMPT).not.toMatch(/\bCLI\b|curl|install|npx/i); + expect(CAP_INFRA_AGENT_PROMPT.match(/\bMCP\b/g)).toEqual(["MCP", "MCP"]); + expect(CAP_INFRA_AGENT_PROMPT).toContain( + "hosted MCP server at https://cap.so/api/developer/mcp", + ); expect(CAP_INFRA_AGENT_PROMPT).not.toContain("cpk_"); expect(CAP_INFRA_AGENT_PROMPT).toContain( "Never ask me to paste the key into the chat", @@ -85,7 +103,7 @@ describe("Cap Infra agent prompt", () => { ); }); - it("is introduced in the console without the Cap CLI or an MCP server", () => { + it("is introduced in the console without the Cap CLI", () => { const source = readFileSync( join( process.cwd(), @@ -93,7 +111,7 @@ describe("Cap Infra agent prompt", () => { ), "utf8", ); - expect(source).not.toMatch(/\bCLI\b|\bMCP\b/); + expect(source).not.toMatch(/\bCLI\b/); expect(source).toContain("secret key"); }); }); diff --git a/apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts b/apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts new file mode 100644 index 00000000000..551c6d50d96 --- /dev/null +++ b/apps/web/__tests__/unit/infra-auto-top-up-sweep.test.ts @@ -0,0 +1,256 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +type Account = { id: string; lastAttemptAt: Date | null; waiting?: boolean }; + +const state = vi.hoisted(() => ({ + accounts: [] as { + id: string; + lastAttemptAt: Date | null; + waiting?: boolean; + }[], + pages: [] as { + sql: string; + params: unknown[]; + after: { key: string; id: string } | null; + limit: number; + }[], + reads: [] as string[], + msPerAccount: 0, + other: [] as { sql: string; params: unknown[] }[], +})); + +vi.mock("@cap/database", async () => { + const { drizzle } = await import("drizzle-orm/mysql-proxy"); + const NEVER = "1970-01-01 00:00:00"; + const keyOf = (account: { lastAttemptAt: Date | null }) => + account.lastAttemptAt + ? account.lastAttemptAt.toISOString().slice(0, 19).replace("T", " ") + : NEVER; + const client = drizzle(async (sql, params) => { + if (sql.startsWith("select `developer_credit_accounts`.`id`, cast(")) { + const limit = Number(params.at(-1)); + const keyset = sql.includes("and `developer_credit_accounts`.`id` > ?))"); + const after = keyset + ? { key: String(params.at(-4)), id: String(params.at(-2)) } + : null; + state.pages.push({ sql, params, after, limit }); + const rows = state.accounts + .filter((account) => !account.waiting) + .map((account) => ({ id: account.id, key: keyOf(account) })) + .filter( + (row) => + !after || + row.key > after.key || + (row.key === after.key && row.id > after.id), + ) + .sort((a, b) => + a.key === b.key ? a.id.localeCompare(b.id) : a.key < b.key ? -1 : 1, + ) + .slice(0, limit); + return { rows: rows.map((row) => [row.id, row.key]) }; + } + if (sql.startsWith("select `developer_credit_accounts`.`id` from")) { + const limit = Number(params.at(-1)); + const cursor = sql.includes("`id` > ?") ? String(params.at(-2)) : null; + state.pages.push({ + sql, + params, + after: cursor ? { key: "", id: cursor } : null, + limit, + }); + return { + rows: state.accounts + .filter((account) => cursor === null || account.id > cursor) + .slice(0, limit) + .map((account) => [account.id]), + }; + } + if (sql.includes("`developer_apps`.`name`")) { + state.reads.push(String(params[0])); + vi.setSystemTime(Date.now() + state.msPerAccount); + return { rows: [] }; + } + state.other.push({ sql, params }); + return { rows: [] }; + }); + return { db: () => client }; +}); +vi.mock("@cap/database/helpers", () => ({ + nanoId: () => "row", + nanoIdLength: 15, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + serverEnv: () => ({}), +})); +vi.mock("@cap/utils", () => ({ stripe: () => ({}) })); +vi.mock("@/lib/infra-billing-emails", () => ({ + sendInfraTopUpFailedEmail: vi.fn(), +})); + +import { + AUTO_TOP_UP_DAILY_LIMIT, + getAutoTopUpFailure, + runDueAutoTopUps, +} from "@/lib/infra-auto-top-up"; + +const NOW = new Date("2026-11-13T06:41:00Z"); +const HOUR_MS = 60 * 60 * 1000; + +const accounts = ( + count: number, + from = 0, + lastAttemptAt: Date | null = null, + waiting = false, +): Account[] => + Array.from({ length: count }, (_, index) => ({ + id: `account-${String(from + index).padStart(4, "0")}`, + lastAttemptAt, + waiting, + })); + +const sqlTime = (date: Date) => + date.toISOString().slice(0, 23).replace("T", " "); + +beforeEach(() => { + vi.useFakeTimers(); + vi.setSystemTime(NOW); + state.accounts = accounts(450); + state.pages = []; + state.reads = []; + state.msPerAccount = 0; + state.other = []; +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("auto top-up sweep selection", () => { + it("pages through due accounts by last attempt, then id", async () => { + await runDueAutoTopUps(NOW); + + expect(state.pages.map(({ after, limit }) => ({ after, limit }))).toEqual([ + { after: null, limit: 200 }, + { after: { key: "1970-01-01 00:00:00", id: "account-0199" }, limit: 200 }, + { after: { key: "1970-01-01 00:00:00", id: "account-0399" }, limit: 200 }, + ]); + expect(state.reads).toHaveLength(450); + }); + + it("asks the database for enabled accounts under their threshold after held credit that nothing makes wait", async () => { + state.accounts = []; + + await runDueAutoTopUps(NOW); + + expect(state.pages).toHaveLength(1); + const { sql, params } = state.pages[0] ?? { sql: "", params: [] }; + expect(sql).toContain("`autoTopUpEnabled` = ?"); + expect(sql).toContain("`stripePaymentMethodId` is not null"); + expect(sql).toContain( + "left join (select `appId`, COALESCE(SUM(`reservedMicroCredits`), 0) as `heldMicroCredits` from `developer_transcripts` where `developer_transcripts`.`status` = ? group by `developer_transcripts`.`appId`) `held` on `held`.`appId` = `developer_credit_accounts`.`appId`", + ); + expect(sql).toContain( + "`balanceMicroCredits` < `developer_credit_accounts`.`autoTopUpThresholdMicroCredits` + coalesce(`heldMicroCredits`, 0)", + ); + expect(sql).toContain( + "(select max(`developer_credit_transactions`.`createdAt`) from `developer_credit_transactions` where `developer_credit_transactions`.`accountId` = `developer_credit_accounts`.`id` and `developer_credit_transactions`.`type` in (?, ?, ?) and `developer_credit_transactions`.`referenceType` in (?, ?, ?))", + ); + expect(sql).toContain( + "not (exists (select 1 from `developer_credit_transactions` `pending_attempt` where `pending_attempt`.`accountId` = `developer_credit_accounts`.`id` and `pending_attempt`.`type` = ? and `pending_attempt`.`referenceType` = ? and `pending_attempt`.`createdAt` > ? and `pending_attempt`.`referenceId` not like ? and not exists (select 1 from `developer_credit_transactions` `settled_attempt` where `settled_attempt`.`accountId` = `pending_attempt`.`accountId` and `settled_attempt`.`referenceId` = `pending_attempt`.`referenceId`", + ); + expect(sql).toContain( + "exists (select 1 from `developer_credit_transactions` `declined_attempt` where `declined_attempt`.`accountId` = `developer_credit_accounts`.`id` and `declined_attempt`.`type` = ? and `declined_attempt`.`referenceType` = ? and `declined_attempt`.`createdAt` > ? and not exists (select 1 from `developer_credit_transactions` `later_top_up` where `later_top_up`.`accountId` = `declined_attempt`.`accountId` and `later_top_up`.`type` = ?", + ); + expect(sql).toContain( + "(select count(*) from `developer_credit_transactions` `recent_top_up` where `recent_top_up`.`accountId` = `developer_credit_accounts`.`id` and `recent_top_up`.`type` = ? and `recent_top_up`.`referenceType` = ? and `recent_top_up`.`createdAt` > ?) >= ?", + ); + expect(sql).toMatch( + /order by attemptKey, `developer_credit_accounts`\.`id` asc limit \?$/, + ); + expect(params).toContain(sqlTime(new Date(NOW.getTime() - 72 * HOUR_MS))); + expect(params).toContain(sqlTime(new Date(NOW.getTime() - 24 * HOUR_MS))); + expect(params).toContain(AUTO_TOP_UP_DAILY_LIMIT); + expect(params).toContain("infra-auto-top-up:%"); + expect(sql).toContain( + "or exists (select 1 from `developer_credit_transactions` `stale_payment` where `stale_payment`.`accountId` = `developer_credit_accounts`.`id` and `stale_payment`.`type` = ? and `stale_payment`.`referenceType` = ? and `stale_payment`.`createdAt` <= ?", + ); + expect(sql).toContain( + "from `developer_credit_accounts` `owner_account` join `developer_credit_transactions` `owner_entry` on `owner_entry`.`accountId` = `owner_account`.`id` where `owner_account`.`ownerId` = `developer_credit_accounts`.`ownerId`", + ); + }); + + it("tries the least recently tried account first", async () => { + state.accounts = [ + { + id: "account-0001", + lastAttemptAt: new Date(NOW.getTime() - 26 * HOUR_MS), + }, + { id: "account-0002", lastAttemptAt: null }, + { + id: "account-0003", + lastAttemptAt: new Date(NOW.getTime() - 30 * HOUR_MS), + }, + { + id: "account-0004", + lastAttemptAt: new Date(NOW.getTime() - 2 * HOUR_MS), + }, + { id: "account-0005", lastAttemptAt: null }, + ]; + + await runDueAutoTopUps(NOW); + + expect(state.reads).toEqual([ + "account-0002", + "account-0005", + "account-0003", + "account-0001", + "account-0004", + ]); + }); + + it("reaches 50 due accounts in one run behind 300 accounts that must wait", async () => { + const tried = accounts(300, 0, new Date(NOW.getTime() - HOUR_MS), true); + const due = accounts(50, 300); + state.accounts = [...tried, ...due]; + state.msPerAccount = 250; + + await runDueAutoTopUps(NOW, { until: NOW.getTime() + 60_000 }); + + expect(state.reads).toEqual(due.map((account) => account.id)); + }); + + it("stops between accounts when its time is up", async () => { + state.msPerAccount = 1000; + + await runDueAutoTopUps(NOW, { until: NOW.getTime() + 10_500 }); + + expect(state.reads).toHaveLength(11); + expect(state.pages).toHaveLength(1); + }); +}); + +describe("last auto top-up lookup", () => { + it("reads the account's top-up rows by type, so the account and type index serves it", async () => { + expect(await getAutoTopUpFailure("account-0001")).toBeNull(); + + expect(state.other).toHaveLength(1); + const [{ sql, params }] = state.other as [ + { sql: string; params: unknown[] }, + ]; + expect(sql).toContain( + "where (`developer_credit_transactions`.`accountId` = ? and `developer_credit_transactions`.`type` in (?, ?) and `developer_credit_transactions`.`referenceType` in (?, ?))", + ); + expect(sql).toContain( + "order by `developer_credit_transactions`.`createdAt` desc limit ?", + ); + expect(params).toEqual([ + "account-0001", + "topup", + "topup_failed", + "auto_top_up", + "auto_top_up_failed", + 1, + ]); + }); +}); diff --git a/apps/web/__tests__/unit/infra-auto-top-up.test.ts b/apps/web/__tests__/unit/infra-auto-top-up.test.ts index 62221bee078..255bb963c4b 100644 --- a/apps/web/__tests__/unit/infra-auto-top-up.test.ts +++ b/apps/web/__tests__/unit/infra-auto-top-up.test.ts @@ -5,12 +5,17 @@ const mocks = vi.hoisted(() => ({ ids: 0, emailsEnabled: false, createPaymentIntent: vi.fn(), + retrievePaymentIntent: vi.fn(), + searchPaymentIntents: vi.fn(), retrieveSetupIntent: vi.fn(), retrievePaymentMethod: vi.fn(), detachPaymentMethod: vi.fn(), sendEmail: vi.fn(), + after: vi.fn(), })); +vi.mock("next/server", () => ({ after: mocks.after })); + vi.mock("@cap/database", () => ({ db: () => mocks.db })); vi.mock("@cap/database/helpers", () => ({ nanoId: () => `row-${++mocks.ids}`, @@ -25,7 +30,11 @@ vi.mock("@cap/env", () => ({ })); vi.mock("@cap/utils", () => ({ stripe: () => ({ - paymentIntents: { create: mocks.createPaymentIntent }, + paymentIntents: { + create: mocks.createPaymentIntent, + retrieve: mocks.retrievePaymentIntent, + search: mocks.searchPaymentIntents, + }, setupIntents: { retrieve: mocks.retrieveSetupIntent }, paymentMethods: { detach: mocks.detachPaymentMethod, @@ -46,6 +55,9 @@ vi.mock("@/lib/developer-credits-product", () => ({ import type Stripe from "stripe"; import { + AUTO_TOP_UP_DAILY_LIMIT, + AUTO_TOP_UP_STRIPE_TIMEOUT_MS, + autoTopUpDescription, autoTopUpFailureText, autoTopUpIdempotencyKey, autoTopUpSettingsError, @@ -53,6 +65,7 @@ import { runAutoTopUp, runDueAutoTopUps, saveCardFromCheckout, + scheduleAutoTopUp, settleAutoTopUpPaymentIntent, } from "@/lib/infra-auto-top-up"; import { @@ -84,6 +97,7 @@ const intent = ( type: "developer_auto_top_up", accountId: "account-1", period: "2026-11-11", + attempt: "0:0", }, last_payment_error: null, ...extra, @@ -110,6 +124,12 @@ const arm = () => { const topUps = () => ledger.rows.filter((row) => row.referenceType === "auto_top_up"); +const reservations = () => + ledger.rows.filter( + (row) => + row.referenceType === "auto_top_up_pending" && + String(row.referenceId).startsWith("infra-auto-top-up:"), + ); const failures = () => ledger.rows.filter((row) => row.referenceType === "auto_top_up_failed"); @@ -119,6 +139,9 @@ beforeEach(() => { mocks.emailsEnabled = false; mocks.createPaymentIntent.mockReset(); mocks.createPaymentIntent.mockResolvedValue(intent("pi_1", "succeeded")); + mocks.retrievePaymentIntent.mockReset(); + mocks.searchPaymentIntents.mockReset(); + mocks.searchPaymentIntents.mockResolvedValue({ data: [] }); mocks.sendEmail.mockReset(); mocks.sendEmail.mockResolvedValue({ data: { id: "email-1" }, error: null }); mocks.retrieveSetupIntent.mockReset(); @@ -181,12 +204,28 @@ describe("auto top-up settings", () => { ).toBe(null); }); - it("keys a charge by account and UTC day", () => { - expect(autoTopUpIdempotencyKey("account-1", "2026-11-11")).toBe( - "infra-auto-top-up:account-1:2026-11-11", + it("keys each charge by the account's recorded top-ups and declines, whatever the settings", () => { + expect( + autoTopUpIdempotencyKey("account-1", { topUps: 3, declines: 1 }), + ).toBe("infra-auto-top-up:account-1:3:1"); + expect( + autoTopUpIdempotencyKey("account-1", { topUps: 3, declines: 2 }), + ).not.toBe( + autoTopUpIdempotencyKey("account-1", { topUps: 3, declines: 1 }), ); }); + it("names the app in the charge description, within Stripe's limit", () => { + expect(autoTopUpDescription(" Acme Video ")).toBe( + "Cap Infra credits for Acme Video, auto top-up", + ); + expect(autoTopUpDescription("")).toBe("Cap Infra credits, auto top-up"); + expect(autoTopUpDescription(null)).toBe("Cap Infra credits, auto top-up"); + const long = autoTopUpDescription("x".repeat(5_000)); + expect(long.length).toBeLessThanOrEqual(1_000); + expect(long.endsWith(", auto top-up")).toBe(true); + }); + it("explains a decline in plain words", () => { expect(autoTopUpFailureText("authentication_required")).toContain("bank"); expect(autoTopUpFailureText("insufficient_funds")).toContain( @@ -197,18 +236,6 @@ describe("auto top-up settings", () => { }); describe("auto top-up charge", () => { - it("never charges an account outside the preview list", async () => { - ledger.account.email = "someone@example.com"; - - expect(await runAutoTopUp("account-1", now())).toEqual({ - state: "skipped", - reason: "not_allowed", - }); - expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); - expect(ledger.rows).toHaveLength(0); - expect(ledger.account.autoTopUpEnabled).toBe(true); - }); - it("does nothing when switched off, without a card, misconfigured or above the threshold", async () => { ledger.account.autoTopUpEnabled = false; expect(await runAutoTopUp("account-1", now())).toMatchObject({ @@ -242,6 +269,44 @@ describe("auto top-up charge", () => { expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); }); + it("never charges the card of a deleted app", async () => { + arm(); + ledger.account.appDeletedAt = new Date("2026-11-10T00:00:00Z"); + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "app_deleted", + }); + + ledger.account.appDeletedAt = null; + ledger.appDeletedBeforeLock = true; + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "app_deleted", + }); + + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + expect(ledger.rows).toHaveLength(0); + }); + + it("does not charge the card of an app whose payment was disputed", async () => { + arm(); + ledger.rows.push({ + id: "disputed", + accountId: "account-1", + type: "adjustment", + amountMicroCredits: 0, + balanceAfterMicroCredits: 30_000, + referenceId: "dp_1", + referenceType: "stripe_dispute", + createdAt: new Date(ledger.now), + }); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "disputed", + }); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + }); it("charges the saved card off session and grants credit keyed by the payment intent", async () => { const result = await runAutoTopUp("account-1", now()); @@ -262,11 +327,16 @@ describe("auto top-up charge", () => { metadata: { type: "developer_auto_top_up", accountId: "account-1", - period: "2026-11-11", + attempt: "0:0", }, }); expect(options).toEqual({ - idempotencyKey: "infra-auto-top-up:account-1:2026-11-11", + idempotencyKey: autoTopUpIdempotencyKey("account-1", { + topUps: 0, + declines: 0, + }), + timeout: AUTO_TOP_UP_STRIPE_TIMEOUT_MS, + maxNetworkRetries: 0, }); expect(ledger.account.balanceMicroCredits).toBe(530_000); expect(topUps()).toEqual([ @@ -279,6 +349,13 @@ describe("auto top-up charge", () => { ]); }); + it("asks Stripe to email the receipt to the account owner", async () => { + await runAutoTopUp("account-1", now()); + + const [params] = mocks.createPaymentIntent.mock.calls[0] ?? []; + expect(params.receipt_email).toBe(ledger.account.email); + }); + it("grants once when the webhook also reports the same payment", async () => { await runAutoTopUp("account-1", now()); @@ -297,31 +374,239 @@ describe("auto top-up charge", () => { expect(ledger.account.balanceMicroCredits).toBe(530_000); }); - it("makes at most one attempt per account in 24 hours", async () => { + it("tops up again the same day once the balance falls below the threshold again", async () => { await runAutoTopUp("account-1", now()); + later(0.1); + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "above_threshold", + }); + ledger.account.balanceMicroCredits = 10; + mocks.createPaymentIntent.mockResolvedValue(intent("pi_2", "succeeded")); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_2", + }); + + const [first, second] = mocks.createPaymentIntent.mock.calls; + expect(second?.[0].metadata.attempt).toBe("1:0"); + expect(second?.[1].idempotencyKey).not.toBe(first?.[1].idempotencyKey); + expect(topUps()).toHaveLength(2); + expect(now().toISOString().slice(0, 10)).toBe("2026-11-11"); + }); + + it("reserves the attempt before calling Stripe and removes the reservation with the grant", async () => { + mocks.createPaymentIntent.mockImplementation(async () => { + expect(reservations()).toEqual([ + expect.objectContaining({ + type: "topup_pending", + amountMicroCredits: 0, + referenceId: "infra-auto-top-up:account-1:0:0", + }), + ]); + return intent("pi_1", "succeeded"); + }); + + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + expect(reservations()).toHaveLength(0); + expect(ledger.rows.map((row) => row.type)).toEqual(["topup"]); + }); + + it("voids the reservation when the request never reached Stripe", async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + mocks.createPaymentIntent.mockRejectedValueOnce( + new Error("STRIPE_SECRET_KEY is not set"), + ); + + expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); + expect(ledger.rows).toHaveLength(0); + errors.mockRestore(); + }); + + it("lets the webhook settle a reservation whose response was lost", async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign(new Error("timeout"), { type: "StripeConnectionError" }), + ); + await runAutoTopUp("account-1", now()); + expect(reservations()).toHaveLength(1); + + await settleAutoTopUpPaymentIntent( + intent("pi_1", "succeeded"), + "succeeded", + ); + + expect(reservations()).toHaveLength(0); + expect(topUps()).toEqual([ + expect.objectContaining({ referenceId: "pi_1" }), + ]); + expect(ledger.account.balanceMicroCredits).toBe(530_000); + errors.mockRestore(); + }); + + it("repeats the same key when retried before the attempt settles", async () => { + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign(new Error("network"), { type: "StripeConnectionError" }), + ); + expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + + const [first, retry] = mocks.createPaymentIntent.mock.calls; + expect(retry?.[1].idempotencyKey).toBe(first?.[1].idempotencyKey); + expect(topUps()).toHaveLength(1); + }); + + it("gives up on a payment request that times out, writes nothing and retries with the same key", async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign( + new Error("Request aborted due to timeout being reached (15000ms)"), + { type: "StripeConnectionError" }, + ), + ); + + expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); + expect(errors).toHaveBeenCalledWith( + "[infra-billing] Auto top-up could not be attempted", + expect.objectContaining({ accountId: "account-1" }), + ); + expect(ledger.rows).toEqual([ + expect.objectContaining({ + type: "topup_pending", + amountMicroCredits: 0, + referenceId: "infra-auto-top-up:account-1:0:0", + metadata: expect.objectContaining({ + amountCents: 500, + customerId: "cus_1", + paymentMethodId: "pm_1", + period: "2026-11-11", + }), + }), + ]); + expect(ledger.account.autoTopUpEnabled).toBe(true); + expect(mocks.sendEmail).not.toHaveBeenCalled(); + + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + const [timedOut, retry] = mocks.createPaymentIntent.mock.calls; + expect(timedOut?.[1].timeout).toBe(15_000); + expect(retry?.[1].idempotencyKey).toBe(timedOut?.[1].idempotencyKey); + errors.mockRestore(); + }); + + it("treats a payment already being made under the same key as in progress", async () => { + mocks.createPaymentIntent.mockRejectedValue( + Object.assign(new Error("in use"), { + type: "StripeInvalidRequestError", + code: "idempotency_key_in_use", + statusCode: 409, + }), + ); - later(1); expect(await runAutoTopUp("account-1", now())).toEqual({ state: "skipped", - reason: "recent_attempt", + reason: "in_progress", }); - later(22.9); + expect(ledger.account.autoTopUpEnabled).toBe(true); + expect(reservations()).toHaveLength(1); + expect(ledger.rows).toHaveLength(1); + expect(mocks.sendEmail).not.toHaveBeenCalled(); + }); + + it("retries an unresolved attempt with the reserved amount and card, not changed settings", async () => { + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign(new Error("Request aborted due to timeout"), { + type: "StripeConnectionError", + }), + ); + vi.spyOn(console, "error").mockImplementationOnce(() => {}); + expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); + + ledger.account.autoTopUpAmountCents = 2_500; + ledger.account.stripePaymentMethodId = "pm_new"; + mocks.createPaymentIntent.mockResolvedValueOnce( + intent("pi_1", "succeeded"), + ); expect(await runAutoTopUp("account-1", now())).toMatchObject({ - reason: "recent_attempt", + state: "succeeded", + paymentIntentId: "pi_1", }); - expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); - later(0.2); - mocks.createPaymentIntent.mockResolvedValue(intent("pi_2", "succeeded")); + const [first, retry] = mocks.createPaymentIntent.mock.calls; + expect(retry?.[0]).toEqual(first?.[0]); + expect(retry?.[0]).toMatchObject({ amount: 500, payment_method: "pm_1" }); + expect(retry?.[1].idempotencyKey).toBe(first?.[1].idempotencyKey); + expect(reservations()).toHaveLength(0); + expect(topUps()).toHaveLength(1); + + ledger.account.balanceMicroCredits = 10; + mocks.createPaymentIntent.mockResolvedValueOnce( + intent("pi_2", "succeeded", { amount_received: 2_500 }), + ); + await runAutoTopUp("account-1", now()); + const next = mocks.createPaymentIntent.mock.calls[2]; + expect(next?.[0]).toMatchObject({ + amount: 2_500, + payment_method: "pm_new", + }); + expect(next?.[1].idempotencyKey).not.toBe(first?.[1].idempotencyKey); + }); + + it("treats Stripe refusing a changed request under the same key as in progress", async () => { + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign( + new Error( + "Keys for idempotent requests can only be used with the same parameters they were first used with.", + ), + { type: "StripeIdempotencyError", statusCode: 400 }, + ), + ); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "in_progress", + }); + expect(reservations()).toHaveLength(1); + expect(ledger.account.autoTopUpEnabled).toBe(true); + expect(mocks.sendEmail).not.toHaveBeenCalled(); + }); + + it(`stops at ${AUTO_TOP_UP_DAILY_LIMIT} automatic top-ups in any 24 hours, then waits`, async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + for (let i = 1; i <= AUTO_TOP_UP_DAILY_LIMIT; i++) { + ledger.account.balanceMicroCredits = 10; + mocks.createPaymentIntent.mockResolvedValue( + intent(`pi_${i}`, "succeeded"), + ); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + later(1); + } + + ledger.account.balanceMicroCredits = 10; + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "daily_limit", + }); + expect(warn).toHaveBeenCalledTimes(1); + expect(mocks.createPaymentIntent).toHaveBeenCalledTimes( + AUTO_TOP_UP_DAILY_LIMIT, + ); + + later(14.5); + mocks.createPaymentIntent.mockResolvedValue(intent("pi_next", "succeeded")); expect(await runAutoTopUp("account-1", now())).toMatchObject({ state: "succeeded", - paymentIntentId: "pi_2", - }); - expect(mocks.createPaymentIntent.mock.calls[1]?.[1]).toEqual({ - idempotencyKey: "infra-auto-top-up:account-1:2026-11-12", + paymentIntentId: "pi_next", }); - expect(topUps()).toHaveLength(2); + warn.mockRestore(); }); it("grants nothing for a processing payment, then once when it succeeds", async () => { @@ -353,9 +638,286 @@ describe("auto top-up charge", () => { expect(topUps()).toHaveLength(1); expect(ledger.account.balanceMicroCredits).toBe(530_000); + + ledger.account.balanceMicroCredits = 10; + mocks.createPaymentIntent.mockResolvedValue(intent("pi_2", "succeeded")); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_2", + }); }); - it("does not start a second payment after a processing one, even on the next UTC day", async () => { + describe("the owner's daily ceiling", () => { + it("waits once automatic top-ups would pass $1,000 in 24 hours, without switching anything off", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + ledger.account.autoTopUpAmountCents = 60_000; + mocks.createPaymentIntent.mockResolvedValue( + intent("pi_1", "succeeded", { amount_received: 60_000 }), + ); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + + ledger.account.balanceMicroCredits = 10; + later(1); + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "owner_daily_limit", + }); + expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); + expect(ledger.account.autoTopUpEnabled).toBe(true); + expect(failures()).toHaveLength(0); + expect(mocks.sendEmail).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + "[infra-billing] Auto top-up waits for its daily limit", + { accountId: "account-1", reason: "owner_daily_limit" }, + ); + + later(23.1); + mocks.createPaymentIntent.mockResolvedValue( + intent("pi_2", "succeeded", { amount_received: 60_000 }), + ); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_2", + }); + warn.mockRestore(); + }); + + it("does not count credit bought by hand", async () => { + ledger.rows.push({ + id: "manual", + accountId: "account-1", + type: "topup", + amountMicroCredits: 90_000_000, + balanceAfterMicroCredits: 90_030_000, + referenceId: "pi_manual", + referenceType: "stripe_payment_intent", + createdAt: new Date(ledger.now), + }); + ledger.account.autoTopUpAmountCents = 60_000; + mocks.createPaymentIntent.mockResolvedValue( + intent("pi_1", "succeeded", { amount_received: 60_000 }), + ); + + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + }); + }); + }); + + describe("recovering an attempt whose answer was lost", () => { + const loseAnswer = async () => { + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + mocks.createPaymentIntent.mockRejectedValueOnce( + Object.assign(new Error("timeout"), { type: "StripeConnectionError" }), + ); + await runAutoTopUp("account-1", now()); + errors.mockRestore(); + mocks.createPaymentIntent.mockClear(); + }; + + it("waits instead of replaying once the reservation is a day old", async () => { + await loseAnswer(); + later(25); + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "awaiting_payment", + }); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + expect(mocks.searchPaymentIntents).not.toHaveBeenCalled(); + }); + + it("after 72 hours, finds the payment in Stripe and grants it once", async () => { + await loseAnswer(); + later(72.1); + mocks.searchPaymentIntents.mockResolvedValue({ + data: [intent("pi_found", "succeeded")], + }); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(mocks.searchPaymentIntents.mock.calls[0]?.[0]).toEqual({ + query: + "metadata['accountId']:'account-1' AND metadata['attempt']:'0:0'", + limit: 1, + }); + expect(topUps()).toEqual([ + expect.objectContaining({ referenceId: "pi_found" }), + ]); + expect(reservations()).toHaveLength(0); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + }); + + it("recovers a deleted app's lost attempt, then never charges it again", async () => { + await loseAnswer(); + later(72.1); + ledger.account.appDeletedAt = now(); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(mocks.searchPaymentIntents).toHaveBeenCalledTimes(1); + expect(ledger.rows).toHaveLength(0); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "app_deleted", + }); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + expect(ledger.rows).toHaveLength(0); + }); + + it("after 72 hours, voids a reservation Stripe never charged", async () => { + await loseAnswer(); + later(72.1); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(ledger.rows).toHaveLength(0); + expect(topUps()).toHaveLength(0); + }); + + it("recovers before the switch, the decline wait or any limit can stop it", async () => { + await loseAnswer(); + ledger.account.autoTopUpEnabled = false; + ledger.account.balanceMicroCredits = 99_000_000; + later(72.1); + mocks.searchPaymentIntents.mockResolvedValue({ + data: [intent("pi_found", "succeeded")], + }); + + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "reconciled", + }); + expect(topUps()).toHaveLength(1); + }); + }); + + describe("a payment still pending after 72 hours", () => { + const leavePending = async () => { + mocks.createPaymentIntent.mockResolvedValue(intent("pi_1", "processing")); + await runAutoTopUp("account-1", now()); + mocks.createPaymentIntent.mockReset(); + }; + + it("asks Stripe once and records a payment that succeeded", async () => { + await leavePending(); + later(71.9); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "awaiting_payment", + }); + expect(mocks.retrievePaymentIntent).not.toHaveBeenCalled(); + + later(0.2); + mocks.retrievePaymentIntent.mockResolvedValue( + intent("pi_1", "succeeded"), + ); + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(mocks.retrievePaymentIntent).toHaveBeenCalledTimes(1); + expect(mocks.retrievePaymentIntent.mock.calls[0]?.[0]).toBe("pi_1"); + expect(topUps()).toEqual([ + expect.objectContaining({ referenceId: "pi_1" }), + ]); + expect(ledger.account.balanceMicroCredits).toBe(530_000); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + + await settleAutoTopUpPaymentIntent( + intent("pi_1", "succeeded"), + "succeeded", + ); + expect(topUps()).toHaveLength(1); + expect(ledger.account.balanceMicroCredits).toBe(530_000); + + ledger.account.balanceMicroCredits = 10; + mocks.createPaymentIntent.mockResolvedValue(intent("pi_2", "succeeded")); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_2", + }); + expect(mocks.retrievePaymentIntent).toHaveBeenCalledTimes(1); + }); + + it("is picked up by the storage sweep once it passes 72 hours, with no other trigger", async () => { + await leavePending(); + later(71); + expect(await runDueAutoTopUps(now())).toEqual({ + attempted: 0, + succeeded: 0, + failed: 0, + }); + expect(mocks.retrievePaymentIntent).not.toHaveBeenCalled(); + + later(1.1); + mocks.retrievePaymentIntent.mockResolvedValue( + intent("pi_1", "succeeded"), + ); + await runDueAutoTopUps(now()); + + expect(mocks.retrievePaymentIntent).toHaveBeenCalledTimes(1); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + expect(topUps()).toEqual([ + expect.objectContaining({ referenceId: "pi_1" }), + ]); + }); + + it("records a payment that failed and switches auto top-up off", async () => { + await leavePending(); + later(72.1); + mocks.retrievePaymentIntent.mockResolvedValue( + intent("pi_1", "requires_payment_method", { + last_payment_error: { + decline_code: "insufficient_funds", + } as Stripe.PaymentIntent.LastPaymentError, + }), + ); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(failures()).toEqual([ + expect.objectContaining({ + referenceId: "pi_1", + metadata: expect.objectContaining({ reason: "insufficient_funds" }), + }), + ]); + expect(topUps()).toHaveLength(0); + expect(ledger.account.autoTopUpEnabled).toBe(false); + expect(ledger.account.balanceMicroCredits).toBe(30_000); + }); + + it("leaves a payment Stripe still reports as processing", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + await leavePending(); + later(80); + mocks.retrievePaymentIntent.mockResolvedValue( + intent("pi_1", "processing"), + ); + + expect(await runAutoTopUp("account-1", now())).toEqual({ + state: "skipped", + reason: "reconciled", + }); + expect(topUps()).toHaveLength(0); + expect(failures()).toHaveLength(0); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "reconciled", + }); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + warn.mockRestore(); + }); + }); + + it("does not start a second payment while one is still processing, however long it takes", async () => { ledger.now = new Date("2026-11-11T23:50:00Z"); mocks.createPaymentIntent.mockResolvedValue(intent("pi_1", "processing")); await runAutoTopUp("account-1", now()); @@ -365,12 +927,21 @@ describe("auto top-up charge", () => { expect(now().toISOString().slice(0, 10)).toBe("2026-11-12"); expect(await runAutoTopUp("account-1", now())).toEqual({ state: "skipped", - reason: "recent_attempt", + reason: "awaiting_payment", }); later(23); expect(await runAutoTopUp("account-1", now())).toMatchObject({ - reason: "recent_attempt", + reason: "awaiting_payment", }); + later(1.5); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "awaiting_payment", + }); + later(46.5); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "awaiting_payment", + }); + expect(mocks.retrievePaymentIntent).not.toHaveBeenCalled(); expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); expect(ledger.rows).toHaveLength(1); @@ -481,9 +1052,25 @@ describe("auto top-up charge", () => { ledger.account.autoTopUpEnabled = true; later(2); expect(await runAutoTopUp("account-1", now())).toMatchObject({ - reason: "recent_attempt", + reason: "recent_decline", + }); + later(21.9); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + reason: "recent_decline", }); expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); + + later(0.2); + mocks.createPaymentIntent.mockResolvedValue(intent("pi_2", "succeeded")); + expect(await runAutoTopUp("account-1", now())).toMatchObject({ + state: "succeeded", + paymentIntentId: "pi_2", + }); + const [declined, next] = mocks.createPaymentIntent.mock.calls; + expect(declined?.[1].idempotencyKey).toBe( + "infra-auto-top-up:account-1:0:0", + ); + expect(next?.[1].idempotencyKey).toBe("infra-auto-top-up:account-1:0:1"); }); it("treats a charge that needs the customer as a failure", async () => { @@ -506,7 +1093,8 @@ describe("auto top-up charge", () => { expect(await runAutoTopUp("account-1", now())).toEqual({ state: "error" }); expect(ledger.account.autoTopUpEnabled).toBe(true); - expect(ledger.rows).toHaveLength(0); + expect(reservations()).toHaveLength(1); + expect(ledger.rows).toHaveLength(1); }); it("ignores payments that are not auto top-ups or did not settle in dollars", async () => { @@ -543,6 +1131,88 @@ describe("auto top-up charge", () => { }); }); +describe("auto top-up after a charge", () => { + const settle = () => new Promise((resolve) => setTimeout(resolve, 0)); + const outsideRequest = () => { + throw new Error("`after` was called outside a request scope"); + }; + + beforeEach(() => { + mocks.after.mockReset(); + }); + + it("waits until the response is sent, then tops up the account", async () => { + const tasks: (() => Promise)[] = []; + mocks.after.mockImplementation((task: () => Promise) => { + tasks.push(task); + }); + + await scheduleAutoTopUp({ accountId: "account-1" }); + await settle(); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + + await tasks[0]?.(); + expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); + expect(topUps()).toHaveLength(1); + }); + + it("finds the account from the app", async () => { + const tasks: (() => Promise)[] = []; + mocks.after.mockImplementation((task: () => Promise) => { + tasks.push(task); + }); + + await scheduleAutoTopUp({ appId: "app-1" }); + await tasks[0]?.(); + + expect(mocks.createPaymentIntent.mock.calls[0]?.[0].metadata).toMatchObject( + { accountId: "account-1", appId: "app-1" }, + ); + }); + + it("outside a request, finishes the top-up before it returns", async () => { + mocks.after.mockImplementation(outsideRequest); + + await scheduleAutoTopUp({ accountId: "account-1" }); + + expect(mocks.after).toHaveBeenCalledTimes(1); + expect(mocks.createPaymentIntent).toHaveBeenCalledTimes(1); + expect(topUps()).toHaveLength(1); + }); + + it("inside a request, returns at once and leaves the top-up to after()", async () => { + mocks.after.mockImplementation(() => {}); + + await scheduleAutoTopUp({ accountId: "account-1" }); + + expect(mocks.after).toHaveBeenCalledTimes(1); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + }); + + it("never throws into the caller and logs what went wrong", async () => { + mocks.after.mockImplementation(outsideRequest); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + const broken = { + ...fakeDb, + select: () => { + throw new Error("database unavailable"); + }, + }; + mocks.db = broken; + + await expect( + scheduleAutoTopUp({ accountId: "account-1" }), + ).resolves.toBeUndefined(); + expect(errors).toHaveBeenCalledWith( + "[infra-billing] Auto top-up after a charge failed", + expect.objectContaining({ target: { accountId: "account-1" } }), + ); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + mocks.db = fakeDb; + errors.mockRestore(); + }); +}); + describe("saving a card from checkout", () => { const session = (extra: Partial = {}) => ({ @@ -658,6 +1328,46 @@ describe("auto top-up sweep", () => { failed: 1, }); }); + + it("does not read an account that is not due", async () => { + ledger.account.balanceMicroCredits = + ledger.account.autoTopUpThresholdMicroCredits; + const select = vi.spyOn(fakeDb, "select"); + + expect(await runDueAutoTopUps(now())).toEqual({ + attempted: 0, + succeeded: 0, + failed: 0, + }); + + expect( + select.mock.calls.filter(([fields]) => "autoTopUpAmountCents" in fields), + ).toHaveLength(0); + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + select.mockRestore(); + }); + + it("still tops up an account that only held credit puts under its threshold", async () => { + ledger.account.balanceMicroCredits = + ledger.account.autoTopUpThresholdMicroCredits + 10; + ledger.heldMicroCredits = 11; + + expect(await runDueAutoTopUps(now())).toEqual({ + attempted: 1, + succeeded: 1, + failed: 0, + }); + }); + + it("starts no top-up once its time is up", async () => { + expect(await runDueAutoTopUps(now(), { until: Date.now() - 1 })).toEqual({ + attempted: 0, + succeeded: 0, + failed: 0, + }); + + expect(mocks.createPaymentIntent).not.toHaveBeenCalled(); + }); }); describe("billing emails", () => { @@ -725,12 +1435,15 @@ describe("billing emails", () => { expect(mocks.sendEmail).toHaveBeenCalledTimes(1); }); - it("never emails an account outside the preview list", async () => { + it("emails any account owner when the switch is on", async () => { mocks.emailsEnabled = true; ledger.account.email = "someone@example.com"; - expect(await outOfCredit()).toBe(false); - expect(await topUpFailed()).toBe(false); - expect(mocks.sendEmail).not.toHaveBeenCalled(); + expect(await outOfCredit()).toBe(true); + expect(await topUpFailed()).toBe(true); + expect(mocks.sendEmail).toHaveBeenCalledTimes(2); + expect(mocks.sendEmail.mock.calls[0]?.[0]).toMatchObject({ + email: "someone@example.com", + }); }); }); diff --git a/apps/web/__tests__/unit/infra-billing-docs-wording.test.ts b/apps/web/__tests__/unit/infra-billing-docs-wording.test.ts new file mode 100644 index 00000000000..a17bfccfc8a --- /dev/null +++ b/apps/web/__tests__/unit/infra-billing-docs-wording.test.ts @@ -0,0 +1,28 @@ +import fs from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const billing = fs.readFileSync( + path.join(process.cwd(), "content/docs/api/billing.mdx"), + "utf8", +); + +describe("billing docs wording", () => { + it("points a US business to the EIN field before checkout", () => { + expect(billing).toContain( + "Stripe's checkout has no US tax ID type, so a US business adds its EIN with \"Add your EIN\" on the console's Billing page before it pays", + ); + }); + + it("says where amounts are exact and that the sidebar shows whole cents", () => { + expect(billing).not.toContain( + "Every amount in the console and the API is exact.", + ); + expect(billing).toContain( + "Every balance and charge on the Billing page, in the ledger and in the API is exact.", + ); + expect(billing).toContain( + "The console sidebar shows the balance in whole cents, rounded down, with the exact amount on hover.", + ); + }); +}); diff --git a/apps/web/__tests__/unit/infra-billing-fake-db.ts b/apps/web/__tests__/unit/infra-billing-fake-db.ts index 85239c33378..32a28e58b3f 100644 --- a/apps/web/__tests__/unit/infra-billing-fake-db.ts +++ b/apps/web/__tests__/unit/infra-billing-fake-db.ts @@ -1,11 +1,12 @@ import { + developerApps, developerCreditAccounts, developerCreditTransactions, developerDailyStorageSnapshots, developerTranscripts, developerVideos, } from "@cap/database/schema"; -import type { SQL } from "drizzle-orm"; +import { type SQL, sql } from "drizzle-orm"; import { MySqlDialect } from "drizzle-orm/mysql-core"; const dialect = new MySqlDialect(); @@ -39,6 +40,8 @@ export type VideoGroup = { count: number; status?: string; maxResolutionTier?: string | null; + dailySeconds?: number; + dailyCount?: number; }; const freshAccount = () => ({ @@ -52,6 +55,7 @@ const freshAccount = () => ({ autoTopUpThresholdMicroCredits: 0, autoTopUpAmountCents: 0, email: "richie@cap.so", + appDeletedAt: null as Date | null, }); export const ledger = { @@ -61,6 +65,7 @@ export const ledger = { snapshots: [] as SnapshotRow[], videos: [] as VideoGroup[], heldMicroCredits: 0, + appDeletedBeforeLock: false, }; export function resetLedger() { @@ -68,6 +73,7 @@ export function resetLedger() { ledger.account = freshAccount(); ledger.rows = []; ledger.snapshots = []; + ledger.appDeletedBeforeLock = false; ledger.videos = []; ledger.heldMicroCredits = 0; } @@ -95,6 +101,12 @@ const asTime = (value: unknown) => { ).getTime(); }; +const availableAfter = (row: LedgerRow) => + Number( + (row.metadata as { availableAfterMicroCredits?: number } | null | undefined) + ?.availableAfterMicroCredits ?? row.balanceAfterMicroCredits, + ); + const newestFirst = (rows: LedgerRow[]) => [...rows].sort( (a, b) => @@ -102,6 +114,90 @@ const newestFirst = (rows: LedgerRow[]) => ledger.rows.indexOf(b) - ledger.rows.indexOf(a), ); +const byId = (a: LedgerRow, b: LedgerRow) => + a.id < b.id ? 1 : a.id > b.id ? -1 : 0; + +const latestWritten = (rows: LedgerRow[]) => + [...rows].sort( + (a, b) => b.createdAt.getTime() - a.createdAt.getTime() || byId(a, b), + ); + +const latestStorageDayRows = () => { + const days = storageRows().filter( + (row) => row.referenceType === "storage_day" && row.referenceId, + ); + const latestDay = days + .map((row) => String(row.referenceId)) + .sort() + .at(-1); + return days.filter((row) => row.referenceId === latestDay).sort(byId); +}; + +const HOUR_MS = 60 * 60 * 1000; + +function autoTopUpWaits() { + const now = ledger.now.getTime(); + const of = (type: string, referenceType: string) => + ledger.rows.filter( + (row) => row.type === type && row.referenceType === referenceType, + ); + const settledIds = new Set( + ledger.rows + .filter( + (row) => + row.referenceType === "auto_top_up" || + row.referenceType === "auto_top_up_failed", + ) + .map((row) => row.referenceId), + ); + const topUps = of("topup", "auto_top_up"); + return { + pending: of("topup_pending", "auto_top_up_pending").some( + (row) => + row.createdAt.getTime() > now - 72 * HOUR_MS && + !String(row.referenceId).startsWith("infra-auto-top-up:") && + !settledIds.has(row.referenceId), + ), + decline: of("topup_failed", "auto_top_up_failed").some( + (row) => + row.createdAt.getTime() > now - 24 * HOUR_MS && + !topUps.some((topUp) => topUp.createdAt > row.createdAt), + ), + limit: + topUps.filter((row) => row.createdAt.getTime() > now - 24 * HOUR_MS) + .length >= 10, + owner: + ledger.rows + .filter( + (row) => + !String(row.referenceId).startsWith("infra-auto-top-up:") && + ((row.type === "topup" && + row.referenceType === "auto_top_up" && + row.createdAt.getTime() > now - 24 * HOUR_MS) || + (row.type === "topup_pending" && + row.referenceType === "auto_top_up_pending" && + !settledIds.has(row.referenceId))), + ) + .reduce( + (total, row) => + total + + (row.type === "topup" + ? row.amountMicroCredits + : Number(row.metadata?.amountCents ?? 0) * 1000), + 0, + ) + + Number( + ledger.rows.find( + (row) => + row.type === "topup_pending" && + String(row.referenceId).startsWith("infra-auto-top-up:"), + )?.metadata?.amountCents ?? ledger.account.autoTopUpAmountCents, + ) * + 1000 > + 100_000_000, + }; +} + function runSelect(query: Query): Record[] { const keys = Object.keys(query.fields).sort().join(","); const rendered = query.where @@ -111,41 +207,272 @@ function runSelect(query: Query): Record[] { if (query.table === developerCreditAccounts) { const { exists, ...account } = ledger.account; if (!exists) return []; + if ( + keys === + "amountCents,balanceMicroCredits,customerId,declineWait,enabled,id,limitWait,ownerWait,paymentMethodId,pendingWait,thresholdMicroCredits" + ) { + if (rendered.params.at(-1) !== account.id) return []; + const waits = autoTopUpWaits(); + return [ + { + id: account.id, + enabled: account.autoTopUpEnabled, + thresholdMicroCredits: account.autoTopUpThresholdMicroCredits, + amountCents: account.autoTopUpAmountCents, + customerId: account.stripeCustomerId, + paymentMethodId: account.stripePaymentMethodId, + balanceMicroCredits: account.balanceMicroCredits, + pendingWait: Number(waits.pending), + declineWait: Number(waits.decline), + limitWait: Number(waits.limit), + ownerWait: Number(waits.owner), + }, + ]; + } + if (keys === "attemptKey,id") { + const waits = autoTopUpWaits(); + const attempts = ledger.rows + .filter((row) => + ["auto_top_up", "auto_top_up_failed", "auto_top_up_pending"].includes( + String(row.referenceType), + ), + ) + .map((row) => row.createdAt.getTime()); + const lastAttempt = attempts.length > 0 ? Math.max(...attempts) : null; + const due = + account.autoTopUpEnabled && + account.stripePaymentMethodId && + account.balanceMicroCredits < + account.autoTopUpThresholdMicroCredits + ledger.heldMicroCredits && + !waits.pending && + !waits.decline && + !waits.limit && + !waits.owner; + return due + ? [ + { + id: account.id, + attemptKey: + lastAttempt === null + ? "1970-01-01 00:00:00" + : new Date(lastAttempt) + .toISOString() + .slice(0, 19) + .replace("T", " "), + }, + ] + : []; + } + if ( + keys === "id" && + /`id` = \?$/.test(rendered.sql) && + rendered.params[0] !== account.id + ) { + return []; + } if ( rendered.sql.includes("autoTopUpThresholdMicroCredits") && keys === "id" && !( account.autoTopUpEnabled && account.stripePaymentMethodId && - account.balanceMicroCredits < account.autoTopUpThresholdMicroCredits + account.balanceMicroCredits < + account.autoTopUpThresholdMicroCredits + ledger.heldMicroCredits ) ) { return []; } + if ("heldMicroCredits" in query.fields) { + const [dispute] = latestWritten( + ledger.rows.filter( + (row) => + row.type === "adjustment" && + ["stripe_dispute", "manual"].includes(String(row.referenceType)), + ), + ); + return [ + { + ...account, + heldMicroCredits: ledger.heldMicroCredits, + disputeType: dispute?.referenceType ?? null, + disputeCreatedAt: dispute?.createdAt ?? null, + }, + ]; + } return [account]; } if (query.table === developerCreditTransactions) { + if (keys === "createdAt") { + const type = rendered.params[1]; + return latestWritten( + ledger.rows.filter((row) => row.type === type), + ).slice(0, 1); + } + if (keys === "createdAt,id,metadata") { + if ( + rendered.sql.includes( + "`developer_credit_transactions`.`referenceId` <= ?", + ) + ) { + return latestStorageDayRows().slice(0, 1); + } + const [, type, at] = rendered.params; + if (at === "delivery_day") { + return ledger.rows + .filter( + (row) => + row.type === type && + row.referenceType === at && + row.metadata?.duringShortfall === undefined, + ) + .sort( + (a, b) => + b.createdAt.getTime() - a.createdAt.getTime() || + String(b.referenceId).localeCompare(String(a.referenceId)) || + byId(a, b), + ) + .slice(0, 1); + } + const ofType = latestWritten( + ledger.rows.filter((row) => row.type === type), + ); + return at === undefined + ? ofType.slice(0, 1) + : ofType.filter((row) => row.createdAt.getTime() === asTime(at)); + } if (keys === "createdAt,metadata") { - return newestFirst(storageRows()).slice(0, 1); + return rendered.sql.includes( + "`developer_credit_transactions`.`referenceId` <= ?", + ) + ? latestStorageDayRows().slice(0, 1) + : latestWritten(storageRows()).slice(0, 1); + } + if (keys === "accountId,createdAt,id,metadata") { + return latestStorageDayRows(); + } + if (keys === "owedMicroCredits") { + const shortSince = rendered.params.at(-1); + const owed = ledger.rows + .filter( + (row) => + row.type === "video_create" && + row.metadata?.shortSince === shortSince && + row.metadata?.resolvedAt === undefined, + ) + .map((row) => { + const owedMicroCredits = Number(row.metadata?.owedMicroCredits) || 0; + const charged = Number(row.metadata?.chargedMicroCredits) || 0; + return Math.max( + charged < owedMicroCredits ? owedMicroCredits : 0, + Number(row.metadata?.outstandingMicroCredits) || 0, + ); + }); + return [{ owedMicroCredits: owed.length ? Math.max(...owed) : null }]; } if (keys === "metadata") { const since = asTime(rendered.params.at(-1)); return storageRows().filter((row) => row.createdAt.getTime() >= since); } + if (keys === "accountId,amountMicroCredits") { + const [accountId, referenceId, ...referenceTypes] = rendered.params; + return ledger.rows.filter( + (row) => + row.type === "topup" && + row.accountId === accountId && + row.referenceId === referenceId && + referenceTypes.includes(row.referenceType), + ); + } + if (keys === "metadata,referenceId,referenceType") { + const allowed = rendered.params.slice(1); + return ledger.rows.filter( + (row) => + allowed.includes(row.type) && allowed.includes(row.referenceType), + ); + } + if (keys === "value") { + const column = (query.fields.value as { name?: string }).name ?? ""; + return latestWritten( + ledger.rows.filter( + (row) => + row.type === "adjustment" && + ["stripe_dispute", "manual"].includes(String(row.referenceType)), + ), + ) + .slice(0, 1) + .map((row) => ({ value: row[column as keyof typeof row] ?? null })); + } + if (keys === "createdAt,referenceType") { + const types = rendered.params.slice(2); + return latestWritten( + ledger.rows.filter( + (row) => + row.type === "adjustment" && types.includes(row.referenceType), + ), + ).slice(0, 1); + } + if (keys === "pendingId,pendingRow,pendingSince") { + return ledger.rows + .filter( + (row) => + row.type === "topup_pending" && + row.referenceType === "auto_top_up_pending", + ) + .map((row) => ({ + pendingRow: row.id, + pendingId: row.referenceId, + pendingSince: row.createdAt, + })); + } + if (keys === "reservation,reservedAt") { + const [accountId, referenceId] = rendered.params; + return ledger.rows + .filter( + (row) => + row.accountId === accountId && + row.referenceId === referenceId && + row.referenceType === "auto_top_up_pending", + ) + .map((row) => ({ + reservation: row.metadata, + reservedAt: row.createdAt, + })); + } + if (keys === "settledId") { + const ids = rendered.params.slice(3); + return ledger.rows + .filter( + (row) => + (row.referenceType === "auto_top_up" || + row.referenceType === "auto_top_up_failed") && + ids.includes(row.referenceId), + ) + .map((row) => ({ settledId: row.referenceId })); + } + if (keys === "declines,topUps") { + const count = (type: string) => + ledger.rows.filter((row) => row.referenceType === type).length; + return [ + { + topUps: count("auto_top_up"), + declines: count("auto_top_up_failed"), + }, + ]; + } if (keys === "createdAt,metadata,type") { const types = rendered.params.slice(1); return newestFirst( ledger.rows.filter((row) => types.includes(row.referenceType)), ).slice(0, 1); } - if (keys === "id" && rendered.sql.includes("balanceAfterMicroCredits")) { - const [, since, , owed] = rendered.params; + if (keys === "id" && rendered.sql.includes("JSON_EXTRACT")) { + const [, since, , , owed] = rendered.params; return ledger.rows.filter( (row) => - row.createdAt.getTime() >= asTime(since) && + row.createdAt.getTime() > asTime(since) && row.amountMicroCredits > 0 && - row.balanceAfterMicroCredits >= Number(owed), + availableAfter(row) >= Number(owed), ); } if (keys === "id") { @@ -166,6 +493,13 @@ function runSelect(query: Query): Record[] { } } + if (query.table === developerApps && keys === "appDeletedAt") { + const deletedAt = ledger.appDeletedBeforeLock + ? ledger.now + : ledger.account.appDeletedAt; + return [{ appDeletedAt: deletedAt }]; + } + if (query.table === developerTranscripts) { return [{ total: ledger.heldMicroCredits }]; } @@ -177,13 +511,26 @@ function runSelect(query: Query): Record[] { } if (query.table === developerVideos) { - return rendered.sql.includes("<> ?") + const videos = rendered.sql.includes("<> ?") ? ledger.videos.filter( (video) => !video.videoId || !rendered.params.includes(video.videoId), ) : ledger.videos; + const firstMonth = (seconds: number, perMinute: number) => + Math.max(1, Math.floor((seconds / 60) * perMinute)); + return videos.map((video) => ({ + ...video, + status: video.status ?? "ready", + dailySeconds: video.dailySeconds ?? video.seconds, + dailyCount: video.dailyCount ?? video.count, + firstMonth1080p: firstMonth(video.seconds, 85), + firstMonth1440p: firstMonth(video.seconds, 145), + firstMonth2160p: firstMonth(video.seconds, 280), + })); } + if (keys === "storagePaidUntil" || keys === "createdAt") return []; + throw new Error(`Unexpected select: ${keys}`); } @@ -195,12 +542,19 @@ function select(fields: Record) { return chain; }, innerJoin: () => chain, + leftJoin: () => chain, where(condition: SQL | undefined) { query.where = condition; const rows = Promise.resolve().then(() => runSelect(query)); const result = Object.assign(rows, { orderBy: () => result, - groupBy: () => result, + groupBy: () => + Object.assign(result, { + as: (alias: string) => ({ + appId: sql.raw(`\`${alias}\`.\`appId\``), + heldMicroCredits: sql.raw("`heldMicroCredits`"), + }), + }), limit: () => result, for: () => result, }); @@ -214,7 +568,45 @@ function update(table: unknown) { return { set(values: Record) { return { - async where() { + async where(condition?: SQL) { + if (table === developerCreditTransactions && values.metadata) { + const params = condition + ? dialect.sqlToQuery(condition).params + : []; + const resolvedAt = dialect + .sqlToQuery(values.metadata as SQL) + .params.find((param) => typeof param === "string"); + const [first, ...rest] = params; + const types = rest.slice(0, -1); + const shortSince = rest.at(-1); + const rows = ledger.rows.filter((entry) => + types.includes("video_create") + ? entry.accountId === first && + types.includes(entry.type) && + entry.metadata?.shortSince === shortSince + : entry.id === first, + ); + let affectedRows = 0; + for (const row of rows) { + if (row.metadata?.resolvedAt !== undefined) continue; + row.metadata = { ...row.metadata, resolvedAt }; + affectedRows++; + } + return [{ affectedRows }]; + } + if (table === developerCreditTransactions && condition) { + const [accountId, referenceId, referenceType, rowId] = + dialect.sqlToQuery(condition).params; + const rows = ledger.rows.filter( + (row) => + row.accountId === accountId && + row.referenceId === referenceId && + row.referenceType === referenceType && + (rowId === undefined || row.id === rowId), + ); + for (const row of rows) Object.assign(row, values); + return [{ affectedRows: rows.length }]; + } if (table !== developerCreditAccounts) return [{ affectedRows: 0 }]; const { balanceMicroCredits, ...rest } = values; if (balanceMicroCredits !== undefined) { @@ -265,10 +657,32 @@ function insert(table: unknown) { }; } +function remove(table: unknown) { + return { + async where(condition: SQL) { + if (table !== developerCreditTransactions) return [{ affectedRows: 0 }]; + const [accountId, referenceId, referenceType, rowId] = + dialect.sqlToQuery(condition).params; + const before = ledger.rows.length; + ledger.rows = ledger.rows.filter( + (row) => + !( + row.accountId === accountId && + row.referenceId === referenceId && + row.referenceType === referenceType && + (rowId === undefined || row.id === rowId) + ), + ); + return [{ affectedRows: before - ledger.rows.length }]; + }, + }; +} + type FakeDb = { select: typeof select; update: typeof update; insert: typeof insert; + delete: typeof remove; transaction: (run: (tx: FakeDb) => Promise) => Promise; }; @@ -276,5 +690,6 @@ export const fakeDb: FakeDb = { select, update, insert, + delete: remove, transaction: (run) => run(fakeDb), }; diff --git a/apps/web/__tests__/unit/infra-billing-gates.test.ts b/apps/web/__tests__/unit/infra-billing-gates.test.ts index cabf48c095d..1c11d356f13 100644 --- a/apps/web/__tests__/unit/infra-billing-gates.test.ts +++ b/apps/web/__tests__/unit/infra-billing-gates.test.ts @@ -9,6 +9,7 @@ const mocks = vi.hoisted(() => ({ createSession: vi.fn(), detach: vi.fn(), ensureCustomer: vi.fn(), + scheduleAutoTopUp: vi.fn(), })); vi.mock("@cap/database", () => ({ @@ -94,7 +95,10 @@ vi.mock("@/lib/infra-auto-top-up", async () => { const settings = await vi.importActual< typeof import("@/lib/infra-auto-top-up") >("@/lib/infra-auto-top-up"); - return { autoTopUpSettingsError: settings.autoTopUpSettingsError }; + return { + autoTopUpSettingsError: settings.autoTopUpSettingsError, + scheduleAutoTopUp: mocks.scheduleAutoTopUp, + }; }); vi.mock("@/app/api/utils", async () => { const { createMiddleware } = await import("hono/factory"); @@ -131,6 +135,8 @@ const account = (extra: Record = {}) => ({ stripePaymentMethodId: "pm_1", autoTopUpThresholdMicroCredits: 500_000, autoTopUpAmountCents: 2500, + autoTopUpEnabled: false, + appName: "Demo", ...extra, }); @@ -141,6 +147,7 @@ const cardCheckout = () => }); beforeEach(() => { + mocks.scheduleAutoTopUp.mockReset(); mocks.user = preview; mocks.selects = []; mocks.sets = []; @@ -179,6 +186,7 @@ describe("creating an upload", () => { }); expect(mocks.admission).toHaveBeenCalledWith("app-1"); expect(mocks.inserts).toHaveLength(0); + expect(mocks.scheduleAutoTopUp).toHaveBeenCalledWith({ appId: "app-1" }); }); it("creates the video for free when the app is admitted", async () => { @@ -189,6 +197,7 @@ describe("creating an upload", () => { expect(response.status).toBe(200); const { data } = await response.json(); expect(data).toMatchObject({ id: "video-1", videoId: "video-1" }); + expect(mocks.scheduleAutoTopUp).not.toHaveBeenCalled(); expect(Object.keys(data).sort()).toEqual([ "corsOrigin", "embedUrl", @@ -218,7 +227,9 @@ describe("creating an upload", () => { }); describe("console upload route", () => { - it("is refused for an account outside the preview list without reading the app", async () => { + it("checks ownership for any signed-in account, not a preview list", async () => { + mocks.selects = [[]]; + expect(await ownsInfraApp(outsider as never, "app-1")).toBe(false); expect(mocks.selects).toHaveLength(0); }); @@ -245,10 +256,11 @@ describe("console upload route", () => { }); describe("saving a card", () => { - it("refuses an account outside the preview list before reaching the payment provider", async () => { + it("treats any signed-in account by ownership before reaching the payment provider", async () => { mocks.user = outsider; + mocks.selects = [[]]; - expect(await cardCheckout()).toEqual({ status: "forbidden" }); + expect(await cardCheckout()).toEqual({ status: "not_found" }); expect(mocks.ensureCustomer).not.toHaveBeenCalled(); expect(mocks.createSession).not.toHaveBeenCalled(); }); @@ -282,6 +294,12 @@ describe("saving a card", () => { customer: "cus_1", mode: "setup", payment_method_types: ["card"], + custom_text: { + submit: { + message: + "Saving this card for Demo charges nothing. Auto top-up stays off until you switch it on.", + }, + }, metadata, setup_intent_data: { metadata }, success_url: @@ -290,6 +308,23 @@ describe("saving a card", () => { ); }); + it("says on the card page that saving a card leaves auto top-up as it is", async () => { + mocks.selects = [[account({ autoTopUpEnabled: true })]]; + + await cardCheckout(); + + expect(mocks.createSession).toHaveBeenCalledWith( + expect.objectContaining({ + custom_text: { + submit: { + message: + "Saving this card for Demo charges nothing now. Auto top-up is on and will use this card.", + }, + }, + }), + ); + }); + it("reports a failure when the payment provider refuses", async () => { mocks.selects = [[account()]]; mocks.createSession.mockRejectedValue(new Error("provider down")); @@ -305,11 +340,12 @@ describe("auto top-up settings action", () => { extra: Partial[0]> = {}, ) => updateDeveloperAutoTopUp({ appId: "app-1", enabled: true, ...extra }); - it("refuses a signed-out user and an account outside the preview list", async () => { + it("refuses a signed-out user, and any account for an app it does not own", async () => { mocks.user = null; await expect(save()).rejects.toThrow("Unauthorized"); mocks.user = outsider; - await expect(save()).rejects.toThrow("Unauthorized"); + mocks.selects = [[]]; + await expect(save()).rejects.toThrow("App not found"); expect(mocks.sets).toHaveLength(0); }); @@ -390,12 +426,16 @@ describe("auto top-up settings action", () => { }); describe("removing a saved card", () => { - it("refuses an account outside the preview list", async () => { - mocks.user = outsider; - + it("refuses a signed-out user, and any account for an app it does not own", async () => { + mocks.user = null; await expect(removeDeveloperSavedCard("app-1")).rejects.toThrow( "Unauthorized", ); + mocks.user = outsider; + mocks.selects = [[]]; + await expect(removeDeveloperSavedCard("app-1")).rejects.toThrow( + "App not found", + ); expect(mocks.detach).not.toHaveBeenCalled(); }); diff --git a/apps/web/__tests__/unit/infra-billing-math.test.ts b/apps/web/__tests__/unit/infra-billing-math.test.ts index 1e6c04c8371..6a6f074a0c6 100644 --- a/apps/web/__tests__/unit/infra-billing-math.test.ts +++ b/apps/web/__tests__/unit/infra-billing-math.test.ts @@ -199,6 +199,10 @@ describe("storage rates", () => { describe("upload admission", () => { const active = { state: "active" as const, pausesAt: null }; + const pastFirstMonth = (monthlyMicroCredits: number) => ({ + dailyCharge: { monthlyMicroCredits }, + firstMonthMicroCredits: 0, + }); it("needs seven days of storage, rounded up, and at least one micro-credit", () => { expect(UPLOAD_ADMISSION_DAYS).toBe(7); @@ -207,12 +211,54 @@ describe("upload admission", () => { expect(uploadAdmissionMicroCredits(4335)).toBe(1012); }); + it("needs the whole first month of every video still to be charged, plus seven days of the rest", () => { + expect(uploadAdmissionMicroCredits(0, 5_600)).toBe(5_600); + expect(uploadAdmissionMicroCredits(0, 3 * 5_600)).toBe(16_800); + expect(uploadAdmissionMicroCredits(85_000, 5_600)).toBe(25_434); + expect(uploadAdmissionMicroCredits(0, 0.4)).toBe(1); + }); + + it("says what the credit is for when videos are still to be charged their first month", () => { + const firstMonthOnly = uploadAdmission({ + balanceMicroCredits: 5_599, + standing: active, + lastOwedMicroCredits: 0, + storage: { + dailyCharge: { monthlyMicroCredits: 0 }, + firstMonthMicroCredits: 5_600, + }, + }); + expect(firstMonthOnly).toMatchObject({ + allowed: false, + reason: "insufficient_credits", + requiredMicroCredits: 5_600, + shortfallMicroCredits: 1, + }); + expect(firstMonthOnly.message).toBe( + "Not enough credit to upload. Uploading needs $0.05600, enough for the first month of storage for each video being uploaded or encoded. The balance is $0.05599, so add at least $0.00001.", + ); + + const both = uploadAdmission({ + balanceMicroCredits: 0, + standing: active, + lastOwedMicroCredits: 0, + storage: { + dailyCharge: { monthlyMicroCredits: 85_000 }, + firstMonthMicroCredits: 16_800, + }, + }); + expect(both.requiredMicroCredits).toBe(36_634); + expect(both.message).toBe( + "Not enough credit to upload. Add at least $0.36634, enough for the first month of storage for each video being uploaded or encoded and 7 days of storage for the videos past their first month.", + ); + }); + it("admits when the balance covers seven days of everything stored", () => { const admission = uploadAdmission({ balanceMicroCredits: 19_834, standing: active, lastOwedMicroCredits: 0, - usage: { monthlyMicroCredits: 85_000 }, + storage: pastFirstMonth(85_000), }); expect(admission).toMatchObject({ allowed: true, @@ -228,13 +274,13 @@ describe("upload admission", () => { balanceMicroCredits: 19_833, standing: active, lastOwedMicroCredits: 0, - usage: { monthlyMicroCredits: 85_000 }, + storage: pastFirstMonth(85_000), }); expect(admission.allowed).toBe(false); expect(admission.reason).toBe("insufficient_credits"); expect(admission.shortfallMicroCredits).toBe(1); expect(admission.message).toBe( - "Uploading needs $0.19834 of credit, enough for 7 days of storage for everything this app stores. The balance is $0.19833, so add at least $0.00001.", + "Not enough credit to upload. Uploading needs $0.19834, enough for 7 days of storage for the videos past their first month. The balance is $0.19833, so add at least $0.00001.", ); }); @@ -243,12 +289,12 @@ describe("upload admission", () => { balanceMicroCredits: 967, standing: active, lastOwedMicroCredits: 0, - usage: { monthlyMicroCredits: 4_675 }, + storage: pastFirstMonth(4_675), }); expect(admission.requiredMicroCredits).toBe(1_091); expect(admission.shortfallMicroCredits).toBe(124); expect(admission.message).toBe( - "Uploading needs $0.01091 of credit, enough for 7 days of storage for everything this app stores. The balance is $0.00967, so add at least $0.00124.", + "Not enough credit to upload. Uploading needs $0.01091, enough for 7 days of storage for the videos past their first month. The balance is $0.00967, so add at least $0.00124.", ); }); @@ -257,12 +303,12 @@ describe("upload admission", () => { balanceMicroCredits: 0, standing: active, lastOwedMicroCredits: 0, - usage: { monthlyMicroCredits: 0 }, + storage: pastFirstMonth(0), }); expect(admission.allowed).toBe(false); expect(admission.reason).toBe("insufficient_credits"); expect(admission.message).toBe( - "This app has no credit. Add credits to start uploading.", + "Not enough credit to upload. Add credits to start uploading.", ); }); @@ -272,7 +318,7 @@ describe("upload admission", () => { balanceMicroCredits: 50, standing: { state, pausesAt: new Date("2026-11-18T00:00:00Z") }, lastOwedMicroCredits: 144, - usage: { monthlyMicroCredits: 4335 }, + storage: pastFirstMonth(4335), }); expect(admission.allowed).toBe(false); expect(admission.reason).toBe("out_of_credit"); @@ -289,7 +335,7 @@ describe("upload admission", () => { balanceMicroCredits: 0, standing: { state: "grace", pausesAt: new Date() }, lastOwedMicroCredits: 5000, - usage: { monthlyMicroCredits: 300 }, + storage: pastFirstMonth(300), }); expect(admission.requiredMicroCredits).toBe(5000); }); @@ -301,12 +347,15 @@ describe("upload admission", () => { balanceMicroCredits: 0, standing: { state: "grace", pausesAt }, lastOwedMicroCredits: 144, - usage: { monthlyMicroCredits: 4335 }, + storage: pastFirstMonth(4335), }), + "app-1", ); expect(body).toMatchObject({ error: "Insufficient credits", code: "out_of_credit", + hint: "Add credits to this app at https://cap.so/dashboard/infra/app-1/billing, then retry.", + docs: "https://cap.so/docs/api/rest-api", standing: "grace", pausesAt: pausesAt.toISOString(), balanceMicroCredits: 0, @@ -515,4 +564,38 @@ describe("standing", () => { .shortSince, ).toBe(null); }); + + it("keeps a carried first month shortfall in grace until the balance covers what is still unpaid", () => { + const carried = parseStorageCharge({ + createdAt: at("2026-11-12T09:00:00Z"), + metadata: { + owedMicroCredits: 850, + chargedMicroCredits: 850, + outstandingMicroCredits: 16_800, + shortSince: shortSince.toISOString(), + fellShort: false, + }, + }); + + expect(carried).toMatchObject({ owedMicroCredits: 16_800, shortSince }); + expect( + resolveStanding({ + balanceMicroCredits: 50, + lastCharge: carried, + restored: false, + now: at("2026-11-12T09:00:00Z"), + }), + ).toEqual({ + state: "grace", + pausesAt: at("2026-11-18T06:41:00Z"), + }); + expect( + resolveStanding({ + balanceMicroCredits: 16_800, + lastCharge: carried, + restored: false, + now: at("2026-11-12T09:00:00Z"), + }).state, + ).toBe("active"); + }); }); diff --git a/apps/web/__tests__/unit/infra-console-harness.ts b/apps/web/__tests__/unit/infra-console-harness.ts index 9d3f6ada2f0..90f01b4e9ac 100644 --- a/apps/web/__tests__/unit/infra-console-harness.ts +++ b/apps/web/__tests__/unit/infra-console-harness.ts @@ -140,7 +140,7 @@ export function stubStorageUploads(options: { finish?: boolean } = {}) { return requests; } -export function stubVideoProbe() { +export function stubVideoProbe({ duration = 12 }: { duration?: number } = {}) { vi.stubGlobal("URL", { ...URL, createObjectURL: () => "blob:probe", @@ -153,7 +153,7 @@ export function stubVideoProbe() { ) => { if (tag !== "video") return create(tag, options); const probe = { - duration: 12, + duration, videoWidth: 1280, videoHeight: 720, preload: "", diff --git a/apps/web/__tests__/unit/infra-console-round-trips.test.ts b/apps/web/__tests__/unit/infra-console-round-trips.test.ts new file mode 100644 index 00000000000..e91c11c7d44 --- /dev/null +++ b/apps/web/__tests__/unit/infra-console-round-trips.test.ts @@ -0,0 +1,265 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const state = vi.hoisted(() => ({ + queries: [] as string[], + appId: "app-1", +})); + +vi.mock("server-only", () => ({})); +vi.mock("@cap/database", async () => { + const { drizzle } = await import("drizzle-orm/mysql-proxy"); + const CREATED = "2026-11-01 00:00:00"; + const accountValues: Record = { + id: "account-1", + appId: "app-1", + ownerId: "user-1", + balanceMicroCredits: 5_000_000, + stripeCustomerId: null, + stripePaymentMethodId: null, + autoTopUpEnabled: 0, + autoTopUpThresholdMicroCredits: 0, + autoTopUpAmountCents: 0, + createdAt: CREATED, + updatedAt: CREATED, + }; + const appValues: Record[] = [ + { id: "app-1", name: "First app", environment: "production" }, + { id: "app-2", name: "Second app", environment: "development" }, + ].map((app) => ({ + ...app, + ownerId: "user-1", + logoUrl: null, + deletedAt: null, + createdAt: CREATED, + updatedAt: CREATED, + })); + const selected = (sql: string) => + sql + .slice("select ".length, sql.indexOf(" from ")) + .split(", ") + .map((column) => column.replaceAll("`", "").split(".").at(-1) ?? ""); + const client = drizzle(async (sql) => { + state.queries.push(sql); + if (sql.includes("from `developer_apps`")) { + return { + rows: appValues.map((app) => selected(sql).map((name) => app[name])), + }; + } + if ( + sql.includes("from `developer_credit_accounts`") && + !sql.includes("select COALESCE") + ) { + return { rows: [selected(sql).map((name) => accountValues[name])] }; + } + if (sql.includes("COALESCE(SUM(") && !sql.includes("group by")) { + return { rows: [[0]] }; + } + if (sql.startsWith("select count(*)")) return { rows: [[0]] }; + return { rows: [] }; + }); + return { db: () => client }; +}); +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: async () => ({ + id: "user-1", + email: "owner@example.com", + activeOrganizationId: null, + }), +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + serverEnv: () => ({}), + NODE_ENV: "production", +})); +vi.mock("next/font/google", () => { + const font = () => ({ className: "", variable: "", style: {} }); + return { Geist: font, Geist_Mono: font }; +}); +vi.mock("next/headers", () => ({ + cookies: async () => ({ get: () => undefined }), + headers: async () => new Headers(), +})); +vi.mock("next/navigation", () => ({ + notFound: () => { + throw new Error("notFound"); + }, + redirect: (to: string) => { + throw new Error(`redirect ${to}`); + }, + usePathname: () => "/dashboard/infra/app-1", + useRouter: () => ({}), +})); +vi.mock("@/lib/infra-access", () => ({ canUseInfra: () => true })); +vi.mock("@/app/api/utils", async () => { + const { createMiddleware } = await import("hono/factory"); + return { + withDeveloperSecretAuth: createMiddleware(async (c, next) => { + c.set("developerAppId", state.appId); + await next(); + }), + }; +}); + +import InfraBillingPage from "@/app/(infra)/dashboard/infra/[appId]/billing/page"; +import InfraKeysPage from "@/app/(infra)/dashboard/infra/[appId]/keys/page"; +import InfraAppLayout from "@/app/(infra)/dashboard/infra/[appId]/layout"; +import InfraOverviewPage from "@/app/(infra)/dashboard/infra/[appId]/page"; +import InfraSettingsPage from "@/app/(infra)/dashboard/infra/[appId]/settings/page"; +import InfraUsagePage from "@/app/(infra)/dashboard/infra/[appId]/usage/page"; +import { app as usageApi } from "@/app/api/developer/v1/[...route]/usage"; +import { getInfraBilling, getInfraUploadAdmission } from "@/lib/infra-standing"; + +const params = { params: Promise.resolve({ appId: "app-1" }) }; + +const storageAggregates = () => + state.queries.filter( + (sql) => sql.includes("from `developer_videos`") && sql.includes("SUM("), + ); +const reads = (table: string) => + state.queries.filter((sql) => sql.includes(`from \`${table}\``)); + +type Element = { props: Record }; + +beforeEach(() => { + state.queries = []; +}); + +describe("storage totals are computed once per request", () => { + it("on the console overview", async () => { + await InfraOverviewPage(params); + + expect(storageAggregates()).toHaveLength(1); + expect(state.queries).toHaveLength(10); + }); + + it("on the console usage page", async () => { + await InfraUsagePage(params); + + expect(storageAggregates()).toHaveLength(1); + }); + + it("on the console billing page", async () => { + await InfraBillingPage({ ...params, searchParams: Promise.resolve({}) }); + + expect(storageAggregates()).toHaveLength(1); + }); + + it("on GET /usage", async () => { + const response = await usageApi.request("/"); + + expect(response.status).toBe(200); + expect(storageAggregates()).toHaveLength(1); + }); + + it("and not at all when an admission is given the totals it already has", async () => { + const billing = await getInfraBilling("app-1"); + state.queries = []; + + const admission = await getInfraUploadAdmission( + "app-1", + undefined, + undefined, + { storage: billing.storage }, + ); + + expect(storageAggregates()).toHaveLength(0); + expect(admission).toEqual(billing.admission); + }); +}); + +describe("the billing page lists the ledger without counting it", () => { + const counts = () => + state.queries.filter((sql) => sql.startsWith("select count(")); + const ledgerReads = () => + reads("developer_credit_transactions").filter((sql) => + sql.includes("inner join `developer_credit_accounts`"), + ); + + it("reads one page and no total", async () => { + const page = (await InfraBillingPage({ + ...params, + searchParams: Promise.resolve({}), + })) as Element; + + expect(ledgerReads()).toHaveLength(1); + expect(ledgerReads()[0]).toContain("limit ?"); + expect(ledgerReads()[0]).not.toContain("offset"); + expect(counts()).toHaveLength(0); + expect(page.props).toMatchObject({ + transactions: [], + ledgerPage: { newer: null, older: null }, + }); + expect(page.props).not.toHaveProperty("totalTransactions"); + }); + + it("starts from the cursor in the URL", async () => { + await InfraBillingPage({ + ...params, + searchParams: Promise.resolve({ before: "1794461929000.abc123" }), + }); + + expect(ledgerReads()).toHaveLength(1); + expect(ledgerReads()[0]).toContain("`createdAt` < ?"); + expect(counts()).toHaveLength(0); + }); +}); + +describe("the console loads only what a page shows", () => { + it("reads the app list and the open app's account for the shell, nothing else", async () => { + const layout = (await InfraAppLayout({ + children: null, + ...params, + })) as Element; + + expect(state.queries).toHaveLength(2); + expect(reads("developer_apps")).toHaveLength(1); + expect(reads("developer_credit_accounts")).toHaveLength(1); + expect(layout.props.balanceMicroCredits).toBe(5_000_000); + const shell = layout.props.children as Element; + expect(shell.props).toMatchObject({ + appId: "app-1", + balanceMicroCredits: 5_000_000, + apps: [ + { id: "app-1", name: "First app", environment: "production" }, + { id: "app-2", name: "Second app", environment: "development" }, + ], + }); + }); + + it("never loads domains, other apps' keys or a video count for the overview", async () => { + await InfraOverviewPage(params); + + expect(reads("developer_app_domains")).toHaveLength(0); + expect(reads("developer_api_keys")).toHaveLength(1); + expect(reads("developer_api_keys")[0]).toContain("`appId` = ?"); + expect( + state.queries.filter((sql) => sql.includes("count(*)")), + ).toHaveLength(0); + }); + + it("reads one app's keys for the keys page", async () => { + const page = (await InfraKeysPage(params)) as Element; + + expect(state.queries).toHaveLength(3); + expect(reads("developer_api_keys")[0]).toContain("`appId` = ?"); + expect(page.props).toEqual({ appId: "app-1", secretKeyPrefix: null }); + }); + + it("counts videos and lists allowed sites only on the settings page, and only for that app", async () => { + const page = (await InfraSettingsPage(params)) as Element; + + expect(state.queries).toHaveLength(4); + const [count] = reads("developer_videos"); + expect(count).toContain("count(*)"); + expect(count).toContain("`appId` = ?"); + const [domains] = reads("developer_app_domains"); + expect(domains).toContain("`appId` = ?"); + expect(page.props).toMatchObject({ + appId: "app-1", + name: "First app", + environment: "production", + balanceMicroCredits: 5_000_000, + videoCount: 0, + }); + }); +}); diff --git a/apps/web/__tests__/unit/infra-console-upload-errors.test.ts b/apps/web/__tests__/unit/infra-console-upload-errors.test.ts new file mode 100644 index 00000000000..a40d9b59299 --- /dev/null +++ b/apps/web/__tests__/unit/infra-console-upload-errors.test.ts @@ -0,0 +1,198 @@ +import type { HttpApi } from "@effect/platform"; +import type { Context, Layer as EffectLayer } from "effect"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; +import { consoleErrorBody } from "@/lib/infra-console-errors"; + +const mocks = vi.hoisted(() => ({ + owns: true, + create: null as null | (() => Promise), + dispose: async () => {}, +})); + +vi.mock("server-only", () => ({})); + +vi.mock("@/lib/infra-video-upload", () => ({ + MAX_PRESIGN_BATCH: 50, + ownsInfraApp: async () => mocks.owns, + createInfraVideo: () => mocks.create?.(), + initiateInfraUpload: async () => ({ status: 200, body: { uploadId: "u" } }), + presignInfraUploadParts: async () => ({ status: 200, body: { parts: [] } }), + completeInfraUpload: async () => ({ status: 200, body: { success: true } }), + abortInfraUpload: async () => ({ status: 200, body: { success: true } }), +})); + +vi.mock("@/lib/server", async () => { + const { HttpAuthMiddleware, Organisation, User } = await import( + "@cap/web-domain" + ); + const { HttpApiBuilder, HttpServer } = await import("@effect/platform"); + const { Effect, Layer, Option } = await import("effect"); + const auth = Layer.succeed( + HttpAuthMiddleware, + Effect.succeed({ + id: User.UserId.make("user"), + email: "user@example.com", + activeOrganizationId: Organisation.OrganisationId.make("org"), + iconUrlOrKey: Option.none(), + }), + ); + return { + apiToHandler: ( + api: EffectLayer.Layer< + HttpApi.Api, + never, + Context.Tag.Identifier + >, + ) => { + const handler = api.pipe( + Layer.provideMerge(auth), + Layer.merge(HttpServer.layerContext), + HttpApiBuilder.toWebHandler, + ); + mocks.dispose = handler.dispose; + return handler.handler; + }, + }; +}); + +const SHAPE = ["code", "docs", "error", "hint", "message"]; +const REFUSAL_MESSAGE = + "Not enough credit to upload. Add credits to start uploading."; + +describe("console upload errors", () => { + let route: typeof import("@/app/api/developer/console/upload/[...route]/route"); + + beforeAll(async () => { + route = await import("@/app/api/developer/console/upload/[...route]/route"); + }); + + beforeEach(() => { + mocks.owns = true; + mocks.create = null; + }); + + afterAll(() => mocks.dispose()); + + const post = (path: string, body: unknown) => + route.POST( + new Request(`http://localhost/api/developer/console/upload/${path}`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }), + ); + + const read = async (response: Response) => { + const text = await response.text(); + expect(text).not.toContain("_tag"); + const body = JSON.parse(text) as Record; + expect(Object.keys(body).sort()).toEqual(SHAPE); + return body; + }; + + it("answers a credit refusal with its plain message and the app's Billing link", async () => { + mocks.create = async () => ({ + status: 402, + body: { + error: "Insufficient credits", + code: "insufficient_credits", + message: REFUSAL_MESSAGE, + hint: "Add credits to this app at https://cap.so/dashboard/infra/app-1/billing, then retry.", + docs: "https://cap.so/docs/api/rest-api", + standing: "active", + pausesAt: null, + balanceMicroCredits: 0, + requiredMicroCredits: 1, + shortfallMicroCredits: 1, + }, + }); + + const response = await post("create", { appId: "app-1" }); + + expect(response.status).toBe(402); + expect(response.headers.get("content-type")).toBe("application/json"); + expect(await read(response)).toEqual({ + error: REFUSAL_MESSAGE, + code: "insufficient_credits", + message: REFUSAL_MESSAGE, + hint: "Add credits to this app at https://cap.so/dashboard/infra/app-1/billing, then retry.", + docs: "https://cap.so/docs/api/rest-api", + }); + }); + + it("words a request the schema rejects plainly, without the decoder's detail", async () => { + const response = await post("presign", { + appId: "app-1", + videoId: "v", + uploadId: "u", + partNumbers: [0], + }); + + expect(response.status).toBe(400); + expect(await read(response)).toMatchObject({ + code: "invalid_request", + message: "The upload request was not valid", + }); + }); + + it("says App not found for an app the user does not own", async () => { + mocks.owns = false; + const response = await post("create", { appId: "someone-else" }); + + expect(response.status).toBe(404); + expect(await read(response)).toMatchObject({ + code: "not_found", + message: "App not found", + }); + }); + + it("hides what went wrong inside when the upload throws", async () => { + mocks.create = async () => { + throw new Error("connect ECONNREFUSED 10.0.0.1:3306"); + }; + const response = await post("create", { appId: "app-1" }); + + expect(response.status).toBe(500); + const body = await read(response); + expect(body).toMatchObject({ + code: "upload_failed", + message: "Upload failed", + }); + expect(JSON.stringify(body)).not.toContain("ECONNREFUSED"); + }); + + it("passes a success through untouched", async () => { + mocks.create = async () => ({ status: 200, body: { videoId: "v_1" } }); + const response = await post("create", { appId: "app-1" }); + + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ videoId: "v_1" }); + }); +}); + +describe("consoleErrorBody", () => { + it.each([ + [400, { _tag: "HttpApiDecodeError", message: "Expected string" }], + [401, { _tag: "Unauthorized" }], + [402, { _tag: "SomethingElse", message: "internal" }], + [429, null], + [500, "not json"], + [503, { _tag: "UploadFailed" }], + ])("keeps a %i in the five-field shape with no internals", (status, raw) => { + const body = consoleErrorBody(status, raw); + + expect(Object.keys(body).sort()).toEqual(SHAPE); + expect(JSON.stringify(body)).not.toContain("_tag"); + expect(body.message).not.toBe("internal"); + expect(body.message).not.toBe("Expected string"); + expect(body.error).toBe(body.message); + }); +}); diff --git a/apps/web/__tests__/unit/infra-console-uploads.test.ts b/apps/web/__tests__/unit/infra-console-uploads.test.ts index 15dfcba70b8..c9d9d58659a 100644 --- a/apps/web/__tests__/unit/infra-console-uploads.test.ts +++ b/apps/web/__tests__/unit/infra-console-uploads.test.ts @@ -1,7 +1,18 @@ // @vitest-environment jsdom -import { createElement, type FunctionComponent, type ReactNode } from "react"; +import { + act, + createElement, + type FunctionComponent, + type ReactNode, +} from "react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + fileTooLargeMessage, + MAX_UPLOAD_BYTES, + MAX_UPLOAD_DURATION_SECS, + VIDEO_TOO_LONG_MESSAGE, +} from "@/lib/infra-upload"; import type { InfraVideoListItem } from "@/lib/infra-video-view"; import { button, @@ -154,6 +165,7 @@ beforeEach(() => { }); afterEach(async () => { + vi.useRealTimers(); await stage.close(); }); @@ -225,6 +237,65 @@ describe("a video uploaded from the console", () => { expect(stage.seen.filter((frame) => FAILURE_TEXT.test(frame))).toEqual([]); }); + it("waits out a completion another request holds, then finishes", async () => { + const answerApi = globalThis.fetch as unknown as ( + url: string, + init: { body: string }, + ) => Promise; + const firstAnswer = deferred(); + let completeCalls = 0; + vi.stubGlobal( + "fetch", + vi.fn(async (url: string, init: { body: string }) => { + if (!url.endsWith("/complete")) return answerApi(url, init); + apiCalls.push("complete"); + completeCalls++; + if (completeCalls === 1) await firstAnswer.promise; + if (Date.now() < busyUntil) { + return new Response( + JSON.stringify({ + error: "Another request is completing this upload.", + code: "completion_in_progress", + }), + { status: 409, headers: { "Retry-After": "5" } }, + ); + } + return new Response(JSON.stringify({ success: true }), { + status: 200, + }); + }), + ); + let busyUntil = Number.POSITIVE_INFINITY; + await show([video("old", "ready")]); + + await chooseFiles([file("talk.mp4")]); + await until(() => completeCalls === 1, "the completion call"); + + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "Date"] }); + busyUntil = Date.now() + 60_000; + firstAnswer.resolve(); + for (let waited = 0; waited < 60_000; waited += 5_000) { + await act(() => vi.advanceTimersByTimeAsync(5_000)); + if (waited === 5_000) { + expect(text()).toContain( + "Still finishing, this can take a few minutes", + ); + } + } + await act(() => vi.advanceTimersByTimeAsync(5_000)); + vi.useRealTimers(); + await until( + () => mocks.toast.success.mock.calls.length === 1, + "the uploaded toast", + ); + + expect(mocks.toast.success).toHaveBeenCalledWith("Uploaded talk.mp4"); + expect(completeCalls).toBe(13); + expect(apiCalls).not.toContain("abort"); + expect(mocks.deleteVideo).not.toHaveBeenCalled(); + expect(text()).not.toMatch(/not confirmed|Still finishing/); + }); + it("still calls an upload this tab did not send unfinished, and offers to remove it", async () => { await show([video("left-behind", "uploading"), video("old", "ready")]); @@ -255,6 +326,35 @@ describe("a file the console cannot upload", () => { "That file is empty.", ); }); + + it("refuses a file over the upload limit before creating a video", async () => { + await show([video("old", "ready")]); + const huge = file("huge.mp4"); + Object.defineProperty(huge, "size", { value: MAX_UPLOAD_BYTES + 1 }); + + await chooseFiles([huge]); + await until( + () => text().includes(fileTooLargeMessage()), + "the size message", + ); + + expect(apiCalls).toEqual([]); + expect(mocks.deleteVideo).not.toHaveBeenCalled(); + }); + + it("refuses a video over the length limit before creating a video", async () => { + stubVideoProbe({ duration: MAX_UPLOAD_DURATION_SECS + 1 }); + await show([video("old", "ready")]); + + await chooseFiles([file("long.mp4")]); + await until( + () => text().includes(VIDEO_TOO_LONG_MESSAGE), + "the length message", + ); + + expect(apiCalls).toEqual([]); + expect(mocks.deleteVideo).not.toHaveBeenCalled(); + }); }); describe("the videos list", () => { diff --git a/apps/web/__tests__/unit/infra-credit-checkout-text.test.ts b/apps/web/__tests__/unit/infra-credit-checkout-text.test.ts new file mode 100644 index 00000000000..e80d15420d0 --- /dev/null +++ b/apps/web/__tests__/unit/infra-credit-checkout-text.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@/lib/developer-credits-product", () => ({ + developerCreditsProductId: () => "prod_credits", +})); + +import { + CREDIT_CHECKOUT_3DS_OVER_CENTS, + creditCheckoutSessionParams, + creditPurchaseDescription, + STRIPE_DESCRIPTION_LIMIT, +} from "@/lib/infra-credit-checkout"; +import { parseCreditUsEin } from "@/lib/infra-credit-us-ein"; + +describe("credit purchase text", () => { + it("uses the app name as stored, trimmed", () => { + expect(creditPurchaseDescription(" Course\n site ")).toBe( + "Cap Infra credits for Course site", + ); + }); + + it("falls back to the plain product name for a blank app name", () => { + expect(creditPurchaseDescription(" ")).toBe("Cap Infra credits"); + }); + + it("caps the description at Stripe's 1,000 characters, like auto top-up", () => { + const description = creditPurchaseDescription("x".repeat(5_000)); + expect(description.length).toBe(STRIPE_DESCRIPTION_LIMIT); + expect(description.startsWith("Cap Infra credits for x")).toBe(true); + }); + + it("falls back when there is no name at all", () => { + expect(creditPurchaseDescription(null)).toBe("Cap Infra credits"); + expect(creditPurchaseDescription(undefined)).toBe("Cap Infra credits"); + }); + + it("keeps the credits product so codes still apply", () => { + const params = creditCheckoutSessionParams({ + customerId: "cus_1", + accountId: "account-1", + appName: "Course site", + amountCents: 1000, + receiptEmail: "owner@example.com", + }); + expect(params.line_items[0]?.price_data.product).toBe("prod_credits"); + expect(params.payment_method_types).toEqual(["card"]); + }); +}); + +describe("3D Secure on large credit purchases", () => { + const paramsFor = (amountCents: number) => + creditCheckoutSessionParams({ + customerId: "cus_1", + accountId: "account-1", + appName: "Course site", + amountCents, + receiptEmail: "owner@example.com", + }); + + it("offers card only on every credit checkout, whatever the amount", () => { + for (const amountCents of [500, 5_000, 100_000, 100_001, 1_000_000]) { + expect(paramsFor(amountCents).payment_method_types).toEqual(["card"]); + } + }); + + it("sends no card options at exactly $1,000", () => { + expect(CREDIT_CHECKOUT_3DS_OVER_CENTS).toBe(100_000); + expect(paramsFor(100_000)).not.toHaveProperty("payment_method_options"); + expect(paramsFor(5_000)).not.toHaveProperty("payment_method_options"); + }); + + it("asks for 3D Secure from $1,000.01", () => { + for (const amountCents of [100_001, 500_000, 1_000_000]) { + const params = paramsFor(amountCents); + expect(params.payment_method_types).toEqual(["card"]); + expect(params.payment_method_options).toEqual({ + card: { request_three_d_secure: "any" }, + }); + } + }); + + it("builds the same session params every time for the same amount, so a retry under one Stripe key matches", () => { + for (const amountCents of [5_000, 100_000, 100_001, 1_000_000]) { + const first = paramsFor(amountCents); + const retry = paramsFor(amountCents); + expect(JSON.stringify(retry)).toBe(JSON.stringify(first)); + } + }); + + it("changes nothing else about the session", () => { + const { + payment_method_options: _options, + line_items: largeItems, + ...large + } = paramsFor(100_001); + const { line_items: smallItems, ...small } = paramsFor(100_000); + expect(large).toEqual(small); + expect(largeItems[0]?.price_data.unit_amount).toBe(100_001); + expect(smallItems[0]?.price_data.unit_amount).toBe(100_000); + }); +}); + +describe("parseCreditUsEin", () => { + it("formats nine digits with or without a dash", () => { + expect(parseCreditUsEin("123456789")).toEqual({ + state: "valid", + ein: "12-3456789", + }); + expect(parseCreditUsEin(" 12-3456789 ")).toEqual({ + state: "valid", + ein: "12-3456789", + }); + }); + + it("treats a missing or blank EIN as none", () => { + expect(parseCreditUsEin(undefined)).toEqual({ state: "none" }); + expect(parseCreditUsEin(" ")).toEqual({ state: "none" }); + }); + + it("refuses anything that is not nine digits", () => { + expect(parseCreditUsEin("12-345")).toEqual({ state: "invalid" }); + expect(parseCreditUsEin("AB-CDEFGHI")).toEqual({ state: "invalid" }); + }); +}); diff --git a/apps/web/__tests__/unit/infra-credit-invoice-webhook.test.ts b/apps/web/__tests__/unit/infra-credit-invoice-webhook.test.ts new file mode 100644 index 00000000000..89640f887ad --- /dev/null +++ b/apps/web/__tests__/unit/infra-credit-invoice-webhook.test.ts @@ -0,0 +1,256 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + constructEvent: vi.fn(), + retrieveSubscription: vi.fn(), + listSubscriptions: vi.fn(), + retrieveCustomer: vi.fn(), + updateCustomer: vi.fn(), + dbCalls: [] as string[], + existingGrants: [] as { id: string }[][], + addCredits: vi.fn(), + sendEmail: vi.fn(), + enqueueLoopsSync: vi.fn(), + attachSsoCheckout: vi.fn(), + getSsoBillingForSubscription: vi.fn(), + syncSsoSubscription: vi.fn(), + saveCardFromCheckout: vi.fn(), +})); + +const chain: Record = new Proxy( + {}, + { + get(_target, key) { + if (key === "then") return undefined; + return (..._args: unknown[]) => { + mocks.dbCalls.push(String(key)); + if (key === "limit") { + return Promise.resolve(mocks.existingGrants.shift() ?? []); + } + return chain; + }; + }, + }, +); + +vi.mock("@cap/database", () => ({ db: () => chain })); +vi.mock("@cap/database/helpers", () => ({ + nanoId: () => "id", + nanoIdLength: 15, +})); +vi.mock("@cap/database/emails/config", () => ({ sendEmail: mocks.sendEmail })); +vi.mock("@cap/database/emails/infra-out-of-credit", () => ({ + InfraOutOfCredit: () => null, +})); +vi.mock("@cap/database/emails/infra-top-up-failed", () => ({ + InfraTopUpFailed: () => null, +})); +vi.mock("@cap/database/loops/queue", () => ({ + enqueueLoopsSync: mocks.enqueueLoopsSync, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + serverEnv: () => ({ + STRIPE_WEBHOOK_SECRET: "whsec_test", + WEB_URL: "https://cap.test", + }), +})); +vi.mock("@cap/utils", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + stripe: () => ({ + webhooks: { constructEvent: mocks.constructEvent }, + subscriptions: { + retrieve: mocks.retrieveSubscription, + list: mocks.listSubscriptions, + }, + customers: { + retrieve: mocks.retrieveCustomer, + update: mocks.updateCustomer, + }, + }), + }; +}); +vi.mock("@/lib/developer-credits", () => ({ + addCreditsToAccount: mocks.addCredits, +})); +vi.mock("@/lib/developer-credits-product", () => ({ + developerCreditsProductId: () => "prod_credits", +})); +vi.mock("@/lib/server-analytics", () => ({ trackServerEvent: vi.fn() })); +vi.mock("@/lib/sso/billing", () => ({ + attachSsoCheckout: mocks.attachSsoCheckout, + getSsoBillingForSubscription: mocks.getSsoBillingForSubscription, + syncSsoSubscription: mocks.syncSsoSubscription, +})); + +import { POST } from "@/app/api/webhooks/stripe/route"; + +const PAYMENT_INTENT = "pi_credit"; +const INVOICE = "in_credit"; + +const session = { + id: "cs_credit", + object: "checkout.session", + mode: "payment", + customer: "cus_owner", + subscription: null, + invoice: INVOICE, + payment_status: "paid", + payment_intent: PAYMENT_INTENT, + amount_total: 2500, + metadata: { + type: "developer_credits", + appId: "app-1", + accountId: "account-1", + amountCents: "2500", + userId: "user-1", + }, +}; + +const invoice = { + id: INVOICE, + object: "invoice", + customer: "cus_owner", + subscription: null, + subscription_details: { metadata: null }, + billing_reason: "manual", + description: "Cap Infra credits", + status: "paid", + payment_intent: PAYMENT_INTENT, + charge: "ch_credit", + attempt_count: 1, + next_payment_attempt: null, + lines: { + data: [ + { + id: "il_credit", + amount: 2500, + price: { id: "price_adhoc", product: "prod_credits" }, + quantity: 1, + }, + ], + }, +}; + +const paymentTag = { type: "developer_credits", accountId: "account-1" }; + +const paymentIntent = { + id: PAYMENT_INTENT, + object: "payment_intent", + status: "succeeded", + amount: 2500, + amount_received: 2500, + currency: "usd", + invoice: INVOICE, + metadata: paymentTag, +}; + +const charge = { + id: "ch_credit", + object: "charge", + payment_intent: PAYMENT_INTENT, + invoice: INVOICE, + amount: 2500, + metadata: paymentTag, +}; + +const invoicePayment = { + id: "inpay_credit", + object: "invoice_payment", + invoice: INVOICE, + payment: { type: "payment_intent", payment_intent: PAYMENT_INTENT }, +}; + +async function deliver(type: string, object: Record) { + mocks.constructEvent.mockReturnValue({ type, data: { object } }); + return POST( + new Request("https://cap.test/api/webhooks/stripe", { + method: "POST", + headers: { "Stripe-Signature": "sig_test" }, + body: "{}", + }), + ); +} + +function expectNothingElseTouched() { + expect(mocks.syncSsoSubscription).not.toHaveBeenCalled(); + expect(mocks.getSsoBillingForSubscription).not.toHaveBeenCalled(); + expect(mocks.attachSsoCheckout).not.toHaveBeenCalled(); + expect(mocks.retrieveSubscription).not.toHaveBeenCalled(); + expect(mocks.listSubscriptions).not.toHaveBeenCalled(); + expect(mocks.retrieveCustomer).not.toHaveBeenCalled(); + expect(mocks.updateCustomer).not.toHaveBeenCalled(); + expect(mocks.enqueueLoopsSync).not.toHaveBeenCalled(); + expect(mocks.sendEmail).not.toHaveBeenCalled(); + expect(mocks.dbCalls).not.toContain("update"); + expect(mocks.dbCalls).not.toContain("insert"); + expect(mocks.dbCalls).not.toContain("delete"); +} + +beforeEach(() => { + vi.clearAllMocks(); + mocks.dbCalls.length = 0; + mocks.existingGrants.length = 0; + mocks.addCredits.mockResolvedValue(2_500_000); +}); + +describe("the invoice Stripe creates for a Cap Infra credit purchase", () => { + it("answers invoice.paid with 200 and touches no billing state", async () => { + const response = await deliver("invoice.paid", invoice); + + expect(response.status).toBe(200); + expect(mocks.addCredits).not.toHaveBeenCalled(); + expect(mocks.dbCalls).toEqual([]); + expectNothingElseTouched(); + }); + + it.each([ + ["invoice.created", invoice], + ["invoice.finalized", invoice], + ["invoice.payment_succeeded", invoice], + ["invoice.sent", invoice], + ["invoice_payment.paid", invoicePayment], + ["charge.succeeded", charge], + ["charge.updated", charge], + ])("leaves %s without side effects", async (type, object) => { + await deliver(type, object); + + expect(mocks.addCredits).not.toHaveBeenCalled(); + expect(mocks.dbCalls).toEqual([]); + expectNothingElseTouched(); + }); + + it("grants the purchase once, from checkout.session.completed, across every event Stripe sends for it", async () => { + const events: [string, Record][] = [ + ["payment_intent.created", { ...paymentIntent, status: "processing" }], + ["payment_intent.succeeded", paymentIntent], + ["charge.succeeded", charge], + ["invoice.created", invoice], + ["invoice.finalized", invoice], + ["charge.updated", charge], + ["checkout.session.completed", session], + ["invoice.payment_succeeded", invoice], + ["invoice.paid", invoice], + ["invoice.sent", invoice], + ["invoice_payment.paid", invoicePayment], + ]; + for (const [type, object] of events) { + await deliver(type, object); + } + mocks.existingGrants.push([{ id: "txn_credit" }]); + await deliver("invoice.paid", invoice); + await deliver("checkout.session.completed", session); + + expect(mocks.addCredits).toHaveBeenCalledTimes(1); + expect(mocks.addCredits).toHaveBeenCalledWith({ + accountId: "account-1", + amountCents: 2500, + referenceId: PAYMENT_INTENT, + referenceType: "stripe_payment_intent", + metadata: { amountCents: 2500, stripeSessionId: "cs_credit" }, + }); + expectNothingElseTouched(); + }); +}); diff --git a/apps/web/__tests__/unit/infra-credit-prompts.test.ts b/apps/web/__tests__/unit/infra-credit-prompts.test.ts new file mode 100644 index 00000000000..d188af82981 --- /dev/null +++ b/apps/web/__tests__/unit/infra-credit-prompts.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest"; +import { + firstCreditOffersAddCredits, + lowBalanceText, +} from "@/lib/infra-credit-prompts"; + +const storing = { + hasAccount: true, + standing: "active" as const, + monthlyStorageMicroCredits: 300_000, + balanceMicroCredits: 40_000, + requiredMicroCredits: 70_000, + shortfallMicroCredits: 30_000, +}; + +describe("lowBalanceText", () => { + it("warns when the balance no longer covers the next upload", () => { + expect(lowBalanceText(storing)).toBe( + "Credit is low. Uploading needs at least $0.70 and this app has $0.40.", + ); + }); + + it("stays quiet while the balance covers the next upload", () => { + expect( + lowBalanceText({ + ...storing, + balanceMicroCredits: 70_000, + shortfallMicroCredits: 0, + }), + ).toBeNull(); + }); + + const storingNothing = { + ...storing, + monthlyStorageMicroCredits: 0, + requiredMicroCredits: 1, + shortfallMicroCredits: 0, + }; + + it("warns an app that stores nothing when it has under $1", () => { + expect( + lowBalanceText({ ...storingNothing, balanceMicroCredits: 967 }), + ).toBe("Credit is low. This app has $0.00967, under $1.00."); + expect( + lowBalanceText({ ...storingNothing, balanceMicroCredits: 99_999 }), + ).toBe("Credit is low. This app has $0.99999, under $1.00."); + }); + + it("stays quiet for an app that stores nothing once it has $1 or more", () => { + expect( + lowBalanceText({ ...storingNothing, balanceMicroCredits: 100_000 }), + ).toBeNull(); + expect( + lowBalanceText({ ...storingNothing, balanceMicroCredits: 2_500_000 }), + ).toBeNull(); + }); + + it("leaves an unfunded app to the Add credits prompt instead of both", () => { + const unfunded = { ...storingNothing, balanceMicroCredits: 0 }; + expect(lowBalanceText(unfunded)).toBeNull(); + expect( + firstCreditOffersAddCredits({ + balanceMicroCredits: 0, + proCreditStatus: "claimed", + }), + ).toBe(true); + }); + + it("keeps the floor to active, funded apps", () => { + const low = { ...storingNothing, balanceMicroCredits: 967 }; + expect(lowBalanceText({ ...low, standing: "grace" })).toBeNull(); + expect(lowBalanceText({ ...low, hasAccount: false })).toBeNull(); + }); + + it("leaves grace and paused apps to the out of credit notice", () => { + expect(lowBalanceText({ ...storing, standing: "grace" })).toBeNull(); + expect(lowBalanceText({ ...storing, standing: "paused" })).toBeNull(); + }); + + it("says nothing without a credit account", () => { + expect(lowBalanceText({ ...storing, hasAccount: false })).toBeNull(); + }); +}); + +describe("firstCreditOffersAddCredits", () => { + it("is true for an unfunded app when there is no Cap Pro credit to claim", () => { + for (const status of ["claimed", "trial", "not_pro"]) { + expect( + firstCreditOffersAddCredits({ + balanceMicroCredits: 0, + proCreditStatus: status, + }), + ).toBe(true); + } + }); + + it("is false once the app has credit or can claim Cap Pro credit", () => { + expect( + firstCreditOffersAddCredits({ + balanceMicroCredits: 1, + proCreditStatus: "claimed", + }), + ).toBe(false); + expect( + firstCreditOffersAddCredits({ + balanceMicroCredits: 0, + proCreditStatus: "available", + }), + ).toBe(false); + }); +}); diff --git a/apps/web/__tests__/unit/infra-credit-reversal.test.ts b/apps/web/__tests__/unit/infra-credit-reversal.test.ts new file mode 100644 index 00000000000..d36e4747c9e --- /dev/null +++ b/apps/web/__tests__/unit/infra-credit-reversal.test.ts @@ -0,0 +1,946 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + db: null as unknown, + ids: 0, + listRefunds: vi.fn(), + retrievePaymentIntent: vi.fn(), + listSessions: vi.fn(), +})); + +vi.mock("@cap/database", () => ({ db: () => mocks.db })); +vi.mock("@cap/database/helpers", () => ({ + nanoId: () => `row-${++mocks.ids}`, + nanoIdLength: 15, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + serverEnv: () => ({ WEB_URL: "https://cap.test" }), +})); +vi.mock("@cap/utils", () => ({ + stripe: () => ({ + refunds: { list: mocks.listRefunds }, + paymentIntents: { retrieve: mocks.retrievePaymentIntent }, + checkout: { sessions: { list: mocks.listSessions } }, + }), +})); +vi.mock("@/lib/developer-credits-product", () => ({ + developerCreditsProductId: () => "prod_credits", +})); + +import type Stripe from "stripe"; +import { addCreditsToAccount } from "@/lib/developer-credits"; +import { transcriptStartRefusal } from "@/lib/developer-transcript-billing"; +import { creditCheckoutSessionParams } from "@/lib/infra-credit-checkout"; +import { + awaitsCreditGrant, + refundedShareMicroCredits, + reverseDisputedPayment, + reverseRefundedCharge, +} from "@/lib/infra-credit-reversal"; +import { describeLedgerRow, serializeLedgerRow } from "@/lib/infra-ledger"; +import { DISPUTED_UPLOAD_MESSAGE, getInfraBilling } from "@/lib/infra-standing"; +import { infraVideoFailure } from "@/lib/infra-video-view"; +import { + advanceDays, + fakeDb, + type LedgerRow, + ledger, + resetLedger, +} from "./infra-billing-fake-db"; + +mocks.db = fakeDb; + +const GRANT = 2_500_000; + +const refund = (id: string, amount: number, status = "succeeded") => + ({ + id, + amount, + status, + created: Number(id.replace(/\D/g, "")), + }) as Stripe.Refund; + +const charge = ( + paymentIntent: string | null, + refunds: Stripe.Refund[] | null, + extra: Partial = {}, +) => + ({ + id: "ch_1", + amount: 2500, + currency: "usd", + payment_intent: paymentIntent, + amount_refunded: (refunds ?? []).reduce( + (total, item) => total + item.amount, + 0, + ), + ...(refunds + ? { refunds: { object: "list", data: refunds, has_more: false } } + : {}), + ...extra, + }) as Stripe.Charge; + +const dispute = (id: string, paymentIntent: string | null) => + ({ + id, + amount: 2500, + charge: "ch_1", + payment_intent: paymentIntent, + reason: "fraudulent", + }) as Stripe.Dispute; + +const topUp = (paymentIntentId = "pi_1", amountCents = 2500) => + addCreditsToAccount({ + accountId: "account-1", + amountCents, + referenceId: paymentIntentId, + referenceType: "stripe_payment_intent", + metadata: { amountCents }, + }); + +const reversals = () => + ledger.rows.filter( + (row) => + row.referenceType === "stripe_refund" || + row.referenceType === "stripe_dispute", + ); + +const described = (row: LedgerRow | undefined) => { + if (!row) throw new Error("No ledger row"); + return describeLedgerRow({ + ...row, + referenceId: row.referenceId ?? null, + referenceType: row.referenceType ?? null, + metadata: row.metadata ?? null, + }); +}; + +const billing = () => getInfraBilling("app-1", undefined, new Date(ledger.now)); + +const addsUp = () => { + let balance = 0; + for (const row of ledger.rows) { + balance += row.amountMicroCredits; + expect(row.balanceAfterMicroCredits, row.id).toBe(balance); + } + expect(ledger.account.balanceMicroCredits).toBe(balance); +}; + +const creditCheckout = (accountId = "account-1") => ({ + type: "developer_credits", + accountId, +}); +const autoTopUp = { type: "developer_auto_top_up", accountId: "account-1" }; +const checkoutPaymentTag = creditCheckoutSessionParams({ + customerId: "cus_1", + accountId: "account-1", + appName: "Demo", + amountCents: 2500, + receiptEmail: "owner@example.com", +}).payment_intent_data.metadata; + +const payments: Record< + string, + { + invoice?: string; + metadata?: Record; + session?: Record; + } +> = { + pi_1: { session: creditCheckout() }, + pi_2: { session: creditCheckout() }, + pi_3: { session: creditCheckout() }, + pi_other_account: { session: creditCheckout("account-2") }, + pi_auto: { metadata: autoTopUp }, + pi_pending: { metadata: autoTopUp }, + pi_subscription: { invoice: "in_1" }, + pi_invoiced: { + invoice: "in_2", + metadata: checkoutPaymentTag, + session: creditCheckout(), + }, + pi_pro_checkout: { session: { type: "pro", userId: "user-1" } }, + pi_unknown: {}, +}; + +beforeEach(() => { + resetLedger(); + mocks.ids = 0; + mocks.listRefunds.mockReset(); + mocks.listRefunds.mockResolvedValue({ data: [] }); + mocks.retrievePaymentIntent.mockReset(); + mocks.retrievePaymentIntent.mockImplementation(async (id: string) => { + const payment = payments[id]; + if (!payment) throw new Error(`No such payment_intent: ${id}`); + return { + id, + invoice: payment.invoice ?? null, + metadata: payment.metadata ?? {}, + }; + }); + mocks.listSessions.mockReset(); + mocks.listSessions.mockImplementation( + async ({ payment_intent }: { payment_intent: string }) => { + const session = payments[payment_intent]?.session; + return { data: session ? [{ id: "cs_1", metadata: session }] : [] }; + }, + ); +}); + +describe("a refunded top-up", () => { + it("takes the whole grant back on a full refund, in one ledger row", async () => { + await topUp(); + + const [result] = await reverseRefundedCharge( + charge("pi_1", [refund("re_1", 2500)]), + ); + + expect(result).toEqual({ + state: "reversed", + deductedMicroCredits: GRANT, + uncollectedMicroCredits: 0, + balanceMicroCredits: 0, + }); + expect(ledger.account.balanceMicroCredits).toBe(0); + expect(reversals()).toHaveLength(1); + expect(reversals()[0]).toMatchObject({ + accountId: "account-1", + type: "refund", + amountMicroCredits: -GRANT, + balanceAfterMicroCredits: 0, + referenceId: "re_1", + referenceType: "stripe_refund", + }); + addsUp(); + }); + + it("takes back each partial refund once and never more than the grant", async () => { + await topUp(); + + await reverseRefundedCharge(charge("pi_1", [refund("re_1", 1000)])); + expect(ledger.account.balanceMicroCredits).toBe(1_500_000); + + await reverseRefundedCharge( + charge("pi_1", [refund("re_2", 500), refund("re_1", 1000)]), + ); + expect(ledger.account.balanceMicroCredits).toBe(1_000_000); + + await reverseRefundedCharge( + charge("pi_1", [ + refund("re_3", 1000), + refund("re_2", 500), + refund("re_1", 1000), + ]), + ); + + expect( + reversals().map((row) => [row.referenceId, row.amountMicroCredits]), + ).toEqual([ + ["re_1", -1_000_000], + ["re_2", -500_000], + ["re_3", -1_000_000], + ]); + expect(ledger.account.balanceMicroCredits).toBe(0); + addsUp(); + }); + + it("takes back exactly the grant when the parts do not divide evenly", async () => { + await topUp("pi_1", 1000); + const paid = { amount: 333 }; + + await reverseRefundedCharge(charge("pi_1", [refund("re_1", 111)], paid)); + await reverseRefundedCharge( + charge("pi_1", [refund("re_1", 111), refund("re_2", 111)], paid), + ); + await reverseRefundedCharge( + charge( + "pi_1", + [refund("re_1", 111), refund("re_2", 111), refund("re_3", 111)], + paid, + ), + ); + + expect( + reversals().reduce((total, row) => total - row.amountMicroCredits, 0), + ).toBe(1_000_000); + expect(ledger.account.balanceMicroCredits).toBe(0); + expect( + refundedShareMicroCredits({ + grantMicroCredits: 1_000_000, + refundedCents: 111, + chargeCents: 333, + }), + ).toBe(333_333); + }); + + it("catches up a refund whose own event never arrived", async () => { + await topUp(); + + await reverseRefundedCharge( + charge("pi_1", [refund("re_2", 500), refund("re_1", 1000)]), + ); + + expect(reversals().map((row) => row.referenceId)).toEqual(["re_1", "re_2"]); + expect(ledger.account.balanceMicroCredits).toBe(1_000_000); + }); + + it("stops at zero when the credit was already spent and says how much it could not take", async () => { + await topUp(); + ledger.account.balanceMicroCredits = 400_000; + ledger.rows.push({ + id: "spent", + accountId: "account-1", + type: "transcription", + amountMicroCredits: -2_100_000, + balanceAfterMicroCredits: 400_000, + referenceId: "transcript-1", + referenceType: "developer_transcript", + createdAt: new Date(ledger.now), + }); + + const [result] = await reverseRefundedCharge( + charge("pi_1", [refund("re_1", 2500)]), + ); + + expect(result).toEqual({ + state: "reversed", + deductedMicroCredits: 400_000, + uncollectedMicroCredits: 2_100_000, + balanceMicroCredits: 0, + }); + expect(ledger.account.balanceMicroCredits).toBe(0); + const row = reversals()[0]; + expect(row).toMatchObject({ + amountMicroCredits: -400_000, + balanceAfterMicroCredits: 0, + metadata: { + requestedMicroCredits: GRANT, + uncollectedMicroCredits: 2_100_000, + }, + }); + expect(described(row)).toEqual({ + label: "Payment reversed", + note: "Credit taken back because the payment was returned. $21.00 of it had already been spent and was not taken back.", + reference: { kind: "Refund", value: "re_1" }, + }); + addsUp(); + + const after = await billing(); + expect(after.standing.state).toBe("active"); + expect(after.admission).toMatchObject({ + allowed: false, + reason: "insufficient_credits", + balanceMicroCredits: 0, + }); + expect(await transcriptStartRefusal("app-1", ledger.now)).toMatchObject({ + code: "insufficient_credits", + }); + }); + + it("records a refund that could take nothing, so a retry stays quiet", async () => { + await topUp(); + ledger.account.balanceMicroCredits = 0; + const event = charge("pi_1", [refund("re_1", 2500)]); + + await reverseRefundedCharge(event); + expect(await reverseRefundedCharge(event)).toEqual([ + { state: "duplicate" }, + ]); + + expect(reversals()).toHaveLength(1); + expect(reversals()[0]).toMatchObject({ + amountMicroCredits: 0, + balanceAfterMicroCredits: 0, + metadata: { uncollectedMicroCredits: GRANT }, + }); + }); + + it("writes nothing twice when the same event is delivered again", async () => { + await topUp(); + const event = charge("pi_1", [refund("re_1", 1000)]); + + await reverseRefundedCharge(event); + const again = await reverseRefundedCharge(event); + + expect(again).toEqual([{ state: "duplicate" }]); + expect(reversals()).toHaveLength(1); + expect(ledger.account.balanceMicroCredits).toBe(1_500_000); + }); + + it("reverses an auto top-up the same way, from the charge alone", async () => { + await addCreditsToAccount({ + accountId: "account-1", + amountCents: 2500, + referenceId: "pi_auto", + referenceType: "auto_top_up", + }); + + await reverseRefundedCharge( + charge("pi_auto", [refund("re_1", 2500)], { metadata: autoTopUp }), + ); + + expect(ledger.account.balanceMicroCredits).toBe(0); + expect(reversals()).toHaveLength(1); + expect(mocks.retrievePaymentIntent).not.toHaveBeenCalled(); + expect(mocks.listSessions).not.toHaveBeenCalled(); + }); + + it("reverses a purchase whose checkout attached an invoice, by the same grant", async () => { + await topUp("pi_invoiced"); + + const refunded = await reverseRefundedCharge( + charge("pi_invoiced", [refund("re_1", 1000)], { + invoice: "in_2", + metadata: checkoutPaymentTag, + } as Partial), + ); + const disputed = await reverseDisputedPayment( + dispute("dp_1", "pi_invoiced"), + ); + + expect(refunded).toEqual([ + expect.objectContaining({ + state: "reversed", + deductedMicroCredits: 1_000_000, + }), + ]); + expect(disputed).toEqual( + expect.objectContaining({ + state: "reversed", + deductedMicroCredits: 1_500_000, + }), + ); + expect(reversals().map((row) => row.metadata?.paymentIntentId)).toEqual([ + "pi_invoiced", + "pi_invoiced", + ]); + expect(ledger.account.balanceMicroCredits).toBe(0); + expect(mocks.listSessions).not.toHaveBeenCalled(); + addsUp(); + }); + + it("reverses an auto top-up whose charge carries no metadata", async () => { + await addCreditsToAccount({ + accountId: "account-1", + amountCents: 2500, + referenceId: "pi_auto", + referenceType: "auto_top_up", + }); + + await reverseDisputedPayment(dispute("dp_1", "pi_auto")); + + expect(ledger.account.balanceMicroCredits).toBe(0); + expect(mocks.listSessions).not.toHaveBeenCalled(); + }); + + it("takes the whole grant back when a code paid for part of it", async () => { + await topUp(); + + await reverseRefundedCharge( + charge("pi_1", [refund("re_1", 500)], { amount: 500 }), + ); + + expect(ledger.account.balanceMicroCredits).toBe(0); + }); + + it("reads the refunds from Stripe when the event does not carry them", async () => { + await topUp(); + mocks.listRefunds.mockResolvedValue({ + data: [refund("re_2", 700, "failed"), refund("re_1", 1000)], + }); + + await reverseRefundedCharge(charge("pi_1", null)); + + expect(mocks.listRefunds).toHaveBeenCalledWith({ + charge: "ch_1", + limit: 100, + }); + expect(reversals().map((row) => row.referenceId)).toEqual(["re_1"]); + expect(ledger.account.balanceMicroCredits).toBe(1_500_000); + }); +}); + +describe("a charge that did not buy credit", () => { + it("ignores a Cap subscription charge, even for a customer with an Infra balance", async () => { + await topUp(); + + const results = await reverseRefundedCharge( + charge("pi_subscription", null, { + invoice: "in_1", + customer: "cus_1", + } as Partial), + ); + expect(mocks.retrievePaymentIntent).not.toHaveBeenCalled(); + const disputed = await reverseDisputedPayment( + dispute("dp_1", "pi_subscription"), + ); + expect(mocks.retrievePaymentIntent).toHaveBeenCalledWith("pi_subscription"); + expect(mocks.listSessions).not.toHaveBeenCalled(); + + expect(results).toEqual([ + { state: "ignored", reason: "not_a_credit_payment" }, + ]); + expect(disputed).toEqual({ + state: "ignored", + reason: "not_a_credit_payment", + }); + expect(mocks.listRefunds).not.toHaveBeenCalled(); + expect(reversals()).toHaveLength(0); + expect(ledger.account.balanceMicroCredits).toBe(GRANT); + expect((await billing()).standing.state).toBe("active"); + }); + + it("ignores an unknown payment and a charge with no payment", async () => { + expect( + await reverseRefundedCharge(charge("pi_unknown", [refund("re_1", 2500)])), + ).toEqual([{ state: "ignored", reason: "not_a_credit_payment" }]); + expect( + await reverseRefundedCharge(charge(null, [refund("re_1", 2500)])), + ).toEqual([{ state: "ignored", reason: "no_payment" }]); + expect(await reverseDisputedPayment(dispute("dp_1", null))).toEqual({ + state: "ignored", + reason: "no_payment", + }); + expect(ledger.rows).toHaveLength(0); + }); + + it("ignores a checkout that was not for credits", async () => { + await topUp(); + + expect( + await reverseRefundedCharge( + charge("pi_pro_checkout", [refund("re_1", 2500)]), + ), + ).toEqual([{ state: "ignored", reason: "not_a_credit_payment" }]); + expect(ledger.account.balanceMicroCredits).toBe(GRANT); + }); + + it("ignores a payment whose checkout names an account that does not exist", async () => { + await topUp("pi_other_account"); + + expect( + await reverseDisputedPayment(dispute("dp_1", "pi_other_account")), + ).toEqual({ state: "ignored", reason: "not_a_credit_payment" }); + expect(reversals()).toHaveLength(0); + expect(ledger.account.balanceMicroCredits).toBe(GRANT); + }); + + it("leaves the event for a retry when Stripe cannot be read", async () => { + await topUp(); + mocks.retrievePaymentIntent.mockRejectedValueOnce(new Error("network")); + + await expect( + reverseDisputedPayment(dispute("dp_1", "pi_1")), + ).rejects.toThrow("network"); + expect(reversals()).toHaveLength(0); + }); + + it("cannot touch a purchase a code covered in full", async () => { + await addCreditsToAccount({ + accountId: "account-1", + amountCents: 2500, + referenceId: "cs_free", + referenceType: "stripe_checkout_session", + }); + + expect( + await reverseRefundedCharge(charge("pi_1", [refund("re_1", 2500)])), + ).toEqual([{ state: "grant_pending" }]); + expect(await reverseDisputedPayment(dispute("dp_1", "pi_2"))).toEqual({ + state: "grant_pending", + }); + expect(await reverseDisputedPayment(dispute("dp_2", "pi_unknown"))).toEqual( + { + state: "ignored", + reason: "not_a_credit_payment", + }, + ); + + expect(reversals()).toHaveLength(0); + expect(ledger.account.balanceMicroCredits).toBe(GRANT); + expect((await billing()).standing.state).toBe("active"); + }); + + it("does not count a pending or declined auto top-up as a grant", async () => { + ledger.rows.push({ + id: "pending", + accountId: "account-1", + type: "topup_pending", + amountMicroCredits: 0, + balanceAfterMicroCredits: 0, + referenceId: "pi_pending", + referenceType: "auto_top_up_pending", + createdAt: new Date(ledger.now), + }); + + expect(await reverseDisputedPayment(dispute("dp_1", "pi_pending"))).toEqual( + { state: "grant_pending" }, + ); + + expect(reversals()).toHaveLength(0); + expect(ledger.rows).toHaveLength(1); + }); +}); + +describe("an event that arrives before its grant", () => { + it("leaves a refund unacknowledged and writes nothing, then reverses it once", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const event = charge("pi_1", [refund("re_1", 1000)]); + + const early = await reverseRefundedCharge(event); + + expect(early).toEqual([{ state: "grant_pending" }]); + expect(awaitsCreditGrant(early)).toBe(true); + expect(ledger.rows).toHaveLength(0); + expect(ledger.account.balanceMicroCredits).toBe(0); + expect(mocks.listRefunds).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledTimes(1); + warn.mockRestore(); + + await topUp(); + const redelivered = await reverseRefundedCharge(event); + + expect(awaitsCreditGrant(redelivered)).toBe(false); + expect(redelivered).toMatchObject([ + { state: "reversed", deductedMicroCredits: 1_000_000 }, + ]); + expect(await reverseRefundedCharge(event)).toEqual([ + { state: "duplicate" }, + ]); + expect(reversals()).toHaveLength(1); + expect(ledger.account.balanceMicroCredits).toBe(1_500_000); + addsUp(); + }); + + it("leaves a dispute unacknowledged and unpaused, then reverses and pauses once", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + ledger.account.autoTopUpEnabled = true; + + const early = await reverseDisputedPayment(dispute("dp_1", "pi_1")); + + expect(early).toEqual({ state: "grant_pending" }); + expect(awaitsCreditGrant(early)).toBe(true); + expect(ledger.rows).toHaveLength(0); + expect(ledger.account.autoTopUpEnabled).toBe(true); + expect((await billing()).standing.state).toBe("active"); + expect(warn).toHaveBeenCalledTimes(1); + warn.mockRestore(); + + await topUp(); + const redelivered = await reverseDisputedPayment(dispute("dp_1", "pi_1")); + + expect(redelivered).toMatchObject({ + state: "reversed", + deductedMicroCredits: GRANT, + }); + expect(await reverseDisputedPayment(dispute("dp_1", "pi_1"))).toEqual({ + state: "duplicate", + }); + expect(reversals()).toHaveLength(1); + expect(ledger.account).toMatchObject({ + balanceMicroCredits: 0, + autoTopUpEnabled: false, + }); + expect((await billing()).standing).toMatchObject({ + state: "paused", + disputed: true, + }); + }); + + it("waits for an auto top-up whose grant is not written yet", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + expect( + await reverseRefundedCharge( + charge("pi_auto", [refund("re_1", 2500)], { metadata: autoTopUp }), + ), + ).toEqual([{ state: "grant_pending" }]); + expect(ledger.rows).toHaveLength(0); + warn.mockRestore(); + }); + + it("still acknowledges a charge that never bought credit", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + const subscription = await reverseRefundedCharge( + charge("pi_subscription", null, { + invoice: "in_1", + } as Partial), + ); + const unknown = await reverseDisputedPayment(dispute("dp_1", "pi_unknown")); + const otherCheckout = await reverseRefundedCharge( + charge("pi_pro_checkout", [refund("re_1", 2500)]), + ); + + expect(awaitsCreditGrant(subscription)).toBe(false); + expect(awaitsCreditGrant(unknown)).toBe(false); + expect(awaitsCreditGrant(otherCheckout)).toBe(false); + expect(warn).not.toHaveBeenCalled(); + expect(ledger.rows).toHaveLength(0); + warn.mockRestore(); + }); +}); + +describe("a charge with more refunds than one page", () => { + const pages = (all: Stripe.Refund[]) => + mocks.listRefunds.mockImplementation( + async ({ starting_after }: { starting_after?: string }) => { + const from = starting_after + ? all.findIndex((item) => item.id === starting_after) + 1 + : 0; + return { + data: all.slice(from, from + 100), + has_more: from + 100 < all.length, + }; + }, + ); + + it("applies every refund once across pages and never passes the grant", async () => { + await topUp(); + const all = Array.from({ length: 250 }, (_, index) => + refund(`re_${index + 1}`, 10), + ).reverse(); + pages(all); + + const results = await reverseRefundedCharge(charge("pi_1", null)); + + expect(mocks.listRefunds).toHaveBeenCalledTimes(3); + expect(mocks.listRefunds.mock.calls.map(([params]) => params)).toEqual([ + { charge: "ch_1", limit: 100 }, + { charge: "ch_1", limit: 100, starting_after: "re_151" }, + { charge: "ch_1", limit: 100, starting_after: "re_51" }, + ]); + expect(results).toHaveLength(250); + expect(results.every((result) => result.state === "reversed")).toBe(true); + expect(new Set(reversals().map((row) => row.referenceId)).size).toBe(250); + expect( + reversals().reduce((total, row) => total - row.amountMicroCredits, 0), + ).toBe(GRANT); + expect(ledger.account.balanceMicroCredits).toBe(0); + addsUp(); + + const again = await reverseRefundedCharge(charge("pi_1", null)); + expect(again.every((result) => result.state === "duplicate")).toBe(true); + expect(reversals()).toHaveLength(250); + }); + + it("takes back only the grant when the pages hold a refund for all of it and more rows", async () => { + await topUp(); + pages( + [ + ...Array.from({ length: 120 }, (_, index) => + refund(`re_${index + 1}`, 20), + ), + refund("re_121", 100), + refund("re_122", 700, "failed"), + ].reverse(), + ); + + const results = await reverseRefundedCharge(charge("pi_1", null)); + + expect(results).toHaveLength(121); + expect( + reversals().reduce((total, row) => total - row.amountMicroCredits, 0), + ).toBe(GRANT); + expect(ledger.account.balanceMicroCredits).toBe(0); + }); + + it("fails instead of cutting the list short, so the event is retried", async () => { + await topUp(); + let served = 0; + mocks.listRefunds.mockImplementation(async () => ({ + data: Array.from({ length: 100 }, () => refund(`re_${++served}`, 1)), + has_more: true, + })); + + await expect(reverseRefundedCharge(charge("pi_1", null))).rejects.toThrow( + "more than 5000 refunds", + ); + expect(reversals()).toHaveLength(0); + expect(ledger.account.balanceMicroCredits).toBe(GRANT); + }); +}); + +describe("a disputed top-up", () => { + const armed = async () => { + await topUp(); + Object.assign(ledger.account, { + stripeCustomerId: "cus_1", + stripePaymentMethodId: "pm_1", + autoTopUpEnabled: true, + autoTopUpThresholdMicroCredits: 500_000, + autoTopUpAmountCents: 500, + }); + }; + + it("takes the remaining grant back and switches auto top-up off", async () => { + await armed(); + + const result = await reverseDisputedPayment(dispute("dp_1", "pi_1")); + + expect(result).toEqual({ + state: "reversed", + deductedMicroCredits: GRANT, + uncollectedMicroCredits: 0, + balanceMicroCredits: 0, + }); + expect(ledger.account).toMatchObject({ + balanceMicroCredits: 0, + autoTopUpEnabled: false, + }); + const row = reversals()[0]; + expect(row).toMatchObject({ + type: "adjustment", + amountMicroCredits: -GRANT, + balanceAfterMicroCredits: 0, + referenceId: "dp_1", + referenceType: "stripe_dispute", + metadata: { reason: "fraudulent", autoTopUpSwitchedOff: true }, + }); + expect(described(row)).toEqual({ + label: "Payment disputed", + note: "Credit taken back for a disputed payment. The app is paused and auto top-up is off until Cap reviews it.", + reference: { kind: "Dispute", value: "dp_1" }, + }); + addsUp(); + }); + + it("takes back only what a partial refund left", async () => { + await armed(); + await reverseRefundedCharge(charge("pi_1", [refund("re_1", 1000)])); + + await reverseDisputedPayment(dispute("dp_1", "pi_1")); + + expect(reversals().map((row) => row.amountMicroCredits)).toEqual([ + -1_000_000, -1_500_000, + ]); + expect(ledger.account.balanceMicroCredits).toBe(0); + + expect( + await reverseRefundedCharge( + charge("pi_1", [refund("re_1", 1000), refund("re_2", 1500)]), + ), + ).toEqual([{ state: "duplicate" }, { state: "nothing_left" }]); + expect(reversals()).toHaveLength(2); + }); + + it("writes nothing twice when the same dispute is delivered again", async () => { + await armed(); + await topUp("pi_2"); + + await reverseDisputedPayment(dispute("dp_1", "pi_1")); + expect(await reverseDisputedPayment(dispute("dp_1", "pi_1"))).toEqual({ + state: "duplicate", + }); + + expect(reversals()).toHaveLength(1); + expect(ledger.account.balanceMicroCredits).toBe(GRANT); + }); + + it("pauses the app at once: no uploads, no new transcripts, no playback", async () => { + await armed(); + await topUp("pi_2"); + expect((await billing()).admission.allowed).toBe(true); + expect(await transcriptStartRefusal("app-1", ledger.now)).toBeNull(); + + await reverseDisputedPayment(dispute("dp_1", "pi_1")); + + const after = await billing(); + expect(ledger.account.balanceMicroCredits).toBe(GRANT); + expect(after.standing).toEqual({ + state: "paused", + pausesAt: new Date(ledger.now), + disputed: true, + }); + expect(after.admission).toMatchObject({ + allowed: false, + reason: "out_of_credit", + message: DISPUTED_UPLOAD_MESSAGE, + }); + expect(infraVideoFailure(DISPUTED_UPLOAD_MESSAGE)).toBe( + DISPUTED_UPLOAD_MESSAGE, + ); + expect(await transcriptStartRefusal("app-1", ledger.now)).toMatchObject({ + code: "out_of_credit", + }); + }); + + it("stays paused when more credit is bought, until Cap reviews it", async () => { + await armed(); + await reverseDisputedPayment(dispute("dp_1", "pi_1")); + + advanceDays(1); + await topUp("pi_3", 5000); + expect((await billing()).standing.state).toBe("paused"); + expect((await billing()).admission.allowed).toBe(false); + + advanceDays(1); + ledger.rows.push({ + id: "reviewed", + accountId: "account-1", + type: "adjustment", + amountMicroCredits: 0, + balanceAfterMicroCredits: ledger.account.balanceMicroCredits, + referenceId: "Dispute reviewed", + referenceType: "manual", + createdAt: new Date(ledger.now), + }); + + const restored = await billing(); + expect(restored.standing).toEqual({ state: "active", pausesAt: null }); + expect(restored.admission.allowed).toBe(true); + }); + + it("records what was already spent and still pauses", async () => { + await armed(); + ledger.account.balanceMicroCredits = 0; + + const result = await reverseDisputedPayment(dispute("dp_1", "pi_1")); + + expect(result).toMatchObject({ + deductedMicroCredits: 0, + uncollectedMicroCredits: GRANT, + balanceMicroCredits: 0, + }); + expect(described(reversals()[0]).note).toBe( + "Credit taken back for a disputed payment. The app is paused and auto top-up is off until Cap reviews it. $25.00 of it had already been spent and was not taken back.", + ); + expect((await billing()).standing.state).toBe("paused"); + }); +}); + +describe("how the rows read in the API", () => { + it("describes both rows without passing metadata through", async () => { + await topUp(); + await reverseRefundedCharge(charge("pi_1", [refund("re_1", 1000)])); + await reverseDisputedPayment(dispute("dp_1", "pi_1")); + + const [refunded, disputed] = reversals().map((row) => + serializeLedgerRow({ + ...row, + referenceId: row.referenceId ?? null, + referenceType: row.referenceType ?? null, + metadata: row.metadata ?? null, + }), + ); + + expect(refunded).toMatchObject({ + type: "refund", + description: "Payment reversed", + amountMicroCredits: -1_000_000, + amountDollars: "-10.00", + referenceType: "stripe_refund", + referenceId: "re_1", + }); + expect(disputed).toMatchObject({ + type: "adjustment", + description: "Payment disputed", + amountMicroCredits: -1_500_000, + balanceAfterMicroCredits: 0, + referenceType: "stripe_dispute", + referenceId: "dp_1", + }); + expect(refunded).not.toHaveProperty("metadata"); + }); +}); diff --git a/apps/web/__tests__/unit/infra-delivered-minutes-render.test.ts b/apps/web/__tests__/unit/infra-delivered-minutes-render.test.ts new file mode 100644 index 00000000000..372eded01c8 --- /dev/null +++ b/apps/web/__tests__/unit/infra-delivered-minutes-render.test.ts @@ -0,0 +1,119 @@ +// @vitest-environment jsdom + +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/database", () => ({ db: () => ({}) })); + +import { + DELIVERY_MAY_CHANGE, + DELIVERY_UNAVAILABLE, + DeliveredMinutes, + NO_DELIVERY, +} from "@/app/(infra)/dashboard/infra/_components/delivered-minutes"; + +let root: Root; +let container: HTMLDivElement; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); +}); + +const rows = () => + Array.from(container.querySelectorAll("tbody tr")).map((tr) => + Array.from(tr.querySelectorAll("td")).map((td) => td.textContent), + ); + +describe("Delivered minutes on the Usage page", () => { + it("says what it is, with no billing claim", () => { + act(() => + root.render( + createElement(DeliveredMinutes, { + delivery: null, + monthName: "November", + }), + ), + ); + + expect(container.textContent).toContain("Delivered minutes"); + expect(container.textContent).toContain( + "How much video your viewers watched.", + ); + expect(container.textContent).not.toMatch(/bill|price|charge|cost|\$/i); + }); + + it("shows one quiet line when nothing has been measured", () => { + act(() => + root.render( + createElement(DeliveredMinutes, { + delivery: null, + monthName: "November", + }), + ), + ); + + expect(container.textContent).toContain(NO_DELIVERY); + expect(container.querySelector("table")).toBeNull(); + expect(container.textContent).not.toContain(DELIVERY_MAY_CHANGE); + expect(container.textContent).not.toMatch(/\$/); + }); + + it("says when the delivery figures could not be read, instead of showing them as empty", () => { + act(() => + root.render( + createElement(DeliveredMinutes, { + delivery: null, + monthName: "November", + unavailable: true, + }), + ), + ); + + expect(container.textContent).toContain(DELIVERY_UNAVAILABLE); + expect(container.textContent).not.toContain(NO_DELIVERY); + expect(container.querySelector("table")).toBeNull(); + }); + + it("shows the minutes for each resolution with any delivery and a total", () => { + act(() => + root.render( + createElement(DeliveredMinutes, { + delivery: { + minutes: 1248.5, + minutesByTier: { + "360p": 12.5, + "720p": 336, + "1080p": 900, + "1440p": 0, + "2160p": 0, + }, + finalThrough: "2026-11-18", + }, + monthName: "November", + }), + ), + ); + + expect(container.querySelector("thead")?.textContent).toBe( + "ResolutionMinutes in November", + ); + expect(rows()).toEqual([ + ["360p", "13"], + ["720p", "336"], + ["1080p", "900"], + ["Total", "1,249"], + ]); + expect(container.textContent).not.toContain(NO_DELIVERY); + expect(container.textContent).toContain(DELIVERY_MAY_CHANGE); + expect(container.textContent).not.toMatch(/\$|cost|price|bill/i); + }); +}); diff --git a/apps/web/__tests__/unit/infra-delivery-billing.test.ts b/apps/web/__tests__/unit/infra-delivery-billing.test.ts new file mode 100644 index 00000000000..5d7ccfdedca --- /dev/null +++ b/apps/web/__tests__/unit/infra-delivery-billing.test.ts @@ -0,0 +1,257 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/database", () => ({ db: () => null })); +process.env.CAP_INFRA_DELIVERY_BILLING_FROM = "2026-11-20"; + +const { deliveryBillingStart, deliveryDayIsFinal, deliverySettleDays } = + await import("@/lib/infra-delivery-billing"); +const { + DELIVERY_MICRO_CREDITS_PER_1000_MINUTES, + deliveryMicroCredits, + groupDeliveredDays, + isRollupNotReadyError, +} = await import("@/lib/infra-delivery-rollup"); +const { INFRA_DELIVERY_BILLED_FROM, INFRA_DELIVERY_DOLLARS_PER_1000_MINUTES } = + await import("@/lib/infra-pricing"); +const { MICRO_CREDITS_PER_DOLLAR } = await import("@/lib/developer-pricing"); +const { describeLedgerRow, ledgerCsvLine } = await import("@/lib/infra-ledger"); + +describe("delivery price", () => { + it("is one flat rate for every resolution", () => { + expect(INFRA_DELIVERY_DOLLARS_PER_1000_MINUTES).toBe(0.95); + expect(DELIVERY_MICRO_CREDITS_PER_1000_MINUTES).toBe(95_000); + }); + + it("charges N/1000 of the price for N minutes, in micro-credits", () => { + for (const minutes of [1, 7, 60, 1000, 12_345, 1_000_000]) { + expect(deliveryMicroCredits(minutes * 60)).toBe( + Math.floor( + (minutes * + Math.round( + INFRA_DELIVERY_DOLLARS_PER_1000_MINUTES * + MICRO_CREDITS_PER_DOLLAR, + )) / + 1000, + ), + ); + } + }); + + it("charges whole micro-credits, floored once over the day's delivered seconds", () => { + expect(deliveryMicroCredits(0)).toBe(0); + expect(deliveryMicroCredits(-60)).toBe(0); + expect(deliveryMicroCredits(Number.NaN)).toBe(0); + expect(deliveryMicroCredits(60)).toBe(95); + expect(deliveryMicroCredits(1)).toBe(1); + expect(deliveryMicroCredits(4234)).toBe(6703); + expect(deliveryMicroCredits(60_000_000)).toBe(95_000_000); + }); +}); + +describe("delivery rollup rows", () => { + it("adds final video tiers per app and day, leaves audio out of the minutes, and holds a day with any row not final", () => { + const days = groupDeliveredDays([ + { + appId: "a", + day: "2026-11-20", + tier: "1080p", + seconds: "3000", + final: 1, + }, + { + appId: "a", + day: "2026-11-20", + tier: "720p", + seconds: 1234, + final: true, + }, + { appId: "a", day: "2026-11-20", tier: "audio", seconds: 99, final: "1" }, + { appId: "a", day: "2026-11-21", tier: "360p", seconds: 60, final: 1 }, + { appId: "a", day: "2026-11-21", tier: "720p", seconds: 60, final: 0 }, + { appId: "b", day: "2026-11-20", tier: "2160p", seconds: 0, final: 1 }, + { + appId: "b", + day: "2026-11-21", + tier: "2160p", + seconds: 10, + final: null, + }, + { appId: null, day: "2026-11-20", tier: "1080p", seconds: 10, final: 1 }, + { appId: "c", day: "20-11-2026", tier: "1080p", seconds: 10, final: 1 }, + ]); + expect(days).toEqual([ + { + appId: "a", + day: "2026-11-20", + seconds: 4234, + secondsByTier: { "1080p": 3000, "720p": 1234 }, + final: true, + }, + { + appId: "a", + day: "2026-11-21", + seconds: 0, + secondsByTier: {}, + final: false, + }, + { + appId: "b", + day: "2026-11-20", + seconds: 0, + secondsByTier: {}, + final: true, + }, + { + appId: "b", + day: "2026-11-21", + seconds: 0, + secondsByTier: {}, + final: false, + }, + ]); + }); + + it("knows a missing table or column however the driver wraps it, and nothing else", () => { + expect(isRollupNotReadyError({ errno: 1146 })).toBe(true); + expect(isRollupNotReadyError({ errno: 1054 })).toBe(true); + expect(isRollupNotReadyError({ code: "ER_BAD_FIELD_ERROR" })).toBe(true); + expect( + isRollupNotReadyError( + new Error("Failed query", { + cause: Object.assign(new Error("no table"), { + code: "ER_NO_SUCH_TABLE", + }), + }), + ), + ).toBe(true); + expect(isRollupNotReadyError({ errno: 1062 })).toBe(false); + expect(isRollupNotReadyError(new Error("Connection lost"))).toBe(false); + expect(isRollupNotReadyError(null)).toBe(false); + }); +}); + +describe("delivery billing flag", () => { + it("never starts before the day the pricing copy says delivery is billed from", () => { + expect(INFRA_DELIVERY_BILLED_FROM).toBe("2026-11-14"); + expect(deliveryBillingStart("2026-11-01")).toBe("2026-11-14"); + expect(deliveryBillingStart("2026-11-13")).toBe("2026-11-14"); + expect(deliveryBillingStart("2026-11-14")).toBe("2026-11-14"); + expect(deliveryBillingStart("2026-11-15")).toBe("2026-11-15"); + }); + + it("reads the first billable day, and is off for anything that is not a real day", () => { + expect(deliveryBillingStart()).toBe("2026-11-20"); + expect(deliveryBillingStart(" 2026-11-20 ")).toBe("2026-11-20"); + for (const value of [ + "", + " ", + "true", + "1", + "2026-11-31", + "2026-13-01", + "2026-1-1", + "2026-11-20T00:00:00Z", + ]) { + expect(deliveryBillingStart(value)).toBeNull(); + } + }); + + it("settles the days that ended two to seven days ago, from the start day on", () => { + const now = new Date("2026-11-25T06:41:00Z"); + expect(deliverySettleDays(now, "2026-01-01")).toEqual([ + "2026-11-18", + "2026-11-19", + "2026-11-20", + "2026-11-21", + "2026-11-22", + "2026-11-23", + ]); + expect(deliverySettleDays(now, "2026-11-22")).toEqual([ + "2026-11-22", + "2026-11-23", + ]); + expect(deliverySettleDays(now, "2026-11-24")).toEqual([]); + }); + + it("treats a day as final only when health marked it final and D+2 has begun", () => { + const settleTime = new Date("2026-11-24T00:00:00Z"); + expect( + deliveryDayIsFinal({ day: "2026-11-22", final: true }, settleTime), + ).toBe(true); + expect( + deliveryDayIsFinal( + { day: "2026-11-22", final: true }, + new Date(settleTime.getTime() - 1), + ), + ).toBe(false); + expect( + deliveryDayIsFinal({ day: "2026-11-22", final: false }, settleTime), + ).toBe(false); + }); +}); + +describe("delivery ledger rows", () => { + const row = (overrides: Record = {}) => ({ + id: "row-1", + type: "delivery_daily", + amountMicroCredits: -6703, + balanceAfterMicroCredits: 993297, + referenceId: "2026-11-20", + referenceType: "delivery_day", + metadata: { + deliveryDate: "2026-11-20", + owedMicroCredits: 6703, + chargedMicroCredits: 6703, + shortSince: null, + minutesByTier: { "720p": 20.57, "1080p": 50, "2160p": 0 }, + }, + createdAt: new Date("2026-11-22T00:41:05Z"), + ...overrides, + }); + + it("names the day and lists the minutes by tier, highest first", () => { + expect(describeLedgerRow(row())).toEqual({ + label: "Delivery", + note: "70.57 minutes delivered (1080p 50, 720p 20.57).", + reference: { kind: "Day", value: "2026-11-20" }, + }); + expect(ledgerCsvLine(row())).toBe( + 'row-1,2026-11-22T00:41:05Z,Delivery,"70.57 minutes delivered (1080p 50, 720p 20.57).",Day,2026-11-20,-0.06703,-6703,9.93297,993297\r\n', + ); + }); + + it("says when the balance paid part or none of the day, like a storage day", () => { + const short = { shortSince: "2026-11-22T00:41:05.000Z" }; + expect( + describeLedgerRow( + row({ + amountMicroCredits: -1000, + metadata: { ...row().metadata, ...short }, + }), + ).note, + ).toBe( + "70.57 minutes delivered (1080p 50, 720p 20.57). The balance paid for part of this day. The rest is not charged later.", + ); + expect( + describeLedgerRow( + row({ + amountMicroCredits: 0, + metadata: { ...row().metadata, ...short }, + }), + ), + ).toMatchObject({ + label: "Delivery, unpaid", + note: "70.57 minutes delivered (1080p 50, 720p 20.57). The balance could not pay for this day. It is not charged later.", + }); + }); + + it("still describes a row with a bad day or unreadable metadata", () => { + expect( + describeLedgerRow(row({ referenceId: "soon", metadata: "{not json" })), + ).toEqual({ + label: "Delivery", + note: null, + reference: { kind: "Day", value: "soon" }, + }); + }); +}); diff --git a/apps/web/__tests__/unit/infra-delivery-metering.test.ts b/apps/web/__tests__/unit/infra-delivery-metering.test.ts new file mode 100644 index 00000000000..6c00388d134 --- /dev/null +++ b/apps/web/__tests__/unit/infra-delivery-metering.test.ts @@ -0,0 +1,1275 @@ +import { Readable } from "node:stream"; +import { createGzip, gzipSync } from "node:zlib"; +import { describe, expect, it } from "vitest"; +import { + cachedDeliveryResolver, + type DeliveryPart, + type DeliveryResolver, + type DeliveryVideo, + deliveredSeconds, + deliveryMp4Tier, + hlsSecondsDelivered, + meterDeliveryLog, + parseDeliveryLine, + parseHlsByteRanges, + readDeliveryParts, + readDeliveryVideo, +} from "@/lib/infra-delivery-log"; +import { + type DeliveryDayState, + type DeliveryLogFile, + type DeliveryLogSource, + type DeliveryRow, + type DeliveryStore, + deliveryDayFinalAt, + deliveryDayPartsWindow, + deliveryLogHour, + readDeliveryLogLocation, + rollupDeliveryDay, + runDeliveryMetering, +} from "@/lib/infra-delivery-metering"; + +const APP = "appaaaaaaaaaaa1"; +const OTHER_APP = "appbbbbbbbbbbb2"; +const VIDEO = "vidaaaaaaaaaaa1"; +const base = `/developer/${APP}/${VIDEO}/enc123456/out1`; + +const PLAYLIST = [ + "#EXTM3U", + "#EXT-X-VERSION:7", + "#EXT-X-TARGETDURATION:4", + "#EXT-X-PLAYLIST-TYPE:VOD", + '#EXT-X-MAP:URI="720p.m4s",BYTERANGE="1000@0"', + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:100000@1000", + "720p.m4s", + "#EXTINF:4.000000,", + "#EXT-X-BYTERANGE:200000@101000", + "720p.m4s", + "#EXTINF:2.500000,", + "#EXT-X-BYTERANGE:50000", + "720p.m4s", + "#EXT-X-ENDLIST", +].join("\n"); + +function line(overrides: Record = {}) { + return JSON.stringify({ + date: "2026-11-12", + time: "10:00:00", + "c-ip": "192.0.2.1", + "cs-method": "GET", + "cs-uri-stem": `${base}/hls/720p.m4s`, + "sc-status": "206", + "sc-bytes": "100400", + "sc-content-len": "100000", + "sc-range-start": "1000", + "sc-range-end": "100999", + "x-edge-result-type": "Hit", + "x-host-header": "example.test", + ...overrides, + }); +} + +const SEGMENT_0 = { "sc-range-start": "1000", "sc-range-end": "100999" }; + +function gz(lines: string[]) { + return Readable.from([gzipSync(`${lines.join("\n")}\n`)]); +} + +const VIDEO_ROW: DeliveryVideo = { + appId: APP, + duration: 10.5, + renditions: [ + { name: "1080p", width: 1920, height: 1080, bytes: 1_050_000 }, + { name: "720p", width: 1280, height: 720, bytes: 351_000 }, + { name: "360p", width: 640, height: 360, bytes: 105_000 }, + ], + mp4Bytes: 10_500_000, + width: 1920, + height: 1080, + maxResolutionTier: "1080p", +}; + +const MP4_ONLY = "vidmp4only00001"; +const MP4_ONLY_ROW: DeliveryVideo = { + appId: APP, + duration: 30, + renditions: [], + mp4Bytes: 60_000_000, + width: 3840, + height: 2160, + maxResolutionTier: "2160p", +}; + +class FakeResolver implements DeliveryResolver { + hlsCalls: string[] = []; + videoCalls: string[] = []; + async hls(key: string) { + this.hlsCalls.push(key); + return key.endsWith("/hls/720p.m3u8") + ? parseHlsByteRanges(PLAYLIST, "720p.m4s") + : null; + } + async video({ videoId }: { videoId: string }) { + this.videoCalls.push(videoId); + if (videoId === MP4_ONLY) return MP4_ONLY_ROW; + return videoId === VIDEO ? VIDEO_ROW : null; + } +} + +describe("parseDeliveryLine", () => { + it.each([["360p"], ["720p"], ["1080p"], ["1440p"], ["2160p"], ["audio"]])( + "counts the %s rendition file", + (tier) => { + expect( + parseDeliveryLine(line({ "cs-uri-stem": `${base}/hls/${tier}.m4s` })), + ).toEqual({ + day: "2026-11-12", + appId: APP, + videoId: VIDEO, + kind: tier, + key: `${base.slice(1)}/hls/${tier}.m4s`, + rangeStart: 1000, + bytes: 100400, + bodyBytes: 100000, + abandoned: false, + }); + }, + ); + + it("counts an MP4 download", () => { + expect( + parseDeliveryLine( + line({ "cs-uri-stem": `${base}/video.mp4`, "sc-status": "200" }), + )?.kind, + ).toBe("mp4"); + }); + + it.each([ + ["/a1b2c3d4e5f6g7h/v1/result.mp4"], + ["/3gwrf77snqt5em1/abc/segments/video/segment_1.m4s"], + [`${base}/poster.jpg`], + [`${base}/hls/master.m3u8`], + [`${base}/hls/720p.m3u8`], + [`${base}/hls/480p.m4s`], + [`${base}/captions/en.vtt`], + [`/developer/${APP}/${VIDEO}/hls/720p.m4s`], + [`/x/developer/${APP}/${VIDEO}/e/o/hls/720p.m4s`], + ])("ignores %s", (path) => { + expect(parseDeliveryLine(line({ "cs-uri-stem": path }))).toBeNull(); + }); + + it.each([ + ["200", true], + ["206", true], + ["304", false], + ["403", false], + ["404", false], + ["416", false], + ["500", false], + ])("status %s delivered=%s", (status, delivered) => { + expect(parseDeliveryLine(line({ "sc-status": status })) !== null).toBe( + delivered, + ); + }); + + it.each([["HEAD"], ["OPTIONS"], ["POST"]])("ignores %s", (method) => { + expect(parseDeliveryLine(line({ "cs-method": method }))).toBeNull(); + }); + + it("marks an edge Error transfer as abandoned and ignores empty responses", () => { + expect( + parseDeliveryLine(line({ "x-edge-result-type": "Error" })), + ).toMatchObject({ abandoned: true, bytes: 100400 }); + expect(parseDeliveryLine(line({ "sc-bytes": "0" }))).toBeNull(); + }); + + it("caps body bytes at what left the edge, the content length and the range", () => { + expect(parseDeliveryLine(line({ "sc-bytes": "600" }))).toMatchObject({ + bytes: 600, + bodyBytes: 600, + }); + expect( + parseDeliveryLine( + line({ "sc-content-len": "-", "sc-range-end": "1499" }), + ), + ).toMatchObject({ bytes: 100400, bodyBytes: 500 }); + expect( + parseDeliveryLine( + line({ + "sc-status": "200", + "sc-content-len": "100000", + "sc-range-start": "-", + "sc-range-end": "-", + }), + ), + ).toMatchObject({ rangeStart: 0, bodyBytes: 100000 }); + }); + + it("rejects malformed lines", () => { + expect(parseDeliveryLine("{not json /developer/")).toBeNull(); + expect(parseDeliveryLine(line({ "sc-bytes": "-" }))).toBeNull(); + expect(parseDeliveryLine(line({ date: "12/11/2026" }))).toBeNull(); + expect(parseDeliveryLine('["/developer/"]')).toBeNull(); + }); + + it("never surfaces the client address", () => { + expect(JSON.stringify(parseDeliveryLine(line()))).not.toContain( + "192.0.2.1", + ); + }); +}); + +describe("HLS segment seconds", () => { + const segments = parseHlsByteRanges(PLAYLIST, "720p.m4s") ?? []; + + it("reads byte ranges, including an implicit offset", () => { + expect(segments).toEqual([ + { start: 1000, end: 101000, duration: 4 }, + { start: 101000, end: 301000, duration: 4 }, + { start: 301000, end: 351000, duration: 2.5 }, + ]); + }); + + it("refuses a playlist that is not one byte-range file", () => { + expect(parseHlsByteRanges(PLAYLIST, "1080p.m4s")).toBeNull(); + expect( + parseHlsByteRanges("#EXTM3U\n#EXTINF:4,\nseg0.m4s\n", "seg0.m4s"), + ).toBeNull(); + expect(parseHlsByteRanges("#EXTM3U\n", "720p.m4s")).toBeNull(); + }); + + it.each([ + ["the init section", 0, 1000, 0], + ["one whole segment", 1000, 100000, 4], + ["two whole segments", 1000, 300000, 8], + ["the whole file", 0, 351000, 10.5], + ["half of a segment, aborted", 101000, 100000, 2], + ["a range starting mid segment", 51000, 50000, 2], + ["init plus first segment", 0, 101000, 4], + ["past the end", 351000, 1000, 0], + ])("%s", (_name, start, length, seconds) => { + expect(hlsSecondsDelivered(segments, start, length)).toBeCloseTo( + seconds, + 9, + ); + }); +}); + +describe("deliveredSeconds", () => { + const parsed = (overrides: Record = {}) => { + const request = parseDeliveryLine(line(overrides)); + if (!request) throw new Error("not parsed"); + return request; + }; + + it("maps a segment request to exact segment seconds", async () => { + expect( + await deliveredSeconds(parsed(SEGMENT_0), new FakeResolver()), + ).toEqual({ + tier: "720p", + seconds: 4, + method: "playlist", + }); + }); + + it("counts an aborted segment pro rata", async () => { + expect( + ( + await deliveredSeconds( + parsed({ "sc-bytes": "25000" }), + new FakeResolver(), + ) + )?.seconds, + ).toBeCloseTo(1, 9); + }); + + it("estimates MP4 seconds from the file's average rate under its top rendition", async () => { + expect( + await deliveredSeconds( + parsed({ + "cs-uri-stem": `${base}/video.mp4`, + "sc-bytes": "3000400", + "sc-content-len": "3000000", + "sc-range-start": "0", + "sc-range-end": "2999999", + }), + new FakeResolver(), + ), + ).toEqual({ tier: "1080p", seconds: 3, method: "mp4-estimate" }); + }); + + it("counts an MP4-only video (HLS failed) under its resolution tier", async () => { + expect( + await deliveredSeconds( + parsed({ + "cs-uri-stem": `/developer/${APP}/${MP4_ONLY}/e/o/video.mp4`, + "sc-status": "200", + "sc-bytes": "4000400", + "sc-content-len": "4000000", + "sc-range-start": "-", + "sc-range-end": "-", + }), + new FakeResolver(), + ), + ).toEqual({ tier: "2160p", seconds: 2, method: "mp4-estimate" }); + }); + + it("drops an MP4 whose video is unknown or belongs to another app", async () => { + const resolver = new FakeResolver(); + expect( + await deliveredSeconds( + parsed({ + "cs-uri-stem": `/developer/${APP}/vidzzzzzzzzzzz9/e/o/video.mp4`, + }), + resolver, + ), + ).toBeNull(); + expect( + await deliveredSeconds( + parsed({ + "cs-uri-stem": `/developer/${OTHER_APP}/${VIDEO}/e/o/video.mp4`, + }), + resolver, + ), + ).toBeNull(); + }); + + it("gives audio no seconds and looks nothing up", async () => { + const resolver = new FakeResolver(); + expect( + await deliveredSeconds( + parsed({ "cs-uri-stem": `${base}/hls/audio.m4s` }), + resolver, + ), + ).toEqual({ tier: "audio", seconds: 0, method: "audio" }); + expect(resolver.hlsCalls).toEqual([]); + expect(resolver.videoCalls).toEqual([]); + }); + + it("estimates from the video row's rendition rate when the playlist is gone", async () => { + expect( + await deliveredSeconds( + parsed({ "cs-uri-stem": `${base}/hls/360p.m4s` }), + new FakeResolver(), + ), + ).toEqual({ tier: "360p", seconds: 10, method: "rendition-estimate" }); + }); + + it("keeps bytes but no seconds when the playlist and the video row are both gone", async () => { + expect( + await deliveredSeconds( + parsed({ + "cs-uri-stem": `/developer/${APP}/vidzzzzzzzzzzz9/e/o/hls/360p.m4s`, + }), + new FakeResolver(), + ), + ).toEqual({ tier: "360p", seconds: 0, method: "no-rate" }); + }); + + it("never estimates from another app's video row", async () => { + expect( + await deliveredSeconds( + parsed({ + "cs-uri-stem": `/developer/${OTHER_APP}/${VIDEO}/e/o/hls/360p.m4s`, + }), + new FakeResolver(), + ), + ).toEqual({ tier: "360p", seconds: 0, method: "no-rate" }); + }); +}); + +describe("cachedDeliveryResolver", () => { + it("reads each playlist and video once per run and retries a failed read", async () => { + const inner = new FakeResolver(); + let fail = true; + const flaky: DeliveryResolver = { + hls: async (key) => { + if (fail) { + fail = false; + throw new Error("transient"); + } + return inner.hls(key); + }, + video: (request) => inner.video(request), + }; + const cached = cachedDeliveryResolver(flaky); + await expect(cached.hls("a/hls/720p.m3u8")).rejects.toThrow(); + await cached.hls("a/hls/720p.m3u8"); + await cached.hls("a/hls/720p.m3u8"); + await cached.video({ videoId: VIDEO, key: "a" }); + await cached.video({ videoId: VIDEO, key: "a" }); + expect(inner.hlsCalls).toHaveLength(1); + expect(inner.videoCalls).toHaveLength(1); + }); +}); + +describe("readDeliveryVideo", () => { + it("keeps known renditions and the MP4 size, and labels the MP4 with the top one", () => { + const video = readDeliveryVideo({ + appId: APP, + duration: 100, + width: 1920, + height: 1080, + maxResolutionTier: "1080p", + renditions: [ + { name: "720p", width: 1280, height: 720, bytes: 5 }, + { name: "1080p", width: 1920, height: 1080, bytes: 9 }, + { name: "480p", width: 854, height: 480, bytes: 3 }, + "junk", + ], + processing: { bytes: { mp4: 120_000_000 } }, + }); + expect(video).toEqual({ + appId: APP, + duration: 100, + width: 1920, + height: 1080, + maxResolutionTier: "1080p", + renditions: [ + { name: "720p", width: 1280, height: 720, bytes: 5 }, + { name: "1080p", width: 1920, height: 1080, bytes: 9 }, + ], + mp4Bytes: 120_000_000, + }); + expect(deliveryMp4Tier(video)).toBe("1080p"); + expect( + deliveryMp4Tier( + readDeliveryVideo({ + appId: APP, + duration: 1, + width: null, + height: null, + maxResolutionTier: "1080p", + renditions: null, + processing: null, + }), + ), + ).toBeNull(); + }); + + // Expected names come from planHlsLadder in apps/media-server, which cannot + // load under this runner, given the web encode plan's ladderMaxTier. + it.each([ + [3840, 2160, "1080p", "1080p"], + [3840, 2160, "1440p", "1440p"], + [3840, 2160, "2160p", "2160p"], + [2560, 1440, "1080p", "1080p"], + [2560, 1440, "1440p", "1440p"], + [2560, 1440, "2160p", "1440p"], + [1920, 1080, "1080p", "1080p"], + [1920, 1080, "1440p", "1080p"], + [1920, 1080, "2160p", "1080p"], + [1080, 1920, "1080p", "1080p"], + [1080, 1920, "1440p", "1080p"], + [1080, 1920, "2160p", "1080p"], + [1920, 804, "1080p", "1080p"], + [1920, 804, "1440p", "1080p"], + [1920, 804, "2160p", "1080p"], + [1280, 720, "1080p", "720p"], + [1280, 720, "1440p", "720p"], + [1280, 720, "2160p", "720p"], + [1000, 562, "1080p", "720p"], + [1000, 562, "1440p", "720p"], + [1000, 562, "2160p", "720p"], + [854, 480, "1080p", "360p"], + [854, 480, "1440p", "360p"], + [854, 480, "2160p", "360p"], + [640, 360, "1080p", "360p"], + [640, 360, "1440p", "360p"], + [640, 360, "2160p", "360p"], + [480, 270, "1080p", "360p"], + [480, 270, "1440p", "360p"], + [480, 270, "2160p", "360p"], + [4096, 1716, "1080p", "720p"], + [4096, 1716, "1440p", "1080p"], + [4096, 1716, "2160p", "2160p"], + [5120, 2880, "1080p", "1080p"], + [5120, 2880, "1440p", "1440p"], + [5120, 2880, "2160p", "2160p"], + [1440, 1080, "1080p", "1080p"], + [1440, 1080, "1440p", "1080p"], + [1440, 1080, "2160p", "1080p"], + [720, 1280, "1080p", "720p"], + [720, 1280, "1440p", "720p"], + [720, 1280, "2160p", "720p"], + [2160, 3840, "1080p", "1080p"], + [2160, 3840, "1440p", "1440p"], + [2160, 3840, "2160p", "2160p"], + [1366, 768, "1080p", "720p"], + [1366, 768, "1440p", "720p"], + [1366, 768, "2160p", "720p"], + [3440, 1440, "1080p", "720p"], + [3440, 1440, "1440p", "1080p"], + [3440, 1440, "2160p", "1440p"], + [1600, 900, "1080p", "1080p"], + [1600, 900, "1440p", "1080p"], + [1600, 900, "2160p", "1080p"], + [2048, 858, "1080p", "1080p"], + [2048, 858, "1440p", "1080p"], + [2048, 858, "2160p", "1080p"], + ] as const)( + "names an HLS-less %ix%i MP4 up to %s as the encoder's top rendition, %s", + (width, height, maxResolutionTier, tier) => { + expect( + deliveryMp4Tier({ + appId: APP, + duration: 10, + width, + height, + maxResolutionTier, + renditions: [], + mp4Bytes: 1, + }), + ).toBe(tier); + }, + ); + + it.each([ + [3840, 2160, "2160p", "2160p"], + [3840, 2160, "1080p", "1080p"], + [1920, 1080, "1440p", "1080p"], + [1080, 1920, "1080p", "1080p"], + [1280, 720, "1080p", "720p"], + [640, 360, "1080p", "360p"], + ] as const)( + "labels an MP4 with no HLS renditions from the encode plan: %ix%i up to %s is %s", + (width, height, maxResolutionTier, tier) => { + expect( + deliveryMp4Tier({ + appId: APP, + duration: 10, + width, + height, + maxResolutionTier, + renditions: [], + mp4Bytes: 1, + }), + ).toBe(tier); + }, + ); +}); + +describe("meterDeliveryLog", () => { + it("aggregates a gzipped log per day, app, video and tier", async () => { + const result = await meterDeliveryLog( + gz([ + line(), + line({ + "sc-range-start": "101000", + "sc-range-end": "300999", + "sc-content-len": "200000", + "sc-bytes": "200400", + }), + line({ "cs-uri-stem": `${base}/hls/audio.m4s`, "sc-bytes": "50" }), + line({ date: "2026-11-13" }), + line({ "sc-status": "403" }), + line({ "cs-uri-stem": "/3gwrf77snqt5em1/x/result.mp4" }), + line({ "x-edge-result-type": "Error", "sc-bytes": "70000" }), + line({ + "x-edge-result-type": "Error", + "cs-uri-stem": `${base}/hls/1080p.m4s`, + "sc-bytes": "9000", + }), + line({ + "x-edge-result-type": "Error", + "sc-status": "403", + "sc-bytes": "5", + }), + "", + ]), + new FakeResolver(), + ); + expect(result.lines).toBe(9); + expect(result.delivered).toBe(4); + expect(result.methods.seconds.playlist).toBe(12); + expect(result.methods.requests).toMatchObject({ playlist: 3, audio: 1 }); + expect(result.parts).toEqual([ + ["2026-11-12", APP, VIDEO, "720p", 2, 300800, 8000, 70000], + ["2026-11-12", APP, VIDEO, "audio", 1, 50, 0, 0], + ["2026-11-13", APP, VIDEO, "720p", 1, 100400, 4000, 0], + ["2026-11-12", APP, VIDEO, "1080p", 0, 0, 0, 9000], + ]); + }); + + it("reads a last line with no trailing newline", async () => { + const result = await meterDeliveryLog( + Readable.from([gzipSync(`${line()}\n${line()}`)]), + new FakeResolver(), + ); + expect(result.parts[0]?.[4]).toBe(2); + }); + + it("fails on a truncated gzip instead of under-counting silently", async () => { + const whole = gzipSync(`${line()}\n`.repeat(2000)); + await expect( + meterDeliveryLog( + Readable.from([whole.subarray(0, Math.floor(whole.length / 2))]), + new FakeResolver(), + ), + ).rejects.toThrow(); + }); + + it("keeps memory bounded on a large log", async () => { + const LINES = 400_000; + function* source() { + for (let i = 0; i < LINES; i++) { + const tier = ["360p", "720p", "1080p", "audio"][(i / 10) % 4]; + yield `${line({ + "cs-uri-stem": + i % 10 === 0 + ? `${base}/hls/${tier}.m4s` + : `/regular${i % 97}/abc/segments/video/segment_${i}.m4s`, + "cs(User-Agent)": "x".repeat(200), + })}\n`; + } + } + global.gc?.(); + const before = process.memoryUsage().heapUsed; + let peak = before; + const timer = setInterval(() => { + peak = Math.max(peak, process.memoryUsage().heapUsed); + }, 5); + const result = await meterDeliveryLog( + Readable.from(source()).pipe(createGzip()), + new FakeResolver(), + ); + clearInterval(timer); + peak = Math.max(peak, process.memoryUsage().heapUsed); + expect(result.lines).toBe(LINES); + expect(result.delivered).toBe(LINES / 10); + expect(result.parts).toHaveLength(4); + expect(peak - before).toBeLessThan(64 * 1024 * 1024); + }, 60_000); +}); + +describe("readDeliveryParts", () => { + it("drops malformed stored parts", () => { + const good: DeliveryPart = ["2026-11-12", APP, VIDEO, "720p", 1, 2, 3, 4]; + expect( + readDeliveryParts([ + good, + ["2026-11-12", APP, VIDEO, "mp4", 1, 2, 3, 4], + ["2026-11-12", APP, VIDEO, "720p", -1, 2, 3, 4], + ["2026-11-12", APP, VIDEO, "720p", 1, 2, 3, 4, 5], + ["bad", APP, VIDEO, "720p", 1, 2, 3, 4], + "x", + ]), + ).toEqual([good]); + expect( + readDeliveryParts([["2026-11-12", APP, VIDEO, "720p", 1, 2, 3]]), + ).toEqual([["2026-11-12", APP, VIDEO, "720p", 1, 2, 3, 0]]); + expect(readDeliveryParts(null)).toEqual([]); + }); +}); + +describe("rollupDeliveryDay", () => { + it("sums one day's parts per app and tier", () => { + expect( + rollupDeliveryDay("2026-11-12", [ + ["2026-11-12", APP, VIDEO, "720p", 10, 1000, 30_400, 7], + ["2026-11-12", APP, "vidbbbbbbbbbbb2", "720p", 2, 200, 10_200, 3], + ["2026-11-12", APP, VIDEO, "audio", 20, 900, 0, 0], + ["2026-11-12", OTHER_APP, VIDEO, "1080p", 1, 5, 4_000, 0], + ["2026-11-13", APP, VIDEO, "720p", 99, 99, 99_000, 99], + ]), + ).toEqual([ + { + appId: APP, + tier: "720p", + seconds: 41, + bytes: 1200, + requests: 12, + abandonedBytes: 10, + }, + { + appId: APP, + tier: "audio", + seconds: 0, + bytes: 900, + requests: 20, + abandonedBytes: 0, + }, + { + appId: OTHER_APP, + tier: "1080p", + seconds: 4, + bytes: 5, + requests: 1, + abandonedBytes: 0, + }, + ]); + }); +}); + +class MemoryLogs implements DeliveryLogSource { + files = new Map(); + opened: string[] = []; + put(name: string, lines: string[], etag = `"${name}"`) { + this.files.set(name, { etag, lines }); + } + async list(day: string): Promise { + return [...this.files.entries()].flatMap(([file, { etag }]) => { + const hour = deliveryLogHour(file); + return file.includes(`.${day}-`) && hour ? [{ file, etag, hour }] : []; + }); + } + async open(file: string) { + this.opened.push(file); + const entry = this.files.get(file); + if (!entry) throw new Error("missing"); + return gz(entry.lines); + } +} + +type StoredRow = DeliveryRow & { + day: string; + final: boolean; + finalizedAt: Date | null; + updatedAt: Date; +}; + +class MemoryStore implements DeliveryStore { + files = new Map< + string, + { etag: string; hour: Date; parts: DeliveryPart[]; processedAt: Date } + >(); + rows: StoredRow[] = []; + day(day: string) { + return this.rows.filter((row) => row.day === day); + } + async processedFiles(files: string[]) { + const known = new Map(); + for (const file of files) { + const row = this.files.get(file); + if (row) + known.set(file, { etag: row.etag, processedAt: row.processedAt }); + } + return known; + } + async saveFile(file: DeliveryLogFile, parts: DeliveryPart[], now: Date) { + this.files.set(file.file, { + etag: file.etag, + hour: file.hour, + parts, + processedAt: now, + }); + } + async partsNear(from: Date, to: Date) { + return [...this.files.values()] + .filter((row) => row.hour >= from && row.hour < to) + .flatMap((row) => row.parts); + } + async dayStates(from: string, to: string) { + const states = new Map(); + for (const row of this.rows) { + if (row.day < from || row.day > to) continue; + const state = states.get(row.day); + states.set(row.day, { + final: (state?.final ?? false) || row.final, + updatedAt: + state && state.updatedAt > row.updatedAt + ? state.updatedAt + : row.updatedAt, + }); + } + return states; + } + async replaceDay( + day: string, + rows: DeliveryRow[], + now: Date, + final: boolean, + ) { + if (this.day(day).some((row) => row.final)) return "final" as const; + this.rows = [ + ...this.rows.filter((row) => row.day !== day), + ...rows.map((row) => ({ + ...row, + day, + final, + finalizedAt: final ? now : null, + updatedAt: now, + })), + ]; + return "written" as const; + } + async finalizeDay( + day: string, + coverage: { file: string; etag: string }[], + now: Date, + ) { + if (this.day(day).some((row) => row.final)) return "already-final" as const; + if ( + coverage.some(({ file, etag }) => this.files.get(file)?.etag !== etag) + ) { + return "incomplete" as const; + } + const { from, to } = deliveryDayPartsWindow(day); + await this.replaceDay( + day, + rollupDeliveryDay(day, await this.partsNear(from, to)), + now, + true, + ); + return "final" as const; + } + async pruneFiles(before: Date) { + for (const [file, row] of this.files) { + if (row.hour < before) this.files.delete(file); + } + } +} + +const at = (iso: string) => new Date(iso); +const run = (logs: MemoryLogs, store: MemoryStore, now: Date, maxFiles = 100) => + runDeliveryMetering({ + source: logs, + store, + resolver: new FakeResolver(), + now, + maxFiles, + outOfTime: () => false, + }); + +const play = (date: string) => line({ date }); + +describe("runDeliveryMetering", () => { + it("recomputes yesterday and today on every run, refreshing updatedAt even when unchanged", async () => { + const logs = new MemoryLogs(); + logs.put("D.2026-11-12-10.a.gz", [play("2026-11-12")]); + logs.put("D.2026-11-13-08.b.gz", [play("2026-11-13")]); + const store = new MemoryStore(); + + const first = await run(logs, store, at("2026-11-13T09:23:00Z")); + expect(first).toMatchObject({ + listed: 2, + processed: 2, + recomputed: ["2026-11-12", "2026-11-13"], + }); + expect(first.methods.requests.playlist).toBe(2); + expect(first.methods.seconds.playlist).toBe(8); + const counts = store.rows.map(({ updatedAt: _, ...row }) => row); + + const second = await run(logs, store, at("2026-11-13T10:23:00Z")); + expect(second).toMatchObject({ + processed: 0, + recomputed: ["2026-11-12", "2026-11-13"], + }); + expect(store.rows.map(({ updatedAt: _, ...row }) => row)).toEqual(counts); + expect( + store.rows.every( + (row) => row.updatedAt.toISOString() === "2026-11-13T10:23:00.000Z", + ), + ).toBe(true); + expect(logs.opened).toHaveLength(2); + expect(store.day("2026-11-12")).toMatchObject([ + { + appId: APP, + tier: "720p", + seconds: 4, + bytes: 100400, + requests: 1, + final: false, + finalizedAt: null, + }, + ]); + }); + + it("replaces a day when a late file arrives instead of adding to it", async () => { + const logs = new MemoryLogs(); + const store = new MemoryStore(); + logs.put("D.2026-11-12-22.a.gz", [play("2026-11-12")]); + await run(logs, store, at("2026-11-12T23:23:00Z")); + await run(logs, store, at("2026-11-13T00:23:00Z")); + logs.put("D.2026-11-12-23.late.gz", [play("2026-11-12")]); + logs.put("D.2026-11-13-00.c.gz", [play("2026-11-12"), play("2026-11-13")]); + const result = await run(logs, store, at("2026-11-13T01:23:00Z")); + expect(result.processed).toBe(2); + expect(store.day("2026-11-12")).toMatchObject([ + { tier: "720p", seconds: 12, bytes: 301200, requests: 3 }, + ]); + expect(store.day("2026-11-13")[0]?.requests).toBe(1); + }); + + it("re-reads a file whose ETag changed and drops its old counts", async () => { + const logs = new MemoryLogs(); + const store = new MemoryStore(); + logs.put("D.2026-11-12-10.a.gz", [play("2026-11-12")], '"1"'); + await run(logs, store, at("2026-11-13T09:00:00Z")); + logs.put( + "D.2026-11-12-10.a.gz", + [play("2026-11-12"), play("2026-11-12")], + '"2"', + ); + await run(logs, store, at("2026-11-13T10:00:00Z")); + expect(store.day("2026-11-12")[0]).toMatchObject({ + seconds: 8, + requests: 2, + }); + }); + + it("handles a bounded number of files per run and picks up the rest next run", async () => { + const logs = new MemoryLogs(); + const store = new MemoryStore(); + for (let hour = 0; hour < 10; hour++) { + logs.put(`D.2026-11-12-${String(hour).padStart(2, "0")}.x.gz`, [ + play("2026-11-12"), + ]); + } + const now = at("2026-11-13T09:00:00Z"); + expect(await run(logs, store, now, 4)).toMatchObject({ + processed: 4, + remaining: 6, + }); + expect(store.day("2026-11-12")[0]?.requests).toBe(4); + await run(logs, store, now, 4); + expect(await run(logs, store, now, 4)).toMatchObject({ + processed: 2, + remaining: 0, + }); + expect(store.day("2026-11-12")[0]).toMatchObject({ + requests: 10, + seconds: 40, + }); + }); + + it("keeps going when one file fails and retries it next run", async () => { + const logs = new MemoryLogs(); + const store = new MemoryStore(); + logs.put("D.2026-11-12-10.a.gz", [play("2026-11-12")]); + logs.put("D.2026-11-12-11.b.gz", [play("2026-11-12")]); + const open = logs.open.bind(logs); + logs.open = async (file) => { + if (file.includes("-11.")) throw new Error("boom"); + return open(file); + }; + const now = at("2026-11-13T09:00:00Z"); + expect(await run(logs, store, now)).toMatchObject({ + processed: 1, + failed: 1, + }); + logs.open = open; + expect(await run(logs, store, now)).toMatchObject({ + processed: 1, + failed: 0, + }); + expect(store.day("2026-11-12")[0]?.requests).toBe(2); + }); + + it("clears stale rows of a day it recomputes", async () => { + const store = new MemoryStore(); + await store.replaceDay( + "2026-11-13", + [ + { + appId: OTHER_APP, + tier: "360p", + seconds: 1, + bytes: 1, + requests: 1, + abandonedBytes: 0, + }, + ], + at("2026-11-13T01:00:00Z"), + false, + ); + await run(new MemoryLogs(), store, at("2026-11-13T09:00:00Z")); + expect(store.day("2026-11-13")).toEqual([]); + }); + + it("leaves a day alone after D+1 and finalizes it at 02:00 UTC on D+2, not 01:59", async () => { + const logs = new MemoryLogs(); + const store = new MemoryStore(); + logs.put("D.2026-11-12-10.a.gz", [play("2026-11-12")]); + await run(logs, store, at("2026-11-13T23:23:00Z")); + const settled = structuredClone(store.day("2026-11-12")); + + const early = await run(logs, store, at("2026-11-14T01:59:59Z")); + expect(early.finalized).toEqual([]); + expect(store.day("2026-11-12")).toEqual(settled); + + expect(deliveryDayFinalAt("2026-11-12").toISOString()).toBe( + "2026-11-14T02:00:00.000Z", + ); + const due = await run(logs, store, at("2026-11-14T02:00:00Z")); + expect(due.finalized).toEqual(["2026-11-12"]); + expect(store.day("2026-11-12")).toEqual( + settled.map((row) => ({ + ...row, + final: true, + finalizedAt: at("2026-11-14T02:00:00Z"), + updatedAt: at("2026-11-14T02:00:00Z"), + })), + ); + }); + + it("reads a late file of D at finalization and includes it in the final totals", async () => { + const logs = new MemoryLogs(); + const store = new MemoryStore(); + logs.put("D.2026-11-12-10.a.gz", [play("2026-11-12")]); + await run(logs, store, at("2026-11-13T23:23:00Z")); + logs.put("D.2026-11-12-23.late.gz", [play("2026-11-12")]); + const early = await run(logs, store, at("2026-11-14T01:59:59Z")); + expect(early.processed).toBe(0); + expect(store.day("2026-11-12")[0]?.requests).toBe(1); + const due = await run(logs, store, at("2026-11-14T02:00:00Z")); + expect(due).toMatchObject({ processed: 1, finalized: ["2026-11-12"] }); + expect(store.day("2026-11-12")[0]).toMatchObject({ + requests: 2, + final: true, + updatedAt: at("2026-11-14T02:00:00Z"), + }); + }); + + it("does not finalize D while a file the last D+1 run left behind at its file cap is unread", async () => { + const logs = new MemoryLogs(); + const store = new MemoryStore(); + for (let hour = 20; hour < 23; hour++) { + logs.put(`D.2026-11-12-${hour}.x.gz`, [play("2026-11-12")]); + } + const last = await run(logs, store, at("2026-11-13T23:23:00Z"), 2); + expect(last).toMatchObject({ processed: 2, remaining: 1 }); + expect(store.day("2026-11-12")[0]?.requests).toBe(2); + + const open = logs.open.bind(logs); + logs.open = async (file) => { + if (file.includes("-22.")) throw new Error("boom"); + return open(file); + }; + const failing = await run(logs, store, at("2026-11-14T02:00:00Z")); + expect(failing).toMatchObject({ + failed: 1, + finalized: [], + deferred: ["2026-11-12"], + }); + expect(store.day("2026-11-12")[0]).toMatchObject({ + requests: 2, + final: false, + }); + + const stillFailing = await run(logs, store, at("2026-11-14T03:00:00Z")); + expect(stillFailing).toMatchObject({ failed: 1, deferred: ["2026-11-12"] }); + + logs.open = open; + const read = await run(logs, store, at("2026-11-14T04:00:00Z")); + expect(read).toMatchObject({ processed: 1, finalized: ["2026-11-12"] }); + expect(store.day("2026-11-12")[0]).toMatchObject({ + requests: 3, + final: true, + }); + }); + + it("recomputes D at finalization when a file of D was read after D's last recompute", async () => { + const logs = new MemoryLogs(); + const store = new MemoryStore(); + logs.put("D.2026-11-12-10.a.gz", [play("2026-11-12")]); + await run(logs, store, at("2026-11-13T22:23:00Z")); + logs.put("D.2026-11-13-00.b.gz", [play("2026-11-12")]); + await run(logs, store, at("2026-11-14T00:23:00Z")); + expect(store.day("2026-11-12")[0]?.requests).toBe(1); + const due = await run(logs, store, at("2026-11-14T02:23:00Z")); + expect(due).toMatchObject({ processed: 0, finalized: ["2026-11-12"] }); + expect(store.day("2026-11-12")[0]).toMatchObject({ + requests: 2, + final: true, + }); + }); + + it("finalizes complete totals when an overlapping run wrote partial totals last", async () => { + const logs = new MemoryLogs(); + const store = new MemoryStore(); + logs.put("D.2026-11-12-10.a.gz", [play("2026-11-12")]); + logs.put("D.2026-11-12-11.b.gz", [play("2026-11-12")]); + + const open = logs.open.bind(logs); + logs.open = async (file) => { + if (file.includes("-11.")) throw new Error("boom"); + return open(file); + }; + await run(logs, store, at("2026-11-13T10:00:30Z")); + const partial = store + .day("2026-11-12") + .map( + ({ + day: _day, + final: _final, + finalizedAt: _at, + updatedAt: _updated, + ...row + }) => row, + ); + expect(partial[0]?.requests).toBe(1); + + logs.open = open; + await run(logs, store, at("2026-11-13T10:00:00Z")); + expect(store.day("2026-11-12")[0]?.requests).toBe(2); + + await store.replaceDay( + "2026-11-12", + partial, + at("2026-11-13T10:00:30Z"), + false, + ); + expect(store.day("2026-11-12")[0]?.requests).toBe(1); + + const due = await run(logs, store, at("2026-11-14T02:00:00Z")); + expect(due.finalized).toEqual(["2026-11-12"]); + expect(store.day("2026-11-12")).toMatchObject([ + { requests: 2, seconds: 8, bytes: 200800, final: true }, + ]); + }); + + it("does not finalize when a covering file changes between listing and finalizing", async () => { + const logs = new MemoryLogs(); + const store = new MemoryStore(); + logs.put("D.2026-11-12-10.a.gz", [play("2026-11-12")]); + await run(logs, store, at("2026-11-13T10:00:00Z")); + const finalize = store.finalizeDay.bind(store); + store.finalizeDay = async (day, coverage, now) => { + const recorded = store.files.get("D.2026-11-12-10.a.gz"); + if (recorded) recorded.etag = '"changed"'; + return finalize(day, coverage, now); + }; + const due = await run(logs, store, at("2026-11-14T02:00:00Z")); + expect(due).toMatchObject({ finalized: [], deferred: ["2026-11-12"] }); + expect(store.day("2026-11-12")[0]?.final).toBe(false); + }); + + it("leaves final rows unchanged when an older run replaces the day afterwards", async () => { + const logs = new MemoryLogs(); + const store = new MemoryStore(); + logs.put("D.2026-11-12-10.a.gz", [play("2026-11-12")]); + await run(logs, store, at("2026-11-13T23:23:00Z")); + await run(logs, store, at("2026-11-14T02:23:00Z")); + const closed = structuredClone(store.day("2026-11-12")); + expect(closed[0]?.final).toBe(true); + expect( + await store.replaceDay( + "2026-11-12", + [ + { + appId: APP, + tier: "720p", + seconds: 1, + bytes: 1, + requests: 1, + abandonedBytes: 0, + }, + ], + at("2026-11-13T23:59:00Z"), + false, + ), + ).toBe("final"); + expect(store.day("2026-11-12")).toEqual(closed); + expect( + await store.finalizeDay("2026-11-12", [], at("2026-11-15T02:00:00Z")), + ).toBe("already-final"); + expect(store.day("2026-11-12")).toEqual(closed); + }); + + it("never rewrites or deletes a final day", async () => { + const logs = new MemoryLogs(); + const store = new MemoryStore(); + logs.put("D.2026-11-12-10.a.gz", [play("2026-11-12")]); + await run(logs, store, at("2026-11-13T23:23:00Z")); + await run(logs, store, at("2026-11-14T02:23:00Z")); + const closed = structuredClone(store.day("2026-11-12")); + expect(closed[0]?.final).toBe(true); + + logs.put("D.2026-11-12-11.later.gz", [play("2026-11-12")]); + const later = await run(logs, store, at("2026-11-15T05:23:00Z")); + expect(later.finalized).not.toContain("2026-11-12"); + expect(store.day("2026-11-12")).toEqual(closed); + expect( + await store.replaceDay( + "2026-11-12", + [], + at("2026-11-15T06:00:00Z"), + false, + ), + ).toBe("final"); + expect(store.day("2026-11-12")).toEqual(closed); + }); + + it("recomputes and finalizes a day in one step when the cron was down during D+1", async () => { + const logs = new MemoryLogs(); + const store = new MemoryStore(); + logs.put("D.2026-11-12-10.a.gz", [play("2026-11-12")]); + await run(logs, store, at("2026-11-12T20:23:00Z")); + expect(store.day("2026-11-12")[0]?.requests).toBe(1); + + logs.put("D.2026-11-12-21.b.gz", [play("2026-11-12")]); + logs.put("D.2026-11-13-00.c.gz", [play("2026-11-12"), play("2026-11-13")]); + const back = await run(logs, store, at("2026-11-14T03:23:00Z")); + expect(back.finalized).toEqual(["2026-11-12"]); + expect(store.day("2026-11-12")).toMatchObject([ + { + requests: 3, + seconds: 12, + final: true, + finalizedAt: at("2026-11-14T03:23:00Z"), + updatedAt: at("2026-11-14T03:23:00Z"), + }, + ]); + expect(store.day("2026-11-13")[0]).toMatchObject({ + requests: 1, + final: false, + }); + }); + + it("defers a catch-up finalize while that day's log files are still unread", async () => { + const logs = new MemoryLogs(); + const store = new MemoryStore(); + for (let hour = 0; hour < 6; hour++) { + logs.put(`D.2026-11-12-${String(hour).padStart(2, "0")}.x.gz`, [ + play("2026-11-12"), + ]); + } + const first = await run(logs, store, at("2026-11-14T03:00:00Z"), 4); + expect(first).toMatchObject({ deferred: ["2026-11-12"], finalized: [] }); + expect(store.day("2026-11-12")).toEqual([]); + const second = await run(logs, store, at("2026-11-14T04:00:00Z"), 4); + expect(second.finalized).toEqual(["2026-11-12"]); + expect(store.day("2026-11-12")[0]).toMatchObject({ + requests: 6, + final: true, + }); + }); + + it("never meters a day before metering started", async () => { + const logs = new MemoryLogs(); + logs.put("D.2026-11-11-10.a.gz", [play("2026-11-11")]); + const store = new MemoryStore(); + const result = await run(logs, store, at("2026-11-14T03:00:00Z")); + expect(result.finalized).toEqual(["2026-11-12"]); + expect(store.day("2026-11-11")).toEqual([]); + expect(logs.opened).toEqual([]); + }); + + it("prunes stored file counts after the log retention window", async () => { + const logs = new MemoryLogs(); + const store = new MemoryStore(); + logs.put("D.2026-11-12-10.a.gz", [play("2026-11-12")]); + await run(logs, store, at("2026-11-13T09:00:00Z")); + await run(new MemoryLogs(), store, at("2026-11-30T00:00:00Z")); + expect(store.files.size).toBe(0); + }); +}); + +describe("delivery log location", () => { + it("reads the bucket and key prefix", () => { + expect( + readDeliveryLogLocation("s3://logs-bucket/AWSLogs/1/CloudFront/DIST."), + ).toEqual({ bucket: "logs-bucket", prefix: "AWSLogs/1/CloudFront/DIST." }); + expect(readDeliveryLogLocation(undefined)).toBeNull(); + expect(readDeliveryLogLocation("logs-bucket/x")).toBeNull(); + }); + + it("reads the hour from a log file name", () => { + expect(deliveryLogHour("DIST.2026-11-12-07.abc123.gz")?.toISOString()).toBe( + "2026-11-12T07:00:00.000Z", + ); + expect(deliveryLogHour("DIST.2026-11-12.abc.gz")).toBeNull(); + }); +}); diff --git a/apps/web/__tests__/unit/infra-delivery.test.ts b/apps/web/__tests__/unit/infra-delivery.test.ts new file mode 100644 index 00000000000..25f3a615a7e --- /dev/null +++ b/apps/web/__tests__/unit/infra-delivery.test.ts @@ -0,0 +1,199 @@ +import type { SQL } from "drizzle-orm"; +import { MySqlDialect } from "drizzle-orm/mysql-core"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + rows: [] as { tier: string; seconds: unknown; finalThrough: unknown }[], + failure: null as unknown, + where: [] as unknown[], + groupBy: 0, +})); + +vi.mock("@cap/database", () => ({ + db: () => { + const chain = { + select: () => chain, + from: () => chain, + where: (condition: unknown) => { + mocks.where.push(condition); + return chain; + }, + groupBy: () => { + mocks.groupBy++; + return mocks.failure + ? Promise.reject(mocks.failure) + : Promise.resolve(mocks.rows); + }, + }; + return chain; + }, +})); + +import { + deliveryJson, + deliveryPeriod, + getDeliveredUsage, + isDeliveryNotReadyError, + sumDeliveryRows, +} from "@/lib/infra-delivery"; + +const dialect = new MySqlDialect(); +const query = (condition: unknown) => dialect.sqlToQuery(condition as SQL); + +beforeEach(() => { + mocks.rows = []; + mocks.failure = null; + mocks.where = []; + mocks.groupBy = 0; +}); + +describe("getDeliveredUsage", () => { + it("returns null when the delivery table does not exist yet", async () => { + mocks.failure = Object.assign(new Error("Failed query"), { + cause: Object.assign(new Error("Table doesn't exist"), { + code: "ER_NO_SUCH_TABLE", + errno: 1146, + }), + }); + + await expect(getDeliveredUsage("app-1")).resolves.toBeNull(); + }); + + it("returns null while the table has no final column yet", async () => { + mocks.failure = Object.assign(new Error("Failed query"), { + cause: Object.assign(new Error("Unknown column 'final'"), { + code: "ER_BAD_FIELD_ERROR", + errno: 1054, + }), + }); + + await expect(getDeliveredUsage("app-1")).resolves.toBeNull(); + }); + + it("throws any other database error", async () => { + mocks.failure = Object.assign(new Error("Failed query"), { + cause: Object.assign(new Error("Lock wait timeout"), { + code: "ER_LOCK_WAIT_TIMEOUT", + errno: 1205, + }), + }); + + await expect(getDeliveredUsage("app-1")).rejects.toThrow("Failed query"); + }); + + it("returns null when nothing was measured this month", async () => { + await expect(getDeliveredUsage("app-1")).resolves.toBeNull(); + }); + + it("sums seconds for each tier", async () => { + mocks.rows = [ + { tier: "720p", seconds: "20160", finalThrough: "2026-11-17" }, + { tier: "1080p", seconds: "54000", finalThrough: "2026-11-18" }, + { tier: "360p", seconds: 750, finalThrough: null }, + ]; + + await expect(getDeliveredUsage("app-1")).resolves.toEqual({ + seconds: 74_910, + secondsByTier: { + "360p": 750, + "720p": 20_160, + "1080p": 54_000, + "1440p": 0, + "2160p": 0, + }, + finalThrough: "2026-11-18", + }); + }); + + it("reads one app over the UTC calendar month, on the primary key prefix, without the audio tier", async () => { + await getDeliveredUsage("app-1", new Date("2026-11-30T23:59:59.999Z")); + + const { sql, params } = query(mocks.where[0]); + expect(sql).toBe( + "(`developer_delivery_daily`.`appId` = ? and `developer_delivery_daily`.`day` >= ? and `developer_delivery_daily`.`day` < ? and `developer_delivery_daily`.`tier` <> ?)", + ); + expect(params).toEqual(["app-1", "2026-11-01", "2026-12-01", "audio"]); + expect(mocks.groupBy).toBe(1); + }); +}); + +describe("deliveryPeriod", () => { + it("starts at 00:00 UTC on the 1st, the same as speech", () => { + expect(deliveryPeriod(new Date("2026-11-01T00:00:00Z"))).toEqual({ + start: new Date("2026-11-01T00:00:00Z"), + startDay: "2026-11-01", + endDay: "2026-12-01", + }); + }); + + it("rolls over the year in December", () => { + expect(deliveryPeriod(new Date("2026-12-31T23:59:59Z"))).toMatchObject({ + startDay: "2026-12-01", + endDay: "2027-01-01", + }); + }); +}); + +describe("delivery rows", () => { + it("ignores tiers it does not know and empty values", () => { + expect( + sumDeliveryRows([ + { tier: "480p", seconds: 600, finalThrough: "2026-11-19" }, + { tier: "audio", seconds: 0, finalThrough: "2026-11-19" }, + { tier: "1080p", seconds: null, finalThrough: "not a day" }, + ]), + ).toEqual({ + seconds: 0, + secondsByTier: { + "360p": 0, + "720p": 0, + "1080p": 0, + "1440p": 0, + "2160p": 0, + }, + finalThrough: null, + }); + expect( + sumDeliveryRows([{ tier: "audio", seconds: 0, finalThrough: null }]), + ).toBeNull(); + }); + + it("reports minutes for each tier and the last final day, with no audio tier", () => { + const json = deliveryJson({ + seconds: 90, + secondsByTier: { + "360p": 0, + "720p": 0, + "1080p": 90, + "1440p": 0, + "2160p": 0, + }, + finalThrough: "2026-11-18", + }); + expect(json).toEqual({ + minutes: 1.5, + minutesByTier: { + "360p": 0, + "720p": 0, + "1080p": 1.5, + "1440p": 0, + "2160p": 0, + }, + finalThrough: "2026-11-18", + }); + expect(Object.keys(json?.minutesByTier ?? {})).not.toContain("audio"); + expect(deliveryJson(null)).toBeNull(); + }); + + it("recognises a missing table or column at any depth", () => { + expect(isDeliveryNotReadyError({ errno: 1146 })).toBe(true); + expect(isDeliveryNotReadyError({ code: "ER_NO_SUCH_TABLE" })).toBe(true); + expect(isDeliveryNotReadyError({ errno: 1054 })).toBe(true); + expect(isDeliveryNotReadyError({ code: "ER_BAD_FIELD_ERROR" })).toBe(true); + expect(isDeliveryNotReadyError({ cause: { cause: { errno: 1146 } } })).toBe( + true, + ); + expect(isDeliveryNotReadyError({ errno: 1205 })).toBe(false); + expect(isDeliveryNotReadyError(null)).toBe(false); + }); +}); diff --git a/apps/web/__tests__/unit/infra-discovery.test.ts b/apps/web/__tests__/unit/infra-discovery.test.ts new file mode 100644 index 00000000000..866e62cc161 --- /dev/null +++ b/apps/web/__tests__/unit/infra-discovery.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, it, vi } from "vitest"; + +vi.mock("server-only", () => ({})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_WEB_URL: "https://cap.so" }, + serverEnv: () => ({ WEB_URL: "https://cap.so" }), +})); + +import { GET as infraCard } from "@/app/.well-known/mcp/infra/route"; +import { GET as infraCardJson } from "@/app/.well-known/mcp/infra/server-card.json/route"; +import { GET as libraryCard } from "@/app/.well-known/mcp/route"; +import { GET as libraryCardJson } from "@/app/.well-known/mcp/server-card.json/route"; +import { GET as infraProtectedResource } from "@/app/.well-known/oauth-protected-resource/api/developer/mcp/route"; +import { GET as libraryProtectedResource } from "@/app/.well-known/oauth-protected-resource/api/mcp/route"; +import { GET as rootProtectedResource } from "@/app/.well-known/oauth-protected-resource/route"; +import { GET as infraLlmsRoute } from "@/app/(infra-site)/infra/llms.txt/route"; +import { GET as infraLlmsFullRoute } from "@/app/(infra-site)/infra/llms-full.txt/route"; +import { GET as infraMarkdownRoute } from "@/app/infra-markdown/route"; +import { CAP_INFRA_AGENT_PROMPT } from "@/data/infra-agent-prompt"; +import { INFRA_DOC_SLUGS } from "@/lib/infra-markdown"; + +const LIBRARY_CARD = `{ + "$schema": "https://static.modelcontextprotocol.io/schemas/v1/server-card.schema.json", + "name": "so.cap/cap", + "title": "Cap", + "description": "Read-only access to your Cap screen recordings: find recordings by title, get status and share links, and read summaries and timestamped transcripts.", + "version": "1.0.0", + "websiteUrl": "https://cap.so/agents", + "repository": { + "source": "github", + "url": "https://github.com/CapSoftware/Cap" + }, + "remotes": [ + { + "type": "streamable-http", + "url": "https://cap.so/api/mcp" + } + ] +}`; + +const LIBRARY_PROTECTED_RESOURCE = + '{"resource":"https://cap.so/api/mcp","authorization_servers":["https://cap.so"],"scopes_supported":["caps:read"],"bearer_methods_supported":["header"]}'; + +const headersOf = (response: Response) => + Object.fromEntries(response.headers.entries()); + +describe("the library MCP discovery documents", () => { + it("stay byte-identical, with their headers", async () => { + const card = libraryCard(); + expect(await card.text()).toBe(LIBRARY_CARD); + expect(headersOf(card)).toEqual({ + "access-control-allow-origin": "*", + "cache-control": "public, max-age=3600", + "content-type": "application/json; charset=utf-8", + }); + + const cardJson = libraryCardJson(); + expect(await cardJson.text()).toBe(LIBRARY_CARD); + expect(cardJson.headers.get("content-type")).toBe( + "application/mcp-server-card+json; charset=utf-8", + ); + + for (const response of [ + libraryProtectedResource(), + rootProtectedResource(), + ]) { + expect(await response.text()).toBe(LIBRARY_PROTECTED_RESOURCE); + expect(response.status).toBe(200); + expect(response.headers.get("cache-control")).toBe("public, max-age=300"); + } + }); +}); + +describe("the Cap Infra MCP server card", () => { + it("is served on its own path and names the hosted server and its bearer key", async () => { + const card = await infraCard().json(); + expect(card).toMatchObject({ + name: "so.cap/infra", + title: "Cap Infra", + websiteUrl: "https://cap.so/docs/api/get-started#10-use-with-agents-mcp", + remotes: [ + { + type: "streamable-http", + url: "https://cap.so/api/developer/mcp", + headers: [ + { name: "Authorization", isRequired: true, isSecret: true }, + ], + }, + ], + }); + const typed = infraCardJson(); + expect(typed.headers.get("content-type")).toBe( + "application/mcp-server-card+json; charset=utf-8", + ); + expect(await typed.json()).toEqual(card); + }); +}); + +describe("protected resource metadata for the Cap Infra MCP server", () => { + it("answers a JSON 404 that says to send the secret key, with no OAuth", async () => { + const response = infraProtectedResource(); + expect(response.status).toBe(404); + expect(response.headers.get("content-type")).toContain("application/json"); + expect(await response.json()).toMatchObject({ + code: "oauth_not_supported", + message: "The Cap Infra MCP server does not use OAuth", + hint: expect.stringContaining("Authorization: Bearer"), + }); + }); +}); + +describe("/infra as Markdown", () => { + it("carries the agent prompt the page copies, and the links an agent needs", async () => { + const response = infraMarkdownRoute( + new Request("https://cap.so/infra-markdown"), + ); + expect(response.status).toBe(200); + expect(headersOf(response)).toMatchObject({ + "content-type": "text/markdown; charset=utf-8", + vary: "Accept", + }); + const markdown = await response.text(); + expect(markdown.startsWith("# Cap Infra\n")).toBe(true); + expect(markdown).toContain(`\`\`\`text\n${CAP_INFRA_AGENT_PROMPT}\n\`\`\``); + for (const link of [ + "https://cap.so/docs/api/get-started", + "https://cap.so/openapi.json", + "https://cap.so/docs/api/webhooks", + "https://cap.so/docs/api/get-started#10-use-with-agents-mcp", + "https://cap.so/.well-known/mcp/infra/server-card.json", + "https://cap.so/llms.txt", + "npx -y @cap.so/infra-mcp", + ]) { + expect(markdown).toContain(link); + } + expect(markdown).not.toMatch(/<[a-z]/i); + }); + + it("links the origin it was asked on", async () => { + const markdown = await infraMarkdownRoute( + new Request("https://preview.example/infra-markdown"), + ).text(); + expect(markdown).toContain("https://preview.example/docs/api/rest-api"); + }); +}); + +describe("/infra/llms.txt", () => { + it("follows the llms.txt layout and links every Infra doc as Markdown", async () => { + const response = infraLlmsRoute( + new Request("https://cap.so/infra/llms.txt"), + ); + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toBe( + "text/plain; charset=utf-8", + ); + const text = await response.text(); + const lines = text.split("\n"); + expect(lines[0]).toBe("# Cap Infra"); + expect(lines[2]?.startsWith("> ")).toBe(true); + for (const slug of INFRA_DOC_SLUGS) { + expect(text).toContain(`](https://cap.so/docs/${slug}.md): `); + } + for (const link of [ + "https://cap.so/infra/llms-full.txt", + "https://cap.so/openapi.json", + "https://cap.so/infra.md", + "https://cap.so/api/developer/mcp", + "https://cap.so/.well-known/mcp/infra/server-card.json", + ]) { + expect(text).toContain(link); + } + const links = lines.filter((line) => line.startsWith("- ")); + for (const line of links) { + expect(line).toMatch(/^- \[[^\]]+\]\([^)]+\)(: .+)?$/); + } + }); + + it("links the origin it was asked on", async () => { + const text = await infraLlmsRoute( + new Request("https://preview.example/infra/llms.txt"), + ).text(); + expect(text).toContain("https://preview.example/docs/api/rest-api.md"); + }); +}); + +describe("/infra/llms-full.txt", () => { + it("holds the Infra overview and every Infra doc in full, in order", async () => { + const response = infraLlmsFullRoute( + new Request("https://cap.so/infra/llms-full.txt"), + ); + expect(response.status).toBe(200); + const text = await response.text(); + expect(text.startsWith("# Cap Infra\n")).toBe(true); + expect(text).toContain(CAP_INFRA_AGENT_PROMPT); + + const positions = INFRA_DOC_SLUGS.map((slug) => + text.indexOf(`Source: https://cap.so/docs/${slug}\n`), + ); + for (const position of positions) expect(position).toBeGreaterThan(0); + expect([...positions].sort((a, b) => a - b)).toEqual(positions); + + for (const heading of [ + "# Get Started", + "# REST API", + "# Webhooks", + "# Billing", + ]) { + expect(text).toContain(`\n${heading}\n`); + } + expect(text).not.toMatch(/^<[A-Z]/m); + expect(text).not.toContain("](/"); + }); +}); diff --git a/apps/web/__tests__/unit/infra-dispute-tie.test.ts b/apps/web/__tests__/unit/infra-dispute-tie.test.ts new file mode 100644 index 00000000000..4d741ccdcc8 --- /dev/null +++ b/apps/web/__tests__/unit/infra-dispute-tie.test.ts @@ -0,0 +1,64 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const holder = vi.hoisted(() => ({ db: null as unknown, ids: 0 })); + +vi.mock("@cap/database", () => ({ db: () => holder.db })); +vi.mock("@cap/database/helpers", () => ({ + nanoId: () => `row-${++holder.ids}`, + nanoIdLength: 15, +})); + +import { fakeDb, ledger, resetLedger } from "./infra-billing-fake-db"; + +holder.db = fakeDb; + +import { + disputedSince, + getInfraAppStanding, + readStandingAccount, +} from "@/lib/infra-standing"; + +const adjustment = (id: string, referenceType: string, createdAt: Date) => ({ + id, + accountId: ledger.account.id, + type: "adjustment", + amountMicroCredits: 0, + balanceAfterMicroCredits: ledger.account.balanceMicroCredits, + referenceId: id, + referenceType, + createdAt, +}); + +beforeEach(() => { + resetLedger(); + holder.ids = 0; +}); + +describe("a dispute and a review written in the same second", () => { + const second = new Date("2026-11-11T06:00:00.000Z"); + + it("is decided by the larger id, not by which was written first", async () => { + ledger.rows.push(adjustment("z-review", "manual", second)); + ledger.rows.push(adjustment("a-dispute", "stripe_dispute", second)); + + expect(await disputedSince(fakeDb as never, ledger.account.id)).toBeNull(); + expect((await readStandingAccount("app-1"))?.disputedAt).toBeNull(); + expect(await getInfraAppStanding("app-1")).toMatchObject({ + state: "active", + }); + }); + + it("is disputed when the dispute has the larger id, whichever was written first", async () => { + ledger.rows.push(adjustment("z-dispute", "stripe_dispute", second)); + ledger.rows.push(adjustment("a-review", "manual", second)); + + expect(await disputedSince(fakeDb as never, ledger.account.id)).toEqual( + second, + ); + expect((await readStandingAccount("app-1"))?.disputedAt).toEqual(second); + expect(await getInfraAppStanding("app-1")).toMatchObject({ + state: "paused", + disputed: true, + }); + }); +}); diff --git a/apps/web/__tests__/unit/infra-embed-ancestors.test.ts b/apps/web/__tests__/unit/infra-embed-ancestors.test.ts new file mode 100644 index 00000000000..c17fd5ed8f2 --- /dev/null +++ b/apps/web/__tests__/unit/infra-embed-ancestors.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it } from "vitest"; +import { normalizeEmbedOrigin } from "@/lib/infra-domain"; +import { + embedFrameAncestors, + embedFrameSource, + embedPlaybackId, + withFrameAncestors, +} from "@/lib/infra-embed-ancestors"; + +describe("embedFrameAncestors", () => { + it("sends nothing for an empty list", () => { + expect(embedFrameAncestors([])).toBeNull(); + }); + + it("allows one site plus the hosted origin", () => { + expect(embedFrameAncestors(["https://myshop.com"])).toBe( + "frame-ancestors 'self' https://myshop.com", + ); + }); + + it("allows several sites, once each", () => { + expect( + embedFrameAncestors([ + "https://myshop.com", + "https://www.myshop.com", + "https://app.example.com:8443", + "https://myshop.com", + ]), + ).toBe( + "frame-ancestors 'self' https://myshop.com https://www.myshop.com https://app.example.com:8443", + ); + }); + + it("keeps localhost over http for testing", () => { + expect( + embedFrameAncestors(["http://localhost:3000", "http://127.0.0.1:8080"]), + ).toBe( + "frame-ancestors 'self' http://localhost:3000 http://127.0.0.1:8080", + ); + }); + + it.each([ + "*.myshop.com", + "https://*.myshop.com", + "*", + "http://myshop.com", + "https://myshop.com/path", + "https://myshop.com/", + "myshop.com", + "https://myshop.com; script-src *", + "https://myshop.com https://evil.example", + "https://user:pass@myshop.com", + "javascript:alert(1)", + "", + "'self'", + "'unsafe-inline'", + "data:", + "HTTPS://MYSHOP.COM", + ])("skips the stored entry %j", (entry) => { + expect(embedFrameSource(entry)).toBeNull(); + expect(embedFrameAncestors([entry, "https://myshop.com"])).toBe( + "frame-ancestors 'self' https://myshop.com", + ); + }); + + it("keeps a set list restrictive when no entry is usable", () => { + expect(embedFrameAncestors(["http://myshop.com", "*.myshop.com"])).toBe( + "frame-ancestors 'self'", + ); + }); +}); + +describe("withFrameAncestors", () => { + const ancestors = "frame-ancestors 'self' https://myshop.com"; + + it("sets the directive when there is no policy", () => { + expect(withFrameAncestors(null, ancestors)).toBe(ancestors); + }); + + it("keeps the other directives of an existing policy", () => { + expect( + withFrameAncestors("default-src 'self'; img-src * data:", ancestors), + ).toBe(`default-src 'self'; img-src * data:; ${ancestors}`); + }); + + it("replaces an existing frame-ancestors directive", () => { + expect( + withFrameAncestors("default-src 'self'; Frame-Ancestors *; ", ancestors), + ).toBe(`default-src 'self'; ${ancestors}`); + }); +}); + +describe("embedPlaybackId", () => { + it.each([ + ["/dev/playback-id-0123456789ab/embed", "playback-id-0123456789ab"], + ["/dev/%70layback-id-0123456789ab/embed", "playback-id-0123456789ab"], + ["/Dev/playback-id-0123456789ab/Embed", "playback-id-0123456789ab"], + ["/DEV/playback-id-0123456789ab/EMBED", "playback-id-0123456789ab"], + ["/dev/playback%2Did%2D0123456789ab/embed", "playback-id-0123456789ab"], + ])("finds the playback id in %s", (path, id) => { + expect(embedPlaybackId(path)).toBe(id); + }); + + it.each([ + "/dev/playback-id-0123456789ab", + "/dev/playback-id-0123456789ab/embed/", + "/dev/playback-id-0123456789ab/embed/x", + "//dev/playback-id-0123456789ab/embed", + "/dev/short/embed", + "/dev/playback%2Fid-0123456789ab/embed", + "/dev/%E0%A4%A/embed", + "/s/playback-id-0123456789ab/embed", + "/embed/playback-id-0123456789ab", + "/%64ev/playback-id-0123456789ab/embed", + "/dev/playback-id-0123456789ab/%65mbed", + ])("ignores %s", (path) => { + expect(embedPlaybackId(path)).toBeNull(); + }); +}); + +describe("normalizeEmbedOrigin", () => { + it.each([ + ["myshop.com", "https://myshop.com"], + ["https://myshop.com/pricing", "https://myshop.com"], + ["localhost:3000", "http://localhost:3000"], + ["http://127.0.0.1:8080", "http://127.0.0.1:8080"], + ])("accepts %s as %s", (input, origin) => { + expect(normalizeEmbedOrigin(input)).toEqual({ ok: true, origin }); + }); + + it.each(["http://myshop.com", "*.myshop.com", "ftp://myshop.com", ""])( + "refuses %j", + (input) => { + expect(normalizeEmbedOrigin(input).ok).toBe(false); + }, + ); +}); diff --git a/apps/web/__tests__/unit/infra-first-month-start.test.ts b/apps/web/__tests__/unit/infra-first-month-start.test.ts new file mode 100644 index 00000000000..b015df66873 --- /dev/null +++ b/apps/web/__tests__/unit/infra-first-month-start.test.ts @@ -0,0 +1,108 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/database", () => ({ db: () => ({}) })); + +import { + clearFirstMonthChargesBeganCache, + FIRST_MONTH_START_SETTLED_MS, + firstMonthChargesBegan, +} from "@/lib/infra-first-month-start"; + +const FIRST_CHARGE = new Date("2026-11-13T10:39:54.000Z"); + +const reader = (results: (Date | null)[]) => { + const reads = { count: 0 }; + const chain = (outer: boolean) => { + const query = { + from: () => query, + where: () => query, + orderBy: () => query, + limit: () => query, + then: outer + ? (resolve: (rows: { createdAt: Date | null }[]) => unknown) => { + reads.count += 1; + const next = results.shift() ?? null; + return Promise.resolve([ + { createdAt: next ? new Date(next.getTime()) : null }, + ]).then(resolve); + } + : undefined, + }; + return query; + }; + let selects = 0; + const database = { + select: (fields: Record) => { + selects += 1; + return chain("createdAt" in fields && !("storagePaidUntil" in fields)); + }, + } as unknown as Parameters[0]; + return { database, reads, selects: () => selects }; +}; + +describe("when first month charging began", () => { + beforeEach(() => { + clearFirstMonthChargesBeganCache(); + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-11-13T15:00:00.000Z")); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("reads the start once it is known, and never queries for it again", async () => { + const first = reader([FIRST_CHARGE]); + expect(await firstMonthChargesBegan(first.database)).toEqual(FIRST_CHARGE); + expect(first.reads.count).toBe(1); + + const second = reader([new Date("2027-01-01T00:00:00.000Z")]); + expect(await firstMonthChargesBegan(second.database)).toEqual(FIRST_CHARGE); + expect(second.selects()).toBe(0); + expect(second.reads.count).toBe(0); + }); + + it("reads again every time while no first month charge exists, then keeps the first one found", async () => { + const database = reader([null, null, FIRST_CHARGE]); + + expect(await firstMonthChargesBegan(database.database)).toBeNull(); + expect(await firstMonthChargesBegan(database.database)).toBeNull(); + expect(await firstMonthChargesBegan(database.database)).toEqual( + FIRST_CHARGE, + ); + expect(await firstMonthChargesBegan(database.database)).toEqual( + FIRST_CHARGE, + ); + expect(database.reads.count).toBe(3); + }); + + it("hands out a copy, so a caller cannot move the start", async () => { + const database = reader([FIRST_CHARGE]); + const read = await firstMonthChargesBegan(database.database); + read?.setTime(0); + const cached = await firstMonthChargesBegan(database.database); + cached?.setTime(0); + + expect(await firstMonthChargesBegan(database.database)).toEqual( + FIRST_CHARGE, + ); + expect(database.reads.count).toBe(1); + }); + + it("keeps reading a start younger than the settle window, so an earlier charge committing late is still found", async () => { + const late = new Date("2026-11-13T14:59:00.000Z"); + const earlier = new Date("2026-11-13T14:58:30.000Z"); + const database = reader([late, earlier, earlier]); + + expect(await firstMonthChargesBegan(database.database)).toEqual(late); + expect(await firstMonthChargesBegan(database.database)).toEqual(earlier); + expect(database.reads.count).toBe(2); + + vi.setSystemTime( + new Date(earlier.getTime() + FIRST_MONTH_START_SETTLED_MS), + ); + expect(await firstMonthChargesBegan(database.database)).toEqual(earlier); + expect(await firstMonthChargesBegan(database.database)).toEqual(earlier); + expect(database.reads.count).toBe(3); + }); +}); diff --git a/apps/web/__tests__/unit/infra-hosted-mcp.test.ts b/apps/web/__tests__/unit/infra-hosted-mcp.test.ts new file mode 100644 index 00000000000..5fb59329576 --- /dev/null +++ b/apps/web/__tests__/unit/infra-hosted-mcp.test.ts @@ -0,0 +1,1003 @@ +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { DEVELOPER_KEY_REQUESTS_PER_MINUTE } from "@/lib/developer-rate-limits"; + +const ORIGIN = "https://cap.test"; +const ENDPOINT = `${ORIGIN}/api/developer/mcp`; + +const state = vi.hoisted(() => ({ + keys: new Map(), + gatedApps: new Set(), + rest: [] as Request[], + nextKey: 0, + presignFails: false, + completeStatus: 200, + completeBody: null as Record | null, +})); + +vi.mock("server-only", () => ({})); +vi.mock("@cap/database", async () => { + const { drizzle } = await import("drizzle-orm/mysql-proxy"); + const database = drizzle(async (query, params) => { + if (query.includes("from `developer_api_keys`")) { + const key = state.keys.get(String(params[0]).replace("hash:", "")); + if (!key || key.revoked || params[1] !== "secret") return { rows: [] }; + return { rows: [[key.appId]] }; + } + return { rows: [{ affectedRows: 1 }] }; + }); + return { db: () => database }; +}); +vi.mock("@cap/database/auth/session", () => ({ getCurrentUser: vi.fn() })); +vi.mock("@cap/database/auth/domain-utils", () => ({ + isBlockedAccountEmail: () => false, +})); +vi.mock("@cap/database/auth/request-site", () => ({ + isCrossSiteCookieWrite: () => false, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_WEB_URL: ORIGIN }, + serverEnv: () => ({ WEB_URL: ORIGIN }), +})); +vi.mock("@/lib/developer-key-hash", () => ({ + hashKey: async (key: string) => `hash:${key}`, +})); +vi.mock("@/lib/rate-limit", async (importOriginal) => ({ + ...(await importOriginal()), + sharedRateLimitDecision: async () => undefined, +})); + +const SIGNED_PART = + "https://storage.test/developer/app1/video-1/video?partNumber=1&X-Amz-Credential=AKIAEXAMPLE&X-Amz-Signature=deadbeefsignature"; + +function appOf(request: Request) { + const key = request.headers.get("authorization")?.split(" ")[1] ?? ""; + return state.keys.get(key)?.appId ?? null; +} + +async function restAnswer(request: Request): Promise { + state.rest.push(request); + const url = new URL(request.url); + const route = `${request.method} ${url.pathname}`; + const appId = appOf(request); + switch (route) { + case "GET /api/developer/v1/usage": + return Response.json({ + data: { + balanceMicroCredits: 5_000_000, + standing: { state: "active" }, + upload: { allowed: true }, + }, + }); + case "GET /api/developer/v1/videos": + return Response.json({ + data: [{ id: "video-1", name: "Demo", status: "ready", duration: 12 }], + total: 1, + }); + case "POST /api/developer/v1/uploads": + return Response.json({ + data: { + id: "video-1", + videoId: "video-1", + token: "cut_uploadtoken123.signature", + shareUrl: `${ORIGIN}/infra/s/video-1`, + embedUrl: `${ORIGIN}/infra/e/video-1`, + }, + }); + case "POST /api/developer/v1/uploads/video-1/token": + return Response.json({ + data: { videoId: "video-1", token: "cut_renewedtoken123.signature" }, + }); + case "POST /api/developer/sdk/v1/upload/multipart/initiate": + return Response.json({ uploadId: "upload-1" }); + case "POST /api/developer/sdk/v1/upload/multipart/presign-part": + if (state.presignFails) { + return Response.json( + { code: "storage_unavailable", message: "Storage failed" }, + { status: 400 }, + ); + } + return Response.json({ + parts: ( + ((await request.clone().json()) as { partNumbers?: number[] }) + .partNumbers ?? [1] + ).map((partNumber) => ({ + partNumber, + presignedUrl: SIGNED_PART.replace( + "partNumber=1", + `partNumber=${partNumber}`, + ), + })), + }); + case "POST /api/developer/sdk/v1/upload/multipart/complete": + if (state.completeStatus !== 200) { + return Response.json( + state.completeBody ?? { + code: "internal_error", + message: "Something went wrong", + }, + { status: state.completeStatus }, + ); + } + return Response.json({ success: true }); + case "GET /api/developer/v1/videos/video-1": + return Response.json({ + data: { id: "video-1", status: "uploading", shareUrl: null }, + }); + case "POST /api/developer/sdk/v1/upload/multipart/abort": + return Response.json({ success: true }); + case "POST /api/developer/v1/transcripts": + if (appId !== null && state.gatedApps.has(appId)) { + return Response.json( + { + code: "infra_not_available", + message: "Cap Infra is not available for this account yet", + hint: "Cap Infra is in preview. Contact hello@cap.so for access.", + }, + { status: 403 }, + ); + } + return Response.json({ + data: { id: "transcript-1", status: "queued", name: "Talk" }, + }); + case "GET /api/developer/v1/transcripts/transcript-1": + return Response.json({ + data: { id: "transcript-1", status: "complete", text: "Hello there" }, + }); + case "GET /api/developer/v1/videos/video-slow": + return new Promise(() => {}); + default: + return Response.json( + { code: "not_found", message: route }, + { status: 404 }, + ); + } +} + +vi.mock("@/app/api/developer/v1/[...route]/app", () => ({ + app: { fetch: restAnswer }, +})); +vi.mock("@/app/api/developer/sdk/v1/[...route]/app", () => ({ + app: { fetch: restAnswer }, +})); + +type Route = typeof import("@/app/api/developer/mcp/route"); +let route: Route; + +beforeAll(async () => { + route = await import("@/app/api/developer/mcp/route"); +}); + +beforeEach(() => { + state.rest.length = 0; + state.gatedApps.clear(); + state.presignFails = false; + state.completeStatus = 200; + state.completeBody = null; +}); + +const newKey = (appId = "app1", revoked = false) => { + const key = `csk_${appId}_unitsecret${++state.nextKey}`; + state.keys.set(key, { appId, revoked }); + return key; +}; + +function mcp( + credential: string | null, + body: unknown, + headers: Record = {}, +) { + return route.POST( + new Request(ENDPOINT, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json, text/event-stream", + "X-Forwarded-For": "203.0.113.7", + ...(credential ? { Authorization: `Bearer ${credential}` } : {}), + ...headers, + }, + body: typeof body === "string" ? body : JSON.stringify(body), + }), + ); +} + +async function rpc( + credential: string, + method: string, + params: Record = {}, +) { + const response = await mcp(credential, { + jsonrpc: "2.0", + id: 1, + method, + params, + }); + expect(response.status).toBe(200); + const text = await response.text(); + const line = text + .split("\n") + .find((entry) => entry.startsWith("data: ") && entry.includes('"id":1')); + return { + raw: text, + message: JSON.parse(line ? line.slice(6) : text) as { + result?: Record; + error?: unknown; + }, + }; +} + +async function callTool( + credential: string, + name: string, + args: Record = {}, +) { + const { raw, message } = await rpc(credential, "tools/call", { + name, + arguments: args, + }); + const result = message.result as { + content: { text: string }[]; + isError?: boolean; + }; + return { + raw, + isError: result.isError === true, + body: parseToolText(result.content[0]?.text ?? "null"), + }; +} + +// The SDK answers a call its schema refuses with plain text, not JSON. +function parseToolText(text: string) { + try { + return JSON.parse(text); + } catch { + return { text }; + } +} + +describe("hosted Cap Infra MCP auth", () => { + const initialize = { + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "test", version: "1.0.0" }, + }, + }; + + it.each([ + [ + "no Authorization header", + null, + 401, + "invalid_secret_key", + "Missing secret key", + ], + [ + "a public key", + "cpk_app1_copied_from_a_page", + 403, + "public_key_not_allowed", + "Public keys cannot call this MCP server", + ], + [ + "an upload token", + "cut_uploadtoken123.signature", + 401, + "invalid_secret_key", + "Invalid secret key", + ], + [ + "a secret key that does not exist", + "csk_app1_neverissued", + 401, + "invalid_secret_key", + "Invalid or revoked secret key", + ], + ])( + "refuses %s like the REST API", + async (_label, credential, status, code, message) => { + const response = await mcp(credential, initialize); + expect(response.status).toBe(status); + if (status === 401) { + expect(response.headers.get("www-authenticate")).toContain("Bearer"); + } else { + expect(response.headers.get("www-authenticate")).toBeNull(); + } + expect(await response.json()).toMatchObject({ code, message }); + expect(state.rest).toHaveLength(0); + }, + ); + + it("links the MCP setup guide, not the REST reference, when refusing a key", async () => { + for (const credential of [ + null, + "cpk_app1_copied", + "csk_app1_neverissued", + ]) { + const body = await (await mcp(credential, initialize)).json(); + expect(body.docs).toBe( + "https://cap.so/docs/api/get-started#10-use-with-agents-mcp", + ); + expect(JSON.stringify(body)).not.toContain("REST API"); + } + }); + + it("says the key is missing after Bearer when the header has none", async () => { + const response = await mcp(null, initialize, { Authorization: "Bearer " }); + expect(response.status).toBe(401); + expect(await response.json()).toMatchObject({ + code: "invalid_secret_key", + hint: expect.stringMatching( + /^The Authorization header has no key after Bearer\./, + ), + }); + }); + + it("sends the REST API's response headers on refusals and answers", async () => { + const refused = await mcp(null, initialize, { + "x-vercel-id": "iad1::req-1", + }); + expect(refused.status).toBe(401); + expect(refused.headers.get("cache-control")).toBe("private, no-store"); + expect(refused.headers.get("x-request-id")).toBe("iad1::req-1"); + expect(refused.headers.get("www-authenticate")).toBe( + 'Bearer realm="cap-infra"', + ); + + const answered = await mcp(newKey(), initialize, { + "x-vercel-id": "iad1::req-2", + }); + expect(answered.status).toBe(200); + expect(answered.headers.get("cache-control")).toBe("private, no-store"); + expect(answered.headers.get("x-request-id")).toBe("iad1::req-2"); + expect(answered.headers.get("www-authenticate")).toBeNull(); + }); + + it("refuses a revoked secret key", async () => { + const response = await mcp(newKey("app1", true), initialize); + expect(response.status).toBe(401); + expect(await response.json()).toMatchObject({ + code: "invalid_secret_key", + message: "Invalid or revoked secret key", + }); + }); + + it("passes the REST preview refusal through for an owner outside the preview", async () => { + const key = newKey("gated"); + state.gatedApps.add("gated"); + const created = await callTool(key, "transcripts_create", { + url: "https://example.com/talk.mp3", + }); + expect(created.isError).toBe(true); + expect(created.body.error).toMatchObject({ + code: "infra_not_available", + status: 403, + }); + }); + + it("answers initialize for a valid secret key without touching REST", async () => { + const response = await mcp(newKey(), initialize); + expect(response.status).toBe(200); + expect(await response.text()).toContain("cap-infra"); + expect(state.rest).toHaveLength(0); + }); +}); + +describe("hosted Cap Infra MCP tools", () => { + it("lists the hosted tools with no local file access", async () => { + const { message } = await rpc(newKey(), "tools/list"); + const tools = ( + message.result as { + tools: { + name: string; + inputSchema: { + properties: Record< + string, + { maximum?: number; description?: string } + >; + additionalProperties?: boolean; + }; + }[]; + } + ).tools; + expect(tools.map((tool) => tool.name).sort()).toEqual([ + "account_transactions", + "account_usage", + "docs_read", + "transcripts_create", + "transcripts_delete", + "transcripts_get", + "transcripts_list", + "transcripts_text", + "transcripts_wait", + "videos_complete_upload", + "videos_create_upload", + "videos_delete", + "videos_get", + "videos_import", + "videos_list", + "videos_wait", + ]); + const create = tools.find((tool) => tool.name === "transcripts_create"); + expect(Object.keys(create?.inputSchema.properties ?? {})).not.toContain( + "path", + ); + for (const listed of tools) { + expect(listed.inputSchema, listed.name).toMatchObject({ + additionalProperties: false, + }); + } + const upload = tools.find((tool) => tool.name === "videos_create_upload"); + expect(JSON.stringify(upload)).not.toContain("cannot read your files"); + for (const name of ["videos_wait", "transcripts_wait"]) { + const wait = tools.find((tool) => tool.name === name); + expect(wait?.inputSchema.properties.timeoutSeconds?.maximum).toBe(45); + expect(wait?.inputSchema.properties.timeoutSeconds?.description).toBe( + "Longest time to wait, in seconds (default and maximum 45). Checks are at least 3 seconds apart, so a timeout below 4 makes one check and returns at once", + ); + } + }); + + it("reads usage and videos through the in-process REST API with the caller's key and address", async () => { + const key = newKey(); + const usage = await callTool(key, "account_usage"); + expect(usage.isError).toBe(false); + expect(usage.body).toMatchObject({ balanceMicroCredits: 5_000_000 }); + + const videos = await callTool(key, "videos_list", { limit: 5 }); + expect(videos.body.videos).toEqual([ + expect.objectContaining({ id: "video-1", status: "ready" }), + ]); + + const [first, second] = state.rest; + expect(new URL(first?.url ?? "").pathname).toBe("/api/developer/v1/usage"); + expect(first?.headers.get("authorization")).toBe(`Bearer ${key}`); + expect(first?.headers.get("x-forwarded-for")).toBe("203.0.113.7"); + expect(new URL(second?.url ?? "").search).toBe("?limit=5&offset=0"); + expect(usage.raw).not.toContain(key); + }); + + it("creates an upload with signed part links and completes it", async () => { + const key = newKey(); + const info = vi.spyOn(console, "info").mockImplementation(() => {}); + const created = await callTool(key, "videos_create_upload", { + contentLength: 1_000_000, + fileName: "demo.mp4", + }); + expect(created.isError).toBe(false); + expect(created.body).toMatchObject({ + videoId: "video-1", + uploadId: "upload-1", + status: "uploading", + parts: [{ partNumber: 1, bytes: 1_000_000, url: SIGNED_PART }], + shareUrl: `${ORIGIN}/infra/s/video-1`, + }); + expect(created.body.steps.join(" ")).toContain("videos_complete_upload"); + expect(created.raw).not.toContain("cut_"); + expect(created.raw).not.toContain(key); + + const initiate = state.rest.find((request) => + request.url.endsWith("/multipart/initiate"), + ); + expect(initiate?.headers.get("authorization")).toBe( + "Bearer cut_uploadtoken123.signature", + ); + expect(await initiate?.json()).toEqual({ + videoId: "video-1", + contentType: "video/mp4", + contentLength: 1_000_000, + }); + + const completed = await callTool(key, "videos_complete_upload", { + videoId: "video-1", + uploadId: "upload-1", + parts: [{ partNumber: 1, etag: '"etag-1"' }], + }); + expect(completed.body).toMatchObject({ + videoId: "video-1", + status: "processing", + }); + const complete = state.rest.find((request) => + request.url.endsWith("/multipart/complete"), + ); + expect(complete?.headers.get("authorization")).toBe( + "Bearer cut_renewedtoken123.signature", + ); + + const logged = info.mock.calls.flat().join("\n"); + expect(logged).toContain("video_created"); + expect(logged).not.toContain(key); + expect(logged).not.toContain("deadbeefsignature"); + expect(logged).not.toContain("cut_uploadtoken123"); + info.mockRestore(); + }); + + it("creates a transcript from a url and reads its text", async () => { + const key = newKey(); + const created = await callTool(key, "transcripts_create", { + url: "https://example.com/talk.mp3", + }); + expect(created.body).toMatchObject({ id: "transcript-1" }); + const sent = state.rest.at(-1); + expect(sent?.method).toBe("POST"); + expect(await sent?.json()).toEqual({ url: "https://example.com/talk.mp3" }); + + const text = await callTool(key, "transcripts_text", { + transcriptId: "transcript-1", + }); + expect(text.body).toEqual({ + id: "transcript-1", + status: "complete", + text: "Hello there", + }); + }); + + it("transcribes a stored video by id through the REST videoId field", async () => { + const created = await callTool(newKey(), "transcripts_create", { + videoId: "video-1", + }); + expect(created.isError).toBe(false); + expect(state.rest).toHaveLength(1); + expect(await state.rest[0]?.json()).toEqual({ videoId: "video-1" }); + }); + + it("aborts the upload it opened for a new video when signing fails", async () => { + state.presignFails = true; + const created = await callTool(newKey(), "videos_create_upload", { + contentLength: 1_000_000, + }); + expect(created.isError).toBe(true); + expect(created.body).toMatchObject({ + videoId: "video-1", + videoStatus: "uploading", + }); + expect( + state.rest.some((request) => request.url.endsWith("/multipart/abort")), + ).toBe(true); + }); + + it("keeps a reused upload and its stored parts when signing fails on retry", async () => { + state.presignFails = true; + const retried = await callTool(newKey(), "videos_create_upload", { + contentLength: 1_000_000, + videoId: "video-1", + }); + expect(retried.isError).toBe(true); + expect( + state.rest.some((request) => request.url.endsWith("/multipart/abort")), + ).toBe(false); + }); + + it("says the outcome is unknown when completing fails on the server", async () => { + state.completeStatus = 500; + const completed = await callTool(newKey(), "videos_complete_upload", { + videoId: "video-1", + uploadId: "upload-1", + parts: [{ partNumber: 1, etag: "etag-1" }], + }); + expect(completed.isError).toBe(true); + expect(completed.body).toMatchObject({ + videoId: "video-1", + videoStatus: "unknown", + next: expect.stringContaining("videos_get"), + }); + }); + + it("tells the caller to upload again when completing removed the file", async () => { + state.completeStatus = 402; + state.completeBody = { + code: "insufficient_credits", + message: "Not enough credit to store this video", + fileRemoved: true, + }; + const completed = await callTool(newKey(), "videos_complete_upload", { + videoId: "video-1", + uploadId: "upload-1", + parts: [{ partNumber: 1, etag: "etag-1" }], + }); + expect(completed.isError).toBe(true); + expect(completed.body.error).toMatchObject({ fileRemoved: true }); + expect(completed.body.next).toContain("upload the file again"); + expect(completed.body.next).not.toContain("parts are kept"); + }); + + it("refuses a local path for a transcript", async () => { + const result = await callTool(newKey(), "transcripts_create", { + path: "/etc/passwd", + }); + expect(result.isError).toBe(true); + expect(result.body.text).toContain('Unrecognized key: "path"'); + expect(result.body.text).toContain("This tool accepts url, videoId,"); + expect(state.rest).toHaveLength(0); + }); + + it("refuses a transcript with no source, with a hint and a status", async () => { + const result = await callTool(newKey(), "transcripts_create", {}); + expect(result.isError).toBe(true); + expect(result.body.error).toMatchObject({ + code: "invalid_source", + status: 400, + hint: expect.stringContaining("videoId for a ready video"), + }); + }); + + it.each([ + ["url", { contentLength: 100, url: "https://example.com/a.mp4" }], + ["videoID", { contentLength: 100, videoID: "video-1" }], + ])( + "refuses an unknown %s argument instead of creating a video", + async (key, args) => { + const result = await callTool(newKey(), "videos_create_upload", args); + expect(result.isError).toBe(true); + expect(result.body.text).toContain(`Unrecognized key: "${key}"`); + expect(result.body.text).toContain( + "This tool accepts contentLength, fileName, contentType, name, userId, maxResolutionTier, metadata, videoId and fromPart.", + ); + expect(state.rest).toHaveLength(0); + }, + ); + + describe("part links for a large file", () => { + const TWO_GB = 2 * 1024 ** 3; + const MAX_BYTES = 36_000_000_000; + const signed = () => + state.rest.filter((request) => + request.url.endsWith("/multipart/presign-part"), + ); + + it("returns the first ten part links, the part count and where to continue", async () => { + const created = await callTool(newKey(), "videos_create_upload", { + contentLength: TWO_GB, + fileName: "launch.mp4", + }); + + expect(created.isError).toBe(false); + expect(created.body).toMatchObject({ + videoId: "video-1", + uploadId: "upload-1", + partBytes: 64 * 1024 * 1024, + partCount: 32, + nextFromPart: 11, + }); + expect( + created.body.parts.map( + (part: { partNumber: number }) => part.partNumber, + ), + ).toEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + expect(created.body.steps.join(" ")).toContain("fromPart 11"); + expect(created.body.next).toContain("fromPart 11"); + expect(created.raw.length).toBeLessThan(8_000); + }); + + it("keeps the largest file's result as small as a 2 GB one", async () => { + const created = await callTool(newKey(), "videos_create_upload", { + contentLength: MAX_BYTES, + }); + + expect(created.body).toMatchObject({ partCount: 100, nextFromPart: 11 }); + expect(created.body.parts).toHaveLength(10); + expect(created.raw.length).toBeLessThan(8_000); + }); + + it("signs the next parts of the same upload from fromPart, without a new video", async () => { + const next = await callTool(newKey(), "videos_create_upload", { + contentLength: TWO_GB, + videoId: "video-1", + fromPart: 31, + }); + + expect(next.isError).toBe(false); + expect(next.body).toMatchObject({ + videoId: "video-1", + uploadId: "upload-1", + partCount: 32, + nextFromPart: null, + }); + expect(next.body.parts).toEqual([ + expect.objectContaining({ partNumber: 31, bytes: 64 * 1024 * 1024 }), + expect.objectContaining({ partNumber: 32, bytes: 64 * 1024 * 1024 }), + ]); + expect(await signed()[0]?.json()).toMatchObject({ + videoId: "video-1", + uploadId: "upload-1", + partNumbers: [31, 32], + }); + expect( + state.rest.some((request) => + request.url.endsWith("/api/developer/v1/uploads"), + ), + ).toBe(false); + }); + + it("refuses fromPart without a videoId before creating anything", async () => { + const result = await callTool(newKey(), "videos_create_upload", { + contentLength: TWO_GB, + fromPart: 11, + }); + + expect(result.isError).toBe(true); + expect(result.body.error).toMatchObject({ + code: "invalid_request", + status: 400, + message: "fromPart needs the videoId of the upload", + }); + expect(state.rest).toHaveLength(0); + }); + + it("refuses a fromPart past the last part before calling Cap", async () => { + const result = await callTool(newKey(), "videos_create_upload", { + contentLength: 1_000_000, + videoId: "video-1", + fromPart: 2, + }); + + expect(result.isError).toBe(true); + expect(result.body.error).toMatchObject({ + code: "invalid_request", + message: + "fromPart is past the last part: a file of 1000000 bytes has 1 part", + }); + expect(state.rest).toHaveLength(0); + }); + + it("gives a small file one link and nothing to continue", async () => { + const created = await callTool(newKey(), "videos_create_upload", { + contentLength: 1_000_000, + }); + + expect(created.body).toMatchObject({ partCount: 1, nextFromPart: null }); + expect(created.body.steps).toHaveLength(2); + }); + }); + + it.each([ + [{ fileName: "notes.txt", contentType: "text/plain" }], + [{ fileName: "notes.txt" }], + [{ contentType: "application/pdf" }], + [{ fileName: "cover.png" }], + [{ contentType: "audio/mpeg" }], + ])( + "refuses a file that is not a video before creating one: %o", + async (file) => { + const result = await callTool(newKey(), "videos_create_upload", { + contentLength: 20, + ...file, + }); + expect(result.isError).toBe(true); + expect(result.body.error).toMatchObject({ + code: "unsupported_content_type", + status: 400, + message: expect.stringMatching( + /^This file could not be read as a video\. Upload MP4, MOV, MKV and WebM/, + ), + }); + expect(state.rest).toHaveLength(0); + }, + ); + + it.each([ + [{ fileName: "talk.mp4", contentType: "video/mp4" }], + [{ fileName: "talk.mov", contentType: "video/quicktime" }], + [{ fileName: "talk.mkv", contentType: "video/x-matroska" }], + [{ fileName: "talk.webm", contentType: "video/webm" }], + [{ fileName: "talk.mkv", contentType: "application/octet-stream" }], + [{ fileName: "talk.mov", contentType: "" }], + [{ fileName: "talk.mkv" }], + [{ fileName: "capture.unknownext" }], + [{}], + ])("still accepts a video upload: %o", async (file) => { + const args: Record = { contentLength: 100, ...file }; + if (args.contentType === "") delete args.contentType; + const result = await callTool(newKey(), "videos_create_upload", args); + expect(result.isError).toBe(false); + expect(result.body).toMatchObject({ + videoId: "video-1", + uploadId: "upload-1", + }); + }); +}); + +describe("hosted Cap Infra MCP docs_read", () => { + const mainHeadings = (markdown: string) => { + const headings: string[] = []; + let fenced = false; + for (const line of markdown.split("\n")) { + if (/^\s*(`{3,}|~{3,})/.test(line)) fenced = !fenced; + else if (!fenced && line.startsWith("## ")) headings.push(line.slice(3)); + } + return headings; + }; + + it("reads the docs in process, as the .md route serves them, with no REST or HTTP call", async () => { + const { infraDocsSource } = await import("@/lib/infra-mcp-server"); + const { GET } = await import("@/app/docs-markdown/[...slug]/route"); + const key = newKey(); + const network = vi.spyOn(globalThis, "fetch"); + for (const page of ["get-started", "rest-api", "webhooks", "billing"]) { + const served = await ( + await GET(new Request(`${ORIGIN}/docs/api/${page}.md`), { + params: Promise.resolve({ slug: ["api", page] }), + }) + ).text(); + expect( + await infraDocsSource(page as "billing", new AbortController().signal), + ).toBe(served); + const outline = await callTool(key, "docs_read", { page }); + expect(outline.isError).toBe(false); + expect(outline.body.pageUrl).toBe(`${ORIGIN}/docs/api/${page}`); + for (const heading of mainHeadings(served)) { + const section = await callTool(key, "docs_read", { + page, + section: heading, + }); + expect(section.body.section, heading).toBe(heading); + } + } + expect(network).not.toHaveBeenCalled(); + network.mockRestore(); + expect(state.rest).toHaveLength(0); + }); + + it("returns the embed options and the webhook signature check", async () => { + const key = newKey(); + const embed = await callTool(key, "docs_read", { + page: "rest-api", + section: "Embed Options", + }); + expect(embed.body.url).toBe(`${ORIGIN}/docs/api/rest-api#embed-options`); + expect(embed.body.markdown).toMatch(/^### Embed Options/); + const signature = await callTool(key, "docs_read", { + page: "webhooks", + query: "verify the signature", + }); + expect(signature.body.section).toBe("Verify the signature"); + expect(signature.body.url).toBe( + `${ORIGIN}/docs/api/webhooks#verify-the-signature`, + ); + }); + + it("refuses unknown keys and needs a valid secret key", async () => { + const refused = await callTool(newKey(), "docs_read", { + page: "billing", + sectionName: "Credits", + }); + expect(refused.isError).toBe(true); + expect(JSON.stringify(refused.body)).toContain( + 'Unrecognized key: \\"sectionName\\"', + ); + const anonymous = await mcp(null, { + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: { name: "docs_read", arguments: { page: "billing" } }, + }); + expect(anonymous.status).toBe(401); + }); +}); + +describe("hosted Cap Infra MCP limits", () => { + it("refuses a body over 64 KB before any MCP work", async () => { + const response = await mcp(newKey(), { + jsonrpc: "2.0", + id: 1, + method: "tools/call", + params: { + name: "videos_list", + arguments: { userId: "x".repeat(70_000) }, + }, + }); + expect(response.status).toBe(413); + expect(await response.json()).toMatchObject({ code: "request_too_large" }); + expect(state.rest).toHaveLength(0); + }); + + it("refuses a body that is not JSON", async () => { + const response = await mcp(newKey(), "hello", { + "Content-Type": "text/plain", + }); + expect(response.status).toBe(415); + }); + + it("answers GET with 405 since the endpoint holds no sessions", async () => { + const response = await route.GET( + new Request(ENDPOINT, { + headers: { Authorization: `Bearer ${newKey()}` }, + }), + ); + expect(response.status).toBe(405); + expect(response.headers.get("allow")).toBe("POST"); + }); + + it("applies the REST API's per-key limit to MCP requests", async () => { + const key = newKey(); + const list = { jsonrpc: "2.0", id: 1, method: "tools/list", params: {} }; + for (let index = 0; index < DEVELOPER_KEY_REQUESTS_PER_MINUTE; index++) { + expect((await mcp(key, list)).status).toBe(200); + } + const limited = await mcp(key, list); + expect(limited.status).toBe(429); + expect(limited.headers.get("retry-after")).not.toBeNull(); + expect(await limited.json()).toMatchObject({ code: "rate_limited" }); + const initialize = await mcp(key, { + jsonrpc: "2.0", + id: 1, + method: "initialize", + params: { + protocolVersion: "2025-11-25", + capabilities: {}, + clientInfo: { name: "test", version: "1.0.0" }, + }, + }); + expect(initialize.status).toBe(429); + expect((await mcp(newKey(), list)).status).toBe(200); + }); + + it("answers a rate-limited tool call with a tool error that says how long to wait", async () => { + const key = newKey(); + const list = { jsonrpc: "2.0", id: 1, method: "tools/list", params: {} }; + for (let index = 0; index < DEVELOPER_KEY_REQUESTS_PER_MINUTE; index++) { + expect((await mcp(key, list)).status).toBe(200); + } + state.rest.length = 0; + const response = await mcp(key, { + jsonrpc: "2.0", + id: 7, + method: "tools/call", + params: { name: "account_usage", arguments: {} }, + }); + expect(response.status).toBe(200); + const retryAfter = Number(response.headers.get("retry-after")); + expect(retryAfter).toBeGreaterThan(0); + expect(response.headers.get("x-ratelimit-remaining")).toBe("0"); + const message = await response.json(); + expect(message).toMatchObject({ + jsonrpc: "2.0", + id: 7, + result: { isError: true }, + }); + const { error } = JSON.parse(message.result.content[0].text); + expect(error).toMatchObject({ + code: "rate_limited", + status: 429, + retryAfterSeconds: retryAfter, + hint: expect.stringMatching( + new RegExp( + `^Wait ${retryAfter} seconds, then call the tool again\\. Each key allows ${DEVELOPER_KEY_REQUESTS_PER_MINUTE} requests per minute\\.$`, + ), + ), + }); + expect(state.rest).toHaveLength(0); + expect((await mcp(key, list)).status).toBe(429); + }); + + it("stops a tool that outlives its budget and says so", async () => { + const { inProcessDeveloperFetch } = await import("@/lib/infra-mcp-rest"); + const { createHostedTools } = await import("@cap/infra-mcp/hosted"); + const tools = createHostedTools({ + apiUrl: ORIGIN, + secretKey: newKey(), + fetch: inProcessDeveloperFetch(new Headers()), + log: () => {}, + budgetMs: 200, + }); + const get = tools.find((tool) => tool.name === "videos_get"); + const startedAt = Date.now(); + const result = await get?.run( + { videoId: "video-slow" }, + { + signal: new AbortController().signal, + progressToken: undefined, + sendProgress: async () => {}, + sendLog: async () => {}, + }, + ); + expect(Date.now() - startedAt).toBeLessThan(2_000); + expect(result?.isError).toBe(true); + expect(JSON.parse(result?.content[0]?.text ?? "{}").error).toMatchObject({ + code: "timeout", + }); + }); +}); diff --git a/apps/web/__tests__/unit/infra-landing.test.ts b/apps/web/__tests__/unit/infra-landing.test.ts new file mode 100644 index 00000000000..135e44b95aa --- /dev/null +++ b/apps/web/__tests__/unit/infra-landing.test.ts @@ -0,0 +1,642 @@ +import fs from "node:fs"; +import path from "node:path"; +import { JSDOM } from "jsdom"; +import * as React from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const env = vi.hoisted(() => ({ + NEXT_PUBLIC_IS_CAP: "true" as string | undefined, + NEXT_PUBLIC_INFRA_FILM_URL: undefined as string | undefined, +})); + +vi.mock("@cap/env", () => ({ buildEnv: env })); +vi.mock("@cap/database", () => ({ db: () => ({}) })); +vi.mock("next/font/google", () => { + const font = () => ({ className: "", variable: "" }); + return { Geist: font, Geist_Mono: font }; +}); +vi.mock("@/app/utils/analytics", () => ({ trackEvent: vi.fn() })); +vi.mock("@tanstack/react-query", async (importOriginal) => ({ + ...(await importOriginal()), + useMutation: () => ({ mutate: vi.fn(), isPending: false, reset: vi.fn() }), +})); +vi.mock("next/navigation", () => ({ + useRouter: () => ({ push: vi.fn(), replace: vi.fn(), refresh: vi.fn() }), + usePathname: () => "/dashboard/infra/app-1/billing", + useSearchParams: () => new URLSearchParams(), +})); +vi.mock("@number-flow/react", async () => { + const { createElement } = await import("react"); + return { + default: ({ value }: { value: number }) => + createElement("span", null, String(value)), + }; +}); +vi.mock("@cap/database/helpers", () => ({ + nanoId: () => "id", + nanoIdLength: 15, +})); + +import type { BillingAccount } from "@/app/(infra)/dashboard/infra/[appId]/billing/billing-view"; +import { parseAmount } from "@/components/pages/infra/amount"; +import { CAP_INFRA_AGENT_PROMPT } from "@/data/infra-agent-prompt"; +import { + INFRA_DELIVERY_BILLED_FROM_TEXT, + INFRA_DELIVERY_DOLLARS_PER_1000_MINUTES, + INFRA_DELIVERY_PRICE, + INFRA_DELIVERY_PRICE_TEXT, + INFRA_DELIVERY_PRICE_UNIT, + INFRA_RESOLUTION_TIERS, + INFRA_SPEECH_DOLLARS_PER_HOUR, + INFRA_STORAGE_DOLLARS_PER_1000_MINUTES_PER_MONTH, +} from "@/lib/infra-pricing"; +import { PRO_MONTHLY_CREDIT_DOLLARS } from "@/lib/infra-pro-credit"; +import { getDocBySlug } from "@/utils/docs"; + +const rest = getDocBySlug("api/rest-api")?.content ?? ""; +const billing = getDocBySlug("api/billing")?.content ?? ""; +const getStarted = getDocBySlug("api/get-started")?.content ?? ""; + +const PAGE_DIR = path.join(process.cwd(), "components/pages/infra"); + +const pageSources = () => + fs + .readdirSync(PAGE_DIR) + .filter((file) => /\.(ts|tsx|css)$/.test(file)) + .map((file) => ({ + file, + text: fs.readFileSync(path.join(PAGE_DIR, file), "utf8"), + })); + +const parse = (html: string) => new JSDOM(html).window.document; + +const load = async (isCap: string | undefined) => { + env.NEXT_PUBLIC_IS_CAP = isCap; + vi.resetModules(); + return import("@/components/pages/infra/content"); +}; + +describe("the Cap Infra landing page", () => { + beforeEach(() => { + env.NEXT_PUBLIC_IS_CAP = "true"; + }); + + it("shows only requests and responses that are in the REST API docs", async () => { + const content = await load("true"); + + expect(rest).not.toBe(""); + for (const sample of [ + content.CREATE_UPLOAD, + content.GET_VIDEO_STATUS, + content.VIDEO_STATUS_RESPONSE, + ...content.CREATE_TRANSCRIPT.map((entry) => entry.code), + content.PLAYER.find((entry) => entry.id === "embed")?.code, + ]) { + expect(rest).toContain(sample); + } + expect(content.VIDEO_FLOW.map((entry) => entry.code)).toEqual([ + `${content.CREATE_UPLOAD}\n\n${content.GET_VIDEO_STATUS}`, + ]); + for (const url of Object.values(content.SAMPLE_PLAYBACK)) { + expect(rest).toContain(url); + } + for (const sample of content.PLAYER) { + expect( + Object.values(content.SAMPLE_PLAYBACK).some((url) => + sample.code.includes(url), + ), + ).toBe(true); + } + }); + + it("takes every price from the pricing module", async () => { + const content = await load("true"); + + expect(content.PRICES.storagePer1000Minutes).toEqual( + INFRA_STORAGE_DOLLARS_PER_1000_MINUTES_PER_MONTH, + ); + expect(content.PRICES.speechPerHour).toBe(INFRA_SPEECH_DOLLARS_PER_HOUR); + expect(content.PRICES.deliveryPer1000Minutes).toBe( + INFRA_DELIVERY_DOLLARS_PER_1000_MINUTES, + ); + expect(content.PRICES.proMonthlyCredit).toBe(PRO_MONTHLY_CREDIT_DOLLARS); + expect(content.TIERS).toEqual(INFRA_RESOLUTION_TIERS); + + const priced = content.PRICE_ROWS.filter((row) => row.rates); + expect(priced.map((row) => row.meter)).toEqual(["Storage"]); + expect(priced[0]?.rates).toEqual( + INFRA_STORAGE_DOLLARS_PER_1000_MINUTES_PER_MONTH, + ); + for (const tier of INFRA_RESOLUTION_TIERS) { + const price = `$${INFRA_STORAGE_DOLLARS_PER_1000_MINUTES_PER_MONTH[tier].toFixed(2)}`; + expect(billing).toContain(`${price} per 1,000 minutes per month`); + expect(content.infraFaqs.some((faq) => faq.answer.includes(price))).toBe( + true, + ); + expect(content.infraSoftwareSchema.offers.description).toContain(price); + } + }); + + it("charges for what billing charges for and nothing else", async () => { + const content = await load("true"); + + expect(billing).toContain( + "Three things are charged: stored video, delivered video and speech to text. Uploading and encoding are free.", + ); + expect( + content.PRICE_ROWS.filter((row) => row.unit).map((row) => row.meter), + ).toEqual(["Storage", "Delivery"]); + + const text = [ + content.infraSeo.description, + content.infraSeo.ogDescription, + content.infraSoftwareSchema.offers.description, + ...content.infraFaqs.flatMap((faq) => [faq.question, faq.answer]), + ...content.VIDEO_OUTPUTS.flatMap((item) => [item.name, item.detail]), + ...pageSources().map((source) => source.text), + ].join("\n"); + + expect(text).not.toMatch(/\bwatched\b|\bwebhooks?\b/i); + expect(text).not.toContain(String.fromCodePoint(0x2014)); + }); + + it("states the first month of storage in the words the docs use", async () => { + const content = await load("true"); + + expect(billing).toContain(content.STORAGE_RULE); + expect(getStarted).toContain(content.STORAGE_RULE); + expect( + content.infraFaqs.find( + (faq) => faq.question === "What does Cap Infra charge for?", + )?.answer, + ).toBe( + `Three things: stored video, delivered video and speech to text. ${content.STORAGE_RULE} ${content.DELIVERY_RULE}`, + ); + expect( + pageSources().find((source) => source.file === "Pricing.tsx")?.text, + ).toContain("{STORAGE_RULE}"); + }); + + it("lets anyone start building on Cap's own deployment", async () => { + const content = await load("true"); + + expect(content.OPEN_ACCESS).toBe(true); + expect(content.START).toEqual({ + label: "Start building", + href: "/dashboard/infra", + }); + expect(content.PREVIEW_NOTE).toBeNull(); + expect( + content.infraFaqs.some((faq) => faq.answer.includes("preview")), + ).toBe(false); + }); + + it("opens the console where anyone can use it", async () => { + const content = await load("false"); + + expect(content.OPEN_ACCESS).toBe(true); + expect(content.START).toEqual({ + label: "Start building", + href: "/dashboard/infra", + }); + expect(content.PREVIEW_NOTE).toBeNull(); + expect( + content.infraFaqs.some((faq) => faq.answer.includes("preview")), + ).toBe(false); + }); + + it("links only to anchors the page has", async () => { + const content = await load("true"); + const anchors = new Set(Object.values(content.ANCHORS)); + + for (const link of content.NAV_LINKS) { + expect(anchors.has(link.href.slice(1)), link.href).toBe(true); + } + for (const source of pageSources()) { + for (const [, name] of source.text.matchAll(/ANCHORS\.(\w+)/g)) { + expect( + Object.keys(content.ANCHORS), + `${source.file}: ${name}`, + ).toContain(name); + } + } + const heading = content.LINKS.uploadDocs.split("#")[1]; + expect(rest).toMatch(/^#+ Multipart Upload$/m); + expect(heading).toBe("multipart-upload"); + }); +}); + +describe("the rendered page", () => { + it("renders every section with headings in order and working anchors", async () => { + const content = await load("true"); + const { InfraPage } = await import("@/components/pages/infra/InfraPage"); + const html = renderToStaticMarkup(React.createElement(InfraPage)); + + const page = parse(html); + const text = page.body.textContent ?? ""; + + const levels = [...page.querySelectorAll("h1,h2,h3,h4,h5,h6")].map( + (heading) => Number(heading.tagName.slice(1)), + ); + expect(levels.filter((level) => level === 1)).toHaveLength(1); + expect(levels[0]).toBe(1); + for (const [index, level] of levels.entries()) { + expect(level - (levels[index - 1] ?? 1)).toBeLessThanOrEqual(1); + } + + const anchors = [...page.querySelectorAll('a[href^="#"]')].map( + (link) => link.getAttribute("href")?.slice(1) ?? "", + ); + expect(anchors.length).toBeGreaterThan(0); + for (const anchor of anchors) { + expect(page.getElementById(anchor), anchor).not.toBeNull(); + } + for (const anchor of Object.values(content.ANCHORS)) { + expect(page.getElementById(anchor), anchor).not.toBeNull(); + } + + const embed = content.PLAYER.find((entry) => entry.id === "embed")?.code; + expect(embed).toContain('loading="lazy"'); + expect(text).toContain(embed); + expect(text).toContain("Start building"); + expect(text).not.toContain("Request access"); + expect(text).not.toContain("in preview"); + for (const image of page.querySelectorAll("img")) { + expect(image.hasAttribute("alt")).toBe(true); + const eager = + image.hasAttribute("fetchpriority") || + (image.getAttribute("src") ?? "").includes("plain-lit"); + if (!eager) expect(image.getAttribute("loading")).toBe("lazy"); + } + }); + + it("prices speech to text on its own row, outside the resolution table", async () => { + await load("true"); + const { Pricing } = await import("@/components/pages/infra/Pricing"); + const page = parse(renderToStaticMarkup(React.createElement(Pricing))); + const speech = `$${INFRA_SPEECH_DOLLARS_PER_HOUR.toFixed(2)}`; + const table = page.querySelector("table"); + + expect(table?.querySelector("caption")?.textContent).toBe( + "Cap Infra prices by resolution", + ); + expect(table?.textContent).not.toContain("Speech to text"); + expect(table?.textContent).not.toContain(speech); + + const row = [...page.querySelectorAll("dl")].find( + (list) => + list.querySelector("dt")?.textContent?.trim() === "Speech to text", + ); + expect(row).toBeDefined(); + expect(row?.closest("table")).toBeNull(); + expect(row?.querySelector("dd")?.textContent).toBe( + `${speech} per audio hour`, + ); + }); + + it("says a month of storage only where billing would agree", async () => { + const content = await load("true"); + const stored = content.PRO_CREDIT_COVERS[0]; + const days31 = + (stored.amount / 1000) * + INFRA_STORAGE_DOLLARS_PER_1000_MINUTES_PER_MONTH["1080p"] * + (31 / 30); + + expect(stored.unit).toBe("minutes stored for 30 days"); + expect( + (stored.amount / 1000) * + INFRA_STORAGE_DOLLARS_PER_1000_MINUTES_PER_MONTH["1080p"], + ).toBeLessThanOrEqual(PRO_MONTHLY_CREDIT_DOLLARS); + expect(days31).toBeGreaterThan(PRO_MONTHLY_CREDIT_DOLLARS); + expect(billing).toContain("A month is counted as 30 days"); + }); +}); + +describe("delivery pricing", () => { + const PRICE_PATTERN = /\$(\d+(?:\.\d+)?) per 1,000 delivered minutes/g; + const MEASURE = + "A delivered minute is a minute of video your viewers' players download. Delivered minutes are the media seconds in the video bytes we actually send, counting part of a segment in proportion; transfers the viewer abandons part-way aren't counted. For HLS it counts the exact seconds of the segments served, from each segment's EXTINF duration and EXT-X-BYTERANGE range. For MP4 downloads it counts the bytes served divided by the file's average bytes per second."; + const DEFINITION = `${MEASURE} Playlists, posters, \`HEAD\` requests, responses other than \`200\` or \`206\` and empty responses are never counted.`; + const PLAIN_DEFINITION = `${MEASURE} Playlists, posters, HEAD requests, responses other than 200 or 206 and empty responses are never counted.`; + const FREE_DELIVERY = [ + /\b(?:delivery|delivered|streaming|streams?|bandwidth|egress|playback|views?)\b[^.\n]*\b(?:free|unmetered|unlimited|included|not charged|no charge)\b/i, + /\b(?:free|unmetered|unlimited|included)\b[^.\n]*\b(?:delivery|streaming|bandwidth|egress)\b/i, + ]; + + const readTree = (dir: string): { file: string; text: string }[] => + fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) return readTree(full); + if (!/\.(ts|tsx|mdx|txt)$/.test(entry.name)) return []; + return [{ file: full, text: fs.readFileSync(full, "utf8") }]; + }); + + const llmsInfra = () => + ["public/llms-full.txt", "public/llms.txt"] + .flatMap((file) => + fs.readFileSync(path.join(process.cwd(), file), "utf8").split("\n"), + ) + .filter((line) => /Cap Infra|\/api\/developer|delivered/.test(line)) + .join("\n"); + + const sourceSurfaces = () => [ + ...pageSources(), + ...readTree(path.join(process.cwd(), "app/(infra)/dashboard/infra")), + ...readTree(path.join(process.cwd(), "../../packages/infra-mcp/src")), + ...["lib/developer-openapi.ts", "data/infra-agent-prompt.ts"].map( + (file) => ({ + file, + text: fs.readFileSync(path.join(process.cwd(), file), "utf8"), + }), + ), + ]; + + it("states the one delivery price exactly once on the /infra page, in the docs, in the llms file and on the console Billing page", async () => { + await load("true"); + const { InfraPage } = await import("@/components/pages/infra/InfraPage"); + const { BillingView } = await import( + "@/app/(infra)/dashboard/infra/[appId]/billing/billing-view" + ); + const { getAllDocs } = await import("@/utils/docs"); + const landing = + parse(renderToStaticMarkup(React.createElement(InfraPage))).body + .textContent ?? ""; + const consoleText = ( + account: BillingAccount | null, + deliveryBilledFrom: string | null = null, + ) => + parse( + renderToStaticMarkup( + React.createElement(BillingView, { + app: { id: "app-1", name: "Fixture" }, + account, + proCredit: { + status: "not_pro", + nextDate: "2026-12-01T00:00:00Z", + }, + standing: { state: "active", pausesAt: null }, + shortfallMicroCredits: 0, + autoTopUpFailure: null, + transactions: [], + ledgerPage: { newer: null, older: null }, + deliveryBilledFrom, + }), + ), + ).body.textContent ?? ""; + const docs = getAllDocs() + .filter((doc) => doc.slug.startsWith("api/")) + .map((doc) => doc.content) + .join("\n"); + const occurrences = (text: string, needle: string) => + text.split(needle).length - 1; + + expect(INFRA_DELIVERY_PRICE).toBe( + `$${INFRA_DELIVERY_DOLLARS_PER_1000_MINUTES.toFixed(2)}`, + ); + expect(INFRA_DELIVERY_PRICE_TEXT).toBe( + `${INFRA_DELIVERY_PRICE} ${INFRA_DELIVERY_PRICE_UNIT}`, + ); + for (const [surface, text] of Object.entries({ + landing, + console: consoleText({ + balanceMicroCredits: 0, + hasCard: false, + autoTopUpEnabled: false, + autoTopUpThresholdMicroCredits: 0, + autoTopUpAmountCents: 0, + }), + docs, + llms: llmsInfra(), + })) { + expect(occurrences(text, INFRA_DELIVERY_PRICE), surface).toBe(1); + for (const match of text.matchAll(PRICE_PATTERN)) { + expect(`$${match[1]}`, surface).toBe(INFRA_DELIVERY_PRICE); + } + expect(occurrences(text, INFRA_DELIVERY_PRICE_UNIT), surface).toBe(1); + expect(text, surface).not.toMatch(/\bmux\b/i); + } + expect(occurrences(docs, INFRA_DELIVERY_PRICE_TEXT)).toBe(1); + expect(occurrences(llmsInfra(), INFRA_DELIVERY_PRICE_TEXT)).toBe(1); + + for (const doc of [billing, getStarted, llmsInfra()]) { + expect(doc).toContain(DEFINITION); + } + expect(landing).toContain(PLAIN_DEFINITION); + expect(occurrences(billing, INFRA_DELIVERY_BILLED_FROM_TEXT)).toBe(1); + expect(INFRA_DELIVERY_BILLED_FROM_TEXT).toBe( + "Delivery is billed from 14 November 2026. Earlier days are not charged.", + ); + expect(occurrences(landing, INFRA_DELIVERY_BILLED_FROM_TEXT)).toBe(1); + expect( + occurrences( + consoleText(null, "2026-11-14"), + INFRA_DELIVERY_BILLED_FROM_TEXT, + ), + ).toBe(1); + expect(consoleText(null, "2026-11-20")).toContain( + "Delivery is billed from 20 November 2026. Earlier days are not charged.", + ); + expect( + occurrences(consoleText(null, null), INFRA_DELIVERY_BILLED_FROM_TEXT), + ).toBe(1); + + for (const source of sourceSurfaces()) { + if (source.file.endsWith(".css")) continue; + expect(source.text, source.file).not.toContain(INFRA_DELIVERY_PRICE); + expect(source.text, source.file).not.toMatch( + new RegExp(PRICE_PATTERN.source), + ); + } + }); + + it("never says delivery, streaming or bandwidth is free on an Infra surface", async () => { + const content = await load("true"); + const { InfraPage } = await import("@/components/pages/infra/InfraPage"); + const page = parse(renderToStaticMarkup(React.createElement(InfraPage))); + + const surfaces = [ + { file: "page", text: page.body.textContent ?? "" }, + { file: "billing", text: billing }, + { file: "rest", text: rest }, + { file: "getStarted", text: getStarted }, + { file: "llms", text: llmsInfra() }, + { + file: "content", + text: [ + content.infraSeo.description, + content.infraSoftwareSchema.offers.description, + ...content.infraFaqs.map((faq) => faq.answer), + ...content.PRICE_ROWS.map((row) => + [row.meter, row.unit, row.price, row.note].join(" "), + ), + ].join("\n"), + }, + ...sourceSurfaces(), + ]; + for (const { file, text } of surfaces) { + for (const pattern of FREE_DELIVERY) { + expect(text, file).not.toMatch(pattern); + } + } + expect(billing).toContain( + "Three things are charged: stored video, delivered video and speech to text.", + ); + expect(billing).not.toMatch(/\bfrom today\b/i); + }); +}); + +describe("the film", () => { + const names = { + hls: "master.m3u8", + mp4: "video.mp4", + poster: "poster.jpg", + } as const; + const under = (base: string) => ({ + hls: `${base}/${names.hls}`, + mp4: `${base}/${names.mp4}`, + poster: `${base}/${names.poster}`, + }); + + beforeEach(() => { + env.NEXT_PUBLIC_INFRA_FILM_URL = undefined; + }); + + it("plays from the default base when no other is set", async () => { + const { FILM_DEFAULT_BASE, filmFiles } = await import( + "@/components/pages/infra/film-source" + ); + + expect(FILM_DEFAULT_BASE).toMatch( + /^https:\/\/cap\.so\/api\/developer\/playback\/[\w-]+$/, + ); + expect(filmFiles(undefined)).toEqual(under(FILM_DEFAULT_BASE)); + expect(filmFiles("")).toEqual(under(FILM_DEFAULT_BASE)); + expect(filmFiles(" ")).toEqual(under(FILM_DEFAULT_BASE)); + }); + + it("plays from the configured base when one is set", async () => { + const { filmFiles } = await import("@/components/pages/infra/film-source"); + + expect(filmFiles("https://media.example.com/film")).toEqual( + under("https://media.example.com/film"), + ); + expect(filmFiles(" https://media.example.com/film// ")).toEqual( + under("https://media.example.com/film"), + ); + }); + + it.each([ + ["unset", undefined], + ["empty", ""], + ["set", "https://media.example.com/film"], + ])("renders the button with the base %s", async (_name, value) => { + env.NEXT_PUBLIC_INFRA_FILM_URL = value; + vi.resetModules(); + const { Hero } = await import("@/components/pages/infra/Hero"); + const page = parse(renderToStaticMarkup(React.createElement(Hero))); + const buttons = [...page.querySelectorAll("button")].map((button) => + button.textContent?.trim(), + ); + + expect(buttons).toContain("Watch the film"); + expect(page.querySelector("dialog")?.hasAttribute("open")).toBe(false); + expect(page.querySelectorAll("video, iframe")).toHaveLength(0); + expect(page.documentElement.outerHTML).not.toContain("/playback/"); + expect(page.documentElement.outerHTML).not.toContain("media.example.com"); + }); +}); + +describe("the agent section", () => { + const CLI = /\bCLI\b|install-cli|\bcap (?:auth|developers|guide|version)\b/i; + + it("describes a CLI only when the prompt uses one", async () => { + const content = await load("true"); + const section = fs.readFileSync(path.join(PAGE_DIR, "Agent.tsx"), "utf8"); + const copy = [content.AGENT_SUMMARY, content.AGENT_TOOLS, section]; + + expect(CLI.test("Your agent installs the Cap CLI")).toBe(true); + expect(CLI.test("run cap developers list --json")).toBe(true); + expect(CLI.test(CAP_INFRA_AGENT_PROMPT)).toBe(false); + if (!CLI.test(CAP_INFRA_AGENT_PROMPT)) { + for (const text of copy) expect(text).not.toMatch(CLI); + } + for (const text of [content.AGENT_SUMMARY, content.AGENT_TOOLS]) { + expect(text).not.toMatch(/\binstalls?\b/i); + } + }); + + it("renders the whole prompt, with its first line as the opening", async () => { + await load("true"); + const { Agent } = await import("@/components/pages/infra/Agent"); + const { CAP_INFRA_AGENT_PROMPT: prompt, CAP_INFRA_POINTER_PROMPT } = + await import("@/data/infra-agent-prompt"); + const html = renderToStaticMarkup(React.createElement(Agent)); + const page = parse(html); + const text = page.body.textContent ?? ""; + const [opening = "", ...rest] = prompt.split("\n"); + + expect(opening).toBe( + "Set up Cap Infra in this project so it can upload and play video.", + ); + expect(page.getElementById("agents")).not.toBeNull(); + expect(text).toContain(opening); + expect(text).toContain(rest.join("\n").trim()); + expect(text).toContain(CAP_INFRA_POINTER_PROMPT); + expect(text).toContain("Copy prompt"); + expect(text).not.toContain("Cap Infra is in preview."); + expect(text).not.toMatch(CLI); + const hrefs = [...page.querySelectorAll("a")].map((link) => + link.getAttribute("href"), + ); + expect(hrefs).toContain("/docs/api/rest-api"); + expect(hrefs).not.toContain("/docs/agents"); + }); + + it("promises only what the prompt asks the agent to do", async () => { + const content = await load("true"); + + expect(content.AGENT_SUMMARY).toContain("reads the API docs"); + expect(CAP_INFRA_AGENT_PROMPT).toContain( + "https://cap.so/docs/api/rest-api", + ); + expect(content.AGENT_SUMMARY).toContain("uploads and playback"); + expect(CAP_INFRA_AGENT_PROMPT).toMatch(/upload/i); + expect(CAP_INFRA_AGENT_PROMPT).toMatch(/embed/i); + expect(content.AGENT_SUMMARY).toContain( + "Add your app's secret key to your project", + ); + expect(CAP_INFRA_AGENT_PROMPT).toContain( + "put its secret key (csk_) in this project's local environment file", + ); + expect(content.AGENT_SUMMARY).toContain("keeps the key on the server"); + expect(CAP_INFRA_AGENT_PROMPT).toContain("Keep the key on the server"); + }); +}); + +describe("the calculator's number field", () => { + it("reads decimals as decimals", () => { + expect(parseAmount("0.5", 100)).toEqual({ draft: "0.5", value: 0.5 }); + expect(parseAmount("1.5", 100)).toEqual({ draft: "1.5", value: 1.5 }); + expect(parseAmount("0.", 100)).toEqual({ draft: "0.", value: 0 }); + expect(parseAmount(".", 100)).toEqual({ draft: ".", value: 0 }); + expect(parseAmount(".25", 100)).toEqual({ draft: ".25", value: 0.25 }); + expect(parseAmount("1.2.3", 100)).toEqual({ draft: "1.23", value: 1.23 }); + expect(parseAmount("0.129", 100)).toEqual({ draft: "0.12", value: 0.12 }); + }); + + it("reads grouped whole numbers and caps at the meter's maximum", () => { + expect(parseAmount("1,500", 10_000)).toEqual({ + draft: "1500", + value: 1500, + }); + expect(parseAmount("", 100)).toEqual({ draft: "", value: 0 }); + expect(parseAmount("abc", 100)).toEqual({ draft: "", value: 0 }); + expect(parseAmount("100", 100)).toEqual({ draft: "100", value: 100 }); + expect(parseAmount("250", 100)).toEqual({ draft: "100", value: 100 }); + expect(parseAmount("200000", 100_000)).toEqual({ + draft: "100000", + value: 100_000, + }); + }); +}); diff --git a/apps/web/__tests__/unit/infra-ledger-paging.test.ts b/apps/web/__tests__/unit/infra-ledger-paging.test.ts new file mode 100644 index 00000000000..1f73fec6a77 --- /dev/null +++ b/apps/web/__tests__/unit/infra-ledger-paging.test.ts @@ -0,0 +1,302 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +type Row = { + id: string; + createdAt: Date; + amountMicroCredits: number; + balanceAfterMicroCredits?: number; +}; + +const state = vi.hoisted(() => ({ + rows: [] as Row[], + queries: [] as string[], +})); + +vi.mock("@cap/database", async () => { + const { drizzle } = await import("drizzle-orm/mysql-proxy"); + const toDate = (value: unknown) => + new Date(`${String(value).replace(" ", "T")}Z`); + const stamp = (date: Date) => + date.toISOString().slice(0, 19).replace("T", " "); + const client = drizzle(async (sql, params) => { + state.queries.push(sql); + const ascending = sql.includes( + "order by `developer_credit_transactions`.`createdAt` asc", + ); + const sorted = [...state.rows].sort( + (a, b) => + a.createdAt.getTime() - b.createdAt.getTime() || + a.id.localeCompare(b.id), + ); + if (!ascending) sorted.reverse(); + let rest = params.slice(1); + let matching = sorted; + if (sql.includes("`createdAt` >= ?")) { + const from = toDate(rest[0]).getTime(); + const to = toDate(rest[1]).getTime(); + matching = sorted.filter( + (row) => + row.createdAt.getTime() >= from && row.createdAt.getTime() <= to, + ); + rest = [matching.length]; + } else if ( + sql.includes("`createdAt` < ?") || + sql.includes("`createdAt` > ?") + ) { + const at = toDate(rest[0]).getTime(); + const id = String(rest[2]); + rest = rest.slice(3); + matching = sorted.filter((row) => { + const time = row.createdAt.getTime(); + return ascending + ? time > at || (time === at && row.id > id) + : time < at || (time === at && row.id < id); + }); + } else if (sql.includes("`createdAt` = ?")) { + const at = toDate(rest[0]).getTime(); + matching = sorted.filter((row) => row.createdAt.getTime() === at); + rest = [matching.length]; + } + const limit = Number(rest[0]); + const offset = sql.includes("offset ?") ? Number(rest[1]) : 0; + return { + rows: matching + .slice(offset, offset + limit) + .map((row) => [ + row.id, + "transcription", + row.amountMicroCredits, + row.balanceAfterMicroCredits ?? 0, + null, + null, + null, + stamp(row.createdAt), + ]), + }; + }); + return { db: () => client }; +}); + +import { LEDGER_PAGE_SIZE, type LedgerRow } from "@/lib/infra-ledger"; +import { + formatLedgerCursor, + listAppLedger, + listAppLedgerPage, + parseLedgerCursor, + readAppLedger, +} from "@/lib/infra-ledger-store"; + +const START = Date.parse("2026-11-01T00:00:00Z"); + +const seed = (count: number) => { + state.rows = Array.from({ length: count }, (_, index) => ({ + id: `txn${String(index).padStart(5, "0")}`, + createdAt: new Date(START + Math.floor(index / 3) * 1000), + amountMicroCredits: -(index + 1), + })); +}; + +const newestFirst = () => + [...state.rows] + .sort( + (a, b) => + b.createdAt.getTime() - a.createdAt.getTime() || + b.id.localeCompare(a.id), + ) + .map((row) => row.id); + +const ids = (page: { rows: { id: string }[] }) => + page.rows.map((row) => row.id).sort(); + +const page = (cursor: { before?: string | null; after?: string | null } = {}) => + listAppLedgerPage("app-1", { + limit: LEDGER_PAGE_SIZE, + before: parseLedgerCursor(cursor.before), + after: parseLedgerCursor(cursor.after), + }); + +beforeEach(() => { + state.queries = []; + seed(120); +}); + +describe("ledger pages by cursor", () => { + it("lists the newest page in one query, with no count", async () => { + const first = await page(); + + expect(first.rows).toHaveLength(LEDGER_PAGE_SIZE); + expect(ids(first)).toEqual(newestFirst().slice(0, 50).sort()); + expect(first.newer).toBeNull(); + expect(first.older).not.toBeNull(); + expect(state.queries).toHaveLength(1); + expect(state.queries.join("\n")).not.toMatch(/count\(/i); + expect(state.queries.join("\n")).not.toContain("offset"); + }); + + it("walks every row once going Older, across rows written in the same second", async () => { + const seen: string[] = []; + let current = await page(); + seen.push(...ids(current)); + while (current.older) { + current = await page({ before: current.older }); + seen.push(...ids(current)); + } + + expect(seen).toHaveLength(120); + expect(new Set(seen).size).toBe(120); + expect(current.rows).toHaveLength(20); + expect(current.older).toBeNull(); + expect(state.queries).toHaveLength(3); + }); + + it("returns to the same page going Newer", async () => { + const first = await page(); + const second = await page({ before: first.older }); + const third = await page({ before: second.older }); + + const backToSecond = await page({ after: third.newer }); + const backToFirst = await page({ after: backToSecond.newer }); + + expect(ids(backToSecond)).toEqual(ids(second)); + expect(backToSecond.older).toBe(second.older); + expect(ids(backToFirst)).toEqual(ids(first)); + expect(backToFirst.newer).toBeNull(); + }); + + it("shows a full newest page when fewer than a page of rows is newer", async () => { + const first = await page(); + const second = await page({ before: first.older }); + state.rows.push({ + id: "txn99999", + createdAt: new Date(START + 3_600_000), + amountMicroCredits: -1, + }); + + const anchor = state.rows.find((row) => row.id === "txn00100"); + if (!anchor) throw new Error("seed row missing"); + const afterOne = await page({ after: formatLedgerCursor(anchor) }); + const backToFirst = await page({ after: second.newer }); + const top = await page({ after: backToFirst.newer }); + + expect(afterOne.rows).toHaveLength(LEDGER_PAGE_SIZE); + expect(afterOne.newer).toBeNull(); + expect(backToFirst.rows).toHaveLength(LEDGER_PAGE_SIZE); + expect(backToFirst.newer).not.toBeNull(); + expect(top.rows).toHaveLength(LEDGER_PAGE_SIZE); + expect(top.newer).toBeNull(); + expect(top.rows.some((row) => row.id === "txn99999")).toBe(true); + }); + + it("keeps a cursor that survives a URL and ignores one that is not a cursor", async () => { + const row = { id: "0abc9xyz", createdAt: new Date(START) }; + const cursor = formatLedgerCursor(row); + + expect(cursor).toBe(`${START}.0abc9xyz`); + expect(encodeURIComponent(cursor)).toBe(cursor); + expect(parseLedgerCursor(cursor)).toEqual(row); + for (const bad of [undefined, null, "", "3", "abc.def", `${START}.ABC`]) { + expect(parseLedgerCursor(bad)).toBeNull(); + } + expect(ids(await page({ before: "not-a-cursor" }))).toEqual( + ids(await page()), + ); + }); + + it("shows no page links for a ledger that fits one page", async () => { + seed(7); + + expect(await page()).toMatchObject({ newer: null, older: null }); + expect((await page()).rows).toHaveLength(7); + }); +}); + +describe("the transactions API keeps limit and offset", () => { + it("lists with limit and offset and never counts", async () => { + const result = await listAppLedger("app-1", { limit: 50, offset: 100 }); + + expect(result.rows).toHaveLength(20); + expect(result.hasMore).toBe(false); + expect( + (await listAppLedger("app-1", { limit: 50, offset: 0 })).hasMore, + ).toBe(true); + expect(state.queries).toHaveLength(4); + expect(state.queries[0]).toContain("limit ? offset ?"); + expect(state.queries.join("\n")).not.toMatch(/count\(/i); + }); +}); + +describe("rows written in the same second across a page edge", () => { + const SECOND = (offset: number) => new Date(START + offset * 1000); + const seedChain = () => { + const groups = [ + { at: SECOND(0), ids: ["q1", "c7"] }, + { at: SECOND(1), ids: ["m4", "z9", "a2", "t6", "f3"] }, + { at: SECOND(2), ids: ["k8", "b5"] }, + ]; + let balance = 1_000_000; + let amount = 11; + state.rows = groups.flatMap(({ at, ids }) => + ids.map((id) => { + amount += 7; + balance -= amount; + return { + id, + createdAt: at, + amountMicroCredits: -amount, + balanceAfterMicroCredits: balance, + }; + }), + ); + }; + const chainBreaks = ( + rows: { amountMicroCredits: number; balanceAfterMicroCredits: number }[], + ) => + rows.slice(0, -1).filter((row, index) => { + const older = rows[index + 1]; + return ( + !older || + row.balanceAfterMicroCredits - row.amountMicroCredits !== + older.balanceAfterMicroCredits + ); + }).length; + const walk = async (limit: number) => { + const rows: LedgerRow[] = []; + for (let offset = 0; ; offset += limit) { + const page = await listAppLedger("app-1", { limit, offset }); + rows.push(...page.rows); + if (!page.hasMore) return rows; + } + }; + + beforeEach(seedChain); + + it("gives the same chain whatever the page size", async () => { + const whole = (await listAppLedger("app-1", { limit: 100, offset: 0 })) + .rows; + + expect(whole).toHaveLength(9); + expect(chainBreaks(whole)).toBe(0); + for (const limit of [1, 2, 3, 4, 5, 8]) { + const paged = await walk(limit); + expect( + paged.map((row) => row.id), + `limit ${limit}`, + ).toEqual(whole.map((row) => row.id)); + } + }); + + it("gives the same chain in an export read in small batches", async () => { + const whole = (await listAppLedger("app-1", { limit: 100, offset: 0 })) + .rows; + for (const batchSize of [2, 3, 4]) { + const exported: LedgerRow[] = []; + for await (const batch of readAppLedger("app-1", batchSize)) { + exported.push(...batch); + } + expect( + exported.map((row) => row.id), + `batch ${batchSize}`, + ).toEqual(whole.map((row) => row.id)); + } + }); +}); diff --git a/apps/web/__tests__/unit/infra-ledger-reference-column.test.ts b/apps/web/__tests__/unit/infra-ledger-reference-column.test.ts new file mode 100644 index 00000000000..cb157e86bb1 --- /dev/null +++ b/apps/web/__tests__/unit/infra-ledger-reference-column.test.ts @@ -0,0 +1,185 @@ +// @vitest-environment jsdom + +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { CreditTransactionTable } from "@/app/(infra)/dashboard/infra/_components/credit-transaction-table"; +import type { LedgerRow } from "@/lib/infra-ledger"; + +let root: Root; +let container: HTMLDivElement; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); +}); + +const row = (overrides: Partial): LedgerRow => ({ + id: "rowid000000001", + type: "transcription", + amountMicroCredits: -33, + balanceAfterMicroCredits: 2_499_967, + referenceId: "transcript00001", + referenceType: "developer_transcript", + metadata: null, + createdAt: new Date("2026-11-13T10:00:00Z"), + ...overrides, +}); + +const desktopReferences = () => + Array.from(container.querySelectorAll("table tbody tr")).map( + (tr) => tr.querySelectorAll("td")[2]?.textContent, + ); + +describe("ledger Reference column", () => { + it("shows the video or transcript the row is about, not the row's own id", () => { + act(() => + root.render( + createElement(CreditTransactionTable, { + transactions: [ + row({}), + row({ + id: "rowid000000002", + type: "storage_daily", + referenceType: "developer_video", + referenceId: "video0000000001", + }), + ], + }), + ), + ); + + expect(desktopReferences()).toEqual([ + "Transcript transcript00001", + "Video video0000000001", + ]); + expect(container.querySelector("table")?.textContent).not.toContain( + "rowid000000001", + ); + }); + + it("shows a dash when the row has no reference", () => { + act(() => + root.render( + createElement(CreditTransactionTable, { + transactions: [ + row({ + type: "adjustment", + amountMicroCredits: 100_000, + referenceId: null, + referenceType: null, + }), + ], + }), + ), + ); + + expect(desktopReferences()).toEqual(["–"]); + expect(container.querySelector("ul")?.textContent).toContain("Reference –"); + }); + + it("truncates a 66-character reference to one line, with the full value on hover", () => { + const session = `cs_test_${"a1B2c3D4E5".repeat(5)}a1B2c3D4`; + expect(session).toHaveLength(66); + act(() => + root.render( + createElement(CreditTransactionTable, { + transactions: [ + row({ + type: "topup", + amountMicroCredits: 2_500_000, + referenceType: "stripe_checkout_session", + referenceId: session, + }), + ], + }), + ), + ); + + const cell = container.querySelectorAll("table tbody tr td")[2]; + const reference = cell?.querySelector("[title]"); + expect(reference?.className.split(" ")).toEqual([ + "block", + "max-w-[124px]", + "xl:inline-block", + "xl:max-w-[220px]", + "xl:truncate", + "xl:align-bottom", + ]); + const [kind, id] = Array.from(reference?.children ?? []); + expect(kind?.className).toBe("block truncate xl:inline"); + expect(id?.className.split(" ")).toEqual( + expect.arrayContaining(["block", "truncate", "xl:inline"]), + ); + expect(reference?.getAttribute("title")).toContain(session); + expect(reference?.textContent).toContain(session); + + const card = container.querySelector("ul li [title]"); + expect(card?.getAttribute("title")).toContain(session); + expect(card?.parentElement?.className).toContain("block truncate"); + }); + + it("lets the Type label wrap at a readable width so it never runs into Reference", () => { + act(() => + root.render( + createElement(CreditTransactionTable, { transactions: [row({})] }), + ), + ); + + const typeCell = container.querySelectorAll("table tbody tr td")[1]; + const classes = typeCell?.className.split(" ") ?? []; + expect(classes).toEqual( + expect.arrayContaining(["w-full", "min-w-[140px]", "!whitespace-normal"]), + ); + expect(classes).not.toContain("max-w-0"); + }); + + it("uses the important wrap so the table cell's nowrap cannot win over it", () => { + act(() => + root.render( + createElement(CreditTransactionTable, { transactions: [row({})] }), + ), + ); + + const classes = + container + .querySelectorAll("table tbody tr td")[1] + ?.className.split(" ") ?? []; + expect(classes).toContain("whitespace-nowrap"); + expect(classes).toContain("!whitespace-normal"); + expect(classes).not.toContain("whitespace-normal"); + }); + + it("keeps the phone card reference on one line", () => { + act(() => + root.render( + createElement(CreditTransactionTable, { transactions: [row({})] }), + ), + ); + + const reference = container.querySelector("ul li [title]"); + expect(reference?.getAttribute("class")).toBeNull(); + for (const part of Array.from(reference?.children ?? [])) { + expect(part.getAttribute("class") ?? "").not.toContain("block"); + } + }); + + it("shows the same reference on phones", () => { + act(() => + root.render( + createElement(CreditTransactionTable, { transactions: [row({})] }), + ), + ); + + const card = container.querySelector("ul li")?.textContent ?? ""; + expect(card).toContain("Reference Transcript transcript00001"); + expect(card).not.toContain("rowid000000001"); + }); +}); diff --git a/apps/web/__tests__/unit/infra-ledger.test.ts b/apps/web/__tests__/unit/infra-ledger.test.ts index 601d8591d4c..c3a87227cf6 100644 --- a/apps/web/__tests__/unit/infra-ledger.test.ts +++ b/apps/web/__tests__/unit/infra-ledger.test.ts @@ -13,7 +13,6 @@ import { LEDGER_CSV_COLUMNS, type LedgerRow, ledgerCsv, - ledgerPageWindow, orderLedgerRows, serializeLedgerRow, } from "@/lib/infra-ledger"; @@ -258,6 +257,60 @@ describe("what each row says", () => { }); }); +describe("a video's first month of storage", () => { + const firstMonth = ( + amount: number, + owed: number, + shortSince: string | null, + ) => + row({ + type: "video_create", + amountMicroCredits: amount, + referenceId: "v1d3o9k2m4x7n1p", + referenceType: "developer_video", + metadata: { + owedMicroCredits: owed, + chargedMicroCredits: -amount, + shortSince, + durationSeconds: 600, + tier: "1080p", + paidUntil: "2026-12-12T00:00:00.000Z", + }, + }); + + it("names a paid month and points at the video", () => { + expect(describeLedgerRow(firstMonth(-850, 850, null))).toEqual({ + label: "Storage, first month", + note: "One month of storage for this video, charged when it became ready. It is charged daily after that.", + reference: { kind: "Video", value: "v1d3o9k2m4x7n1p" }, + }); + }); + + it("names a paid month as paid while it carries an earlier unpaid month", () => { + const row = firstMonth(-850, 850, "2026-11-12T14:05:09.000Z"); + expect( + describeLedgerRow({ + ...row, + metadata: { + ...(row.metadata as Record), + outstandingMicroCredits: 16_800, + }, + }).label, + ).toBe("Storage, first month"); + }); + + it("says when the balance paid part of the month or none of it", () => { + const since = "2026-11-12T14:05:09.000Z"; + expect(describeLedgerRow(firstMonth(-300, 850, since)).label).toBe( + "Storage, first month, part paid", + ); + expect(describeLedgerRow(firstMonth(0, 850, since))).toMatchObject({ + label: "Storage, first month, unpaid", + note: "The balance could not pay for this video's first month. It is not charged later.", + }); + }); +}); + describe("the API shape", () => { it("carries exact dollars beside the micro-credits", () => { expect(serializeLedgerRow(row())).toEqual({ @@ -462,7 +515,7 @@ describe("CSV export", () => { "2026-11-12T14:20:00Z", "Transcription", "", - "developer_transcript", + "Transcript", "8kq2m4x7n1p3v5w", "-0.00033", "-33", @@ -536,6 +589,34 @@ describe("CSV export", () => { }, ); + it("labels references the way the console does", () => { + const adjustment = row({ + type: "adjustment", + referenceType: "manual", + referenceId: "test-grant-1", + }); + const { records } = csvTable(ledgerCsv([adjustment])); + const { note, reference } = describeLedgerRow(adjustment); + + expect(records[0]?.cell("note")).toBe(note); + expect(records[0]?.cell("reference_type")).toBe("Note"); + expect(records[0]?.cell("reference_id")).toBe("test-grant-1"); + expect(reference).toEqual({ kind: "Note", value: "test-grant-1" }); + }); + + it("shows a declined day the way the console does", () => { + const declined = row({ + type: "topup_failed", + amountMicroCredits: 0, + referenceType: "auto_top_up_failed", + referenceId: "declined-2026-11-12", + }); + const { records } = csvTable(ledgerCsv([declined])); + + expect(records[0]?.cell("reference_type")).toBe("Day"); + expect(records[0]?.cell("reference_id")).toBe("2026-11-12"); + }); + it("leaves negative amounts as numbers", () => { const { records } = csvTable(ledgerCsv([row()])); expect(records[0]?.cell("amount_usd")).toBe("-0.00033"); @@ -546,19 +627,3 @@ describe("CSV export", () => { expect(ledgerCsv([])).toBe(`${LEDGER_CSV_COLUMNS.join(",")}\r\n`); }); }); - -describe("paging the console ledger", () => { - it.each([ - [undefined, 312, { page: 1, offset: 0, from: 1, to: 50, pages: 7 }], - ["2", 312, { page: 2, offset: 50, from: 51, to: 100, pages: 7 }], - ["7", 312, { page: 7, offset: 300, from: 301, to: 312, pages: 7 }], - ["99", 312, { page: 7, offset: 300, from: 301, to: 312, pages: 7 }], - ["0", 312, { page: 1, offset: 0, from: 1, to: 50, pages: 7 }], - ["-3", 312, { page: 1, offset: 0, from: 1, to: 50, pages: 7 }], - ["abc", 312, { page: 1, offset: 0, from: 1, to: 50, pages: 7 }], - ["2", 0, { page: 1, offset: 0, from: 0, to: 0, pages: 1 }], - ["1", 50, { page: 1, offset: 0, from: 1, to: 50, pages: 1 }], - ])("reads page %j of %i rows", (raw, total, expected) => { - expect(ledgerPageWindow(raw, total)).toEqual(expected); - }); -}); diff --git a/apps/web/__tests__/unit/infra-out-of-credit-email.test.ts b/apps/web/__tests__/unit/infra-out-of-credit-email.test.ts new file mode 100644 index 00000000000..80897c5f1c4 --- /dev/null +++ b/apps/web/__tests__/unit/infra-out-of-credit-email.test.ts @@ -0,0 +1,90 @@ +import { renderToStaticMarkup } from "react-dom/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ sendEmail: vi.fn() })); + +vi.mock("@cap/database", () => { + const chain = { + select: () => chain, + from: () => chain, + innerJoin: () => chain, + where: () => chain, + limit: async () => [ + { appId: "app-1", appName: "Acme", email: "owner@example.com" }, + ], + }; + return { db: () => chain }; +}); +vi.mock("@cap/database/emails/config", () => ({ sendEmail: mocks.sendEmail })); +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ + CAP_INFRA_BILLING_EMAILS: true, + WEB_URL: "https://cap.so", + }), +})); +vi.mock("@/lib/infra-access", () => ({ canUseInfra: () => true })); + +import { InfraOutOfCredit } from "@cap/database/emails/infra-out-of-credit"; +import { sendInfraOutOfCreditEmail } from "@/lib/infra-billing-emails"; + +const shortSince = new Date("2026-11-13T06:41:00Z"); +const send = () => + sendInfraOutOfCreditEmail({ + accountId: "account-1", + shortSince, + pausesAt: new Date(shortSince.getTime() + 7 * 24 * 60 * 60 * 1000), + }); + +beforeEach(() => { + mocks.sendEmail.mockReset(); +}); + +describe("the out-of-credit email", () => { + it("reports a send only when the email service accepted it", async () => { + mocks.sendEmail.mockResolvedValueOnce({ data: { id: "e-1" }, error: null }); + expect(await send()).toBe(true); + + mocks.sendEmail.mockResolvedValueOnce({ + data: null, + error: { name: "validation_error", message: "rejected" }, + }); + expect(await send()).toBe(false); + + mocks.sendEmail.mockResolvedValueOnce(undefined); + expect(await send()).toBe(false); + }); + + it("uses one dedupe key per account and shortfall day, and pauses seven days after the shortfall", async () => { + mocks.sendEmail.mockResolvedValue({ data: { id: "e-1" }, error: null }); + await send(); + + expect(mocks.sendEmail).toHaveBeenCalledWith( + expect.objectContaining({ + idempotencyKey: "infra-out-of-credit-account-1-2026-11-13", + }), + ); + const html = renderToStaticMarkup(mocks.sendEmail.mock.calls[0]?.[0].react); + expect(html).toContain("November 20"); + }); + + it("words the shortfall for any storage charge", () => { + const html = renderToStaticMarkup( + InfraOutOfCredit({ + email: "owner@example.com", + appName: "Acme", + billingUrl: "https://cap.so/billing", + pausesOn: "November 20", + }), + ); + const text = html.replace(/<[^>]+>/g, " ").replace(/\s+/g, " "); + + expect(text).toContain( + "A storage charge for Acme was more than its Cap Infra balance, so the balance is now zero.", + ); + expect(text).toContain( + "Adding credits brings the app straight back. What it could not pay is not charged afterwards.", + ); + expect(text).not.toMatch(/\btoday\b|\bdaily\b|\bdays it could not\b/i); + expect(html).not.toContain(String.fromCodePoint(0x2014)); + }); +}); diff --git a/apps/web/__tests__/unit/infra-pro-credit-eligibility.test.ts b/apps/web/__tests__/unit/infra-pro-credit-eligibility.test.ts new file mode 100644 index 00000000000..0dff6ada8bc --- /dev/null +++ b/apps/web/__tests__/unit/infra-pro-credit-eligibility.test.ts @@ -0,0 +1,298 @@ +import { createElement, type ReactNode } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +type Subscriber = { + id: string; + email: string; + stripeSubscriptionStatus?: string | null; + thirdPartyStripeSubscriptionId?: string | null; +}; + +const mocks = vi.hoisted(() => ({ + isCap: true, + user: null as Subscriber | null, + claimedThisMonth: false, + claim: vi.fn(), + hasClaimed: vi.fn(), + payer: vi.fn(), +})); + +vi.mock("@cap/env", () => ({ + buildEnv: { + get NEXT_PUBLIC_IS_CAP() { + return mocks.isCap; + }, + }, + serverEnv: () => ({ WEB_URL: "https://web.example" }), +})); +vi.mock("@cap/database", () => ({ + db: () => { + const chain = { + select: () => chain, + from: () => chain, + innerJoin: () => chain, + where: () => chain, + limit: async () => [{ id: "account-1" }], + }; + return chain; + }, +})); +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: async () => mocks.user, +})); +vi.mock("@/lib/infra-access", () => ({ canUseInfra: () => true })); +vi.mock("@/lib/infra-pro-credit", async (importOriginal) => ({ + ...(await importOriginal()), + claimProMonthlyCredit: mocks.claim, + hasClaimedProCredit: mocks.hasClaimed, + nextProCreditDate: () => new Date("2026-12-01T00:00:00Z"), + proCreditPayer: mocks.payer, +})); +vi.mock("@/lib/developer-video-start", () => ({ + redriveDeveloperVideoProcessing: vi.fn(), +})); +vi.mock("next/headers", () => ({ + cookies: async () => ({ get: () => null }), + headers: async () => ({ get: () => null }), +})); +vi.mock("next/navigation", () => ({ + redirect: (path: string) => { + throw new Error(`redirect:${path}`); + }, + notFound: () => { + throw new Error("not-found"); + }, + useRouter: () => ({ refresh: () => undefined }), +})); +vi.mock("next/link", () => ({ + default: ({ href, children }: { href: string; children: ReactNode }) => + createElement("a", { href }, children), +})); +vi.mock("sonner", () => ({ toast: { success: vi.fn(), error: vi.fn() } })); +vi.mock("@/lib/EffectRuntime", () => ({ + useEffectMutation: () => ({ mutate: () => undefined, isPending: false }), +})); + +const { PRO_CREDIT_TRIAL_MESSAGE, proCreditEligibility } = await import( + "@/lib/infra-pro-credit-eligibility" +); +const { claimDeveloperProCredit } = await import( + "@/actions/developers/claim-pro-credit" +); +const { getInfraProCredit } = await import( + "@/app/(infra)/dashboard/infra/infra-data" +); +const { ProCredit } = await import( + "@/app/(infra)/dashboard/infra/_components/pro-credit" +); +const { FirstCredit } = await import( + "@/app/(infra)/dashboard/infra/_components/get-started" +); + +const subscriber = (fields: Partial): Subscriber => ({ + id: "user-1", + email: "owner@example.com", + stripeSubscriptionStatus: null, + thirdPartyStripeSubscriptionId: null, + ...fields, +}); + +const PAID: Array<[string, Partial]> = [ + ["active", { stripeSubscriptionStatus: "active" }], + ["past_due", { stripeSubscriptionStatus: "past_due" }], + ["complete", { stripeSubscriptionStatus: "complete" }], + ["paid", { stripeSubscriptionStatus: "paid" }], + ["third-party", { thirdPartyStripeSubscriptionId: "sub_org" }], + [ + "third-party beside a personal trial", + { + stripeSubscriptionStatus: "trialing", + thirdPartyStripeSubscriptionId: "sub_org", + }, + ], +]; + +beforeEach(() => { + mocks.isCap = true; + mocks.user = null; + mocks.claim.mockReset(); + mocks.claim.mockResolvedValue({ + claimed: true, + balanceMicroCredits: 500_000, + }); + mocks.payer.mockReset(); + mocks.payer.mockResolvedValue({ + state: "ok", + subscription: { id: "sub_paying", seats: 2 }, + }); + mocks.hasClaimed.mockReset(); + mocks.hasClaimed.mockResolvedValue(false); +}); + +describe("who the Cap Pro credit is for", () => { + it("is not for a trial", () => { + expect( + proCreditEligibility( + subscriber({ stripeSubscriptionStatus: "trialing" }), + ), + ).toBe("trial"); + }); + + it.each(PAID)("is for a %s subscription", (_label, fields) => { + expect(proCreditEligibility(subscriber(fields))).toBe("paid"); + }); + + it.each([null, "canceled", "unpaid", "incomplete"])( + "is not for an account without Cap Pro (%s)", + (status) => { + expect( + proCreditEligibility(subscriber({ stripeSubscriptionStatus: status })), + ).toBe("not_pro"); + expect(proCreditEligibility(null)).toBe("not_pro"); + }, + ); + + it("stays open on a self-hosted install, where everyone has Pro", () => { + mocks.isCap = false; + expect(proCreditEligibility(subscriber({}))).toBe("paid"); + }); + + it("says when the credit starts in one sentence", () => { + expect(PRO_CREDIT_TRIAL_MESSAGE).toBe( + "Your $5 monthly Infra credit starts with your first paid month of Cap Pro.", + ); + }); +}); + +describe("claiming the Cap Pro credit", () => { + it("refuses a trialing account and grants nothing", async () => { + mocks.user = subscriber({ stripeSubscriptionStatus: "trialing" }); + + expect(await claimDeveloperProCredit("app-1")).toEqual({ + success: false, + message: PRO_CREDIT_TRIAL_MESSAGE, + }); + expect(mocks.claim).not.toHaveBeenCalled(); + }); + + it("refuses an account without Cap Pro", async () => { + mocks.user = subscriber({ stripeSubscriptionStatus: "canceled" }); + + expect(await claimDeveloperProCredit("app-1")).toEqual({ + success: false, + message: "This credit comes with Cap Pro.", + }); + expect(mocks.claim).not.toHaveBeenCalled(); + }); + + it.each(PAID)("grants a %s subscription", async (_label, fields) => { + mocks.user = subscriber(fields); + + expect(await claimDeveloperProCredit("app-1")).toEqual({ + balanceMicroCredits: 500_000, + }); + expect(mocks.claim).toHaveBeenCalledTimes(1); + expect(mocks.claim).toHaveBeenCalledWith( + expect.objectContaining({ + userId: "user-1", + accountId: "account-1", + subscription: { id: "sub_paying", seats: 2 }, + }), + ); + }); + + it("refuses a seat whose team subscription is not paid, and grants nothing", async () => { + mocks.user = subscriber({ thirdPartyStripeSubscriptionId: "sub_org" }); + mocks.payer.mockResolvedValue({ + state: "refused", + message: PRO_CREDIT_TRIAL_MESSAGE, + }); + + expect(await claimDeveloperProCredit("app-1")).toEqual({ + success: false, + message: PRO_CREDIT_TRIAL_MESSAGE, + }); + expect(mocks.claim).not.toHaveBeenCalled(); + }); + + it("still refuses a second claim in the same month", async () => { + mocks.user = subscriber({ stripeSubscriptionStatus: "active" }); + mocks.claim.mockResolvedValue({ claimed: false }); + + expect(await claimDeveloperProCredit("app-1")).toEqual({ + success: false, + message: + "This month's credit is already claimed. It is one per account, not one per app.", + }); + }); +}); + +describe("what the console offers", () => { + it("does not offer the claim to a trialing account", async () => { + mocks.user = subscriber({ stripeSubscriptionStatus: "trialing" }); + + expect(await getInfraProCredit()).toEqual({ + status: "trial", + nextDate: "2026-12-01T00:00:00.000Z", + }); + expect(mocks.hasClaimed).not.toHaveBeenCalled(); + }); + + it.each(PAID)( + "offers the claim to a %s subscription", + async (_label, fields) => { + mocks.user = subscriber(fields); + + expect((await getInfraProCredit()).status).toBe("available"); + mocks.hasClaimed.mockResolvedValue(true); + expect((await getInfraProCredit()).status).toBe("claimed"); + }, + ); + + it("keeps the upgrade note for an account without Cap Pro", async () => { + mocks.user = subscriber({}); + + expect((await getInfraProCredit()).status).toBe("not_pro"); + }); + + const billingNote = (status: "available" | "claimed" | "trial" | "not_pro") => + renderToStaticMarkup( + createElement(ProCredit, { + appId: "app-1", + credit: { status, nextDate: "2026-12-01T00:00:00.000Z" }, + }), + ); + + it("shows a trial one sentence and no claim button", () => { + const html = billingNote("trial"); + + expect(html).toBe( + `

${PRO_CREDIT_TRIAL_MESSAGE}

`, + ); + expect(html).not.toContain(" { + expect(billingNote("available")).toContain("Claim $5 credit"); + expect(billingNote("claimed")).not.toContain(" { + const html = renderToStaticMarkup( + createElement(FirstCredit, { + appId: "app-1", + balanceMicroCredits: 0, + proCredit: { status: "trial", nextDate: "2026-12-01T00:00:00.000Z" }, + }), + ); + + expect(html).toContain("Add credits"); + expect(html).toContain(PRO_CREDIT_TRIAL_MESSAGE); + expect(html).not.toContain("Claim"); + }); +}); diff --git a/apps/web/__tests__/unit/infra-pro-credit-payer.test.ts b/apps/web/__tests__/unit/infra-pro-credit-payer.test.ts new file mode 100644 index 00000000000..d0385d7d7a0 --- /dev/null +++ b/apps/web/__tests__/unit/infra-pro-credit-payer.test.ts @@ -0,0 +1,184 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + isCap: true, + retrieve: vi.fn(), +})); + +vi.mock("@cap/database", () => ({ + db: () => { + const chain = { + select: () => chain, + from: () => chain, + where: () => chain, + limit: async () => [{ id: "owner" }], + }; + return chain; + }, +})); +vi.mock("@cap/database/schema", () => ({ + users: { id: "id", email: "email" }, + developerCreditAccounts: {}, + developerCreditTransactions: {}, +})); +vi.mock("drizzle-orm", () => ({ + and: vi.fn(), + count: vi.fn(), + eq: vi.fn(), + sql: vi.fn(), +})); +vi.mock("@cap/database/helpers", () => ({ + nanoId: () => "txn", + nanoIdLength: 15, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { + get NEXT_PUBLIC_IS_CAP() { + return mocks.isCap; + }, + }, +})); +vi.mock("@cap/utils", () => ({ + userIsPro: () => true, + stripe: () => ({ subscriptions: { retrieve: mocks.retrieve } }), +})); + +import type { User } from "@cap/web-domain"; +import { proCreditPayer } from "@/lib/infra-pro-credit"; + +const subscription = ( + id: string, + status: string, + quantities: number[] = [1], +) => ({ + id, + status, + items: { data: quantities.map((quantity) => ({ quantity })) }, + customer: { + id: "cus_1", + email: "owner@example.com", + metadata: { userId: "owner" }, + }, +}); + +const member = (fields: Record = {}) => ({ + id: "user-1" as User.UserId, + stripeSubscriptionId: null, + stripeSubscriptionStatus: null, + thirdPartyStripeSubscriptionId: null, + ...fields, +}); + +const direct = member({ + stripeSubscriptionId: "sub_own", + stripeSubscriptionStatus: "active", +}); +const seated = member({ thirdPartyStripeSubscriptionId: "sub_team" }); + +beforeEach(() => { + mocks.isCap = true; + mocks.retrieve.mockReset(); +}); + +describe("the subscription that pays for a Cap Pro credit", () => { + it("is a direct subscriber's own subscription", async () => { + mocks.retrieve.mockResolvedValue(subscription("sub_own", "active")); + + expect(await proCreditPayer(direct)).toEqual({ + state: "ok", + subscription: { id: "sub_own", seats: 1, payerId: "user-1" }, + }); + expect(mocks.retrieve).toHaveBeenCalledWith("sub_own", { + expand: ["customer"], + }); + }); + + it("is the team subscription for a member on a seat, with its paid seats", async () => { + mocks.retrieve.mockResolvedValue( + subscription("sub_team", "active", [3, 2]), + ); + + expect(await proCreditPayer(seated)).toEqual({ + state: "ok", + subscription: { id: "sub_team", seats: 5, payerId: "owner" }, + }); + expect(mocks.retrieve).toHaveBeenCalledWith("sub_team", { + expand: ["customer"], + }); + }); + + it("is the member's own subscription when they have both", async () => { + mocks.retrieve.mockResolvedValue(subscription("sub_own", "active")); + + await proCreditPayer({ + ...direct, + thirdPartyStripeSubscriptionId: "sub_team", + }); + + expect(mocks.retrieve).toHaveBeenCalledWith("sub_own", { + expand: ["customer"], + }); + }); + + it("falls back to the team subscription when the member's own is a trial", async () => { + mocks.retrieve.mockResolvedValue(subscription("sub_team", "active")); + + await proCreditPayer({ + ...seated, + stripeSubscriptionId: "sub_own", + stripeSubscriptionStatus: "trialing", + }); + + expect(mocks.retrieve).toHaveBeenCalledWith("sub_team", { + expand: ["customer"], + }); + }); + + it("refuses a seat on a team subscription that is still a trial", async () => { + mocks.retrieve.mockResolvedValue(subscription("sub_team", "trialing")); + + expect(await proCreditPayer(seated)).toEqual({ + state: "refused", + message: + "Your $5 monthly Infra credit starts with your first paid month of Cap Pro.", + }); + }); + + it("still allows a subscription that is past due", async () => { + mocks.retrieve.mockResolvedValue(subscription("sub_team", "past_due")); + + expect((await proCreditPayer(seated)).state).toBe("ok"); + }); + + it.each(["canceled", "unpaid", "incomplete", "incomplete_expired", "paused"])( + "refuses a seat whose subscription is %s", + async (status) => { + mocks.retrieve.mockResolvedValue(subscription("sub_team", status)); + + expect(await proCreditPayer(seated)).toEqual({ + state: "refused", + message: "This credit comes with Cap Pro.", + }); + }, + ); + + it("grants nothing when the subscription cannot be read", async () => { + mocks.retrieve.mockRejectedValue(new Error("network")); + + expect(await proCreditPayer(seated)).toEqual({ + state: "refused", + message: + "Could not check your Cap Pro subscription. Try again in a moment.", + }); + }); + + it("does not ask Stripe on a self-hosted install", async () => { + mocks.isCap = false; + + expect(await proCreditPayer(member())).toEqual({ + state: "ok", + subscription: null, + }); + expect(mocks.retrieve).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/__tests__/unit/infra-pro-credit-seats.test.ts b/apps/web/__tests__/unit/infra-pro-credit-seats.test.ts new file mode 100644 index 00000000000..5ab64f65130 --- /dev/null +++ b/apps/web/__tests__/unit/infra-pro-credit-seats.test.ts @@ -0,0 +1,455 @@ +import * as Db from "@cap/database/schema"; +import type { SQL } from "drizzle-orm"; +import { MySqlDialect, type MySqlTable } from "drizzle-orm/mysql-core"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +type Member = { + id: string; + email: string; + stripeSubscriptionId: string | null; + stripeSubscriptionStatus: string | null; + thirdPartyStripeSubscriptionId: string | null; +}; + +type StripeSubscription = { + id: string; + status: string; + items: { data: Array<{ quantity?: number }> }; + customer: { id: string; email: string | null; metadata: { userId?: string } }; +}; + +const mocks = vi.hoisted(() => ({ + db: vi.fn(), + user: null as Member | null, + isCap: true, + now: new Date("2026-11-12T10:00:00Z"), + subscriptions: new Map(), + retrieve: vi.fn(), +})); + +vi.mock("@cap/database", async (importOriginal) => ({ + ...(await importOriginal()), + db: mocks.db, +})); +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: async () => mocks.user, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { + get NEXT_PUBLIC_IS_CAP() { + return mocks.isCap; + }, + }, + serverEnv: () => ({}), +})); +vi.mock("@cap/utils", () => ({ + userIsPro: (user?: Partial | null) => + !mocks.isCap || + Boolean(user?.thirdPartyStripeSubscriptionId) || + ["active", "trialing", "complete", "paid", "past_due"].includes( + user?.stripeSubscriptionStatus ?? "", + ), + stripe: () => ({ subscriptions: { retrieve: mocks.retrieve } }), +})); +vi.mock("@/lib/infra-access", () => ({ canUseInfra: () => true })); + +const { claimDeveloperProCredit } = await import( + "@/actions/developers/claim-pro-credit" +); + +type Row = Record; + +const dialect = new MySqlDialect(); +const render = (value: SQL | undefined) => + value ? dialect.sqlToQuery(value) : { sql: "", params: [] as unknown[] }; + +function makeDatabase(people: Member[]) { + const accounts = people.map((person) => ({ + id: `account-${person.id}`, + appId: `app-${person.id}`, + ownerId: person.id, + balanceMicroCredits: 0, + })); + const rows: Row[] = []; + const locks = new Map>(); + const locked: string[] = []; + const ownerOf = (accountId: unknown) => + accounts.find((account) => account.id === accountId)?.ownerId; + + const makeClient = (held: Array<() => void> | null) => { + const lock = async (userId: string) => { + if (!held) return; + locked.push(userId); + const previous = locks.get(userId) ?? Promise.resolve(); + let release: () => void = () => {}; + locks.set( + userId, + new Promise((resolve) => { + release = resolve; + }), + ); + held.push(release); + await previous; + }; + + return { + select: (fields: Record) => { + const keys = Object.keys(fields).sort().join(","); + let table: MySqlTable; + let condition: SQL | undefined; + const read = (): Row[] => { + const { sql, params } = render(condition); + if (table === Db.users) { + return people + .filter((person) => + sql.includes("`email`") + ? person.email === params[0] + : person.id === params[0], + ) + .map((person) => ({ id: person.id })); + } + if (table === Db.developerCreditAccounts && keys === "total") { + const [, period, subscriptionId] = params; + return [ + { + total: rows.filter( + (row) => + row.referenceType === "pro_monthly_credit" && + row.referenceId === period && + (row.metadata as Row).subscriptionId === subscriptionId, + ).length, + }, + ]; + } + if (table === Db.developerCreditAccounts && keys === "id") { + const [appId, ownerId] = params; + return accounts + .filter( + (account) => + account.appId === appId && account.ownerId === ownerId, + ) + .map((account) => ({ id: account.id })); + } + if (table === Db.developerCreditAccounts) { + return accounts.filter((account) => account.id === params[0]); + } + if (table === Db.developerCreditTransactions) { + const [ownerId, , period] = params; + return rows.filter( + (row) => + ownerOf(row.accountId) === ownerId && + row.referenceId === period, + ); + } + if (table === Db.developerTranscripts) return [{ total: 0 }]; + throw new Error("Unexpected select"); + }; + const settle = () => + Object.assign(Promise.resolve().then(read), { + limit: () => settle(), + for: async () => { + if (table === Db.users) { + await lock(String(render(condition).params[0])); + } + return read(); + }, + }); + const query = { + from: (value: MySqlTable) => { + table = value; + return query; + }, + innerJoin: () => query, + where: (value: SQL | undefined) => { + condition = value; + return settle(); + }, + }; + return query; + }, + update: () => ({ + set: (values: { balanceMicroCredits: SQL }) => ({ + where: async (condition: SQL | undefined) => { + const account = accounts.find( + (candidate) => candidate.id === render(condition).params[0], + ); + if (account) { + account.balanceMicroCredits += Number( + render(values.balanceMicroCredits).params.at(-1), + ); + } + }, + }), + }), + insert: () => ({ + values: async (value: Row) => { + await new Promise((resolve) => setTimeout(resolve, 20)); + rows.push(value); + }, + }), + }; + }; + + return { + accounts, + rows, + locked, + balanceOf: (person: Member) => + accounts.find((account) => account.ownerId === person.id) + ?.balanceMicroCredits, + client: { + ...makeClient(null), + transaction: async (run: (tx: unknown) => Promise) => { + const held: Array<() => void> = []; + try { + return await run(makeClient(held)); + } finally { + for (const release of held) release(); + } + }, + }, + }; +} + +const person = (id: string, fields: Partial = {}): Member => ({ + id, + email: `${id}@example.com`, + stripeSubscriptionId: null, + stripeSubscriptionStatus: null, + thirdPartyStripeSubscriptionId: null, + ...fields, +}); + +const SEATS_CLAIMED = { + success: false, + message: + "This month's Cap Pro credits for your team are all claimed. There is one for each paid seat.", +}; +const GRANTED = { balanceMicroCredits: 500_000 }; + +describe("the Cap Pro credit follows the paid seats", () => { + let owner: Member; + let anna: Member; + let ben: Member; + let cara: Member; + let solo: Member; + let database: ReturnType; + + const team = (seats: number, status = "active") => + mocks.subscriptions.set("sub_team", { + id: "sub_team", + status, + items: { data: [{ quantity: seats }] }, + customer: { + id: "cus_team", + email: owner.email, + metadata: { userId: owner.id }, + }, + }); + + const claimAs = (member: Member) => { + mocks.user = member; + return claimDeveloperProCredit(`app-${member.id}`); + }; + + const holdSeat = (member: Member) => { + for (const other of [anna, ben, cara]) { + other.thirdPartyStripeSubscriptionId = + other === member ? "sub_team" : null; + } + }; + + beforeEach(() => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date("2026-11-12T10:00:00Z")); + mocks.isCap = true; + owner = person("owner", { + stripeSubscriptionId: "sub_team", + stripeSubscriptionStatus: "active", + }); + anna = person("anna"); + ben = person("ben"); + cara = person("cara"); + solo = person("solo", { + stripeSubscriptionId: "sub_solo", + stripeSubscriptionStatus: "active", + }); + database = makeDatabase([owner, anna, ben, cara, solo]); + mocks.db.mockReturnValue(database.client); + mocks.subscriptions = new Map(); + team(1); + mocks.subscriptions.set("sub_solo", { + id: "sub_solo", + status: "active", + items: { data: [{ quantity: 1 }] }, + customer: { id: "cus_solo", email: solo.email, metadata: {} }, + }); + mocks.retrieve.mockReset(); + mocks.retrieve.mockImplementation(async (id: string) => { + const subscription = mocks.subscriptions.get(id); + if (!subscription) throw new Error("No such subscription"); + return subscription; + }); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("grants one claim a month when one seat is rotated across three members", async () => { + holdSeat(anna); + expect(await claimAs(anna)).toEqual(GRANTED); + holdSeat(ben); + expect(await claimAs(ben)).toEqual(SEATS_CLAIMED); + holdSeat(cara); + expect(await claimAs(cara)).toEqual(SEATS_CLAIMED); + + expect(database.rows).toHaveLength(1); + expect(database.balanceOf(anna)).toBe(500_000); + expect(database.balanceOf(ben)).toBe(0); + expect(database.balanceOf(cara)).toBe(0); + }); + + it("counts the owner's own claim against the same seat", async () => { + expect(await claimAs(owner)).toEqual(GRANTED); + holdSeat(anna); + expect(await claimAs(anna)).toEqual(SEATS_CLAIMED); + expect(database.rows).toHaveLength(1); + }); + + it("grants two claims a month on a two-seat subscription", async () => { + team(2); + + holdSeat(anna); + expect(await claimAs(anna)).toEqual(GRANTED); + holdSeat(ben); + expect(await claimAs(ben)).toEqual(GRANTED); + holdSeat(cara); + expect(await claimAs(cara)).toEqual(SEATS_CLAIMED); + + expect(database.rows).toHaveLength(2); + }); + + it("grants one when two members claim through one seat at the same moment", async () => { + anna.thirdPartyStripeSubscriptionId = "sub_team"; + ben.thirdPartyStripeSubscriptionId = "sub_team"; + mocks.user = anna; + const first = claimDeveloperProCredit("app-anna"); + mocks.user = ben; + const second = claimDeveloperProCredit("app-ben"); + mocks.user = null; + + const results = await Promise.all([first, second]); + + expect( + results.filter((result) => "balanceMicroCredits" in result), + ).toHaveLength(1); + expect(results.filter((result) => "success" in result)).toEqual([ + SEATS_CLAIMED, + ]); + expect(database.rows).toHaveLength(1); + expect(database.locked.filter((id) => id === "owner")).toHaveLength(2); + }); + + it("opens the seat again the next month", async () => { + holdSeat(anna); + await claimAs(anna); + holdSeat(ben); + expect(await claimAs(ben)).toEqual(SEATS_CLAIMED); + + vi.setSystemTime(new Date("2026-12-01T00:00:01Z")); + expect(await claimAs(ben)).toEqual(GRANTED); + holdSeat(anna); + expect(await claimAs(anna)).toEqual(SEATS_CLAIMED); + }); + + it("leaves a direct subscriber unaffected", async () => { + holdSeat(anna); + await claimAs(anna); + + expect(await claimAs(solo)).toEqual(GRANTED); + expect(await claimAs(solo)).toEqual({ + success: false, + message: + "This month's credit is already claimed. It is one per account, not one per app.", + }); + expect(database.balanceOf(solo)).toBe(500_000); + expect(database.locked.filter((id) => id === "solo")).toHaveLength(2); + expect(database.rows.at(-1)).toMatchObject({ + type: "pro_credit", + referenceId: "2026-11", + referenceType: "pro_monthly_credit", + metadata: { period: "2026-11", subscriptionId: "sub_solo", seats: 1 }, + }); + }); + + it("records the paying subscription on a member's claim", async () => { + holdSeat(anna); + await claimAs(anna); + + expect(database.rows[0]).toMatchObject({ + accountId: "account-anna", + referenceId: "2026-11", + metadata: { subscriptionId: "sub_team", seats: 1 }, + }); + }); + + it("refuses a member whose team subscription is still on a trial", async () => { + team(1, "trialing"); + holdSeat(anna); + + expect(await claimAs(anna)).toEqual({ + success: false, + message: + "Your $5 monthly Infra credit starts with your first paid month of Cap Pro.", + }); + expect(database.rows).toHaveLength(0); + }); + + it("still grants while the subscription is past due", async () => { + team(1, "past_due"); + holdSeat(anna); + + expect(await claimAs(anna)).toEqual(GRANTED); + }); + + it("refuses a seat whose subscription has ended", async () => { + team(1, "canceled"); + holdSeat(anna); + + expect(await claimAs(anna)).toEqual({ + success: false, + message: "This credit comes with Cap Pro.", + }); + expect(database.rows).toHaveLength(0); + }); + + it("grants nothing when the subscription cannot be read", async () => { + mocks.retrieve.mockRejectedValue(new Error("network")); + holdSeat(anna); + + expect(await claimAs(anna)).toEqual({ + success: false, + message: + "Could not check your Cap Pro subscription. Try again in a moment.", + }); + expect(database.rows).toHaveLength(0); + }); + + it("finds the payer by email when the customer carries no user id", async () => { + const subscription = mocks.subscriptions.get("sub_team"); + if (subscription) subscription.customer.metadata = {}; + holdSeat(anna); + + expect(await claimAs(anna)).toEqual(GRANTED); + expect(database.locked).toEqual(["owner", "anna"]); + }); + + it("does not ask Stripe on a self-hosted install", async () => { + mocks.isCap = false; + + expect(await claimAs(anna)).toEqual(GRANTED); + expect(mocks.retrieve).not.toHaveBeenCalled(); + expect(database.rows[0]).toMatchObject({ metadata: { period: "2026-11" } }); + }); +}); diff --git a/apps/web/__tests__/unit/infra-pro-credit.test.ts b/apps/web/__tests__/unit/infra-pro-credit.test.ts index a79b654ad3b..ca737debe1f 100644 --- a/apps/web/__tests__/unit/infra-pro-credit.test.ts +++ b/apps/web/__tests__/unit/infra-pro-credit.test.ts @@ -4,6 +4,8 @@ type Row = Record; const state = vi.hoisted(() => ({ balance: 0, + heldMicroCredits: 0, + reads: [] as string[], owner: "user-1", transactions: [] as Record[], lock: Promise.resolve(), @@ -38,13 +40,23 @@ vi.mock("@cap/database", () => ({ const chain = { innerJoin: () => chain, where: (conditions: Row[]) => ({ - limit: () => - Object.assign(Promise.resolve(read(table, conditions)), { - for: async () => { - await lock(); - return read(table, conditions); - }, - }), + limit: () => ({ + then: ( + resolve: (rows: Row[]) => unknown, + reject: (error: unknown) => unknown, + ) => { + state.reads.push(`read ${table.table}`); + return Promise.resolve(read(table, conditions)).then( + resolve, + reject, + ); + }, + for: async () => { + state.reads.push(`lock ${table.table}`); + await lock(); + return read(table, conditions); + }, + }), }), }; return chain; @@ -73,6 +85,12 @@ vi.mock("@cap/database", () => ({ })); vi.mock("@cap/database/helpers", () => ({ nanoId: () => "txn" })); +vi.mock("@cap/env", () => ({ buildEnv: { NEXT_PUBLIC_IS_CAP: "true" } })); +vi.mock("@cap/utils", () => ({ stripe: () => ({}), userIsPro: () => true })); +vi.mock("@/lib/infra-balance", () => ({ + AVAILABLE_AFTER_FIELD: "availableAfterMicroCredits", + heldTranscriptMicroCredits: async () => state.heldMicroCredits, +})); vi.mock("@cap/database/schema", () => ({ users: { table: "users", id: "id" }, developerCreditAccounts: { @@ -110,6 +128,8 @@ const claim = (now = november, accountId = "account-1") => describe("Cap Pro monthly Infra credit", () => { beforeEach(() => { state.balance = 0; + state.heldMicroCredits = 0; + state.reads = []; state.owner = "user-1"; state.transactions = []; state.lock = Promise.resolve(); @@ -133,6 +153,28 @@ describe("Cap Pro monthly Infra credit", () => { ).toBe("2027-01-01T00:00:00.000Z"); }); + it("records the credit available after the claim, net of held transcript credit", async () => { + state.balance = 100_000; + state.heldMicroCredits = 60_000; + + await claim(); + + expect(state.transactions[0]?.metadata).toMatchObject({ + period: "2026-11", + availableAfterMicroCredits: 100_000 + 500_000 - 60_000, + }); + }); + + it("locks the credit account before its first plain read, so the held credit it reads is current", async () => { + await claim(); + + const firstPlainRead = state.reads.findIndex((entry) => + entry.startsWith("read "), + ); + expect(state.reads.indexOf("lock accounts")).toBeGreaterThanOrEqual(0); + expect(state.reads.indexOf("lock accounts")).toBeLessThan(firstPlainRead); + }); + it("adds the credit and writes one ledger row", async () => { const result = await claim(); expect(result).toEqual({ claimed: true, balanceMicroCredits: 500_000 }); @@ -155,7 +197,10 @@ describe("Cap Pro monthly Infra credit", () => { it("refuses a second claim in the same month, even for another app", async () => { await claim(); - expect(await claim(november, "account-2")).toEqual({ claimed: false }); + expect(await claim(november, "account-2")).toEqual({ + claimed: false, + reason: "already_claimed", + }); expect(state.balance).toBe(500_000); }); diff --git a/apps/web/__tests__/unit/infra-storage-billing.test.ts b/apps/web/__tests__/unit/infra-storage-billing.test.ts index f6ad01f874e..8ccfbf2f04b 100644 --- a/apps/web/__tests__/unit/infra-storage-billing.test.ts +++ b/apps/web/__tests__/unit/infra-storage-billing.test.ts @@ -20,8 +20,8 @@ import { holder.db = fakeDb; import { + firstMonthStorageMicroCredits, storageUsage, - uploadAdmissionMicroCredits, } from "@/lib/developer-pricing"; import { admitDeveloperVideoUpload } from "@/lib/developer-video-charge"; import { availableBalance } from "@/lib/infra-balance"; @@ -29,8 +29,10 @@ import { getInfraAppStanding, getInfraBilling, getInfraUploadAdmission, + latestStorageCharge, + latestStorageCharges, } from "@/lib/infra-standing"; -import { getAppStorageUsage } from "@/lib/infra-storage"; +import { getAppStorage } from "@/lib/infra-storage"; import { chargeStorageDay } from "@/lib/infra-storage-billing"; const DAY_MS = 24 * 60 * 60 * 1000; @@ -47,11 +49,25 @@ const charge = async () => chargeStorageDay({ appId: "app-1", accountId: "account-1", - usage: await getAppStorageUsage("app-1"), + usage: (await getAppStorage("app-1")).dailyCharge, dateKey: dateKey(), now: new Date(ledger.now), }); +const finish = async (day: string) => + chargeStorageDay({ + appId: "app-1", + accountId: "account-1", + usage: (await getAppStorage("app-1")).dailyCharge, + dateKey: day, + now: new Date(ledger.now), + retryUnpaid: false, + }); + +const later = (hours: number) => { + ledger.now = new Date(ledger.now.getTime() + hours * 60 * 60 * 1000); +}; + const credit = (microCredits: number) => { ledger.account.balanceMicroCredits += microCredits; ledger.rows.push({ @@ -231,6 +247,163 @@ describe("daily storage charge", () => { expect(ledger.snapshots.at(-1)?.microCreditsCharged).toBe(0); }); + it("leaves a billed day alone when it is finished later", async () => { + credit(500_000); + expect(await charge()).toMatchObject({ state: "billed" }); + const snapshot = { ...ledger.snapshots[0] }; + const day = dateKey(); + later(20); + + expect(await finish(day)).toEqual({ state: "skipped" }); + + expect(storageRows()).toHaveLength(1); + expect(ledger.snapshots).toEqual([snapshot]); + expect(ledger.account.balanceMicroCredits).toBe(500_000 - 144); + }); + + it("leaves a short day alone when it is finished later", async () => { + credit(50); + expect(await charge()).toMatchObject({ + state: "billed", + chargedMicroCredits: 50, + fellShort: true, + }); + const snapshot = { ...ledger.snapshots[0] }; + const day = dateKey(); + credit(500_000); + later(20); + + expect(await finish(day)).toEqual({ state: "skipped" }); + + expect(storageRows()).toHaveLength(1); + expect(ledger.snapshots).toEqual([snapshot]); + }); + + it("does not write an unpaid day again when it is finished later, even after a top-up", async () => { + await charge(); + advanceDays(1); + expect(await charge()).toMatchObject({ state: "unpaid" }); + const day = dateKey(); + const recorded = { + ...ledger.snapshots.find((snapshot) => snapshot.snapshotDate === day), + }; + credit(500_000); + later(20); + + expect(await finish(day)).toEqual({ state: "skipped" }); + + expect( + ledger.snapshots.find((snapshot) => snapshot.snapshotDate === day), + ).toEqual(recorded); + expect(storageRows()).toHaveLength(1); + expect(ledger.account.balanceMicroCredits).toBe(500_000); + }); + + it("records an unpaid day once when it is first reached while finishing", async () => { + await charge(); + advanceDays(1); + + expect(await finish(dateKey())).toMatchObject({ state: "unpaid" }); + expect(await finish(dateKey())).toEqual({ state: "skipped" }); + + expect( + ledger.snapshots.filter( + (snapshot) => snapshot.snapshotDate === dateKey(), + ), + ).toHaveLength(1); + }); + + it("charges a day over an idle visit marker and replaces it", async () => { + credit(500_000); + ledger.snapshots = [ + { + snapshotDate: dateKey(), + totalDurationMinutes: 0, + videoCount: 0, + microCreditsCharged: 0, + processedAt: new Date(ledger.now), + }, + ]; + + expect(await charge()).toMatchObject({ + state: "billed", + chargedMicroCredits: 144, + }); + + expect(storageRows()).toHaveLength(1); + expect(ledger.snapshots).toEqual([ + expect.objectContaining({ + snapshotDate: dateKey(), + videoCount: 3, + microCreditsCharged: 144, + }), + ]); + expect(ledger.snapshots[0]?.totalDurationMinutes).toBeGreaterThan(0); + }); + + it("reads the latest billed day, not the last write, when two days are written in the same second", async () => { + const yesterday = new Date(ledger.now.getTime() - DAY_MS) + .toISOString() + .slice(0, 10); + const shortSince = new Date(ledger.now); + ledger.account.balanceMicroCredits = 0; + ledger.rows.push( + { + id: "row-a", + accountId: "account-1", + type: "storage_daily", + amountMicroCredits: -50, + balanceAfterMicroCredits: 0, + referenceId: dateKey(), + referenceType: "storage_day", + metadata: { + snapshotDate: dateKey(), + owedMicroCredits: 144, + chargedMicroCredits: 50, + shortSince: shortSince.toISOString(), + }, + createdAt: new Date(ledger.now), + }, + { + id: "row-b", + accountId: "account-1", + type: "storage_daily", + amountMicroCredits: -144, + balanceAfterMicroCredits: 50, + referenceId: yesterday, + referenceType: "storage_day", + metadata: { + snapshotDate: yesterday, + owedMicroCredits: 144, + chargedMicroCredits: 144, + shortSince: null, + }, + createdAt: new Date(ledger.now), + }, + ); + + expect(await latestStorageCharge(fakeDb as never, "account-1")).toEqual({ + id: "row-a", + createdAt: new Date(ledger.now), + owedMicroCredits: 144, + shortSince, + }); + expect( + (await latestStorageCharges(fakeDb as never, ["account-1"])).get( + "account-1", + ), + ).toEqual({ + id: "row-a", + createdAt: new Date(ledger.now), + owedMicroCredits: 144, + shortSince, + }); + expect(await standing()).toMatchObject({ state: "grace" }); + + advanceDays(1); + expect(await charge()).toEqual({ state: "unpaid", shortSince }); + }); + it("does not bill an app that stores nothing", async () => { ledger.videos = []; ledger.account.balanceMicroCredits = 1_000_000; @@ -335,7 +508,7 @@ describe("admission prices a video the way the storage job will", () => { ledger.videos = []; }); - it("requires exactly 7 days of what the job charges for the same row", async () => { + it("requires exactly the first month the charge at ready takes for the same row", async () => { ledger.account.balanceMicroCredits = 1_000_000; ledger.videos = [ { @@ -348,7 +521,7 @@ describe("admission prices a video the way the storage job will", () => { maxResolutionTier: capped.maxResolutionTier, }, ]; - const jobUsage = await getAppStorageUsage("app-1"); + const jobUsage = (await getAppStorage("app-1")).dailyCharge; ledger.videos = []; const admission = await admitDeveloperVideoUpload(capped); @@ -359,16 +532,20 @@ describe("admission prices a video the way the storage job will", () => { "2160p": 0, }); expect(admission.requiredMicroCredits).toBe( - uploadAdmissionMicroCredits(jobUsage.monthlyMicroCredits), + firstMonthStorageMicroCredits({ + durationSeconds: capped.durationSeconds, + tier: "1080p", + }), ); - expect(admission.requiredMicroCredits).toBe(1190); + expect(admission.requiredMicroCredits).toBe(jobUsage.monthlyMicroCredits); + expect(admission.requiredMicroCredits).toBe(5100); }); it("admits a 4K source capped at 1080p on a balance that covers 1080p storage, and refuses one that does not", async () => { - ledger.account.balanceMicroCredits = 1190; + ledger.account.balanceMicroCredits = 5100; expect((await admitDeveloperVideoUpload(capped)).allowed).toBe(true); - ledger.account.balanceMicroCredits = 1189; + ledger.account.balanceMicroCredits = 5099; expect(await admitDeveloperVideoUpload(capped)).toMatchObject({ allowed: false, reason: "insufficient_credits", @@ -377,14 +554,238 @@ describe("admission prices a video the way the storage job will", () => { }); it("needs 4K credit when the video is allowed to be stored at 4K", async () => { - ledger.account.balanceMicroCredits = 1190; + ledger.account.balanceMicroCredits = 5100; expect( await admitDeveloperVideoUpload({ ...capped, maxResolutionTier: "2160p", }), - ).toMatchObject({ allowed: false, requiredMicroCredits: 3920 }); + ).toMatchObject({ allowed: false, requiredMicroCredits: 16_800 }); + }); +}); + +describe("first months charged in the same second", () => { + const firstMonth = (amount: number, shortSince: Date | null) => ({ + id: `first-${ledger.rows.length}`, + accountId: "account-1", + type: "video_create", + amountMicroCredits: -amount, + balanceAfterMicroCredits: 0, + referenceId: `video-${ledger.rows.length}`, + referenceType: "developer_video", + metadata: { + owedMicroCredits: 850, + chargedMicroCredits: amount, + shortSince: shortSince?.toISOString() ?? null, + }, + createdAt: new Date(ledger.now), + }); + + it("keeps the app out of credit whichever of the two rows comes first", async () => { + for (const order of ["paid first", "short first"]) { + resetLedger(); + ledger.videos = []; + const paid = firstMonth(850, null); + const short = firstMonth(0, new Date(ledger.now)); + ledger.rows = order === "paid first" ? [paid, short] : [short, paid]; + + expect(await standing(), order).toMatchObject({ state: "grace" }); + } + }); +}); + +describe("a first month the balance could not pay", () => { + it("stays in effect through later paid days until credit covers what it owed", async () => { + ledger.videos = [ + { appId: "app-1", width: 1920, height: 1080, seconds: 600, count: 1 }, + ]; + const shortSince = new Date(ledger.now); + ledger.rows.push({ + id: "first-month", + accountId: "account-1", + type: "video_create", + amountMicroCredits: 0, + balanceAfterMicroCredits: 0, + referenceId: "video-1", + referenceType: "developer_video", + metadata: { + owedMicroCredits: 16_800, + chargedMicroCredits: 0, + shortSince: shortSince.toISOString(), + }, + createdAt: new Date(ledger.now), + }); + expect(await standing()).toMatchObject({ state: "grace" }); + + advanceDays(1); + credit(1_000); + expect(await charge()).toMatchObject({ + state: "billed", + chargedMicroCredits: 28, + }); + expect(await standing()).toEqual({ + state: "grace", + pausesAt: new Date(shortSince.getTime() + 7 * DAY_MS), + }); + expect((await admission()).reason).toBe("out_of_credit"); + + credit(16_800); + expect(await standing()).toEqual({ state: "active", pausesAt: null }); + }); +}); + +describe("a shortfall the available balance comes to cover without new credit", () => { + it("is latched as resolved once, stays resolved through later paid days, and a new shortfall starts its own grace", async () => { + ledger.videos = [ + { appId: "app-1", width: 1920, height: 1080, seconds: 600, count: 1 }, + ]; + ledger.account.balanceMicroCredits = 900; + ledger.heldMicroCredits = 900; + const shortSince = new Date(ledger.now); + const month = { + id: "first-month-held", + accountId: "account-1", + type: "video_create", + amountMicroCredits: 0, + balanceAfterMicroCredits: 900, + referenceId: "video-1", + referenceType: "developer_video", + metadata: { + owedMicroCredits: 850, + chargedMicroCredits: 0, + shortSince: shortSince.toISOString(), + } as Record, + createdAt: new Date(ledger.now), + }; + ledger.rows.push(month); + expect(await standing()).toMatchObject({ state: "grace" }); + + ledger.heldMicroCredits = 0; + expect(await standing()).toEqual({ state: "active", pausesAt: null }); + + advanceDays(1); + expect(await charge()).toMatchObject({ + state: "billed", + chargedMicroCredits: 28, + shortSince: null, + }); + const latched = month.metadata.resolvedAt; + expect(latched).toBe(new Date(ledger.now).toISOString()); + + advanceDays(1); + expect(await charge()).toMatchObject({ + state: "billed", + chargedMicroCredits: 28, + }); + expect(ledger.account.balanceMicroCredits).toBe(844); + expect(month.metadata.resolvedAt).toBe(latched); + + advanceDays(8); + expect(await standing()).toEqual({ state: "active", pausesAt: null }); + + ledger.account.balanceMicroCredits = 0; + expect(await charge()).toMatchObject({ + state: "billed", + chargedMicroCredits: 0, + fellShort: true, + }); + expect(await standing()).toEqual({ + state: "grace", + pausesAt: new Date(ledger.now.getTime() + 7 * DAY_MS), + }); + expect(month.metadata.resolvedAt).toBe(latched); + }); +}); + +describe("an unpaid first month is not masked by daily rows", () => { + const unpaidMonth = () => { + const shortSince = new Date(ledger.now); + ledger.rows.push({ + id: "first-month-unpaid", + accountId: "account-1", + type: "video_create", + amountMicroCredits: 0, + balanceAfterMicroCredits: 0, + referenceId: "video-1", + referenceType: "developer_video", + metadata: { + owedMicroCredits: 16_800, + chargedMicroCredits: 0, + shortSince: shortSince.toISOString(), + }, + createdAt: new Date(ledger.now), + }); + return shortSince; + }; + + it("keeps a first month that settled short in effect over a newer fully paid storage_daily row", async () => { + ledger.videos = [ + { appId: "app-1", width: 1920, height: 1080, seconds: 600, count: 1 }, + ]; + const shortSince = unpaidMonth(); + advanceDays(1); + credit(1_000); + expect(await charge()).toMatchObject({ + state: "billed", + chargedMicroCredits: 28, + shortSince: null, + }); + + expect( + await latestStorageCharge(fakeDb as never, "account-1"), + ).toMatchObject({ firstMonth: true, owedMicroCredits: 16_800, shortSince }); + expect((await standing()).state).not.toBe("active"); + }); + + it("gives grace, not active, when a paid storage_day row lands in the same second as a short first month", async () => { + const shortSince = unpaidMonth(); + ledger.account.balanceMicroCredits = 500; + ledger.rows.push({ + id: "storage-day-paid", + accountId: "account-1", + type: "storage_daily", + amountMicroCredits: -28, + balanceAfterMicroCredits: 500, + referenceId: dateKey(), + referenceType: "storage_day", + metadata: { + snapshotDate: dateKey(), + owedMicroCredits: 28, + chargedMicroCredits: 28, + shortSince: null, + }, + createdAt: new Date(ledger.now), + }); + + expect(await standing()).toEqual({ + state: "grace", + pausesAt: new Date(shortSince.getTime() + 7 * DAY_MS), + }); + }); + + it("keeps the shortfall when a partly paid daily row is followed by a day with nothing charged daily", async () => { + ledger.videos = [ + { appId: "app-1", width: 1920, height: 1080, seconds: 600, count: 1 }, + ]; + const shortSince = unpaidMonth(); + advanceDays(1); + credit(10); + expect(await charge()).toMatchObject({ + state: "billed", + chargedMicroCredits: 10, + shortSince, + }); + + advanceDays(1); + ledger.videos = []; + const rowsBefore = ledger.rows.length; + expect(await charge()).toEqual({ state: "skipped" }); + expect(ledger.rows).toHaveLength(rowsBefore); + expect(await standing()).toEqual({ + state: "grace", + pausesAt: new Date(shortSince.getTime() + 7 * DAY_MS), + }); }); }); @@ -549,7 +950,7 @@ describe("which videos are billed after the pipeline measures them", () => { it("charges storage for ready videos only, at the lower of the measured tier and the max tier", async () => { ledger.account.balanceMicroCredits = 1_000_000; - const usage = await getAppStorageUsage("app-1"); + const usage = (await getAppStorage("app-1")).stored; const result = await charge(); expect(usage.minutesByTier).toEqual({ @@ -561,17 +962,17 @@ describe("which videos are billed after the pipeline measures them", () => { expect(result).toMatchObject({ state: "billed", chargedMicroCredits: 31 }); }); - it("counts a still-processing video toward upload admission but not the storage charge", async () => { + it("counts a still-processing video toward upload admission at its first month, but not as stored or charged daily", async () => { ledger.account.balanceMicroCredits = 1_000_000; - const { usage } = await billing(); + const { storage, admission } = await billing(); - expect(usage.minutesByTier).toEqual({ - "1080p": 11, - "1440p": 0, - "2160p": 10, - }); - expect((await getAppStorageUsage("app-1")).minutesByTier["2160p"]).toBe(0); + expect(storage.firstMonthMicroCredits).toBe(10 * 280); + expect(storage.stored.minutesByTier["2160p"]).toBe(0); + expect(storage.dailyCharge.minutesByTier["2160p"]).toBe(0); + expect(admission.requiredMicroCredits).toBe( + Math.ceil(10 * 280 + (11 * 85 * 7) / 30), + ); }); it("leaves a video out of its own check when its measured length is admitted", async () => { @@ -589,10 +990,10 @@ describe("which videos are billed after the pipeline measures them", () => { const doubled = await getInfraUploadAdmission("app-1", measured); expect(own.requiredMicroCredits).toBe( - Math.ceil(((11 * 85 + 11 * 280) * 7) / 30), + Math.ceil(11 * 280 + (11 * 85 * 7) / 30), ); expect(doubled.requiredMicroCredits).toBe( - Math.ceil(((11 * 85 + 21 * 280) * 7) / 30), + Math.ceil(21 * 280 + (11 * 85 * 7) / 30), ); }); }); @@ -612,17 +1013,19 @@ describe("available balance", () => { describe("upload admission against what is stored", () => { it("counts the video being admitted on top of what is stored", async () => { - const stored = await getAppStorageUsage("app-1"); - const withNew = await getAppStorageUsage("app-1", { + const stored = await getAppStorage("app-1"); + const withNew = await getAppStorage("app-1", { durationSeconds: 6000, width: 3840, height: 2160, tier: "2160p", }); - expect(stored.monthlyMicroCredits).toBe(4335); - expect(withNew.monthlyMicroCredits).toBe(4335 + 28_000); - expect(withNew.videoCount).toBe(4); + expect(stored.dailyCharge.monthlyMicroCredits).toBe(4335); + expect(stored.firstMonthMicroCredits).toBe(0); + expect(withNew.dailyCharge.monthlyMicroCredits).toBe(4335); + expect(withNew.firstMonthMicroCredits).toBe(28_000); + expect(withNew.stored.videoCount).toBe(3); ledger.account.balanceMicroCredits = 1012; expect((await getInfraUploadAdmission("app-1")).allowed).toBe(true); @@ -635,13 +1038,13 @@ describe("upload admission against what is stored", () => { expect(refused).toMatchObject({ allowed: false, reason: "insufficient_credits", - requiredMicroCredits: 7545, - shortfallMicroCredits: 6533, + requiredMicroCredits: 29_012, + shortfallMicroCredits: 28_000, }); }); it("matches the pure storage math", async () => { - expect(await getAppStorageUsage("app-1")).toEqual( + expect((await getAppStorage("app-1")).stored).toEqual( storageUsage([ { video: { diff --git a/apps/web/__tests__/unit/infra-storage-created-before.test.ts b/apps/web/__tests__/unit/infra-storage-created-before.test.ts new file mode 100644 index 00000000000..698ec5c3078 --- /dev/null +++ b/apps/web/__tests__/unit/infra-storage-created-before.test.ts @@ -0,0 +1,56 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const state = vi.hoisted(() => ({ + queries: [] as { sql: string; params: unknown[] }[], +})); + +vi.mock("@cap/database", async () => { + const { drizzle } = await import("drizzle-orm/mysql-proxy"); + const client = drizzle(async (sql, params) => { + state.queries.push({ sql, params }); + return { rows: [] }; + }); + return { db: () => client }; +}); + +import { getAppStorage, getStorageUsageByApp } from "@/lib/infra-storage"; + +beforeEach(() => { + state.queries = []; +}); + +describe("stored video bounded by creation time", () => { + it("adds the bound to the same aggregate query", async () => { + const dayEnd = new Date("2026-11-13T00:00:00.000Z"); + + await getStorageUsageByApp(["app-1", "app-2"], undefined, undefined, { + createdBefore: dayEnd, + }); + + expect(state.queries).toHaveLength(1); + const [{ sql, params }] = state.queries as [ + { sql: string; params: unknown[] }, + ]; + expect(sql).toContain("COALESCE(SUM(`duration`), 0)"); + expect(sql).toContain("and `developer_videos`.`createdAt` < ?)"); + expect(sql).toContain("group by"); + expect(params.slice(-3)).toEqual([ + "app-1", + "app-2", + "2026-11-13 00:00:00.000", + ]); + }); + + it("reads every stored video when no bound is given", async () => { + await getStorageUsageByApp(["app-1"]); + await getAppStorage("app-1"); + + const aggregates = state.queries.filter(({ sql }) => + sql.includes("group by"), + ); + expect(aggregates).toHaveLength(2); + for (const { sql } of aggregates) { + expect(sql.slice(sql.indexOf(" where "))).not.toContain("createdAt"); + } + }); +}); diff --git a/apps/web/__tests__/unit/infra-storage-cron-resume.test.ts b/apps/web/__tests__/unit/infra-storage-cron-resume.test.ts new file mode 100644 index 00000000000..4a0f5804dd9 --- /dev/null +++ b/apps/web/__tests__/unit/infra-storage-cron-resume.test.ts @@ -0,0 +1,993 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +type ChargeCall = { + accountId: string; + dateKey: string; + retryUnpaid: boolean | undefined; + dailyMicroCredits: number; +}; + +const state = vi.hoisted(() => ({ + accounts: [] as { id: string; appId: string; createdAt: Date }[], + appCreatedAt: new Map(), + videos: new Map(), + markers: new Map(), + markerInserts: [] as string[], + usageReads: [] as { appIds: string[]; createdBefore: Date | undefined }[], + deletedApps: new Set(), + unpaid: new Set(), + short: new Set(), + everShort: new Set(), + charged: new Map(), + snapshotWrites: new Map(), + chargeCalls: [] as ChargeCall[], + accountReads: [] as { cursor: string | null; limit: number | null }[], + accountSql: [] as string[], + longestIdList: 0, + secondsPerCharge: 0, + topUps: vi.fn(), + delivery: vi.fn(), + emails: vi.fn(), + afters: [] as Promise[], + continuations: [] as { url: string; authorization: string | null }[], + continuationStatus: 200 as number | "network", +})); + +vi.mock("next/server", async (original) => ({ + ...(await original()), + after: (task: Promise) => { + state.afters.push(task); + }, +})); +vi.mock("@cap/database", async () => { + const { drizzle } = await import("drizzle-orm/mysql-proxy"); + const client = drizzle(async (sql, params) => { + if (sql.includes("from `developer_credit_accounts`")) { + const limit = /limit \?/.test(sql) ? Number(params.at(-1)) : null; + const cursor = /`id` > \?/.test(sql) ? String(params[0]) : null; + state.accountReads.push({ cursor, limit }); + state.accountSql.push(sql); + const after = state.accounts.filter( + (account) => cursor === null || account.id > cursor, + ); + const stamp = (date: Date) => + date.toISOString().slice(0, 19).replace("T", " "); + return { + rows: (limit === null ? after : after.slice(0, limit)).map( + (account) => [account.id, account.appId, stamp(account.createdAt)], + ), + }; + } + if (sql.includes("from `developer_apps`")) { + const bounded = sql.includes("`developer_apps`.`createdAt` < ?"); + const bound = bounded + ? new Date(`${String(params.at(-1)).replace(" ", "T")}Z`) + : null; + return { + rows: (bounded ? params.slice(0, -1) : params) + .map(String) + .filter((appId) => !state.deletedApps.has(appId)) + .filter( + (appId) => + !bound || + (state.appCreatedAt.get(appId) ?? new Date(0)).getTime() < + bound.getTime(), + ) + .map((appId) => [appId]), + }; + } + if (sql.includes("from `developer_credit_transactions`")) { + const ids = params.slice(0, -2).map(String); + const date = String(params.at(-2)); + state.longestIdList = Math.max(state.longestIdList, ids.length); + return { + rows: ids + .filter((id) => state.charged.has(`${id}|${date}`)) + .map((id) => [id]), + }; + } + if (sql.includes("from `developer_daily_storage_snapshots`")) { + const appIds = params.slice(0, -1).map(String); + const date = String(params.at(-1)); + return { + rows: appIds + .filter( + (appId) => + state.snapshotWrites.has(`${appId}|${date}`) || + state.markers.has(`${appId}|${date}`), + ) + .map((appId) => [appId]), + }; + } + throw new Error(`Unexpected query: ${sql}`); + }); + return { db: () => client }; +}); +vi.mock("@/lib/infra-storage", () => ({ + getStorageUsageByApp: async ( + appIds: string[], + _extra?: unknown, + _executor?: unknown, + options: { createdBefore?: Date } = {}, + ) => { + state.usageReads.push({ appIds, createdBefore: options.createdBefore }); + return new Map( + appIds.map((appId) => { + const videos = ( + state.videos.get(appId) ?? [new Date("2026-11-01T00:00:00Z")] + ) + .map((video) => + video instanceof Date + ? { createdAt: video, readyAt: video } + : video, + ) + .filter( + ({ createdAt, readyAt }) => + readyAt.getTime() <= Date.now() && + (!options.createdBefore || + createdAt.getTime() < options.createdBefore.getTime()), + ).length; + return [ + appId, + { + videoCount: videos, + totalMinutes: videos * 10, + minutesByTier: { "1080p": videos * 10, "1440p": 0, "2160p": 0 }, + monthlyMicroCredits: videos * 840, + dailyMicroCredits: videos * 28, + }, + ]; + }), + ); + }, +})); +vi.mock("@/lib/infra-storage-billing", () => ({ + recordStorageDayVisits: async ({ + appIds, + dateKey, + }: { + appIds: string[]; + dateKey: string; + }) => { + for (const appId of appIds) { + const key = `${appId}|${dateKey}`; + state.markerInserts.push(key); + if (!state.snapshotWrites.has(key) && !state.markers.has(key)) { + state.markers.set(key, 1); + } + } + }, + chargeStorageDay: async ({ + appId, + accountId, + dateKey, + retryUnpaid, + usage, + }: { + appId: string; + accountId: string; + dateKey: string; + retryUnpaid?: boolean; + usage: { dailyMicroCredits: number }; + }) => { + state.chargeCalls.push({ + accountId, + dateKey, + retryUnpaid, + dailyMicroCredits: usage.dailyMicroCredits, + }); + vi.setSystemTime(Date.now() + state.secondsPerCharge * 1000); + const key = `${accountId}|${dateKey}`; + const snapshot = `${appId}|${dateKey}`; + if (state.charged.has(key)) return { state: "skipped" }; + const writeSnapshot = () => { + state.markers.delete(snapshot); + state.snapshotWrites.set( + snapshot, + (state.snapshotWrites.get(snapshot) ?? 0) + 1, + ); + }; + if ( + retryUnpaid === false && + (state.markers.has(snapshot) || + (state.unpaid.has(accountId) && state.snapshotWrites.has(snapshot))) + ) { + return { state: "skipped" }; + } + if (state.unpaid.has(accountId)) { + writeSnapshot(); + return { state: "unpaid", shortSince: new Date("2026-11-01T00:00:00Z") }; + } + if (usage.dailyMicroCredits <= 0) return { state: "skipped" }; + state.charged.set(key, 1); + writeSnapshot(); + const short = state.short.has(accountId); + const fellShort = short && !state.everShort.has(accountId); + if (short) state.everShort.add(accountId); + return { + state: "billed", + owedMicroCredits: 28, + chargedMicroCredits: short ? 0 : 28, + balanceMicroCredits: short ? 0 : 972, + shortSince: short ? new Date("2026-11-12T00:00:00Z") : null, + fellShort, + }; + }, +})); +vi.mock("@/lib/infra-standing", () => ({ + INFRA_GRACE_DAYS: 7, + STORAGE_DAY_REFERENCE_TYPE: "storage_day", + latestStorageCharges: async () => new Map(), +})); +vi.mock("@/lib/infra-storage-minimum", () => ({ + chargeMissedFirstMonths: async () => ({ charged: 0, short: 0, failed: 0 }), + shortfallsAwaitingNotice: async () => new Map(), + openFirstMonthShortfalls: async () => new Set(), + markShortfallsNotified: async () => {}, + markStorageDayNotified: async () => {}, +})); +vi.mock("@/lib/infra-auto-top-up", () => ({ runDueAutoTopUps: state.topUps })); +vi.mock("@/lib/infra-billing-emails", () => ({ + sendInfraOutOfCreditEmail: state.emails, +})); +vi.mock("@/lib/infra-delivery-billing", () => ({ + deliveryBillingStart: () => "2026-11-01", + markDeliveryDayNotified: async () => {}, + settleDeliveryDays: state.delivery, +})); + +import { + GET, + MAX_STORAGE_HOPS, + maxDuration, +} from "@/app/api/cron/developer-storage/route"; + +const ACCOUNTS = 450; +const ORIGIN = "https://cap.example"; +const PATH = "/api/cron/developer-storage"; + +type Body = { + success?: boolean; + error?: string; + date: string; + complete: boolean; + continued: boolean; + hop: number; + appsProcessed: number; + appsShort: number; +}; + +const call = async (query = "") => { + const response = await GET( + new Request(`${ORIGIN}${PATH}${query}`, { + headers: { authorization: "Bearer cron-secret" }, + }), + ); + return { status: response.status, body: (await response.json()) as Body }; +}; + +const run = async (query = "") => (await call(query)).body; + +const lastContinuation = () => { + const last = state.continuations.at(-1); + if (!last) throw new Error("no continuation was started"); + return new URL(last.url); +}; + +const followChain = async ( + first: Body, + between: () => void = () => {}, + limit = MAX_STORAGE_HOPS + 2, +) => { + const bodies = [first]; + let body = first; + while (body.continued) { + if (bodies.length > limit) throw new Error("chain did not end"); + between(); + const next = lastContinuation(); + body = await run(next.search); + bodies.push(body); + } + return bodies; +}; + +const chargedFor = (date: string) => + [...state.charged.keys()] + .filter((key) => key.endsWith(`|${date}`)) + .map((key) => key.split("|")[0]) + .sort(); + +const callsFor = (date: string) => + state.chargeCalls.filter((call) => call.dateKey === date); + +const allIds = () => state.accounts.map((account) => account.id); + +const settle = (date: string) => { + for (const id of allIds()) state.charged.set(`${id}|${date}`, 1); +}; + +beforeEach(() => { + process.env.CRON_SECRET = "cron-secret"; + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-11-12T06:41:00Z")); + state.accounts = Array.from({ length: ACCOUNTS }, (_, index) => { + const serial = String(index).padStart(4, "0"); + return { + id: `account-${serial}`, + appId: `app-${serial}`, + createdAt: new Date("2026-10-01T00:00:00Z"), + }; + }); + state.appCreatedAt = new Map(); + state.videos = new Map(); + state.markers = new Map(); + state.markerInserts = []; + state.usageReads = []; + state.deletedApps = new Set(); + state.unpaid = new Set(); + state.short = new Set(); + state.everShort = new Set(); + state.charged = new Map(); + state.snapshotWrites = new Map(); + settle("2026-11-11"); + state.chargeCalls = []; + state.accountReads = []; + state.accountSql = []; + state.longestIdList = 0; + state.secondsPerCharge = 0; + state.afters = []; + state.continuations = []; + state.continuationStatus = 200; + state.emails.mockReset(); + state.topUps.mockReset(); + state.topUps.mockResolvedValue({ attempted: 0, succeeded: 0, failed: 0 }); + vi.stubGlobal( + "fetch", + vi.fn(async (url: URL | string, init?: RequestInit) => { + state.continuations.push({ + url: String(url), + authorization: new Headers(init?.headers).get("authorization"), + }); + if (state.continuationStatus === "network") { + throw new TypeError("fetch failed"); + } + return new Response(null, { status: state.continuationStatus }); + }), + ); +}); + +afterEach(() => { + vi.useRealTimers(); + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe("storage cron paging", () => { + it("reads accounts a page at a time, by id, for yesterday and then today, and never sends every id at once", async () => { + const body = await run(); + + expect(body).toMatchObject({ + date: "2026-11-12", + complete: true, + continued: false, + appsProcessed: ACCOUNTS, + }); + const pages = [ + { cursor: null, limit: 200 }, + { cursor: "account-0199", limit: 200 }, + { cursor: "account-0399", limit: 200 }, + ]; + expect(state.accountReads).toEqual([...pages, ...pages]); + expect(state.longestIdList).toBe(200); + expect(state.continuations).toEqual([]); + expect(callsFor("2026-11-11")).toEqual([]); + }); + + it("pages on the account id alone and leaves out accounts of deleted apps", async () => { + state.deletedApps = new Set(["app-0003", "app-0250"]); + + const body = await run(); + + expect(body).toMatchObject({ complete: true, appsProcessed: ACCOUNTS - 2 }); + expect(chargedFor("2026-11-12")).not.toContain("account-0003"); + expect(chargedFor("2026-11-12")).not.toContain("account-0250"); + expect(state.accountSql.every((sql) => !sql.includes("join"))).toBe(true); + }); +}); + +describe("a cut-off sweep continues itself", () => { + it("hands the date, the last attempted account and the next hop to a request to itself", async () => { + state.secondsPerCharge = 1.6; + const startedAt = Date.now(); + + const body = await run(); + + expect(body).toMatchObject({ + date: "2026-11-12", + complete: false, + continued: true, + hop: 0, + appsProcessed: 130, + }); + expect(Date.now() - startedAt).toBeLessThan(maxDuration * 1000); + expect(state.afters).toHaveLength(1); + await Promise.all(state.afters); + expect(state.continuations).toHaveLength(1); + const next = lastContinuation(); + expect(next.origin).toBe(ORIGIN); + expect(next.pathname).toBe(PATH); + expect(Object.fromEntries(next.searchParams)).toEqual({ + date: "2026-11-12", + after: "account-0129", + hop: "1", + }); + expect(state.continuations[0]?.authorization).toBe("Bearer cron-secret"); + }); + + it("resumes a page cut off part way at the first account it did not attempt", async () => { + state.secondsPerCharge = 1.6; + await run(); + const attempted = state.chargeCalls.map((call) => call.accountId); + state.chargeCalls = []; + state.accountReads = []; + state.secondsPerCharge = 0; + + const body = await run(lastContinuation().search); + + expect(attempted.at(-1)).toBe("account-0129"); + expect(state.chargeCalls[0]?.accountId).toBe("account-0130"); + expect(state.accountReads).toEqual([ + { cursor: "account-0129", limit: 200 }, + { cursor: "account-0329", limit: 200 }, + ]); + expect(body).toMatchObject({ + complete: true, + continued: false, + hop: 1, + appsProcessed: ACCOUNTS - 130, + }); + expect(chargedFor("2026-11-12")).toEqual(allIds()); + }); + + it("never charges again an account an earlier hop charged", async () => { + state.secondsPerCharge = 1.6; + const bodies = await followChain(await run()); + + expect(bodies.length).toBeGreaterThan(2); + expect(bodies.at(-1)).toMatchObject({ complete: true, continued: false }); + const attempted = callsFor("2026-11-12").map((call) => call.accountId); + expect(new Set(attempted).size).toBe(attempted.length); + expect(chargedFor("2026-11-12")).toEqual(allIds()); + expect(bodies.reduce((total, body) => total + body.appsProcessed, 0)).toBe( + ACCOUNTS, + ); + }); + + it("finishes the day a continuation started even after midnight, then sweeps the new day", async () => { + vi.setSystemTime(new Date("2026-11-12T23:56:00Z")); + state.secondsPerCharge = 1.6; + + const first = await run(); + expect(first).toMatchObject({ date: "2026-11-12", continued: true }); + vi.setSystemTime(new Date("2026-11-13T00:02:00Z")); + state.secondsPerCharge = 0; + + const bodies = await followChain(first); + + expect(bodies.at(-1)).toMatchObject({ + date: "2026-11-13", + complete: true, + }); + expect(chargedFor("2026-11-12")).toEqual(allIds()); + expect(chargedFor("2026-11-13")).toEqual(allIds()); + expect(callsFor("2026-11-12")).toHaveLength(ACCOUNTS); + expect(callsFor("2026-11-13")).toHaveLength(ACCOUNTS); + }); + + it("reaches funded accounts behind a run of unpaid accounts longer than one run's budget", async () => { + state.unpaid = new Set(allIds().slice(0, 300)); + state.secondsPerCharge = 1.6; + + const bodies = await followChain(await run()); + + const funded = allIds().slice(300); + expect(chargedFor("2026-11-12")).toEqual(funded); + expect(bodies.at(-1)).toMatchObject({ complete: true }); + const unpaidCalls = callsFor("2026-11-12").filter((call) => + state.unpaid.has(call.accountId), + ); + expect(unpaidCalls).toHaveLength(300); + expect(new Set(unpaidCalls.map((call) => call.accountId)).size).toBe(300); + }); +}); + +describe("a broken chain is finished by the next day's first run", () => { + const breakOnLastTickOfTheDay = async () => { + vi.setSystemTime(new Date("2026-11-12T18:41:00Z")); + state.secondsPerCharge = 1.6; + state.continuationStatus = "network"; + vi.spyOn(console, "error").mockImplementation(() => {}); + const lastTick = await run(); + await Promise.all(state.afters); + expect(lastTick).toMatchObject({ + date: "2026-11-12", + complete: false, + continued: true, + }); + expect(console.error).toHaveBeenCalledWith( + "[developer-storage] Continuation not started", + expect.objectContaining({ date: "2026-11-12", after: "account-0129" }), + ); + const firstDayCalls = callsFor("2026-11-12").map((call) => call.accountId); + vi.setSystemTime(new Date("2026-11-13T00:41:00Z")); + state.secondsPerCharge = 0; + state.continuationStatus = 200; + state.chargeCalls = []; + return firstDayCalls; + }; + + it("charges the accounts after the cursor for that day, each once, then sweeps today", async () => { + const chargedBeforeTheBreak = await breakOnLastTickOfTheDay(); + + const nextDay = await run(); + + expect(nextDay).toMatchObject({ + date: "2026-11-13", + complete: true, + continued: false, + hop: 0, + appsProcessed: ACCOUNTS - 130 + ACCOUNTS, + }); + expect(chargedFor("2026-11-12")).toEqual(allIds()); + expect(chargedFor("2026-11-13")).toEqual(allIds()); + const finished = callsFor("2026-11-12").map((call) => call.accountId); + expect(finished).toEqual(allIds().slice(130)); + expect(finished.some((id) => chargedBeforeTheBreak.includes(id))).toBe( + false, + ); + expect( + callsFor("2026-11-12").every((call) => call.retryUnpaid === false), + ).toBe(true); + expect( + callsFor("2026-11-13").every((call) => call.retryUnpaid === true), + ).toBe(true); + }); + + it("does not charge or email again an account that fell short before the break", async () => { + state.short = new Set(["account-0005", "account-0006", "account-0200"]); + await breakOnLastTickOfTheDay(); + expect(state.emails).toHaveBeenCalledTimes(2); + + await run(); + + expect(state.emails).toHaveBeenCalledTimes(3); + const emailed = state.emails.mock.calls.map( + ([notice]) => (notice as { accountId: string }).accountId, + ); + expect(emailed.sort()).toEqual([ + "account-0005", + "account-0006", + "account-0200", + ]); + expect(callsFor("2026-11-12").map((call) => call.accountId)).not.toContain( + "account-0005", + ); + }); + + it("writes no second snapshot for an account that was unpaid before the break", async () => { + state.unpaid = new Set([ + "account-0010", + "account-0011", + "account-0300", + "account-0301", + ]); + await breakOnLastTickOfTheDay(); + expect(state.snapshotWrites.get("app-0010|2026-11-12")).toBe(1); + expect(state.snapshotWrites.has("app-0300|2026-11-12")).toBe(false); + + await run(); + + const finished = callsFor("2026-11-12").map((call) => call.accountId); + expect(finished).not.toContain("account-0010"); + expect(finished).not.toContain("account-0011"); + expect(finished).toContain("account-0300"); + for (const app of ["app-0010", "app-0011", "app-0300", "app-0301"]) { + expect(state.snapshotWrites.get(`${app}|2026-11-12`)).toBe(1); + } + expect(state.snapshotWrites.get("app-0010|2026-11-13")).toBe(1); + }); + + it("costs a settled yesterday one read per table per page and no charge", async () => { + settle("2026-11-12"); + vi.setSystemTime(new Date("2026-11-13T00:41:00Z")); + state.unpaid = new Set(allIds().slice(0, 50)); + for (const id of allIds().slice(0, 50)) { + state.charged.delete(`${id}|2026-11-12`); + state.snapshotWrites.set(`${id.replace("account", "app")}|2026-11-12`, 1); + } + + await run(); + + expect(callsFor("2026-11-12")).toEqual([]); + for (const id of allIds().slice(0, 50)) { + expect( + state.snapshotWrites.get(`${id.replace("account", "app")}|2026-11-12`), + ).toBe(1); + } + }); +}); + +describe("the finishing pass counts only what existed on that day", () => { + const nextDayTick = () => { + vi.setSystemTime(new Date("2026-11-13T00:41:00Z")); + settle("2026-11-12"); + for (const id of [ + "account-0300", + "account-0301", + "account-0302", + "account-0303", + ]) { + state.charged.delete(`${id}|2026-11-12`); + } + }; + const account = (id: string) => { + const found = state.accounts.find((row) => row.id === id); + if (!found) throw new Error(`no ${id}`); + return found; + }; + + it("does not charge yesterday for an account whose first video arrived after midnight", async () => { + nextDayTick(); + state.videos.set("app-0300", [new Date("2026-11-13T00:30:00Z")]); + + await run(); + + expect(chargedFor("2026-11-12")).not.toContain("account-0300"); + expect(callsFor("2026-11-12").map((call) => call.accountId)).not.toContain( + "account-0300", + ); + expect(chargedFor("2026-11-13")).toContain("account-0300"); + }); + + it("does not charge yesterday for an account or app created after midnight", async () => { + nextDayTick(); + account("account-0301").createdAt = new Date("2026-11-13T00:10:00Z"); + state.appCreatedAt.set("app-0302", new Date("2026-11-13T00:10:00Z")); + + await run(); + + const finished = callsFor("2026-11-12").map((call) => call.accountId); + expect(finished).not.toContain("account-0301"); + expect(finished).not.toContain("account-0302"); + expect(finished.sort()).toEqual(["account-0300", "account-0303"]); + expect(chargedFor("2026-11-13")).toEqual( + expect.arrayContaining(["account-0301", "account-0302"]), + ); + }); + + it("leaves a video created after midnight out of yesterday's charge and keeps the earlier ones", async () => { + nextDayTick(); + state.videos.set("app-0303", [ + new Date("2026-11-05T00:00:00Z"), + new Date("2026-11-12T23:59:59Z"), + new Date("2026-11-13T00:20:00Z"), + ]); + + await run(); + + const [yesterday] = callsFor("2026-11-12").filter( + (call) => call.accountId === "account-0303", + ); + const [today] = callsFor("2026-11-13").filter( + (call) => call.accountId === "account-0303", + ); + expect(yesterday?.dailyMicroCredits).toBe(56); + expect(today?.dailyMicroCredits).toBe(84); + }); + + it("bounds only the finishing pass, at the midnight after that day", async () => { + nextDayTick(); + + await run(); + + const bounds = state.usageReads.map((read) => + read.createdBefore?.toISOString(), + ); + expect(bounds.filter(Boolean)).toEqual(["2026-11-13T00:00:00.000Z"]); + expect(bounds.filter((bound) => bound === undefined)).toHaveLength(3); + }); +}); + +describe("a visit on a day leaves a marker when nothing is charged", () => { + const tick = (at: string) => { + vi.setSystemTime(new Date(at)); + return run(); + }; + const insertsFor = (app: string, date: string) => + state.markerInserts.filter((key) => key === `${app}|${date}`).length; + + it("does not charge a day for an app that was empty at every run of it, when its first video becomes ready after midnight", async () => { + state.videos.set("app-0300", [ + { + createdAt: new Date("2026-11-12T23:00:00Z"), + readyAt: new Date("2026-11-13T00:20:00Z"), + }, + ]); + for (const at of [ + "2026-11-12T06:41:00Z", + "2026-11-12T12:41:00Z", + "2026-11-12T18:41:00Z", + ]) { + await tick(at); + } + expect(state.markers.has("app-0300|2026-11-12")).toBe(true); + + await tick("2026-11-13T00:41:00Z"); + + expect(callsFor("2026-11-12").map((call) => call.accountId)).not.toContain( + "account-0300", + ); + expect(chargedFor("2026-11-12")).not.toContain("account-0300"); + expect(chargedFor("2026-11-13")).toContain("account-0300"); + }); + + it("charges a video that becomes ready after the last run of its day from the next day, as main does", async () => { + state.videos.set("app-0300", [ + { + createdAt: new Date("2026-11-12T19:00:00Z"), + readyAt: new Date("2026-11-12T19:05:00Z"), + }, + ]); + for (const at of [ + "2026-11-12T06:41:00Z", + "2026-11-12T12:41:00Z", + "2026-11-12T18:41:00Z", + ]) { + await tick(at); + } + expect(state.markers.has("app-0300|2026-11-12")).toBe(true); + + await tick("2026-11-13T00:41:00Z"); + await tick("2026-11-13T06:41:00Z"); + + expect(callsFor("2026-11-12").map((call) => call.accountId)).not.toContain( + "account-0300", + ); + expect(chargedFor("2026-11-12")).not.toContain("account-0300"); + expect( + callsFor("2026-11-13").filter( + (call) => call.accountId === "account-0300", + ), + ).toEqual([expect.objectContaining({ dailyMicroCredits: 28 })]); + expect(chargedFor("2026-11-13")).toContain("account-0300"); + }); + + it("charges a day for an app the sweep never reached that day, for videos created before midnight", async () => { + state.videos.set("app-0300", [ + new Date("2026-11-12T10:00:00Z"), + { + createdAt: new Date("2026-11-13T00:05:00Z"), + readyAt: new Date("2026-11-13T00:06:00Z"), + }, + ]); + vi.setSystemTime(new Date("2026-11-12T18:41:00Z")); + state.secondsPerCharge = 1.6; + state.continuationStatus = "network"; + vi.spyOn(console, "error").mockImplementation(() => {}); + const lastTick = await run(); + await Promise.all(state.afters); + expect(lastTick.continued).toBe(true); + expect(state.markers.has("app-0300|2026-11-12")).toBe(false); + state.secondsPerCharge = 0; + + await tick("2026-11-13T00:41:00Z"); + + const [finished] = callsFor("2026-11-12").filter( + (call) => call.accountId === "account-0300", + ); + expect(finished?.dailyMicroCredits).toBe(28); + expect(chargedFor("2026-11-12")).toContain("account-0300"); + const [today] = callsFor("2026-11-13").filter( + (call) => call.accountId === "account-0300", + ); + expect(today?.dailyMicroCredits).toBe(56); + }); + + it("writes one marker per idle app and day, however often the app is visited", async () => { + const idle = ["app-0400", "app-0401", "app-0402", "app-0403", "app-0404"]; + for (const app of idle) state.videos.set(app, []); + + for (const at of [ + "2026-11-12T06:41:00Z", + "2026-11-12T12:41:00Z", + "2026-11-12T18:41:00Z", + "2026-11-13T00:41:00Z", + "2026-11-13T06:41:00Z", + ]) { + await tick(at); + } + + for (const app of idle) { + expect(insertsFor(app, "2026-11-12")).toBe(1); + expect(insertsFor(app, "2026-11-13")).toBe(1); + expect(state.markers.get(`${app}|2026-11-12`)).toBe(1); + } + expect(state.markerInserts).toHaveLength(idle.length * 2); + expect( + state.chargeCalls.filter((call) => + idle.includes(call.accountId.replace("account", "app")), + ), + ).toEqual([]); + }); + + it("still charges the same day when storage arrives after an idle marker", async () => { + state.videos.set("app-0410", [ + { + createdAt: new Date("2026-11-12T09:00:00Z"), + readyAt: new Date("2026-11-12T10:00:00Z"), + }, + ]); + await tick("2026-11-12T06:41:00Z"); + expect(state.markers.has("app-0410|2026-11-12")).toBe(true); + expect(chargedFor("2026-11-12")).not.toContain("account-0410"); + + await tick("2026-11-12T12:41:00Z"); + + const charged = callsFor("2026-11-12").filter( + (call) => call.accountId === "account-0410", + ); + expect(charged).toEqual([ + expect.objectContaining({ dailyMicroCredits: 28, retryUnpaid: true }), + ]); + expect(chargedFor("2026-11-12")).toContain("account-0410"); + expect(state.snapshotWrites.get("app-0410|2026-11-12")).toBe(1); + expect(state.markers.has("app-0410|2026-11-12")).toBe(false); + }); +}); + +describe("the chain is bounded", () => { + it("stops at the hop limit and logs it", async () => { + state.secondsPerCharge = 1.6; + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + + const body = await run( + `?date=2026-11-12&after=account-0099&hop=${MAX_STORAGE_HOPS}`, + ); + + expect(body).toMatchObject({ + complete: false, + continued: false, + hop: MAX_STORAGE_HOPS, + }); + expect(state.afters).toEqual([]); + expect(state.continuations).toEqual([]); + expect(errors).toHaveBeenCalledWith( + "[developer-storage] Sweep stopped at the hop limit", + expect.objectContaining({ date: "2026-11-12", hop: MAX_STORAGE_HOPS }), + ); + }); + + it("logs a continuation the route refused", async () => { + state.secondsPerCharge = 1.6; + state.continuationStatus = 500; + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + + await run(); + await Promise.all(state.afters); + + expect(errors).toHaveBeenCalledWith( + "[developer-storage] Continuation refused", + expect.objectContaining({ status: 500, hop: 1 }), + ); + }); + + it("logs a handoff that is still running, with the next cursor, as information", async () => { + state.secondsPerCharge = 1.6; + vi.stubGlobal( + "fetch", + vi.fn(async () => { + throw new DOMException("The operation timed out.", "TimeoutError"); + }), + ); + const info = vi.spyOn(console, "info").mockImplementation(() => {}); + const errors = vi.spyOn(console, "error").mockImplementation(() => {}); + + await run(); + await Promise.all(state.afters); + + expect(info).toHaveBeenCalledWith( + "[developer-storage] Continuation still running at handoff", + { date: "2026-11-12", after: "account-0129", hop: 1 }, + ); + expect(errors).not.toHaveBeenCalled(); + }); + + it("refuses a date other than today or yesterday and any malformed continuation", async () => { + for (const query of [ + "?date=2026-11-10&hop=1", + "?date=2026-11-13&hop=1", + "?date=2026-1-12&hop=1", + "?date=yesterday&hop=1", + "?date=2026-11-12", + "?hop=1", + "?after=account-0001", + "?date=2026-11-12&hop=0", + `?date=2026-11-12&hop=${MAX_STORAGE_HOPS + 1}`, + "?date=2026-11-12&hop=1&after=a%20b", + "?date=2026-11-12&hop=1x", + ]) { + const { status, body } = await call(query); + expect({ query, status, error: body.error }).toEqual({ + query, + status: 400, + error: "Invalid continuation", + }); + } + expect(state.chargeCalls).toEqual([]); + expect(state.topUps).not.toHaveBeenCalled(); + expect((await call("?date=2026-11-11&hop=1")).status).toBe(200); + }); +}); + +describe("auto top-ups run once per sweep", () => { + it("runs before charging on the scheduled run and after charging on the last hop only", async () => { + state.secondsPerCharge = 1.6; + const startedAt = Date.now(); + + const first = await run(); + expect(state.topUps).toHaveBeenCalledTimes(1); + const [[, before]] = state.topUps.mock.calls as [[Date, { until: number }]]; + expect(before.until - startedAt).toBeGreaterThan(0); + expect(before.until - startedAt).toBeLessThan(maxDuration * 1000); + + state.topUps.mockClear(); + const second = await run(lastContinuation().search); + expect(second.continued).toBe(true); + expect(state.topUps).not.toHaveBeenCalled(); + + state.secondsPerCharge = 0; + const bodies = await followChain(second); + expect(bodies.at(-1)).toMatchObject({ complete: true }); + expect(state.topUps).toHaveBeenCalledTimes(1); + expect(first.continued).toBe(true); + }); + + it("runs both passes on a scheduled run that finishes", async () => { + await run(); + + expect(state.topUps).toHaveBeenCalledTimes(2); + }); +}); + +describe("delivery days settle once per sweep", () => { + it("settles after storage on the last hop only, inside the run's time budget", async () => { + state.secondsPerCharge = 1.6; + const order: string[] = []; + state.delivery.mockImplementation(async () => { + order.push("delivery"); + return { billed: 0 }; + }); + state.topUps.mockImplementation(async () => { + order.push("top-up"); + return { attempted: 0, succeeded: 0, failed: 0 }; + }); + + const first = await run(); + expect(first.continued).toBe(true); + expect(state.delivery).not.toHaveBeenCalled(); + + state.secondsPerCharge = 0; + const bodies = await followChain(first); + expect(bodies.at(-1)).toMatchObject({ + complete: true, + delivery: { billed: 0 }, + }); + expect(state.delivery).toHaveBeenCalledTimes(1); + expect(order.slice(-2)).toEqual(["delivery", "top-up"]); + const [[options]] = state.delivery.mock.calls as [ + [{ startDay: string; outOfTime: () => boolean; now: Date }], + ]; + expect(options.startDay).toBe("2026-11-01"); + expect(options.outOfTime()).toBe(false); + }); +}); diff --git a/apps/web/__tests__/unit/infra-storage-latest-day.test.ts b/apps/web/__tests__/unit/infra-storage-latest-day.test.ts new file mode 100644 index 00000000000..ddefa622d16 --- /dev/null +++ b/apps/web/__tests__/unit/infra-storage-latest-day.test.ts @@ -0,0 +1,130 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const state = vi.hoisted(() => ({ + queries: [] as { sql: string; params: unknown[] }[], + hold: null as Promise | null, +})); + +vi.mock("@cap/database", async () => { + const { drizzle } = await import("drizzle-orm/mysql-proxy"); + const client = drizzle(async (sql, params) => { + state.queries.push({ sql, params }); + if (state.hold) await state.hold; + return { rows: [] }; + }); + return { db: () => client }; +}); + +import { db } from "@cap/database"; +import { + latestStorageCharge, + latestStorageCharges, +} from "@/lib/infra-standing"; + +const LATEST_DAY = + "(select `developer_credit_transactions`.`accountId`, `developer_credit_transactions`.`referenceId` from `developer_credit_transactions` where `developer_credit_transactions`.`accountId` = ? and `developer_credit_transactions`.`referenceType` = ? and `developer_credit_transactions`.`referenceId` <= ? order by `developer_credit_transactions`.`referenceId` desc limit 1)"; + +beforeEach(() => { + state.queries = []; + state.hold = null; + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-11-13T23:59:59Z")); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("latest storage day", () => { + it("orders one account's storage day rows by the day they bill, up to tomorrow", async () => { + await latestStorageCharge(db() as never, "account-1"); + + expect(state.queries).toHaveLength(2); + const [firstMonth, { sql, params }] = state.queries as [ + { sql: string; params: unknown[] }, + { sql: string; params: unknown[] }, + ]; + expect(firstMonth.params).toEqual(["account-1", "video_create", 1]); + expect(firstMonth.sql).not.toContain("referenceId"); + expect(sql).toBe( + "select `id`, `createdAt`, `metadata` from `developer_credit_transactions` where (`developer_credit_transactions`.`accountId` = ? and (`developer_credit_transactions`.`type` = ? and `developer_credit_transactions`.`referenceType` = ?) and `developer_credit_transactions`.`referenceId` <= ?) order by `developer_credit_transactions`.`referenceId` desc, `developer_credit_transactions`.`id` desc limit ?", + ); + expect(params).toEqual([ + "account-1", + "storage_daily", + "storage_day", + "2026-11-14", + 1, + ]); + }); + + it("reads the latest delivery day alongside it only while delivery billing has a start day", async () => { + vi.stubEnv("CAP_INFRA_DELIVERY_BILLING_FROM", "2026-11-14"); + try { + await latestStorageCharge(db() as never, "account-1"); + } finally { + vi.unstubAllEnvs(); + } + + expect(state.queries).toHaveLength(3); + const delivery = state.queries[2] as { sql: string; params: unknown[] }; + expect(delivery.sql).toBe( + "select `id`, `createdAt`, `metadata` from `developer_credit_transactions` where ((`developer_credit_transactions`.`accountId` = ? and `developer_credit_transactions`.`type` = ? and `developer_credit_transactions`.`referenceType` = ? and JSON_EXTRACT(`developer_credit_transactions`.`metadata`, '$.duringShortfall') IS NULL) and `developer_credit_transactions`.`createdAt` = ((select `createdAt` from `developer_credit_transactions` where (`developer_credit_transactions`.`accountId` = ? and `developer_credit_transactions`.`type` = ? and `developer_credit_transactions`.`referenceType` = ? and JSON_EXTRACT(`developer_credit_transactions`.`metadata`, '$.duringShortfall') IS NULL) order by `developer_credit_transactions`.`createdAt` desc limit ?))) order by `developer_credit_transactions`.`referenceId` desc, `developer_credit_transactions`.`id` desc limit ?", + ); + expect(delivery.params).toEqual([ + "account-1", + "delivery_daily", + "delivery_day", + "account-1", + "delivery_daily", + "delivery_day", + 1, + 1, + ]); + }); + + it("finds every account's latest day in one statement, one bounded lookup per account", async () => { + await latestStorageCharges(db() as never, ["account-1", "account-2"]); + + expect(state.queries).toHaveLength(1); + const [{ sql, params }] = state.queries as [ + { sql: string; params: unknown[] }, + ]; + expect(sql).toContain( + `in (select latest.accountId, latest.referenceId from (${LATEST_DAY} union all ${LATEST_DAY}) latest)`, + ); + expect(params).toEqual([ + "storage_daily", + "storage_day", + "account-1", + "storage_day", + "2026-11-14", + "account-2", + "storage_day", + "2026-11-14", + ]); + }); + + it("reads nothing for no accounts", async () => { + expect(await latestStorageCharges(db() as never, [])).toEqual(new Map()); + expect(state.queries).toEqual([]); + }); + + it("asks for the latest first month and the latest storage day at once, not one after the other", async () => { + let release = () => {}; + state.hold = new Promise((resolve) => { + release = resolve; + }); + + const charge = latestStorageCharge(db() as never, "account-1"); + await vi.waitFor(() => expect(state.queries).toHaveLength(2)); + expect(state.queries.map(({ params }) => params[1])).toEqual([ + "video_create", + "storage_daily", + ]); + + release(); + expect(await charge).toBeNull(); + expect(state.queries).toHaveLength(2); + }); +}); diff --git a/apps/web/__tests__/unit/infra-storage-once.test.ts b/apps/web/__tests__/unit/infra-storage-once.test.ts new file mode 100644 index 00000000000..1587fb62d49 --- /dev/null +++ b/apps/web/__tests__/unit/infra-storage-once.test.ts @@ -0,0 +1,136 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const holder = vi.hoisted(() => ({ db: null as unknown })); + +vi.mock("@cap/database", () => ({ db: () => holder.db })); +vi.mock("@cap/database/helpers", () => ({ + nanoId: () => "row", + nanoIdLength: 15, +})); + +import { fakeDb, ledger, resetLedger } from "./infra-billing-fake-db"; + +holder.db = fakeDb; + +import { + billableVideo, + firstMonthStorageMicroCredits, +} from "@/lib/developer-pricing"; +import { getInfraBilling, infraBillingJson } from "@/lib/infra-standing"; +import { getAppStorage, getAppStorageUsage } from "@/lib/infra-storage"; + +const newVideo = billableVideo({ + duration: 900, + width: 3840, + height: 2160, + maxResolutionTier: "2160p", +}); + +const videoReads = (select: { mock: { results: { value: unknown }[] } }) => + select.mock.results.length; + +beforeEach(() => { + resetLedger(); + ledger.account.balanceMicroCredits = 40_000; + ledger.videos = [ + { + videoId: "v1", + appId: "app-1", + width: 1280, + height: 720, + seconds: 61.5, + count: 3, + }, + { + videoId: "v2", + appId: "app-1", + width: 2560, + height: 1440, + seconds: 600.25, + count: 2, + }, + { + videoId: "v3", + appId: "app-1", + width: 3840, + height: 2160, + seconds: 1234.125, + count: 1, + maxResolutionTier: "1080p", + }, + { + videoId: "v4", + appId: "app-1", + width: 1920, + height: 1080, + seconds: 300, + count: 4, + status: "processing", + }, + { + videoId: "v5", + appId: "app-1", + width: 1920, + height: 1080, + seconds: 77, + count: 1, + status: "error", + }, + ]; +}); + +describe("one read of stored video serves both totals", () => { + it("gives the same stored, daily and first month totals as separate reads", async () => { + for (const [video, options] of [ + [undefined, {}], + [newVideo, {}], + [newVideo, { excludeVideoId: "v2" }], + [undefined, { excludeVideoId: "v4" }], + ] as const) { + const storage = await getAppStorage("app-1", video, undefined, options); + + expect(storage.stored).toEqual( + await getAppStorageUsage("app-1", undefined, undefined, options), + ); + expect(storage.dailyCharge).toEqual( + await getAppStorageUsage("app-1", undefined, undefined, { + ...options, + dailyChargeAt: new Date(ledger.now), + }), + ); + expect(storage.firstMonthMicroCredits).toBe( + (options.excludeVideoId === "v4" ? 0 : 425) + + (video ? firstMonthStorageMicroCredits(video) : 0), + ); + } + }); + + it("keeps processing video out of what is shown as stored and in what admission counts", async () => { + const { stored, firstMonthMicroCredits } = await getAppStorage("app-1"); + + expect(stored.videoCount).toBe(6); + expect(firstMonthMicroCredits).toBe(5 * 85); + }); + + it("reads video once for billing and returns what the pages show", async () => { + const now = new Date(ledger.now); + const select = vi.spyOn(fakeDb, "select"); + const onVideos = () => + select.mock.calls.filter(([fields]) => "seconds" in fields).length; + + const billing = await getInfraBilling("app-1", undefined, now); + + expect(onVideos()).toBe(1); + expect(videoReads(select)).toBeGreaterThan(1); + select.mockRestore(); + expect(billing.stored).toEqual(await getAppStorageUsage("app-1")); + expect(billing.storage.dailyCharge).toEqual( + await getAppStorageUsage("app-1", undefined, undefined, { + dailyChargeAt: now, + }), + ); + expect(infraBillingJson(billing).storage.minutesByTier).toEqual( + billing.stored.minutesByTier, + ); + }); +}); diff --git a/apps/web/__tests__/unit/infra-storage-snapshot-readers.test.ts b/apps/web/__tests__/unit/infra-storage-snapshot-readers.test.ts new file mode 100644 index 00000000000..b52178485df --- /dev/null +++ b/apps/web/__tests__/unit/infra-storage-snapshot-readers.test.ts @@ -0,0 +1,149 @@ +import { User } from "@cap/web-domain"; +import { Effect } from "effect"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +type Snapshot = { + snapshotDate: string; + totalDurationMinutes: number; + videoCount: number; + microCreditsCharged: number; +}; + +const state = vi.hoisted(() => ({ + queries: [] as string[], + snapshots: [] as { + snapshotDate: string; + totalDurationMinutes: number; + videoCount: number; + microCreditsCharged: number; + }[], +})); + +vi.mock("server-only", () => ({})); +vi.mock("@cap/database", async () => { + const { drizzle } = await import("drizzle-orm/mysql-proxy"); + const STAMP = "2026-11-01 00:00:00"; + const client = drizzle(async (sql, params) => { + state.queries.push(sql); + if (sql.includes("from `developer_daily_storage_snapshots`")) { + const skipsZero = sql.includes( + "`developer_daily_storage_snapshots`.`totalDurationMinutes` > ?", + ); + const since = params.find( + (value) => + typeof value === "string" && /^\d{4}-\d{2}-\d{2}$/.test(value), + ); + const rows = state.snapshots + .filter((row) => !skipsZero || row.totalDurationMinutes > 0) + .filter((row) => !since || row.snapshotDate >= String(since)) + .sort((a, b) => b.snapshotDate.localeCompare(a.snapshotDate)); + if ( + sql.startsWith( + "select `developer_daily_storage_snapshots`.`totalDurationMinutes`", + ) || + sql.startsWith("select `totalDurationMinutes` from") + ) { + return { + rows: rows.slice(0, 1).map((row) => [row.totalDurationMinutes]), + }; + } + return { + rows: rows.map((row) => [ + row.snapshotDate, + row.totalDurationMinutes, + row.videoCount, + row.microCreditsCharged, + ]), + }; + } + if (sql.includes("from `developer_apps`")) { + return { + rows: [["app-1", "First app", "production", null, STAMP, STAMP]], + }; + } + if (sql.includes("from `developer_credit_accounts`")) { + return { rows: [[5_000_000, 0, 0, 0, 0]] }; + } + if (sql.includes("COUNT(*)")) return { rows: [[3]] }; + return { rows: [] }; + }); + return { db: () => client }; +}); + +import { getInfraStorageSnapshots } from "@/app/(infra)/dashboard/infra/infra-data"; +import { summarizeStorageMonth } from "@/lib/infra-usage"; +import { AgentManagement } from "../../../../packages/web-backend/src/AgentManagement"; + +const charged = (snapshotDate: string, minutes: number): Snapshot => ({ + snapshotDate, + totalDurationMinutes: minutes, + videoCount: 2, + microCreditsCharged: 120, +}); +const unpaid = (snapshotDate: string): Snapshot => ({ + snapshotDate, + totalDurationMinutes: 30, + videoCount: 2, + microCreditsCharged: 0, +}); +const marker = (snapshotDate: string): Snapshot => ({ + snapshotDate, + totalDurationMinutes: 0, + videoCount: 0, + microCreditsCharged: 0, +}); + +const appContext = () => + Effect.runPromise( + Effect.gen(function* () { + const agents = yield* AgentManagement; + return yield* agents.getDeveloperAppContext( + User.UserId.make("user-1"), + "app-1", + ); + }).pipe(Effect.provide(AgentManagement.Default)), + ); + +beforeEach(() => { + state.queries = []; + state.snapshots = [ + charged("2026-11-09", 40), + unpaid("2026-11-10"), + charged("2026-11-11", 45), + ]; +}); + +describe("readers of daily storage snapshots ignore visit markers", () => { + it("the console usage history and month summary are the same with markers on other days", async () => { + const now = new Date("2026-11-13T12:00:00Z"); + const before = await getInfraStorageSnapshots("app-1", "2026-10-01"); + state.snapshots.push(marker("2026-11-12"), marker("2026-11-13")); + + const after = await getInfraStorageSnapshots("app-1", "2026-10-01"); + + expect(after).toEqual(before); + expect(after.map((row) => row.snapshotDate)).toEqual([ + "2026-11-11", + "2026-11-10", + "2026-11-09", + ]); + expect(summarizeStorageMonth(after, now)).toEqual( + summarizeStorageMonth(before, now), + ); + expect(summarizeStorageMonth(after, now)).toEqual({ + chargedMicroCredits: 240, + billedDays: 3, + }); + }); + + it("the agent API reports the last stored minutes, not a later marker", async () => { + const before = await appContext(); + state.snapshots.push(marker("2026-11-12")); + + const after = await appContext(); + + expect(before.storageMinutes).toBe(45); + expect(after.storageMinutes).toBe(45); + expect(after).toEqual(before); + }); +}); diff --git a/apps/web/__tests__/unit/infra-storage-visit-marker.test.ts b/apps/web/__tests__/unit/infra-storage-visit-marker.test.ts new file mode 100644 index 00000000000..0b0ac5ff01f --- /dev/null +++ b/apps/web/__tests__/unit/infra-storage-visit-marker.test.ts @@ -0,0 +1,68 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const state = vi.hoisted(() => ({ + queries: [] as { sql: string; params: unknown[] }[], +})); + +vi.mock("@cap/database", async () => { + const { drizzle } = await import("drizzle-orm/mysql-proxy"); + const client = drizzle(async (sql, params) => { + state.queries.push({ sql, params }); + return { rows: [{ affectedRows: 0, insertId: 0 }] }; + }); + return { db: () => client }; +}); +vi.mock("@cap/database/helpers", () => ({ + nanoId: () => "marker-id", + nanoIdLength: 15, +})); + +import { recordStorageDayVisits } from "@/lib/infra-storage-billing"; + +beforeEach(() => { + state.queries = []; +}); + +describe("storage day visit markers", () => { + it("writes every visited app of a page in one statement and changes nothing that exists", async () => { + await recordStorageDayVisits({ + appIds: ["app-1", "app-2"], + dateKey: "2026-11-12", + now: new Date("2026-11-12T06:41:00Z"), + }); + + expect(state.queries).toHaveLength(1); + const [{ sql, params }] = state.queries as [ + { sql: string; params: unknown[] }, + ]; + expect(sql).toMatch( + /^insert into `developer_daily_storage_snapshots` .* values \(.*\), \(.*\) on duplicate key update `appId` = `developer_daily_storage_snapshots`\.`appId`$/, + ); + expect(params).toEqual([ + "marker-id", + "app-1", + "2026-11-12", + 0, + 0, + 0, + "2026-11-12 06:41:00.000", + "marker-id", + "app-2", + "2026-11-12", + 0, + 0, + 0, + "2026-11-12 06:41:00.000", + ]); + }); + + it("does nothing for a page with no idle visits", async () => { + await recordStorageDayVisits({ + appIds: [], + dateKey: "2026-11-12", + now: new Date("2026-11-12T06:41:00Z"), + }); + + expect(state.queries).toEqual([]); + }); +}); diff --git a/apps/web/__tests__/unit/infra-supported-formats.test.ts b/apps/web/__tests__/unit/infra-supported-formats.test.ts new file mode 100644 index 00000000000..3bf695100bc --- /dev/null +++ b/apps/web/__tests__/unit/infra-supported-formats.test.ts @@ -0,0 +1,78 @@ +import fs from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { developerOpenApiSpec } from "@/lib/developer-openapi"; +import { + INFRA_SUPPORTED_VIDEO_FORMATS, + INFRA_UPLOAD_CONTENT_TYPES, + INFRA_VIDEO_FILE_ACCEPT, + isRefusedUploadContentType, +} from "@/lib/infra-video-formats"; +import { SUPPORTED_VIDEO_FORMATS as MCP_SUPPORTED_VIDEO_FORMATS } from "../../../../packages/infra-mcp/src/files"; + +const SENTENCE = + "MP4, MOV, MKV and WebM files with H.264, HEVC, VP8, VP9, AV1, ProRes, MPEG-4 or Motion JPEG video"; + +const doc = (name: string) => + fs.readFileSync( + path.join(process.cwd(), "content/docs/api", `${name}.mdx`), + "utf8", + ); + +describe("supported video formats", () => { + it("is one sentence, word for word", () => { + expect(INFRA_SUPPORTED_VIDEO_FORMATS).toBe(SENTENCE); + expect(MCP_SUPPORTED_VIDEO_FORMATS).toBe(SENTENCE); + }); + + it.each(["rest-api", "get-started"])("appears verbatim in %s.mdx", (name) => { + expect(doc(name)).toContain(SENTENCE); + }); + + it("puts the note under Create Upload in the reference", () => { + const reference = doc("rest-api"); + const createUpload = reference.indexOf("### Create Upload"); + const note = reference.indexOf(`Cap Infra accepts ${SENTENCE}.`); + const nextSection = reference.indexOf("\n### ", createUpload + 1); + expect(createUpload).toBeGreaterThan(-1); + expect(note).toBeGreaterThan(createUpload); + expect(note).toBeLessThan(nextSection); + }); + + it("no longer offers AVI as an upload content type", () => { + expect(INFRA_UPLOAD_CONTENT_TYPES).not.toContain("video/avi"); + expect(doc("rest-api")).not.toMatch(/`video\/x-matroska`, `video\/avi`/); + expect(isRefusedUploadContentType("video/avi")).toBe(true); + expect(isRefusedUploadContentType("video/webm;codecs=vp9")).toBe(false); + expect(isRefusedUploadContentType(undefined)).toBe(false); + }); + + it("lists the same content types in the OpenAPI enum", () => { + const spec = developerOpenApiSpec("https://cap.so") as unknown as { + components: { + schemas: { + InitiateUploadRequest: { + properties: { contentType: { enum: string[] } }; + }; + }; + }; + }; + expect( + spec.components.schemas.InitiateUploadRequest.properties.contentType.enum, + ).toEqual([...INFRA_UPLOAD_CONTENT_TYPES]); + }); + + it("lets the file picker choose only those containers", () => { + expect(INFRA_VIDEO_FILE_ACCEPT.split(",")).toEqual([ + ".mp4", + ".mov", + ".mkv", + ".webm", + "video/mp4", + "video/quicktime", + "video/x-matroska", + "video/webm", + ]); + expect(INFRA_VIDEO_FILE_ACCEPT).not.toContain("video/*"); + }); +}); diff --git a/apps/web/__tests__/unit/infra-top-up-amount.test.ts b/apps/web/__tests__/unit/infra-top-up-amount.test.ts index 068f0654552..9fd5412d9d1 100644 --- a/apps/web/__tests__/unit/infra-top-up-amount.test.ts +++ b/apps/web/__tests__/unit/infra-top-up-amount.test.ts @@ -1,13 +1,21 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("@cap/env", () => ({ buildEnv: { NEXT_PUBLIC_IS_CAP: "true" } })); + +import { PRO_CREDIT_COVERS } from "@/components/pages/infra/content"; import { resolveTopUpAmount, + TOP_UP_AMOUNT_PLACEHOLDER, TOP_UP_MAX_CENTS, TOP_UP_MIN_CENTS, + TOP_UP_RANGE_TEXT, topUpButtonLabel, + topUpCovers, + topUpCoversText, } from "@/lib/infra-top-up"; const NOT_AN_AMOUNT = "Enter an amount in dollars, for example 25 or 12.50."; -const OUT_OF_RANGE = "Enter an amount from $5 to $1,000."; +const OUT_OF_RANGE = "Enter an amount from $5 to $10,000."; const PRESET = 2_500; describe("the amount a top-up will charge", () => { @@ -25,8 +33,9 @@ describe("the amount a top-up will charge", () => { ["1,00", NOT_AN_AMOUNT], ["4.99", OUT_OF_RANGE], ["0", OUT_OF_RANGE], - ["1000.01", OUT_OF_RANGE], - ["1001", OUT_OF_RANGE], + ["10000.01", OUT_OF_RANGE], + ["10001", OUT_OF_RANGE], + ["10,000.01", OUT_OF_RANGE], ["99999999999999999999", OUT_OF_RANGE], ])( "blocks %j with a reason and never falls back to the preset", @@ -53,6 +62,10 @@ describe("the amount a top-up will charge", () => { ["$20", 2_000, "Add $20"], [" 25 ", 2_500, "Add $25"], ["999.99", 99_999, "Add $999.99"], + ["1000.01", 100_001, "Add $1,000.01"], + ["10000", 1_000_000, "Add $10,000"], + ["10,000.00", 1_000_000, "Add $10,000"], + ["9,999.99", 999_999, "Add $9,999.99"], ])("takes %j as a typed amount", (typed, cents, label) => { const amount = resolveTopUpAmount(typed, PRESET); expect(amount).toEqual({ amountCents: cents, error: null, custom: true }); @@ -94,6 +107,30 @@ describe("the amount a top-up will charge", () => { it("keeps the limits the checkout enforces", () => { expect(TOP_UP_MIN_CENTS).toBe(500); - expect(TOP_UP_MAX_CENTS).toBe(100_000); + expect(TOP_UP_MAX_CENTS).toBe(1_000_000); + expect(TOP_UP_RANGE_TEXT).toBe("$5 to $10,000"); + expect(TOP_UP_AMOUNT_PLACEHOLDER).toBe("Other, 5 to 10,000"); + }); +}); + +describe("what a top-up buys", () => { + it("works $5 out the same way as the landing page", () => { + const covers = topUpCovers(5); + expect([covers.storedMinutes, covers.transcribedHours]).toEqual( + PRO_CREDIT_COVERS.map((cover) => cover.amount), + ); + }); + + it("rounds $25 down like the landing page", () => { + expect(topUpCovers(25)).toEqual({ + storedMinutes: 29_400, + transcribedHours: 250, + }); + }); + + it("says what $5 and $25 buy in one line", () => { + expect(topUpCoversText()).toBe( + "At 1080p, $5 stores 5,800 minutes for 30 days or transcribes 50 hours. $25 stores 29,400 minutes or transcribes 250 hours.", + ); }); }); diff --git a/apps/web/__tests__/unit/infra-top-up-ein-hint.test.ts b/apps/web/__tests__/unit/infra-top-up-ein-hint.test.ts new file mode 100644 index 00000000000..1a1b5f403de --- /dev/null +++ b/apps/web/__tests__/unit/infra-top-up-ein-hint.test.ts @@ -0,0 +1,97 @@ +// @vitest-environment jsdom + +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ replace: vi.fn(), refresh: vi.fn(), push: vi.fn() }), + useSearchParams: () => new URLSearchParams(), +})); +vi.mock("sonner", () => ({ toast: { success: vi.fn(), error: vi.fn() } })); + +import { TopUp } from "@/app/(infra)/dashboard/infra/[appId]/billing/top-up"; + +let root: Root; +let container: HTMLDivElement; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); + act(() => + root.render( + createElement(QueryClientProvider, { + client: new QueryClient(), + children: createElement(TopUp, { appId: "app-1", autoTopUpHref: null }), + }), + ), + ); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); +}); + +const einInput = () => + container.querySelector('input[placeholder^="US EIN"]'); + +describe("US EIN hint on the top-up form", () => { + it("tells a US business to add its EIN before paying", () => { + expect(container.textContent).toContain( + "US business? Add your EIN before you pay so it is on your invoice.", + ); + expect(einInput()).toBeNull(); + }); + + it("opens the EIN field from the hint and then steps aside", () => { + const link = Array.from(container.querySelectorAll("button")).find( + (button) => button.textContent === "Add your EIN", + ); + expect(link).toBeDefined(); + + act(() => link?.click()); + + expect(einInput()).not.toBeNull(); + expect(container.textContent).not.toContain("US business?"); + }); + + it("offers the EIN field only from the hint, not a second button", () => { + const buttons = Array.from(container.querySelectorAll("button")).map( + (button) => button.textContent, + ); + expect(buttons.filter((text) => text?.includes("EIN"))).toEqual([ + "Add your EIN", + ]); + }); + + it("lets the buyer clear the EIN and still pay", () => { + const link = Array.from(container.querySelectorAll("button")).find( + (button) => button.textContent === "Add your EIN", + ); + act(() => link?.click()); + const input = einInput(); + if (!input) throw new Error("no EIN field"); + const type = (value: string) => { + const setter = Object.getOwnPropertyDescriptor( + HTMLInputElement.prototype, + "value", + )?.set; + act(() => { + setter?.call(input, value); + input.dispatchEvent(new Event("input", { bubbles: true })); + }); + }; + const submit = () => + container.querySelector('button[type="submit"]'); + + type("12-34"); + expect(submit()?.disabled).toBe(true); + type(""); + expect(submit()?.disabled).toBe(false); + expect(container.querySelector('[role="alert"]')).toBeNull(); + }); +}); diff --git a/apps/web/__tests__/unit/infra-top-up-nudge.test.ts b/apps/web/__tests__/unit/infra-top-up-nudge.test.ts new file mode 100644 index 00000000000..504edf10cea --- /dev/null +++ b/apps/web/__tests__/unit/infra-top-up-nudge.test.ts @@ -0,0 +1,42 @@ +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const navigation = vi.hoisted(() => ({ query: "" })); + +vi.mock("next/navigation", () => ({ + useRouter: () => ({ replace: vi.fn(), refresh: vi.fn() }), + useSearchParams: () => new URLSearchParams(navigation.query), +})); +vi.mock("sonner", () => ({ toast: { success: vi.fn(), error: vi.fn() } })); +vi.mock("@tanstack/react-query", () => ({ + useMutation: () => ({ mutate: vi.fn(), isPending: false }), +})); + +import { TopUp } from "@/app/(infra)/dashboard/infra/[appId]/billing/top-up"; + +const NUDGE = + "to add credit as soon as the balance falls below a threshold you set."; + +const render = (autoTopUpHref: string | null) => + renderToStaticMarkup(createElement(TopUp, { appId: "app-1", autoTopUpHref })); + +describe("the nudge after a purchase", () => { + beforeEach(() => { + navigation.query = ""; + }); + + it("points to auto top-up after a purchase while it is off", () => { + navigation.query = "purchase=success"; + const html = render("#auto-top-up"); + + expect(html).toContain(NUDGE); + expect(html).toContain('href="#auto-top-up"'); + }); + + it("stays out of the way before a purchase and when auto top-up is on", () => { + expect(render("#auto-top-up")).not.toContain(NUDGE); + navigation.query = "purchase=success"; + expect(render(null)).not.toContain(NUDGE); + }); +}); diff --git a/apps/web/__tests__/unit/infra-upload-drop-copy.test.ts b/apps/web/__tests__/unit/infra-upload-drop-copy.test.ts new file mode 100644 index 00000000000..0e877ffb0c8 --- /dev/null +++ b/apps/web/__tests__/unit/infra-upload-drop-copy.test.ts @@ -0,0 +1,45 @@ +// @vitest-environment jsdom + +import React, { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/app/(infra)/dashboard/infra/_components/uploads", () => ({ + useUploads: () => ({ + appId: "app-1", + active: null, + waiting: 0, + failures: [], + add: vi.fn(), + cancel: vi.fn(), + }), +})); + +import { UploadDrop } from "@/app/(infra)/dashboard/infra/_components/upload-drop"; + +let root: Root; +let container: HTMLDivElement; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); +}); + +describe("UploadDrop copy", () => { + it("says encoding is free and that uploading needs a balance", () => { + act(() => root.render(React.createElement(UploadDrop))); + const text = (container.textContent ?? "").replace(/\s+/g, " "); + + expect(text).toContain( + "Encoding is free. Stored and delivered video are billed by the minute, and uploading needs a balance that covers its first month.", + ); + expect(text).not.toMatch(/uploads are free/i); + }); +}); diff --git a/apps/web/__tests__/unit/infra-upload-drop-credits.test.ts b/apps/web/__tests__/unit/infra-upload-drop-credits.test.ts new file mode 100644 index 00000000000..ca63ec0a3c5 --- /dev/null +++ b/apps/web/__tests__/unit/infra-upload-drop-credits.test.ts @@ -0,0 +1,68 @@ +// @vitest-environment jsdom + +import React, { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("@/app/(infra)/dashboard/infra/_components/uploads", () => ({ + useUploads: () => ({ + appId: "app-1", + active: null, + waiting: 0, + failures: [ + { + id: "failure-1", + name: "lesson.mp4", + message: "This app has no credit. Add credits to start uploading.", + needsCredits: true, + skipped: 0, + }, + ], + add: vi.fn(), + cancel: vi.fn(), + }), +})); + +import { UploadDrop } from "@/app/(infra)/dashboard/infra/_components/upload-drop"; + +let root: Root; +let container: HTMLDivElement; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); +}); + +const creditLinks = () => + Array.from(container.querySelectorAll("a")).filter( + (link) => link.textContent === "Add credits", + ); + +describe("UploadDrop credit link", () => { + it("links to Billing from a refused upload by default", () => { + act(() => root.render(React.createElement(UploadDrop))); + + expect(creditLinks()).toHaveLength(1); + expect(creditLinks()[0]?.getAttribute("href")).toBe( + "/dashboard/infra/app-1/billing", + ); + }); + + it("leaves the link out when the page already shows an Add credits button", () => { + act(() => + root.render(React.createElement(UploadDrop, { creditsLink: false })), + ); + + expect(creditLinks()).toHaveLength(0); + expect(container.textContent).toContain( + "This app has no credit. Add credits to start uploading.", + ); + }); +}); diff --git a/apps/web/__tests__/unit/infra-upload-idempotency.test.ts b/apps/web/__tests__/unit/infra-upload-idempotency.test.ts new file mode 100644 index 00000000000..ca81abc2313 --- /dev/null +++ b/apps/web/__tests__/unit/infra-upload-idempotency.test.ts @@ -0,0 +1,291 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +type Row = Record; + +const state = vi.hoisted(() => ({ + videos: new Map>(), + inserts: 0, + nextId: 0, + admission: vi.fn(), +})); + +vi.mock("@cap/database", async () => { + const { drizzle } = await import("drizzle-orm/mysql-proxy"); + const columnsOf = (list: string) => + [...list.matchAll(/`([A-Za-z0-9]+)`(?:,|\))/g)].map( + (match) => match[1] ?? "", + ); + const client = drizzle(async (sql, params) => { + if (sql.startsWith("insert into `developer_videos`")) { + const columns = columnsOf( + sql.slice(sql.indexOf("("), sql.indexOf(")") + 1), + ); + const values = sql + .slice(sql.indexOf("values (") + 8, sql.lastIndexOf(")")) + .split(", "); + const row: Row = {}; + let param = 0; + columns.forEach((column, index) => { + row[column] = values[index] === "?" ? params[param++] : null; + }); + const id = String(row.id); + if (state.videos.has(id)) { + if (!sql.includes("on duplicate key update")) { + throw new Error("Duplicate entry for key PRIMARY"); + } + return { rows: [{ affectedRows: 0 }] }; + } + state.inserts += 1; + state.videos.set(id, { + ...row, + metadata: + typeof row.metadata === "string" + ? JSON.parse(row.metadata) + : row.metadata, + duration: null, + deletedAt: null, + status: "uploading", + }); + return { rows: [{ affectedRows: 1 }] }; + } + if (sql.includes("from `developer_videos`")) { + const fields = sql + .slice("select ".length, sql.indexOf(" from ")) + .split(", ") + .map((field) => field.replaceAll("`", "").split(".").pop() ?? ""); + const video = state.videos.get(String(params[0])); + if (!video) return { rows: [] }; + return { + rows: [fields.map((field) => video[field] ?? null)], + }; + } + throw new Error(`Unexpected query: ${sql}`); + }); + return { db: () => client }; +}); +vi.mock("@cap/database/helpers", () => ({ + nanoId: () => `random-video-${++state.nextId}`, + nanoIdLength: 15, +})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_WEB_URL: "https://cap.test" }, + serverEnv: () => ({ WEB_URL: "https://cap.test" }), +})); +vi.mock("@cap/web-backend", () => ({ + provideOptionalAuth: (effect: T) => effect, + S3Buckets: {}, +})); +vi.mock("@/lib/server", () => ({ runPromise: async () => undefined })); +vi.mock("@/lib/infra-standing", async (original) => ({ + ...(await original()), + getInfraUploadAdmission: state.admission, +})); +vi.mock("@/lib/developer-video-start", () => ({ + claimDeveloperVideoUpload: vi.fn(), + releaseDeveloperVideoUpload: vi.fn(), + startDeveloperVideoProcessing: vi.fn(), + redriveDeveloperVideoProcessing: vi.fn(), +})); + +import { + developerVideoIdForIdempotencyKey, + legacyDeveloperVideoIdForIdempotencyKey, +} from "@/lib/developer-ids"; +import { createInfraVideo } from "@/lib/infra-video-upload"; + +const admitted = { + allowed: true, + reason: null, + standing: { state: "active", pausesAt: null }, + balanceMicroCredits: 1_000_000, + requiredMicroCredits: 1, + shortfallMicroCredits: 0, + message: null, +}; + +const KEY = "retry-upload-0001"; +const request = { + appId: "app-1", + name: "Standup", + userId: "user_42", + metadata: { project: "alpha", tags: ["a", "b"] }, + maxResolutionTier: "1440p" as const, + idempotencyKey: KEY, +}; + +beforeEach(() => { + state.videos = new Map(); + state.inserts = 0; + state.nextId = 0; + state.admission.mockReset(); + state.admission.mockResolvedValue(admitted); +}); + +describe("create upload with an Idempotency-Key", () => { + it("returns the same video for a repeated request and makes one row", async () => { + const first = await createInfraVideo(request); + const second = await createInfraVideo({ + ...request, + metadata: { tags: ["a", "b"], project: "alpha" }, + }); + + expect(first.status).toBe(200); + expect(second).toEqual(first); + expect(first.status === 200 && first.body.videoId).toBe( + developerVideoIdForIdempotencyKey("app-1", KEY), + ); + expect(state.inserts).toBe(1); + expect(state.videos.size).toBe(1); + }); + + it("gives a new keyed video an id in the alphabet of every other video id", async () => { + const created = await createInfraVideo(request); + + expect(created.status === 200 && created.body.videoId).toMatch( + /^[0-9a-hjkmnp-tv-z]{15}$/, + ); + }); + + it("keeps returning a video created under the key before ids changed", async () => { + const legacy = legacyDeveloperVideoIdForIdempotencyKey("app-1", KEY); + state.videos.set(legacy, { + id: legacy, + appId: "app-1", + s3Key: `developer/app-1/${legacy}/video`, + name: request.name, + externalUserId: request.userId, + metadata: request.metadata, + maxResolutionTier: request.maxResolutionTier, + playbackId: "playback-legacy-0001", + duration: null, + deletedAt: null, + status: "uploading", + }); + + const repeated = await createInfraVideo(request); + + expect(repeated.status === 200 && repeated.body.videoId).toBe(legacy); + expect(state.inserts).toBe(0); + expect(state.videos.size).toBe(1); + expect( + await createInfraVideo({ ...request, name: "Another name" }), + ).toMatchObject({ status: 409 }); + }); + + it("never stores Cap's takedown marker from a client's metadata, and replays the request", async () => { + const withMarker = { + ...request, + metadata: { + project: "alpha", + tags: ["a", "b"], + __capTakedown: { at: "2026-11-13T12:00:00.000Z", metadata: null }, + }, + }; + const first = await createInfraVideo(withMarker); + const id = developerVideoIdForIdempotencyKey("app-1", KEY); + + expect(first.status).toBe(200); + expect(state.videos.get(id)?.metadata).toEqual({ + project: "alpha", + tags: ["a", "b"], + }); + expect(await createInfraVideo(withMarker)).toEqual(first); + expect(state.inserts).toBe(1); + }); + + it("replays an unchanged request for a video Cap took down", async () => { + const first = await createInfraVideo(request); + const id = developerVideoIdForIdempotencyKey("app-1", KEY); + const stored = state.videos.get(id); + if (!stored) throw new Error("missing video"); + stored.metadata = { + __capTakedown: { + at: "2026-11-13T12:00:00.000Z", + metadata: stored.metadata, + }, + }; + + expect(await createInfraVideo(request)).toEqual(first); + expect(state.inserts).toBe(1); + }); + + it("makes one video when the same request races itself", async () => { + const results = await Promise.all( + Array.from({ length: 5 }, () => createInfraVideo(request)), + ); + + expect(new Set(results.map((result) => result.status))).toEqual( + new Set([200]), + ); + expect( + new Set(results.map((result) => JSON.stringify(result.body))).size, + ).toBe(1); + expect(state.videos.size).toBe(1); + }); + + it.each([ + ["name", { name: "Retro" }], + ["userId", { userId: "user_43" }], + ["maxResolutionTier", { maxResolutionTier: "2160p" as const }], + ["metadata", { metadata: { project: "beta" } }], + ])("refuses the same key with a different %s", async (field, change) => { + await createInfraVideo(request); + + const conflict = await createInfraVideo({ ...request, ...change }); + + expect(conflict).toMatchObject({ + status: 409, + body: { + code: "idempotency_conflict", + message: `Idempotency key was already used with a different ${field}`, + }, + }); + expect(state.videos.size).toBe(1); + }); + + it("refuses a key whose video was deleted and answers a completed upload like a token renewal", async () => { + await createInfraVideo(request); + const id = developerVideoIdForIdempotencyKey("app-1", KEY); + const stored = state.videos.get(id); + if (!stored) throw new Error("video missing"); + + stored.duration = 12; + expect(await createInfraVideo(request)).toMatchObject({ + status: 409, + body: { code: "upload_already_completed" }, + }); + + stored.deletedAt = "2026-11-13 10:00:00"; + expect(await createInfraVideo(request)).toMatchObject({ + status: 409, + body: { + code: "idempotency_conflict", + message: "Idempotency key belongs to a video that was deleted", + }, + }); + expect(state.videos.size).toBe(1); + }); + + it("checks upload admission on a repeat too", async () => { + await createInfraVideo(request); + state.admission.mockResolvedValueOnce({ + ...admitted, + allowed: false, + reason: "insufficient_credits", + message: "Add credit", + }); + + expect(await createInfraVideo(request)).toMatchObject({ status: 402 }); + expect(state.admission).toHaveBeenCalledTimes(2); + expect(state.videos.size).toBe(1); + }); + + it("keeps the same key separate per app, and creates a new video each time without a key", async () => { + await createInfraVideo(request); + await createInfraVideo({ ...request, appId: "app-2" }); + await createInfraVideo({ ...request, idempotencyKey: undefined }); + await createInfraVideo({ ...request, idempotencyKey: undefined }); + + expect(state.videos.size).toBe(4); + }); +}); diff --git a/apps/web/__tests__/unit/infra-upload.test.ts b/apps/web/__tests__/unit/infra-upload.test.ts index 6b05c262efb..501f559b45f 100644 --- a/apps/web/__tests__/unit/infra-upload.test.ts +++ b/apps/web/__tests__/unit/infra-upload.test.ts @@ -1,12 +1,12 @@ import { describe, expect, it } from "vitest"; import { - BUSY_COMPLETE_ATTEMPTS, COMPLETION_IN_PROGRESS, completeRetryPlan, isDefinitiveRejection, partBytesFor, presignBatch, retryAfterMs, + STALE_UPLOAD_CLAIM_MS, startsLikeAnImageOrDocument, } from "@/lib/infra-upload"; @@ -93,31 +93,40 @@ describe("a completion another request is already running", () => { expect(isDefinitiveRejection(500, null)).toBe(false); }); - it("is retried more times, at a steady pace", () => { - expect(completeRetryPlan(1, COMPLETION_IN_PROGRESS, 3)).toEqual({ - retry: true, - waitMs: 3000, + const busy = (busyWaitedMs: number, retryAfter: string | null = "5") => + completeRetryPlan({ + attempt: 1, + code: COMPLETION_IN_PROGRESS, + attempts: 3, + retryAfter, + busyWaitedMs, }); - expect( - completeRetryPlan(BUSY_COMPLETE_ATTEMPTS - 1, COMPLETION_IN_PROGRESS, 3) - .retry, - ).toBe(true); - expect( - completeRetryPlan(BUSY_COMPLETE_ATTEMPTS, COMPLETION_IN_PROGRESS, 3) - .retry, - ).toBe(false); + + it("waits as long as the server's Retry-After asks", () => { + expect(busy(0)).toEqual({ retry: true, waitMs: 5_000, busy: true }); + expect(busy(0, "30").waitMs).toBe(30_000); + expect(busy(0, null).waitMs).toBe(5_000); + }); + + it("keeps waiting for as long as the server can hold a completion", () => { + expect(busy(60_000).retry).toBe(true); + expect(busy(STALE_UPLOAD_CLAIM_MS).retry).toBe(true); + expect(busy(STALE_UPLOAD_CLAIM_MS + 55_000).retry).toBe(true); + expect(busy(STALE_UPLOAD_CLAIM_MS + 55_001).retry).toBe(false); }); it("keeps the usual plan for a lost response", () => { - expect(completeRetryPlan(1, null, 3)).toEqual({ - retry: true, - waitMs: 1500, - }); - expect(completeRetryPlan(2, null, 3)).toEqual({ - retry: true, - waitMs: 3000, - }); - expect(completeRetryPlan(3, null, 3).retry).toBe(false); + const lost = (attempt: number) => + completeRetryPlan({ + attempt, + code: null, + attempts: 3, + retryAfter: null, + busyWaitedMs: 0, + }); + expect(lost(1)).toEqual({ retry: true, waitMs: 1500, busy: false }); + expect(lost(2)).toEqual({ retry: true, waitMs: 3000, busy: false }); + expect(lost(3).retry).toBe(false); }); }); diff --git a/apps/web/__tests__/unit/infra-usage.test.ts b/apps/web/__tests__/unit/infra-usage.test.ts index 0560ffa7c5a..b7dbb4d0020 100644 --- a/apps/web/__tests__/unit/infra-usage.test.ts +++ b/apps/web/__tests__/unit/infra-usage.test.ts @@ -1,9 +1,12 @@ import { describe, expect, it } from "vitest"; import { + formatAudioLength, + formatBalanceCents, formatBytes, formatCharge, formatCredits, formatDuration, + formatDurationOrDash, summarizeStorageMonth, utcDaysAgoKey, utcMonthStart, @@ -104,3 +107,56 @@ describe("formatting", () => { expect(formatDuration(3723)).toBe("1:02:03"); }); }); + +describe("formatAudioLength", () => { + it("shows seconds under a minute so a small charge never reads as 0 hours", () => { + expect(formatAudioLength(12)).toBe("12 audio seconds"); + expect(formatAudioLength(1)).toBe("1 audio second"); + }); + + it("shows minutes under an hour", () => { + expect(formatAudioLength(270)).toBe("4.5 audio minutes"); + expect(formatAudioLength(60)).toBe("1 audio minute"); + expect(formatAudioLength(2_400)).toBe("40 audio minutes"); + }); + + it("shows hours from one hour up", () => { + expect(formatAudioLength(3_600)).toBe("1 audio hour"); + expect(formatAudioLength(5_400)).toBe("1.5 audio hours"); + expect(formatAudioLength(54_000)).toBe("15 audio hours"); + }); + + it("reads zero as minutes", () => { + expect(formatAudioLength(0)).toBe("0 audio minutes"); + }); +}); + +describe("formatDurationOrDash", () => { + it("shows a dash when nothing was measured", () => { + expect(formatDurationOrDash(null)).toBe("–"); + expect(formatDurationOrDash(undefined)).toBe("–"); + }); + + it("shows a measured length as minutes and seconds", () => { + expect(formatDurationOrDash(120)).toBe("2:00"); + }); +}); + +describe("formatBalanceCents", () => { + it("rounds down to whole cents so it never shows more than the balance", () => { + expect(formatBalanceCents(999_816)).toBe("$9.99"); + expect(formatBalanceCents(9_998_160)).toBe("$99.98"); + }); + + it("shows a positive balance under a cent as under a cent", () => { + expect(formatBalanceCents(999)).toBe("<$0.01"); + }); + + it("shows zero as zero", () => { + expect(formatBalanceCents(0)).toBe("$0.00"); + }); + + it("groups thousands", () => { + expect(formatBalanceCents(123_456_789_000)).toBe("$1,234,567.89"); + }); +}); diff --git a/apps/web/__tests__/unit/infra-video-fields.test.ts b/apps/web/__tests__/unit/infra-video-fields.test.ts index 8398eafb6e9..c3e3927eb01 100644 --- a/apps/web/__tests__/unit/infra-video-fields.test.ts +++ b/apps/web/__tests__/unit/infra-video-fields.test.ts @@ -65,6 +65,7 @@ import { getInfraApps, requireInfraApp, } from "@/app/(infra)/dashboard/infra/infra-data"; +import { developerVideoReportedDuration } from "@/lib/developer-video-api"; const dialect = new MySqlDialect(); const query = (condition: unknown) => dialect.sqlToQuery(condition as SQL); @@ -88,7 +89,7 @@ beforeEach(() => { }); describe("console video list", () => { - it("reads named columns and never the stored key or processing state", async () => { + it("reads named columns and never the stored key", async () => { await getDeveloperAppVideos("app-1"); const [selection] = mocks.selections; @@ -99,7 +100,9 @@ describe("console video list", () => { "id", "legacy", "name", + "processing", "status", + "takenDown", "width", ]); expect(query(selection?.legacy).sql).toMatch( @@ -162,6 +165,56 @@ describe("console video list", () => { }); }); +describe("console video duration", () => { + const declaredOnly = () => + developerVideo({ + status: "errored", + duration: 120, + processing: processingState({ + probe: null, + hls: { state: "failed", reason: "unreadable" }, + mp4: { state: "failed", reason: "unreadable" }, + }), + }); + + it("leaves out a declared length the API also leaves out, in the list", async () => { + const measured = developerVideo({ + id: "video-2", + status: "errored", + duration: 60, + processing: processingState({ + hls: { state: "failed", reason: "unreadable" }, + mp4: { state: "failed", reason: "unreadable" }, + }), + }); + mocks.rows = [ + [ + { ...declaredOnly(), legacy: 0 }, + { ...measured, legacy: 0 }, + ], + ]; + + const [failed, probed] = await getDeveloperAppVideos("app-1"); + + expect(failed?.state).toBe("failed"); + expect(failed?.duration).toBeNull(); + expect(probed?.duration).toBe(60); + }); + + it("leaves it out on the video page too", async () => { + mocks.rows = [[declaredOnly()]]; + + const video = await getDeveloperAppVideo("app-1", "video-1"); + + expect(video?.state).toBe("failed"); + expect(video?.duration).toBeNull(); + }); + + it("matches what the API returns for the same video", () => { + expect(developerVideoReportedDuration(declaredOnly())).toBeNull(); + }); +}); + describe("console video page", () => { it("looks the video up inside the app and skips deleted ones", async () => { await expect(getDeveloperAppVideo("app-1", "video-1")).resolves.toBeNull(); diff --git a/apps/web/__tests__/unit/infra-video-import.test.ts b/apps/web/__tests__/unit/infra-video-import.test.ts new file mode 100644 index 00000000000..bd60773f2ac --- /dev/null +++ b/apps/web/__tests__/unit/infra-video-import.test.ts @@ -0,0 +1,605 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const dnsState = vi.hoisted(() => ({ + answers: new Map(), + calls: [] as string[], +})); + +vi.mock("node:dns", async () => { + const actual = await vi.importActual("node:dns"); + const lookup = ( + hostname: string, + options: unknown, + callback: ( + error: NodeJS.ErrnoException | null, + address: unknown, + family?: number, + ) => void, + ) => { + dnsState.calls.push(hostname); + const answers = dnsState.answers.get(hostname); + if (!answers) { + const error: NodeJS.ErrnoException = new Error(`ENOTFOUND ${hostname}`); + error.code = "ENOTFOUND"; + callback(error, undefined); + return; + } + const entries = answers.map((address) => ({ + address, + family: address.includes(":") ? 6 : 4, + })); + const all = + typeof options === "object" && + options !== null && + (options as { all?: boolean }).all; + if (all) callback(null, entries); + else callback(null, entries[0]?.address, entries[0]?.family); + }; + return { ...actual, default: { ...actual, lookup }, lookup }; +}); + +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + NODE_ENV: "production", + serverEnv: () => ({}), +})); +vi.mock("@cap/database", () => ({ + db: () => { + throw new Error("No database in unit tests"); + }, +})); +vi.mock("@/lib/infra-video-upload", () => ({})); +vi.mock("@/lib/developer-webhook-dispatch", () => ({})); +vi.mock("@/lib/infra-auto-top-up", () => ({})); +vi.mock("@/lib/infra-standing", () => ({})); +vi.mock("@/lib/developer-video-charge", () => ({})); + +const { + checkImportUrl, + headStop, + ImportStop, + importStopFor, + isMissingImportTable, + openImportSource, + preflightImport, + streamSourceToParts, +} = await import("@/lib/infra-video-import"); +const { isBlockedS3Endpoint } = await import("@/lib/s3-endpoint-policy"); +const { requestPublicOnly } = await import("@/lib/storage-fetch"); + +const resolvesTo = + (table: Record) => + (url: string): Promise => + isBlockedS3Endpoint(url, async (hostname) => table[hostname] ?? []); + +const publicNames = resolvesTo({ + "media.example.com": ["93.184.216.34"], + "cdn.example.com": ["93.184.216.35"], + "mixed.example.com": ["93.184.216.34", "10.0.0.5"], + "private.example.com": ["10.1.2.3"], + "metadata.example.com": ["169.254.169.254"], + "loopback6.example.com": ["::1"], + "mapped.example.com": ["::ffff:127.0.0.1"], +}); + +const MP4_HEAD = new Uint8Array([ + 0x00, 0x00, 0x00, 0x18, 0x66, 0x74, 0x79, 0x70, 0x69, 0x73, 0x6f, 0x6d, 0x00, + 0x00, 0x02, 0x00, +]); + +const streamOf = (chunks: Uint8Array[]) => + new ReadableStream({ + start(controller) { + for (const chunk of chunks) controller.enqueue(chunk); + controller.close(); + }, + }); + +const okResponse = ( + chunks: Uint8Array[], + headers: Record = { "content-type": "video/mp4" }, +) => new Response(streamOf(chunks), { status: 200, headers }); + +const redirect = (location: string, status = 302) => + new Response(null, { status, headers: { location } }); + +const codeOf = async (work: Promise) => { + try { + await work; + return "ok"; + } catch (error) { + return error instanceof ImportStop ? error.code : `raw:${String(error)}`; + } +}; + +afterEach(() => { + dnsState.answers.clear(); + dnsState.calls.length = 0; + vi.useRealTimers(); +}); + +describe("checkImportUrl", () => { + it("accepts a public https URL", async () => { + expect( + await checkImportUrl("https://media.example.com/a.mp4", { + isBlocked: publicNames, + }), + ).toBeNull(); + }); + + it.each([ + ["plain http", "http://media.example.com/a.mp4"], + ["credentials", "https://user:pass@media.example.com/a.mp4"], + ["a username alone", "https://user@media.example.com/a.mp4"], + ["an explicit port", "https://media.example.com:8443/a.mp4"], + ["the default port written out", "https://media.example.com:443x/a.mp4"], + ["localhost", "https://localhost/a.mp4"], + ["a localhost subdomain", "https://x.localhost/a.mp4"], + [".local", "https://nas.local/a.mp4"], + [".internal", "https://metadata.google.internal/computeMetadata/v1/"], + ["a file URL", "file:///etc/passwd"], + ["a gopher URL", "gopher://media.example.com/"], + ["a data URL", "data:video/mp4;base64,AAAA"], + ["no host", "https:///a.mp4"], + [ + "a URL over 2,048 characters", + `https://media.example.com/${"a".repeat(2_100)}`, + ], + ["IPv4 loopback", "https://127.0.0.1/a.mp4"], + ["IPv4 loopback, short form", "https://127.1/a.mp4"], + ["IPv4 loopback, decimal form", "https://2130706433/a.mp4"], + ["IPv4 loopback, hex form", "https://0x7f000001/a.mp4"], + ["IPv4 loopback, octal form", "https://0177.0.0.1/a.mp4"], + ["metadata, decimal form", "https://2852039166/latest/meta-data/"], + ["metadata, hex dotted form", "https://0xa9.0xfe.0xa9.0xfe/latest/"], + ["this network, not zero", "https://0.1.2.3/a.mp4"], + ["carrier NAT, top of range", "https://100.127.255.254/a.mp4"], + [ + "GCP metadata name", + "https://metadata.google.internal/computeMetadata/v1/", + ], + ["a bare metadata name", "https://metadata/latest/"], + ["cloud metadata", "https://169.254.169.254/latest/meta-data/"], + ["RFC 1918 10/8", "https://10.0.0.1/a.mp4"], + ["RFC 1918 172.16/12", "https://172.16.5.4/a.mp4"], + ["RFC 1918 192.168/16", "https://192.168.1.1/a.mp4"], + ["carrier NAT", "https://100.64.0.1/a.mp4"], + ["this network", "https://0.0.0.0/a.mp4"], + ["multicast", "https://224.0.0.1/a.mp4"], + ["IPv6 loopback", "https://[::1]/a.mp4"], + ["IPv6 unspecified", "https://[::]/a.mp4"], + ["IPv6 unique local", "https://[fd00::1]/a.mp4"], + ["IPv6 link local", "https://[fe80::1]/a.mp4"], + ["IPv4-mapped loopback", "https://[::ffff:127.0.0.1]/a.mp4"], + ["IPv4-mapped metadata in hex", "https://[::ffff:a9fe:a9fe]/a.mp4"], + ["IPv4-compatible loopback", "https://[::7f00:1]/a.mp4"], + ["NAT64 metadata", "https://[64:ff9b::a9fe:a9fe]/a.mp4"], + ["6to4 wrapping a private address", "https://[2002:a00:1::1]/a.mp4"], + ["Teredo", "https://[2001:0:4136:e378:8000:63bf:3fff:fdd2]/a.mp4"], + ["a name that resolves to 10/8", "https://private.example.com/a.mp4"], + ["a name that resolves to metadata", "https://metadata.example.com/a.mp4"], + ["a name with one private answer", "https://mixed.example.com/a.mp4"], + ["a name that resolves to ::1", "https://loopback6.example.com/a.mp4"], + [ + "a name that resolves to a mapped address", + "https://mapped.example.com/a.mp4", + ], + ["a name that does not resolve", "https://nowhere.example.com/a.mp4"], + ])("refuses %s", async (_label, url) => { + const refused = await checkImportUrl(url, { isBlocked: publicNames }); + expect(refused?.code).toBe("url_not_allowed"); + }); + + it("explains each refusal in words", async () => { + const messages = await Promise.all( + [ + "http://media.example.com/a.mp4", + "https://user:pass@media.example.com/a.mp4", + "https://media.example.com:8443/a.mp4", + "https://169.254.169.254/", + ].map( + async (url) => + (await checkImportUrl(url, { isBlocked: publicNames }))?.message, + ), + ); + expect(messages).toEqual([ + "The URL must start with https://", + "The URL must not contain a username or password", + "The URL must use the default https port", + "The URL must point to a public address", + ]); + }); +}); + +describe("redirects", () => { + it("follows up to three redirects, checking every hop", async () => { + const isBlocked = vi.fn(publicNames); + const request = vi + .fn() + .mockResolvedValueOnce(redirect("https://cdn.example.com/1")) + .mockResolvedValueOnce(redirect("/2", 301)) + .mockResolvedValueOnce(redirect("https://media.example.com/3", 307)) + .mockResolvedValueOnce(okResponse([MP4_HEAD])); + const found = await preflightImport("https://media.example.com/0", { + isBlocked, + request, + }); + expect(found.contentType).toBe("video/mp4"); + expect(isBlocked.mock.calls.map(([url]) => url)).toEqual([ + "https://media.example.com/0", + "https://cdn.example.com/1", + "https://cdn.example.com/2", + "https://media.example.com/3", + ]); + expect(request).toHaveBeenCalledTimes(4); + }); + + it("stops at the fourth redirect", async () => { + const request = vi + .fn() + .mockImplementation(async () => redirect("https://cdn.example.com/x")); + const stop = await preflightImport("https://media.example.com/0", { + isBlocked: publicNames, + request, + }).catch((error) => error); + expect(stop).toBeInstanceOf(ImportStop); + expect(stop.code).toBe("url_unreachable"); + expect(request).toHaveBeenCalledTimes(4); + }); + + it.each([ + ["a private literal", "https://10.0.0.1/a.mp4"], + ["cloud metadata", "http://169.254.169.254/latest/meta-data/"], + ["metadata over https", "https://169.254.169.254/latest/meta-data/"], + ["a name that resolves privately", "https://private.example.com/a.mp4"], + ["plain http", "http://cdn.example.com/a.mp4"], + ["localhost", "https://localhost/a.mp4"], + ["a credentialed URL", "https://a:b@cdn.example.com/a.mp4"], + ["an odd port", "https://cdn.example.com:2375/a.mp4"], + ])( + "refuses a redirect to %s without requesting it", + async (_label, target) => { + const request = vi.fn().mockResolvedValueOnce(redirect(target)); + expect( + await codeOf( + preflightImport("https://media.example.com/0", { + isBlocked: publicNames, + request, + }), + ), + ).toBe("url_not_allowed"); + expect(request).toHaveBeenCalledTimes(1); + }, + ); +}); + +describe("DNS rebinding", () => { + it("refuses at connect when the name answers privately after the check", async () => { + const checked: string[] = []; + const isBlocked = (url: string) => + isBlockedS3Endpoint(url, async (hostname) => { + checked.push(hostname); + return ["93.184.216.34"]; + }); + dnsState.answers.set("rebind.example.com", ["127.0.0.1"]); + + expect( + await codeOf( + preflightImport("https://rebind.example.com/a.mp4", { + isBlocked, + request: requestPublicOnly, + }), + ), + ).toBe("url_not_allowed"); + expect(checked).toEqual(["rebind.example.com"]); + expect(dnsState.calls).toEqual(["rebind.example.com"]); + }); + + it.each([ + ["metadata", ["169.254.169.254"]], + ["IPv6 loopback", ["::1"]], + ["an IPv4-mapped loopback", ["::ffff:127.0.0.1"]], + ["NAT64 metadata", ["64:ff9b::a9fe:a9fe"]], + ["one private answer among public ones", ["93.184.216.34", "10.0.0.7"]], + ])("refuses a connect-time answer of %s", async (_label, answers) => { + dnsState.answers.set("rebind.example.com", answers); + expect( + await codeOf( + preflightImport("https://rebind.example.com/a.mp4", { + isBlocked: async () => false, + request: requestPublicOnly, + }), + ), + ).toBe("url_not_allowed"); + }); +}); + +describe("what the URL serves", () => { + const serve = (response: Response) => ({ + isBlocked: publicNames, + request: vi.fn().mockResolvedValueOnce(response), + }); + const html = new TextEncoder().encode("hi"); + const playlist = new TextEncoder().encode("#EXTM3U\n#EXT-X-VERSION:3\n"); + const png = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + const pdf = new TextEncoder().encode("%PDF-1.7\n"); + + it.each([ + ["a web page", okResponse([html], {})], + [ + "a page served as text/html", + okResponse([MP4_HEAD], { "content-type": "text/html; charset=utf-8" }), + ], + ["a playlist", okResponse([playlist], {})], + ["a PNG", okResponse([png], {})], + [ + "a PDF", + okResponse([pdf], { "content-type": "application/octet-stream" }), + ], + ["an image type", okResponse([MP4_HEAD], { "content-type": "image/jpeg" })], + [ + "an AVI type", + okResponse([MP4_HEAD], { "content-type": "video/x-msvideo" }), + ], + ["JSON", okResponse([MP4_HEAD], { "content-type": "application/json" })], + ])("refuses %s as unsupported_format", async (_label, response) => { + expect( + await codeOf( + preflightImport("https://media.example.com/a", serve(response)), + ), + ).toBe("unsupported_format"); + }); + + it.each([ + ["a 404", new Response("nope", { status: 404 })], + ["a 500", new Response("nope", { status: 500 })], + ["an empty body", okResponse([])], + ["a redirect with no location", new Response(null, { status: 302 })], + ])("reports %s as url_unreachable", async (_label, response) => { + expect( + await codeOf( + preflightImport("https://media.example.com/a", serve(response)), + ), + ).toBe("url_unreachable"); + }); + + it("reports a declared length over the limit as too_large before reading", async () => { + const body = new ReadableStream({ pull() {} }); + const response = new Response(body, { + status: 200, + headers: { "content-length": "999", "content-type": "video/mp4" }, + }); + expect( + await codeOf( + preflightImport("https://media.example.com/a", { + ...serve(response), + maxBytes: 998, + }), + ), + ).toBe("too_large"); + }); + + it("never repeats what the server sent in its answer", async () => { + const stop = await preflightImport( + "https://media.example.com/a", + serve( + okResponse([MP4_HEAD], { + "content-type": "text/plain; x=", + }), + ), + ).catch((error) => error); + expect(stop.code).toBe("unsupported_format"); + expect(stop.message).toBe("The URL returned a file that is not a video"); + const page = await preflightImport( + "https://media.example.com/a", + serve(new Response("secret internal page", { status: 403 })), + ).catch((error) => error); + expect(page.message).toBe("The URL answered 403"); + }); + + it("accepts octet-stream and no type when the bytes look like a video", async () => { + const variants: Record[] = [ + { "content-type": "application/octet-stream" }, + {}, + ]; + for (const headers of variants) { + expect( + await codeOf( + preflightImport( + "https://media.example.com/a", + serve(okResponse([MP4_HEAD], headers)), + ), + ), + ).toBe("ok"); + } + }); +}); + +describe("timeouts", () => { + it("gives up when the source stops sending", async () => { + const body = new ReadableStream({ + start(controller) { + controller.enqueue(MP4_HEAD); + }, + }); + const source = await openImportSource("https://media.example.com/a", { + isBlocked: publicNames, + request: async () => new Response(body, { status: 200 }), + stallTimeoutMs: 50, + }); + const started = Date.now(); + expect((await source.read()).done).toBe(false); + const error = await source.read().catch((caught) => caught); + const stop = importStopFor(error, source.stopped()); + source.close(); + expect(stop.code).toBe("timeout"); + expect(stop.message).toMatch(/stopped sending data/); + expect(Date.now() - started).toBeLessThan(2_000); + }); + + it("gives up when the headers never arrive", async () => { + expect( + await codeOf( + openImportSource("https://media.example.com/a", { + isBlocked: publicNames, + request: () => new Promise(() => {}), + stallTimeoutMs: 50, + }), + ), + ).toBe("timeout"); + }); + + it("gives up at the total deadline even while bytes keep coming", async () => { + let timer: ReturnType | undefined; + const body = new ReadableStream({ + start(controller) { + timer = setInterval(() => controller.enqueue(MP4_HEAD), 10); + }, + cancel() { + clearInterval(timer); + }, + }); + const source = await openImportSource("https://media.example.com/a", { + isBlocked: publicNames, + request: async () => new Response(body, { status: 200 }), + stallTimeoutMs: 1_000, + totalTimeoutMs: 150, + }); + let error: unknown; + try { + for (;;) await source.read(); + } catch (caught) { + error = caught; + } + const stop = importStopFor(error, source.stopped()); + source.close(); + clearInterval(timer); + expect(stop.code).toBe("timeout"); + expect(stop.message).toMatch(/did not finish downloading/); + }); +}); + +describe("streamSourceToParts", () => { + const chunks = (sizes: number[]) => + sizes.map((size, index) => new Uint8Array(size).fill(index + 1)); + + const sourceOf = (list: Uint8Array[]) => { + const reader = streamOf(list).getReader(); + return { read: () => reader.read() }; + }; + + it("cuts the stream into fixed parts with one upload in flight at a time", async () => { + let inFlight = 0; + let most = 0; + const sizes: number[] = []; + const writer = { + put: async (partNumber: number, bytes: Uint8Array) => { + inFlight++; + most = Math.max(most, inFlight); + sizes[partNumber - 1] = bytes.length; + await new Promise((resolve) => setTimeout(resolve, 5)); + inFlight--; + return `etag-${partNumber}`; + }, + }; + const result = await streamSourceToParts( + sourceOf(chunks([7, 3, 12, 1, 9])), + { chunks: [], length: 0, done: false }, + writer, + { maxBytes: 1_000, partBytes: 10 }, + ); + expect(result.totalBytes).toBe(32); + expect(sizes).toEqual([10, 10, 10, 2]); + expect(result.parts.map((part) => part.etag)).toEqual([ + "etag-1", + "etag-2", + "etag-3", + "etag-4", + ]); + expect(most).toBe(1); + }); + + it("stops mid-stream once the size cap is passed, with no Content-Length", async () => { + const put = vi.fn(async (partNumber: number) => `etag-${partNumber}`); + let delivered = 0; + const endless = { + read: async () => { + delivered += 8; + return { done: false as const, value: new Uint8Array(8) }; + }, + }; + const error = await streamSourceToParts( + endless, + { chunks: [], length: 0, done: false }, + { put }, + { maxBytes: 50, partBytes: 16 }, + ).catch((caught) => caught); + expect(error).toBeInstanceOf(ImportStop); + expect(error.code).toBe("too_large"); + expect(delivered).toBe(56); + expect(put).toHaveBeenCalledTimes(3); + }); + + it("reports a part that fails while the next read is still waiting", async () => { + const unhandled = vi.fn(); + process.on("unhandledRejection", unhandled); + try { + let reads = 0; + const slow = { + read: async () => { + reads++; + if (reads > 2) return { done: true as const, value: undefined }; + await new Promise((resolve) => setTimeout(resolve, 30)); + return { done: false as const, value: new Uint8Array(16) }; + }, + }; + const error = await streamSourceToParts( + slow, + { chunks: [], length: 0, done: false }, + { put: async () => Promise.reject(new Error("storage down")) }, + { maxBytes: 1_000, partBytes: 16 }, + ).catch((caught) => caught); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(error).toBeInstanceOf(Error); + expect(error.message).toBe("storage down"); + expect(unhandled).not.toHaveBeenCalled(); + } finally { + process.off("unhandledRejection", unhandled); + } + }); + + it("keeps the bytes it already read for the sniff", async () => { + const put = vi.fn(async (partNumber: number, bytes: Uint8Array) => { + expect(Array.from(bytes)).toEqual([1, 1, 2, 2, 2]); + return `etag-${partNumber}`; + }); + const result = await streamSourceToParts( + sourceOf([new Uint8Array([2, 2, 2])]), + { chunks: [new Uint8Array([1, 1])], length: 2, done: false }, + { put }, + { maxBytes: 100, partBytes: 64 }, + ); + expect(result.totalBytes).toBe(5); + expect(put).toHaveBeenCalledTimes(1); + }); +}); + +describe("headStop", () => { + it("lets a real MP4 head through", () => { + expect(headStop(MP4_HEAD)).toBeNull(); + }); +}); + +describe("isMissingImportTable", () => { + it("knows the MySQL missing-table error, wrapped or not", () => { + const mysql = Object.assign( + new Error("Table 'cap.developer_video_imports' doesn't exist"), + { errno: 1146, code: "ER_NO_SUCH_TABLE" }, + ); + expect(isMissingImportTable(mysql)).toBe(true); + expect( + isMissingImportTable(new Error("Failed query", { cause: mysql })), + ).toBe(true); + expect(isMissingImportTable(new Error("Connection lost"))).toBe(false); + }); +}); diff --git a/apps/web/__tests__/unit/infra-video-view.test.ts b/apps/web/__tests__/unit/infra-video-view.test.ts index d6e1efcedb3..7d39ffda169 100644 --- a/apps/web/__tests__/unit/infra-video-view.test.ts +++ b/apps/web/__tests__/unit/infra-video-view.test.ts @@ -13,13 +13,16 @@ import { developerVideoPlaybackUrls, } from "@/lib/developer-video-api"; import { + DEFAULT_EMBED_OPTIONS, INFRA_VIDEO_DISPLAY_LABELS, INFRA_VIDEO_FAILURE_FALLBACK, INFRA_VIDEO_PAGE_MAX, INFRA_VIDEO_PUBLIC_FAILURE, + INFRA_VIDEO_PUBLIC_FAILURE_DETAIL, infraRefreshDelayMs, infraVideoDisplayState, infraVideoEmbedCode, + infraVideoEmbedUrl, infraVideoFailure, infraVideoIsChanging, infraVideoListLimit, @@ -35,6 +38,7 @@ const row = (overrides: Parameters[0] = {}) => { return { ...video, legacy: video.playbackId === null && video.processing === null, + takenDown: false, }; }; @@ -49,6 +53,7 @@ const LIST_FIELDS = [ const DETAIL_FIELDS = [ ...LIST_FIELDS, + "captions", "embedCode", "failure", "height", @@ -114,6 +119,17 @@ describe("infraVideoFailure", () => { } }); + it("passes on the sentence about a disputed payment unchanged", () => { + const disputed = + "A payment for this app was disputed, so uploads are blocked and playback is paused. Contact us to restore it."; + + expect(infraVideoFailure(disputed)).toBe(disputed); + expect(infraVideoFailure(` ${disputed}\n`)).toBe(disputed); + expect(infraVideoFailure("A payment failed upstream")).toBe( + INFRA_VIDEO_FAILURE_FALLBACK, + ); + }); + it("replaces encoder wording with a sentence a customer can act on", () => { for (const reason of [ "Encoded MP4 was not stored", @@ -129,11 +145,19 @@ describe("infraVideoFailure", () => { }); describe("the sentence a viewer of the public page gets", () => { - it("is one sentence with no advice and nothing about the owner's account", () => { - expect(INFRA_VIDEO_PUBLIC_FAILURE).toBe("This video is not available."); - expect(INFRA_VIDEO_PUBLIC_FAILURE).not.toMatch( - /credit|balance|billing|payment|upload|export|owner/i, + it("is a heading and one line with no advice and nothing about the owner's account", () => { + expect(INFRA_VIDEO_PUBLIC_FAILURE).toBe("This video isn't available"); + expect(INFRA_VIDEO_PUBLIC_FAILURE_DETAIL).toBe( + "There is nothing to play at this link.", ); + for (const text of [ + INFRA_VIDEO_PUBLIC_FAILURE, + INFRA_VIDEO_PUBLIC_FAILURE_DETAIL, + ]) { + expect(text).not.toMatch( + /credit|balance|billing|payment|upload|export|owner|process/i, + ); + } }); }); @@ -176,6 +200,7 @@ describe("console video payload", () => { expect(detail.posterUrl).toContain("/playback-id-0123456789ab/"); expect(detail.stillEncoding).toBe(false); expect(detail.failure).toBeNull(); + expect(detail.captions).toEqual([]); const sent = JSON.stringify(detail); expect(sent).not.toContain("developer/app-1/video-1/video"); @@ -230,7 +255,7 @@ describe("console video payload", () => { expect(detail.shareUrl).toBe(PAGE); expect(detail.embedCode).toBe( - ``, + ``, ); }); @@ -260,6 +285,68 @@ describe("console video payload", () => { ).toMatchObject({ shareUrl: null, embedCode: null }); }); + it("builds the snippet from the console's embed options", () => { + const video = { name: "Demo", width: 1920, height: 1080 }; + const base = { ...DEFAULT_EMBED_OPTIONS }; + + expect(infraVideoEmbedCode(`${PAGE}/embed`, video, base)).toBe( + infraVideoEmbedCode(`${PAGE}/embed`, video), + ); + expect(infraVideoEmbedUrl(`${PAGE}/embed`, base)).toBe(`${PAGE}/embed`); + expect( + infraVideoEmbedUrl(`${PAGE}/embed`, { + color: "#F97316", + controls: false, + autoplay: true, + captions: true, + qualityMenu: false, + }), + ).toBe( + `${PAGE}/embed?color=f97316&controls=0&autoplay=1&muted=1&captions=1&quality_menu=0`, + ); + expect( + infraVideoEmbedUrl(`${PAGE}/embed`, { ...base, captions: true }), + ).toBe(`${PAGE}/embed?captions=1`); + }); + + it("keeps the iframe's attributes whatever the options", () => { + const video = { name: "Demo", width: 1920, height: 1080 }; + const attributes = (code: string) => code.replace(/ src="[^"]*"/, ""); + + expect( + attributes( + infraVideoEmbedCode(`${PAGE}/embed`, video, { + color: "#f97316", + controls: false, + autoplay: true, + captions: true, + qualityMenu: false, + }), + ), + ).toBe(attributes(infraVideoEmbedCode(`${PAGE}/embed`, video))); + }); + + it.each([ + 'red" onload="alert(1)', + "#12345g", + "3b82f6&autoplay=1", + "", + ])("leaves the hostile colour %j out of the snippet", (color) => { + const code = infraVideoEmbedCode( + `${PAGE}/embed`, + { name: "Demo", width: 1920, height: 1080 }, + { ...DEFAULT_EMBED_OPTIONS, color }, + ); + + expect(code).toBe( + infraVideoEmbedCode(`${PAGE}/embed`, { + name: "Demo", + width: 1920, + height: 1080, + }), + ); + }); + it("keeps a video name from breaking out of the title attribute", () => { const code = infraVideoEmbedCode(`${PAGE}/embed`, { name: 'Demo "one" & more', diff --git a/apps/web/__tests__/unit/infra-webhook-console.test.ts b/apps/web/__tests__/unit/infra-webhook-console.test.ts new file mode 100644 index 00000000000..4b13af56be0 --- /dev/null +++ b/apps/web/__tests__/unit/infra-webhook-console.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { infraWebhookLocks } from "@/lib/infra-webhook-console"; + +const idle = { adding: false, creating: false, secretShown: false }; + +describe("webhook console locks", () => { + it("allows every action when nothing is open", () => { + expect(infraWebhookLocks(idle)).toEqual({ + canAdd: true, + canRotate: true, + canDelete: true, + }); + }); + + it("blocks rotate and delete while an add is in flight, so its secret cannot be replaced", () => { + expect( + infraWebhookLocks({ ...idle, adding: true, creating: true }), + ).toEqual({ canAdd: false, canRotate: false, canDelete: false }); + }); + + it("blocks rotate while the add form is open", () => { + expect(infraWebhookLocks({ ...idle, adding: true })).toMatchObject({ + canAdd: false, + canRotate: false, + canDelete: true, + }); + }); + + it("blocks add and rotate until a shown secret is dismissed", () => { + expect(infraWebhookLocks({ ...idle, secretShown: true })).toEqual({ + canAdd: false, + canRotate: false, + canDelete: true, + }); + }); + + it("never lets a second secret start while one is shown or on its way", () => { + for (const adding of [false, true]) { + for (const creating of [false, true]) { + for (const secretShown of [false, true]) { + const locks = infraWebhookLocks({ adding, creating, secretShown }); + const secretBusy = adding || creating || secretShown; + expect(locks.canAdd || locks.canRotate).toBe(!secretBusy); + } + } + } + }); +}); diff --git a/apps/web/__tests__/unit/loom-download-url.test.ts b/apps/web/__tests__/unit/loom-download-url.test.ts index a6786025939..2acb181a51d 100644 --- a/apps/web/__tests__/unit/loom-download-url.test.ts +++ b/apps/web/__tests__/unit/loom-download-url.test.ts @@ -3,6 +3,7 @@ import { getLoomDownloadUrl, getReusableLoomDownloadUrl, LoomDownloadTemporaryError, + LoomDownloadUnavailableError, } from "@/lib/loom-download-url"; afterEach(() => { @@ -165,6 +166,35 @@ describe("getLoomDownloadUrl", () => { }, ); + it.each([ + { age: 25 * 60 * 60 * 1000, error: LoomDownloadUnavailableError }, + { age: 10 * 60 * 1000, error: LoomDownloadTemporaryError }, + ])( + "stops retrying once a video has had no source for a day: $age", + async ({ age, error }) => { + vi.stubGlobal( + "fetch", + vi.fn(async (input) => + String(input) === "https://www.loom.com/graphql" + ? Response.json({ + data: { + getVideo: { + id: "video", + createdAt: new Date(Date.now() - age).toISOString(), + downloadable: true, + download_enabled: true, + nullableRawCdnUrl: null, + __typename: "RegularUserVideo", + }, + }, + }) + : new Response("Conflict", { status: 409 }), + ), + ); + await expect(getLoomDownloadUrl("video")).rejects.toBeInstanceOf(error); + }, + ); + it.each([ { id: "video", downloadable: false, download_enabled: false }, { id: "different-video", downloadable: true, download_enabled: true }, diff --git a/apps/web/__tests__/unit/loom-import.test.ts b/apps/web/__tests__/unit/loom-import.test.ts index 1dc0947db2a..b8f857a4ec9 100644 --- a/apps/web/__tests__/unit/loom-import.test.ts +++ b/apps/web/__tests__/unit/loom-import.test.ts @@ -303,6 +303,33 @@ function withLimit(value: unknown) { }; } +const UNFINISHED_LOOM_MESSAGE = + "Loom has no video file for this recording, so it can't be imported. The recording may not have finished uploading to Loom."; + +function mockUnfinishedLoomRecording() { + vi.mocked(fetch).mockImplementation(async (input) => { + const url = typeof input === "string" ? input : input.toString(); + if (url === "https://www.loom.com/graphql") { + return Response.json({ + data: { + getVideo: { + id: "loom-abc1234567", + name: "Unfinished recording", + createdAt: new Date( + Date.now() - 3 * 24 * 60 * 60 * 1000, + ).toISOString(), + downloadable: true, + download_enabled: true, + nullableRawCdnUrl: null, + __typename: "RegularUserVideo", + }, + }, + }); + } + return new Response("Conflict", { status: 409 }); + }); +} + describe("importFromLoom", () => { beforeEach(() => { vi.clearAllMocks(); @@ -575,6 +602,18 @@ describe("importFromLoom", () => { }); }); + it("reports a Loom recording with no source file as a permanent failure", async () => { + mockUnfinishedLoomRecording(); + + const { downloadLoomVideo } = await import("@/actions/loom"); + + const result = await downloadLoomVideo( + "https://www.loom.com/share/loom-abc1234567", + ); + + expect(result).toEqual({ success: false, error: UNFINISHED_LOOM_MESSAGE }); + }); + it("returns streaming Loom URLs for browser conversion instead of proxying", async () => { const fetchMock = vi.mocked(fetch); fetchMock.mockImplementation(async (input) => { @@ -1294,6 +1333,43 @@ describe("importFromLoom", () => { expect(startMock).not.toHaveBeenCalled(); }); + it("does not mark CSV rows for a Loom recording with no source file as retryable", async () => { + whereMock.mockImplementation((conditions: unknown) => { + if (JSON.stringify(conditions).includes("sourceId")) { + return Promise.resolve([]); + } + + return withLimit([{ userId: "member-123", email: "member@example.com" }]); + }); + mockUnfinishedLoomRecording(); + + const { importFromLoomCsv } = await import("@/actions/loom"); + + const result = await importFromLoomCsv({ + orgId: "org-1" as never, + rows: [ + { + rowNumber: 2, + loomUrl: "https://www.loom.com/share/loom-abc1234567", + userEmail: "member@example.com", + }, + ], + }); + + expect(result.results).toStrictEqual([ + { + rowNumber: 2, + userEmail: "member@example.com", + spaceName: undefined, + success: false, + videoId: undefined, + error: UNFINISHED_LOOM_MESSAGE, + }, + ]); + expect(result.results[0]).not.toHaveProperty("errorCode"); + expect(startMock).not.toHaveBeenCalled(); + }); + it("limits CSV imports to 500 rows", async () => { const fetchMock = vi.mocked(fetch); const { importFromLoomCsv } = await import("@/actions/loom"); diff --git a/apps/web/__tests__/unit/media-client-jobs.test.ts b/apps/web/__tests__/unit/media-client-jobs.test.ts index 36394e9fac3..3b7d66ee2ce 100644 --- a/apps/web/__tests__/unit/media-client-jobs.test.ts +++ b/apps/web/__tests__/unit/media-client-jobs.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { + cancelMediaServerVideoJob, readMediaServerVideoJob, startMediaServerVideoJob, tryProbeVideoViaMediaServer, @@ -249,7 +250,17 @@ describe("tryProbeVideoViaMediaServer", () => { setup: () => mockFetch.mockRejectedValue(networkError("ECONNREFUSED")), }, { name: "a 503", setup: () => mockFetch.mockResolvedValue(json({}, 503)) }, - { name: "a 504", setup: () => mockFetch.mockResolvedValue(json({}, 504)) }, + { + name: "a 504 from in front of the media server", + setup: () => mockFetch.mockResolvedValue(json({}, 504)), + }, + { + name: "a 504 without a JSON body", + setup: () => + mockFetch.mockResolvedValue( + new Response("Gateway Timeout", { status: 504 }), + ), + }, ])("reports $name as unreachable without retrying", async ({ setup }) => { setup(); @@ -259,6 +270,24 @@ describe("tryProbeVideoViaMediaServer", () => { expect(mockFetch).toHaveBeenCalledTimes(1); }); + it("reports the media server giving up on the file as timed out", async () => { + mockFetch.mockResolvedValue( + json( + { + error: "Request timed out", + code: "TIMEOUT", + details: "Operation timed out", + }, + 504, + ), + ); + + expect(await tryProbeVideoViaMediaServer("https://storage/video")).toEqual({ + status: "timed_out", + }); + expect(mockFetch).toHaveBeenCalledTimes(1); + }); + it("reports a file the media server cannot read as rejected", async () => { mockFetch.mockResolvedValue( json({ error: "Probe failed", details: "Invalid data found" }, 500), @@ -289,3 +318,72 @@ describe("tryProbeVideoViaMediaServer", () => { }); }); }); + +describe("cancelMediaServerVideoJob", () => { + it("asks the media server to cancel the job with the shared secret", async () => { + mockFetch.mockResolvedValue(json({ success: true })); + + expect(await cancelMediaServerVideoJob("job/1")).toBe("cancelled"); + + const [url, init] = mockFetch.mock.calls[0] ?? []; + expect(url).toBe("http://media.example/video/process/job%2F1/cancel"); + expect(init).toMatchObject({ + method: "POST", + headers: { "x-media-server-secret": "test-secret" }, + redirect: "error", + }); + expect(init.signal).toBeInstanceOf(AbortSignal); + }); + + it.each([ + { + name: "a job that already finished", + response: () => + json({ error: "Job already finished", code: "INVALID_STATE" }, 400), + result: "finished", + }, + { + name: "a job no media server knows", + response: () => json({ error: "Job not found", code: "NOT_FOUND" }, 404), + result: "missing", + }, + { + name: "a 404 that did not come from the media server", + response: () => new Response("Application not found", { status: 404 }), + result: "failed", + }, + { + name: "a server error", + response: () => json({ error: "boom" }, 500), + result: "failed", + }, + ])("reports $name as $result", async ({ response, result }) => { + mockFetch.mockResolvedValue(response()); + + expect(await cancelMediaServerVideoJob("job-1")).toBe(result); + }); + + it("reports a media server it cannot reach as failed", async () => { + mockFetch.mockRejectedValue(networkError("ECONNREFUSED")); + + expect(await cancelMediaServerVideoJob("job-1")).toBe("failed"); + }); + + it("gives up after five seconds", async () => { + const timeout = vi.spyOn(AbortSignal, "timeout"); + timeout.mockImplementation(() => AbortSignal.abort()); + mockFetch.mockImplementation( + (_url: string, init: RequestInit) => + new Promise((_resolve, reject) => { + if (init.signal?.aborted) reject(init.signal.reason); + init.signal?.addEventListener("abort", () => + reject(init.signal?.reason), + ); + }), + ); + + expect(await cancelMediaServerVideoJob("job-1")).toBe("failed"); + expect(timeout).toHaveBeenCalledWith(5_000); + timeout.mockRestore(); + }); +}); diff --git a/apps/web/__tests__/unit/media-container.test.ts b/apps/web/__tests__/unit/media-container.test.ts index 701c3e2c0b5..1b3170ec170 100644 --- a/apps/web/__tests__/unit/media-container.test.ts +++ b/apps/web/__tests__/unit/media-container.test.ts @@ -1,8 +1,10 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { isSelfContainedMedia, looksLikePlaylist, + MEDIA_CONTAINER_SNIFF_BYTES, sniffMedia, + sniffMediaUrl, } from "@/lib/media-container"; const text = (value: string) => new TextEncoder().encode(value); @@ -166,3 +168,247 @@ describe("playlist check", () => { expect(sniffMedia(withAscii([0, 0, 0, 0x20], "ftypisom"))).toBe("media"); }); }); + +const syncsafe = (size: number) => [ + (size >> 21) & 0x7f, + (size >> 14) & 0x7f, + (size >> 7) & 0x7f, + size & 0x7f, +]; +const FOOTER = 0x10; +const id3 = (bodyBytes = 0, { flags = 0, version = 4 } = {}) => + Uint8Array.from([ + ...text("ID3"), + version, + 0, + flags, + ...syncsafe(bodyBytes), + ...new Array(bodyBytes).fill(0), + ...(flags & FOOTER + ? [...text("3DI"), version, 0, flags, ...syncsafe(bodyBytes)] + : []), + ]); +const join = (...parts: Uint8Array[]) => + Uint8Array.from(parts.flatMap((part) => [...part])); +const padded = (head: number[], length = 64) => + Uint8Array.from([...head, ...new Array(length).fill(0)]); + +const MPEG_FRAMES = padded([0xff, 0xfb, 0x90, 0x00]); +const ADTS_FRAMES = padded([0xff, 0xf1, 0x50, 0x80]); +const FFCONCAT = text("ffconcat version 1.0\nfile 'http://127.0.0.1/x'\n"); +const HLS = text("#EXTM3U\n#EXT-X-VERSION:3\nhttp://10.0.0.1/a.ts\n"); +const DASH = text( + '', +); +const DASH_WITHOUT_PROLOG = text(''); +const PLAYLISTS: [string, Uint8Array][] = [ + ["concat script", FFCONCAT], + ["hls playlist", HLS], + ["dash manifest", DASH], + ["dash manifest without a prolog", DASH_WITHOUT_PROLOG], +]; +const ALBUM_ART_BYTES = 100_000; + +describe("a file that starts with an ID3 tag", () => { + const tags: [string, Uint8Array][] = [ + ["an empty tag", id3()], + ["a tag with frames", id3(300)], + ["a tag with a footer", id3(40, { flags: FOOTER })], + ["a version 2.2 tag", id3(0, { version: 2 })], + ["a version 2.3 tag", id3(120, { version: 3 })], + ["two tags in a row", join(id3(20), id3(30))], + ]; + + it.each(tags)("refuses a playlist behind %s", (_name, tag) => { + for (const [name, playlist] of PLAYLISTS) { + const sample = join(tag, playlist); + expect(isSelfContainedMedia(sample), name).toBe(false); + expect(looksLikePlaylist(sample), name).toBe(true); + expect(sniffMedia(sample), name).toBe("playlist"); + } + }); + + it.each(tags)("accepts audio behind %s", (_name, tag) => { + const audio: [string, Uint8Array][] = [ + ["mpeg audio frames", MPEG_FRAMES], + ["adts frames", ADTS_FRAMES], + ["flac", text("fLaC\u0000\u0000\u0000\u0022")], + ["padding before the first frame", join(new Uint8Array(32), MPEG_FRAMES)], + ]; + for (const [name, frames] of audio) { + const sample = join(tag, frames); + expect(isSelfContainedMedia(sample), name).toBe(true); + expect(looksLikePlaylist(sample), name).toBe(false); + expect(sniffMedia(sample), name).toBe("media"); + } + }); + + it("accepts an mp3 with no tag", () => { + expect(sniffMedia(MPEG_FRAMES)).toBe("media"); + expect(sniffMedia(ADTS_FRAMES)).toBe("media"); + }); + + it("accepts a tag of album art that fills the bytes it was given", () => { + const head = join(id3(ALBUM_ART_BYTES), MPEG_FRAMES).subarray( + 0, + MEDIA_CONTAINER_SNIFF_BYTES, + ); + + expect(isSelfContainedMedia(head)).toBe(true); + expect(sniffMedia(head)).toBe("media"); + }); + + it("accepts a tag that ends exactly where the bytes do", () => { + expect(sniffMedia(id3(300))).toBe("media"); + expect(sniffMedia(id3())).toBe("media"); + }); + + it("reads a header it cannot parse as it did before", () => { + const cutShort = text("ID3\u0004\u0000\u0000\u0000"); + const sizeNotSyncsafe = join( + Uint8Array.from([...text("ID3"), 4, 0, 0, 0x80, 0, 0, 0]), + FFCONCAT, + ); + + expect(sniffMedia(cutShort)).toBe("media"); + expect(sniffMedia(sizeNotSyncsafe)).toBe("media"); + }); + + it("still refuses a playlist with no tag in front", () => { + for (const [name, playlist] of PLAYLISTS) { + expect(sniffMedia(playlist), name).toBe("playlist"); + } + }); +}); + +describe("sniffing a stored file by URL", () => { + const URL = "https://storage.test/developer/app/video"; + + function serve(file: Uint8Array, { ranges = true } = {}) { + const fetched = vi.fn(async (_url: string, init?: RequestInit) => { + const range = new Headers(init?.headers).get("range") ?? ""; + const [, from = "0", to = "0"] = /^bytes=(\d+)-(\d+)$/.exec(range) ?? []; + const start = Number(from); + if (!ranges && start > 0) + return new Response(file.slice(), { status: 200 }); + if (start >= file.length) return new Response(null, { status: 416 }); + return new Response(file.slice(start, Number(to) + 1), { status: 206 }); + }); + vi.stubGlobal("fetch", fetched); + return { + ranges: () => + fetched.mock.calls.map(([, init]) => + new Headers(init?.headers).get("range"), + ), + }; + } + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("reads the first bytes once for a file with no tag", async () => { + const stored = serve(withAscii([0, 0, 0, 0x20], "ftypisom", 8000)); + + expect(await sniffMediaUrl(URL)).toBe("media"); + expect(stored.ranges()).toEqual(["bytes=0-4095"]); + }); + + it("reads the first bytes once when a short tag leaves the audio in view", async () => { + const stored = serve(join(id3(60), padded([0xff, 0xfb, 0x90, 0x00], 8000))); + + expect(await sniffMediaUrl(URL)).toBe("media"); + expect(stored.ranges()).toEqual(["bytes=0-4095"]); + }); + + it("reads again at the end of a tag of album art and accepts the audio there", async () => { + const stored = serve(join(id3(ALBUM_ART_BYTES), MPEG_FRAMES)); + + expect(await sniffMediaUrl(URL)).toBe("media"); + expect(stored.ranges()).toEqual([ + "bytes=0-4095", + `bytes=${ALBUM_ART_BYTES + 10}-${ALBUM_ART_BYTES + 10 + 4095}`, + ]); + }); + + it.each(PLAYLISTS)( + "refuses a %s behind a tag longer than the first read", + async (_name, playlist) => { + const stored = serve(join(id3(ALBUM_ART_BYTES), playlist)); + + expect(await sniffMediaUrl(URL)).toBe("playlist"); + expect(stored.ranges()).toHaveLength(2); + }, + ); + + it.each(PLAYLISTS)( + "refuses a %s behind a short tag", + async (_name, playlist) => { + const stored = serve(join(id3(), playlist)); + + expect(await sniffMediaUrl(URL)).toBe("playlist"); + expect(stored.ranges()).toEqual(["bytes=0-4095"]); + }, + ); + + it("accepts audio that sits behind padding after a tag of album art, as a short tag does", async () => { + const padding = new Uint8Array(32); + serve(join(id3(ALBUM_ART_BYTES), padding, MPEG_FRAMES)); + expect(await sniffMediaUrl(URL)).toBe("media"); + + serve(join(id3(60), padding, MPEG_FRAMES)); + expect(await sniffMediaUrl(URL)).toBe("media"); + }); + + it("still refuses a playlist behind padding after a tag of album art", async () => { + serve(join(id3(ALBUM_ART_BYTES), text("\n\n "), HLS)); + + expect(await sniffMediaUrl(URL)).toBe("playlist"); + }); + + it("refuses a playlist that starts in the last bytes of the first read", async () => { + const tag = id3(MEDIA_CONTAINER_SNIFF_BYTES - 10 - 3); + const stored = serve(join(tag, HLS, new Uint8Array(8000))); + + expect(await sniffMediaUrl(URL)).toBe("playlist"); + expect(stored.ranges()).toEqual([ + "bytes=0-4095", + `bytes=${tag.length}-${tag.length + 4095}`, + ]); + }); + + it("follows several long tags to what comes after them", async () => { + const tags = join(id3(9000), id3(9000), id3(9000)); + + serve(join(tags, HLS)); + expect(await sniffMediaUrl(URL)).toBe("playlist"); + + const stored = serve(join(tags, MPEG_FRAMES)); + expect(await sniffMediaUrl(URL)).toBe("media"); + expect(stored.ranges()).toHaveLength(4); + }); + + it("accepts a tag that says it is longer than the file, as before", async () => { + const stored = serve(id3(ALBUM_ART_BYTES).subarray(0, 6000)); + + expect(await sniffMediaUrl(URL)).toBe("media"); + expect(stored.ranges()).toHaveLength(2); + }); + + it("accepts the file as before when storage answers the second read with the whole file", async () => { + serve(join(id3(ALBUM_ART_BYTES), HLS), { ranges: false }); + + expect(await sniffMediaUrl(URL)).toBe("media"); + }); + + it("fails when a read is refused", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(null, { status: 403 })), + ); + + await expect(sniffMediaUrl(URL)).rejects.toThrow( + "Could not read the stored file (403)", + ); + }); +}); diff --git a/apps/web/__tests__/unit/messenger-launcher-dashboard.test.ts b/apps/web/__tests__/unit/messenger-launcher-dashboard.test.ts new file mode 100644 index 00000000000..764d1f66b7a --- /dev/null +++ b/apps/web/__tests__/unit/messenger-launcher-dashboard.test.ts @@ -0,0 +1,115 @@ +// @vitest-environment jsdom + +import { act, createElement } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const state = vi.hoisted(() => ({ pathname: "/" as string | null })); + +vi.mock("next/navigation", () => ({ usePathname: () => state.pathname })); +vi.mock("@/actions/messenger", () => ({ + fetchMessengerConversation: vi.fn(async () => null), + fetchMessengerConversations: vi.fn(async () => []), + createMessengerConversation: vi.fn(async () => null), + sendMessengerUserMessage: vi.fn(), +})); +vi.mock("next/image", () => ({ + default: ({ src, alt }: { src: string; alt: string }) => + createElement("img", { src, alt }), +})); + +import { MessengerWidget } from "@/app/Layout/MessengerWidget"; + +let root: Root; +let container: HTMLDivElement; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + vi.stubGlobal("requestAnimationFrame", () => 1); + container = document.createElement("div"); + document.body.appendChild(container); + root = createRoot(container); +}); + +afterEach(() => { + act(() => root.unmount()); + container.remove(); + state.pathname = "/"; +}); + +const render = (pathname: string | null) => { + state.pathname = pathname; + act(() => root.render(createElement(MessengerWidget))); + const launcher = container.querySelector("button[aria-label]"); + if (!launcher) throw new Error("launcher not rendered"); + return launcher; +}; + +const open = async (launcher: Element) => { + await act(async () => { + (launcher as HTMLButtonElement).click(); + }); + return container.querySelector(".cap-messenger-panel"); +}; + +describe("messenger launcher above the dashboard's mobile tab bar", () => { + it.each([ + "/dashboard", + "/dashboard/caps", + "/dashboard/settings/organization", + ])( + "lifts the launcher and the panel below the lg breakpoint on %s", + async (pathname) => { + const launcher = render(pathname); + expect(launcher.className).toContain("max-lg:bottom-[84px]"); + expect(launcher.className).toContain("bottom-5"); + + const panel = await open(launcher); + expect(panel?.classList.contains("cap-messenger-panel-dashboard")).toBe( + true, + ); + }, + ); + + it.each(["/", "/pricing", "/s/abc123", "/dashboardish", null])( + "leaves the launcher and panel where they were on %s", + async (pathname) => { + const launcher = render(pathname); + expect(launcher.className).not.toContain("max-lg:"); + expect(launcher.className).toContain("bottom-5"); + + const panel = await open(launcher); + expect(panel?.classList.contains("cap-messenger-panel-dashboard")).toBe( + false, + ); + }, + ); + + const widgetCss = () => + ( + render("/dashboard/caps").parentElement?.querySelector("style") + ?.textContent ?? "" + ).replace(/\s+/g, " "); + + it("lifts the panel by the tab bar's 64 px below the lg breakpoint", () => { + const css = widgetCss(); + const block = css.match( + /@media \(max-width: 1023\.98px\) \{ \.cap-messenger-panel-dashboard \{ ([^}]*) \} \}/, + ); + expect(block?.[1]).toContain("bottom: 152px;"); + expect(block?.[1]).toContain("height: min(760px, calc(100dvh - 172px));"); + expect(css).toContain("bottom: 88px;"); + expect(css).toContain("height: min(760px, calc(100dvh - 108px));"); + }); + + it("keeps the full screen panel on phones above the dashboard rule", () => { + const css = widgetCss(); + const dashboardRule = css.indexOf(".cap-messenger-panel-dashboard"); + const phoneRule = css.indexOf("@media (max-width: 480px)"); + expect(dashboardRule).toBeGreaterThan(-1); + expect(phoneRule).toBeGreaterThan(dashboardRule); + expect(css.slice(phoneRule)).toMatch( + /\.cap-messenger-panel \{ width: 100vw; height: 100dvh; right: 0; bottom: 0;/, + ); + }); +}); diff --git a/apps/web/__tests__/unit/messenger-poll-ordering.test.ts b/apps/web/__tests__/unit/messenger-poll-ordering.test.ts index 7488e0cada9..862227de41d 100644 --- a/apps/web/__tests__/unit/messenger-poll-ordering.test.ts +++ b/apps/web/__tests__/unit/messenger-poll-ordering.test.ts @@ -18,6 +18,7 @@ vi.mock("next/image", () => ({ default: ({ src, alt }: { src: string; alt: string }) => createElement("img", { src, alt }), })); +vi.mock("next/navigation", () => ({ usePathname: () => "/" })); vi.mock("next/link", () => ({ default: ({ children, href }: { children: ReactNode; href: string }) => createElement("a", { href }, children), diff --git a/apps/web/__tests__/unit/move-items-dialog.test.ts b/apps/web/__tests__/unit/move-items-dialog.test.ts index a47c922b12d..6144d2c0531 100644 --- a/apps/web/__tests__/unit/move-items-dialog.test.ts +++ b/apps/web/__tests__/unit/move-items-dialog.test.ts @@ -1,7 +1,8 @@ // @vitest-environment jsdom -import { Folder, Organisation, Video } from "@cap/web-domain"; +import { Folder, Organisation, Space, Video } from "@cap/web-domain"; import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { Option } from "effect"; import React, { act } from "react"; import { createRoot, type Root } from "react-dom/client"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; @@ -14,6 +15,7 @@ const mocks = vi.hoisted(() => ({ moveFolder: vi.fn(), refresh: vi.fn(), close: vi.fn(), + create: vi.fn(), })); vi.mock("@/actions/folders/move-items", () => ({ getOwnedVideoMoveDestinations: mocks.destinations, @@ -25,9 +27,22 @@ vi.mock("@/actions/folders/move-items", () => ({ vi.mock("next/navigation", () => ({ useRouter: () => ({ refresh: mocks.refresh }), })); +vi.mock("@/app/(org)/dashboard/DashboardContext", () => ({ + useDashboardContext: () => ({ + activeOrganization: { organization: { id: "org" } }, + }), +})); vi.mock("@/app/Layout/AuthContext", () => ({ useCurrentUser: () => ({ id: "owner", activeOrganizationId: "org" }), })); +vi.mock("@/lib/EffectRuntime", async () => { + const { useMutation } = await import("@tanstack/react-query"); + return { + useRpcClient: () => ({ FolderCreate: mocks.create }), + useEffectMutation: (options: Parameters[0]) => + useMutation(options), + }; +}); vi.mock("@cap/utils", async () => await import("@cap/utils/helpers")); vi.mock("@cap/ui", async () => ({ ...(await import("../../../../packages/ui/src/components/Button")), @@ -46,12 +61,21 @@ const folderId = Folder.FolderId.make("team-folder"); beforeEach(() => { vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + mocks.destinations.mockReset(); + mocks.create.mockReset(); + mocks.place.mockReset(); mocks.destinations.mockResolvedValue([ - { name: "My Caps", location: { type: "personal" }, folders: [] }, { - name: "All team members", + name: "My Caps", + location: { type: "personal" }, + folders: [], + canCreateFolder: true, + }, + { + name: "Acme", location: { type: "organization" }, folders: [{ id: folderId, name: "Team folder", parentId: null }], + canCreateFolder: true, }, ]); mocks.folders.mockResolvedValue([]); @@ -104,12 +128,24 @@ function button(label: string) { if (!element) throw new Error(`Button not found: ${label}`); return element; } +function destination(name: string) { + const element = [ + ...document.querySelectorAll("button[aria-pressed]"), + ].find((candidate) => candidate.textContent?.startsWith(name)); + if (!element) throw new Error(`Destination not found: ${name}`); + return element; +} async function chooseTeam() { - const select = document.querySelector("select"); - if (!select) throw new Error("Location selector not found"); + await act(async () => destination("Acme").click()); +} +async function type(input: HTMLInputElement, value: string) { + const setValue = Object.getOwnPropertyDescriptor( + HTMLInputElement.prototype, + "value", + )?.set; await act(async () => { - select.value = "organization"; - select.dispatchEvent(new Event("change", { bubbles: true })); + setValue?.call(input, value); + input.dispatchEvent(new Event("input", { bubbles: true })); }); } @@ -118,11 +154,13 @@ describe("moving Caps from My Caps to a team location", () => { await render(); expect(button("Move").disabled).toBe(true); await chooseTeam(); + expect(destination("Acme").getAttribute("aria-pressed")).toBe("true"); + expect(destination("My Caps").getAttribute("aria-pressed")).toBe("false"); expect(document.body.textContent).toContain( - "People with access to All team members", + "People in Acme will be able to watch it", ); await act(async () => button("Team folder").click()); - await act(async () => button("Share & move").click()); + await act(async () => button("Add to Acme").click()); expect(mocks.place).toHaveBeenCalledWith({ videoIds: [videoId], folderId, @@ -134,8 +172,8 @@ describe("moving Caps from My Caps to a team location", () => { it("allows the team root even when the Cap starts at the personal root", async () => { await render(); await chooseTeam(); - expect(button("Share & move").disabled).toBe(false); - await act(async () => button("Share & move").click()); + expect(button("Add to Acme").disabled).toBe(false); + await act(async () => button("Add to Acme").click()); expect(mocks.place).toHaveBeenCalledWith({ videoIds: [videoId], folderId: null, @@ -148,7 +186,7 @@ describe("moving Caps from My Caps to a team location", () => { await render(undefined, organizationId); expect(mocks.destinations).toHaveBeenCalledWith(organizationId); await chooseTeam(); - await act(async () => button("Share & move").click()); + await act(async () => button("Add to Acme").click()); expect(mocks.place).toHaveBeenCalledWith({ videoIds: [videoId], folderId: null, @@ -159,7 +197,7 @@ describe("moving Caps from My Caps to a team location", () => { it("keeps folder moves within their original location", async () => { await render({ type: "folder", folderId, currentParentId: null }); - expect(document.querySelector("select")).toBeNull(); + expect(document.querySelector("[aria-pressed]")).toBeNull(); expect(mocks.destinations).not.toHaveBeenCalled(); expect(mocks.folders).toHaveBeenCalledWith({ type: "personal" }); }); @@ -175,7 +213,7 @@ describe("reopening the Move dialog", () => { mocks.destinations.mockResolvedValue([ { name: "My Caps", location: { type: "personal" }, folders: [] }, { - name: "All team members", + name: "Acme", location: { type: "organization" }, folders: [ { id: folderId, name: "Team folder", parentId: null }, @@ -194,3 +232,103 @@ describe("reopening the Move dialog", () => { expect(document.body.textContent).toContain("New team folder"); }); }); + +describe("creating a folder while moving", () => { + it("creates the folder in the chosen space and selects it", async () => { + const newFolderId = Folder.FolderId.make("seo-folder"); + mocks.create.mockResolvedValue(undefined); + await render(); + await chooseTeam(); + await act(async () => button("New folder").click()); + const input = document.querySelector( + 'input[aria-label="New folder name"]', + ); + if (!input) throw new Error("Folder name field missing"); + await type(input, "SEO"); + + mocks.destinations.mockResolvedValue([ + { name: "My Caps", location: { type: "personal" }, folders: [] }, + { + name: "Acme", + location: { type: "organization" }, + folders: [ + { id: folderId, name: "Team folder", parentId: null }, + { id: newFolderId, name: "SEO", parentId: null }, + ], + canCreateFolder: true, + }, + ]); + await act(async () => button("Create").click()); + await act(async () => { + await new Promise((resolve) => setTimeout(resolve, 30)); + }); + + const call = mocks.create.mock.calls[0]?.[0]; + expect(call?.name).toBe("SEO"); + expect(call?.color).toBe("normal"); + expect(call?.public).toBe(false); + expect(call && Option.getOrNull(call.spaceId)).toBe("org"); + expect(call && Option.getOrNull(call.parentId)).toBeNull(); + + await act(async () => button("Add to Acme").click()); + expect(mocks.place).toHaveBeenCalledWith({ + videoIds: [videoId], + folderId: newFolderId, + location: { type: "organization" }, + }); + }); + + it("hides folder creation where the person cannot create folders", async () => { + mocks.destinations.mockResolvedValue([ + { name: "My Caps", location: { type: "personal" }, folders: [] }, + { + name: "Acme", + location: { type: "organization" }, + folders: [], + canCreateFolder: false, + }, + ]); + await render(); + await chooseTeam(); + expect( + [...document.querySelectorAll("button")].some( + (element) => element.textContent?.trim() === "New folder", + ), + ).toBe(false); + }); +}); + +describe("choosing between many places", () => { + it("filters places by name once there are more than six", async () => { + const spaces = Array.from({ length: 8 }, (_, index) => ({ + name: index === 5 ? "Customer support" : `Team ${index + 1}`, + location: { + type: "space" as const, + spaceId: Space.SpaceId.make(`space-${index}`), + }, + folders: [], + })); + mocks.destinations.mockResolvedValue([ + { name: "My Caps", location: { type: "personal" }, folders: [] }, + ...spaces, + ]); + await render(); + const input = document.querySelector( + 'input[aria-label="Search places"]', + ); + if (!input) throw new Error("Place search missing"); + expect(document.activeElement).not.toBe(input); + await type(input, "support"); + const visible = [...document.querySelectorAll("button[aria-pressed]")].map( + (element) => element.getAttribute("title"), + ); + expect(visible).toEqual(["Customer support"]); + }); + + it("leaves place search out for small workspaces", async () => { + await render(); + expect( + document.querySelector('input[aria-label="Search places"]'), + ).toBeNull(); + }); +}); diff --git a/apps/web/__tests__/unit/move-items.test.ts b/apps/web/__tests__/unit/move-items.test.ts index 943bb6a6b81..bcec7e90bb0 100644 --- a/apps/web/__tests__/unit/move-items.test.ts +++ b/apps/web/__tests__/unit/move-items.test.ts @@ -1,8 +1,15 @@ -import { Folder, Organisation, Space } from "@cap/web-domain"; +import { Folder, Organisation, Space, Video } from "@cap/web-domain"; import { describe, expect, it } from "vitest"; import { buildMoveFolderDestinationRows, + describeDraggedCaps, + MAX_MOVE_ITEMS, + moveLocationFolderSpaceId, + parseCapDrag, + parseFolderDrag, resolveMoveLocation, + serializeCapDrag, + serializeFolderDrag, } from "@/lib/move-items"; const folderId = Folder.FolderId.make; @@ -65,4 +72,75 @@ describe("move item helpers", () => { expect(rows).toHaveLength(2); }); + + it("maps a move location to the space id new folders are created in", () => { + const organizationId = Organisation.OrganisationId.make("org-1"); + expect( + moveLocationFolderSpaceId({ type: "personal" }, organizationId), + ).toBe(null); + expect( + moveLocationFolderSpaceId({ type: "organization" }, organizationId), + ).toBe("org-1"); + expect( + moveLocationFolderSpaceId( + { type: "space", spaceId: Space.SpaceId.make("space-1") }, + organizationId, + ), + ).toBe("space-1"); + }); +}); + +describe("drag payloads", () => { + const cap = { id: Video.VideoId.make("a"), name: "Demo" }; + + it("drags the whole selection when the dragged Cap is part of it", () => { + const selected = ["a", "b", "b", "c"].map((id) => Video.VideoId.make(id)); + const payload = parseCapDrag(serializeCapDrag(cap, selected)); + expect(payload).toEqual({ id: "a", name: "Demo", ids: ["a", "b", "c"] }); + expect(payload && describeDraggedCaps(payload)).toBe("3 Caps"); + }); + + it("drags only the Cap itself when it is not selected", () => { + const payload = parseCapDrag( + serializeCapDrag(cap, [Video.VideoId.make("b")]), + ); + expect(payload?.ids).toEqual(["a"]); + expect(payload && describeDraggedCaps(payload)).toBe('"Demo"'); + }); + + it("reads payloads written before multi-select dragging", () => { + expect(parseCapDrag(JSON.stringify({ id: "a", name: "Demo" }))).toEqual({ + id: "a", + name: "Demo", + ids: ["a"], + }); + }); + + it("caps the number of dragged Caps at the move limit", () => { + const selected = Array.from({ length: MAX_MOVE_ITEMS + 5 }, (_, index) => + Video.VideoId.make(index === 0 ? "a" : `v${index}`), + ); + expect(parseCapDrag(serializeCapDrag(cap, selected))?.ids).toHaveLength( + MAX_MOVE_ITEMS, + ); + }); + + it.each(["", "not json", "[]", "null", '{"name":"x"}', '{"id":3}'])( + "ignores a malformed Cap payload %#", + (raw) => { + expect(parseCapDrag(raw)).toBeNull(); + }, + ); + + it("round-trips a folder payload and rejects one without a location", () => { + const payload = { + id: folderId("f1"), + name: "SEO", + locationKey: "personal", + }; + expect(parseFolderDrag(serializeFolderDrag(payload))).toEqual(payload); + expect(parseFolderDrag(JSON.stringify({ id: "f1", name: "SEO" }))).toBe( + null, + ); + }); }); diff --git a/apps/web/__tests__/unit/move-owned-videos.test.ts b/apps/web/__tests__/unit/move-owned-videos.test.ts index ffc09227f1d..18ff6219dd9 100644 --- a/apps/web/__tests__/unit/move-owned-videos.test.ts +++ b/apps/web/__tests__/unit/move-owned-videos.test.ts @@ -295,7 +295,15 @@ describe("placing owned Caps in team folders", () => { it("limits the selectable folders to personal and accessible team locations", async () => { results = [ - [{ id: spaceId, name: "Team space" }], + [{ name: "Acme" }], + [ + { + id: spaceId, + name: "Team space", + createdById: "someone", + memberRole: "member", + }, + ], [ { id: "mine", name: "Personal", parentId: null, spaceId: null }, { id: folderId, name: "Team folder", parentId: null, spaceId: "org" }, @@ -307,14 +315,88 @@ describe("placing owned Caps in team folders", () => { destinations.map((group) => [ group.name, group.folders.map((folder) => folder.id), + group.canCreateFolder, ]), ).toEqual([ - ["My Caps", ["mine"]], - ["All team members", [folderId]], - ["Team space", ["space-folder"]], + ["My Caps", ["mine"], true], + ["Acme", [folderId], false], + ["Team space", ["space-folder"], false], + ]); + expect(destinations[1]?.folders[0]).toEqual({ + id: folderId, + name: "Team folder", + parentId: null, + color: undefined, + }); + expect(filters[0]?.params).toEqual(["org"]); + expect(filters[1]?.params).toEqual(["org", "owner", "owner"]); + expect(filters[2]?.params).toEqual(["org", "owner", "org", spaceId]); + }); + + it("offers folder creation only where the backend would allow it", async () => { + results = [ + [{ name: "Acme" }], + [ + { + id: spaceId, + name: "Own space", + createdById: "owner", + memberRole: null, + }, + ], + [], + ]; + const asMember = await getOwnedVideoMoveDestinations(); + expect(asMember.map((group) => group.canCreateFolder)).toEqual([ + true, + false, + true, + ]); + + mocks.organization.mockResolvedValue({ role: "admin" }); + results = [ + [{ name: " " }], + [ + { + id: spaceId, + name: "Team space", + createdById: "someone", + memberRole: null, + }, + ], + [], + ]; + const asAdmin = await getOwnedVideoMoveDestinations(); + expect(asAdmin.map((group) => group.canCreateFolder)).toEqual([ + true, + true, + true, + ]); + expect(asAdmin[1]?.name).toBe("Everyone in the organization"); + }); + + it("never offers folder creation in an organization that is not active", async () => { + mocks.organization.mockResolvedValue({ role: "owner" }); + results = [ + [{ name: "Other" }], + [ + { + id: spaceId, + name: "Team space", + createdById: "owner", + memberRole: "admin", + }, + ], + [], + ]; + const destinations = await getOwnedVideoMoveDestinations( + Organisation.OrganisationId.make("other-org"), + ); + expect(destinations.map((group) => group.canCreateFolder)).toEqual([ + false, + false, + false, ]); - expect(filters[0]?.params).toEqual(["org", "owner", "owner"]); - expect(filters[1]?.params).toEqual(["org", "owner", "org", spaceId]); }); it("scopes every check and write to a requested organization the owner belongs to", async () => { @@ -359,11 +441,12 @@ describe("placing owned Caps in team folders", () => { it("lists destinations in a requested organization", async () => { const organizationId = Organisation.OrganisationId.make("cap-org"); - results = [[], []]; + results = [[], [], []]; await getOwnedVideoMoveDestinations(organizationId); expect(mocks.organization).toHaveBeenCalledWith("owner", organizationId); - expect(filters[0]?.params).toEqual([organizationId, "owner", "owner"]); - expect(filters[1]?.params).toEqual([ + expect(filters[0]?.params).toEqual([organizationId]); + expect(filters[1]?.params).toEqual([organizationId, "owner", "owner"]); + expect(filters[2]?.params).toEqual([ organizationId, "owner", organizationId, diff --git a/apps/web/__tests__/unit/pro-checkout-webhook.test.ts b/apps/web/__tests__/unit/pro-checkout-webhook.test.ts index b7f4e7bb0e7..7aa34ccb28f 100644 --- a/apps/web/__tests__/unit/pro-checkout-webhook.test.ts +++ b/apps/web/__tests__/unit/pro-checkout-webhook.test.ts @@ -499,6 +499,31 @@ describe("Stripe webhook: Cap Pro checkout fulfilment", () => { }); }); + it("ends Pro by email when the customer's userId names a deleted account", async () => { + state.users = [{ ...owner(), stripeSubscriptionStatus: "past_due" }]; + const canceled = fakeSubscription("sub_own", "cus_own", 5, "canceled"); + stripeSubscriptions([canceled]); + mockStripe.customers.retrieve.mockResolvedValue({ + id: "cus_own", + email: "owner@example.com", + metadata: { userId: "user-gone" }, + }); + mockStripe.webhooks.constructEvent.mockReturnValue( + subscriptionEvent("customer.subscription.deleted", canceled), + ); + + expect((await POST(request())).status).toBe(200); + expect(state.users[0]).toMatchObject({ + id: "user-own", + stripeSubscriptionId: "sub_own", + stripeSubscriptionStatus: "canceled", + inviteQuota: 1, + }); + expect(mockStripe.customers.update).toHaveBeenCalledWith("cus_own", { + metadata: { userId: "user-own" }, + }); + }); + it("moves the account onto the customer's other live subscription when the tracked one is deleted", async () => { const canceled = fakeSubscription("sub_own", "cus_own", 5, "canceled"); stripeSubscriptions([ diff --git a/apps/web/__tests__/unit/process-developer-video-workflow.test.ts b/apps/web/__tests__/unit/process-developer-video-workflow.test.ts index 9d684cc34eb..b7e5638693b 100644 --- a/apps/web/__tests__/unit/process-developer-video-workflow.test.ts +++ b/apps/web/__tests__/unit/process-developer-video-workflow.test.ts @@ -222,4 +222,60 @@ describe("process developer video workflow", () => { ], ]); }); + + it("keeps a video waiting past three hours while every worker answers busy, then encodes it", async () => { + mocks.prepare.mockResolvedValue("ready"); + mocks.dispatch.mockResolvedValue("busy"); + const started = Date.now(); + const busyFor = 5 * 60 * 60; + let waited = 0; + const now = vi + .spyOn(Date, "now") + .mockImplementation(() => started + waited * 1000); + mocks.sleep.mockImplementation(async (wait: string) => { + waited += Number.parseInt(wait, 10); + }); + mocks.readParts.mockImplementation(async () => + waited < busyFor + ? { mp4: "dispatch", hls: "dispatch" } + : { mp4: "done", hls: "done" }, + ); + + const result = await run(); + now.mockRestore(); + + expect(result).toEqual({ success: true }); + expect(waited).toBeGreaterThanOrEqual(busyFor); + expect(mocks.fail).not.toHaveBeenCalled(); + expect(mocks.sweep).toHaveBeenCalledWith("video-1", "encode-1"); + }); + + it("still stops at the three hour deadline when one output waits for room while the other has gone quiet", async () => { + mocks.prepare.mockResolvedValue("ready"); + mocks.dispatch.mockResolvedValue("busy"); + mocks.check.mockResolvedValue("unreachable"); + mocks.readParts.mockResolvedValue({ mp4: "dispatch", hls: "check" }); + const started = Date.now(); + let waited = 0; + const now = vi + .spyOn(Date, "now") + .mockImplementation(() => started + waited * 1000); + mocks.sleep.mockImplementation(async (wait: string) => { + waited += Number.parseInt(wait, 10); + }); + + const result = await run(); + now.mockRestore(); + + expect(result).toEqual({ success: true }); + expect(waited).toBeGreaterThanOrEqual(3 * 60 * 60); + expect(waited).toBeLessThan(3 * 60 * 60 + 60); + expect(mocks.fail.mock.calls).toEqual([ + [ + "video-1", + "encode-1", + "Processing took too long and was stopped. Upload the video again.", + ], + ]); + }); }); diff --git a/apps/web/__tests__/unit/processing-retry-limit.test.ts b/apps/web/__tests__/unit/processing-retry-limit.test.ts new file mode 100644 index 00000000000..0c6276b4aaa --- /dev/null +++ b/apps/web/__tests__/unit/processing-retry-limit.test.ts @@ -0,0 +1,168 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + user: vi.fn(), + video: vi.fn(), + update: vi.fn(), + startAiGeneration: vi.fn(), + aiEnabled: vi.fn(), +})); + +vi.mock("@cap/database", () => ({ + db: () => ({ + select: () => ({ + from: () => ({ + where: () => ({ limit: async () => mocks.video() }), + }), + }), + update: () => ({ + set: (values: unknown) => ({ + where: async () => mocks.update(values), + }), + }), + }), +})); +vi.mock("@cap/database/auth/request-site", () => ({ + isCrossSiteCookieWrite: () => false, +})); +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: mocks.user, +})); +vi.mock("@cap/database/schema", () => ({ users: {}, videos: {} })); +vi.mock("next/cache", () => ({ revalidatePath: vi.fn() })); +vi.mock("@/lib/generate-ai", () => ({ + startAiGeneration: mocks.startAiGeneration, +})); +vi.mock("@/utils/flags", () => ({ isAiGenerationEnabled: mocks.aiEnabled })); + +import { POST as retryAi } from "@/app/api/videos/[videoId]/retry-ai/route"; +import { POST as retryTranscription } from "@/app/api/videos/[videoId]/retry-transcription/route"; +import { + PROCESSING_RETRY_RATE_MESSAGE, + processingRetryLimited, +} from "@/lib/processing-retry-limit"; + +const RETRIES_PER_HOUR = 30; + +let userCounter = 0; +let userId = ""; + +const request = () => + new Request("http://localhost/api/videos/video-1/retry", { method: "POST" }); +const params = { params: Promise.resolve({ videoId: "video-1" }) } as never; + +beforeEach(() => { + vi.clearAllMocks(); + userCounter++; + userId = `owner-${userCounter}`; + mocks.user.mockResolvedValue({ id: userId }); + mocks.video.mockImplementation(() => [ + { + id: "video-1", + ownerId: userId, + transcriptionStatus: "COMPLETE", + metadata: { aiGenerationStatus: "ERROR" }, + email: "owner@example.com", + }, + ]); + mocks.aiEnabled.mockResolvedValue(true); + mocks.startAiGeneration.mockResolvedValue({ success: true, message: "ok" }); +}); + +describe("processing retry limit", () => { + it("allows normal retries and refuses a loop of them", async () => { + for (let attempt = 0; attempt < RETRIES_PER_HOUR; attempt++) { + await expect(processingRetryLimited(userId)).resolves.toBe(false); + } + await expect(processingRetryLimited(userId)).resolves.toBe(true); + await expect(processingRetryLimited(`${userId}-other`)).resolves.toBe( + false, + ); + }); + + it("retries AI generation for the owner", async () => { + const response = await retryAi(request(), params); + + expect(response.status).toBe(200); + expect(mocks.startAiGeneration).toHaveBeenCalledOnce(); + }); + + it("stops AI generation retries past the limit with a clear error", async () => { + for (let attempt = 0; attempt < RETRIES_PER_HOUR; attempt++) { + expect((await retryAi(request(), params)).status).toBe(200); + } + + const response = await retryAi(request(), params); + + expect(response.status).toBe(429); + expect(await response.json()).toEqual({ + error: PROCESSING_RETRY_RATE_MESSAGE, + }); + expect(mocks.startAiGeneration).toHaveBeenCalledTimes(RETRIES_PER_HOUR); + }); + + it("retries transcription for the owner", async () => { + const response = await retryTranscription(request(), params); + + expect(response.status).toBe(200); + expect(mocks.update).toHaveBeenCalledExactlyOnceWith({ + transcriptionStatus: null, + }); + }); + + it("stops transcription retries past the limit without resetting the status", async () => { + for (let attempt = 0; attempt < RETRIES_PER_HOUR; attempt++) { + expect((await retryTranscription(request(), params)).status).toBe(200); + } + + const response = await retryTranscription(request(), params); + + expect(response.status).toBe(429); + expect(await response.json()).toEqual({ + error: PROCESSING_RETRY_RATE_MESSAGE, + }); + expect(mocks.update).toHaveBeenCalledTimes(RETRIES_PER_HOUR); + }); + + it("shares one budget between both share page retries", async () => { + for (let attempt = 0; attempt < RETRIES_PER_HOUR; attempt++) { + await retryTranscription(request(), params); + } + + expect((await retryAi(request(), params)).status).toBe(429); + expect(mocks.startAiGeneration).not.toHaveBeenCalled(); + }); + + it("charges new agent reruns before the mutation and never reruns on a replay", () => { + const source = readFileSync( + join(process.cwd(), "app/api/v1/[...route]/route.ts"), + "utf8", + ); + const start = source.indexOf('.handle("processCap"'); + const handler = source.slice( + start, + source.indexOf('.handle("replaceTranscript"', start), + ); + const at = (text: string) => { + const index = handler.indexOf(text); + expect(index, text).toBeGreaterThan(-1); + return index; + }; + + expect(handler).toMatch( + /const chargesRerun = chargesProcessingRerun\(\{\s*target: payload\.target,\s*retry: payload\.retry,\s*transcriptionStatus: row\.transcriptionStatus,\s*metadata,\s*\}\);/, + ); + expect(handler).toMatch( + /runAgentMutation\(\{\s*admit: Effect\.gen\(function\* \(\) \{\s*newRequest = true;\s*if \(\s*chargesRerun &&\s*\(yield\* Effect\.promise\(\(\) =>\s*processingRetryLimited\(principal\.id, headers\),\s*\)\)\s*\)\s*\{\s*return yield\* rateLimited\(\s*requestId,\s*PROCESSING_RETRY_WINDOW_MS,\s*\);/, + ); + expect(handler).toMatch( + /state\.target !== "transcript" &&\s*\(newRequest \|\| !isAiGenerationRerun\(metadata\)\)\s*\)\s*\{\s*const result = yield\* Effect\.tryPromise\(\(\) =>\s*startAiGeneration\(/, + ); + expect(at("const chargesRerun")).toBeGreaterThan( + at("isAiGenerationEnabled("), + ); + expect(at("admit:")).toBeLessThan(at("transcribeVideo(")); + }); +}); diff --git a/apps/web/__tests__/unit/proxy-embed-ancestors.test.ts b/apps/web/__tests__/unit/proxy-embed-ancestors.test.ts new file mode 100644 index 00000000000..2dfec5f3b59 --- /dev/null +++ b/apps/web/__tests__/unit/proxy-embed-ancestors.test.ts @@ -0,0 +1,188 @@ +import { NextRequest } from "next/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + clearEmbedOriginsCache, + EMBED_ORIGINS_CACHE_MS, +} from "@/lib/developer-embed-origins"; +import { proxy } from "../../proxy"; + +const state = vi.hoisted(() => ({ + domains: [] as string[], + fail: false, + lookups: 0, +})); + +vi.mock("@cap/database", () => ({ + db: () => ({ + select: () => ({ + from: () => ({ + where: async () => [], + innerJoin: () => ({ + where: async () => { + state.lookups += 1; + if (state.fail) throw new Error("database unavailable"); + return state.domains.map((domain) => ({ domain })); + }, + }), + }), + }), + }), +})); + +vi.mock("@cap/database/schema", () => ({ + organizations: {}, + developerAppDomains: {}, + developerVideos: {}, +})); + +vi.mock("drizzle-orm", () => ({ eq: () => undefined })); + +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + serverEnv: () => ({ + WEB_URL: "https://cap.so", + VERCEL_URL_HOST: undefined, + VERCEL_BRANCH_URL_HOST: undefined, + VERCEL_PROJECT_PRODUCTION_URL_HOST: undefined, + }), +})); + +const PLAYBACK_ID = "playback-id-0123456789ab"; +const EMBED = `https://cap.so/dev/${PLAYBACK_ID}/embed`; + +const send = (url: string, headers: Record = {}) => + proxy(new NextRequest(url, { headers })); + +const headerList = (response: Response) => [...response.headers.entries()]; + +beforeEach(() => { + state.domains = []; + state.fail = false; + state.lookups = 0; + clearEmbedOriginsCache(); + vi.spyOn(console, "error").mockImplementation(() => {}); +}); + +afterEach(() => { + vi.useRealTimers(); +}); + +describe("embed frame-ancestors", () => { + it("leaves the embed response untouched when the list is empty", async () => { + const embed = await send(`${EMBED}?autoplay=1`); + const hosted = await send(`https://cap.so/dev/${PLAYBACK_ID}`); + expect(state.lookups).toBe(1); + expect(embed.status).toBe(200); + expect(headerList(embed)).toEqual(headerList(hosted)); + expect(headerList(embed)).toEqual([["x-middleware-next", "1"]]); + }); + + it("allows one site", async () => { + state.domains = ["https://myshop.com"]; + const response = await send(EMBED); + expect(response.status).toBe(200); + expect(response.headers.get("content-security-policy")).toBe( + "frame-ancestors 'self' https://myshop.com", + ); + expect(response.headers.get("x-frame-options")).toBeNull(); + expect(response.headers.get("set-cookie")).toBeNull(); + }); + + it("allows several sites and skips stored entries it cannot honour", async () => { + state.domains = [ + "https://myshop.com", + "*.myshop.com", + "http://myshop.com", + "https://docs.myshop.com", + ]; + const response = await send(EMBED); + expect(response.headers.get("content-security-policy")).toBe( + "frame-ancestors 'self' https://myshop.com https://docs.myshop.com", + ); + }); + + it.each([ + "/dev/%70layback-id-0123456789ab/embed", + "/Dev/playback-id-0123456789ab/Embed", + "/DEV/playback-id-0123456789ab/EMBED?autoplay=1", + ])("restricts the same embed at %s", async (path) => { + state.domains = ["https://myshop.com"]; + const response = await send(`https://cap.so${path}`); + expect(state.lookups).toBe(1); + expect(response.headers.get("content-security-policy")).toBe( + "frame-ancestors 'self' https://myshop.com", + ); + }); + + it("leaves a differently cased embed untouched when the list is empty", async () => { + const main = await send( + `https://cap.so/Dev/${PLAYBACK_ID}/Embed?dub_id=click_123`, + ); + expect(state.lookups).toBe(1); + expect(main.headers.get("content-security-policy")).toBeNull(); + expect(main.headers.get("set-cookie")).toContain("dub_id=click_123"); + }); + + it("sends one policy header for the RSC request too", async () => { + state.domains = ["https://myshop.com"]; + const response = await send(EMBED, { rsc: "1" }); + const policies = [...response.headers.entries()].filter( + ([name]) => name === "content-security-policy", + ); + expect(policies).toEqual([ + ["content-security-policy", "frame-ancestors 'self' https://myshop.com"], + ]); + }); + + it("serves the embed as before when the list cannot be read", async () => { + state.domains = ["https://myshop.com"]; + state.fail = true; + const response = await send(EMBED); + expect(response.status).toBe(200); + expect(headerList(response)).toEqual([["x-middleware-next", "1"]]); + }); + + it("reads an app's list once a minute per instance", async () => { + vi.useFakeTimers({ toFake: ["Date"] }); + state.domains = ["https://myshop.com"]; + await send(EMBED); + state.domains = []; + const cached = await send(EMBED); + expect(state.lookups).toBe(1); + expect(cached.headers.get("content-security-policy")).toBe( + "frame-ancestors 'self' https://myshop.com", + ); + + vi.setSystemTime(Date.now() + EMBED_ORIGINS_CACHE_MS + 1); + const fresh = await send(EMBED); + expect(state.lookups).toBe(2); + expect(fresh.headers.get("content-security-policy")).toBeNull(); + }); + + it("does not keep a failed read", async () => { + state.domains = ["https://myshop.com"]; + state.fail = true; + await send(EMBED); + state.fail = false; + const response = await send(EMBED); + expect(state.lookups).toBe(2); + expect(response.headers.get("content-security-policy")).toBe( + "frame-ancestors 'self' https://myshop.com", + ); + }); + + it.each([ + `/dev/${PLAYBACK_ID}`, + `/dev/${PLAYBACK_ID}/embed/extra`, + "/dev/short/embed", + `/s/${PLAYBACK_ID}`, + `/embed/${PLAYBACK_ID}`, + "/", + "/pricing", + ])("never restricts framing of %s", async (path) => { + state.domains = ["https://myshop.com"]; + const response = await send(`https://cap.so${path}`); + expect(state.lookups).toBe(0); + expect(response.headers.get("content-security-policy")).toBeNull(); + }); +}); diff --git a/apps/web/__tests__/unit/proxy-frame-headers.test.ts b/apps/web/__tests__/unit/proxy-frame-headers.test.ts index e2050cffe0a..5954a2451eb 100644 --- a/apps/web/__tests__/unit/proxy-frame-headers.test.ts +++ b/apps/web/__tests__/unit/proxy-frame-headers.test.ts @@ -4,11 +4,20 @@ import { proxy } from "../../proxy"; vi.mock("@cap/database", () => ({ db: () => ({ - select: () => ({ from: () => ({ where: async () => [] }) }), + select: () => ({ + from: () => ({ + where: async () => [], + innerJoin: () => ({ where: async () => [] }), + }), + }), }), })); -vi.mock("@cap/database/schema", () => ({ organizations: {} })); +vi.mock("@cap/database/schema", () => ({ + organizations: {}, + developerAppDomains: {}, + developerVideos: {}, +})); vi.mock("@cap/env", () => ({ buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, diff --git a/apps/web/__tests__/unit/proxy-infra-path.test.ts b/apps/web/__tests__/unit/proxy-infra-path.test.ts index 7ba7994e2d3..5d67d7d87a7 100644 --- a/apps/web/__tests__/unit/proxy-infra-path.test.ts +++ b/apps/web/__tests__/unit/proxy-infra-path.test.ts @@ -22,6 +22,10 @@ vi.mock("@cap/env", () => ({ vi.mock("@/lib/agent-markdown", () => ({ agentMarkdownResponse: async () => null, + docsMarkdownSlug: () => null, + DOCS_MARKDOWN_ROUTE: "/docs-markdown", + isInfraMarkdownRequest: () => false, + INFRA_MARKDOWN_ROUTE: "/infra-markdown", })); const FORWARDED = "x-middleware-request-x-cap-infra-path"; diff --git a/apps/web/__tests__/unit/regular-mcp-unchanged.test.ts b/apps/web/__tests__/unit/regular-mcp-unchanged.test.ts new file mode 100644 index 00000000000..0a3de1e8f08 --- /dev/null +++ b/apps/web/__tests__/unit/regular-mcp-unchanged.test.ts @@ -0,0 +1,60 @@ +import { createHash } from "node:crypto"; +import { describe, expect, it, vi } from "vitest"; + +vi.mock("server-only", () => ({})); +vi.mock("@cap/env", () => ({ + buildEnv: { NEXT_PUBLIC_IS_CAP: "true" }, + serverEnv: () => ({ WEB_URL: "https://cap.so" }), +})); +vi.mock("@/lib/mcp-auth", () => ({ + authenticateMcpBearer: async (authorization: string | null) => + authorization === "Bearer valid" + ? { userId: "owner", clientId: "client" } + : null, + mcpIssuer: () => "https://cap.so", + mcpResource: () => "https://cap.so/api/mcp", + mcpProtectedResourceMetadataUrl: () => + "https://cap.so/.well-known/oauth-protected-resource/api/mcp", +})); +vi.mock("@/lib/mcp-data", () => ({ + listMcpCaps: async () => ({ caps: [], nextCursor: null }), + getMcpCap: async () => null, + getMcpCapContext: async () => null, +})); + +const sha256 = (text: string) => + createHash("sha256").update(text).digest("hex"); + +describe("regular Cap MCP is unchanged by the Infra MCP", () => { + it("lists the same tools and serves the same server card", async () => { + const { POST } = await import("@/app/api/mcp/route"); + const { GET } = await import("@/app/.well-known/mcp/route"); + const listed = await POST( + new Request("https://cap.so/api/mcp", { + method: "POST", + headers: { + Authorization: "Bearer valid", + "Content-Type": "application/json", + Accept: "application/json, text/event-stream", + }, + body: JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "tools/list", + params: {}, + }), + }), + ); + const data = (await listed.text()) + .split("\n") + .find((line) => line.startsWith("data: ")); + const tools = JSON.stringify(JSON.parse(data?.slice(6) ?? "").result.tools); + const card = await (await GET()).text(); + expect(sha256(tools)).toBe( + "3777d45b0877cb63f1899bc0fde71996aac5dee16bf295e09637e49e6ab6dca3", + ); + expect(sha256(card)).toBe( + "5fa2dde2ab13f218a289a3f0eabdb3596cd57114b314fcfcefba632fa139426d", + ); + }); +}); diff --git a/apps/web/__tests__/unit/reject-on-failure.test.ts b/apps/web/__tests__/unit/reject-on-failure.test.ts new file mode 100644 index 00000000000..b08ca160983 --- /dev/null +++ b/apps/web/__tests__/unit/reject-on-failure.test.ts @@ -0,0 +1,23 @@ +import { Cause, Exit } from "effect"; +import { describe, expect, it } from "vitest"; +import { rejectOnFailure } from "@/lib/effect-react-query"; + +describe("rejectOnFailure", () => { + it("resolves with the value of a successful exit", async () => { + await expect(rejectOnFailure(Exit.succeed("done"))).resolves.toBe("done"); + }); + + it("rejects with the cause of a failed exit so toast.promise shows its error", async () => { + const failure = Exit.fail("storage unavailable"); + + const result = await rejectOnFailure(failure).then( + () => "resolved", + (cause) => cause, + ); + + expect(Cause.isCause(result)).toBe(true); + expect(Cause.failureOption(result as Cause.Cause)).toEqual( + expect.objectContaining({ value: "storage unavailable" }), + ); + }); +}); diff --git a/apps/web/__tests__/unit/remove-organization-videos.test.ts b/apps/web/__tests__/unit/remove-organization-videos.test.ts index 0039691472a..8c111087725 100644 --- a/apps/web/__tests__/unit/remove-organization-videos.test.ts +++ b/apps/web/__tests__/unit/remove-organization-videos.test.ts @@ -22,8 +22,30 @@ const colleagueVideo = Video.VideoId.make("video-colleague"); let memberRole: string | null; let sharedRows: { videoId: Video.VideoId; ownerId: string }[]; +const folderRows = [ + { id: "personal-folder", organizationId, spaceId: null }, + { id: "org-folder", organizationId, spaceId: organizationId }, + { id: "space-folder", organizationId, spaceId: "space-1" }, + { id: "other-org-folder", organizationId: "org-2", spaceId: "org-2" }, +]; const writes: { table: string; params: unknown[] }[] = []; +// Applies a compiled `and` of `column = ?` / `column is null` comparisons on +// `folders` to a row, so the test checks the filter the action really builds. +function matchesFolderFilter( + row: Record, + sql: string, + params: unknown[], +) { + if (/ or |not /i.test(sql)) throw new Error(`unsupported filter: ${sql}`); + const comparisons = [...sql.matchAll(/`folders`\.`(\w+)` (= \?|is null)/g)]; + if (comparisons.length === 0) throw new Error(`unsupported filter: ${sql}`); + let next = 0; + return comparisons.every(([, column = "", op]) => + op === "is null" ? row[column] == null : row[column] === params[next++], + ); +} + beforeEach(() => { writes.length = 0; memberRole = "member"; @@ -35,6 +57,7 @@ beforeEach(() => { mocks.select.mockImplementation(() => { let table = ""; let params: unknown[] = []; + let sql = ""; const rows = () => { if (table === "organizations") return [{ id: organizationId, ownerId: "org-owner" }]; @@ -42,7 +65,10 @@ beforeEach(() => { return memberRole ? [{ id: "membership", role: memberRole }] : []; if (table === "shared_videos") return sharedRows.filter((row) => params.includes(row.videoId)); - if (table === "folders") return [{ id: "folder-1" }]; + if (table === "folders") + return folderRows + .filter((row) => matchesFolderFilter(row, sql, params)) + .map(({ id }) => ({ id })); return []; }; const query = Object.assign( @@ -54,7 +80,9 @@ beforeEach(() => { }, leftJoin: () => query, where(filter: SQL) { - params = dialect.sqlToQuery(filter).params; + const compiled = dialect.sqlToQuery(filter); + params = compiled.params; + sql = compiled.sql; return query; }, limit: () => query, @@ -95,6 +123,15 @@ describe("removing videos from an organization", () => { expect(writes).toEqual([]); }); + it("only unfiles videos from the organization's folders, not personal ones", async () => { + await removeVideosFromOrganization(organizationId, [ownVideo]); + + expect(writes).toContainEqual({ + table: "videos", + params: [ownVideo, "org-folder"], + }); + }); + it("only removes and unfiles the member's own videos from a mixed request", async () => { const result = await removeVideosFromOrganization(organizationId, [ ownVideo, @@ -104,7 +141,7 @@ describe("removing videos from an organization", () => { expect(result).toMatchObject({ success: true }); expect(writes).toEqual([ { table: "shared_videos", params: [organizationId, ownVideo] }, - { table: "videos", params: [ownVideo, "folder-1"] }, + { table: "videos", params: [ownVideo, "org-folder"] }, ]); }); @@ -122,7 +159,7 @@ describe("removing videos from an organization", () => { expect(result).toMatchObject({ success: true }); expect(writes).toEqual([ { table: "shared_videos", params: [organizationId, colleagueVideo] }, - { table: "videos", params: [colleagueVideo, "folder-1"] }, + { table: "videos", params: [colleagueVideo, "org-folder"] }, ]); }); diff --git a/apps/web/__tests__/unit/s3-endpoint-policy.test.ts b/apps/web/__tests__/unit/s3-endpoint-policy.test.ts index 92b46116268..c6fcf21e065 100644 --- a/apps/web/__tests__/unit/s3-endpoint-policy.test.ts +++ b/apps/web/__tests__/unit/s3-endpoint-policy.test.ts @@ -61,6 +61,41 @@ describe("isPrivateAddress", () => { ])("allows %s", (address) => { expect(isPrivateAddress(address)).toBe(false); }); + + // The webhook ranges (documentation, 6to4, Teredo, discard, local NAT64) + // must not catch the public addresses storage providers answer with. + it.each([ + "3.5.140.2", + "52.92.16.1", + "104.16.0.1", + "162.159.140.1", + "172.67.1.1", + "34.120.0.1", + "2001:1::1", + "2001:db9::1", + "2003::1", + "2600:9000::1", + "2606:4700::1111", + "2a06:98c1::1", + "2a00:1450::1", + ])("still allows the public storage address %s", (address) => { + expect(isPrivateAddress(address)).toBe(false); + }); + + it.each([ + "192.0.2.1", + "192.88.99.1", + "198.51.100.1", + "203.0.113.1", + "64:ff9b:1::a00:1", + "100::1", + "2001::1", + "2001:0:ffff::1", + "2001:db8::1", + "2002:a9fe:a9fe::1", + ])("blocks the reserved address %s", (address) => { + expect(isPrivateAddress(address)).toBe(true); + }); }); describe("isBlockedS3Endpoint", () => { @@ -92,6 +127,43 @@ describe("isBlockedS3Endpoint", () => { ).resolves.toBe(true); }); + // Main refused a name when any address in its answer was private. That + // stays true for those ranges. The ranges added for webhooks only drop + // their own addresses, so the public one is still used. + it.each([ + ["a 6to4 AAAA", ["52.216.1.1", "2002:a00:1::1"]], + ["a Teredo AAAA", ["52.216.1.1", "2001:0:4136:e378::1"]], + ["a local-use NAT64 AAAA", ["52.216.1.1", "64:ff9b:1::a00:1"]], + ["a documentation address", ["203.0.113.9", "52.216.1.1"]], + ["a discard prefix AAAA", ["2600:1f18::1", "100::1"]], + ])("allows a public address next to %s", async (_label, addresses) => { + await expect( + isBlockedS3Endpoint("https://s3.example.com", resolveTo(addresses)), + ).resolves.toBe(false); + }); + + it.each([ + ["loopback", ["52.216.1.1", "127.0.0.1"]], + ["a private v4", ["52.216.1.1", "192.168.0.7"]], + ["unique local v6", ["2600:1f18::1", "fd00::1"]], + ["NAT64 of a private v4", ["52.216.1.1", "64:ff9b::a00:1"]], + ["a mapped private v4", ["52.216.1.1", "::ffff:10.0.0.1"]], + ["a reserved and a private address", ["2002:a00:1::1", "10.0.0.1"]], + ])("still blocks a public address next to %s", async (_label, addresses) => { + await expect( + isBlockedS3Endpoint("https://s3.example.com", resolveTo(addresses)), + ).resolves.toBe(true); + }); + + it("blocks a name whose only addresses are reserved", async () => { + await expect( + isBlockedS3Endpoint( + "https://s3.example.com", + resolveTo(["2002:a00:1::1", "192.0.2.1"]), + ), + ).resolves.toBe(true); + }); + it("blocks hostnames that do not resolve", async () => { await expect( isBlockedS3Endpoint("https://s3.example.com", async () => { diff --git a/apps/web/__tests__/unit/sentry-server-propagation.test.ts b/apps/web/__tests__/unit/sentry-server-propagation.test.ts new file mode 100644 index 00000000000..8aee14dfd5f --- /dev/null +++ b/apps/web/__tests__/unit/sentry-server-propagation.test.ts @@ -0,0 +1,29 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const init = vi.hoisted(() => vi.fn()); +vi.mock("@sentry/nextjs", () => ({ init })); + +describe("server Sentry setup", () => { + beforeEach(() => { + init.mockReset(); + vi.resetModules(); + vi.stubEnv( + "NEXT_PUBLIC_SENTRY_DSN", + "https://public@example.ingest.sentry.io/1", + ); + }); + + it.each([ + ["node", "../../sentry.server.config"], + ["edge", "../../sentry.edge.config"], + ])( + "adds no trace headers to outgoing requests on the %s runtime", + async (_runtime, path) => { + await import(path); + expect(init).toHaveBeenCalledTimes(1); + const options = init.mock.calls[0]?.[0]; + expect(options.tracePropagationTargets).toEqual([]); + expect(options.tracesSampleRate).toBe(0); + }, + ); +}); diff --git a/apps/web/__tests__/unit/share-cap-authorization.test.ts b/apps/web/__tests__/unit/share-cap-authorization.test.ts index e37fa9ac99e..f4126677277 100644 --- a/apps/web/__tests__/unit/share-cap-authorization.test.ts +++ b/apps/web/__tests__/unit/share-cap-authorization.test.ts @@ -232,4 +232,24 @@ describe("sharing Caps to private spaces", () => { }); expect(writes()).toEqual([]); }); + + it("replaces the full share list by default", async () => { + spaceRole = "member"; + organizationShareExists = true; + await expect(shareCap({ capId, spaceIds: [spaceId] })).resolves.toEqual({ + success: true, + }); + expect(organizationShareExists).toBe(false); + }); + + it("only adds the space when existing shares must be kept", async () => { + spaceRole = "member"; + organizationShareExists = true; + await expect( + shareCap({ capId, spaceIds: [spaceId], keepExisting: true }), + ).resolves.toEqual({ success: true }); + expect(organizationShareExists).toBe(true); + expect(writes().map(([query]) => query.split(" ")[0])).toEqual(["insert"]); + expect(writes()[0]?.[0]).toMatch(/^insert into `space_videos`/); + }); }); diff --git a/apps/web/__tests__/unit/share-comment-time-hydration.test.ts b/apps/web/__tests__/unit/share-comment-time-hydration.test.ts new file mode 100644 index 00000000000..0cb3b99cb8e --- /dev/null +++ b/apps/web/__tests__/unit/share-comment-time-hydration.test.ts @@ -0,0 +1,289 @@ +// @vitest-environment jsdom + +import { act, createElement, type ReactElement, type ReactNode } from "react"; +import { createRoot, hydrateRoot } from "react-dom/client"; +import { renderToString } from "react-dom/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + RelativeTime, + ShareNowProvider, +} from "@/app/s/[videoId]/_components/RelativeTime"; +import CommentComponent from "@/app/s/[videoId]/_components/tabs/Activity/Comment"; +import { formatTimeAgo } from "@/app/s/[videoId]/_components/tabs/Activity/utils"; +import { fromNow } from "@/app/s/[videoId]/_components/utils/from-now"; +import type { CommentType } from "@/app/s/[videoId]/Share"; + +vi.mock("next/navigation", () => ({ + useSearchParams: () => new URLSearchParams(), +})); +vi.mock("@/app/Layout/AuthContext", () => ({ + useCurrentUser: () => null, +})); +vi.mock("@/components/Tooltip", () => ({ + Tooltip: ({ children }: { children: ReactNode }) => children, +})); +vi.mock("@/components/LinkifiedText", () => ({ + LinkifiedText: ({ text }: { text: string }) => text, +})); +vi.mock("motion/react", () => ({ + motion: { + div: ({ + children, + className, + }: { + children: ReactNode; + className?: string; + }) => createElement("div", { className }, children), + }, +})); + +const CREATED_AT = new Date("2026-11-13T12:00:00.000Z"); +const SERVER_NOW = CREATED_AT.getTime() + 6 * 60_000; +const SKEWED_CLIENT_NOW = SERVER_NOW + 90 * 60_000; + +const comment = { + id: "c1", + authorId: "u1", + authorName: "Ada", + authorImage: null, + content: "Nice cut", + type: "text", + timestamp: null, + parentCommentId: null, + videoId: "v1", + createdAt: CREATED_AT, + updatedAt: CREATED_AT, + sending: false, +} as unknown as CommentType; + +const commentTree = (serverNow: number | null) => { + const node = createElement(CommentComponent, { + comment, + replies: [], + onReply: () => undefined, + replyingToId: null, + handleReply: async () => undefined, + onCancelReply: () => undefined, + onDelete: () => undefined, + }); + return serverNow === null + ? node + : createElement(ShareNowProvider, { now: serverNow }, node); +}; + +async function serverThenClient( + tree: () => ReactElement, + serverNow: number, + clientNow: number, +) { + vi.setSystemTime(serverNow); + const html = renderToString(tree()); + const container = document.createElement("div"); + container.innerHTML = html; + document.body.append(container); + vi.setSystemTime(clientNow); + const recoverableErrors: unknown[] = []; + let root: ReturnType | undefined; + await act(async () => { + root = hydrateRoot(container, tree(), { + onRecoverableError: (error) => recoverableErrors.push(error), + }); + }); + return { + html, + container, + recoverableErrors, + unmount: () => act(() => root?.unmount()), + }; +} + +const timeText = (root: ParentNode) => root.querySelector("time")?.textContent; + +describe("share page relative times hydrate without a mismatch", () => { + let cleanup: (() => void) | undefined; + + beforeEach(() => { + ( + globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean } + ).IS_REACT_ACT_ENVIRONMENT = true; + vi.useFakeTimers({ toFake: ["Date"] }); + }); + + afterEach(() => { + cleanup?.(); + cleanup = undefined; + document.body.innerHTML = ""; + vi.useRealTimers(); + }); + + it("reproduces the mismatch when the time is computed during render", async () => { + const RenderTimeAgo = () => + createElement("p", null, formatTimeAgo(CREATED_AT)); + const result = await serverThenClient( + () => createElement(RenderTimeAgo), + SERVER_NOW, + SKEWED_CLIENT_NOW, + ); + cleanup = result.unmount; + expect(String(result.recoverableErrors[0])).toMatch( + /server rendered text didn't match the client/, + ); + }); + + it("hydrates a comment cleanly when the viewer's clock is 90 minutes off", async () => { + const result = await serverThenClient( + () => commentTree(SERVER_NOW), + SERVER_NOW, + SKEWED_CLIENT_NOW, + ); + cleanup = result.unmount; + expect(result.recoverableErrors).toEqual([]); + expect(result.html).toContain( + '', + ); + expect(timeText(result.container)).toBe("1h ago"); + }); + + it("hydrates cleanly when a minute boundary falls between the renders", async () => { + const serverNow = CREATED_AT.getTime() + 59_900; + const result = await serverThenClient( + () => commentTree(serverNow), + serverNow, + CREATED_AT.getTime() + 60_100, + ); + cleanup = result.unmount; + expect(result.recoverableErrors).toEqual([]); + expect(result.html).toContain(">now"); + expect(timeText(result.container)).toBe("1m ago"); + }); + + it("keeps the header's fromNow wording and hydrates cleanly", async () => { + const tree = () => + createElement( + ShareNowProvider, + { now: SERVER_NOW }, + createElement(RelativeTime, { date: CREATED_AT, format: fromNow }), + ); + const result = await serverThenClient(tree, SERVER_NOW, SKEWED_CLIENT_NOW); + cleanup = result.unmount; + expect(result.recoverableErrors).toEqual([]); + expect(result.html).toContain(">6 minutes ago"); + expect(timeText(result.container)).toBe("2 hours ago"); + }); + + it("falls back to a fixed UTC date without a provider", async () => { + const result = await serverThenClient( + () => commentTree(null), + SERVER_NOW, + SKEWED_CLIENT_NOW, + ); + cleanup = result.unmount; + expect(result.recoverableErrors).toEqual([]); + expect(result.html).toContain(">Nov 13, 2026"); + expect(timeText(result.container)).toBe("1h ago"); + }); + + it("keeps an invalid date empty with a provider and after hydration", async () => { + const invalid = new Date("not a date"); + const tree = () => + createElement( + ShareNowProvider, + { now: SERVER_NOW }, + createElement(RelativeTime, { date: invalid, format: formatTimeAgo }), + ); + const result = await serverThenClient(tree, SERVER_NOW, SKEWED_CLIENT_NOW); + cleanup = result.unmount; + expect(result.recoverableErrors).toEqual([]); + expect(result.html).toBe(""); + expect(timeText(result.container)).toBe(""); + }); + + it("renders an invalid date without throwing", () => { + vi.setSystemTime(SERVER_NOW); + const html = renderToString( + createElement(RelativeTime, { + date: new Date("not a date"), + format: formatTimeAgo, + }), + ); + expect(html).toBe(""); + }); + + it("formats the same with an explicit now as with the clock", () => { + vi.setSystemTime(SERVER_NOW); + expect(formatTimeAgo(CREATED_AT, new Date(SERVER_NOW))).toBe( + formatTimeAgo(CREATED_AT), + ); + }); +}); + +describe("share page relative times keep updating", () => { + beforeEach(() => { + ( + globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean } + ).IS_REACT_ACT_ENVIRONMENT = true; + vi.useFakeTimers({ toFake: ["Date", "setInterval", "clearInterval"] }); + vi.setSystemTime(CREATED_AT.getTime() + 30_000); + }); + + afterEach(() => { + document.body.innerHTML = ""; + vi.useRealTimers(); + }); + + const mount = async (count: number) => { + const container = document.createElement("div"); + document.body.append(container); + const root = createRoot(container); + const render = async (visible: number) => { + await act(async () => { + root.render( + createElement( + "div", + null, + Array.from({ length: visible }, (_, index) => + createElement(RelativeTime, { + key: index, + date: CREATED_AT, + format: formatTimeAgo, + }), + ), + ), + ); + }); + }; + await render(count); + return { container, root, render }; + }; + + const texts = (container: HTMLElement) => + [...container.querySelectorAll("time")].map((time) => time.textContent); + + it("advances every timestamp once a minute from one shared timer", async () => { + const { container, root } = await mount(2); + expect(texts(container)).toEqual(["now", "now"]); + expect(vi.getTimerCount()).toBe(1); + await act(async () => { + vi.advanceTimersByTime(60_000); + }); + expect(texts(container)).toEqual(["1m ago", "1m ago"]); + await act(async () => { + vi.advanceTimersByTime(60_000); + }); + expect(texts(container)).toEqual(["2m ago", "2m ago"]); + await act(async () => root.unmount()); + }); + + it("stops the timer only after the last timestamp unmounts", async () => { + const { root, render } = await mount(2); + expect(vi.getTimerCount()).toBe(1); + await render(1); + expect(vi.getTimerCount()).toBe(1); + await render(0); + expect(vi.getTimerCount()).toBe(0); + await render(1); + expect(vi.getTimerCount()).toBe(1); + await act(async () => root.unmount()); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/apps/web/__tests__/unit/share-comments-rendering.test.ts b/apps/web/__tests__/unit/share-comments-rendering.test.ts index 95b6ec629ed..d9845bf4a1f 100644 --- a/apps/web/__tests__/unit/share-comments-rendering.test.ts +++ b/apps/web/__tests__/unit/share-comments-rendering.test.ts @@ -365,6 +365,34 @@ describe("share comments rendering", () => { expect(container.querySelector("#comment-saved")).toBeNull(); }); + it("offers delete only once your comment is saved, and says so when deleting fails", async () => { + comments = [ + ...discussion(1), + { ...comment("pending", "root-0"), sending: true }, + ]; + await render(); + expect(row("pending").querySelector('[title="Delete comment"]')).toBeNull(); + + mocks.remove.mockRejectedValueOnce(new Error("Offline")); + mocks.showError.mockReset(); + setComments.mockReset(); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(window, "confirm").mockReturnValue(true); + await click( + row("reply-0").querySelector('[title="Delete comment"] button'), + ); + + expect(mocks.remove).toHaveBeenCalledWith({ + commentId: "reply-0", + parentId: "root-0", + videoId: "video", + }); + expect(mocks.showError).toHaveBeenCalledExactlyOnceWith( + "Couldn't delete the comment. Please try again.", + ); + expect(setComments).not.toHaveBeenCalled(); + }); + it.each(["comment", "reply"] as const)( "keeps a failed %s draft for retry and clears it only after saving", async (kind) => { diff --git a/apps/web/__tests__/unit/shared-caps-past-last-page.test.ts b/apps/web/__tests__/unit/shared-caps-past-last-page.test.ts new file mode 100644 index 00000000000..b9e488f61ce --- /dev/null +++ b/apps/web/__tests__/unit/shared-caps-past-last-page.test.ts @@ -0,0 +1,155 @@ +// @vitest-environment jsdom + +import React, { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + search: "", + marker: (name: string) => () => `[${name}]`, +})); + +vi.mock("next/navigation", () => ({ + usePathname: () => "/dashboard/spaces/space-1", + useRouter: () => ({ push: vi.fn(), refresh: vi.fn() }), + useSearchParams: () => new URLSearchParams(mocks.search), +})); +vi.mock("@cap/ui", () => ({ Button: () => null })); +vi.mock("@/components/FontAwesomeIcon", () => ({ + FontAwesomeIcon: () => null, +})); +vi.mock("@/components/VideoThumbnail", () => ({ + SeededThumbnailsProvider: ({ children }: { children: React.ReactNode }) => + React.createElement(React.Fragment, null, children), +})); +vi.mock("@/lib/Queries/Analytics", () => ({ + useVideosAnalyticsQuery: () => ({ data: {}, isLoading: false }), +})); +vi.mock("@/app/(org)/dashboard/Contexts", () => ({ + useDashboardContext: () => ({ + activeOrganization: { + organization: { id: "org-1", name: "Org", ownerId: "owner-1" }, + ownerIsPro: true, + }, + }), +})); +vi.mock("@/app/(org)/dashboard/_components/CollectionShareControl", () => ({ + CollectionShareControl: () => null, +})); +vi.mock("@/app/(org)/dashboard/_components/Navbar/SpaceDialog", () => ({ + default: () => null, +})); +vi.mock("@/app/(org)/dashboard/caps/components/CapPagination", () => ({ + CapPagination: mocks.marker("pagination"), +})); +vi.mock("@/app/(org)/dashboard/caps/components/FoldersSection", () => ({ + FoldersSection: () => null, +})); +vi.mock("@/app/(org)/dashboard/caps/components/NewFolderDialog", () => ({ + NewFolderDialog: () => null, +})); +vi.mock("@/app/(org)/dashboard/caps/components/SelectedCapsBar", () => ({ + SelectedCapsBar: () => null, +})); +vi.mock( + "@/app/(org)/dashboard/spaces/[spaceId]/components/AddVideosDialog", + () => ({ AddVideosDialog: () => null }), +); +vi.mock( + "@/app/(org)/dashboard/spaces/[spaceId]/components/AddVideosToOrganizationDialog", + () => ({ AddVideosToOrganizationDialog: () => null }), +); +vi.mock( + "@/app/(org)/dashboard/spaces/[spaceId]/components/EmptySharedCapState", + () => ({ EmptySharedCapState: mocks.marker("empty-state") }), +); +vi.mock( + "@/app/(org)/dashboard/spaces/[spaceId]/components/MembersIndicator", + () => ({ MembersIndicator: () => null }), +); +vi.mock( + "@/app/(org)/dashboard/spaces/[spaceId]/components/OrganizationIndicator", + () => ({ OrganizationIndicator: () => null }), +); +vi.mock( + "@/app/(org)/dashboard/spaces/[spaceId]/components/SharedCapCard", + () => ({ SharedCapCard: mocks.marker("cap") }), +); + +import { SharedCaps } from "@/app/(org)/dashboard/spaces/[spaceId]/SharedCaps"; + +let root: Root; +let container: HTMLDivElement; + +beforeEach(() => { + vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); +}); +afterEach(async () => { + await act(async () => root.unmount()); + container.remove(); +}); + +const cap = (index: number) => ({ + id: `video-${index}`, + ownerId: "owner-1", + name: `Cap ${index}`, + createdAt: new Date(0), + totalComments: 0, + totalReactions: 0, + ownerName: "Owner", +}); + +const renderPage = async ( + search: string, + count: number, + caps: number, + folderCount = 0, +) => { + mocks.search = search; + const props = { + data: Array.from({ length: caps }, (_, index) => cap(index)), + count, + spaceId: "space-1", + analyticsEnabled: false, + folders: Array.from({ length: folderCount }, (_, index) => ({ + id: `folder-${index}`, + })), + } as unknown as Parameters[0]; + await act(async () => root.render(React.createElement(SharedCaps, props))); + const has = (name: string) => + container.textContent?.includes(`[${name}]`) ?? false; + return { empty: has("empty-state"), pagination: has("pagination") }; +}; + +describe("space page past its last page", () => { + it("shows the empty state for a space with no videos", async () => { + expect(await renderPage("", 0, 0)).toEqual({ + empty: true, + pagination: false, + }); + }); + + it("shows pagination, not the empty state, when the page is past the end", async () => { + expect(await renderPage("page=3", 20, 0)).toEqual({ + empty: false, + pagination: true, + }); + }); + + it("shows pagination past the end of a space that also has folders", async () => { + expect(await renderPage("page=3", 20, 0, 1)).toEqual({ + empty: false, + pagination: true, + }); + }); + + it("still paginates a full page", async () => { + expect(await renderPage("", 20, 15)).toEqual({ + empty: false, + pagination: true, + }); + }); +}); diff --git a/apps/web/__tests__/unit/sharing-dialog-link-access.test.ts b/apps/web/__tests__/unit/sharing-dialog-link-access.test.ts index 323ffa903a2..7d0df38fb8c 100644 --- a/apps/web/__tests__/unit/sharing-dialog-link-access.test.ts +++ b/apps/web/__tests__/unit/sharing-dialog-link-access.test.ts @@ -9,12 +9,13 @@ import { SharingDialog } from "@/app/(org)/dashboard/caps/components/SharingDial const mocks = vi.hoisted(() => ({ share: vi.fn(), + setPassword: vi.fn(), toast: { error: vi.fn(), success: vi.fn(), info: vi.fn(), warning: vi.fn() }, })); vi.mock("@/actions/caps/share", () => ({ shareCap: mocks.share })); vi.mock("@/actions/videos/password", () => ({ - setVideoPassword: vi.fn(), + setVideoPassword: mocks.setPassword, removeVideoPassword: vi.fn(), })); vi.mock("@/actions/videos/viewer-invites", () => ({ @@ -48,6 +49,8 @@ beforeEach(() => { vi.stubGlobal("IS_REACT_ACT_ENVIRONMENT", true); mocks.share.mockReset(); mocks.share.mockResolvedValue({ success: true }); + mocks.setPassword.mockReset(); + mocks.setPassword.mockResolvedValue({ success: true }); mocks.toast.success.mockReset(); container = document.createElement("div"); document.body.append(container); @@ -200,4 +203,55 @@ describe("sharing dialog link access", () => { public: true, }); }); + + it("keeps unsaved link and password edits through a page refresh", async () => { + const props = { + allowedEmailDomain: "company.com", + allowLinkOverride: true, + }; + await render(props); + await click("Anyone with the link"); + + const passwordSwitch = Array.from( + document.querySelectorAll("button[role='switch']"), + ).find((element) => + element.parentElement?.textContent?.includes( + "Restrict access with a password", + ), + ); + if (!passwordSwitch) throw new Error("Missing password switch"); + await act(async () => passwordSwitch.click()); + const passwordInput = () => { + const input = document.querySelector( + "input[type='password']", + ); + if (!input) throw new Error("Missing password input"); + return input; + }; + const setter = Object.getOwnPropertyDescriptor( + HTMLInputElement.prototype, + "value", + )?.set; + await act(async () => { + setter?.call(passwordInput(), "secret-pass"); + passwordInput().dispatchEvent(new Event("input", { bubbles: true })); + }); + + // Inviting or removing a viewer refreshes the page, which renders the + // dialog again with a new sharedSpaces array. + await render(props); + expect(button("Anyone with the link")?.getAttribute("aria-pressed")).toBe( + "true", + ); + expect(passwordInput().value).toBe("secret-pass"); + + await click("Save"); + expect(mocks.share.mock.calls[0]?.[0]).toEqual({ + capId: "video-id", + spaceIds: [], + public: true, + linkAccess: "anyone", + }); + expect(mocks.setPassword).toHaveBeenCalledWith("video-id", "secret-pass"); + }); }); diff --git a/apps/web/__tests__/unit/sharing-dialog-viewer-invites.test.ts b/apps/web/__tests__/unit/sharing-dialog-viewer-invites.test.ts index ac548350051..4d8de5d9343 100644 --- a/apps/web/__tests__/unit/sharing-dialog-viewer-invites.test.ts +++ b/apps/web/__tests__/unit/sharing-dialog-viewer-invites.test.ts @@ -237,4 +237,36 @@ describe("sharing dialog viewer invitations", () => { expect(mocks.toast.error).toHaveBeenCalledWith("Unable to invite viewer"); expectNoSuccessEffects(); }); + + it("keeps unsaved edits when the page refreshes, and resets on reopen", async () => { + const renderOpen = (isOpen: boolean) => + act(async () => { + root.render( + createElement( + QueryClientProvider, + { client: queryClient }, + createElement(SharingDialog, { + isOpen, + onClose: vi.fn(), + capId: Video.VideoId.make("video-id"), + capName: "Test video", + sharedSpaces: [], + onSharingUpdated: vi.fn(), + isPublic: false, + }), + ), + ); + }); + await render(); + await enterEmail("someone@example.com"); + + await renderOpen(true); + await flushUpdates(); + expect(emailInput().value).toBe("someone@example.com"); + + await renderOpen(false); + await renderOpen(true); + await flushUpdates(); + expect(emailInput().value).toBe(""); + }); }); diff --git a/apps/web/__tests__/unit/signed-baa-webhook.test.ts b/apps/web/__tests__/unit/signed-baa-webhook.test.ts index 07a5ca85c76..bb718a40dd4 100644 --- a/apps/web/__tests__/unit/signed-baa-webhook.test.ts +++ b/apps/web/__tests__/unit/signed-baa-webhook.test.ts @@ -947,9 +947,8 @@ describe("Signed BAA Payment Link webhooks", () => { }); mockStripe.subscriptions.list.mockResolvedValue({ data: [pro] }); if (eventType.endsWith("deleted")) { - mockDbChain.limit.mockResolvedValueOnce([linkedRecord]); - mockDbChain.where - .mockReturnValueOnce(mockDbChain) + mockDbChain.limit + .mockResolvedValueOnce([linkedRecord]) .mockResolvedValueOnce([owner]); } else { mockDbChain.limit diff --git a/apps/web/__tests__/unit/site-navigation.test.ts b/apps/web/__tests__/unit/site-navigation.test.ts new file mode 100644 index 00000000000..7b0a2a7c681 --- /dev/null +++ b/apps/web/__tests__/unit/site-navigation.test.ts @@ -0,0 +1,107 @@ +// @vitest-environment jsdom +import { act, createElement } from "react"; +import { createRoot } from "react-dom/client"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +( + globalThis as typeof globalThis & { IS_REACT_ACT_ENVIRONMENT: boolean } +).IS_REACT_ACT_ENVIRONMENT = true; + +vi.mock("next/navigation", () => ({ usePathname: () => "/" })); + +const { DesktopNavLinks } = await import("@/app/(site)/DesktopNavLinks"); +const { default: MobileMenu } = await import("@/components/ui/MobileMenu"); + +const mounted: Array<() => void> = []; + +const mount = (element: ReturnType) => { + const container = document.createElement("div"); + document.body.appendChild(container); + const root = createRoot(container); + act(() => root.render(element)); + mounted.push(() => { + act(() => root.unmount()); + container.remove(); + }); + return container; +}; + +afterEach(() => { + for (const unmount of mounted.splice(0)) unmount(); + vi.useRealTimers(); + vi.unstubAllGlobals(); +}); + +const topLevelLabels = (nav: Element) => + Array.from(nav.querySelectorAll(":scope > ul > li")).map((item) => + item.querySelector(":scope > a, :scope > button")?.textContent?.trim(), + ); + +describe("site navigation", () => { + it("puts Infra after Agents in the desktop header with the same link styles", () => { + const container = mount(createElement(DesktopNavLinks)); + const nav = container.querySelector('nav[aria-label="Main"]'); + if (!nav) throw new Error("desktop nav missing"); + + expect(topLevelLabels(nav)).toEqual([ + "Product", + "Agents", + "Infra", + "Testimonials", + "Help", + "About", + "Blog", + "Pricing", + ]); + + const infra = nav.querySelector(':scope > ul > li > a[href="/infra"]'); + const agents = nav.querySelector(':scope > ul > li > a[href="/agents"]'); + expect(infra?.textContent).toBe("Infra"); + expect(infra?.className).toBe(agents?.className); + expect(infra?.parentElement?.className).toBe( + agents?.parentElement?.className, + ); + }); + + it("puts Infra after Agents in the phone menu with the same link styles", () => { + vi.useFakeTimers(); + vi.stubGlobal("matchMedia", () => ({ + matches: false, + addEventListener: () => {}, + removeEventListener: () => {}, + })); + const container = mount(createElement(MobileMenu)); + const trigger = container.querySelector('button[aria-label="Open menu"]'); + if (!(trigger instanceof HTMLButtonElement)) { + throw new Error("menu trigger missing"); + } + act(() => trigger.click()); + + const nav = document.body.querySelector('nav[aria-label="Primary"]'); + if (!nav) throw new Error("phone nav missing"); + const links = Array.from(nav.querySelectorAll("a")); + + expect(links.map((link) => link.getAttribute("href"))).toEqual([ + "/pricing", + "/download", + "/agents", + "/infra", + "/migrate-from-loom", + "/blog", + "/changelog", + "/about", + "/testimonials", + "/faq", + ]); + + const infra = links.find((link) => link.getAttribute("href") === "/infra"); + const agents = links.find( + (link) => link.getAttribute("href") === "/agents", + ); + expect(infra?.textContent).toBe("Infra"); + expect(infra?.className).toBe(agents?.className); + expect(infra?.parentElement?.className).toBe( + agents?.parentElement?.className, + ); + }); +}); diff --git a/apps/web/__tests__/unit/space-folder-page-access.test.ts b/apps/web/__tests__/unit/space-folder-page-access.test.ts index 30db476467c..38dcef5c4a0 100644 --- a/apps/web/__tests__/unit/space-folder-page-access.test.ts +++ b/apps/web/__tests__/unit/space-folder-page-access.test.ts @@ -5,6 +5,9 @@ import FolderPage from "@/app/(org)/dashboard/spaces/[spaceId]/folder/[folderId] const mocks = vi.hoisted(() => ({ spaceOrOrg: null as unknown, folder: null as unknown, + canManage: true, + orgRole: "owner", + NewSubfolderButton: () => null, })); vi.mock("server-only", () => ({})); @@ -51,10 +54,10 @@ vi.mock("@/lib/folder", async () => { }); vi.mock("@/lib/org-pro", () => ({ isOrganizationOwnerPro: async () => true })); vi.mock("@/actions/organization/authorization", () => ({ - getOrganizationAccess: async () => ({ role: "owner" }), + getOrganizationAccess: async () => ({ role: mocks.orgRole }), })); vi.mock("@/actions/organization/space-authorization", () => ({ - getSpaceAccess: async () => ({ canManage: true }), + getSpaceAccess: async () => ({ canManage: mocks.canManage }), })); vi.mock("@/app/(org)/dashboard/_components/CollectionShareControl", () => ({ CollectionShareControl: () => null, @@ -65,7 +68,7 @@ vi.mock("@/app/(org)/dashboard/caps/components/FoldersSection", () => ({ vi.mock("@/app/(org)/dashboard/folder/[id]/components", () => ({ BreadcrumbItem: () => null, ClientMyCapsLink: () => null, - NewSubfolderButton: () => null, + NewSubfolderButton: mocks.NewSubfolderButton, })); vi.mock( "@/app/(org)/dashboard/folder/[id]/components/FolderVideosSection", @@ -108,8 +111,17 @@ const load = (spaceId: string) => beforeEach(() => { mocks.spaceOrOrg = null; mocks.folder = null; + mocks.canManage = true; + mocks.orgRole = "owner"; }); +const renders = (node: unknown, type: unknown): boolean => { + if (Array.isArray(node)) return node.some((child) => renders(child, type)); + if (!node || typeof node !== "object" || !("props" in node)) return false; + const element = node as { type: unknown; props: { children?: unknown } }; + return element.type === type || renders(element.props.children, type); +}; + describe("space folder page scope", () => { it("renders a folder that belongs to the space", async () => { mocks.spaceOrOrg = SPACE; @@ -149,3 +161,30 @@ describe("space folder page scope", () => { }, ); }); + +describe("space folder page New Subfolder button", () => { + it("shows it to someone who can manage the space", async () => { + mocks.spaceOrOrg = SPACE; + mocks.folder = folder("space-1"); + + expect(renders(await load("space-1"), mocks.NewSubfolderButton)).toBe(true); + }); + + it("hides it from a space member who cannot create folders there", async () => { + mocks.spaceOrOrg = SPACE; + mocks.folder = folder("space-1"); + mocks.canManage = false; + + expect(renders(await load("space-1"), mocks.NewSubfolderButton)).toBe( + false, + ); + }); + + it("hides it from an organization member on the organization entry", async () => { + mocks.spaceOrOrg = ORGANIZATION; + mocks.folder = folder("org-1"); + mocks.orgRole = "member"; + + expect(renders(await load("org-1"), mocks.NewSubfolderButton)).toBe(false); + }); +}); diff --git a/apps/web/__tests__/unit/space-member-organization.test.ts b/apps/web/__tests__/unit/space-member-organization.test.ts index 93313a68065..c126c399866 100644 --- a/apps/web/__tests__/unit/space-member-organization.test.ts +++ b/apps/web/__tests__/unit/space-member-organization.test.ts @@ -114,6 +114,7 @@ beforeEach(() => { return query; }, limit: () => query, + for: () => query, }, ); return query; diff --git a/apps/web/__tests__/unit/storage-fetch.test.ts b/apps/web/__tests__/unit/storage-fetch.test.ts index 685a88a900f..b48b210894c 100644 --- a/apps/web/__tests__/unit/storage-fetch.test.ts +++ b/apps/web/__tests__/unit/storage-fetch.test.ts @@ -135,6 +135,29 @@ describe("requestPublicOnly", () => { } }); + it("still refuses a body that is not text", async () => { + await expect( + requestPublicOnly(`http://localhost:${port}/`, { + method: "PUT", + body: new Blob(["x"]), + }), + ).rejects.toThrow("only send a text request body"); + }); + + it("refuses a private address written as a literal before connecting", async () => { + lookupState.allowLoopback = true; + try { + await expect( + requestPublicOnly(`http://127.0.0.1:${port}/`), + ).rejects.toBeInstanceOf(StorageFetchRefusedError); + await expect( + requestPublicOnly(`http://[::1]:${port}/`), + ).rejects.toBeInstanceOf(StorageFetchRefusedError); + } finally { + lookupState.allowLoopback = false; + } + }); + it("refuses non-http schemes", async () => { await expect(requestPublicOnly("file:///etc/hosts")).rejects.toBeInstanceOf( StorageFetchRefusedError, diff --git a/apps/web/__tests__/unit/subfolder-dialog-space.test.ts b/apps/web/__tests__/unit/subfolder-dialog-space.test.ts index 1c726cf7e8c..f0e4d209bc5 100644 --- a/apps/web/__tests__/unit/subfolder-dialog-space.test.ts +++ b/apps/web/__tests__/unit/subfolder-dialog-space.test.ts @@ -85,7 +85,7 @@ async function createIn(spaceId?: Space.SpaceIdOrOrganisationId) { input.dispatchEvent(new Event("input", { bubbles: true })); }); const buttons = [...document.querySelectorAll("button")]; - const color = buttons.find((b) => b.textContent?.trim() === "Blue"); + const color = buttons.find((b) => b.getAttribute("aria-label") === "Blue"); const create = buttons.find((b) => b.textContent?.trim() === "Create"); if (!color || !create) throw new Error("Dialog buttons missing"); await act(async () => color.click()); @@ -109,4 +109,38 @@ describe("creating a subfolder inside a space folder", () => { expect(call && Option.getOrNull(call.spaceId)).toBe("listed-space"); }); + + it("creates with the default color when no color is picked, and on Enter", async () => { + await act(async () => { + root.render( + React.createElement(SubfolderDialog, { + open: true, + onOpenChange: () => {}, + parentFolderId, + }), + ); + }); + const input = document.querySelector( + 'input[placeholder="Subfolder name"]', + ); + if (!input) throw new Error("Name field missing"); + const setValue = Object.getOwnPropertyDescriptor( + HTMLInputElement.prototype, + "value", + )?.set; + await act(async () => { + setValue?.call(input, "SEO"); + input.dispatchEvent(new Event("input", { bubbles: true })); + }); + await act(async () => { + input.dispatchEvent( + new KeyboardEvent("keydown", { key: "Enter", bubbles: true }), + ); + }); + const call = mocks.create.mock.calls[0]?.[0] as + | { name: string; color: string } + | undefined; + expect(call?.name).toBe("SEO"); + expect(call?.color).toBe("normal"); + }); }); diff --git a/apps/web/__tests__/unit/workspace-creation-limits.test.ts b/apps/web/__tests__/unit/workspace-creation-limits.test.ts new file mode 100644 index 00000000000..21ac11c4c13 --- /dev/null +++ b/apps/web/__tests__/unit/workspace-creation-limits.test.ts @@ -0,0 +1,315 @@ +import { getTableName } from "drizzle-orm"; +import type { MySqlTable } from "drizzle-orm/mysql-core"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const state = vi.hoisted(() => ({ + ownedOrganizations: 0, + spacesInOrganization: 0, + locks: new Map>(), + lockRows: true, + events: [] as string[], + rateLimited: new Set(), + putObject: vi.fn(), + deleteObject: vi.fn(), + uploadSpaceIcon: vi.fn(), +})); + +const pause = () => new Promise((resolve) => setTimeout(resolve, 0)); + +function makeClient(releases: (() => void)[]) { + const acquire = async (id: string) => { + if (!state.lockRows) return; + const previous = state.locks.get(id) ?? Promise.resolve(); + let release = () => {}; + const held = new Promise((resolve) => { + release = resolve; + }); + state.locks.set( + id, + previous.then(() => held), + ); + await previous; + releases.push(release); + }; + const client = { + select: (fields?: Record) => { + let table = ""; + const rows = async () => { + await pause(); + if (fields && "total" in fields) { + state.events.push(`count:${table}`); + return [ + { + total: + table === "organizations" + ? state.ownedOrganizations + : state.spacesInOrganization, + }, + ]; + } + return []; + }; + const query = { + from(value: MySqlTable) { + table = getTableName(value); + return query; + }, + where: () => query, + limit: () => query, + for: async () => { + state.events.push(`lock:${table}`); + await acquire(table); + return []; + }, + // biome-ignore lint/suspicious/noThenProperty: drizzle queries are thenable + then: ( + resolve: (value: unknown) => unknown, + reject: (error: unknown) => unknown, + ) => rows().then(resolve, reject), + }; + return query; + }, + insert: (value: MySqlTable) => ({ + values: async () => { + const table = getTableName(value); + state.events.push(`insert:${table}`); + await pause(); + if (table === "organizations") state.ownedOrganizations++; + if (table === "spaces") state.spacesInOrganization++; + }, + }), + update: () => ({ set: () => ({ where: async () => undefined }) }), + transaction: async (run: (tx: unknown) => Promise) => { + const txReleases: (() => void)[] = []; + try { + return await run(makeClient(txReleases)); + } finally { + for (const release of txReleases) release(); + } + }, + }; + return client; +} + +vi.mock("@cap/database", () => ({ db: () => makeClient([]) })); +vi.mock("@cap/database/auth/session", () => ({ + getCurrentUser: async () => ({ + id: "user-1", + activeOrganizationId: "org-1", + }), +})); +vi.mock("@cap/database/crypto", () => ({ + hashPassword: async (value: string) => `hashed:${value}`, +})); +vi.mock("@cap/utils", () => ({ userIsPro: () => true })); +vi.mock("@cap/web-backend", async () => { + const { Effect } = await import("effect"); + return { + S3Buckets: { + getBucketAccess: () => + Effect.succeed([ + { + putObject: (key: string) => Effect.sync(() => state.putObject(key)), + deleteObject: (key: string) => + Effect.sync(() => state.deleteObject(key)), + }, + ]), + }, + }; +}); +vi.mock("@/lib/server", async () => { + const { Effect } = await import("effect"); + return { runPromise: Effect.runPromise }; +}); +vi.mock("@/lib/org-pro", () => ({ isOrganizationOwnerPro: async () => true })); +vi.mock("@/actions/organization/upload-space-icon", () => ({ + uploadSpaceIcon: state.uploadSpaceIcon, +})); +vi.mock("next/cache", () => ({ revalidatePath: vi.fn() })); +vi.mock("@/lib/rate-limit", () => ({ + RATE_LIMIT_IDS: { + ORGANIZATION_CREATE: "rl_organization_create", + SPACE_CREATE: "rl_space_create", + }, + isRateLimited: async (ruleId: string, opts: { key: string }) => + state.rateLimited.has(`${ruleId}|${opts.key}`), +})); + +import { createSpace } from "@/actions/organization/create-space"; +import { createOrganization } from "@/components/forms/server"; +import { + ORGANIZATION_LIMIT, + ORGANIZATION_LIMIT_MESSAGE, + ORGANIZATION_RATE_MESSAGE, + SPACE_LIMIT, + SPACE_LIMIT_MESSAGE, + SPACE_RATE_MESSAGE, +} from "@/lib/workspace-creation-limits"; + +function form(entries: Record) { + const data = new FormData(); + for (const [key, value] of Object.entries(entries)) data.append(key, value); + return data; +} + +const icon = () => + new File([new Uint8Array(16)], "icon.png", { type: "image/png" }); + +beforeEach(() => { + state.ownedOrganizations = 0; + state.spacesInOrganization = 0; + state.locks.clear(); + state.lockRows = true; + state.events.length = 0; + state.rateLimited.clear(); + state.putObject.mockReset(); + state.deleteObject.mockReset(); + state.uploadSpaceIcon.mockReset(); + state.uploadSpaceIcon.mockResolvedValue({ iconUrl: "icon-key" }); +}); + +describe("organization creation limits", () => { + it("creates an organization with an icon for a normal user", async () => { + state.ownedOrganizations = 1; + + const result = await createOrganization( + form({ name: "Acme", icon: icon() }), + ); + + expect(result).toMatchObject({ success: true }); + expect(state.putObject).toHaveBeenCalledOnce(); + expect(state.events).toEqual([ + "count:organizations", + "lock:users", + "count:organizations", + "insert:organizations", + "insert:organization_members", + ]); + }); + + it("refuses a user creating organizations too quickly before any upload", async () => { + state.rateLimited.add("rl_organization_create|organization-create:user-1"); + + const result = await createOrganization( + form({ name: "Acme", icon: icon() }), + ); + + expect(result).toEqual({ + success: false, + error: ORGANIZATION_RATE_MESSAGE, + }); + expect(state.putObject).not.toHaveBeenCalled(); + expect(state.events).not.toContain("insert:organizations"); + }); + + it("refuses a user at the cap before any upload", async () => { + state.ownedOrganizations = ORGANIZATION_LIMIT; + + const result = await createOrganization( + form({ name: "Acme", icon: icon() }), + ); + + expect(result).toEqual({ + success: false, + error: ORGANIZATION_LIMIT_MESSAGE, + }); + expect(state.putObject).not.toHaveBeenCalled(); + expect(state.events).not.toContain("insert:organizations"); + }); + + it("never passes the cap with concurrent creates", async () => { + state.ownedOrganizations = ORGANIZATION_LIMIT - 3; + + const results = await Promise.all( + Array.from({ length: 20 }, (_, index) => + createOrganization(form({ name: `Acme ${index}` })), + ), + ); + + expect(results.filter((result) => result.success)).toHaveLength(3); + expect(state.ownedOrganizations).toBe(ORGANIZATION_LIMIT); + }); + + it("removes the icons of creates refused at the cap", async () => { + state.ownedOrganizations = ORGANIZATION_LIMIT - 1; + + const results = await Promise.all( + Array.from({ length: 5 }, (_, index) => + createOrganization(form({ name: `Acme ${index}`, icon: icon() })), + ), + ); + + expect(results.filter((result) => result.success)).toHaveLength(1); + expect(state.putObject).toHaveBeenCalledTimes(5); + expect(state.deleteObject).toHaveBeenCalledTimes(4); + const kept = state.putObject.mock.calls + .map(([key]) => key) + .filter( + (key) => !state.deleteObject.mock.calls.some(([gone]) => gone === key), + ); + expect(kept).toHaveLength(1); + }); + + it("needs the row lock to hold the cap", async () => { + state.lockRows = false; + state.ownedOrganizations = ORGANIZATION_LIMIT - 3; + + await Promise.all( + Array.from({ length: 20 }, (_, index) => + createOrganization(form({ name: `Acme ${index}` })), + ), + ); + + expect(state.ownedOrganizations).toBeGreaterThan(ORGANIZATION_LIMIT); + }); +}); + +describe("space creation limits", () => { + it("creates a space with an icon for a normal user", async () => { + state.spacesInOrganization = 4; + + const result = await createSpace(form({ name: "Design", icon: icon() })); + + expect(result).toMatchObject({ success: true, iconUrl: "icon-key" }); + expect(state.uploadSpaceIcon).toHaveBeenCalledOnce(); + expect(state.events).toEqual([ + "lock:organizations", + "count:spaces", + "insert:spaces", + "insert:space_members", + ]); + }); + + it("refuses a user creating spaces too quickly", async () => { + state.rateLimited.add("rl_space_create|space-create:user-1"); + + const result = await createSpace(form({ name: "Design", icon: icon() })); + + expect(result).toEqual({ success: false, error: SPACE_RATE_MESSAGE }); + expect(state.uploadSpaceIcon).not.toHaveBeenCalled(); + expect(state.events).toEqual([]); + }); + + it("refuses an organization at the cap without storing anything", async () => { + state.spacesInOrganization = SPACE_LIMIT; + + const result = await createSpace(form({ name: "Design", icon: icon() })); + + expect(result).toEqual({ success: false, error: SPACE_LIMIT_MESSAGE }); + expect(state.uploadSpaceIcon).not.toHaveBeenCalled(); + expect(state.events).not.toContain("insert:spaces"); + }); + + it("never passes the cap with concurrent creates", async () => { + state.spacesInOrganization = SPACE_LIMIT - 2; + + const results = await Promise.all( + Array.from({ length: 10 }, (_, index) => + createSpace(form({ name: `Design ${index}` })), + ), + ); + + expect(results.filter((result) => result.success)).toHaveLength(2); + expect(state.spacesInOrganization).toBe(SPACE_LIMIT); + }); +}); diff --git a/apps/web/__tests__/unit/workspace-creation-rate-limits.test.ts b/apps/web/__tests__/unit/workspace-creation-rate-limits.test.ts new file mode 100644 index 00000000000..0fa549f65cc --- /dev/null +++ b/apps/web/__tests__/unit/workspace-creation-rate-limits.test.ts @@ -0,0 +1,30 @@ +import { User } from "@cap/web-domain"; +import { describe, expect, it } from "vitest"; +import { + organizationCreationLimited, + spaceCreationLimited, +} from "@/lib/workspace-creation-limits"; + +const SPACES_PER_HOUR = 300; +const ORGANIZATIONS_PER_HOUR = 20; + +describe("workspace creation rate limits", () => { + it("lets one user create the largest observed burst of spaces and refuses past 300 an hour", async () => { + const userId = User.UserId.make("space-burst-user"); + for (let created = 0; created < SPACES_PER_HOUR; created++) { + await expect(spaceCreationLimited(userId)).resolves.toBe(false); + } + await expect(spaceCreationLimited(userId)).resolves.toBe(true); + await expect( + spaceCreationLimited(User.UserId.make("another-user")), + ).resolves.toBe(false); + }); + + it("refuses a user past 20 organizations an hour", async () => { + const userId = User.UserId.make("organization-burst-user"); + for (let created = 0; created < ORGANIZATIONS_PER_HOUR; created++) { + await expect(organizationCreationLimited(userId)).resolves.toBe(false); + } + await expect(organizationCreationLimited(userId)).resolves.toBe(true); + }); +}); diff --git a/apps/web/actions/caps/share.ts b/apps/web/actions/caps/share.ts index d992ea77031..214842ed9a1 100644 --- a/apps/web/actions/caps/share.ts +++ b/apps/web/actions/caps/share.ts @@ -26,6 +26,7 @@ interface ShareCapParams { spaceIds: Space.SpaceIdOrOrganisationId[]; public?: boolean; linkAccess?: LinkAccessChoice; + keepExisting?: boolean; } export async function shareCap({ @@ -33,6 +34,7 @@ export async function shareCap({ spaceIds, public: isPublic, linkAccess, + keepExisting = false, }: ShareCapParams) { try { const user = await getCurrentUser(); @@ -113,9 +115,11 @@ export async function shareCap({ .from(sharedVideos) .where(eq(sharedVideos.videoId, capId)); - const orgIdsToRemove = currentSharedOrganizations - .filter((s) => !organizationIds.includes(s.organizationId)) - .map((s) => s.organizationId); + const orgIdsToRemove = keepExisting + ? [] + : currentSharedOrganizations + .filter((s) => !organizationIds.includes(s.organizationId)) + .map((s) => s.organizationId); if (orgIdsToRemove.length > 0) { await db() .delete(sharedVideos) @@ -144,9 +148,11 @@ export async function shareCap({ const spacesIds = spacesData.map((space) => space.id); - const spaceIdsToRemove = currentSpaceVideos - .filter((s) => !spacesIds.includes(s.spaceId)) - .map((s) => s.spaceId); + const spaceIdsToRemove = keepExisting + ? [] + : currentSpaceVideos + .filter((s) => !spacesIds.includes(s.spaceId)) + .map((s) => s.spaceId); if (spaceIdsToRemove.length > 0) { await db() .delete(spaceVideos) diff --git a/apps/web/actions/developers/add-domain.ts b/apps/web/actions/developers/add-domain.ts index 25b526c0d3e..cbc61b22023 100644 --- a/apps/web/actions/developers/add-domain.ts +++ b/apps/web/actions/developers/add-domain.ts @@ -8,13 +8,13 @@ import { and, eq, isNull } from "drizzle-orm"; import { revalidatePath } from "next/cache"; import { canUseInfra } from "@/lib/infra-access"; import { actionFailure } from "@/lib/infra-action-result"; -import { normalizeAllowedOrigin } from "@/lib/infra-domain"; +import { normalizeEmbedOrigin } from "@/lib/infra-domain"; export async function addDeveloperDomain(appId: string, domain: string) { const user = await getCurrentUser(); if (!user || !canUseInfra(user.email)) throw new Error("Unauthorized"); - const allowed = normalizeAllowedOrigin(domain); + const allowed = normalizeEmbedOrigin(domain); if (!allowed.ok) return actionFailure(allowed.message); const { origin } = allowed; diff --git a/apps/web/actions/developers/claim-pro-credit.ts b/apps/web/actions/developers/claim-pro-credit.ts index c94402b8833..88fdd567d83 100644 --- a/apps/web/actions/developers/claim-pro-credit.ts +++ b/apps/web/actions/developers/claim-pro-credit.ts @@ -3,18 +3,29 @@ import { db } from "@cap/database"; import { getCurrentUser } from "@cap/database/auth/session"; import { developerApps, developerCreditAccounts } from "@cap/database/schema"; -import { userIsPro } from "@cap/utils"; import { and, eq, isNull } from "drizzle-orm"; import { canUseInfra } from "@/lib/infra-access"; import { actionFailure } from "@/lib/infra-action-result"; -import { claimProMonthlyCredit } from "@/lib/infra-pro-credit"; +import { + claimProMonthlyCredit, + PRO_CREDIT_CLAIMED_MESSAGE, + PRO_CREDIT_NOT_PRO_MESSAGE, + PRO_CREDIT_SEATS_CLAIMED_MESSAGE, + proCreditPayer, +} from "@/lib/infra-pro-credit"; +import { + PRO_CREDIT_TRIAL_MESSAGE, + proCreditEligibility, +} from "@/lib/infra-pro-credit-eligibility"; export async function claimDeveloperProCredit(appId: string) { const user = await getCurrentUser(); if (!user || !canUseInfra(user.email)) throw new Error("Unauthorized"); - if (!userIsPro(user)) { - return actionFailure("This credit comes with Cap Pro."); + const eligibility = proCreditEligibility(user); + if (eligibility === "not_pro") { + return actionFailure(PRO_CREDIT_NOT_PRO_MESSAGE); } + if (eligibility === "trial") return actionFailure(PRO_CREDIT_TRIAL_MESSAGE); const [account] = await db() .select({ id: developerCreditAccounts.id }) @@ -34,15 +45,21 @@ export async function claimDeveloperProCredit(appId: string) { if (!account) throw new Error("App not found"); + const payer = await proCreditPayer(user); + if (payer.state === "refused") return actionFailure(payer.message); + const result = await claimProMonthlyCredit({ userId: user.id, accountId: account.id, now: new Date(), + subscription: payer.subscription, }); if (!result.claimed) { return actionFailure( - "This month's credit is already claimed. It is one per account, not one per app.", + result.reason === "seats_claimed" + ? PRO_CREDIT_SEATS_CLAIMED_MESSAGE + : PRO_CREDIT_CLAIMED_MESSAGE, ); } diff --git a/apps/web/actions/developers/create-app.ts b/apps/web/actions/developers/create-app.ts index 2e57c8f9012..a46ce0c8482 100644 --- a/apps/web/actions/developers/create-app.ts +++ b/apps/web/actions/developers/create-app.ts @@ -9,6 +9,14 @@ import { developerApps, developerCreditAccounts, } from "@cap/database/schema"; +import { + DEVELOPER_APP_LIMIT_MESSAGE, + DEVELOPER_APP_RATE_MESSAGE, + developerAppCreationLimited, + developerAppEnvironment, + developerAppName, + reserveDeveloperAppSlot, +} from "@/lib/developer-app-limits"; import { hashKey } from "@/lib/developer-key-hash"; import { canUseInfra } from "@/lib/infra-access"; import { actionFailure } from "@/lib/infra-action-result"; @@ -20,7 +28,13 @@ export async function createDeveloperApp(data: { const user = await getCurrentUser(); if (!user || !canUseInfra(user.email)) throw new Error("Unauthorized"); - if (!data.name.trim()) return actionFailure("Give the app a name."); + const named = developerAppName(data.name); + if (!named.ok) return actionFailure(named.message); + const chosen = developerAppEnvironment(data.environment); + if (!chosen.ok) return actionFailure(chosen.message); + if (await developerAppCreationLimited(user.id)) { + return actionFailure(DEVELOPER_APP_RATE_MESSAGE); + } const appId = nanoId(); const publicKeyRaw = `cpk_${nanoIdLong()}`; @@ -31,12 +45,14 @@ export async function createDeveloperApp(data: { const encryptedPublicKey = await encrypt(publicKeyRaw); const encryptedSecretKey = await encrypt(secretKeyRaw); - await db().transaction(async (tx) => { + const created = await db().transaction(async (tx) => { + if (!(await reserveDeveloperAppSlot(tx, user.id))) return false; + await tx.insert(developerApps).values({ id: appId, ownerId: user.id, - name: data.name.trim(), - environment: data.environment, + name: named.name, + environment: chosen.environment, }); await tx.insert(developerApiKeys).values([ @@ -63,8 +79,11 @@ export async function createDeveloperApp(data: { appId, ownerId: user.id, }); + return true; }); + if (!created) return actionFailure(DEVELOPER_APP_LIMIT_MESSAGE); + return { appId, publicKey: publicKeyRaw, diff --git a/apps/web/actions/developers/delete-app.ts b/apps/web/actions/developers/delete-app.ts index 83b011db127..ac10d1c1da1 100644 --- a/apps/web/actions/developers/delete-app.ts +++ b/apps/web/actions/developers/delete-app.ts @@ -9,7 +9,10 @@ import { } from "@cap/database/schema"; import { and, eq, isNull } from "drizzle-orm"; import { revalidatePath } from "next/cache"; +import { purgeDeveloperPlaybackCache } from "@/lib/developer-video-cache"; +import { cancelDeveloperVideoEncodes } from "@/lib/developer-video-cancel"; import { canUseInfra } from "@/lib/infra-access"; +import { settleFirstMonthsForDeletedApp } from "@/lib/infra-storage-minimum"; export async function deleteDeveloperApp(appId: string) { const user = await getCurrentUser(); @@ -29,10 +32,11 @@ export async function deleteDeveloperApp(appId: string) { if (!app) throw new Error("App not found"); + const deletedAt = new Date(Math.floor(Date.now() / 1000) * 1000); await db().transaction(async (tx) => { await tx .update(developerApiKeys) - .set({ revokedAt: new Date() }) + .set({ revokedAt: deletedAt }) .where( and( eq(developerApiKeys.appId, appId), @@ -42,12 +46,12 @@ export async function deleteDeveloperApp(appId: string) { await tx .update(developerApps) - .set({ deletedAt: new Date() }) + .set({ deletedAt }) .where(eq(developerApps.id, appId)); await tx .update(developerVideos) - .set({ deletedAt: new Date() }) + .set({ deletedAt }) .where( and( eq(developerVideos.appId, appId), @@ -55,6 +59,9 @@ export async function deleteDeveloperApp(appId: string) { ), ); }); + await purgeDeveloperPlaybackCache({ appId }); + await settleFirstMonthsForDeletedApp(appId, deletedAt); + await cancelDeveloperVideoEncodes({ appId, deletedAt }); revalidatePath("/dashboard/infra", "layout"); return { success: true }; diff --git a/apps/web/actions/developers/delete-video.ts b/apps/web/actions/developers/delete-video.ts index 83304bedb48..261e02e0c8a 100644 --- a/apps/web/actions/developers/delete-video.ts +++ b/apps/web/actions/developers/delete-video.ts @@ -5,7 +5,10 @@ import { getCurrentUser } from "@cap/database/auth/session"; import { developerApps, developerVideos } from "@cap/database/schema"; import { and, eq, isNull } from "drizzle-orm"; import { revalidatePath } from "next/cache"; +import { purgeDeveloperPlaybackCache } from "@/lib/developer-video-cache"; +import { cancelDeveloperVideoEncodes } from "@/lib/developer-video-cancel"; import { canUseInfra } from "@/lib/infra-access"; +import { settleFirstMonthAtDelete } from "@/lib/infra-storage-minimum"; export async function deleteDeveloperVideo(appId: string, videoId: string) { const user = await getCurrentUser(); @@ -25,12 +28,21 @@ export async function deleteDeveloperVideo(appId: string, videoId: string) { if (!app) throw new Error("App not found"); - await db() + const [deleted] = await db() .update(developerVideos) .set({ deletedAt: new Date() }) .where( - and(eq(developerVideos.id, videoId), eq(developerVideos.appId, appId)), + and( + eq(developerVideos.id, videoId), + eq(developerVideos.appId, appId), + isNull(developerVideos.deletedAt), + ), ); + if (deleted.affectedRows > 0) { + await purgeDeveloperPlaybackCache({ videoId }); + await settleFirstMonthAtDelete(videoId); + } + await cancelDeveloperVideoEncodes({ appId, videoId }); revalidatePath("/dashboard/infra", "layout"); return { success: true }; diff --git a/apps/web/actions/developers/regenerate-keys.ts b/apps/web/actions/developers/regenerate-keys.ts index 7c824d317e2..8b9628c4cea 100644 --- a/apps/web/actions/developers/regenerate-keys.ts +++ b/apps/web/actions/developers/regenerate-keys.ts @@ -7,8 +7,13 @@ import { nanoId, nanoIdLong } from "@cap/database/helpers"; import { developerApiKeys, developerApps } from "@cap/database/schema"; import { and, eq, isNull } from "drizzle-orm"; import { revalidatePath } from "next/cache"; +import { + DEVELOPER_KEY_ROTATION_RATE_MESSAGE, + developerKeyRotationLimited, +} from "@/lib/developer-app-limits"; import { hashKey } from "@/lib/developer-key-hash"; import { canUseInfra } from "@/lib/infra-access"; +import { actionFailure } from "@/lib/infra-action-result"; export async function regenerateDeveloperKeys(appId: string) { const user = await getCurrentUser(); @@ -28,6 +33,10 @@ export async function regenerateDeveloperKeys(appId: string) { if (!app) throw new Error("App not found"); + if (await developerKeyRotationLimited(user.id)) { + return actionFailure(DEVELOPER_KEY_ROTATION_RATE_MESSAGE); + } + const publicKeyRaw = `cpk_${nanoIdLong()}`; const secretKeyRaw = `csk_${nanoIdLong()}`; const publicKeyHash = await hashKey(publicKeyRaw); diff --git a/apps/web/actions/developers/update-app.ts b/apps/web/actions/developers/update-app.ts index 20f6dc5046f..a2f458b9139 100644 --- a/apps/web/actions/developers/update-app.ts +++ b/apps/web/actions/developers/update-app.ts @@ -5,6 +5,11 @@ import { getCurrentUser } from "@cap/database/auth/session"; import { developerApps } from "@cap/database/schema"; import { and, eq, isNull } from "drizzle-orm"; import { revalidatePath } from "next/cache"; +import { + developerAppEnvironment, + developerAppLogoUrl, + developerAppName, +} from "@/lib/developer-app-limits"; import { canUseInfra } from "@/lib/infra-access"; import { actionFailure } from "@/lib/infra-action-result"; @@ -33,12 +38,20 @@ export async function updateDeveloperApp(data: { const updates: Partial = {}; if (data.name !== undefined) { - const trimmed = data.name.trim(); - if (!trimmed) return actionFailure("Give the app a name."); - updates.name = trimmed; + const named = developerAppName(data.name); + if (!named.ok) return actionFailure(named.message); + updates.name = named.name; + } + if (data.environment !== undefined) { + const chosen = developerAppEnvironment(data.environment); + if (!chosen.ok) return actionFailure(chosen.message); + updates.environment = chosen.environment; + } + if (data.logoUrl !== undefined) { + const logo = developerAppLogoUrl(data.logoUrl); + if (!logo.ok) return actionFailure(logo.message); + updates.logoUrl = logo.logoUrl; } - if (data.environment !== undefined) updates.environment = data.environment; - if (data.logoUrl !== undefined) updates.logoUrl = data.logoUrl; if (Object.keys(updates).length > 0) { await db() diff --git a/apps/web/actions/developers/webhooks.ts b/apps/web/actions/developers/webhooks.ts new file mode 100644 index 00000000000..0175d1c6b05 --- /dev/null +++ b/apps/web/actions/developers/webhooks.ts @@ -0,0 +1,206 @@ +"use server"; + +import { db } from "@cap/database"; +import { getCurrentUser } from "@cap/database/auth/session"; +import { developerApps } from "@cap/database/schema"; +import { and, eq, isNull } from "drizzle-orm"; +import { revalidatePath } from "next/cache"; +import { headers } from "next/headers"; +import { developerOwnerRequestLimited } from "@/app/api/utils"; +import { + createWebhookEndpointInput, + updateWebhookEndpointInput, +} from "@/lib/developer-webhook-input"; +import { + createWebhookEndpoint, + deleteWebhookEndpoint, + rotateWebhookEndpointSecret, + sendWebhookTestEvent, + serializeWebhookEndpoint, + serializeWebhookTestResult, + updateWebhookEndpoint, + type WebhookFailure, +} from "@/lib/developer-webhooks"; +import { canUseInfra } from "@/lib/infra-access"; +import type { ActionFailure } from "@/lib/infra-action-result"; + +export type InfraWebhookFailure = ActionFailure & { + status: WebhookFailure["status"] | 429; + code: string; + hint: string | null; +}; + +const refuse = ( + status: InfraWebhookFailure["status"], + code: string, + message: string, + hint: string | null = null, +): InfraWebhookFailure => ({ success: false, status, code, message, hint }); + +const fromLibrary = (failure: WebhookFailure) => + refuse(failure.status, failure.code, failure.message, failure.hint); + +const appNotFound = () => + refuse( + 404, + "app_not_found", + "App not found", + "Check that the app exists and belongs to your account.", + ); + +const webhookNotFound = () => + refuse( + 404, + "webhook_not_found", + "Webhook endpoint not found", + "Check the endpoint ID and that it belongs to this app. Deleted endpoints are not returned.", + ); + +type Access = + | { ok: true; appId: string } + | { ok: false; failure: InfraWebhookFailure }; + +// Same scope as the REST API, which resolves the app from its secret key: an +// app the signed-in user owns and has not deleted. Anything else is a 404, so +// another account's app ID reveals nothing. +async function ownedApp(appId: unknown): Promise { + const user = await getCurrentUser(); + if (!user || !canUseInfra(user.email)) throw new Error("Unauthorized"); + if (typeof appId !== "string" || appId === "") { + return { ok: false, failure: appNotFound() }; + } + + const [app] = await db() + .select({ id: developerApps.id }) + .from(developerApps) + .where( + and( + eq(developerApps.id, appId), + eq(developerApps.ownerId, user.id), + isNull(developerApps.deletedAt), + ), + ) + .limit(1); + if (!app) return { ok: false, failure: appNotFound() }; + + if (await developerOwnerRequestLimited(user.id, await headers())) { + return { + ok: false, + failure: refuse( + 429, + "rate_limited", + "Too many requests at once", + "Your account's API requests count here too. Try again in a minute.", + ), + }; + } + return { ok: true, appId: app.id }; +} + +const validEndpointId = (value: unknown): value is string => + typeof value === "string" && value !== ""; + +const refresh = () => revalidatePath("/dashboard/infra", "layout"); + +export async function createInfraWebhook( + appId: string, + input: { url: string; events: string[] }, +) { + const access = await ownedApp(appId); + if (!access.ok) return access.failure; + + const parsed = createWebhookEndpointInput.safeParse({ + url: input?.url, + events: input?.events, + }); + if (!parsed.success) { + return refuse( + 400, + "invalid_request", + parsed.error.issues[0]?.message ?? "Invalid request", + ); + } + + const result = await createWebhookEndpoint(access.appId, parsed.data); + if (!result.ok) return fromLibrary(result); + refresh(); + return { + success: true as const, + endpoint: serializeWebhookEndpoint(result.endpoint), + secret: result.secret, + }; +} + +export async function setInfraWebhookEnabled( + appId: string, + endpointId: string, + enabled: boolean, +) { + const access = await ownedApp(appId); + if (!access.ok) return access.failure; + if (!validEndpointId(endpointId)) return webhookNotFound(); + + const parsed = updateWebhookEndpointInput.safeParse({ enabled }); + if (!parsed.success) { + return refuse( + 400, + "invalid_request", + parsed.error.issues[0]?.message ?? "Invalid request", + ); + } + + const result = await updateWebhookEndpoint( + access.appId, + endpointId, + parsed.data, + ); + if (!result.ok) return fromLibrary(result); + refresh(); + return { + success: true as const, + endpoint: serializeWebhookEndpoint(result.endpoint), + }; +} + +export async function rotateInfraWebhookSecret( + appId: string, + endpointId: string, +) { + const access = await ownedApp(appId); + if (!access.ok) return access.failure; + if (!validEndpointId(endpointId)) return webhookNotFound(); + + const result = await rotateWebhookEndpointSecret(access.appId, endpointId); + if (!result.ok) return fromLibrary(result); + refresh(); + return { + success: true as const, + endpoint: serializeWebhookEndpoint(result.endpoint), + secret: result.secret, + }; +} + +export async function deleteInfraWebhook(appId: string, endpointId: string) { + const access = await ownedApp(appId); + if (!access.ok) return access.failure; + if (!validEndpointId(endpointId)) return webhookNotFound(); + + const result = await deleteWebhookEndpoint(access.appId, endpointId); + if (!result.ok) return fromLibrary(result); + refresh(); + return { success: true as const }; +} + +export async function sendInfraWebhookTest(appId: string, endpointId: string) { + const access = await ownedApp(appId); + if (!access.ok) return access.failure; + if (!validEndpointId(endpointId)) return webhookNotFound(); + + const result = await sendWebhookTestEvent(access.appId, endpointId); + if (!result.ok) return fromLibrary(result); + refresh(); + return { + success: true as const, + test: serializeWebhookTestResult(result.eventId, result.result), + }; +} diff --git a/apps/web/actions/folders/move-items.ts b/apps/web/actions/folders/move-items.ts index 80162d23333..4c1f55f70a6 100644 --- a/apps/web/actions/folders/move-items.ts +++ b/apps/web/actions/folders/move-items.ts @@ -5,6 +5,7 @@ import { getCurrentUser } from "@cap/database/auth/session"; import { nanoId } from "@cap/database/helpers"; import { folders, + organizations, sharedVideos, spaceMembers, spaces, @@ -120,7 +121,12 @@ export async function getMoveFolderDestinations( await requireMoveAccess(user, location); return db() - .select({ id: folders.id, name: folders.name, parentId: folders.parentId }) + .select({ + id: folders.id, + name: folders.name, + parentId: folders.parentId, + color: folders.color, + }) .from(folders) .where(getFolderScope(user, location)) .orderBy(asc(folders.name)); @@ -133,8 +139,20 @@ export async function getOwnedVideoMoveDestinations( const organizationId = requestedOrganizationId ?? user?.activeOrganizationId; if (!user || !organizationId) throw new Error("Unauthorized"); const access = await requireOrganizationAccess(user.id, organizationId); + const canManageOrganization = canManageOrganizationSettings(access.role); + const organizationName = db() + .select({ name: organizations.name }) + .from(organizations) + .where(eq(organizations.id, organizationId)) + .limit(1) + .then((rows) => rows); const availableSpaces = await db() - .select({ id: spaces.id, name: spaces.name }) + .select({ + id: spaces.id, + name: spaces.name, + createdById: spaces.createdById, + memberRole: spaceMembers.role, + }) .from(spaces) .leftJoin( spaceMembers, @@ -146,7 +164,7 @@ export async function getOwnedVideoMoveDestinations( .where( and( eq(spaces.organizationId, organizationId), - canManageOrganizationSettings(access.role) + canManageOrganization ? undefined : or( eq(spaces.createdById, user.id), @@ -160,6 +178,7 @@ export async function getOwnedVideoMoveDestinations( id: folders.id, name: folders.name, parentId: folders.parentId, + color: folders.color, spaceId: folders.spaceId, }) .from(folders) @@ -176,26 +195,42 @@ export async function getOwnedVideoMoveDestinations( ), ) .orderBy(asc(folders.name)); + const [organization] = await organizationName; + // New folders always land in the active organization, so the dialog only + // offers to create one when that is the organization being browsed. + const creatable = organizationId === user.activeOrganizationId; + const pick = ( + spaceId: (typeof organizationFolders)[number]["spaceId"], + ): MoveFolderDestination[] => + organizationFolders + .filter((folder) => folder.spaceId === spaceId) + .map(({ id, name, parentId, color }) => ({ id, name, parentId, color })); return [ { location: { type: "personal" }, name: "My Caps", - folders: organizationFolders.filter((folder) => folder.spaceId === null), + folders: pick(null), + canCreateFolder: creatable, }, { location: { type: "organization" }, - name: "All team members", - folders: organizationFolders.filter( - (folder) => folder.spaceId === organizationId, - ), + name: organization?.name?.trim() || "Everyone in the organization", + folders: pick(organizationId), + canCreateFolder: creatable && canManageOrganization, }, ...availableSpaces.map( (space): MoveDestinationGroup => ({ location: { type: "space", spaceId: space.id }, name: space.name, - folders: organizationFolders.filter( - (folder) => folder.spaceId === space.id, - ), + folders: pick(space.id), + canCreateFolder: + creatable && + (canManageOrganization || + getEffectiveSpaceRole({ + userId: user.id, + createdById: space.createdById, + memberRole: space.memberRole, + }) === "admin"), }), ), ]; diff --git a/apps/web/actions/folders/moveVideoToFolder.ts b/apps/web/actions/folders/moveVideoToFolder.ts index 215d9202bd5..01f95e445c9 100644 --- a/apps/web/actions/folders/moveVideoToFolder.ts +++ b/apps/web/actions/folders/moveVideoToFolder.ts @@ -2,9 +2,28 @@ import { getCurrentUser } from "@cap/database/auth/session"; import type { Folder, Space, Video } from "@cap/web-domain"; -import { moveVideos } from "@/actions/folders/move-items"; +import { moveFolder, moveVideos } from "@/actions/folders/move-items"; import { resolveMoveLocation } from "@/lib/move-items"; +export async function moveVideosToFolder({ + videoIds, + folderId, + spaceId, +}: { + videoIds: Video.VideoId[]; + folderId: Folder.FolderId | null; + spaceId?: Space.SpaceIdOrOrganisationId | null; +}) { + const user = await getCurrentUser(); + if (!user?.activeOrganizationId) throw new Error("Unauthorized"); + + return moveVideos({ + videoIds, + folderId, + location: resolveMoveLocation(spaceId, user.activeOrganizationId), + }); +} + export async function moveVideoToFolder({ videoId, folderId, @@ -13,13 +32,25 @@ export async function moveVideoToFolder({ videoId: Video.VideoId; folderId: Folder.FolderId | null; spaceId?: Space.SpaceIdOrOrganisationId | null; +}) { + return moveVideosToFolder({ videoIds: [videoId], folderId, spaceId }); +} + +export async function moveFolderToFolder({ + folderId, + parentId, + spaceId, +}: { + folderId: Folder.FolderId; + parentId: Folder.FolderId | null; + spaceId?: Space.SpaceIdOrOrganisationId | null; }) { const user = await getCurrentUser(); if (!user?.activeOrganizationId) throw new Error("Unauthorized"); - return moveVideos({ - videoIds: [videoId], + return moveFolder({ folderId, + parentId, location: resolveMoveLocation(spaceId, user.activeOrganizationId), }); } diff --git a/apps/web/actions/organization/create-space.ts b/apps/web/actions/organization/create-space.ts index 04e0d94a338..1cfaa468f24 100644 --- a/apps/web/actions/organization/create-space.ts +++ b/apps/web/actions/organization/create-space.ts @@ -17,6 +17,12 @@ import { import { and, eq } from "drizzle-orm"; import { revalidatePath } from "next/cache"; import { isOrganizationOwnerPro } from "@/lib/org-pro"; +import { + reserveSpaceSlot, + SPACE_LIMIT_MESSAGE, + SPACE_RATE_MESSAGE, + spaceCreationLimited, +} from "@/lib/workspace-creation-limits"; import { filterUsersInOrganization } from "./organization-member-ids"; import { getSpaceSettingsFromFormData, @@ -92,6 +98,10 @@ export async function createSpace( }; } + if (await spaceCreationLimited(user.id)) { + return { success: false, error: SPACE_RATE_MESSAGE }; + } + const existingSpace = await db() .select({ id: spaces.id }) .from(spaces) @@ -140,7 +150,15 @@ export async function createSpace( ? await hashPassword(password.trim()) : null; - await db().transaction(async (tx) => { + const created = await db().transaction(async (tx) => { + if ( + !(await reserveSpaceSlot( + tx, + Organisation.OrganisationId.make(user.activeOrganizationId), + )) + ) + return false; + await tx.insert(spaces).values({ id: spaceId, name, @@ -163,7 +181,11 @@ export async function createSpace( }); await tx.insert(spaceMembers).values(spaceMembersToInsert); + return true; }); + if (!created) { + return { success: false, error: SPACE_LIMIT_MESSAGE }; + } const iconFile = formData.get("icon") as File | null; diff --git a/apps/web/actions/organizations/remove-videos.ts b/apps/web/actions/organizations/remove-videos.ts index 71a08f71455..14e52fc8e1a 100644 --- a/apps/web/actions/organizations/remove-videos.ts +++ b/apps/web/actions/organizations/remove-videos.ts @@ -106,12 +106,17 @@ export async function removeVideosFromOrganization( ), ); - // Clear folderId for videos that are being removed from the organization and are currently in folders within that organization - // First, get all folder IDs that belong to this organization + // Clear folderId for videos that are being removed from the organization and are currently in the organization's own folders. + // Personal folders (spaceId null) also carry the organizationId, and a Cap's place in My Caps must survive unsharing it. const organizationFolders = await db() .select({ id: folders.id }) .from(folders) - .where(eq(folders.organizationId, organizationId)); + .where( + and( + eq(folders.organizationId, organizationId), + eq(folders.spaceId, organizationId), + ), + ); const organizationFolderIds = organizationFolders.map((f) => f.id); diff --git a/apps/web/app/(docs)/docs/[[...slug]]/page.tsx b/apps/web/app/(docs)/docs/[[...slug]]/page.tsx index 28dc276a968..16c22f2769f 100644 --- a/apps/web/app/(docs)/docs/[[...slug]]/page.tsx +++ b/apps/web/app/(docs)/docs/[[...slug]]/page.tsx @@ -43,7 +43,10 @@ export async function generateMetadata( return { title: pageTitle, description, - alternates: { canonical: `https://cap.so${docPath}` }, + alternates: { + canonical: `https://cap.so${docPath}`, + types: { "text/markdown": `https://cap.so${docPath}.md` }, + }, openGraph: { title: pageTitle, description, diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/audio/audio-client.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/audio/audio-client.tsx index a884da97978..38eb3f2c760 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/audio/audio-client.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/audio/audio-client.tsx @@ -31,10 +31,12 @@ import { ConfirmDialog } from "../../_components/dialog"; import { Block, Button, + CARD_ROWS, Empty, FOCUS, LIT, MONO, + PANEL, PageHeader, TD, TEXT_LINK, @@ -569,8 +571,14 @@ export function AudioClient({ ) : ( -
- +
+
diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx index 3ad9dc0b54b..d16fa649792 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/auto-top-up.tsx @@ -143,11 +143,11 @@ export function AutoTopUp({

Saving a card charges nothing and leaves auto top-up off. Once you - switch it on, Cap adds ${initialAmount} from that card whenever this + switch it on, Cap adds ${initialAmount} from that card as soon as this app's balance falls below ${initialThreshold}. You can change both - amounts, the card is charged at most once a day, and auto top-up - switches itself off if a charge is declined. Remove the card at any - time. + amounts. The card is charged at most ten times in any 24 hours, and at + most $1,000 a day across all your apps. Auto top-up switches itself + off if a charge is declined. Remove the card at any time.

); @@ -263,8 +263,9 @@ export function AutoTopUp({

- A card is saved for this app. It is charged at most once a day, and auto - top-up switches itself off if a charge is declined. + A card is saved for this app. It is charged at most ten times in any 24 + hours, and at most $1,000 a day across all your apps. Auto top-up + switches itself off if a charge is declined.

); diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/billing-view.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/billing-view.tsx index d72659cb07b..77297c4a5ab 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/billing-view.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/billing-view.tsx @@ -1,7 +1,11 @@ import clsx from "clsx"; import Link from "next/link"; -import type { LedgerRow, ledgerPageWindow } from "@/lib/infra-ledger"; +import type { LedgerRow } from "@/lib/infra-ledger"; import { + deliveryBilledFromText, + INFRA_DELIVERY_BILLED_FROM, + INFRA_DELIVERY_PRICE, + INFRA_DELIVERY_PRICE_UNIT, INFRA_RESOLUTION_TIERS, INFRA_SPEECH_DOLLARS_PER_HOUR, INFRA_STORAGE_DOLLARS_PER_1000_MINUTES_PER_MONTH, @@ -31,6 +35,11 @@ import { TopUp } from "./top-up"; const dollars = (value: number) => `$${value.toFixed(2)}`; const flatRates = [ + { + name: "Delivery", + unit: INFRA_DELIVERY_PRICE_UNIT, + price: INFRA_DELIVERY_PRICE, + }, { name: "Speech to text", unit: "per audio hour", @@ -49,6 +58,7 @@ const RATE_PRICE = clsx( ); const LEDGER_ANCHOR = "transactions"; +const AUTO_TOP_UP_ANCHOR = "auto-top-up"; export type BillingAccount = { balanceMicroCredits: number; @@ -66,8 +76,8 @@ export function BillingView({ shortfallMicroCredits, autoTopUpFailure, transactions, - totalTransactions, ledgerPage, + deliveryBilledFrom = null, }: { app: { id: string; name: string }; account: BillingAccount | null; @@ -76,12 +86,13 @@ export function BillingView({ shortfallMicroCredits: number; autoTopUpFailure: { day: string; reason: string } | null; transactions: LedgerRow[]; - totalTransactions: number; - ledgerPage: ReturnType; + ledgerPage: { newer: string | null; older: string | null }; + deliveryBilledFrom?: string | null; }) { const billingPath = infraAppPath(app.id, "billing"); - const ledgerHref = (page: number) => - `${billingPath}${page > 1 ? `?page=${page}` : ""}#${LEDGER_ANCHOR}`; + const ledgerHref = (direction: "before" | "after", cursor: string) => + `${billingPath}?${direction}=${encodeURIComponent(cursor)}#${LEDGER_ANCHOR}`; + const hasTransactions = transactions.length > 0 || ledgerPage.newer !== null; return ( <> @@ -112,7 +123,12 @@ export function BillingView({
{account ? ( <> - +
@@ -174,8 +190,11 @@ export function BillingView({
Name

- Storage is charged once a day for each video at its resolution. - Uploading needs a balance that covers seven days of storage.{" "} + Each video is charged one month of storage when it becomes ready, + then once a day at its resolution. Uploading needs a balance that + covers that first month. Delivered video is charged once a day, when + the day's figures become final.{" "} + {`${deliveryBilledFromText(deliveryBilledFrom ?? INFRA_DELIVERY_BILLED_FROM)} `} {account && ( - - - +

+ + + +
)}
@@ -207,7 +228,7 @@ export function BillingView({ title="Transactions" description="Every credit and charge, newest first. Times are in UTC." action={ - totalTransactions > 0 && ( + hasTransactions && ( - {ledgerPage.pages > 1 && ( + {(ledgerPage.newer || ledgerPage.older) && ( )} diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/page.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/page.tsx index 2e7c2fc833c..fbf88534dc4 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/page.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/page.tsx @@ -3,8 +3,13 @@ import { autoTopUpFailureText, getAutoTopUpFailure, } from "@/lib/infra-auto-top-up"; -import { LEDGER_PAGE_SIZE, ledgerPageWindow } from "@/lib/infra-ledger"; -import { countAppLedger, listAppLedger } from "@/lib/infra-ledger-store"; +import { deliveryBillingStart } from "@/lib/infra-delivery-flag"; +import { LEDGER_PAGE_SIZE } from "@/lib/infra-ledger"; +import { + type LedgerPage, + listAppLedgerPage, + parseLedgerCursor, +} from "@/lib/infra-ledger-store"; import { getInfraBilling } from "@/lib/infra-standing"; import { getInfraProCredit, requireInfraApp } from "../../infra-data"; import { BillingView } from "./billing-view"; @@ -13,6 +18,11 @@ export const metadata: Metadata = { title: "Billing | Cap Infra", }; +const EMPTY_LEDGER: LedgerPage = { rows: [], newer: null, older: null }; + +const first = (value: string | string[] | undefined) => + Array.isArray(value) ? value[0] : value; + const failureDay = new Intl.DateTimeFormat("en-US", { month: "long", day: "numeric", @@ -24,33 +34,30 @@ export default async function InfraBillingPage({ searchParams, }: { params: Promise<{ appId: string }>; - searchParams: Promise<{ page?: string | string[] }>; + searchParams: Promise<{ + before?: string | string[]; + after?: string | string[]; + }>; }) { const [{ appId }, query] = await Promise.all([params, searchParams]); const { app } = await requireInfraApp(appId); const account = app.creditAccount; - const [total, proCredit, billing, failure] = await Promise.all([ - account ? countAppLedger(app.id) : Promise.resolve(0), + const [ledger, proCredit, billing, failure] = await Promise.all([ + account + ? listAppLedgerPage(app.id, { + limit: LEDGER_PAGE_SIZE, + before: parseLedgerCursor(first(query.before)), + after: parseLedgerCursor(first(query.after)), + }) + : Promise.resolve(EMPTY_LEDGER), getInfraProCredit(), getInfraBilling(app.id), account ? getAutoTopUpFailure(account.id) : Promise.resolve(null), ]); - const ledgerPage = ledgerPageWindow( - Array.isArray(query.page) ? query.page[0] : query.page, - total, - ); - const transactions = - total > 0 - ? ( - await listAppLedger(app.id, { - limit: LEDGER_PAGE_SIZE, - offset: ledgerPage.offset, - }) - ).rows - : []; return ( ); } diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/top-up.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/top-up.tsx index 6a0c40eb2ad..c71aadf446d 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/billing/top-up.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/billing/top-up.tsx @@ -6,16 +6,29 @@ import { useRouter, useSearchParams } from "next/navigation"; import { useEffect, useId, useState } from "react"; import { toast } from "sonner"; import { CREDIT_CODE_MAX_LENGTH } from "@/lib/infra-credit-code"; +import { + CREDIT_US_EIN_INVALID, + CREDIT_US_EIN_MAX_LENGTH, + parseCreditUsEin, +} from "@/lib/infra-credit-us-ein"; import { infraAppPath } from "@/lib/infra-routes"; import { resolveTopUpAmount, + TOP_UP_AMOUNT_PLACEHOLDER, TOP_UP_DEFAULT_CENTS, TOP_UP_PRESETS_CENTS, topUpButtonLabel, + topUpCoversText, } from "@/lib/infra-top-up"; -import { Button, FOCUS, Input, MONO } from "../../_components/ui"; +import { Button, FOCUS, Input, MONO, TEXT_LINK } from "../../_components/ui"; -export function TopUp({ appId }: { appId: string }) { +export function TopUp({ + appId, + autoTopUpHref, +}: { + appId: string; + autoTopUpHref: string | null; +}) { const router = useRouter(); const searchParams = useSearchParams(); const amountId = useId(); @@ -26,6 +39,14 @@ export function TopUp({ appId }: { appId: string }) { const codeNoteId = useId(); const [codeOpen, setCodeOpen] = useState(false); const [code, setCode] = useState(""); + const einId = useId(); + const einNoteId = useId(); + const einErrorId = useId(); + const [einOpen, setEinOpen] = useState(false); + const [usEin, setUsEin] = useState(""); + const [purchased] = useState( + () => searchParams.get("purchase") === "success", + ); useEffect(() => { if (searchParams.get("purchase") === "success") { @@ -44,6 +65,7 @@ export function TopUp({ appId }: { appId: string }) { appId, amountCents, ...(code.trim() ? { code: code.trim() } : {}), + ...(ein.state === "valid" ? { usEin: ein.ein } : {}), }), }); const data: { url?: string; error?: string } = await res @@ -62,13 +84,15 @@ export function TopUp({ appId }: { appId: string }) { }); const amount = resolveTopUpAmount(custom, selected); + const ein = parseCreditUsEin(einOpen ? usEin : undefined); + const einInvalid = ein.state === "invalid"; return (
{ e.preventDefault(); - if (amount.amountCents === null) return; + if (amount.amountCents === null || einInvalid) return; checkout.mutate(amount.amountCents); }} > @@ -115,11 +139,27 @@ export function TopUp({ appId }: { appId: string }) { aria-invalid={amount.error ? true : undefined} aria-describedby={amount.error ? amountErrorId : undefined} onChange={(e) => setCustom(e.target.value)} - placeholder="Other, 5 to 1,000" + placeholder={TOP_UP_AMOUNT_PLACEHOLDER} className="h-[38px] w-[176px] pl-6" />
+

+ {topUpCoversText()} +

+ {!einOpen && ( +

+ US business?{" "} + {" "} + before you pay so it is on your invoice. +

+ )} {amount.error && (

)} + {einOpen && ( +

+ +
+ setUsEin(e.target.value)} + placeholder="US EIN, 12-3456789" + className={clsx(MONO, "h-[38px]")} + /> +
+ {einInvalid ? ( + + ) : ( +

+ For a US business. It is saved to your billing details, so it + shows on the invoice for this purchase and later ones. +

+ )} +
+ )}
)}
+ {purchased && autoTopUpHref && ( +

+ Switch on{" "} + + auto top-up + {" "} + to add credit as soon as the balance falls below a threshold you set. +

+ )} ); } diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/keys/api-keys-client.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/keys/api-keys-client.tsx index 91188eb8569..c0afad852f4 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/keys/api-keys-client.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/keys/api-keys-client.tsx @@ -5,6 +5,7 @@ import { useRouter } from "next/navigation"; import { useState } from "react"; import { toast } from "sonner"; import { regenerateDeveloperKeys } from "@/actions/developers/regenerate-keys"; +import { refusalMessage, settled } from "@/lib/infra-action-result"; import { KeyRow } from "../../_components/copy"; import { KeysReveal } from "../../_components/create-app"; import { ConfirmDialog } from "../../_components/dialog"; @@ -22,14 +23,15 @@ export function ApiKeysClient({ const [confirmOpen, setConfirmOpen] = useState(false); const regenerate = useMutation({ - mutationFn: () => regenerateDeveloperKeys(appId), + mutationFn: () => settled(regenerateDeveloperKeys(appId)), onSuccess: (result) => { setNewSecretKey(result.secretKey); setConfirmOpen(false); toast.success("New secret key generated"); router.refresh(); }, - onError: () => toast.error("Could not generate a new secret key"), + onError: (error) => + toast.error(refusalMessage(error, "Could not generate a new secret key")), }); return ( diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/keys/page.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/keys/page.tsx index 29f111b4628..e9fda712863 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/keys/page.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/keys/page.tsx @@ -1,4 +1,8 @@ -import { infraPageMetadata, requireInfraApp } from "../../infra-data"; +import { + getInfraApiKeys, + infraPageMetadata, + requireInfraApp, +} from "../../infra-data"; import { ApiKeysClient } from "./api-keys-client"; export const generateMetadata = infraPageMetadata("API keys"); @@ -10,7 +14,8 @@ export default async function InfraKeysPage({ }) { const { appId } = await params; const { app } = await requireInfraApp(appId); - const secretKey = app.apiKeys.find((k) => k.keyType === "secret"); + const apiKeys = await getInfraApiKeys(app.id); + const secretKey = apiKeys.find((k) => k.keyType === "secret"); return ( - + {children} diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/page.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/page.tsx index a08cce5fc5a..3d52fe12030 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/page.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/page.tsx @@ -1,8 +1,11 @@ import clsx from "clsx"; import Link from "next/link"; +import { + firstCreditOffersAddCredits, + lowBalanceText, +} from "@/lib/infra-credit-prompts"; import { infraAppPath } from "@/lib/infra-routes"; import { getInfraBilling } from "@/lib/infra-standing"; -import { getAppStorageUsage } from "@/lib/infra-storage"; import { formatCharge, formatCredits, @@ -28,6 +31,7 @@ import { UploadDrop } from "../_components/upload-drop"; import { LiveVideoStatus, WatchVideos } from "../_components/video-live"; import { getDeveloperAppVideos, + getInfraApiKeys, getInfraProCredit, getInfraStorageChargesSince, infraPageMetadata, @@ -56,20 +60,29 @@ export default async function InfraOverviewPage({ const { app } = await requireInfraApp(appId); const monthStart = utcMonthStart(new Date()); - const [usage, monthStorageCharges, recentVideos, proCredit, billing] = + const [monthStorageCharges, recentVideos, proCredit, billing, apiKeys] = await Promise.all([ - getAppStorageUsage(app.id), app.creditAccount ? getInfraStorageChargesSince(app.creditAccount.id, monthStart) : Promise.resolve(0), getDeveloperAppVideos(app.id, { limit: 6 }), getInfraProCredit(), getInfraBilling(app.id), + getInfraApiKeys(app.id), ]); + const usage = billing.stored; const balance = app.creditAccount?.balanceMicroCredits ?? 0; - const secretKey = app.apiKeys.find((k) => k.keyType === "secret"); + const secretKey = apiKeys.find((k) => k.keyType === "secret"); const empty = recentVideos.length === 0; + const lowBalance = lowBalanceText({ + hasAccount: !!app.creditAccount, + standing: billing.standing.state, + monthlyStorageMicroCredits: billing.stored.monthlyMicroCredits, + balanceMicroCredits: billing.admission.balanceMicroCredits, + requiredMicroCredits: billing.admission.requiredMicroCredits, + shortfallMicroCredits: billing.admission.shortfallMicroCredits, + }); return ( <> @@ -94,6 +107,21 @@ export default async function InfraOverviewPage({ action /> + {lowBalance && ( +

+ {lowBalance}{" "} + + Go to Billing + +

+ )} +
- + setName(e.target.value)} className="max-w-[420px]" /> @@ -106,6 +109,8 @@ export function AppSettingsClient({ + {domains} + } confirmLabel="Delete app" diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/settings/domains-section.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/settings/domains-section.tsx new file mode 100644 index 00000000000..10d837862fd --- /dev/null +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/settings/domains-section.tsx @@ -0,0 +1,112 @@ +"use client"; + +import { useRouter } from "next/navigation"; +import { useId, useState } from "react"; +import { toast } from "sonner"; +import { addDeveloperDomain } from "@/actions/developers/add-domain"; +import { useEffectMutation } from "@/lib/EffectRuntime"; +import { normalizeEmbedOrigin } from "@/lib/infra-domain"; +import { DomainRow } from "../../_components/domain-row"; +import { actionMessage, runAction } from "../../_components/run-action"; +import { Button, Input, Split } from "../../_components/ui"; + +export function DomainsSection({ + appId, + domains, +}: { + appId: string; + domains: { id: string; domain: string }[]; +}) { + const router = useRouter(); + const inputId = useId(); + const problemId = useId(); + const [newDomain, setNewDomain] = useState(""); + const [problem, setProblem] = useState(null); + + const add = useEffectMutation({ + mutationFn: (origin: string) => + runAction(() => addDeveloperDomain(appId, origin)), + onSuccess: (result) => { + setNewDomain(""); + toast.success(`${result.domain} added`); + router.refresh(); + }, + onError: (cause) => + setProblem(actionMessage(cause, "Could not add the site. Try again.")), + }); + + return ( + +
{ + e.preventDefault(); + if (!newDomain.trim() || add.isPending) return; + const allowed = normalizeEmbedOrigin(newDomain); + if (!allowed.ok) { + setProblem(allowed.message); + return; + } + setProblem(null); + add.mutate(allowed.origin); + }} + > + + { + setNewDomain(e.target.value); + setProblem(null); + }} + placeholder="myshop.com" + autoComplete="off" + autoCapitalize="none" + spellCheck={false} + inputMode="url" + aria-invalid={problem ? true : undefined} + aria-describedby={problem ? problemId : undefined} + className="min-w-[12rem] flex-1" + /> + +
+ {problem && ( + + )} + {domains.length > 0 ? ( +
    + {domains.map((d) => ( + + ))} +
+ ) : ( +

+ Any site can embed your player. +

+ )} +
+ ); +} diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/settings/page.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/settings/page.tsx index 0a5d7181ce6..10208dfaf81 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/settings/page.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/settings/page.tsx @@ -1,5 +1,11 @@ -import { infraPageMetadata, requireInfraApp } from "../../infra-data"; +import { + getInfraAppDomains, + getInfraVideoCount, + infraPageMetadata, + requireInfraApp, +} from "../../infra-data"; import { AppSettingsClient } from "./app-settings-client"; +import { DomainsSection } from "./domains-section"; export const generateMetadata = infraPageMetadata("Settings"); @@ -10,6 +16,10 @@ export default async function InfraSettingsPage({ }) { const { appId } = await params; const { app } = await requireInfraApp(appId); + const [videoCount, domains] = await Promise.all([ + getInfraVideoCount(app.id), + getInfraAppDomains(app.id), + ]); return ( } /> ); } diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/usage/page.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/usage/page.tsx index d14aabd0604..7d10da7c142 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/usage/page.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/usage/page.tsx @@ -4,12 +4,14 @@ import { storageMicroCreditsPerMinutePerMonth, } from "@/lib/developer-pricing"; import { getTranscriptionUsageSince } from "@/lib/developer-transcript-store"; +import { deliveryJson, getDeliveredUsage } from "@/lib/infra-delivery"; import { INFRA_RESOLUTION_TIERS, INFRA_STORAGE_DOLLARS_PER_1000_MINUTES_PER_MONTH, } from "@/lib/infra-pricing"; -import { getAppStorageUsage } from "@/lib/infra-storage"; +import { getAppStorage } from "@/lib/infra-storage"; import { + formatAudioLength, formatCharge, formatMinutes, INFRA_TIER_LABELS, @@ -17,6 +19,7 @@ import { utcDaysAgoKey, utcMonthStart, } from "@/lib/infra-usage"; +import { DeliveredMinutes } from "../../_components/delivered-minutes"; import { Block, Empty, @@ -30,6 +33,7 @@ import { TR, } from "../../_components/ui"; import { + getInfraFirstMonthChargesSince, getInfraStorageSnapshots, infraPageMetadata, requireInfraApp, @@ -68,15 +72,29 @@ export default async function InfraUsagePage({ const { app } = await requireInfraApp(appId); const now = new Date(); - const [usage, snapshots, transcription] = await Promise.all([ - getAppStorageUsage(app.id), - getInfraStorageSnapshots(app.id, utcDaysAgoKey(now, HISTORY_DAYS)), - getTranscriptionUsageSince(app.id, utcMonthStart(now)).catch((error) => { - console.error("[infra-usage] Transcription usage unavailable", error); - return null; - }), - ]); - const audioHours = (transcription?.audioSeconds ?? 0) / 3600; + const [storage, snapshots, transcription, firstMonths, delivered] = + await Promise.all([ + getAppStorage(app.id), + getInfraStorageSnapshots(app.id, utcDaysAgoKey(now, HISTORY_DAYS)), + getTranscriptionUsageSince(app.id, utcMonthStart(now)).catch((error) => { + console.error("[infra-usage] Transcription usage unavailable", error); + return null; + }), + app.creditAccount + ? getInfraFirstMonthChargesSince( + app.creditAccount.id, + utcMonthStart(now), + ) + : Promise.resolve({ chargedMicroCredits: 0, videos: 0 }), + getDeliveredUsage(app.id, now).then( + (usage) => ({ usage, failed: false }), + (error) => { + console.error("[infra-usage] Delivery usage unavailable", error); + return { usage: null, failed: true }; + }, + ), + ]); + const usage = storage.stored; const storedTiers = INFRA_RESOLUTION_TIERS.filter( (tier) => usage.minutesByTier[tier] > 0, ); @@ -92,7 +110,7 @@ export default async function InfraUsagePage({ <> @@ -107,18 +125,27 @@ export default async function InfraUsagePage({ />
0 + ? [ + `${firstMonths.videos} first ${firstMonths.videos === 1 ? "month" : "months"}`, + ] + : []), + ].join(", ")} + value={formatCharge( + month.chargedMicroCredits + firstMonths.chargedMicroCredits, + )} /> {transcription && (
)} @@ -223,6 +250,12 @@ export default async function InfraUsagePage({ )} + + ; + +export function EmbedOptions({ + embedUrl, + video, + hasCaptions, +}: { + embedUrl: string; + video: { name: string; width: number | null; height: number | null }; + hasCaptions: boolean; +}) { + const [options, setOptions] = useState(DEFAULT_EMBED_OPTIONS); + const colorId = useId(); + const set = (name: Switch) => (on: boolean) => + setOptions((current) => ({ ...current, [name]: on })); + + return ( + <> +
+ + {options.color && ( + + )} + + + {hasCaptions && ( + + )} + +
+ +

+ Corner radius, a quality cap, hiding other buttons and events your page + can listen to are in{" "} + + Embed Options + + . +

+ + ); +} diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/videos/[videoId]/video-client.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/videos/[videoId]/video-client.tsx index e6f53818d12..8305fd91747 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/videos/[videoId]/video-client.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/videos/[videoId]/video-client.tsx @@ -9,9 +9,10 @@ import { toast } from "sonner"; import { deleteDeveloperVideo } from "@/actions/developers/delete-video"; import { renameDeveloperVideo } from "@/actions/developers/rename-video"; import { CapPlayer } from "@/components/cap-player/CapPlayer"; +import { DEVELOPER_VIDEO_REPORT_EMAIL } from "@/lib/developer-video-takedown"; import { useEffectMutation } from "@/lib/EffectRuntime"; import { infraAppPath } from "@/lib/infra-routes"; -import { formatBytes, formatDuration } from "@/lib/infra-usage"; +import { formatBytes, formatDurationOrDash } from "@/lib/infra-usage"; import { type InfraVideoDetail, infraVideoDisplayState, @@ -32,6 +33,7 @@ import { } from "../../../_components/ui"; import { useLocalUpload } from "../../../_components/uploads"; import { VideoStatus } from "../../../_components/video-status"; +import { EmbedOptions } from "./embed-options"; const createdFormat = new Intl.DateTimeFormat("en-US", { month: "short", @@ -85,6 +87,7 @@ function Player({ video }: { video: InfraVideoDetail }) { source={source} title={video.name} poster={video.posterUrl ?? undefined} + captions={video.captions} aspectRatio={ video.width && video.height ? `${video.width} / ${video.height}` @@ -185,7 +188,7 @@ export function VideoClient({ ); const playable = video.hlsUrl !== null || video.mp4Url !== null; - const duration = formatDuration(video.duration); + const duration = formatDurationOrDash(video.duration); const size = formatBytes(video.storedBytes); const links = [ { label: "HLS", value: video.hlsUrl }, @@ -286,7 +289,14 @@ export function VideoClient({
- {video.state === "ready" && playable && playbackPaused ? ( + {video.state === "taken_down" ? ( + + Cap took this video down after a report, so it does not play here, + on its page or anywhere it is embedded. It is still stored and + billed like your other videos. If you think this is a mistake, + email {DEVELOPER_VIDEO_REPORT_EMAIL}. + + ) : video.state === "ready" && playable && playbackPaused ? (
- {duration && {duration}} + {duration} {size && ( {size} @@ -345,6 +355,14 @@ export function VideoClient({ {video.width}×{video.height} ) : null} + {video.captions.length > 0 && ( + + {video.captions.map((caption) => caption.label).join(", ")} + + )} {createdFormat.format(new Date(video.createdAt))} @@ -364,7 +382,11 @@ export function VideoClient({ description="A hosted player for this video. Send the link to a viewer, or paste the embed code into a page." > - + 0} + /> )} diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/videos/page.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/videos/page.tsx index 7954e248d81..63650476b2f 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/videos/page.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/videos/page.tsx @@ -1,3 +1,4 @@ +import { firstCreditOffersAddCredits } from "@/lib/infra-credit-prompts"; import { infraVideoListLimit, infraVideoListPage, @@ -39,17 +40,26 @@ export default async function InfraVideosPage({ ); if (videos.length === 0 && !userId) { + const balanceMicroCredits = app.creditAccount?.balanceMicroCredits ?? 0; + const proCredit = await getInfraProCredit(); return ( <> - + diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/videos/videos-client.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/videos/videos-client.tsx index e97112c6436..bb671cef824 100644 --- a/apps/web/app/(infra)/dashboard/infra/[appId]/videos/videos-client.tsx +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/videos/videos-client.tsx @@ -9,7 +9,7 @@ import { useId, useState, useTransition } from "react"; import { toast } from "sonner"; import { deleteDeveloperVideo } from "@/actions/developers/delete-video"; import { infraAppPath } from "@/lib/infra-routes"; -import { formatDuration } from "@/lib/infra-usage"; +import { formatDurationOrDash } from "@/lib/infra-usage"; import { INFRA_VIDEO_PAGE_MAX, type InfraVideoListItem, @@ -19,10 +19,12 @@ import { CopyButton } from "../../_components/copy"; import { ConfirmDialog } from "../../_components/dialog"; import { Button, + CARD_ROWS, Empty, FOCUS, Input, MONO, + PANEL, PageHeader, TD, TH, @@ -154,8 +156,14 @@ export function VideosClient({
) : ( -
- +
+
@@ -212,7 +220,7 @@ export function VideosClient({ "hidden text-right text-[13px] tabular-nums sm:table-cell", )} > - {formatDuration(video.duration) ?? NONE} + {formatDurationOrDash(video.duration)} - + -
Name {dateFormat.format(new Date(video.createdAt))} diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/webhooks/page.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/webhooks/page.tsx new file mode 100644 index 00000000000..e6140e6f2dc --- /dev/null +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/webhooks/page.tsx @@ -0,0 +1,51 @@ +import { + MAX_WEBHOOK_ENDPOINTS_PER_APP, + WEBHOOK_DISABLED_BY_FAILURES, + WEBHOOK_EVENT_TYPES, +} from "@/lib/developer-webhook-events"; +import { + listWebhookEndpoints, + serializeWebhookEndpoint, +} from "@/lib/developer-webhooks"; +import { infraPageMetadata, requireInfraApp } from "../../infra-data"; +import { WebhooksClient } from "./webhooks-client"; + +export const generateMetadata = infraPageMetadata("Webhooks"); + +export default async function InfraWebhooksPage({ + params, +}: { + params: Promise<{ appId: string }>; +}) { + const { appId } = await params; + const { app } = await requireInfraApp(appId); + const { endpoints } = await listWebhookEndpoints(app.id, { + limit: MAX_WEBHOOK_ENDPOINTS_PER_APP, + offset: 0, + }); + + return ( + { + const endpoint = serializeWebhookEndpoint(row); + return { + id: endpoint.id, + url: endpoint.url, + events: endpoint.events, + enabled: endpoint.enabled, + disabledByFailures: + endpoint.disabledReason === WEBHOOK_DISABLED_BY_FAILURES, + secretPrefix: endpoint.secretPrefix, + lastDeliveryAt: endpoint.lastDeliveryAt, + lastDeliveryStatus: endpoint.lastDeliveryStatus, + lastResponseStatus: endpoint.lastResponseStatus, + lastErrorCode: endpoint.lastErrorCode, + consecutiveFailures: endpoint.consecutiveFailures, + }; + })} + /> + ); +} diff --git a/apps/web/app/(infra)/dashboard/infra/[appId]/webhooks/webhooks-client.tsx b/apps/web/app/(infra)/dashboard/infra/[appId]/webhooks/webhooks-client.tsx new file mode 100644 index 00000000000..9ebcba24527 --- /dev/null +++ b/apps/web/app/(infra)/dashboard/infra/[appId]/webhooks/webhooks-client.tsx @@ -0,0 +1,642 @@ +"use client"; + +import { useMutation } from "@tanstack/react-query"; +import clsx from "clsx"; +import Link from "next/link"; +import { useRouter } from "next/navigation"; +import { useEffect, useId, useState } from "react"; +import { toast } from "sonner"; +import { + createInfraWebhook, + deleteInfraWebhook, + type InfraWebhookFailure, + rotateInfraWebhookSecret, + sendInfraWebhookTest, + setInfraWebhookEnabled, +} from "@/actions/developers/webhooks"; +import { infraWebhookLocks } from "@/lib/infra-webhook-console"; +import { CopyTextButton, KeyRow } from "../../_components/copy"; +import { ConfirmDialog } from "../../_components/dialog"; +import { + Button, + FAINT, + Input, + MONO, + MUTED, + PANEL, + PageHeader, + Panel, + Split, + TEXT_LINK, + Toggle, +} from "../../_components/ui"; + +export type InfraWebhookEndpoint = { + id: string; + url: string; + events: string[]; + enabled: boolean; + disabledByFailures: boolean; + secretPrefix: string; + lastDeliveryAt: Date | null; + lastDeliveryStatus: "succeeded" | "failed" | null; + lastResponseStatus: number | null; + lastErrorCode: string | null; + consecutiveFailures: number; +}; + +type Revealed = { + endpointId: string; + url: string; + secret: string; + rotated: boolean; +}; + +type TestOutcome = { + delivered: boolean; + responseStatus: number | null; + errorCode: string | null; + durationMs: number; +}; + +const deliveryFormat = new Intl.DateTimeFormat("en-US", { + month: "short", + day: "numeric", + hour: "2-digit", + minute: "2-digit", + hourCycle: "h23", + timeZone: "UTC", +}); + +const ERROR_TEXT: Record = { + timeout: "no answer within 10 seconds", + connection_failed: "could not connect", + redirect: "answered with a redirect, which is not followed", + http_error: "answered with an error", + private_address: "the host is not a public address", + invalid_url: "the URL is not valid", + https_required: "the URL is not https", + credentials_in_url: "the URL holds a username or password", + unsupported_port: "the URL uses a port other than 443", +}; + +const failureText = (status: number | null, errorCode: string | null) => + status !== null && (errorCode === null || errorCode === "http_error") + ? `HTTP ${status}` + : (ERROR_TEXT[errorCode ?? ""] ?? "failed"); + +function lastDelivery(endpoint: InfraWebhookEndpoint) { + if (!endpoint.lastDeliveryAt || !endpoint.lastDeliveryStatus) { + return "No deliveries yet"; + } + const when = `${deliveryFormat.format(new Date(endpoint.lastDeliveryAt))} UTC`; + if (endpoint.lastDeliveryStatus === "succeeded") { + return endpoint.lastResponseStatus === null + ? `Last delivery ${when}: delivered` + : `Last delivery ${when}: HTTP ${endpoint.lastResponseStatus}`; + } + const streak = + endpoint.consecutiveFailures > 1 + ? `, ${endpoint.consecutiveFailures} failures in a row` + : ""; + return `Last delivery ${when} failed: ${failureText(endpoint.lastResponseStatus, endpoint.lastErrorCode)}${streak}`; +} + +function testText(test: TestOutcome) { + const time = `${test.durationMs.toLocaleString("en-US")} ms`; + return test.delivered + ? `Test delivered: HTTP ${test.responseStatus} in ${time}` + : `Test failed: ${failureText(test.responseStatus, test.errorCode)} after ${time}`; +} + +const isFailure = (result: { + success: boolean; +}): result is InfraWebhookFailure => result.success === false; + +type Problem = { message: string; hint: string | null }; + +const showRefusal = (failure: Problem) => + toast.error( + failure.message, + failure.hint ? { description: failure.hint } : {}, + ); + +export function WebhooksClient({ + appId, + limit, + eventTypes, + endpoints, +}: { + appId: string; + limit: number; + eventTypes: readonly string[]; + endpoints: InfraWebhookEndpoint[]; +}) { + const router = useRouter(); + const [adding, setAdding] = useState(false); + const [creating, setCreating] = useState(false); + const [revealed, setRevealed] = useState(null); + const full = endpoints.length >= limit; + const locks = infraWebhookLocks({ + adding, + creating, + secretShown: revealed !== null, + }); + + const startAdding = () => setAdding(true); + + return ( + <> + + Cap sends a signed POST to your server when a video is ready or + fails.{" "} + + Verify the signature + + . + + } + action={ + endpoints.length > 0 && locks.canAdd && !full ? ( + + ) : undefined + } + /> + + {(revealed || adding) && ( +
+ {revealed && ( + + setRevealed(null)} + /> + + )} + + {adding && ( + + setAdding(false)} + onPendingChange={setCreating} + onCreated={(endpointId, url, secret) => { + setAdding(false); + setCreating(false); + setRevealed({ endpointId, url, secret, rotated: false }); + router.refresh(); + }} + /> + + )} +
+ )} + + {endpoints.length === 0 ? ( + !adding && + !revealed && ( +
+

+ No endpoints yet. +

+ +
+ ) + ) : ( +
+

Endpoints

+ +
    + {endpoints.map((endpoint) => ( + + setRevealed((current) => + current?.endpointId === endpoint.id ? null : current, + ) + } + onRotated={(secret) => { + setAdding(false); + setRevealed({ + endpointId: endpoint.id, + url: endpoint.url, + secret, + rotated: true, + }); + }} + /> + ))} +
+
+ {full && ( +

+ An app can have {limit} endpoints. Delete one to add another. +

+ )} +
+ )} + + ); +} + +function SecretReveal({ + revealed, + onDone, +}: { + revealed: Revealed; + onDone: () => void; +}) { + return ( +
+

+ {revealed.url} +

+
+ +
+
+ + +
+

+ {revealed.rotated + ? "The old secret has stopped working. Update your server now." + : "Your server checks each request's Cap-Signature header with it."} +

+
+ ); +} + +function AddEndpointForm({ + appId, + eventTypes, + onCancel, + onPendingChange, + onCreated, +}: { + appId: string; + eventTypes: readonly string[]; + onCancel: () => void; + onPendingChange: (pending: boolean) => void; + onCreated: (endpointId: string, url: string, secret: string) => void; +}) { + const urlId = useId(); + const errorId = useId(); + const [url, setUrl] = useState(""); + const [events, setEvents] = useState([...eventTypes]); + const [error, setError] = useState(null); + + const create = useMutation({ + mutationFn: () => createInfraWebhook(appId, { url: url.trim(), events }), + onSuccess: (result) => { + if (isFailure(result)) { + setError(result); + return; + } + toast.success("Endpoint added"); + onCreated(result.endpoint.id, result.endpoint.url, result.secret); + }, + onError: () => + setError({ message: "Could not add the endpoint", hint: "Try again." }), + }); + + const ready = url.trim().length > 0 && !create.isPending; + + useEffect(() => { + onPendingChange(create.isPending); + }, [create.isPending, onPendingChange]); + + return ( +
{ + event.preventDefault(); + if (!ready) return; + setError(null); + create.mutate(); + }} + > +
+ + { + setUrl(event.target.value); + if (error) setError(null); + }} + placeholder="https://example.com/webhooks/cap" + className={clsx(MONO, "max-w-[520px] text-[13.5px]")} + /> +
+
+ + Events + +
+ {eventTypes.map((type) => ( + { + setEvents((current) => + on + ? eventTypes.filter( + (item) => item === type || current.includes(item), + ) + : current.filter((item) => item !== type), + ); + if (error) setError(null); + }} + /> + ))} +
+
+ {error && ( + + )} +
+ + +
+
+ ); +} + +function EndpointState({ endpoint }: { endpoint: InfraWebhookEndpoint }) { + const failing = endpoint.disabledByFailures; + return ( + + + ); +} + +function EndpointRow({ + appId, + endpoint, + canRotate, + canDelete, + onRotated, + onDeleted, +}: { + appId: string; + endpoint: InfraWebhookEndpoint; + canRotate: boolean; + canDelete: boolean; + onRotated: (secret: string) => void; + onDeleted: () => void; +}) { + const router = useRouter(); + const [test, setTest] = useState(null); + const [confirm, setConfirm] = useState<"rotate" | "delete" | null>(null); + + const refuse = (result: InfraWebhookFailure) => { + showRefusal(result); + if (result.status === 404) router.refresh(); + }; + + const sendTest = useMutation({ + mutationFn: () => sendInfraWebhookTest(appId, endpoint.id), + onMutate: () => setTest(null), + onSuccess: (result) => { + if (isFailure(result)) return refuse(result); + setTest(result.test); + router.refresh(); + }, + onError: () => toast.error("Could not send the test event"), + }); + + const toggle = useMutation({ + mutationFn: () => + setInfraWebhookEnabled(appId, endpoint.id, !endpoint.enabled), + onSuccess: (result) => { + if (isFailure(result)) return refuse(result); + toast.success( + result.endpoint.enabled ? "Endpoint turned on" : "Endpoint turned off", + ); + router.refresh(); + }, + onError: () => toast.error("Could not change the endpoint"), + }); + + const rotate = useMutation({ + mutationFn: () => rotateInfraWebhookSecret(appId, endpoint.id), + onSuccess: (result) => { + setConfirm(null); + if (isFailure(result)) return refuse(result); + toast.success("New signing secret generated"); + onRotated(result.secret); + router.refresh(); + }, + onError: () => toast.error("Could not rotate the signing secret"), + }); + + const remove = useMutation({ + mutationFn: () => deleteInfraWebhook(appId, endpoint.id), + onSuccess: (result) => { + setConfirm(null); + if (isFailure(result)) return refuse(result); + toast.success("Endpoint deleted"); + onDeleted(); + router.refresh(); + }, + onError: () => toast.error("Could not delete the endpoint"), + }); + + const busy = + sendTest.isPending || + toggle.isPending || + rotate.isPending || + remove.isPending; + + return ( +
  • +
    +

    + {endpoint.url} +

    + + + +
    +

    + {endpoint.events.join(", ")} + + , signing secret + + {endpoint.secretPrefix} + + +

    + {endpoint.disabledByFailures && !endpoint.enabled && ( +

    + Turned off by Cap after repeated failed deliveries. Fix the endpoint, + send a test, then turn it on. +

    + )} +

    + {lastDelivery(endpoint)} +

    + + {test ? testText(test) : ""} + +
    + + + + +
    + + rotate.mutate()} + onCancel={() => { + if (!rotate.isPending) setConfirm(null); + }} + /> + + Cap stops sending events to{" "} + {endpoint.url}{" "} + straight away. This cannot be undone. + + } + confirmLabel="Delete endpoint" + loading={remove.isPending} + onConfirm={() => remove.mutate()} + onCancel={() => { + if (!remove.isPending) setConfirm(null); + }} + /> +
  • + ); +} diff --git a/apps/web/app/(infra)/dashboard/infra/_components/app-switcher.tsx b/apps/web/app/(infra)/dashboard/infra/_components/app-switcher.tsx index 69816b8b561..5b1946eb1ba 100644 --- a/apps/web/app/(infra)/dashboard/infra/_components/app-switcher.tsx +++ b/apps/web/app/(infra)/dashboard/infra/_components/app-switcher.tsx @@ -46,17 +46,20 @@ export function AppSwitcher({ - + {!compact && } + {current.name} {!compact && ( - + )}
    {apps.map((app) => ( @@ -77,6 +80,7 @@ export function AppSwitcher({ className={ITEM} onSelect={() => selectApp(app.id)} > + {app.name} @@ -104,3 +108,18 @@ export function AppSwitcher({ ); } + +function AppMark({ name, small = false }: { name: string; small?: boolean }) { + const letter = Array.from(name.trim())[0]?.toUpperCase() ?? "A"; + return ( + + ); +} diff --git a/apps/web/app/(infra)/dashboard/infra/_components/create-app.tsx b/apps/web/app/(infra)/dashboard/infra/_components/create-app.tsx index ac021d93d99..ee5f13bdf7b 100644 --- a/apps/web/app/(infra)/dashboard/infra/_components/create-app.tsx +++ b/apps/web/app/(infra)/dashboard/infra/_components/create-app.tsx @@ -5,6 +5,7 @@ import { useRouter } from "next/navigation"; import { useId, useState } from "react"; import { toast } from "sonner"; import { createDeveloperApp } from "@/actions/developers/create-app"; +import { DEVELOPER_APP_NAME_MAX_LENGTH } from "@/lib/developer-app-rules"; import { useEffectMutation } from "@/lib/EffectRuntime"; import { infraAppPath } from "@/lib/infra-routes"; import { CopyTextButton, KeyRow } from "./copy"; @@ -140,7 +141,7 @@ export function AppNameForm({ setName(e.target.value)} diff --git a/apps/web/app/(infra)/dashboard/infra/_components/credit-transaction-table.tsx b/apps/web/app/(infra)/dashboard/infra/_components/credit-transaction-table.tsx index 75857d31326..68e1cc8b8db 100644 --- a/apps/web/app/(infra)/dashboard/infra/_components/credit-transaction-table.tsx +++ b/apps/web/app/(infra)/dashboard/infra/_components/credit-transaction-table.tsx @@ -14,26 +14,49 @@ function amountText(microCredits: number) { return `${sign}${formatCharge(microCredits)}`; } -function Detail({ row }: { row: LedgerRow }) { - const { note, reference } = describeLedgerRow(row); - if (!note && !reference) return null; +export const NO_REFERENCE = "–"; + +export const LEDGER_TYPE_CELL = "w-full min-w-[140px] !whitespace-normal"; + +export const LEDGER_REFERENCE_CELL = + "block max-w-[124px] xl:inline-block xl:max-w-[220px] xl:truncate xl:align-bottom"; + +function Note({ row }: { row: LedgerRow }) { + const { note } = describeLedgerRow(row); + if (!note) return null; return ( - <> - {reference && ( - - {reference.kind}{" "} - {reference.value} - - )} - {note && ( - - {note} - - )} - + + {note} + + ); +} + +export function Reference({ + row, + className, + stacked = false, +}: { + row: LedgerRow; + className?: string; + stacked?: boolean; +}) { + const { reference } = describeLedgerRow(row); + if (!reference) return <>{NO_REFERENCE}; + return ( + + + {reference.kind} + {" "} + + {reference.value} + + ); } @@ -66,11 +89,13 @@ export function CreditTransactionTable({ {formatLedgerTime(tx.createdAt)}
    + {describeLedgerRow(tx).label} - + + + {tx.id}
    - - Reference{" "} - {tx.id} + Reference +
    ))} diff --git a/apps/web/app/(infra)/dashboard/infra/_components/delivered-minutes.tsx b/apps/web/app/(infra)/dashboard/infra/_components/delivered-minutes.tsx new file mode 100644 index 00000000000..b299a348568 --- /dev/null +++ b/apps/web/app/(infra)/dashboard/infra/_components/delivered-minutes.tsx @@ -0,0 +1,94 @@ +import clsx from "clsx"; +import { + DELIVERY_TIERS, + type DeliveryJson, + type DeliveryTier, +} from "@/lib/infra-delivery"; +import { formatMinutes } from "@/lib/infra-usage"; +import { Block, Empty, MONO, TD, TH, TR } from "./ui"; + +export const DELIVERY_TIER_LABELS: Record = { + "360p": "360p", + "720p": "720p", + "1080p": "1080p", + "1440p": "1440p", + "2160p": "4K", +}; + +export const NO_DELIVERY = "No delivery measured yet."; + +export const DELIVERY_UNAVAILABLE = "Delivery usage unavailable."; + +export const DELIVERY_MAY_CHANGE = "Today and yesterday may still change."; + +export function DeliveredMinutes({ + delivery, + monthName, + unavailable = false, +}: { + delivery: DeliveryJson | null; + monthName: string; + unavailable?: boolean; +}) { + const tiers = delivery + ? DELIVERY_TIERS.filter((tier) => delivery.minutesByTier[tier] > 0) + : []; + return ( + + {unavailable ? ( + {DELIVERY_UNAVAILABLE} + ) : !delivery ? ( + {NO_DELIVERY} + ) : ( + <> + + + + + + + + + {tiers.map((tier) => ( + + + + + ))} + + + + + +
    Resolution + Minutes in {monthName} +
    + {DELIVERY_TIER_LABELS[tier]} + + {formatMinutes(delivery.minutesByTier[tier])} +
    Total + {formatMinutes(delivery.minutes)} +
    +

    + {DELIVERY_MAY_CHANGE} +

    + + )} +
    + ); +} diff --git a/apps/web/app/(infra)/dashboard/infra/_components/dialog.tsx b/apps/web/app/(infra)/dashboard/infra/_components/dialog.tsx index ca82c7a5d91..f2e67d9d385 100644 --- a/apps/web/app/(infra)/dashboard/infra/_components/dialog.tsx +++ b/apps/web/app/(infra)/dashboard/infra/_components/dialog.tsx @@ -33,7 +33,7 @@ export function Modal({ }} > - + { opener.current = document.activeElement; @@ -45,7 +45,7 @@ export function Modal({ target.focus(); } }} - className={`${infraSans.variable} ${infraMono.variable} fixed left-1/2 top-1/2 z-[601] w-[calc(100%-32px)] max-w-[440px] -translate-x-1/2 -translate-y-1/2 rounded-[14px] bg-[#0B0C0E] p-6 text-[#E8ECF4] shadow-[inset_0_0_0_1px_rgba(222,230,246,0.12),0_24px_80px_rgba(0,0,0,0.6)] [font-family:var(--font-infra-sans),system-ui,sans-serif] focus:outline-none`} + className={`${infraSans.variable} ${infraMono.variable} fixed left-1/2 top-1/2 z-[601] w-[calc(100%-32px)] max-w-[440px] -translate-x-1/2 -translate-y-1/2 rounded-[16px] bg-[#111216] p-6 text-[#E8ECF4] shadow-[inset_0_0_0_1px_rgba(222,230,246,0.12),0_24px_80px_rgba(0,0,0,0.6)] [font-family:var(--font-infra-sans),system-ui,sans-serif] focus:outline-none`} > {title} diff --git a/apps/web/app/(infra)/dashboard/infra/_components/domain-row.tsx b/apps/web/app/(infra)/dashboard/infra/_components/domain-row.tsx new file mode 100644 index 00000000000..1fd346147dc --- /dev/null +++ b/apps/web/app/(infra)/dashboard/infra/_components/domain-row.tsx @@ -0,0 +1,57 @@ +"use client"; + +import clsx from "clsx"; +import { X } from "lucide-react"; +import { useRouter } from "next/navigation"; +import { toast } from "sonner"; +import { removeDeveloperDomain } from "@/actions/developers/remove-domain"; +import { useEffectMutation } from "@/lib/EffectRuntime"; +import { runAction } from "./run-action"; +import { FOCUS, MONO } from "./ui"; + +export function DomainRow({ + appId, + domainId, + domain, +}: { + appId: string; + domainId: string; + domain: string; +}) { + const router = useRouter(); + const removeMutation = useEffectMutation({ + mutationFn: () => runAction(() => removeDeveloperDomain(appId, domainId)), + onSuccess: () => { + toast.success(`${domain} removed`); + router.refresh(); + }, + onError: () => { + toast.error("Could not remove the site"); + }, + }); + + return ( +
  • + + {domain} + + +
  • + ); +} diff --git a/apps/web/app/(infra)/dashboard/infra/_components/first-run.tsx b/apps/web/app/(infra)/dashboard/infra/_components/first-run.tsx index 5ee9b3f0388..ecf5b49e437 100644 --- a/apps/web/app/(infra)/dashboard/infra/_components/first-run.tsx +++ b/apps/web/app/(infra)/dashboard/infra/_components/first-run.tsx @@ -3,6 +3,7 @@ import clsx from "clsx"; import Link from "next/link"; import { useState } from "react"; +import { PRO_CREDIT_TRIAL_MESSAGE } from "@/lib/infra-pro-credit-eligibility"; import { infraAppPath } from "@/lib/infra-routes"; import type { InfraProCredit } from "../infra-data"; import { AgentSetup } from "./agent-setup"; @@ -66,8 +67,10 @@ export function FirstRun({

    Infra is separate from Cap. Your recordings, plan and team stay exactly as they are. - {proCredit.status !== "not_pro" && - " Your Cap Pro plan adds $5 of Infra credit every month."} + {proCredit.status === "trial" + ? ` ${PRO_CREDIT_TRIAL_MESSAGE}` + : proCredit.status !== "not_pro" && + " Your Cap Pro plan adds $5 of Infra credit every month."}

    @@ -121,7 +124,8 @@ export function FirstRun({ {!created ? ( Uploading needs a balance. Credits start at $5 - {proCredit.status === "not_pro" + {proCredit.status === "not_pro" || + proCredit.status === "trial" ? "." : ", and your Cap Pro plan covers the first $5 each month."} diff --git a/apps/web/app/(infra)/dashboard/infra/_components/get-started.tsx b/apps/web/app/(infra)/dashboard/infra/_components/get-started.tsx index 00c859b4f83..fd3f31c3bb2 100644 --- a/apps/web/app/(infra)/dashboard/infra/_components/get-started.tsx +++ b/apps/web/app/(infra)/dashboard/infra/_components/get-started.tsx @@ -1,5 +1,6 @@ import clsx from "clsx"; import Link from "next/link"; +import { PRO_CREDIT_TRIAL_MESSAGE } from "@/lib/infra-pro-credit-eligibility"; import { infraAppPath } from "@/lib/infra-routes"; import type { InfraProCredit } from "../infra-data"; import { CodeBlock } from "./copy"; @@ -35,7 +36,9 @@ export function FirstCredit({ {proCredit.status === "claimed" ? "Uploading needs a balance. This month's Cap Pro credit is already claimed, and it is one per account, not one per app." - : "Uploading needs a balance. Credits start at $5, and Cap Pro includes $5 every month."} + : proCredit.status === "trial" + ? `Uploading needs a balance. ${PRO_CREDIT_TRIAL_MESSAGE}` + : "Uploading needs a balance. Credits start at $5, and Cap Pro includes $5 every month."}
    ); diff --git a/apps/web/app/(infra)/dashboard/infra/_components/pro-credit.tsx b/apps/web/app/(infra)/dashboard/infra/_components/pro-credit.tsx index 2c1da8f999e..d5988f2e133 100644 --- a/apps/web/app/(infra)/dashboard/infra/_components/pro-credit.tsx +++ b/apps/web/app/(infra)/dashboard/infra/_components/pro-credit.tsx @@ -6,6 +6,7 @@ import { useState } from "react"; import { toast } from "sonner"; import { claimDeveloperProCredit } from "@/actions/developers/claim-pro-credit"; import { useEffectMutation } from "@/lib/EffectRuntime"; +import { PRO_CREDIT_TRIAL_MESSAGE } from "@/lib/infra-pro-credit-eligibility"; import type { InfraProCredit } from "../infra-data"; import { actionMessage, runAction } from "./run-action"; import { Button, TEXT_LINK } from "./ui"; @@ -53,6 +54,10 @@ export function ProCredit({ ); } + if (credit.status === "trial") { + return

    {PRO_CREDIT_TRIAL_MESSAGE}

    ; + } + if (claimedHere || credit.status === "claimed") { return (

    diff --git a/apps/web/app/(infra)/dashboard/infra/_components/shell.tsx b/apps/web/app/(infra)/dashboard/infra/_components/shell.tsx index 9227d257258..96c91ff4c71 100644 --- a/apps/web/app/(infra)/dashboard/infra/_components/shell.tsx +++ b/apps/web/app/(infra)/dashboard/infra/_components/shell.tsx @@ -1,6 +1,20 @@ "use client"; import clsx from "clsx"; +import { + ArrowLeft, + AudioLines, + BookOpen, + ChartNoAxesColumn, + Clapperboard, + CreditCard, + KeyRound, + LayoutGrid, + type LucideIcon, + Plus, + Settings2, + Webhook, +} from "lucide-react"; import Link from "next/link"; import { usePathname } from "next/navigation"; import { useCallback, useEffect, useId, useRef, useState } from "react"; @@ -8,13 +22,14 @@ import { Lockup } from "@/components/infra/Lockup"; import { INFRA_BASE_PATH, INFRA_SECTIONS, + type InfraSection, infraAppPath, parseInfraPathname, } from "@/lib/infra-routes"; -import { formatCredits } from "@/lib/infra-usage"; +import { formatBalanceCents, formatCredits } from "@/lib/infra-usage"; import type { InfraAppSummary } from "../infra-data"; import { AppSwitcher } from "./app-switcher"; -import { FOCUS, FOCUS_INSET, LIT, MONO } from "./ui"; +import { FOCUS, FOCUS_INSET, LIT, MONO, SURFACE } from "./ui"; import { useMayLeave, useMayLeavePage } from "./uploads"; type Navigation = { preventDefault: () => void }; @@ -30,6 +45,7 @@ export function Brand({ @@ -38,17 +54,37 @@ export function Brand({ } const QUIET_LINK = clsx( - "rounded-[6px] text-[13px] text-[rgba(232,236,244,0.5)] transition-colors duration-200 hover:text-[#E8ECF4]", + "inline-flex items-center gap-1.5 rounded-[6px] text-[13px] text-[rgba(232,236,244,0.56)] transition-colors duration-200 hover:text-[#E8ECF4]", FOCUS, ); +const SECTION_ICONS: Record = { + "": LayoutGrid, + videos: Clapperboard, + audio: AudioLines, + usage: ChartNoAxesColumn, + billing: CreditCard, + keys: KeyRound, + webhooks: Webhook, + settings: Settings2, +}; + +const ACCOUNT_SECTIONS = new Set([ + "billing", + "keys", + "webhooks", + "settings", +]); + export function Shell({ apps, appId, + balanceMicroCredits, children, }: { apps: InfraAppSummary[]; appId: string; + balanceMicroCredits: number; children: React.ReactNode; }) { const pathname = usePathname(); @@ -106,75 +142,127 @@ export function Shell({ > Skip to content -