diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..533028a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,132 @@ +# Tests on x86_64 and aarch64, then publishes the OCI image to +# ghcr.io//helios on pushes to master and on v* tags. +name: ci + +on: + push: + branches: [master] + tags: ["v*"] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + test: + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + system: x86_64-linux + arch: amd64 + - runner: ubuntu-24.04-arm + system: aarch64-linux + arch: arm64 + runs-on: ${{ matrix.runner }} + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: DeterminateSystems/nix-installer-action@3138316df39ed29be04236d7ffc686fa525866aa # v23 + with: + # Upstream Nix, as users of the module and image run. + determinate: false + extra-conf: | + system-features = nixos-test benchmark big-parallel kvm + + - name: Format + run: nix develop -c cargo fmt --check + - name: Clippy + run: nix develop -c cargo clippy --release --all-targets -- -D warnings + - name: Rust tests + run: nix develop -c cargo test --release + - name: Zig tests + run: nix develop -c ./scripts/test-zig.sh + - name: End to end + run: nix develop -c sh -c 'cargo build --release && ./scripts/e2e.sh' + + # The NixOS and Docker VM tests need KVM, which only the x86_64 + # runners have. + - name: Enable KVM + if: matrix.system == 'x86_64-linux' + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Nix checks + if: matrix.system == 'x86_64-linux' + run: nix flake check -L + - name: Nix build + if: matrix.system != 'x86_64-linux' + run: nix build -L .#checks.${{ matrix.system }}.package .#docker + + # Builds and pushes each architecture's image. Kept apart from the tests, + # which run pull request code, so only pushes ever hold a write token. + push: + if: github.event_name == 'push' + needs: test + strategy: + matrix: + include: + - runner: ubuntu-24.04 + arch: amd64 + - runner: ubuntu-24.04-arm + arch: arm64 + runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: DeterminateSystems/nix-installer-action@3138316df39ed29be04236d7ffc686fa525866aa # v23 + with: + determinate: false + + - name: Push image + env: + ARCH: ${{ matrix.arch }} + TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + version=$(nix eval --raw .#helios.version) + if [[ "$GITHUB_REF_TYPE" == tag && "$GITHUB_REF_NAME" != "v$version" ]]; then + echo "tag $GITHUB_REF_NAME does not match package version $version" >&2 + exit 1 + fi + image="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/helios" + nix run --inputs-from . nixpkgs#skopeo -- copy \ + --dest-creds "$GITHUB_ACTOR:$TOKEN" \ + "docker-archive:$(nix build --print-out-paths .#docker)" \ + "docker://$image:sha-${GITHUB_SHA::12}-$ARCH" + + # Joins the per-architecture images into one multi-arch tag. + image: + if: github.event_name == 'push' + needs: push + runs-on: ubuntu-24.04 + permissions: + packages: write + steps: + - name: Publish manifest + env: + TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "$TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + image="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/helios" + sha="sha-${GITHUB_SHA::12}" + tags=(-t "$image:$sha") + if [[ "$GITHUB_REF_TYPE" == tag ]]; then + tags+=(-t "$image:${GITHUB_REF_NAME#v}" -t "$image:latest") + else + tags+=(-t "$image:$GITHUB_REF_NAME") + fi + docker buildx imagetools create "${tags[@]}" "$image:$sha-amd64" "$image:$sha-arm64" diff --git a/.gitignore b/.gitignore index f4e6192..c8709fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,17 @@ -node_modules/ -.pnpm-store/ -.wrangler/ -coverage/ -dist/ -*.log .DS_Store +*.log result +result-* + +# Rust +/target +/bench/flake-lock-rs/target + +# Zig +.zig-cache/ +zig-out/ + +# Benchmark output (bench/*.sh, perf, poop): keep results out of the tree +/bench/**/*.txt +*.tsv +perf.data* diff --git a/AGENTS.md b/AGENTS.md index c3995a3..7727c5f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,31 +1,72 @@ # Helios -This repository is a `pnpm` workspace for Cloudflare-native services. +A self-hosted Nix binary cache: Rust server and CLI over a Zig core. ## Workspace Layout -- `workers/*`: deployable Cloudflare Workers -- `apps/*`: operator or end-user applications that sit beside Workers -- `packages/*`: shared code used by workers or apps +- `pkg/*`: Zig packages for Nix formats and the NAR pipeline, one concern + each. Each builds and tests on its own (`zig build test`). +- `core/`: libhelios, the C ABI over `pkg/*`, compiled to a static library + by `crates/helios-core/build.rs`. `core/include/helios.h` is the contract. +- `crates/helios-core`: safe Rust bindings to libhelios. +- `crates/helios-server`, `crates/helios-cli`, `crates/helios-daemon`: the + binaries. `crates/helios-log` sets up their logging. The daemon drives + the server over its maintenance socket and never writes the database + itself. +- `nix/`: the NixOS module (`services.helios`), the OCI image, and their VM + tests. +- `bench/`: comparisons against the Haskell Nix libraries, the Rust + nix-flake-lock crate and Nix C++. Not part of the Cargo workspace. -Keep deployable runtime code inside `workers/`. Move shared protocol, parsing, and storage helpers into `packages/` only after they are reused. +Put Nix format logic in a `pkg/*` package and expose it through +`core/src/root.zig` only when Rust needs it. Keep calls across the C ABI +coarse, such as a whole NAR or narinfo per call. ## Tooling -- Use `pnpm`, not `npm`, for dependency management and scripts. -- Do not commit `package-lock.json` files. -- If `pnpm` or `wrangler` are missing locally, use `nix shell nixpkgs#nodejs nixpkgs#pnpm nixpkgs#wrangler`. -- Run workspace commands from the repository root unless a package-specific command is clearer. +- Use `nix develop` for zig 0.17, cargo, zstd and pkg-config. +- Zig 0.17 APIs changed a lot; check `zig env` for the std source rather + than relying on older examples. +- System libraries: zstd and sqlite (pkg-config). +- `HELIOS_ZIG_CPU` sets libhelios's `-Dcpu` (default `native`; the Nix + package uses `baseline`). -## Cloudflare Rules +## Dependencies -- Before changing Workers, R2, D1, Durable Objects, Queues, or platform limits, fetch current Cloudflare docs. -- Prefer platform-native APIs such as `fetch`, Web Streams, and Web Crypto. -- Do not enable `nodejs_compat` unless a concrete dependency requires it. -- After changing a worker's bindings in `wrangler.jsonc`, run that package's `cf-typegen` script. +Keep the dependency tree small; every crate is build time, audit surface and +supply-chain risk. Before adding one, check whether libhelios, std or a few +dozen lines cover it, and disable default features. Deliberately absent: +reqwest (and `url`/`idna`), aws-lc-rs, parking_lot, uuid, +hmac/sha2 (libhelios provides SHA-256, HMAC and randomness) and mimalloc +(measured: no gain here). + +Logging is `tracing` with `tracing-subscriber` (fmt, ansi and json only; no +`env-filter`, whose regex engine `Targets` makes unnecessary) and +`tracing-journald`, set up once in `crates/helios-log`. + +## Performance Rules + +- Measure before and after; `bench/run.sh` and `bench/flake-lock.sh` + compare against the reference implementations on the same corpus. + `bench/poop.sh` compares whole processes with poop (wall time, peak RSS, + cycles, cache and branch misses), including `helios push` against + `nix copy`. Benchmark output stays out of the tree. +- Keep the narinfo read path free of SQLite on misses and of per-request + signing. +- Keep a scalar reference for every SIMD routine, with a test asserting the + two agree. ## Verification -- Install dependencies with `pnpm install` from the repo root. -- Run `pnpm check` before finishing substantial changes. -- For worker-only iteration, use `pnpm --filter dev`. +CI (`.github/workflows/ci.yml`) runs all of this on x86_64 and aarch64 +(the VM tests on x86_64 only) and publishes the image on pushes. + +- `cargo fmt --check` and `cargo clippy --release --all-targets -- -D warnings` +- `cargo test --release` +- `./scripts/test-zig.sh` +- `nix build .#checks.x86_64-linux.nixos` after changing `nix/`: a VM test of + the NixOS module. `.#checks.x86_64-linux.docker` likewise for the OCI + image. +- `./scripts/e2e.sh` before finishing changes to the server, the CLI or wire + formats; it substitutes a real closure through Nix with signatures + required. diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..e28ccef --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1054 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "find-msvc-tools" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "helios-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64", + "bytes", + "clap", + "futures-util", + "helios-core", + "helios-log", + "http", + "http-body-util", + "hyper", + "rustls", + "rustls-native-certs", + "serde", + "serde_json", + "tokio", + "tokio-rustls", + "tracing", +] + +[[package]] +name = "helios-core" +version = "0.1.0" +dependencies = [ + "pkg-config", +] + +[[package]] +name = "helios-daemon" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "clap", + "futures-util", + "helios-core", + "helios-log", + "http", + "http-body-util", + "hyper", + "libc", + "serde", + "serde_json", + "tokio", + "tracing", +] + +[[package]] +name = "helios-log" +version = "0.1.0" +dependencies = [ + "tracing", + "tracing-journald", + "tracing-subscriber", +] + +[[package]] +name = "helios-server" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "base64", + "bytes", + "clap", + "futures-util", + "helios-core", + "helios-log", + "quick_cache", + "rusqlite", + "serde", + "serde_json", + "tokio", + "tracing", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lazy_static" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick_cache" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403c1a912fec895cafb223201e368234842acb9220aaf08ab042ae89ba5f135c" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "rustls" +version = "0.23.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tokio" +version = "1.53.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-journald" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d3a81ed245bfb62592b1e2bc153e77656d94ee6a0497683a65a12ccaf2438d0" +dependencies = [ + "libc", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "nu-ansi-term", + "serde", + "serde_json", + "sharded-slab", + "thread_local", + "tracing-core", + "tracing-serde", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..afbb718 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,14 @@ +[workspace] +resolver = "3" +members = ["crates/*"] + +[workspace.package] +version = "0.1.0" +edition = "2024" +license-file = "LICENSE" +publish = false + +[profile.release] +lto = "fat" +codegen-units = 1 +panic = "abort" diff --git a/README.md b/README.md index 46d98b1..effcd3e 100644 --- a/README.md +++ b/README.md @@ -1,87 +1,74 @@ # Helios -A Cloudflare-native Nix binary cache. +A self-hosted Nix binary cache for a single server: a Rust HTTP server and +CLI on top of a Zig core library for NAR serialisation, hashing, +compression, narinfo and signing. -[![Deploy to Cloudflare](https://deploy.workers.cloudflare.com/button)](https://deploy.workers.cloudflare.com/?url=https://github.com/Polymath-AS/helios) -## Workspace +## Layout ``` -workers/cache/ Cloudflare Worker (main service) -packages/cache-domain/ Shared types and input parsers -apps/cli/ CLI for pushing store paths -scripts/ smoke-test.sh -``` - -## Setup - -```bash -nix develop # drops you into a shell with node, pnpm, wrangler -pnpm install +pkg/ Zig packages: nix-base32, nix-store-path, nix-archive, + nix-narinfo, nix-derivation, nix-flake-lock +core/ libhelios: C ABI over pkg/*, built as a static library +crates/ helios-core (Rust bindings), helios-server, helios-cli, + helios-daemon (auto-GC, scrub, backups, metrics) +nix/ NixOS module (services.helios) and its VM test +bench/ benchmarks against the Haskell and Rust Nix libraries and Nix C++ +scripts/ e2e.sh (end-to-end test), test-zig.sh (Zig unit tests) ``` ## Develop ```bash -pnpm dev # runs the worker locally via wrangler -pnpm check # type-check + tests across the workspace -``` - -## Deploy - -See [docs.md](docs.md) for the full reference. Short version: - -```bash -wrangler r2 bucket create helios-cache -wrangler d1 create helios-cache -# update database_id in workers/cache/wrangler.jsonc -wrangler d1 migrations apply helios-cache --remote -wrangler secret put JWT_SECRET -wrangler secret put ADMIN_SECRET -wrangler secret put SIGNING_PRIVATE_KEY -wrangler secret put SIGNING_KEY_NAME -pnpm deploy +nix develop # zig, rust, zstd, pkg-config +cargo build --release # builds libhelios via zig automatically +cargo test --release +./scripts/test-zig.sh # unit tests for every pkg/* +./scripts/e2e.sh # server + CLI + real `nix copy` substitution ``` -## Push store paths +`nix build` produces `helios` (CLI) and `helios-server`. -Build the CLI (or `nix run .` for one-off use): +## Run the server ```bash -nix build # produces ./result/bin/helios +nix key generate-secret --key-name cache.example.com-1 > /var/lib/helios/signing.key +openssl rand -hex 32 > /var/lib/helios/jwt.secret +openssl rand -hex 32 > /var/lib/helios/admin.secret + +helios-server \ + --listen 127.0.0.1:8080 \ + --data-dir /var/lib/helios \ + --signing-key-file /var/lib/helios/signing.key \ + --jwt-secret-file /var/lib/helios/jwt.secret \ + --admin-secret-file /var/lib/helios/admin.secret + +helios-server --print-public-key --signing-key-file /var/lib/helios/signing.key ``` -Save credentials once: +## Push and substitute ```bash -helios login prod https://your-worker.workers.dev "$PUSH_TOKEN" -``` +helios login admin https://cache.example.com "$(cat admin.secret)" +helios cache create main +helios token create ci --caches main --perms push,pull # prints the token once -Push a single path: - -```bash -helios push main /nix/store/abc...-hello -``` - -Push a full closure: - -```bash +helios login ci https://cache.example.com "$TOKEN" helios push main --closure /run/current-system ``` -Push a flake output closure: - ```bash -helios push main --closure .#nixosConfigurations.myhost.config.system.build.toplevel +helios use main # this user's Nix pulls from the cache +helios use main --print # settings for NixOS or /etc/nix/nix.conf ``` -See [docs.md](docs.md) for token management and parallelism options. +On NixOS, use the flake's `nixosModules.default` (`services.helios`): the +server, the maintenance daemon, and push-on-build for build machines. +Elsewhere, run the `ghcr.io/polymath-as/helios` image: the server and daemon +in one container that generates its keys on first start. -## Use as a substituter - -```nix -# configuration.nix or flake -nix.settings.substituters = [ "https://your-worker.workers.dev/main" ]; -``` +See [docs.md](docs.md) for the NixOS module, configuration, private caches, +the HTTP API and deployment behind a reverse proxy. ## License diff --git a/apps/.gitkeep b/apps/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/apps/cli/package.json b/apps/cli/package.json deleted file mode 100644 index a48b7cf..0000000 --- a/apps/cli/package.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "@helios/cli", - "version": "0.0.0", - "private": true, - "type": "module", - "bin": { - "helios": "./dist/main.js" - }, - "scripts": { - "build": "tsc", - "check": "tsc --noEmit" - }, - "dependencies": { - "@helios/cache-domain": "workspace:*" - }, - "devDependencies": { - "@types/node": "^26.1.1", - "typescript": "^7.0.2" - } -} diff --git a/apps/cli/src/api.ts b/apps/cli/src/api.ts deleted file mode 100644 index 4f75b19..0000000 --- a/apps/cli/src/api.ts +++ /dev/null @@ -1,278 +0,0 @@ -import { createReadStream } from "node:fs"; -import { stat } from "node:fs/promises"; -import { runPool } from "./pool.js"; - -export interface ApiClient { - readonly server: string; - readonly token: string; -} - -export function createClient(server: string, token: string): ApiClient { - return { server: server.replace(/\/$/, ""), token }; -} - -async function request( - client: ApiClient, - method: string, - path: string, - body?: unknown, - headers?: Record, -): Promise { - const url = `${client.server}${path}`; - const reqHeaders: Record = { - authorization: `Bearer ${client.token}`, - ...headers, - }; - - const init: RequestInit = { method, headers: reqHeaders }; - - if (body !== undefined && typeof body === "object" && !(body instanceof ReadableStream)) { - reqHeaders["content-type"] = "application/json"; - init.body = JSON.stringify(body); - } else if (body instanceof ReadableStream) { - init.body = body; - init.duplex = "half"; - } - - return fetch(url, init); -} - -// System boundary: Response.json() returns Promise in Node's lib types. -// We trust the shape of our own API responses at this boundary. -async function jsonBody(resp: Response): Promise { - return resp.json() as Promise; -} - -export interface SessionResponse { - readonly sessionId: string; - readonly r2Key: string; - readonly uploadMethod: string; - readonly expiresAt: string; -} - -export async function getMissingPaths( - client: ApiClient, - cache: string, - storePathHashes: string[], -): Promise { - const resp = await request(client, "POST", "/_api/v1/get-missing-paths", { - cache, - storePathHashes, - }); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`get-missing-paths failed (${resp.status}): ${text}`); - } - const data = await jsonBody<{ missing: string[] }>(resp); - return data.missing; -} - -export async function createUploadSession( - client: ApiClient, - cache: string, - params: { - storePath: string; - storePathHash: string; - narHash: string; - narSize: number; - fileHash: string; - fileSize: number; - compression: string; - references: string[]; - deriver?: string; - system?: string; - }, -): Promise { - const resp = await request( - client, - "POST", - `/_api/v1/caches/${cache}/upload-sessions`, - params, - ); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`create-session failed (${resp.status}): ${text}`); - } - return jsonBody(resp); -} - -export async function uploadBlob( - client: ApiClient, - sessionId: string, - filePath: string, -): Promise { - const fileInfo = await stat(filePath); - const stream = createReadStream(filePath); - const webStream = ReadableStream.from(stream); - - const resp = await request(client, "PUT", `/_api/v1/uploads/${sessionId}/blob`, webStream, { - "content-type": "application/octet-stream", - "content-length": String(fileInfo.size), - }); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`blob upload failed (${resp.status}): ${text}`); - } -} - -export async function completeUpload( - client: ApiClient, - sessionId: string, -): Promise { - const resp = await request(client, "POST", `/_api/v1/uploads/${sessionId}/complete`, {}); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`complete failed (${resp.status}): ${text}`); - } -} - -const MULTIPART_THRESHOLD = 90 * 1024 * 1024; // 90MB - -export function shouldUseMultipart(fileSize: number): boolean { - return fileSize > MULTIPART_THRESHOLD; -} - -export async function initiateMultipart( - client: ApiClient, - sessionId: string, -): Promise<{ uploadId: string; r2Key: string }> { - const resp = await request(client, "POST", `/_api/v1/uploads/${sessionId}/multipart`, {}); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`multipart initiation failed (${resp.status}): ${text}`); - } - return jsonBody<{ uploadId: string; r2Key: string }>(resp); -} - -export async function uploadPart( - client: ApiClient, - sessionId: string, - partNumber: number, - body: ReadableStream, - contentLength: number, -): Promise<{ partNumber: number; etag: string }> { - const resp = await request( - client, - "PUT", - `/_api/v1/uploads/${sessionId}/part/${String(partNumber)}`, - body, - { - "content-type": "application/octet-stream", - "content-length": String(contentLength), - }, - ); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`part upload failed (${resp.status}): ${text}`); - } - return jsonBody<{ partNumber: number; etag: string }>(resp); -} - -// Parts of a single blob upload in parallel. Kept modest because pushPaths -// already uploads several blobs concurrently. -const PART_UPLOAD_CONCURRENCY = 4; - -export async function uploadBlobMultipart( - client: ApiClient, - sessionId: string, - filePath: string, -): Promise { - const fileInfo = await stat(filePath); - const fileSize = Number(fileInfo.size); - const partSize = 90 * 1024 * 1024; // 90MB parts - - await initiateMultipart(client, sessionId); - - const totalParts = Math.ceil(fileSize / partSize); - const partNumbers = Array.from({ length: totalParts }, (_, i) => i + 1); - - // Parts are independent byte ranges of the same file, so they can be - // read and uploaded concurrently. - await runPool(partNumbers, PART_UPLOAD_CONCURRENCY, async (partNumber) => { - const start = (partNumber - 1) * partSize; - const end = Math.min(start + partSize, fileSize); - const length = end - start; - - const stream = createReadStream(filePath, { start, end: end - 1 }); - const webStream = ReadableStream.from(stream); - - await uploadPart(client, sessionId, partNumber, webStream, length); - }); -} - -export async function publishPath( - client: ApiClient, - sessionId: string, -): Promise<{ published: boolean; alreadyExisted?: boolean }> { - const resp = await request(client, "POST", `/_api/v1/uploads/${sessionId}/publish`, {}); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`publish failed (${resp.status}): ${text}`); - } - return jsonBody<{ published: boolean; alreadyExisted?: boolean }>(resp); -} - -// ── Token Management (Admin API) ── - -export interface TokenInfo { - readonly jti: string; - readonly subject: string; - readonly caches: string[]; - readonly perms: string[]; - readonly createdAt: string; - readonly expiresAt: string; - readonly createdBy: string; - readonly revokedAt: string | null; - readonly revokedBy: string | null; - readonly revocationReason: string | null; -} - -export interface CreateTokenResponse { - readonly token: string; - readonly jti: string; - readonly subject: string; - readonly caches: string[]; - readonly perms: string[]; - readonly expiresAt: string; -} - -export async function createToken( - client: ApiClient, - params: { - subject: string; - caches: string[]; - perms: string[]; - expiresInDays?: number; - }, -): Promise { - const resp = await request(client, "POST", "/_api/v1/admin/tokens", params); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`token create failed (${resp.status}): ${text}`); - } - return jsonBody(resp); -} - -export async function listTokens( - client: ApiClient, -): Promise { - const resp = await request(client, "GET", "/_api/v1/admin/tokens"); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`token list failed (${resp.status}): ${text}`); - } - const data = await jsonBody<{ tokens: TokenInfo[] }>(resp); - return data.tokens; -} - -export async function revokeToken( - client: ApiClient, - jti: string, - reason: string, -): Promise { - const resp = await request(client, "POST", `/_api/v1/admin/tokens/${jti}/revoke`, { reason }); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`token revoke failed (${resp.status}): ${text}`); - } -} diff --git a/apps/cli/src/config.ts b/apps/cli/src/config.ts deleted file mode 100644 index bbe8346..0000000 --- a/apps/cli/src/config.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { readFile, writeFile, mkdir } from "node:fs/promises"; -import { homedir } from "node:os"; -import { join } from "node:path"; - -interface ServerConfig { - readonly server: string; - readonly token: string; -} - -interface HeliosConfig { - readonly defaultServer?: string; - readonly servers: Record; -} - -const CONFIG_DIR = join(homedir(), ".config", "helios"); -const CONFIG_PATH = join(CONFIG_DIR, "config.json"); - -export async function loadConfig(): Promise { - try { - const raw = await readFile(CONFIG_PATH, "utf-8"); - return JSON.parse(raw); - } catch { - return { servers: {} }; - } -} - -export async function saveConfig(config: HeliosConfig): Promise { - await mkdir(CONFIG_DIR, { recursive: true }); - await writeFile(CONFIG_PATH, JSON.stringify(config, null, 2) + "\n", { - mode: 0o600, - }); -} - -export async function login( - name: string, - server: string, - token: string, -): Promise { - const config = await loadConfig(); - const servers = { ...config.servers, [name]: { server, token } }; - await saveConfig({ ...config, defaultServer: name, servers }); -} - -export async function getServer( - name?: string, -): Promise { - const config = await loadConfig(); - const key = name ?? config.defaultServer; - if (!key) { - throw new Error("No server specified and no default configured. Run: helios login "); - } - const entry = config.servers[key]; - if (!entry) { - throw new Error(`Server '${key}' not found in config. Run: helios login `); - } - return entry; -} diff --git a/apps/cli/src/main.ts b/apps/cli/src/main.ts deleted file mode 100644 index f596de4..0000000 --- a/apps/cli/src/main.ts +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env node - -import { login, getServer } from "./config.js"; -import { createClient, createToken, listTokens, revokeToken } from "./api.js"; -import { pushPaths } from "./push.js"; -import { getClosurePaths } from "./nix.js"; - -const USAGE = `helios - Nix binary cache CLI - -Commands: - helios login Save server credentials - helios push Push store paths to a cache - helios push --closure Push a store path and its closure - helios token create Create a new API token - helios token list List all API tokens - helios token revoke Revoke an API token - -Options: - --server Use a specific server (default: last logged-in) - --jobs Number of parallel uploads (default: 8) - --help Show this help - -Token create options: - --caches Comma-separated cache names or "*" (default: *) - --perms Comma-separated permissions (default: push) - --expires Token lifetime in days (default: 90) - -Examples: - helios login prod https://cache.example.com my-admin-secret - helios push main /nix/store/abc...-hello - helios push main --closure /run/current-system - helios token create ci-runner --caches main --perms push --expires 90 - helios token list - helios token revoke a1b2c3d4-... "employee offboarded" -`; - -async function main(): Promise { - const args = process.argv.slice(2); - - if (args.length === 0 || args.includes("--help") || args.includes("-h")) { - console.log(USAGE); - process.exit(0); - } - - const command = args[0]; - - if (command === "login") { - if (args.length < 4) { - console.error("Usage: helios login "); - process.exit(1); - } - await login(args[1], args[2], args[3]); - console.log(`Logged in to '${args[1]}' at ${args[2]}`); - return; - } - - if (command === "push") { - if (args.length < 3) { - console.error("Usage: helios push | helios push --closure "); - process.exit(1); - } - - const filteredArgs = [...args]; - - // Extract --server flag - let serverName: string | undefined; - const serverIdx = filteredArgs.indexOf("--server"); - if (serverIdx !== -1 && serverIdx + 1 < filteredArgs.length) { - serverName = filteredArgs[serverIdx + 1]; - filteredArgs.splice(serverIdx, 2); - } - - // Extract --jobs flag - let concurrency: number | undefined; - const jobsIdx = filteredArgs.indexOf("--jobs"); - if (jobsIdx !== -1 && jobsIdx + 1 < filteredArgs.length) { - concurrency = parseInt(filteredArgs[jobsIdx + 1], 10); - filteredArgs.splice(jobsIdx, 2); - } - - if (concurrency !== undefined && (!Number.isInteger(concurrency) || concurrency < 1)) { - console.error("--jobs must be a positive integer"); - process.exit(1); - } - - const serverConfig = await getServer(serverName); - const client = createClient(serverConfig.server, serverConfig.token); - const cache = filteredArgs[1]; - - const progress = (current: number, total: number, name: string, status: string) => { - console.log(`[${String(current)}/${String(total)}] ${name} ${status}`); - }; - - const pushOpts = { concurrency, onProgress: progress }; - - const closureIdx = filteredArgs.indexOf("--closure"); - let result; - if (closureIdx !== -1) { - const rootPath = filteredArgs[closureIdx + 1]; - if (!rootPath) { - console.error("--closure requires a store path or installable"); - process.exit(1); - } - console.log(`Resolving closure for ${rootPath}...`); - const paths = await getClosurePaths(rootPath); - console.log(`Found ${String(paths.length)} paths in closure`); - result = await pushPaths(client, cache, paths, pushOpts); - } else { - const paths = filteredArgs.slice(2); - if (paths.length === 0) { - console.error("No paths specified"); - process.exit(1); - } - result = await pushPaths(client, cache, paths, pushOpts); - } - - console.log(""); - console.log(`Done: ${String(result.pushed)} pushed, ${String(result.skipped)} skipped, ${String(result.failed)} failed`); - - if (result.failed > 0) { - process.exit(1); - } - return; - } - - if (command === "token") { - const subcommand = args[1]; - - // Extract --server flag - let serverName: string | undefined; - const serverIdx = args.indexOf("--server"); - if (serverIdx !== -1 && serverIdx + 1 < args.length) { - serverName = args[serverIdx + 1]; - } - - const serverConfig = await getServer(serverName); - const client = createClient(serverConfig.server, serverConfig.token); - - if (subcommand === "create") { - const subject = args[2]; - if (!subject) { - console.error("Usage: helios token create [--caches ...] [--perms ...] [--expires ]"); - process.exit(1); - } - - let cachesStr = "*"; - const cachesIdx = args.indexOf("--caches"); - if (cachesIdx !== -1 && cachesIdx + 1 < args.length) { - cachesStr = args[cachesIdx + 1]; - } - - let permsStr = "push"; - const permsIdx = args.indexOf("--perms"); - if (permsIdx !== -1 && permsIdx + 1 < args.length) { - permsStr = args[permsIdx + 1]; - } - - let expiresInDays: number | undefined; - const expiresIdx = args.indexOf("--expires"); - if (expiresIdx !== -1 && expiresIdx + 1 < args.length) { - expiresInDays = parseInt(args[expiresIdx + 1], 10); - if (!Number.isInteger(expiresInDays) || expiresInDays < 1) { - console.error("--expires must be a positive integer"); - process.exit(1); - } - } - - const caches = cachesStr.split(",").map(s => s.trim()).filter(Boolean); - const perms = permsStr.split(",").map(s => s.trim()).filter(Boolean); - - const result = await createToken(client, { subject, caches, perms, expiresInDays }); - console.log(`Token created for '${result.subject}'`); - console.log(` JTI: ${result.jti}`); - console.log(` Caches: ${result.caches.join(", ")}`); - console.log(` Perms: ${result.perms.join(", ")}`); - console.log(` Expires: ${result.expiresAt}`); - console.log(""); - console.log(result.token); - return; - } - - if (subcommand === "list") { - const tokens = await listTokens(client); - if (tokens.length === 0) { - console.log("No tokens found"); - return; - } - for (const t of tokens) { - const status = t.revokedAt ? `revoked (${t.revokedAt})` : "active"; - console.log(`${t.jti} ${t.subject} [${t.perms.join(",")}] caches=[${t.caches.join(",")}] ${status} expires=${t.expiresAt}`); - } - return; - } - - if (subcommand === "revoke") { - const jti = args[2]; - const reason = args.slice(3).join(" "); - if (!jti || !reason) { - console.error("Usage: helios token revoke "); - process.exit(1); - } - await revokeToken(client, jti, reason); - console.log(`Token ${jti} revoked`); - return; - } - - console.error("Usage: helios token "); - process.exit(1); - } - - console.error(`Unknown command: ${command}`); - console.error("Run 'helios --help' for usage"); - process.exit(1); -} - -main().catch((err: unknown) => { - console.error(err instanceof Error ? err.message : String(err)); - process.exit(1); -}); diff --git a/apps/cli/src/nix.ts b/apps/cli/src/nix.ts deleted file mode 100644 index d5e74e7..0000000 --- a/apps/cli/src/nix.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { execFile, spawn } from "node:child_process"; -import { createHash } from "node:crypto"; -import { createWriteStream } from "node:fs"; -import { promisify } from "node:util"; - -const exec = promisify(execFile); - -export interface PathInfo { - readonly storePath: string; - readonly storePathHash: string; - readonly narHash: string; - readonly narSize: number; - readonly references: string[]; - readonly deriver?: string; -} - -interface NixPathInfoJson { - readonly narHash: string; - readonly narSize: number; - readonly references: string[]; - readonly deriver?: string; -} - -// nix path-info output for a large batch (references lists included) can far -// exceed execFile's default 1 MiB stdout buffer. -const EXEC_MAX_BUFFER_BYTES = 64 * 1024 * 1024; - -// Bound argv size per nix invocation; store paths are ~90 bytes each. -const PATH_INFO_BATCH_SIZE = 500; - -function toPathInfo(storePath: string, entry: NixPathInfoJson): PathInfo { - const basename = storePath.split("/").pop(); - if (!basename) { - throw new Error(`Invalid store path: ${storePath}`); - } - const storePathHash = basename.split("-")[0]; - - const refs = entry.references.map((r: string) => { - const refBase = r.split("/").pop(); - if (!refBase) { - throw new Error(`Invalid reference path: ${r}`); - } - return refBase.split("-")[0]; - }); - - return { - storePath, - storePathHash, - narHash: entry.narHash, - narSize: entry.narSize, - references: refs, - deriver: entry.deriver, - }; -} - -/** - * Query path info for many store paths with a bounded number of nix - * invocations, instead of one process spawn per path. - */ -export async function getPathInfos( - storePaths: readonly string[], -): Promise> { - const infos = new Map(); - - for (let i = 0; i < storePaths.length; i += PATH_INFO_BATCH_SIZE) { - const chunk = storePaths.slice(i, i + PATH_INFO_BATCH_SIZE); - const { stdout } = await exec("nix", ["path-info", "--json", ...chunk], { - maxBuffer: EXEC_MAX_BUFFER_BYTES, - }); - const parsed: Record = JSON.parse(stdout); - - for (const storePath of chunk) { - const entry = parsed[storePath]; - if (!entry) { - throw new Error(`nix path-info returned no data for ${storePath}`); - } - infos.set(storePath, toPathInfo(storePath, entry)); - } - } - - return infos; -} - -export async function getClosurePaths(storePath: string): Promise { - const { stdout } = await exec("nix", ["path-info", "-r", storePath], { - maxBuffer: EXEC_MAX_BUFFER_BYTES, - }); - return stdout.trim().split("\n").filter(Boolean); -} - -export interface CompressedNar { - readonly fileHash: string; - readonly fileSize: number; -} - -/** - * Dump a store path as a NAR, compress it with zstd, and write it to - * outputPath. The compressed stream is hashed and counted while it is - * written, so callers do not need to re-read the file to compute the - * upload hash and size. - */ -export async function dumpAndCompress( - storePath: string, - outputPath: string, -): Promise { - const nix = spawn("nix", ["store", "dump-path", storePath], { - stdio: ["ignore", "pipe", "ignore"], - }); - const zstd = spawn("zstd", ["-q", "-c"], { - stdio: ["pipe", "pipe", "ignore"], - }); - - nix.stdout.pipe(zstd.stdin); - - // Observe the piped chunks to hash and count them; pipe() still owns - // backpressure to the file stream. - const hash = createHash("sha256"); - let fileSize = 0; - zstd.stdout.on("data", (chunk: Buffer) => { - hash.update(chunk); - fileSize += chunk.length; - }); - - const out = createWriteStream(outputPath); - zstd.stdout.pipe(out); - - await new Promise((resolve, reject) => { - let nixExited = false; - let zstdExited = false; - let fileFlushed = false; - let settled = false; - - const fail = (err: Error) => { - if (!settled) { - settled = true; - reject(err); - } - }; - - const tryResolve = () => { - if (nixExited && zstdExited && fileFlushed && !settled) { - settled = true; - resolve(); - } - }; - - nix.on("error", fail); - zstd.on("error", fail); - out.on("error", fail); - - nix.on("close", (code) => { - nixExited = true; - if (code !== 0) { - fail(new Error(`nix store dump-path exited with ${String(code)}`)); - return; - } - tryResolve(); - }); - - zstd.on("close", (code) => { - zstdExited = true; - if (code !== 0) { - fail(new Error(`zstd exited with ${String(code)}`)); - return; - } - tryResolve(); - }); - - out.on("finish", () => { - fileFlushed = true; - tryResolve(); - }); - }); - - return { fileHash: hash.digest("hex"), fileSize }; -} diff --git a/apps/cli/src/pool.ts b/apps/cli/src/pool.ts deleted file mode 100644 index a42cd03..0000000 --- a/apps/cli/src/pool.ts +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Run fn over items with bounded concurrency. Items are claimed in order; - * the returned promise settles after every item has been processed. - * Rejections propagate to the caller. - */ -export async function runPool( - items: readonly T[], - concurrency: number, - fn: (item: T, index: number) => Promise, -): Promise { - let next = 0; - - async function worker(): Promise { - while (next < items.length) { - const idx = next; - next++; - await fn(items[idx], idx); - } - } - - const workers: Promise[] = []; - for (let i = 0; i < Math.min(concurrency, items.length); i++) { - workers.push(worker()); - } - await Promise.all(workers); -} diff --git a/apps/cli/src/push.ts b/apps/cli/src/push.ts deleted file mode 100644 index e4cba5e..0000000 --- a/apps/cli/src/push.ts +++ /dev/null @@ -1,147 +0,0 @@ -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import type { ApiClient } from "./api.js"; -import { - getMissingPaths, - createUploadSession, - uploadBlob, - shouldUseMultipart, - uploadBlobMultipart, - completeUpload, - publishPath, -} from "./api.js"; -import { getPathInfos, dumpAndCompress } from "./nix.js"; -import type { PathInfo } from "./nix.js"; -import { runPool } from "./pool.js"; - -const DEFAULT_CONCURRENCY = 8; - -export interface PushResult { - readonly pushed: number; - readonly skipped: number; - readonly failed: number; - readonly errors: string[]; -} - -interface PushOptions { - readonly concurrency?: number; - readonly onProgress?: (current: number, total: number, name: string, status: string) => void; -} - -async function pushSinglePath( - client: ApiClient, - cache: string, - storePath: string, - info: PathInfo, -): Promise<{ sizeKb: number }> { - const narDir = await mkdtemp(join(tmpdir(), "helios-nar-")); - const narFile = join(narDir, "nar.zst"); - - try { - // Hash and size are computed while the compressed NAR is written, - // avoiding a second full read of the file. - const { fileHash, fileSize } = await dumpAndCompress(storePath, narFile); - - const session = await createUploadSession(client, cache, { - storePath: info.storePath, - storePathHash: info.storePathHash, - narHash: info.narHash, - narSize: info.narSize, - fileHash, - fileSize, - compression: "zstd", - references: info.references, - }); - - if (shouldUseMultipart(fileSize)) { - await uploadBlobMultipart(client, session.sessionId, narFile); - } else { - await uploadBlob(client, session.sessionId, narFile); - } - await completeUpload(client, session.sessionId); - await publishPath(client, session.sessionId); - - return { sizeKb: Math.round(fileSize / 1024) }; - } finally { - await rm(narDir, { recursive: true, force: true }); - } -} - -export async function pushPaths( - client: ApiClient, - cache: string, - storePaths: string[], - options?: PushOptions, -): Promise { - const concurrency = options?.concurrency ?? DEFAULT_CONCURRENCY; - const onProgress = options?.onProgress; - - const hashToPath = new Map(); - const hashes: string[] = []; - for (const p of storePaths) { - const basename = p.split("/").pop(); - if (!basename) continue; - const hash = basename.split("-")[0]; - hashToPath.set(hash, p); - hashes.push(hash); - } - - // Check missing in sequential batches (server chunks internally for D1 limits) - const missingSet = new Set(); - for (let i = 0; i < hashes.length; i += 500) { - const chunk = hashes.slice(i, i + 500); - const missing = await getMissingPaths(client, cache, chunk); - for (const h of missing) { - missingSet.add(h); - } - } - - const skipped = storePaths.length - missingSet.size; - let pushed = 0; - let failed = 0; - const errors: string[] = []; - let completed = 0; - - const missingHashes = hashes.filter((h) => missingSet.has(h)); - const total = missingHashes.length; - - if (total === 0) { - return { pushed: 0, skipped, failed: 0, errors: [] }; - } - - // One batched nix path-info call for all missing paths, instead of one - // process spawn per path inside the upload pool. - const missingPaths: string[] = []; - for (const hash of missingHashes) { - const storePath = hashToPath.get(hash); - if (storePath) missingPaths.push(storePath); - } - const pathInfos = await getPathInfos(missingPaths); - - await runPool(missingHashes, concurrency, async (hash) => { - const storePath = hashToPath.get(hash); - if (!storePath) return; - const name = storePath.split("/").pop() ?? hash; - - try { - const info = pathInfos.get(storePath); - if (!info) { - throw new Error(`missing path info for ${storePath}`); - } - const result = await pushSinglePath(client, cache, storePath, info); - completed++; - onProgress?.(completed, total, name, `ok (${String(result.sizeKb)}KB)`); - pushed++; - } catch (err) { - completed++; - const msg = err instanceof Error ? err.message : String(err); - errors.push(`${name}: ${msg}`); - onProgress?.(completed, total, name, `FAIL: ${msg}`); - failed++; - } - }); - - return { pushed, skipped, failed, errors }; -} - diff --git a/apps/cli/tsconfig.json b/apps/cli/tsconfig.json deleted file mode 100644 index 1a87488..0000000 --- a/apps/cli/tsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "compilerOptions": { - "target": "es2024", - "lib": ["es2024"], - "module": "nodenext", - "moduleResolution": "nodenext", - "outDir": "dist", - "rootDir": "src", - "strict": true, - "isolatedModules": true, - "forceConsistentCasingInFileNames": true, - "skipLibCheck": true, - "declaration": false, - "sourceMap": false - }, - "include": ["src/**/*.ts"] -} diff --git a/bench/compression.sh b/bench/compression.sh new file mode 100755 index 0000000..6b5bf86 --- /dev/null +++ b/bench/compression.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Compares NAR compression codecs that Nix clients can decode (the narinfo +# `Compression` field), plus how much a closure dedups across NARs. +# Every NAR is compressed on its own, the way a cache serves it. Times are +# single-threaded CPU seconds (user+sys), summed over the closure; jobs run +# in parallel across NARs. +# +# Usage: bench/compression.sh [store-path] +# CODECS="zstd-3 xz-6" only these codecs (default: all below) +# JOBS=N parallel jobs (default: nproc) +# OUT=file.tsv keep the per-NAR results (default: discarded) +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +# Pin every codec to the flake's nixpkgs so runs are comparable. +if [ -z "${HELIOS_BENCH_SHELL:-}" ]; then + export HELIOS_BENCH_SHELL=1 + exec nix shell --inputs-from "$ROOT" nixpkgs#{zstd,xz,brotli,lz4,lzop,lzip,gzip,bzip2,coreutils,findutils,gawk} -c "$0" "$@" +fi + +TARGET="$(readlink -f "${1:-/run/current-system}")" +JOBS="${JOBS:-$(nproc)}" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +# name|narinfo Compression|compress|decompress. zstd windows stay at 2^27 +# and brotli at its default, the largest a stock Nix decoder accepts. +ALL=( + "none|none|cat|cat" + "lz4-1|lz4|lz4 -q -1 -c|lz4 -q -d -c" + "lz4-9|lz4|lz4 -q -9 -c|lz4 -q -d -c" + "lzop-1|lzop|lzop -q -1 -c|lzop -q -d -c" + "gzip-6|gzip|gzip -6 -c|gzip -d -c" + "gzip-9|gzip|gzip -9 -c|gzip -d -c" + "bzip2-9|bzip2|bzip2 -9 -c|bzip2 -d -c" + "zstd-1|zstd|zstd -q -T1 -1 -c|zstd -q -d -c" + "zstd-2|zstd|zstd -q -T1 -2 -c|zstd -q -d -c" + "zstd-3|zstd|zstd -q -T1 -3 -c|zstd -q -d -c" + "zstd-9|zstd|zstd -q -T1 -9 -c|zstd -q -d -c" + "zstd-19|zstd|zstd -q -T1 -19 -c|zstd -q -d -c" + "zstd-3-long|zstd|zstd -q -T1 -3 --long=27 -c|zstd -q -d -c" + "zstd-6-long|zstd|zstd -q -T1 -6 --long=27 -c|zstd -q -d -c" + "zstd-9-long|zstd|zstd -q -T1 -9 --long=27 -c|zstd -q -d -c" + "zstd-12-long|zstd|zstd -q -T1 -12 --long=27 -c|zstd -q -d -c" + "zstd-19-long|zstd|zstd -q -T1 -19 --long=27 -c|zstd -q -d -c" + "zstd-22-long|zstd|zstd -q -T1 --ultra -22 --long=27 -c|zstd -q -d -c" + "br-6|br|brotli -q 6 -c|brotli -d -c" + "br-11|br|brotli -q 11 -c|brotli -d -c" + "xz-6|xz|xz -T1 -6 -c|xz -d -c" + "xz-9e|xz|xz -T1 -9e -c|xz -d -c" + "lzip-6|lzip|lzip -q -6 -c|lzip -q -d -c" +) +: >"$WORK/codecs" +for c in "${ALL[@]}"; do + if [ -z "${CODECS:-}" ] || [[ " $CODECS " == *" ${c%%|*} "* ]]; then echo "$c" >>"$WORK/codecs"; fi +done + +echo "dumping closure of $TARGET..." >&2 +mkdir -p "$WORK/nar" +nix-store -qR "$TARGET" >"$WORK/paths" +xargs -P "$JOBS" -I{} sh -c 'nix-store --dump "$1" >"$2/nar/$(basename "$1").nar"' _ {} "$WORK" <"$WORK/paths" +RAW=$(du -cb "$WORK"/nar/*.nar | tail -1 | cut -f1) +echo "$(wc -l <"$WORK/paths") paths, $((RAW / 1048576)) MiB of NAR" >&2 + +# One job per (codec, NAR), largest NARs first so the pool drains evenly. +job() { + IFS='|' read -r name _ comp decomp <<<"$1" + local nar=$2 tmp; tmp="$(mktemp -p "$WORK")" + TIMEFORMAT='%3U %3S' + local ct dt + ct=$( { time $comp <"$nar" >"$tmp"; } 2>&1 ) + dt=$( { time $decomp <"$tmp" >/dev/null; } 2>&1 ) + $decomp <"$tmp" | cmp -s - "$nar" || { echo "roundtrip failed: $name $nar" >&2; exit 1; } + printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$name" "$(basename "$nar")" "$(stat -c%s "$nar")" "$(stat -c%s "$tmp")" \ + "$(awk '{print $1+$2}' <<<"$ct")" "$(awk '{print $1+$2}' <<<"$dt")" + rm -f "$tmp" +} +export -f job; export WORK +ls -S "$WORK"/nar/*.nar >"$WORK/nars" +while read -r c; do while read -r n; do printf '%s\0%s\0' "$c" "$n"; done <"$WORK/nars"; done <"$WORK/codecs" \ + | xargs -0 -n2 -P "$JOBS" bash -c 'job "$1" "$2"' _ >"$WORK/results.tsv" +[ -n "${OUT:-}" ] && cp "$WORK/results.tsv" "$OUT" + +echo +printf '%-14s %-6s %11s %7s %9s %12s %12s\n' codec narinfo "size MiB" ratio "vs zstd-3" "comp MB/s" "decomp MB/s" +awk -F'\t' -v order="$(cut -d'|' -f1,2 "$WORK/codecs" | tr '\n' ' ')" ' + { raw[$1]+=$3; out[$1]+=$4; ct[$1]+=$5; dt[$1]+=$6 } + END { + n = split(order, o, " "); base = ("zstd-3" in out) ? out["zstd-3"] : 0 + for (i = 1; i <= n; i++) { + split(o[i], f, "|"); c = f[1] + printf "%-14s %-6s %11.1f %7.2f %9s %12.0f %12.0f\n", c, f[2], out[c]/1048576, raw[c]/out[c], + base ? sprintf("%+.1f%%", (out[c]/base-1)*100) : "-", + ct[c] ? raw[c]/ct[c]/1e6 : 0, dt[c] ? raw[c]/dt[c]/1e6 : 0 + } + }' "$WORK/results.tsv" + +# Headroom beyond per-NAR compression, which only the storage side can use. +echo +echo "storage-side redundancy (not servable as-is):" +find "$WORK/nar" -name '*.nar' -print0 | xargs -0 cat | zstd -q -T"$JOBS" -9 --long=31 -c | wc -c \ + | awk -v raw="$RAW" '{printf " whole closure as one zstd-9 --long=31 stream: %.1f MiB (%.2fx)\n", $1/1048576, raw/$1}' +xargs -I{} find {} -type f -printf '%s\t%p\n' <"$WORK/paths" | LC_ALL=C sort -t $'\t' -k2,2 >"$WORK/sizes" +xargs -I{} find {} -type f -print0 <"$WORK/paths" | xargs -0 -n 2000 -P "$JOBS" sha256sum \ + | awk '{h=$1; $1=""; sub(/^ \*?/, ""); print $0 "\t" h}' | LC_ALL=C sort -t $'\t' -k1,1 >"$WORK/hashes" +LC_ALL=C join -t $'\t' -1 2 -2 1 "$WORK/sizes" "$WORK/hashes" \ + | awk -F'\t' '{ tot+=$2; if (!seen[$3]++) uniq+=$2 } END { + printf " regular files: %.1f MiB, %.1f MiB unique by content (%.1f%% duplicate)\n", tot/1048576, uniq/1048576, (1-uniq/tot)*100 }' diff --git a/bench/flake-lock-rs/Cargo.lock b/bench/flake-lock-rs/Cargo.lock new file mode 100644 index 0000000..8a97319 --- /dev/null +++ b/bench/flake-lock-rs/Cargo.lock @@ -0,0 +1,32 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "flake-lock-rs" +version = "0.0.0" +dependencies = [ + "nix-flake-lock", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "nix-flake-lock" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7815d99678fd80c5d646d68ab0360132754e4c0ba0c5e203bc7b4f353de6d9" +dependencies = [ + "itoa", + "memchr", +] diff --git a/bench/flake-lock-rs/Cargo.toml b/bench/flake-lock-rs/Cargo.toml new file mode 100644 index 0000000..2d4d2e5 --- /dev/null +++ b/bench/flake-lock-rs/Cargo.toml @@ -0,0 +1,15 @@ +# Reference harness: the nix-flake-lock crate (Rust), pinned. Not part of the workspace. +[package] +name = "flake-lock-rs" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +nix-flake-lock = "=0.1.0" + +[profile.release] +lto = "fat" +codegen-units = 1 + +[workspace] diff --git a/bench/flake-lock-rs/src/main.rs b/bench/flake-lock-rs/src/main.rs new file mode 100644 index 0000000..5de1dc7 --- /dev/null +++ b/bench/flake-lock-rs/src/main.rs @@ -0,0 +1,180 @@ +//! Reference side of bench/flake-lock.sh. +//! +//! - `canon ...`: for each file print `== ` then either the +//! canonical bytes plus `-- valid` / `-- invalid`, or `-- parse error`. +//! - `bench ...`: `: ` lines. +//! - `gen `: write the large / adversarial fixtures from upstream's benches. +//! - `fuzz ...`: write `n` mutated copies of the inputs. + +use std::fmt::Write as _; +use std::hint::black_box; +use std::io::Write as _; +use std::time::Instant; + +use nix_flake_lock::LockFile; + +fn read(path: &str) -> Vec { + std::fs::read(path).unwrap_or_else(|e| panic!("{path}: {e}")) +} + +pub fn iterations(len: usize) -> usize { + (4_000_000 / len.max(1)).clamp(3, 200_000) +} + +fn best(iters: usize, mut f: F) -> u128 { + for _ in 0..iters.min(1000) { + f(); + } + (0..7) + .map(|_| { + let t = Instant::now(); + for _ in 0..iters { + f(); + } + t.elapsed().as_nanos() + }) + .min() + .unwrap() +} + +fn large(nodes: usize, reverse: bool) -> String { + let idx = |p: usize| if reverse { nodes - p - 1 } else { p }; + let mut json = String::from("{\"nodes\":{\"root\":{\"inputs\":{"); + for p in 0..nodes { + if p != 0 { + json.push(','); + } + let i = idx(p); + write!(json, "\"n{i:06}\":\"n{i:06}\"").unwrap(); + } + json.push_str("}}"); + for p in 0..nodes { + let i = idx(p); + write!(json, ",\"n{i:06}\":{{\"locked\":{{\"path\":\"x\",\"type\":\"path\"}},\"original\":{{\"path\":\"x\",\"type\":\"path\"}}}}").unwrap(); + } + json.push_str("},\"root\":\"root\",\"version\":7}"); + json +} + +fn colliding(n: usize) -> String { + let mut json = String::from("{\"nodes\":{\"root\":{\"inputs\":{"); + for i in 0..n { + if i != 0 { + json.push(','); + } + write!(json, "\"dep{i:06}\":\"dep{i:06}\"").unwrap(); + } + json.push_str("}}"); + for i in 0..n { + write!(json, ",\"dep{i:06}\":{{\"inputs\":{{\"systems\":\"systems{i:06}\"}},\"locked\":{{\"path\":\"x\",\"type\":\"path\"}},\"original\":{{\"path\":\"x\",\"type\":\"path\"}}}},\"systems{i:06}\":{{\"locked\":{{\"path\":\"x\",\"type\":\"path\"}},\"original\":{{\"path\":\"x\",\"type\":\"path\"}}}}").unwrap(); + } + json.push_str("},\"root\":\"root\",\"version\":7}"); + json +} + +/// xorshift64*: deterministic mutations shared by nothing but this tool. +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + self.0.wrapping_mul(0x2545F4914F6CDD1D) + } + fn below(&mut self, n: usize) -> usize { + (self.next() % n.max(1) as u64) as usize + } +} + +fn mutate(rng: &mut Rng, mut doc: Vec) -> Vec { + const TOKENS: &[&[u8]] = &[ + b"\"", b"\\", b"\\u00e9", b"\\ud83d\\ude00", b"\\ud800", b"{", b"}", b"[", b"]", b",", b":", + b"null", b"true", b"false", b"0", b"01", b"1.5", b"-1", b"18446744073709551616", b"\"root\"", + b"\"inputs\"", b"\"follows\"", b"[\"a\"]", b"\"version\"", b"\xc3\xa9", b"\xff", b"\x01", b" ", b"\n", + ]; + for _ in 0..1 + rng.below(3) { + let at = rng.below(doc.len() + 1); + match rng.below(4) { + 0 => { + let t = TOKENS[rng.below(TOKENS.len())]; + doc.splice(at..at, t.iter().copied()); + } + 1 if !doc.is_empty() => { + let end = (at + 1 + rng.below(8)).min(doc.len()); + doc.drain(at.min(doc.len())..end); + } + 2 if at < doc.len() => doc[at] = TOKENS[rng.below(TOKENS.len())][0], + _ => { + // Duplicate a slice (duplicate keys, repeated members). + if doc.len() > 2 { + let a = rng.below(doc.len()); + let b = (a + rng.below(64)).min(doc.len()); + let piece = doc[a..b].to_vec(); + doc.splice(at..at, piece); + } + } + } + } + doc +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let out = std::io::stdout(); + let mut out = std::io::BufWriter::new(out.lock()); + match args.first().map(String::as_str) { + Some("canon") => { + for f in &args[1..] { + let bytes = read(f); + writeln!(out, "== {f}").unwrap(); + match LockFile::parse(&bytes) { + Ok(lock) => { + out.write_all(&lock.to_nix_bytes()).unwrap(); + writeln!(out, "-- {}", if lock.validate().is_ok() { "valid" } else { "invalid" }).unwrap(); + } + Err(_) => writeln!(out, "-- parse error").unwrap(), + } + } + } + Some("bench") => { + for f in &args[1..] { + let bytes = read(f); + let name = std::path::Path::new(f).file_name().unwrap().to_string_lossy().into_owned(); + let iters = iterations(bytes.len()); + let lock = LockFile::parse(&bytes).expect("bench inputs parse"); + let ns = best(iters, || { + black_box(LockFile::parse(black_box(&bytes)).unwrap()); + }); + writeln!(out, "parse:{name} {iters} {ns}").unwrap(); + let ns = best(iters, || { + black_box(black_box(&lock).to_nix_bytes()); + }); + writeln!(out, "serialize:{name} {iters} {ns}").unwrap(); + let ns = best(iters, || { + black_box(black_box(&lock).validate()).ok(); + }); + writeln!(out, "validate:{name} {iters} {ns}").unwrap(); + } + } + Some("gen") => { + let dir = &args[1]; + std::fs::write(format!("{dir}/large-sorted-8000.lock"), large(8000, false)).unwrap(); + std::fs::write(format!("{dir}/large-reverse-8000.lock"), large(8000, true)).unwrap(); + std::fs::write(format!("{dir}/colliding-4000.lock"), colliding(4000)).unwrap(); + } + Some("fuzz") => { + let dir = &args[1]; + let n: usize = args[2].parse().unwrap(); + let mut rng = Rng(args[3].parse::().unwrap() | 1); + let seeds: Vec> = args[4..].iter().map(|f| read(f)).collect(); + for i in 0..n { + let seed = seeds[rng.below(seeds.len())].clone(); + std::fs::write(format!("{dir}/fuzz-{i:05}.lock"), mutate(&mut rng, seed)).unwrap(); + } + } + _ => { + eprintln!("usage: flake-lock-rs canon|bench|gen|fuzz ..."); + std::process::exit(2); + } + } +} diff --git a/bench/flake-lock.sh b/bench/flake-lock.sh new file mode 100755 index 0000000..95ff79b --- /dev/null +++ b/bench/flake-lock.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# pkg/nix-flake-lock (Zig) vs the nix-flake-lock crate (Rust, pinned in +# bench/flake-lock-rs): a differential check that both produce identical +# canonical output and validation results, then a single-threaded benchmark. +# +# Corpus: upstream's fixtures and generated benchmark inputs, every +# flake.lock under $FLAKE_LOCK_SEARCH (default: $HOME), and FUZZ mutated +# documents. Usage (inside `nix develop`): bench/flake-lock.sh +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SEARCH="${FLAKE_LOCK_SEARCH:-$HOME}" +FUZZ="${FUZZ:-20000}" +W="$(mktemp -d)" +trap 'rm -rf "$W"' EXIT + +cargo build -q --release --manifest-path "$ROOT/bench/flake-lock-rs/Cargo.toml" +(cd "$ROOT/bench/zig" && zig build -Doptimize=ReleaseFast -Dcpu=native) +RS="$ROOT/bench/flake-lock-rs/target/release/flake-lock-rs" +ZIG="$ROOT/bench/zig/zig-out/bin/flake-lock" + +mkdir -p "$W/fixtures" "$W/real" "$W/fuzz" +UPSTREAM="$(find "${CARGO_HOME:-$HOME/.cargo}/registry/src" -path '*/nix-flake-lock-0.1.0/tests/fixtures' -type d | head -1)" +cp "$UPSTREAM"/*.lock "$W/fixtures/" +"$RS" gen "$W/fixtures" +i=0 +while IFS= read -r -d '' f; do + cp "$f" "$W/real/$(printf '%04d' $i).lock"; i=$((i + 1)) +done < <(find "$SEARCH" -name flake.lock -not -path '*/.cache/*' -not -path '*/node_modules/*' -size -16M -print0 2>/dev/null) +mapfile -t SEEDS < <(ls "$W"/real/*.lock 2>/dev/null | head -50 || true) +"$RS" fuzz "$W/fuzz" "$FUZZ" 42 "$W"/fixtures/{tiny,typical,legacy-v5,legacy-v6}.lock "${SEEDS[@]}" + +echo "== differential: $(ls "$W/fixtures" | wc -l) fixtures, $i real-world lock files, $FUZZ mutations" +find "$W/fixtures" "$W/real" "$W/fuzz" -name '*.lock' | sort >"$W/all" +xargs "$RS" canon <"$W/all" >"$W/rs.out" +FL_MODE=canon FL_LIST="$W/all" "$ZIG" >"$W/zig.out" +if cmp -s "$W/rs.out" "$W/zig.out"; then + echo "identical output on $(wc -l <"$W/all") documents" \ + "($(grep -c -- '-- valid' "$W/rs.out") valid, $(grep -c -- '-- invalid' "$W/rs.out") invalid follows, $(grep -c -- '-- parse error' "$W/rs.out") rejected)" +else + echo "MISMATCH"; diff <(cat "$W/rs.out") <(cat "$W/zig.out") | head -40; exit 1 +fi + +echo +echo "== benchmark (best of 7, ns per operation)" +mapfile -t BIGGEST < <(ls -S "$W"/real/*.lock 2>/dev/null | head -3 || true) +for f in "${BIGGEST[@]}"; do cp "$f" "$W/fixtures/real-$(basename "$f")"; done +find "$W/fixtures" -name '*.lock' | sort >"$W/bench" +xargs "$RS" bench <"$W/bench" | sort >"$W/rs.bench" +FL_MODE=bench FL_LIST="$W/bench" "$ZIG" | sort >"$W/zig.bench" +printf '%-38s %9s %12s %12s %8s\n' operation bytes "zig ns" "rust ns" speedup +join "$W/zig.bench" "$W/rs.bench" | while read -r key zi zn ri rn; do + file="${key#*:}"; bytes=$(stat -c %s "$W/fixtures/$file") + awk -v k="$key" -v b="$bytes" -v z="$zn" -v zi="$zi" -v r="$rn" -v ri="$ri" \ + 'BEGIN { zz = z / zi; rr = r / ri; printf "%-38s %9d %12.0f %12.0f %7.2fx\n", k, b, zz, rr, rr / zz }' +done diff --git a/bench/haskell/Bench.hs b/bench/haskell/Bench.hs new file mode 100644 index 0000000..69e5249 --- /dev/null +++ b/bench/haskell/Bench.hs @@ -0,0 +1,128 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE PackageImports #-} + +-- Benchmarks for the Haskell Nix libraries +-- (nix-narinfo, nix-derivation, hnix-store-core, hnix-store-nar, ed25519). +-- Mirrors bench/zig/src/main.zig: same corpus, same output format +-- (` `), best of 7 rounds after a warm-up. +-- +-- Inputs are pre-decoded to Text outside the timed region, which favours +-- these libraries (our Zig parsers work on raw bytes). + +module Main (main) where + +import Control.Exception (evaluate) +import Control.Monad (forM, replicateM, when) +import qualified "crypton" Crypto.Hash as H +import qualified Crypto.Sign.Ed25519 as Ed +import qualified Data.Attoparsec.Text as A +import qualified Data.ByteString as BS +import qualified Data.ByteString.Base64 as B64 +import qualified Data.ByteString.Char8 as BC +import Data.IORef +import qualified Data.Map.Strict as Map +import qualified Data.Set as Set +import qualified Data.Text as T +import qualified Data.Text.Encoding as TE +import qualified Data.Text.Lazy as TL +import qualified Data.Text.Lazy.Builder as TB +import Data.Word (Word64) +import GHC.Clock (getMonotonicTimeNSec) +import qualified Nix.Derivation as D +import Nix.NarInfo (NarInfo (..)) +import qualified Nix.NarInfo as NI +import qualified "hnix-store-core" System.Nix.Base32 as B32 +import qualified "hnix-store-nar" System.Nix.Nar.Streamer as Nar +import qualified "hnix-store-core" System.Nix.StorePath as SP +import System.Environment (getEnv, lookupEnv) + +rounds :: Int +rounds = 7 + +records :: FilePath -> IO [BS.ByteString] +records path = filter (not . BS.null) . BS.split 0 <$> BS.readFile path + +report :: String -> Int -> Word64 -> IO () +report name ops ns = putStrLn (name ++ " " ++ show ops ++ " " ++ show ns) + +-- Applies `f` to `x` afresh every round. NOINLINE plus -fno-full-laziness +-- keep GHC from sharing one evaluation across rounds. +{-# NOINLINE bench #-} +bench :: String -> Int -> (a -> Int) -> a -> IO () +bench name ops f x = do + _ <- evaluate (f x) + times <- replicateM rounds $ do + t0 <- getMonotonicTimeNSec + !_ <- evaluate (f x) + t1 <- getMonotonicTimeNSec + pure (t1 - t0) + report name ops (minimum times) + +forceNarInfo :: NI.SimpleNarInfo -> Int +forceNarInfo ni = + length (storePath ni) + T.length (url ni) + T.length (compression ni) + T.length (fileHash ni) + + fromIntegral (fileSize ni) + T.length (narHash ni) + fromIntegral (narSize ni) + + sum (map length (Set.toList (references ni))) + + maybe 0 T.length (deriver ni) + maybe 0 T.length (system ni) + maybe 0 T.length (sig ni) + +forceDrv :: D.Derivation FilePath T.Text -> Int +forceDrv d = + sum [length (D.path o) + T.length (D.hashAlgo o) + T.length (D.hash o) + T.length k | (k, o) <- Map.toList (D.outputs d)] + + sum [length p + sum (map T.length (Set.toList os)) | (p, os) <- Map.toList (D.inputDrvs d)] + + sum (map length (Set.toList (D.inputSrcs d))) + + T.length (D.platform d) + T.length (D.builder d) + sum (fmap T.length (D.args d)) + + sum [T.length k + T.length v | (k, v) <- Map.toList (D.env d)] + +parseText :: A.Parser a -> T.Text -> a +parseText p t = either error id (A.parseOnly p t) + +-- Nix secret key file: ":". +signer :: BS.ByteString -> (BS.ByteString, Ed.SecretKey) +signer key = + let (name, rest) = BC.break (== ':') (BC.strip key) + in (name, Ed.SecretKey (either error id (B64.decode (BS.drop 1 rest)))) + +main :: IO () +main = do + dir <- getEnv "BENCH_DIR" + only <- maybe "" id <$> lookupEnv "BENCH_ONLY" + paths <- records (dir ++ "/paths.bin") + + when (only == "nar") $ do + t0 <- getMonotonicTimeNSec + total <- fmap sum . forM paths $ \p -> do + ref <- newIORef (H.hashInit :: H.Context H.SHA256) + size <- newIORef (0 :: Int) + Nar.dumpPath (BC.unpack p) $ \chunk -> do + modifyIORef' ref (`H.hashUpdate` chunk) + modifyIORef' size (+ BS.length chunk) + digest <- H.hashFinalize <$> readIORef ref + _ <- evaluate (length (show digest)) + readIORef size + t1 <- getMonotonicTimeNSec + report "nar-dump-sha256" total (t1 - t0) + + when (only /= "nar") $ do + narinfos <- map TE.decodeUtf8 <$> records (dir ++ "/narinfo.bin") + drvs <- map TE.decodeUtf8 <$> records (dir ++ "/drv.bin") + let parsed = map (parseText NI.parseNarInfo) narinfos + _ <- evaluate (sum (map forceNarInfo parsed)) + let hashes32 = [T.take 32 (T.drop 11 (T.pack (storePath ni))) | ni <- parsed] + hashes52 = [T.drop 7 (narHash ni) | ni <- parsed] + raw20 = [either error id (B32.decode h) | h <- hashes32] + fingerprints = + [ TE.encodeUtf8 (T.intercalate ";" ["1", T.pack (storePath ni), narHash ni, T.pack (show (narSize ni)), T.unwords (map T.pack (Set.toList (references ni)))]) + | ni <- parsed + ] + _ <- evaluate (sum (map BS.length raw20) + sum (map BS.length fingerprints) + sum (map T.length hashes52)) + (keyName, sk) <- signer <$> BS.readFile (dir ++ "/key") + + bench "narinfo-parse" (length narinfos) (sum . map (forceNarInfo . parseText NI.parseNarInfo)) narinfos + bench "narinfo-render" (length parsed) (sum . map (fromIntegral . TL.length . TB.toLazyText . NI.buildNarInfo)) parsed + bench "ed25519-sign" (length fingerprints) (\fps -> sum [BS.length (keyName <> ":" <> B64.encode (Ed.unSignature (Ed.dsign sk fp))) | fp <- fps]) fingerprints + bench "drv-parse" (length drvs) (sum . map (forceDrv . parseText D.parseDerivation)) drvs + bench "base32-decode-20" (length hashes32) (\hs -> sum [either error BS.length (B32.decode h) | h <- hs]) hashes32 + bench "base32-decode-32" (length hashes52) (\hs -> sum [either error BS.length (B32.decode h) | h <- hs]) hashes52 + bench "base32-encode-20" (length raw20) (sum . map (T.length . B32.encode)) raw20 + bench "storepath-parse" (length paths) (\ps -> sum [either error (\sp -> SP.storePathHash sp `seq` SP.storePathName sp `seq` 1) (SP.parsePath "/nix/store" p) | p <- ps]) paths diff --git a/bench/haskell/ghc.nix b/bench/haskell/ghc.nix new file mode 100644 index 0000000..0289f20 --- /dev/null +++ b/bench/haskell/ghc.nix @@ -0,0 +1,12 @@ +# GHC with the Haskell Nix libraries (nix-narinfo, nix-derivation, hnix-store). +{ pkgs ? import { } }: +pkgs.haskellPackages.ghcWithPackages (p: [ + p.nix-narinfo + p.nix-derivation + p.hnix-store-core + p.hnix-store-nar + p.ed25519 + p.crypton + p.base64-bytestring + p.attoparsec +]) diff --git a/bench/poop.sh b/bench/poop.sh new file mode 100755 index 0000000..61033a7 --- /dev/null +++ b/bench/poop.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Whole-process comparisons with poop (https://github.com/andrewrk/poop): +# wall time, peak RSS, cycles, instructions, cache and branch misses, on the +# same work, counting child processes: +# +# 1. NAR serialise + SHA-256 of a closure: pkg/nix-archive (one pass of +# the bench binary) vs `nix hash path` +# 2. flake.lock canonicalisation and validation: pkg/nix-flake-lock vs the +# nix-flake-lock crate, on every flake.lock under $FLAKE_LOCK_SEARCH +# 3. Pushing a closure: `helios push` to a fresh local server (started, +# populated and stopped each run) vs `nix copy` to a fresh zstd file +# cache, both compressing at level 3 and signing. Nix reads the store +# itself (read-only), so none of its work hides in the nix-daemon. +# +# poop counts user-space events only; I/O shows in wall time. Results go to +# the terminal, never to files in the tree. +# +# Usage (inside `nix develop`): bench/poop.sh [store-path] +# default store path: python3's closure (about 200 MiB of NAR) +# DURATION=ms per command (default 5000) +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SEARCH="${FLAKE_LOCK_SEARCH:-$HOME}" +DURATION="${DURATION:-5000}" +W="$(mktemp -d)" +trap 'rm -rf "$W"' EXIT + +echo "building..." >&2 +(cd "$ROOT" && cargo build --release -q) +(cd "$ROOT/bench/zig" && zig build -Doptimize=ReleaseFast -Dcpu=native) +cargo build -q --release --manifest-path "$ROOT/bench/flake-lock-rs/Cargo.toml" +POOP="$(nix build --no-link --print-out-paths --inputs-from "$ROOT" nixpkgs#poop)/bin/poop" +TARGET="$(readlink -f "${1:-$(nix build --no-link --print-out-paths --inputs-from "$ROOT" nixpkgs#python3)}")" +BIN="$ROOT/target/release" + +nix-store -qR "$TARGET" >"$W/paths" +while read -r p; do printf '%s\0' "$p"; done <"$W/paths" >"$W/paths.bin" +: >"$W/narinfo.bin" +: >"$W/drv.bin" +nix key generate-secret --key-name poop-1 >"$W/key" +find "$SEARCH" -name flake.lock -not -path '*/.cache/*' -not -path '*/node_modules/*' -size -16M 2>/dev/null >"$W/locks" || true +echo "closure of $TARGET: $(wc -l <"$W/paths") paths, $(nix path-info -rs "$TARGET" | awk '{s+=$2} END {printf "%.0f", s/1048576}') MiB of NAR" >&2 +echo "flake.lock files: $(wc -l <"$W/locks")" >&2 + +# poop splits each command on spaces without a shell, so every side runs +# from a small script; both sides pay the same shell start. +script() { + local name=$1 + shift + printf '#!/bin/sh\nset -e\n%s\n' "$*" >"$W/$name" + chmod +x "$W/$name" +} + +script helios-nar "BENCH_DIR=$W BENCH_ONLY=nar-once exec $ROOT/bench/zig/zig-out/bin/bench" +script nix-nar "exec xargs nix hash path --type sha256 <$W/paths >/dev/null 2>&1" + +script helios-flake-lock "FL_MODE=canon FL_LIST=$W/locks exec $ROOT/bench/zig/zig-out/bin/flake-lock >/dev/null" +script rust-flake-lock "exec xargs $ROOT/bench/flake-lock-rs/target/release/flake-lock-rs canon <$W/locks >/dev/null" + +script helios-push " +d=\$(mktemp -d -p $W) +$BIN/helios-server generate-secrets --dir \$d/s >/dev/null 2>&1 +HELIOS_LOG=error $BIN/helios-server --listen 127.0.0.1:18299 --data-dir \$d/data \\ + --signing-key-file \$d/s/signing-key --jwt-secret-file \$d/s/jwt-secret --admin-secret-file \$d/s/admin-secret & +pid=\$! +until curl -sf http://127.0.0.1:18299/healthz >/dev/null; do sleep 0.01; done +export XDG_CONFIG_HOME=\$d/cfg HELIOS_LOG=error +$BIN/helios login admin http://127.0.0.1:18299 \$(cat \$d/s/admin-secret) >/dev/null 2>&1 +$BIN/helios cache create main >/dev/null +$BIN/helios login ci http://127.0.0.1:18299 \$($BIN/helios token create ci --caches main --perms push 2>/dev/null | sed -n 's/.*\"token\": *\"\\([^\"]*\\)\".*/\\1/p') >/dev/null 2>&1 +$BIN/helios push main --closure \"\$@\" $TARGET >/dev/null 2>&1 +kill \$pid; wait \$pid || true +rm -rf \$d" +# The same, without the long window: zstd level 3 exactly as Nix runs it. +sed 's|--closure "\$@"|--closure --window-log 0|' "$W/helios-push" >"$W/helios-push-plain" +chmod +x "$W/helios-push-plain" +script nix-copy " +d=\$(mktemp -d -p $W) +nix copy --extra-experimental-features read-only-local-store --from 'local?read-only=true' --to \"file://\$d?compression=zstd&compression-level=3¶llel-compression=true&secret-key=$W/key\" $TARGET +rm -rf \$d" + +echo +echo "== NAR serialise + SHA-256" +"$POOP" -d "$DURATION" "$W/nix-nar" "$W/helios-nar" +echo +echo "== flake.lock canonicalise + validate" +"$POOP" -d "$DURATION" "$W/rust-flake-lock" "$W/helios-flake-lock" +echo +echo "== push a closure (compress at zstd 3, sign, store)" +"$POOP" -d "$DURATION" "$W/nix-copy" "$W/helios-push-plain" "$W/helios-push" diff --git a/bench/run.sh b/bench/run.sh new file mode 100755 index 0000000..c46f4d2 --- /dev/null +++ b/bench/run.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Benchmarks pkg/* (Zig) against the Haskell Nix libraries (nix-narinfo, +# nix-derivation, hnix-store) and Nix's own C++ NAR serialiser. +# Single-threaded, same corpus for both. +# +# Usage: bench/run.sh [store-path] (inside `nix develop`) +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +TARGET="${1:-/run/current-system}" +WORK="$(mktemp -d)" +trap 'kill ${SERVER_PID:-} 2>/dev/null || true; rm -rf "$WORK"' EXIT + +echo "building..." >&2 +(cd "$ROOT" && cargo build --release -q) +(cd "$ROOT/bench/zig" && zig build -Doptimize=ReleaseFast -Dcpu=native) +GHC="$(nix build --no-link --print-out-paths --impure --expr "import $ROOT/bench/haskell/ghc.nix {}")/bin/ghc" +"$GHC" -O2 -fno-full-laziness -rtsopts -outputdir "$WORK/hs" -o "$WORK/bench-hs" "$ROOT/bench/haskell/Bench.hs" >/dev/null + +echo "building corpus from $TARGET..." >&2 +C="$WORK/corpus"; mkdir -p "$C" +nix key generate-secret --key-name bench-1 >"$C/key" +head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' >"$WORK/secret" +"$ROOT/target/release/helios-server" --listen 127.0.0.1:18099 --data-dir "$WORK/data" \ + --signing-key-file "$C/key" --jwt-secret-file "$WORK/secret" --admin-secret-file "$WORK/secret" 2>/dev/null & +SERVER_PID=$! +for _ in $(seq 50); do curl -sf http://127.0.0.1:18099/healthz >/dev/null && break; sleep 0.1; done +export XDG_CONFIG_HOME="$WORK/config" +H="$ROOT/target/release/helios" +"$H" login a http://127.0.0.1:18099 "$(cat "$WORK/secret")" >/dev/null +"$H" cache create main >/dev/null +"$H" login ci http://127.0.0.1:18099 "$("$H" --server a token create bench --caches main --perms push 2>/dev/null | jq -r .token)" >/dev/null +"$H" push main -r "$TARGET" >/dev/null 2>&1 +nix-store -qR "$TARGET" >"$WORK/paths" +while read -r p; do printf '%s\0' "$p"; done <"$WORK/paths" >"$C/paths.bin" +while read -r p; do curl -s "http://127.0.0.1:18099/main/$(basename "$p" | cut -c1-32).narinfo"; printf '\0'; done <"$WORK/paths" >"$C/narinfo.bin" +mapfile -t DRVS < <(find /nix/store -maxdepth 1 -name '*.drv' -print 2>/dev/null | head -3000 || true) +for d in "${DRVS[@]}"; do cat "$d"; printf '\0'; done >"$C/drv.bin" +kill "$SERVER_PID"; SERVER_PID= + +export BENCH_DIR="$C" +ZIG="$ROOT/bench/zig/zig-out/bin/bench" +"$ZIG" 2>&1 | sort >"$WORK/zig.txt" +"$WORK/bench-hs" +RTS -A64m -RTS | sort >"$WORK/hs.txt" + +# NAR: warm the page cache once, then time each implementation on the closure. +BENCH_ONLY=nar "$ZIG" >/dev/null 2>&1 +BENCH_ONLY=nar "$ZIG" 2>&1 >/dev/null | sed 's/^/zig /' >"$WORK/nar.txt" +BENCH_ONLY=nar "$WORK/bench-hs" +RTS -A64m -RTS | sed 's/^/hs /' >>"$WORK/nar.txt" +t0=$(date +%s%N); xargs nix hash path --type sha256 <"$WORK/paths" >/dev/null; t1=$(date +%s%N) +echo "cpp nar-dump-sha256 $(awk '{print $3}' "$WORK/nar.txt" | head -1) $((t1 - t0))" >>"$WORK/nar.txt" + +echo +echo "corpus: $(wc -l <"$WORK/paths") store paths, $(tr -cd '\0' <"$C/drv.bin" | wc -c) .drv files" +echo +printf '%-26s %14s %14s %9s\n' benchmark "helios ns/op" "haskell ns/op" speedup +join "$WORK/zig.txt" "$WORK/hs.txt" | awk '{z=$3/$2; h=$5/$4; printf "%-26s %14.1f %14.1f %8.1fx\n", $1, z, h, h/z}' +join -v1 "$WORK/zig.txt" "$WORK/hs.txt" | awk '{printf "%-26s %14.1f %14s\n", $1, $3/$2, "-"}' +echo +printf '%-30s %12s\n' "NAR serialise + SHA-256" "GiB/s (1 core)" +awk '{printf "%-30s %12.2f\n", ($1=="zig"?"helios (pkg/nix-archive)":($1=="hs"?"hnix-store-nar + crypton":"nix hash path (Nix C++)")), $3/$4*1e9/1073741824}' "$WORK/nar.txt" diff --git a/bench/zig/build.zig b/bench/zig/build.zig new file mode 100644 index 0000000..c72ba88 --- /dev/null +++ b/bench/zig/build.zig @@ -0,0 +1,38 @@ +const std = @import("std"); + +const packages = [_][2][]const u8{ + .{ "nix-base32", "nix_base32" }, + .{ "nix-store-path", "nix_store_path" }, + .{ "nix-narinfo", "nix_narinfo" }, + .{ "nix-derivation", "nix_derivation" }, + .{ "nix-archive", "nix_archive" }, +}; + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + const mod = b.createModule(.{ + .root_source_file = b.path("src/main.zig"), + .target = target, + .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, + .link_libc = true, + }); + inline for (packages) |p| mod.addImport(p[0], b.dependency(p[1], .{ .target = target, .optimize = optimize }).module(p[0])); + mod.linkSystemLibrary("zstd", .{}); + b.installArtifact(b.addExecutable(.{ .name = "bench", .root_module = mod })); + + const fl_mod = b.createModule(.{ + .root_source_file = b.path("src/flake_lock.zig"), + .target = target, + .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, + .link_libc = true, + }); + fl_mod.addImport("nix-flake-lock", b.dependency("nix_flake_lock", .{ .target = target, .optimize = optimize }).module("nix-flake-lock")); + b.installArtifact(b.addExecutable(.{ .name = "flake-lock", .root_module = fl_mod })); +} diff --git a/bench/zig/build.zig.zon b/bench/zig/build.zig.zon new file mode 100644 index 0000000..5e0b4b6 --- /dev/null +++ b/bench/zig/build.zig.zon @@ -0,0 +1,15 @@ +.{ + .name = .helios_bench, + .version = "0.1.0", + .fingerprint = 0xc67ae359dd92ff1e, + .minimum_zig_version = "0.17.0", + .dependencies = .{ + .nix_base32 = .{ .path = "../../pkg/nix-base32" }, + .nix_store_path = .{ .path = "../../pkg/nix-store-path" }, + .nix_narinfo = .{ .path = "../../pkg/nix-narinfo" }, + .nix_derivation = .{ .path = "../../pkg/nix-derivation" }, + .nix_archive = .{ .path = "../../pkg/nix-archive" }, + .nix_flake_lock = .{ .path = "../../pkg/nix-flake-lock" }, + }, + .paths = .{ "build.zig", "build.zig.zon", "src" }, +} diff --git a/bench/zig/src/flake_lock.zig b/bench/zig/src/flake_lock.zig new file mode 100644 index 0000000..b8c04e3 --- /dev/null +++ b/bench/zig/src/flake_lock.zig @@ -0,0 +1,113 @@ +//! Zig side of bench/flake-lock.sh, mirroring bench/flake-lock-rs. +//! FL_MODE=canon|bench, FL_LIST=. Output on stdout. + +const std = @import("std"); +const linux = std.os.linux; +const fl = @import("nix-flake-lock"); + +const gpa = std.heap.c_allocator; + +fn now() u64 { + var ts: linux.timespec = undefined; + _ = linux.clock_gettime(.MONOTONIC, &ts); + return @as(u64, @intCast(ts.sec)) * std.time.ns_per_s + @as(u64, @intCast(ts.nsec)); +} + +fn readFile(path: []const u8) ![]u8 { + const z = try gpa.dupeSentinel(u8, path, 0); + defer gpa.free(z); + const fd: i32 = @intCast(@as(isize, @bitCast(linux.openat(linux.AT.FDCWD, z, .{ .ACCMODE = .RDONLY }, 0)))); + if (fd < 0) return error.Open; + defer _ = linux.close(fd); + var list: std.ArrayList(u8) = .empty; + var buf: [1 << 16]u8 = undefined; + while (true) { + const n = linux.read(fd, &buf, buf.len); + if (linux.errno(n) != .SUCCESS) return error.Read; + if (n == 0) break; + try list.appendSlice(gpa, buf[0..n]); + } + return list.toOwnedSlice(gpa); +} + +fn flush(out: *std.ArrayList(u8)) void { + var off: usize = 0; + while (off < out.items.len) { + const n = linux.write(1, out.items[off..].ptr, out.items.len - off); + if (linux.errno(n) != .SUCCESS) return; + off += n; + } + out.clearRetainingCapacity(); +} + +fn iterations(len: usize) usize { + return std.math.clamp(4_000_000 / @max(len, 1), 3, 200_000); +} + +fn best(iters: usize, ctx: anytype, comptime f: fn (@TypeOf(ctx)) void) u64 { + for (0..@min(iters, 1000)) |_| f(ctx); + var b: u64 = std.math.maxInt(u64); + for (0..7) |_| { + const t0 = now(); + for (0..iters) |_| f(ctx); + b = @min(b, now() - t0); + } + return b; +} + +const ParseCtx = struct { bytes: []const u8 }; + +fn parseFresh(c: ParseCtx) void { + // Parse and free each time: the same malloc traffic the Rust API incurs. + var doc = fl.parse(gpa, c.bytes, .{}, null) catch unreachable; + std.mem.doNotOptimizeAway(doc.lock.nodes.ptr); + doc.deinit(); +} + +const LockCtx = struct { lock: *const fl.LockFile }; + +fn serialize(c: LockCtx) void { + var out: std.ArrayList(u8) = .empty; + defer out.deinit(gpa); + c.lock.writeNix(gpa, &out) catch unreachable; + std.mem.doNotOptimizeAway(out.items.ptr); +} + +fn validate(c: LockCtx) void { + c.lock.validate(gpa) catch {}; +} + +pub fn main() !void { + const mode = std.mem.span(std.c.getenv("FL_MODE") orelse return error.NoMode); + const list = try readFile(std.mem.span(std.c.getenv("FL_LIST") orelse return error.NoList)); + var out: std.ArrayList(u8) = .empty; + var it = std.mem.tokenizeScalar(u8, list, '\n'); + while (it.next()) |path| { + const bytes = try readFile(path); + defer gpa.free(bytes); + if (std.mem.eql(u8, mode, "canon")) { + try out.print(gpa, "== {s}\n", .{path}); + if (fl.parse(gpa, bytes, .{}, null)) |doc_| { + var doc = doc_; + defer doc.deinit(); + const lock = &doc.lock; + try lock.writeNix(gpa, &out); + const ok = if (lock.validate(gpa)) true else |_| false; + try out.print(gpa, "-- {s}\n", .{if (ok) "valid" else "invalid"}); + } else |_| { + try out.appendSlice(gpa, "-- parse error\n"); + } + if (out.items.len > 1 << 20) flush(&out); + continue; + } + const name = std.Io.Dir.path.basename(path); + const iters = iterations(bytes.len); + var doc = try fl.parse(gpa, bytes, .{}, null); + defer doc.deinit(); + const lock = &doc.lock; + try out.print(gpa, "parse:{s} {d} {d}\n", .{ name, iters, best(iters, ParseCtx{ .bytes = bytes }, parseFresh) }); + try out.print(gpa, "serialize:{s} {d} {d}\n", .{ name, iters, best(iters, LockCtx{ .lock = lock }, serialize) }); + try out.print(gpa, "validate:{s} {d} {d}\n", .{ name, iters, best(iters, LockCtx{ .lock = lock }, validate) }); + } + flush(&out); +} diff --git a/bench/zig/src/main.zig b/bench/zig/src/main.zig new file mode 100644 index 0000000..e92d074 --- /dev/null +++ b/bench/zig/src/main.zig @@ -0,0 +1,292 @@ +//! Benchmarks for the pkg/* Nix libraries. Mirrors bench/haskell/Bench.hs: +//! both read the same NUL-separated corpus files from $BENCH_DIR and print +//! ` ` lines. + +const std = @import("std"); +const linux = std.os.linux; +const base32 = @import("nix-base32"); +const store_path = @import("nix-store-path"); +const narinfo = @import("nix-narinfo"); +const derivation = @import("nix-derivation"); +const archive = @import("nix-archive"); + +const gpa = std.heap.c_allocator; +const rounds = 7; + +fn now() u64 { + var ts: linux.timespec = undefined; + _ = linux.clock_gettime(.MONOTONIC, &ts); + return @as(u64, @intCast(ts.sec)) * std.time.ns_per_s + @as(u64, @intCast(ts.nsec)); +} + +fn readFile(path: [:0]const u8) ![]u8 { + const fd: i32 = @intCast(linux.openat(linux.AT.FDCWD, path, .{ .ACCMODE = .RDONLY }, 0)); + if (fd < 0) return error.Open; + defer _ = linux.close(fd); + var list: std.ArrayList(u8) = .empty; + var buf: [1 << 16]u8 = undefined; + while (true) { + const n = linux.read(fd, &buf, buf.len); + if (linux.errno(n) != .SUCCESS) return error.Read; + if (n == 0) break; + try list.appendSlice(gpa, buf[0..n]); + } + return list.toOwnedSlice(gpa); +} + +fn records(dir: []const u8, name: []const u8) ![]const []const u8 { + const path = try gpa.printSentinel("{s}/{s}", .{ dir, name }, 0); + const data = try readFile(path); + var out: std.ArrayList([]const u8) = .empty; + var it = std.mem.splitScalar(u8, data, 0); + while (it.next()) |r| if (r.len > 0) try out.append(gpa, r); + return out.toOwnedSlice(gpa); +} + +fn report(name: []const u8, ops: usize, ns: u64) void { + std.debug.print("{s} {d} {d}\n", .{ name, ops, ns }); +} + +/// Runs `body` `rounds` times and reports the fastest round. +fn bench(name: []const u8, ops: usize, ctx: anytype, comptime body: fn (@TypeOf(ctx)) anyerror!void) !void { + try body(ctx); // warm up + var best: u64 = std.math.maxInt(u64); + for (0..rounds) |_| { + const t0 = now(); + try body(ctx); + best = @min(best, now() - t0); + } + report(name, ops, best); +} + +const Ctx = struct { + narinfos: []const []const u8, + parsed: []narinfo.NarInfo, + drvs: []const []const u8, + paths: []const []const u8, + hashes32: []const []const u8, + hashes52: []const []const u8, + raw20: [][20]u8, + fingerprints: []const []const u8, + signer: *narinfo.Signer, +}; + +fn narinfoParse(c: *const Ctx) !void { + for (c.narinfos) |t| std.mem.doNotOptimizeAway(try narinfo.parse(t)); +} + +fn narinfoRender(c: *const Ctx) !void { + for (c.parsed) |p| { + var nar: [32]u8 = undefined; + var file: [32]u8 = undefined; + try base32.decode(&nar, p.nar_hash["sha256:".len..]); + try base32.decode(&file, p.file_hash["sha256:".len..]); + const text = try narinfo.render(gpa, .{ + .store_path = p.store_path, + .nar_hash = &nar, + .nar_size = p.nar_size, + .file_hash = &file, + .file_size = p.file_size orelse 0, + .compression = p.compression, + .references = p.references, + .deriver = p.deriver, + .system = p.system, + }, null); + std.mem.doNotOptimizeAway(text.ptr); + gpa.free(text); + } +} + +fn sign(c: *const Ctx) !void { + var out: std.ArrayList(u8) = .empty; + defer out.deinit(gpa); + for (c.fingerprints) |fp| { + out.clearRetainingCapacity(); + try c.signer.sign(&out, gpa, fp); + std.mem.doNotOptimizeAway(out.items.ptr); + } +} + +fn drvParse(c: *const Ctx) !void { + var arena: std.heap.ArenaAllocator = .init(gpa); + defer arena.deinit(); + for (c.drvs) |d| { + std.mem.doNotOptimizeAway(try derivation.parse(arena.allocator(), d)); + _ = arena.reset(.retain_capacity); + } +} + +fn decode32(c: *const Ctx) !void { + for (c.hashes32) |h| { + var out: [20]u8 = undefined; + try base32.decode(&out, h); + std.mem.doNotOptimizeAway(&out); + } +} + +fn decode32Scalar(c: *const Ctx) !void { + for (c.hashes32) |h| { + var out: [20]u8 = undefined; + try base32.decodeScalar(&out, h); + std.mem.doNotOptimizeAway(&out); + } +} + +fn decode52(c: *const Ctx) !void { + for (c.hashes52) |h| { + var out: [32]u8 = undefined; + try base32.decode(&out, h); + std.mem.doNotOptimizeAway(&out); + } +} + +fn encode20(c: *const Ctx) !void { + for (c.raw20) |*r| { + var out: [32]u8 = undefined; + base32.encode(&out, r); + std.mem.doNotOptimizeAway(&out); + } +} + +fn storePathParse(c: *const Ctx) !void { + for (c.paths) |p| { + const base = store_path.baseName(p) orelse return error.Invalid; + { + const h = store_path.hashOf(base); + std.mem.doNotOptimizeAway(&h); + } + } +} + +fn storePathValid(c: *const Ctx) !void { + for (c.paths) |p| { + if (!store_path.isValidBaseName(p[store_path.store_dir.len..])) return error.Invalid; + } +} + +fn encode20Scalar(c: *const Ctx) !void { + for (c.raw20) |*r| { + var out: [32]u8 = undefined; + base32.encodeScalar(&out, r); + std.mem.doNotOptimizeAway(&out); + } +} + +fn storePathValidScalar(c: *const Ctx) !void { + for (c.paths) |p| { + const base = p[store_path.store_dir.len..]; + if (!(base32.isValidScalar(base[0..32]) and store_path.isValidNameScalar(base[33..]))) return error.Invalid; + } +} + +fn discard(_: ?*anyopaque, _: [*]const u8, _: usize) callconv(.c) c_int { + return 0; +} + +fn narHash(c: *const Ctx) !void { + for (c.paths) |p| { + var sink = try archive.Sink.init(gpa, .{ .level = 0 }, discard, null); + defer sink.deinit(gpa); + const z = try gpa.dupeSentinel(u8, p, 0); + defer gpa.free(z); + try archive.dump(&sink, z); + std.mem.doNotOptimizeAway(try sink.finish()); + } +} + +pub fn main() !void { + const dir = std.mem.span(std.c.getenv("BENCH_DIR") orelse return error.NoBenchDir); + const only = if (std.c.getenv("BENCH_ONLY")) |s| std.mem.span(s) else ""; + if (std.mem.eql(u8, only, "sha")) return shaThroughput(); + + const narinfos = try records(dir, "narinfo.bin"); + const parsed = try gpa.alloc(narinfo.NarInfo, narinfos.len); + for (narinfos, parsed) |t, *p| p.* = try narinfo.parse(t); + + const raw20 = try gpa.alloc([20]u8, parsed.len); + const hashes32 = try gpa.alloc([]const u8, parsed.len); + const hashes52 = try gpa.alloc([]const u8, parsed.len); + const fingerprints = try gpa.alloc([]const u8, parsed.len); + for (parsed, 0..) |p, i| { + hashes32[i] = p.store_path[store_path.store_dir.len..][0..32]; + hashes52[i] = p.nar_hash["sha256:".len..]; + try base32.decode(&raw20[i], hashes32[i]); + fingerprints[i] = try gpa.print("1;{s};{s};{d};{s}", .{ p.store_path, p.nar_hash, p.nar_size, p.references }); + } + const key = try readFile(try gpa.printSentinel("{s}/key", .{dir}, 0)); + const ctx: Ctx = .{ + .narinfos = narinfos, + .parsed = parsed, + .drvs = try records(dir, "drv.bin"), + .paths = try records(dir, "paths.bin"), + .hashes32 = hashes32, + .hashes52 = hashes52, + .raw20 = raw20, + .fingerprints = fingerprints, + .signer = try narinfo.Signer.parse(gpa, key), + }; + + // One untimed pass, for whole-process tools (bench/poop.sh). + if (std.mem.eql(u8, only, "nar-once")) return narHash(&ctx); + if (std.mem.eql(u8, only, "nar")) { + var bytes: u64 = 0; + for (ctx.paths) |p| { + var sink = try archive.Sink.init(gpa, .{ .level = 0 }, discard, null); + defer sink.deinit(gpa); + try archive.dump(&sink, try gpa.dupeSentinel(u8, p, 0)); + bytes += (try sink.finish()).nar_size; + } + const t0 = now(); + try narHash(&ctx); + report("nar-dump-sha256", @intCast(bytes), now() - t0); + return; + } + + try bench("narinfo-parse", narinfos.len, &ctx, narinfoParse); + try bench("narinfo-render", parsed.len, &ctx, narinfoRender); + try bench("ed25519-sign", fingerprints.len, &ctx, sign); + try bench("drv-parse", ctx.drvs.len, &ctx, drvParse); + try bench("base32-decode-20", hashes32.len, &ctx, decode32); + try bench("base32-decode-20-scalar", hashes32.len, &ctx, decode32Scalar); + try bench("base32-decode-32", hashes52.len, &ctx, decode52); + try bench("base32-encode-20", raw20.len, &ctx, encode20); + try bench("storepath-parse", ctx.paths.len, &ctx, storePathParse); + try bench("storepath-validate-simd", ctx.paths.len, &ctx, storePathValid); + try bench("storepath-validate-scalar", ctx.paths.len, &ctx, storePathValidScalar); + try bench("base32-encode-20-scalar", raw20.len, &ctx, encode20Scalar); +} + +/// Raw SHA-256 throughput over a 64 MiB buffer, for comparison with +/// `openssl speed -evp sha256`. +fn shaOnce(comptime H: type, name: []const u8, buf: []const u8) void { + var best: u64 = std.math.maxInt(u64); + for (0..5) |_| { + const t0 = now(); + var d: [32]u8 = undefined; + H.hash(buf, &d, .{}); + std.mem.doNotOptimizeAway(d); + best = @min(best, now() - t0); + } + report(name, buf.len, best); +} + +pub fn shaThroughput() void { + const big = gpa.alloc(u8, 64 << 20) catch unreachable; + @memset(big, 0xab); + shaOnce(std.crypto.hash.sha2.Sha256, "sha256-std", big); + shaOnce(archive.Sha256, "sha256-multiblock", big); +} + +pub fn shaThroughputOld() void { + const buf = gpa.alloc(u8, 64 << 20) catch unreachable; + @memset(buf, 0xab); + var best: u64 = std.math.maxInt(u64); + for (0..5) |_| { + const t0 = now(); + var d: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(buf, &d, .{}); + std.mem.doNotOptimizeAway(d); + best = @min(best, now() - t0); + } + report("sha256-64MiB", buf.len, best); +} diff --git a/core/build.zig b/core/build.zig new file mode 100644 index 0000000..ce6d6ec --- /dev/null +++ b/core/build.zig @@ -0,0 +1,39 @@ +//! libhelios: the C ABI over the pkg/* Nix packages, linked into the Rust +//! server and CLI as a static library. + +const std = @import("std"); + +const packages = [_][]const u8{ "nix-base32", "nix-store-path", "nix-archive", "nix-narinfo" }; + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mod = b.createModule(.{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, + .link_libc = true, + // Linked into Rust binaries, which are PIE. + .pic = true, + }); + inline for (packages) |name| { + const dep = b.dependency(comptime underscored(name), .{ .target = target, .optimize = optimize }); + mod.addImport(name, dep.module(name)); + } + + const lib = b.addLibrary(.{ .name = "helios", .linkage = .static, .root_module = mod }); + lib.bundle_compiler_rt = false; + lib.installHeader(b.path("include/helios.h"), "helios.h"); + b.installArtifact(lib); +} + +fn underscored(comptime name: []const u8) []const u8 { + comptime var out: [name.len]u8 = undefined; + for (name, 0..) |c, i| out[i] = if (c == '-') '_' else c; + const final = out; + return &final; +} diff --git a/core/build.zig.zon b/core/build.zig.zon new file mode 100644 index 0000000..72b2285 --- /dev/null +++ b/core/build.zig.zon @@ -0,0 +1,13 @@ +.{ + .name = .libhelios, + .version = "0.1.0", + .fingerprint = 0x27391ab28c1ecfe2, + .minimum_zig_version = "0.17.0", + .dependencies = .{ + .nix_base32 = .{ .path = "../pkg/nix-base32" }, + .nix_store_path = .{ .path = "../pkg/nix-store-path" }, + .nix_archive = .{ .path = "../pkg/nix-archive" }, + .nix_narinfo = .{ .path = "../pkg/nix-narinfo" }, + }, + .paths = .{ "build.zig", "build.zig.zon", "src", "include" }, +} diff --git a/core/include/helios.h b/core/include/helios.h new file mode 100644 index 0000000..d04d1c8 --- /dev/null +++ b/core/include/helios.h @@ -0,0 +1,126 @@ +/* libhelios: NAR pipeline and Nix binary cache formats. */ +#ifndef HELIOS_H +#define HELIOS_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define HL_OK 0 +#define HL_E_IO (-1) +#define HL_E_UNSUPPORTED_FILE (-2) +#define HL_E_ZSTD (-3) +#define HL_E_ABORTED (-4) +#define HL_E_INVALID (-5) +#define HL_E_NOMEM (-6) +#define HL_E_CHANGED (-7) +#define HL_E_UNSUPPORTED_OS (-8) +#define HL_E_TRUNCATED (-9) +#define HL_E_NOT_NAR (-10) +#define HL_E_BUFFER (-11) + +const char *hl_strerror(int rc); +/* errno of the last failed syscall on the calling thread. */ +int hl_last_errno(void); + +/* ── nix32 ── */ + +/* Writes ceil(len * 8 / 5) chars (32 for 20 bytes, 52 for 32 bytes). */ +size_t hl_nix32_encode(const uint8_t *bytes, size_t len, char *out); +int hl_nix32_decode(const char *text, size_t text_len, uint8_t *out, size_t out_len); +bool hl_store_basename_valid(const char *base, size_t len); + +/* ── Compressing NAR pipeline ── */ + +typedef struct { + uint8_t file_hash[32]; /* sha256 of the compressed stream */ + uint64_t file_size; + uint8_t nar_hash[32]; /* sha256 of the uncompressed NAR */ + uint64_t nar_size; +} hl_digest; + +typedef struct { + int level; /* zstd level; 0 disables compression */ + int threads; /* zstd worker threads; 0 = calling thread */ + uint64_t nar_size; /* exact NAR size, pledged to zstd; 0 if unknown */ + int window_log; /* 0: the level's window; 10-27: long-distance matching */ + bool hash; /* false skips SHA-256; the digest then has sizes only */ +} hl_dump_options; + +/* Return non-zero to abort the pipeline with HL_E_ABORTED. */ +typedef int (*hl_write_fn)(void *ctx, const uint8_t *buf, size_t len); + +/* Serialise `path` as a NAR, compress it, and stream it to `write`. + * Linux only; returns HL_E_UNSUPPORTED_OS elsewhere. */ +int hl_nar_dump(const char *path, const hl_dump_options *opts, hl_write_fn write, void *ctx, hl_digest *out); + +/* Same pipeline, fed with NAR bytes from elsewhere (e.g. nix store dump-path). */ +typedef struct hl_compressor hl_compressor; +hl_compressor *hl_compressor_new(const hl_dump_options *opts, hl_write_fn write, void *ctx); +int hl_compressor_update(hl_compressor *c, const uint8_t *data, size_t len); +int hl_compressor_finish(hl_compressor *c, hl_digest *out); +void hl_compressor_free(hl_compressor *c); + +/* ── Upload verifier ── */ + +#define HL_COMPRESSION_NONE 0 +#define HL_COMPRESSION_ZSTD 1 + +typedef struct hl_verifier hl_verifier; +hl_verifier *hl_verifier_new(int compression); +int hl_verifier_update(hl_verifier *v, const uint8_t *data, size_t len); +/* Fails with HL_E_TRUNCATED or HL_E_NOT_NAR on bad input. */ +int hl_verifier_finish(hl_verifier *v, hl_digest *out); +void hl_verifier_free(hl_verifier *v); + +/* ── Signing and narinfo ── */ + +typedef struct hl_signer hl_signer; +/* Parses a Nix secret key: ":". */ +hl_signer *hl_signer_new(const char *key, size_t len); +void hl_signer_free(hl_signer *s); +/* Writes ":". Sets *out_len even on HL_E_BUFFER. */ +int hl_signer_public_key(const hl_signer *s, char *out, size_t cap, size_t *out_len); +/* A detached Ed25519 signature over msg, as `:`. */ +int hl_signer_sign(const hl_signer *s, const uint8_t *msg, size_t msg_len, char *out, size_t cap, size_t *out_len); +/* Generates a new secret key in the same format from the OS CSPRNG. */ +int hl_signer_generate(const char *name, size_t name_len, char *out, size_t cap, size_t *out_len); + +typedef struct { + const char *ptr; + size_t len; +} hl_str; + +typedef struct { + hl_str store_path; /* /nix/store/- */ + const uint8_t *nar_hash; /* 32 bytes */ + uint64_t nar_size; + const uint8_t *file_hash; /* 32 bytes */ + uint64_t file_size; + hl_str compression; /* zstd | none | xz | bzip2 */ + hl_str references; /* space-separated basenames */ + hl_str deriver; /* basename or empty */ + hl_str system; /* or empty */ +} hl_narinfo_input; + +/* Validates every field and renders a (signed, if signer != NULL) narinfo + * into a heap buffer that must be released with hl_free. */ +int hl_narinfo_render(const hl_narinfo_input *in, const hl_signer *signer, char **out, size_t *out_len); +void hl_free(char *ptr, size_t len); + +/* ── Primitives ── */ + +void hl_sha256(const uint8_t *data, size_t len, uint8_t out[32]); +void hl_hmac_sha256(const uint8_t *key, size_t key_len, const uint8_t *msg, size_t msg_len, uint8_t out[32]); +/* Fills buf from the OS CSPRNG. */ +int hl_random(uint8_t *buf, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/core/src/root.zig b/core/src/root.zig new file mode 100644 index 0000000..7dfccd7 --- /dev/null +++ b/core/src/root.zig @@ -0,0 +1,278 @@ +//! C ABI for libhelios. See include/helios.h for the contract. + +const std = @import("std"); +const nix32 = @import("nix-base32"); +const store_path = @import("nix-store-path"); +const archive = @import("nix-archive"); + +const narinfo = @import("nix-narinfo"); + +const Sink = archive.Sink; +const Digest = archive.Digest; +const WriteFn = archive.WriteFn; +const gpa = std.heap.c_allocator; + +pub const HL_OK: c_int = 0; +pub const HL_E_IO: c_int = -1; +pub const HL_E_UNSUPPORTED_FILE: c_int = -2; +pub const HL_E_ZSTD: c_int = -3; +pub const HL_E_ABORTED: c_int = -4; +pub const HL_E_INVALID: c_int = -5; +pub const HL_E_NOMEM: c_int = -6; +pub const HL_E_CHANGED: c_int = -7; +pub const HL_E_UNSUPPORTED_OS: c_int = -8; +pub const HL_E_TRUNCATED: c_int = -9; +pub const HL_E_NOT_NAR: c_int = -10; +pub const HL_E_BUFFER: c_int = -11; + +fn code(err: anyerror) c_int { + return switch (err) { + error.Io, error.NameTooLong => HL_E_IO, + error.UnsupportedFileType => HL_E_UNSUPPORTED_FILE, + error.Zstd => HL_E_ZSTD, + error.Aborted => HL_E_ABORTED, + error.OutOfMemory => HL_E_NOMEM, + error.FileChanged => HL_E_CHANGED, + error.Truncated => HL_E_TRUNCATED, + error.NotNar => HL_E_NOT_NAR, + else => HL_E_INVALID, + }; +} + +export fn hl_strerror(rc: c_int) [*:0]const u8 { + return switch (rc) { + HL_OK => "ok", + HL_E_IO => "I/O error", + HL_E_UNSUPPORTED_FILE => "unsupported file type in store path", + HL_E_ZSTD => "zstd error", + HL_E_ABORTED => "aborted by write callback", + HL_E_INVALID => "invalid input", + HL_E_NOMEM => "out of memory", + HL_E_CHANGED => "file changed while reading", + HL_E_UNSUPPORTED_OS => "NAR dumping is not supported on this OS", + HL_E_TRUNCATED => "compressed stream is truncated", + HL_E_NOT_NAR => "not a NAR archive", + HL_E_BUFFER => "output buffer too small", + else => "unknown error", + }; +} + +export fn hl_last_errno() c_int { + return archive.lastErrno(); +} + +// ── nix32 ── + +export fn hl_nix32_encode(bytes: [*]const u8, len: usize, out: [*]u8) usize { + const n = nix32.encodedLen(len); + nix32.encode(out[0..n], bytes[0..len]); + return n; +} + +export fn hl_nix32_decode(text: [*]const u8, text_len: usize, out: [*]u8, out_len: usize) c_int { + nix32.decode(out[0..out_len], text[0..text_len]) catch return HL_E_INVALID; + return HL_OK; +} + +export fn hl_store_basename_valid(base: [*]const u8, len: usize) bool { + return store_path.isValidBaseName(base[0..len]); +} + +// ── Compressing NAR pipeline ── + +pub const DumpOptions = extern struct { + level: c_int, + threads: c_int, + nar_size: u64, + window_log: c_int, + hash: bool, +}; + +fn sinkOptions(opts: *const DumpOptions) archive.Options { + return .{ .level = opts.level, .threads = opts.threads, .nar_size = opts.nar_size, .window_log = opts.window_log, .hash = opts.hash }; +} + +export fn hl_nar_dump(path: [*:0]const u8, opts: *const DumpOptions, write: WriteFn, ctx: ?*anyopaque, out: *Digest) c_int { + if (comptime !archive.supported) return HL_E_UNSUPPORTED_OS; + var sink = Sink.init(gpa, sinkOptions(opts), write, ctx) catch |e| return code(e); + defer sink.deinit(gpa); + archive.dump(&sink, path) catch |e| return code(e); + out.* = sink.finish() catch |e| return code(e); + return HL_OK; +} + +export fn hl_compressor_new(opts: *const DumpOptions, write: WriteFn, ctx: ?*anyopaque) ?*Sink { + const sink = gpa.create(Sink) catch return null; + sink.* = Sink.init(gpa, sinkOptions(opts), write, ctx) catch { + gpa.destroy(sink); + return null; + }; + return sink; +} + +export fn hl_compressor_update(sink: *Sink, data: [*]const u8, len: usize) c_int { + sink.write(data[0..len]) catch |e| return code(e); + return HL_OK; +} + +export fn hl_compressor_finish(sink: *Sink, out: *Digest) c_int { + out.* = sink.finish() catch |e| return code(e); + return HL_OK; +} + +export fn hl_compressor_free(sink: ?*Sink) void { + const s = sink orelse return; + s.deinit(gpa); + gpa.destroy(s); +} + +// ── Upload verifier ── + +export fn hl_verifier_new(compression: c_int) ?*archive.Verifier { + const kind = std.enums.fromInt(archive.Compression, compression) orelse return null; + const v = gpa.create(archive.Verifier) catch return null; + v.* = archive.Verifier.init(gpa, kind) catch { + gpa.destroy(v); + return null; + }; + return v; +} + +export fn hl_verifier_update(v: *archive.Verifier, data: [*]const u8, len: usize) c_int { + v.update(data[0..len]) catch |e| return code(e); + return HL_OK; +} + +export fn hl_verifier_finish(v: *archive.Verifier, out: *Digest) c_int { + out.* = v.finish() catch |e| return code(e); + return HL_OK; +} + +export fn hl_verifier_free(v: ?*archive.Verifier) void { + const p = v orelse return; + p.deinit(gpa); + gpa.destroy(p); +} + +// ── Signing and narinfo ── + +export fn hl_signer_new(key: [*]const u8, len: usize) ?*narinfo.Signer { + return narinfo.Signer.parse(gpa, key[0..len]) catch null; +} + +export fn hl_signer_free(s: ?*narinfo.Signer) void { + if (s) |p| p.destroy(gpa); +} + +/// Generates a Nix secret key named `name` from the OS CSPRNG. +export fn hl_signer_generate(name: [*]const u8, name_len: usize, out: ?[*]u8, cap: usize, out_len: *usize) c_int { + var seed: [32]u8 = undefined; + const rc = hl_random(&seed, seed.len); + if (rc != HL_OK) return rc; + defer std.crypto.secureZero(u8, &seed); + var list: std.ArrayList(u8) = .empty; + defer { + std.crypto.secureZero(u8, list.items); + list.deinit(gpa); + } + narinfo.Signer.generate(&list, gpa, name[0..name_len], seed) catch |e| return code(e); + return copyOut(list.items, out, cap, out_len); +} + +fn copyOut(bytes: []const u8, out: ?[*]u8, cap: usize, out_len: *usize) c_int { + out_len.* = bytes.len; + if (bytes.len > cap or out == null) return HL_E_BUFFER; + @memcpy(out.?[0..bytes.len], bytes); + return HL_OK; +} + +export fn hl_signer_public_key(s: *const narinfo.Signer, out: ?[*]u8, cap: usize, out_len: *usize) c_int { + var list: std.ArrayList(u8) = .empty; + defer list.deinit(gpa); + s.publicKey(&list, gpa) catch |e| return code(e); + return copyOut(list.items, out, cap, out_len); +} + +/// A detached signature over `msg`, as `:`. +export fn hl_signer_sign(s: *const narinfo.Signer, msg: ?[*]const u8, msg_len: usize, out: ?[*]u8, cap: usize, out_len: *usize) c_int { + var list: std.ArrayList(u8) = .empty; + defer list.deinit(gpa); + const m: []const u8 = if (msg) |p| p[0..msg_len] else ""; + s.sign(&list, gpa, m) catch |e| return code(e); + return copyOut(list.items, out, cap, out_len); +} + +pub const Str = extern struct { + ptr: ?[*]const u8, + len: usize, + + fn slice(self: Str) []const u8 { + return if (self.ptr) |p| p[0..self.len] else ""; + } +}; + +pub const NarinfoInput = extern struct { + store_path: Str, + nar_hash: *const [32]u8, + nar_size: u64, + file_hash: *const [32]u8, + file_size: u64, + compression: Str, + references: Str, + deriver: Str, + system: Str, +}; + +/// Renders into a heap buffer released with hl_free. +export fn hl_narinfo_render(in: *const NarinfoInput, signer: ?*const narinfo.Signer, out: *?[*]u8, out_len: *usize) c_int { + const text = narinfo.render(gpa, .{ + .store_path = in.store_path.slice(), + .nar_hash = in.nar_hash, + .nar_size = in.nar_size, + .file_hash = in.file_hash, + .file_size = in.file_size, + .compression = in.compression.slice(), + .references = in.references.slice(), + .deriver = in.deriver.slice(), + .system = in.system.slice(), + }, signer) catch |e| return code(e); + out.* = text.ptr; + out_len.* = text.len; + return HL_OK; +} + +export fn hl_free(ptr: ?[*]u8, len: usize) void { + if (ptr) |p| gpa.free(p[0..len]); +} + + +// ── Small primitives for the Rust side (keeps hmac/sha2/uuid crates out) ── + +const builtin = @import("builtin"); + +export fn hl_sha256(data: [*]const u8, len: usize, out: *[32]u8) void { + archive.Sha256.hash(data[0..len], out, .{}); +} + +export fn hl_hmac_sha256(key: [*]const u8, key_len: usize, msg: [*]const u8, msg_len: usize, out: *[32]u8) void { + std.crypto.auth.hmac.sha2.HmacSha256.create(out, msg[0..msg_len], key[0..key_len]); +} + +/// Fills `buf` from the OS CSPRNG. +export fn hl_random(buf: [*]u8, len: usize) c_int { + if (builtin.target.os.tag == .linux) { + const linux = std.os.linux; + var off: usize = 0; + while (off < len) { + const rc = linux.getrandom(buf + off, len - off, 0); + switch (linux.errno(rc)) { + .SUCCESS => off += rc, + .INTR => {}, + else => return HL_E_IO, + } + } + } else { + std.c.arc4random_buf(buf, len); + } + return HL_OK; +} diff --git a/crates/helios-cli/Cargo.toml b/crates/helios-cli/Cargo.toml new file mode 100644 index 0000000..67463c2 --- /dev/null +++ b/crates/helios-cli/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "helios-cli" +version.workspace = true +edition.workspace = true +license-file.workspace = true +publish.workspace = true + +[[bin]] +name = "helios" +path = "src/main.rs" + +[dependencies] +anyhow = "1.0.104" +base64 = "0.23.1" +bytes = "1.12.1" +clap = { version = "4.6.7", default-features = false, features = ["std", "derive", "env", "help", "usage", "error-context"] } +futures-util = { version = "0.3.34", default-features = false, features = ["std"] } +helios-core = { path = "../helios-core" } +helios-log = { path = "../helios-log" } +http = "1.5" +http-body-util = "0.1.5" +# HTTP/1.1 client only; TLS via rustls with the ring provider (no aws-lc-rs). +hyper = { version = "1.11", default-features = false, features = ["client", "http1"] } +rustls = { version = "0.23.45", default-features = false, features = ["ring", "std", "tls12"] } +rustls-native-certs = "0.8.4" +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.151" +tokio = { version = "1.53.2", features = ["rt-multi-thread", "net", "sync", "process", "io-util", "time"] } +tracing = { version = "0.1.44", default-features = false, features = ["std"] } +tokio-rustls = { version = "0.26.6", default-features = false, features = ["ring", "tls12"] } diff --git a/crates/helios-cli/src/api.rs b/crates/helios-cli/src/api.rs new file mode 100644 index 0000000..ff485df --- /dev/null +++ b/crates/helios-cli/src/api.rs @@ -0,0 +1,200 @@ +use anyhow::{Context, bail}; +use http::{Method, StatusCode}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::config::Server; +use crate::transport::{self, Body}; + +pub struct Client { + http: transport::Client, + token: String, +} + +#[derive(Serialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct PathSpec { + pub store_path: String, + pub nar_hash: String, + pub nar_size: u64, + pub references: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub deriver: Option, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Uploaded { + pub file_size: u64, + pub nar_hash: String, + pub nar_size: u64, +} + +pub enum Publish { + Done { published: u64 }, + MissingNars(Vec), +} + +fn error_text(status: StatusCode, body: &[u8]) -> String { + let msg = serde_json::from_slice::(body) + .ok() + .and_then(|v| v.get("error").and_then(Value::as_str).map(str::to_owned)) + .unwrap_or_else(|| String::from_utf8_lossy(body).into_owned()); + format!("{status}: {msg}") +} + +impl Client { + pub fn new(server: &Server) -> anyhow::Result { + Ok(Self { http: transport::Client::new(server.server.trim_end_matches('/'))?, token: server.token.clone() }) + } + + async fn call(&self, method: Method, path: &str, content_type: Option<&str>, body: Body) -> anyhow::Result { + let (status, bytes) = self.http.send(method, &format!("/_api/v2{path}"), &self.token, content_type, body).await?; + if !status.is_success() { + bail!("{}", error_text(status, &bytes)); + } + serde_json::from_slice(&bytes).with_context(|| format!("decoding response from {path}")) + } + + async fn post(&self, path: &str, body: &Value) -> anyhow::Result { + self.call(Method::POST, path, Some("application/json"), transport::full(serde_json::to_vec(body)?)).await + } + + pub async fn missing(&self, cache: &str, hashes: &[&str]) -> anyhow::Result> { + #[derive(Deserialize)] + struct R { + missing: Vec, + } + let r: R = self.post(&format!("/caches/{cache}/missing"), &json!({ "hashes": hashes })).await?; + Ok(r.missing) + } + + pub async fn known(&self, cache: &str, nar_hashes: &[&str]) -> anyhow::Result> { + #[derive(Deserialize)] + struct R { + known: Vec, + } + let r: R = self.post(&format!("/caches/{cache}/nars/known"), &json!({ "narHashes": nar_hashes })).await?; + Ok(r.known) + } + + /// `compression` is the narinfo name: `zstd`, or `none` for a raw NAR. + pub async fn upload(&self, cache: &str, compression: &str, body: Body) -> anyhow::Result { + self.call(Method::PUT, &format!("/caches/{cache}/nar?compression={compression}"), Some("application/x-nix-nar"), body).await + } + + pub async fn publish_build_traces(&self, cache: &str, entries: &[Value]) -> anyhow::Result { + self.post(&format!("/caches/{cache}/build-traces"), &json!({ "entries": entries })).await + } + + pub async fn pin(&self, cache: &str, store_paths: &[String]) -> anyhow::Result { + self.post(&format!("/caches/{cache}/pins"), &json!({ "storePaths": store_paths })).await + } + + pub async fn unpin(&self, cache: &str, store_path: &str) -> anyhow::Result { + let base = store_path.strip_prefix("/nix/store/").unwrap_or(store_path); + self.call(Method::DELETE, &format!("/caches/{cache}/pins/{base}"), None, transport::full(Bytes::new())).await + } + + pub async fn pins(&self, cache: &str) -> anyhow::Result { + self.call(Method::GET, &format!("/caches/{cache}/pins"), None, transport::full(Bytes::new())).await + } + + pub async fn cache_info(&self, cache: &str) -> anyhow::Result { + self.call(Method::GET, &format!("/caches/{cache}"), None, transport::full(Bytes::new())).await + } + + /// Starts a chunked upload; returns its id. + pub async fn upload_create(&self, cache: &str, compression: &str) -> anyhow::Result { + #[derive(Deserialize)] + struct R { + id: String, + } + let r: R = + self.call(Method::POST, &format!("/caches/{cache}/uploads?compression={compression}"), None, transport::full(Bytes::new())).await?; + Ok(r.id) + } + + /// Appends `chunk` at `offset`, retrying transient failures. After a + /// failed request the chunk may or may not have landed; the server's + /// offset says which, and a chunk is never appended twice. + pub async fn upload_append(&self, cache: &str, id: &str, offset: u64, chunk: Bytes) -> anyhow::Result { + #[derive(Deserialize)] + struct R { + offset: u64, + } + let end = offset + chunk.len() as u64; + let path = format!("/_api/v2/caches/{cache}/uploads/{id}"); + let mut delay = std::time::Duration::from_millis(250); + for attempt in 1.. { + let failure = + match self.http.send(Method::PATCH, &format!("{path}?offset={offset}"), &self.token, None, transport::full(chunk.clone())).await { + Ok((status, bytes)) if status.is_success() => return Ok(serde_json::from_slice::(&bytes)?.offset), + Ok((status, bytes)) if status == StatusCode::CONFLICT => { + let at = serde_json::from_slice::(&bytes).ok().and_then(|v| v["offset"].as_u64()); + match at { + Some(at) if at == end => return Ok(end), + _ => bail!("{}", error_text(status, &bytes)), + } + } + Ok((status, bytes)) if status.is_server_error() || status == StatusCode::TOO_MANY_REQUESTS => error_text(status, &bytes), + Ok((status, bytes)) => bail!("{}", error_text(status, &bytes)), + Err(e) => format!("{e:#}"), + }; + if attempt == 5 { + bail!("uploading a chunk at offset {offset}: {failure}"); + } + tracing::warn!("uploading a chunk at offset {offset} failed, retrying in {delay:?}: {failure}"); + tokio::time::sleep(delay).await; + delay *= 2; + // The request may have landed before the failure; if so, move on. + if let Ok(r) = self.call::(Method::GET, &format!("/caches/{cache}/uploads/{id}"), None, transport::full(Bytes::new())).await { + match r.offset { + at if at == end => return Ok(end), + at if at == offset => {} + at => bail!("server is at offset {at}, expected {offset} or {end}"), + } + } + } + unreachable!() + } + + pub async fn upload_complete(&self, cache: &str, id: &str) -> anyhow::Result { + self.call(Method::POST, &format!("/caches/{cache}/uploads/{id}/complete"), None, transport::full(Bytes::new())).await + } + + pub async fn upload_abort(&self, cache: &str, id: &str) { + let _ = + self.http.send(Method::DELETE, &format!("/_api/v2/caches/{cache}/uploads/{id}"), &self.token, None, transport::full(Bytes::new())).await; + } + + pub async fn publish(&self, cache: &str, paths: &[PathSpec]) -> anyhow::Result { + let body = transport::full(serde_json::to_vec(&json!({ "paths": paths }))?); + let (status, bytes) = + self.http.send(Method::POST, &format!("/_api/v2/caches/{cache}/paths"), &self.token, Some("application/json"), body).await?; + if status == StatusCode::CONFLICT { + let v: Value = serde_json::from_slice(&bytes)?; + if v["error"] == "nar_required" { + let missing = v["missing"].as_array().context("missing list")?.iter().filter_map(|m| m.as_str().map(str::to_owned)).collect(); + return Ok(Publish::MissingNars(missing)); + } + bail!("publish conflict: {}", v["error"]); + } + if !status.is_success() { + bail!("{}", error_text(status, &bytes)); + } + let v: Value = serde_json::from_slice(&bytes)?; + Ok(Publish::Done { published: v["published"].as_u64().unwrap_or(0) }) + } + + pub async fn admin_get(&self, path: &str) -> anyhow::Result { + self.call(Method::GET, path, None, transport::full(Bytes::new())).await + } + + pub async fn admin_post(&self, path: &str, body: Value) -> anyhow::Result { + self.post(path, &body).await + } +} + +use bytes::Bytes; diff --git a/crates/helios-cli/src/config.rs b/crates/helios-cli/src/config.rs new file mode 100644 index 0000000..93e11cc --- /dev/null +++ b/crates/helios-cli/src/config.rs @@ -0,0 +1,108 @@ +//! `~/.config/helios/config.json`. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, bail}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Clone)] +pub struct Server { + pub server: String, + pub token: String, +} + +#[derive(Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct Config { + #[serde(skip_serializing_if = "Option::is_none")] + pub default_server: Option, + #[serde(default)] + pub servers: BTreeMap, +} + +fn path() -> anyhow::Result { + let base = match std::env::var_os("XDG_CONFIG_HOME") { + Some(dir) if !dir.is_empty() => PathBuf::from(dir), + _ => PathBuf::from(std::env::var_os("HOME").context("HOME is not set")?).join(".config"), + }; + Ok(base.join("helios").join("config.json")) +} + +pub fn load() -> anyhow::Result { + match std::fs::read(path()?) { + Ok(raw) => Ok(serde_json::from_slice(&raw).context("parsing helios config")?), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Config::default()), + Err(e) => Err(e.into()), + } +} + +pub fn login(name: &str, server: &str, token: &str) -> anyhow::Result<()> { + let mut cfg = load()?; + cfg.servers.insert(name.into(), Server { server: server.trim_end_matches('/').into(), token: token.into() }); + cfg.default_server = Some(name.into()); + let path = path()?; + std::fs::create_dir_all(path.parent().expect("config path has a parent"))?; + let mut body = serde_json::to_vec_pretty(&cfg)?; + body.push(b'\n'); + write_secret(&path, &body) +} + +/// Replaces `path` with `contents`, readable by the owner only: a fresh +/// 0600 file in the same directory, synced and renamed over the old one, so +/// a crash leaves the old or the new file and never a torn or +/// world-readable one. A symlink is followed, so the file it names is +/// replaced. +pub fn write_secret(path: &Path, contents: &[u8]) -> anyhow::Result<()> { + use std::io::Write; + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + + let path = std::fs::canonicalize(path).unwrap_or_else(|_| path.to_owned()); + let dir = path.parent().with_context(|| format!("{} has no directory", path.display()))?; + let name = path.file_name().with_context(|| format!("{} has no file name", path.display()))?.to_string_lossy(); + let tmp = dir.join(format!(".{name}.{}.tmp", helios_core::uuid_v4())); + let written = (|| { + let mut file = std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(&tmp)?; + // The mode above is subject to the umask; this is not. + file.set_permissions(std::fs::Permissions::from_mode(0o600))?; + file.write_all(contents)?; + file.sync_all()?; + std::fs::rename(&tmp, &path)?; + std::fs::File::open(dir)?.sync_all() + })(); + if written.is_err() { + let _ = std::fs::remove_file(&tmp); + } + written.with_context(|| format!("writing {}", path.display())) +} + +pub fn server(name: Option<&str>) -> anyhow::Result { + let cfg = load()?; + let Some(key) = name.map(str::to_owned).or(cfg.default_server) else { + bail!("no server configured; run: helios login "); + }; + cfg.servers.get(&key).cloned().with_context(|| format!("server '{key}' not found; run: helios login {key} ")) +} + +#[cfg(test)] +mod tests { + use super::write_secret; + use std::os::unix::fs::PermissionsExt; + + #[test] + fn secrets_replace_files_privately() { + let dir = std::env::temp_dir().join(format!("helios-config-{}", helios_core::uuid_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("netrc"); + std::fs::write(&file, "old").unwrap(); + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o644)).unwrap(); + let link = dir.join("link"); + std::os::unix::fs::symlink(&file, &link).unwrap(); + write_secret(&link, b"new").unwrap(); + assert_eq!(std::fs::read_to_string(&file).unwrap(), "new"); + assert_eq!(std::fs::metadata(&file).unwrap().permissions().mode() & 0o777, 0o600); + assert!(std::fs::symlink_metadata(&link).unwrap().file_type().is_symlink()); + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 2, "no temporary files left"); + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/crates/helios-cli/src/main.rs b/crates/helios-cli/src/main.rs new file mode 100644 index 0000000..08db0e0 --- /dev/null +++ b/crates/helios-cli/src/main.rs @@ -0,0 +1,300 @@ +mod api; +mod config; +mod nix; +mod push; +mod substituter; +mod transport; +mod watch; + +use anyhow::Context; +use clap::{Parser, Subcommand}; +use serde_json::{Value, json}; + +/// Helios Nix binary cache CLI. +#[derive(Parser)] +#[command(version)] +struct Cli { + /// Server name from `helios login` (default: the last one logged in). + #[arg(long, global = true, env = "HELIOS_SERVER")] + server: Option, + + /// Server URL, instead of a saved login (for services and CI). + #[arg(long, global = true, env = "HELIOS_URL")] + url: Option, + + /// File holding the token for --url (or set HELIOS_TOKEN). + #[arg(long, global = true, env = "HELIOS_TOKEN_FILE")] + token_file: Option, + + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Save server credentials (a push token, or the admin secret for admin commands). + Login { name: String, url: String, token: String }, + /// Push store paths (or installables like `.#foo`) to a cache. + Push { + cache: String, + #[arg(required = true)] + installables: Vec, + /// Push the full closure. + #[arg(long, short = 'r')] + closure: bool, + /// Parallel uploads. + #[arg(long, short = 'j', default_value_t = 8)] + jobs: usize, + /// zstd level, 1-19, or 0 to upload uncompressed. Default: the + /// server's (3 unless configured). + #[arg(long, value_parser = clap::value_parser!(i32).range(0..=19))] + level: Option, + /// zstd window for long-distance matching, as a power of two + /// (10-27), or 0 for the level's own. Default: the server's (27). + #[arg(long, value_parser = parse_window_log)] + window_log: Option, + /// Pin the given paths afterwards, so auto-GC keeps them and their + /// closures. + #[arg(long)] + pin: bool, + /// Upload NARs in chunks of this many MiB, for proxies that cap + /// request bodies (Cloudflare, Cloud Run). 0 sends each NAR in one + /// request; so does a NAR that fits in one chunk. + #[arg(long, default_value_t = 32, value_parser = clap::value_parser!(u64).range(0..=64))] + chunk_size: u64, + }, + /// Push paths as they are built: drain a spool filled by `queue-paths`. + WatchStore { + cache: String, + /// Spool directory shared with the post-build hook. + #[arg(long, default_value = "/var/lib/helios-watch-store/spool")] + spool: std::path::PathBuf, + /// Parallel uploads. + #[arg(long, short = 'j', default_value_t = 8)] + jobs: usize, + /// zstd level, 1-19, or 0 to upload uncompressed. Default: the + /// server's (3 unless configured). + #[arg(long, value_parser = clap::value_parser!(i32).range(0..=19))] + level: Option, + /// zstd window for long-distance matching, as a power of two + /// (10-27), or 0 for the level's own. Default: the server's (27). + #[arg(long, value_parser = parse_window_log)] + window_log: Option, + /// Upload NARs in chunks of this many MiB, for proxies that cap + /// request bodies (Cloudflare, Cloud Run). 0 sends each NAR in one + /// request; so does a NAR that fits in one chunk. + #[arg(long, default_value_t = 32, value_parser = clap::value_parser!(u64).range(0..=64))] + chunk_size: u64, + }, + /// Protect store paths, with their closures in the cache, from auto-GC. + Pin { + cache: String, + /// Store paths, or installables to resolve with Nix. + #[arg(required = true)] + paths: Vec, + }, + /// Remove pins. + Unpin { + cache: String, + #[arg(required = true)] + paths: Vec, + }, + /// List a cache's pins. + Pins { cache: String }, + /// Configure Nix to substitute from a cache: its URL, signing key and, + /// for a private cache, this login's token in netrc. + Use { + cache: String, + /// Print system-wide settings (nix.conf, NixOS, netrc) instead of + /// writing the user's nix.conf. + #[arg(long)] + print: bool, + }, + /// Spool store paths for `watch-store` (Nix post-build-hook; reads $OUT_PATHS). + QueuePaths { + #[arg(long, default_value = "/var/lib/helios-watch-store/spool")] + spool: std::path::PathBuf, + /// Paths to queue; defaults to $OUT_PATHS. + paths: Vec, + }, + /// Manage caches (admin). + #[command(subcommand)] + Cache(CacheCmd), + /// Manage API tokens (admin). + #[command(subcommand)] + Token(TokenCmd), +} + +#[derive(Subcommand)] +enum CacheCmd { + Create { + name: String, + /// Require a pull token to read. + #[arg(long)] + private: bool, + }, + List, +} + +#[derive(Subcommand)] +enum TokenCmd { + Create { + subject: String, + /// Comma-separated cache names, or "*" for all of them. + #[arg(long)] + caches: String, + /// Comma-separated: pull, push. Read-only unless push is asked for; + /// push does not imply pull, so a builder can upload without reading + /// private caches. + #[arg(long, default_value = "pull")] + perms: String, + /// Lifetime in days (1-365). + #[arg(long, default_value_t = 90)] + expires: i64, + }, + List, + Revoke { + jti: String, + reason: String, + }, +} + +fn split(s: &str) -> Vec { + s.split(',').map(str::trim).filter(|s| !s.is_empty()).map(str::to_owned).collect() +} + +fn print(v: &Value) { + println!("{}", serde_json::to_string_pretty(v).unwrap_or_default()); +} + +async fn run(cli: Cli) -> anyhow::Result<()> { + if let Command::Login { name, url, token } = &cli.command { + config::login(name, url, token)?; + tracing::info!("logged in to '{name}' at {url}"); + return Ok(()); + } + // Runs inside the nix-daemon's post-build hook: no server needed, and it + // must return quickly. + if let Command::QueuePaths { spool, paths } = &cli.command { + let joined = if paths.is_empty() { std::env::var("OUT_PATHS").unwrap_or_default() } else { paths.join(" ") }; + watch::queue(spool, &joined)?; + return Ok(()); + } + let server = match &cli.url { + Some(url) => { + let token = match &cli.token_file { + Some(file) => std::fs::read_to_string(file).with_context(|| format!("reading {}", file.display()))?, + None => std::env::var("HELIOS_TOKEN").context("--url needs --token-file or HELIOS_TOKEN")?, + }; + config::Server { server: url.trim_end_matches('/').to_owned(), token: token.trim().to_owned() } + } + None => config::server(cli.server.as_deref())?, + }; + let client = api::Client::new(&server)?; + match cli.command { + Command::Login { .. } | Command::QueuePaths { .. } => unreachable!(), + Command::Push { cache, installables, closure, jobs, level, window_log, chunk_size, pin } => { + let (level, window_log) = compression(&client, &cache, level, window_log).await; + let opts = push::Options { jobs, level, window_log, closure, chunk_size: (chunk_size << 20) as usize }; + push::push(&client, &cache, &installables, opts).await?; + if pin { + pin_paths(&client, &cache, &installables).await?; + } + } + Command::Pin { cache, paths } => pin_paths(&client, &cache, &paths).await?, + Command::Unpin { cache, paths } => { + for path in store_paths(&paths).await? { + client.unpin(&cache, &path).await?; + tracing::info!("unpinned {path}"); + } + } + Command::Pins { cache } => print(&client.pins(&cache).await?), + Command::Use { cache, print } => { + let plan = substituter::plan(&client, &server, &cache).await?; + if print { plan.print() } else { plan.apply()? } + } + Command::WatchStore { cache, spool, jobs, level, window_log, chunk_size } => { + let (level, window_log) = compression(&client, &cache, level, window_log).await; + let opts = push::Options { jobs, level, window_log, closure: true, chunk_size: (chunk_size << 20) as usize }; + watch::watch(&client, &cache, &spool, opts).await?; + } + Command::Cache(CacheCmd::Create { name, private }) => { + print(&client.admin_post("/admin/caches", json!({ "name": name, "public": !private })).await?); + } + Command::Cache(CacheCmd::List) => print(&client.admin_get("/admin/caches").await?), + Command::Token(TokenCmd::Create { subject, caches, perms, expires }) => { + let v = client + .admin_post( + "/admin/tokens", + json!({ "subject": subject, "caches": split(&caches), "perms": split(&perms), "expiresInDays": expires }), + ) + .await?; + tracing::warn!("store this token now; it is not shown again"); + print(&v); + } + Command::Token(TokenCmd::List) => print(&client.admin_get("/admin/tokens").await?), + Command::Token(TokenCmd::Revoke { jti, reason }) => { + print(&client.admin_post(&format!("/admin/tokens/{jti}/revoke"), json!({ "reason": reason })).await?); + } + } + Ok(()) +} + +fn main() { + // Nix environments point at their CA bundle with NIX_SSL_CERT_FILE. + if std::env::var_os("SSL_CERT_FILE").is_none() + && let Some(file) = std::env::var_os("NIX_SSL_CERT_FILE") + { + // SAFETY: single-threaded; the runtime has not started yet. + unsafe { std::env::set_var("SSL_CERT_FILE", file) }; + } + helios_log::init(helios_log::Style::Cli); + let cli = Cli::parse(); + let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build().expect("starting the tokio runtime"); + if let Err(e) = runtime.block_on(run(cli)) { + tracing::error!("{e:#}"); + std::process::exit(1); + } +} + +/// Store paths as given, or resolved by Nix when any is an installable. +async fn store_paths(args: &[String]) -> anyhow::Result> { + if args.iter().all(|a| a.starts_with("/nix/store/")) { + return Ok(args.to_vec()); + } + Ok(nix::path_infos(args, false).await?.into_iter().map(|i| i.path).collect()) +} + +async fn pin_paths(client: &api::Client, cache: &str, args: &[String]) -> anyhow::Result<()> { + let paths = store_paths(args).await?; + let r = client.pin(cache, &paths).await?; + tracing::info!("pinned {} paths in '{cache}' ({} already were)", r["pinned"], r["alreadyPinned"]); + Ok(()) +} + +fn parse_window_log(s: &str) -> Result { + match s.parse::() { + Ok(n) if n == 0 || (10..=27).contains(&n) => Ok(n), + _ => Err("expected 0, or 10-27 (27, a 128 MiB window, is the largest a stock Nix decoder accepts)".into()), + } +} + +/// The level and window to compress with: what was asked for, else the +/// server's defaults for the cache, else level 3 with a 2^27 window. +async fn compression(client: &api::Client, cache: &str, level: Option, window_log: Option) -> (i32, i32) { + let server = if level.is_none() || window_log.is_none() { + match client.cache_info(cache).await { + Ok(info) => info.compression, + Err(e) => { + tracing::debug!("no compression defaults from the server: {e:#}"); + None + } + } + } else { + None + }; + let level = level.or(server.map(|c| c.level)).unwrap_or(3); + // An uncompressed upload has no window. + let window_log = if level == 0 { 0 } else { window_log.or(server.map(|c| c.window_log)).unwrap_or(27) }; + (level, window_log) +} diff --git a/crates/helios-cli/src/nix.rs b/crates/helios-cli/src/nix.rs new file mode 100644 index 0000000..0628354 --- /dev/null +++ b/crates/helios-cli/src/nix.rs @@ -0,0 +1,240 @@ +//! Store path metadata from `nix path-info`: one process for the whole set. + +use std::collections::BTreeMap; + +use anyhow::{Context, bail}; +use serde::Deserialize; +use tokio::process::Command; + +#[derive(Debug, Clone)] +pub struct PathInfo { + pub path: String, + pub hash: String, + pub nar_hash: String, + pub nar_size: u64, + /// Full store paths. + pub references: Vec, + pub deriver: Option, + /// Set for content-addressed paths, such as CA derivation outputs. + pub ca: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct RawInfo { + #[serde(default)] + path: Option, + nar_hash: String, + nar_size: u64, + #[serde(default)] + references: Vec, + #[serde(default)] + deriver: Option, + #[serde(default)] + ca: Option, +} + +const STORE_DIR: &str = "/nix/store/"; + +fn full(p: &str) -> String { + if p.starts_with('/') { p.to_owned() } else { format!("{STORE_DIR}{p}") } +} + +/// Installables beyond this many go to Nix on stdin rather than argv, +/// which the kernel caps (E2BIG). +const MAX_ARGS: usize = 1_000; + +/// Runs `cmd` with `installables` as arguments, or on stdin (`--stdin`) +/// when there are too many for one command line. +async fn output(mut cmd: Command, installables: &[String]) -> std::io::Result { + use tokio::io::AsyncWriteExt; + cmd.stdout(std::process::Stdio::piped()).stderr(std::process::Stdio::piped()); + if installables.len() <= MAX_ARGS { + // Not our stdin: nothing should wait on whatever the caller left open. + return cmd.arg("--").args(installables).stdin(std::process::Stdio::null()).output().await; + } + let mut child = cmd.arg("--stdin").stdin(std::process::Stdio::piped()).spawn()?; + let mut stdin = child.stdin.take().expect("stdin is piped"); + let input = installables.join("\n") + "\n"; + // Feed stdin while collecting the output, or a full pipe deadlocks both. + let (written, out) = futures_util::future::join( + async move { + let r = stdin.write_all(input.as_bytes()).await; + drop(stdin); + r + }, + child.wait_with_output(), + ) + .await; + let out = out?; + // Nix closes the pipe early when it fails; its stderr says why. + if out.status.success() { + written?; + } + Ok(out) +} + +async fn run(args: &[&str], installables: &[String]) -> anyhow::Result { + let mut cmd = Command::new("nix"); + cmd.args(["--extra-experimental-features", "nix-command flakes", "path-info"]).args(args); + output(cmd, installables).await.context("running nix path-info (is nix on PATH?)") +} + +pub async fn path_infos(installables: &[String], closure: bool) -> anyhow::Result> { + let mut args = vec!["--json", "--json-format", "1"]; + if closure { + args.push("--recursive"); + } + let mut out = run(&args, installables).await?; + if !out.status.success() && String::from_utf8_lossy(&out.stderr).contains("json-format") { + // Older Nix without --json-format. + args.retain(|a| *a != "--json-format" && *a != "1"); + out = run(&args, installables).await?; + } + if !out.status.success() { + bail!("nix path-info failed: {}", String::from_utf8_lossy(&out.stderr).trim()); + } + parse(&out.stdout) +} + +fn parse(stdout: &[u8]) -> anyhow::Result> { + let value: serde_json::Value = serde_json::from_slice(stdout).context("parsing nix path-info output")?; + let mut raw: BTreeMap = BTreeMap::new(); + match value { + serde_json::Value::Object(map) => { + for (path, info) in map { + if info.is_null() { + bail!("{path} is not a valid store path"); + } + raw.insert(full(&path), serde_json::from_value(info)?); + } + } + serde_json::Value::Array(items) => { + for item in items { + let info: RawInfo = serde_json::from_value(item)?; + let path = info.path.clone().context("path-info entry without a path")?; + raw.insert(full(&path), info); + } + } + _ => bail!("unexpected nix path-info output"), + } + raw.into_iter() + .map(|(path, info)| { + let base = path.strip_prefix(STORE_DIR).with_context(|| format!("{path} is not in /nix/store"))?; + let hash = base.get(..32).with_context(|| format!("{path} has no store hash"))?.to_owned(); + Ok(PathInfo { + hash, + nar_hash: info.nar_hash, + nar_size: info.nar_size, + references: info.references.iter().map(|r| full(r)).collect(), + deriver: info.deriver.map(|d| full(&d)), + ca: info.ca, + path, + }) + }) + .collect() +} + +/// Dependencies before dependents, so a partially pushed closure never +/// exposes a path whose references are missing. +pub fn topo_order(infos: Vec) -> Vec { + use std::collections::HashMap; + let index: HashMap = infos.iter().enumerate().map(|(i, p)| (p.path.clone(), i)).collect(); + let mut pending: Vec = vec![0; infos.len()]; + let mut dependents: Vec> = vec![Vec::new(); infos.len()]; + for (i, info) in infos.iter().enumerate() { + for r in &info.references { + if let Some(&j) = index.get(r) + && j != i + { + pending[i] += 1; + dependents[j].push(i); + } + } + } + let mut ready: Vec = (0..infos.len()).filter(|&i| pending[i] == 0).collect(); + let mut order = Vec::with_capacity(infos.len()); + while let Some(i) = ready.pop() { + order.push(i); + for &d in &dependents[i] { + pending[d] -= 1; + if pending[d] == 0 { + ready.push(d); + } + } + } + // Cycles cannot occur in a valid store; keep anything left over anyway. + let placed: std::collections::HashSet = order.iter().copied().collect(); + order.extend((0..infos.len()).filter(|i| !placed.contains(i))); + let mut slots: Vec> = infos.into_iter().map(Some).collect(); + order.into_iter().map(|i| slots[i].take().expect("each index placed once")).collect() +} + +/// Build trace entries (realisations) for these derivations' outputs, as +/// this Nix prints them: `{"key", "value"}` from 2.35 on, `{"id", "outPath"}` +/// before. Empty when this Nix has CA derivations disabled. +pub async fn build_traces(derivers: &[String]) -> anyhow::Result> { + let outputs: Vec = derivers.iter().map(|d| format!("{d}^*")).collect(); + let mut out = None; + // `store build-trace` is the current name; `realisation` the old one. + for sub in [&["store", "build-trace", "info"][..], &["realisation", "info"][..]] { + let mut cmd = Command::new("nix"); + cmd.args(["--extra-experimental-features", "nix-command ca-derivations"]).args(sub).arg("--json"); + let o = output(cmd, &outputs).await.context("running nix store build-trace info")?; + if o.status.success() { + out = Some(o); + break; + } + tracing::debug!("nix {}: {}", sub.join(" "), String::from_utf8_lossy(&o.stderr).trim()); + } + let Some(out) = out else { return Ok(Vec::new()) }; + let entries: Vec = serde_json::from_slice(&out.stdout).context("parsing build trace entries")?; + // Store paths given as installables come back as `{"opaquePath"}`. + Ok(entries.into_iter().filter(|e| e.get("key").is_some() || e.get("id").is_some()).collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn info(name: &str, refs: &[&str]) -> PathInfo { + PathInfo { + path: format!("{STORE_DIR}{name}"), + hash: String::new(), + nar_hash: String::new(), + nar_size: 0, + references: refs.iter().map(|r| format!("{STORE_DIR}{r}")).collect(), + deriver: None, + ca: None, + } + } + + #[test] + fn dependencies_come_first() { + let order = topo_order(vec![info("app", &["lib", "app"]), info("lib", &["libc"]), info("libc", &[])]); + let names: Vec<_> = order.iter().map(|p| p.path.trim_start_matches(STORE_DIR)).collect(); + assert_eq!(names, ["libc", "lib", "app"]); + } + + #[test] + fn parses_both_json_formats() { + let obj = br#"{"/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-x":{"narHash":"sha256-AAAA","narSize":8,"references":[],"deriver":null}}"#; + assert_eq!(parse(obj).unwrap()[0].hash, "a".repeat(32)); + let arr = br#"[{"path":"/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-y","narHash":"sha256:x","narSize":8,"references":["/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-y"]}]"#; + assert_eq!(parse(arr).unwrap()[0].references.len(), 1); + } + + #[test] + fn many_installables_go_on_stdin() { + let rt = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + let run = |n: usize| { + let mut cmd = Command::new("sh"); + cmd.args(["-c", r#"printf '%s ' "$@"; cat | wc -l"#, "sh"]); + let args: Vec = (0..n).map(|i| format!("p{i}")).collect(); + let out = rt.block_on(output(cmd, &args)).unwrap(); + String::from_utf8(out.stdout).unwrap() + }; + assert_eq!(run(2).split_whitespace().collect::>(), ["--", "p0", "p1", "0"]); + assert_eq!(run(MAX_ARGS + 1).split_whitespace().collect::>(), ["--stdin", &(MAX_ARGS + 1).to_string()]); + } +} diff --git a/crates/helios-cli/src/push.rs b/crates/helios-cli/src/push.rs new file mode 100644 index 0000000..ac8ff03 --- /dev/null +++ b/crates/helios-cli/src/push.rs @@ -0,0 +1,375 @@ +//! `helios push`: NARs are serialised and compressed by libhelios and +//! streamed straight into the upload request, with no temp files and no +//! `nix store dump-path | zstd` subprocesses. The server hashes what it +//! receives, so the client does not hash again. + +use std::collections::HashSet; +use std::path::Path; +use std::time::Instant; + +use anyhow::{Context, bail}; +use base64::Engine; +use bytes::{Bytes, BytesMut}; +use futures_util::StreamExt; +use helios_core::{DumpOptions, HL_E_UNSUPPORTED_OS}; +use tokio::sync::mpsc; + +use crate::api::{Client, PathSpec, Publish, Uploaded}; +use crate::nix::{self, PathInfo}; + +const QUERY_BATCH: usize = 50_000; +const PUBLISH_BATCH: usize = 1_000; + +#[derive(Clone, Copy)] +pub struct Options { + pub jobs: usize, + pub level: i32, + /// zstd window for long-distance matching (10-27), or 0 for the level's. + pub window_log: i32, + pub closure: bool, + /// Bytes per chunked-upload request; 0 streams each NAR in one request. + pub chunk_size: usize, +} + +fn parse_sha256(s: &str) -> Option<[u8; 32]> { + if let Some(b64) = s.strip_prefix("sha256-") { + return base64::engine::general_purpose::STANDARD.decode(b64).ok()?.try_into().ok(); + } + helios_core::nix32_decode::<32>(s.strip_prefix("sha256:")?) +} + +fn name(path: &str) -> &str { + path.strip_prefix("/nix/store/").and_then(|b| b.get(33..)).unwrap_or(path) +} + +fn human(bytes: u64) -> String { + const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; + let mut v = bytes as f64; + let mut unit = 0; + while v >= 1024.0 && unit < UNITS.len() - 1 { + v /= 1024.0; + unit += 1; + } + if unit == 0 { format!("{bytes} B") } else { format!("{v:.1} {}", UNITS[unit]) } +} + +/// Produce the compressed NAR on a blocking thread, feeding `tx`. +fn produce(path: &str, opts: DumpOptions, tx: mpsc::Sender>) -> anyhow::Result { + let send = |chunk: &[u8]| tx.blocking_send(Ok(Bytes::copy_from_slice(chunk))).is_ok(); + match helios_core::dump_nar(Path::new(path), &opts, send) { + Err(e) if e.code == HL_E_UNSUPPORTED_OS => {} + other => return other.with_context(|| format!("serialising {path}")), + } + // Non-Linux: let Nix serialise, libhelios compresses. + use std::io::Read; + let mut child = std::process::Command::new("nix-store") + .args(["--dump", path]) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::piped()) + .spawn() + .context("running nix-store --dump")?; + let mut stdout = child.stdout.take().context("nix-store stdout")?; + let mut compressor = helios_core::Compressor::new(&opts, send)?; + let mut buf = vec![0u8; 1 << 20]; + loop { + let n = stdout.read(&mut buf)?; + if n == 0 { + break; + } + compressor.update(&buf[..n])?; + } + if !child.wait()?.success() { + bail!("nix-store --dump {path} failed"); + } + Ok(compressor.finish()?) +} + +/// Sends the compressed NAR arriving on `rx` in `chunk_size` pieces, or in +/// one request if it fits in one chunk. +async fn send_chunked( + client: &Client, + cache: &str, + compression: &str, + rx: &mut mpsc::Receiver>, + chunk_size: usize, +) -> anyhow::Result { + let mut buf = BytesMut::new(); + let mut session: Option = None; + let mut offset = 0u64; + let result = async { + loop { + let done = match rx.recv().await { + Some(chunk) => { + buf.extend_from_slice(&chunk?); + false + } + None => true, + }; + if done && session.is_none() && buf.len() <= chunk_size { + return client.upload(cache, compression, crate::transport::full(buf.split().freeze())).await; + } + while buf.len() >= chunk_size || (done && !buf.is_empty()) { + let chunk = buf.split_to(buf.len().min(chunk_size)).freeze(); + let id = match &session { + Some(id) => id.clone(), + None => session.insert(client.upload_create(cache, compression).await?).clone(), + }; + offset = client.upload_append(cache, &id, offset, chunk).await?; + } + if done { + let id = session.as_deref().context("chunked upload without a session")?; + return client.upload_complete(cache, id).await; + } + } + } + .await; + if let (Err(_), Some(id)) = (&result, &session) { + client.upload_abort(cache, id).await; + } + result +} + +async fn upload_one(client: &Client, cache: &str, info: &PathInfo, opts: DumpOptions, chunk_size: usize) -> anyhow::Result { + let compression = if opts.level == 0 { "none" } else { "zstd" }; + let (tx, mut rx) = mpsc::channel(8); + let path = info.path.clone(); + // Also reports whether the upload had already stopped reading, which + // tells a cause (local serialisation failure) from an effect. + let producer = tokio::task::spawn_blocking(move || { + let digest = produce(&path, opts, tx.clone()); + (digest, tx.is_closed()) + }); + let uploaded = if chunk_size == 0 { + let chunks = futures_util::stream::unfold(rx, |mut rx| async move { rx.recv().await.map(|c| (c, rx)) }); + client.upload(cache, compression, crate::transport::stream(chunks)).await + } else { + let uploaded = send_chunked(client, cache, compression, &mut rx, chunk_size).await; + drop(rx); // Stops the producer if the upload gave up early. + uploaded + }; + let (digest, upload_stopped) = producer.await?; + // A failed upload aborts the producer, so then the upload's error is + // the cause; a failed producer cuts the stream short, and the server's + // complaint about it is only the effect. + let (uploaded, digest) = match (uploaded, digest) { + (Err(e), Err(_)) if upload_stopped => return Err(e), + (_, Err(e)) => return Err(e), + (uploaded, Ok(digest)) => (uploaded?, digest), + }; + + // The NAR is not hashed here: the server hashes what it received, and + // that is checked against the Nix database, which still catches a + // corrupt local store (the NAR is then never published). + let verified = parse_sha256(&uploaded.nar_hash); + if verified.is_none() || verified != parse_sha256(&info.nar_hash) { + bail!("{}: NAR hash {} differs from the Nix database ({})", info.path, uploaded.nar_hash, info.nar_hash); + } + if uploaded.nar_size != digest.nar_size || uploaded.file_size != digest.file_size { + bail!("{}: server verified different content than was sent", info.path); + } + Ok(uploaded.file_size) +} + +/// Uploads `paths`, returning the bytes sent and the paths that failed. +async fn upload_all(client: &Client, cache: &str, paths: &[&PathInfo], opts: &Options) -> (u64, HashSet) { + let total = paths.len(); + let threads = (std::thread::available_parallelism().map_or(4, |n| n.get()) / opts.jobs.max(1)).max(1) as i32; + let mut done = 0usize; + let mut bytes = 0u64; + let mut failures = HashSet::new(); + let mut results = futures_util::stream::iter(paths.iter().map(|info| async move { + let dump = DumpOptions { level: opts.level, threads, nar_size: info.nar_size, window_log: opts.window_log, hash: false }; + let started = Instant::now(); + (info, upload_one(client, cache, info, dump, opts.chunk_size).await, started.elapsed()) + })) + .buffer_unordered(opts.jobs.max(1)); + while let Some((info, result, took)) = results.next().await { + done += 1; + match result { + Ok(size) => { + bytes += size; + tracing::info!("[{done}/{total}] {} {} → {} ({:.2}s)", name(&info.path), human(info.nar_size), human(size), took.as_secs_f64()); + } + Err(e) => { + tracing::error!("[{done}/{total}] {} failed: {e:#}", name(&info.path)); + failures.insert(info.path.clone()); + } + } + } + (bytes, failures) +} + +/// Drops from `batch` the paths in `bad` and those referring to one, +/// adding the latter to `bad`. `batch` comes dependencies first, so one +/// pass reaches every dependent, here and in later batches. +fn without_bad<'a>(batch: &'a [PathInfo], bad: &mut HashSet) -> Vec<&'a PathInfo> { + let mut kept = Vec::with_capacity(batch.len()); + for info in batch { + if bad.contains(&info.path) || info.references.iter().any(|r| *r != info.path && bad.contains(r)) { + bad.insert(info.path.clone()); + } else { + kept.push(info); + } + } + kept +} + +pub async fn push(client: &Client, cache: &str, installables: &[String], opts: Options) -> anyhow::Result<()> { + let infos = nix::path_infos(installables, opts.closure).await?; + // Outputs of content-addressed derivations need build traces as well, + // or Nix cannot tell which paths a CA derivation produced. + let derivers: Vec = + infos.iter().filter(|i| i.ca.is_some()).filter_map(|i| i.deriver.clone()).collect::>().into_iter().collect(); + let mut paths: HashSet = infos.iter().map(|i| i.path.clone()).collect(); + let total = paths.len(); + let unpublished = push_infos(client, cache, infos, opts).await?; + paths.retain(|p| !unpublished.contains(p)); + if !derivers.is_empty() { + push_build_traces(client, cache, &derivers, &paths).await?; + } + if !unpublished.is_empty() { + bail!("{} of {total} paths not pushed: they, or paths they refer to, failed to upload", unpublished.len()); + } + Ok(()) +} + +async fn push_build_traces(client: &Client, cache: &str, derivers: &[String], pushed: &HashSet) -> anyhow::Result<()> { + let entries = match nix::build_traces(derivers).await { + Ok(entries) => entries, + Err(e) => { + tracing::warn!("not pushing build traces: {e:#}"); + return Ok(()); + } + }; + // Only outputs this push published; the server refuses the rest. + let entries: Vec = entries + .into_iter() + .filter(|e| { + // `value.outPath` from Nix 2.35 on, `outPath` before. + e["value"]["outPath"] + .as_str() + .or(e["outPath"].as_str()) + .is_some_and(|p| pushed.contains(&if p.starts_with('/') { p.to_owned() } else { format!("/nix/store/{p}") })) + }) + .collect(); + let mut published = 0; + for chunk in entries.chunks(PUBLISH_BATCH) { + published += client.publish_build_traces(cache, chunk).await?["published"].as_u64().unwrap_or(0); + } + if !entries.is_empty() { + tracing::info!("published {published} build traces to '{cache}' ({} already there)", entries.len() as u64 - published); + } + Ok(()) +} + +/// Uploads and publishes what is missing. A path that fails to upload is +/// left out with everything referring to it, and the rest is published; +/// returns the paths left out. +async fn push_infos(client: &Client, cache: &str, infos: Vec, opts: Options) -> anyhow::Result> { + let started = Instant::now(); + let total = infos.len(); + + let mut missing: HashSet = HashSet::new(); + let hashes: Vec<&str> = infos.iter().map(|i| i.hash.as_str()).collect(); + for chunk in hashes.chunks(QUERY_BATCH) { + missing.extend(client.missing(cache, chunk).await?); + } + let todo: Vec = infos.into_iter().filter(|i| missing.contains(&i.hash)).collect(); + if todo.is_empty() { + tracing::info!("all {total} paths already in '{cache}'"); + return Ok(HashSet::new()); + } + + // Paths whose NAR the server already holds (from any cache) skip the upload. + let mut known: HashSet = HashSet::new(); + let nar_hashes: Vec<&str> = todo.iter().map(|i| i.nar_hash.as_str()).collect(); + for chunk in nar_hashes.chunks(QUERY_BATCH) { + known.extend(client.known(cache, chunk).await?); + } + let uploads: Vec<&PathInfo> = todo.iter().filter(|i| !known.contains(&i.nar_hash)).collect(); + tracing::info!("{total} paths, {} missing from '{cache}': uploading {}, reusing {} NARs", todo.len(), uploads.len(), todo.len() - uploads.len()); + let (mut uploaded_bytes, mut bad) = upload_all(client, cache, &uploads, &opts).await; + + let ordered = nix::topo_order(todo); + let mut published = 0u64; + for batch in ordered.chunks(PUBLISH_BATCH) { + let mut retried = false; + loop { + let kept = without_bad(batch, &mut bad); + if kept.is_empty() { + break; + } + let specs: Vec = kept + .iter() + .map(|i| PathSpec { + store_path: i.path.clone(), + nar_hash: i.nar_hash.clone(), + nar_size: i.nar_size, + references: i.references.clone(), + deriver: i.deriver.clone(), + }) + .collect(); + match client.publish(cache, &specs).await? { + Publish::Done { published: n } => { + published += n; + break; + } + // A NAR vanished between the check and the publish (e.g. GC); upload and retry once. + Publish::MissingNars(paths) if !retried => { + retried = true; + let wanted: HashSet<&str> = paths.iter().map(String::as_str).collect(); + let redo: Vec<&PathInfo> = kept.into_iter().filter(|i| wanted.contains(i.path.as_str())).collect(); + let (bytes, failed) = upload_all(client, cache, &redo, &opts).await; + uploaded_bytes += bytes; + bad.extend(failed); + } + Publish::MissingNars(paths) => { + let wanted: HashSet = paths.into_iter().collect(); + let lacking: Vec = kept.iter().filter(|i| wanted.contains(&i.path)).map(|i| i.path.clone()).collect(); + // Each round drops at least one path, so this ends. + if lacking.is_empty() { + bail!("server lacks NARs for {} paths not in this batch", wanted.len()); + } + tracing::error!("server still lacks NARs for {} paths; leaving them out", lacking.len()); + bad.extend(lacking); + } + } + } + } + + let secs = started.elapsed().as_secs_f64(); + tracing::info!( + "published {published} paths to '{cache}', uploaded {} in {secs:.2}s ({}/s)", + human(uploaded_bytes), + human((uploaded_bytes as f64 / secs.max(1e-3)) as u64) + ); + Ok(bad) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn info(name: &str, refs: &[&str]) -> PathInfo { + PathInfo { + path: name.to_owned(), + hash: String::new(), + nar_hash: String::new(), + nar_size: 0, + references: refs.iter().map(|r| (*r).to_owned()).collect(), + deriver: None, + ca: None, + } + } + + #[test] + fn failed_paths_hold_back_their_dependents() { + let first = [info("libc", &["libc"]), info("lib", &["libc"]), info("other", &[])]; + let second = [info("app", &["lib", "other"]), info("tool", &["other"])]; + let mut bad: HashSet = ["lib".to_owned()].into(); + let names = |v: Vec<&PathInfo>| v.into_iter().map(|i| i.path.clone()).collect::>(); + assert_eq!(names(without_bad(&first, &mut bad)), ["libc", "other"]); + assert_eq!(names(without_bad(&second, &mut bad)), ["tool"]); + assert!(bad.contains("app")); + } +} diff --git a/crates/helios-cli/src/substituter.rs b/crates/helios-cli/src/substituter.rs new file mode 100644 index 0000000..2c0e34b --- /dev/null +++ b/crates/helios-cli/src/substituter.rs @@ -0,0 +1,344 @@ +//! `helios use `: configure Nix to substitute from a cache. +//! +//! The substituter URL is the one given to `helios login`, never an address +//! the server reports about itself: behind a proxy the server does not know +//! the address clients reach it at (compare zhaofengli/attic#325). + +use std::io::Write; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, bail}; +use serde::Deserialize; + +use crate::api::Client; +use crate::config::Server; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CacheInfo { + pub public: bool, + pub public_key: Option, + /// The server's compression defaults; absent from older servers. + #[serde(default)] + pub compression: Option, +} + +#[derive(Deserialize, Clone, Copy)] +#[serde(rename_all = "camelCase")] +pub struct Compression { + pub level: i32, + pub window_log: i32, +} + +pub struct Plan { + url: String, + public_key: Option, + /// `(host, token)` for private caches, which Nix reads from netrc. + netrc: Option<(String, String)>, +} + +pub async fn plan(client: &Client, server: &Server, cache: &str) -> anyhow::Result { + let info = client.cache_info(cache).await?; + let netrc = (!info.public).then(|| Ok::<_, anyhow::Error>((host(&server.server)?.to_owned(), server.token.clone()))).transpose()?; + Ok(Plan { url: format!("{}/{cache}", server.server), public_key: info.public_key, netrc }) +} + +/// The netrc `machine` for a URL: its host, without scheme, port or path. +fn host(url: &str) -> anyhow::Result<&str> { + let rest = url.split_once("://").map_or(url, |(_, r)| r); + let authority = rest.split('/').next().unwrap_or(rest); + let host = authority.rsplit_once('@').map_or(authority, |(_, h)| h); + let host = match host.strip_prefix('[') { + Some(v6) => v6.split(']').next().unwrap_or(v6), + None => host.split(':').next().unwrap_or(host), + }; + if host.is_empty() { + bail!("no host in server URL {url}"); + } + Ok(host) +} + +fn nix_config_dir() -> anyhow::Result { + let base = match std::env::var_os("XDG_CONFIG_HOME") { + Some(dir) if !dir.is_empty() => PathBuf::from(dir), + _ => PathBuf::from(std::env::var_os("HOME").context("HOME is not set")?).join(".config"), + }; + Ok(base.join("nix")) +} + +impl Plan { + fn conf_lines(&self, netrc_file: Option<&Path>) -> Vec { + let mut lines = vec![format!("extra-substituters = {}", self.url)]; + if let Some(key) = &self.public_key { + lines.push(format!("extra-trusted-public-keys = {key}")); + } + if let (Some(_), Some(file)) = (&self.netrc, netrc_file) { + lines.push(format!("netrc-file = {}", file.display())); + } + lines + } + + /// Settings for a NixOS configuration or `/etc/nix/nix.conf`, which + /// apply to every user. + pub fn print(&self) { + println!("# nix.conf"); + for line in self.conf_lines(self.netrc.as_ref().map(|_| Path::new("/etc/nix/netrc"))) { + println!("{line}"); + } + println!("\n# NixOS"); + println!("nix.settings = {{"); + println!(" extra-substituters = [ \"{}\" ];", self.url); + if let Some(key) = &self.public_key { + println!(" extra-trusted-public-keys = [ \"{key}\" ];"); + } + if self.netrc.is_some() { + println!(" netrc-file = \"/etc/nix/netrc\"; # holding the netrc entry below"); + } + println!("}};"); + if let Some((host, token)) = &self.netrc { + println!("\n# netrc (keep it readable by root only)\nmachine {host}\nlogin helios\npassword {token}"); + } + } + + /// Adds the settings to the user's nix.conf (and netrc), leaving lines + /// already there alone. + pub fn apply(&self) -> anyhow::Result<()> { + let dir = nix_config_dir()?; + std::fs::create_dir_all(&dir)?; + let conf = dir.join("nix.conf"); + let existing = + std::fs::read_to_string(&conf).or_else(|e| if e.kind() == std::io::ErrorKind::NotFound { Ok(String::new()) } else { Err(e) })?; + + let mut netrc_file = None; + if let Some((host, token)) = &self.netrc { + netrc_file = self.write_netrc(&dir, &existing, host, token)?; + } + + let missing: Vec = self + .conf_lines(netrc_file.as_deref()) + .into_iter() + .filter(|line| { + let key = line.split('=').next().unwrap_or("").trim(); + // netrc-file is single-valued: keep the one that is there. + !existing.lines().any(|l| l.trim() == line || (key == "netrc-file" && l.trim().starts_with("netrc-file"))) + }) + .collect(); + if !missing.is_empty() { + let mut out = std::fs::OpenOptions::new().append(true).create(true).open(&conf)?; + if !existing.is_empty() && !existing.ends_with('\n') { + writeln!(out)?; + } + for line in &missing { + writeln!(out, "{line}")?; + } + } + tracing::info!("configured {} in {}", self.url, conf.display()); + if self.public_key.is_none() { + tracing::warn!("the server has no signing key; Nix refuses its paths unless require-sigs is off"); + } + if !nix_trusts_user() { + tracing::warn!( + "the Nix daemon does not trust this user, so it ignores substituters set here; \ + add them system-wide instead (helios use --print shows how) or list this user in trusted-users" + ); + } + Ok(()) + } + + /// Adds the token to the netrc file Nix reads. Nix reads only one, so + /// pointing it at a new one would hide the entries of the one it reads + /// now (by default `/etc/nix/netrc`). Returns the file to name in the + /// user's nix.conf, if any. + fn write_netrc(&self, dir: &Path, conf: &str, host: &str, token: &str) -> anyhow::Result> { + // One set in the user's nix.conf already is the one. + if let Some(file) = conf.lines().find_map(conf_netrc_file) { + add_netrc(&file, None, host, token)?; + tracing::info!("added the token for {host} to {}", file.display()); + return Ok(Some(file)); + } + let effective = effective_netrc_file(); + if let Some(file) = &effective + && std::fs::OpenOptions::new().append(true).open(file).is_ok() + { + add_netrc(file, None, host, token)?; + tracing::info!("added the token for {host} to {}", file.display()); + return Ok(None); + } + let own = dir.join("netrc"); + let mut base = None; + if let Some(file) = effective.filter(|f| *f != own) { + match std::fs::read_to_string(&file) { + Ok(text) if !own.exists() => { + tracing::info!("copying the entries of {} to {}, which Nix reads instead from now on", file.display(), own.display()); + base = Some(text); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Ok(_) => { + tracing::warn!("Nix reads {} instead of {} from now on; copy over any entries it still needs", own.display(), file.display()) + } + Err(e) => tracing::warn!( + "Nix reads {} instead of {} from now on, which could not be read to copy its entries over ({e}); \ + copy any it still needs by hand", + own.display(), + file.display() + ), + } + } + add_netrc(&own, base, host, token)?; + tracing::info!("added the token for {host} to {}", own.display()); + Ok(Some(own)) + } +} + +/// Sets the token for `host` in a netrc file. `base` is the content to +/// start from instead of the file's own, when the file is new. +fn add_netrc(file: &Path, base: Option, host: &str, token: &str) -> anyhow::Result<()> { + let existing = match base { + Some(base) => base, + None => match std::fs::read_to_string(file) { + Ok(text) => text, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(e) => return Err(e).with_context(|| format!("reading {}", file.display())), + }, + }; + crate::config::write_secret(file, edit_netrc(&existing, host, token).as_bytes()) +} + +/// `netrc` with the entry for `machine host` set to the token. The old +/// entry's lines go (its usual forms: on one line, or `login`, `password` +/// and `account` on lines of their own), the new entry takes their place, +/// or comes before any `default` entry, which ends the entries curl reads. +/// Every other line, with comments, macros and quoting, stays as it was. +fn edit_netrc(netrc: &str, host: &str, token: &str) -> String { + let mut kept: Vec<&str> = Vec::new(); + let mut at: Option = None; + let (mut removing, mut in_macro) = (false, false); + for line in netrc.split_inclusive('\n') { + let words: Vec<&str> = line.split_whitespace().collect(); + if in_macro { + // A macro body runs to the next empty line. + if words.is_empty() { + (in_macro, removing) = (false, false); + kept.push(line); + } else if !removing { + kept.push(line); + } + continue; + } + match words.first().copied() { + None => {} + Some(w) if w.starts_with('#') => {} + Some(first @ ("machine" | "default")) => { + removing = false; + if first == "default" { + at.get_or_insert(kept.len()); + } + // Only an entry alone on its line; one sharing a line with + // others is left alone. + let ours = first == "machine" && words.get(1) == Some(&host) && !words[2..].iter().any(|w| matches!(*w, "machine" | "default")); + if ours { + at.get_or_insert(kept.len()); + removing = true; + in_macro = words.contains(&"macdef"); + continue; + } + } + Some("login" | "password" | "account" | "macdef") if removing => { + in_macro = words.contains(&"macdef"); + continue; + } + Some(_) => removing = false, + } + in_macro = words.contains(&"macdef") && !words[0].starts_with('#'); + kept.push(line); + } + let at = at.unwrap_or(kept.len()); + let mut out: String = kept[..at].concat(); + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } + out.push_str(&format!("machine {host} login helios password {token}\n")); + out.push_str(&kept[at..].concat()); + out +} + +/// The netrc file Nix reads, as `nix config show` reports it with every +/// configuration file applied. +fn effective_netrc_file() -> Option { + let nix = |args: &[&str]| { + let out = std::process::Command::new("nix") + .args(["--extra-experimental-features", "nix-command"]) + .args(args) + .stderr(std::process::Stdio::null()) + .stdin(std::process::Stdio::null()) + .output() + .ok()?; + out.status.success().then(|| String::from_utf8_lossy(&out.stdout).into_owned()) + }; + let value = match nix(&["config", "show", "netrc-file"]) { + Some(v) => v.trim().to_owned(), + // Before `nix config show`. + None => nix(&["show-config"])?.lines().find_map(conf_netrc_file)?.to_string_lossy().into_owned(), + }; + (!value.is_empty()).then(|| PathBuf::from(value)) +} + +/// The value of a `netrc-file = ...` line. +fn conf_netrc_file(line: &str) -> Option { + line.trim().strip_prefix("netrc-file")?.trim_start().strip_prefix('=').map(|v| PathBuf::from(v.trim())) +} + +fn nix_trusts_user() -> bool { + let Ok(out) = std::process::Command::new("nix") + .args(["--extra-experimental-features", "nix-command", "store", "info", "--json"]) + .stdin(std::process::Stdio::null()) + .output() + else { + return true; // No way to tell; do not warn. + }; + serde_json::from_slice::(&out.stdout).map_or(true, |v| v["trusted"] != false && v["trusted"] != 0) +} + +#[cfg(test)] +mod tests { + use super::{edit_netrc, host}; + + #[test] + fn hosts() { + assert_eq!(host("https://cache.example.com").unwrap(), "cache.example.com"); + assert_eq!(host("http://127.0.0.1:8080/").unwrap(), "127.0.0.1"); + assert_eq!(host("http://[::1]:8080").unwrap(), "::1"); + assert_eq!(host("https://u@cache.example.com:443/x").unwrap(), "cache.example.com"); + assert!(host("http://").is_err()); + } + + #[test] + fn netrc_entries_go_before_default() { + let netrc = "machine a login x password y\ndefault login anonymous password me@example.com\n"; + assert_eq!( + edit_netrc(netrc, "h", "T"), + "machine a login x password y\nmachine h login helios password T\ndefault login anonymous password me@example.com\n" + ); + // An entry stranded after `default` by an older helios moves up. + let stale = "default login x password y\nmachine h login helios password old\n"; + assert_eq!(edit_netrc(stale, "h", "T"), "machine h login helios password T\ndefault login x password y\n"); + assert_eq!(edit_netrc("", "h", "T"), "machine h login helios password T\n"); + assert_eq!(edit_netrc("machine a login x password y", "h", "T"), "machine a login x password y\nmachine h login helios password T\n"); + } + + #[test] + fn netrc_edits_leave_other_lines_alone() { + let netrc = "# tokens\nmachine h\n login old\n # rotated yearly\n password old\n\nmachine o login l password \"a b\"\n"; + assert_eq!( + edit_netrc(netrc, "h", "T"), + "# tokens\nmachine h login helios password T\n # rotated yearly\n\nmachine o login l password \"a b\"\n" + ); + let macros = "machine o login l password p macdef init\nmachine h is not an entry here\n\nmachine h login a password b\n"; + assert_eq!( + edit_netrc(macros, "h", "T"), + "machine o login l password p macdef init\nmachine h is not an entry here\n\nmachine h login helios password T\n" + ); + let own_macro = "machine h login a password b\nmacdef init\ncd /x\n\nmachine o login l password p\n"; + assert_eq!(edit_netrc(own_macro, "h", "T"), "machine h login helios password T\n\nmachine o login l password p\n"); + } +} diff --git a/crates/helios-cli/src/transport.rs b/crates/helios-cli/src/transport.rs new file mode 100644 index 0000000..4d2cc5d --- /dev/null +++ b/crates/helios-cli/src/transport.rs @@ -0,0 +1,339 @@ +//! A small HTTP/1.1 client for one Helios server: hyper over TCP, rustls +//! (ring) for https, system CA roots, and a keep-alive connection pool. +//! It replaces reqwest, which pulled in URL/IDNA handling, proxy support +//! and aws-lc-rs that a CLI talking to one configured server never needs. + +use std::io; +use std::net::IpAddr; +use std::pin::Pin; +use std::sync::{Arc, Mutex, PoisonError}; +use std::task::{Context, Poll}; +use std::time::Duration; + +use anyhow::{Context as _, bail}; +use bytes::Bytes; +use http::{HeaderValue, Method, Request, Response, header}; +use http_body_util::combinators::UnsyncBoxBody; +use http_body_util::{BodyExt, Full, StreamBody}; +use hyper::body::{Frame, Incoming}; +use hyper::client::conn::http1::SendRequest; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio::net::{TcpSocket, TcpStream}; +use tokio::time::{Instant, Sleep}; + +pub type Body = UnsyncBoxBody; + +pub fn full(bytes: impl Into) -> Body { + Full::new(bytes.into()).map_err(|never| match never {}).boxed_unsync() +} + +pub fn stream(s: S) -> Body +where + S: futures_util::Stream> + Send + 'static, +{ + use futures_util::StreamExt; + StreamBody::new(s.map(|r| r.map(Frame::data))).boxed_unsync() +} + +struct Target { + tls: bool, + host: String, + port: u16, + /// Path prefix without a trailing slash, e.g. "" or "/helios". + prefix: String, +} + +fn parse_base(url: &str) -> anyhow::Result { + let (tls, rest) = if let Some(r) = url.strip_prefix("https://") { + (true, r) + } else if let Some(r) = url.strip_prefix("http://") { + (false, r) + } else { + bail!("server URL must start with http:// or https://: {url}"); + }; + let (authority, path) = rest.split_once('/').map_or((rest, ""), |(a, p)| (a, p)); + let (host, port) = if let Some(v6) = authority.strip_prefix('[') { + let (h, after) = v6.split_once(']').context("unterminated IPv6 address")?; + (h.to_owned(), after.strip_prefix(':')) + } else { + match authority.rsplit_once(':') { + Some((h, p)) => (h.to_owned(), Some(p)), + None => (authority.to_owned(), None), + } + }; + if host.is_empty() { + bail!("server URL has no host: {url}"); + } + let port = match port { + Some(p) => p.parse().with_context(|| format!("invalid port in {url}"))?, + None if tls => 443, + None => 80, + }; + let prefix = format!("/{}", path.trim_matches('/')); + Ok(Target { tls, host, port, prefix: if prefix == "/" { String::new() } else { prefix } }) +} + +fn tls_config() -> anyhow::Result> { + let mut roots = rustls::RootCertStore::empty(); + let native = rustls_native_certs::load_native_certs(); + for cert in native.certs { + let _ = roots.add(cert); + } + if roots.is_empty() { + bail!("no CA certificates found (set SSL_CERT_FILE or NIX_SSL_CERT_FILE)"); + } + let config = rustls::ClientConfig::builder_with_provider(Arc::new(rustls::crypto::ring::default_provider())) + .with_safe_default_protocol_versions()? + .with_root_certificates(roots) + .with_no_client_auth(); + Ok(Arc::new(config)) +} + +/// hyper's I/O traits over a tokio stream. +struct Io(T); + +impl hyper::rt::Read for Io { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, mut buf: hyper::rt::ReadBufCursor<'_>) -> Poll> { + // SAFETY: tokio's ReadBuf only writes initialised bytes into the + // uninitialised tail, and we advance by exactly what was filled. + let filled = unsafe { + let mut rb = ReadBuf::uninit(buf.as_mut()); + match Pin::new(&mut self.0).poll_read(cx, &mut rb) { + Poll::Ready(Ok(())) => rb.filled().len(), + other => return other, + } + }; + unsafe { buf.advance(filled) }; + Poll::Ready(Ok(())) + } +} + +impl hyper::rt::Write for Io { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + Pin::new(&mut self.0).poll_write(cx, buf) + } + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.0).poll_flush(cx) + } + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.0).poll_shutdown(cx) + } +} + +/// Connecting, including the TLS and HTTP handshakes. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(30); +/// Connecting to one of the host's addresses, before trying the next. +const ADDRESS_TIMEOUT: Duration = Duration::from_secs(10); +/// A connection that moves no bytes either way for this long is dead (a +/// half-open connection otherwise hangs forever). Large uploads are fine: +/// they keep making progress. +const IDLE_TIMEOUT: Duration = Duration::from_secs(300); + +/// A stream that fails reads and writes once it has made no progress in +/// either direction for `limit`. +struct Idle { + inner: T, + limit: Duration, + timer: Pin>, + last: Instant, +} + +impl Idle { + fn new(inner: T, limit: Duration) -> Self { + Self { inner, limit, timer: Box::pin(tokio::time::sleep(limit)), last: Instant::now() } + } + + /// Records progress, or checks the deadline when there was none. + fn track(&mut self, cx: &mut Context<'_>, poll: Poll>) -> Poll> { + if poll.is_ready() { + self.last = Instant::now(); + return poll; + } + loop { + if self.timer.as_mut().poll(cx).is_pending() { + return Poll::Pending; + } + let deadline = self.last + self.limit; + if Instant::now() >= deadline { + return Poll::Ready(Err(io::Error::new(io::ErrorKind::TimedOut, format!("connection idle for {:?}", self.limit)))); + } + self.timer.as_mut().reset(deadline); + } + } +} + +impl AsyncRead for Idle { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let poll = Pin::new(&mut self.inner).poll_read(cx, buf); + self.track(cx, poll) + } +} + +impl AsyncWrite for Idle { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + let poll = Pin::new(&mut self.inner).poll_write(cx, buf); + self.track(cx, poll) + } + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let poll = Pin::new(&mut self.inner).poll_flush(cx); + self.track(cx, poll) + } + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let poll = Pin::new(&mut self.inner).poll_shutdown(cx); + self.track(cx, poll) + } +} + +pub struct Client { + target: Target, + host_header: HeaderValue, + tls: Option>, + idle: Mutex>>, +} + +impl Client { + pub fn new(base: &str) -> anyhow::Result { + let target = parse_base(base)?; + let default_port = if target.tls { 443 } else { 80 }; + let host = if target.host.contains(':') { format!("[{}]", target.host) } else { target.host.clone() }; + let host_header = if target.port == default_port { host } else { format!("{host}:{}", target.port) }; + let tls = if target.tls { Some(tls_config()?) } else { None }; + Ok(Self { host_header: HeaderValue::from_str(&host_header)?, target, tls, idle: Mutex::new(Vec::new()) }) + } + + pub fn path(&self, p: &str) -> String { + format!("{}{p}", self.target.prefix) + } + + /// TCP to the first of the host's addresses that answers, with + /// keepalive on so the kernel notices a peer that went away. + async fn tcp(&self) -> io::Result { + let mut last = io::Error::new(io::ErrorKind::NotFound, "host has no addresses"); + for addr in tokio::net::lookup_host((self.target.host.as_str(), self.target.port)).await? { + let socket = if addr.is_ipv4() { TcpSocket::new_v4() } else { TcpSocket::new_v6() }?; + socket.set_keepalive(true)?; + socket.set_nodelay(true)?; + match tokio::time::timeout(ADDRESS_TIMEOUT, socket.connect(addr)).await { + Ok(Ok(tcp)) => return Ok(tcp), + Ok(Err(e)) => last = e, + Err(_) => last = io::Error::new(io::ErrorKind::TimedOut, format!("{addr} did not answer in {ADDRESS_TIMEOUT:?}")), + } + } + Err(last) + } + + async fn connect(&self) -> anyhow::Result> { + let connected = tokio::time::timeout(CONNECT_TIMEOUT, self.handshake()).await; + connected + .unwrap_or_else(|_| Err(anyhow::anyhow!("timed out after {CONNECT_TIMEOUT:?}"))) + .with_context(|| format!("connecting to {}:{}", self.target.host, self.target.port)) + } + + async fn handshake(&self) -> anyhow::Result> { + let tcp = Idle::new(self.tcp().await?, IDLE_TIMEOUT); + let sender = match &self.tls { + None => { + let (sender, conn) = hyper::client::conn::http1::handshake(Io(tcp)).await?; + tokio::spawn(conn); + sender + } + Some(config) => { + let name = match self.target.host.parse::() { + Ok(ip) => rustls::pki_types::ServerName::IpAddress(ip.into()), + Err(_) => rustls::pki_types::ServerName::try_from(self.target.host.clone())?, + }; + let tls = tokio_rustls::TlsConnector::from(config.clone()).connect(name, tcp).await?; + let (sender, conn) = hyper::client::conn::http1::handshake(Io(tls)).await?; + tokio::spawn(conn); + sender + } + }; + Ok(sender) + } + + fn checkout(&self) -> Option> { + let mut idle = self.idle.lock().unwrap_or_else(PoisonError::into_inner); + while let Some(s) = idle.pop() { + if !s.is_closed() { + return Some(s); + } + } + None + } + + /// Sends a request and reads the whole response body. + pub async fn send( + &self, + method: Method, + path: &str, + token: &str, + content_type: Option<&str>, + body: Body, + ) -> anyhow::Result<(http::StatusCode, Bytes)> { + let mut req = Request::builder() + .method(method) + .uri(self.path(path)) + .header(header::HOST, self.host_header.clone()) + .header(header::USER_AGENT, concat!("helios-cli/", env!("CARGO_PKG_VERSION"))) + .header(header::AUTHORIZATION, format!("Bearer {token}")); + if let Some(ct) = content_type { + req = req.header(header::CONTENT_TYPE, ct); + } + let req = req.body(body)?; + let mut sender = match self.checkout() { + Some(s) => s, + None => self.connect().await?, + }; + sender.ready().await?; + let resp: Response = sender.send_request(req).await?; + let status = resp.status(); + let bytes = resp.into_body().collect().await?.to_bytes(); + if !sender.is_closed() { + self.idle.lock().unwrap_or_else(PoisonError::into_inner).push(sender); + } + Ok((status, bytes)) + } +} + +#[cfg(test)] +mod tests { + use super::{Idle, parse_base}; + use std::time::Duration; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + #[test] + fn idle_connections_time_out() { + let rt = tokio::runtime::Builder::new_current_thread().enable_time().build().unwrap(); + rt.block_on(async { + let (a, mut b) = tokio::io::duplex(64); + let mut a = Idle::new(a, Duration::from_millis(100)); + let writer = tokio::spawn(async move { + for _ in 0..3 { + tokio::time::sleep(Duration::from_millis(60)).await; + b.write_all(b"x").await.unwrap(); + } + b + }); + // Progress keeps it alive past the limit. + let mut buf = [0u8; 4]; + for _ in 0..3 { + assert_eq!(a.read(&mut buf).await.unwrap(), 1); + } + let _open = writer.await.unwrap(); + let err = a.read(&mut buf).await.unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::TimedOut); + }); + } + + #[test] + fn parses_base_urls() { + let t = parse_base("https://cache.example.com").unwrap(); + assert!(t.tls && t.host == "cache.example.com" && t.port == 443 && t.prefix.is_empty()); + let t = parse_base("http://127.0.0.1:8080/helios/").unwrap(); + assert!(!t.tls && t.host == "127.0.0.1" && t.port == 8080 && t.prefix == "/helios"); + let t = parse_base("https://[::1]:9443").unwrap(); + assert!(t.host == "::1" && t.port == 9443); + assert!(parse_base("ftp://x").is_err()); + assert!(parse_base("https://:80").is_err()); + } +} diff --git a/crates/helios-cli/src/watch.rs b/crates/helios-cli/src/watch.rs new file mode 100644 index 0000000..c91c605 --- /dev/null +++ b/crates/helios-cli/src/watch.rs @@ -0,0 +1,210 @@ +//! Push-on-build. Nix's post-build-hook runs `helios queue-paths`, which +//! drops the built paths into a spool directory and returns at once, so +//! builds never wait on the network. `helios watch-store` drains the spool +//! and pushes each batch's closure, retrying with backoff while the +//! server is unreachable. +//! +//! Spool entries are files of newline-separated store paths, written to a +//! temporary name and renamed into place so readers never see half a file. +//! A batch that fails while the server is reachable is retried one entry +//! at a time, so one bad entry cannot hold up the rest; an entry that keeps +//! failing on its own moves to `failed/` in the spool. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::Context; + +use crate::api::Client; +use crate::push; + +const POLL: Duration = Duration::from_secs(1); +const MAX_BACKOFF: Duration = Duration::from_secs(60); +/// Paths named by one batch (each with its closure); an entry naming more +/// goes alone. +const MAX_BATCH_PATHS: usize = 1_000; +/// Failed pushes of an entry on its own, with the server reachable, before +/// it is set aside. +const MAX_STRIKES: u32 = 5; +const FAILED_DIR: &str = "failed"; + +fn valid(path: &str) -> bool { + path.strip_prefix("/nix/store/").is_some_and(helios_core::store_basename_valid) +} + +/// Writes `paths` (whitespace-separated, as in $OUT_PATHS) as one spool entry. +pub fn queue(spool: &Path, paths: &str) -> anyhow::Result { + let paths: Vec<&str> = paths.split_whitespace().filter(|p| valid(p)).collect(); + if paths.is_empty() { + return Ok(0); + } + std::fs::create_dir_all(spool).with_context(|| format!("creating {}", spool.display()))?; + let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_nanos()); + let name = format!("{stamp:020}-{}", std::process::id()); + let tmp = spool.join(format!(".{name}")); + std::fs::write(&tmp, paths.join("\n") + "\n")?; + std::fs::rename(&tmp, spool.join(name))?; + Ok(paths.len()) +} + +/// Spool entries in arrival order, skipping ones still being written. +fn pending(spool: &Path) -> anyhow::Result> { + let mut entries: Vec = match std::fs::read_dir(spool) { + Ok(rd) => rd + .flatten() + .filter(|e| e.file_type().is_ok_and(|t| t.is_file())) + .map(|e| e.path()) + .filter(|p| p.file_name().and_then(|n| n.to_str()).is_some_and(|n| !n.starts_with('.'))) + .collect(), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(e) => return Err(e.into()), + }; + entries.sort(); + Ok(entries) +} + +/// Moves an entry out of the queue into `failed/`, for a person to look at. +fn set_aside(spool: &Path, entry: &Path, why: &str) { + let dir = spool.join(FAILED_DIR); + let moved = std::fs::create_dir_all(&dir).and_then(|()| std::fs::rename(entry, dir.join(entry.file_name().unwrap_or_default()))); + match moved { + Ok(()) => tracing::warn!(entry = %entry.display(), to = %dir.display(), "{why}; set the entry aside"), + Err(e) => tracing::error!(entry = %entry.display(), error = %e, "{why}; could not set the entry aside"), + } +} + +/// The oldest entries and their paths, up to `MAX_BATCH_PATHS` paths (at +/// least one entry). Unreadable entries are set aside. +fn next_batch(spool: &Path, entries: &[PathBuf]) -> Vec<(PathBuf, Vec)> { + let mut batch = Vec::new(); + let mut count = 0; + for entry in entries { + let text = match std::fs::read_to_string(entry) { + Ok(text) => text, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, + Err(e) => { + set_aside(spool, entry, &format!("unreadable spool entry: {e}")); + continue; + } + }; + let paths: Vec = text.lines().filter(|p| valid(p)).map(str::to_owned).collect(); + if !batch.is_empty() && count + paths.len() > MAX_BATCH_PATHS { + break; + } + count += paths.len(); + batch.push((entry.clone(), paths)); + } + batch +} + +async fn push_paths(client: &Client, cache: &str, paths: impl Iterator, opts: push::Options) -> anyhow::Result<()> { + let mut paths: Vec = paths.cloned().collect(); + paths.sort(); + paths.dedup(); + // Paths garbage-collected since they were built cannot be pushed. + paths.retain(|p| Path::new(p).exists()); + if paths.is_empty() { + return Ok(()); + } + push::push(client, cache, &paths, push::Options { closure: true, ..opts }).await +} + +/// Whether the server answers this client, so a failure is the entry's +/// fault rather than the network's or the server's. +async fn reachable(client: &Client, cache: &str) -> bool { + client.missing(cache, &[]).await.is_ok() +} + +pub async fn watch(client: &Client, cache: &str, spool: &Path, opts: push::Options) -> anyhow::Result<()> { + tracing::info!(spool = %spool.display(), cache, "watching for paths to push"); + let mut backoff = POLL; + let mut strikes: HashMap = HashMap::new(); + loop { + let entries = pending(spool)?; + let batch = next_batch(spool, &entries); + if batch.is_empty() { + tokio::time::sleep(POLL).await; + continue; + } + let Err(e) = push_paths(client, cache, batch.iter().flat_map(|(_, p)| p), opts).await else { + for (entry, _) in &batch { + let _ = std::fs::remove_file(entry); + strikes.remove(entry); + } + backoff = POLL; + continue; + }; + tracing::warn!(entries = batch.len(), error = format!("{e:#}"), "push failed"); + // With the server reachable, find the entries at fault and push the others. + if reachable(client, cache).await { + for (entry, paths) in &batch { + if batch.len() > 1 { + match push_paths(client, cache, paths.iter(), opts).await { + Ok(()) => { + let _ = std::fs::remove_file(entry); + strikes.remove(entry); + continue; + } + Err(e) => { + tracing::warn!(entry = %entry.display(), error = format!("{e:#}"), "push of one entry failed"); + if !reachable(client, cache).await { + break; + } + } + } + } + let n = strikes.entry(entry.clone()).or_default(); + *n += 1; + if *n >= MAX_STRIKES { + strikes.remove(entry); + set_aside(spool, entry, &format!("push failed {MAX_STRIKES} times")); + } + } + } + tracing::info!(retry_in = ?backoff, "retrying failed pushes"); + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(MAX_BACKOFF); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn queue_writes_valid_paths_atomically() { + let dir = std::env::temp_dir().join(format!("helios-spool-{}", helios_core::uuid_v4())); + let n = queue(&dir, "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello /etc/passwd\n/nix/store/bad").unwrap(); + assert_eq!(n, 1); + let entries = pending(&dir).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(std::fs::read_to_string(&entries[0]).unwrap(), "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello\n"); + assert_eq!(queue(&dir, "").unwrap(), 0); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn batches_are_bounded_and_skip_unreadable_entries() { + let dir = std::env::temp_dir().join(format!("helios-spool-{}", helios_core::uuid_v4())); + let path = "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello"; + let many = vec![path; MAX_BATCH_PATHS].join(" "); + queue(&dir, path).unwrap(); + // A directory where an entry should be: reading it fails. + std::fs::create_dir_all(dir.join("00000000000000000000-0")).unwrap(); + queue(&dir, &many).unwrap(); + queue(&dir, path).unwrap(); + let entries = pending(&dir).unwrap(); + assert_eq!(entries.len(), 3, "directories are not entries"); + let mut unreadable = entries.clone(); + unreadable.insert(0, dir.join("00000000000000000000-0")); + let batch = next_batch(&dir, &unreadable); + assert_eq!(batch.len(), 1, "the large entry waits for the next batch"); + assert!(dir.join(FAILED_DIR).join("00000000000000000000-0").exists(), "unreadable entry set aside"); + std::fs::remove_file(&batch[0].0).unwrap(); + let batch = next_batch(&dir, &pending(&dir).unwrap()); + assert_eq!(batch.len(), 1); + assert_eq!(batch[0].1.len(), MAX_BATCH_PATHS); + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/crates/helios-core/Cargo.toml b/crates/helios-core/Cargo.toml new file mode 100644 index 0000000..54e76b3 --- /dev/null +++ b/crates/helios-core/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "helios-core" +description = "Safe Rust bindings to libhelios (Zig): NAR pipeline, narinfo, signing" +version.workspace = true +edition.workspace = true +license-file.workspace = true +publish.workspace = true +links = "helios" + +[dependencies] + +[build-dependencies] +pkg-config = "0.3.34" + diff --git a/crates/helios-core/build.rs b/crates/helios-core/build.rs new file mode 100644 index 0000000..4f4d013 --- /dev/null +++ b/crates/helios-core/build.rs @@ -0,0 +1,56 @@ +//! Builds libhelios (Zig) as a static library and links it with libzstd. +//! +//! Environment: +//! - `ZIG`: zig executable (default `zig`) +//! - `HELIOS_ZIG_CPU`: `-Dcpu` value (default `native`; use `baseline` for +//! portable binaries, which loses SHA-NI accelerated hashing) +//! - `HELIOS_ZIG_TARGET`: `-Dtarget` value when cross-compiling + +use std::env; +use std::path::PathBuf; +use std::process::Command; + +fn main() { + let manifest = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let core = manifest.join("../../core").canonicalize().expect("core/ directory"); + let out = PathBuf::from(env::var("OUT_DIR").unwrap()); + let prefix = out.join("zig-out"); + + for path in ["build.zig", "build.zig.zon", "src", "include"] { + println!("cargo:rerun-if-changed={}", core.join(path).display()); + } + for pkg in std::fs::read_dir(core.join("../pkg")).expect("pkg/ directory").flatten() { + println!("cargo:rerun-if-changed={}", pkg.path().display()); + } + for var in ["ZIG", "HELIOS_ZIG_CPU", "HELIOS_ZIG_TARGET"] { + println!("cargo:rerun-if-env-changed={var}"); + } + + let zig = env::var("ZIG").unwrap_or_else(|_| "zig".into()); + let cpu = env::var("HELIOS_ZIG_CPU").unwrap_or_else(|_| "native".into()); + let mut cmd = Command::new(&zig); + cmd.current_dir(&core) + .arg("build") + .arg("-Doptimize=ReleaseFast") + .arg(format!("-Dcpu={cpu}")) + .arg("--prefix") + .arg(&prefix) + .arg("--cache-dir") + .arg(out.join("zig-cache")); + if let Ok(target) = env::var("HELIOS_ZIG_TARGET") { + cmd.arg(format!("-Dtarget={target}")); + } + if env::var_os("ZIG_GLOBAL_CACHE_DIR").is_none() { + cmd.env("ZIG_GLOBAL_CACHE_DIR", out.join("zig-global-cache")); + } + let status = cmd.status().unwrap_or_else(|e| panic!("failed to run `{zig} build` (is zig 0.17 on PATH?): {e}")); + assert!(status.success(), "zig build failed for libhelios"); + + println!("cargo:rustc-link-search=native={}", prefix.join("lib").display()); + println!("cargo:rustc-link-lib=static=helios"); + println!("cargo:include={}", prefix.join("include").display()); + + if pkg_config::Config::new().atleast_version("1.5").probe("libzstd").is_err() { + println!("cargo:rustc-link-lib=zstd"); + } +} diff --git a/crates/helios-core/src/lib.rs b/crates/helios-core/src/lib.rs new file mode 100644 index 0000000..95b41cf --- /dev/null +++ b/crates/helios-core/src/lib.rs @@ -0,0 +1,621 @@ +//! Safe bindings to libhelios, the Zig core: the compressing NAR pipeline, +//! the upload verifier, nix32, narinfo rendering and Ed25519 signing. + +use std::ffi::{CStr, CString, c_int, c_void}; +use std::fmt; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; + +mod sys { + use std::ffi::{c_char, c_int, c_void}; + + #[repr(C)] + #[derive(Clone, Copy, Default)] + pub struct Digest { + pub file_hash: [u8; 32], + pub file_size: u64, + pub nar_hash: [u8; 32], + pub nar_size: u64, + } + + #[repr(C)] + pub struct DumpOptions { + pub level: c_int, + pub threads: c_int, + pub nar_size: u64, + pub window_log: c_int, + pub hash: bool, + } + + #[repr(C)] + pub struct Str { + pub ptr: *const u8, + pub len: usize, + } + + #[repr(C)] + pub struct NarinfoInput { + pub store_path: Str, + pub nar_hash: *const [u8; 32], + pub nar_size: u64, + pub file_hash: *const [u8; 32], + pub file_size: u64, + pub compression: Str, + pub references: Str, + pub deriver: Str, + pub system: Str, + } + + pub type WriteFn = unsafe extern "C" fn(ctx: *mut c_void, buf: *const u8, len: usize) -> c_int; + + pub enum Compressor {} + pub enum Verifier {} + pub enum Signer {} + + unsafe extern "C" { + pub fn hl_strerror(rc: c_int) -> *const c_char; + pub fn hl_last_errno() -> c_int; + pub fn hl_nix32_encode(bytes: *const u8, len: usize, out: *mut u8) -> usize; + pub fn hl_nix32_decode(text: *const u8, text_len: usize, out: *mut u8, out_len: usize) -> c_int; + pub fn hl_store_basename_valid(base: *const u8, len: usize) -> bool; + pub fn hl_nar_dump(path: *const c_char, opts: *const DumpOptions, write: WriteFn, ctx: *mut c_void, out: *mut Digest) -> c_int; + pub fn hl_compressor_new(opts: *const DumpOptions, write: WriteFn, ctx: *mut c_void) -> *mut Compressor; + pub fn hl_compressor_update(c: *mut Compressor, data: *const u8, len: usize) -> c_int; + pub fn hl_compressor_finish(c: *mut Compressor, out: *mut Digest) -> c_int; + pub fn hl_compressor_free(c: *mut Compressor); + pub fn hl_verifier_new(compression: c_int) -> *mut Verifier; + pub fn hl_verifier_update(v: *mut Verifier, data: *const u8, len: usize) -> c_int; + pub fn hl_verifier_finish(v: *mut Verifier, out: *mut Digest) -> c_int; + pub fn hl_verifier_free(v: *mut Verifier); + pub fn hl_signer_new(key: *const u8, len: usize) -> *mut Signer; + pub fn hl_signer_free(s: *mut Signer); + pub fn hl_signer_public_key(s: *const Signer, out: *mut u8, cap: usize, out_len: *mut usize) -> c_int; + pub fn hl_signer_sign(s: *const Signer, msg: *const u8, msg_len: usize, out: *mut u8, cap: usize, out_len: *mut usize) -> c_int; + pub fn hl_signer_generate(name: *const u8, name_len: usize, out: *mut u8, cap: usize, out_len: *mut usize) -> c_int; + pub fn hl_narinfo_render(input: *const NarinfoInput, signer: *const Signer, out: *mut *mut u8, out_len: *mut usize) -> c_int; + pub fn hl_free(ptr: *mut u8, len: usize); + pub fn hl_sha256(data: *const u8, len: usize, out: *mut [u8; 32]); + pub fn hl_hmac_sha256(key: *const u8, key_len: usize, msg: *const u8, msg_len: usize, out: *mut [u8; 32]); + pub fn hl_random(buf: *mut u8, len: usize) -> c_int; + } +} + +pub const HL_E_IO: i32 = -1; +pub const HL_E_ABORTED: i32 = -4; +pub const HL_E_INVALID: i32 = -5; +pub const HL_E_UNSUPPORTED_OS: i32 = -8; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Error { + pub code: i32, + /// OS errno for I/O failures, 0 otherwise. + pub errno: i32, +} + +impl Error { + fn from_rc(code: c_int) -> Self { + let errno = if code == HL_E_IO { unsafe { sys::hl_last_errno() } } else { 0 }; + Self { code, errno } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let msg = unsafe { CStr::from_ptr(sys::hl_strerror(self.code)) }; + write!(f, "{}", msg.to_string_lossy())?; + if self.errno != 0 { + write!(f, ": {}", std::io::Error::from_raw_os_error(self.errno))?; + } + Ok(()) + } +} + +impl std::error::Error for Error {} + +fn check(rc: c_int) -> Result<(), Error> { + if rc == 0 { Ok(()) } else { Err(Error::from_rc(rc)) } +} + +// ── nix32 ── + +pub fn nix32_encode(bytes: &[u8]) -> String { + let len = if bytes.is_empty() { 0 } else { (bytes.len() * 8 - 1) / 5 + 1 }; + let mut out = vec![0u8; len]; + unsafe { sys::hl_nix32_encode(bytes.as_ptr(), bytes.len(), out.as_mut_ptr()) }; + // The nix32 alphabet is ASCII. + unsafe { String::from_utf8_unchecked(out) } +} + +pub fn nix32_decode(text: &str) -> Option<[u8; N]> { + let mut out = [0u8; N]; + let rc = unsafe { sys::hl_nix32_decode(text.as_ptr(), text.len(), out.as_mut_ptr(), N) }; + (rc == 0).then_some(out) +} + +/// `<32 nix32 chars>-` with only the characters Nix allows. +pub fn store_basename_valid(base: &str) -> bool { + unsafe { sys::hl_store_basename_valid(base.as_ptr(), base.len()) } +} + +// ── Pipeline ── + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Digest { + pub file_hash: [u8; 32], + pub file_size: u64, + pub nar_hash: [u8; 32], + pub nar_size: u64, +} + +impl From for Digest { + fn from(d: sys::Digest) -> Self { + Self { file_hash: d.file_hash, file_size: d.file_size, nar_hash: d.nar_hash, nar_size: d.nar_size } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct DumpOptions { + /// zstd level; 0 disables compression. + pub level: i32, + /// zstd worker threads; 0 compresses on the calling thread. + pub threads: i32, + /// The exact NAR size, 0 if unknown. zstd fits its window to it and + /// records it in the frame; a stream of another length fails. + pub nar_size: u64, + /// 0 keeps the level's window; 10-27 enables long-distance matching + /// with a 2^window_log window. + pub window_log: i32, + /// false skips both SHA-256 passes; the digest then carries sizes only, + /// for an uploader whose server verifies the content anyway. + pub hash: bool, +} + +impl Default for DumpOptions { + fn default() -> Self { + Self { level: 3, threads: 0, nar_size: 0, window_log: 0, hash: true } + } +} + +impl DumpOptions { + fn raw(&self) -> sys::DumpOptions { + sys::DumpOptions { level: self.level, threads: self.threads, nar_size: self.nar_size, window_log: self.window_log, hash: self.hash } + } +} + +unsafe extern "C" fn trampoline bool>(ctx: *mut c_void, buf: *const u8, len: usize) -> c_int { + let f = unsafe { &mut *(ctx as *mut F) }; + let chunk = unsafe { std::slice::from_raw_parts(buf, len) }; + // Never unwind into Zig. + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| f(chunk))) { + Ok(true) => 0, + _ => 1, + } +} + +/// Serialise `path` as a NAR, compress it and stream the output to `write`. +/// Return `false` from `write` to abort. Blocking; Linux only. +pub fn dump_nar bool>(path: &Path, opts: &DumpOptions, mut write: F) -> Result { + let cpath = CString::new(path.as_os_str().as_bytes()).map_err(|_| Error { code: HL_E_INVALID, errno: 0 })?; + let mut out = sys::Digest::default(); + let rc = unsafe { sys::hl_nar_dump(cpath.as_ptr(), &opts.raw(), trampoline::, &mut write as *mut F as *mut c_void, &mut out) }; + check(rc)?; + Ok(out.into()) +} + +/// The same pipeline, fed with NAR bytes produced elsewhere. +pub struct Compressor bool> { + raw: *mut sys::Compressor, + // Boxed so the pointer handed to Zig stays stable. + _write: Box, +} + +impl bool> Compressor { + pub fn new(opts: &DumpOptions, write: F) -> Result { + let mut write = Box::new(write); + let ctx = &mut *write as *mut F as *mut c_void; + let raw = unsafe { sys::hl_compressor_new(&opts.raw(), trampoline::, ctx) }; + if raw.is_null() { + return Err(Error { code: -6, errno: 0 }); + } + Ok(Self { raw, _write: write }) + } + + pub fn update(&mut self, data: &[u8]) -> Result<(), Error> { + check(unsafe { sys::hl_compressor_update(self.raw, data.as_ptr(), data.len()) }) + } + + pub fn finish(self) -> Result { + let mut out = sys::Digest::default(); + check(unsafe { sys::hl_compressor_finish(self.raw, &mut out) })?; + Ok(out.into()) + } +} + +impl bool> Drop for Compressor { + fn drop(&mut self) { + unsafe { sys::hl_compressor_free(self.raw) } + } +} + +// ── Verifier ── + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Compression { + None, + Zstd, +} + +impl Compression { + pub fn parse(s: &str) -> Option { + match s { + "zstd" => Some(Self::Zstd), + "none" => Some(Self::None), + _ => None, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::None => "none", + Self::Zstd => "zstd", + } + } + + pub fn extension(self) -> &'static str { + match self { + Self::None => ".nar", + Self::Zstd => ".nar.zst", + } + } +} + +/// Hashes an uploaded compressed NAR and the NAR it decompresses to. +pub struct Verifier { + raw: *mut sys::Verifier, +} + +// The verifier owns its state exclusively; it is only ever used from one thread at a time. +unsafe impl Send for Verifier {} + +impl Verifier { + pub fn new(compression: Compression) -> Result { + let kind = match compression { + Compression::None => 0, + Compression::Zstd => 1, + }; + let raw = unsafe { sys::hl_verifier_new(kind) }; + if raw.is_null() { + return Err(Error { code: -6, errno: 0 }); + } + Ok(Self { raw }) + } + + pub fn update(&mut self, data: &[u8]) -> Result<(), Error> { + check(unsafe { sys::hl_verifier_update(self.raw, data.as_ptr(), data.len()) }) + } + + pub fn finish(self) -> Result { + let mut out = sys::Digest::default(); + check(unsafe { sys::hl_verifier_finish(self.raw, &mut out) })?; + Ok(out.into()) + } +} + +impl Drop for Verifier { + fn drop(&mut self) { + unsafe { sys::hl_verifier_free(self.raw) } + } +} + +// ── Signing and narinfo ── + +pub struct Signer { + raw: *mut sys::Signer, +} + +// Immutable after construction; signing only reads the key pair. +unsafe impl Send for Signer {} +unsafe impl Sync for Signer {} + +impl Signer { + /// Parses a Nix secret key (`nix key generate-secret` output). + pub fn new(secret_key: &str) -> Option { + let raw = unsafe { sys::hl_signer_new(secret_key.as_ptr(), secret_key.len()) }; + (!raw.is_null()).then_some(Self { raw }) + } + + /// A new Nix secret key named `name`, in `nix key generate-secret` format. + pub fn generate(name: &str) -> Result { + // The name, ':' and 88 base64 characters: one call, so one key. + let buf = out_string(name.len() + 89, |out, cap, len| unsafe { sys::hl_signer_generate(name.as_ptr(), name.len(), out, cap, len) })?; + Ok(String::from_utf8(buf).expect("key name was valid UTF-8")) + } + + /// `:`, the value for `trusted-public-keys`. + pub fn public_key(&self) -> String { + let buf = out_string(128, |out, cap, len| unsafe { sys::hl_signer_public_key(self.raw, out, cap, len) }).expect("public key renders"); + String::from_utf8(buf).expect("key name was valid UTF-8") + } + + /// A detached signature over `msg`, as `:`. + pub fn sign(&self, msg: &[u8]) -> String { + let buf = out_string(160, |out, cap, len| unsafe { sys::hl_signer_sign(self.raw, msg.as_ptr(), msg.len(), out, cap, len) }) + .expect("signing succeeds"); + String::from_utf8(buf).expect("key name was valid UTF-8") + } +} + +/// Calls `f` with a buffer of `cap` bytes, and once more with the length it +/// asked for if that was too small: the key name has no fixed bound. +fn out_string(cap: usize, f: impl Fn(*mut u8, usize, &mut usize) -> c_int) -> Result, Error> { + const HL_E_BUFFER: c_int = -11; + let mut buf = vec![0u8; cap]; + let mut len = 0usize; + let mut rc = f(buf.as_mut_ptr(), buf.len(), &mut len); + if rc == HL_E_BUFFER { + buf.resize(len, 0); + rc = f(buf.as_mut_ptr(), buf.len(), &mut len); + } + check(rc)?; + buf.truncate(len); + Ok(buf) +} + +impl Drop for Signer { + fn drop(&mut self) { + unsafe { sys::hl_signer_free(self.raw) } + } +} + +pub struct NarinfoInput<'a> { + pub store_path: &'a str, + pub nar_hash: &'a [u8; 32], + pub nar_size: u64, + pub file_hash: &'a [u8; 32], + pub file_size: u64, + pub compression: &'a str, + /// Space-separated basenames. + pub references: &'a str, + pub deriver: &'a str, + pub system: &'a str, +} + +fn s(v: &str) -> sys::Str { + sys::Str { ptr: v.as_ptr(), len: v.len() } +} + +/// Validates every field and renders the narinfo, signed if `signer` is set. +pub fn render_narinfo(input: &NarinfoInput<'_>, signer: Option<&Signer>) -> Result, Error> { + let raw = sys::NarinfoInput { + store_path: s(input.store_path), + nar_hash: input.nar_hash, + nar_size: input.nar_size, + file_hash: input.file_hash, + file_size: input.file_size, + compression: s(input.compression), + references: s(input.references), + deriver: s(input.deriver), + system: s(input.system), + }; + let mut ptr: *mut u8 = std::ptr::null_mut(); + let mut len = 0usize; + let signer_ptr = signer.map_or(std::ptr::null(), |s| s.raw as *const sys::Signer); + check(unsafe { sys::hl_narinfo_render(&raw, signer_ptr, &mut ptr, &mut len) })?; + let text = unsafe { std::slice::from_raw_parts(ptr, len) }.to_vec(); + unsafe { sys::hl_free(ptr, len) }; + Ok(text) +} + +// ── Primitives ── + +pub fn sha256(data: &[u8]) -> [u8; 32] { + let mut out = [0u8; 32]; + unsafe { sys::hl_sha256(data.as_ptr(), data.len(), &mut out) }; + out +} + +pub fn hmac_sha256(key: &[u8], msg: &[u8]) -> [u8; 32] { + let mut out = [0u8; 32]; + unsafe { sys::hl_hmac_sha256(key.as_ptr(), key.len(), msg.as_ptr(), msg.len(), &mut out) }; + out +} + +/// Constant-time equality for equal-length byte strings. +pub fn ct_eq(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + let diff = a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)); + std::hint::black_box(diff) == 0 +} + +/// Fills `buf` from the OS CSPRNG. +pub fn random_bytes(buf: &mut [u8]) { + let rc = unsafe { sys::hl_random(buf.as_mut_ptr(), buf.len()) }; + assert_eq!(rc, 0, "the OS random number generator failed"); +} + +/// A random (version 4) UUID in canonical hyphenated form. +pub fn uuid_v4() -> String { + let mut b = [0u8; 16]; + random_bytes(&mut b); + b[6] = (b[6] & 0x0f) | 0x40; + b[8] = (b[8] & 0x3f) | 0x80; + let hex: String = b.iter().map(|x| format!("{x:02x}")).collect(); + format!("{}-{}-{}-{}-{}", &hex[0..8], &hex[8..12], &hex[12..16], &hex[16..20], &hex[20..32]) +} + +#[cfg(test)] +mod tests { + use super::*; + + const KEY: &str = "test-1:suJsWimvBNFUIc0JJE18OfOMygH/f2GuTC2/XwD6rPKkV+1VKilMIkXl6Ax9hqeKpUF/BSOH+Fqng4tqZlirhw=="; + + /// A scratch directory removed on drop (keeps tempfile out of the tree). + struct TempDir(std::path::PathBuf); + + impl TempDir { + fn new() -> Self { + let dir = std::env::temp_dir().join(format!("helios-core-test-{}", uuid_v4())); + std::fs::create_dir(&dir).unwrap(); + Self(dir) + } + fn path(&self) -> &Path { + &self.0 + } + } + + impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + #[test] + fn primitives() { + // RFC 4231 test case 2. + let mac = hmac_sha256(b"Jefe", b"what do ya want for nothing?"); + assert_eq!(nix32_encode(&mac).len(), 52); + assert_eq!(mac.iter().map(|b| format!("{b:02x}")).collect::(), "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843"); + assert_eq!( + sha256(b"abc").iter().map(|b| format!("{b:02x}")).collect::(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + assert!(ct_eq(b"same", b"same") && !ct_eq(b"same", b"diff") && !ct_eq(b"a", b"ab")); + let (a, b) = (uuid_v4(), uuid_v4()); + assert_ne!(a, b); + assert_eq!(a.len(), 36); + assert_eq!(&a[14..15], "4"); + // nix32 lengths and the error path across the C ABI. + let bytes = [7u8; 20]; + assert_eq!(nix32_decode::<20>(&nix32_encode(&bytes)), Some(bytes)); + assert_eq!(nix32_decode::<20>("not-valid"), None); + } + + /// The Signer binding: generation, parsing, and buffers that grow for + /// long key names. Signing itself is tested in pkg/nix-narinfo. + #[test] + fn signer() { + let key = Signer::generate("gen-1").unwrap(); + assert!(key.starts_with("gen-1:")); + assert_ne!(key, Signer::generate("gen-1").unwrap()); + assert!(Signer::new(&key).unwrap().public_key().starts_with("gen-1:")); + assert!(Signer::generate("bad:name").is_err()); + assert!(Signer::new(KEY).unwrap().public_key().starts_with("test-1:")); + assert!(Signer::new("garbage").is_none()); + + let name = "k".repeat(600); + let long = Signer::new(&Signer::generate(&name).unwrap()).unwrap(); + assert_eq!(long.public_key().len(), name.len() + 1 + 44); + assert_eq!(long.sign(b"msg").len(), name.len() + 1 + 88); + } + + #[test] + fn detached_signature_matches_the_narinfo_one() { + // The same key over the same fingerprint gives the narinfo's Sig. + let signer = Signer::new(KEY).unwrap(); + let path = "/nix/store/7rjj86a15146cq1d3qy068lml7n8ykzm-hello-2.12"; + let nar_hash = [7u8; 32]; + let narinfo = render_narinfo( + &NarinfoInput { + store_path: path, + nar_hash: &nar_hash, + nar_size: 1234, + file_hash: &[9u8; 32], + file_size: 100, + compression: "zstd", + references: "", + deriver: "", + system: "", + }, + Some(&signer), + ) + .unwrap(); + let text = String::from_utf8(narinfo).unwrap(); + let sig = text.lines().find_map(|l| l.strip_prefix("Sig: ")).unwrap(); + let fingerprint = format!("1;{path};sha256:{};1234;", nix32_encode(&nar_hash)); + assert_eq!(signer.sign(fingerprint.as_bytes()), sig); + } + + #[test] + fn dump_matches_compressor_and_verifier() { + let dir = TempDir::new(); + let root = dir.path().join("pkg"); + std::fs::create_dir_all(root.join("bin")).unwrap(); + std::fs::write(root.join("bin/hello"), b"#!/bin/sh\necho hi\n").unwrap(); + std::fs::set_permissions(root.join("bin/hello"), std::os::unix::fs::PermissionsExt::from_mode(0o755)).unwrap(); + std::fs::write(root.join("big"), vec![b'a'; 5 << 20]).unwrap(); + std::os::unix::fs::symlink("bin/hello", root.join("link")).unwrap(); + + let mut compressed = Vec::new(); + let digest = dump_nar(&root, &DumpOptions { level: 3, threads: 2, ..Default::default() }, |c| { + compressed.extend_from_slice(c); + true + }) + .unwrap(); + assert_eq!(digest.file_size, compressed.len() as u64); + + let mut v = Verifier::new(Compression::Zstd).unwrap(); + for chunk in compressed.chunks(65536) { + v.update(chunk).unwrap(); + } + let verified = v.finish().unwrap(); + assert_eq!(verified, digest); + + let aborted = dump_nar(&root, &DumpOptions::default(), |_| false); + assert_eq!(aborted.unwrap_err().code, HL_E_ABORTED); + } + + /// Files are mapped a 16 MiB window at a time; one spanning several + /// windows and ending mid-page must still serialise exactly. + #[test] + fn large_files_serialise_across_mapping_windows() { + let dir = TempDir::new(); + let file = dir.path().join("big"); + let data: Vec = (0..(40usize << 20) + 12345).map(|i| (i * 31 % 251) as u8).collect(); + std::fs::write(&file, &data).unwrap(); + + let mut nar = Vec::new(); + let digest = dump_nar(&file, &DumpOptions { level: 0, ..Default::default() }, |c| { + nar.extend_from_slice(c); + true + }) + .unwrap(); + + let mut expected = Vec::new(); + let mut str = |s: &[u8]| { + expected.extend_from_slice(&(s.len() as u64).to_le_bytes()); + expected.extend_from_slice(s); + expected.resize(expected.len().next_multiple_of(8), 0); + }; + for s in [&b"nix-archive-1"[..], b"(", b"type", b"regular", b"contents", &data, b")"] { + str(s); + } + assert!(nar == expected, "NAR differs from the reference encoding"); + assert_eq!(digest.nar_hash, sha256(&expected)); + } + + /// Our NAR serialiser must agree byte-for-byte with Nix: compare NAR + /// hashes against the Nix database for a real closure. + #[test] + fn nar_hash_matches_nix_store() { + use std::process::Command; + let Ok(out) = Command::new("nix-store").args(["-qR", "/run/current-system/sw/bin/nix"]).output() else { + eprintln!("nix-store not available; skipping"); + return; + }; + if !out.status.success() { + eprintln!("no local system closure; skipping"); + return; + } + let paths: Vec = String::from_utf8(out.stdout).unwrap().lines().map(str::to_owned).collect(); + assert!(!paths.is_empty()); + let hashes = Command::new("nix-store").args(["-q", "--hash"]).args(&paths).output().unwrap(); + let hashes = String::from_utf8(hashes.stdout).unwrap(); + for (path, expected) in paths.iter().zip(hashes.lines()) { + let digest = + dump_nar(Path::new(path), &DumpOptions { level: 1, ..Default::default() }, |_| true).unwrap_or_else(|e| panic!("{path}: {e}")); + assert_eq!(format!("sha256:{}", nix32_encode(&digest.nar_hash)), expected, "{path}"); + } + } +} diff --git a/crates/helios-daemon/Cargo.toml b/crates/helios-daemon/Cargo.toml new file mode 100644 index 0000000..009e87a --- /dev/null +++ b/crates/helios-daemon/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "helios-daemon" +description = "Maintenance daemon for helios-server: auto-GC, integrity scrub, SQLite maintenance, metrics" +version.workspace = true +edition.workspace = true +license-file.workspace = true +publish.workspace = true + +[dependencies] +anyhow = "1.0.104" +bytes = "1.12.1" +clap = { version = "4.6.7", default-features = false, features = ["std", "derive", "env", "help", "usage", "error-context"] } +futures-util = { version = "0.3.34", default-features = false, features = ["std"] } +helios-core = { path = "../helios-core" } +helios-log = { path = "../helios-log" } +http = "1.5" +http-body-util = "0.1.5" +hyper = { version = "1.11", default-features = false, features = ["client", "server", "http1"] } +libc = "0.2.190" +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.151" +tokio = { version = "1.53.2", features = ["rt-multi-thread", "net", "time", "signal", "sync", "io-util"] } +tracing = { version = "0.1.44", default-features = false, features = ["std"] } diff --git a/crates/helios-daemon/src/autogc.rs b/crates/helios-daemon/src/autogc.rs new file mode 100644 index 0000000..b31061a --- /dev/null +++ b/crates/helios-daemon/src/autogc.rs @@ -0,0 +1,187 @@ +//! Keeps the cache under a byte quota (and optionally the disk above a +//! free-space floor) by evicting least-recently-used paths. +//! +//! Eviction starts above the high watermark and stops at the low one, so +//! it runs in occasional batches rather than on every upload. + +use serde::Deserialize; +use serde_json::json; + +use crate::state::{Daemon, add, now, set}; + +#[derive(Clone, Copy, Debug)] +pub struct Policy { + /// Target NAR bytes; 0 disables the quota. + pub quota: u64, + pub high: f64, + pub low: f64, + /// Keep at least this much free on the data filesystem; 0 disables. + pub min_free: u64, +} + +#[derive(Deserialize)] +struct Stats { + blobs: Blobs, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Blobs { + bytes: u64, + /// NARs no path uses any more; GC deletes them once their upload + /// grace period ends, so they no longer count against the quota. + unreferenced_bytes: u64, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Candidate { + cache: u32, + hash: String, + nar_bytes: u64, +} + +#[derive(Deserialize)] +struct Lru { + paths: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Evicted { + evicted: u64, + freed_bytes: u64, +} + +/// Bytes to free now, for each reason; 0 when within policy. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Need { + pub quota: u64, + pub disk: u64, +} + +pub fn to_free(policy: &Policy, used: u64, free_disk: u64) -> Need { + let quota = + if policy.quota > 0 && used as f64 > policy.quota as f64 * policy.high { used - (policy.quota as f64 * policy.low) as u64 } else { 0 }; + let disk = if policy.min_free > 0 && free_disk < policy.min_free { + // Overshoot a little so the next upload does not trigger it again. + (policy.min_free as f64 * 1.1) as u64 - free_disk + } else { + 0 + }; + Need { quota, disk } +} + +/// One run evicts at most this share of what helios holds for the +/// free-space floor, so a disk filled by something else drains the cache +/// slowly, with warnings, rather than at once. +const MAX_DISK_SHARE: f64 = 0.25; + +/// Bytes this run should evict. The quota is helios's own, and is met in +/// full; the free-space floor only as far as evicting helios data can +/// meet it, and within the per-run cap. +pub fn budget(need: Need, held: u64) -> u64 { + let disk = if need.disk > held { 0 } else { need.disk.min((held as f64 * MAX_DISK_SHARE) as u64) }; + need.quota.max(disk) +} + +/// Upper bound on eviction rounds per run, as a backstop. +const MAX_ROUNDS: usize = 10_000; + +/// Referenced NAR bytes and bytes to free, from fresh server stats. +async fn measure(d: &Daemon, policy: &Policy) -> anyhow::Result<(u64, Need)> { + let stats: Stats = d.client.get("/v1/stats").await?; + let referenced = stats.blobs.bytes.saturating_sub(stats.blobs.unreferenced_bytes); + let free_disk = crate::state::disk_space(&d.data_dir).map(|(free, _)| free).unwrap_or(u64::MAX); + // Unreferenced NARs will be deleted soon; count them as free already. + let free_soon = free_disk.saturating_add(stats.blobs.unreferenced_bytes); + Ok((referenced, to_free(policy, referenced, free_soon))) +} + +pub async fn run(d: &Daemon, policy: &Policy) -> anyhow::Result<()> { + add(&d.metrics.gc_runs, 1); + set(&d.metrics.gc_last_run, now()); + let (referenced, need) = measure(d, policy).await?; + if need.disk > referenced { + tracing::warn!( + short = need.disk, + held = referenced, + "the data filesystem lacks more free space than helios holds; not evicting for it, as something else fills the disk" + ); + } + let mut left = budget(need, referenced); + if left == 0 { + return Ok(()); + } + if left < need.quota.max(need.disk) && need.disk <= referenced { + tracing::warn!(short = need.disk, evicting = left, "low on disk space; evicting only part of what is needed in this run"); + } + tracing::info!(referenced, need = left, "evicting"); + let (mut evicted, mut freed) = (0u64, 0u64); + for _ in 0..MAX_ROUNDS { + let lru: Lru = d.client.get("/v1/lru?limit=500").await?; + // Enough of the oldest paths to cover what is left, by NAR size. + let mut batch = Vec::new(); + let mut planned = 0u64; + for c in lru.paths { + planned += c.nar_bytes; + batch.push(json!({ "cache": c.cache, "hash": c.hash })); + if planned >= left { + break; + } + } + if batch.is_empty() { + break; + } + let r: Evicted = d.client.post("/v1/evict", json!({ "paths": batch })).await?; + if r.evicted == 0 { + break; + } + evicted += r.evicted; + freed += r.freed_bytes; + // NARs shared with paths still in use stay; re-measure rather than + // trusting the plan, and never go past this run's budget. + let (referenced, need) = measure(d, policy).await?; + left = left.saturating_sub(planned).min(budget(need, referenced)); + if left == 0 { + break; + } + } + if left > 0 { + tracing::warn!(need = left, "cannot free enough: what is left is pinned, or shared with paths still in use"); + } + add(&d.metrics.gc_evicted_paths, evicted); + add(&d.metrics.gc_freed_bytes, freed); + tracing::info!(paths = evicted, freed_now = freed, "evicted (the rest is freed after the upload grace period)"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const GIB: u64 = 1 << 30; + + #[test] + fn watermarks() { + let p = Policy { quota: 100 * GIB, high: 0.9, low: 0.8, min_free: 0 }; + assert_eq!(to_free(&p, 89 * GIB, u64::MAX), Need { quota: 0, disk: 0 }); + assert_eq!(to_free(&p, 95 * GIB, u64::MAX), Need { quota: 15 * GIB, disk: 0 }); + assert_eq!(budget(to_free(&p, 95 * GIB, u64::MAX), 95 * GIB), 15 * GIB); + } + + #[test] + fn free_space_floor() { + let p = Policy { quota: 0, high: 0.9, low: 0.8, min_free: 10 * GIB }; + assert_eq!(to_free(&p, 1, 20 * GIB), Need { quota: 0, disk: 0 }); + assert_eq!(to_free(&p, 1, 5 * GIB), Need { quota: 0, disk: 6 * GIB }); + let need = Need { quota: 0, disk: 6 * GIB }; + // Evicting all of it would not be enough: something else fills the disk. + assert_eq!(budget(need, 5 * GIB), 0); + // A run evicts a share at most. + assert_eq!(budget(need, 8 * GIB), 2 * GIB); + assert_eq!(budget(need, 100 * GIB), 6 * GIB); + // The quota is met in full regardless. + assert_eq!(budget(Need { quota: 7 * GIB, disk: 6 * GIB }, 8 * GIB), 7 * GIB); + } +} diff --git a/crates/helios-daemon/src/client.rs b/crates/helios-daemon/src/client.rs new file mode 100644 index 0000000..9bc147a --- /dev/null +++ b/crates/helios-daemon/src/client.rs @@ -0,0 +1,69 @@ +//! Client for helios-server's maintenance API on its Unix socket. Requests +//! are rare (minutes apart), so each one gets a fresh connection. + +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::{Context, bail}; +use bytes::Bytes; +use http::{Method, Request, header}; +use http_body_util::{BodyExt, Full}; +use serde::de::DeserializeOwned; +use serde_json::Value; +use tokio::net::UnixStream; + +use crate::io::Io; + +/// Queries: a server that takes longer is stuck. +const GET_TIMEOUT: Duration = Duration::from_secs(60); +/// Operations, which may be slow (a backup of a large database, a large +/// eviction) but must not hang a task forever. +const POST_TIMEOUT: Duration = Duration::from_secs(30 * 60); + +#[derive(Clone)] +pub struct Client { + socket: PathBuf, +} + +impl Client { + pub fn new(socket: PathBuf) -> Self { + Self { socket } + } + + async fn call(&self, method: Method, path: &str, body: Option) -> anyhow::Result { + let limit = if method == Method::GET { GET_TIMEOUT } else { POST_TIMEOUT }; + match tokio::time::timeout(limit, self.request(method, path, body)).await { + Ok(result) => result, + Err(_) => bail!("{path}: no response in {limit:?}"), + } + } + + async fn request(&self, method: Method, path: &str, body: Option) -> anyhow::Result { + let stream = UnixStream::connect(&self.socket).await.with_context(|| format!("connecting to {}", self.socket.display()))?; + let (mut sender, conn) = hyper::client::conn::http1::handshake(Io(stream)).await?; + tokio::spawn(conn); + let mut req = Request::builder().method(method).uri(path).header(header::HOST, "helios"); + let payload = match body { + Some(v) => { + req = req.header(header::CONTENT_TYPE, "application/json"); + Bytes::from(serde_json::to_vec(&v)?) + } + None => Bytes::new(), + }; + let resp = sender.send_request(req.body(Full::new(payload))?).await?; + let status = resp.status(); + let bytes = resp.into_body().collect().await?.to_bytes(); + if !status.is_success() { + bail!("{path}: {status}: {}", String::from_utf8_lossy(&bytes)); + } + serde_json::from_slice(&bytes).with_context(|| format!("decoding {path}")) + } + + pub async fn get(&self, path: &str) -> anyhow::Result { + self.call(Method::GET, path, None).await + } + + pub async fn post(&self, path: &str, body: Value) -> anyhow::Result { + self.call(Method::POST, path, Some(body)).await + } +} diff --git a/crates/helios-daemon/src/io.rs b/crates/helios-daemon/src/io.rs new file mode 100644 index 0000000..6433b7a --- /dev/null +++ b/crates/helios-daemon/src/io.rs @@ -0,0 +1,38 @@ +//! hyper's I/O traits over tokio streams (hyper-util would bring a crate +//! for these forty lines). + +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +pub struct Io(pub T); + +impl hyper::rt::Read for Io { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, mut buf: hyper::rt::ReadBufCursor<'_>) -> Poll> { + // SAFETY: tokio's ReadBuf only writes initialised bytes into the + // uninitialised tail, and we advance by exactly what was filled. + let filled = unsafe { + let mut rb = ReadBuf::uninit(buf.as_mut()); + match Pin::new(&mut self.0).poll_read(cx, &mut rb) { + Poll::Ready(Ok(())) => rb.filled().len(), + other => return other, + } + }; + unsafe { buf.advance(filled) }; + Poll::Ready(Ok(())) + } +} + +impl hyper::rt::Write for Io { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + Pin::new(&mut self.0).poll_write(cx, buf) + } + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.0).poll_flush(cx) + } + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.0).poll_shutdown(cx) + } +} diff --git a/crates/helios-daemon/src/main.rs b/crates/helios-daemon/src/main.rs new file mode 100644 index 0000000..c1eb115 --- /dev/null +++ b/crates/helios-daemon/src/main.rs @@ -0,0 +1,276 @@ +//! helios-daemon: maintenance for a helios-server, driven over its admin +//! socket. It never writes the database itself; it decides what to do and +//! the server does it, so the server's in-memory state stays consistent. + +mod autogc; +mod client; +mod io; +mod maint; +mod metrics; +mod scrub; +mod state; + +use std::future::Future; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::AtomicU64; +use std::time::Duration; + +use clap::Parser; +use tracing::Instrument; + +use crate::state::{Daemon, add}; + +/// Maintenance daemon for helios-server. +#[derive(Parser, Debug)] +#[command(version)] +struct Args { + /// helios-server's maintenance socket (its --admin-socket). + #[arg(long, env = "HELIOS_ADMIN_SOCKET")] + socket: PathBuf, + + /// helios-server's data directory, read for scrubbing and disk space. + #[arg(long, env = "HELIOS_DATA_DIR", default_value = "/var/lib/helios")] + data_dir: PathBuf, + + /// Directory for the daemon's own state, such as where an unfinished + /// scrub resumes (default: systemd's $STATE_DIRECTORY; without either, + /// a restart begins the scrub again). + #[arg(long, env = "HELIOS_DAEMON_STATE_DIR")] + state_dir: Option, + + /// Keep stored NARs under this size (e.g. 500G); 0 disables the quota. + #[arg(long, env = "HELIOS_QUOTA", default_value = "0", value_parser = parse_size)] + quota: u64, + + /// Start evicting above this fraction of the quota. + #[arg(long, env = "HELIOS_QUOTA_HIGH", default_value_t = 0.9)] + quota_high: f64, + + /// Evict down to this fraction of the quota. + #[arg(long, env = "HELIOS_QUOTA_LOW", default_value_t = 0.8)] + quota_low: f64, + + /// Also evict while the data filesystem has less free space than this (e.g. 20G); 0 disables. + #[arg(long, env = "HELIOS_MIN_FREE", default_value = "0", value_parser = parse_size)] + min_free: u64, + + /// Time between auto-GC checks. + #[arg(long, env = "HELIOS_GC_INTERVAL", default_value = "5m", value_parser = parse_duration)] + gc_interval: Duration, + + /// Time between full integrity scrubs; 0 disables. + #[arg(long, env = "HELIOS_SCRUB_INTERVAL", default_value = "7d", value_parser = parse_duration)] + scrub_interval: Duration, + + /// Scrub read rate limit (e.g. 64M per second); 0 is unlimited. + #[arg(long, env = "HELIOS_SCRUB_RATE", default_value = "64M", value_parser = parse_size)] + scrub_rate: u64, + + /// Time between WAL checkpoints. + #[arg(long, env = "HELIOS_CHECKPOINT_INTERVAL", default_value = "15m", value_parser = parse_duration)] + checkpoint_interval: Duration, + + /// Time between query planner statistics updates (PRAGMA optimize). + #[arg(long, env = "HELIOS_OPTIMIZE_INTERVAL", default_value = "1d", value_parser = parse_duration)] + optimize_interval: Duration, + + /// Time between database backups; 0 disables. + #[arg(long, env = "HELIOS_BACKUP_INTERVAL", default_value = "1d", value_parser = parse_duration)] + backup_interval: Duration, + + /// Database backups to keep. + #[arg(long, env = "HELIOS_BACKUP_KEEP", default_value_t = 7)] + backup_keep: usize, + + /// Serve Prometheus metrics on this address. + #[arg(long, env = "HELIOS_METRICS_LISTEN")] + metrics_listen: Option, +} + +/// Bytes with an optional binary suffix: 1024, 64M, 500G, 2T. +fn parse_size(s: &str) -> Result { + let s = s.trim(); + let (num, mult) = match s.char_indices().last() { + Some((i, c)) if c.is_ascii_alphabetic() => { + let mult = match c.to_ascii_uppercase() { + 'K' => 1u64 << 10, + 'M' => 1 << 20, + 'G' => 1 << 30, + 'T' => 1 << 40, + _ => return Err(format!("unknown size suffix in {s}")), + }; + (&s[..i], mult) + } + _ => (s, 1), + }; + let n: f64 = num.parse().map_err(|_| format!("invalid size {s}"))?; + if n < 0.0 { + return Err(format!("invalid size {s}")); + } + Ok((n * mult as f64) as u64) +} + +/// A duration with a unit: 30s, 5m, 24h, 7d. A bare 0 disables. +fn parse_duration(s: &str) -> Result { + let s = s.trim(); + if s == "0" { + return Ok(Duration::ZERO); + } + let split = s.find(|c: char| c.is_ascii_alphabetic()).ok_or_else(|| format!("{s}: missing unit (s, m, h or d)"))?; + let n: u64 = s[..split].parse().map_err(|_| format!("invalid duration {s}"))?; + let unit = match &s[split..] { + "s" => 1, + "m" => 60, + "h" => 3600, + "d" => 86400, + other => return Err(format!("unknown duration unit {other}")), + }; + Ok(Duration::from_secs(n * unit)) +} + +/// Runs `task` every `every`, logging and counting failures instead of +/// exiting: the server may be restarting. +async fn periodic(name: &'static str, every: Duration, delay_first: bool, errors: &AtomicU64, mut task: F) +where + F: FnMut() -> Fut, + Fut: Future>, +{ + if delay_first { + tokio::time::sleep(every).await; + } + let mut interval = tokio::time::interval(every); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + interval.tick().await; + if let Err(e) = task().instrument(tracing::info_span!("task", name)).await { + add(errors, 1); + tracing::error!(task = name, error = format!("{e:#}"), "task failed"); + } + } +} + +async fn shutdown() { + use tokio::signal::unix::{SignalKind, signal}; + let mut term = signal(SignalKind::terminate()).expect("installing SIGTERM handler"); + let term = std::pin::pin!(term.recv()); + let int = std::pin::pin!(tokio::signal::ctrl_c()); + futures_util::future::select(term, int).await; +} + +fn main() -> std::process::ExitCode { + helios_log::init(helios_log::Style::Service); + // Fatal errors go through the logger too, so JSON and journald get them. + match run() { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(e) => { + tracing::error!("{e:#}"); + std::process::ExitCode::FAILURE + } + } +} + +fn run() -> anyhow::Result<()> { + let args = Args::parse(); + anyhow::ensure!( + (0.0..=1.0).contains(&args.quota_low) && args.quota_low < args.quota_high && args.quota_high <= 1.0, + "watermarks must satisfy 0 <= low < high <= 1" + ); + let policy = autogc::Policy { quota: args.quota, high: args.quota_high, low: args.quota_low, min_free: args.min_free }; + // systemd's StateDirectory= may name several directories; the first is ours. + let state_dir = args.state_dir.clone().or_else(|| { + let dirs = std::env::var_os("STATE_DIRECTORY")?; + std::env::split_paths(&dirs).next().filter(|p| !p.as_os_str().is_empty()) + }); + if let Some(dir) = &state_dir { + std::fs::create_dir_all(dir).map_err(|e| anyhow::anyhow!("creating {}: {e}", dir.display()))?; + } + let d = Arc::new(Daemon { + client: client::Client::new(args.socket.clone()), + data_dir: args.data_dir.clone(), + state_dir, + metrics: Default::default(), + stopping: Default::default(), + }); + + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?; + let stopping = d.stopping.clone(); + let result = runtime.block_on(async move { + let mut tasks = Vec::new(); + if policy.quota > 0 || policy.min_free > 0 { + let d = d.clone(); + tasks.push(tokio::spawn(async move { + periodic("auto-gc", args.gc_interval.max(Duration::from_secs(1)), false, &d.metrics.errors_gc, || autogc::run(&d, &policy)).await + })); + } + if !args.scrub_interval.is_zero() { + let d = d.clone(); + let rate = args.scrub_rate; + // Wait a full interval first so restarts do not re-read + // everything, unless a pass was cut short. + let delay = !scrub::resuming(&d); + tasks.push(tokio::spawn( + async move { periodic("scrub", args.scrub_interval, delay, &d.metrics.errors_scrub, || scrub::run(&d, rate)).await }, + )); + } + { + let d = d.clone(); + tasks.push(tokio::spawn(async move { + periodic("checkpoint", args.checkpoint_interval.max(Duration::from_secs(1)), true, &d.metrics.errors_db, || maint::checkpoint(&d)) + .await + })); + } + { + let d = d.clone(); + tasks.push(tokio::spawn(async move { + periodic("optimize", args.optimize_interval.max(Duration::from_secs(1)), true, &d.metrics.errors_db, || maint::optimize(&d)).await + })); + } + if !args.backup_interval.is_zero() { + let d = d.clone(); + let keep = args.backup_keep; + tasks.push(tokio::spawn(async move { + periodic("backup", args.backup_interval, false, &d.metrics.errors_db, || maint::backup(&d, keep)).await + })); + } + if let Some(addr) = args.metrics_listen { + let d = d.clone(); + tasks.push(tokio::spawn(async move { + if let Err(e) = metrics::serve(d, addr).await { + tracing::error!(error = format!("{e:#}"), "metrics server failed"); + } + })); + } + tracing::info!(tasks = tasks.len(), "helios-daemon started"); + shutdown().await; + tracing::info!("shutting down"); + Ok(()) + }); + // Blocking work (a scrub read) sees this and stops; do not wait long + // for anything that does not. + stopping.store(true, std::sync::atomic::Ordering::Relaxed); + runtime.shutdown_timeout(Duration::from_secs(5)); + result +} + +#[cfg(test)] +mod tests { + use super::{parse_duration, parse_size}; + use std::time::Duration; + + #[test] + fn durations_and_sizes() { + assert_eq!(parse_duration("30s"), Ok(Duration::from_secs(30))); + assert_eq!(parse_duration("7d"), Ok(Duration::from_secs(7 * 86400))); + assert_eq!(parse_duration("0"), Ok(Duration::ZERO)); + assert!(parse_duration("5").is_err()); + assert!(parse_duration("5w").is_err()); + assert_eq!(parse_size("0"), Ok(0)); + assert_eq!(parse_size("1024"), Ok(1024)); + assert_eq!(parse_size("64M"), Ok(64 << 20)); + assert_eq!(parse_size("1.5g"), Ok(3 << 29)); + assert!(parse_size("5X").is_err()); + assert!(parse_size("-1").is_err()); + } +} diff --git a/crates/helios-daemon/src/maint.rs b/crates/helios-daemon/src/maint.rs new file mode 100644 index 0000000..fac804a --- /dev/null +++ b/crates/helios-daemon/src/maint.rs @@ -0,0 +1,29 @@ +//! SQLite upkeep, performed by the server on request: WAL checkpoints (so +//! the WAL does not grow without bound under constant reads), query +//! planner statistics, and rotated online backups. + +use serde_json::{Value, json}; + +use crate::state::{Daemon, add, now, set}; + +pub async fn checkpoint(d: &Daemon) -> anyhow::Result<()> { + let r: Value = d.client.post("/v1/db/checkpoint", json!({})).await?; + add(&d.metrics.db_checkpoints, 1); + if r["busy"] == true { + tracing::warn!("checkpoint could not complete while readers were active"); + } + Ok(()) +} + +pub async fn optimize(d: &Daemon) -> anyhow::Result<()> { + let _: Value = d.client.post("/v1/db/optimize", json!({})).await?; + Ok(()) +} + +pub async fn backup(d: &Daemon, keep: usize) -> anyhow::Result<()> { + let r: Value = d.client.post("/v1/db/backup", json!({ "keep": keep })).await?; + add(&d.metrics.db_backups, 1); + set(&d.metrics.db_last_backup, now()); + tracing::info!(path = r["path"].as_str().unwrap_or("?"), bytes = r["bytes"].as_u64(), "backed up the database"); + Ok(()) +} diff --git a/crates/helios-daemon/src/metrics.rs b/crates/helios-daemon/src/metrics.rs new file mode 100644 index 0000000..bfa37d9 --- /dev/null +++ b/crates/helios-daemon/src/metrics.rs @@ -0,0 +1,124 @@ +//! Prometheus text exposition on `--metrics-listen`: the server's counters +//! (fetched per scrape over the admin socket), the daemon's own, and disk +//! space of the data filesystem. + +use std::fmt::Write as _; +use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use bytes::Bytes; +use http::{Response, StatusCode, header}; +use http_body_util::Full; +use hyper::service::service_fn; +use serde_json::Value; + +use crate::io::Io; +use crate::state::Daemon; + +fn line(out: &mut String, name: &str, kind: &str, help: &str, samples: &[(&str, u64)]) { + let _ = writeln!(out, "# HELP {name} {help}\n# TYPE {name} {kind}"); + for (labels, v) in samples { + let _ = writeln!(out, "{name}{labels} {v}"); + } +} + +fn get(c: &AtomicU64) -> u64 { + c.load(Ordering::Relaxed) +} + +pub async fn render(d: &Daemon) -> String { + let mut out = String::new(); + let server: Option = d.client.get("/v1/stats").await.ok(); + line(&mut out, "helios_up", "gauge", "Whether helios-server answered on the admin socket.", &[("", u64::from(server.is_some()))]); + if let Some(s) = &server { + let n = |v: &Value| v.as_u64().unwrap_or(0); + let caches: Vec<(String, u64)> = s["caches"] + .as_array() + .into_iter() + .flatten() + .map(|c| (format!("{{cache=\"{}\"}}", c["name"].as_str().unwrap_or("")), n(&c["paths"]))) + .collect(); + let caches: Vec<(&str, u64)> = caches.iter().map(|(l, v)| (l.as_str(), *v)).collect(); + line(&mut out, "helios_cache_paths", "gauge", "Published paths per cache.", &caches); + line(&mut out, "helios_blobs", "gauge", "Stored NAR files.", &[("", n(&s["blobs"]["count"]))]); + line(&mut out, "helios_blob_bytes", "gauge", "Bytes of stored NAR files.", &[("", n(&s["blobs"]["bytes"]))]); + let r = &s["requests"]; + line( + &mut out, + "helios_narinfo_requests_total", + "counter", + "Narinfo lookups since the server started.", + &[("{result=\"hit\"}", n(&r["narinfoHits"])), ("{result=\"miss\"}", n(&r["narinfoMisses"]))], + ); + line(&mut out, "helios_nar_requests_total", "counter", "NAR download requests.", &[("", n(&r["nars"]))]); + line(&mut out, "helios_uploads_total", "counter", "Verified NAR uploads.", &[("", n(&r["uploads"]))]); + line(&mut out, "helios_upload_bytes_total", "counter", "Bytes of verified NAR uploads.", &[("", n(&r["uploadBytes"]))]); + line(&mut out, "helios_published_paths_total", "counter", "Paths published.", &[("", n(&r["publishedPaths"]))]); + } + if let Ok((free, total)) = crate::state::disk_space(&d.data_dir) { + line(&mut out, "helios_disk_free_bytes", "gauge", "Free bytes on the data filesystem.", &[("", free)]); + line(&mut out, "helios_disk_size_bytes", "gauge", "Size of the data filesystem.", &[("", total)]); + } + let m = &d.metrics; + line(&mut out, "helios_gc_runs_total", "counter", "Auto-GC checks.", &[("", get(&m.gc_runs))]); + line(&mut out, "helios_gc_evicted_paths_total", "counter", "Paths evicted by auto-GC.", &[("", get(&m.gc_evicted_paths))]); + line(&mut out, "helios_gc_freed_bytes_total", "counter", "NAR bytes freed by auto-GC.", &[("", get(&m.gc_freed_bytes))]); + line(&mut out, "helios_gc_last_run_timestamp_seconds", "gauge", "Last auto-GC check.", &[("", get(&m.gc_last_run))]); + line( + &mut out, + "helios_scrub_blobs_total", + "counter", + "Blobs checked by the integrity scrub.", + &[ + ("{result=\"ok\"}", get(&m.scrub_ok)), + ("{result=\"corrupt\"}", get(&m.scrub_corrupt)), + ("{result=\"missing\"}", get(&m.scrub_missing)), + ("{result=\"unreadable\"}", get(&m.scrub_unreadable)), + ], + ); + line(&mut out, "helios_scrub_bytes_total", "counter", "Bytes read by the integrity scrub.", &[("", get(&m.scrub_bytes))]); + line(&mut out, "helios_scrub_last_complete_timestamp_seconds", "gauge", "End of the last full scrub pass.", &[("", get(&m.scrub_last_complete))]); + line(&mut out, "helios_db_checkpoints_total", "counter", "WAL checkpoints.", &[("", get(&m.db_checkpoints))]); + line(&mut out, "helios_db_backups_total", "counter", "Database backups.", &[("", get(&m.db_backups))]); + line(&mut out, "helios_db_last_backup_timestamp_seconds", "gauge", "Last database backup.", &[("", get(&m.db_last_backup))]); + line( + &mut out, + "helios_maintenance_errors_total", + "counter", + "Failed maintenance runs.", + &[("{task=\"gc\"}", get(&m.errors_gc)), ("{task=\"scrub\"}", get(&m.errors_scrub)), ("{task=\"db\"}", get(&m.errors_db))], + ); + out +} + +pub async fn serve(d: Arc, addr: SocketAddr) -> anyhow::Result<()> { + let listener = tokio::net::TcpListener::bind(addr).await?; + tracing::info!(url = %format_args!("http://{addr}/metrics"), "metrics listening"); + loop { + // Accepting fails transiently (EMFILE, a connection reset before + // it was accepted); the endpoint must outlive that. + let stream = match listener.accept().await { + Ok((stream, _)) => stream, + Err(e) => { + tracing::warn!(error = %e, "accepting a metrics connection failed"); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + continue; + } + }; + let d = d.clone(); + tokio::spawn(async move { + let svc = service_fn(move |req: http::Request| { + let d = d.clone(); + async move { + if req.uri().path() == "/metrics" { + Response::builder().header(header::CONTENT_TYPE, "text/plain; version=0.0.4").body(Full::new(Bytes::from(render(&d).await))) + } else { + Response::builder().status(StatusCode::NOT_FOUND).body(Full::new(Bytes::from_static(b"not found\n"))) + } + } + }); + let _ = hyper::server::conn::http1::Builder::new().serve_connection(Io(stream), svc).await; + }); + } +} diff --git a/crates/helios-daemon/src/scrub.rs b/crates/helios-daemon/src/scrub.rs new file mode 100644 index 0000000..b494841 --- /dev/null +++ b/crates/helios-daemon/src/scrub.rs @@ -0,0 +1,293 @@ +//! Integrity scrub: re-reads every blob, decompresses and re-hashes it, and +//! compares the file hash, NAR hash and NAR size with the database. A blob +//! that is missing or corrupt is quarantined, which unpublishes its paths +//! so clients never download bad data. Reads are rate limited, and a pass +//! cut short resumes where it stopped. + +use std::io::Read; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use helios_core::{Compression, Verifier}; +use serde::Deserialize; +use serde_json::json; + +use crate::state::{Daemon, add, now, set}; + +#[derive(Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +struct Blob { + id: i64, + file_hash: String, + compression: String, + file_size: u64, + nar_hash: String, + nar_size: u64, + path: Option, +} + +#[derive(Deserialize)] +struct Page { + blobs: Vec, +} + +enum Verdict { + Ok, + Missing, + Corrupt(String), + /// The file is there but reading it failed (EIO, EACCES): not proof of + /// corruption, so it is counted and reported rather than quarantined. + Unreadable(String), + /// The daemon is shutting down. + Stopped, +} + +/// Holds reads to `rate` bytes per second (0 = unlimited) over a pass. +struct Throttle { + rate: u64, + started: Instant, + bytes: AtomicU64, +} + +impl Throttle { + /// Accounts for `n` bytes read, sleeping until they are due; false + /// once the daemon is stopping. + fn take(&self, n: u64, stopping: &AtomicBool) -> bool { + let total = self.bytes.fetch_add(n, Ordering::Relaxed) + n; + loop { + if stopping.load(Ordering::Relaxed) { + return false; + } + if self.rate == 0 { + return true; + } + let due = Duration::from_secs_f64(total as f64 / self.rate as f64); + match due.checked_sub(self.started.elapsed()) { + Some(wait) if !wait.is_zero() => std::thread::sleep(wait.min(Duration::from_millis(100))), + _ => return true, + } + } + } +} + +fn check(data_dir: &Path, b: &Blob, throttle: &Throttle, stopping: &AtomicBool) -> anyhow::Result { + let (Some(rel), Some(compression)) = (&b.path, Compression::parse(&b.compression)) else { + return Ok(Verdict::Corrupt(format!("unsupported compression {}", b.compression))); + }; + let mut file = match std::fs::File::open(data_dir.join(rel)) { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Verdict::Missing), + Err(e) => return Ok(Verdict::Unreadable(format!("opening: {e}"))), + }; + let mut v = Verifier::new(compression)?; + let mut buf = vec![0u8; 1 << 20]; + loop { + let n = match file.read(&mut buf) { + Ok(0) => break, + Ok(n) => n, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => return Ok(Verdict::Unreadable(format!("reading: {e}"))), + }; + if let Err(e) = v.update(&buf[..n]) { + return Ok(Verdict::Corrupt(format!("undecodable: {e}"))); + } + if !throttle.take(n as u64, stopping) { + return Ok(Verdict::Stopped); + } + } + let d = match v.finish() { + Ok(d) => d, + Err(e) => return Ok(Verdict::Corrupt(format!("undecodable: {e}"))), + }; + let problems: Vec<&str> = [ + (d.file_size != b.file_size, "file size"), + (helios_core::nix32_encode(&d.file_hash) != b.file_hash, "file hash"), + (helios_core::nix32_encode(&d.nar_hash) != b.nar_hash, "NAR hash"), + (d.nar_size != b.nar_size, "NAR size"), + ] + .into_iter() + .filter_map(|(bad, what)| bad.then_some(what)) + .collect(); + Ok(if problems.is_empty() { Verdict::Ok } else { Verdict::Corrupt(format!("{} mismatch", problems.join(", "))) }) +} + +/// The id of the last blob checked in the current pass, so a pass cut +/// short (by a restart, or a server that stayed away) resumes where it +/// stopped. Kept in the state directory when there is one. +static CURSOR: AtomicI64 = AtomicI64::new(0); +const CURSOR_FILE: &str = "scrub-cursor"; + +fn load_cursor(d: &Daemon) -> i64 { + let Some(dir) = &d.state_dir else { return CURSOR.load(Ordering::Relaxed) }; + match std::fs::read_to_string(dir.join(CURSOR_FILE)) { + Ok(text) => text.trim().parse().unwrap_or(0), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => 0, + Err(e) => { + tracing::warn!(error = %e, "reading the scrub position failed; starting over"); + 0 + } + } +} + +fn save_cursor(d: &Daemon, id: i64) { + CURSOR.store(id, Ordering::Relaxed); + let Some(dir) = &d.state_dir else { return }; + let tmp = dir.join(format!(".{CURSOR_FILE}")); + if let Err(e) = std::fs::write(&tmp, format!("{id}\n")).and_then(|()| std::fs::rename(&tmp, dir.join(CURSOR_FILE))) { + tracing::warn!(error = %e, dir = %dir.display(), "saving the scrub position failed"); + } +} + +/// Whether a pass was cut short and should resume now rather than after a +/// full interval. +pub fn resuming(d: &Daemon) -> bool { + load_cursor(d) > 0 +} + +/// Retries `f` through a short server outage (a restart) before giving up. +async fn retrying(what: &str, mut f: F) -> anyhow::Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let mut delay = Duration::from_secs(2); + for attempt in 1.. { + match f().await { + Ok(v) => return Ok(v), + Err(e) if attempt < ATTEMPTS => { + tracing::warn!(error = format!("{e:#}"), retry_in = ?delay, "{what} failed"); + tokio::time::sleep(delay).await; + delay *= 2; + } + Err(e) => return Err(e), + } + } + unreachable!() +} + +const ATTEMPTS: u32 = 5; +const SAVE_EVERY: Duration = Duration::from_secs(10); + +/// One full pass over every blob, resuming one cut short. `rate` is bytes +/// per second (0 = unlimited). +pub async fn run(d: &Daemon, rate: u64) -> anyhow::Result<()> { + let started = Instant::now(); + let throttle = Arc::new(Throttle { rate, started, bytes: AtomicU64::new(0) }); + let mut after = load_cursor(d); + if after > 0 { + tracing::info!(after, "resuming the scrub"); + } + let (mut bytes, mut blobs, mut saved) = (0u64, 0u64, Instant::now()); + loop { + let path = format!("/v1/blobs?after={after}&limit=500"); + let page: Page = retrying("listing blobs", || d.client.get(&path)).await?; + if page.blobs.is_empty() { + break; + } + for b in page.blobs { + let (dir, blob, throttle, stopping) = (d.data_dir.clone(), b.clone(), throttle.clone(), d.stopping.clone()); + let verdict = tokio::task::spawn_blocking(move || check(&dir, &blob, &throttle, &stopping)).await??; + let reason = match verdict { + Verdict::Stopped => return Ok(()), + Verdict::Ok => { + add(&d.metrics.scrub_ok, 1); + None + } + Verdict::Missing => { + add(&d.metrics.scrub_missing, 1); + Some("file missing".to_owned()) + } + Verdict::Corrupt(why) => { + add(&d.metrics.scrub_corrupt, 1); + Some(why) + } + Verdict::Unreadable(why) => { + add(&d.metrics.scrub_unreadable, 1); + tracing::warn!(blob = %b.file_hash, reason = %why, "unreadable blob, skipping"); + None + } + }; + blobs += 1; + bytes += b.file_size; + add(&d.metrics.scrub_bytes, b.file_size); + if let Some(reason) = reason { + tracing::error!(blob = %b.file_hash, %reason, "bad blob, quarantining"); + let body = json!({ "fileHash": b.file_hash, "reason": reason }); + // A blob GC removed since it was listed is already gone. + retrying("quarantining", || async { + match d.client.post::("/v1/quarantine", body.clone()).await { + Err(e) if e.to_string().contains("404") => Ok(()), + r => r.map(drop), + } + }) + .await?; + } + after = b.id; + if saved.elapsed() >= SAVE_EVERY { + save_cursor(d, after); + saved = Instant::now(); + } + } + save_cursor(d, after); + } + save_cursor(d, 0); + set(&d.metrics.scrub_last_complete, now()); + tracing::info!(blobs, bytes, elapsed = ?started.elapsed(), "scrub finished"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn blob(path: &str) -> Blob { + Blob { + id: 1, + file_hash: String::new(), + compression: "zstd".into(), + file_size: 0, + nar_hash: String::new(), + nar_size: 0, + path: Some(path.into()), + } + } + + #[test] + fn read_errors_are_not_corruption() { + let dir = std::env::temp_dir().join(format!("helios-scrub-{}", helios_core::uuid_v4())); + std::fs::create_dir_all(dir.join("nar")).unwrap(); + let throttle = Throttle { rate: 0, started: Instant::now(), bytes: AtomicU64::new(0) }; + let stopping = AtomicBool::new(false); + // Reading a directory fails with EISDIR. + assert!(matches!(check(&dir, &blob("nar"), &throttle, &stopping).unwrap(), Verdict::Unreadable(_))); + assert!(matches!(check(&dir, &blob("gone"), &throttle, &stopping).unwrap(), Verdict::Missing)); + std::fs::write(dir.join("junk"), b"not zstd").unwrap(); + assert!(matches!(check(&dir, &blob("junk"), &throttle, &stopping).unwrap(), Verdict::Corrupt(_))); + stopping.store(true, Ordering::Relaxed); + assert!(!throttle.take(1, &stopping)); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn the_scrub_position_survives_a_restart() { + let dir = std::env::temp_dir().join(format!("helios-scrub-{}", helios_core::uuid_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let d = Daemon { + client: crate::client::Client::new(dir.join("sock")), + data_dir: dir.clone(), + state_dir: Some(dir.clone()), + metrics: Default::default(), + stopping: Default::default(), + }; + assert!(!resuming(&d)); + save_cursor(&d, 42); + CURSOR.store(0, Ordering::Relaxed); + assert_eq!(load_cursor(&d), 42); + assert!(resuming(&d)); + save_cursor(&d, 0); + assert!(!resuming(&d)); + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/crates/helios-daemon/src/state.rs b/crates/helios-daemon/src/state.rs new file mode 100644 index 0000000..7e859ff --- /dev/null +++ b/crates/helios-daemon/src/state.rs @@ -0,0 +1,62 @@ +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + +use crate::client::Client; + +/// Daemon-side counters, exported on /metrics. +#[derive(Default)] +pub struct Metrics { + pub gc_runs: AtomicU64, + pub gc_evicted_paths: AtomicU64, + pub gc_freed_bytes: AtomicU64, + pub gc_last_run: AtomicU64, + pub scrub_ok: AtomicU64, + pub scrub_corrupt: AtomicU64, + pub scrub_missing: AtomicU64, + pub scrub_unreadable: AtomicU64, + pub scrub_bytes: AtomicU64, + pub scrub_last_complete: AtomicU64, + pub db_checkpoints: AtomicU64, + pub db_backups: AtomicU64, + pub db_last_backup: AtomicU64, + pub errors_gc: AtomicU64, + pub errors_scrub: AtomicU64, + pub errors_db: AtomicU64, +} + +pub fn add(counter: &AtomicU64, n: u64) { + counter.fetch_add(n, Ordering::Relaxed); +} + +pub fn set(counter: &AtomicU64, v: u64) { + counter.store(v, Ordering::Relaxed); +} + +pub fn now() -> u64 { + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs()) +} + +pub struct Daemon { + pub client: Client, + pub data_dir: PathBuf, + /// Where the daemon keeps its own state, such as the scrub position. + pub state_dir: Option, + pub metrics: Metrics, + /// Set on shutdown, for long blocking work to stop early. + pub stopping: Arc, +} + +/// Free and total bytes of the filesystem holding `path`. +// statvfs fields are c_ulong: the casts matter on 32-bit targets. +#[allow(clippy::unnecessary_cast)] +pub fn disk_space(path: &std::path::Path) -> std::io::Result<(u64, u64)> { + use std::os::unix::ffi::OsStrExt; + let c = std::ffi::CString::new(path.as_os_str().as_bytes())?; + let mut s: libc::statvfs = unsafe { std::mem::zeroed() }; + if unsafe { libc::statvfs(c.as_ptr(), &mut s) } != 0 { + return Err(std::io::Error::last_os_error()); + } + let frag = s.f_frsize as u64; + Ok((s.f_bavail as u64 * frag, s.f_blocks as u64 * frag)) +} diff --git a/crates/helios-log/Cargo.toml b/crates/helios-log/Cargo.toml new file mode 100644 index 0000000..3ec9470 --- /dev/null +++ b/crates/helios-log/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "helios-log" +description = "Logging setup shared by the helios binaries" +version.workspace = true +edition.workspace = true +license-file.workspace = true +publish.workspace = true + +[dependencies] +tracing = { version = "0.1.44", default-features = false, features = ["std"] } +tracing-journald = "0.3.2" +tracing-subscriber = { version = "0.3.23", default-features = false, features = ["std", "fmt", "ansi", "json"] } diff --git a/crates/helios-log/src/lib.rs b/crates/helios-log/src/lib.rs new file mode 100644 index 0000000..7acea2a --- /dev/null +++ b/crates/helios-log/src/lib.rs @@ -0,0 +1,115 @@ +//! Logging setup shared by the helios binaries, over `tracing`. +//! +//! `HELIOS_LOG` filters events: a level (`error`, `warn`, `info`, `debug`, +//! `trace`; default `info`), optionally with per-target directives such as +//! `info,helios_server=debug`. `HELIOS_LOG_FORMAT` picks the output: +//! +//! - `journald`: structured entries with priorities, event fields and span +//! fields as journal fields. The default when stderr is the journal. +//! - `json`: one object per line on stderr, for container log collectors. +//! - `text`: human-readable lines on stderr. The default otherwise. + +use std::io::IsTerminal; + +use tracing::{Event, Level, Subscriber}; +use tracing_subscriber::Layer; +use tracing_subscriber::filter::{LevelFilter, Targets}; +use tracing_subscriber::fmt::format::Writer; +use tracing_subscriber::fmt::{self, FmtContext, FormatEvent, FormatFields}; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::registry::{LookupSpan, Registry}; +use tracing_subscriber::util::SubscriberInitExt; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Style { + /// A long-running service: text lines carry a timestamp, level, spans + /// and target. + Service, + /// A command run by hand: text lines are the message alone, prefixed + /// with `warning:` or `error:` the way command-line tools report. + Cli, +} + +type Output = Box + Send + Sync>; + +/// Installs the global subscriber. Call once, first thing in `main`. +pub fn init(style: Style) { + let (filter, bad_filter) = match std::env::var("HELIOS_LOG") { + Ok(spec) if !spec.trim().is_empty() => match spec.parse::() { + Ok(filter) => (filter, None), + Err(e) => (default_filter(), Some(format!("ignoring HELIOS_LOG={spec:?}: {e}"))), + }, + _ => (default_filter(), None), + }; + + let format = std::env::var("HELIOS_LOG_FORMAT").unwrap_or_default(); + let (output, fallback) = match format.as_str() { + "json" => (json(), None), + "text" => (text(style), None), + "journald" => journald().map_or_else(|e| (text(style), Some(format!("journald unavailable, logging to stderr: {e}"))), |j| (j, None)), + "" | "auto" if stderr_is_journal() => journald().map_or_else(|_| (text(style), None), |j| (j, None)), + "" | "auto" => (text(style), None), + other => (text(style), Some(format!("unknown HELIOS_LOG_FORMAT {other:?}; expected text, json or journald"))), + }; + + let _ = tracing_subscriber::registry().with(output.with_filter(filter)).try_init(); + for problem in [bad_filter, fallback].into_iter().flatten() { + tracing::warn!("{problem}"); + } +} + +/// Whether stderr is the journal: systemd sets JOURNAL_STREAM to its +/// `device:inode`, but children inherit the variable, such as a shell +/// started from a service, so only a match counts. +fn stderr_is_journal() -> bool { + use std::os::fd::AsFd; + use std::os::unix::fs::MetadataExt; + let Some((dev, ino)) = std::env::var("JOURNAL_STREAM").ok().and_then(|s| s.split_once(':').map(|(d, i)| (d.parse::(), i.parse::()))) + else { + return false; + }; + let Ok(stderr) = std::io::stderr().as_fd().try_clone_to_owned() else { return false }; + let Ok(meta) = std::fs::File::from(stderr).metadata() else { return false }; + dev == Ok(meta.dev()) && ino == Ok(meta.ino()) +} + +fn default_filter() -> Targets { + Targets::new().with_default(LevelFilter::INFO) +} + +fn journald() -> std::io::Result { + Ok(Box::new(tracing_journald::layer()?)) +} + +fn json() -> Output { + Box::new(fmt::layer().json().flatten_event(true).with_current_span(true).with_span_list(false).with_writer(std::io::stderr)) +} + +fn text(style: Style) -> Output { + let ansi = std::io::stderr().is_terminal(); + match style { + Style::Service => Box::new(fmt::layer().with_ansi(ansi).with_writer(std::io::stderr)), + Style::Cli => Box::new(fmt::layer().event_format(CliFormat).with_ansi(ansi).with_writer(std::io::stderr)), + } +} + +/// `message key=value`, with a level prefix except on `info`. +struct CliFormat; + +impl FormatEvent for CliFormat +where + S: Subscriber + for<'a> LookupSpan<'a>, + N: for<'a> FormatFields<'a> + 'static, +{ + fn format_event(&self, ctx: &FmtContext<'_, S, N>, mut writer: Writer<'_>, event: &Event<'_>) -> std::fmt::Result { + match *event.metadata().level() { + Level::ERROR => write!(writer, "error: ")?, + Level::WARN => write!(writer, "warning: ")?, + Level::INFO => {} + Level::DEBUG => write!(writer, "debug: ")?, + Level::TRACE => write!(writer, "trace: ")?, + } + ctx.format_fields(writer.by_ref(), event)?; + writeln!(writer) + } +} diff --git a/crates/helios-server/Cargo.toml b/crates/helios-server/Cargo.toml new file mode 100644 index 0000000..8a6fd38 --- /dev/null +++ b/crates/helios-server/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "helios-server" +version.workspace = true +edition.workspace = true +license-file.workspace = true +publish.workspace = true + +[features] +default = [] +# Compile SQLite into the binary instead of linking the system library. +bundled-sqlite = ["rusqlite/bundled"] + +[dependencies] +anyhow = "1.0.104" +# HTTP/1.1 only: TLS and HTTP/2 end at the reverse proxy. +axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "json"] } +base64 = "0.23.1" +bytes = "1.12.1" +clap = { version = "4.6.7", default-features = false, features = ["std", "derive", "env", "help", "usage", "error-context"] } +futures-util = { version = "0.3.34", default-features = false, features = ["std"] } +helios-core = { path = "../helios-core" } +helios-log = { path = "../helios-log" } +quick_cache = { version = "0.7.0", default-features = false } +rusqlite = { version = "0.40.2", default-features = false, features = ["cache", "array", "fallible_uint"] } +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.151" +tokio = { version = "1.53.2", features = ["rt-multi-thread", "net", "fs", "signal", "sync", "time", "io-util"] } +tracing = { version = "0.1.44", default-features = false, features = ["std"] } diff --git a/crates/helios-server/src/admin.rs b/crates/helios-server/src/admin.rs new file mode 100644 index 0000000..ddb16e8 --- /dev/null +++ b/crates/helios-server/src/admin.rs @@ -0,0 +1,216 @@ +//! Admin API: caches and API tokens. Authenticated with the admin secret. + +use axum::Json; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use rusqlite::params; +use serde::Deserialize; +use serde_json::json; + +use crate::audit::Audit; +use crate::auth::{self, AUDIENCE, Claims, ISSUER, MAX_CACHES_PER_TOKEN, MAX_PERMS_PER_TOKEN, Perm}; +use crate::error::{ApiError, ApiResult}; +use crate::state::Locked; +use crate::state::{Shared, TokenState}; + +const MAX_TOKEN_LIFETIME_DAYS: i64 = 365; +const DEFAULT_TOKEN_LIFETIME_DAYS: i64 = 90; +const MAX_SUBJECT_LEN: usize = 256; +const MAX_REASON_LEN: usize = 512; + +pub(crate) fn valid_cache_name(name: &str) -> bool { + let b = name.as_bytes(); + (1..=64).contains(&b.len()) + && b.iter().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == b'-') + && b[0] != b'-' + && b[b.len() - 1] != b'-' +} + +#[derive(Deserialize)] +pub struct CreateCacheReq { + name: String, + #[serde(default = "default_true")] + public: bool, +} + +fn default_true() -> bool { + true +} + +pub async fn create_cache( + State(st): State, + audit: Audit, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult<(StatusCode, Json)> { + let who = st.authorize_admin(&headers)?; + if !valid_cache_name(&req.name) { + return Err(ApiError::bad_request("cache name must be 1-64 lowercase alphanumerics or hyphens, no leading/trailing hyphen")); + } + let st2 = st.clone(); + let name = req.name.clone(); + let created = tokio::task::spawn_blocking(move || st2.insert_cache(&name, req.public)).await??; + let status = if created.is_some() { StatusCode::CREATED } else { StatusCode::CONFLICT }; + audit.log(&st, &who, "cache.create", Some(&req.name), status, json!({ "public": req.public })); + match created { + Some(info) => Ok((status, Json(json!({ "name": req.name, "public": info.public })))), + None => Err(ApiError::new(status, "cache already exists")), + } +} + +pub async fn list_caches(State(st): State, headers: HeaderMap) -> ApiResult> { + st.authorize_admin(&headers)?; + let mut caches: Vec<_> = st.caches.rd().iter().map(|(name, c)| json!({ "name": name, "public": c.public })).collect(); + caches.sort_by(|a, b| a["name"].as_str().cmp(&b["name"].as_str())); + Ok(Json(json!({ "caches": caches }))) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CreateTokenReq { + subject: String, + caches: Vec, + perms: Vec, + expires_in_days: Option, +} + +pub async fn create_token( + State(st): State, + audit: Audit, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult<(StatusCode, Json)> { + let who = st.authorize_admin(&headers)?; + let secret = st.cfg.jwt_secret.clone().ok_or_else(|| ApiError::new(StatusCode::SERVICE_UNAVAILABLE, "token signing is not configured"))?; + if req.subject.is_empty() || req.subject.len() > MAX_SUBJECT_LEN { + return Err(ApiError::bad_request("subject must be 1-256 characters")); + } + let mut caches = req.caches; + caches.sort(); + caches.dedup(); + if caches.is_empty() || caches.len() > MAX_CACHES_PER_TOKEN || caches.iter().any(|c| c != "*" && !valid_cache_name(c)) { + return Err(ApiError::bad_request(format!("caches must be 1-{MAX_CACHES_PER_TOKEN} cache names or \"*\""))); + } + let mut perms = req.perms; + perms.sort(); + perms.dedup(); + if perms.is_empty() || perms.len() > MAX_PERMS_PER_TOKEN { + return Err(ApiError::bad_request("perms must be a non-empty list of \"pull\" and/or \"push\"")); + } + let days = req.expires_in_days.unwrap_or(DEFAULT_TOKEN_LIFETIME_DAYS); + if !(1..=MAX_TOKEN_LIFETIME_DAYS).contains(&days) { + return Err(ApiError::bad_request(format!("expiresInDays must be between 1 and {MAX_TOKEN_LIFETIME_DAYS}"))); + } + + let now = crate::db::now(); + let claims = Claims { + jti: helios_core::uuid_v4(), + sub: req.subject, + iss: ISSUER.into(), + aud: AUDIENCE.into(), + caches, + perms, + iat: now, + exp: now + days * 86400, + }; + let token = auth::sign(&claims, &secret); + let db = st.db.clone(); + let row = claims.clone(); + tokio::task::spawn_blocking(move || { + db.write(|conn| { + conn.execute( + "INSERT INTO tokens (jti, subject, caches, perms, created_at, expires_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + row.jti, + row.sub, + serde_json::to_string(&row.caches).unwrap_or_default(), + serde_json::to_string(&row.perms).unwrap_or_default(), + row.iat, + row.exp + ], + ) + }) + }) + .await??; + st.tokens.wr().insert(claims.jti.as_str().into(), TokenState { expires_at: claims.exp, revoked: false }); + audit.log(&st, &who, "token.create", None, StatusCode::CREATED, json!({ "jti": claims.jti, "subject": claims.sub })); + Ok(( + StatusCode::CREATED, + Json(json!({ + "token": token, + "jti": claims.jti, + "subject": claims.sub, + "caches": claims.caches, + "perms": claims.perms, + "expiresAt": claims.exp, + })), + )) +} + +pub async fn list_tokens(State(st): State, headers: HeaderMap) -> ApiResult> { + st.authorize_admin(&headers)?; + let db = st.db.clone(); + let tokens = tokio::task::spawn_blocking(move || { + db.read(|conn| { + let mut stmt = conn.prepare( + "SELECT jti, subject, caches, perms, created_at, expires_at, revoked_at, revoked_by, revocation_reason + FROM tokens ORDER BY created_at", + )?; + let rows = stmt.query_map([], |r| { + let caches: String = r.get(2)?; + let perms: String = r.get(3)?; + Ok(json!({ + "jti": r.get::<_, String>(0)?, + "subject": r.get::<_, String>(1)?, + "caches": serde_json::from_str::(&caches).unwrap_or_default(), + "perms": serde_json::from_str::(&perms).unwrap_or_default(), + "createdAt": r.get::<_, i64>(4)?, + "expiresAt": r.get::<_, i64>(5)?, + "revokedAt": r.get::<_, Option>(6)?, + "revokedBy": r.get::<_, Option>(7)?, + "revocationReason": r.get::<_, Option>(8)?, + })) + })?; + rows.collect::>>() + }) + }) + .await??; + Ok(Json(json!({ "tokens": tokens }))) +} + +#[derive(Deserialize)] +pub struct RevokeReq { + reason: String, +} + +pub async fn revoke_token( + State(st): State, + Path(jti): Path, + audit: Audit, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult> { + let who = st.authorize_admin(&headers)?; + if req.reason.is_empty() || req.reason.len() > MAX_REASON_LEN { + return Err(ApiError::bad_request("reason must be 1-512 characters")); + } + let db = st.db.clone(); + let id = jti.clone(); + let changed = tokio::task::spawn_blocking(move || { + db.write(|conn| { + conn.execute( + "UPDATE tokens SET revoked_at = ?2, revoked_by = 'admin', revocation_reason = ?3 WHERE jti = ?1 AND revoked_at IS NULL", + params![id, crate::db::now(), req.reason], + ) + }) + }) + .await??; + if changed == 0 { + return Err(ApiError::new(StatusCode::NOT_FOUND, "token not found or already revoked")); + } + if let Some(t) = st.tokens.wr().get_mut(jti.as_str()) { + t.revoked = true; + } + audit.log(&st, &who, "token.revoke", None, StatusCode::OK, json!({ "jti": jti })); + Ok(Json(json!({ "revoked": true, "jti": jti }))) +} diff --git a/crates/helios-server/src/audit.rs b/crates/helios-server/src/audit.rs new file mode 100644 index 0000000..2c1fef8 --- /dev/null +++ b/crates/helios-server/src/audit.rs @@ -0,0 +1,92 @@ +//! Audit logging off the request path: events go through a channel and are +//! inserted in batched transactions. + +use std::net::{IpAddr, SocketAddr}; +use std::time::Duration; + +use axum::extract::{ConnectInfo, FromRequestParts}; +use axum::http::StatusCode; +use axum::http::request::Parts; +use rusqlite::params; +use tokio::sync::mpsc; + +use crate::auth::Identity; +use crate::db::Db; +use crate::state::Shared; + +pub struct AuditEvent { + ts: i64, + actor: String, + action: &'static str, + cache: Option, + detail: String, + ip: Option, + status: u16, +} + +/// Request extractor carrying the client IP for audit events. +pub struct Audit { + ip: Option, +} + +impl FromRequestParts for Audit { + type Rejection = std::convert::Infallible; + + async fn from_request_parts(parts: &mut Parts, st: &Shared) -> Result { + // The proxy appends the peer it saw; everything left of that is + // whatever the client claimed, so only the last entry is trusted. + let forwarded = + st.cfg.trust_proxy.then(|| parts.headers.get("x-forwarded-for")?.to_str().ok()?.rsplit(',').next()?.trim().parse().ok()).flatten(); + let peer = parts.extensions.get::>().map(|c| c.0.ip()); + Ok(Self { ip: forwarded.or(peer) }) + } +} + +impl Audit { + pub fn log(&self, st: &Shared, who: &Identity, action: &'static str, cache: Option<&str>, status: StatusCode, detail: serde_json::Value) { + let _ = st.audit.send(AuditEvent { + ts: crate::db::now(), + actor: who.actor().to_owned(), + action, + cache: cache.map(str::to_owned), + detail: detail.to_string(), + ip: self.ip.map(|ip| ip.to_string()), + status: status.as_u16(), + }); + } +} + +pub async fn run(db: Db, mut rx: mpsc::UnboundedReceiver) { + let mut batch = Vec::with_capacity(256); + loop { + if rx.recv_many(&mut batch, 1024).await == 0 { + return; + } + // Coalesce bursts into one transaction. + tokio::time::sleep(Duration::from_millis(200)).await; + while let Ok(ev) = rx.try_recv() { + batch.push(ev); + } + let events = std::mem::take(&mut batch); + let db = db.clone(); + let result = tokio::task::spawn_blocking(move || { + db.write(|conn| -> rusqlite::Result<()> { + let tx = conn.transaction()?; + { + let mut stmt = tx + .prepare_cached("INSERT INTO audit_log (ts, actor, action, cache, detail, ip, status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)")?; + for e in &events { + stmt.execute(params![e.ts, e.actor, e.action, e.cache, e.detail, e.ip, e.status])?; + } + } + tx.commit() + }) + }) + .await; + match result { + Ok(Ok(())) => {} + Ok(Err(e)) => tracing::error!(error = %e, "writing audit log"), + Err(e) => tracing::error!(error = %e, "audit writer panicked"), + } + } +} diff --git a/crates/helios-server/src/auth.rs b/crates/helios-server/src/auth.rs new file mode 100644 index 0000000..2e84d12 --- /dev/null +++ b/crates/helios-server/src/auth.rs @@ -0,0 +1,207 @@ +//! HS256 API tokens and the admin bearer secret. Revocation is checked +//! against the in-memory token table, so it takes effect immediately. + +use axum::http::{HeaderMap, StatusCode, header}; +use base64::Engine; +use base64::engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD}; +use serde::{Deserialize, Serialize}; + +use crate::error::ApiError; +use crate::state::Locked; +use crate::state::{AppState, CacheInfo}; + +pub const ISSUER: &str = "helios-cache"; +pub const AUDIENCE: &str = "helios-cache"; +const HEADER_B64: &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"; // {"alg":"HS256","typ":"JWT"} +const MAX_TOKEN_BYTES: usize = 4096; +const MAX_CLOCK_SKEW: i64 = 60; +pub const MAX_CACHES_PER_TOKEN: usize = 50; +pub const MAX_PERMS_PER_TOKEN: usize = 10; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Perm { + Pull, + Push, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Claims { + pub jti: String, + pub sub: String, + pub iss: String, + pub aud: String, + pub caches: Vec, + pub perms: Vec, + pub iat: i64, + pub exp: i64, +} + +impl std::fmt::Display for Perm { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Perm::Pull => "pull", + Perm::Push => "push", + }) + } +} + +impl Claims { + pub fn allows(&self, cache: &str, perm: Perm) -> bool { + self.perms.contains(&perm) && self.caches.iter().any(|c| c == "*" || c == cache) + } +} + +pub enum Identity { + Token(Claims), + Admin, +} + +impl Identity { + pub fn actor(&self) -> &str { + match self { + Identity::Token(c) => &c.jti, + Identity::Admin => "admin", + } + } +} + +pub fn sign(claims: &Claims, secret: &[u8]) -> String { + let payload = URL_SAFE_NO_PAD.encode(serde_json::to_vec(claims).expect("claims serialise")); + let input = format!("{HEADER_B64}.{payload}"); + let sig = URL_SAFE_NO_PAD.encode(helios_core::hmac_sha256(secret, input.as_bytes())); + format!("{input}.{sig}") +} + +fn verify(token: &str, secret: &[u8], now: i64) -> Option { + if token.len() > MAX_TOKEN_BYTES { + return None; + } + let (signed, sig_b64) = token.rsplit_once('.')?; + let (header, payload_b64) = signed.split_once('.')?; + if header != HEADER_B64 || payload_b64.contains('.') { + return None; + } + let sig = URL_SAFE_NO_PAD.decode(sig_b64).ok()?; + if !helios_core::ct_eq(&helios_core::hmac_sha256(secret, signed.as_bytes()), &sig) { + return None; + } + + let claims: Claims = serde_json::from_slice(&URL_SAFE_NO_PAD.decode(payload_b64).ok()?).ok()?; + let valid = !claims.jti.is_empty() + && !claims.sub.is_empty() + && claims.iss == ISSUER + && claims.aud == AUDIENCE + && claims.iat <= now + MAX_CLOCK_SKEW + && claims.exp > claims.iat + && claims.exp > now - MAX_CLOCK_SKEW + && (1..=MAX_CACHES_PER_TOKEN).contains(&claims.caches.len()) + && (1..=MAX_PERMS_PER_TOKEN).contains(&claims.perms.len()) + && claims.caches.iter().all(|c| !c.is_empty()); + valid.then_some(claims) +} + +/// Bearer token, or the password of HTTP Basic auth (what Nix sends from netrc). +fn credential(headers: &HeaderMap) -> Option { + let value = headers.get(header::AUTHORIZATION)?.to_str().ok()?; + let (scheme, rest) = value.split_once(' ')?; + let rest = rest.trim(); + if scheme.eq_ignore_ascii_case("bearer") { + return Some(rest.to_owned()); + } + if scheme.eq_ignore_ascii_case("basic") { + let decoded = String::from_utf8(STANDARD.decode(rest).ok()?).ok()?; + let (_, password) = decoded.split_once(':')?; + return Some(password.to_owned()); + } + None +} + +fn unauthorized(message: &str) -> ApiError { + ApiError::new(StatusCode::UNAUTHORIZED, message) +} + +fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { + // Hash first so the comparison does not leak the secret's length. + helios_core::ct_eq(&helios_core::sha256(a), &helios_core::sha256(b)) +} + +impl AppState { + fn token_claims(&self, headers: &HeaderMap) -> Result { + let secret = self.cfg.jwt_secret.as_deref().ok_or_else(|| ApiError::new(StatusCode::FORBIDDEN, "token auth is not configured"))?; + let token = credential(headers).ok_or_else(|| unauthorized("a token is required"))?; + let now = crate::db::now(); + // Say why a token is refused: its holder already knows its claims, + // and a bare "unauthorized" leaves them guessing. + let claims = verify(&token, secret, now).ok_or_else(|| unauthorized("invalid or expired token"))?; + match self.tokens.rd().get(claims.jti.as_str()) { + Some(t) if t.revoked => Err(unauthorized("token has been revoked")), + Some(t) if t.expires_at > now => Ok(claims), + Some(_) => Err(unauthorized("token has expired")), + None => Err(unauthorized("token was not issued by this server")), + } + } + + /// Requires a token granting `perm` on `cache`. + pub fn authorize(&self, headers: &HeaderMap, cache: &str, perm: Perm) -> Result { + let claims = self.token_claims(headers)?; + if !claims.allows(cache, perm) { + return Err(ApiError::new(StatusCode::FORBIDDEN, format!("token does not grant {perm} on cache '{cache}'"))); + } + Ok(Identity::Token(claims)) + } + + /// Public caches are readable by anyone; private ones need `pull`. + pub fn authorize_read(&self, headers: &HeaderMap, name: &str, cache: CacheInfo) -> Result<(), ApiError> { + if cache.public { + return Ok(()); + } + self.authorize(headers, name, Perm::Pull).map(|_| ()) + } + + pub fn authorize_admin(&self, headers: &HeaderMap) -> Result { + let secret = self.cfg.admin_secret.as_deref().ok_or_else(|| ApiError::new(StatusCode::FORBIDDEN, "the admin API is not configured"))?; + let given = credential(headers).ok_or_else(|| unauthorized("the admin secret is required"))?; + if !constant_time_eq(given.as_bytes(), secret) { + return Err(ApiError::new(StatusCode::FORBIDDEN, "the admin API takes the admin secret, not a cache token")); + } + Ok(Identity::Admin) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn claims(now: i64) -> Claims { + Claims { + jti: "id".into(), + sub: "ci".into(), + iss: ISSUER.into(), + aud: AUDIENCE.into(), + caches: vec!["main".into()], + perms: vec![Perm::Push], + iat: now, + exp: now + 3600, + } + } + + #[test] + fn tokens() { + // The pre-encoded header must stay what it claims to be. + assert_eq!(URL_SAFE_NO_PAD.encode(br#"{"alg":"HS256","typ":"JWT"}"#), HEADER_B64); + let now = 1_700_000_000; + let token = sign(&claims(now), b"0123456789abcdef"); + assert!(verify(&token, b"0123456789abcdef", now).is_some()); + assert!(verify(&token, b"0123456789abcdeX", now).is_none()); + assert!(verify(&token, b"0123456789abcdef", now + 7200).is_none()); + let mut tampered = token.clone(); + tampered.insert(token.find('.').unwrap() + 3, 'x'); + assert!(verify(&tampered, b"0123456789abcdef", now).is_none()); + + let c = claims(0); + assert!(c.allows("main", Perm::Push)); + assert!(!c.allows("main", Perm::Pull)); + assert!(!c.allows("other", Perm::Push)); + } +} diff --git a/crates/helios-server/src/chunked.rs b/crates/helios-server/src/chunked.rs new file mode 100644 index 0000000..e4dd7ad --- /dev/null +++ b/crates/helios-server/src/chunked.rs @@ -0,0 +1,268 @@ +//! Chunked uploads, for NARs larger than a proxy in front of the server +//! accepts in one request (Cloudflare's free plan stops at 100 MB, Cloud +//! Run at 32 MiB). +//! +//! 1. `POST /_api/v2/caches/{cache}/uploads?compression=zstd` → `{id, offset: 0}` +//! 2. `PATCH /_api/v2/caches/{cache}/uploads/{id}?offset=N` with the next +//! chunk as the body → `{offset}`. A chunk is appended whole or not at +//! all, so after a failed request the client asks for the offset and +//! resends from there; a stale offset gets 409 with the current one. +//! 3. `GET /_api/v2/caches/{cache}/uploads/{id}` → `{offset}` +//! 4. `POST /_api/v2/caches/{cache}/uploads/{id}/complete` → as `PUT /nar` +//! 5. `DELETE /_api/v2/caches/{cache}/uploads/{id}` abandons the upload. +//! +//! Chunks are verified as they arrive, as in a single-request upload. +//! Sessions live in memory: after a server restart the client starts the +//! NAR again. A session idle for an hour is dropped with its file; until +//! then, GC leaves the file alone. + +use std::io::Write; +use std::path::PathBuf; +use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; + +use axum::Json; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode, Uri}; +use bytes::Bytes; +use helios_core::{Compression, Verifier}; +use serde_json::json; + +use crate::audit::Audit; +use crate::auth::{Identity, Perm}; +use crate::error::{ApiError, ApiResult}; +use crate::push::{UploadResp, cache_for, store_blob}; +use crate::state::Shared; + +const MAX_SESSIONS: usize = 1024; +/// Per token, so one client cannot take every slot. +const MAX_SESSIONS_PER_ACTOR: usize = 32; +const IDLE_TIMEOUT_SECS: i64 = 3600; +const FILE_PREFIX: &str = "upload-"; + +pub struct Session { + cache: Box, + actor: String, + compression: Compression, + tmp: PathBuf, + /// Bytes appended so far; read without waiting for a chunk in progress. + offset: AtomicU64, + /// Unix time of the last request. + touched: AtomicI64, + io: Mutex, +} + +struct Io { + file: std::fs::File, + /// `None` once the stream is found invalid or the upload is finished. + verifier: Option, +} + +impl Session { + fn idle(&self, now: i64) -> bool { + now - self.touched.load(Ordering::Relaxed) > IDLE_TIMEOUT_SECS + } +} + +pub type Sessions = Mutex, Arc>>; + +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { + m.lock().unwrap_or_else(PoisonError::into_inner) +} + +fn query<'a>(uri: &'a Uri, key: &str) -> Option<&'a str> { + uri.query()?.split('&').find_map(|kv| kv.strip_prefix(key)?.strip_prefix('=')) +} + +fn not_found() -> ApiError { + ApiError::new(StatusCode::NOT_FOUND, "upload not found; it expired or the server restarted, so start the NAR again") +} + +/// Drops idle sessions and their files. Returns how many. +pub fn expire(st: &Shared) -> usize { + let now = crate::db::now(); + let expired: Vec> = { + let mut sessions = lock(&st.uploads); + let ids: Vec> = sessions.iter().filter(|(_, s)| s.idle(now)).map(|(id, _)| id.clone()).collect(); + ids.iter().filter_map(|id| sessions.remove(id)).collect() + }; + for s in &expired { + let _ = std::fs::remove_file(&s.tmp); + } + expired.len() +} + +/// Whether a file in the tmp directory belongs to a live upload, which GC +/// must leave alone however old its mtime. +pub fn is_live(st: &Shared, file_name: &str) -> bool { + file_name.strip_prefix(FILE_PREFIX).is_some_and(|id| lock(&st.uploads).contains_key(id)) +} + +/// The session `id`, if it belongs to `cache` and to whoever is asking and +/// has not gone idle. +fn session(st: &Shared, cache: &str, id: &str, who: &Identity) -> ApiResult> { + let s = lock(&st.uploads).get(id).cloned().ok_or_else(not_found)?; + if &*s.cache != cache || s.actor != who.actor() { + return Err(not_found()); + } + let now = crate::db::now(); + if s.idle(now) { + drop_session(st, id); + return Err(not_found()); + } + s.touched.store(now, Ordering::Relaxed); + Ok(s) +} + +fn drop_session(st: &Shared, id: &str) { + if let Some(s) = lock(&st.uploads).remove(id) { + let _ = std::fs::remove_file(&s.tmp); + } +} + +pub async fn create( + State(st): State, + Path(name): Path, + uri: Uri, + headers: HeaderMap, +) -> ApiResult<(StatusCode, Json)> { + cache_for(&st, &name)?; + let who = st.authorize(&headers, &name, Perm::Push)?; + let compression = + Compression::parse(query(&uri, "compression").unwrap_or("zstd")).ok_or_else(|| ApiError::bad_request("compression must be zstd or none"))?; + expire(&st); + { + let sessions = lock(&st.uploads); + if sessions.len() >= MAX_SESSIONS { + return Err(ApiError::new(StatusCode::SERVICE_UNAVAILABLE, "too many uploads in progress")); + } + if sessions.values().filter(|s| s.actor == who.actor()).count() >= MAX_SESSIONS_PER_ACTOR { + return Err(ApiError::new( + StatusCode::TOO_MANY_REQUESTS, + format!("at most {MAX_SESSIONS_PER_ACTOR} uploads in progress per token; complete or abandon one"), + )); + } + } + + let id = helios_core::uuid_v4(); + let tmp = st.tmp_dir().join(format!("{FILE_PREFIX}{id}")); + let verifier = Verifier::new(compression).map_err(ApiError::internal)?; + let file = tokio::task::spawn_blocking({ + let tmp = tmp.clone(); + move || std::fs::File::create_new(tmp) + }) + .await? + .map_err(ApiError::internal)?; + let new = Session { + cache: name.into(), + actor: who.actor().to_owned(), + compression, + tmp, + offset: AtomicU64::new(0), + touched: AtomicI64::new(crate::db::now()), + io: Mutex::new(Io { file, verifier: Some(verifier) }), + }; + lock(&st.uploads).insert(id.as_str().into(), Arc::new(new)); + Ok((StatusCode::CREATED, Json(json!({ "id": id, "offset": 0 })))) +} + +pub async fn append( + State(st): State, + Path((name, id)): Path<(String, String)>, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> ApiResult> { + let at: u64 = query(&uri, "offset").and_then(|o| o.parse().ok()).ok_or_else(|| ApiError::bad_request("offset is required"))?; + let who = st.authorize(&headers, &name, Perm::Push)?; + let s = session(&st, &name, &id, &who)?; + let max = st.cfg.max_upload_bytes; + let result = tokio::task::spawn_blocking(move || -> ApiResult { + let mut io = lock(&s.io); + let io = &mut *io; + let offset = s.offset.load(Ordering::Acquire); + if at != offset { + return Err(ApiError::new(StatusCode::CONFLICT, "offset mismatch").with(json!({ "offset": offset }))); + } + if offset + body.len() as u64 > max { + return Err(ApiError::new(StatusCode::PAYLOAD_TOO_LARGE, "upload too large")); + } + let verifier = io.verifier.as_mut().ok_or_else(|| ApiError::new(StatusCode::CONFLICT, "upload is completing or was abandoned"))?; + if let Err(e) = verifier.update(&body) { + io.verifier = None; + return Err(ApiError::bad_request(format!("invalid NAR stream: {e}"))); + } + // A short write leaves the file ahead of the offset; later chunks + // would then land in the wrong place, so give up on the upload. + if let Err(e) = io.file.write_all(&body) { + io.verifier = None; + return Err(ApiError::internal(e)); + } + let offset = offset + body.len() as u64; + s.offset.store(offset, Ordering::Release); + Ok(offset) + }) + .await?; + match result { + Ok(offset) => Ok(Json(json!({ "offset": offset }))), + Err(e) => { + if e.status != StatusCode::CONFLICT { + drop_session(&st, &id); + } + Err(e) + } + } +} + +pub async fn status(State(st): State, Path((name, id)): Path<(String, String)>, headers: HeaderMap) -> ApiResult> { + let who = st.authorize(&headers, &name, Perm::Push)?; + let s = session(&st, &name, &id, &who)?; + Ok(Json(json!({ "offset": s.offset.load(Ordering::Acquire) }))) +} + +pub async fn complete( + State(st): State, + Path((name, id)): Path<(String, String)>, + audit: Audit, + headers: HeaderMap, +) -> ApiResult<(StatusCode, Json)> { + let cache_id = cache_for(&st, &name)?.id; + let who = st.authorize(&headers, &name, Perm::Push)?; + let s = session(&st, &name, &id, &who)?; + // Taken out of the map first, so a concurrent chunk cannot extend it. + lock(&st.uploads).remove(id.as_str()); + let (tmp, compression) = (s.tmp.clone(), s.compression); + let digest = tokio::task::spawn_blocking(move || { + let mut io = lock(&s.io); + match io.verifier.take() { + Some(v) => v.finish().map_err(|e| ApiError::bad_request(format!("invalid NAR stream: {e}"))), + None => Err(ApiError::bad_request("upload is no longer accepting data")), + } + .and_then(|d| io.file.sync_data().map(|()| d).map_err(ApiError::internal)) + }) + .await?; + let digest = match digest { + Ok(d) => d, + Err(err) => { + let _ = tokio::fs::remove_file(&tmp).await; + audit.log(&st, &who, "nar.upload", Some(&name), err.status, json!({ "error": err.message, "chunked": true })); + return Err(err); + } + }; + let resp = store_blob(&st, cache_id, tmp, digest, compression).await?; + audit.log( + &st, + &who, + "nar.upload", + Some(&name), + StatusCode::CREATED, + json!({ "fileHash": resp.file_hash, "size": resp.file_size, "chunked": true }), + ); + Ok((StatusCode::CREATED, Json(resp))) +} +pub async fn abort(State(st): State, Path((name, id)): Path<(String, String)>, headers: HeaderMap) -> ApiResult { + let who = st.authorize(&headers, &name, Perm::Push)?; + session(&st, &name, &id, &who)?; + drop_session(&st, &id); + Ok(StatusCode::NO_CONTENT) +} diff --git a/crates/helios-server/src/config.rs b/crates/helios-server/src/config.rs new file mode 100644 index 0000000..d095f8c --- /dev/null +++ b/crates/helios-server/src/config.rs @@ -0,0 +1,189 @@ +use std::net::SocketAddr; +use std::path::PathBuf; +use std::time::Duration; + +use clap::{Parser, Subcommand}; + +/// Helios Nix binary cache server. +#[derive(Parser, Debug, Clone)] +#[command(version)] +pub struct Args { + #[command(subcommand)] + pub command: Option, + + /// Address to listen on. + #[arg(long, env = "HELIOS_LISTEN", default_value = "127.0.0.1:8080")] + pub listen: SocketAddr, + + /// Directory for the database and NAR files. + #[arg(long, env = "HELIOS_DATA_DIR", default_value = "/var/lib/helios")] + pub data_dir: PathBuf, + + /// Nix secret key file (`nix key generate-secret`) used to sign narinfo. + #[arg(long, env = "HELIOS_SIGNING_KEY_FILE")] + pub signing_key_file: Option, + + /// File holding the HMAC secret for API tokens (JWT HS256). + #[arg(long, env = "HELIOS_JWT_SECRET_FILE")] + pub jwt_secret_file: Option, + + /// File holding the bearer secret for the admin API. + #[arg(long, env = "HELIOS_ADMIN_SECRET_FILE")] + pub admin_secret_file: Option, + + /// Serve NARs by delegating to a reverse proxy (nginx X-Accel-Redirect) + /// under this internal location, e.g. `/_nar`. The location must alias + /// `/nar/`. Lets the proxy use sendfile for zero-copy downloads. + #[arg(long, env = "HELIOS_ACCEL_REDIRECT")] + pub accel_redirect: Option, + + /// Trust X-Forwarded-For for audit log client IPs. + #[arg(long, env = "HELIOS_TRUST_PROXY", default_value_t = false)] + pub trust_proxy: bool, + + /// Rendered narinfo bodies kept in memory. + #[arg(long, env = "HELIOS_NARINFO_CACHE_ENTRIES", default_value_t = 262_144)] + pub narinfo_cache_entries: usize, + + /// Largest accepted compressed NAR upload, in bytes. + #[arg(long, env = "HELIOS_MAX_UPLOAD_BYTES", default_value_t = 64 << 30)] + pub max_upload_bytes: u64, + + /// Seconds an uploaded NAR is kept before it must be published. Protects + /// pushes in flight from GC and eviction. + #[arg(long, env = "HELIOS_UPLOAD_GRACE_SECONDS", default_value_t = 3600)] + pub upload_grace_seconds: u64, + + /// Hours between garbage collection runs. + #[arg(long, env = "HELIOS_GC_INTERVAL_HOURS", default_value_t = 6)] + pub gc_interval_hours: u64, + + /// Days to keep audit log entries. + #[arg(long, env = "HELIOS_AUDIT_RETENTION_DAYS", default_value_t = 30)] + pub audit_retention_days: u64, + + /// Unix socket for the maintenance API used by helios-daemon. Access is + /// controlled by the socket's permissions (created 0660). + #[arg(long, env = "HELIOS_ADMIN_SOCKET")] + pub admin_socket: Option, + + /// Caches to create at startup, comma-separated: `name` for a public + /// cache, `name:private` for a private one. A listed cache that exists + /// takes the listed visibility; unlisted caches are left alone. + #[arg(long, env = "HELIOS_CACHES", value_delimiter = ',', value_parser = parse_declared_cache)] + pub caches: Vec, + + /// zstd level clients compress with unless told otherwise (0 uploads + /// uncompressed). Level 3 with a 2^27 window gives NARs about 9% smaller + /// than plain level 2 while compressing faster than most uplinks. + #[arg(long, env = "HELIOS_COMPRESSION_LEVEL", default_value_t = 3, value_parser = clap::value_parser!(i32).range(0..=19))] + pub compression_level: i32, + + /// zstd window for long-distance matching clients use unless told + /// otherwise, as a power of two (10-27), or 0 for the level's own. + /// 27 (128 MiB) is the largest a stock Nix decoder accepts. + #[arg(long, env = "HELIOS_COMPRESSION_WINDOW_LOG", default_value_t = 27, value_parser = parse_window_log)] + pub compression_window_log: i32, + + /// Print the public key for `trusted-public-keys` and exit. + #[arg(long)] + pub print_public_key: bool, +} + +#[derive(Debug, Clone)] +pub struct DeclaredCache { + pub name: String, + pub public: bool, +} + +fn parse_window_log(s: &str) -> Result { + match s.parse::() { + Ok(n) if n == 0 || (10..=27).contains(&n) => Ok(n), + _ => Err("expected 0, or 10-27".into()), + } +} + +fn parse_declared_cache(s: &str) -> Result { + let (name, public) = match s.trim().split_once(':') { + None => (s.trim(), true), + Some((name, "public")) => (name, true), + Some((name, "private")) => (name, false), + Some((_, other)) => return Err(format!("unknown visibility {other:?}; expected public or private")), + }; + if !crate::admin::valid_cache_name(name) { + return Err(format!("invalid cache name {name:?}: 1-64 of a-z, 0-9 and -, not starting or ending with -")); + } + Ok(DeclaredCache { name: name.to_owned(), public }) +} + +#[derive(Subcommand, Debug, Clone)] +pub enum Command { + /// Create whichever of signing-key, jwt-secret and admin-secret are + /// missing in DIR (mode 0600), and write public-key. Safe to rerun. + GenerateSecrets { + #[arg(long, env = "HELIOS_SECRETS_DIR")] + dir: PathBuf, + /// Name for a new signing key, as it appears in `trusted-public-keys`. + #[arg(long, env = "HELIOS_KEY_NAME", default_value = "helios-1")] + key_name: String, + }, +} + +#[derive(Debug, Clone)] +pub struct Config { + pub data_dir: PathBuf, + pub jwt_secret: Option>, + pub admin_secret: Option>, + pub accel_redirect: Option, + pub trust_proxy: bool, + pub narinfo_cache_entries: usize, + pub max_upload_bytes: u64, + pub upload_grace: Duration, + pub gc_interval: Duration, + pub audit_retention: Duration, + pub compression_level: i32, + pub compression_window_log: i32, +} + +pub fn read_secret(path: &Option) -> anyhow::Result>> { + let Some(path) = path else { return Ok(None) }; + let raw = std::fs::read(path).map_err(|e| anyhow::anyhow!("reading {}: {e}", path.display()))?; + let trimmed = raw.trim_ascii().to_vec(); + anyhow::ensure!(trimmed.len() >= 16, "{} must hold at least 16 bytes", path.display()); + Ok(Some(trimmed)) +} + +impl Config { + pub fn from_args(args: &Args) -> anyhow::Result { + Ok(Self { + data_dir: args.data_dir.clone(), + jwt_secret: read_secret(&args.jwt_secret_file)?, + admin_secret: read_secret(&args.admin_secret_file)?, + accel_redirect: args.accel_redirect.as_ref().map(|s| s.trim_end_matches('/').to_owned()), + trust_proxy: args.trust_proxy, + narinfo_cache_entries: args.narinfo_cache_entries.max(16), + max_upload_bytes: args.max_upload_bytes, + upload_grace: Duration::from_secs(args.upload_grace_seconds), + gc_interval: Duration::from_secs(args.gc_interval_hours.max(1) * 3600), + audit_retention: Duration::from_secs(args.audit_retention_days * 86400), + compression_level: args.compression_level, + compression_window_log: args.compression_window_log, + }) + } +} + +#[cfg(test)] +mod tests { + use super::parse_declared_cache; + + #[test] + fn declared_caches() { + let c = parse_declared_cache("main").unwrap(); + assert_eq!((c.name.as_str(), c.public), ("main", true)); + assert!(!parse_declared_cache("team:private").unwrap().public); + assert!(parse_declared_cache("team:public").unwrap().public); + assert!(parse_declared_cache("team:secret").is_err()); + assert!(parse_declared_cache("Bad_Name").is_err()); + assert!(parse_declared_cache("").is_err()); + } +} diff --git a/crates/helios-server/src/db.rs b/crates/helios-server/src/db.rs new file mode 100644 index 0000000..a9b9514 --- /dev/null +++ b/crates/helios-server/src/db.rs @@ -0,0 +1,167 @@ +//! SQLite in WAL mode: one writer connection behind a mutex, and one +//! read-only connection per blocking-pool thread. + +use std::cell::RefCell; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use rusqlite::{Connection, OpenFlags}; +use std::sync::{Mutex, PoisonError}; + +const MIGRATIONS: &[&str] = &[ + r#" +CREATE TABLE caches ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + is_public INTEGER NOT NULL DEFAULT 1, + created_at INTEGER NOT NULL +); +CREATE TABLE blobs ( + id INTEGER PRIMARY KEY, + file_hash BLOB NOT NULL UNIQUE, + file_size INTEGER NOT NULL, + compression TEXT NOT NULL, + nar_hash BLOB NOT NULL, + nar_size INTEGER NOT NULL, + created_at INTEGER NOT NULL +); +CREATE INDEX blobs_nar_hash ON blobs (nar_hash); +CREATE TABLE paths ( + id INTEGER PRIMARY KEY, + cache_id INTEGER NOT NULL REFERENCES caches (id), + hash BLOB NOT NULL, + blob_id INTEGER NOT NULL REFERENCES blobs (id), + store_path TEXT NOT NULL, + narinfo BLOB NOT NULL, + created_at INTEGER NOT NULL +); +CREATE UNIQUE INDEX paths_cache_hash ON paths (cache_id, hash); +CREATE INDEX paths_blob ON paths (blob_id, cache_id); +CREATE TABLE tokens ( + jti TEXT PRIMARY KEY, + subject TEXT NOT NULL, + caches TEXT NOT NULL, + perms TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + revoked_at INTEGER, + revoked_by TEXT, + revocation_reason TEXT +); +CREATE TABLE audit_log ( + id INTEGER PRIMARY KEY, + ts INTEGER NOT NULL, + actor TEXT NOT NULL, + action TEXT NOT NULL, + cache TEXT, + detail TEXT NOT NULL, + ip TEXT, + status INTEGER NOT NULL +); +CREATE INDEX audit_log_ts ON audit_log (ts); +"#, + // Last narinfo hit per path, for least-recently-used eviction. + r#" +ALTER TABLE paths ADD COLUMN accessed_at INTEGER NOT NULL DEFAULT 0; +UPDATE paths SET accessed_at = created_at; +CREATE INDEX paths_accessed ON paths (accessed_at); +"#, + // Paths auto-GC must keep, with their closures. + r#" +CREATE TABLE pins ( + cache_id INTEGER NOT NULL REFERENCES caches (id), + hash BLOB NOT NULL, + store_path TEXT NOT NULL, + created_at INTEGER NOT NULL, + created_by TEXT NOT NULL, + PRIMARY KEY (cache_id, hash) +); +"#, + // Build traces of content-addressed derivations, rendered and signed. + r#" +CREATE TABLE build_traces ( + cache_id INTEGER NOT NULL REFERENCES caches (id), + id TEXT NOT NULL, + out_path TEXT NOT NULL, + body BLOB NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (cache_id, id) +); +"#, + // Caches each blob was uploaded to or published in: a cache may only + // reuse a blob it holds, or one any public cache serves. + r#" +CREATE TABLE blob_caches ( + blob_id INTEGER NOT NULL REFERENCES blobs (id), + cache_id INTEGER NOT NULL REFERENCES caches (id), + PRIMARY KEY (blob_id, cache_id) +) WITHOUT ROWID; +INSERT OR IGNORE INTO blob_caches (blob_id, cache_id) SELECT DISTINCT blob_id, cache_id FROM paths; +"#, +]; + +const PRAGMAS: &str = " +PRAGMA busy_timeout = 5000; +PRAGMA synchronous = NORMAL; +PRAGMA temp_store = MEMORY; +PRAGMA mmap_size = 4294967296; +PRAGMA cache_size = -65536; +"; + +#[derive(Clone)] +pub struct Db { + path: Arc, + writer: Arc>, +} + +thread_local! { + static READER: RefCell, Connection)>> = const { RefCell::new(None) }; +} + +impl Db { + pub fn open(path: &Path) -> anyhow::Result { + let mut conn = Connection::open(path)?; + conn.pragma_update(None, "journal_mode", "WAL")?; + conn.execute_batch(PRAGMAS)?; + migrate(&mut conn)?; + Ok(Self { path: Arc::new(path.to_owned()), writer: Arc::new(Mutex::new(conn)) }) + } + + /// Runs `f` on the writer connection. Blocking: call from spawn_blocking. + pub fn write(&self, f: impl FnOnce(&mut Connection) -> T) -> T { + f(&mut self.writer.lock().unwrap_or_else(PoisonError::into_inner)) + } + + /// Runs `f` on this thread's read-only connection. Blocking. + pub fn read(&self, f: impl FnOnce(&Connection) -> rusqlite::Result) -> rusqlite::Result { + READER.with(|cell| { + let mut slot = cell.borrow_mut(); + let stale = !matches!(&*slot, Some((p, _)) if Arc::ptr_eq(p, &self.path)); + if stale { + let conn = Connection::open_with_flags( + &*self.path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX | OpenFlags::SQLITE_OPEN_URI, + )?; + conn.execute_batch(PRAGMAS)?; + rusqlite::vtab::array::load_module(&conn)?; + *slot = Some((self.path.clone(), conn)); + } + f(&slot.as_ref().expect("reader initialised").1) + }) + } +} + +fn migrate(conn: &mut Connection) -> rusqlite::Result<()> { + let version: usize = conn.pragma_query_value(None, "user_version", |r| r.get(0))?; + for (i, sql) in MIGRATIONS.iter().enumerate().skip(version) { + let tx = conn.transaction()?; + tx.execute_batch(sql)?; + tx.pragma_update(None, "user_version", i + 1)?; + tx.commit()?; + } + Ok(()) +} + +pub fn now() -> i64 { + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs() as i64) +} diff --git a/crates/helios-server/src/error.rs b/crates/helios-server/src/error.rs new file mode 100644 index 0000000..2f14228 --- /dev/null +++ b/crates/helios-server/src/error.rs @@ -0,0 +1,64 @@ +use axum::http::{HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use serde_json::json; + +#[derive(Debug)] +pub struct ApiError { + pub status: StatusCode, + pub message: String, + pub extra: Option, +} + +impl ApiError { + pub fn new(status: StatusCode, message: impl Into) -> Self { + Self { status, message: message.into(), extra: None } + } + + pub fn bad_request(message: impl Into) -> Self { + Self::new(StatusCode::BAD_REQUEST, message) + } + + pub fn not_found() -> Self { + Self::new(StatusCode::NOT_FOUND, "not found") + } + + pub fn internal(err: impl std::fmt::Display) -> Self { + tracing::error!(error = %err, "internal error"); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, "internal error") + } + + pub fn with(mut self, extra: serde_json::Value) -> Self { + self.extra = Some(extra); + self + } +} + +impl From for ApiError { + fn from(err: rusqlite::Error) -> Self { + Self::internal(err) + } +} + +impl From for ApiError { + fn from(err: tokio::task::JoinError) -> Self { + Self::internal(err) + } +} + +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + let mut body = json!({ "error": self.message }); + if let (Some(serde_json::Value::Object(extra)), serde_json::Value::Object(map)) = (self.extra, &mut body) { + map.extend(extra); + } + let mut resp = (self.status, axum::Json(body)).into_response(); + let headers = resp.headers_mut(); + headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + if self.status == StatusCode::UNAUTHORIZED { + headers.insert(header::WWW_AUTHENTICATE, HeaderValue::from_static("Basic realm=\"helios\"")); + } + resp + } +} + +pub type ApiResult = Result; diff --git a/crates/helios-server/src/gc.rs b/crates/helios-server/src/gc.rs new file mode 100644 index 0000000..72a5abf --- /dev/null +++ b/crates/helios-server/src/gc.rs @@ -0,0 +1,151 @@ +//! Periodic garbage collection: stale temp uploads, unreferenced blobs past +//! a grace period, expired tokens, and old audit log entries. Also flushes +//! recorded narinfo hits to `paths.accessed_at` for LRU eviction. + +use std::time::{Duration, SystemTime}; + +use helios_core::Compression; +use rusqlite::params; + +use crate::state::Locked; +use crate::state::Shared; + +const TMP_MAX_AGE: Duration = Duration::from_secs(3600); +const BLOB_BATCH: usize = 1000; +const ACCESS_FLUSH: Duration = Duration::from_secs(60); + +#[derive(Debug, Default, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub struct GcStats { + pub tmp_files: usize, + pub blobs: usize, + pub bytes: u64, + pub tokens: usize, + pub audit_rows: usize, +} + +pub async fn run_forever(st: Shared) { + tokio::time::sleep(Duration::from_secs(60)).await; + let mut interval = tokio::time::interval(st.cfg.gc_interval); + loop { + interval.tick().await; + let st2 = st.clone(); + match tokio::task::spawn_blocking(move || collect(&st2)).await { + Ok(Ok(stats)) => tracing::info!(?stats, "gc finished"), + Ok(Err(e)) => tracing::error!(error = %e, "gc failed"), + Err(e) => tracing::error!(error = %e, "gc panicked"), + } + } +} + +pub fn collect(st: &Shared) -> anyhow::Result { + let mut stats = GcStats::default(); + let now = crate::db::now(); + + // Idle chunked uploads first: their files are then ordinary stale ones, + // and a live upload's file is skipped however old its mtime. + crate::chunked::expire(st); + if let Ok(entries) = std::fs::read_dir(st.tmp_dir()) { + for entry in entries.flatten() { + if crate::chunked::is_live(st, &entry.file_name().to_string_lossy()) { + continue; + } + let stale = entry + .metadata() + .and_then(|m| m.modified()) + .map(|t| SystemTime::now().duration_since(t).unwrap_or_default() > TMP_MAX_AGE) + .unwrap_or(false); + if stale && std::fs::remove_file(entry.path()).is_ok() { + stats.tmp_files += 1; + } + } + } + + (stats.blobs, stats.bytes) = collect_blobs(st)?; + + stats.tokens = st.db.write(|conn| conn.execute("DELETE FROM tokens WHERE expires_at < ?1", [now]))?; + st.tokens.wr().retain(|_, t| t.expires_at >= now); + + let audit_cutoff = now - st.cfg.audit_retention.as_secs() as i64; + stats.audit_rows = st.db.write(|conn| conn.execute("DELETE FROM audit_log WHERE ts < ?1", [audit_cutoff]))?; + Ok(stats) +} + +/// Deletes blobs no path references, once past the upload grace period, so +/// a push that uploads first and publishes afterwards is never raced. +/// Returns (blobs, bytes) removed. +pub fn collect_blobs(st: &Shared) -> anyhow::Result<(usize, u64)> { + let cutoff = crate::db::now() - st.cfg.upload_grace.as_secs() as i64; + let (mut total, mut freed) = (0usize, 0u64); + loop { + // Row delete and unlink share the writer lock with upload's + // rename+upsert, so a re-uploaded blob can never lose its file. + let n = st.db.write(|conn| -> anyhow::Result { + let tx = conn.transaction()?; + let victims: Vec<(i64, Vec, String, u64)> = { + let mut stmt = tx.prepare_cached( + "SELECT id, file_hash, compression, file_size FROM blobs b WHERE created_at <= ?1 + AND NOT EXISTS (SELECT 1 FROM paths p WHERE p.blob_id = b.id) LIMIT ?2", + )?; + stmt.query_map(params![cutoff, BLOB_BATCH], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))?.collect::>()? + }; + { + let mut del = tx.prepare_cached("DELETE FROM blobs WHERE id = ?1")?; + let mut unlink = tx.prepare_cached("DELETE FROM blob_caches WHERE blob_id = ?1")?; + for (id, ..) in &victims { + unlink.execute([id])?; + del.execute([id])?; + } + } + tx.commit()?; + for (_, hash, compression, size) in &victims { + let (Ok(hash), Some(compression)) = (<[u8; 32]>::try_from(hash.as_slice()), Compression::parse(compression)) else { + continue; + }; + match std::fs::remove_file(st.nar_path(&hash, compression)) { + Ok(()) => freed += size, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => tracing::warn!(error = %e, "removing blob file"), + } + } + Ok(victims.len()) + })?; + total += n; + if n < BLOB_BATCH { + return Ok((total, freed)); + } + } +} + +/// Writes recorded narinfo hits to `paths.accessed_at` in one transaction. +pub fn flush_access(st: &Shared) -> rusqlite::Result { + let keys = st.access.drain(); + if keys.is_empty() { + return Ok(0); + } + let now = crate::db::now(); + st.db.write(|conn| { + let tx = conn.transaction()?; + { + let mut stmt = tx.prepare_cached("UPDATE paths SET accessed_at = ?1 WHERE cache_id = ?2 AND hash = ?3")?; + for k in &keys { + stmt.execute(params![now, k.cache, &k.hash[..]])?; + } + } + tx.commit()?; + Ok(keys.len()) + }) +} + +pub async fn run_access_flush(st: Shared) { + let mut interval = tokio::time::interval(ACCESS_FLUSH); + loop { + interval.tick().await; + let st2 = st.clone(); + match tokio::task::spawn_blocking(move || flush_access(&st2)).await { + Ok(Ok(_)) => {} + Ok(Err(e)) => tracing::error!(error = %e, "flushing access times"), + Err(e) => tracing::error!(error = %e, "access flush panicked"), + } + } +} diff --git a/crates/helios-server/src/internal.rs b/crates/helios-server/src/internal.rs new file mode 100644 index 0000000..9df63a1 --- /dev/null +++ b/crates/helios-server/src/internal.rs @@ -0,0 +1,350 @@ +//! Maintenance API for helios-daemon, served only on a Unix socket +//! (`--admin-socket`) whose file permissions are the access control. +//! +//! The server stays the only process that changes the database, so its +//! in-memory index and narinfo cache always agree with it; the daemon +//! decides what to do and asks. + +use std::path::PathBuf; + +use axum::extract::State; +use axum::http::{StatusCode, Uri}; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use helios_core::Compression; +use rusqlite::{OptionalExtension, params}; +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::error::{ApiError, ApiResult}; +use crate::state::{Locked, PathKey, Shared}; + +pub fn router(st: Shared) -> Router { + Router::new() + .route("/v1/stats", get(stats)) + .route("/v1/lru", get(lru)) + .route("/v1/evict", post(evict)) + .route("/v1/blobs", get(blobs)) + .route("/v1/quarantine", post(quarantine)) + .route("/v1/gc", post(gc)) + .route("/v1/db/checkpoint", post(checkpoint)) + .route("/v1/db/optimize", post(optimize)) + .route("/v1/db/backup", post(backup)) + .with_state(st) +} + +async fn blocking(f: impl FnOnce() -> ApiResult + Send + 'static) -> ApiResult { + tokio::task::spawn_blocking(f).await? +} + +fn internal(e: impl std::fmt::Display) -> ApiError { + ApiError::internal(e) +} + +/// A numeric query parameter (the public API has no query strings to speak of). +fn param(uri: &Uri, name: &str) -> Option { + uri.query()?.split('&').find_map(|kv| kv.strip_prefix(name)?.strip_prefix('=')?.parse().ok()) +} + +async fn stats(State(st): State) -> ApiResult> { + let db_st = st.clone(); + let (caches, blob_count, blob_bytes, unreferenced) = blocking(move || { + db_st + .db + .read(|conn| { + let mut stmt = conn.prepare_cached( + "SELECT c.name, c.is_public, count(p.id) FROM caches c LEFT JOIN paths p ON p.cache_id = c.id GROUP BY c.id ORDER BY c.name", + )?; + let caches: Vec = stmt + .query_map([], |r| Ok(json!({ "name": r.get::<_, String>(0)?, "public": r.get::<_, bool>(1)?, "paths": r.get::<_, u64>(2)? })))? + .collect::>()?; + let (count, bytes): (u64, u64) = + conn.query_row("SELECT count(*), coalesce(sum(file_size), 0) FROM blobs", [], |r| Ok((r.get(0)?, r.get(1)?)))?; + // Blobs no path uses: GC deletes them once their upload grace ends. + let unreferenced: u64 = conn.query_row( + "SELECT coalesce(sum(file_size), 0) FROM blobs b WHERE NOT EXISTS (SELECT 1 FROM paths p WHERE p.blob_id = b.id)", + [], + |r| r.get(0), + )?; + Ok((caches, count, bytes, unreferenced)) + }) + .map_err(internal) + }) + .await?; + let c = &st.counters; + Ok(Json(json!({ + "caches": caches, + "blobs": { "count": blob_count, "bytes": blob_bytes, "unreferencedBytes": unreferenced }, + "indexedPaths": st.index.rd().len(), + "narinfoCacheEntries": st.narinfo.len(), + "requests": { + "narinfoHits": c.narinfo_hits.get(), + "narinfoMisses": c.narinfo_misses.get(), + "nars": c.nar_requests.get(), + "uploads": c.uploads.get(), + "uploadBytes": c.upload_bytes.get(), + "publishedPaths": c.published_paths.get(), + }, + }))) +} + +/// Least recently accessed paths first, with the size of their NAR. +/// Pinned paths and their closures are never candidates. +async fn lru(State(st): State, uri: Uri) -> ApiResult> { + let limit = param(&uri, "limit").unwrap_or(500).clamp(1, 10_000); + let paths = blocking(move || { + // Pending hits must count before anything is chosen for eviction. + crate::gc::flush_access(&st).map_err(internal)?; + st.db + .read(|conn| { + let protected = crate::pins::protected(conn)?; + // No LIMIT: rows stream, and pinned ones are skipped until + // enough candidates are found. + let mut stmt = conn.prepare_cached( + "SELECT p.cache_id, p.hash, p.store_path, p.accessed_at, b.file_size + FROM paths p JOIN blobs b ON b.id = p.blob_id ORDER BY p.accessed_at, p.id", + )?; + let mut rows = stmt.query([])?; + let mut out = Vec::new(); + while out.len() < limit + && let Some(r) = rows.next()? + { + let cache: u32 = r.get(0)?; + let hash: Vec = r.get(1)?; + if <[u8; 20]>::try_from(hash.as_slice()).is_ok_and(|hash| protected.contains(&PathKey { cache, hash })) { + continue; + } + out.push(json!({ + "cache": cache, + "hash": helios_core::nix32_encode(&hash), + "storePath": r.get::<_, String>(2)?, + "accessedAt": r.get::<_, i64>(3)?, + "narBytes": r.get::<_, u64>(4)?, + })); + } + Ok(out) + }) + .map_err(internal) + }) + .await?; + Ok(Json(json!({ "paths": paths }))) +} + +#[derive(Deserialize)] +struct EvictPath { + cache: u32, + hash: String, +} + +#[derive(Deserialize)] +struct EvictReq { + paths: Vec, +} + +fn forget(st: &Shared, keys: &[PathKey]) { + let mut index = st.index.wr(); + st.narinfo_epoch.fetch_add(1, std::sync::atomic::Ordering::Release); + for k in keys { + index.remove(k); + st.narinfo.remove(k); + } +} + +/// Deletes paths, then any blobs that no longer back a path. Pinned paths +/// are kept, even when asked for: a pin may be newer than the candidate list. +async fn evict(State(st): State, Json(req): Json) -> ApiResult> { + let mut keys: Vec = req + .paths + .iter() + .map(|p| helios_core::nix32_decode::<20>(&p.hash).map(|hash| PathKey { cache: p.cache, hash })) + .collect::>() + .ok_or_else(|| ApiError::bad_request("invalid path hash"))?; + blocking(move || { + let evicted = st.db.write(|conn| -> rusqlite::Result { + let tx = conn.transaction()?; + let protected = crate::pins::protected(&tx)?; + keys.retain(|k| !protected.contains(k)); + let mut n = 0; + { + let mut del = tx.prepare_cached("DELETE FROM paths WHERE cache_id = ?1 AND hash = ?2")?; + for k in &keys { + n += del.execute(params![k.cache, &k.hash[..]])?; + } + } + tx.commit()?; + // Under the writer lock, so a publish cannot interleave, and + // before the files go. + forget(&st, &keys); + Ok(n) + }); + let evicted = evicted.map_err(internal)?; + let (blobs, bytes) = crate::gc::collect_blobs(&st).map_err(internal)?; + tracing::info!(paths = evicted, blobs, bytes, "evicted"); + Ok(Json(json!({ "evicted": evicted, "freedBlobs": blobs, "freedBytes": bytes }))) + }) + .await +} + +/// Blobs in id order, for the integrity scrub. +async fn blobs(State(st): State, uri: Uri) -> ApiResult> { + let after: i64 = param(&uri, "after").unwrap_or(0); + let limit = param(&uri, "limit").unwrap_or(1000).clamp(1, 10_000); + let data_dir = st.cfg.data_dir.clone(); + let blobs = blocking(move || { + st.db + .read(|conn| { + let mut stmt = conn.prepare_cached( + "SELECT id, file_hash, compression, file_size, nar_hash, nar_size FROM blobs WHERE id > ?1 ORDER BY id LIMIT ?2", + )?; + stmt.query_map(params![after, limit], |r| { + let file_hash: Vec = r.get(1)?; + let nar_hash: Vec = r.get(4)?; + let compression: String = r.get(2)?; + let path = <[u8; 32]>::try_from(file_hash.as_slice()) + .ok() + .zip(Compression::parse(&compression)) + .map(|(h, c)| st.nar_path(&h, c).strip_prefix(&data_dir).map(PathBuf::from).unwrap_or_default()); + Ok(json!({ + "id": r.get::<_, i64>(0)?, + "fileHash": helios_core::nix32_encode(&file_hash), + "compression": compression, + "fileSize": r.get::<_, u64>(3)?, + "narHash": helios_core::nix32_encode(&nar_hash), + "narSize": r.get::<_, u64>(5)?, + "path": path, + })) + })? + .collect::>>() + }) + .map_err(internal) + }) + .await?; + Ok(Json(json!({ "blobs": blobs }))) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct QuarantineReq { + file_hash: String, + reason: String, +} + +/// Takes a corrupt or missing blob out of service: every path it backs is +/// unpublished, and its file is moved aside for inspection. +async fn quarantine(State(st): State, Json(req): Json) -> ApiResult> { + let file_hash = helios_core::nix32_decode::<32>(&req.file_hash).ok_or_else(|| ApiError::bad_request("invalid file hash"))?; + blocking(move || { + let found = st.db.write(|conn| -> rusqlite::Result>> { + let tx = conn.transaction()?; + let Some((id, compression)) = tx + .query_row("SELECT id, compression FROM blobs WHERE file_hash = ?1", [&file_hash[..]], |r| { + Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)) + }) + .optional()? + else { + return Ok(None); + }; + let keys: Vec = { + let mut stmt = tx.prepare_cached("SELECT cache_id, hash FROM paths WHERE blob_id = ?1")?; + stmt.query_map([id], |r| Ok((r.get::<_, u32>(0)?, r.get::<_, Vec>(1)?)))? + .filter_map(|row| row.ok().and_then(|(cache, h)| <[u8; 20]>::try_from(h.as_slice()).ok().map(|hash| PathKey { cache, hash }))) + .collect() + }; + tx.execute("DELETE FROM paths WHERE blob_id = ?1", [id])?; + tx.execute("DELETE FROM blob_caches WHERE blob_id = ?1", [id])?; + tx.execute("DELETE FROM blobs WHERE id = ?1", [id])?; + tx.commit()?; + forget(&st, &keys); + // Still under the writer lock: a re-upload of the same NAR must + // not have its good file moved aside by this. + if let Some(compression) = Compression::parse(&compression) { + let src = st.nar_path(&file_hash, compression); + let dir = st.cfg.data_dir.join("quarantine"); + let moved = std::fs::create_dir_all(&dir).and_then(|()| std::fs::rename(&src, dir.join(src.file_name().unwrap_or_default()))); + if let Err(e) = moved + && e.kind() != std::io::ErrorKind::NotFound + { + tracing::warn!(error = %e, "moving quarantined blob"); + } + } + Ok(Some(keys)) + }); + let Some(keys) = found.map_err(internal)? else { + return Err(ApiError::new(StatusCode::NOT_FOUND, "blob not found")); + }; + tracing::warn!(blob = %req.file_hash, reason = %req.reason, unpublished = keys.len(), "quarantined blob"); + Ok(Json(json!({ "pathsRemoved": keys.len() }))) + }) + .await +} + +async fn gc(State(st): State) -> ApiResult> { + blocking(move || { + let stats = crate::gc::collect(&st).map_err(internal)?; + Ok(Json(serde_json::to_value(stats).map_err(internal)?)) + }) + .await +} + +async fn checkpoint(State(st): State) -> ApiResult> { + blocking(move || { + let (busy, log, done) = st + .db + .write(|conn| { + conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?, r.get::<_, i64>(2)?))) + }) + .map_err(internal)?; + Ok(Json(json!({ "busy": busy != 0, "walPages": log, "checkpointedPages": done }))) + }) + .await +} + +async fn optimize(State(st): State) -> ApiResult> { + blocking(move || { + st.db.write(|conn| conn.execute_batch("PRAGMA optimize;")).map_err(internal)?; + Ok(Json(json!({ "optimized": true }))) + }) + .await +} + +#[derive(Deserialize)] +struct BackupReq { + /// Backups to retain, newest first. + keep: usize, +} + +/// A consistent online copy of the database under `/backups`, via +/// VACUUM INTO on a read connection so writers are not blocked. +async fn backup(State(st): State, Json(req): Json) -> ApiResult> { + blocking(move || { + let dir = st.cfg.data_dir.join("backups"); + std::fs::create_dir_all(&dir).map_err(internal)?; + // Milliseconds: VACUUM INTO refuses an existing file. Names still sort + // by age, after older backups named in seconds. + let millis = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_millis()); + let dest = dir.join(format!("helios-{millis}.db")); + let dest_str = dest.to_str().ok_or_else(|| internal("backup path is not UTF-8"))?.to_owned(); + if let Err(e) = st.db.read(|conn| conn.execute("VACUUM INTO ?1", [&dest_str])) { + // A partial file must not count towards --backup-keep. + let _ = std::fs::remove_file(&dest); + return Err(internal(e)); + } + let bytes = std::fs::metadata(&dest).map(|m| m.len()).unwrap_or(0); + + let mut existing: Vec = std::fs::read_dir(&dir) + .map_err(internal)? + .flatten() + .map(|e| e.path()) + .filter(|p| p.file_name().and_then(|n| n.to_str()).is_some_and(|n| n.starts_with("helios-") && n.ends_with(".db"))) + .collect(); + existing.sort(); + let mut removed = 0; + while existing.len() > req.keep.max(1) { + if std::fs::remove_file(existing.remove(0)).is_ok() { + removed += 1; + } + } + Ok(Json(json!({ "path": dest, "bytes": bytes, "removed": removed }))) + }) + .await +} diff --git a/crates/helios-server/src/main.rs b/crates/helios-server/src/main.rs new file mode 100644 index 0000000..9fa39cb --- /dev/null +++ b/crates/helios-server/src/main.rs @@ -0,0 +1,223 @@ +mod admin; +mod audit; +mod auth; +mod chunked; +mod config; +mod db; +mod error; +mod gc; +mod internal; +mod pins; +mod push; +mod read; +mod state; +mod stats; +mod traces; + +use std::net::SocketAddr; +use std::sync::Arc; + +use axum::Router; +use axum::extract::{DefaultBodyLimit, State}; +use axum::http::StatusCode; +use axum::routing::{get, post, put}; +use clap::Parser; +use serde_json::json; +use tracing::Instrument; + +use crate::config::{Args, Config}; +use crate::state::{AppState, Shared}; + +fn load_signer(args: &Args) -> anyhow::Result> { + let Some(path) = &args.signing_key_file else { return Ok(None) }; + let text = std::fs::read_to_string(path).map_err(|e| anyhow::anyhow!("reading {}: {e}", path.display()))?; + let signer = helios_core::Signer::new(&text).ok_or_else(|| anyhow::anyhow!("{} is not a Nix secret key", path.display()))?; + Ok(Some(signer)) +} + +pub fn router(st: Shared) -> Router { + let api = Router::new() + .route("/caches/{cache}", get(read::cache_info)) + .route("/caches/{cache}/missing", post(push::missing)) + .route("/caches/{cache}/nars/known", post(push::known)) + .route("/caches/{cache}/nar", put(push::upload).layer(DefaultBodyLimit::disable())) + .route("/caches/{cache}/build-traces", post(traces::publish)) + .route("/caches/{cache}/pins", get(pins::list).post(pins::add)) + .route("/caches/{cache}/pins/{path}", axum::routing::delete(pins::remove)) + .route("/caches/{cache}/uploads", post(chunked::create)) + .route("/caches/{cache}/uploads/{id}", get(chunked::status).patch(chunked::append).delete(chunked::abort)) + .route("/caches/{cache}/uploads/{id}/complete", post(chunked::complete)) + .route("/caches/{cache}/paths", post(push::publish)) + .route("/admin/caches", post(admin::create_cache).get(admin::list_caches)) + .route("/admin/tokens", post(admin::create_token).get(admin::list_tokens)) + .route("/admin/tokens/{jti}/revoke", post(admin::revoke_token)) + .layer(DefaultBodyLimit::max(64 << 20)); + + Router::new() + .route("/", get(|| async { axum::Json(json!({ "service": "helios", "status": "ok" })) })) + .route("/healthz", get(healthz)) + .nest("/_api/v2", api) + .fallback(read::handle) + .layer(axum::middleware::from_fn(trace_request)) + .with_state(st) +} + +/// Requests log at debug. Server errors log at warn with the request, which +/// the handler's own error event does not name. +async fn trace_request(req: axum::extract::Request, next: axum::middleware::Next) -> axum::response::Response { + let method = req.method().clone(); + let uri = req.uri().clone(); + let started = std::time::Instant::now(); + let resp = next.run(req).instrument(tracing::debug_span!("request", %method, path = uri.path())).await; + let status = resp.status(); + if status.is_server_error() { + tracing::warn!(%method, path = uri.path(), status = status.as_u16(), elapsed = ?started.elapsed(), "request failed"); + } else { + tracing::debug!(%method, path = uri.path(), status = status.as_u16(), elapsed = ?started.elapsed(), "request"); + } + resp +} + +async fn healthz(State(st): State) -> (StatusCode, axum::Json) { + let db = st.db.clone(); + let ok = tokio::task::spawn_blocking(move || db.read(|c| c.query_row("SELECT 1", [], |_| Ok(()))).is_ok()).await.unwrap_or(false); + let status = if ok { StatusCode::OK } else { StatusCode::SERVICE_UNAVAILABLE }; + (status, axum::Json(json!({ "ok": ok, "service": "helios" }))) +} + +pub async fn build_state(args: &Args) -> anyhow::Result { + let cfg = Config::from_args(args)?; + for dir in ["nar", "tmp"] { + let dir = cfg.data_dir.join(dir); + std::fs::create_dir_all(&dir).map_err(|e| anyhow::anyhow!("creating {}: {e}", dir.display()))?; + } + let db = db::Db::open(&cfg.data_dir.join("helios.db"))?; + let signer = load_signer(args)?; + if signer.is_none() { + tracing::warn!("no signing key configured; narinfo will be unsigned"); + } + let (audit_tx, audit_rx) = tokio::sync::mpsc::unbounded_channel(); + tokio::spawn(audit::run(db.clone(), audit_rx)); + let st = Arc::new(AppState::load(cfg, db, signer, audit_tx)?); + for c in &args.caches { + if let Some(change) = st.declare_cache(&c.name, c.public)? { + tracing::info!(cache = c.name, public = c.public, "declared cache {change}"); + } + } + tokio::spawn(gc::run_forever(st.clone())); + tokio::spawn(gc::run_access_flush(st.clone())); + Ok(st) +} + +fn main() -> std::process::ExitCode { + helios_log::init(helios_log::Style::Service); + // Fatal errors go through the logger too, so JSON and journald get them. + match run() { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(e) => { + tracing::error!("{e:#}"); + std::process::ExitCode::FAILURE + } + } +} + +fn run() -> anyhow::Result<()> { + let args = Args::parse(); + + if let Some(config::Command::GenerateSecrets { dir, key_name }) = &args.command { + return generate_secrets(dir, key_name); + } + if args.print_public_key { + let signer = load_signer(&args)?.ok_or_else(|| anyhow::anyhow!("--signing-key-file is required"))?; + println!("{}", signer.public_key()); + return Ok(()); + } + + tokio::runtime::Builder::new_multi_thread().enable_all().build()?.block_on(async move { + let st = build_state(&args).await?; + let (stop_tx, stop_rx) = tokio::sync::watch::channel(false); + tokio::spawn(async move { + shutdown_signal().await; + let _ = stop_tx.send(true); + }); + let stopped = |mut rx: tokio::sync::watch::Receiver| async move { + let _ = rx.wait_for(|stop| *stop).await; + }; + + let admin = match &args.admin_socket { + Some(path) => { + let _ = std::fs::remove_file(path); + let listener = tokio::net::UnixListener::bind(path)?; + // Group access only: the socket directory decides who that is. + std::fs::set_permissions(path, std::os::unix::fs::PermissionsExt::from_mode(0o660))?; + tracing::info!(socket = %path.display(), "maintenance API listening"); + let serve = axum::serve(listener, internal::router(st.clone())).with_graceful_shutdown(stopped(stop_rx.clone())); + Some(tokio::spawn(async move { serve.await })) + } + None => None, + }; + + let listener = tokio::net::TcpListener::bind(args.listen).await?; + tracing::info!(addr = %args.listen, "listening"); + axum::serve(listener, router(st.clone()).into_make_service_with_connect_info::()) + .with_graceful_shutdown(stopped(stop_rx)) + .await?; + if let Some(admin) = admin { + admin.await??; + } + // Keep the hits recorded since the last flush. + tokio::task::spawn_blocking(move || gc::flush_access(&st)).await??; + Ok(()) + }) +} + +/// SIGTERM (systemd stop) or SIGINT: finish in-flight requests, then exit. +async fn shutdown_signal() { + use tokio::signal::unix::{SignalKind, signal}; + let mut term = signal(SignalKind::terminate()).expect("installing SIGTERM handler"); + let term = std::pin::pin!(term.recv()); + let int = std::pin::pin!(tokio::signal::ctrl_c()); + futures_util::future::select(term, int).await; + tracing::info!("shutting down"); +} + +/// Writes `contents` to `path` only if it does not exist, readable by the +/// owner alone. Returns whether it was created. +fn create_secret(path: &std::path::Path, contents: impl FnOnce() -> anyhow::Result) -> anyhow::Result { + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; + if path.exists() { + return Ok(false); + } + let mut file = std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(path)?; + file.write_all(contents()?.as_bytes())?; + file.write_all(b"\n")?; + Ok(true) +} + +fn random_hex() -> anyhow::Result { + let mut bytes = [0u8; 32]; + helios_core::random_bytes(&mut bytes); + Ok(bytes.iter().map(|b| format!("{b:02x}")).collect()) +} + +fn generate_secrets(dir: &std::path::Path, key_name: &str) -> anyhow::Result<()> { + use std::os::unix::fs::PermissionsExt; + std::fs::create_dir_all(dir)?; + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))?; + let signing = dir.join("signing-key"); + for (name, created) in [ + ("signing-key", create_secret(&signing, || Ok(helios_core::Signer::generate(key_name)?))?), + ("jwt-secret", create_secret(&dir.join("jwt-secret"), random_hex)?), + ("admin-secret", create_secret(&dir.join("admin-secret"), random_hex)?), + ] { + if created { + tracing::info!(path = %dir.join(name).display(), "created secret"); + } + } + let key = std::fs::read_to_string(&signing)?; + let public = helios_core::Signer::new(&key).ok_or_else(|| anyhow::anyhow!("{} is not a Nix secret key", signing.display()))?.public_key(); + std::fs::write(dir.join("public-key"), format!("{public}\n"))?; + println!("{public}"); + Ok(()) +} diff --git a/crates/helios-server/src/pins.rs b/crates/helios-server/src/pins.rs new file mode 100644 index 0000000..76eb76a --- /dev/null +++ b/crates/helios-server/src/pins.rs @@ -0,0 +1,168 @@ +//! Pins: store paths auto-GC must not evict, with their closures in the +//! cache. A pinned system is no use without its dependencies, so a pin +//! protects everything it references, directly or not, including +//! dependencies pushed after the pin. +//! +//! - `GET /_api/v2/caches/{cache}/pins` (pull) +//! - `POST /_api/v2/caches/{cache}/pins` `{"storePaths": [...]}` (push) +//! - `DELETE /_api/v2/caches/{cache}/pins/{store path basename}` (push) +//! +//! Pins only hold back eviction. A path the integrity scrub finds corrupt +//! is still unpublished. + +use std::collections::HashSet; + +use axum::Json; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use rusqlite::{Connection, params}; +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::audit::Audit; +use crate::auth::Perm; +use crate::error::{ApiError, ApiResult}; +use crate::push::cache_for; +use crate::state::{Locked, PathKey, Shared}; + +const MAX_PINS_PER_REQUEST: usize = 1000; +/// Each eviction walks every pinned closure under the writer lock. +const MAX_PINS_PER_CACHE: i64 = 10_000; + +/// `/nix/store/-` or its basename, as (hash, basename). +fn parse_path(s: &str) -> Option<([u8; 20], &str)> { + let base = s.strip_prefix("/nix/store/").unwrap_or(s); + if !helios_core::store_basename_valid(base) { + return None; + } + Some((helios_core::nix32_decode::<20>(&base[..32])?, base)) +} + +/// Every path a pin protects: pinned paths and their closures, following +/// the References of each narinfo within its cache. +pub fn protected(conn: &Connection) -> rusqlite::Result> { + let mut seen = HashSet::new(); + let mut todo: Vec = conn + .prepare_cached("SELECT cache_id, hash FROM pins")? + .query_map([], |r| Ok((r.get::<_, u32>(0)?, r.get::<_, Vec>(1)?)))? + .filter_map(|row| row.ok().and_then(|(cache, hash)| Some(PathKey { cache, hash: hash.try_into().ok()? }))) + .collect(); + let mut narinfo = conn.prepare_cached("SELECT narinfo FROM paths WHERE cache_id = ?1 AND hash = ?2")?; + while let Some(key) = todo.pop() { + if !seen.insert(key) { + continue; + } + let text: Option> = match narinfo.query_row(params![key.cache, &key.hash[..]], |r| r.get(0)) { + Ok(t) => Some(t), + Err(rusqlite::Error::QueryReturnedNoRows) => None, // Pinned before it was pushed. + Err(e) => return Err(e), + }; + let Some(text) = text else { continue }; + let refs = std::str::from_utf8(&text).unwrap_or("").lines().find_map(|l| l.strip_prefix("References:")).unwrap_or(""); + for r in refs.split_ascii_whitespace() { + if let Some((hash, _)) = parse_path(r) { + todo.push(PathKey { cache: key.cache, hash }); + } + } + } + Ok(seen) +} + +pub async fn list(State(st): State, Path(name): Path, headers: HeaderMap) -> ApiResult> { + let cache = cache_for(&st, &name)?; + st.authorize(&headers, &name, Perm::Pull)?; + let pins = tokio::task::spawn_blocking(move || { + st.db.read(|conn| { + conn.prepare_cached("SELECT store_path, created_at, created_by FROM pins WHERE cache_id = ?1 ORDER BY store_path")? + .query_map([cache.id], |r| { + Ok(json!({ "storePath": format!("/nix/store/{}", r.get::<_, String>(0)?), "createdAt": r.get::<_, i64>(1)?, "createdBy": r.get::<_, String>(2)? })) + })? + .collect::>>() + }) + }) + .await??; + Ok(Json(json!({ "pins": pins }))) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AddReq { + store_paths: Vec, +} + +pub async fn add( + State(st): State, + Path(name): Path, + audit: Audit, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult<(StatusCode, Json)> { + let cache = cache_for(&st, &name)?; + let who = st.authorize(&headers, &name, Perm::Push)?; + if req.store_paths.is_empty() || req.store_paths.len() > MAX_PINS_PER_REQUEST { + return Err(ApiError::bad_request(format!("storePaths must hold 1-{MAX_PINS_PER_REQUEST} store paths"))); + } + let parsed: Vec<([u8; 20], String)> = req + .store_paths + .iter() + .map(|p| parse_path(p).map(|(h, base)| (h, base.to_owned())).ok_or_else(|| ApiError::bad_request(format!("invalid store path: {p}")))) + .collect::>()?; + // Only published paths: a pin stops at what the cache lacks, so pinning + // ahead of the push would protect less than it seems to. + let missing: Vec = { + let index = st.index.rd(); + parsed.iter().filter(|(hash, _)| !index.contains(&PathKey { cache: cache.id, hash: *hash })).map(|(_, b)| format!("/nix/store/{b}")).collect() + }; + if !missing.is_empty() { + return Err( + ApiError::new(StatusCode::CONFLICT, "pin paths after pushing them; these are not in the cache").with(json!({ "missing": missing })) + ); + } + let actor = who.actor().to_owned(); + let st2 = st.clone(); + let added = tokio::task::spawn_blocking(move || { + st2.db.write(|conn| -> ApiResult { + let tx = conn.transaction()?; + let pinned: i64 = tx.query_row("SELECT count(*) FROM pins WHERE cache_id = ?1", [cache.id], |r| r.get(0))?; + if pinned + parsed.len() as i64 > MAX_PINS_PER_CACHE { + return Err(ApiError::bad_request(format!("a cache holds at most {MAX_PINS_PER_CACHE} pins; it has {pinned}"))); + } + let mut n = 0; + { + let mut insert = tx.prepare_cached( + "INSERT INTO pins (cache_id, hash, store_path, created_at, created_by) VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT (cache_id, hash) DO NOTHING", + )?; + for (hash, base) in &parsed { + n += insert.execute(params![cache.id, &hash[..], base, crate::db::now(), actor])?; + } + } + tx.commit()?; + Ok(n) + }) + }) + .await??; + audit.log(&st, &who, "pin.add", Some(&name), StatusCode::CREATED, json!({ "pinned": added, "requested": req.store_paths.len() })); + Ok((StatusCode::CREATED, Json(json!({ "pinned": added, "alreadyPinned": req.store_paths.len() - added })))) +} + +pub async fn remove( + State(st): State, + Path((name, path)): Path<(String, String)>, + audit: Audit, + headers: HeaderMap, +) -> ApiResult> { + let cache = cache_for(&st, &name)?; + let who = st.authorize(&headers, &name, Perm::Push)?; + let (hash, _) = parse_path(&path).ok_or_else(|| ApiError::bad_request(format!("invalid store path: {path}")))?; + let st2 = st.clone(); + let removed = tokio::task::spawn_blocking(move || { + st2.db.write(|conn| conn.execute("DELETE FROM pins WHERE cache_id = ?1 AND hash = ?2", params![cache.id, &hash[..]])) + }) + .await??; + if removed == 0 { + return Err(ApiError::new(StatusCode::NOT_FOUND, "not pinned")); + } + audit.log(&st, &who, "pin.remove", Some(&name), StatusCode::OK, json!({ "storePath": path })); + Ok(Json(json!({ "unpinned": removed }))) +} diff --git a/crates/helios-server/src/push.rs b/crates/helios-server/src/push.rs new file mode 100644 index 0000000..7f20930 --- /dev/null +++ b/crates/helios-server/src/push.rs @@ -0,0 +1,537 @@ +//! The push API. +//! +//! 1. `POST /_api/v2/caches/{cache}/missing` → which store paths are absent (memory only). +//! 2. `POST /_api/v2/caches/{cache}/nars/known` → which NAR hashes already have a blob. +//! 3. `PUT /_api/v2/caches/{cache}/nar?compression=zstd` → stream a compressed NAR. +//! The server hashes and decompresses it on the fly and records the verified +//! NAR hash; clients cannot claim content they did not upload. +//! 4. `POST /_api/v2/caches/{cache}/paths` → publish a batch in one transaction. + +use std::io::Write; +use std::rc::Rc; + +use axum::Json; +use axum::body::Body; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode, Uri}; +use base64::Engine; +use bytes::Bytes; +use futures_util::StreamExt; +use helios_core::{Compression, NarinfoInput, Verifier}; +use rusqlite::types::Value; +use rusqlite::{OptionalExtension, params}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +use crate::audit::Audit; +use crate::auth::Perm; +use crate::error::{ApiError, ApiResult}; +use crate::state::Locked; +use crate::state::{CacheInfo, PathKey, Shared}; + +const MAX_MISSING_BATCH: usize = 100_000; +const MAX_PUBLISH_BATCH: usize = 5_000; +const STORE_DIR: &str = "/nix/store/"; + +/// SQL condition on blob `b`: the cache bound to parameter `?n` may reuse +/// it without uploading. That holds for a blob uploaded to or published in +/// that cache, and for one a public cache serves, which anyone can fetch. +/// Anything else would let a push token for one cache publish, and so +/// read, another cache's private NARs by their hash. +fn reusable(n: u8) -> String { + format!( + "(EXISTS (SELECT 1 FROM blob_caches bc WHERE bc.blob_id = b.id AND bc.cache_id = ?{n}) + OR EXISTS (SELECT 1 FROM paths p JOIN caches c ON c.id = p.cache_id WHERE p.blob_id = b.id AND c.is_public = 1))" + ) +} + +pub(crate) fn cache_for(st: &Shared, name: &str) -> ApiResult { + st.cache(name).ok_or_else(|| ApiError::new(StatusCode::NOT_FOUND, "cache not found")) +} + +// ── Missing paths ── + +#[derive(Deserialize)] +pub struct MissingReq { + hashes: Vec, +} + +pub async fn missing( + State(st): State, + Path(name): Path, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult> { + let cache = cache_for(&st, &name)?; + st.authorize(&headers, &name, Perm::Push)?; + if req.hashes.len() > MAX_MISSING_BATCH { + return Err(ApiError::bad_request(format!("at most {MAX_MISSING_BATCH} hashes per request"))); + } + let index = st.index.rd(); + let missing: Vec<&String> = req + .hashes + .iter() + .filter(|h| match helios_core::nix32_decode::<20>(h) { + Some(hash) => !index.contains(&PathKey { cache: cache.id, hash }), + None => true, + }) + .collect(); + Ok(Json(json!({ "missing": missing }))) +} + +// ── Known NARs ── + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct KnownReq { + nar_hashes: Vec, +} + +pub async fn known( + State(st): State, + Path(name): Path, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult> { + let cache = cache_for(&st, &name)?; + st.authorize(&headers, &name, Perm::Push)?; + if req.nar_hashes.len() > MAX_MISSING_BATCH { + return Err(ApiError::bad_request(format!("at most {MAX_MISSING_BATCH} hashes per request"))); + } + let parsed: Vec> = req.nar_hashes.iter().map(|h| parse_sha256(h)).collect(); + let db = st.db.clone(); + let values: Vec = parsed.iter().flatten().map(|h| Value::Blob(h.to_vec())).collect(); + let found = tokio::task::spawn_blocking(move || { + db.read(|conn| { + let mut stmt = conn.prepare_cached(&format!("SELECT DISTINCT nar_hash FROM blobs b WHERE nar_hash IN rarray(?1) AND {}", reusable(2)))?; + let rows = stmt.query_map(params![Rc::new(values), cache.id], |r| r.get::<_, Vec>(0))?; + rows.collect::>>>() + }) + }) + .await??; + let known: Vec<&String> = req.nar_hashes.iter().zip(&parsed).filter(|(_, h)| h.is_some_and(|h| found.contains(&h[..]))).map(|(s, _)| s).collect(); + Ok(Json(json!({ "known": known }))) +} + +// ── Upload ── + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UploadResp { + pub(crate) file_hash: String, + pub(crate) file_size: u64, + nar_hash: String, + nar_size: u64, +} + +pub async fn upload( + State(st): State, + Path(name): Path, + uri: Uri, + audit: Audit, + headers: HeaderMap, + body: Body, +) -> ApiResult<(StatusCode, Json)> { + let cache = cache_for(&st, &name)?; + let identity = st.authorize(&headers, &name, Perm::Push)?; + let requested = uri.query().unwrap_or("").split('&').find_map(|kv| kv.strip_prefix("compression=")).unwrap_or("zstd"); + let compression = Compression::parse(requested).ok_or_else(|| ApiError::bad_request("compression must be zstd or none"))?; + + let tmp = st.tmp_dir().join(helios_core::uuid_v4()); + let (tx, mut rx) = tokio::sync::mpsc::channel::(32); + let worker_tmp = tmp.clone(); + // Disk writes, hashing and decompression happen off the async runtime. + // A bad stream is the client's error (400); a failed write is ours (500). + let worker = tokio::task::spawn_blocking(move || -> ApiResult { + let invalid = |e: helios_core::Error| ApiError::bad_request(format!("invalid NAR stream: {e}")); + let file = std::fs::File::create_new(&worker_tmp).map_err(ApiError::internal)?; + let mut out = std::io::BufWriter::with_capacity(1 << 20, file); + let mut verifier = Verifier::new(compression).map_err(ApiError::internal)?; + while let Some(chunk) = rx.blocking_recv() { + verifier.update(&chunk).map_err(invalid)?; + out.write_all(&chunk).map_err(ApiError::internal)?; + } + let file = out.into_inner().map_err(|e| ApiError::internal(e.into_error()))?; + file.sync_data().map_err(ApiError::internal)?; + verifier.finish().map_err(invalid) + }); + + let mut stream = body.into_data_stream(); + let mut received: u64 = 0; + let mut failure: Option = None; + while let Some(chunk) = stream.next().await { + let chunk = match chunk { + Ok(c) => c, + Err(e) => { + failure = Some(ApiError::bad_request(format!("reading body: {e}"))); + break; + } + }; + received += chunk.len() as u64; + if received > st.cfg.max_upload_bytes { + failure = Some(ApiError::new(StatusCode::PAYLOAD_TOO_LARGE, "upload too large")); + break; + } + if tx.send(chunk).await.is_err() { + break; // Worker failed; its error is reported below. + } + } + drop(tx); + let result = worker.await?; + + let digest = match (failure, result) { + (Some(err), _) => { + let _ = tokio::fs::remove_file(&tmp).await; + audit.log(&st, &identity, "nar.upload", Some(&name), err.status, json!({ "error": err.message })); + return Err(err); + } + (None, Err(err)) => { + let _ = tokio::fs::remove_file(&tmp).await; + audit.log(&st, &identity, "nar.upload", Some(&name), err.status, json!({ "error": err.message })); + return Err(err); + } + (None, Ok(d)) => d, + }; + + let resp = store_blob(&st, cache.id, tmp, digest, compression).await?; + audit.log(&st, &identity, "nar.upload", Some(&name), StatusCode::CREATED, json!({ "fileHash": resp.file_hash, "size": resp.file_size })); + Ok((StatusCode::CREATED, Json(resp))) +} + +/// Moves a verified upload at `tmp` into the blob store and records it as +/// held by `cache_id`: uploading a NAR entitles that cache to reuse it. +pub(crate) async fn store_blob( + st: &Shared, + cache_id: u32, + tmp: std::path::PathBuf, + digest: helios_core::Digest, + compression: Compression, +) -> ApiResult { + let dest = st.nar_path(&digest.file_hash, compression); + let st2 = st.clone(); + tokio::task::spawn_blocking(move || -> ApiResult<()> { + let parent = dest.parent().expect("blob paths have a parent"); + std::fs::create_dir_all(parent).map_err(ApiError::internal)?; + // Rename and row upsert happen under the writer lock so GC can never + // delete the file between them. Renaming over an existing blob is + // harmless: the content is identical by hash. + st2.db.write(|conn| -> ApiResult<()> { + match std::fs::rename(&tmp, &dest) { + Ok(()) => {} + // GC removed an upload left idle past the grace period. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Err(ApiError::new(StatusCode::NOT_FOUND, "the upload expired before it completed; upload the NAR again")); + } + Err(e) => return Err(ApiError::internal(e)), + } + // The rename must survive a crash before a row refers to it. + std::fs::File::open(parent).and_then(|d| d.sync_all()).map_err(ApiError::internal)?; + let recorded = (|| -> rusqlite::Result<()> { + let tx = conn.transaction()?; + tx.prepare_cached( + "INSERT INTO blobs (file_hash, file_size, compression, nar_hash, nar_size, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT (file_hash) DO UPDATE SET created_at = excluded.created_at", + )? + .execute(params![ + &digest.file_hash[..], + digest.file_size, + compression.as_str(), + &digest.nar_hash[..], + digest.nar_size, + crate::db::now() + ])?; + tx.prepare_cached("INSERT OR IGNORE INTO blob_caches (blob_id, cache_id) SELECT id, ?2 FROM blobs WHERE file_hash = ?1")? + .execute(params![&digest.file_hash[..], cache_id])?; + tx.commit() + })(); + if let Err(e) = recorded { + // A file without a row is never collected; remove it unless + // an earlier upload of the same NAR owns it. + if conn.query_row("SELECT 1 FROM blobs WHERE file_hash = ?1", [&digest.file_hash[..]], |_| Ok(())).is_err() { + let _ = std::fs::remove_file(&dest); + } + return Err(e.into()); + } + Ok(()) + }) + }) + .await??; + + let resp = UploadResp { + file_hash: helios_core::nix32_encode(&digest.file_hash), + file_size: digest.file_size, + nar_hash: format!("sha256:{}", helios_core::nix32_encode(&digest.nar_hash)), + nar_size: digest.nar_size, + }; + st.counters.uploads.inc(); + st.counters.upload_bytes.add(resp.file_size); + Ok(resp) +} + +// ── Publish ── + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PathSpec { + store_path: String, + nar_hash: String, + nar_size: u64, + #[serde(default)] + references: Vec, + deriver: Option, + system: Option, + /// Pin a specific uploaded blob (nix32 file hash); otherwise any blob + /// with a matching verified NAR hash is used. + file_hash: Option, +} + +#[derive(Deserialize)] +pub struct PublishReq { + paths: Vec, +} + +/// Accepts `sha256:`, `sha256:` and SRI `sha256-`. +pub fn parse_sha256(s: &str) -> Option<[u8; 32]> { + if let Some(b64) = s.strip_prefix("sha256-") { + return base64::engine::general_purpose::STANDARD.decode(b64).ok()?.try_into().ok(); + } + let h = s.strip_prefix("sha256:").unwrap_or(s); + match h.len() { + 52 => helios_core::nix32_decode::<32>(h), + 64 => { + let mut out = [0u8; 32]; + for (i, byte) in out.iter_mut().enumerate() { + *byte = u8::from_str_radix(h.get(i * 2..i * 2 + 2)?, 16).ok()?; + } + Some(out) + } + _ => None, + } +} + +fn basename(s: &str) -> &str { + s.strip_prefix(STORE_DIR).unwrap_or(s) +} + +struct Prepared { + key: PathKey, + store_path: String, + nar_hash: [u8; 32], + nar_size: u64, + file_hash: Option<[u8; 32]>, + references: String, + deriver: String, + system: String, +} + +struct Rendered { + key: PathKey, + store_path: String, + blob_id: i64, + file_hash: [u8; 32], + narinfo: Vec, +} + +fn prepare(cache: CacheInfo, spec: PathSpec) -> Result { + let base = spec + .store_path + .strip_prefix(STORE_DIR) + .filter(|b| helios_core::store_basename_valid(b)) + .ok_or_else(|| format!("invalid store path: {}", spec.store_path))?; + let hash = helios_core::nix32_decode::<20>(&base[..32]).ok_or("invalid store path hash")?; + let nar_hash = parse_sha256(&spec.nar_hash).ok_or_else(|| format!("{}: invalid narHash", spec.store_path))?; + let file_hash = match &spec.file_hash { + Some(h) => Some(parse_sha256(h).ok_or_else(|| format!("{}: invalid fileHash", spec.store_path))?), + None => None, + }; + let mut references = String::new(); + for r in &spec.references { + let r = basename(r); + if !helios_core::store_basename_valid(r) { + return Err(format!("{}: invalid reference {r}", spec.store_path)); + } + if !references.is_empty() { + references.push(' '); + } + references.push_str(r); + } + Ok(Prepared { + key: PathKey { cache: cache.id, hash }, + store_path: spec.store_path, + nar_hash, + nar_size: spec.nar_size, + file_hash, + references, + deriver: spec.deriver.as_deref().map(basename).unwrap_or_default().to_owned(), + system: spec.system.unwrap_or_default(), + }) +} + +pub async fn publish( + State(st): State, + Path(name): Path, + audit: Audit, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult<(StatusCode, Json)> { + let cache = cache_for(&st, &name)?; + let identity = st.authorize(&headers, &name, Perm::Push)?; + if req.paths.len() > MAX_PUBLISH_BATCH { + return Err(ApiError::bad_request(format!("at most {MAX_PUBLISH_BATCH} paths per request"))); + } + + let mut prepared = Vec::with_capacity(req.paths.len()); + let mut already = 0usize; + { + let index = st.index.rd(); + for spec in req.paths { + let p = prepare(cache, spec).map_err(ApiError::bad_request)?; + if index.contains(&p.key) { + already += 1; + } else { + prepared.push(p); + } + } + } + + let st2 = st.clone(); + let outcome = tokio::task::spawn_blocking(move || render_and_insert(&st2, prepared)).await??; + let (rendered_keys, missing) = match outcome { + Outcome::Published(keys) => (keys, Vec::new()), + Outcome::MissingNars(missing) => (Vec::new(), missing), + }; + if !missing.is_empty() { + let err = ApiError::new(StatusCode::CONFLICT, "nar_required").with(json!({ "missing": missing })); + audit.log(&st, &identity, "path.publish", Some(&name), err.status, json!({ "missingNars": missing.len() })); + return Err(err); + } + + let published = rendered_keys.len(); + st.counters.published_paths.add(published as u64); + audit.log(&st, &identity, "path.publish", Some(&name), StatusCode::CREATED, json!({ "published": published, "alreadyExisted": already })); + Ok((StatusCode::CREATED, Json(json!({ "published": published, "alreadyExisted": already })))) +} + +enum Outcome { + Published(Vec), + MissingNars(Vec), +} + +struct Blob { + id: i64, + file_hash: [u8; 32], + file_size: u64, + compression: String, + nar_size: u64, +} + +fn render_and_insert(st: &Shared, prepared: Vec) -> ApiResult { + // Resolve blobs on a read connection, outside the writer lock. + let blobs: Vec> = st.db.read(|conn| { + let mut by_nar = conn.prepare_cached(&format!( + "SELECT id, file_hash, file_size, compression, nar_size FROM blobs b + WHERE nar_hash = ?1 AND {} ORDER BY compression = 'zstd' DESC, id LIMIT 1", + reusable(2) + ))?; + let mut by_file = conn.prepare_cached(&format!( + "SELECT id, file_hash, file_size, compression, nar_size FROM blobs b WHERE file_hash = ?1 AND nar_hash = ?3 AND {}", + reusable(2) + ))?; + let row = |r: &rusqlite::Row<'_>| { + Ok(Blob { + id: r.get(0)?, + file_hash: r.get::<_, Vec>(1)?.try_into().unwrap_or([0; 32]), + file_size: r.get(2)?, + compression: r.get(3)?, + nar_size: r.get(4)?, + }) + }; + prepared + .iter() + .map(|p| match &p.file_hash { + Some(fh) => by_file.query_row(params![&fh[..], p.key.cache, &p.nar_hash[..]], row).optional(), + None => by_nar.query_row(params![&p.nar_hash[..], p.key.cache], row).optional(), + }) + .collect() + })?; + + let missing: Vec = prepared.iter().zip(&blobs).filter(|(_, b)| b.is_none()).map(|(p, _)| p.store_path.clone()).collect(); + if !missing.is_empty() { + return Ok(Outcome::MissingNars(missing)); + } + + // Render and sign in parallel; Ed25519 dominates the cost of a publish. + let work: Vec<(Prepared, Blob)> = prepared.into_iter().zip(blobs.into_iter().map(Option::unwrap)).collect(); + let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(work.len().div_ceil(64).max(1)); + let chunk = work.len().div_ceil(threads).max(1); + let rendered: Vec> = std::thread::scope(|scope| { + let handles: Vec<_> = + work.chunks(chunk).map(|part| scope.spawn(move || part.iter().map(|(p, b)| render(st, p, b)).collect::>())).collect(); + handles.into_iter().flat_map(|h| h.join().expect("render thread panicked")).collect() + }); + let rendered: Vec = rendered.into_iter().collect::>()?; + + let now = crate::db::now(); + st.db.write(|conn| -> ApiResult { + let tx = conn.transaction()?; + let mut keys = Vec::with_capacity(rendered.len()); + let mut gone = Vec::new(); + { + // By file hash too: SQLite reuses the rowid of a deleted blob. + let mut blob_exists = tx.prepare_cached("SELECT 1 FROM blobs WHERE id = ?1 AND file_hash = ?2")?; + let mut insert = tx.prepare_cached( + "INSERT INTO paths (cache_id, hash, blob_id, store_path, narinfo, created_at, accessed_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?6) ON CONFLICT (cache_id, hash) DO NOTHING", + )?; + let mut link = tx.prepare_cached("INSERT OR IGNORE INTO blob_caches (blob_id, cache_id) VALUES (?1, ?2)")?; + for r in &rendered { + // Re-checked under the writer lock: GC may have collected the blob. + if !blob_exists.exists(params![r.blob_id, &r.file_hash[..]])? { + gone.push(r.store_path.clone()); + continue; + } + if insert.execute(params![r.key.cache, &r.key.hash[..], r.blob_id, r.store_path, r.narinfo, now])? > 0 { + keys.push(r.key); + } + link.execute(params![r.blob_id, r.key.cache])?; + } + } + if !gone.is_empty() { + // Nothing is committed; the client uploads those NARs again. + return Ok(Outcome::MissingNars(gone)); + } + tx.commit()?; + // Under the writer lock, so an eviction cannot run between the + // commit and the index update. + let mut index = st.index.wr(); + st.narinfo_epoch.fetch_add(1, std::sync::atomic::Ordering::Release); + for key in &keys { + index.insert(*key); + st.narinfo.remove(key); + } + Ok(Outcome::Published(keys)) + }) +} + +fn render(st: &Shared, p: &Prepared, b: &Blob) -> ApiResult { + if b.nar_size != p.nar_size { + return Err(ApiError::bad_request(format!("{}: narSize {} does not match the uploaded NAR", p.store_path, p.nar_size))); + } + let narinfo = helios_core::render_narinfo( + &NarinfoInput { + store_path: &p.store_path, + nar_hash: &p.nar_hash, + nar_size: b.nar_size, + file_hash: &b.file_hash, + file_size: b.file_size, + compression: &b.compression, + references: &p.references, + deriver: &p.deriver, + system: &p.system, + }, + st.signer.as_ref(), + ) + .map_err(|e| ApiError::bad_request(format!("{}: {e}", p.store_path)))?; + Ok(Rendered { key: p.key, store_path: p.store_path.clone(), blob_id: b.id, file_hash: b.file_hash, narinfo }) +} diff --git a/crates/helios-server/src/read.rs b/crates/helios-server/src/read.rs new file mode 100644 index 0000000..0140813 --- /dev/null +++ b/crates/helios-server/src/read.rs @@ -0,0 +1,240 @@ +//! The substituter read path: `//nix-cache-info`, +//! `//.narinfo`, `//nar/.nar[.zst]` and +//! `//build-trace-v2//.doi` (or `realisations/.doi`). +//! URLs are parsed by hand, without allocating. + +use axum::body::Body; +use axum::extract::State; +use axum::http::{HeaderMap, HeaderValue, Method, StatusCode, Uri, header}; +use axum::response::{IntoResponse, Response}; +use bytes::Bytes; +use helios_core::Compression; +use rusqlite::{OptionalExtension, params}; + +use crate::error::{ApiError, ApiResult}; +use crate::state::Locked; +use crate::state::{PathKey, Shared}; + +const NIX_CACHE_INFO: &str = "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 40\n"; +/// Cache-Control for each kind of response. +struct Caching { + /// NARs are named by their content, so they never change. + nar: HeaderValue, + /// narinfo and build traces can be evicted, quarantined or re-pushed. + metadata: HeaderValue, + /// Most narinfo requests are misses (Nix asks every substituter about + /// every path); new paths still appear quickly. + miss: HeaderValue, +} + +const PUBLIC: Caching = Caching { + nar: HeaderValue::from_static("public, max-age=31536000, immutable"), + metadata: HeaderValue::from_static("public, max-age=3600"), + miss: HeaderValue::from_static("public, max-age=60"), +}; + +/// Nothing from a private cache may be kept by a shared proxy, and the +/// client's own copy is revalidated. +const PRIVATE: Caching = Caching { + nar: HeaderValue::from_static("private, no-cache"), + metadata: HeaderValue::from_static("private, no-cache"), + miss: HeaderValue::from_static("private, no-cache"), +}; + +/// Before the cache is known (unknown cache or URL). +const NEGATIVE: HeaderValue = HeaderValue::from_static("public, max-age=60"); + +enum Route<'a> { + CacheInfo, + Narinfo(&'a str), + Nar(&'a str, Compression), + /// `build-trace-v2/...` or `realisations/...` + BuildTrace(&'a str), +} + +fn route(rest: &str) -> Option> { + if rest == "nix-cache-info" { + return Some(Route::CacheInfo); + } + if rest.starts_with("build-trace-v2/") || rest.starts_with("realisations/") { + return Some(Route::BuildTrace(rest)); + } + if let Some(hash) = rest.strip_suffix(".narinfo") { + return (hash.len() == 32).then_some(Route::Narinfo(hash)); + } + let file = rest.strip_prefix("nar/")?; + if let Some(hash) = file.strip_suffix(".nar.zst") { + return Some(Route::Nar(hash, Compression::Zstd)); + } + file.strip_suffix(".nar").map(|hash| Route::Nar(hash, Compression::None)) +} + +fn not_found(cache_control: HeaderValue) -> Response { + (StatusCode::NOT_FOUND, [(header::CACHE_CONTROL, cache_control)], "not found").into_response() +} + +pub async fn handle(State(st): State, method: Method, uri: Uri, headers: HeaderMap) -> Response { + match serve(st, method, uri, headers).await { + Ok(resp) => resp, + Err(err) => err.into_response(), + } +} + +async fn serve(st: Shared, method: Method, uri: Uri, headers: HeaderMap) -> ApiResult { + if method != Method::GET && method != Method::HEAD { + return Ok((StatusCode::METHOD_NOT_ALLOWED, [(header::ALLOW, "GET, HEAD")]).into_response()); + } + let path = uri.path().trim_start_matches('/'); + let Some((name, rest)) = path.split_once('/') else { + return Ok(not_found(NEGATIVE)); + }; + let Some(route) = route(rest) else { + return Ok(not_found(NEGATIVE)); + }; + let Some(cache) = st.cache(name) else { + return Ok(not_found(NEGATIVE)); + }; + st.authorize_read(&headers, name, cache)?; + let caching = if cache.public { &PUBLIC } else { &PRIVATE }; + + match route { + Route::CacheInfo => Ok(( + [(header::CONTENT_TYPE, HeaderValue::from_static("text/x-nix-cache-info")), (header::CACHE_CONTROL, caching.metadata.clone())], + NIX_CACHE_INFO, + ) + .into_response()), + Route::BuildTrace(id) => Ok(match crate::traces::lookup(&st, cache.id, id) { + Some(body) => { + ([(header::CONTENT_TYPE, HeaderValue::from_static("application/json")), (header::CACHE_CONTROL, caching.metadata.clone())], body) + .into_response() + } + None => not_found(caching.miss.clone()), + }), + Route::Narinfo(hash) => { + let Some(hash) = helios_core::nix32_decode::<20>(hash) else { + return Ok(not_found(caching.miss.clone())); + }; + let key = PathKey { cache: cache.id, hash }; + if !st.index.rd().contains(&key) { + st.counters.narinfo_misses.inc(); + return Ok(not_found(caching.miss.clone())); + } + st.counters.narinfo_hits.inc(); + st.access.touch(key); + let body = match st.narinfo.get(&key) { + Some(body) => body, + None => { + let epoch = st.narinfo_epoch.load(std::sync::atomic::Ordering::Acquire); + let db = st.db.clone(); + let loaded = tokio::task::spawn_blocking(move || { + db.read(|conn| { + conn.prepare_cached("SELECT narinfo FROM paths WHERE cache_id = ?1 AND hash = ?2")? + .query_row(params![key.cache, &key.hash[..]], |r| r.get::<_, Vec>(0)) + .optional() + }) + }) + .await??; + let Some(text) = loaded else { + return Ok(not_found(caching.miss.clone())); + }; + let body = Bytes::from(text); + // Checked under the index lock that publish and eviction + // hold while they bump the epoch. + let index = st.index.rd(); + if index.contains(&key) && st.narinfo_epoch.load(std::sync::atomic::Ordering::Acquire) == epoch { + st.narinfo.insert(key, body.clone()); + } + body + } + }; + Ok(([(header::CONTENT_TYPE, HeaderValue::from_static("text/x-nix-narinfo")), (header::CACHE_CONTROL, caching.metadata.clone())], body) + .into_response()) + } + Route::Nar(hash, compression) => { + st.counters.nar_requests.inc(); + let Some(file_hash) = helios_core::nix32_decode::<32>(hash) else { + return Ok(not_found(caching.miss.clone())); + }; + let db = st.db.clone(); + let cache_id = cache.id; + // Content-addressed, but only served from caches that publish it. + let size = tokio::task::spawn_blocking(move || { + db.read(|conn| { + conn.prepare_cached( + "SELECT b.file_size FROM blobs b WHERE b.file_hash = ?1 AND b.compression = ?2 + AND EXISTS (SELECT 1 FROM paths p WHERE p.blob_id = b.id AND p.cache_id = ?3)", + )? + .query_row(params![&file_hash[..], compression.as_str(), cache_id], |r| r.get::<_, u64>(0)) + .optional() + }) + }) + .await??; + let Some(size) = size else { + return Ok(not_found(caching.miss.clone())); + }; + nar_response(&st, &file_hash, compression, size, &method, caching).await + } + } +} + +async fn nar_response( + st: &Shared, + file_hash: &[u8; 32], + compression: Compression, + size: u64, + method: &Method, + caching: &Caching, +) -> ApiResult { + let mut builder = Response::builder().header(header::CONTENT_TYPE, "application/x-nix-nar").header(header::CACHE_CONTROL, caching.nar.clone()); + + if let Some(prefix) = &st.cfg.accel_redirect { + let name = helios_core::nix32_encode(file_hash); + let location = format!("{prefix}/{}/{name}{}", &name[..2], compression.extension()); + builder = builder.header("x-accel-redirect", location); + return builder.body(Body::empty()).map_err(ApiError::internal); + } + + builder = builder.header(header::CONTENT_LENGTH, size); + if method == Method::HEAD { + return builder.body(Body::empty()).map_err(ApiError::internal); + } + let file = match tokio::fs::File::open(st.nar_path(file_hash, compression)).await { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(not_found(caching.miss.clone())), + Err(e) => return Err(ApiError::internal(e)), + }; + // 512 KiB reads; the stream ends after EOF or the first error. + let stream = futures_util::stream::unfold(Some(file), |file| async move { + use tokio::io::AsyncReadExt; + let mut file = file?; + let mut buf = bytes::BytesMut::with_capacity(512 * 1024); + match file.read_buf(&mut buf).await { + Ok(0) => None, + Ok(_) => Some((Ok(buf.freeze()), Some(file))), + Err(e) => Some((Err(e), None)), + } + }); + builder.body(Body::from_stream(stream)).map_err(ApiError::internal) +} + +/// `GET /_api/v2/caches/{cache}`: what a client needs to substitute from the +/// cache. Readable by whoever can read the cache itself. +pub async fn cache_info( + State(st): State, + axum::extract::Path(name): axum::extract::Path, + headers: HeaderMap, +) -> ApiResult> { + let cache = st.cache(&name).ok_or_else(ApiError::not_found)?; + // Pushers read it too, for the compression defaults; a push-only token + // learns nothing here it could not from pushing. + if st.authorize_read(&headers, &name, cache).is_err() { + st.authorize(&headers, &name, crate::auth::Perm::Push)?; + } + let public_key = st.signer.as_ref().map(helios_core::Signer::public_key); + Ok(axum::Json(serde_json::json!({ + "name": name, + "public": cache.public, + "publicKey": public_key, + "compression": { "level": st.cfg.compression_level, "windowLog": st.cfg.compression_window_log }, + }))) +} diff --git a/crates/helios-server/src/state.rs b/crates/helios-server/src/state.rs new file mode 100644 index 0000000..c9f710d --- /dev/null +++ b/crates/helios-server/src/state.rs @@ -0,0 +1,263 @@ +//! Shared server state. The hot read path (narinfo lookups) is served from +//! memory: a presence index answers misses without touching SQLite, and an +//! LRU holds rendered narinfo bodies for hits. + +use std::collections::{HashMap, HashSet}; +use std::hash::{BuildHasherDefault, Hash, Hasher}; +use std::path::PathBuf; +use std::sync::Arc; + +use bytes::Bytes; +use helios_core::{Compression, Signer}; +use rusqlite::params; +use std::sync::{PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard}; +use tokio::sync::mpsc; + +use crate::audit::AuditEvent; +use crate::config::Config; +use crate::db::Db; + +/// A published path: cache id plus the 20-byte decoded store path hash. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct PathKey { + pub cache: u32, + pub hash: [u8; 20], +} + +/// Per-process key for `PathKey` hashes. Store path hashes come from +/// clients, so a fixed hash would let a pusher publish paths that all +/// collide and slow every lookup to a crawl. +static HASH_KEY: std::sync::LazyLock<[u64; 3]> = std::sync::LazyLock::new(|| { + let mut bytes = [0u8; 24]; + helios_core::random_bytes(&mut bytes); + let word = |i: usize| u64::from_le_bytes(bytes[i * 8..i * 8 + 8].try_into().expect("8 bytes")); + [word(0), word(1), word(2)] +}); + +/// The 128-bit product folded to 64 bits, as in foldhash and wyhash. +fn fold_mul(a: u64, b: u64) -> u64 { + let full = u128::from(a) * u128::from(b); + (full as u64) ^ ((full >> 64) as u64) +} + +impl Hash for PathKey { + fn hash(&self, state: &mut H) { + let k = &*HASH_KEY; + let word = |i: usize| u64::from_le_bytes(self.hash[i..i + 8].try_into().expect("8 bytes")); + let tail = u64::from(u32::from_le_bytes(self.hash[16..20].try_into().expect("4 bytes"))) | (u64::from(self.cache) << 32); + state.write_u64(fold_mul(fold_mul(word(0) ^ k[0], word(8) ^ k[1]) ^ tail, k[2])); + } +} + +#[derive(Default)] +pub struct PassThroughHasher(u64); + +impl Hasher for PassThroughHasher { + fn finish(&self) -> u64 { + self.0 + } + fn write(&mut self, bytes: &[u8]) { + for &b in bytes { + self.0 = self.0.rotate_left(8) ^ u64::from(b); + } + } + fn write_u64(&mut self, n: u64) { + self.0 = n; + } +} + +pub type PathIndex = HashSet>; + +#[derive(Clone, Copy)] +pub struct CacheInfo { + pub id: u32, + pub public: bool, +} + +#[derive(Clone, Copy)] +pub struct TokenState { + pub expires_at: i64, + pub revoked: bool, +} + +pub struct AppState { + pub cfg: Config, + pub db: Db, + pub signer: Option, + pub caches: RwLock, CacheInfo>>, + pub index: RwLock, + pub narinfo: quick_cache::sync::Cache>, + pub tokens: RwLock, TokenState>>, + pub audit: mpsc::UnboundedSender, + pub counters: crate::stats::Counters, + pub access: crate::stats::Access, + pub uploads: crate::chunked::Sessions, + pub traces: RwLock, + /// Bumped, under the index write lock, whenever paths are published or + /// forgotten. A narinfo read from the database is cached only if this + /// did not move meanwhile, so a body from before an eviction or re-push + /// is never cached after it. + pub narinfo_epoch: std::sync::atomic::AtomicU64, +} + +pub type Shared = Arc; + +impl AppState { + pub fn load(cfg: Config, db: Db, signer: Option, audit: mpsc::UnboundedSender) -> anyhow::Result { + let (caches, index, tokens, traces) = db.read(|conn| { + let mut caches = HashMap::new(); + let mut stmt = conn.prepare("SELECT id, name, is_public FROM caches")?; + let mut rows = stmt.query([])?; + while let Some(r) = rows.next()? { + let name: String = r.get(1)?; + caches.insert(name.into_boxed_str(), CacheInfo { id: r.get(0)?, public: r.get::<_, i64>(2)? != 0 }); + } + + let count: usize = conn.query_row("SELECT count(*) FROM paths", [], |r| r.get(0))?; + let mut index = PathIndex::with_capacity_and_hasher(count + count / 4, Default::default()); + let mut stmt = conn.prepare("SELECT cache_id, hash FROM paths")?; + let mut rows = stmt.query([])?; + while let Some(r) = rows.next()? { + let hash = r.get_ref(1)?.as_blob()?; + if let Ok(hash) = <[u8; 20]>::try_from(hash) { + index.insert(PathKey { cache: r.get(0)?, hash }); + } + } + + let mut tokens = HashMap::new(); + let mut stmt = conn.prepare("SELECT jti, expires_at, revoked_at IS NOT NULL FROM tokens")?; + let mut rows = stmt.query([])?; + while let Some(r) = rows.next()? { + let jti: String = r.get(0)?; + tokens.insert(jti.into_boxed_str(), TokenState { expires_at: r.get(1)?, revoked: r.get(2)? }); + } + Ok((caches, index, tokens, crate::traces::load(conn)?)) + })?; + + tracing::info!(caches = caches.len(), paths = index.len(), tokens = tokens.len(), "loaded state"); + let narinfo = quick_cache::sync::Cache::with( + cfg.narinfo_cache_entries, + cfg.narinfo_cache_entries as u64, + quick_cache::UnitWeighter, + Default::default(), + quick_cache::sync::DefaultLifecycle::default(), + ); + Ok(Self { + cfg, + db, + signer, + caches: RwLock::new(caches), + index: RwLock::new(index), + narinfo, + tokens: RwLock::new(tokens), + audit, + counters: Default::default(), + access: Default::default(), + uploads: Default::default(), + traces: RwLock::new(traces), + narinfo_epoch: Default::default(), + }) + } + + pub fn cache(&self, name: &str) -> Option { + self.caches.rd().get(name).copied() + } + + pub fn nar_path(&self, file_hash: &[u8; 32], compression: Compression) -> PathBuf { + let name = helios_core::nix32_encode(file_hash); + self.cfg.data_dir.join("nar").join(&name[..2]).join(format!("{name}{}", compression.extension())) + } + + pub fn tmp_dir(&self) -> PathBuf { + self.cfg.data_dir.join("tmp") + } + + /// Creates `name`, or gives an existing cache the visibility `public`. + /// Returns what changed, if anything. + pub fn declare_cache(&self, name: &str, public: bool) -> rusqlite::Result> { + if self.insert_cache(name, public)?.is_some() { + return Ok(Some("created")); + } + if self.cache(name).is_some_and(|c| c.public == public) { + return Ok(None); + } + self.db.write(|conn| conn.execute("UPDATE caches SET is_public = ?2 WHERE name = ?1", params![name, public]))?; + if let Some(c) = self.caches.wr().get_mut(name) { + c.public = public; + } + Ok(Some(if public { "made public" } else { "made private" })) + } + + pub fn insert_cache(&self, name: &str, public: bool) -> rusqlite::Result> { + let id = self.db.write(|conn| -> rusqlite::Result> { + conn.execute( + "INSERT INTO caches (name, is_public, created_at) VALUES (?1, ?2, ?3) ON CONFLICT (name) DO NOTHING", + params![name, public, crate::db::now()], + )?; + if conn.changes() == 0 { + return Ok(None); + } + Ok(Some(conn.last_insert_rowid() as u32)) + })?; + Ok(id.map(|id| { + let info = CacheInfo { id, public }; + self.caches.wr().insert(name.into(), info); + info + })) + } +} + +/// Lock access that ignores poisoning: every critical section here leaves +/// the data consistent, so a panic elsewhere must not wedge the server. +pub trait Locked { + fn rd(&self) -> RwLockReadGuard<'_, T>; + fn wr(&self) -> RwLockWriteGuard<'_, T>; +} + +impl Locked for RwLock { + fn rd(&self) -> RwLockReadGuard<'_, T> { + self.read().unwrap_or_else(PoisonError::into_inner) + } + fn wr(&self) -> RwLockWriteGuard<'_, T> { + self.write().unwrap_or_else(PoisonError::into_inner) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn declared_caches_are_created_and_take_the_declared_visibility() { + let dir = std::env::temp_dir().join(format!("helios-state-test-{}", helios_core::uuid_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let cfg = Config { + data_dir: dir.clone(), + jwt_secret: None, + admin_secret: None, + accel_redirect: None, + trust_proxy: false, + narinfo_cache_entries: 16, + max_upload_bytes: 1 << 20, + upload_grace: std::time::Duration::from_secs(1), + gc_interval: std::time::Duration::from_secs(3600), + audit_retention: std::time::Duration::from_secs(3600), + compression_level: 3, + compression_window_log: 27, + }; + let db = Db::open(&dir.join("helios.db")).unwrap(); + let (tx, _rx) = mpsc::unbounded_channel(); + let st = AppState::load(cfg.clone(), db, None, tx).unwrap(); + + assert_eq!(st.declare_cache("team", false).unwrap(), Some("created")); + assert_eq!(st.declare_cache("team", false).unwrap(), None); + assert_eq!(st.declare_cache("team", true).unwrap(), Some("made public")); + assert!(st.cache("team").unwrap().public); + + // The change is in the database, not only in memory. + let (tx, _rx) = mpsc::unbounded_channel(); + let reloaded = AppState::load(cfg, Db::open(&dir.join("helios.db")).unwrap(), None, tx).unwrap(); + assert!(reloaded.cache("team").unwrap().public); + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/crates/helios-server/src/stats.rs b/crates/helios-server/src/stats.rs new file mode 100644 index 0000000..cdc42aa --- /dev/null +++ b/crates/helios-server/src/stats.rs @@ -0,0 +1,116 @@ +//! Request counters and path access tracking, kept off shared cache lines: +//! the narinfo path runs at over a million requests per second, where a +//! single atomic or lock touched by every core would become the bottleneck. + +use std::collections::HashSet; +use std::hash::BuildHasherDefault; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Mutex, PoisonError}; + +use crate::state::{PassThroughHasher, PathKey}; + +const STRIPES: usize = 32; + +#[repr(align(128))] +#[derive(Default)] +struct Padded(AtomicU64); + +static NEXT_SLOT: AtomicUsize = AtomicUsize::new(0); + +thread_local! { + /// Each thread increments its own stripe. + static SLOT: usize = NEXT_SLOT.fetch_add(1, Ordering::Relaxed) % STRIPES; +} + +/// A monotonically increasing counter striped across cache lines. +#[derive(Default)] +pub struct Counter { + stripes: [Padded; STRIPES], +} + +impl Counter { + pub fn add(&self, n: u64) { + let slot = SLOT.with(|s| *s); + self.stripes[slot].0.fetch_add(n, Ordering::Relaxed); + } + + pub fn inc(&self) { + self.add(1); + } + + pub fn get(&self) -> u64 { + self.stripes.iter().map(|s| s.0.load(Ordering::Relaxed)).sum() + } +} + +#[derive(Default)] +pub struct Counters { + pub narinfo_hits: Counter, + pub narinfo_misses: Counter, + pub nar_requests: Counter, + pub uploads: Counter, + pub upload_bytes: Counter, + pub published_paths: Counter, +} + +type KeySet = HashSet>; + +const SHARDS: usize = 64; + +/// Paths served since the last flush. Sharded by hash so concurrent +/// requests rarely share a lock. +pub struct Access { + shards: Vec>, +} + +impl Default for Access { + fn default() -> Self { + Self { shards: (0..SHARDS).map(|_| Mutex::new(KeySet::default())).collect() } + } +} + +impl Access { + pub fn touch(&self, key: PathKey) { + let shard = &self.shards[usize::from(key.hash[19]) % SHARDS]; + shard.lock().unwrap_or_else(PoisonError::into_inner).insert(key); + } + + /// Takes every recorded key, leaving the tracker empty. + pub fn drain(&self) -> Vec { + let mut out = Vec::new(); + for shard in &self.shards { + let taken = std::mem::take(&mut *shard.lock().unwrap_or_else(PoisonError::into_inner)); + out.extend(taken); + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn counters_sum_across_threads() { + let c = std::sync::Arc::new(Counter::default()); + let threads: Vec<_> = (0..8) + .map(|_| { + let c = c.clone(); + std::thread::spawn(move || (0..1000).for_each(|_| c.inc())) + }) + .collect(); + threads.into_iter().for_each(|t| t.join().unwrap()); + assert_eq!(c.get(), 8000); + } + + #[test] + fn access_deduplicates_and_drains() { + let a = Access::default(); + let k = PathKey { cache: 1, hash: [7; 20] }; + a.touch(k); + a.touch(k); + a.touch(PathKey { cache: 2, hash: [7; 20] }); + assert_eq!(a.drain().len(), 2); + assert!(a.drain().is_empty()); + } +} diff --git a/crates/helios-server/src/traces.rs b/crates/helios-server/src/traces.rs new file mode 100644 index 0000000..275adb4 --- /dev/null +++ b/crates/helios-server/src/traces.rs @@ -0,0 +1,286 @@ +//! Build traces ("realisations"): which output path a content-addressed +//! derivation produced, so Nix can substitute CA derivations instead of +//! building them (compare zhaofengli/attic#188). +//! +//! Nix has two formats, and a cache serves whichever its pushers produce: +//! +//! - Nix 2.35 and later: `//build-trace-v2//.doi`, +//! keyed by derivation path, with structured signatures. +//! - Before 2.35: `//realisations/sha256:!.doi`, keyed by +//! the derivation's hash modulo, which only Nix computes. +//! +//! CA derivations are experimental in Nix, and so are both formats. +//! +//! `POST /_api/v2/caches/{cache}/build-traces` takes entries as `nix store +//! build-trace info --json` (or the older `nix realisation info --json`) +//! prints them. Each output path must already be published in the cache. +//! Client signatures are dropped and the entry is signed with the cache's +//! key, as narinfo is; entries are rendered and signed once, at publish +//! time, and served from memory. + +use std::collections::HashMap; + +use axum::Json; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use bytes::Bytes; +use rusqlite::params; +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::audit::Audit; +use crate::auth::Perm; +use crate::error::{ApiError, ApiResult}; +use crate::push::cache_for; +use crate::state::{Locked, PathKey, Shared}; + +const MAX_ENTRIES: usize = 5_000; + +/// An entry's file name without `.doi`: `/` for the current +/// format, `sha256:!` for the old one. The two cannot collide. +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct TraceKey { + pub cache: u32, + pub id: Box, +} + +pub struct Trace { + /// The output path, which must still be published for the entry to be + /// served. + pub out: PathKey, + pub body: Bytes, +} + +pub type Traces = HashMap; + +fn out_key(cache: u32, out: &str) -> Option { + let hash = helios_core::store_basename_valid(out).then(|| helios_core::nix32_decode::<20>(&out[..32])).flatten()?; + Some(PathKey { cache, hash }) +} + +pub fn load(conn: &rusqlite::Connection) -> rusqlite::Result { + let mut traces = Traces::new(); + let mut stmt = conn.prepare("SELECT cache_id, id, out_path, body FROM build_traces")?; + let mut rows = stmt.query([])?; + while let Some(r) = rows.next()? { + let cache: u32 = r.get(0)?; + let (id, out): (String, String) = (r.get(1)?, r.get(2)?); + if let Some(key) = out_key(cache, &out) { + traces.insert(TraceKey { cache, id: id.into() }, Trace { out: key, body: Bytes::from(r.get::<_, Vec>(3)?) }); + } + } + Ok(traces) +} + +/// Nix output names: the characters of store path names. +fn output_name_valid(s: &str) -> bool { + (1..=255).contains(&s.len()) && s.bytes().all(|c| c.is_ascii_alphanumeric() || b"+-._?=".contains(&c)) +} + +fn basename(s: &str) -> &str { + s.strip_prefix("/nix/store/").unwrap_or(s) +} + +/// An entry in either format, validated. +enum Parsed { + /// ``, ``. + Current(String, String), + /// `sha256:!`. + Legacy(String), +} + +impl Parsed { + fn id(&self) -> String { + match self { + Parsed::Current(drv, output) => format!("{drv}/{output}"), + Parsed::Legacy(id) => id.clone(), + } + } +} + +fn legacy_id_valid(id: &str) -> bool { + let Some((hash, output)) = id.split_once('!') else { return false }; + hash.strip_prefix("sha256:").is_some_and(|h| h.len() == 64 && h.bytes().all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c))) + && output_name_valid(output) +} + +/// The signed file body. The signature covers what Nix's +/// `UnkeyedRealisation::fingerprint` produces for the format: the entry's +/// JSON with keys sorted and the signatures removed. Every value is limited +/// to store path and hash characters, so no JSON escaping is involved. +fn render(st: &Shared, entry: &Parsed, out: &str) -> Vec { + let sign = |fingerprint: String| st.signer.as_ref().map(|s| s.sign(fingerprint.as_bytes())); + let body = match entry { + Parsed::Current(drv, output) => { + let sig = sign(format!(r#"{{"key":{{"drvPath":"{drv}","outputName":"{output}"}},"value":{{"outPath":"{out}"}}}}"#)); + let signatures: Vec = sig + .iter() + .map(|s| { + let (name, sig) = s.split_once(':').expect("signatures are name:base64"); + json!({ "keyName": name, "sig": sig }) + }) + .collect(); + json!({ "outPath": out, "signatures": signatures }) + } + Parsed::Legacy(id) => { + let sig = sign(format!(r#"{{"dependentRealisations":{{}},"id":"{id}","outPath":"{out}"}}"#)); + json!({ "dependentRealisations": {}, "id": id, "outPath": out, "signatures": sig.into_iter().collect::>() }) + } + }; + serde_json::to_vec(&body).expect("serialisable") +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct EntryKey { + drv_path: String, + output_name: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct EntryValue { + out_path: String, +} + +/// `{"key": {...}, "value": {...}}` (Nix 2.35+) or `{"id", "outPath"}`. +#[derive(Deserialize)] +#[serde(untagged)] +enum Entry { + #[serde(rename_all = "camelCase")] + Current { key: EntryKey, value: EntryValue }, + #[serde(rename_all = "camelCase")] + Legacy { id: String, out_path: String }, +} + +#[derive(Deserialize)] +pub struct PublishReq { + entries: Vec, +} + +pub async fn publish( + State(st): State, + Path(name): Path, + audit: Audit, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult<(StatusCode, Json)> { + let cache = cache_for(&st, &name)?; + let who = st.authorize(&headers, &name, Perm::Push)?; + if req.entries.len() > MAX_ENTRIES { + return Err(ApiError::bad_request(format!("at most {MAX_ENTRIES} entries per request"))); + } + + let mut rows = Vec::with_capacity(req.entries.len()); + let mut missing = Vec::new(); + { + let index = st.index.rd(); + for e in &req.entries { + let (parsed, out) = match e { + Entry::Current { key, value } => { + let drv = basename(&key.drv_path); + if !(helios_core::store_basename_valid(drv) && drv.ends_with(".drv")) { + return Err(ApiError::bad_request(format!("invalid derivation path: {}", key.drv_path))); + } + if !output_name_valid(&key.output_name) { + return Err(ApiError::bad_request(format!("invalid output name: {}", key.output_name))); + } + (Parsed::Current(drv.to_owned(), key.output_name.clone()), basename(&value.out_path)) + } + Entry::Legacy { id, out_path } => { + if !legacy_id_valid(id) { + return Err(ApiError::bad_request(format!("invalid build trace id: {id}"))); + } + (Parsed::Legacy(id.clone()), basename(out_path)) + } + }; + let key = out_key(cache.id, out).ok_or_else(|| ApiError::bad_request(format!("invalid output path: {out}")))?; + if !index.contains(&key) { + missing.push(format!("/nix/store/{out}")); + continue; + } + // Signed later, off the index lock. + rows.push((parsed, out.to_owned(), key)); + } + } + if !missing.is_empty() { + return Err(ApiError::new(StatusCode::CONFLICT, "output paths are not published in this cache").with(json!({ "missing": missing }))); + } + + let st2 = st.clone(); + let inserted = tokio::task::spawn_blocking(move || -> ApiResult> { + let rendered: Vec<_> = rows.into_iter().map(|(parsed, out, key)| (parsed.id(), render(&st2, &parsed, &out), out, key)).collect(); + let now = crate::db::now(); + st2.db + .write(|conn| -> rusqlite::Result> { + let tx = conn.transaction()?; + let mut inserted = Vec::new(); + { + let mut upsert = tx.prepare_cached( + "INSERT INTO build_traces (cache_id, id, out_path, body, created_at) VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT (cache_id, id) DO UPDATE SET out_path = excluded.out_path, body = excluded.body, created_at = excluded.created_at", + )?; + let (traces, index) = (st2.traces.rd(), st2.index.rd()); + for (id, body, out, key) in rendered { + let trace_key = TraceKey { cache: cache.id, id: id.as_str().into() }; + // The first entry for an output wins while its path is + // served, so a non-deterministic build cannot flip it; + // once that path is gone, a newer build may take over. + if traces.get(&trace_key).is_some_and(|t| index.contains(&t.out)) { + continue; + } + upsert.execute(params![cache.id, id, out, body, now])?; + inserted.push((trace_key, Trace { out: key, body: body.into() })); + } + } + tx.commit()?; + Ok(inserted) + }) + .map_err(ApiError::from) + }) + .await??; + + let published = inserted.len(); + { + let mut traces = st.traces.wr(); + for (k, v) in inserted { + traces.insert(k, v); + } + } + let already = req.entries.len() - published; + audit.log(&st, &who, "build_trace.publish", Some(&name), StatusCode::CREATED, json!({ "published": published, "alreadyExisted": already })); + Ok((StatusCode::CREATED, Json(json!({ "published": published, "alreadyExisted": already })))) +} + +/// The body for a request path below the cache, if it names a build trace +/// whose output is still published. +pub fn lookup(st: &Shared, cache: u32, rest: &str) -> Option { + let (current, id) = match rest.strip_prefix("build-trace-v2/") { + Some(id) => (true, id), + None => (false, rest.strip_prefix("realisations/")?), + }; + let id = id.strip_suffix(".doi")?; + // Old clients may percent-encode the `:` and `!` of a hash-keyed id. + let id = if current { id.to_owned() } else { id.replace("%3A", ":").replace("%3a", ":").replace("%21", "!") }; + if current == legacy_id_valid(&id) { + return None; // Each format only under its own prefix. + } + let traces = st.traces.rd(); + let trace = traces.get(&TraceKey { cache, id: id.into() })?; + st.index.rd().contains(&trace.out).then(|| trace.body.clone()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn legacy_ids() { + let hex = "a".repeat(64); + assert!(legacy_id_valid(&format!("sha256:{hex}!out"))); + assert!(!legacy_id_valid(&format!("sha256:{hex}"))); + assert!(!legacy_id_valid(&format!("sha1:{hex}!out"))); + assert!(!legacy_id_valid(&format!("sha256:{}!out", "A".repeat(64)))); + assert!(!legacy_id_valid(&format!("sha256:{hex}!bad/name"))); + } +} diff --git a/docs.md b/docs.md index 0c1fbb5..aee54c7 100644 --- a/docs.md +++ b/docs.md @@ -1,97 +1,435 @@ # Helios Documentation -## Deployment +## Storage -```bash -wrangler r2 bucket create helios-cache -wrangler d1 create helios-cache -# set database_id in workers/cache/wrangler.jsonc -wrangler d1 migrations apply helios-cache --remote -wrangler secret put JWT_SECRET -wrangler secret put ADMIN_SECRET -wrangler secret put SIGNING_PRIVATE_KEY -wrangler secret put SIGNING_KEY_NAME -pnpm deploy +Everything lives under `--data-dir`: + +| Path | Contents | +|------|----------| +| `helios.db` | SQLite (WAL): caches, blobs, published paths, tokens, audit log | +| `nar//.nar.zst` | Content-addressed compressed NARs | +| `tmp/` | In-flight uploads, renamed into `nar/` once verified | +| `backups/` | Database backups made on request of helios-daemon | +| `quarantine/` | NARs the integrity scrub found missing or corrupt | + +Uploads are decompressed and hashed by the server as they stream in, so a +path is only published against a NAR whose hash the server computed itself. +Narinfo is rendered and signed once, at publish time. + +## Server configuration + +Every flag can also be set through its environment variable. + +| Flag | Env | Default | +|------|-----|---------| +| `--listen` | `HELIOS_LISTEN` | `127.0.0.1:8080` | +| `--data-dir` | `HELIOS_DATA_DIR` | `/var/lib/helios` | +| `--signing-key-file` | `HELIOS_SIGNING_KEY_FILE` | unset: narinfo is unsigned | +| `--jwt-secret-file` | `HELIOS_JWT_SECRET_FILE` | unset: no API tokens | +| `--admin-secret-file` | `HELIOS_ADMIN_SECRET_FILE` | unset: no admin API | +| `--accel-redirect` | `HELIOS_ACCEL_REDIRECT` | unset: server streams NARs itself | +| `--trust-proxy` | `HELIOS_TRUST_PROXY` | `false` (audit IPs from `X-Forwarded-For`) | +| `--narinfo-cache-entries` | `HELIOS_NARINFO_CACHE_ENTRIES` | `262144` | +| `--max-upload-bytes` | `HELIOS_MAX_UPLOAD_BYTES` | 64 GiB | +| `--gc-interval-hours` | `HELIOS_GC_INTERVAL_HOURS` | `6` | +| `--audit-retention-days` | `HELIOS_AUDIT_RETENTION_DAYS` | `30` | +| `--upload-grace-seconds` | `HELIOS_UPLOAD_GRACE_SECONDS` | `3600`: an unpublished upload is kept this long | +| `--admin-socket` | `HELIOS_ADMIN_SOCKET` | unset: no maintenance API (see helios-daemon) | +| `--caches` | `HELIOS_CACHES` | unset: caches to create at startup, such as `main,team:private`; a listed cache that exists takes the listed visibility, and unlisted caches are left alone | +| `--compression-level` | `HELIOS_COMPRESSION_LEVEL` | `3`: zstd level clients use unless told otherwise (0: uncompressed) | +| `--compression-window-log` | `HELIOS_COMPRESSION_WINDOW_LOG` | `27`: long-distance matching window clients use (0: the level's own) | + +Secret files must hold at least 16 bytes. + +## Logging + +The server, daemon and CLI log through `tracing`, configured the same way: + +| Variable | | +|----------|--| +| `HELIOS_LOG` | a level (`error`, `warn`, `info`, `debug`, `trace`; default `info`), optionally with per-target directives, such as `info,helios_server=debug` | +| `HELIOS_LOG_FORMAT` | `journald`, `json` or `text`. The default is `journald` under systemd and `text` otherwise | + +Under journald, events are structured entries: their fields (`blob`, +`cache`, `bytes` and so on) are journal fields, so `journalctl -u helios +PRIORITY=4` or `journalctl BLOB=` filters on them. `json` writes one +object per line for container log collectors. Requests log at `debug`; a +request that fails with a 5xx logs at `warn` with its method and path. + +The server links the system SQLite; build with `--features bundled-sqlite` +to compile SQLite in instead. + +## NixOS + +The flake exports `nixosModules.default`. It runs the server as a hardened +systemd service and, when `domain` is set, puts nginx in front of it with +TLS from Let's Encrypt and zero-copy NAR downloads: + +```nix +# flake.nix +{ + inputs.helios.url = "github:Polymath-AS/helios"; + outputs = { nixpkgs, helios, ... }: { + nixosConfigurations.cache = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ helios.nixosModules.default ./configuration.nix ]; + }; + }; +} +``` + +```nix +# configuration.nix +{ + services.helios = { + enable = true; + domain = "cache.example.com"; + signingKeyFile = "/run/secrets/helios-signing-key"; + jwtSecretFile = "/run/secrets/helios-jwt-secret"; + adminSecretFile = "/run/secrets/helios-admin-secret"; + }; + security.acme = { + acceptTerms = true; + defaults.email = "ops@example.com"; + }; +} ``` -| Secret | Purpose | -|--------|---------| -| `JWT_SECRET` | HMAC-SHA256 key for JWT token signing | -| `ADMIN_SECRET` | Bearer token for the admin API | -| `SIGNING_PRIVATE_KEY` | Ed25519 key for narinfo signing | -| `SIGNING_KEY_NAME` | Key name prefix for signatures | +Caches can be declared instead of created by hand: -## Setup +```nix +services.helios.caches = { + main = { }; + team.public = false; # reads need a token with pull +}; +``` -Configure the CLI with your server URL and admin secret: +A declared cache that exists takes the declared visibility. Removing one from +the list leaves it and its paths in place; there is no declarative delete. -```bash -helios login prod https://your-worker.workers.dev $ADMIN_SECRET +Secret files are handed to the service as systemd credentials, so they can +be root-only and come from any secret manager (agenix, sops-nix, or files +placed by hand). Without `domain`, the module runs only the server on +`listen`, for use behind another proxy or on a private network. + +| Option | Default | | +|--------|---------|--| +| `listen` | `127.0.0.1:8080` | keep on loopback behind nginx | +| `dataDir` | `/var/lib/helios` | any other path, such as a mounted disk, is created and allowed in the sandbox | +| `domain` | `null` | enables nginx for this host name | +| `nginx.acme` | `true` | set `false` for plain HTTP or your own certificates | +| `openFirewall` | on with nginx | opens 80 and 443 | +| `logLevel` | `info` | | +| `settings.*` | | `narinfoCacheEntries`, `maxUploadBytes`, `gcIntervalHours`, `auditRetentionDays`, `compressionLevel`, `compressionWindowLog` | +| `caches..public` | `true` | caches created at startup; see below | + +`services.helios.daemon` (on by default) runs helios-daemon beside the +server; see [Maintenance](#maintenance). To keep the cache under a size: + +```nix +services.helios.daemon = { + quota = "500G"; + minFree = "20G"; + metrics.listen = "127.0.0.1:9120"; +}; ``` -## Using as a substituter +`services.helios.watchStore` goes on build machines, and needs no server +there. It installs a Nix post-build hook, so every path the machine builds is +pushed to a cache: ```nix -nix.settings.substituters = [ "https://your-worker.workers.dev/main" ]; +services.helios.watchStore = { + enable = true; + url = "https://cache.example.com"; + cache = "main"; + tokenFile = "/run/secrets/helios-push-token"; +}; ``` -## Managing tokens +`overlays.default` adds `pkgs.helios`. `nix flake check` runs a VM test of +all of the above: push, signed substitution, sandboxing, metrics, backups, +a scrub that catches a corrupted NAR, watch-store, eviction and shutdown. -JWT tokens scope write access to specific caches. The signed JWT is -returned once on creation and never stored. Save it immediately. +## Docker -```bash -# Create a token scoped to the "main" cache with push access -helios token create ci-runner --caches main --perms push --expires 90 +`ghcr.io/polymath-as/helios` is an OCI image with the server and the +maintenance daemon for x86_64 and aarch64, running as uid 10000 with all +state in `/var/lib/helios`. CI publishes `latest` and the version for each +`v*` tag, `master` for the branch, and `sha-` for every build. To +build it yourself, `nix build .#docker` gives a tarball for `docker load`. -# Create a token with access to all caches -helios token create admin-bot --caches "*" --perms push,pull +```sh +docker run -d --name helios -p 8080:8080 -v helios:/var/lib/helios \ + -e HELIOS_QUOTA=500G ghcr.io/polymath-as/helios:latest +docker exec helios helios-public-key # for trusted-public-keys +docker exec helios helios-admin cache create main +docker exec helios helios-admin token create ci --caches main --perms push,pull +``` -# List all tokens -helios token list +On first start the entrypoint generates a signing key (named by +`HELIOS_KEY_NAME`, default `helios-1`), a token secret and an admin secret +into `/var/lib/helios/secrets`, and logs the public key. It never overwrites +them, so keep the volume and the keys survive upgrades. To provide your own +instead, for example as Docker secrets, set all three of +`HELIOS_SIGNING_KEY_FILE`, `HELIOS_JWT_SECRET_FILE` and +`HELIOS_ADMIN_SECRET_FILE`. + +Both binaries take their flags as `HELIOS_*` environment variables (see +`helios-server --help` and `helios-daemon --help`), such as `HELIOS_QUOTA`, +`HELIOS_CACHES=main,team:private`, `HELIOS_MIN_FREE`, `HELIOS_METRICS_LISTEN`, +`HELIOS_LOG` and +`HELIOS_LOG_FORMAT=json` (see [Logging](#logging)). Set +`HELIOS_DAEMON=0` to run the server alone. `docker stop` shuts both down +cleanly; if either exits on its own, the container exits non-zero so the +restart policy takes over. The image has no reverse proxy: put one in front +for TLS. `nix build .#checks.x86_64-linux.docker` runs the image in a VM +under real Docker. + +## Reverse proxy and zero-copy downloads + +Outside NixOS, terminate TLS in a reverse proxy. With `--accel-redirect /_nar`, the server +answers NAR requests with an `X-Accel-Redirect` header after the access +check, and nginx serves the file with `sendfile`: -# Revoke a token -helios token revoke $JTI "employee offboarded" +```nginx +location / { + proxy_pass http://127.0.0.1:8080; + proxy_request_buffering off; # stream uploads + client_max_body_size 0; +} +location /_nar/ { + internal; + alias /var/lib/helios/nar/; +} ``` -| Option | Default | Description | -|--------|---------|-------------| -| `--caches` | `*` | Comma-separated cache names or `*` for all | -| `--perms` | `push` | Comma-separated: `push` and/or `pull` | -| `--expires` | `90` | Token lifetime in days (1-365) | +## Caches and tokens -## Pushing store paths +```bash +helios cache create main # public: anyone can read +helios cache create team --private # reads need a token with `pull` +helios cache list + +helios token create reader --caches team # read-only (the default) +helios token create ci --caches main --perms push --expires 90 +helios token list +helios token revoke "reason" # takes effect immediately +``` -Log in with a push token (not the admin secret): +Admin commands use a server logged in with the admin secret; pass +`--server ` to pick one. Tokens are HS256 JWTs scoped to cache names +(or `*`) and to `pull` and/or `push`. Tokens are read-only unless created +with `push`, and `push` does not imply `pull`: a builder can upload to a +private cache without being able to read it. A refused request says why, +such as `token does not grant push on cache 'main'` or `token has been +revoked`. + +## Substituting + +`helios use` configures Nix to pull from a cache, using the URL given to +`helios login` (not an address the server reports, which behind a proxy is +the wrong one): ```bash -helios login prod https://your-worker.workers.dev $PUSH_TOKEN +helios login reader https://cache.example.com "$READ_TOKEN" +helios use team # adds to ~/.config/nix/nix.conf and netrc +helios use team --print # system-wide settings, for NixOS or /etc/nix ``` -Then push: +It adds the substituter and the cache's signing key and, for a private +cache, a netrc entry with the login's token (Nix sends it as HTTP Basic +auth; the server reads the password). Running it again changes nothing, and +logging in with a new token replaces the old entry. The Nix daemon ignores +substituters from users it does not trust; `helios use` warns when that is +the case, and `--print` gives the `nix.settings` to use instead. + +## Pushing ```bash -# Single path -helios push main /nix/store/abc...-hello +helios push main /nix/store/...-hello # single paths +helios push main --closure .#nixosConfigurations.host.config.system.build.toplevel +helios push main --closure /run/current-system --jobs 8 --level 9 +``` + +The CLI serialises, zstd-compresses and hashes each NAR in one pass and +streams it straight into the upload, without temp files. It checks every +NAR hash against the Nix database. NARs the cache already holds, or that a +public cache serves, are reused instead of uploaded; a NAR only another +private cache holds is uploaded again, since reusing it would expose it. +Paths are published dependencies-first, in batches. + +A NAR larger than `--chunk-size` (MiB, default 32) is uploaded in chunks of +that size, so pushes work through proxies that cap request bodies, such as +Cloudflare (100 MB on the free plan) and Cloud Run (32 MiB). Each chunk is +retried on its own after a network or server error, and the server's offset +decides whether it has to be sent again. `--chunk-size 0` streams every NAR +in one request. + +## Compression + +NARs are compressed with zstd at level 3 with long-distance matching over a +128 MiB window (`--window-log 27`). On an 8.2 GiB NixOS system closure that +stores 3357 MiB, against 3697 MiB for plain zstd level 2 and 3568 MiB for +level 3: 9% and 6% smaller. It costs about half the single-core speed of +level 2 (129 against 260 MB/s), but push compresses on every core, so even +a two-core machine stays ahead of most uplinks, and fewer bytes means a +faster push. + +| Level | Size vs level 2 | MB/s per core | +|-------|-----------------|---------------| +| 3, long window (default) | -9.2% | 129 | +| 6, long window | -12.2% | 63 | +| 9, long window | -14.3% | 33 | +| 19, long window | -21.2% | 3 | + +`--level` (0-19; 0 uploads uncompressed) and `--window-log` (10-27, or 0 +for the level's own window) override the server's defaults, which are set +with `HELIOS_COMPRESSION_LEVEL` and `HELIOS_COMPRESSION_WINDOW_LOG`; clients +without either flag use them. Every NAR is compressed with its exact size +pledged to zstd, so the window never exceeds the NAR and a decoder allocates +no more than the NAR needs. 27 is the largest window a stock Nix decoder +accepts. `bench/compression.sh` reproduces the measurements on any closure. + +## HTTP API + +Substituter endpoints (`GET`/`HEAD`): + +| Path | | +|------|--| +| `//nix-cache-info` | | +| `//.narinfo` | misses are answered from memory | +| `//nar/.nar.zst` | only served by caches that publish it | +| `//build-trace-v2//.doi`, `//realisations/.doi` | build traces of CA derivations, from memory | + +`GET /_api/v2/caches/` returns `{"name", "public", "publicKey"}` to +anyone who can read the cache. + +Push endpoints (bearer token with `push` on the cache): + +| Method and path | Body | | +|-----------------|------|--| +| `POST /_api/v2/caches//missing` | `{"hashes": [...]}` | store path hashes absent from the cache | +| `POST /_api/v2/caches//nars/known` | `{"narHashes": [...]}` | NAR hashes the server already has | +| `PUT /_api/v2/caches//nar?compression=zstd` | compressed NAR | returns the verified hashes and sizes | +| `POST /_api/v2/caches//uploads?compression=zstd` | | starts a chunked upload: `{"id", "offset": 0}` | +| `PATCH /_api/v2/caches//uploads/?offset=` | the next chunk, up to 64 MiB | appends it whole or not at all: `{"offset"}`; a stale offset gets `409` with the current one | +| `GET /_api/v2/caches//uploads/` | | `{"offset"}`, to resume after a failed request | +| `POST /_api/v2/caches//uploads//complete` | | as `PUT /nar` | +| `DELETE /_api/v2/caches//uploads/` | | abandons it | +| `POST /_api/v2/caches//paths` | `{"paths": [...]}` | publishes a batch in one transaction; `409 nar_required` lists paths without an uploaded NAR | + +Chunks are verified as they arrive, as in a single `PUT`. An upload belongs +to the token that started it, is dropped after an hour without a chunk or +after an invalid one, and does not survive a server restart (the client +then starts that NAR again). + +`POST /_api/v2/caches//build-traces` (`push`) takes `{"entries": [...]}` +as `nix store build-trace info --json` (or `nix realisation info --json`) +prints them. -# Full closure -helios push main --closure /run/current-system +Pin endpoints: `GET /_api/v2/caches//pins` (`pull`), `POST +/_api/v2/caches//pins` with `{"storePaths": [...]}` and `DELETE +/_api/v2/caches//pins/` (`push`). -# Flake output closure -helios push main --closure .#nixosConfigurations.myhost.config.system.build.toplevel +Admin endpoints (bearer admin secret): `GET/POST /_api/v2/admin/caches`, +`GET/POST /_api/v2/admin/tokens`, `POST /_api/v2/admin/tokens//revoke`. + +## Garbage collection + +Every `--gc-interval-hours`, the server removes stale temp uploads, blobs no +path references (once past `--upload-grace-seconds`, so an upload is never +collected before it is published), expired tokens, and old audit log rows. + +## Maintenance + +helios-daemon keeps a cache healthy without touching the database itself: +it decides what to do and asks the server, over the Unix socket given to +the server as `--admin-socket`, so the server's in-memory index always +agrees with SQLite. The socket's permissions are its access control. + +- **Auto-GC.** With `--quota`, stored NARs are kept under that size. + Eviction starts above `--quota-high` (0.9) and removes the least recently + used paths down to `--quota-low` (0.8). Recency comes from narinfo hits, + which the server records in memory and flushes once a minute. + `--min-free` also evicts while the disk is short on space. NARs another + path still uses stay, and an evicted NAR is deleted once its upload grace + period has passed. + Pinned paths stay too; see [Pins](#pins). +- **Integrity scrub.** Every `--scrub-interval` (7d), every NAR is read at up + to `--scrub-rate` (64M per second), then decompressed and re-hashed. A + missing or corrupt NAR is quarantined, which unpublishes its paths, so no + client downloads bad data. +- **SQLite.** WAL checkpoints (`--checkpoint-interval`, 15m), `PRAGMA + optimize` (`--optimize-interval`, 1d) and online backups to + `/backups` (`--backup-interval`, 1d; `--backup-keep`, 7). +- **Metrics.** `--metrics-listen` serves Prometheus metrics: + - cache paths, NAR bytes, and narinfo hits and misses; + - uploads; + - auto-GC, scrub and backup results; + - disk space. + +Intervals take a unit (`30s`, `5m`, `24h`, `7d`) and sizes an optional +binary suffix (`64M`, `500G`). A failed run is logged, counted in +`helios_maintenance_errors_total`, and retried on the next tick. + +The maintenance API (`/v1/stats`, `/v1/lru`, `/v1/evict`, `/v1/blobs`, +`/v1/quarantine`, `/v1/gc`, `/v1/db/checkpoint`, `/v1/db/optimize`, +`/v1/db/backup`) is internal and may change between versions. + +## Pins + +A pin keeps a store path and its closure in a cache through auto-GC, for +releases or systems that must stay substitutable however rarely they are +fetched: + +```bash +helios push main --closure --pin .#nixosConfigurations.host.config.system.build.toplevel +helios pin main /nix/store/...-release # or an installable +helios pins main +helios unpin main /nix/store/...-release ``` -Use `--jobs ` to control parallelism (default 8). +The closure is followed through the References of the published narinfo +when auto-GC runs, so dependencies pushed after the pin are protected too. +It stops at a dependency the cache does not have: pin after pushing the +closure (as `--pin` does). Pinning and unpinning need `push` on the cache, +listing needs `pull`. Pins only hold back eviction: the integrity scrub +still unpublishes a corrupt NAR, and pushing the closure again restores it. +When what is left over quota is pinned, the daemon logs that it cannot +free enough. + +## Content-addressed derivations + +A content-addressed (CA) derivation's output path is only known once it is +built, so Nix looks it up in the cache's build traces ("realisations") +before it can substitute. `helios push` publishes them for the CA outputs it +pushes, when Nix has the experimental `ca-derivations` feature enabled. -## Migrating from AUTH_TOKEN +The cache serves whichever format the pushing Nix produces: Nix 2.35 and +later use `//build-trace-v2//.doi`, earlier versions +`//realisations/sha256:!.doi`. Push and substitute +with the same side of that change. An entry is accepted only when its output +path is published in the cache, and is signed with the cache's key over the +fingerprint Nix defines. Current Nix does not check those signatures when +substituting (a wrong one is accepted too); what it checks is the output +path's signed narinfo. Both formats are experimental upstream and may +change. -1. Set `JWT_SECRET` and `ADMIN_SECRET` via `wrangler secret put` -2. Log in with the admin secret: `helios login prod $URL $ADMIN_SECRET` -3. Create scoped tokens: `helios token create ci-runner --caches main` -4. Update publishers to log in with their JWT -5. Remove `AUTH_TOKEN` to disable legacy auth +## Watch-store -## Audit logging +On a build machine, `helios queue-paths` works as Nix's `post-build-hook`. +It spools each build's outputs and returns at once, so builds never wait on +the network. `helios watch-store ` pushes the spooled closures and +retries with backoff while the server is unreachable. Both take `--spool`, +and the CLI accepts `--url` with `--token-file` (or `HELIOS_URL` and +`HELIOS_TOKEN`) instead of a saved login. + +## Benchmarks + +```bash +bench/run.sh # pkg/* vs the Haskell Nix libraries and Nix C++ +bench/flake-lock.sh # pkg/nix-flake-lock vs the nix-flake-lock crate +``` -All write operations and admin actions are logged to D1 with actor, -action, cache name, IP, and status code. Logs are retained for 30 days. +Both use the local store as their corpus and run single-threaded. diff --git a/flake.lock b/flake.lock index dfdfdf9..4b67419 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1776548001, - "narHash": "sha256-ZSK0NL4a1BwVbbTBoSnWgbJy9HeZFXLYQizjb2DPF24=", + "lastModified": 1791048980, + "narHash": "sha256-KgItSKML8Xvte0B7/uGnBDsYzSnnKHcOaiUbgWBXLXw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b12141ef619e0a9c1c84dc8c684040326f27cdcc", + "rev": "a7868a727837f3c09cee2ce0ca671c76b1589fed", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 969a727..f7987eb 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,5 @@ { - description = "Helios – Cloudflare-native Nix binary cache"; + description = "Helios: self-hosted Nix binary cache (Rust server and CLI, Zig core)"; inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; @@ -7,88 +7,92 @@ outputs = { self, nixpkgs }: let - systems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]; + # NAR serialisation is implemented for Linux only. + systems = [ "x86_64-linux" "aarch64-linux" ]; forAllSystems = fn: nixpkgs.lib.genAttrs systems (system: fn nixpkgs.legacyPackages.${system}); - pnpmDepsFor = pkgs: pkgs.fetchPnpmDeps { + helios' = pkgs: pkgs.rustPlatform.buildRustPackage { pname = "helios"; - version = "0.0.0"; - src = self; - hash = "sha256-FsWEGFPLmT7jU7QIv2FGqVHOdNMUqvqmnWmog2Wpfuw="; - fetcherVersion = 3; + version = "0.1.0"; + # Only what the build reads, so docs and module edits do not rebuild it. + src = pkgs.lib.cleanSourceWith { + src = self; + filter = path: _type: + let + rel = pkgs.lib.removePrefix "${toString self}/" (toString path); + top = builtins.head (pkgs.lib.splitString "/" rel); + base = baseNameOf path; + in + builtins.elem top [ "Cargo.toml" "Cargo.lock" "core" "crates" "pkg" ] + && !(builtins.elem base [ "target" ".zig-cache" "zig-out" ]); + }; + cargoLock.lockFile = ./Cargo.lock; + + nativeBuildInputs = [ pkgs.zig_0_17 pkgs.pkg-config ]; + buildInputs = [ pkgs.zstd pkgs.sqlite ]; + + # Nix builds must not depend on the build machine's CPU. `baseline` + # loses SHA-NI accelerated hashing; for a dedicated host, build with + # HELIOS_ZIG_CPU=native (or e.g. x86_64_v3+sha) instead. + HELIOS_ZIG_CPU = "baseline"; + # zig is a build tool of helios-core's build.rs, not the package builder. + dontUseZigBuild = true; + dontUseZigCheck = true; + dontUseZigInstall = true; + + meta = { + description = "Self-hosted Nix binary cache"; + mainProgram = "helios"; + }; }; in { - packages = forAllSystems (pkgs: { - default = pkgs.stdenv.mkDerivation { - pname = "helios"; - version = "0.0.0"; - src = self; - - nativeBuildInputs = [ - pkgs.nodejs_22 - pkgs.pnpm - pkgs.pnpmConfigHook - pkgs.makeWrapper - ]; - - pnpmDeps = pnpmDepsFor pkgs; - - buildPhase = '' - pnpm --filter @helios/cli exec tsc - ''; - - installPhase = '' - mkdir -p $out/lib/helios $out/bin + packages = forAllSystems (pkgs: rec { + helios = helios' pkgs; + default = helios; + # OCI image with the server and maintenance daemon; see nix/docker.nix. + docker = import ./nix/docker.nix { inherit pkgs helios; }; + }); - # Copy the full workspace structure so pnpm symlinks resolve - cp -r apps $out/lib/helios/apps - cp -r packages $out/lib/helios/packages - cp -r workers $out/lib/helios/workers - cp -r node_modules $out/lib/helios/node_modules - cp package.json $out/lib/helios/package.json + overlays.default = final: _prev: { helios = helios' final; }; - makeWrapper ${pkgs.nodejs_22}/bin/node $out/bin/helios \ - --add-flags "$out/lib/helios/apps/cli/dist/main.js" \ - --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.nix pkgs.zstd ]} - ''; + # services.helios; the package defaults to this flake's build. + nixosModules.default = + { lib, pkgs, ... }: + { + imports = [ ./nix/module.nix ]; + services.helios.package = lib.mkDefault (helios' pkgs); }; - }); + nixosModules.helios = self.nixosModules.default; devShells = forAllSystems (pkgs: { default = pkgs.mkShell { packages = [ - pkgs.nodejs_22 - pkgs.pnpm - pkgs.wrangler + pkgs.zig_0_17 + pkgs.cargo + pkgs.rustc + pkgs.clippy + pkgs.rustfmt + pkgs.pkg-config + pkgs.zstd + pkgs.sqlite + pkgs.jq + pkgs.curl ]; }; }); checks = forAllSystems (pkgs: { - default = pkgs.stdenv.mkDerivation { - name = "helios-check"; - src = self; - - nativeBuildInputs = [ - pkgs.nodejs_22 - pkgs.pnpm - pkgs.pnpmConfigHook - ]; - - pnpmDeps = pnpmDepsFor pkgs; - - buildPhase = '' - # Domain package: full check (tsc + unit tests) - pnpm --filter @helios/cache-domain check - - # Worker: type-check only (integration tests need workerd runtime) - pnpm --filter @helios/cache-worker exec tsc --noEmit - ''; - - installPhase = '' - touch $out - ''; + package = self.packages.${pkgs.stdenv.hostPlatform.system}.helios; + nixos = import ./nix/test.nix { + inherit pkgs; + module = ./nix/module.nix; + helios = self.packages.${pkgs.stdenv.hostPlatform.system}.helios; + }; + docker = import ./nix/docker-test.nix { + inherit pkgs; + image = self.packages.${pkgs.stdenv.hostPlatform.system}.docker; + helios = self.packages.${pkgs.stdenv.hostPlatform.system}.helios; }; }); }; diff --git a/nix/docker-test.nix b/nix/docker-test.nix new file mode 100644 index 0000000..01b5b61 --- /dev/null +++ b/nix/docker-test.nix @@ -0,0 +1,65 @@ +# VM test for the OCI image: load it into Docker, run it with generated +# secrets, administer it with docker exec, push to it from the host, and +# substitute back through Nix with signatures required. +{ pkgs, image, helios }: + +pkgs.testers.runNixOSTest { + name = "helios-docker"; + + nodes.machine = { + virtualisation.docker.enable = true; + virtualisation.diskSize = 4096; + virtualisation.memorySize = 2048; + environment.systemPackages = [ + helios + pkgs.hello + ]; + nix.settings.experimental-features = [ "nix-command" ]; + }; + + testScript = '' + import json + + machine.wait_for_unit("docker.service") + tag = machine.succeed("docker load < ${image} | sed -n 's/^Loaded image: //p'").strip() + machine.succeed(f"docker run -d --name helios -p 8080:8080 -v helios:/var/lib/helios {tag}") + machine.wait_until_succeeds("curl -sf http://127.0.0.1:8080/healthz") + + with subtest("runs unprivileged with generated secrets"): + assert machine.succeed("docker inspect -f '{{.Config.User}}' helios").strip() == "10000:10000" + # The container's init (the entrypoint) runs as the image user. + pid = machine.succeed("docker inspect -f '{{.State.Pid}}' helios").strip() + machine.succeed(f"grep -Eq '^Uid:\\s+10000\\s' /proc/{pid}/status") + machine.wait_until_succeeds("docker top helios | grep -q helios-daemon") + pubkey = machine.succeed("docker exec helios helios-public-key").strip() + assert pubkey.startswith("helios-1:"), pubkey + + with subtest("administer with docker exec, push from the host"): + machine.succeed("docker exec helios helios-admin cache create main") + token = json.loads(machine.succeed("docker exec helios helios-admin token create ci --caches main --perms push,pull 2>/dev/null"))["token"] + machine.succeed(f"helios login ci http://127.0.0.1:8080 {token}") + machine.succeed("helios push main --closure ${pkgs.hello}") + + with subtest("substitute with signatures required"): + machine.succeed( + f"NIX_CONFIG='trusted-public-keys = {pubkey}\nrequire-sigs = true' " + "nix copy --from http://127.0.0.1:8080/main --to /tmp/store ${pkgs.hello}" + ) + + with subtest("docker stop is clean; data and keys persist"): + machine.succeed("docker stop helios") + assert machine.succeed("docker inspect -f '{{.State.ExitCode}}' helios").strip() == "0" + machine.succeed("docker start helios") + machine.wait_until_succeeds("curl -sf http://127.0.0.1:8080/healthz") + machine.succeed("curl -sf http://127.0.0.1:8080/main/$(basename ${pkgs.hello} | cut -c1-32).narinfo") + assert machine.succeed("docker exec helios helios-public-key").strip() == pubkey + # The daemon waited for the new server, not the last run's socket. + machine.wait_until_succeeds("docker logs helios 2>&1 | grep -c 'helios-daemon started' | grep -qx 2") + machine.fail("docker logs helios 2>&1 | grep -q 'Connection refused'") + + with subtest("docker stop during startup is clean"): + # Without a handler yet, PID 1 would ignore the TERM until the kill. + machine.succeed("docker stop helios && docker start helios && timeout 20 docker stop -t 60 helios") + assert machine.succeed("docker inspect -f '{{.State.ExitCode}}' helios").strip() == "0" + ''; +} diff --git a/nix/docker.nix b/nix/docker.nix new file mode 100644 index 0000000..f346c20 --- /dev/null +++ b/nix/docker.nix @@ -0,0 +1,152 @@ +# OCI image: helios-server and helios-daemon in one container, running as +# an unprivileged user, with everything persistent under /var/lib/helios. +# +# docker load < $(nix build .#docker --print-out-paths) +# docker run -d -p 8080:8080 -v helios:/var/lib/helios helios: +# +# On first start, without HELIOS_*_FILE secrets configured, the entrypoint +# generates a signing key, token secret and admin secret into +# /var/lib/helios/secrets. +{ pkgs, helios }: + +let + uid = "10000"; + + entrypoint = pkgs.writeShellApplication { + name = "helios-entrypoint"; + runtimeInputs = [ helios pkgs.coreutils ]; + text = '' + data="''${HELIOS_DATA_DIR:-/var/lib/helios}" + secrets="$data/secrets" + + # docker stop: forward SIGTERM so both shut down cleanly. Installed + # first because bash is PID 1, which ignores signals without a + # handler, and a stop can come before either process has started. + stopping=0 + server= + daemon= + # shellcheck disable=SC2329 # invoked by the trap below + stop() { + stopping=1 + if [[ -n "$server$daemon" ]]; then + kill -TERM ''${server:+"$server"} ''${daemon:+"$daemon"} 2>/dev/null || true + fi + } + trap stop TERM INT + + # Generate secrets on first start unless they are provided (for + # example as Docker secrets under /run/secrets). + if [[ -z "''${HELIOS_SIGNING_KEY_FILE:-}''${HELIOS_JWT_SECRET_FILE:-}''${HELIOS_ADMIN_SECRET_FILE:-}" ]]; then + pubkey=$(helios-server generate-secrets --dir "$secrets" --key-name "''${HELIOS_KEY_NAME:-helios-1}") + echo "helios: signing key $pubkey (secrets in $secrets)" >&2 + export HELIOS_SIGNING_KEY_FILE="$secrets/signing-key" + export HELIOS_JWT_SECRET_FILE="$secrets/jwt-secret" + export HELIOS_ADMIN_SECRET_FILE="$secrets/admin-secret" + fi + + if [[ "''${1:-serve}" != serve ]]; then + exec "$@" + fi + if (( stopping )); then + exit 0 + fi + + export HELIOS_ADMIN_SOCKET="''${HELIOS_ADMIN_SOCKET:-/run/helios/admin.sock}" + # A restarted container keeps the last run's socket; without this the + # wait below would pass before the server is listening. + rm -f "$HELIOS_ADMIN_SOCKET" + helios-server & + server=$! + # The trap may have run between the fork and the assignment. + if (( stopping )); then stop; fi + if [[ "''${HELIOS_DAEMON:-1}" != 0 ]]; then + for _ in {1..100}; do + if [[ -S "$HELIOS_ADMIN_SOCKET" ]] || (( stopping )); then break; fi + sleep 0.1 + done + if (( ! stopping )); then + helios-daemon --socket "$HELIOS_ADMIN_SOCKET" --data-dir "$data" --state-dir "$data/daemon" & + daemon=$! + if (( stopping )); then stop; fi + fi + fi + + # If either process exits on its own, take the other down too. + status=0 + wait -n || status=$? + kill -TERM "$server" ''${daemon:+"$daemon"} 2>/dev/null || true + wait || true + if (( stopping )); then + exit 0 + fi + exit $(( status == 0 ? 1 : status )) + ''; + }; + + # docker exec helpers; the image has no shell utilities of its own. + admin = pkgs.writeShellApplication { + name = "helios-admin"; + runtimeInputs = [ helios ]; + text = '' + # The CLI, pointed at this container's server with the admin secret: + # docker exec helios helios-admin cache create main + listen="''${HELIOS_LISTEN:-0.0.0.0:8080}" + exec helios --url "http://127.0.0.1:''${listen##*:}" \ + --token-file "''${HELIOS_ADMIN_SECRET_FILE:-''${HELIOS_DATA_DIR:-/var/lib/helios}/secrets/admin-secret}" "$@" + ''; + }; + publicKey = pkgs.writeShellApplication { + name = "helios-public-key"; + runtimeInputs = [ helios ]; + text = '' + exec helios-server --print-public-key \ + --signing-key-file "''${HELIOS_SIGNING_KEY_FILE:-''${HELIOS_DATA_DIR:-/var/lib/helios}/secrets/signing-key}" + ''; + }; + + nss = pkgs.dockerTools.fakeNss.override { + extraPasswdLines = [ "helios:x:${uid}:${uid}:helios:/var/lib/helios:/bin/false" ]; + extraGroupLines = [ "helios:x:${uid}:" ]; + }; +in +pkgs.dockerTools.buildLayeredImage { + name = "helios"; + tag = helios.version; + contents = [ + helios + entrypoint + admin + publicKey + pkgs.cacert + ]; + # Copies, not store symlinks: docker exec resolves the user through + # /etc/passwd and /etc/group and refuses absolute symlinks there. + fakeRootCommands = '' + mkdir -p etc var/lib/helios run/helios tmp + cp -L ${nss}/etc/passwd ${nss}/etc/group ${nss}/etc/nsswitch.conf etc/ + chown ${uid}:${uid} var/lib/helios run/helios + chmod 0750 var/lib/helios run/helios + chmod 1777 tmp + ''; + config = { + Entrypoint = [ "${entrypoint}/bin/helios-entrypoint" ]; + Cmd = [ "serve" ]; + User = "${uid}:${uid}"; + WorkingDir = "/var/lib/helios"; + Env = [ + "HELIOS_LISTEN=0.0.0.0:8080" + "HELIOS_DATA_DIR=/var/lib/helios" + "HOME=/var/lib/helios" + "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + ]; + ExposedPorts."8080/tcp" = { }; + Volumes."/var/lib/helios" = { }; + Labels = { + "org.opencontainers.image.title" = "helios"; + "org.opencontainers.image.description" = "Self-hosted Nix binary cache"; + "org.opencontainers.image.version" = helios.version; + # Links the GHCR package to the repository. + "org.opencontainers.image.source" = "https://github.com/Polymath-AS/helios"; + }; + }; +} diff --git a/nix/module.nix b/nix/module.nix new file mode 100644 index 0000000..a78f858 --- /dev/null +++ b/nix/module.nix @@ -0,0 +1,503 @@ +# services.helios: the Helios binary cache server, optionally behind nginx +# with ACME TLS. NAR downloads go through nginx's sendfile via +# X-Accel-Redirect after the server has checked access. +{ + config, + lib, + pkgs, + ... +}: + +let + cfg = config.services.helios; + inherit (lib) + mkEnableOption + mkIf + mkOption + optional + optionalString + types + ; + + dataDir = cfg.dataDir; + defaultDataDir = "/var/lib/helios"; + secret = name: file: optional (file != null) "${name}:${file}"; + + # Secrets are runtime paths given as strings: a path literal, or a string + # built from one, would put the secret in the world-readable store. + secretFileAssertion = option: file: { + assertion = file == null || (lib.hasPrefix "/" file && !(lib.hasPrefix "${builtins.storeDir}/" file)); + message = "services.helios.${option} must be an absolute path outside the Nix store, such as \"/run/secrets/helios\", given as a string: store paths are world-readable."; + }; + + # The maintenance socket lives in a setgid directory, so it belongs to + # group helios-admin: the daemon can reach it, nginx (group helios) cannot. + adminDir = "/run/helios-admin"; + adminSocket = "${adminDir}/admin.sock"; + + daemonArgs = lib.cli.toCommandLineShellGNU { } ( + { + socket = adminSocket; + data-dir = dataDir; + gc-interval = cfg.daemon.gcInterval; + scrub-interval = if cfg.daemon.scrub.interval == null then "0" else cfg.daemon.scrub.interval; + scrub-rate = cfg.daemon.scrub.rate; + checkpoint-interval = cfg.daemon.checkpointInterval; + optimize-interval = cfg.daemon.optimizeInterval; + backup-interval = if cfg.daemon.backup.interval == null then "0" else cfg.daemon.backup.interval; + backup-keep = cfg.daemon.backup.keep; + quota-high = cfg.daemon.quotaHigh; + quota-low = cfg.daemon.quotaLow; + } + // lib.optionalAttrs (cfg.daemon.quota != null) { quota = cfg.daemon.quota; } + // lib.optionalAttrs (cfg.daemon.minFree != null) { min-free = cfg.daemon.minFree; } + // lib.optionalAttrs (cfg.daemon.metrics.listen != null) { metrics-listen = cfg.daemon.metrics.listen; } + ); + + watchSpool = "/var/lib/helios-watch-store/spool"; + # Runs as root inside the nix-daemon after every build; a failing + # post-build hook fails the build, so queueing problems are only logged. + queueHook = pkgs.writeShellScript "helios-queue-paths" '' + ${cfg.package}/bin/helios queue-paths --spool ${watchSpool} \ + || echo "helios: could not queue $OUT_PATHS for pushing" >&2 + exit 0 + ''; + + hardening = { + NoNewPrivileges = true; + CapabilityBoundingSet = ""; + AmbientCapabilities = ""; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectKernelLogs = true; + ProtectControlGroups = true; + ProtectClock = true; + ProtectHostname = true; + ProtectProc = "invisible"; + ProcSubset = "pid"; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + LockPersonality = true; + MemoryDenyWriteExecute = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ "@system-service" "~@privileged" ]; + }; + credential = flag: name: file: optionalString (file != null) " ${flag} %d/${name}"; +in +{ + options.services.helios = { + enable = mkEnableOption "the Helios Nix binary cache"; + + package = mkOption { + type = types.package; + description = "Package providing `helios-server`."; + }; + + dataDir = mkOption { + type = types.str; + default = defaultDataDir; + description = '' + Where the database and NARs live. The default is managed by systemd + (StateDirectory); another path, such as a mounted data disk, is + created with the right ownership and made writable in the sandbox. + ''; + }; + + listen = mkOption { + type = types.str; + default = "127.0.0.1:8080"; + description = "Address the server listens on. Keep it on loopback when nginx is enabled."; + }; + + signingKeyFile = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Nix secret key (`nix key generate-secret`) used to sign narinfo. Read by + systemd as a credential, so it may be root-only (for example an agenix + secret). Without it narinfo is unsigned. + ''; + }; + + jwtSecretFile = mkOption { + type = types.nullOr types.str; + default = null; + description = "HMAC secret for API tokens (at least 16 bytes). Without it no tokens can be issued or used."; + }; + + adminSecretFile = mkOption { + type = types.nullOr types.str; + default = null; + description = "Bearer secret for the admin API (at least 16 bytes). Without it the admin API is disabled."; + }; + + caches = mkOption { + type = types.attrsOf (types.submodule { + options.public = mkOption { + type = types.bool; + default = true; + description = "Whether anyone may read the cache; a private cache needs a token with pull."; + }; + }); + default = { }; + example = { main = { }; team.public = false; }; + description = '' + Caches to create at startup. A declared cache that already exists takes + the declared visibility. Removing a cache from here does not delete it + or its paths. + ''; + }; + + logLevel = mkOption { + type = types.str; + default = "info"; + example = "info,helios_server=debug"; + description = "HELIOS_LOG for the server and daemon: a level, optionally with per-target directives."; + }; + + settings = { + narinfoCacheEntries = mkOption { + type = types.ints.positive; + default = 262144; + description = "Rendered narinfo bodies kept in memory."; + }; + maxUploadBytes = mkOption { + type = types.ints.positive; + default = 64 * 1024 * 1024 * 1024; + description = "Largest accepted compressed NAR."; + }; + gcIntervalHours = mkOption { + type = types.ints.positive; + default = 6; + }; + auditRetentionDays = mkOption { + type = types.ints.unsigned; + default = 30; + }; + compressionLevel = mkOption { + type = types.ints.between 0 19; + default = 3; + description = "zstd level clients compress with unless told otherwise; 0 uploads uncompressed."; + }; + compressionWindowLog = mkOption { + type = types.addCheck types.int (n: n == 0 || (n >= 10 && n <= 27)); + default = 27; + description = "zstd long-distance matching window clients use, as a power of two (10-27), or 0 for the level's own."; + }; + }; + + domain = mkOption { + type = types.nullOr types.str; + default = null; + example = "cache.example.com"; + description = "Public host name. When set, nginx serves the cache on it."; + }; + + nginx = { + enable = mkOption { + type = types.bool; + default = cfg.domain != null; + defaultText = lib.literalExpression "config.services.helios.domain != null"; + description = "Put nginx in front of the server (TLS, HTTP/2, zero-copy NAR downloads)."; + }; + acme = mkOption { + type = types.bool; + default = true; + description = "Obtain a certificate for `domain` from Let's Encrypt. Requires `security.acme` terms and email."; + }; + }; + + openFirewall = mkOption { + type = types.bool; + default = cfg.nginx.enable; + defaultText = lib.literalExpression "config.services.helios.nginx.enable"; + description = "Open TCP 80 and 443."; + }; + + daemon = { + enable = mkOption { + type = types.bool; + default = true; + description = '' + Run helios-daemon beside the server: WAL checkpoints, database + backups and an integrity scrub, plus auto-GC once a quota is set. + ''; + }; + quota = mkOption { + type = types.nullOr types.str; + default = null; + example = "500G"; + description = "Keep stored NARs under this size by evicting least-recently-used paths. Null disables auto-GC by size."; + }; + quotaHigh = mkOption { + type = types.float; + default = 0.9; + description = "Start evicting above this fraction of the quota."; + }; + quotaLow = mkOption { + type = types.float; + default = 0.8; + description = "Evict down to this fraction of the quota."; + }; + minFree = mkOption { + type = types.nullOr types.str; + default = null; + example = "20G"; + description = "Also evict while the data filesystem has less free space than this."; + }; + gcInterval = mkOption { + type = types.str; + default = "5m"; + }; + scrub = { + interval = mkOption { + type = types.nullOr types.str; + default = "7d"; + description = "Time between full integrity scrubs; null disables."; + }; + rate = mkOption { + type = types.str; + default = "64M"; + description = "Scrub read rate per second; \"0\" is unlimited."; + }; + }; + checkpointInterval = mkOption { + type = types.str; + default = "15m"; + }; + optimizeInterval = mkOption { + type = types.str; + default = "1d"; + }; + backup = { + interval = mkOption { + type = types.nullOr types.str; + default = "1d"; + description = "Time between database backups (to /backups); null disables."; + }; + keep = mkOption { + type = types.ints.positive; + default = 7; + }; + }; + metrics.listen = mkOption { + type = types.nullOr types.str; + default = null; + example = "127.0.0.1:9120"; + description = "Serve Prometheus metrics on this address."; + }; + }; + + watchStore = { + enable = mkEnableOption "pushing every locally built path to a Helios cache (post-build hook)"; + url = mkOption { + type = types.str; + example = "https://cache.example.com"; + description = "Helios server to push to."; + }; + cache = mkOption { + type = types.str; + example = "main"; + }; + tokenFile = mkOption { + type = types.str; + description = "File holding a push token for `cache`; read as a systemd credential."; + }; + jobs = mkOption { + type = types.ints.positive; + default = 8; + description = "Parallel uploads."; + }; + }; + }; + + config = lib.mkMerge [ + (mkIf cfg.enable { + assertions = [ + { + assertion = cfg.nginx.enable -> cfg.domain != null; + message = "services.helios.nginx.enable requires services.helios.domain"; + } + (secretFileAssertion "signingKeyFile" cfg.signingKeyFile) + (secretFileAssertion "jwtSecretFile" cfg.jwtSecretFile) + (secretFileAssertion "adminSecretFile" cfg.adminSecretFile) + ]; + + users.users.helios = { + isSystemUser = true; + group = "helios"; + home = dataDir; + }; + users.groups.helios = { }; + + systemd.services.helios = { + description = "Helios Nix binary cache"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + environment = { + HELIOS_LOG = cfg.logLevel; + HELIOS_LISTEN = cfg.listen; + HELIOS_DATA_DIR = dataDir; + HELIOS_NARINFO_CACHE_ENTRIES = toString cfg.settings.narinfoCacheEntries; + HELIOS_MAX_UPLOAD_BYTES = toString cfg.settings.maxUploadBytes; + HELIOS_GC_INTERVAL_HOURS = toString cfg.settings.gcIntervalHours; + HELIOS_AUDIT_RETENTION_DAYS = toString cfg.settings.auditRetentionDays; + HELIOS_COMPRESSION_LEVEL = toString cfg.settings.compressionLevel; + HELIOS_COMPRESSION_WINDOW_LOG = toString cfg.settings.compressionWindowLog; + } + // lib.optionalAttrs (cfg.caches != { }) { + HELIOS_CACHES = lib.concatStringsSep "," (lib.mapAttrsToList (name: c: if c.public then name else "${name}:private") cfg.caches); + } + // lib.optionalAttrs cfg.daemon.enable { + HELIOS_ADMIN_SOCKET = adminSocket; + } + // lib.optionalAttrs cfg.nginx.enable { + HELIOS_ACCEL_REDIRECT = "/_nar"; + HELIOS_TRUST_PROXY = "true"; + }; + + serviceConfig = { + ExecStart = + "${cfg.package}/bin/helios-server" + + credential "--signing-key-file" "signing-key" cfg.signingKeyFile + + credential "--jwt-secret-file" "jwt-secret" cfg.jwtSecretFile + + credential "--admin-secret-file" "admin-secret" cfg.adminSecretFile; + LoadCredential = + secret "signing-key" cfg.signingKeyFile + ++ secret "jwt-secret" cfg.jwtSecretFile + ++ secret "admin-secret" cfg.adminSecretFile; + + User = "helios"; + Group = "helios"; + # nginx (in group helios) reads NARs for X-Accel-Redirect. + StateDirectory = mkIf (dataDir == defaultDataDir) "helios"; + StateDirectoryMode = "0750"; + ReadWritePaths = optional (dataDir != defaultDataDir) dataDir ++ optional cfg.daemon.enable adminDir; + UMask = "0027"; + Restart = "on-failure"; + RestartSec = 2; + LimitNOFILE = 65536; + + # The server only needs its state directory and the network. + RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ]; + } + // hardening; + }; + + systemd.tmpfiles.rules = mkIf (dataDir != defaultDataDir) [ "d '${dataDir}' 0750 helios helios -" ]; + + users.users.nginx = mkIf cfg.nginx.enable { extraGroups = [ "helios" ]; }; + + services.nginx = mkIf cfg.nginx.enable { + enable = true; + recommendedProxySettings = true; + recommendedTlsSettings = true; + recommendedOptimisation = true; + virtualHosts.${cfg.domain} = { + enableACME = cfg.nginx.acme; + forceSSL = cfg.nginx.acme; + locations."/" = { + proxyPass = "http://${cfg.listen}"; + extraConfig = '' + # Stream uploads straight to the server instead of spooling them. + proxy_request_buffering off; + client_max_body_size 0; + proxy_read_timeout 1h; + proxy_send_timeout 1h; + ''; + }; + locations."/_nar/" = { + alias = "${dataDir}/nar/"; + extraConfig = '' + internal; + ''; + }; + }; + }; + + networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ 80 443 ]; + }) + + (mkIf (cfg.enable && cfg.daemon.enable) { + users.users.helios-daemon = { + isSystemUser = true; + group = "helios-admin"; + # Reads NARs for the scrub. + extraGroups = [ "helios" ]; + }; + users.groups.helios-admin = { }; + + systemd.tmpfiles.rules = [ "d ${adminDir} 2750 helios helios-admin -" ]; + + systemd.services.helios-daemon = { + description = "Helios cache maintenance"; + wantedBy = [ "multi-user.target" ]; + after = [ "helios.service" ]; + wants = [ "helios.service" ]; + environment.HELIOS_LOG = cfg.logLevel; + serviceConfig = { + ExecStart = "${cfg.package}/bin/helios-daemon ${daemonArgs}"; + User = "helios-daemon"; + # Where an unfinished scrub resumes after a restart. + StateDirectory = "helios-daemon"; + Group = "helios-admin"; + SupplementaryGroups = [ "helios" ]; + Restart = "on-failure"; + RestartSec = 5; + # Maintenance must not compete with serving. + Nice = 10; + CPUSchedulingPolicy = "batch"; + IOSchedulingClass = "idle"; + # Reaches the server over its Unix socket; the network only for metrics. + PrivateNetwork = cfg.daemon.metrics.listen == null; + RestrictAddressFamilies = [ "AF_UNIX" ] ++ lib.optionals (cfg.daemon.metrics.listen != null) [ "AF_INET" "AF_INET6" ]; + } + // hardening; + }; + }) + + (mkIf cfg.watchStore.enable { + assertions = [ (secretFileAssertion "watchStore.tokenFile" cfg.watchStore.tokenFile) ]; + + users.users.helios-watch-store = { + isSystemUser = true; + group = "helios-watch-store"; + home = "/var/lib/helios-watch-store"; + }; + users.groups.helios-watch-store = { }; + + nix.settings.post-build-hook = queueHook; + systemd.tmpfiles.rules = [ "d ${watchSpool} 0750 helios-watch-store helios-watch-store -" ]; + + systemd.services.helios-watch-store = { + description = "Push locally built paths to ${cfg.watchStore.url}"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" "nix-daemon.socket" ]; + wants = [ "network-online.target" ]; + path = [ config.nix.package ]; + environment = { + HELIOS_URL = cfg.watchStore.url; + HOME = "/var/lib/helios-watch-store"; + }; + serviceConfig = { + ExecStart = "${cfg.package}/bin/helios --token-file %d/token watch-store ${lib.escapeShellArg cfg.watchStore.cache} --spool ${watchSpool} --jobs ${toString cfg.watchStore.jobs}"; + LoadCredential = [ "token:${cfg.watchStore.tokenFile}" ]; + User = "helios-watch-store"; + Group = "helios-watch-store"; + StateDirectory = "helios-watch-store"; + StateDirectoryMode = "0750"; + Restart = "always"; + RestartSec = 5; + Nice = 10; + RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ]; + } + // hardening; + }; + }) + ]; +} diff --git a/nix/test.nix b/nix/test.nix new file mode 100644 index 0000000..c86f19b --- /dev/null +++ b/nix/test.nix @@ -0,0 +1,229 @@ +# NixOS VM test for services.helios: a server behind nginx with the +# maintenance daemon, and a client that pushes a closure with the CLI, +# substitutes it back through Nix with signatures required, and pushes +# new builds automatically with watchStore. +{ pkgs, module, helios }: + +let + system = pkgs.stdenv.hostPlatform.system; +in +pkgs.testers.runNixOSTest { + name = "helios"; + + nodes.server = + { lib, ... }: + { + imports = [ module ]; + services.helios = { + enable = true; + package = helios; + domain = "server"; + # A custom location exercises the tmpfiles and sandbox path handling. + dataDir = "/srv/helios"; + caches = { + main = { }; + team.public = false; + }; + nginx.acme = false; + signingKeyFile = "/etc/helios/signing-key"; + jwtSecretFile = "/etc/helios/jwt-secret"; + adminSecretFile = "/etc/helios/admin-secret"; + daemon = { + metrics.listen = "127.0.0.1:9120"; + checkpointInterval = "5s"; + scrub = { + interval = "5s"; + rate = "0"; + }; + backup = { + interval = "5s"; + keep = 2; + }; + }; + }; + # Started by the test once the secrets exist. + systemd.services.helios.wantedBy = lib.mkForce [ ]; + systemd.services.helios-daemon.wantedBy = lib.mkForce [ ]; + nix.settings.experimental-features = [ "nix-command" ]; + }; + + nodes.client = + { lib, ... }: + { + imports = [ module ]; + services.helios = { + package = helios; + watchStore = { + enable = true; + url = "http://server"; + cache = "main"; + tokenFile = "/etc/helios-push-token"; + }; + }; + systemd.services.helios-watch-store.wantedBy = lib.mkForce [ ]; + environment.systemPackages = [ + helios + pkgs.hello + ]; + # A derivation that builds offline in the VM. storePath gives the + # builder string context, so the build sandbox includes busybox. + environment.etc."watched.nix".text = '' + derivation { + name = "watched"; + system = "${system}"; + builder = "''${builtins.storePath "${pkgs.busybox}"}/bin/sh"; + args = [ "-c" "echo built > $out" ]; + } + ''; + # Content-addressed derivations, whose outputs Nix finds through build + # traces. `top` depends on `dep`, so both traces are needed. + environment.etc."ca.nix".text = '' + let + sh = "''${builtins.storePath "${pkgs.busybox}"}/bin/sh"; + ca = name: script: derivation { + inherit name; + system = "${system}"; + builder = sh; + args = [ "-c" script ]; + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + }; + in rec { + dep = ca "ca-dep" "echo dep > $out"; + top = ca "ca-top" "echo top ''${dep} > $out"; + } + ''; + nix.settings.experimental-features = [ "nix-command" "ca-derivations" ]; + virtualisation.memorySize = 2048; + }; + + testScript = '' + import json + import re + + start_all() + + def narinfo_url(path): + return f"http://server/main/{path.split('/')[-1][:32]}.narinfo" + + def metric(name): + out = server.succeed("curl -sf http://127.0.0.1:9120/metrics") + for line in out.splitlines(): + if line.startswith(name + " ") or line.startswith(name + "{"): + return line + raise Exception(f"{name} not in metrics:\n{out}") + + with subtest("secrets and startup"): + server.succeed( + "install -d -m 0700 /etc/helios", + "nix key generate-secret --key-name test-1 > /etc/helios/signing-key", + "head -c 32 /dev/urandom | base64 > /etc/helios/jwt-secret", + "head -c 32 /dev/urandom | base64 > /etc/helios/admin-secret", + "chmod 0400 /etc/helios/*", + "systemctl start helios helios-daemon", + ) + server.wait_for_unit("helios.service") + server.wait_for_unit("helios-daemon.service") + server.wait_for_unit("nginx.service") + server.wait_for_open_port(8080) + server.wait_for_open_port(80) + client.wait_for_unit("multi-user.target") + pubkey = server.succeed("nix key convert-secret-to-public < /etc/helios/signing-key").strip() + admin = server.succeed("cat /etc/helios/admin-secret").strip() + + with subtest("sandboxing"): + # The unit reads its secrets as credentials, not from /etc. + server.fail("sudo -u helios cat /etc/helios/signing-key") + props = server.succeed("systemctl show helios -p NoNewPrivileges -p ProtectSystem -p User") + assert "NoNewPrivileges=yes" in props and "ProtectSystem=strict" in props and "User=helios" in props, props + # nginx may read NARs but not reach the maintenance socket. + server.succeed("stat -c %G /run/helios-admin/admin.sock | grep -qx helios-admin") + server.fail("sudo -u nginx curl -sf --unix-socket /run/helios-admin/admin.sock http://x/v1/stats") + server.succeed("sudo -u helios-daemon curl -sf --unix-socket /run/helios-admin/admin.sock http://x/v1/stats") + + with subtest("admin and push"): + client.succeed(f"helios login admin http://server {admin}") + caches = {c["name"]: c["public"] for c in json.loads(client.succeed("helios cache list"))["caches"]} + assert caches == {"main": True, "team": False}, caches + token = json.loads(client.succeed("helios token create ci --caches main --perms push,pull 2>/dev/null"))["token"] + client.succeed(f"helios login ci http://server {token}") + client.succeed("helios push main --closure ${pkgs.hello}") + out = client.succeed("helios push main --closure ${pkgs.hello} 2>&1") + assert "already in 'main'" in out, out + + with subtest("substitute with signatures required"): + client.succeed( + f"NIX_CONFIG='trusted-public-keys = {pubkey}\nrequire-sigs = true' " + "nix copy --from http://server/main --to /tmp/store ${pkgs.hello}" + ) + client.succeed(f"nix store verify --store /tmp/store --trusted-public-keys '{pubkey}' -r ${pkgs.hello}") + + with subtest("nginx serves NARs from disk"): + url = client.succeed(f"curl -sf {narinfo_url('${pkgs.hello}')} | sed -n 's/^URL: //p'").strip() + client.succeed(f"curl -sf -o /dev/null http://server/main/{url}") + # The internal location is not reachable directly. + client.fail("curl -sf -o /dev/null http://server/_nar/") + server.succeed("find /srv/helios/nar -name '*.nar.zst' | grep -q .") + server.succeed("stat -c %U:%G:%a /srv/helios | grep -qx helios:helios:750") + + with subtest("state survives restarts"): + server.succeed("systemctl restart helios") + server.wait_for_open_port(8080) + client.succeed(f"curl -sf {narinfo_url('${pkgs.hello}')}") + + with subtest("daemon: metrics, backups, scrub"): + server.wait_until_succeeds("curl -sf http://127.0.0.1:9120/metrics | grep -qx 'helios_up 1'") + assert int(metric('helios_cache_paths{cache="main"}').split()[-1]) > 0 + server.wait_until_succeeds("test $(ls /srv/helios/backups | wc -l) -eq 2", timeout=60) + # Corrupt a dependency's NAR; the scrub must take it out of service. + url = client.succeed(f"curl -sf {narinfo_url('${pkgs.glibc}')} | sed -n 's/^URL: nar\\///p'").strip() + server.succeed(f"printf garbage | dd of=/srv/helios/nar/{url[:2]}/{url} bs=1 seek=100 conv=notrunc") + server.wait_until_succeeds("ls /srv/helios/quarantine | grep -q .", timeout=60) + client.fail(f"curl -sf {narinfo_url('${pkgs.glibc}')}") + assert metric('helios_scrub_blobs_total{result="corrupt"}').endswith(" 1") + + with subtest("watch-store pushes new builds"): + client.succeed(f"echo {token} > /etc/helios-push-token", "systemctl start helios-watch-store") + built = client.succeed("nix-build /etc/watched.nix --no-out-link").strip() + client.wait_until_succeeds(f"curl -sf {narinfo_url(built)}", timeout=60) + + with subtest("content-addressed derivations substitute through build traces"): + top = client.succeed("nix-build /etc/ca.nix -A top --no-out-link").strip() + out = client.succeed(f"helios push main --closure {top} 2>&1") + assert re.search(r"published [1-9][0-9]* build traces", out), out + # A fresh store may build nothing, so both outputs have to be found + # through the signed traces. + client.succeed("nix copy --no-check-sigs --to /tmp/ca-store ${pkgs.busybox}") + substituted = client.succeed( + "nix-build --store /tmp/ca-store /etc/ca.nix -A top --no-out-link --max-jobs 0 " + f"--option substituters http://server/main --option trusted-public-keys '{pubkey}' --option require-sigs true" + ).strip() + assert substituted == top, (substituted, top) + + with subtest("auto-gc evicts least recently used paths, except pinned closures"): + # The scrub unpublished glibc; a re-push restores the whole closure, + # which --pin then protects. + client.succeed("helios push main --closure --pin ${pkgs.hello}") + client.succeed(f"curl -sf {narinfo_url('${pkgs.glibc}')}") + pinned = client.succeed("nix-store -qR ${pkgs.hello}").split() + before = int(metric('helios_cache_paths{cache="main"}').split()[-1]) + server.succeed( + "sudo -u helios-daemon timeout 5 ${helios}/bin/helios-daemon " + "--socket /run/helios-admin/admin.sock --data-dir /srv/helios " + "--quota 1K --gc-interval 1s --scrub-interval 0 --backup-interval 0 || test $? -eq 124" + ) + after = int(metric('helios_cache_paths{cache="main"}').split()[-1]) + assert after < before, (before, after) + for p in pinned: + client.succeed(f"curl -sf http://server/main/{p[11:43]}.narinfo") + assert after == len(pinned), (after, pinned) + + with subtest("SIGTERM shuts down cleanly"): + server.succeed("systemctl stop helios-daemon helios") + server.succeed("journalctl -u helios | grep -q 'shutting down'") + server.succeed("journalctl -u helios-daemon | grep -q 'shutting down'") + server.fail("systemctl is-failed helios") + server.fail("systemctl is-failed helios-daemon") + ''; +} diff --git a/package.json b/package.json deleted file mode 100644 index 67a4975..0000000 --- a/package.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "packageManager": "pnpm@11.15.1", - "type": "module", - "name": "helios", - "private": true, - "license": "SEE LICENSE IN LICENSE", - "description": "Cloudflare-native Nix binary cache", - "scripts": { - "build": "echo 'wrangler bundles at deploy time'", - "deploy": "pnpm exec wrangler d1 migrations apply CACHE_DB --remote && pnpm exec wrangler deploy", - "dev": "pnpm --filter @helios/cache-worker dev", - "check": "pnpm -r check" - }, - "cloudflare": { - "bindings": { - "AUTH_TOKEN": { - "description": "Bearer token for write operations (push). Generate with `openssl rand -hex 32`." - }, - "SIGNING_PRIVATE_KEY": { - "description": "Ed25519 private key in PEM format for signing narinfo. Generate with `nix key generate-secret --key-name your-cache-1`." - } - } - }, - "devDependencies": { - "wrangler": "^4.112.0" - } -} diff --git a/packages/cache-domain/package.json b/packages/cache-domain/package.json deleted file mode 100644 index cce22ec..0000000 --- a/packages/cache-domain/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "@helios/cache-domain", - "version": "0.0.0", - "private": true, - "type": "module", - "exports": { - ".": "./src/index.ts" - }, - "scripts": { - "test": "vitest run", - "check": "tsc --noEmit && vitest run" - }, - "devDependencies": { - "typescript": "^7.0.2", - "vitest": "~4.1.10" - } -} diff --git a/packages/cache-domain/src/index.ts b/packages/cache-domain/src/index.ts deleted file mode 100644 index 1d90619..0000000 --- a/packages/cache-domain/src/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -export type { - Brand, - CacheName, - StorePathHash, - NarHash, - FileHash, - Compression, - R2ObjectKey, - NarInfoFields, - ParseError, -} from './types.js'; - -export { - parseCacheName, - parseStorePathHash, - parseNarHash, - parseFileHash, - parseCompression, - buildR2ObjectKey, -} from './parsers.js'; diff --git a/packages/cache-domain/src/parsers.ts b/packages/cache-domain/src/parsers.ts deleted file mode 100644 index 15405d8..0000000 --- a/packages/cache-domain/src/parsers.ts +++ /dev/null @@ -1,70 +0,0 @@ -import type { - CacheName, - Compression, - FileHash, - NarHash, - ParseError, - R2ObjectKey, - StorePathHash, -} from './types.js'; - -const NIX_BASE32_CHARS = "0123456789abcdfghijklmnpqrsvwxyz"; -const NIX_BASE32_RE = new RegExp(`^[${NIX_BASE32_CHARS}]+$`); -const HEX_RE = /^[0-9a-f]{64}$/; -const CACHE_NAME_RE = /^[a-z0-9]([a-z0-9-]{0,62}[a-z0-9])?$/; -const VALID_COMPRESSIONS = new Set(["zstd", "xz", "bzip2", "none"]); - -function err(message: string): ParseError { - return { kind: "ParseError", message }; -} - -export function parseCacheName(raw: string): CacheName | ParseError { - if (!CACHE_NAME_RE.test(raw)) { - return err( - "CacheName must be 1-64 lowercase alphanumeric chars or hyphens, no leading/trailing hyphen", - ); - } - return raw as CacheName; -} - -export function parseStorePathHash(raw: string): StorePathHash | ParseError { - if (raw.length !== 32) { - return err("StorePathHash must be exactly 32 characters"); - } - if (!NIX_BASE32_RE.test(raw)) { - return err("StorePathHash contains invalid nix base-32 characters"); - } - return raw as StorePathHash; -} - -export function parseNarHash(raw: string): NarHash | ParseError { - if (!raw.startsWith("sha256:")) { - return err("NarHash must start with 'sha256:'"); - } - const hash = raw.slice(7); - if (hash.length !== 52) { - return err("NarHash hash part must be exactly 52 characters"); - } - if (!NIX_BASE32_RE.test(hash)) { - return err("NarHash contains invalid nix base-32 characters"); - } - return raw as NarHash; -} - -export function parseFileHash(raw: string): FileHash | ParseError { - if (!HEX_RE.test(raw)) { - return err("FileHash must be exactly 64 lowercase hex characters"); - } - return raw as FileHash; -} - -export function parseCompression(raw: string): Compression | ParseError { - if (!VALID_COMPRESSIONS.has(raw as Compression)) { - return err(`Compression must be one of: ${[...VALID_COMPRESSIONS].join(", ")}`); - } - return raw as Compression; -} - -export function buildR2ObjectKey(fileHash: FileHash, compression: Compression): R2ObjectKey { - return `nars/sha256/${fileHash}/${compression}.nar` as R2ObjectKey; -} diff --git a/packages/cache-domain/src/types.ts b/packages/cache-domain/src/types.ts deleted file mode 100644 index 3ecb3ad..0000000 --- a/packages/cache-domain/src/types.ts +++ /dev/null @@ -1,27 +0,0 @@ -export type Brand = T & { readonly __brand: B }; - -export type CacheName = Brand; -export type StorePathHash = Brand; -export type NarHash = Brand; -export type FileHash = Brand; -export type Compression = "zstd" | "xz" | "bzip2" | "none"; -export type R2ObjectKey = Brand; - -export interface NarInfoFields { - readonly storePath: string; - readonly storePathHash: StorePathHash; - readonly narHash: NarHash; - readonly narSize: number; - readonly fileHash: FileHash; - readonly fileSize: number; - readonly compression: Compression; - readonly references: readonly StorePathHash[]; - readonly deriver?: string; - readonly system?: string; - readonly signatures: readonly string[]; -} - -export interface ParseError { - readonly kind: "ParseError"; - readonly message: string; -} diff --git a/packages/cache-domain/test/parsers.test.ts b/packages/cache-domain/test/parsers.test.ts deleted file mode 100644 index 0f022ef..0000000 --- a/packages/cache-domain/test/parsers.test.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { - parseCacheName, - parseStorePathHash, - parseNarHash, - parseFileHash, - parseCompression, - buildR2ObjectKey, -} from '../src/index.js'; -import type { FileHash, Compression } from '../src/index.js'; - -function isParseError(result: unknown): result is { kind: 'ParseError'; message: string } { - return ( - typeof result === 'object' && - result !== null && - (result as { kind?: string }).kind === 'ParseError' - ); -} - -describe('parseCacheName', () => { - it.each(['my-cache', 'a', 'a1b2', 'abc', 'a-b'])('accepts valid name %j', (input) => { - const result = parseCacheName(input); - expect(isParseError(result)).toBe(false); - expect(result).toBe(input); - }); - - it.each([ - { input: '', label: 'empty string' }, - { input: 'My-Cache', label: 'uppercase' }, - { input: '-leading', label: 'leading hyphen' }, - { input: 'trailing-', label: 'trailing hyphen' }, - { input: 'has spaces', label: 'spaces' }, - { input: 'a'.repeat(65), label: '65-char string' }, - { input: 'has_underscore', label: 'underscore' }, - ])('rejects $label', ({ input }) => { - const result = parseCacheName(input); - expect(isParseError(result)).toBe(true); - }); -}); - -describe('parseStorePathHash', () => { - it('accepts 32 valid nix-base32 chars', () => { - const input = '0v9z1234abcdfghijklmnpqrsvwxyz01'; - const result = parseStorePathHash(input); - expect(isParseError(result)).toBe(false); - expect(result).toBe(input); - }); - - it.each([ - { input: '0'.repeat(31), label: '31 chars (too short)' }, - { input: '0'.repeat(33), label: '33 chars (too long)' }, - { input: '0'.repeat(31) + 'e', label: "contains 'e'" }, - { input: '0'.repeat(31) + 'o', label: "contains 'o'" }, - { input: '0'.repeat(31) + 't', label: "contains 't'" }, - { input: '0'.repeat(31) + 'u', label: "contains 'u'" }, - { input: '0'.repeat(31) + 'A', label: 'uppercase letters' }, - ])('rejects $label', ({ input }) => { - const result = parseStorePathHash(input); - expect(isParseError(result)).toBe(true); - }); -}); - -describe('parseNarHash', () => { - it('accepts sha256: prefix + 52 valid nix-base32 chars (zeros)', () => { - const input = 'sha256:' + '0'.repeat(52); - const result = parseNarHash(input); - expect(isParseError(result)).toBe(false); - expect(result).toBe(input); - }); - - it('accepts sha256: prefix + 52 mixed valid nix-base32 chars', () => { - const input = 'sha256:' + 'abcdfghijklmnpqrsvwxyz0123456789abcdfghijklmnpqrsvwx'; - const result = parseNarHash(input); - expect(isParseError(result)).toBe(false); - expect(result).toBe(input); - }); - - it.each([ - { input: '0'.repeat(52), label: 'no sha256: prefix' }, - { input: 'md5:' + '0'.repeat(52), label: 'md5: prefix' }, - { input: 'sha256:' + '0'.repeat(51), label: '51 hash chars' }, - { input: 'sha256:' + '0'.repeat(53), label: '53 hash chars' }, - { input: 'sha256:' + '0'.repeat(51) + 'e', label: "hash contains 'e'" }, - ])('rejects $label', ({ input }) => { - const result = parseNarHash(input); - expect(isParseError(result)).toBe(true); - }); -}); - -describe('parseFileHash', () => { - it('accepts 64 lowercase hex chars (repeated a)', () => { - const input = 'a'.repeat(64); - const result = parseFileHash(input); - expect(isParseError(result)).toBe(false); - expect(result).toBe(input); - }); - - it('accepts 64 lowercase hex chars (mixed)', () => { - const input = '0123456789abcdef'.repeat(4); - const result = parseFileHash(input); - expect(isParseError(result)).toBe(false); - expect(result).toBe(input); - }); - - it.each([ - { input: 'a'.repeat(63), label: '63 chars' }, - { input: 'a'.repeat(65), label: '65 chars' }, - { input: 'a'.repeat(63) + 'g', label: "contains 'g'" }, - { input: 'A'.repeat(64), label: 'uppercase hex' }, - ])('rejects $label', ({ input }) => { - const result = parseFileHash(input); - expect(isParseError(result)).toBe(true); - }); -}); - -describe('parseCompression', () => { - it.each(['zstd', 'xz', 'bzip2', 'none'] as const)('accepts %j', (input) => { - const result = parseCompression(input); - expect(isParseError(result)).toBe(false); - expect(result).toBe(input); - }); - - it.each([ - { input: 'gzip', label: 'gzip' }, - { input: '', label: 'empty string' }, - { input: 'ZSTD', label: 'uppercase ZSTD' }, - ])('rejects $label', ({ input }) => { - const result = parseCompression(input); - expect(isParseError(result)).toBe(true); - }); -}); - -describe('buildR2ObjectKey', () => { - const fileHash = 'a'.repeat(64) as FileHash; - - it('builds key with zstd compression', () => { - const result = buildR2ObjectKey(fileHash, 'zstd' as Compression); - expect(result).toBe(`nars/sha256/${'a'.repeat(64)}/zstd.nar`); - }); - - it('builds key with none compression', () => { - const result = buildR2ObjectKey(fileHash, 'none' as Compression); - expect(result).toBe(`nars/sha256/${'a'.repeat(64)}/none.nar`); - }); -}); diff --git a/packages/cache-domain/tsconfig.json b/packages/cache-domain/tsconfig.json deleted file mode 100644 index 17187e0..0000000 --- a/packages/cache-domain/tsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "compilerOptions": { - "target": "es2024", - "lib": ["es2024"], - "module": "es2022", - "moduleResolution": "Bundler", - "resolveJsonModule": true, - "noEmit": true, - "isolatedModules": true, - "allowSyntheticDefaultImports": true, - "forceConsistentCasingInFileNames": true, - "strict": true, - "skipLibCheck": true - }, - "include": ["src/**/*.ts"], - "exclude": ["test"] -} diff --git a/packages/cache-domain/vitest.config.mts b/packages/cache-domain/vitest.config.mts deleted file mode 100644 index 43e56f4..0000000 --- a/packages/cache-domain/vitest.config.mts +++ /dev/null @@ -1,7 +0,0 @@ -import { defineConfig } from 'vitest/config'; - -export default defineConfig({ - test: { - include: ['test/**/*.test.ts'], - }, -}); diff --git a/pkg/nix-archive/build.zig b/pkg/nix-archive/build.zig new file mode 100644 index 0000000..ad848ee --- /dev/null +++ b/pkg/nix-archive/build.zig @@ -0,0 +1,20 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mod = b.addModule("nix-archive", .{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, + }); + mod.link_libc = true; + + const tests = b.addTest(.{ .root_module = mod }); + tests.root_module.linkSystemLibrary("zstd", .{}); + b.step("test", "Run unit tests").dependOn(&b.addRunArtifact(tests).step); +} diff --git a/pkg/nix-archive/build.zig.zon b/pkg/nix-archive/build.zig.zon new file mode 100644 index 0000000..b3505b5 --- /dev/null +++ b/pkg/nix-archive/build.zig.zon @@ -0,0 +1,9 @@ +.{ + .name = .nix_archive, + .version = "0.1.0", + .fingerprint = 0xcb3a212dbdaff4ec, + .minimum_zig_version = "0.17.0", + .dependencies = .{ + }, + .paths = .{ "build.zig", "build.zig.zon", "src" }, +} diff --git a/pkg/nix-archive/src/root.zig b/pkg/nix-archive/src/root.zig new file mode 100644 index 0000000..d6a8f86 --- /dev/null +++ b/pkg/nix-archive/src/root.zig @@ -0,0 +1,31 @@ +//! NAR (Nix ARchive) production and verification. +//! +//! - `dump`: serialise a filesystem tree straight from the kernel (Linux). +//! - `Sink`: hash, count and zstd-compress a NAR stream in one pass. +//! - `Verifier`: hash and decompress an uploaded stream, checking it is a NAR. + +const walk = @import("walk.zig"); +const sink = @import("sink.zig"); +const verify = @import("verify.zig"); + +pub const Sink = sink.Sink; +pub const Digest = sink.Digest; +pub const WriteFn = sink.WriteFn; +pub const Options = sink.Options; +pub const SinkError = sink.Error; + +pub const dump = walk.dump; +pub const supported = walk.supported; +pub const DumpError = walk.Error; +pub fn lastErrno() i32 { + return walk.last_errno; +} + +pub const Verifier = verify.Verifier; +pub const Compression = verify.Compression; +pub const Sha256 = @import("sha256.zig").Sha256; + +test { + _ = @import("tests.zig"); + _ = @import("sha256.zig"); +} diff --git a/pkg/nix-archive/src/sha256.zig b/pkg/nix-archive/src/sha256.zig new file mode 100644 index 0000000..fa37c27 --- /dev/null +++ b/pkg/nix-archive/src/sha256.zig @@ -0,0 +1,173 @@ +//! Multi-block SHA-256 for x86-64 SHA-NI. +//! +//! std's SHA-NI path compresses one block per call: it loads and shuffles +//! the state from memory, runs the rounds, and stores it back, every 64 +//! bytes. Here the state stays in two xmm registers (ABEF / CDGH) for the +//! whole buffer, and message words are byte-swapped with one shuffle per +//! 16 bytes. The per-block instruction sequence is std's. Without SHA-NI +//! this is std's implementation. + +const std = @import("std"); +const builtin = @import("builtin"); + +pub const accelerated = builtin.target.cpu.arch == .x86_64 and + builtin.zig_backend != .stage2_c and + builtin.target.cpu.hasAll(.x86, &.{ .sha, .avx2 }); + +pub const Sha256 = if (accelerated) Fast else std.crypto.hash.sha2.Sha256; + +const V4u32 = @Vector(4, u32); + +const K = [64]u32{ + 0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5, 0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5, + 0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3, 0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174, + 0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC, 0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA, + 0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7, 0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967, + 0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13, 0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85, + 0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3, 0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070, + 0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5, 0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3, + 0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208, 0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2, +}; + +const iv = [8]u32{ 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19 }; + +inline fn loadBe(bytes: *const [16]u8) V4u32 { + const v: @Vector(16, u8) = bytes.*; + const swapped = @shuffle(u8, v, undefined, [16]i32{ 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12 }); + return @bitCast(swapped); +} + +/// Compresses `blocks` (a multiple of 64 bytes) into `state`. +fn compress(state: *[8]u32, blocks: []const u8) void { + var x: V4u32 = .{ state[5], state[4], state[1], state[0] }; + var y: V4u32 = .{ state[7], state[6], state[3], state[2] }; + var off: usize = 0; + while (off < blocks.len) : (off += 64) { + const x0 = x; + const y0 = y; + var m: [16]V4u32 = undefined; + inline for (0..4) |i| m[i] = loadBe(blocks[off + 16 * i ..][0..16]); + + inline for (0..16) |k| { + if (k < 12) { + var tmp = m[k]; + m[k + 4] = asm ( + \\ sha256msg1 %[w4_7], %[tmp] + \\ vpalignr $0x4, %[w8_11], %[w12_15], %[result] + \\ paddd %[tmp], %[result] + \\ sha256msg2 %[w12_15], %[result] + : [tmp] "=&x" (tmp), + [result] "=&x" (-> V4u32), + : [_] "0" (tmp), + [w4_7] "x" (m[k + 1]), + [w8_11] "x" (m[k + 2]), + [w12_15] "x" (m[k + 3]), + ); + } + const w: V4u32 = m[k] +% @as(V4u32, K[4 * k ..][0..4].*); + y = asm ("sha256rnds2 %[x], %[y]" + : [y] "=x" (-> V4u32), + : [_] "0" (y), + [x] "x" (x), + [_] "{xmm0}" (w), + ); + x = asm ("sha256rnds2 %[y], %[x]" + : [x] "=x" (-> V4u32), + : [_] "0" (x), + [y] "x" (y), + [_] "{xmm0}" (@as(V4u32, @bitCast(@as(u128, @bitCast(w)) >> 64))), + ); + } + x +%= x0; + y +%= y0; + } + state[0] = x[3]; + state[1] = x[2]; + state[4] = x[1]; + state[5] = x[0]; + state[2] = y[3]; + state[3] = y[2]; + state[6] = y[1]; + state[7] = y[0]; +} + +pub const Fast = struct { + pub const digest_length = 32; + pub const Options = struct {}; + + s: [8]u32 = iv, + buf: [64]u8 = undefined, + buf_len: u8 = 0, + total_len: u64 = 0, + + pub fn init(_: Options) Fast { + return .{}; + } + + pub fn hash(b: []const u8, out: *[digest_length]u8, options: Options) void { + var d = Fast.init(options); + d.update(b); + d.final(out); + } + + pub fn update(d: *Fast, b: []const u8) void { + var off: usize = 0; + if (d.buf_len != 0) { + const take = @min(64 - d.buf_len, b.len); + @memcpy(d.buf[d.buf_len..][0..take], b[0..take]); + d.buf_len += @intCast(take); + off = take; + if (d.buf_len < 64) { + d.total_len += b.len; + return; + } + compress(&d.s, &d.buf); + d.buf_len = 0; + } + const full = (b.len - off) / 64 * 64; + if (full > 0) compress(&d.s, b[off..][0..full]); + off += full; + const rest = b[off..]; + @memcpy(d.buf[0..rest.len], rest); + d.buf_len = @intCast(rest.len); + d.total_len += b.len; + } + + pub fn final(d: *Fast, out: *[digest_length]u8) void { + const bits = d.total_len * 8; + d.buf[d.buf_len] = 0x80; + @memset(d.buf[d.buf_len + 1 ..], 0); + if (d.buf_len >= 56) { + compress(&d.s, &d.buf); + @memset(&d.buf, 0); + } + std.mem.writeInt(u64, d.buf[56..64], bits, .big); + compress(&d.s, &d.buf); + for (d.s, 0..) |word, i| std.mem.writeInt(u32, out[4 * i ..][0..4], word, .big); + } +}; + +test "matches std SHA-256 for every length and split" { + const Std = std.crypto.hash.sha2.Sha256; + var prng: std.Random.DefaultPrng = .init(9); + const r = prng.random(); + var data: [1000]u8 = undefined; + r.bytes(&data); + for (0..data.len) |n| { + var want: [32]u8 = undefined; + Std.hash(data[0..n], &want, .{}); + var got: [32]u8 = undefined; + Sha256.hash(data[0..n], &got, .{}); + try std.testing.expectEqualSlices(u8, &want, &got); + + var h = Sha256.init(.{}); + var off: usize = 0; + while (off < n) { + const step = @min(n - off, r.uintAtMost(usize, 150)); + h.update(data[off..][0..step]); + off += step; + } + h.final(&got); + try std.testing.expectEqualSlices(u8, &want, &got); + } +} diff --git a/pkg/nix-archive/src/sink.zig b/pkg/nix-archive/src/sink.zig new file mode 100644 index 0000000..0e15fa5 --- /dev/null +++ b/pkg/nix-archive/src/sink.zig @@ -0,0 +1,184 @@ +//! The compressing sink: NAR bytes go in, get hashed and counted, are +//! zstd-compressed, and the compressed bytes are hashed, counted and +//! handed to a caller-supplied write callback. Everything happens in a +//! single pass; zstd worker threads do the compression when enabled. + +const std = @import("std"); +const zstd = @import("zstd.zig"); +const Sha256 = @import("sha256.zig").Sha256; + +pub const WriteFn = *const fn (ctx: ?*anyopaque, buf: [*]const u8, len: usize) callconv(.c) c_int; + +pub const Digest = extern struct { + file_hash: [32]u8, + file_size: u64, + nar_hash: [32]u8, + nar_size: u64, +}; + +pub const Error = error{ Zstd, Aborted, OutOfMemory }; + +pub const Options = struct { + /// 0 disables compression (the output is the raw NAR). + level: c_int = 3, + /// zstd worker threads; 0 compresses on the calling thread. + threads: c_int = 0, + /// The exact NAR size, 0 when unknown. It is pledged to zstd, which then + /// sizes its window to the NAR and records the size in the frame, so a + /// decoder allocates no more than the NAR needs. A stream of another + /// length fails. + nar_size: u64 = 0, + /// 0 keeps the level's window. Otherwise long-distance matching with a + /// window of 2^window_log bytes (at most 27, the largest a stock Nix + /// decoder accepts), which finds repeats across a large NAR. + window_log: c_int = 0, + /// false skips both SHA-256 passes; the digest then carries sizes only. + /// For an uploader whose server verifies the content anyway. + hash: bool = true, +}; + +const in_capacity = 1 << 20; + +/// Writes at least this large bypass the input buffer. +pub const direct_threshold = in_capacity; + +/// NAR streams below this size are compressed on the calling thread: +/// spinning up zstd workers costs more than it saves. +const multithread_threshold = 4 << 20; + +pub const Sink = struct { + hash: bool = true, + nar_hasher: Sha256 = .init(.{}), + nar_size: u64 = 0, + file_hasher: Sha256 = .init(.{}), + file_size: u64 = 0, + cctx: ?*zstd.CCtx = null, + in_buf: []u8, + in_len: usize = 0, + out_buf: []u8, + write_fn: WriteFn, + write_ctx: ?*anyopaque, + + pub fn init(allocator: std.mem.Allocator, options: Options, write_fn: WriteFn, write_ctx: ?*anyopaque) Error!Sink { + const in_buf = try allocator.alloc(u8, in_capacity); + errdefer allocator.free(in_buf); + + var cctx: ?*zstd.CCtx = null; + // At function scope so a failed out_buf allocation frees it too. + errdefer _ = zstd.ZSTD_freeCCtx(cctx); + var out_len: usize = 0; + if (options.level != 0) { + cctx = zstd.ZSTD_createCCtx() orelse return error.OutOfMemory; + if (zstd.isError(zstd.ZSTD_CCtx_setParameter(cctx.?, zstd.c_compressionLevel, options.level))) return error.Zstd; + if (options.threads > 0 and (options.nar_size == 0 or options.nar_size >= multithread_threshold)) { + // Fails harmlessly on a single-threaded libzstd build. + _ = zstd.ZSTD_CCtx_setParameter(cctx.?, zstd.c_nbWorkers, options.threads); + } + if (options.window_log != 0) { + if (options.window_log < 10 or options.window_log > 27) return error.Zstd; + if (zstd.isError(zstd.ZSTD_CCtx_setParameter(cctx.?, zstd.c_enableLongDistanceMatching, 1))) return error.Zstd; + if (zstd.isError(zstd.ZSTD_CCtx_setParameter(cctx.?, zstd.c_windowLog, options.window_log))) return error.Zstd; + } + if (options.nar_size > 0) { + if (zstd.isError(zstd.ZSTD_CCtx_setPledgedSrcSize(cctx.?, options.nar_size))) return error.Zstd; + } + out_len = zstd.ZSTD_CStreamOutSize(); + } + const out_buf = try allocator.alloc(u8, out_len); + + return .{ + .hash = options.hash, + .cctx = cctx, + .in_buf = in_buf, + .out_buf = out_buf, + .write_fn = write_fn, + .write_ctx = write_ctx, + }; + } + + pub fn deinit(self: *Sink, allocator: std.mem.Allocator) void { + if (self.cctx) |cctx| _ = zstd.ZSTD_freeCCtx(cctx); + allocator.free(self.in_buf); + allocator.free(self.out_buf); + self.* = undefined; + } + + /// Buffered write of NAR bytes. + pub fn write(self: *Sink, bytes: []const u8) Error!void { + if (self.in_len + bytes.len <= self.in_buf.len) { + @memcpy(self.in_buf[self.in_len..][0..bytes.len], bytes); + self.in_len += bytes.len; + return; + } + try self.flush(); + if (bytes.len >= self.in_buf.len) { + try self.feed(bytes); + } else { + @memcpy(self.in_buf[0..bytes.len], bytes); + self.in_len = bytes.len; + } + } + + /// Free tail of the input buffer, for callers that read straight into it. + pub fn spare(self: *Sink) []u8 { + return self.in_buf[self.in_len..]; + } + + pub fn commit(self: *Sink, n: usize) Error!void { + self.in_len += n; + if (self.in_len == self.in_buf.len) try self.flush(); + } + + pub fn flush(self: *Sink) Error!void { + if (self.in_len == 0) return; + const pending = self.in_buf[0..self.in_len]; + self.in_len = 0; + try self.feed(pending); + } + + fn feed(self: *Sink, data: []const u8) Error!void { + if (self.hash) self.nar_hasher.update(data); + self.nar_size += data.len; + const cctx = self.cctx orelse return self.emit(data); + var input: zstd.InBuffer = .{ .src = data.ptr, .size = data.len, .pos = 0 }; + while (input.pos < input.size) { + var output: zstd.OutBuffer = .{ .dst = self.out_buf.ptr, .size = self.out_buf.len, .pos = 0 }; + if (zstd.isError(zstd.ZSTD_compressStream2(cctx, &output, &input, zstd.e_continue))) return error.Zstd; + if (output.pos > 0) try self.emit(self.out_buf[0..output.pos]); + } + } + + fn emit(self: *Sink, data: []const u8) Error!void { + // Uncompressed output is the NAR itself: hash it once, not twice. + if (self.hash and self.cctx != null) self.file_hasher.update(data); + self.file_size += data.len; + if (self.write_fn(self.write_ctx, data.ptr, data.len) != 0) return error.Aborted; + } + + pub fn finish(self: *Sink) Error!Digest { + try self.flush(); + if (self.cctx) |cctx| { + var input: zstd.InBuffer = .{ .src = null, .size = 0, .pos = 0 }; + while (true) { + var output: zstd.OutBuffer = .{ .dst = self.out_buf.ptr, .size = self.out_buf.len, .pos = 0 }; + const remaining = zstd.ZSTD_compressStream2(cctx, &output, &input, zstd.e_end); + if (zstd.isError(remaining)) return error.Zstd; + if (output.pos > 0) try self.emit(self.out_buf[0..output.pos]); + if (remaining == 0) break; + } + } + var digest: Digest = undefined; + if (!self.hash) { + @memset(&digest.nar_hash, 0); + @memset(&digest.file_hash, 0); + } else { + self.nar_hasher.final(&digest.nar_hash); + if (self.cctx != null) self.file_hasher.final(&digest.file_hash) else { + digest.file_hash = digest.nar_hash; + } + } + digest.nar_size = self.nar_size; + digest.file_size = self.file_size; + return digest; + } +}; diff --git a/pkg/nix-archive/src/tests.zig b/pkg/nix-archive/src/tests.zig new file mode 100644 index 0000000..8726b27 --- /dev/null +++ b/pkg/nix-archive/src/tests.zig @@ -0,0 +1,135 @@ +const std = @import("std"); +const Sink = @import("sink.zig").Sink; +const Options = @import("sink.zig").Options; +const Verifier = @import("verify.zig").Verifier; + +const Collect = struct { + list: std.ArrayList(u8) = .empty, + + fn write(ctx: ?*anyopaque, buf: [*]const u8, len: usize) callconv(.c) c_int { + const self: *Collect = @ptrCast(@alignCast(ctx.?)); + self.list.appendSlice(std.testing.allocator, buf[0..len]) catch return 1; + return 0; + } +}; + +fn fakeNar(a: std.mem.Allocator, body_len: usize) ![]u8 { + var nar: std.ArrayList(u8) = .empty; + try nar.appendSlice(a, "\x0d\x00\x00\x00\x00\x00\x00\x00nix-archive-1\x00\x00\x00"); + var prng: std.Random.DefaultPrng = .init(42); + for (0..body_len) |i| try nar.append(a, if (i % 7 == 0) prng.random().int(u8) else 'x'); + return nar.toOwnedSlice(a); +} + +fn roundTrip(level: c_int, threads: c_int, body_len: usize) !void { + try roundTripWith(.{ .level = level, .threads = threads }, body_len, false); +} + +fn roundTripWith(options: Options, body_len: usize, pledge: bool) !void { + const level = options.level; + const a = std.testing.allocator; + const nar = try fakeNar(a, body_len); + defer a.free(nar); + + var opts = options; + if (pledge) opts.nar_size = nar.len; + var out: Collect = .{}; + defer out.list.deinit(a); + var sink = try Sink.init(a, opts, Collect.write, &out); + defer sink.deinit(a); + // Uneven writes exercise the buffering paths. + var off: usize = 0; + var step: usize = 1; + while (off < nar.len) : (step = step * 3 + 1) { + const n = @min(step, nar.len - off); + try sink.write(nar[off..][0..n]); + off += n; + } + const produced = try sink.finish(); + + var expected_nar: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(nar, &expected_nar, .{}); + try std.testing.expectEqualSlices(u8, &expected_nar, &produced.nar_hash); + try std.testing.expectEqual(@as(u64, nar.len), produced.nar_size); + try std.testing.expectEqual(@as(u64, out.list.items.len), produced.file_size); + + var v = try Verifier.init(a, if (level == 0) .none else .zstd); + defer v.deinit(a); + var i: usize = 0; + while (i < out.list.items.len) : (i += 4093) { + try v.update(out.list.items[i..@min(i + 4093, out.list.items.len)]); + } + const verified = try v.finish(); + try std.testing.expectEqualSlices(u8, &produced.nar_hash, &verified.nar_hash); + try std.testing.expectEqualSlices(u8, &produced.file_hash, &verified.file_hash); + try std.testing.expectEqual(produced.nar_size, verified.nar_size); + try std.testing.expectEqual(produced.file_size, verified.file_size); +} + +test "compress then verify round-trips" { + try roundTrip(3, 0, 1000); + try roundTrip(3, 0, 3 << 20); + try roundTrip(1, 4, 9 << 20); + try roundTrip(0, 0, 5000); +} + +test "long-distance matching with an exact pledged size round-trips" { + try roundTripWith(.{ .level = 3, .window_log = 27 }, 3 << 20, true); + try roundTripWith(.{ .level = 9, .threads = 4, .window_log = 27 }, 9 << 20, true); + try roundTripWith(.{ .level = 3 }, 1000, true); +} + +test "invalid options and mismatched pledges are refused" { + const a = std.testing.allocator; + var none: Collect = .{}; + try std.testing.expectError(error.Zstd, Sink.init(a, .{ .window_log = 28 }, Collect.write, &none)); + + var out: Collect = .{}; + defer out.list.deinit(a); + var sink = try Sink.init(a, .{ .nar_size = 10 }, Collect.write, &out); + defer sink.deinit(a); + try sink.write("only five"); + try std.testing.expectError(error.Zstd, sink.finish()); +} + +test "verifier rejects truncated and non-NAR input" { + const a = std.testing.allocator; + const nar = try fakeNar(a, 100_000); + defer a.free(nar); + var out: Collect = .{}; + defer out.list.deinit(a); + var sink = try Sink.init(a, .{}, Collect.write, &out); + defer sink.deinit(a); + try sink.write(nar); + _ = try sink.finish(); + + var v = try Verifier.init(a, .zstd); + defer v.deinit(a); + try v.update(out.list.items[0 .. out.list.items.len / 2]); + try std.testing.expectError(error.Truncated, v.finish()); + + var raw = try Verifier.init(a, .none); + defer raw.deinit(a); + try raw.update("definitely not a nar archive...."); + try std.testing.expectError(error.NotNar, raw.finish()); +} + +test "hash = false writes the same stream and sizes without hashing" { + const a = std.testing.allocator; + const nar = try fakeNar(a, 100_000); + defer a.free(nar); + var outs: [2]Collect = .{ .{}, .{} }; + defer for (&outs) |*o| o.list.deinit(a); + var digests: [2]@import("sink.zig").Digest = undefined; + for ([_]bool{ true, false }, &outs, &digests) |hash, *out, *digest| { + var sink = try Sink.init(a, .{ .level = 3, .hash = hash }, Collect.write, out); + defer sink.deinit(a); + try sink.write(nar); + digest.* = try sink.finish(); + } + try std.testing.expectEqualSlices(u8, outs[0].list.items, outs[1].list.items); + try std.testing.expectEqual(digests[0].nar_size, digests[1].nar_size); + try std.testing.expectEqual(digests[0].file_size, digests[1].file_size); + try std.testing.expectEqualSlices(u8, &@as([32]u8, @splat(0)), &digests[1].nar_hash); + try std.testing.expectEqualSlices(u8, &@as([32]u8, @splat(0)), &digests[1].file_hash); +} diff --git a/pkg/nix-archive/src/verify.zig b/pkg/nix-archive/src/verify.zig new file mode 100644 index 0000000..0976ae9 --- /dev/null +++ b/pkg/nix-archive/src/verify.zig @@ -0,0 +1,79 @@ +//! Server-side upload verification: feed the compressed NAR as it arrives +//! and get back the hash/size of both the compressed file and the NAR it +//! decompresses to. The server signs narinfo from these values rather than +//! trusting what the client claims. + +const std = @import("std"); +const zstd = @import("zstd.zig"); +const Digest = @import("sink.zig").Digest; +const Sha256 = @import("sha256.zig").Sha256; + +pub const Compression = enum(c_int) { none = 0, zstd = 1 }; + +pub const Error = error{ Zstd, Truncated, NotNar, OutOfMemory }; + +const nar_magic = "\x0d\x00\x00\x00\x00\x00\x00\x00nix-archive-1\x00\x00\x00"; + +pub const Verifier = struct { + file_hasher: Sha256 = .init(.{}), + file_size: u64 = 0, + nar_hasher: Sha256 = .init(.{}), + nar_size: u64 = 0, + dctx: ?*zstd.DCtx = null, + out_buf: []u8 = &.{}, + /// Last ZSTD_decompressStream result; 0 means a frame just ended. + frame_state: usize = 0, + head: [nar_magic.len]u8 = undefined, + + pub fn init(allocator: std.mem.Allocator, compression: Compression) Error!Verifier { + var v: Verifier = .{}; + if (compression == .zstd) { + v.dctx = zstd.ZSTD_createDCtx() orelse return error.OutOfMemory; + errdefer _ = zstd.ZSTD_freeDCtx(v.dctx); + v.out_buf = try allocator.alloc(u8, zstd.ZSTD_DStreamOutSize() * 4); + } + return v; + } + + pub fn deinit(self: *Verifier, allocator: std.mem.Allocator) void { + if (self.dctx) |dctx| _ = zstd.ZSTD_freeDCtx(dctx); + allocator.free(self.out_buf); + self.* = undefined; + } + + fn nar(self: *Verifier, data: []const u8) void { + if (self.nar_size < self.head.len) { + const n = @min(data.len, self.head.len - self.nar_size); + @memcpy(self.head[self.nar_size..][0..n], data[0..n]); + } + self.nar_hasher.update(data); + self.nar_size += data.len; + } + + pub fn update(self: *Verifier, data: []const u8) Error!void { + self.file_hasher.update(data); + self.file_size += data.len; + const dctx = self.dctx orelse return self.nar(data); + var input: zstd.InBuffer = .{ .src = data.ptr, .size = data.len, .pos = 0 }; + while (true) { + var output: zstd.OutBuffer = .{ .dst = self.out_buf.ptr, .size = self.out_buf.len, .pos = 0 }; + const rc = zstd.ZSTD_decompressStream(dctx, &output, &input); + if (zstd.isError(rc)) return error.Zstd; + self.frame_state = rc; + if (output.pos > 0) self.nar(self.out_buf[0..output.pos]); + // Done once the input is consumed and zstd has no more buffered output. + if (input.pos == input.size and output.pos < output.size) break; + } + } + + pub fn finish(self: *Verifier) Error!Digest { + if (self.dctx != null and self.frame_state != 0) return error.Truncated; + if (self.nar_size < self.head.len or !std.mem.eql(u8, &self.head, nar_magic)) return error.NotNar; + var digest: Digest = undefined; + self.file_hasher.final(&digest.file_hash); + self.nar_hasher.final(&digest.nar_hash); + digest.file_size = self.file_size; + digest.nar_size = self.nar_size; + return digest; + } +}; diff --git a/pkg/nix-archive/src/walk.zig b/pkg/nix-archive/src/walk.zig new file mode 100644 index 0000000..d5807ee --- /dev/null +++ b/pkg/nix-archive/src/walk.zig @@ -0,0 +1,210 @@ +//! NAR serialisation straight from the filesystem, without spawning +//! `nix store dump-path`. Walks with openat/getdents64/statx relative to +//! directory fds, and reads file contents directly into the sink's input +//! buffer so file data is copied exactly once out of the kernel. + +const std = @import("std"); +const builtin = @import("builtin"); +const Sink = @import("sink.zig").Sink; + +pub const supported = builtin.target.os.tag == .linux; + +pub const Error = error{ + Io, + UnsupportedFileType, + FileChanged, + NameTooLong, +} || @import("sink.zig").Error; + +/// errno of the last failed syscall on this thread, for diagnostics. +pub threadlocal var last_errno: i32 = 0; + +const linux = std.os.linux; + +fn check(rc: usize) Error!usize { + const e = linux.errno(rc); + if (e == .SUCCESS) return rc; + last_errno = @backingInt(e); + return error.Io; +} + +fn closeFd(fd: i32) void { + _ = linux.close(fd); +} + +/// Files at least this large are mmapped; sink.write feeds them unbuffered. +const mmap_threshold = @import("sink.zig").direct_threshold; +/// Bytes mapped at once; a page multiple, so every window offset is aligned. +const mmap_window = 16 << 20; + +const Kind = enum { regular, executable, symlink, directory }; + +const Node = struct { + kind: Kind, + size: u64, +}; + +fn statAt(dirfd: i32, name: [*:0]const u8) Error!Node { + var stx: linux.Statx = undefined; + _ = try check(linux.statx(dirfd, name, linux.AT.SYMLINK_NOFOLLOW, .{ .TYPE = true, .MODE = true, .SIZE = true }, &stx)); + const mode: u32 = stx.mode; + return switch (mode & linux.S.IFMT) { + linux.S.IFREG => .{ .kind = if (mode & 0o100 != 0) .executable else .regular, .size = stx.size }, + linux.S.IFLNK => .{ .kind = .symlink, .size = stx.size }, + linux.S.IFDIR => .{ .kind = .directory, .size = 0 }, + else => error.UnsupportedFileType, + }; +} + +const Writer = struct { + sink: *Sink, + arena: std.mem.Allocator, + /// getdents64 buffer, shared: a directory is fully listed before recursing. + dents: []u8, + + fn str(self: *Writer, s: []const u8) Error!void { + try self.len(s.len); + try self.sink.write(s); + try self.pad(s.len); + } + + fn len(self: *Writer, n: u64) Error!void { + var buf: [8]u8 = undefined; + std.mem.writeInt(u64, &buf, n, .little); + try self.sink.write(&buf); + } + + fn pad(self: *Writer, n: u64) Error!void { + const zeros: [8]u8 = @splat(0); + const rem: usize = @intCast(n % 8); + if (rem != 0) try self.sink.write(zeros[0 .. 8 - rem]); + } + + fn node(self: *Writer, dirfd: i32, name: [*:0]const u8, n: Node) Error!void { + try self.str("("); + try self.str("type"); + switch (n.kind) { + .regular, .executable => { + try self.str("regular"); + if (n.kind == .executable) { + try self.str("executable"); + try self.str(""); + } + try self.str("contents"); + try self.contents(dirfd, name, n.size); + }, + .symlink => { + try self.str("symlink"); + try self.str("target"); + var buf: [4096]u8 = undefined; + const got = try check(linux.readlinkat(dirfd, name, &buf, buf.len)); + if (got >= buf.len) return error.NameTooLong; + try self.str(buf[0..got]); + }, + .directory => { + try self.str("directory"); + try self.directory(dirfd, name); + }, + } + try self.str(")"); + } + + fn contents(self: *Writer, dirfd: i32, name: [*:0]const u8, size: u64) Error!void { + const fd: i32 = @intCast(try check(linux.openat(dirfd, name, .{ .ACCMODE = .RDONLY, .NOFOLLOW = true, .CLOEXEC = true }, 0))); + defer closeFd(fd); + try self.len(size); + // Large files are hashed straight out of the page cache: no + // kernel-to-user copy, and the sink feeds the mapping without + // buffering it. Store paths are immutable, so the mapping is stable. + // A window at a time, so peak RSS does not grow with the file. + var done: u64 = 0; + if (size >= mmap_threshold) { + while (done < size) { + const window: usize = @intCast(@min(size - done, mmap_window)); + const rc = linux.mmap(null, window, .{ .READ = true }, .{ .TYPE = .PRIVATE, .POPULATE = true }, fd, @intCast(done)); + if (linux.errno(rc) != .SUCCESS) { + // Read the rest instead, from where the mappings stopped. + _ = try check(linux.lseek(fd, @intCast(done), linux.SEEK.SET)); + break; + } + const ptr: [*]u8 = @ptrFromInt(rc); + defer _ = linux.munmap(ptr, window); + try self.sink.write(ptr[0..window]); + done += window; + } + } + var remaining = size - done; + while (remaining > 0) { + var dst = self.sink.spare(); + if (dst.len == 0) { + try self.sink.flush(); + dst = self.sink.spare(); + } + const want: usize = @intCast(@min(remaining, dst.len)); + const rc = linux.read(fd, dst.ptr, want); + const e = linux.errno(rc); + if (e == .INTR) continue; + const got = try check(rc); + if (got == 0) return error.FileChanged; + remaining -= got; + try self.sink.commit(got); + } + try self.pad(size); + } + + fn directory(self: *Writer, parent: i32, name: [*:0]const u8) Error!void { + const fd: i32 = @intCast(try check(linux.openat(parent, name, .{ .ACCMODE = .RDONLY, .DIRECTORY = true, .NOFOLLOW = true, .CLOEXEC = true }, 0))); + defer closeFd(fd); + + const Entry = struct { name: [:0]const u8, dtype: u8 }; + var entries: std.ArrayList(Entry) = .empty; + const buf = self.dents; + while (true) { + const n = try check(linux.getdents64(fd, buf.ptr, @intCast(buf.len))); + if (n == 0) break; + var off: usize = 0; + while (off < n) { + const reclen = std.mem.readInt(u16, buf[off + 16 ..][0..2], builtin.target.cpu.arch.endian()); + const dtype = buf[off + 18]; + const entry_name = std.mem.sliceTo(@as([*:0]const u8, @ptrCast(&buf[off + 19])), 0); + off += reclen; + if (std.mem.eql(u8, entry_name, ".") or std.mem.eql(u8, entry_name, "..")) continue; + try entries.append(self.arena, .{ .name = try self.arena.dupeSentinel(u8, entry_name, 0), .dtype = dtype }); + } + } + + // NAR entries are ordered bytewise, like Nix's std::map. + std.mem.sort(Entry, entries.items, {}, struct { + fn lessThan(_: void, a: Entry, b: Entry) bool { + return std.mem.lessThan(u8, a.name, b.name); + } + }.lessThan); + + for (entries.items) |entry| { + const child: Node = switch (entry.dtype) { + linux.DT.DIR => .{ .kind = .directory, .size = 0 }, + linux.DT.LNK => .{ .kind = .symlink, .size = 0 }, + else => try statAt(fd, entry.name.ptr), + }; + try self.str("entry"); + try self.str("("); + try self.str("name"); + try self.str(entry.name); + try self.str("node"); + try self.node(fd, entry.name.ptr, child); + try self.str(")"); + } + } +}; + +/// Serialise `path` as a NAR into `sink`. +pub fn dump(sink: *Sink, path: [*:0]const u8) Error!void { + if (!supported) @compileError("NAR dumping is only implemented for Linux"); + var arena_state: std.heap.ArenaAllocator = .init(std.heap.c_allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var w: Writer = .{ .sink = sink, .arena = arena, .dents = try arena.alloc(u8, 64 * 1024) }; + const root = try statAt(linux.AT.FDCWD, path); + try w.str("nix-archive-1"); + try w.node(linux.AT.FDCWD, path, root); +} diff --git a/pkg/nix-archive/src/zstd.zig b/pkg/nix-archive/src/zstd.zig new file mode 100644 index 0000000..f9de889 --- /dev/null +++ b/pkg/nix-archive/src/zstd.zig @@ -0,0 +1,43 @@ +//! Minimal hand-written bindings for the libzstd streaming API. + +pub const CCtx = opaque {}; +pub const DCtx = opaque {}; + +pub const InBuffer = extern struct { + src: ?*const anyopaque, + size: usize, + pos: usize, +}; + +pub const OutBuffer = extern struct { + dst: ?*anyopaque, + size: usize, + pos: usize, +}; + +pub const c_compressionLevel: c_int = 100; +pub const c_checksumFlag: c_int = 201; +pub const c_nbWorkers: c_int = 400; +pub const c_windowLog: c_int = 101; +pub const c_enableLongDistanceMatching: c_int = 160; + +pub const e_continue: c_int = 0; +pub const e_end: c_int = 2; + +pub extern "c" fn ZSTD_createCCtx() ?*CCtx; +pub extern "c" fn ZSTD_freeCCtx(cctx: ?*CCtx) usize; +pub extern "c" fn ZSTD_CCtx_setParameter(cctx: *CCtx, param: c_int, value: c_int) usize; +pub extern "c" fn ZSTD_compressStream2(cctx: *CCtx, output: *OutBuffer, input: *InBuffer, end_op: c_int) usize; +pub extern "c" fn ZSTD_CCtx_setPledgedSrcSize(cctx: *CCtx, pledged_src_size: c_ulonglong) usize; +pub extern "c" fn ZSTD_CStreamOutSize() usize; + +pub extern "c" fn ZSTD_createDCtx() ?*DCtx; +pub extern "c" fn ZSTD_freeDCtx(dctx: ?*DCtx) usize; +pub extern "c" fn ZSTD_decompressStream(dctx: *DCtx, output: *OutBuffer, input: *InBuffer) usize; +pub extern "c" fn ZSTD_DStreamOutSize() usize; + +pub extern "c" fn ZSTD_isError(code: usize) c_uint; + +pub fn isError(code: usize) bool { + return ZSTD_isError(code) != 0; +} diff --git a/pkg/nix-base32/build.zig b/pkg/nix-base32/build.zig new file mode 100644 index 0000000..dee0106 --- /dev/null +++ b/pkg/nix-base32/build.zig @@ -0,0 +1,18 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mod = b.addModule("nix-base32", .{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, + }); + + const tests = b.addTest(.{ .root_module = mod }); + b.step("test", "Run unit tests").dependOn(&b.addRunArtifact(tests).step); +} diff --git a/pkg/nix-base32/build.zig.zon b/pkg/nix-base32/build.zig.zon new file mode 100644 index 0000000..21afb7c --- /dev/null +++ b/pkg/nix-base32/build.zig.zon @@ -0,0 +1,9 @@ +.{ + .name = .nix_base32, + .version = "0.1.0", + .fingerprint = 0x9c08ba94cedadc5b, + .minimum_zig_version = "0.17.0", + .dependencies = .{ + }, + .paths = .{ "build.zig", "build.zig.zon", "src" }, +} diff --git a/pkg/nix-base32/src/root.zig b/pkg/nix-base32/src/root.zig new file mode 100644 index 0000000..de246e9 --- /dev/null +++ b/pkg/nix-base32/src/root.zig @@ -0,0 +1,261 @@ +//! Nix's base-32 encoding: a custom alphabet (no e, o, t, u) with the +//! digits emitted from the most significant 5-bit group down. +//! +//! Character classification is vectorised (Zig @Vector): a whole store path +//! hash (32 chars) is validated and mapped to 5-bit digits in one pass. +//! The scalar table path handles tails and is kept as the reference. + +const std = @import("std"); + +pub const alphabet = "0123456789abcdfghijklmnpqrsvwxyz"; + +const invalid: u8 = 0xff; + +const reverse: [256]u8 = blk: { + var table: [256]u8 = @splat(invalid); + for (alphabet, 0..) |c, i| table[c] = @intCast(i); + break :blk table; +}; + +/// Lanes per vector step. 32 covers a store path hash exactly and lowers to +/// one AVX2 register (two SSE registers, one half of an AVX-512 one). +pub const lanes = 32; +const V = @Vector(lanes, u8); +const Mask = @Int(.unsigned, lanes); + +fn splat(c: u8) V { + return @splat(c); +} + +fn bits(b: @Vector(lanes, bool)) Mask { + return @bitCast(b); +} + +/// Maps 32 characters to their 5-bit digit values. Returns a bitmask with a +/// set bit for every lane that is not in the alphabet. +/// +/// Digits map to c - '0'. Letters map to c - 'a' + 10 minus one for every +/// skipped letter (e, o, t, u) below c. +inline fn classify(v: V, out: *V) Mask { + const is_digit = bits(v >= splat('0')) & bits(v <= splat('9')); + const is_lower = bits(v >= splat('a')) & bits(v <= splat('z')); + const skipped = bits(v == splat('e')) | bits(v == splat('o')) | bits(v == splat('t')) | bits(v == splat('u')); + const valid = is_digit | (is_lower & ~skipped); + + const one = splat(1); + const zero = splat(0); + const gaps = @select(u8, v > splat('e'), one, zero) + + @select(u8, v > splat('o'), one, zero) + + @select(u8, v > splat('t'), one, zero) + + @select(u8, v > splat('u'), one, zero); + const letter = v -% splat('a' - 10) -% gaps; + const digit = v -% splat('0'); + out.* = @select(u8, @as(@Vector(lanes, bool), @bitCast(is_digit)), digit, letter); + return ~valid; +} + +pub fn encodedLen(byte_len: usize) usize { + if (byte_len == 0) return 0; + return (byte_len * 8 - 1) / 5 + 1; +} + +/// Writes exactly `encodedLen(bytes.len)` characters into `out`. +/// +/// 5 bytes hold exactly 8 digits, so full groups are unpacked from one u64 +/// without cross-byte carries; only the last partial group goes bit by bit. +pub fn encode(out: []u8, bytes: []const u8) void { + const len = encodedLen(bytes.len); + std.debug.assert(out.len >= len); + var k: usize = 0; + while (5 * k + 5 <= bytes.len) : (k += 1) { + const v = std.mem.readInt(u40, bytes[5 * k ..][0..5], .little); + inline for (0..8) |i| out[len - 1 - (8 * k + i)] = alphabet[@as(u5, @truncate(v >> (5 * i)))]; + } + for (8 * k..len) |n| out[len - 1 - n] = alphabet[digitAt(bytes, n)]; +} + +fn digitAt(bytes: []const u8, n: usize) u8 { + const b = n * 5; + const i = b / 8; + const j: u3 = @intCast(b % 8); + var c: u16 = bytes[i] >> j; + if (i + 1 < bytes.len) c |= @as(u16, bytes[i + 1]) << (@as(u4, 8) - j); + return @intCast(c & 0x1f); +} + +pub fn encodeScalar(out: []u8, bytes: []const u8) void { + const len = encodedLen(bytes.len); + for (0..len) |n| out[len - 1 - n] = alphabet[digitAt(bytes, n)]; +} + +pub const DecodeError = error{InvalidNix32}; + +/// Converts characters to digit values in place order, vector-at-a-time. +fn digits(dst: []u8, in: []const u8) DecodeError!void { + var i: usize = 0; + while (i + lanes <= in.len) : (i += lanes) { + var d: V = undefined; + if (classify(in[i..][0..lanes].*, &d) != 0) return error.InvalidNix32; + dst[i..][0..lanes].* = d; + } + for (in[i..], dst[i..in.len]) |c, *d| { + d.* = reverse[c]; + if (d.* == invalid) return error.InvalidNix32; + } +} + +/// Packs digits (most significant first) into little-endian bytes: full +/// 8-digit groups become 5 bytes via one u64, the tail goes bit by bit. +fn pack(out: []u8, d: []const u8) DecodeError!void { + const len = d.len; + var k: usize = 0; + while (8 * k + 8 <= len and 5 * k + 5 <= out.len) : (k += 1) { + // One load puts digit 8k+i in byte i (big-endian, as the digits run + // most significant first). Three mask-and-shift steps then close + // the 3-bit gaps between them: pairs into 10 bits, pairs of those + // into 20, then the two halves into 40. Unlike shifting each digit + // in, this does not depend on how the backend extracts bytes. + const word = std.mem.readInt(u64, d[len - 8 - 8 * k ..][0..8], .big); + const pairs = (word & 0x001F001F001F001F) | ((word & 0x1F001F001F001F00) >> 3); + const quads = (pairs & 0x000003FF000003FF) | ((pairs & 0x03FF000003FF0000) >> 6); + const v: u40 = @truncate((quads & 0xFFFFF) | ((quads >> 32) << 20)); + std.mem.writeInt(u40, out[5 * k ..][0..5], v, .little); + } + @memset(out[5 * k ..], 0); + return packBits(out, d, 8 * k); +} + +fn packBits(out: []u8, d: []const u8, from: usize) DecodeError!void { + for (from..d.len) |n| { + const digit = d[d.len - n - 1]; + const b = n * 5; + const i = b / 8; + const j: u3 = @intCast(b % 8); + out[i] |= digit << j; + const carry: u8 = @intCast(@as(u16, digit) >> (@as(u4, 8) - j)); + if (i + 1 < out.len) { + out[i + 1] |= carry; + } else if (carry != 0) { + return error.InvalidNix32; + } + } +} + +/// Decodes `in` into exactly `out.len` bytes. `in.len` must equal +/// `encodedLen(out.len)`, and no bits may overflow the output. +pub fn decode(out: []u8, in: []const u8) DecodeError!void { + if (in.len != encodedLen(out.len)) return error.InvalidNix32; + var buf: [128]u8 = undefined; + if (in.len > buf.len) return decodeScalar(out, in); + try digits(buf[0..in.len], in); + try pack(out, buf[0..in.len]); +} + +/// Table-driven reference implementation. +pub fn decodeScalar(out: []u8, in: []const u8) DecodeError!void { + if (in.len != encodedLen(out.len)) return error.InvalidNix32; + var buf: [256]u8 = undefined; + if (in.len > buf.len) return error.InvalidNix32; + for (in, buf[0..in.len]) |c, *d| { + d.* = reverse[c]; + if (d.* == invalid) return error.InvalidNix32; + } + @memset(out, 0); + try packBits(out, buf[0..in.len], 0); +} + +pub fn isValid(in: []const u8) bool { + var i: usize = 0; + var scratch: V = undefined; + while (i + lanes <= in.len) : (i += lanes) { + if (classify(in[i..][0..lanes].*, &scratch) != 0) return false; + } + for (in[i..]) |c| if (reverse[c] == invalid) return false; + return true; +} + +pub fn isValidScalar(in: []const u8) bool { + for (in) |c| if (reverse[c] == invalid) return false; + return true; +} + +test "sha256 of empty string round-trips" { + try std.testing.expectEqual(@as(usize, 32), encodedLen(20)); // a store path hash + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash("", &digest, .{}); + var text: [52]u8 = undefined; + encode(&text, &digest); + try std.testing.expectEqualStrings("0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73", &text); + var back: [32]u8 = undefined; + try decode(&back, &text); + try std.testing.expectEqualSlices(u8, &digest, &back); +} + +test "rejects invalid characters and overflow" { + var out: [20]u8 = undefined; + try std.testing.expectError(error.InvalidNix32, decode(&out, &@as([32]u8, @splat('e')))); + // 52 chars carry 260 bits; the top 4 must be zero for a 32-byte digest. + var digest: [32]u8 = undefined; + try std.testing.expectError(error.InvalidNix32, decode(&digest, &@as([52]u8, @splat('z')))); +} + +test "vector classifier agrees with the table for every byte in every lane" { + for (0..256) |c| { + for (0..lanes) |lane| { + var in: [lanes]u8 = @splat('0'); + in[lane] = @intCast(c); + try std.testing.expectEqual(isValidScalar(&in), isValid(&in)); + var d: V = undefined; + const bad = classify(in, &d); + if (bad == 0) { + const arr: [lanes]u8 = d; + try std.testing.expectEqual(reverse[c], arr[lane]); + } + } + } +} + +test "vector and scalar decode agree on random inputs" { + var prng: std.Random.DefaultPrng = .init(7); + const r = prng.random(); + for (0..10_000) |_| { + var bytes: [20]u8 = undefined; + r.bytes(&bytes); + var text: [32]u8 = undefined; + encode(&text, &bytes); + if (r.boolean()) text[r.uintLessThan(usize, 32)] = r.int(u8); + var a: [20]u8 = undefined; + var b: [20]u8 = undefined; + const ra = decode(&a, &text); + const rb = decodeScalar(&b, &text); + if (rb) |_| { + try ra; + try std.testing.expectEqualSlices(u8, &b, &a); + } else |_| { + try std.testing.expectError(error.InvalidNix32, ra); + } + } +} + +test "grouped encode/decode agree with the bitwise reference at every length" { + var prng: std.Random.DefaultPrng = .init(11); + const r = prng.random(); + for (1..64) |n| { + for (0..200) |_| { + var bytes: [64]u8 = undefined; + r.bytes(bytes[0..n]); + var fast: [128]u8 = undefined; + var slow: [128]u8 = undefined; + const len = encodedLen(n); + encode(fast[0..len], bytes[0..n]); + encodeScalar(slow[0..len], bytes[0..n]); + try std.testing.expectEqualStrings(slow[0..len], fast[0..len]); + var back: [64]u8 = undefined; + try decode(back[0..n], fast[0..len]); + try std.testing.expectEqualSlices(u8, bytes[0..n], back[0..n]); + var back_slow: [64]u8 = undefined; + try decodeScalar(back_slow[0..n], fast[0..len]); + try std.testing.expectEqualSlices(u8, bytes[0..n], back_slow[0..n]); + } + } +} diff --git a/pkg/nix-derivation/build.zig b/pkg/nix-derivation/build.zig new file mode 100644 index 0000000..95a2656 --- /dev/null +++ b/pkg/nix-derivation/build.zig @@ -0,0 +1,18 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mod = b.addModule("nix-derivation", .{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, + }); + + const tests = b.addTest(.{ .root_module = mod }); + b.step("test", "Run unit tests").dependOn(&b.addRunArtifact(tests).step); +} diff --git a/pkg/nix-derivation/build.zig.zon b/pkg/nix-derivation/build.zig.zon new file mode 100644 index 0000000..a25f4d5 --- /dev/null +++ b/pkg/nix-derivation/build.zig.zon @@ -0,0 +1,9 @@ +.{ + .name = .nix_derivation, + .version = "0.1.0", + .fingerprint = 0x7a251e78739a22ca, + .minimum_zig_version = "0.17.0", + .dependencies = .{ + }, + .paths = .{ "build.zig", "build.zig.zon", "src" }, +} diff --git a/pkg/nix-derivation/src/root.zig b/pkg/nix-derivation/src/root.zig new file mode 100644 index 0000000..457b903 --- /dev/null +++ b/pkg/nix-derivation/src/root.zig @@ -0,0 +1,208 @@ +//! Parser for Nix store derivations (`.drv`, ATerm syntax): +//! +//! Derive([outputs],[input drvs],[input srcs],"system","builder",[args],[env]) +//! +//! Strings without escapes are returned as slices of the input (zero-copy); +//! escaped strings are unescaped into the arena. The string scanner looks +//! for '"' or '\\' 32 bytes at a time. + +const std = @import("std"); + +pub const Error = error{ Syntax, OutOfMemory }; + +pub const Output = struct { + name: []const u8, + path: []const u8, + hash_algo: []const u8, + hash: []const u8, +}; + +pub const InputDrv = struct { + path: []const u8, + outputs: []const []const u8, +}; + +pub const EnvVar = struct { + name: []const u8, + value: []const u8, +}; + +pub const Derivation = struct { + outputs: []const Output, + input_drvs: []const InputDrv, + input_srcs: []const []const u8, + platform: []const u8, + builder: []const u8, + args: []const []const u8, + env: []const EnvVar, +}; + +const lanes = 32; +const V = @Vector(lanes, u8); +const Mask = @Int(.unsigned, lanes); + +/// Index of the first '"' or '\\' at or after `start`. +pub fn findQuoteOrEscape(s: []const u8, start: usize) ?usize { + var i = start; + const quote: V = @splat('"'); + const backslash: V = @splat('\\'); + while (i + lanes <= s.len) : (i += lanes) { + const v: V = s[i..][0..lanes].*; + const m: Mask = @as(Mask, @bitCast(v == quote)) | @as(Mask, @bitCast(v == backslash)); + if (m != 0) return i + @ctz(m); + } + while (i < s.len) : (i += 1) if (s[i] == '"' or s[i] == '\\') return i; + return null; +} + +const Parser = struct { + s: []const u8, + pos: usize = 0, + arena: std.mem.Allocator, + + fn expect(p: *Parser, comptime lit: []const u8) Error!void { + if (!std.mem.startsWith(u8, p.s[p.pos..], lit)) return error.Syntax; + p.pos += lit.len; + } + + fn peek(p: *Parser) Error!u8 { + if (p.pos >= p.s.len) return error.Syntax; + return p.s[p.pos]; + } + + fn string(p: *Parser) Error![]const u8 { + try p.expect("\""); + const start = p.pos; + var i = findQuoteOrEscape(p.s, start) orelse return error.Syntax; + if (p.s[i] == '"') { + p.pos = i + 1; + return p.s[start..i]; + } + // Escaped: copy the clean prefix, then unescape the rest. + var out: std.ArrayList(u8) = .empty; + try out.appendSlice(p.arena, p.s[start..i]); + while (true) { + if (p.s[i] == '"') { + p.pos = i + 1; + return out.toOwnedSlice(p.arena); + } + // Backslash escape. + if (i + 1 >= p.s.len) return error.Syntax; + try out.append(p.arena, switch (p.s[i + 1]) { + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + else => |c| c, + }); + const next = findQuoteOrEscape(p.s, i + 2) orelse return error.Syntax; + try out.appendSlice(p.arena, p.s[i + 2 .. next]); + i = next; + } + } + + fn list(p: *Parser, comptime T: type, comptime item: fn (*Parser) Error!T) Error![]const T { + try p.expect("["); + var items: std.ArrayList(T) = .empty; + if (try p.peek() == ']') { + p.pos += 1; + return &.{}; + } + while (true) { + try items.append(p.arena, try item(p)); + switch (try p.peek()) { + ',' => p.pos += 1, + ']' => { + p.pos += 1; + return items.toOwnedSlice(p.arena); + }, + else => return error.Syntax, + } + } + } + + fn output(p: *Parser) Error!Output { + try p.expect("("); + const name = try p.string(); + try p.expect(","); + const path = try p.string(); + try p.expect(","); + const algo = try p.string(); + try p.expect(","); + const hash = try p.string(); + try p.expect(")"); + return .{ .name = name, .path = path, .hash_algo = algo, .hash = hash }; + } + + fn inputDrv(p: *Parser) Error!InputDrv { + try p.expect("("); + const path = try p.string(); + try p.expect(","); + const outs = try p.list([]const u8, string); + try p.expect(")"); + return .{ .path = path, .outputs = outs }; + } + + fn envVar(p: *Parser) Error!EnvVar { + try p.expect("("); + const name = try p.string(); + try p.expect(","); + const value = try p.string(); + try p.expect(")"); + return .{ .name = name, .value = value }; + } +}; + +/// Parses a derivation. All allocations go to `arena`; free it as a whole. +pub fn parse(arena: std.mem.Allocator, text: []const u8) Error!Derivation { + var p: Parser = .{ .s = text, .arena = arena }; + try p.expect("Derive("); + const outputs = try p.list(Output, Parser.output); + try p.expect(","); + const input_drvs = try p.list(InputDrv, Parser.inputDrv); + try p.expect(","); + const input_srcs = try p.list([]const u8, Parser.string); + try p.expect(","); + const platform = try p.string(); + try p.expect(","); + const builder = try p.string(); + try p.expect(","); + const args = try p.list([]const u8, Parser.string); + try p.expect(","); + const env = try p.list(EnvVar, Parser.envVar); + try p.expect(")"); + if (p.pos != text.len) return error.Syntax; + return .{ + .outputs = outputs, + .input_drvs = input_drvs, + .input_srcs = input_srcs, + .platform = platform, + .builder = builder, + .args = args, + .env = env, + }; +} + +test "parses a derivation with escapes" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const text = + \\Derive([("out","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello","","")],[("/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-bash.drv",["out"])],["/nix/store/cccccccccccccccccccccccccccccccc-builder.sh"],"x86_64-linux","/bin/sh",["-e","/builder.sh"],[("buildPhase","echo \"hi\"\nmake -j\\$N"),("name","hello")]) + ; + const d = try parse(arena.allocator(), text); + try std.testing.expectEqual(@as(usize, 1), d.outputs.len); + try std.testing.expectEqualStrings("out", d.input_drvs[0].outputs[0]); + try std.testing.expectEqualStrings("echo \"hi\"\nmake -j\\$N", d.env[0].value); + try std.testing.expectEqualStrings("x86_64-linux", d.platform); + try std.testing.expectError(error.Syntax, parse(arena.allocator(), "Derive([],")); +} + +test "vector quote search matches scalar" { + var buf: [100]u8 = @splat('x'); + for (0..buf.len) |i| { + for ([_]u8{ '"', '\\' }) |c| { + buf[i] = c; + for (0..i + 1) |start| try std.testing.expectEqual(@as(?usize, i), findQuoteOrEscape(&buf, start)); + buf[i] = 'x'; + } + } +} diff --git a/pkg/nix-flake-lock/build.zig b/pkg/nix-flake-lock/build.zig new file mode 100644 index 0000000..4d6b4eb --- /dev/null +++ b/pkg/nix-flake-lock/build.zig @@ -0,0 +1,18 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mod = b.addModule("nix-flake-lock", .{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, + }); + + const tests = b.addTest(.{ .root_module = mod }); + b.step("test", "Run unit tests").dependOn(&b.addRunArtifact(tests).step); +} diff --git a/pkg/nix-flake-lock/build.zig.zon b/pkg/nix-flake-lock/build.zig.zon new file mode 100644 index 0000000..0693adc --- /dev/null +++ b/pkg/nix-flake-lock/build.zig.zon @@ -0,0 +1,9 @@ +.{ + .name = .nix_flake_lock, + .version = "0.1.0", + .fingerprint = 0x6174916c0100c5fd, + .minimum_zig_version = "0.17.0", + .dependencies = .{ + }, + .paths = .{ "build.zig", "build.zig.zon", "src" }, +} diff --git a/pkg/nix-flake-lock/src/bump.zig b/pkg/nix-flake-lock/src/bump.zig new file mode 100644 index 0000000..73ecf56 --- /dev/null +++ b/pkg/nix-flake-lock/src/bump.zig @@ -0,0 +1,104 @@ +//! Single-threaded bump allocator. std.heap.ArenaAllocator is lock-free and +//! thread-safe since Zig 0.16, which puts atomics on every allocation; parsing +//! allocates from one thread only, so plain pointer bumping is enough. +//! Chunks are chained through a header at their start, so a document that +//! fits in its first chunk costs exactly one malloc. + +const std = @import("std"); +const Allocator = std.mem.Allocator; + +const Header = struct { next: ?*Header, len: usize }; + +pub const Bump = struct { + gpa: Allocator, + head: ?*Header = null, + cur: usize = 0, + end: usize = 0, + + pub fn init(gpa: Allocator, first_chunk: usize) Allocator.Error!Bump { + var b: Bump = .{ .gpa = gpa }; + try b.grow(first_chunk); + return b; + } + + pub fn deinit(b: *Bump) void { + var h = b.head; + while (h) |chunk| { + h = chunk.next; + const bytes: [*]align(@alignOf(Header)) u8 = @ptrCast(chunk); + b.gpa.free(bytes[0..chunk.len]); + } + b.* = undefined; + } + + fn grow(b: *Bump, need: usize) Allocator.Error!void { + const last = if (b.head) |h| h.len else 0; + const size = @max(need + @sizeOf(Header) + 64, last * 2, 4096); + const mem = try b.gpa.alignedAlloc(u8, .of(Header), size); + const h: *Header = @ptrCast(mem.ptr); + h.* = .{ .next = b.head, .len = size }; + b.head = h; + b.cur = @intFromPtr(mem.ptr) + @sizeOf(Header); + b.end = @intFromPtr(mem.ptr) + size; + } + + inline fn raw(b: *Bump, len: usize, alignment: usize) Allocator.Error![*]u8 { + var start = std.mem.alignForward(usize, b.cur, alignment); + if (start + len > b.end) { + @branchHint(.unlikely); + try b.grow(len + alignment); + start = std.mem.alignForward(usize, b.cur, alignment); + } + b.cur = start + len; + return @ptrFromInt(start); + } + + pub inline fn alloc(b: *Bump, comptime T: type, n: usize) Allocator.Error![]T { + const p: [*]T = @ptrCast(@alignCast(try b.raw(n * @sizeOf(T), @alignOf(T)))); + return p[0..n]; + } + + pub inline fn dupe(b: *Bump, comptime T: type, items: []const T) Allocator.Error![]T { + const out = try b.alloc(T, items.len); + @memcpy(out, items); + return out; + } + + pub fn allocator(b: *Bump) Allocator { + return .{ .ptr = b, .vtable = &.{ .alloc = vAlloc, .resize = vResize, .remap = vRemap, .free = vFree } }; + } + + fn vAlloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, _: usize) ?[*]u8 { + const b: *Bump = @ptrCast(@alignCast(ctx)); + return b.raw(len, alignment.toByteUnits()) catch null; + } + + fn vResize(ctx: *anyopaque, memory: []u8, _: std.mem.Alignment, new_len: usize, _: usize) bool { + const b: *Bump = @ptrCast(@alignCast(ctx)); + // Only the most recent allocation can grow or shrink in place. + if (@intFromPtr(memory.ptr) + memory.len != b.cur) return new_len <= memory.len; + if (@intFromPtr(memory.ptr) + new_len > b.end) return false; + b.cur = @intFromPtr(memory.ptr) + new_len; + return true; + } + + fn vRemap(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ra: usize) ?[*]u8 { + return if (vResize(ctx, memory, alignment, new_len, ra)) memory.ptr else null; + } + + fn vFree(_: *anyopaque, _: []u8, _: std.mem.Alignment, _: usize) void {} +}; + +test "bump allocations are aligned and grow across chunks" { + var b = try Bump.init(std.testing.allocator, 16); + defer b.deinit(); + for (0..1000) |i| { + const s = try b.alloc(u64, i % 7 + 1); + try std.testing.expect(@intFromPtr(s.ptr) % 8 == 0); + s[0] = i; + _ = try b.alloc(u8, 3); + } + var list: std.ArrayList(u32) = .empty; + for (0..10_000) |i| try list.append(b.allocator(), @intCast(i)); + try std.testing.expectEqual(@as(u32, 9999), list.items[9999]); +} diff --git a/pkg/nix-flake-lock/src/root.zig b/pkg/nix-flake-lock/src/root.zig new file mode 100644 index 0000000..88690b3 --- /dev/null +++ b/pkg/nix-flake-lock/src/root.zig @@ -0,0 +1,1385 @@ +//! Nix `flake.lock` parsing (versions 5-7), `follows` validation and +//! resolution, and canonical version 7 serialisation, without Nix. +//! +//! Semantics follow the nix-flake-lock crate (Rust): last duplicate key wins, +//! version 5 `info` is overlaid onto `locked`, unreachable nodes are +//! dropped, and output is byte-identical to Nix's canonical writer. +//! +//! Speed comes from single-pass scanning: strings are scanned 32 bytes at a +//! time for '"', '\\', control and non-ASCII bytes at once (UTF-8 is only +//! validated where it may legally occur, inside strings), whitespace runs are +//! skipped with a vector compare, lists are collected in reusable scratch +//! stacks and copied once into an arena, and canonical names borrow from +//! the input instead of allocating. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Bump = @import("bump.zig").Bump; + +pub const Limits = struct { + max_bytes: usize = 16 * 1024 * 1024, + max_nodes: usize = 1_000_000, + max_depth: usize = 256, + max_path_components: usize = 65_536, +}; + +pub const ParseError = error{ + InputTooLarge, + InvalidUtf8, + UnexpectedEof, + Syntax, + TrailingCharacters, + ControlCharacter, + InvalidEscape, + InvalidUnicodeEscape, + InvalidNumber, + UnsupportedVersion, + MissingField, + MissingNode, + TooManyNodes, + NestingTooDeep, + PathTooLong, + UnsupportedInputAttributeType, + CycleToRoot, + OutOfMemory, +}; + +pub const ValidationError = error{ FollowsCycle, MissingFollowsTarget, OutOfMemory }; + +/// Where a parse error happened. +pub const Diagnostic = struct { offset: usize = 0 }; + +pub const Value = union(enum) { + string: []const u8, + integer: u64, + boolean: bool, +}; + +pub const Attr = struct { name: []const u8, value: Value }; + +pub const Path = []const []const u8; + +pub const Edge = union(enum) { + node: u32, + follows: Path, +}; + +pub const Input = struct { name: []const u8, edge: Edge }; + +pub const Locked = struct { + original: []const Attr, + locked: []const Attr, + is_flake: bool, + parent: ?Path, + + pub fn get(attrs: []const Attr, name: []const u8) ?Value { + const i = findSorted(Attr, attrs, name) orelse return null; + return attrs[i].value; + } +}; + +pub const Node = struct { + /// Key in the source document (not graph identity). + key: []const u8, + /// Sorted by name. + inputs: []const Input, + /// Null for the root node. + locked: ?Locked, + + pub fn input(self: *const Node, name: []const u8) ?*const Input { + const i = findSorted(Input, self.inputs, name) orelse return null; + return &self.inputs[i]; + } +}; + +pub const root_id: u32 = 0; + +pub const LockFile = struct { + source_version: u8, + /// Reachable nodes; the root is at index 0. + nodes: []const Node, + + pub fn resolve(self: *const LockFile, gpa: Allocator, path: Path) ValidationError!?u32 { + var r: Resolver = try .init(gpa, self); + defer r.deinit(); + return r.resolve(path, false); + } + + fn hasFollows(self: *const LockFile) bool { + for (self.nodes) |n| for (n.inputs) |in| switch (in.edge) { + .follows => |p| if (p.len > 0) return true, + .node => {}, + }; + return false; + } + + /// Checks that every `follows` target exists and no chain is cyclic. + pub fn validate(self: *const LockFile, gpa: Allocator) ValidationError!void { + // Only `follows` edges can fail, and every parsed node is reachable, + // so a graph without non-empty follows edges is valid as is. + if (!self.hasFollows()) return; + var first: [4096]u8 = undefined; + var fallback: std.heap.BufferFirstAllocator = .init(&first, gpa); + var bump = try Bump.init(fallback.allocator(), 2 * 1024); + defer bump.deinit(); + var r: Resolver = try .init(bump.allocator(), self); + const visited = try bump.alloc(bool, self.nodes.len); + @memset(visited, false); + visited[root_id] = true; + // Each node is pushed at most once. + const stack_buf = try bump.alloc(u32, self.nodes.len); + var stack: std.ArrayList(u32) = .initBuffer(stack_buf); + stack.appendAssumeCapacity(root_id); + while (stack.pop()) |id| { + const node = &self.nodes[id]; + var i = node.inputs.len; + while (i > 0) { + i -= 1; + switch (node.inputs[i].edge) { + .node => |child| if (!visited[child]) { + visited[child] = true; + stack.appendAssumeCapacity(child); + }, + .follows => |target| if (target.len > 0) { + if (try r.resolve(target, true) == null) return error.MissingFollowsTarget; + }, + } + } + } + } + + /// Appends canonical, two-space-indented version 7 JSON (with the + /// trailing newline Nix writes). + pub fn writeNix(self: *const LockFile, gpa: Allocator, out: *std.ArrayList(u8)) Allocator.Error!void { + var w: Writer = .{ .gpa = gpa, .out = out, .lock = self }; + return w.write(); + } +}; + +fn findSorted(comptime T: type, items: []const T, name: []const u8) ?usize { + var lo: usize = 0; + var hi: usize = items.len; + while (lo < hi) { + const mid = lo + (hi - lo) / 2; + switch (std.mem.order(u8, items[mid].name, name)) { + .eq => return mid, + .lt => lo = mid + 1, + .gt => hi = mid, + } + } + return null; +} + +// ── SIMD scanning ── + +const lanes = 32; +const V = @Vector(lanes, u8); +const Mask = @Int(.unsigned, lanes); + +inline fn splat(c: u8) V { + return @splat(c); +} + +inline fn bits(b: @Vector(lanes, bool)) Mask { + return @bitCast(b); +} + +/// First byte at or after `i` that ends or interrupts a plain string run: +/// '"', '\\', a control byte, or a non-ASCII byte. +fn scanString(s: []const u8, start: usize) ?usize { + var i = start; + while (i + lanes <= s.len) : (i += lanes) { + const v: V = s[i..][0..lanes].*; + const m = bits(v == splat('"')) | bits(v == splat('\\')) | bits(v < splat(0x20)) | bits(v >= splat(0x80)); + if (m != 0) return i + @ctz(m); + } + while (i < s.len) : (i += 1) { + const c = s[i]; + if (c == '"' or c == '\\' or c < 0x20 or c >= 0x80) return i; + } + return null; +} + +fn isSpace(c: u8) bool { + return c == ' ' or c == '\n' or c == '\r' or c == '\t'; +} + +/// First non-whitespace byte at or after `i`. +fn skipSpace(s: []const u8, start: usize) usize { + var i = start; + if (i >= s.len or !isSpace(s[i])) return i; + while (i + lanes <= s.len) : (i += lanes) { + const v: V = s[i..][0..lanes].*; + const ws = bits(v == splat(' ')) | bits(v == splat('\n')) | bits(v == splat('\r')) | bits(v == splat('\t')); + if (ws != ~@as(Mask, 0)) return i + @ctz(~ws); + } + while (i < s.len and isSpace(s[i])) i += 1; + return i; +} + +/// Bytes that must be escaped in JSON output. +fn scanEscape(s: []const u8, start: usize) usize { + var i = start; + while (i + lanes <= s.len) : (i += lanes) { + const v: V = s[i..][0..lanes].*; + const m = bits(v == splat('"')) | bits(v == splat('\\')) | bits(v < splat(0x20)); + if (m != 0) return i + @ctz(m); + } + while (i < s.len) : (i += 1) { + const c = s[i]; + if (c == '"' or c == '\\' or c < 0x20) return i; + } + return s.len; +} + +// ── Parser ── + +const WireEdge = union(enum) { node: []const u8, follows: Path }; + +const WireInput = struct { + offset: usize, + name: []const u8, + edge: WireEdge, + target: u32 = 0, +}; + +const WireNode = struct { + offset: usize, + key: []const u8, + inputs: []WireInput = &.{}, + original: ?[]Attr = null, + locked: ?[]Attr = null, + info: ?[]Attr = null, + is_flake: bool = true, + parent: ?Path = null, +}; + +fn Keyed(comptime T: type) type { + return struct { offset: usize, item: T }; +} + +/// Sort by name and keep the last occurrence of each duplicate, like Nix. +fn sortDedup(comptime T: type, items: []Keyed(T), nameOf: fn (*const T) []const u8) usize { + const Ctx = struct { + fn lessThan(_: void, a: Keyed(T), b: Keyed(T)) bool { + return switch (std.mem.order(u8, nameOf(&a.item), nameOf(&b.item))) { + .lt => true, + .gt => false, + .eq => a.offset > b.offset, + }; + } + }; + if (items.len < 2) return items.len; + // Inputs are usually already sorted: skip the sort when they are. + var sorted = true; + for (items[1..], items[0 .. items.len - 1]) |b, a| { + if (std.mem.order(u8, nameOf(&a.item), nameOf(&b.item)) != .lt) { + sorted = false; + break; + } + } + if (sorted) return items.len; + // Offsets are unique, so an unstable sort gives the same order. + std.mem.sortUnstable(Keyed(T), items, {}, Ctx.lessThan); + var w: usize = 1; + for (items[1..]) |it| { + if (!std.mem.eql(u8, nameOf(&it.item), nameOf(&items[w - 1].item))) { + items[w] = it; + w += 1; + } + } + return w; +} + +fn attrName(a: *const Attr) []const u8 { + return a.name; +} +fn inputName(i: *const WireInput) []const u8 { + return i.name; +} +fn nodeKey(n: *const WireNode) []const u8 { + return n.key; +} + +const Parser = struct { + s: []const u8, + pos: usize = 0, + depth: usize = 0, + limits: Limits, + bump: *Bump, + /// The bump allocator as a std Allocator, for growable lists. + arena: Allocator, + diag: ?*Diagnostic, + attr_scratch: std.ArrayList(Keyed(Attr)) = .empty, + input_scratch: std.ArrayList(Keyed(WireInput)) = .empty, + path_scratch: std.ArrayList([]const u8) = .empty, + + fn fail(p: *Parser, err: ParseError, at: usize) ParseError { + if (p.diag) |d| d.offset = at; + return err; + } + + inline fn ws(p: *Parser) void { + p.pos = skipSpace(p.s, p.pos); + } + + inline fn peek(p: *Parser) ?u8 { + return if (p.pos < p.s.len) p.s[p.pos] else null; + } + + inline fn consume(p: *Parser, c: u8) bool { + if (p.pos < p.s.len and p.s[p.pos] == c) { + p.pos += 1; + return true; + } + return false; + } + + fn expect(p: *Parser, c: u8) ParseError!void { + if (p.consume(c)) return; + return p.fail(if (p.pos >= p.s.len) error.UnexpectedEof else error.Syntax, p.pos); + } + + fn enter(p: *Parser) ParseError!void { + p.depth += 1; + if (p.depth > p.limits.max_depth) return p.fail(error.NestingTooDeep, p.pos); + } + + fn leave(p: *Parser) void { + p.depth -= 1; + } + + /// After a member: true when the object/array closed, false after ','. + fn next(p: *Parser, close: u8) ParseError!bool { + p.ws(); + if (p.consume(close)) { + p.leave(); + return true; + } + try p.expect(','); + p.ws(); + return false; + } + + fn startsWith(p: *Parser, lit: []const u8) bool { + return std.mem.startsWith(u8, p.s[p.pos..], lit); + } + + fn string(p: *Parser) ParseError![]const u8 { + try p.expect('"'); + const start = p.pos; + var decoded: ?std.ArrayList(u8) = null; + var seg = start; + while (true) { + const i = scanString(p.s, p.pos) orelse return p.fail(error.UnexpectedEof, p.s.len); + const c = p.s[i]; + switch (c) { + '"' => { + p.pos = i + 1; + if (decoded) |*d| { + try d.appendSlice(p.arena, p.s[seg..i]); + return d.items; + } + return p.s[start..i]; + }, + '\\' => { + if (decoded == null) decoded = .empty; + try decoded.?.appendSlice(p.arena, p.s[seg..i]); + p.pos = i + 1; + try p.escape(&decoded.?); + seg = p.pos; + }, + 0...0x1f => return p.fail(error.ControlCharacter, i), + else => { + const len = std.unicode.utf8ByteSequenceLength(c) catch return p.fail(error.InvalidUtf8, i); + if (i + len > p.s.len) return p.fail(error.InvalidUtf8, i); + const seq = p.s[i..]; + _ = switch (len) { + 2 => std.unicode.utf8Decode2(seq[0..2].*), + 3 => std.unicode.utf8Decode3(seq[0..3].*), + 4 => std.unicode.utf8Decode4(seq[0..4].*), + else => return p.fail(error.InvalidUtf8, i), + } catch return p.fail(error.InvalidUtf8, i); + p.pos = i + len; + }, + } + } + } + + fn escape(p: *Parser, out: *std.ArrayList(u8)) ParseError!void { + if (p.pos >= p.s.len) return p.fail(error.UnexpectedEof, p.pos); + const e = p.s[p.pos]; + p.pos += 1; + const simple: ?u8 = switch (e) { + '"' => '"', + '\\' => '\\', + '/' => '/', + 'b' => 0x08, + 'f' => 0x0c, + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + 'u' => null, + else => return p.fail(error.InvalidEscape, p.pos), + }; + if (simple) |c| return out.append(p.arena, c); + const first = try p.hexQuad(); + var cp: u21 = first; + if (first >= 0xd800 and first <= 0xdbff) { + if (!p.consume('\\') or !p.consume('u')) return p.fail(error.InvalidUnicodeEscape, p.pos); + const second = try p.hexQuad(); + if (second < 0xdc00 or second > 0xdfff) return p.fail(error.InvalidUnicodeEscape, p.pos); + cp = 0x10000 + ((@as(u21, first) - 0xd800) << 10) + (second - 0xdc00); + } else if (first >= 0xdc00 and first <= 0xdfff) { + return p.fail(error.InvalidUnicodeEscape, p.pos); + } + var buf: [4]u8 = undefined; + const n = std.unicode.utf8Encode(cp, &buf) catch return p.fail(error.InvalidUnicodeEscape, p.pos); + try out.appendSlice(p.arena, buf[0..n]); + } + + fn hexQuad(p: *Parser) ParseError!u16 { + if (p.pos + 4 > p.s.len) return p.fail(error.UnexpectedEof, p.pos); + var v: u16 = 0; + for (p.s[p.pos..][0..4]) |c| { + const d: u16 = switch (c) { + '0'...'9' => c - '0', + 'a'...'f' => c - 'a' + 10, + 'A'...'F' => c - 'A' + 10, + else => return p.fail(error.InvalidUnicodeEscape, p.pos), + }; + v = (v << 4) | d; + p.pos += 1; + } + return v; + } + + fn unsigned(p: *Parser) ParseError!u64 { + const start = p.pos; + const first = p.peek() orelse return p.fail(error.UnexpectedEof, p.pos); + if (first < '0' or first > '9') return p.fail(error.InvalidNumber, start); + p.pos += 1; + if (first == '0' and p.peek() != null and std.ascii.isDigit(p.peek().?)) return p.fail(error.InvalidNumber, start); + var v: u64 = first - '0'; + while (p.peek()) |c| { + if (c < '0' or c > '9') break; + p.pos += 1; + v = std.math.mul(u64, v, 10) catch return p.fail(error.InvalidNumber, start); + v = std.math.add(u64, v, c - '0') catch return p.fail(error.InvalidNumber, start); + } + if (p.peek()) |c| if (c == '.' or c == 'e' or c == 'E') return p.fail(error.InvalidNumber, start); + return v; + } + + fn boolean(p: *Parser) ParseError!bool { + if (p.startsWith("true")) { + p.pos += 4; + return true; + } + if (p.startsWith("false")) { + p.pos += 5; + return false; + } + return p.fail(error.Syntax, p.pos); + } + + fn null_(p: *Parser) ParseError!void { + if (!p.startsWith("null")) return p.fail(error.Syntax, p.pos); + p.pos += 4; + } + + fn skipValue(p: *Parser) ParseError!void { + p.ws(); + const c = p.peek() orelse return p.fail(error.UnexpectedEof, p.pos); + switch (c) { + '"' => _ = try p.string(), + '{' => { + p.pos += 1; + try p.enter(); + p.ws(); + if (p.consume('}')) return p.leave(); + while (true) { + _ = try p.string(); + p.ws(); + try p.expect(':'); + try p.skipValue(); + if (try p.next('}')) return; + } + }, + '[' => { + p.pos += 1; + try p.enter(); + p.ws(); + if (p.consume(']')) return p.leave(); + while (true) { + try p.skipValue(); + if (try p.next(']')) return; + } + }, + 't', 'f' => _ = try p.boolean(), + 'n' => try p.null_(), + '-', '0'...'9' => try p.skipNumber(), + else => return p.fail(error.Syntax, p.pos), + } + } + + fn skipNumber(p: *Parser) ParseError!void { + const start = p.pos; + _ = p.consume('-'); + const d = p.peek() orelse return p.fail(error.InvalidNumber, start); + p.pos += 1; + if (d == '0') { + if (p.peek() != null and std.ascii.isDigit(p.peek().?)) return p.fail(error.InvalidNumber, start); + } else if (d >= '1' and d <= '9') { + while (p.peek() != null and std.ascii.isDigit(p.peek().?)) p.pos += 1; + } else return p.fail(error.InvalidNumber, start); + if (p.consume('.')) { + if (p.peek() == null or !std.ascii.isDigit(p.peek().?)) return p.fail(error.InvalidNumber, start); + while (p.peek() != null and std.ascii.isDigit(p.peek().?)) p.pos += 1; + } + if (p.peek()) |e| if (e == 'e' or e == 'E') { + p.pos += 1; + if (p.peek()) |sign| if (sign == '+' or sign == '-') { + p.pos += 1; + }; + if (p.peek() == null or !std.ascii.isDigit(p.peek().?)) return p.fail(error.InvalidNumber, start); + while (p.peek() != null and std.ascii.isDigit(p.peek().?)) p.pos += 1; + }; + } + + fn key(p: *Parser) ParseError![]const u8 { + const k = try p.string(); + p.ws(); + try p.expect(':'); + p.ws(); + return k; + } + + fn path(p: *Parser) ParseError!Path { + try p.expect('['); + try p.enter(); + p.ws(); + if (p.consume(']')) { + p.leave(); + return &.{}; + } + const mark = p.path_scratch.items.len; + defer p.path_scratch.shrinkRetainingCapacity(mark); + while (true) { + try p.path_scratch.append(p.arena, try p.string()); + if (p.path_scratch.items.len - mark > p.limits.max_path_components) return p.fail(error.PathTooLong, p.pos); + if (try p.next(']')) break; + } + return p.bump.dupe([]const u8, p.path_scratch.items[mark..]); + } + + fn attrs(p: *Parser) ParseError![]Attr { + try p.expect('{'); + try p.enter(); + p.ws(); + if (p.consume('}')) { + p.leave(); + return &.{}; + } + const mark = p.attr_scratch.items.len; + defer p.attr_scratch.shrinkRetainingCapacity(mark); + while (true) { + const offset = p.pos; + const name = try p.key(); + const c = p.peek() orelse return p.fail(error.UnexpectedEof, p.pos); + const value: Value = switch (c) { + '"' => .{ .string = try p.string() }, + 't', 'f' => .{ .boolean = try p.boolean() }, + '0'...'9' => .{ .integer = try p.unsigned() }, + else => return p.fail(error.UnsupportedInputAttributeType, p.pos), + }; + try p.attr_scratch.append(p.arena, .{ .offset = offset, .item = .{ .name = name, .value = value } }); + if (try p.next('}')) break; + } + const items = p.attr_scratch.items[mark..]; + const n = sortDedup(Attr, items, attrName); + const out = try p.bump.alloc(Attr, n); + for (items[0..n], out) |k, *o| o.* = k.item; + return out; + } + + fn inputs(p: *Parser) ParseError![]WireInput { + try p.expect('{'); + try p.enter(); + p.ws(); + if (p.consume('}')) { + p.leave(); + return &.{}; + } + const mark = p.input_scratch.items.len; + defer p.input_scratch.shrinkRetainingCapacity(mark); + while (true) { + const offset = p.pos; + const name = try p.key(); + const edge: WireEdge = if (p.peek() == '[') .{ .follows = try p.path() } else .{ .node = try p.string() }; + try p.input_scratch.append(p.arena, .{ .offset = offset, .item = .{ .offset = offset, .name = name, .edge = edge } }); + if (try p.next('}')) break; + } + const items = p.input_scratch.items[mark..]; + const n = sortDedup(WireInput, items, inputName); + const out = try p.bump.alloc(WireInput, n); + for (items[0..n], out) |k, *o| o.* = k.item; + return out; + } + + fn node(p: *Parser, offset: usize, k: []const u8) ParseError!WireNode { + try p.expect('{'); + try p.enter(); + var n: WireNode = .{ .offset = offset, .key = k }; + p.ws(); + if (p.consume('}')) { + p.leave(); + return n; + } + while (true) { + const field = try p.key(); + if (eq(field, "inputs")) { + n.inputs = try p.inputs(); + } else if (eq(field, "original")) { + n.original = try p.attrs(); + } else if (eq(field, "locked")) { + n.locked = try p.attrs(); + } else if (eq(field, "info")) { + n.info = try p.attrs(); + } else if (eq(field, "flake")) { + n.is_flake = try p.boolean(); + } else if (eq(field, "parent")) { + if (p.startsWith("null")) { + try p.null_(); + n.parent = null; + } else n.parent = try p.path(); + } else try p.skipValue(); + if (try p.next('}')) return n; + } + } + + fn nodes(p: *Parser) ParseError![]WireNode { + try p.expect('{'); + try p.enter(); + var list: std.ArrayList(Keyed(WireNode)) = .empty; + defer list.deinit(p.arena); + p.ws(); + if (p.consume('}')) { + p.leave(); + } else while (true) { + const offset = p.pos; + const k = try p.key(); + try list.append(p.arena, .{ .offset = offset, .item = try p.node(offset, k) }); + if (try p.next('}')) break; + } + const n = sortDedup(WireNode, list.items, nodeKey); + if (n > p.limits.max_nodes) return p.fail(error.TooManyNodes, p.pos); + const out = try p.bump.alloc(WireNode, n); + for (list.items[0..n], out) |k, *o| o.* = k.item; + return out; + } +}; + +fn eq(a: []const u8, comptime b: []const u8) bool { + return a.len == b.len and std.mem.eql(u8, a, b); +} + +/// Overlays `newer` onto `older` (both sorted by name); `newer` wins. +fn overlay(arena: Allocator, older: []const Attr, newer: []const Attr) Allocator.Error![]const Attr { + var out: std.ArrayList(Attr) = try .initCapacity(arena, older.len + newer.len); + var i: usize = 0; + var j: usize = 0; + while (i < older.len and j < newer.len) { + switch (std.mem.order(u8, older[i].name, newer[j].name)) { + .lt => { + out.appendAssumeCapacity(older[i]); + i += 1; + }, + .eq => { + out.appendAssumeCapacity(newer[j]); + i += 1; + j += 1; + }, + .gt => { + out.appendAssumeCapacity(newer[j]); + j += 1; + }, + } + } + out.appendSliceAssumeCapacity(older[i..]); + out.appendSliceAssumeCapacity(newer[j..]); + return out.items; +} + +fn findNode(nodes: []const WireNode, k: []const u8) ?usize { + var lo: usize = 0; + var hi: usize = nodes.len; + while (lo < hi) { + const mid = lo + (hi - lo) / 2; + switch (std.mem.order(u8, nodes[mid].key, k)) { + .eq => return mid, + .lt => lo = mid + 1, + .gt => hi = mid, + } + } + return null; +} + +/// A parsed lock file and the memory backing it. Unescaped strings borrow +/// from the input, which must outlive the document. +pub const Document = struct { + lock: LockFile, + bump: Bump, + + pub fn deinit(self: *Document) void { + self.bump.deinit(); + self.* = undefined; + } +}; + +/// Parses a lock file. All allocations come from one bump allocator owned +/// by the returned document; a typical file needs a single malloc. +pub fn parse(gpa: Allocator, bytes: []const u8, limits: Limits, diag: ?*Diagnostic) ParseError!Document { + if (bytes.len > limits.max_bytes) return error.InputTooLarge; + var bump = try Bump.init(gpa, bytes.len * 2 + 4096); + errdefer bump.deinit(); + var p: Parser = .{ .s = bytes, .limits = limits, .bump = &bump, .arena = bump.allocator(), .diag = diag }; + + var version: ?u64 = null; + var root_key: ?[]const u8 = null; + var wire: ?[]WireNode = null; + p.ws(); + try p.expect('{'); + try p.enter(); + p.ws(); + if (p.consume('}')) { + p.leave(); + } else while (true) { + const k = try p.key(); + if (eq(k, "version")) { + version = try p.unsigned(); + } else if (eq(k, "root")) { + root_key = try p.string(); + } else if (eq(k, "nodes")) { + wire = try p.nodes(); + } else try p.skipValue(); + if (try p.next('}')) break; + } + const v = version orelse return p.fail(error.MissingField, p.pos); + if (v < 5 or v > 7) return p.fail(error.UnsupportedVersion, p.pos); + const rk = root_key orelse return p.fail(error.MissingField, p.pos); + const wn = wire orelse return p.fail(error.MissingField, p.pos); + p.ws(); + if (p.pos != bytes.len) return p.fail(error.TrailingCharacters, p.pos); + const lock = try build(&p, @intCast(v), rk, wn); + return .{ .lock = lock, .bump = bump }; +} + +fn build(p: *Parser, version: u8, root_key: []const u8, wire: []WireNode) ParseError!LockFile { + const gpa = p.arena; + const root_old = findNode(wire, root_key) orelse return p.fail(error.MissingNode, 0); + + const new_id = try gpa.alloc(u32, wire.len); + defer gpa.free(new_id); + const unseen = std.math.maxInt(u32); + @memset(new_id, unseen); + var stack: std.ArrayList(u32) = .empty; + defer stack.deinit(gpa); + try stack.append(gpa, @intCast(root_old)); + var reachable: usize = 0; + // Mark reachability first (0 = seen), assign dense ids afterwards so the + // arena order matches source key order, like the reference. + while (stack.pop()) |idx| { + if (new_id[idx] != unseen) continue; + new_id[idx] = 0; + reachable += 1; + for (wire[idx].inputs) |*in| switch (in.edge) { + .node => |target| { + const t = findNode(wire, target) orelse return p.fail(error.MissingNode, in.offset); + if (t == root_old) return p.fail(error.CycleToRoot, in.offset); + in.target = @intCast(t); + try stack.append(gpa, @intCast(t)); + }, + .follows => {}, + }; + } + if (reachable > p.limits.max_nodes or reachable > std.math.maxInt(u32)) return p.fail(error.TooManyNodes, 0); + var next_id: u32 = 1; + for (new_id, 0..) |*id, old| { + if (id.* == unseen) continue; + if (old == root_old) { + id.* = root_id; + } else { + id.* = next_id; + next_id += 1; + } + } + + const nodes = try p.bump.alloc(Node, reachable); + for (wire, 0..) |*w, old| { + const id = new_id[old]; + if (id == unseen) continue; + const ins = try p.bump.alloc(Input, w.inputs.len); + for (w.inputs, ins) |in, *o| o.* = .{ + .name = in.name, + .edge = switch (in.edge) { + .node => .{ .node = new_id[in.target] }, + .follows => |f| .{ .follows = f }, + }, + }; + var locked: ?Locked = null; + if (old != root_old) { + const l = w.locked orelse return p.fail(error.MissingField, w.offset); + const original = w.original orelse return p.fail(error.MissingField, w.offset); + const merged: []const Attr = if (w.info) |info| try overlay(p.arena, l, info) else l; + locked = .{ .original = original, .locked = merged, .is_flake = w.is_flake, .parent = w.parent }; + } + nodes[id] = .{ .key = w.key, .inputs = ins, .locked = locked }; + } + return .{ .source_version = version, .nodes = nodes }; +} + +// ── follows resolution ── + +const Resolver = struct { + gpa: Allocator, + lock: *const LockFile, + /// Input `i` of node `n` is edge `first_input[n] + i`, so the per-edge + /// state below is plain arrays: no hashing on the resolve path. + first_input: []u32, + /// Edges on the chain being resolved, for cycle detection. + active: []bool, + /// Resolved follows target per edge, or `unknown`. + cache: []u32, + steps: std.ArrayList(Step) = .empty, + + const unknown = std.math.maxInt(u32); + + const Step = union(enum) { component: []const u8, finish: u32 }; + + fn init(gpa: Allocator, lock: *const LockFile) Allocator.Error!Resolver { + const first_input = try gpa.alloc(u32, lock.nodes.len); + errdefer gpa.free(first_input); + var edges: u32 = 0; + for (lock.nodes, first_input) |n, *first| { + first.* = edges; + edges += @intCast(n.inputs.len); + } + const active = try gpa.alloc(bool, edges); + errdefer gpa.free(active); + @memset(active, false); + const cache = try gpa.alloc(u32, edges); + @memset(cache, unknown); + return .{ .gpa = gpa, .lock = lock, .first_input = first_input, .active = active, .cache = cache }; + } + + fn deinit(r: *Resolver) void { + r.gpa.free(r.first_input); + r.gpa.free(r.active); + r.gpa.free(r.cache); + r.steps.deinit(r.gpa); + } + + fn resolve(r: *Resolver, path: Path, use_cache: bool) ValidationError!?u32 { + r.steps.clearRetainingCapacity(); + // Every active edge has its finish step still on the stack; an early + // return clears them so the next call starts clean. + defer for (r.steps.items) |step| switch (step) { + .finish => |e| r.active[e] = false, + .component => {}, + }; + var i = path.len; + while (i > 0) { + i -= 1; + try r.steps.append(r.gpa, .{ .component = path[i] }); + } + var current: u32 = root_id; + while (r.steps.pop()) |step| switch (step) { + .component => |name| { + const node = &r.lock.nodes[current]; + const idx = findSorted(Input, node.inputs, name) orelse return null; + switch (node.inputs[idx].edge) { + .node => |child| current = child, + .follows => |target| { + const edge = r.first_input[current] + @as(u32, @intCast(idx)); + if (use_cache and r.cache[edge] != unknown) { + current = r.cache[edge]; + continue; + } + if (r.active[edge]) return error.FollowsCycle; + r.active[edge] = true; + try r.steps.append(r.gpa, .{ .finish = edge }); + var j = target.len; + while (j > 0) { + j -= 1; + try r.steps.append(r.gpa, .{ .component = target[j] }); + } + current = root_id; + }, + } + }, + .finish => |edge| { + r.active[edge] = false; + if (use_cache) r.cache[edge] = current; + }, + }; + return current; + } +}; + +// ── Canonical writer ── + +const Writer = struct { + gpa: Allocator, + out: *std.ArrayList(u8), + lock: *const LockFile, + + // Capacity is reserved per node (see `reserveNode`), so every write + // below is an unchecked copy. + inline fn raw(w: *Writer, s: []const u8) void { + w.out.appendSliceAssumeCapacity(s); + } + + inline fn byte(w: *Writer, c: u8) void { + w.out.appendAssumeCapacity(c); + } + + inline fn indent(w: *Writer, level: usize) void { + w.out.appendNTimesAssumeCapacity(' ', 2 * level); + } + + fn str(w: *Writer, s: []const u8) void { + const hex = "0123456789abcdef"; + w.byte('"'); + var start: usize = 0; + while (true) { + const i = scanEscape(s, start); + w.raw(s[start..i]); + if (i == s.len) break; + const c = s[i]; + switch (c) { + '"' => w.raw("\\\""), + '\\' => w.raw("\\\\"), + 0x08 => w.raw("\\b"), + 0x0c => w.raw("\\f"), + '\n' => w.raw("\\n"), + '\r' => w.raw("\\r"), + '\t' => w.raw("\\t"), + else => w.raw(&.{ '\\', 'u', '0', '0', hex[c >> 4], hex[c & 15] }), + } + start = i + 1; + } + w.byte('"'); + } + + fn int(w: *Writer, value: u64) void { + var buf: [20]u8 = undefined; + var i: usize = buf.len; + var v = value; + while (true) { + i -= 1; + buf[i] = @intCast('0' + v % 10); + v /= 10; + if (v == 0) break; + } + w.raw(buf[i..]); + } + + fn field(w: *Writer, level: usize, comptime name: []const u8) void { + w.indent(level); + w.raw("\"" ++ name ++ "\": "); + } + + fn sep(w: *Writer, written: usize, total: usize) void { + if (written != total) w.byte(','); + w.byte('\n'); + } + + fn path(w: *Writer, p: Path, level: usize) void { + if (p.len == 0) return w.raw("[]"); + w.raw("[\n"); + for (p, 0..) |c, i| { + w.indent(level + 1); + w.str(c); + if (i + 1 != p.len) w.byte(','); + w.byte('\n'); + } + w.indent(level); + w.byte(']'); + } + + fn attrs(w: *Writer, a: []const Attr, level: usize) void { + var len: usize = 0; + for (a) |x| len += @intFromBool(!eq(x.name, "__final")); + if (len == 0) return w.raw("{}"); + w.raw("{\n"); + var i: usize = 0; + for (a) |x| { + if (eq(x.name, "__final")) continue; + w.indent(level + 1); + w.str(x.name); + w.raw(": "); + switch (x.value) { + .string => |s| w.str(s), + .integer => |n| w.int(n), + .boolean => |b| w.raw(if (b) "true" else "false"), + } + i += 1; + if (i != len) w.byte(','); + w.byte('\n'); + } + w.indent(level); + w.byte('}'); + } + + /// Upper bound on the bytes `node` writes: escaping expands a byte to at + /// most 6, and a line has at most 2 * (level + 3) indentation plus a few + /// bytes of punctuation. + fn reserveNode(w: *Writer, n: *const Node, nm: []const []const u8, name: []const u8) Allocator.Error!void { + const line = 2 * 6 + 16; + var bound: usize = 6 * name.len + 4 * line; + for (n.inputs) |in| { + bound += 6 * in.name.len + line; + switch (in.edge) { + .node => |id| bound += 6 * nm[id].len, + .follows => |p| for (p) |c| { + bound += 6 * c.len + line; + }, + } + } + if (n.locked) |l| { + for ([_][]const Attr{ l.locked, l.original }) |a| for (a) |x| { + bound += 6 * x.name.len + line + switch (x.value) { + .string => |s| 6 * s.len, + else => 20, + }; + }; + if (l.parent) |p| for (p) |c| { + bound += 6 * c.len + line; + }; + } + try w.out.ensureUnusedCapacity(w.gpa, bound); + } + + fn node(w: *Writer, n: *const Node, nm: []const []const u8, level: usize) void { + const has_flake = if (n.locked) |l| !l.is_flake else false; + const has_inputs = n.inputs.len > 0; + const has_parent = if (n.locked) |l| l.parent != null else false; + const total = @as(usize, @intFromBool(has_flake)) + @intFromBool(has_inputs) + @as(usize, if (n.locked != null) 2 else 0) + @intFromBool(has_parent); + if (total == 0) return w.raw("{}"); + w.raw("{\n"); + var written: usize = 0; + if (has_flake) { + w.field(level + 1, "flake"); + w.raw("false"); + written += 1; + w.sep(written, total); + } + if (has_inputs) { + w.field(level + 1, "inputs"); + w.raw("{\n"); + for (n.inputs, 0..) |in, i| { + w.indent(level + 2); + w.str(in.name); + w.raw(": "); + switch (in.edge) { + .node => |id| w.str(nm[id]), + .follows => |p| w.path(p, level + 2), + } + if (i + 1 != n.inputs.len) w.byte(','); + w.byte('\n'); + } + w.indent(level + 1); + w.byte('}'); + written += 1; + w.sep(written, total); + } + if (n.locked) |l| { + w.field(level + 1, "locked"); + w.attrs(l.locked, level + 1); + written += 1; + w.sep(written, total); + w.field(level + 1, "original"); + w.attrs(l.original, level + 1); + written += 1; + w.sep(written, total); + if (l.parent) |p| { + w.field(level + 1, "parent"); + w.path(p, level + 1); + written += 1; + w.sep(written, total); + } + } + w.indent(level); + w.byte('}'); + } + + /// Nix's naming: depth-first from the root, each node named after the + /// first input that reaches it, with `_2`, `_3`, ... on collisions. + fn names(w: *Writer, tmp: Allocator) Allocator.Error![]const []const u8 { + const lock = w.lock; + const out = try tmp.alloc([]const u8, lock.nodes.len); + const named = try tmp.alloc(bool, lock.nodes.len); + @memset(named, false); + out[root_id] = "root"; + named[root_id] = true; + // Small graphs check collisions by scanning assigned names; a hash + // set only pays off beyond a few dozen nodes. + const small = lock.nodes.len <= 48; + var assigned: std.ArrayList([]const u8) = .empty; + var used: std.StringHashMapUnmanaged(void) = .empty; + if (small) { + try assigned.ensureTotalCapacity(tmp, lock.nodes.len); + assigned.appendAssumeCapacity("root"); + } else { + try used.ensureTotalCapacity(tmp, @intCast(lock.nodes.len)); + used.putAssumeCapacity("root", {}); + } + var suffix: std.StringHashMapUnmanaged(u64) = .empty; + const Frame = struct { node: u32, next: u32 }; + var stack: std.ArrayList(Frame) = .empty; + try stack.append(tmp, .{ .node = root_id, .next = 0 }); + while (stack.items.len > 0) { + const top = &stack.items[stack.items.len - 1]; + const n = &lock.nodes[top.node]; + if (top.next >= n.inputs.len) { + _ = stack.pop(); + continue; + } + const in = n.inputs[top.next]; + top.next += 1; + const child = switch (in.edge) { + .node => |c| c, + .follows => continue, + }; + if (named[child]) continue; + const taken = struct { + fn check(s: bool, a: []const []const u8, u: *const std.StringHashMapUnmanaged(void), name: []const u8) bool { + if (!s) return u.contains(name); + for (a) |x| if (std.mem.eql(u8, x, name)) return true; + return false; + } + }.check; + var name = in.name; + if (taken(small, assigned.items, &used, name)) { + const gop = try suffix.getOrPut(tmp, in.name); + if (!gop.found_existing) gop.value_ptr.* = 2; + while (true) { + const candidate = try tmp.print("{s}_{d}", .{ in.name, gop.value_ptr.* }); + gop.value_ptr.* += 1; + if (!taken(small, assigned.items, &used, candidate)) { + name = candidate; + break; + } + } + } + if (small) assigned.appendAssumeCapacity(name) else try used.put(tmp, name, {}); + out[child] = name; + named[child] = true; + try stack.append(tmp, .{ .node = child, .next = 0 }); + } + return out; + } + + fn write(w: *Writer) Allocator.Error!void { + var first: [8192]u8 = undefined; + var fallback: std.heap.BufferFirstAllocator = .init(&first, w.gpa); + var bump = try Bump.init(fallback.allocator(), 4096); + defer bump.deinit(); + const tmp = bump.allocator(); + const nm = try w.names(tmp); + const order = try bump.alloc(u32, nm.len); + for (order, 0..) |*o, i| o.* = @intCast(i); + std.mem.sortUnstable(u32, order, nm, struct { + fn lessThan(ctx: []const []const u8, a: u32, b: u32) bool { + return std.mem.lessThan(u8, ctx[a], ctx[b]); + } + }.lessThan); + try w.out.ensureUnusedCapacity(w.gpa, 64); + w.raw("{\n \"nodes\": {"); + for (order, 0..) |id, i| { + const n = &w.lock.nodes[id]; + try w.reserveNode(n, nm, nm[id]); + w.raw(if (i == 0) "\n" else ",\n"); + w.indent(2); + w.str(nm[id]); + w.raw(": "); + w.node(n, nm, 2); + } + try w.out.ensureUnusedCapacity(w.gpa, 64); + w.raw("\n },\n \"root\": \"root\",\n \"version\": 7\n}\n"); + } +}; + +// ── Tests ── + +const testing = std.testing; + +const typical = + \\{ + \\ "nodes": { + \\ "nixpkgs": { + \\ "locked": { + \\ "lastModified": 1750000000, + \\ "narHash": "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + \\ "owner": "NixOS", + \\ "repo": "nixpkgs", + \\ "rev": "0123456789abcdef0123456789abcdef01234567", + \\ "type": "github" + \\ }, + \\ "original": { + \\ "owner": "NixOS", + \\ "ref": "nixos-unstable", + \\ "repo": "nixpkgs", + \\ "type": "github" + \\ } + \\ }, + \\ "root": { + \\ "inputs": { + \\ "nixpkgs": "nixpkgs", + \\ "tools": "tools" + \\ } + \\ }, + \\ "tools": { + \\ "inputs": { + \\ "nixpkgs": [ + \\ "nixpkgs" + \\ ] + \\ }, + \\ "locked": { + \\ "lastModified": 1750000001, + \\ "narHash": "sha256-BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=", + \\ "owner": "example", + \\ "repo": "tools", + \\ "rev": "89abcdef0123456789abcdef0123456789abcdef", + \\ "type": "github" + \\ }, + \\ "original": { + \\ "owner": "example", + \\ "repo": "tools", + \\ "type": "github" + \\ } + \\ } + \\ }, + \\ "root": "root", + \\ "version": 7 + \\} + \\ +; + +test "canonical documents round-trip byte for byte" { + var doc = try parse(testing.allocator, typical, .{}, null); + defer doc.deinit(); + try doc.lock.validate(testing.allocator); + var out: std.ArrayList(u8) = .empty; + defer out.deinit(testing.allocator); + try doc.lock.writeNix(testing.allocator, &out); + try testing.expectEqualStrings(typical, out.items); + try testing.expectEqual(@as(?u32, doc.lock.nodes[root_id].input("nixpkgs").?.edge.node), try doc.lock.resolve(testing.allocator, &.{ "tools", "nixpkgs" })); +} + +test "version 5 info overlay, duplicate keys and renaming" { + const src = + \\{"version":5,"root":"r","nodes":{"r":{"inputs":{"a":"x","b":"y"}}, + \\"x":{"locked":{"type":"path","path":"1"},"original":{"type":"path"},"info":{"lastModified":7}}, + \\"y":{"inputs":{"a":"z"},"locked":{"type":"path"},"original":{"type":"path"},"flake":false}, + \\"z":{"locked":{"k":"old","k":"new"},"original":{}}, "unused":{"broken":true}}} + ; + var doc = try parse(testing.allocator, src, .{}, null); + defer doc.deinit(); + try testing.expectEqual(@as(usize, 4), doc.lock.nodes.len); + var out: std.ArrayList(u8) = .empty; + defer out.deinit(testing.allocator); + try doc.lock.writeNix(testing.allocator, &out); + try testing.expect(std.mem.find(u8, out.items, "\"lastModified\": 7,") != null); + try testing.expect(std.mem.find(u8, out.items, "\"a_2\": {") != null); + try testing.expect(std.mem.find(u8, out.items, "\"k\": \"new\"") != null); + try testing.expect(std.mem.find(u8, out.items, "\"flake\": false,") != null); + // Re-parsing canonical output is a fixed point. + var doc2 = try parse(testing.allocator, out.items, .{}, null); + defer doc2.deinit(); + var again: std.ArrayList(u8) = .empty; + defer again.deinit(testing.allocator); + try doc2.lock.writeNix(testing.allocator, &again); + try testing.expectEqualStrings(out.items, again.items); +} + +test "large graphs use the hashed collision path" { + var src: std.ArrayList(u8) = .empty; + defer src.deinit(testing.allocator); + try src.appendSlice(testing.allocator, "{\"version\":7,\"root\":\"root\",\"nodes\":{\"root\":{\"inputs\":{"); + for (0..100) |i| try src.print(testing.allocator, "{s}\"d{d}\":\"d{d}\"", .{ if (i == 0) "" else ",", i, i }); + try src.appendSlice(testing.allocator, "}}"); + for (0..100) |i| try src.print(testing.allocator, ",\"d{d}\":{{\"inputs\":{{\"s\":\"s{d}\"}},\"locked\":{{}},\"original\":{{}}}},\"s{d}\":{{\"locked\":{{}},\"original\":{{}}}}", .{ i, i, i }); + try src.appendSlice(testing.allocator, "}}"); + var doc = try parse(testing.allocator, src.items, .{}, null); + defer doc.deinit(); + var out: std.ArrayList(u8) = .empty; + defer out.deinit(testing.allocator); + try doc.lock.writeNix(testing.allocator, &out); + try testing.expect(std.mem.find(u8, out.items, "\"s_100\": {") != null); + try testing.expect(std.mem.find(u8, out.items, "\"s_101\"") == null); +} + +test "rejects malformed and adversarial inputs" { + const a = testing.allocator; + const cases = .{ + .{ "{\"version\":4,\"root\":\"r\",\"nodes\":{\"r\":{}}}", error.UnsupportedVersion }, + .{ "{\"version\":7,\"root\":\"r\",\"nodes\":{\"r\":{\"inputs\":{\"a\":\"r\"}}}}", error.CycleToRoot }, + .{ "{\"version\":7,\"root\":\"r\",\"nodes\":{\"r\":{\"inputs\":{\"a\":\"q\"}}}}", error.MissingNode }, + .{ "{\"version\":7,\"root\":\"r\",\"nodes\":{\"r\":{}}} x", error.TrailingCharacters }, + .{ "{\"version\":07,\"root\":\"r\",\"nodes\":{}}", error.InvalidNumber }, + .{ "{\"version\":7,\"root\":\"r\\x\",\"nodes\":{}}", error.InvalidEscape }, + .{ "{\"version\":7,\"root\":\"\xff\",\"nodes\":{}}", error.InvalidUtf8 }, + .{ "{\"version\":7,\"root\":\"a\x01\",\"nodes\":{}}", error.ControlCharacter }, + .{ "{\"version\":7,\"root\":\"r\",\"nodes\":{\"r\":{\"inputs\":{\"a\":\"b\"}},\"b\":{\"locked\":{}}}}", error.MissingField }, + .{ "{\"version\":7,\"root\":\"r\",\"nodes\":{\"r\":{\"inputs\":{\"a\":\"b\"}},\"b\":{\"locked\":{\"x\":null},\"original\":{}}}}", error.UnsupportedInputAttributeType }, + }; + inline for (cases) |c| try testing.expectError(c[1], parse(a, c[0], .{}, null)); + try testing.expectError(error.NestingTooDeep, parse(a, "{\"x\":" ++ &@as([300]u8, @splat('[')) ++ &@as([300]u8, @splat(']')) ++ "}", .{}, null)); +} + +test "follows validation detects cycles and missing targets" { + var cyc = try parse(testing.allocator, + \\{"version":7,"root":"r","nodes":{"r":{"inputs":{"a":["b"],"b":["a"]}}}} + , .{}, null); + defer cyc.deinit(); + try testing.expectError(error.FollowsCycle, cyc.lock.validate(testing.allocator)); + var missing = try parse(testing.allocator, + \\{"version":7,"root":"r","nodes":{"r":{"inputs":{"a":["nope"]}}}} + , .{}, null); + defer missing.deinit(); + try testing.expectError(error.MissingFollowsTarget, missing.lock.validate(testing.allocator)); +} + +test "unicode escapes and string escaping on output" { + var doc = try parse(testing.allocator, + \\{"version":7,"root":"r","nodes":{"r":{"inputs":{"a":"b"}},"b":{"locked":{"s":"\u00e9\ud83d\ude00 \"q\" \t é"},"original":{}}}} + , .{}, null); + defer doc.deinit(); + const s = Locked.get(doc.lock.nodes[1].locked.?.locked, "s").?.string; + try testing.expectEqualStrings("é😀 \"q\" \t é", s); + var out: std.ArrayList(u8) = .empty; + defer out.deinit(testing.allocator); + try doc.lock.writeNix(testing.allocator, &out); + try testing.expect(std.mem.find(u8, out.items, "\"é😀 \\\"q\\\" \\t é\"") != null); +} + +test "vector scanners agree with scalar definitions" { + var buf: [80]u8 = @splat('a'); + for (0..buf.len) |i| { + for ([_]u8{ '"', '\\', 0x01, 0x80, 0xff }) |c| { + buf[i] = c; + try testing.expectEqual(@as(?usize, i), scanString(&buf, 0)); + if (c < 0x80) try testing.expectEqual(i, scanEscape(&buf, 0)); + buf[i] = 'a'; + } + var sp: [80]u8 = @splat(' '); + sp[i] = 'x'; + try testing.expectEqual(i, skipSpace(&sp, 0)); + } +} + +test { + _ = @import("bump.zig"); +} diff --git a/pkg/nix-narinfo/build.zig b/pkg/nix-narinfo/build.zig new file mode 100644 index 0000000..a591f02 --- /dev/null +++ b/pkg/nix-narinfo/build.zig @@ -0,0 +1,20 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mod = b.addModule("nix-narinfo", .{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, + }); + mod.addImport("nix-base32", b.dependency("nix_base32", .{ .target = target, .optimize = optimize }).module("nix-base32")); + mod.addImport("nix-store-path", b.dependency("nix_store_path", .{ .target = target, .optimize = optimize }).module("nix-store-path")); + + const tests = b.addTest(.{ .root_module = mod }); + b.step("test", "Run unit tests").dependOn(&b.addRunArtifact(tests).step); +} diff --git a/pkg/nix-narinfo/build.zig.zon b/pkg/nix-narinfo/build.zig.zon new file mode 100644 index 0000000..0752b8d --- /dev/null +++ b/pkg/nix-narinfo/build.zig.zon @@ -0,0 +1,11 @@ +.{ + .name = .nix_narinfo, + .version = "0.1.0", + .fingerprint = 0xf254156c7cc4c9ba, + .minimum_zig_version = "0.17.0", + .dependencies = .{ + .nix_base32 = .{ .path = "../nix-base32" }, + .nix_store_path = .{ .path = "../nix-store-path" }, + }, + .paths = .{ "build.zig", "build.zig.zon", "src" }, +} diff --git a/pkg/nix-narinfo/src/basemul.zig b/pkg/nix-narinfo/src/basemul.zig new file mode 100644 index 0000000..563418e --- /dev/null +++ b/pkg/nix-narinfo/src/basemul.zig @@ -0,0 +1,116 @@ +//! Constant-time fixed-base scalar multiplication for Ed25519 signing, using +//! the comb method from ref10 (SUPERCOP's ge_scalarmult_base): +//! +//! s = sum e_i 16^i, e_i in [-8, 8] +//! s*B = sum_{odd i} e_i 16^i B * 16 + sum_{even i} e_i 16^i B +//! +//! with a table of j * 256^k * B (j = 1..8, k = 0..31) in affine "Niels" +//! form. That is 64 mixed additions and 4 doublings, against 252 doublings +//! and 64 additions for std's 4-bit window. Table entries are selected with +//! conditional moves over all 8 candidates, so memory access and timing do +//! not depend on the (secret) scalar. + +const std = @import("std"); +const Curve = std.crypto.ecc.Edwards25519; +const Fe = Curve.Fe; + +const Niels = struct { + yplusx: Fe, + yminusx: Fe, + xy2d: Fe, + + const identity: Niels = .{ .yplusx = Fe.one, .yminusx = Fe.one, .xy2d = Fe.zero }; + + fn fromPoint(p: Curve) Niels { + const zinv = p.z.invert(); + const x = p.x.mul(zinv); + const y = p.y.mul(zinv); + return .{ .yplusx = y.add(x), .yminusx = y.sub(x), .xy2d = x.mul(y).mul(Fe.edwards25519d2) }; + } + + fn cMov(self: *Niels, other: Niels, c: u64) void { + self.yplusx.cMov(other.yplusx, c); + self.yminusx.cMov(other.yminusx, c); + self.xy2d.cMov(other.xy2d, c); + } +}; + +pub const Table = [32][8]Niels; + +/// Builds the comb table for the Ed25519 base point (~1 ms; do it once). +pub fn table() Table { + var t: Table = undefined; + var base = Curve.basePoint; + for (0..32) |k| { + var acc = base; + for (0..8) |j| { + t[k][j] = Niels.fromPoint(acc); + acc = acc.add(base); + } + for (0..8) |_| base = base.dbl(); + } + return t; +} + +/// Mixed addition: p + q with q affine (ref10 ge_madd), 7 multiplications. +fn madd(p: Curve, q: Niels) Curve { + const a = p.y.sub(p.x).mul(q.yminusx); + const b = p.y.add(p.x).mul(q.yplusx); + const c = p.t.mul(q.xy2d); + const d = p.z.add(p.z); + const x = b.sub(a); + const y = b.add(a); + const z = d.add(c); + const t = d.sub(c); + return .{ .x = x.mul(t), .y = y.mul(z), .z = z.mul(t), .t = x.mul(y) }; +} + +fn select(row: *const [8]Niels, digit: i8) Niels { + const negative: u8 = @intFromBool(digit < 0); + const abs: u8 = @intCast(@as(i16, digit) * (1 - 2 * @as(i16, negative))); + var out = Niels.identity; + for (row, 1..) |entry, j| { + const eq: u64 = ((@as(u64, abs ^ @as(u8, @intCast(j))) -% 1) >> 63) & 1; + out.cMov(entry, eq); + } + // -(x, y) = (-x, y): swap y+x with y-x and negate 2dxy. + const neg: Niels = .{ .yplusx = out.yminusx, .yminusx = out.yplusx, .xy2d = out.xy2d.neg() }; + out.cMov(neg, negative); + return out; +} + +/// s * B for a scalar `s` < 2^255 (already reduced mod L). +pub fn mulBase(t: *const Table, s: [32]u8) Curve { + var e: [64]i8 = undefined; + for (s, 0..) |byte, i| { + e[2 * i] = @intCast(byte & 15); + e[2 * i + 1] = @intCast(byte >> 4); + } + var carry: i8 = 0; + for (e[0..63]) |*d| { + d.* += carry; + carry = (d.* + 8) >> 4; + d.* -= carry << 4; + } + e[63] += carry; + + var h = Curve.identityElement; + var i: usize = 1; + while (i < 64) : (i += 2) h = madd(h, select(&t[i / 2], e[i])); + h = h.dbl().dbl().dbl().dbl(); + i = 0; + while (i < 64) : (i += 2) h = madd(h, select(&t[i / 2], e[i])); + return h; +} + +test "comb multiplication matches std for random scalars" { + const t = table(); + var prng: std.Random.DefaultPrng = .init(5); + for (0..300) |_| { + var wide: [64]u8 = undefined; + prng.random().bytes(&wide); + const s = Curve.scalar.reduce64(wide); + const expected = try Curve.basePoint.mul(s); + try std.testing.expectEqualSlices(u8, &expected.toBytes(), &mulBase(&t, s).toBytes()); + } +} diff --git a/pkg/nix-narinfo/src/parse.zig b/pkg/nix-narinfo/src/parse.zig new file mode 100644 index 0000000..0fac67b --- /dev/null +++ b/pkg/nix-narinfo/src/parse.zig @@ -0,0 +1,157 @@ +//! Zero-copy narinfo parser. Fields are slices into the input. +//! +//! Lines are split with a vector newline search (compare 32 bytes against +//! '\n', @ctz the mask); store paths are validated with the vectorised +//! checks in nix-store-path. + +const std = @import("std"); +const store_path = @import("nix-store-path"); + +pub const max_sigs = 8; + +pub const Error = error{ MissingField, InvalidLine, InvalidNumber, InvalidStorePath, InvalidReference, TooManySignatures }; + +pub const NarInfo = struct { + store_path: []const u8 = "", + url: []const u8 = "", + compression: []const u8 = "", + file_hash: []const u8 = "", + file_size: ?u64 = null, + nar_hash: []const u8 = "", + nar_size: u64 = 0, + /// Space-separated basenames, validated. + references: []const u8 = "", + deriver: []const u8 = "", + system: []const u8 = "", + ca: []const u8 = "", + sig_buf: [max_sigs][]const u8 = undefined, + sig_count: u8 = 0, + + pub fn sigs(self: *const NarInfo) []const []const u8 { + return self.sig_buf[0..self.sig_count]; + } + + pub fn referenceIterator(self: *const NarInfo) std.mem.TokenIterator(u8, .scalar) { + return std.mem.tokenizeScalar(u8, self.references, ' '); + } +}; + +const lanes = 32; +const V = @Vector(lanes, u8); +const Mask = @Int(.unsigned, lanes); + +pub fn findNewline(s: []const u8, start: usize) ?usize { + var i = start; + const nl: V = @splat('\n'); + while (i + lanes <= s.len) : (i += lanes) { + const v: V = s[i..][0..lanes].*; + const m: Mask = @bitCast(v == nl); + if (m != 0) return i + @ctz(m); + } + while (i < s.len) : (i += 1) if (s[i] == '\n') return i; + return null; +} + +fn number(s: []const u8) Error!u64 { + return std.fmt.parseInt(u64, s, 10) catch error.InvalidNumber; +} + +fn eq(a: []const u8, comptime b: []const u8) bool { + return a.len == b.len and std.mem.eql(u8, a, b); +} + +pub fn parse(text: []const u8) Error!NarInfo { + var info: NarInfo = .{}; + var have_size = false; + var pos: usize = 0; + while (pos < text.len) { + const end = findNewline(text, pos) orelse text.len; + const line = text[pos..end]; + pos = end + 1; + if (line.len == 0) continue; + const colon = std.mem.findScalar(u8, line, ':') orelse return error.InvalidLine; + const key = line[0..colon]; + // Nix skips exactly ": "; tolerate a bare ":" before end of line. + const value = if (colon + 2 <= line.len) line[colon + 2 ..] else ""; + switch (key.len) { + 2 => if (eq(key, "CA")) { + info.ca = value; + }, + 3 => if (eq(key, "URL")) { + info.url = value; + } else if (eq(key, "Sig")) { + if (info.sig_count == max_sigs) return error.TooManySignatures; + info.sig_buf[info.sig_count] = value; + info.sig_count += 1; + }, + 6 => if (eq(key, "System")) { + info.system = value; + }, + 7 => if (eq(key, "NarHash")) { + info.nar_hash = value; + } else if (eq(key, "NarSize")) { + info.nar_size = try number(value); + have_size = true; + } else if (eq(key, "Deriver")) { + if (value.len > 0 and !eq(value, "unknown-deriver") and !store_path.isValidBaseName(value)) return error.InvalidStorePath; + info.deriver = value; + }, + 8 => if (eq(key, "FileHash")) { + info.file_hash = value; + } else if (eq(key, "FileSize")) { + info.file_size = try number(value); + }, + 9 => if (eq(key, "StorePath")) { + if (store_path.baseName(value) == null) return error.InvalidStorePath; + info.store_path = value; + }, + 10 => if (eq(key, "References")) { + var it = std.mem.tokenizeScalar(u8, value, ' '); + while (it.next()) |r| if (!store_path.isValidBaseName(r)) return error.InvalidReference; + info.references = value; + }, + 11 => if (eq(key, "Compression")) { + info.compression = value; + }, + else => {}, + } + } + if (info.store_path.len == 0 or info.url.len == 0 or info.nar_hash.len == 0 or !have_size) return error.MissingField; + if (info.compression.len == 0) info.compression = "bzip2"; // Nix's historical default + return info; +} + +test "parses a narinfo" { + const text = + \\StorePath: /nix/store/p7jg95rzvfalb95k3mskk0jqxc9d724n-libunistring-1.4.1 + \\URL: nar/0k1zvrlb2c66njmrbc972fpz7a4ncqr1w4rillf0qwa4cjqa1m6c.nar.zst + \\Compression: zstd + \\FileHash: sha256:0k1zvrlb2c66njmrbc972fpz7a4ncqr1w4rillf0qwa4cjqa1m6c + \\FileSize: 796947 + \\NarHash: sha256:162jnvprzbbglkdxvh3vgcy35j34wva4ni1zj71826vw25ycbdf0 + \\NarSize: 2078944 + \\References: p7jg95rzvfalb95k3mskk0jqxc9d724n-libunistring-1.4.1 + \\Deriver: b3bvrzp605zk7v12ar918r07500gnayv-libunistring-1.4.1.drv + \\Sig: t-1:vkWbgswaWiUPIgtNgxNf9AYTerK7QJU2O4TfykgO9F8N6akQpbpnF5fNvsWw2ENiX0vqo6zBB4XzdAcT4uAkDQ== + \\ + ; + const info = try parse(text); + try std.testing.expectEqualStrings("zstd", info.compression); + try std.testing.expectEqual(@as(u64, 2078944), info.nar_size); + try std.testing.expectEqual(@as(?u64, 796947), info.file_size); + try std.testing.expectEqual(@as(usize, 1), info.sigs().len); + var it = info.referenceIterator(); + try std.testing.expectEqualStrings("p7jg95rzvfalb95k3mskk0jqxc9d724n-libunistring-1.4.1", it.next().?); + try std.testing.expectError(error.MissingField, parse("StorePath: /nix/store/p7jg95rzvfalb95k3mskk0jqxc9d724n-x\n")); + try std.testing.expectError(error.InvalidStorePath, parse("StorePath: /nix/store/bad\n")); +} + +test "vector newline search matches scalar" { + var buf: [100]u8 = @splat('x'); + for (0..buf.len) |i| { + buf[i] = '\n'; + for (0..i + 1) |start| try std.testing.expectEqual(@as(?usize, i), findNewline(&buf, start)); + buf[i] = 'x'; + } + try std.testing.expectEqual(@as(?usize, null), findNewline(&buf, 0)); +} diff --git a/pkg/nix-narinfo/src/render.zig b/pkg/nix-narinfo/src/render.zig new file mode 100644 index 0000000..d4cf831 --- /dev/null +++ b/pkg/nix-narinfo/src/render.zig @@ -0,0 +1,359 @@ +//! Narinfo rendering and Ed25519 fingerprint signing. Every field that ends +//! up in a narinfo line is validated here, so no caller-supplied string can +//! inject extra lines. + +const std = @import("std"); +const nix32 = @import("nix-base32"); +const store_path = @import("nix-store-path"); +const Ed25519 = std.crypto.sign.Ed25519; +const Curve = std.crypto.ecc.Edwards25519; +const basemul = @import("basemul.zig"); + +const store_dir = store_path.store_dir; +const baseName = store_path.baseName; +const isValidBaseName = store_path.isValidBaseName; + +pub const Error = error{ InvalidStorePath, InvalidReference, InvalidDeriver, InvalidSystem, InvalidCompression, InvalidKey, OutOfMemory }; + +fn isValidSystem(s: []const u8) bool { + if (s.len == 0 or s.len > 64) return false; + for (s) |c| if (!(std.ascii.isAlphanumeric(c) or c == '_' or c == '-' or c == '.')) return false; + return true; +} + +pub const Signer = struct { + name: []u8, + key_pair: Ed25519.KeyPair, + /// RFC 8032 expanded secret, derived once. std's KeyPair.sign re-derives + /// the public key (a full scalar multiplication) on every call to check + /// for a mismatch; our key pair is validated once at load instead. + scalar: [32]u8, + prefix: [32]u8, + public: [32]u8, + /// Comb table for the base point: signing does 64 additions, not 252 doublings. + table: basemul.Table, + + /// Writes a new Nix secret key, `:` (the + /// `nix key generate-secret` format), derived from a random `seed`. + pub fn generate(out: *std.ArrayList(u8), allocator: std.mem.Allocator, name: []const u8, seed: [32]u8) Error!void { + if (name.len == 0) return error.InvalidKey; + for (name) |c| if (c == ':' or c == '\n' or c == ' ') return error.InvalidKey; + const key_pair = Ed25519.KeyPair.generateDeterministic(seed) catch return error.InvalidKey; + const bytes = key_pair.secret_key.toBytes(); + var b64: [std.base64.standard.Encoder.calcSize(bytes.len)]u8 = undefined; + _ = std.base64.standard.Encoder.encode(&b64, &bytes); + try out.appendSlice(allocator, name); + try out.append(allocator, ':'); + try out.appendSlice(allocator, &b64); + } + + /// Parses a Nix secret key: `:`. + pub fn parse(allocator: std.mem.Allocator, text: []const u8) Error!*Signer { + const trimmed = std.mem.trim(u8, text, " \t\r\n"); + const colon = std.mem.findScalar(u8, trimmed, ':') orelse return error.InvalidKey; + const name = trimmed[0..colon]; + if (name.len == 0) return error.InvalidKey; + for (name) |c| if (c == '\n' or c == ' ') return error.InvalidKey; + var raw: [Ed25519.SecretKey.encoded_length]u8 = undefined; + const decoder = std.base64.standard.Decoder; + const b64 = trimmed[colon + 1 ..]; + const n = decoder.calcSizeForSlice(b64) catch return error.InvalidKey; + if (n != raw.len) return error.InvalidKey; + decoder.decode(&raw, b64) catch return error.InvalidKey; + const secret = Ed25519.SecretKey.fromBytes(raw) catch return error.InvalidKey; + // std's fromSecretKey checks the embedded public half only under + // runtime safety; derive it from the seed in every build mode. + const key_pair = Ed25519.KeyPair.generateDeterministic(secret.seed()) catch return error.InvalidKey; + if (!std.mem.eql(u8, &key_pair.public_key.toBytes(), &secret.publicKeyBytes())) return error.InvalidKey; + + var expanded: [64]u8 = undefined; + std.crypto.hash.sha2.Sha512.hash(raw[0..32], &expanded, .{}); + var scalar = expanded[0..32].*; + Curve.scalar.clamp(&scalar); + + const self = try allocator.create(Signer); + errdefer allocator.destroy(self); + self.* = .{ + .name = try allocator.dupe(u8, name), + .key_pair = key_pair, + .scalar = scalar, + .prefix = expanded[32..64].*, + .public = key_pair.public_key.toBytes(), + .table = basemul.table(), + }; + return self; + } + + pub fn destroy(self: *Signer, allocator: std.mem.Allocator) void { + allocator.free(self.name); + allocator.destroy(self); + } + + /// Deterministic Ed25519 (RFC 8032 §5.1.6): one base-point multiplication. + pub fn signRaw(self: *const Signer, msg: []const u8) Error![64]u8 { + const Sha512 = std.crypto.hash.sha2.Sha512; + var h = Sha512.init(.{}); + h.update(&self.prefix); + h.update(msg); + var r64: [64]u8 = undefined; + h.final(&r64); + const r = Curve.scalar.reduce64(r64); + const big_r = basemul.mulBase(&self.table, r).toBytes(); + + h = Sha512.init(.{}); + h.update(&big_r); + h.update(&self.public); + h.update(msg); + var k64: [64]u8 = undefined; + h.final(&k64); + const k = Curve.scalar.reduce64(k64); + + var sig: [64]u8 = undefined; + sig[0..32].* = big_r; + sig[32..64].* = Curve.scalar.mulAdd(k, self.scalar, r); + return sig; + } + + /// `:`. + pub fn sign(self: *const Signer, out: *std.ArrayList(u8), allocator: std.mem.Allocator, msg: []const u8) Error!void { + const bytes = try self.signRaw(msg); + var b64: [std.base64.standard.Encoder.calcSize(bytes.len)]u8 = undefined; + _ = std.base64.standard.Encoder.encode(&b64, &bytes); + try out.appendSlice(allocator, self.name); + try out.append(allocator, ':'); + try out.appendSlice(allocator, &b64); + } + + /// `:`, the value for `trusted-public-keys`. + pub fn publicKey(self: *const Signer, out: *std.ArrayList(u8), allocator: std.mem.Allocator) Error!void { + const bytes = self.key_pair.public_key.toBytes(); + var b64: [std.base64.standard.Encoder.calcSize(bytes.len)]u8 = undefined; + _ = std.base64.standard.Encoder.encode(&b64, &bytes); + try out.appendSlice(allocator, self.name); + try out.append(allocator, ':'); + try out.appendSlice(allocator, &b64); + } +}; + +pub const Input = struct { + store_path: []const u8, + nar_hash: *const [32]u8, + nar_size: u64, + file_hash: *const [32]u8, + file_size: u64, + compression: []const u8, + /// Space-separated store path basenames, any order. + references: []const u8, + /// Deriver basename, or empty. + deriver: []const u8, + /// System, or empty. + system: []const u8, +}; + +fn fileExtension(compression: []const u8) Error![]const u8 { + if (std.mem.eql(u8, compression, "zstd")) return ".nar.zst"; + if (std.mem.eql(u8, compression, "none")) return ".nar"; + if (std.mem.eql(u8, compression, "xz")) return ".nar.xz"; + if (std.mem.eql(u8, compression, "bzip2")) return ".nar.bz2"; + return error.InvalidCompression; +} + +fn lessThanStr(_: void, a: []const u8, b: []const u8) bool { + return std.mem.lessThan(u8, a, b); +} + +/// Renders a complete narinfo. `signer` may be null for an unsigned cache. +pub fn render(allocator: std.mem.Allocator, in: Input, signer: ?*const Signer) Error![]u8 { + _ = baseName(in.store_path) orelse return error.InvalidStorePath; + const ext = try fileExtension(in.compression); + if (in.deriver.len > 0 and !(isValidBaseName(in.deriver) and std.mem.endsWith(u8, in.deriver, ".drv"))) return error.InvalidDeriver; + if (in.system.len > 0 and !isValidSystem(in.system)) return error.InvalidSystem; + + var refs: std.ArrayList([]const u8) = .empty; + defer refs.deinit(allocator); + var it = std.mem.tokenizeScalar(u8, in.references, ' '); + while (it.next()) |r| { + if (!isValidBaseName(r)) return error.InvalidReference; + try refs.append(allocator, r); + } + std.mem.sort([]const u8, refs.items, {}, lessThanStr); + // Nix reads references into a set, so a duplicate would break the Sig. + var unique: usize = 0; + for (refs.items) |r| { + if (unique > 0 and std.mem.eql(u8, refs.items[unique - 1], r)) continue; + refs.items[unique] = r; + unique += 1; + } + refs.shrinkRetainingCapacity(unique); + + var nar_hash: [52]u8 = undefined; + nix32.encode(&nar_hash, in.nar_hash); + var file_hash: [52]u8 = undefined; + nix32.encode(&file_hash, in.file_hash); + + var out: std.ArrayList(u8) = .empty; + errdefer out.deinit(allocator); + const w = struct { + fn line(o: *std.ArrayList(u8), a: std.mem.Allocator, parts: []const []const u8) Error!void { + for (parts) |p| try o.appendSlice(a, p); + try o.append(a, '\n'); + } + }.line; + var num_buf: [2][20]u8 = undefined; + const nar_size = std.mem.print(&num_buf[0], "{d}", .{in.nar_size}) catch unreachable; + const file_size = std.mem.print(&num_buf[1], "{d}", .{in.file_size}) catch unreachable; + + try w(&out, allocator, &.{ "StorePath: ", in.store_path }); + try w(&out, allocator, &.{ "URL: nar/", &file_hash, ext }); + try w(&out, allocator, &.{ "Compression: ", in.compression }); + try w(&out, allocator, &.{ "FileHash: sha256:", &file_hash }); + try w(&out, allocator, &.{ "FileSize: ", file_size }); + try w(&out, allocator, &.{ "NarHash: sha256:", &nar_hash }); + try w(&out, allocator, &.{ "NarSize: ", nar_size }); + // Always "References: " with the space, even when empty: Nix's parser + // skips two bytes after the colon and would otherwise eat the newline. + try out.appendSlice(allocator, "References: "); + for (refs.items, 0..) |r, i| { + if (i > 0) try out.append(allocator, ' '); + try out.appendSlice(allocator, r); + } + try out.append(allocator, '\n'); + if (in.deriver.len > 0) try w(&out, allocator, &.{ "Deriver: ", in.deriver }); + if (in.system.len > 0) try w(&out, allocator, &.{ "System: ", in.system }); + + if (signer) |s| { + // 1;;sha256:;; + var fp: std.ArrayList(u8) = .empty; + defer fp.deinit(allocator); + try fp.appendSlice(allocator, "1;"); + try fp.appendSlice(allocator, in.store_path); + try fp.appendSlice(allocator, ";sha256:"); + try fp.appendSlice(allocator, &nar_hash); + try fp.append(allocator, ';'); + try fp.appendSlice(allocator, nar_size); + try fp.append(allocator, ';'); + for (refs.items, 0..) |r, i| { + if (i > 0) try fp.append(allocator, ','); + try fp.appendSlice(allocator, store_dir); + try fp.appendSlice(allocator, r); + } + try out.appendSlice(allocator, "Sig: "); + try s.sign(&out, allocator, fp.items); + try out.append(allocator, '\n'); + } + + return out.toOwnedSlice(allocator); +} + + +test "renders and signs a narinfo" { + const a = std.testing.allocator; + // Secret key from `nix key generate-secret --key-name test-1`. + const signer = try Signer.parse(a, "test-1:suJsWimvBNFUIc0JJE18OfOMygH/f2GuTC2/XwD6rPKkV+1VKilMIkXl6Ax9hqeKpUF/BSOH+Fqng4tqZlirhw=="); + defer signer.destroy(a); + const nar: [32]u8 = @splat(1); + const file: [32]u8 = @splat(2); + const text = try render(a, .{ + .store_path = "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello", + .nar_hash = &nar, + .nar_size = 1234, + .file_hash = &file, + .file_size = 99, + .compression = "zstd", + .references = "1mdqa9w1p6cmli6976v4wi0sw9r4p5pr-b 0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello", + .deriver = "", + .system = "x86_64-linux", + }, signer); + defer a.free(text); + try std.testing.expect(std.mem.find(u8, text, "References: 0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello 1mdqa9w1p6cmli6976v4wi0sw9r4p5pr-b\n") != null); + try std.testing.expect(std.mem.find(u8, text, "\nSig: test-1:") != null); + const empty = try render(a, .{ + .store_path = "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello", + .nar_hash = &nar, + .nar_size = 1, + .file_hash = &file, + .file_size = 1, + .compression = "zstd", + .references = "", + .deriver = "0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello.drv", + .system = "", + }, null); + defer a.free(empty); + try std.testing.expect(std.mem.find(u8, empty, "\nReferences: \nDeriver: ") != null); + try std.testing.expectError(error.InvalidSystem, render(a, .{ + .store_path = "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello", + .nar_hash = &nar, + .nar_size = 1, + .file_hash = &file, + .file_size = 1, + .compression = "zstd", + .references = "", + .deriver = "", + .system = "x86\nSig: evil", + }, null)); +} + +test "fast signing is byte-identical to std Ed25519" { + const a = std.testing.allocator; + const signer = try Signer.parse(a, "test-1:suJsWimvBNFUIc0JJE18OfOMygH/f2GuTC2/XwD6rPKkV+1VKilMIkXl6Ax9hqeKpUF/BSOH+Fqng4tqZlirhw=="); + defer signer.destroy(a); + var prng: std.Random.DefaultPrng = .init(3); + var msg: [300]u8 = undefined; + for (0..200) |i| { + prng.random().bytes(msg[0 .. i + 1]); + const ours = try signer.signRaw(msg[0 .. i + 1]); + const theirs = (try signer.key_pair.sign(msg[0 .. i + 1], null)).toBytes(); + try std.testing.expectEqualSlices(u8, &theirs, &ours); + } +} + +test "generated keys parse and sign" { + const a = std.testing.allocator; + var text: std.ArrayList(u8) = .empty; + defer text.deinit(a); + try Signer.generate(&text, a, "gen-1", @as([32]u8, @splat(42))); + const signer = try Signer.parse(a, text.items); + defer signer.destroy(a); + const theirs = (try signer.key_pair.sign("msg", null)).toBytes(); + try std.testing.expectEqualSlices(u8, &theirs, &(try signer.signRaw("msg"))); + try std.testing.expectError(error.InvalidKey, Signer.generate(&text, a, "bad:name", @as([32]u8, @splat(1)))); +} + +test "rejects a secret key whose public half does not match its seed" { + const a = std.testing.allocator; + const prefix = "test-1:"; + const good = prefix ++ "suJsWimvBNFUIc0JJE18OfOMygH/f2GuTC2/XwD6rPKkV+1VKilMIkXl6Ax9hqeKpUF/BSOH+Fqng4tqZlirhw=="; + var raw: [64]u8 = undefined; + try std.base64.standard.Decoder.decode(&raw, good[prefix.len..]); + // A valid point, but the public key of another seed. + raw[32..64].* = (try Ed25519.KeyPair.generateDeterministic(@as([32]u8, @splat(7)))).public_key.toBytes(); + var text: [good.len]u8 = undefined; + @memcpy(text[0..prefix.len], prefix); + _ = std.base64.standard.Encoder.encode(text[prefix.len..], &raw); + try std.testing.expectError(error.InvalidKey, Signer.parse(a, &text)); +} + +test "duplicate references are rendered and signed once" { + const a = std.testing.allocator; + const signer = try Signer.parse(a, "test-1:suJsWimvBNFUIc0JJE18OfOMygH/f2GuTC2/XwD6rPKkV+1VKilMIkXl6Ax9hqeKpUF/BSOH+Fqng4tqZlirhw=="); + defer signer.destroy(a); + const nar: [32]u8 = @splat(1); + const file: [32]u8 = @splat(2); + var in: Input = .{ + .store_path = "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello", + .nar_hash = &nar, + .nar_size = 1234, + .file_hash = &file, + .file_size = 99, + .compression = "zstd", + .references = "1mdqa9w1p6cmli6976v4wi0sw9r4p5pr-b 0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello 1mdqa9w1p6cmli6976v4wi0sw9r4p5pr-b", + .deriver = "", + .system = "", + }; + const dup = try render(a, in, signer); + defer a.free(dup); + in.references = "0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello 1mdqa9w1p6cmli6976v4wi0sw9r4p5pr-b"; + const once = try render(a, in, signer); + defer a.free(once); + try std.testing.expectEqualStrings(once, dup); +} diff --git a/pkg/nix-narinfo/src/root.zig b/pkg/nix-narinfo/src/root.zig new file mode 100644 index 0000000..a5df7fb --- /dev/null +++ b/pkg/nix-narinfo/src/root.zig @@ -0,0 +1,19 @@ +//! Narinfo: parsing, validated rendering, and fingerprint signing. + +const render_mod = @import("render.zig"); +const parse_mod = @import("parse.zig"); + +pub const Signer = render_mod.Signer; +pub const Input = render_mod.Input; +pub const RenderError = render_mod.Error; +pub const render = render_mod.render; + +pub const NarInfo = parse_mod.NarInfo; +pub const ParseError = parse_mod.Error; +pub const parse = parse_mod.parse; + +test { + _ = render_mod; + _ = parse_mod; + _ = @import("basemul.zig"); +} diff --git a/pkg/nix-store-path/build.zig b/pkg/nix-store-path/build.zig new file mode 100644 index 0000000..eaa2a57 --- /dev/null +++ b/pkg/nix-store-path/build.zig @@ -0,0 +1,19 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mod = b.addModule("nix-store-path", .{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, + }); + mod.addImport("nix-base32", b.dependency("nix_base32", .{ .target = target, .optimize = optimize }).module("nix-base32")); + + const tests = b.addTest(.{ .root_module = mod }); + b.step("test", "Run unit tests").dependOn(&b.addRunArtifact(tests).step); +} diff --git a/pkg/nix-store-path/build.zig.zon b/pkg/nix-store-path/build.zig.zon new file mode 100644 index 0000000..3508b82 --- /dev/null +++ b/pkg/nix-store-path/build.zig.zon @@ -0,0 +1,10 @@ +.{ + .name = .nix_store_path, + .version = "0.1.0", + .fingerprint = 0xdcfce087c7a4494d, + .minimum_zig_version = "0.17.0", + .dependencies = .{ + .nix_base32 = .{ .path = "../nix-base32" }, + }, + .paths = .{ "build.zig", "build.zig.zon", "src" }, +} diff --git a/pkg/nix-store-path/src/root.zig b/pkg/nix-store-path/src/root.zig new file mode 100644 index 0000000..e73c8a4 --- /dev/null +++ b/pkg/nix-store-path/src/root.zig @@ -0,0 +1,98 @@ +//! Store path parsing and validation, matching what Nix's StorePath accepts: +//! `/nix/store/<32 nix32 chars>-`, name 1-211 chars of +//! `[A-Za-z0-9+-._?=]`, not starting with '.'. +//! +//! The name check is vectorised: 32 characters are classified per step. + +const std = @import("std"); +const base32 = @import("nix-base32"); + +pub const store_dir = "/nix/store/"; +pub const hash_len = 32; +pub const max_name_len = 211; + +const lanes = 32; +const V = @Vector(lanes, u8); +const Mask = @Int(.unsigned, lanes); + +fn splat(c: u8) V { + return @splat(c); +} + +fn bits(b: @Vector(lanes, bool)) Mask { + return @bitCast(b); +} + +inline fn invalidNameLanes(v: V) Mask { + const lower = v | splat(0x20); // folds A-Z onto a-z; no other byte lands in a-z from outside letters + const alpha = bits(lower >= splat('a')) & bits(lower <= splat('z')); + const digit = bits(v >= splat('0')) & bits(v <= splat('9')); + const special = bits(v == splat('+')) | bits(v == splat('-')) | bits(v == splat('.')) | + bits(v == splat('_')) | bits(v == splat('?')) | bits(v == splat('=')); + return ~(alpha | digit | special); +} + +fn isNameChar(c: u8) bool { + return std.ascii.isAlphanumeric(c) or switch (c) { + '+', '-', '.', '_', '?', '=' => true, + else => false, + }; +} + +// Out of line: Zig 0.17 inlines this vector loop into callers such as +// baseName, which made store path parsing about 7% slower than a call. +pub noinline fn isValidName(name: []const u8) bool { + if (name.len == 0 or name.len > max_name_len or name[0] == '.') return false; + var i: usize = 0; + while (i + lanes <= name.len) : (i += lanes) { + if (invalidNameLanes(name[i..][0..lanes].*) != 0) return false; + } + for (name[i..]) |c| if (!isNameChar(c)) return false; + return true; +} + +pub fn isValidNameScalar(name: []const u8) bool { + if (name.len == 0 or name.len > max_name_len or name[0] == '.') return false; + for (name) |c| if (!isNameChar(c)) return false; + return true; +} + +/// `<32 nix32 chars>-`. +pub fn isValidBaseName(base: []const u8) bool { + if (base.len < hash_len + 2) return false; + return base[hash_len] == '-' and base32.isValid(base[0..hash_len]) and isValidName(base[hash_len + 1 ..]); +} + +/// The basename of a full store path, or null if it is not a valid one. +pub fn baseName(store_path: []const u8) ?[]const u8 { + if (!std.mem.startsWith(u8, store_path, store_dir)) return null; + const base = store_path[store_dir.len..]; + return if (isValidBaseName(base)) base else null; +} + +/// The decoded 20-byte hash of a valid basename. +pub fn hashOf(base: []const u8) ?[20]u8 { + if (base.len < hash_len) return null; + var out: [20]u8 = undefined; + base32.decode(&out, base[0..hash_len]) catch return null; + return out; +} + +test "validates base names" { + try std.testing.expect(isValidBaseName("0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello-2.12")); + try std.testing.expect(!isValidBaseName("0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello\nSig: x")); + try std.testing.expect(!isValidBaseName("0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-.hidden")); + try std.testing.expect(!isValidBaseName("emdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello")); + try std.testing.expect(baseName("/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello") != null); + try std.testing.expect(baseName("/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello/bin") == null); +} + +test "vector name check agrees with scalar for every byte in every lane" { + for (0..256) |c| { + for (0..lanes + 3) |pos| { + var name: [lanes + 3]u8 = @splat('a'); + name[pos] = @intCast(c); + try std.testing.expectEqual(isValidNameScalar(&name), isValidName(&name)); + } + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml deleted file mode 100644 index ed212ed..0000000 --- a/pnpm-lock.yaml +++ /dev/null @@ -1,2723 +0,0 @@ -lockfileVersion: '9.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - -importers: - - .: - devDependencies: - wrangler: - specifier: ^4.112.0 - version: 4.112.0 - - apps/cli: - dependencies: - '@helios/cache-domain': - specifier: workspace:* - version: link:../../packages/cache-domain - devDependencies: - '@types/node': - specifier: ^26.1.1 - version: 26.1.1 - typescript: - specifier: ^7.0.2 - version: 7.0.2 - - packages/cache-domain: - devDependencies: - typescript: - specifier: ^7.0.2 - version: 7.0.2 - vitest: - specifier: ~4.1.10 - version: 4.1.10(@types/node@26.1.1)(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0)) - - workers/cache: - dependencies: - '@helios/cache-domain': - specifier: workspace:* - version: link:../../packages/cache-domain - aws4fetch: - specifier: ^1.0.20 - version: 1.0.20 - drizzle-orm: - specifier: ^0.45.2 - version: 0.45.2 - devDependencies: - '@cloudflare/vitest-pool-workers': - specifier: ^0.18.6 - version: 0.18.6(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10(@types/node@26.1.1)(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0))) - '@types/node': - specifier: ^26.1.1 - version: 26.1.1 - drizzle-kit: - specifier: ^0.31.10 - version: 0.31.10 - typescript: - specifier: ^7.0.2 - version: 7.0.2 - vitest: - specifier: ~4.1.10 - version: 4.1.10(@types/node@26.1.1)(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0)) - wrangler: - specifier: ^4.112.0 - version: 4.112.0 - -packages: - - '@cloudflare/kv-asset-handler@0.5.0': - resolution: {integrity: sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==} - engines: {node: '>=22.0.0'} - - '@cloudflare/unenv-preset@2.16.1': - resolution: {integrity: sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==} - peerDependencies: - unenv: 2.0.0-rc.24 - workerd: '>1.20260305.0 <2.0.0-0' - peerDependenciesMeta: - workerd: - optional: true - - '@cloudflare/vitest-pool-workers@0.18.6': - resolution: {integrity: sha512-6JGqaQsQRZIVq/6jEC4ouJnShZriPIJ2X0yGndwMm+SiPP93pJi5Dp30dYAoztNrNJC7wWK7ec5slLfMBMZ8jA==} - peerDependencies: - '@vitest/runner': ^4.1.0 - '@vitest/snapshot': ^4.1.0 - vitest: ^4.1.0 - - '@cloudflare/workerd-darwin-64@1.20260714.1': - resolution: {integrity: sha512-ZWXqAN8G7Cx9hMRQuk+59ziJhR3j1F4iO+Qs8aHdfKZ3Dq5Yi/57xvkJTgCGBnW1YU/L78r8f6HEy51bwbTpNw==} - engines: {node: '>=16'} - cpu: [x64] - os: [darwin] - - '@cloudflare/workerd-darwin-arm64@1.20260714.1': - resolution: {integrity: sha512-tueWxWC3wyCbMG6zRAxsMXX0YLgrRWbiAPYFQ2uJ7dUH8G+5E7UTWaQS9B1HdJ0bpKFW1NWxhs1o2noKVFSUYg==} - engines: {node: '>=16'} - cpu: [arm64] - os: [darwin] - - '@cloudflare/workerd-linux-64@1.20260714.1': - resolution: {integrity: sha512-1VChTZRb0l0F7R4e1G5RtLKV4oFi6x+rQgxh2+yu887j3l/3TLgatuv1L8/5zhc9gKEhATTxOh0e52Rtd9dDWQ==} - engines: {node: '>=16'} - cpu: [x64] - os: [linux] - - '@cloudflare/workerd-linux-arm64@1.20260714.1': - resolution: {integrity: sha512-rMm3G+NirG2UdgHIRDdF1asNC6FqgIzZzkRG+VDhhDGcVxAQwvrMT1E38BivEvHr3G04MB4AfhcOczX0+GtRkQ==} - engines: {node: '>=16'} - cpu: [arm64] - os: [linux] - - '@cloudflare/workerd-windows-64@1.20260714.1': - resolution: {integrity: sha512-cGqnU3Hg2YZS/k3SAqrMp1DjpdsyFde72tWltdl6ZT9+SFz/Zrk/8gyTU1TcxC4YApXeNVH5TyU5cOGPgUJ0pg==} - engines: {node: '>=16'} - cpu: [x64] - os: [win32] - - '@cspotcode/source-map-support@0.8.1': - resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} - engines: {node: '>=12'} - - '@drizzle-team/brocli@0.10.2': - resolution: {integrity: sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w==} - - '@emnapi/runtime@1.10.0': - resolution: {integrity: sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==} - - '@esbuild-kit/core-utils@3.3.2': - resolution: {integrity: sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==} - deprecated: 'Merged into tsx: https://tsx.hirok.io' - - '@esbuild-kit/esm-loader@2.6.5': - resolution: {integrity: sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==} - deprecated: 'Merged into tsx: https://tsx.hirok.io' - - '@esbuild/aix-ppc64@0.25.12': - resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [aix] - - '@esbuild/aix-ppc64@0.27.7': - resolution: {integrity: sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [aix] - - '@esbuild/aix-ppc64@0.28.1': - resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [aix] - - '@esbuild/android-arm64@0.18.20': - resolution: {integrity: sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ==} - engines: {node: '>=12'} - cpu: [arm64] - os: [android] - - '@esbuild/android-arm64@0.25.12': - resolution: {integrity: sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [android] - - '@esbuild/android-arm64@0.27.7': - resolution: {integrity: sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==} - engines: {node: '>=18'} - cpu: [arm64] - os: [android] - - '@esbuild/android-arm64@0.28.1': - resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [android] - - '@esbuild/android-arm@0.18.20': - resolution: {integrity: sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw==} - engines: {node: '>=12'} - cpu: [arm] - os: [android] - - '@esbuild/android-arm@0.25.12': - resolution: {integrity: sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==} - engines: {node: '>=18'} - cpu: [arm] - os: [android] - - '@esbuild/android-arm@0.27.7': - resolution: {integrity: sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==} - engines: {node: '>=18'} - cpu: [arm] - os: [android] - - '@esbuild/android-arm@0.28.1': - resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} - engines: {node: '>=18'} - cpu: [arm] - os: [android] - - '@esbuild/android-x64@0.18.20': - resolution: {integrity: sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg==} - engines: {node: '>=12'} - cpu: [x64] - os: [android] - - '@esbuild/android-x64@0.25.12': - resolution: {integrity: sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==} - engines: {node: '>=18'} - cpu: [x64] - os: [android] - - '@esbuild/android-x64@0.27.7': - resolution: {integrity: sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==} - engines: {node: '>=18'} - cpu: [x64] - os: [android] - - '@esbuild/android-x64@0.28.1': - resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} - engines: {node: '>=18'} - cpu: [x64] - os: [android] - - '@esbuild/darwin-arm64@0.18.20': - resolution: {integrity: sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA==} - engines: {node: '>=12'} - cpu: [arm64] - os: [darwin] - - '@esbuild/darwin-arm64@0.25.12': - resolution: {integrity: sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [darwin] - - '@esbuild/darwin-arm64@0.27.7': - resolution: {integrity: sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [darwin] - - '@esbuild/darwin-arm64@0.28.1': - resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} - engines: {node: '>=18'} - cpu: [arm64] - os: [darwin] - - '@esbuild/darwin-x64@0.18.20': - resolution: {integrity: sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [darwin] - - '@esbuild/darwin-x64@0.25.12': - resolution: {integrity: sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==} - engines: {node: '>=18'} - cpu: [x64] - os: [darwin] - - '@esbuild/darwin-x64@0.27.7': - resolution: {integrity: sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [darwin] - - '@esbuild/darwin-x64@0.28.1': - resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [darwin] - - '@esbuild/freebsd-arm64@0.18.20': - resolution: {integrity: sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw==} - engines: {node: '>=12'} - cpu: [arm64] - os: [freebsd] - - '@esbuild/freebsd-arm64@0.25.12': - resolution: {integrity: sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [freebsd] - - '@esbuild/freebsd-arm64@0.27.7': - resolution: {integrity: sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==} - engines: {node: '>=18'} - cpu: [arm64] - os: [freebsd] - - '@esbuild/freebsd-arm64@0.28.1': - resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [freebsd] - - '@esbuild/freebsd-x64@0.18.20': - resolution: {integrity: sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [freebsd] - - '@esbuild/freebsd-x64@0.25.12': - resolution: {integrity: sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [freebsd] - - '@esbuild/freebsd-x64@0.27.7': - resolution: {integrity: sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [freebsd] - - '@esbuild/freebsd-x64@0.28.1': - resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [freebsd] - - '@esbuild/linux-arm64@0.18.20': - resolution: {integrity: sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA==} - engines: {node: '>=12'} - cpu: [arm64] - os: [linux] - - '@esbuild/linux-arm64@0.25.12': - resolution: {integrity: sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==} - engines: {node: '>=18'} - cpu: [arm64] - os: [linux] - - '@esbuild/linux-arm64@0.27.7': - resolution: {integrity: sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==} - engines: {node: '>=18'} - cpu: [arm64] - os: [linux] - - '@esbuild/linux-arm64@0.28.1': - resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} - engines: {node: '>=18'} - cpu: [arm64] - os: [linux] - - '@esbuild/linux-arm@0.18.20': - resolution: {integrity: sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg==} - engines: {node: '>=12'} - cpu: [arm] - os: [linux] - - '@esbuild/linux-arm@0.25.12': - resolution: {integrity: sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==} - engines: {node: '>=18'} - cpu: [arm] - os: [linux] - - '@esbuild/linux-arm@0.27.7': - resolution: {integrity: sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==} - engines: {node: '>=18'} - cpu: [arm] - os: [linux] - - '@esbuild/linux-arm@0.28.1': - resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} - engines: {node: '>=18'} - cpu: [arm] - os: [linux] - - '@esbuild/linux-ia32@0.18.20': - resolution: {integrity: sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA==} - engines: {node: '>=12'} - cpu: [ia32] - os: [linux] - - '@esbuild/linux-ia32@0.25.12': - resolution: {integrity: sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==} - engines: {node: '>=18'} - cpu: [ia32] - os: [linux] - - '@esbuild/linux-ia32@0.27.7': - resolution: {integrity: sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==} - engines: {node: '>=18'} - cpu: [ia32] - os: [linux] - - '@esbuild/linux-ia32@0.28.1': - resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} - engines: {node: '>=18'} - cpu: [ia32] - os: [linux] - - '@esbuild/linux-loong64@0.18.20': - resolution: {integrity: sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg==} - engines: {node: '>=12'} - cpu: [loong64] - os: [linux] - - '@esbuild/linux-loong64@0.25.12': - resolution: {integrity: sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==} - engines: {node: '>=18'} - cpu: [loong64] - os: [linux] - - '@esbuild/linux-loong64@0.27.7': - resolution: {integrity: sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==} - engines: {node: '>=18'} - cpu: [loong64] - os: [linux] - - '@esbuild/linux-loong64@0.28.1': - resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} - engines: {node: '>=18'} - cpu: [loong64] - os: [linux] - - '@esbuild/linux-mips64el@0.18.20': - resolution: {integrity: sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ==} - engines: {node: '>=12'} - cpu: [mips64el] - os: [linux] - - '@esbuild/linux-mips64el@0.25.12': - resolution: {integrity: sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==} - engines: {node: '>=18'} - cpu: [mips64el] - os: [linux] - - '@esbuild/linux-mips64el@0.27.7': - resolution: {integrity: sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==} - engines: {node: '>=18'} - cpu: [mips64el] - os: [linux] - - '@esbuild/linux-mips64el@0.28.1': - resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} - engines: {node: '>=18'} - cpu: [mips64el] - os: [linux] - - '@esbuild/linux-ppc64@0.18.20': - resolution: {integrity: sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA==} - engines: {node: '>=12'} - cpu: [ppc64] - os: [linux] - - '@esbuild/linux-ppc64@0.25.12': - resolution: {integrity: sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [linux] - - '@esbuild/linux-ppc64@0.27.7': - resolution: {integrity: sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [linux] - - '@esbuild/linux-ppc64@0.28.1': - resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [linux] - - '@esbuild/linux-riscv64@0.18.20': - resolution: {integrity: sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A==} - engines: {node: '>=12'} - cpu: [riscv64] - os: [linux] - - '@esbuild/linux-riscv64@0.25.12': - resolution: {integrity: sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==} - engines: {node: '>=18'} - cpu: [riscv64] - os: [linux] - - '@esbuild/linux-riscv64@0.27.7': - resolution: {integrity: sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==} - engines: {node: '>=18'} - cpu: [riscv64] - os: [linux] - - '@esbuild/linux-riscv64@0.28.1': - resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} - engines: {node: '>=18'} - cpu: [riscv64] - os: [linux] - - '@esbuild/linux-s390x@0.18.20': - resolution: {integrity: sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ==} - engines: {node: '>=12'} - cpu: [s390x] - os: [linux] - - '@esbuild/linux-s390x@0.25.12': - resolution: {integrity: sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==} - engines: {node: '>=18'} - cpu: [s390x] - os: [linux] - - '@esbuild/linux-s390x@0.27.7': - resolution: {integrity: sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==} - engines: {node: '>=18'} - cpu: [s390x] - os: [linux] - - '@esbuild/linux-s390x@0.28.1': - resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} - engines: {node: '>=18'} - cpu: [s390x] - os: [linux] - - '@esbuild/linux-x64@0.18.20': - resolution: {integrity: sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w==} - engines: {node: '>=12'} - cpu: [x64] - os: [linux] - - '@esbuild/linux-x64@0.25.12': - resolution: {integrity: sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==} - engines: {node: '>=18'} - cpu: [x64] - os: [linux] - - '@esbuild/linux-x64@0.27.7': - resolution: {integrity: sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==} - engines: {node: '>=18'} - cpu: [x64] - os: [linux] - - '@esbuild/linux-x64@0.28.1': - resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} - engines: {node: '>=18'} - cpu: [x64] - os: [linux] - - '@esbuild/netbsd-arm64@0.25.12': - resolution: {integrity: sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [netbsd] - - '@esbuild/netbsd-arm64@0.27.7': - resolution: {integrity: sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==} - engines: {node: '>=18'} - cpu: [arm64] - os: [netbsd] - - '@esbuild/netbsd-arm64@0.28.1': - resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [netbsd] - - '@esbuild/netbsd-x64@0.18.20': - resolution: {integrity: sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A==} - engines: {node: '>=12'} - cpu: [x64] - os: [netbsd] - - '@esbuild/netbsd-x64@0.25.12': - resolution: {integrity: sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [netbsd] - - '@esbuild/netbsd-x64@0.27.7': - resolution: {integrity: sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==} - engines: {node: '>=18'} - cpu: [x64] - os: [netbsd] - - '@esbuild/netbsd-x64@0.28.1': - resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} - engines: {node: '>=18'} - cpu: [x64] - os: [netbsd] - - '@esbuild/openbsd-arm64@0.25.12': - resolution: {integrity: sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openbsd] - - '@esbuild/openbsd-arm64@0.27.7': - resolution: {integrity: sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openbsd] - - '@esbuild/openbsd-arm64@0.28.1': - resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openbsd] - - '@esbuild/openbsd-x64@0.18.20': - resolution: {integrity: sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg==} - engines: {node: '>=12'} - cpu: [x64] - os: [openbsd] - - '@esbuild/openbsd-x64@0.25.12': - resolution: {integrity: sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==} - engines: {node: '>=18'} - cpu: [x64] - os: [openbsd] - - '@esbuild/openbsd-x64@0.27.7': - resolution: {integrity: sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==} - engines: {node: '>=18'} - cpu: [x64] - os: [openbsd] - - '@esbuild/openbsd-x64@0.28.1': - resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} - engines: {node: '>=18'} - cpu: [x64] - os: [openbsd] - - '@esbuild/openharmony-arm64@0.25.12': - resolution: {integrity: sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openharmony] - - '@esbuild/openharmony-arm64@0.27.7': - resolution: {integrity: sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openharmony] - - '@esbuild/openharmony-arm64@0.28.1': - resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openharmony] - - '@esbuild/sunos-x64@0.18.20': - resolution: {integrity: sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [sunos] - - '@esbuild/sunos-x64@0.25.12': - resolution: {integrity: sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==} - engines: {node: '>=18'} - cpu: [x64] - os: [sunos] - - '@esbuild/sunos-x64@0.27.7': - resolution: {integrity: sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==} - engines: {node: '>=18'} - cpu: [x64] - os: [sunos] - - '@esbuild/sunos-x64@0.28.1': - resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [sunos] - - '@esbuild/win32-arm64@0.18.20': - resolution: {integrity: sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg==} - engines: {node: '>=12'} - cpu: [arm64] - os: [win32] - - '@esbuild/win32-arm64@0.25.12': - resolution: {integrity: sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [win32] - - '@esbuild/win32-arm64@0.27.7': - resolution: {integrity: sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==} - engines: {node: '>=18'} - cpu: [arm64] - os: [win32] - - '@esbuild/win32-arm64@0.28.1': - resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} - engines: {node: '>=18'} - cpu: [arm64] - os: [win32] - - '@esbuild/win32-ia32@0.18.20': - resolution: {integrity: sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g==} - engines: {node: '>=12'} - cpu: [ia32] - os: [win32] - - '@esbuild/win32-ia32@0.25.12': - resolution: {integrity: sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==} - engines: {node: '>=18'} - cpu: [ia32] - os: [win32] - - '@esbuild/win32-ia32@0.27.7': - resolution: {integrity: sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==} - engines: {node: '>=18'} - cpu: [ia32] - os: [win32] - - '@esbuild/win32-ia32@0.28.1': - resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} - engines: {node: '>=18'} - cpu: [ia32] - os: [win32] - - '@esbuild/win32-x64@0.18.20': - resolution: {integrity: sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [win32] - - '@esbuild/win32-x64@0.25.12': - resolution: {integrity: sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==} - engines: {node: '>=18'} - cpu: [x64] - os: [win32] - - '@esbuild/win32-x64@0.27.7': - resolution: {integrity: sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==} - engines: {node: '>=18'} - cpu: [x64] - os: [win32] - - '@esbuild/win32-x64@0.28.1': - resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} - engines: {node: '>=18'} - cpu: [x64] - os: [win32] - - '@img/colour@1.1.0': - resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} - engines: {node: '>=18'} - - '@img/sharp-darwin-arm64@0.34.5': - resolution: {integrity: sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [darwin] - - '@img/sharp-darwin-x64@0.34.5': - resolution: {integrity: sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [darwin] - - '@img/sharp-libvips-darwin-arm64@1.2.4': - resolution: {integrity: sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==} - cpu: [arm64] - os: [darwin] - - '@img/sharp-libvips-darwin-x64@1.2.4': - resolution: {integrity: sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==} - cpu: [x64] - os: [darwin] - - '@img/sharp-libvips-linux-arm64@1.2.4': - resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} - cpu: [arm64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-arm@1.2.4': - resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} - cpu: [arm] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-ppc64@1.2.4': - resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} - cpu: [ppc64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-riscv64@1.2.4': - resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} - cpu: [riscv64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-s390x@1.2.4': - resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} - cpu: [s390x] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-x64@1.2.4': - resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linuxmusl-arm64@1.2.4': - resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@img/sharp-libvips-linuxmusl-x64@1.2.4': - resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} - cpu: [x64] - os: [linux] - libc: [musl] - - '@img/sharp-linux-arm64@0.34.5': - resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-arm@0.34.5': - resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-ppc64@0.34.5': - resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [ppc64] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-riscv64@0.34.5': - resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [riscv64] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-s390x@0.34.5': - resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [s390x] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-x64@0.34.5': - resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@img/sharp-linuxmusl-arm64@0.34.5': - resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@img/sharp-linuxmusl-x64@0.34.5': - resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [linux] - libc: [musl] - - '@img/sharp-wasm32@0.34.5': - resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [wasm32] - - '@img/sharp-win32-arm64@0.34.5': - resolution: {integrity: sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [win32] - - '@img/sharp-win32-ia32@0.34.5': - resolution: {integrity: sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [ia32] - os: [win32] - - '@img/sharp-win32-x64@0.34.5': - resolution: {integrity: sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [win32] - - '@jridgewell/resolve-uri@3.1.2': - resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} - engines: {node: '>=6.0.0'} - - '@jridgewell/sourcemap-codec@1.5.5': - resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} - - '@jridgewell/trace-mapping@0.3.9': - resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} - - '@poppinss/colors@4.1.6': - resolution: {integrity: sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==} - - '@poppinss/dumper@0.6.5': - resolution: {integrity: sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==} - - '@poppinss/exception@1.2.3': - resolution: {integrity: sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==} - - '@rollup/rollup-android-arm-eabi@4.60.2': - resolution: {integrity: sha512-dnlp69efPPg6Uaw2dVqzWRfAWRnYVb1XJ8CyyhIbZeaq4CA5/mLeZ1IEt9QqQxmbdvagjLIm2ZL8BxXv5lH4Yw==} - cpu: [arm] - os: [android] - - '@rollup/rollup-android-arm64@4.60.2': - resolution: {integrity: sha512-OqZTwDRDchGRHHm/hwLOL7uVPB9aUvI0am/eQuWMNyFHf5PSEQmyEeYYheA0EPPKUO/l0uigCp+iaTjoLjVoHg==} - cpu: [arm64] - os: [android] - - '@rollup/rollup-darwin-arm64@4.60.2': - resolution: {integrity: sha512-UwRE7CGpvSVEQS8gUMBe1uADWjNnVgP3Iusyda1nSRwNDCsRjnGc7w6El6WLQsXmZTbLZx9cecegumcitNfpmA==} - cpu: [arm64] - os: [darwin] - - '@rollup/rollup-darwin-x64@4.60.2': - resolution: {integrity: sha512-gjEtURKLCC5VXm1I+2i1u9OhxFsKAQJKTVB8WvDAHF+oZlq0GTVFOlTlO1q3AlCTE/DF32c16ESvfgqR7343/g==} - cpu: [x64] - os: [darwin] - - '@rollup/rollup-freebsd-arm64@4.60.2': - resolution: {integrity: sha512-Bcl6CYDeAgE70cqZaMojOi/eK63h5Me97ZqAQoh77VPjMysA/4ORQBRGo3rRy45x4MzVlU9uZxs8Uwy7ZaKnBw==} - cpu: [arm64] - os: [freebsd] - - '@rollup/rollup-freebsd-x64@4.60.2': - resolution: {integrity: sha512-LU+TPda3mAE2QB0/Hp5VyeKJivpC6+tlOXd1VMoXV/YFMvk/MNk5iXeBfB4MQGRWyOYVJ01625vjkr0Az98OJQ==} - cpu: [x64] - os: [freebsd] - - '@rollup/rollup-linux-arm-gnueabihf@4.60.2': - resolution: {integrity: sha512-2QxQrM+KQ7DAW4o22j+XZ6RKdxjLD7BOWTP0Bv0tmjdyhXSsr2Ul1oJDQqh9Zf5qOwTuTc7Ek83mOFaKnodPjg==} - cpu: [arm] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-arm-musleabihf@4.60.2': - resolution: {integrity: sha512-TbziEu2DVsTEOPif2mKWkMeDMLoYjx95oESa9fkQQK7r/Orta0gnkcDpzwufEcAO2BLBsD7mZkXGFqEdMRRwfw==} - cpu: [arm] - os: [linux] - libc: [musl] - - '@rollup/rollup-linux-arm64-gnu@4.60.2': - resolution: {integrity: sha512-bO/rVDiDUuM2YfuCUwZ1t1cP+/yqjqz+Xf2VtkdppefuOFS2OSeAfgafaHNkFn0t02hEyXngZkxtGqXcXwO8Rg==} - cpu: [arm64] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-arm64-musl@4.60.2': - resolution: {integrity: sha512-hr26p7e93Rl0Za+JwW7EAnwAvKkehh12BU1Llm9Ykiibg4uIr2rbpxG9WCf56GuvidlTG9KiiQT/TXT1yAWxTA==} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@rollup/rollup-linux-loong64-gnu@4.60.2': - resolution: {integrity: sha512-pOjB/uSIyDt+ow3k/RcLvUAOGpysT2phDn7TTUB3n75SlIgZzM6NKAqlErPhoFU+npgY3/n+2HYIQVbF70P9/A==} - cpu: [loong64] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-loong64-musl@4.60.2': - resolution: {integrity: sha512-2/w+q8jszv9Ww1c+6uJT3OwqhdmGP2/4T17cu8WuwyUuuaCDDJ2ojdyYwZzCxx0GcsZBhzi3HmH+J5pZNXnd+Q==} - cpu: [loong64] - os: [linux] - libc: [musl] - - '@rollup/rollup-linux-ppc64-gnu@4.60.2': - resolution: {integrity: sha512-11+aL5vKheYgczxtPVVRhdptAM2H7fcDR5Gw4/bTcteuZBlH4oP9f5s9zYO9aGZvoGeBpqXI/9TZZihZ609wKw==} - cpu: [ppc64] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-ppc64-musl@4.60.2': - resolution: {integrity: sha512-i16fokAGK46IVZuV8LIIwMdtqhin9hfYkCh8pf8iC3QU3LpwL+1FSFGej+O7l3E/AoknL6Dclh2oTdnRMpTzFQ==} - cpu: [ppc64] - os: [linux] - libc: [musl] - - '@rollup/rollup-linux-riscv64-gnu@4.60.2': - resolution: {integrity: sha512-49FkKS6RGQoriDSK/6E2GkAsAuU5kETFCh7pG4yD/ylj9rKhTmO3elsnmBvRD4PgJPds5W2PkhC82aVwmUcJ7A==} - cpu: [riscv64] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-riscv64-musl@4.60.2': - resolution: {integrity: sha512-mjYNkHPfGpUR00DuM1ZZIgs64Hpf4bWcz9Z41+4Q+pgDx73UwWdAYyf6EG/lRFldmdHHzgrYyge5akFUW0D3mQ==} - cpu: [riscv64] - os: [linux] - libc: [musl] - - '@rollup/rollup-linux-s390x-gnu@4.60.2': - resolution: {integrity: sha512-ALyvJz965BQk8E9Al/JDKKDLH2kfKFLTGMlgkAbbYtZuJt9LU8DW3ZoDMCtQpXAltZxwBHevXz5u+gf0yA0YoA==} - cpu: [s390x] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-x64-gnu@4.60.2': - resolution: {integrity: sha512-UQjrkIdWrKI626Du8lCQ6MJp/6V1LAo2bOK9OTu4mSn8GGXIkPXk/Vsp4bLHCd9Z9Iz2OTEaokUE90VweJgIYQ==} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-x64-musl@4.60.2': - resolution: {integrity: sha512-bTsRGj6VlSdn/XD4CGyzMnzaBs9bsRxy79eTqTCBsA8TMIEky7qg48aPkvJvFe1HyzQ5oMZdg7AnVlWQSKLTnw==} - cpu: [x64] - os: [linux] - libc: [musl] - - '@rollup/rollup-openbsd-x64@4.60.2': - resolution: {integrity: sha512-6d4Z3534xitaA1FcMWP7mQPq5zGwBmGbhphh2DwaA1aNIXUu3KTOfwrWpbwI4/Gr0uANo7NTtaykFyO2hPuFLg==} - cpu: [x64] - os: [openbsd] - - '@rollup/rollup-openharmony-arm64@4.60.2': - resolution: {integrity: sha512-NetAg5iO2uN7eB8zE5qrZ3CSil+7IJt4WDFLcC75Ymywq1VZVD6qJ6EvNLjZ3rEm6gB7XW5JdT60c6MN35Z85Q==} - cpu: [arm64] - os: [openharmony] - - '@rollup/rollup-win32-arm64-msvc@4.60.2': - resolution: {integrity: sha512-NCYhOotpgWZ5kdxCZsv6Iudx0wX8980Q/oW4pNFNihpBKsDbEA1zpkfxJGC0yugsUuyDZ7gL37dbzwhR0VI7pQ==} - cpu: [arm64] - os: [win32] - - '@rollup/rollup-win32-ia32-msvc@4.60.2': - resolution: {integrity: sha512-RXsaOqXxfoUBQoOgvmmijVxJnW2IGB0eoMO7F8FAjaj0UTywUO/luSqimWBJn04WNgUkeNhh7fs7pESXajWmkg==} - cpu: [ia32] - os: [win32] - - '@rollup/rollup-win32-x64-gnu@4.60.2': - resolution: {integrity: sha512-qdAzEULD+/hzObedtmV6iBpdL5TIbKVztGiK7O3/KYSf+HIzU257+MX1EXJcyIiDbMAqmbwaufcYPvyRryeZtA==} - cpu: [x64] - os: [win32] - - '@rollup/rollup-win32-x64-msvc@4.60.2': - resolution: {integrity: sha512-Nd/SgG27WoA9e+/TdK74KnHz852TLa94ovOYySo/yMPuTmpckK/jIF2jSwS3g7ELSKXK13/cVdmg1Z/DaCWKxA==} - cpu: [x64] - os: [win32] - - '@sindresorhus/is@7.2.0': - resolution: {integrity: sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==} - engines: {node: '>=18'} - - '@speed-highlight/core@1.2.15': - resolution: {integrity: sha512-BMq1K3DsElxDWawkX6eLg9+CKJrTVGCBAWVuHXVUV2u0s2711qiChLSId6ikYPfxhdYocLNt3wWwSvDiTvFabw==} - - '@standard-schema/spec@1.1.0': - resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} - - '@types/chai@5.2.3': - resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} - - '@types/deep-eql@4.0.2': - resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} - - '@types/estree@1.0.8': - resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} - - '@types/node@26.1.1': - resolution: {integrity: sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==} - - '@typescript/typescript-aix-ppc64@7.0.2': - resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} - engines: {node: '>=16.20.0'} - cpu: [ppc64] - os: [aix] - - '@typescript/typescript-darwin-arm64@7.0.2': - resolution: {integrity: sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==} - engines: {node: '>=16.20.0'} - cpu: [arm64] - os: [darwin] - - '@typescript/typescript-darwin-x64@7.0.2': - resolution: {integrity: sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==} - engines: {node: '>=16.20.0'} - cpu: [x64] - os: [darwin] - - '@typescript/typescript-freebsd-arm64@7.0.2': - resolution: {integrity: sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==} - engines: {node: '>=16.20.0'} - cpu: [arm64] - os: [freebsd] - - '@typescript/typescript-freebsd-x64@7.0.2': - resolution: {integrity: sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==} - engines: {node: '>=16.20.0'} - cpu: [x64] - os: [freebsd] - - '@typescript/typescript-linux-arm64@7.0.2': - resolution: {integrity: sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==} - engines: {node: '>=16.20.0'} - cpu: [arm64] - os: [linux] - - '@typescript/typescript-linux-arm@7.0.2': - resolution: {integrity: sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==} - engines: {node: '>=16.20.0'} - cpu: [arm] - os: [linux] - - '@typescript/typescript-linux-loong64@7.0.2': - resolution: {integrity: sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==} - engines: {node: '>=16.20.0'} - cpu: [loong64] - os: [linux] - - '@typescript/typescript-linux-mips64el@7.0.2': - resolution: {integrity: sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==} - engines: {node: '>=16.20.0'} - cpu: [mips64el] - os: [linux] - - '@typescript/typescript-linux-ppc64@7.0.2': - resolution: {integrity: sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==} - engines: {node: '>=16.20.0'} - cpu: [ppc64] - os: [linux] - - '@typescript/typescript-linux-riscv64@7.0.2': - resolution: {integrity: sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==} - engines: {node: '>=16.20.0'} - cpu: [riscv64] - os: [linux] - - '@typescript/typescript-linux-s390x@7.0.2': - resolution: {integrity: sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==} - engines: {node: '>=16.20.0'} - cpu: [s390x] - os: [linux] - - '@typescript/typescript-linux-x64@7.0.2': - resolution: {integrity: sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==} - engines: {node: '>=16.20.0'} - cpu: [x64] - os: [linux] - - '@typescript/typescript-netbsd-arm64@7.0.2': - resolution: {integrity: sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==} - engines: {node: '>=16.20.0'} - cpu: [arm64] - os: [netbsd] - - '@typescript/typescript-netbsd-x64@7.0.2': - resolution: {integrity: sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==} - engines: {node: '>=16.20.0'} - cpu: [x64] - os: [netbsd] - - '@typescript/typescript-openbsd-arm64@7.0.2': - resolution: {integrity: sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==} - engines: {node: '>=16.20.0'} - cpu: [arm64] - os: [openbsd] - - '@typescript/typescript-openbsd-x64@7.0.2': - resolution: {integrity: sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==} - engines: {node: '>=16.20.0'} - cpu: [x64] - os: [openbsd] - - '@typescript/typescript-sunos-x64@7.0.2': - resolution: {integrity: sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==} - engines: {node: '>=16.20.0'} - cpu: [x64] - os: [sunos] - - '@typescript/typescript-win32-arm64@7.0.2': - resolution: {integrity: sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==} - engines: {node: '>=16.20.0'} - cpu: [arm64] - os: [win32] - - '@typescript/typescript-win32-x64@7.0.2': - resolution: {integrity: sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==} - engines: {node: '>=16.20.0'} - cpu: [x64] - os: [win32] - - '@vitest/expect@4.1.10': - resolution: {integrity: sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==} - - '@vitest/mocker@4.1.10': - resolution: {integrity: sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow==} - peerDependencies: - msw: ^2.4.9 - vite: ^6.0.0 || ^7.0.0 || ^8.0.0 - peerDependenciesMeta: - msw: - optional: true - vite: - optional: true - - '@vitest/pretty-format@4.1.10': - resolution: {integrity: sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==} - - '@vitest/runner@4.1.10': - resolution: {integrity: sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==} - - '@vitest/snapshot@4.1.10': - resolution: {integrity: sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw==} - - '@vitest/spy@4.1.10': - resolution: {integrity: sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw==} - - '@vitest/utils@4.1.10': - resolution: {integrity: sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==} - - assertion-error@2.0.1: - resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} - engines: {node: '>=12'} - - aws4fetch@1.0.20: - resolution: {integrity: sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g==} - - blake3-wasm@2.1.5: - resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==} - - buffer-from@1.1.2: - resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} - - chai@6.2.2: - resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} - engines: {node: '>=18'} - - cjs-module-lexer@1.2.3: - resolution: {integrity: sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==} - - convert-source-map@2.0.0: - resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} - - cookie@1.1.1: - resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} - engines: {node: '>=18'} - - detect-libc@2.1.2: - resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} - engines: {node: '>=8'} - - drizzle-kit@0.31.10: - resolution: {integrity: sha512-7OZcmQUrdGI+DUNNsKBn1aW8qSoKuTH7d0mYgSP8bAzdFzKoovxEFnoGQp2dVs82EOJeYycqRtciopszwUf8bw==} - hasBin: true - - drizzle-orm@0.45.2: - resolution: {integrity: sha512-kY0BSaTNYWnoDMVoyY8uxmyHjpJW1geOmBMdSSicKo9CIIWkSxMIj2rkeSR51b8KAPB7m+qysjuHme5nKP+E5Q==} - peerDependencies: - '@aws-sdk/client-rds-data': '>=3' - '@cloudflare/workers-types': '>=4' - '@electric-sql/pglite': '>=0.2.0' - '@libsql/client': '>=0.10.0' - '@libsql/client-wasm': '>=0.10.0' - '@neondatabase/serverless': '>=0.10.0' - '@op-engineering/op-sqlite': '>=2' - '@opentelemetry/api': ^1.4.1 - '@planetscale/database': '>=1.13' - '@prisma/client': '*' - '@tidbcloud/serverless': '*' - '@types/better-sqlite3': '*' - '@types/pg': '*' - '@types/sql.js': '*' - '@upstash/redis': '>=1.34.7' - '@vercel/postgres': '>=0.8.0' - '@xata.io/client': '*' - better-sqlite3: '>=7' - bun-types: '*' - expo-sqlite: '>=14.0.0' - gel: '>=2' - knex: '*' - kysely: '*' - mysql2: '>=2' - pg: '>=8' - postgres: '>=3' - prisma: '*' - sql.js: '>=1' - sqlite3: '>=5' - peerDependenciesMeta: - '@aws-sdk/client-rds-data': - optional: true - '@cloudflare/workers-types': - optional: true - '@electric-sql/pglite': - optional: true - '@libsql/client': - optional: true - '@libsql/client-wasm': - optional: true - '@neondatabase/serverless': - optional: true - '@op-engineering/op-sqlite': - optional: true - '@opentelemetry/api': - optional: true - '@planetscale/database': - optional: true - '@prisma/client': - optional: true - '@tidbcloud/serverless': - optional: true - '@types/better-sqlite3': - optional: true - '@types/pg': - optional: true - '@types/sql.js': - optional: true - '@upstash/redis': - optional: true - '@vercel/postgres': - optional: true - '@xata.io/client': - optional: true - better-sqlite3: - optional: true - bun-types: - optional: true - expo-sqlite: - optional: true - gel: - optional: true - knex: - optional: true - kysely: - optional: true - mysql2: - optional: true - pg: - optional: true - postgres: - optional: true - prisma: - optional: true - sql.js: - optional: true - sqlite3: - optional: true - - error-stack-parser-es@1.0.5: - resolution: {integrity: sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==} - - es-module-lexer@2.3.1: - resolution: {integrity: sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==} - - esbuild@0.18.20: - resolution: {integrity: sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA==} - engines: {node: '>=12'} - hasBin: true - - esbuild@0.25.12: - resolution: {integrity: sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==} - engines: {node: '>=18'} - hasBin: true - - esbuild@0.27.7: - resolution: {integrity: sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==} - engines: {node: '>=18'} - hasBin: true - - esbuild@0.28.1: - resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} - engines: {node: '>=18'} - hasBin: true - - estree-walker@3.0.3: - resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} - - expect-type@1.3.0: - resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} - engines: {node: '>=12.0.0'} - - fdir@6.5.0: - resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} - engines: {node: '>=12.0.0'} - peerDependencies: - picomatch: ^3 || ^4 - peerDependenciesMeta: - picomatch: - optional: true - - fsevents@2.3.3: - resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} - engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} - os: [darwin] - - get-tsconfig@4.14.0: - resolution: {integrity: sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==} - - kleur@4.1.5: - resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} - engines: {node: '>=6'} - - magic-string@0.30.21: - resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} - - miniflare@4.20260714.0: - resolution: {integrity: sha512-MYlTCLdWCPqvrYY2uLwOjXwmglXuiHE3TGGkbOW4BwjUPa1r07E0iuHwrNDIs/sxK21r+o90Jx58AV2KeNdJZw==} - engines: {node: '>=22.0.0'} - hasBin: true - - nanoid@3.3.11: - resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} - engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} - hasBin: true - - obug@2.1.4: - resolution: {integrity: sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==} - engines: {node: '>=12.20.0'} - - path-to-regexp@6.3.0: - resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} - - pathe@2.0.3: - resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} - - picocolors@1.1.1: - resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} - - picomatch@4.0.4: - resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} - engines: {node: '>=12'} - - postcss@8.5.10: - resolution: {integrity: sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==} - engines: {node: ^10 || ^12 || >=14} - - resolve-pkg-maps@1.0.0: - resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} - - rollup@4.60.2: - resolution: {integrity: sha512-J9qZyW++QK/09NyN/zeO0dG/1GdGfyp9lV8ajHnRVLfo/uFsbji5mHnDgn/qYdUHyCkM2N+8VyspgZclfAh0eQ==} - engines: {node: '>=18.0.0', npm: '>=8.0.0'} - hasBin: true - - semver@7.7.4: - resolution: {integrity: sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==} - engines: {node: '>=10'} - hasBin: true - - sharp@0.34.5: - resolution: {integrity: sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - - siginfo@2.0.0: - resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} - - source-map-js@1.2.1: - resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} - engines: {node: '>=0.10.0'} - - source-map-support@0.5.21: - resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} - - source-map@0.6.1: - resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} - engines: {node: '>=0.10.0'} - - stackback@0.0.2: - resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} - - std-env@4.2.0: - resolution: {integrity: sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==} - - supports-color@10.2.2: - resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} - engines: {node: '>=18'} - - tinybench@2.9.0: - resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} - - tinyexec@1.2.4: - resolution: {integrity: sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==} - engines: {node: '>=18'} - - tinyglobby@0.2.16: - resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==} - engines: {node: '>=12.0.0'} - - tinyrainbow@3.1.0: - resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} - engines: {node: '>=14.0.0'} - - tslib@2.8.1: - resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} - - tsx@4.21.0: - resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} - engines: {node: '>=18.0.0'} - hasBin: true - - typescript@7.0.2: - resolution: {integrity: sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==} - engines: {node: '>=16.20.0'} - hasBin: true - - undici-types@8.3.0: - resolution: {integrity: sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==} - - undici@7.28.0: - resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==} - engines: {node: '>=20.18.1'} - - unenv@2.0.0-rc.24: - resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} - - vite@7.3.2: - resolution: {integrity: sha512-Bby3NOsna2jsjfLVOHKes8sGwgl4TT0E6vvpYgnAYDIF/tie7MRaFthmKuHx1NSXjiTueXH3do80FMQgvEktRg==} - engines: {node: ^20.19.0 || >=22.12.0} - hasBin: true - peerDependencies: - '@types/node': ^20.19.0 || >=22.12.0 - jiti: '>=1.21.0' - less: ^4.0.0 - lightningcss: ^1.21.0 - sass: ^1.70.0 - sass-embedded: ^1.70.0 - stylus: '>=0.54.8' - sugarss: ^5.0.0 - terser: ^5.16.0 - tsx: ^4.8.1 - yaml: ^2.4.2 - peerDependenciesMeta: - '@types/node': - optional: true - jiti: - optional: true - less: - optional: true - lightningcss: - optional: true - sass: - optional: true - sass-embedded: - optional: true - stylus: - optional: true - sugarss: - optional: true - terser: - optional: true - tsx: - optional: true - yaml: - optional: true - - vitest@4.1.10: - resolution: {integrity: sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==} - engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} - hasBin: true - peerDependencies: - '@edge-runtime/vm': '*' - '@opentelemetry/api': ^1.9.0 - '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 - '@vitest/browser-playwright': 4.1.10 - '@vitest/browser-preview': 4.1.10 - '@vitest/browser-webdriverio': 4.1.10 - '@vitest/coverage-istanbul': 4.1.10 - '@vitest/coverage-v8': 4.1.10 - '@vitest/ui': 4.1.10 - happy-dom: '*' - jsdom: '*' - vite: ^6.0.0 || ^7.0.0 || ^8.0.0 - peerDependenciesMeta: - '@edge-runtime/vm': - optional: true - '@opentelemetry/api': - optional: true - '@types/node': - optional: true - '@vitest/browser-playwright': - optional: true - '@vitest/browser-preview': - optional: true - '@vitest/browser-webdriverio': - optional: true - '@vitest/coverage-istanbul': - optional: true - '@vitest/coverage-v8': - optional: true - '@vitest/ui': - optional: true - happy-dom: - optional: true - jsdom: - optional: true - - why-is-node-running@2.3.0: - resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} - engines: {node: '>=8'} - hasBin: true - - workerd@1.20260714.1: - resolution: {integrity: sha512-oIbQzfdyl9UQUnG6XLegcSq0Mgt/7WKDbFOoqGgOWCS+/fhyGB460uKEgdAQQ9RHCO/ttcNCX/KiMIQzdoeu3Q==} - engines: {node: '>=16'} - hasBin: true - - wrangler@4.112.0: - resolution: {integrity: sha512-5H+XUD0TySCv1LuktFHDIEOkboH2nTfQs+35L+USt3MtntjDTMVIJprLgQcL2WBjulOyjxpd1vyTiSTJVW5MjQ==} - engines: {node: '>=22.0.0'} - hasBin: true - peerDependencies: - '@cloudflare/workers-types': ^5.20260714.1 - peerDependenciesMeta: - '@cloudflare/workers-types': - optional: true - - ws@8.21.0: - resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - - youch-core@0.3.3: - resolution: {integrity: sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==} - - youch@4.1.0-beta.10: - resolution: {integrity: sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==} - - zod@3.25.76: - resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} - -snapshots: - - '@cloudflare/kv-asset-handler@0.5.0': {} - - '@cloudflare/unenv-preset@2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260714.1)': - dependencies: - unenv: 2.0.0-rc.24 - optionalDependencies: - workerd: 1.20260714.1 - - '@cloudflare/vitest-pool-workers@0.18.6(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10(@types/node@26.1.1)(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0)))': - dependencies: - '@vitest/runner': 4.1.10 - '@vitest/snapshot': 4.1.10 - cjs-module-lexer: 1.2.3 - esbuild: 0.28.1 - miniflare: 4.20260714.0 - vitest: 4.1.10(@types/node@26.1.1)(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0)) - wrangler: 4.112.0 - zod: 3.25.76 - transitivePeerDependencies: - - '@cloudflare/workers-types' - - bufferutil - - utf-8-validate - - '@cloudflare/workerd-darwin-64@1.20260714.1': - optional: true - - '@cloudflare/workerd-darwin-arm64@1.20260714.1': - optional: true - - '@cloudflare/workerd-linux-64@1.20260714.1': - optional: true - - '@cloudflare/workerd-linux-arm64@1.20260714.1': - optional: true - - '@cloudflare/workerd-windows-64@1.20260714.1': - optional: true - - '@cspotcode/source-map-support@0.8.1': - dependencies: - '@jridgewell/trace-mapping': 0.3.9 - - '@drizzle-team/brocli@0.10.2': {} - - '@emnapi/runtime@1.10.0': - dependencies: - tslib: 2.8.1 - optional: true - - '@esbuild-kit/core-utils@3.3.2': - dependencies: - esbuild: 0.18.20 - source-map-support: 0.5.21 - - '@esbuild-kit/esm-loader@2.6.5': - dependencies: - '@esbuild-kit/core-utils': 3.3.2 - get-tsconfig: 4.14.0 - - '@esbuild/aix-ppc64@0.25.12': - optional: true - - '@esbuild/aix-ppc64@0.27.7': - optional: true - - '@esbuild/aix-ppc64@0.28.1': - optional: true - - '@esbuild/android-arm64@0.18.20': - optional: true - - '@esbuild/android-arm64@0.25.12': - optional: true - - '@esbuild/android-arm64@0.27.7': - optional: true - - '@esbuild/android-arm64@0.28.1': - optional: true - - '@esbuild/android-arm@0.18.20': - optional: true - - '@esbuild/android-arm@0.25.12': - optional: true - - '@esbuild/android-arm@0.27.7': - optional: true - - '@esbuild/android-arm@0.28.1': - optional: true - - '@esbuild/android-x64@0.18.20': - optional: true - - '@esbuild/android-x64@0.25.12': - optional: true - - '@esbuild/android-x64@0.27.7': - optional: true - - '@esbuild/android-x64@0.28.1': - optional: true - - '@esbuild/darwin-arm64@0.18.20': - optional: true - - '@esbuild/darwin-arm64@0.25.12': - optional: true - - '@esbuild/darwin-arm64@0.27.7': - optional: true - - '@esbuild/darwin-arm64@0.28.1': - optional: true - - '@esbuild/darwin-x64@0.18.20': - optional: true - - '@esbuild/darwin-x64@0.25.12': - optional: true - - '@esbuild/darwin-x64@0.27.7': - optional: true - - '@esbuild/darwin-x64@0.28.1': - optional: true - - '@esbuild/freebsd-arm64@0.18.20': - optional: true - - '@esbuild/freebsd-arm64@0.25.12': - optional: true - - '@esbuild/freebsd-arm64@0.27.7': - optional: true - - '@esbuild/freebsd-arm64@0.28.1': - optional: true - - '@esbuild/freebsd-x64@0.18.20': - optional: true - - '@esbuild/freebsd-x64@0.25.12': - optional: true - - '@esbuild/freebsd-x64@0.27.7': - optional: true - - '@esbuild/freebsd-x64@0.28.1': - optional: true - - '@esbuild/linux-arm64@0.18.20': - optional: true - - '@esbuild/linux-arm64@0.25.12': - optional: true - - '@esbuild/linux-arm64@0.27.7': - optional: true - - '@esbuild/linux-arm64@0.28.1': - optional: true - - '@esbuild/linux-arm@0.18.20': - optional: true - - '@esbuild/linux-arm@0.25.12': - optional: true - - '@esbuild/linux-arm@0.27.7': - optional: true - - '@esbuild/linux-arm@0.28.1': - optional: true - - '@esbuild/linux-ia32@0.18.20': - optional: true - - '@esbuild/linux-ia32@0.25.12': - optional: true - - '@esbuild/linux-ia32@0.27.7': - optional: true - - '@esbuild/linux-ia32@0.28.1': - optional: true - - '@esbuild/linux-loong64@0.18.20': - optional: true - - '@esbuild/linux-loong64@0.25.12': - optional: true - - '@esbuild/linux-loong64@0.27.7': - optional: true - - '@esbuild/linux-loong64@0.28.1': - optional: true - - '@esbuild/linux-mips64el@0.18.20': - optional: true - - '@esbuild/linux-mips64el@0.25.12': - optional: true - - '@esbuild/linux-mips64el@0.27.7': - optional: true - - '@esbuild/linux-mips64el@0.28.1': - optional: true - - '@esbuild/linux-ppc64@0.18.20': - optional: true - - '@esbuild/linux-ppc64@0.25.12': - optional: true - - '@esbuild/linux-ppc64@0.27.7': - optional: true - - '@esbuild/linux-ppc64@0.28.1': - optional: true - - '@esbuild/linux-riscv64@0.18.20': - optional: true - - '@esbuild/linux-riscv64@0.25.12': - optional: true - - '@esbuild/linux-riscv64@0.27.7': - optional: true - - '@esbuild/linux-riscv64@0.28.1': - optional: true - - '@esbuild/linux-s390x@0.18.20': - optional: true - - '@esbuild/linux-s390x@0.25.12': - optional: true - - '@esbuild/linux-s390x@0.27.7': - optional: true - - '@esbuild/linux-s390x@0.28.1': - optional: true - - '@esbuild/linux-x64@0.18.20': - optional: true - - '@esbuild/linux-x64@0.25.12': - optional: true - - '@esbuild/linux-x64@0.27.7': - optional: true - - '@esbuild/linux-x64@0.28.1': - optional: true - - '@esbuild/netbsd-arm64@0.25.12': - optional: true - - '@esbuild/netbsd-arm64@0.27.7': - optional: true - - '@esbuild/netbsd-arm64@0.28.1': - optional: true - - '@esbuild/netbsd-x64@0.18.20': - optional: true - - '@esbuild/netbsd-x64@0.25.12': - optional: true - - '@esbuild/netbsd-x64@0.27.7': - optional: true - - '@esbuild/netbsd-x64@0.28.1': - optional: true - - '@esbuild/openbsd-arm64@0.25.12': - optional: true - - '@esbuild/openbsd-arm64@0.27.7': - optional: true - - '@esbuild/openbsd-arm64@0.28.1': - optional: true - - '@esbuild/openbsd-x64@0.18.20': - optional: true - - '@esbuild/openbsd-x64@0.25.12': - optional: true - - '@esbuild/openbsd-x64@0.27.7': - optional: true - - '@esbuild/openbsd-x64@0.28.1': - optional: true - - '@esbuild/openharmony-arm64@0.25.12': - optional: true - - '@esbuild/openharmony-arm64@0.27.7': - optional: true - - '@esbuild/openharmony-arm64@0.28.1': - optional: true - - '@esbuild/sunos-x64@0.18.20': - optional: true - - '@esbuild/sunos-x64@0.25.12': - optional: true - - '@esbuild/sunos-x64@0.27.7': - optional: true - - '@esbuild/sunos-x64@0.28.1': - optional: true - - '@esbuild/win32-arm64@0.18.20': - optional: true - - '@esbuild/win32-arm64@0.25.12': - optional: true - - '@esbuild/win32-arm64@0.27.7': - optional: true - - '@esbuild/win32-arm64@0.28.1': - optional: true - - '@esbuild/win32-ia32@0.18.20': - optional: true - - '@esbuild/win32-ia32@0.25.12': - optional: true - - '@esbuild/win32-ia32@0.27.7': - optional: true - - '@esbuild/win32-ia32@0.28.1': - optional: true - - '@esbuild/win32-x64@0.18.20': - optional: true - - '@esbuild/win32-x64@0.25.12': - optional: true - - '@esbuild/win32-x64@0.27.7': - optional: true - - '@esbuild/win32-x64@0.28.1': - optional: true - - '@img/colour@1.1.0': {} - - '@img/sharp-darwin-arm64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-darwin-arm64': 1.2.4 - optional: true - - '@img/sharp-darwin-x64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-darwin-x64': 1.2.4 - optional: true - - '@img/sharp-libvips-darwin-arm64@1.2.4': - optional: true - - '@img/sharp-libvips-darwin-x64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-arm64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-arm@1.2.4': - optional: true - - '@img/sharp-libvips-linux-ppc64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-riscv64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-s390x@1.2.4': - optional: true - - '@img/sharp-libvips-linux-x64@1.2.4': - optional: true - - '@img/sharp-libvips-linuxmusl-arm64@1.2.4': - optional: true - - '@img/sharp-libvips-linuxmusl-x64@1.2.4': - optional: true - - '@img/sharp-linux-arm64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-arm64': 1.2.4 - optional: true - - '@img/sharp-linux-arm@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-arm': 1.2.4 - optional: true - - '@img/sharp-linux-ppc64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-ppc64': 1.2.4 - optional: true - - '@img/sharp-linux-riscv64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-riscv64': 1.2.4 - optional: true - - '@img/sharp-linux-s390x@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-s390x': 1.2.4 - optional: true - - '@img/sharp-linux-x64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-x64': 1.2.4 - optional: true - - '@img/sharp-linuxmusl-arm64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 - optional: true - - '@img/sharp-linuxmusl-x64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linuxmusl-x64': 1.2.4 - optional: true - - '@img/sharp-wasm32@0.34.5': - dependencies: - '@emnapi/runtime': 1.10.0 - optional: true - - '@img/sharp-win32-arm64@0.34.5': - optional: true - - '@img/sharp-win32-ia32@0.34.5': - optional: true - - '@img/sharp-win32-x64@0.34.5': - optional: true - - '@jridgewell/resolve-uri@3.1.2': {} - - '@jridgewell/sourcemap-codec@1.5.5': {} - - '@jridgewell/trace-mapping@0.3.9': - dependencies: - '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.5 - - '@poppinss/colors@4.1.6': - dependencies: - kleur: 4.1.5 - - '@poppinss/dumper@0.6.5': - dependencies: - '@poppinss/colors': 4.1.6 - '@sindresorhus/is': 7.2.0 - supports-color: 10.2.2 - - '@poppinss/exception@1.2.3': {} - - '@rollup/rollup-android-arm-eabi@4.60.2': - optional: true - - '@rollup/rollup-android-arm64@4.60.2': - optional: true - - '@rollup/rollup-darwin-arm64@4.60.2': - optional: true - - '@rollup/rollup-darwin-x64@4.60.2': - optional: true - - '@rollup/rollup-freebsd-arm64@4.60.2': - optional: true - - '@rollup/rollup-freebsd-x64@4.60.2': - optional: true - - '@rollup/rollup-linux-arm-gnueabihf@4.60.2': - optional: true - - '@rollup/rollup-linux-arm-musleabihf@4.60.2': - optional: true - - '@rollup/rollup-linux-arm64-gnu@4.60.2': - optional: true - - '@rollup/rollup-linux-arm64-musl@4.60.2': - optional: true - - '@rollup/rollup-linux-loong64-gnu@4.60.2': - optional: true - - '@rollup/rollup-linux-loong64-musl@4.60.2': - optional: true - - '@rollup/rollup-linux-ppc64-gnu@4.60.2': - optional: true - - '@rollup/rollup-linux-ppc64-musl@4.60.2': - optional: true - - '@rollup/rollup-linux-riscv64-gnu@4.60.2': - optional: true - - '@rollup/rollup-linux-riscv64-musl@4.60.2': - optional: true - - '@rollup/rollup-linux-s390x-gnu@4.60.2': - optional: true - - '@rollup/rollup-linux-x64-gnu@4.60.2': - optional: true - - '@rollup/rollup-linux-x64-musl@4.60.2': - optional: true - - '@rollup/rollup-openbsd-x64@4.60.2': - optional: true - - '@rollup/rollup-openharmony-arm64@4.60.2': - optional: true - - '@rollup/rollup-win32-arm64-msvc@4.60.2': - optional: true - - '@rollup/rollup-win32-ia32-msvc@4.60.2': - optional: true - - '@rollup/rollup-win32-x64-gnu@4.60.2': - optional: true - - '@rollup/rollup-win32-x64-msvc@4.60.2': - optional: true - - '@sindresorhus/is@7.2.0': {} - - '@speed-highlight/core@1.2.15': {} - - '@standard-schema/spec@1.1.0': {} - - '@types/chai@5.2.3': - dependencies: - '@types/deep-eql': 4.0.2 - assertion-error: 2.0.1 - - '@types/deep-eql@4.0.2': {} - - '@types/estree@1.0.8': {} - - '@types/node@26.1.1': - dependencies: - undici-types: 8.3.0 - - '@typescript/typescript-aix-ppc64@7.0.2': - optional: true - - '@typescript/typescript-darwin-arm64@7.0.2': - optional: true - - '@typescript/typescript-darwin-x64@7.0.2': - optional: true - - '@typescript/typescript-freebsd-arm64@7.0.2': - optional: true - - '@typescript/typescript-freebsd-x64@7.0.2': - optional: true - - '@typescript/typescript-linux-arm64@7.0.2': - optional: true - - '@typescript/typescript-linux-arm@7.0.2': - optional: true - - '@typescript/typescript-linux-loong64@7.0.2': - optional: true - - '@typescript/typescript-linux-mips64el@7.0.2': - optional: true - - '@typescript/typescript-linux-ppc64@7.0.2': - optional: true - - '@typescript/typescript-linux-riscv64@7.0.2': - optional: true - - '@typescript/typescript-linux-s390x@7.0.2': - optional: true - - '@typescript/typescript-linux-x64@7.0.2': - optional: true - - '@typescript/typescript-netbsd-arm64@7.0.2': - optional: true - - '@typescript/typescript-netbsd-x64@7.0.2': - optional: true - - '@typescript/typescript-openbsd-arm64@7.0.2': - optional: true - - '@typescript/typescript-openbsd-x64@7.0.2': - optional: true - - '@typescript/typescript-sunos-x64@7.0.2': - optional: true - - '@typescript/typescript-win32-arm64@7.0.2': - optional: true - - '@typescript/typescript-win32-x64@7.0.2': - optional: true - - '@vitest/expect@4.1.10': - dependencies: - '@standard-schema/spec': 1.1.0 - '@types/chai': 5.2.3 - '@vitest/spy': 4.1.10 - '@vitest/utils': 4.1.10 - chai: 6.2.2 - tinyrainbow: 3.1.0 - - '@vitest/mocker@4.1.10(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0))': - dependencies: - '@vitest/spy': 4.1.10 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 7.3.2(@types/node@26.1.1)(tsx@4.21.0) - - '@vitest/pretty-format@4.1.10': - dependencies: - tinyrainbow: 3.1.0 - - '@vitest/runner@4.1.10': - dependencies: - '@vitest/utils': 4.1.10 - pathe: 2.0.3 - - '@vitest/snapshot@4.1.10': - dependencies: - '@vitest/pretty-format': 4.1.10 - '@vitest/utils': 4.1.10 - magic-string: 0.30.21 - pathe: 2.0.3 - - '@vitest/spy@4.1.10': {} - - '@vitest/utils@4.1.10': - dependencies: - '@vitest/pretty-format': 4.1.10 - convert-source-map: 2.0.0 - tinyrainbow: 3.1.0 - - assertion-error@2.0.1: {} - - aws4fetch@1.0.20: {} - - blake3-wasm@2.1.5: {} - - buffer-from@1.1.2: {} - - chai@6.2.2: {} - - cjs-module-lexer@1.2.3: {} - - convert-source-map@2.0.0: {} - - cookie@1.1.1: {} - - detect-libc@2.1.2: {} - - drizzle-kit@0.31.10: - dependencies: - '@drizzle-team/brocli': 0.10.2 - '@esbuild-kit/esm-loader': 2.6.5 - esbuild: 0.25.12 - tsx: 4.21.0 - - drizzle-orm@0.45.2: {} - - error-stack-parser-es@1.0.5: {} - - es-module-lexer@2.3.1: {} - - esbuild@0.18.20: - optionalDependencies: - '@esbuild/android-arm': 0.18.20 - '@esbuild/android-arm64': 0.18.20 - '@esbuild/android-x64': 0.18.20 - '@esbuild/darwin-arm64': 0.18.20 - '@esbuild/darwin-x64': 0.18.20 - '@esbuild/freebsd-arm64': 0.18.20 - '@esbuild/freebsd-x64': 0.18.20 - '@esbuild/linux-arm': 0.18.20 - '@esbuild/linux-arm64': 0.18.20 - '@esbuild/linux-ia32': 0.18.20 - '@esbuild/linux-loong64': 0.18.20 - '@esbuild/linux-mips64el': 0.18.20 - '@esbuild/linux-ppc64': 0.18.20 - '@esbuild/linux-riscv64': 0.18.20 - '@esbuild/linux-s390x': 0.18.20 - '@esbuild/linux-x64': 0.18.20 - '@esbuild/netbsd-x64': 0.18.20 - '@esbuild/openbsd-x64': 0.18.20 - '@esbuild/sunos-x64': 0.18.20 - '@esbuild/win32-arm64': 0.18.20 - '@esbuild/win32-ia32': 0.18.20 - '@esbuild/win32-x64': 0.18.20 - - esbuild@0.25.12: - optionalDependencies: - '@esbuild/aix-ppc64': 0.25.12 - '@esbuild/android-arm': 0.25.12 - '@esbuild/android-arm64': 0.25.12 - '@esbuild/android-x64': 0.25.12 - '@esbuild/darwin-arm64': 0.25.12 - '@esbuild/darwin-x64': 0.25.12 - '@esbuild/freebsd-arm64': 0.25.12 - '@esbuild/freebsd-x64': 0.25.12 - '@esbuild/linux-arm': 0.25.12 - '@esbuild/linux-arm64': 0.25.12 - '@esbuild/linux-ia32': 0.25.12 - '@esbuild/linux-loong64': 0.25.12 - '@esbuild/linux-mips64el': 0.25.12 - '@esbuild/linux-ppc64': 0.25.12 - '@esbuild/linux-riscv64': 0.25.12 - '@esbuild/linux-s390x': 0.25.12 - '@esbuild/linux-x64': 0.25.12 - '@esbuild/netbsd-arm64': 0.25.12 - '@esbuild/netbsd-x64': 0.25.12 - '@esbuild/openbsd-arm64': 0.25.12 - '@esbuild/openbsd-x64': 0.25.12 - '@esbuild/openharmony-arm64': 0.25.12 - '@esbuild/sunos-x64': 0.25.12 - '@esbuild/win32-arm64': 0.25.12 - '@esbuild/win32-ia32': 0.25.12 - '@esbuild/win32-x64': 0.25.12 - - esbuild@0.27.7: - optionalDependencies: - '@esbuild/aix-ppc64': 0.27.7 - '@esbuild/android-arm': 0.27.7 - '@esbuild/android-arm64': 0.27.7 - '@esbuild/android-x64': 0.27.7 - '@esbuild/darwin-arm64': 0.27.7 - '@esbuild/darwin-x64': 0.27.7 - '@esbuild/freebsd-arm64': 0.27.7 - '@esbuild/freebsd-x64': 0.27.7 - '@esbuild/linux-arm': 0.27.7 - '@esbuild/linux-arm64': 0.27.7 - '@esbuild/linux-ia32': 0.27.7 - '@esbuild/linux-loong64': 0.27.7 - '@esbuild/linux-mips64el': 0.27.7 - '@esbuild/linux-ppc64': 0.27.7 - '@esbuild/linux-riscv64': 0.27.7 - '@esbuild/linux-s390x': 0.27.7 - '@esbuild/linux-x64': 0.27.7 - '@esbuild/netbsd-arm64': 0.27.7 - '@esbuild/netbsd-x64': 0.27.7 - '@esbuild/openbsd-arm64': 0.27.7 - '@esbuild/openbsd-x64': 0.27.7 - '@esbuild/openharmony-arm64': 0.27.7 - '@esbuild/sunos-x64': 0.27.7 - '@esbuild/win32-arm64': 0.27.7 - '@esbuild/win32-ia32': 0.27.7 - '@esbuild/win32-x64': 0.27.7 - - esbuild@0.28.1: - optionalDependencies: - '@esbuild/aix-ppc64': 0.28.1 - '@esbuild/android-arm': 0.28.1 - '@esbuild/android-arm64': 0.28.1 - '@esbuild/android-x64': 0.28.1 - '@esbuild/darwin-arm64': 0.28.1 - '@esbuild/darwin-x64': 0.28.1 - '@esbuild/freebsd-arm64': 0.28.1 - '@esbuild/freebsd-x64': 0.28.1 - '@esbuild/linux-arm': 0.28.1 - '@esbuild/linux-arm64': 0.28.1 - '@esbuild/linux-ia32': 0.28.1 - '@esbuild/linux-loong64': 0.28.1 - '@esbuild/linux-mips64el': 0.28.1 - '@esbuild/linux-ppc64': 0.28.1 - '@esbuild/linux-riscv64': 0.28.1 - '@esbuild/linux-s390x': 0.28.1 - '@esbuild/linux-x64': 0.28.1 - '@esbuild/netbsd-arm64': 0.28.1 - '@esbuild/netbsd-x64': 0.28.1 - '@esbuild/openbsd-arm64': 0.28.1 - '@esbuild/openbsd-x64': 0.28.1 - '@esbuild/openharmony-arm64': 0.28.1 - '@esbuild/sunos-x64': 0.28.1 - '@esbuild/win32-arm64': 0.28.1 - '@esbuild/win32-ia32': 0.28.1 - '@esbuild/win32-x64': 0.28.1 - - estree-walker@3.0.3: - dependencies: - '@types/estree': 1.0.8 - - expect-type@1.3.0: {} - - fdir@6.5.0(picomatch@4.0.4): - optionalDependencies: - picomatch: 4.0.4 - - fsevents@2.3.3: - optional: true - - get-tsconfig@4.14.0: - dependencies: - resolve-pkg-maps: 1.0.0 - - kleur@4.1.5: {} - - magic-string@0.30.21: - dependencies: - '@jridgewell/sourcemap-codec': 1.5.5 - - miniflare@4.20260714.0: - dependencies: - '@cspotcode/source-map-support': 0.8.1 - sharp: 0.34.5 - undici: 7.28.0 - workerd: 1.20260714.1 - ws: 8.21.0 - youch: 4.1.0-beta.10 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - nanoid@3.3.11: {} - - obug@2.1.4: {} - - path-to-regexp@6.3.0: {} - - pathe@2.0.3: {} - - picocolors@1.1.1: {} - - picomatch@4.0.4: {} - - postcss@8.5.10: - dependencies: - nanoid: 3.3.11 - picocolors: 1.1.1 - source-map-js: 1.2.1 - - resolve-pkg-maps@1.0.0: {} - - rollup@4.60.2: - dependencies: - '@types/estree': 1.0.8 - optionalDependencies: - '@rollup/rollup-android-arm-eabi': 4.60.2 - '@rollup/rollup-android-arm64': 4.60.2 - '@rollup/rollup-darwin-arm64': 4.60.2 - '@rollup/rollup-darwin-x64': 4.60.2 - '@rollup/rollup-freebsd-arm64': 4.60.2 - '@rollup/rollup-freebsd-x64': 4.60.2 - '@rollup/rollup-linux-arm-gnueabihf': 4.60.2 - '@rollup/rollup-linux-arm-musleabihf': 4.60.2 - '@rollup/rollup-linux-arm64-gnu': 4.60.2 - '@rollup/rollup-linux-arm64-musl': 4.60.2 - '@rollup/rollup-linux-loong64-gnu': 4.60.2 - '@rollup/rollup-linux-loong64-musl': 4.60.2 - '@rollup/rollup-linux-ppc64-gnu': 4.60.2 - '@rollup/rollup-linux-ppc64-musl': 4.60.2 - '@rollup/rollup-linux-riscv64-gnu': 4.60.2 - '@rollup/rollup-linux-riscv64-musl': 4.60.2 - '@rollup/rollup-linux-s390x-gnu': 4.60.2 - '@rollup/rollup-linux-x64-gnu': 4.60.2 - '@rollup/rollup-linux-x64-musl': 4.60.2 - '@rollup/rollup-openbsd-x64': 4.60.2 - '@rollup/rollup-openharmony-arm64': 4.60.2 - '@rollup/rollup-win32-arm64-msvc': 4.60.2 - '@rollup/rollup-win32-ia32-msvc': 4.60.2 - '@rollup/rollup-win32-x64-gnu': 4.60.2 - '@rollup/rollup-win32-x64-msvc': 4.60.2 - fsevents: 2.3.3 - - semver@7.7.4: {} - - sharp@0.34.5: - dependencies: - '@img/colour': 1.1.0 - detect-libc: 2.1.2 - semver: 7.7.4 - optionalDependencies: - '@img/sharp-darwin-arm64': 0.34.5 - '@img/sharp-darwin-x64': 0.34.5 - '@img/sharp-libvips-darwin-arm64': 1.2.4 - '@img/sharp-libvips-darwin-x64': 1.2.4 - '@img/sharp-libvips-linux-arm': 1.2.4 - '@img/sharp-libvips-linux-arm64': 1.2.4 - '@img/sharp-libvips-linux-ppc64': 1.2.4 - '@img/sharp-libvips-linux-riscv64': 1.2.4 - '@img/sharp-libvips-linux-s390x': 1.2.4 - '@img/sharp-libvips-linux-x64': 1.2.4 - '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 - '@img/sharp-libvips-linuxmusl-x64': 1.2.4 - '@img/sharp-linux-arm': 0.34.5 - '@img/sharp-linux-arm64': 0.34.5 - '@img/sharp-linux-ppc64': 0.34.5 - '@img/sharp-linux-riscv64': 0.34.5 - '@img/sharp-linux-s390x': 0.34.5 - '@img/sharp-linux-x64': 0.34.5 - '@img/sharp-linuxmusl-arm64': 0.34.5 - '@img/sharp-linuxmusl-x64': 0.34.5 - '@img/sharp-wasm32': 0.34.5 - '@img/sharp-win32-arm64': 0.34.5 - '@img/sharp-win32-ia32': 0.34.5 - '@img/sharp-win32-x64': 0.34.5 - - siginfo@2.0.0: {} - - source-map-js@1.2.1: {} - - source-map-support@0.5.21: - dependencies: - buffer-from: 1.1.2 - source-map: 0.6.1 - - source-map@0.6.1: {} - - stackback@0.0.2: {} - - std-env@4.2.0: {} - - supports-color@10.2.2: {} - - tinybench@2.9.0: {} - - tinyexec@1.2.4: {} - - tinyglobby@0.2.16: - dependencies: - fdir: 6.5.0(picomatch@4.0.4) - picomatch: 4.0.4 - - tinyrainbow@3.1.0: {} - - tslib@2.8.1: - optional: true - - tsx@4.21.0: - dependencies: - esbuild: 0.27.7 - get-tsconfig: 4.14.0 - optionalDependencies: - fsevents: 2.3.3 - - typescript@7.0.2: - optionalDependencies: - '@typescript/typescript-aix-ppc64': 7.0.2 - '@typescript/typescript-darwin-arm64': 7.0.2 - '@typescript/typescript-darwin-x64': 7.0.2 - '@typescript/typescript-freebsd-arm64': 7.0.2 - '@typescript/typescript-freebsd-x64': 7.0.2 - '@typescript/typescript-linux-arm': 7.0.2 - '@typescript/typescript-linux-arm64': 7.0.2 - '@typescript/typescript-linux-loong64': 7.0.2 - '@typescript/typescript-linux-mips64el': 7.0.2 - '@typescript/typescript-linux-ppc64': 7.0.2 - '@typescript/typescript-linux-riscv64': 7.0.2 - '@typescript/typescript-linux-s390x': 7.0.2 - '@typescript/typescript-linux-x64': 7.0.2 - '@typescript/typescript-netbsd-arm64': 7.0.2 - '@typescript/typescript-netbsd-x64': 7.0.2 - '@typescript/typescript-openbsd-arm64': 7.0.2 - '@typescript/typescript-openbsd-x64': 7.0.2 - '@typescript/typescript-sunos-x64': 7.0.2 - '@typescript/typescript-win32-arm64': 7.0.2 - '@typescript/typescript-win32-x64': 7.0.2 - - undici-types@8.3.0: {} - - undici@7.28.0: {} - - unenv@2.0.0-rc.24: - dependencies: - pathe: 2.0.3 - - vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0): - dependencies: - esbuild: 0.27.7 - fdir: 6.5.0(picomatch@4.0.4) - picomatch: 4.0.4 - postcss: 8.5.10 - rollup: 4.60.2 - tinyglobby: 0.2.16 - optionalDependencies: - '@types/node': 26.1.1 - fsevents: 2.3.3 - tsx: 4.21.0 - - vitest@4.1.10(@types/node@26.1.1)(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0)): - dependencies: - '@vitest/expect': 4.1.10 - '@vitest/mocker': 4.1.10(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0)) - '@vitest/pretty-format': 4.1.10 - '@vitest/runner': 4.1.10 - '@vitest/snapshot': 4.1.10 - '@vitest/spy': 4.1.10 - '@vitest/utils': 4.1.10 - es-module-lexer: 2.3.1 - expect-type: 1.3.0 - magic-string: 0.30.21 - obug: 2.1.4 - pathe: 2.0.3 - picomatch: 4.0.4 - std-env: 4.2.0 - tinybench: 2.9.0 - tinyexec: 1.2.4 - tinyglobby: 0.2.16 - tinyrainbow: 3.1.0 - vite: 7.3.2(@types/node@26.1.1)(tsx@4.21.0) - why-is-node-running: 2.3.0 - optionalDependencies: - '@types/node': 26.1.1 - transitivePeerDependencies: - - msw - - why-is-node-running@2.3.0: - dependencies: - siginfo: 2.0.0 - stackback: 0.0.2 - - workerd@1.20260714.1: - optionalDependencies: - '@cloudflare/workerd-darwin-64': 1.20260714.1 - '@cloudflare/workerd-darwin-arm64': 1.20260714.1 - '@cloudflare/workerd-linux-64': 1.20260714.1 - '@cloudflare/workerd-linux-arm64': 1.20260714.1 - '@cloudflare/workerd-windows-64': 1.20260714.1 - - wrangler@4.112.0: - dependencies: - '@cloudflare/kv-asset-handler': 0.5.0 - '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260714.1) - blake3-wasm: 2.1.5 - esbuild: 0.28.1 - miniflare: 4.20260714.0 - path-to-regexp: 6.3.0 - unenv: 2.0.0-rc.24 - workerd: 1.20260714.1 - optionalDependencies: - fsevents: 2.3.3 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - ws@8.21.0: {} - - youch-core@0.3.3: - dependencies: - '@poppinss/exception': 1.2.3 - error-stack-parser-es: 1.0.5 - - youch@4.1.0-beta.10: - dependencies: - '@poppinss/colors': 4.1.6 - '@poppinss/dumper': 0.6.5 - '@speed-highlight/core': 1.2.15 - cookie: 1.1.1 - youch-core: 0.3.3 - - zod@3.25.76: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml deleted file mode 100644 index 94830da..0000000 --- a/pnpm-workspace.yaml +++ /dev/null @@ -1,8 +0,0 @@ -packages: - - apps/* - - packages/* - - workers/* -allowBuilds: - esbuild: true - sharp: true - workerd: true diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..3121924 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,2 @@ +max_width = 150 +use_small_heuristics = "Max" diff --git a/scripts/e2e.sh b/scripts/e2e.sh new file mode 100755 index 0000000..5cac065 --- /dev/null +++ b/scripts/e2e.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash +# End-to-end test: start helios-server on a scratch data dir, push a real +# closure with the CLI, then have Nix substitute it back into a fresh store +# with signature checking on. +# +# Usage: scripts/e2e.sh [store-path] (run inside `nix develop`, after `cargo build --release`) +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +BIN="${BIN:-$ROOT/target/release}" +PORT="${PORT:-18080}" +URL="http://127.0.0.1:$PORT" +TARGET="${1:-$(readlink -f "$(command -v zstd)" | cut -d/ -f1-4)}" + +WORK="$(mktemp -d)" +cleanup() { + [ -n "${SERVER_PID:-}" ] && kill "$SERVER_PID" 2>/dev/null || true + chmod -R u+w "$WORK" 2>/dev/null || true + rm -rf "$WORK" +} +trap cleanup EXIT + +PASS=0 +FAIL=0 +check() { + if [ "$2" = "$3" ]; then + echo " ok: $1" + PASS=$((PASS + 1)) + else + echo " FAIL: $1 (expected=$2 got=$3)" + FAIL=$((FAIL + 1)) + fi +} +status() { curl -s -o /dev/null -w '%{http_code}' "$@"; } + +nix key generate-secret --key-name e2e-1 >"$WORK/key" +head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' >"$WORK/jwt" +head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' >"$WORK/admin" +PUBKEY="$("$BIN/helios-server" --print-public-key --signing-key-file "$WORK/key")" + +# Debug logs each request, which the chunked-upload checks count. +HELIOS_LOG=warn,helios_server=debug "$BIN/helios-server" 2>"$WORK/server.log" \ + --listen "127.0.0.1:$PORT" \ + --data-dir "$WORK/data" \ + --signing-key-file "$WORK/key" \ + --jwt-secret-file "$WORK/jwt" \ + --admin-secret-file "$WORK/admin" \ + --admin-socket "$WORK/admin.sock" & +SERVER_PID=$! +for _ in $(seq 50); do curl -sf "$URL/healthz" >/dev/null && break; sleep 0.1; done + +export XDG_CONFIG_HOME="$WORK/config" +# Nix caches narinfo per substituter URL; keep runs (and their keys) apart. +export XDG_CACHE_HOME="$WORK/cache" +helios() { "$BIN/helios" "$@"; } + +echo "=== admin" +helios login admin "$URL" "$(cat "$WORK/admin")" >/dev/null +helios cache create main >/dev/null +helios cache create secret --private >/dev/null +helios cache create chunked >/dev/null +helios cache create plain >/dev/null +PUSH_TOKEN="$(helios token create ci --caches main,secret,chunked,plain --perms push,pull 2>/dev/null | jq -r .token)" +PULL_ONLY="$(helios token create reader --caches secret 2>/dev/null | jq -r .token)" +PUSH_ONLY="$(helios token create builder --caches secret --perms push 2>/dev/null | jq -r .token)" +check "tokens are read-only by default" '["pull"]' "$(helios token list | jq -c '.tokens[] | select(.subject=="reader") | .perms')" +check "admin rejects push token" 403 "$(status -H "authorization: Bearer $PUSH_TOKEN" "$URL/_api/v2/admin/tokens")" + +echo "=== chunked upload" + +helios login ci "$URL" "$PUSH_TOKEN" >/dev/null +# 1 MiB chunks: the closure's larger NARs go in pieces, the rest in one request. +helios push chunked --closure --chunk-size 1 "$TARGET" 2>&1 | tail -1 +check "large NARs were sent in chunks" true "$([ "$(grep -c 'method=PATCH' "$WORK/server.log")" -gt 1 ] && echo true)" +NIX_CONFIG="trusted-public-keys = $PUBKEY +require-sigs = true" nix copy --from "$URL/chunked" --to "$WORK/chunked-store" "$TARGET" 2>&1 | tail -3 +check "chunked closure substitutes" "$(nix-store -qR "$TARGET" | wc -l)" "$(nix-store --store "$WORK/chunked-store" -qR "$TARGET" | wc -l)" + +# Level 0: raw NARs, served as such. A fresh path, so no NAR is reused. +head -c 100000 /dev/urandom >"$WORK/plain-file" +PLAIN="$(nix-store --add "$WORK/plain-file")" +helios push plain --level 0 "$PLAIN" 2>&1 | tail -1 +check "level 0 publishes uncompressed NARs" "none" "$(curl -s "$URL/plain/$(basename "$PLAIN" | cut -c1-32).narinfo" | sed -n 's/^Compression: //p')" +NIX_CONFIG="trusted-public-keys = $PUBKEY +require-sigs = true" nix copy --from "$URL/plain" --to "$WORK/plain-store" "$PLAIN" 2>&1 | tail -3 +check "an uncompressed NAR substitutes" "$(nix-store -q --hash "$PLAIN")" "$(nix-store --store "$WORK/plain-store" -q --hash "$PLAIN")" +check "the server advertises compression defaults to pushers" '3 27' "$(curl -s -H "authorization: Bearer $PUSH_ONLY" "$URL/_api/v2/caches/secret" | jq -r '"\(.compression.level) \(.compression.windowLog)"')" + +AUTH=(-H "authorization: Bearer $PUSH_TOKEN") +new_upload() { curl -s -X POST "${AUTH[@]}" "$URL/_api/v2/caches/chunked/uploads" | jq -r .id; } +NAR="$WORK/one.nar.zst" +nix-store --dump "$TARGET" | zstd -q >"$NAR" +NAR_BYTES="$(stat -c %s "$NAR")" +HALF=$((NAR_BYTES / 2)) +ID="$(new_upload)" +head -c "$HALF" "$NAR" >"$WORK/part1" +tail -c +$((HALF + 1)) "$NAR" >"$WORK/part2" +check "chunk at the wrong offset" 409 "$(status -X PATCH "${AUTH[@]}" --data-binary @"$WORK/part2" "$URL/_api/v2/caches/chunked/uploads/$ID?offset=$HALF")" +curl -s -o /dev/null -X PATCH "${AUTH[@]}" --data-binary @"$WORK/part1" "$URL/_api/v2/caches/chunked/uploads/$ID?offset=0" +check "resume reads the offset" "$HALF" "$(curl -s "${AUTH[@]}" "$URL/_api/v2/caches/chunked/uploads/$ID" | jq .offset)" +check "another token cannot see the upload" 404 "$(status -H "authorization: Bearer $PUSH_ONLY" "$URL/_api/v2/caches/secret/uploads/$ID")" +curl -s -o /dev/null -X PATCH "${AUTH[@]}" --data-binary @"$WORK/part2" "$URL/_api/v2/caches/chunked/uploads/$ID?offset=$HALF" +EXPECT="sha256:$(nix-store -q --hash "$TARGET" | cut -d: -f2)" +check "completed upload has the NAR hash" "$EXPECT" "$(curl -s -X POST "${AUTH[@]}" "$URL/_api/v2/caches/chunked/uploads/$ID/complete" | jq -r .narHash)" +check "completed upload is gone" 404 "$(status "${AUTH[@]}" "$URL/_api/v2/caches/chunked/uploads/$ID")" +ID="$(new_upload)" +check "garbage chunk rejected" 400 "$(status -X PATCH "${AUTH[@]}" --data-binary 'not zstd' "$URL/_api/v2/caches/chunked/uploads/$ID?offset=0")" +check "rejected upload is dropped" 404 "$(status "${AUTH[@]}" "$URL/_api/v2/caches/chunked/uploads/$ID")" + +echo "=== push $TARGET" +helios login ci "$URL" "$PUSH_TOKEN" >/dev/null +helios push main --closure "$TARGET" +CLOSURE_SIZE="$(nix-store -qR "$TARGET" | wc -l)" +AGAIN="$(helios push main --closure "$TARGET" 2>&1)" +check "second push is a no-op" "all $CLOSURE_SIZE paths already in 'main'" "$AGAIN" + +echo "=== read path" +HASH="$(basename "$TARGET" | cut -c1-32)" +check "narinfo HEAD" 200 "$(status -I "$URL/main/$HASH.narinfo")" +check "narinfo miss" 404 "$(status "$URL/main/00000000000000000000000000000000.narinfo")" +check "unknown cache" 404 "$(status "$URL/nope/$HASH.narinfo")" +NAR_URL="$(curl -s "$URL/main/$HASH.narinfo" | sed -n 's/^URL: //p')" +check "nar not served from a cache that lacks it" 404 "$(status -H "authorization: Bearer $PUSH_TOKEN" "$URL/secret/$NAR_URL")" + +echo "=== substitute into a fresh store (signatures required)" +NIX_CONFIG="trusted-public-keys = $PUBKEY +require-sigs = true" nix copy --from "$URL/main" --to "$WORK/store" "$TARGET" 2>&1 | tail -3 +check "substituted closure is valid" "$CLOSURE_SIZE" "$(nix-store --store "$WORK/store" -qR "$TARGET" | wc -l)" +nix store verify --store "$WORK/store" --trusted-public-keys "$PUBKEY" -r "$TARGET" && check "nix store verify" 0 0 + +echo "=== private cache" +helios push secret "$TARGET" >/dev/null 2>&1 +check "private narinfo without auth" 401 "$(status "$URL/secret/$HASH.narinfo")" +check "private narinfo with netrc-style basic auth" 200 "$(status -u "reader:$PULL_ONLY" "$URL/secret/$HASH.narinfo")" +check "pull-only token cannot push" 403 "$(status -X POST -H "authorization: Bearer $PULL_ONLY" -H 'content-type: application/json' -d '{"hashes":[]}' "$URL/_api/v2/caches/secret/missing")" +check "push-only token cannot read" 403 "$(status -u "builder:$PUSH_ONLY" "$URL/secret/$HASH.narinfo")" +check "refusals say why" "token does not grant push on cache 'secret'" "$(curl -s -X POST -H "authorization: Bearer $PULL_ONLY" -H 'content-type: application/json' -d '{"hashes":[]}' "$URL/_api/v2/caches/secret/missing" | jq -r .error)" + +echo "=== helios use" +helios login reader "$URL" "$PULL_ONLY" >/dev/null 2>&1 +helios use secret 2>/dev/null +helios use secret 2>/dev/null +NIXCONF="$XDG_CONFIG_HOME/nix/nix.conf" +check "nix.conf gets the login URL, once" 1 "$(grep -cx "extra-substituters = $URL/secret" "$NIXCONF")" +check "netrc is private" 600 "$(stat -c %a "$XDG_CONFIG_HOME/nix/netrc")" +check "netrc holds the token once" 1 "$(grep -c "password $PULL_ONLY" "$XDG_CONFIG_HOME/nix/netrc")" +# Signatures checked against the configured key, fetched with netrc auth. +nix store verify --no-contents --store "$URL/secret" "$TARGET" && check "nix reads the private cache with this config" 0 0 +check "and not without it" fails "$(XDG_CONFIG_HOME="$WORK/empty" nix store verify --no-contents --store "$URL/secret" "$TARGET" >/dev/null 2>&1 || echo fails)" +check "--print shows system-wide settings" 1 "$(helios use secret --print | grep -c "^machine 127.0.0.1 *\$")" +helios login ci "$URL" "$PUSH_TOKEN" >/dev/null 2>&1 + +echo "=== pins" +# Auto-GC takes its candidates from /v1/lru; main is cache 1, chunked cache 3. +lru_count() { curl -s --unix-socket "$WORK/admin.sock" "http://x/v1/lru?limit=10000" | jq --argjson c "$1" --argjson closure "$(nix-store -qR "$TARGET" | jq -R . | jq -s .)" '[.paths[] | select(.cache == $c and ((.storePath | if startswith("/nix/store/") then . else "/nix/store/" + . end) as $p | $closure | index($p)))] | length'; } +check "the closure starts as a GC candidate" "$CLOSURE_SIZE" "$(lru_count 1)" +helios pin main "$TARGET" 2>/dev/null +check "pins are listed" "$TARGET" "$(helios pins main | jq -r '.pins[0].storePath')" +check "a pinned closure is not a GC candidate" 0 "$(lru_count 1)" +check "the same paths in another cache still are" "$CLOSURE_SIZE" "$(lru_count 3)" +check "pull-only token cannot pin" 403 "$(status -X POST -H "authorization: Bearer $PULL_ONLY" -H 'content-type: application/json' -d "{\"storePaths\":[\"$TARGET\"]}" "$URL/_api/v2/caches/secret/pins")" +check "invalid pin rejected" 400 "$(status -X POST "${AUTH[@]}" -H 'content-type: application/json' -d '{"storePaths":["/etc/passwd"]}' "$URL/_api/v2/caches/main/pins")" +helios unpin main "$TARGET" 2>/dev/null +check "unpinned closure is a candidate again" "$CLOSURE_SIZE" "$(lru_count 1)" + +echo "=== private NARs stay private" +OTHER="$(helios --server admin token create other --caches main --perms push 2>/dev/null | jq -r .token)" +head -c 4096 /dev/urandom >"$WORK/secret-file" +nix-store --dump "$WORK/secret-file" | zstd -q >"$WORK/secret.nar.zst" +SECRET_NAR="$(curl -s -X PUT -H "authorization: Bearer $PUSH_TOKEN" --data-binary @"$WORK/secret.nar.zst" "$URL/_api/v2/caches/secret/nar" | jq -r .narHash)" +SECRET_SIZE="$(nix-store --dump "$WORK/secret-file" | wc -c)" +FAKE_PATH="/nix/store/$(printf 'a%.0s' $(seq 32))-leak" +LEAK="{\"paths\":[{\"storePath\":\"$FAKE_PATH\",\"narHash\":\"$SECRET_NAR\",\"narSize\":$SECRET_SIZE}]}" +check "another cache does not see a private NAR as known" "[]" "$(curl -s -X POST -H "authorization: Bearer $OTHER" -H 'content-type: application/json' -d "{\"narHashes\":[\"$SECRET_NAR\"]}" "$URL/_api/v2/caches/main/nars/known" | jq -c .known)" +check "nor can it publish a path over it" 409 "$(status -X POST -H "authorization: Bearer $OTHER" -H 'content-type: application/json' -d "$LEAK" "$URL/_api/v2/caches/main/paths")" +curl -s -o /dev/null -X PUT -H "authorization: Bearer $OTHER" --data-binary @"$WORK/secret.nar.zst" "$URL/_api/v2/caches/main/nar" +check "uploading the NAR itself entitles it" 201 "$(status -X POST -H "authorization: Bearer $OTHER" -H 'content-type: application/json' -d "$LEAK" "$URL/_api/v2/caches/main/paths")" + +echo "=== build traces" +OUT_BASE="$(basename "$TARGET")" +HEX="$(printf '%064d' 0 | tr 0 a)" +traces() { curl -s -o /dev/null -w '%{http_code}' -X POST "${AUTH[@]}" -H 'content-type: application/json' -d "{\"entries\":[$1]}" "$URL/_api/v2/caches/main/build-traces"; } +check "a current-format trace publishes" 201 "$(traces "{\"key\":{\"drvPath\":\"$HASH-x.drv\",\"outputName\":\"out\"},\"value\":{\"outPath\":\"$OUT_BASE\",\"signatures\":[]}}")" +check "a legacy trace publishes" 201 "$(traces "{\"id\":\"sha256:$HEX!out\",\"outPath\":\"$OUT_BASE\",\"signatures\":[],\"dependentRealisations\":{}}")" +check "a trace needs its output published" 409 "$(traces "{\"id\":\"sha256:$HEX!dev\",\"outPath\":\"00000000000000000000000000000000-x\"}")" +check "an invalid trace id is refused" 400 "$(traces "{\"id\":\"sha1:abc!out\",\"outPath\":\"$OUT_BASE\"}")" +CURRENT="$(curl -sf "$URL/main/build-trace-v2/$HASH-x.drv/out.doi")" +check "current trace served, signed by the cache" "$OUT_BASE ${PUBKEY%%:*}" "$(echo "$CURRENT" | jq -r '.outPath + " " + .signatures[0].keyName')" +LEGACY="$(curl -sf "$URL/main/realisations/sha256:$HEX!out.doi")" +check "legacy trace served, signed by the cache" "sha256:$HEX!out ${PUBKEY%%:*}" "$(echo "$LEGACY" | jq -r '.id + " " + (.signatures[0] | split(":")[0])')" +check "legacy trace served percent-encoded" 200 "$(status "$URL/main/realisations/sha256%3A$HEX%21out.doi")" +check "each format only under its prefix" 404 "$(status "$URL/main/build-trace-v2/sha256:$HEX!out.doi")" + +echo "=== abuse" +check "garbage upload rejected" 400 "$(status -X PUT -H "authorization: Bearer $PUSH_TOKEN" --data-binary 'not a zstd stream' "$URL/_api/v2/caches/main/nar")" +check "raw non-NAR upload rejected" 400 "$(status -X PUT -H "authorization: Bearer $PUSH_TOKEN" --data-binary 'hello' "$URL/_api/v2/caches/main/nar?compression=none")" +INJECT='{"paths":[{"storePath":"/nix/store/'"$HASH"'-x\nSig: evil","narHash":"sha256:'"$(printf 0%.0s $(seq 52))"'","narSize":1}]}' +check "narinfo injection rejected" 400 "$(status -X POST -H "authorization: Bearer $PUSH_TOKEN" -H 'content-type: application/json' -d "$INJECT" "$URL/_api/v2/caches/main/paths")" +FAKE='{"paths":[{"storePath":"/nix/store/00000000000000000000000000000000-fake","narHash":"sha256:'"$(printf 0%.0s $(seq 52))"'","narSize":1}]}' +check "publish without uploaded NAR" 409 "$(status -X POST -H "authorization: Bearer $PUSH_TOKEN" -H 'content-type: application/json' -d "$FAKE" "$URL/_api/v2/caches/main/paths")" +JTI="$(helios --server admin token list | jq -r '.tokens[] | select(.subject=="reader") | .jti')" +helios --server admin token revoke "$JTI" "e2e" >/dev/null +check "revocation is immediate" 401 "$(status -u "reader:$PULL_ONLY" "$URL/secret/$HASH.narinfo")" +check "revoked token says so" "token has been revoked" "$(curl -s -u "reader:$PULL_ONLY" "$URL/secret/$HASH.narinfo" | jq -r .error)" + +echo +echo "passed=$PASS failed=$FAIL" +[ "$FAIL" -eq 0 ] diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh deleted file mode 100755 index dd2f7d5..0000000 --- a/scripts/smoke-test.sh +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Smoke test for the Helios Nix binary cache worker. -# -# Usage: -# ./scripts/smoke-test.sh -# -# Example: -# ./scripts/smoke-test.sh https://helios-cache.your-subdomain.workers.dev "your-token" -# ./scripts/smoke-test.sh https://cache.polmath.no "your-token" - -BASE="${1:?usage: smoke-test.sh }" -TOKEN="${2:?usage: smoke-test.sh }" - -# Strip trailing slash -BASE="${BASE%/}" - -# Resolve worker dir for wrangler d1 seeding -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -WORKER_DIR="$(cd "$SCRIPT_DIR/../workers/cache" && pwd)" - -PASS=0 -FAIL=0 - -check() { - local desc="$1" expected="$2" actual="$3" - if [ "$actual" = "$expected" ]; then - echo " ok: $desc" - PASS=$((PASS + 1)) - else - echo " FAIL: $desc (expected=$expected got=$actual)" - FAIL=$((FAIL + 1)) - fi -} - -echo "=== Smoke test against $BASE ===" -echo - -# ── 1. Health check ── -echo "1. Health check" -STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/healthz") -check "/healthz returns 200" "200" "$STATUS" - -BODY=$(curl -s "$BASE/healthz") -check "/healthz has ok=true" "true" "$(echo "$BODY" | grep -o '"ok" *: *true' | head -1 | grep -o 'true')" -echo - -# ── 2. Seed a cache via D1 ── -echo "2. Seeding test cache via D1" -CACHE_NAME="smoke-$(date +%s)" -wrangler d1 execute helios-cache --remote --yes \ - -c "$WORKER_DIR/wrangler.jsonc" \ - --command "INSERT OR IGNORE INTO caches (name, is_public) VALUES ('$CACHE_NAME', 1)" \ - > /dev/null 2>&1 -echo " created cache: $CACHE_NAME" -echo - -# ── 3. Cache info ── -echo "3. nix-cache-info" -STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/$CACHE_NAME/nix-cache-info") -check "returns 200" "200" "$STATUS" - -BODY=$(curl -s "$BASE/$CACHE_NAME/nix-cache-info") -check "contains StoreDir" "StoreDir: /nix/store" "$(echo "$BODY" | head -1)" -echo - -# ── 4. 404 for missing narinfo ── -echo "4. Missing narinfo returns 404" -STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/$CACHE_NAME/00000000000000000000000000000000.narinfo") -check "returns 404" "404" "$STATUS" -echo - -# ── 5. Full upload → publish → read flow ── -echo "5. Upload, complete, publish, read" - -FILE_HASH="$(openssl rand -hex 32)" -STORE_PATH_HASH="00000000000000000000000000000000" -NAR_HASH="sha256:1111111111111111111111111111111111111111111111111111" -NAR_CONTENT="nix-archive-smoke-test-payload1234" -FILE_SIZE=${#NAR_CONTENT} - -# 5a. Create upload session -echo " 5a. Creating upload session" -SESSION_STATUS=$(curl -s -o /tmp/smoke-session.json -w "%{http_code}" -X POST "$BASE/_api/v1/caches/$CACHE_NAME/upload-sessions" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d "{ - \"storePath\": \"/nix/store/${STORE_PATH_HASH}-smoke-test\", - \"storePathHash\": \"$STORE_PATH_HASH\", - \"narHash\": \"$NAR_HASH\", - \"narSize\": 2048, - \"fileHash\": \"$FILE_HASH\", - \"fileSize\": $FILE_SIZE, - \"compression\": \"none\" - }") - -SESSION_RESP=$(cat /tmp/smoke-session.json) -rm -f /tmp/smoke-session.json -SESSION_ID=$(echo "$SESSION_RESP" | grep -o '"sessionId":"[^"]*"' | cut -d'"' -f4) -R2_KEY=$(echo "$SESSION_RESP" | grep -o '"r2Key":"[^"]*"' | cut -d'"' -f4) - -if [ -z "$SESSION_ID" ]; then - echo " FAIL: could not create session (status=$SESSION_STATUS): $SESSION_RESP" - FAIL=$((FAIL + 1)) -else - check "session created (201)" "201" "$SESSION_STATUS" - echo " session=$SESSION_ID r2Key=$R2_KEY" - - # 5b. Upload blob via API - echo " 5b. Uploading blob via API" - UPLOAD_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ - -X PUT "$BASE/_api/v1/uploads/$SESSION_ID/blob" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/octet-stream" \ - --data-binary "$NAR_CONTENT") - check "blob upload returns 200" "200" "$UPLOAD_STATUS" - - # 5c. Complete - echo " 5c. Completing upload" - COMPLETE_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ - "$BASE/_api/v1/uploads/$SESSION_ID/complete" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{}') - check "complete returns 200" "200" "$COMPLETE_STATUS" - - # 5d. Publish - echo " 5d. Publishing path" - PUBLISH_RESP=$(curl -s -X POST "$BASE/_api/v1/uploads/$SESSION_ID/publish" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{}') - PUBLISH_STATUS=$(echo "$PUBLISH_RESP" | grep -o '"published":true' | head -1) - check "publish returns published=true" '"published":true' "$PUBLISH_STATUS" - - # 5e. Read narinfo - echo " 5e. Reading narinfo" - NARINFO_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/$CACHE_NAME/$STORE_PATH_HASH.narinfo") - check "narinfo returns 200" "200" "$NARINFO_STATUS" - - NARINFO=$(curl -s "$BASE/$CACHE_NAME/$STORE_PATH_HASH.narinfo") - check "narinfo contains StorePath" "/nix/store/${STORE_PATH_HASH}-smoke-test" \ - "$(echo "$NARINFO" | grep '^StorePath:' | cut -d' ' -f2)" - check "narinfo contains NarHash" "$NAR_HASH" \ - "$(echo "$NARINFO" | grep '^NarHash:' | cut -d' ' -f2)" - - # 5f. Read NAR blob - echo " 5f. Reading NAR blob" - NAR_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/$CACHE_NAME/nar/$FILE_HASH/none.nar") - check "NAR download returns 200" "200" "$NAR_STATUS" - - NAR_BODY=$(curl -s "$BASE/$CACHE_NAME/nar/$FILE_HASH/none.nar") - check "NAR body matches uploaded content" "$NAR_CONTENT" "$NAR_BODY" - - # 5g. Idempotent publish - echo " 5g. Duplicate publish is idempotent" - REPUB=$(curl -s -X POST "$BASE/_api/v1/uploads/$SESSION_ID/publish" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{}') - check "re-publish has alreadyExisted" '"alreadyExisted":true' \ - "$(echo "$REPUB" | grep -o '"alreadyExisted":true' | head -1)" -fi -echo - -# ── 6. Auth enforcement ── -echo "6. Auth enforcement" -UNAUTH=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ - "$BASE/_api/v1/caches/$CACHE_NAME/upload-sessions" \ - -H "Content-Type: application/json" \ - -d '{}') -check "write without auth returns 401" "401" "$UNAUTH" - -BAD_AUTH=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ - "$BASE/_api/v1/caches/$CACHE_NAME/upload-sessions" \ - -H "Authorization: Bearer wrong-token" \ - -H "Content-Type: application/json" \ - -d '{}') -check "write with bad token returns 403" "403" "$BAD_AUTH" - -PUBLIC_READ=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/$CACHE_NAME/nix-cache-info") -check "read without auth returns 200" "200" "$PUBLIC_READ" -echo - -# ── 7. get-missing-paths ── -echo "7. get-missing-paths" -MISSING_RESP=$(curl -s -X POST "$BASE/_api/v1/get-missing-paths" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d "{\"cache\": \"$CACHE_NAME\", \"storePathHashes\": [\"$STORE_PATH_HASH\", \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"]}") -check "published hash not in missing" "" \ - "$(echo "$MISSING_RESP" | grep -o "\"$STORE_PATH_HASH\"" | head -1)" -check "unknown hash is in missing" '"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"' \ - "$(echo "$MISSING_RESP" | grep -o '"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"' | head -1)" -echo - -# ── Summary ── -echo "=== Results: $PASS passed, $FAIL failed ===" -if [ "$FAIL" -gt 0 ]; then - exit 1 -fi diff --git a/scripts/test-zig.sh b/scripts/test-zig.sh new file mode 100755 index 0000000..65a0205 --- /dev/null +++ b/scripts/test-zig.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Runs the unit tests of every Zig package, then builds libhelios. Tests run +# in Debug and in ReleaseFast, which libhelios ships with: std skips some +# checks without runtime safety. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +for pkg in "$ROOT"/pkg/*/; do + for mode in Debug ReleaseFast; do + echo "== $(basename "$pkg") ($mode)" + (cd "$pkg" && zig build test -Doptimize=$mode -Dcpu=native --summary failures) + done +done +echo "== libhelios" +(cd "$ROOT/core" && zig build -Doptimize=ReleaseFast -Dcpu=native) diff --git a/workers/cache/.editorconfig b/workers/cache/.editorconfig deleted file mode 100644 index a727df3..0000000 --- a/workers/cache/.editorconfig +++ /dev/null @@ -1,12 +0,0 @@ -# http://editorconfig.org -root = true - -[*] -indent_style = tab -end_of_line = lf -charset = utf-8 -trim_trailing_whitespace = true -insert_final_newline = true - -[*.yml] -indent_style = space diff --git a/workers/cache/.gitignore b/workers/cache/.gitignore deleted file mode 100644 index 4138168..0000000 --- a/workers/cache/.gitignore +++ /dev/null @@ -1,167 +0,0 @@ -# Logs - -logs -_.log -npm-debug.log_ -yarn-debug.log* -yarn-error.log* -lerna-debug.log* -.pnpm-debug.log* - -# Diagnostic reports (https://nodejs.org/api/report.html) - -report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json - -# Runtime data - -pids -_.pid -_.seed -\*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover - -lib-cov - -# Coverage directory used by tools like istanbul - -coverage -\*.lcov - -# nyc test coverage - -.nyc_output - -# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) - -.grunt - -# Bower dependency directory (https://bower.io/) - -bower_components - -# node-waf configuration - -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) - -build/Release - -# Dependency directories - -node_modules/ -jspm_packages/ - -# Snowpack dependency directory (https://snowpack.dev/) - -web_modules/ - -# TypeScript cache - -\*.tsbuildinfo - -# Optional npm cache directory - -.npm - -# Optional eslint cache - -.eslintcache - -# Optional stylelint cache - -.stylelintcache - -# Microbundle cache - -.rpt2_cache/ -.rts2_cache_cjs/ -.rts2_cache_es/ -.rts2_cache_umd/ - -# Optional REPL history - -.node_repl_history - -# Output of 'npm pack' - -\*.tgz - -# Yarn Integrity file - -.yarn-integrity - -# parcel-bundler cache (https://parceljs.org/) - -.cache -.parcel-cache - -# Next.js build output - -.next -out - -# Nuxt.js build / generate output - -.nuxt -dist - -# Gatsby files - -.cache/ - -# Comment in the public line in if your project uses Gatsby and not Next.js - -# https://nextjs.org/blog/next-9-1#public-directory-support - -# public - -# vuepress build output - -.vuepress/dist - -# vuepress v2.x temp and cache directory - -.temp -.cache - -# Docusaurus cache and generated files - -.docusaurus - -# Serverless directories - -.serverless/ - -# FuseBox cache - -.fusebox/ - -# DynamoDB Local files - -.dynamodb/ - -# TernJS port file - -.tern-port - -# Stores VSCode versions used for testing VSCode extensions - -.vscode-test - -# yarn v2 - -.yarn/cache -.yarn/unplugged -.yarn/build-state.yml -.yarn/install-state.gz -.pnp.\* - -# wrangler project - -.dev.vars* -!.dev.vars.example -.env* -!.env.example -.wrangler/ diff --git a/workers/cache/.prettierrc b/workers/cache/.prettierrc deleted file mode 100644 index 5c7b5d3..0000000 --- a/workers/cache/.prettierrc +++ /dev/null @@ -1,6 +0,0 @@ -{ - "printWidth": 140, - "singleQuote": true, - "semi": true, - "useTabs": true -} diff --git a/workers/cache/drizzle.config.ts b/workers/cache/drizzle.config.ts deleted file mode 100644 index d4cfa11..0000000 --- a/workers/cache/drizzle.config.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { defineConfig } from "drizzle-kit"; - -export default defineConfig({ - schema: "./src/db/schema.ts", - out: "./migrations", - dialect: "sqlite", -}); diff --git a/workers/cache/migrations/0000_cultured_night_nurse.sql b/workers/cache/migrations/0000_cultured_night_nurse.sql deleted file mode 100644 index 434ee57..0000000 --- a/workers/cache/migrations/0000_cultured_night_nurse.sql +++ /dev/null @@ -1,79 +0,0 @@ -CREATE TABLE `blob_objects` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `file_hash` text NOT NULL, - `file_size` integer NOT NULL, - `compression` text NOT NULL, - `r2_key` text NOT NULL, - `created_at` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL -); ---> statement-breakpoint -CREATE UNIQUE INDEX `blob_objects_r2_key_unique` ON `blob_objects` (`r2_key`);--> statement-breakpoint -CREATE UNIQUE INDEX `blob_objects_file_hash_compression_unique` ON `blob_objects` (`file_hash`,`compression`);--> statement-breakpoint -CREATE TABLE `caches` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `name` text NOT NULL, - `is_public` integer DEFAULT 1 NOT NULL, - `created_at` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL -); ---> statement-breakpoint -CREATE UNIQUE INDEX `caches_name_unique` ON `caches` (`name`);--> statement-breakpoint -CREATE TABLE `gc_marks` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `target_type` text NOT NULL, - `target_id` text NOT NULL, - `reason` text NOT NULL, - `marked_at` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL -); ---> statement-breakpoint -CREATE UNIQUE INDEX `gc_marks_target_unique` ON `gc_marks` (`target_type`,`target_id`);--> statement-breakpoint -CREATE TABLE `published_paths` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `cache_id` integer NOT NULL, - `store_path_hash` text NOT NULL, - `store_path` text NOT NULL, - `nar_hash` text NOT NULL, - `nar_size` integer NOT NULL, - `blob_object_id` integer NOT NULL, - `references_json` text DEFAULT '[]' NOT NULL, - `deriver` text, - `system` text, - `signatures_json` text DEFAULT '[]' NOT NULL, - `created_at` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL, - FOREIGN KEY (`cache_id`) REFERENCES `caches`(`id`) ON UPDATE no action ON DELETE no action, - FOREIGN KEY (`blob_object_id`) REFERENCES `blob_objects`(`id`) ON UPDATE no action ON DELETE no action -); ---> statement-breakpoint -CREATE UNIQUE INDEX `published_paths_cache_store_unique` ON `published_paths` (`cache_id`,`store_path_hash`);--> statement-breakpoint -CREATE INDEX `idx_published_paths_lookup` ON `published_paths` (`cache_id`,`store_path_hash`);--> statement-breakpoint -CREATE TABLE `upload_parts` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `session_id` text NOT NULL, - `part_number` integer NOT NULL, - `etag` text NOT NULL, - `size` integer NOT NULL, - FOREIGN KEY (`session_id`) REFERENCES `upload_sessions`(`id`) ON UPDATE no action ON DELETE no action -); ---> statement-breakpoint -CREATE UNIQUE INDEX `upload_parts_session_part_unique` ON `upload_parts` (`session_id`,`part_number`);--> statement-breakpoint -CREATE TABLE `upload_sessions` ( - `id` text PRIMARY KEY NOT NULL, - `cache_id` integer NOT NULL, - `store_path_hash` text NOT NULL, - `store_path` text NOT NULL, - `nar_hash` text NOT NULL, - `nar_size` integer NOT NULL, - `file_hash` text NOT NULL, - `file_size` integer NOT NULL, - `compression` text NOT NULL, - `references_json` text DEFAULT '[]' NOT NULL, - `deriver` text, - `system` text, - `status` text DEFAULT 'pending' NOT NULL, - `r2_upload_key` text, - `r2_upload_id` text, - `created_at` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL, - `expires_at` text NOT NULL, - FOREIGN KEY (`cache_id`) REFERENCES `caches`(`id`) ON UPDATE no action ON DELETE no action -); ---> statement-breakpoint -CREATE INDEX `idx_upload_sessions_status` ON `upload_sessions` (`status`,`expires_at`); \ No newline at end of file diff --git a/workers/cache/migrations/0001_overjoyed_exiles.sql b/workers/cache/migrations/0001_overjoyed_exiles.sql deleted file mode 100644 index a68632c..0000000 --- a/workers/cache/migrations/0001_overjoyed_exiles.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP INDEX `idx_published_paths_lookup`;--> statement-breakpoint -CREATE INDEX `idx_published_paths_blob_object` ON `published_paths` (`blob_object_id`); \ No newline at end of file diff --git a/workers/cache/migrations/0002_ordinary_wiccan.sql b/workers/cache/migrations/0002_ordinary_wiccan.sql deleted file mode 100644 index a8dae49..0000000 --- a/workers/cache/migrations/0002_ordinary_wiccan.sql +++ /dev/null @@ -1,27 +0,0 @@ -CREATE TABLE `api_tokens` ( - `jti` text PRIMARY KEY NOT NULL, - `subject` text NOT NULL, - `caches_json` text NOT NULL, - `perms_json` text NOT NULL, - `created_at` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL, - `expires_at` text NOT NULL, - `created_by` text NOT NULL, - `revoked_at` text, - `revoked_by` text, - `revocation_reason` text -); ---> statement-breakpoint -CREATE TABLE `audit_logs` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `timestamp` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL, - `actor` text NOT NULL, - `action` text NOT NULL, - `cache_name` text, - `detail` text DEFAULT '{}' NOT NULL, - `ip` text, - `status` integer NOT NULL -); ---> statement-breakpoint -CREATE INDEX `idx_audit_logs_actor` ON `audit_logs` (`actor`);--> statement-breakpoint -CREATE INDEX `idx_audit_logs_cache` ON `audit_logs` (`cache_name`);--> statement-breakpoint -CREATE INDEX `idx_audit_logs_time` ON `audit_logs` (`timestamp`); \ No newline at end of file diff --git a/workers/cache/migrations/meta/0000_snapshot.json b/workers/cache/migrations/meta/0000_snapshot.json deleted file mode 100644 index 2e46b24..0000000 --- a/workers/cache/migrations/meta/0000_snapshot.json +++ /dev/null @@ -1,554 +0,0 @@ -{ - "version": "6", - "dialect": "sqlite", - "id": "201f9766-f457-4259-b02e-f3aa0e4150c0", - "prevId": "00000000-0000-0000-0000-000000000000", - "tables": { - "blob_objects": { - "name": "blob_objects", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "file_hash": { - "name": "file_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_size": { - "name": "file_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "compression": { - "name": "compression", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "r2_key": { - "name": "r2_key", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "blob_objects_r2_key_unique": { - "name": "blob_objects_r2_key_unique", - "columns": [ - "r2_key" - ], - "isUnique": true - }, - "blob_objects_file_hash_compression_unique": { - "name": "blob_objects_file_hash_compression_unique", - "columns": [ - "file_hash", - "compression" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "caches": { - "name": "caches", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "is_public": { - "name": "is_public", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "caches_name_unique": { - "name": "caches_name_unique", - "columns": [ - "name" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "gc_marks": { - "name": "gc_marks", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "target_type": { - "name": "target_type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "target_id": { - "name": "target_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "reason": { - "name": "reason", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "marked_at": { - "name": "marked_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "gc_marks_target_unique": { - "name": "gc_marks_target_unique", - "columns": [ - "target_type", - "target_id" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "published_paths": { - "name": "published_paths", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "cache_id": { - "name": "cache_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path_hash": { - "name": "store_path_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path": { - "name": "store_path", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_hash": { - "name": "nar_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_size": { - "name": "nar_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "blob_object_id": { - "name": "blob_object_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "references_json": { - "name": "references_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "deriver": { - "name": "deriver", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "system": { - "name": "system", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "signatures_json": { - "name": "signatures_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "published_paths_cache_store_unique": { - "name": "published_paths_cache_store_unique", - "columns": [ - "cache_id", - "store_path_hash" - ], - "isUnique": true - }, - "idx_published_paths_lookup": { - "name": "idx_published_paths_lookup", - "columns": [ - "cache_id", - "store_path_hash" - ], - "isUnique": false - } - }, - "foreignKeys": { - "published_paths_cache_id_caches_id_fk": { - "name": "published_paths_cache_id_caches_id_fk", - "tableFrom": "published_paths", - "tableTo": "caches", - "columnsFrom": [ - "cache_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - }, - "published_paths_blob_object_id_blob_objects_id_fk": { - "name": "published_paths_blob_object_id_blob_objects_id_fk", - "tableFrom": "published_paths", - "tableTo": "blob_objects", - "columnsFrom": [ - "blob_object_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "upload_parts": { - "name": "upload_parts", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "session_id": { - "name": "session_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "part_number": { - "name": "part_number", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "etag": { - "name": "etag", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "size": { - "name": "size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "upload_parts_session_part_unique": { - "name": "upload_parts_session_part_unique", - "columns": [ - "session_id", - "part_number" - ], - "isUnique": true - } - }, - "foreignKeys": { - "upload_parts_session_id_upload_sessions_id_fk": { - "name": "upload_parts_session_id_upload_sessions_id_fk", - "tableFrom": "upload_parts", - "tableTo": "upload_sessions", - "columnsFrom": [ - "session_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "upload_sessions": { - "name": "upload_sessions", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "cache_id": { - "name": "cache_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path_hash": { - "name": "store_path_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path": { - "name": "store_path", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_hash": { - "name": "nar_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_size": { - "name": "nar_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_hash": { - "name": "file_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_size": { - "name": "file_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "compression": { - "name": "compression", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "references_json": { - "name": "references_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "deriver": { - "name": "deriver", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "system": { - "name": "system", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "r2_upload_key": { - "name": "r2_upload_key", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "r2_upload_id": { - "name": "r2_upload_id", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - }, - "expires_at": { - "name": "expires_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "idx_upload_sessions_status": { - "name": "idx_upload_sessions_status", - "columns": [ - "status", - "expires_at" - ], - "isUnique": false - } - }, - "foreignKeys": { - "upload_sessions_cache_id_caches_id_fk": { - "name": "upload_sessions_cache_id_caches_id_fk", - "tableFrom": "upload_sessions", - "tableTo": "caches", - "columnsFrom": [ - "cache_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - } - }, - "views": {}, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} - } -} \ No newline at end of file diff --git a/workers/cache/migrations/meta/0001_snapshot.json b/workers/cache/migrations/meta/0001_snapshot.json deleted file mode 100644 index a0812fe..0000000 --- a/workers/cache/migrations/meta/0001_snapshot.json +++ /dev/null @@ -1,553 +0,0 @@ -{ - "version": "6", - "dialect": "sqlite", - "id": "7e8a49f8-b25b-4ee7-8029-ba438ac76987", - "prevId": "201f9766-f457-4259-b02e-f3aa0e4150c0", - "tables": { - "blob_objects": { - "name": "blob_objects", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "file_hash": { - "name": "file_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_size": { - "name": "file_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "compression": { - "name": "compression", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "r2_key": { - "name": "r2_key", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "blob_objects_r2_key_unique": { - "name": "blob_objects_r2_key_unique", - "columns": [ - "r2_key" - ], - "isUnique": true - }, - "blob_objects_file_hash_compression_unique": { - "name": "blob_objects_file_hash_compression_unique", - "columns": [ - "file_hash", - "compression" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "caches": { - "name": "caches", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "is_public": { - "name": "is_public", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "caches_name_unique": { - "name": "caches_name_unique", - "columns": [ - "name" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "gc_marks": { - "name": "gc_marks", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "target_type": { - "name": "target_type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "target_id": { - "name": "target_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "reason": { - "name": "reason", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "marked_at": { - "name": "marked_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "gc_marks_target_unique": { - "name": "gc_marks_target_unique", - "columns": [ - "target_type", - "target_id" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "published_paths": { - "name": "published_paths", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "cache_id": { - "name": "cache_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path_hash": { - "name": "store_path_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path": { - "name": "store_path", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_hash": { - "name": "nar_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_size": { - "name": "nar_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "blob_object_id": { - "name": "blob_object_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "references_json": { - "name": "references_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "deriver": { - "name": "deriver", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "system": { - "name": "system", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "signatures_json": { - "name": "signatures_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "published_paths_cache_store_unique": { - "name": "published_paths_cache_store_unique", - "columns": [ - "cache_id", - "store_path_hash" - ], - "isUnique": true - }, - "idx_published_paths_blob_object": { - "name": "idx_published_paths_blob_object", - "columns": [ - "blob_object_id" - ], - "isUnique": false - } - }, - "foreignKeys": { - "published_paths_cache_id_caches_id_fk": { - "name": "published_paths_cache_id_caches_id_fk", - "tableFrom": "published_paths", - "tableTo": "caches", - "columnsFrom": [ - "cache_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - }, - "published_paths_blob_object_id_blob_objects_id_fk": { - "name": "published_paths_blob_object_id_blob_objects_id_fk", - "tableFrom": "published_paths", - "tableTo": "blob_objects", - "columnsFrom": [ - "blob_object_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "upload_parts": { - "name": "upload_parts", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "session_id": { - "name": "session_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "part_number": { - "name": "part_number", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "etag": { - "name": "etag", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "size": { - "name": "size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "upload_parts_session_part_unique": { - "name": "upload_parts_session_part_unique", - "columns": [ - "session_id", - "part_number" - ], - "isUnique": true - } - }, - "foreignKeys": { - "upload_parts_session_id_upload_sessions_id_fk": { - "name": "upload_parts_session_id_upload_sessions_id_fk", - "tableFrom": "upload_parts", - "tableTo": "upload_sessions", - "columnsFrom": [ - "session_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "upload_sessions": { - "name": "upload_sessions", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "cache_id": { - "name": "cache_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path_hash": { - "name": "store_path_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path": { - "name": "store_path", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_hash": { - "name": "nar_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_size": { - "name": "nar_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_hash": { - "name": "file_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_size": { - "name": "file_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "compression": { - "name": "compression", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "references_json": { - "name": "references_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "deriver": { - "name": "deriver", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "system": { - "name": "system", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "r2_upload_key": { - "name": "r2_upload_key", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "r2_upload_id": { - "name": "r2_upload_id", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - }, - "expires_at": { - "name": "expires_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "idx_upload_sessions_status": { - "name": "idx_upload_sessions_status", - "columns": [ - "status", - "expires_at" - ], - "isUnique": false - } - }, - "foreignKeys": { - "upload_sessions_cache_id_caches_id_fk": { - "name": "upload_sessions_cache_id_caches_id_fk", - "tableFrom": "upload_sessions", - "tableTo": "caches", - "columnsFrom": [ - "cache_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - } - }, - "views": {}, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} - } -} \ No newline at end of file diff --git a/workers/cache/migrations/meta/0002_snapshot.json b/workers/cache/migrations/meta/0002_snapshot.json deleted file mode 100644 index 694a7ac..0000000 --- a/workers/cache/migrations/meta/0002_snapshot.json +++ /dev/null @@ -1,724 +0,0 @@ -{ - "version": "6", - "dialect": "sqlite", - "id": "f6ad1415-55bd-4c46-8984-34fa48eaea35", - "prevId": "7e8a49f8-b25b-4ee7-8029-ba438ac76987", - "tables": { - "api_tokens": { - "name": "api_tokens", - "columns": { - "jti": { - "name": "jti", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "subject": { - "name": "subject", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "caches_json": { - "name": "caches_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "perms_json": { - "name": "perms_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - }, - "expires_at": { - "name": "expires_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_by": { - "name": "created_by", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "revoked_at": { - "name": "revoked_at", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "revoked_by": { - "name": "revoked_by", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "revocation_reason": { - "name": "revocation_reason", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "audit_logs": { - "name": "audit_logs", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "timestamp": { - "name": "timestamp", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - }, - "actor": { - "name": "actor", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "action": { - "name": "action", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "cache_name": { - "name": "cache_name", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "detail": { - "name": "detail", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'{}'" - }, - "ip": { - "name": "ip", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "idx_audit_logs_actor": { - "name": "idx_audit_logs_actor", - "columns": [ - "actor" - ], - "isUnique": false - }, - "idx_audit_logs_cache": { - "name": "idx_audit_logs_cache", - "columns": [ - "cache_name" - ], - "isUnique": false - }, - "idx_audit_logs_time": { - "name": "idx_audit_logs_time", - "columns": [ - "timestamp" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "blob_objects": { - "name": "blob_objects", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "file_hash": { - "name": "file_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_size": { - "name": "file_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "compression": { - "name": "compression", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "r2_key": { - "name": "r2_key", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "blob_objects_r2_key_unique": { - "name": "blob_objects_r2_key_unique", - "columns": [ - "r2_key" - ], - "isUnique": true - }, - "blob_objects_file_hash_compression_unique": { - "name": "blob_objects_file_hash_compression_unique", - "columns": [ - "file_hash", - "compression" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "caches": { - "name": "caches", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "is_public": { - "name": "is_public", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "caches_name_unique": { - "name": "caches_name_unique", - "columns": [ - "name" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "gc_marks": { - "name": "gc_marks", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "target_type": { - "name": "target_type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "target_id": { - "name": "target_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "reason": { - "name": "reason", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "marked_at": { - "name": "marked_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "gc_marks_target_unique": { - "name": "gc_marks_target_unique", - "columns": [ - "target_type", - "target_id" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "published_paths": { - "name": "published_paths", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "cache_id": { - "name": "cache_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path_hash": { - "name": "store_path_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path": { - "name": "store_path", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_hash": { - "name": "nar_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_size": { - "name": "nar_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "blob_object_id": { - "name": "blob_object_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "references_json": { - "name": "references_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "deriver": { - "name": "deriver", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "system": { - "name": "system", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "signatures_json": { - "name": "signatures_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "published_paths_cache_store_unique": { - "name": "published_paths_cache_store_unique", - "columns": [ - "cache_id", - "store_path_hash" - ], - "isUnique": true - }, - "idx_published_paths_blob_object": { - "name": "idx_published_paths_blob_object", - "columns": [ - "blob_object_id" - ], - "isUnique": false - } - }, - "foreignKeys": { - "published_paths_cache_id_caches_id_fk": { - "name": "published_paths_cache_id_caches_id_fk", - "tableFrom": "published_paths", - "tableTo": "caches", - "columnsFrom": [ - "cache_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - }, - "published_paths_blob_object_id_blob_objects_id_fk": { - "name": "published_paths_blob_object_id_blob_objects_id_fk", - "tableFrom": "published_paths", - "tableTo": "blob_objects", - "columnsFrom": [ - "blob_object_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "upload_parts": { - "name": "upload_parts", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "session_id": { - "name": "session_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "part_number": { - "name": "part_number", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "etag": { - "name": "etag", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "size": { - "name": "size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "upload_parts_session_part_unique": { - "name": "upload_parts_session_part_unique", - "columns": [ - "session_id", - "part_number" - ], - "isUnique": true - } - }, - "foreignKeys": { - "upload_parts_session_id_upload_sessions_id_fk": { - "name": "upload_parts_session_id_upload_sessions_id_fk", - "tableFrom": "upload_parts", - "tableTo": "upload_sessions", - "columnsFrom": [ - "session_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "upload_sessions": { - "name": "upload_sessions", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "cache_id": { - "name": "cache_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path_hash": { - "name": "store_path_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path": { - "name": "store_path", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_hash": { - "name": "nar_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_size": { - "name": "nar_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_hash": { - "name": "file_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_size": { - "name": "file_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "compression": { - "name": "compression", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "references_json": { - "name": "references_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "deriver": { - "name": "deriver", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "system": { - "name": "system", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "r2_upload_key": { - "name": "r2_upload_key", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "r2_upload_id": { - "name": "r2_upload_id", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - }, - "expires_at": { - "name": "expires_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "idx_upload_sessions_status": { - "name": "idx_upload_sessions_status", - "columns": [ - "status", - "expires_at" - ], - "isUnique": false - } - }, - "foreignKeys": { - "upload_sessions_cache_id_caches_id_fk": { - "name": "upload_sessions_cache_id_caches_id_fk", - "tableFrom": "upload_sessions", - "tableTo": "caches", - "columnsFrom": [ - "cache_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - } - }, - "views": {}, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} - } -} \ No newline at end of file diff --git a/workers/cache/migrations/meta/_journal.json b/workers/cache/migrations/meta/_journal.json deleted file mode 100644 index 2e084e2..0000000 --- a/workers/cache/migrations/meta/_journal.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "version": "7", - "dialect": "sqlite", - "entries": [ - { - "idx": 0, - "version": "6", - "when": 1776949256837, - "tag": "0000_cultured_night_nurse", - "breakpoints": true - }, - { - "idx": 1, - "version": "6", - "when": 1776966587890, - "tag": "0001_overjoyed_exiles", - "breakpoints": true - }, - { - "idx": 2, - "version": "6", - "when": 1777117423084, - "tag": "0002_ordinary_wiccan", - "breakpoints": true - } - ] -} \ No newline at end of file diff --git a/workers/cache/package.json b/workers/cache/package.json deleted file mode 100644 index a744c3e..0000000 --- a/workers/cache/package.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "@helios/cache-worker", - "version": "0.0.0", - "private": true, - "scripts": { - "deploy": "wrangler deploy", - "dev": "wrangler dev", - "test": "vitest run", - "cf-typegen": "wrangler types", - "check": "tsc --noEmit && vitest run" - }, - "devDependencies": { - "@cloudflare/vitest-pool-workers": "^0.18.6", - "@types/node": "^26.1.1", - "drizzle-kit": "^0.31.10", - "typescript": "^7.0.2", - "vitest": "~4.1.10", - "wrangler": "^4.112.0" - }, - "dependencies": { - "@helios/cache-domain": "workspace:*", - "aws4fetch": "^1.0.20", - "drizzle-orm": "^0.45.2" - } -} diff --git a/workers/cache/src/admin.ts b/workers/cache/src/admin.ts deleted file mode 100644 index 60c6609..0000000 --- a/workers/cache/src/admin.ts +++ /dev/null @@ -1,162 +0,0 @@ -import type { WorkerConfig } from "./config.js"; -import { signJwt, isTokenPermission, MAX_CACHES_PER_TOKEN, MAX_PERMS_PER_TOKEN, TOKEN_ISSUER, TOKEN_AUDIENCE } from "./jwt.js"; -import type { TokenClaims } from "./jwt.js"; -import type { TokenPermission } from "./db/types.js"; -import { - createApiToken, - listApiTokens, - revokeApiToken, -} from "./db/repository.js"; -import { jsonResponse, errorResponse, parseJsonBody } from "./responses.js"; - -const MAX_TOKEN_LIFETIME_DAYS = 365; -const DEFAULT_TOKEN_LIFETIME_DAYS = 90; -const MAX_SUBJECT_LENGTH = 256; -const MAX_REASON_LENGTH = 512; - -// ── Create Token ── - -export async function handleCreateToken( - request: Request, - config: WorkerConfig, -): Promise { - if (!config.jwtSecret) { - return errorResponse("Service unavailable", 503); - } - - const body = await parseJsonBody<{ - subject: string; - caches: string[]; - perms: string[]; - expiresInDays?: number; - }>(request); - if (body instanceof Response) return body; - - if (!body.subject || typeof body.subject !== "string") { - return errorResponse("subject must be a non-empty string", 400); - } - - if (body.subject.length > MAX_SUBJECT_LENGTH) { - return errorResponse("subject exceeds maximum length", 400); - } - - if (!Array.isArray(body.caches) || body.caches.length === 0) { - return errorResponse("caches must be a non-empty array of cache names", 400); - } - if (body.caches.length > MAX_CACHES_PER_TOKEN) { - return errorResponse(`caches exceeds maximum of ${MAX_CACHES_PER_TOKEN}`, 400); - } - for (const c of body.caches) { - if (typeof c !== "string" || c.length === 0) { - return errorResponse("each cache entry must be a non-empty string", 400); - } - } - - if (!Array.isArray(body.perms) || body.perms.length === 0) { - return errorResponse("perms must be a non-empty array", 400); - } - if (body.perms.length > MAX_PERMS_PER_TOKEN) { - return errorResponse(`perms exceeds maximum of ${MAX_PERMS_PER_TOKEN}`, 400); - } - const validatedPerms: TokenPermission[] = []; - for (const p of body.perms) { - if (typeof p !== "string" || !isTokenPermission(p)) { - return errorResponse(`invalid permission: ${String(p)}, expected "pull" or "push"`, 400); - } - validatedPerms.push(p); - } - - const expiresInDays = body.expiresInDays ?? DEFAULT_TOKEN_LIFETIME_DAYS; - if (!Number.isFinite(expiresInDays) || !Number.isInteger(expiresInDays) || expiresInDays < 1 || expiresInDays > MAX_TOKEN_LIFETIME_DAYS) { - return errorResponse(`expiresInDays must be between 1 and ${MAX_TOKEN_LIFETIME_DAYS}`, 400); - } - - const dedupedCaches = [...new Set(body.caches)]; - const dedupedPerms = [...new Set(validatedPerms)]; - - const now = Date.now() / 1000; - const jti = crypto.randomUUID(); - const exp = now + expiresInDays * 86400; - - const claims: TokenClaims = { - jti, - sub: body.subject, - iss: TOKEN_ISSUER, - aud: TOKEN_AUDIENCE, - caches: dedupedCaches, - perms: dedupedPerms, - iat: Math.floor(now), - exp: Math.floor(exp), - }; - - const token = await signJwt(claims, config.jwtSecret); - - const expiresAt = new Date(exp * 1000).toISOString(); - - await createApiToken(config.db, { - jti, - subject: body.subject, - cachesJson: JSON.stringify(dedupedCaches), - permsJson: JSON.stringify(dedupedPerms), - expiresAt, - createdBy: "admin", - }); - - return jsonResponse({ - token, - jti, - subject: body.subject, - caches: dedupedCaches, - perms: dedupedPerms, - expiresAt, - }, 201); -} - -// ── List Tokens ── - -export async function handleListTokens( - config: WorkerConfig, -): Promise { - const tokens = await listApiTokens(config.db); - const result = tokens.map((t) => ({ - jti: t.jti, - subject: t.subject, - caches: JSON.parse(t.cachesJson), - perms: JSON.parse(t.permsJson), - createdAt: t.createdAt, - expiresAt: t.expiresAt, - createdBy: t.createdBy, - revokedAt: t.revokedAt, - revokedBy: t.revokedBy, - revocationReason: t.revocationReason, - })); - return jsonResponse({ tokens: result }); -} - -// ── Revoke Token ── - -export async function handleRevokeToken( - request: Request, - config: WorkerConfig, - jti: string, -): Promise { - const body = await parseJsonBody<{ - reason: string; - }>(request); - if (body instanceof Response) return body; - - if (!body.reason || typeof body.reason !== "string") { - return errorResponse("reason must be a non-empty string", 400); - } - - if (body.reason.length > MAX_REASON_LENGTH) { - return errorResponse("reason exceeds maximum length", 400); - } - - const revoked = await revokeApiToken(config.db, jti, "admin", body.reason); - if (!revoked) { - return errorResponse("Token not found or already revoked", 404); - } - - return jsonResponse({ revoked: true, jti }); -} diff --git a/workers/cache/src/auth.ts b/workers/cache/src/auth.ts deleted file mode 100644 index d273eaf..0000000 --- a/workers/cache/src/auth.ts +++ /dev/null @@ -1,166 +0,0 @@ -import type { WorkerConfig } from "./config.js"; -import type { TokenClaims } from "./jwt.js"; -import { verifyJwt } from "./jwt.js"; -import { isTokenActive } from "./db/repository.js"; -import type { DrizzleD1Database } from "drizzle-orm/d1"; - -// Module-scope cache of recently confirmed-active token IDs (jti → cache -// entry deadline in ms). Workers isolates are reused across requests on the -// same edge node, so this avoids a D1 revocation-check round trip on every -// write request. Tradeoff: a freshly revoked token keeps working for up to -// TOKEN_ACTIVE_TTL_MS on warm isolates. Only positive results are cached; -// revocation is permanent, so negative results gain nothing from caching. -const TOKEN_ACTIVE_TTL_MS = 30_000; -const TOKEN_ACTIVE_CACHE_MAX_ENTRIES = 10_000; -const tokenActiveUntil = new Map(); - -async function isTokenActiveCached(db: DrizzleD1Database, jti: string): Promise { - const now = Date.now(); - const deadline = tokenActiveUntil.get(jti); - if (deadline !== undefined && deadline > now) { - return true; - } - - const active = await isTokenActive(db, jti); - if (active) { - // Bound memory: reset rather than evict; the cache refills on demand. - if (tokenActiveUntil.size >= TOKEN_ACTIVE_CACHE_MAX_ENTRIES) { - tokenActiveUntil.clear(); - } - tokenActiveUntil.set(jti, now + TOKEN_ACTIVE_TTL_MS); - } else { - tokenActiveUntil.delete(jti); - } - return active; -} - -/** Build a JSON error response. Always sets cache-control: no-store so auth failures don't get cached. */ -function authError(message: string, status: number): Response { - const headers: Record = { - "content-type": "application/json; charset=utf-8", - "cache-control": "no-store", - }; - if (status === 401) { - headers["www-authenticate"] = "Bearer"; - } - return new Response(JSON.stringify({ error: message }), { status, headers }); -} - -/** Extract bearer token from Authorization header, or return an error response. */ -function extractBearer(request: Request): string | Response { - const authHeader = request.headers.get("authorization"); - if (!authHeader) { - return authError("Unauthorized", 401); - } - const spaceIdx = authHeader.indexOf(" "); - if (spaceIdx === -1) { - return authError("Unauthorized", 401); - } - const scheme = authHeader.slice(0, spaceIdx); - const token = authHeader.slice(spaceIdx + 1); - if (scheme.toLowerCase() !== "bearer" || token.length === 0) { - return authError("Unauthorized", 401); - } - return token; -} - -const tsEncoder = new TextEncoder(); - -/** - * Timing-safe string comparison. Hashes both inputs with SHA-256 then - * compares digests byte-by-byte with constant-time XOR accumulation. - * Prevents timing side-channel attacks on secret comparison. - */ -async function timingSafeEqual(a: string, b: string): Promise { - const [da, db] = await Promise.all([ - crypto.subtle.digest("SHA-256", tsEncoder.encode(a)), - crypto.subtle.digest("SHA-256", tsEncoder.encode(b)), - ]); - const ba = new Uint8Array(da); - const bb = new Uint8Array(db); - let diff = a.length ^ b.length; - for (let i = 0; i < ba.length; i++) diff |= ba[i] ^ bb[i]; - return diff === 0; -} - -/** Represents the authenticated caller for write operations. */ -export type AuthIdentity = - | { readonly kind: "jwt"; readonly claims: TokenClaims } - | { readonly kind: "legacy" } - | { readonly kind: "admin" }; - -export type AuthResult = - | { readonly kind: "ok"; readonly identity: AuthIdentity } - | { readonly kind: "error"; readonly response: Response }; - -/** - * Authenticate a write request. Checks in order: - * 1. If JWT_SECRET is configured, try JWT verification + active check - * 2. If AUTH_TOKEN is configured, try static bearer token match - * 3. If neither is configured, fail closed (403) - */ -export async function authenticateWrite( - request: Request, - config: WorkerConfig, -): Promise { - // Require at least one auth method to be configured, otherwise reject. - if (!config.jwtSecret && !config.authToken) { - return { kind: "error", response: authError("Forbidden", 403) }; - } - - const bearer = extractBearer(request); - if (typeof bearer !== "string") { - return { kind: "error", response: bearer }; - } - - // Prefer JWT when a secret is set; fall back to the legacy token if JWT verification fails. - if (config.jwtSecret) { - const result = await verifyJwt(bearer, config.jwtSecret); - if (result.kind === "ok") { - const active = await isTokenActiveCached(config.db, result.claims.jti); - if (!active) { - return { kind: "error", response: authError("Unauthorized", 401) }; - } - return { kind: "ok", identity: { kind: "jwt", claims: result.claims } }; - } - if (!config.authToken) { - return { kind: "error", response: authError("Unauthorized", 401) }; - } - } - - if (config.authToken && await timingSafeEqual(bearer, config.authToken)) { - return { kind: "ok", identity: { kind: "legacy" } }; - } - - return { kind: "error", response: authError("Forbidden", 403) }; -} - -/** Authenticate an admin request using ADMIN_SECRET. */ -export async function authenticateAdmin( - request: Request, - config: WorkerConfig, -): Promise { - if (!config.adminSecret) { - return { kind: "error", response: authError("Forbidden", 403) }; - } - - const bearer = extractBearer(request); - if (typeof bearer !== "string") { - return { kind: "error", response: bearer }; - } - - if (!(await timingSafeEqual(bearer, config.adminSecret))) { - return { kind: "error", response: authError("Forbidden", 403) }; - } - - return { kind: "ok", identity: { kind: "admin" } }; -} - -/** Extract the actor identifier for audit logging. */ -export function actorId(identity: AuthIdentity): string { - switch (identity.kind) { - case "jwt": return identity.claims.jti; - case "legacy": return "legacy"; - case "admin": return "admin"; - } -} diff --git a/workers/cache/src/config.ts b/workers/cache/src/config.ts deleted file mode 100644 index 74458d6..0000000 --- a/workers/cache/src/config.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { drizzle } from "drizzle-orm/d1"; -import type { DrizzleD1Database } from "drizzle-orm/d1"; - -export interface WorkerConfig { - readonly bucket: R2Bucket; - readonly db: DrizzleD1Database; - readonly signingKeyName: string; - readonly signingPrivateKey: string; - readonly authToken: string; - readonly r2AccessKeyId: string; - readonly r2SecretAccessKey: string; - readonly r2Endpoint: string; - readonly r2BucketName: string; - readonly jwtSecret: string; - readonly adminSecret: string; - readonly ctx?: ExecutionContext; -} - -export function resolveConfig(env: Env): WorkerConfig { - if (!env.CACHE_BUCKET) { - throw new Error("Missing required binding: CACHE_BUCKET"); - } - if (!env.CACHE_DB) { - throw new Error("Missing required binding: CACHE_DB"); - } - return { - bucket: env.CACHE_BUCKET, - db: drizzle(env.CACHE_DB), - signingKeyName: env.SIGNING_KEY_NAME || "helios-cache-1", - signingPrivateKey: env.SIGNING_PRIVATE_KEY || "", - authToken: env.AUTH_TOKEN || "", - r2AccessKeyId: env.R2_ACCESS_KEY_ID || "", - r2SecretAccessKey: env.R2_SECRET_ACCESS_KEY || "", - r2Endpoint: env.R2_ENDPOINT || "", - r2BucketName: env.R2_BUCKET_NAME || "helios-cache", - jwtSecret: env.JWT_SECRET || "", - adminSecret: env.ADMIN_SECRET || "", - }; -} diff --git a/workers/cache/src/db/index.ts b/workers/cache/src/db/index.ts deleted file mode 100644 index 778281e..0000000 --- a/workers/cache/src/db/index.ts +++ /dev/null @@ -1,58 +0,0 @@ -export { - caches, - blobObjects, - publishedPaths, - uploadSessions, - uploadParts, - gcMarks, - apiTokens, - auditLogs, -} from './schema.js'; - -export type { - Cache, - BlobObject, - PublishedPath, - UploadSession, - UploadPart, - GcMark, - ApiToken, - AuditLog, - TokenPermission, - UploadSessionStatus, -} from './types.js'; - -export { - findCacheByName, - findCacheById, - createCache, - findBlobObject, - findBlobObjectById, - createBlobObject, - findPublishedPath, - findPublishedPathWithBlob, - findPublishedHashes, - createPublishedPath, - findUploadSession, - createUploadSession, - transitionSessionStatus, - transitionToMultipart, - findExpiredSessions, - upsertUploadPart, - upsertUploadParts, - findUploadParts, - createGcMark, - findGcMarks, - deleteGcMark, - findUnreferencedBlobObjects, - deleteBlobObject, - deleteUploadSession, - createApiToken, - findApiToken, - listApiTokens, - revokeApiToken, - isTokenActive, - deleteExpiredApiTokens, - createAuditLog, - deleteExpiredAuditLogs, -} from './repository.js'; diff --git a/workers/cache/src/db/repository.ts b/workers/cache/src/db/repository.ts deleted file mode 100644 index 346bb94..0000000 --- a/workers/cache/src/db/repository.ts +++ /dev/null @@ -1,507 +0,0 @@ -import { eq, and, notInArray, lt, inArray, isNull } from "drizzle-orm"; -import type { DrizzleD1Database } from "drizzle-orm/d1"; -import { - caches, - blobObjects, - publishedPaths, - uploadSessions, - uploadParts, - gcMarks, - apiTokens, - auditLogs, -} from "./schema.js"; -import type { - Cache, - BlobObject, - PublishedPath, - UploadSession, - UploadPart, - GcMark, - ApiToken, - UploadSessionStatus, -} from "./types.js"; - -// ── Caches ── - -export async function findCacheByName( - db: DrizzleD1Database, - name: string, -): Promise { - return db.select().from(caches).where(eq(caches.name, name)).get(); -} - -export async function findCacheById( - db: DrizzleD1Database, - id: number, -): Promise { - return db.select().from(caches).where(eq(caches.id, id)).get(); -} - -export async function createCache( - db: DrizzleD1Database, - name: string, -): Promise { - return db - .insert(caches) - .values({ name }) - .returning() - .get(); -} - -// ── Blob Objects ── - -export async function findBlobObject( - db: DrizzleD1Database, - fileHash: string, - compression: string, -): Promise { - return db - .select() - .from(blobObjects) - .where(and(eq(blobObjects.fileHash, fileHash), eq(blobObjects.compression, compression))) - .get(); -} - -export async function findBlobObjectById( - db: DrizzleD1Database, - id: number, -): Promise { - return db.select().from(blobObjects).where(eq(blobObjects.id, id)).get(); -} - -export async function createBlobObject( - db: DrizzleD1Database, - params: { - readonly fileHash: string; - readonly fileSize: number; - readonly compression: string; - readonly r2Key: string; - }, -): Promise { - return db - .insert(blobObjects) - .values(params) - .returning() - .get(); -} - -// ── Published Paths ── - -export async function findPublishedPathWithBlob( - db: DrizzleD1Database, - cacheId: number, - storePathHash: string, -): Promise<{ path: PublishedPath; blob: BlobObject } | undefined> { - const row = await db - .select({ - path: publishedPaths, - blob: blobObjects, - }) - .from(publishedPaths) - .innerJoin(blobObjects, eq(publishedPaths.blobObjectId, blobObjects.id)) - .where( - and( - eq(publishedPaths.cacheId, cacheId), - eq(publishedPaths.storePathHash, storePathHash), - ), - ) - .get(); - return row; -} - -export async function findPublishedPath( - db: DrizzleD1Database, - cacheId: number, - storePathHash: string, -): Promise { - return db - .select() - .from(publishedPaths) - .where( - and( - eq(publishedPaths.cacheId, cacheId), - eq(publishedPaths.storePathHash, storePathHash), - ), - ) - .get(); -} - -export async function createPublishedPath( - db: DrizzleD1Database, - params: { - readonly cacheId: number; - readonly storePathHash: string; - readonly storePath: string; - readonly narHash: string; - readonly narSize: number; - readonly blobObjectId: number; - readonly referencesJson: string; - readonly deriver: string | null; - readonly system: string | null; - readonly signaturesJson: string; - }, -): Promise { - return db - .insert(publishedPaths) - .values(params) - .returning() - .get(); -} - -export async function findPublishedHashes( - db: DrizzleD1Database, - cacheId: number, - storePathHashes: string[], -): Promise { - if (storePathHashes.length === 0) return []; - - // D1 limits bound parameters to 100 per query; reserve 1 for cacheId - const chunks: string[][] = []; - for (let i = 0; i < storePathHashes.length; i += 99) { - chunks.push(storePathHashes.slice(i, i + 99)); - } - - // Send all chunk queries in a single D1 batch round trip instead of one - // sequential round trip per chunk. - const queries = chunks.map((chunk) => - db - .select({ storePathHash: publishedPaths.storePathHash }) - .from(publishedPaths) - .where( - and( - eq(publishedPaths.cacheId, cacheId), - inArray(publishedPaths.storePathHash, chunk), - ), - ), - ); - - const [firstQuery, ...restQueries] = queries; - if (firstQuery === undefined) return []; - const chunkResults = await db.batch([firstQuery, ...restQueries]); - - const results: string[] = []; - for (const rows of chunkResults) { - for (const r of rows) { - results.push(r.storePathHash); - } - } - - return results; -} - -// ── Upload Sessions ── - -export async function findUploadSession( - db: DrizzleD1Database, - sessionId: string, -): Promise { - return db - .select() - .from(uploadSessions) - .where(eq(uploadSessions.id, sessionId)) - .get(); -} - -export async function createUploadSession( - db: DrizzleD1Database, - params: { - readonly id: string; - readonly cacheId: number; - readonly storePathHash: string; - readonly storePath: string; - readonly narHash: string; - readonly narSize: number; - readonly fileHash: string; - readonly fileSize: number; - readonly compression: string; - readonly referencesJson: string; - readonly deriver: string | null; - readonly system: string | null; - readonly r2UploadKey: string | null; - readonly r2UploadId: string | null; - readonly expiresAt: string; - }, -): Promise { - return db - .insert(uploadSessions) - .values(params) - .returning() - .get(); -} - -export async function transitionSessionStatus( - db: DrizzleD1Database, - sessionId: string, - fromStatus: UploadSessionStatus, - toStatus: UploadSessionStatus, -): Promise { - return db - .update(uploadSessions) - .set({ status: toStatus }) - .where(and(eq(uploadSessions.id, sessionId), eq(uploadSessions.status, fromStatus))) - .returning() - .get(); -} - -export async function transitionToMultipart( - db: DrizzleD1Database, - sessionId: string, - r2UploadId: string, -): Promise { - return db - .update(uploadSessions) - .set({ status: "uploading", r2UploadId }) - .where(and(eq(uploadSessions.id, sessionId), eq(uploadSessions.status, "pending"))) - .returning() - .get(); -} - -export async function findExpiredSessions( - db: DrizzleD1Database, - now: string, -): Promise { - return db - .select() - .from(uploadSessions) - .where( - and( - notInArray(uploadSessions.status, ["completed", "expired"]), - lt(uploadSessions.expiresAt, now), - ), - ) - .all(); -} - -// ── Upload Parts ── - -export async function upsertUploadPart( - db: DrizzleD1Database, - params: { - readonly sessionId: string; - readonly partNumber: number; - readonly etag: string; - readonly size: number; - }, -): Promise { - return db - .insert(uploadParts) - .values(params) - .onConflictDoUpdate({ - target: [uploadParts.sessionId, uploadParts.partNumber], - set: { etag: params.etag, size: params.size }, - }) - .returning() - .get(); -} - -export async function upsertUploadParts( - db: DrizzleD1Database, - sessionId: string, - parts: ReadonlyArray<{ - readonly partNumber: number; - readonly etag: string; - readonly size: number; - }>, -): Promise { - // One D1 batch round trip for all parts instead of one round trip each. - const statements = parts.map((part) => - db - .insert(uploadParts) - .values({ sessionId, partNumber: part.partNumber, etag: part.etag, size: part.size }) - .onConflictDoUpdate({ - target: [uploadParts.sessionId, uploadParts.partNumber], - set: { etag: part.etag, size: part.size }, - }), - ); - - const [firstStatement, ...restStatements] = statements; - if (firstStatement === undefined) return; - await db.batch([firstStatement, ...restStatements]); -} - -export async function findUploadParts( - db: DrizzleD1Database, - sessionId: string, -): Promise { - return db - .select() - .from(uploadParts) - .where(eq(uploadParts.sessionId, sessionId)) - .orderBy(uploadParts.partNumber) - .all(); -} - -// ── Blob Object Cleanup ── - -export async function findUnreferencedBlobObjects( - db: DrizzleD1Database, -): Promise { - const referenced = db - .select({ id: publishedPaths.blobObjectId }) - .from(publishedPaths); - - return db - .select() - .from(blobObjects) - .where(notInArray(blobObjects.id, referenced)) - .all(); -} - -export async function deleteBlobObject( - db: DrizzleD1Database, - id: number, -): Promise { - await db.delete(blobObjects).where(eq(blobObjects.id, id)).run(); -} - -// ── Upload Session Cleanup ── - -export async function deleteUploadSession( - db: DrizzleD1Database, - sessionId: string, -): Promise { - await db.delete(uploadParts).where(eq(uploadParts.sessionId, sessionId)).run(); - await db.delete(uploadSessions).where(eq(uploadSessions.id, sessionId)).run(); -} - -// ── GC Marks ── - -export async function createGcMark( - db: DrizzleD1Database, - params: { - readonly targetType: string; - readonly targetId: string; - readonly reason: string; - }, -): Promise { - return db - .insert(gcMarks) - .values(params) - .returning() - .get(); -} - -export async function findGcMarks( - db: DrizzleD1Database, - targetType: string, -): Promise { - return db - .select() - .from(gcMarks) - .where(eq(gcMarks.targetType, targetType)) - .all(); -} - -export async function deleteGcMark( - db: DrizzleD1Database, - targetType: string, - targetId: string, -): Promise { - await db - .delete(gcMarks) - .where(and(eq(gcMarks.targetType, targetType), eq(gcMarks.targetId, targetId))) - .run(); -} - -// ── API Tokens ── - -export async function createApiToken( - db: DrizzleD1Database, - params: { - readonly jti: string; - readonly subject: string; - readonly cachesJson: string; - readonly permsJson: string; - readonly expiresAt: string; - readonly createdBy: string; - }, -): Promise { - return db - .insert(apiTokens) - .values(params) - .returning() - .get(); -} - -export async function findApiToken( - db: DrizzleD1Database, - jti: string, -): Promise { - return db.select().from(apiTokens).where(eq(apiTokens.jti, jti)).get(); -} - -export async function listApiTokens( - db: DrizzleD1Database, -): Promise { - return db.select().from(apiTokens).all(); -} - -export async function revokeApiToken( - db: DrizzleD1Database, - jti: string, - revokedBy: string, - reason: string, -): Promise { - return db - .update(apiTokens) - .set({ - revokedAt: new Date().toISOString(), - revokedBy, - revocationReason: reason, - }) - .where(and(eq(apiTokens.jti, jti), isNull(apiTokens.revokedAt))) - .returning() - .get(); -} - -export async function isTokenActive( - db: DrizzleD1Database, - jti: string, -): Promise { - const row = await db - .select({ jti: apiTokens.jti }) - .from(apiTokens) - .where(and(eq(apiTokens.jti, jti), isNull(apiTokens.revokedAt))) - .get(); - return row !== undefined; -} - -export async function deleteExpiredApiTokens( - db: DrizzleD1Database, - now: string, -): Promise { - const result = await db - .delete(apiTokens) - .where(lt(apiTokens.expiresAt, now)) - .run(); - return result.meta.changes ?? 0; -} - -// ── Audit Logs ── - -export async function createAuditLog( - db: DrizzleD1Database, - params: { - readonly actor: string; - readonly action: string; - readonly cacheName: string | null; - readonly detail: string; - readonly ip: string | null; - readonly status: number; - }, -): Promise { - // No RETURNING: audit inserts are fire-and-forget on the hot write path, - // and skipping the row round trip keeps them as cheap as possible. - await db.insert(auditLogs).values(params).run(); -} - -export async function deleteExpiredAuditLogs( - db: DrizzleD1Database, - cutoff: string, -): Promise { - const result = await db - .delete(auditLogs) - .where(lt(auditLogs.timestamp, cutoff)) - .run(); - return result.meta.changes ?? 0; -} diff --git a/workers/cache/src/db/schema.ts b/workers/cache/src/db/schema.ts deleted file mode 100644 index 6f32dbd..0000000 --- a/workers/cache/src/db/schema.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { sqliteTable, text, integer, uniqueIndex, index } from "drizzle-orm/sqlite-core"; -import { sql } from "drizzle-orm"; - -const timestamp = (name: string) => - text(name).notNull().default(sql`(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))`); - -export const caches = sqliteTable("caches", { - id: integer("id").primaryKey({ autoIncrement: true }), - name: text("name").notNull().unique(), - isPublic: integer("is_public").notNull().default(1), - createdAt: timestamp("created_at"), -}); - -export const blobObjects = sqliteTable("blob_objects", { - id: integer("id").primaryKey({ autoIncrement: true }), - fileHash: text("file_hash").notNull(), - fileSize: integer("file_size").notNull(), - compression: text("compression").notNull(), - r2Key: text("r2_key").notNull().unique(), - createdAt: timestamp("created_at"), -}, (table) => [ - uniqueIndex("blob_objects_file_hash_compression_unique").on(table.fileHash, table.compression), -]); - -export const publishedPaths = sqliteTable("published_paths", { - id: integer("id").primaryKey({ autoIncrement: true }), - cacheId: integer("cache_id").notNull().references(() => caches.id), - storePathHash: text("store_path_hash").notNull(), - storePath: text("store_path").notNull(), - narHash: text("nar_hash").notNull(), - narSize: integer("nar_size").notNull(), - blobObjectId: integer("blob_object_id").notNull().references(() => blobObjects.id), - referencesJson: text("references_json").notNull().default("[]"), - deriver: text("deriver"), - system: text("system"), - signaturesJson: text("signatures_json").notNull().default("[]"), - createdAt: timestamp("created_at"), -}, (table) => [ - uniqueIndex("published_paths_cache_store_unique").on(table.cacheId, table.storePathHash), - index("idx_published_paths_blob_object").on(table.blobObjectId), -]); - -export const uploadSessions = sqliteTable("upload_sessions", { - id: text("id").primaryKey(), - cacheId: integer("cache_id").notNull().references(() => caches.id), - storePathHash: text("store_path_hash").notNull(), - storePath: text("store_path").notNull(), - narHash: text("nar_hash").notNull(), - narSize: integer("nar_size").notNull(), - fileHash: text("file_hash").notNull(), - fileSize: integer("file_size").notNull(), - compression: text("compression").notNull(), - referencesJson: text("references_json").notNull().default("[]"), - deriver: text("deriver"), - system: text("system"), - status: text("status").notNull().default("pending"), - r2UploadKey: text("r2_upload_key"), - r2UploadId: text("r2_upload_id"), - createdAt: timestamp("created_at"), - expiresAt: text("expires_at").notNull(), -}, (table) => [ - index("idx_upload_sessions_status").on(table.status, table.expiresAt), -]); - -export const uploadParts = sqliteTable("upload_parts", { - id: integer("id").primaryKey({ autoIncrement: true }), - sessionId: text("session_id").notNull().references(() => uploadSessions.id), - partNumber: integer("part_number").notNull(), - etag: text("etag").notNull(), - size: integer("size").notNull(), -}, (table) => [ - uniqueIndex("upload_parts_session_part_unique").on(table.sessionId, table.partNumber), -]); - -export const gcMarks = sqliteTable("gc_marks", { - id: integer("id").primaryKey({ autoIncrement: true }), - targetType: text("target_type").notNull(), - targetId: text("target_id").notNull(), - reason: text("reason").notNull(), - markedAt: text("marked_at").notNull().default(sql`(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))`), -}, (table) => [ - uniqueIndex("gc_marks_target_unique").on(table.targetType, table.targetId), -]); - -export const apiTokens = sqliteTable("api_tokens", { - jti: text("jti").primaryKey(), - subject: text("subject").notNull(), - cachesJson: text("caches_json").notNull(), - permsJson: text("perms_json").notNull(), - createdAt: timestamp("created_at"), - expiresAt: text("expires_at").notNull(), - createdBy: text("created_by").notNull(), - revokedAt: text("revoked_at"), - revokedBy: text("revoked_by"), - revocationReason: text("revocation_reason"), -}); - -export const auditLogs = sqliteTable("audit_logs", { - id: integer("id").primaryKey({ autoIncrement: true }), - timestamp: timestamp("timestamp"), - actor: text("actor").notNull(), - action: text("action").notNull(), - cacheName: text("cache_name"), - detail: text("detail").notNull().default("{}"), - ip: text("ip"), - status: integer("status").notNull(), -}, (table) => [ - index("idx_audit_logs_actor").on(table.actor), - index("idx_audit_logs_cache").on(table.cacheName), - index("idx_audit_logs_time").on(table.timestamp), -]); diff --git a/workers/cache/src/db/types.ts b/workers/cache/src/db/types.ts deleted file mode 100644 index 241ba2c..0000000 --- a/workers/cache/src/db/types.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { InferSelectModel } from "drizzle-orm"; -import type { - caches, - blobObjects, - publishedPaths, - uploadSessions, - uploadParts, - gcMarks, - apiTokens, - auditLogs, -} from "./schema.js"; - -export type Cache = InferSelectModel; - -export type BlobObject = InferSelectModel; - -export type PublishedPath = InferSelectModel; - -export type UploadSession = InferSelectModel; - -export type UploadPart = InferSelectModel; - -export type GcMark = InferSelectModel; - -export type ApiToken = InferSelectModel; - -export type AuditLog = InferSelectModel; - -export type TokenPermission = "pull" | "push"; - -export type UploadSessionStatus = "pending" | "uploading" | "completed" | "expired"; diff --git a/workers/cache/src/env.d.ts b/workers/cache/src/env.d.ts deleted file mode 100644 index 3cb78b7..0000000 --- a/workers/cache/src/env.d.ts +++ /dev/null @@ -1,13 +0,0 @@ -// Secrets set via `wrangler secret put` — not in wrangler.jsonc vars. -declare namespace Cloudflare { - interface Env { - AUTH_TOKEN?: string; - SIGNING_PRIVATE_KEY?: string; - R2_ACCESS_KEY_ID?: string; - R2_SECRET_ACCESS_KEY?: string; - R2_ENDPOINT?: string; - R2_BUCKET_NAME?: string; - JWT_SECRET?: string; - ADMIN_SECRET?: string; - } -} diff --git a/workers/cache/src/gc.ts b/workers/cache/src/gc.ts deleted file mode 100644 index ded69ce..0000000 --- a/workers/cache/src/gc.ts +++ /dev/null @@ -1,112 +0,0 @@ -import type { WorkerConfig } from "./config.js"; -import { - findExpiredSessions, - transitionSessionStatus, - deleteUploadSession, - findUnreferencedBlobObjects, - deleteBlobObject, - deleteExpiredApiTokens, - deleteExpiredAuditLogs, -} from "./db/repository.js"; -import type { UploadSessionStatus } from "./db/types.js"; - -const VALID_TRANSITION_STATUSES = new Set(["pending", "uploading"]); - -function isTransitionableStatus(s: string): s is UploadSessionStatus { - return VALID_TRANSITION_STATUSES.has(s); -} - -const AUDIT_LOG_RETENTION_DAYS = 30; - -export interface GcResult { - readonly expiredSessions: number; - readonly deletedBlobs: number; - readonly expiredTokens: number; - readonly expiredAuditLogs: number; - readonly errors: string[]; -} - -export async function runGarbageCollection(config: WorkerConfig): Promise { - const errors: string[] = []; - let expiredSessions = 0; - let deletedBlobs = 0; - let expiredTokens = 0; - let expiredAuditLogs = 0; - - // Phase 1: Expire abandoned upload sessions - const now = new Date().toISOString(); - const expired = await findExpiredSessions(config.db, now); - - for (const session of expired) { - try { - // findExpiredSessions already excludes "completed" and "expired", - // but validate to satisfy the type system without an assertion. - if (!isTransitionableStatus(session.status)) continue; - - const transitioned = await transitionSessionStatus( - config.db, - session.id, - session.status, - "expired", - ); - if (!transitioned) continue; - - if (session.r2UploadId && session.r2UploadKey) { - try { - const multipart = config.bucket.resumeMultipartUpload(session.r2UploadKey, session.r2UploadId); - await multipart.abort(); - } catch { - // Multipart may already be completed or aborted - } - } - - if (session.r2UploadKey) { - try { - await config.bucket.delete(session.r2UploadKey); - } catch { - // R2 object may not exist - } - } - - await deleteUploadSession(config.db, session.id); - expiredSessions++; - } catch (err) { - errors.push(`Failed to expire session ${session.id}: ${err instanceof Error ? err.message : String(err)}`); - } - } - - // Phase 2: Delete unreferenced blob objects - const unreferenced = await findUnreferencedBlobObjects(config.db); - - for (const blob of unreferenced) { - try { - // Delete DB record first to prevent re-referencing during R2 delete - await deleteBlobObject(config.db, blob.id); - try { - await config.bucket.delete(blob.r2Key); - } catch { - // R2 object may already be gone - } - deletedBlobs++; - } catch (err) { - errors.push(`Failed to delete blob ${blob.id}: ${err instanceof Error ? err.message : String(err)}`); - } - } - - // Phase 3: Delete expired API tokens - try { - expiredTokens = await deleteExpiredApiTokens(config.db, now); - } catch (err) { - errors.push(`Failed to delete expired tokens: ${err instanceof Error ? err.message : String(err)}`); - } - - // Phase 4: Delete audit logs older than retention period - try { - const cutoff = new Date(Date.now() - AUDIT_LOG_RETENTION_DAYS * 86400_000).toISOString(); - expiredAuditLogs = await deleteExpiredAuditLogs(config.db, cutoff); - } catch (err) { - errors.push(`Failed to delete expired audit logs: ${err instanceof Error ? err.message : String(err)}`); - } - - return { expiredSessions, deletedBlobs, expiredTokens, expiredAuditLogs, errors }; -} diff --git a/workers/cache/src/index.ts b/workers/cache/src/index.ts deleted file mode 100644 index daed64d..0000000 --- a/workers/cache/src/index.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { resolveConfig } from "./config.js"; -import { handleRequest } from "./router.js"; -import { runGarbageCollection } from "./gc.js"; -import { logRequest } from "./logging.js"; - -export default { - async fetch(request, env, ctx): Promise { - const start = Date.now(); - try { - const config = { ...resolveConfig(env), ctx }; - const url = new URL(request.url); - const response = await handleRequest(request, config); - if (response.status >= 400 || url.pathname.startsWith("/_api/")) { - logRequest(request, response, start); - } - return response; - } catch (err) { - console.error("Unhandled error", { error: err instanceof Error ? err.message : String(err) }); - const response = new Response(JSON.stringify({ error: "Internal server error" }), { - status: 500, - headers: { "content-type": "application/json" }, - }); - logRequest(request, response, start); - return response; - } - }, - - async scheduled(event, env, ctx): Promise { - const config = resolveConfig(env); - const result = await runGarbageCollection(config); - - if (result.errors.length > 0) { - console.error("GC completed with errors", { - expiredSessions: result.expiredSessions, - deletedBlobs: result.deletedBlobs, - expiredTokens: result.expiredTokens, - expiredAuditLogs: result.expiredAuditLogs, - errors: result.errors, - }); - } else { - console.log("GC completed", { - expiredSessions: result.expiredSessions, - deletedBlobs: result.deletedBlobs, - expiredTokens: result.expiredTokens, - expiredAuditLogs: result.expiredAuditLogs, - }); - } - }, -} satisfies ExportedHandler; diff --git a/workers/cache/src/jwt.ts b/workers/cache/src/jwt.ts deleted file mode 100644 index 3569ba5..0000000 --- a/workers/cache/src/jwt.ts +++ /dev/null @@ -1,248 +0,0 @@ -import type { TokenPermission } from "./db/types.js"; - -export type TokenClaims = { - readonly jti: string; - readonly sub: string; - readonly iss: string; - readonly aud: string; - readonly caches: readonly string[]; - readonly perms: readonly TokenPermission[]; - readonly exp: number; - readonly iat: number; -}; - -export type JwtVerifyResult = - | { readonly kind: "ok"; readonly claims: TokenClaims } - | { readonly kind: "error"; readonly reason: string }; - -function base64UrlEncode(data: Uint8Array): string { - let binary = ""; - for (let i = 0; i < data.length; i++) { - binary += String.fromCharCode(data[i]); - } - return btoa(binary).replaceAll("+", "-").replaceAll("/", "_").replaceAll("=", ""); -} - -function base64UrlDecode(str: string): Uint8Array { - const base64 = str.replaceAll("-", "+").replaceAll("_", "/"); - const paddingNeeded = (4 - (base64.length % 4)) % 4; - const padded = paddingNeeded > 0 ? base64 + "====".slice(0, paddingNeeded) : base64; - const binary = atob(padded); - const bytes = new Uint8Array(binary.length); - for (let i = 0; i < binary.length; i++) { - bytes[i] = binary.charCodeAt(i); - } - return bytes; -} - -const HMAC_ALGO = { name: "HMAC", hash: "SHA-256" } as const; -const encoder = new TextEncoder(); -const decoder = new TextDecoder(); - -// Cache the imported CryptoKey at module scope. Workers isolates are reused -// across requests on the same edge node, so this avoids a costly importKey -// call on every verify. -let cachedSecret = ""; -let cachedKey: CryptoKey | undefined; - -async function getKey(secret: string): Promise { - if (secret === cachedSecret && cachedKey !== undefined) { - return cachedKey; - } - const key = await crypto.subtle.importKey( - "raw", - encoder.encode(secret), - HMAC_ALGO, - false, - ["sign", "verify"], - ); - cachedSecret = secret; - cachedKey = key; - return key; -} - -const EXPECTED_HEADER_JSON = '{"alg":"HS256","typ":"JWT"}'; -const HEADER_B64 = base64UrlEncode(encoder.encode(EXPECTED_HEADER_JSON)); -export const TOKEN_ISSUER = "helios-cache"; -export const TOKEN_AUDIENCE = "helios-cache"; -const MAX_TOKEN_BYTES = 4096; -const MAX_CLOCK_SKEW_SECONDS = 60; -export const MAX_CACHES_PER_TOKEN = 50; -export const MAX_PERMS_PER_TOKEN = 10; - -export const VALID_PERMISSIONS: ReadonlySet = new Set(["pull", "push"]); - -export function isTokenPermission(value: string): value is TokenPermission { - return VALID_PERMISSIONS.has(value); -} - -function validateClaims(obj: Record): JwtVerifyResult { - const jti = obj["jti"]; - if (typeof jti !== "string" || jti.length === 0) { - return { kind: "error", reason: "invalid token" }; - } - - const sub = obj["sub"]; - if (typeof sub !== "string" || sub.length === 0) { - return { kind: "error", reason: "invalid token" }; - } - - const exp = obj["exp"]; - if (typeof exp !== "number") { - return { kind: "error", reason: "invalid token" }; - } - - const iat = obj["iat"]; - if (typeof iat !== "number") { - return { kind: "error", reason: "invalid token" }; - } - - const iss = obj["iss"]; - if (typeof iss !== "string" || iss !== TOKEN_ISSUER) { - return { kind: "error", reason: "invalid token" }; - } - - const aud = obj["aud"]; - if (typeof aud !== "string" || aud !== TOKEN_AUDIENCE) { - return { kind: "error", reason: "invalid token" }; - } - - if (!Number.isSafeInteger(exp) || !Number.isSafeInteger(iat)) { - return { kind: "error", reason: "invalid token" }; - } - - // Reject tokens issued in the future, allowing a small clock skew between nodes. - const now = Date.now() / 1000; - if (iat > now + MAX_CLOCK_SKEW_SECONDS) { - return { kind: "error", reason: "invalid token" }; - } - - if (exp <= iat) { - return { kind: "error", reason: "invalid token" }; - } - - const rawCaches = obj["caches"]; - if (!Array.isArray(rawCaches) || rawCaches.length === 0) { - return { kind: "error", reason: "invalid token" }; - } - if (rawCaches.length > MAX_CACHES_PER_TOKEN) { - return { kind: "error", reason: "invalid token" }; - } - const validatedCaches: string[] = []; - for (let i = 0; i < rawCaches.length; i++) { - const entry: unknown = rawCaches[i]; - if (typeof entry !== "string" || entry.length === 0) { - return { kind: "error", reason: "invalid token" }; - } - validatedCaches.push(entry); - } - - const rawPerms = obj["perms"]; - if (!Array.isArray(rawPerms) || rawPerms.length === 0) { - return { kind: "error", reason: "invalid token" }; - } - if (rawPerms.length > MAX_PERMS_PER_TOKEN) { - return { kind: "error", reason: "invalid token" }; - } - const validatedPerms: TokenPermission[] = []; - for (let i = 0; i < rawPerms.length; i++) { - const perm: unknown = rawPerms[i]; - if (typeof perm !== "string" || !isTokenPermission(perm)) { - return { kind: "error", reason: "invalid token" }; - } - validatedPerms.push(perm); - } - - return { - kind: "ok", - claims: { jti, sub, iss, aud, caches: validatedCaches, perms: validatedPerms, exp, iat }, - }; -} - -export async function verifyJwt(token: string, secret: string): Promise { - if (token.length > MAX_TOKEN_BYTES) { - return { kind: "error", reason: "invalid token" }; - } - - const firstDot = token.indexOf("."); - if (firstDot === -1) { - return { kind: "error", reason: "invalid token" }; - } - const secondDot = token.indexOf(".", firstDot + 1); - if (secondDot === -1 || token.indexOf(".", secondDot + 1) !== -1) { - return { kind: "error", reason: "invalid token" }; - } - - const headerB64 = token.slice(0, firstDot); - const payloadB64 = token.slice(firstDot + 1, secondDot); - const signatureB64 = token.slice(secondDot + 1); - - // Compare the encoded header directly — saves a base64 decode and string parse. - if (headerB64 !== HEADER_B64) { - return { kind: "error", reason: "invalid token" }; - } - - // Parse the payload first so we can short-circuit on expiry before paying for HMAC verify. - let payloadJson: string; - try { - payloadJson = decoder.decode(base64UrlDecode(payloadB64)); - } catch { - return { kind: "error", reason: "invalid token" }; - } - - let rawParsed: unknown; - try { - rawParsed = JSON.parse(payloadJson); - } catch { - return { kind: "error", reason: "invalid token" }; - } - - if (typeof rawParsed !== "object" || rawParsed === null) { - return { kind: "error", reason: "invalid token" }; - } - - const claims: Record = Object.fromEntries(Object.entries(rawParsed)); - - const earlyExp = claims["exp"]; - if (typeof earlyExp === "number" && earlyExp <= Date.now() / 1000 - MAX_CLOCK_SKEW_SECONDS) { - return { kind: "error", reason: "invalid token" }; - } - - const key = await getKey(secret); - let signature: Uint8Array; - try { - signature = base64UrlDecode(signatureB64); - } catch { - return { kind: "error", reason: "invalid token" }; - } - - const data = encoder.encode(token.slice(0, secondDot)); - // Copy into a fresh ArrayBuffer because Uint8Array.buffer is ArrayBufferLike, - // which the CF runtime's BufferSource type does not accept. - const sigBuf = new ArrayBuffer(signature.byteLength); - new Uint8Array(sigBuf).set(signature); - const valid = await crypto.subtle.verify("HMAC", key, sigBuf, data); - if (!valid) { - return { kind: "error", reason: "invalid token" }; - } - - return validateClaims(claims); -} - -export async function signJwt(claims: TokenClaims, secret: string): Promise { - const payloadB64 = base64UrlEncode(encoder.encode(JSON.stringify(claims))); - - const key = await getKey(secret); - const signingInput = `${HEADER_B64}.${payloadB64}`; - const signature = new Uint8Array(await crypto.subtle.sign("HMAC", key, encoder.encode(signingInput))); - - return `${signingInput}.${base64UrlEncode(signature)}`; -} - -export function hasPermission(claims: TokenClaims, perm: TokenPermission): boolean { - return claims.perms.includes(perm); -} - -export function hasCacheAccess(claims: TokenClaims, cacheName: string): boolean { - return claims.caches.includes("*") || claims.caches.includes(cacheName); -} diff --git a/workers/cache/src/logging.ts b/workers/cache/src/logging.ts deleted file mode 100644 index b3e0abd..0000000 --- a/workers/cache/src/logging.ts +++ /dev/null @@ -1,15 +0,0 @@ -export function logRequest( - request: Request, - response: Response, - startTime: number, -): void { - const duration = Date.now() - startTime; - const url = new URL(request.url); - console.log(JSON.stringify({ - method: request.method, - path: url.pathname, - status: response.status, - duration_ms: duration, - cf_ray: request.headers.get("cf-ray") ?? undefined, - })); -} diff --git a/workers/cache/src/narinfo.ts b/workers/cache/src/narinfo.ts deleted file mode 100644 index 416bc72..0000000 --- a/workers/cache/src/narinfo.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { PublishedPath, BlobObject } from "./db/types.js"; - -export function renderNarinfo( - path: PublishedPath, - blob: BlobObject, - refs: readonly string[], - storedSigs: readonly string[], - signatures: readonly string[] = [], -): string { - const lines: string[] = [ - `StorePath: ${path.storePath}`, - `URL: nar/${blob.fileHash}/${blob.compression}.nar`, - `Compression: ${blob.compression}`, - `FileHash: sha256:${blob.fileHash}`, - `FileSize: ${blob.fileSize}`, - `NarHash: ${path.narHash}`, - `NarSize: ${path.narSize}`, - ]; - - if (refs.length > 0) { - lines.push(`References: ${refs.join(" ")}`); - } - - if (path.deriver) { - lines.push(`Deriver: ${path.deriver}`); - } - - if (path.system) { - lines.push(`System: ${path.system}`); - } - - for (const sig of storedSigs) { - lines.push(`Sig: ${sig}`); - } - for (const sig of signatures) { - lines.push(`Sig: ${sig}`); - } - - return lines.join("\n") + "\n"; -} diff --git a/workers/cache/src/presign.ts b/workers/cache/src/presign.ts deleted file mode 100644 index 0c46d75..0000000 --- a/workers/cache/src/presign.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { AwsClient } from "aws4fetch"; - -let cachedClient: AwsClient | undefined; -let cachedCredentials: string | undefined; - -function getClient(accessKeyId: string, secretAccessKey: string): AwsClient { - const key = `${accessKeyId}:${secretAccessKey}`; - if (cachedClient && cachedCredentials === key) { - return cachedClient; - } - cachedClient = new AwsClient({ - accessKeyId, - secretAccessKey, - service: "s3", - region: "auto", - }); - cachedCredentials = key; - return cachedClient; -} - -export async function createPresignedUrl( - endpoint: string, - accessKeyId: string, - secretAccessKey: string, - bucket: string, - key: string, - expiresIn: number, -): Promise { - const client = getClient(accessKeyId, secretAccessKey); - const url = new URL(`/${bucket}/${key}`, endpoint); - url.searchParams.set("X-Amz-Expires", String(expiresIn)); - - const signed = await client.sign(new Request(url, { method: "GET" }), { - aws: { signQuery: true }, - }); - - return signed.url; -} diff --git a/workers/cache/src/responses.ts b/workers/cache/src/responses.ts deleted file mode 100644 index 5f77e6e..0000000 --- a/workers/cache/src/responses.ts +++ /dev/null @@ -1,23 +0,0 @@ -const JSON_HEADERS: Record = { - "content-type": "application/json; charset=utf-8", - "cache-control": "no-store", -}; - -export function jsonResponse(body: unknown, status = 200): Response { - return new Response(JSON.stringify(body), { - status, - headers: JSON_HEADERS, - }); -} - -export function errorResponse(message: string, status: number): Response { - return jsonResponse({ error: message }, status); -} - -export async function parseJsonBody(request: Request): Promise { - try { - return await request.json(); - } catch { - return errorResponse("Invalid JSON in request body", 400); - } -} diff --git a/workers/cache/src/router.ts b/workers/cache/src/router.ts deleted file mode 100644 index bfda217..0000000 --- a/workers/cache/src/router.ts +++ /dev/null @@ -1,452 +0,0 @@ -import type { WorkerConfig } from "./config.js"; -import { findCacheByName, findPublishedPathWithBlob, createAuditLog } from "./db/repository.js"; -import { renderNarinfo } from "./narinfo.js"; -import { authenticateWrite, authenticateAdmin, actorId } from "./auth.js"; -import type { AuthIdentity } from "./auth.js"; -import { hasPermission, hasCacheAccess } from "./jwt.js"; -import { errorResponse } from "./responses.js"; -import { computeFingerprint, signNarinfo } from "./signing.js"; -import { parseCacheName, parseStorePathHash, parseFileHash, parseCompression } from "@helios/cache-domain"; -import type { CacheName } from "@helios/cache-domain"; -import { - handleCreateSession, - handleMultipart, - handleUploadPart, - handleUploadBlob, - handleComplete, - handlePublish, - handleGetMissingPaths, -} from "./uploads.js"; -import { createPresignedUrl } from "./presign.js"; -import { handleCreateToken, handleListTokens, handleRevokeToken } from "./admin.js"; - -// Pre-compiled route patterns. Compiled once per isolate, not per request. -const RE_SESSION_CREATE = /^\/_api\/v1\/caches\/([^/]+)\/upload-sessions$/; -const RE_MULTIPART = /^\/_api\/v1\/uploads\/([^/]+)\/multipart$/; -const RE_PART_UPLOAD = /^\/_api\/v1\/uploads\/([^/]+)\/part\/(\d+)$/; -const RE_BLOB = /^\/_api\/v1\/uploads\/([^/]+)\/blob$/; -const RE_COMPLETE = /^\/_api\/v1\/uploads\/([^/]+)\/complete$/; -const RE_PUBLISH = /^\/_api\/v1\/uploads\/([^/]+)\/publish$/; -const RE_REVOKE = /^\/_api\/v1\/admin\/tokens\/([^/]+)\/revoke$/; - -// Module-scope cache: cache name → D1 cache ID. -// Workers are short-lived isolates reused across requests on the same edge node, -// so this avoids a ~20ms D1 round trip on every narinfo warm hit. -const cacheNameToId = new Map(); - -async function resolveCacheId( - db: WorkerConfig["db"], - name: CacheName, -): Promise { - const cached = cacheNameToId.get(name); - if (cached !== undefined) return cached; - - const row = await findCacheByName(db, name); - if (!row) return undefined; - - cacheNameToId.set(name, row.id); - return row.id; -} - -export async function handleRequest( - request: Request, - config: WorkerConfig, -): Promise { - const url = new URL(request.url); - const method = request.method; - - if (url.pathname === "/") { - return json({ service: "helios-cache", status: "ok" }); - } - - if (url.pathname === "/healthz") { - return handleHealthz(config); - } - - // Admin API routes - if (url.pathname.startsWith("/_api/v1/admin/")) { - return handleAdminApi(request, config, url.pathname); - } - - // Write API routes (POST and PUT) - if (url.pathname.startsWith("/_api/v1/")) { - if (method !== "POST" && method !== "PUT") { - return new Response("Method Not Allowed", { - status: 405, - headers: { allow: "POST, PUT" }, - }); - } - return handleWriteApi(request, config, url.pathname); - } - - // Read paths (GET/HEAD only) - if (method !== "GET" && method !== "HEAD") { - return new Response("Method Not Allowed", { - status: 405, - headers: { allow: "GET, HEAD" }, - }); - } - - const segments = url.pathname.split("/").filter(Boolean); - if (segments.length < 2) { - return new Response("Not Found", { status: 404 }); - } - - // nix-cache-info is static — skip D1 lookup for the cache - if (segments[1] === "nix-cache-info") { - return NIX_CACHE_INFO_RESPONSE(); - } - - // NAR downloads are content-addressed — skip cache name D1 lookup - if (segments[1] === "nar" && segments.length === 4 && segments[3].endsWith(".nar")) { - return handleNarDownload(config, segments[2], segments[3].slice(0, -".nar".length), request, method); - } - - const cacheName = parseCacheName(segments[0]); - if (typeof cacheName === "object") { - return new Response("Not Found", { status: 404 }); - } - - if (segments[1].endsWith(".narinfo")) { - const hashStr = segments[1].slice(0, -".narinfo".length); - return handleNarinfo(config, cacheName, hashStr, method, request); - } - - return new Response("Not Found", { status: 404 }); -} - -function json(body: Record, status = 200): Response { - return new Response(JSON.stringify(body), { - status, - headers: { "content-type": "application/json; charset=utf-8" }, - }); -} - -function writeAuditLog( - config: WorkerConfig, - identity: AuthIdentity, - action: string, - cacheName: string | null, - status: number, - detail: string, - ip: string | null, -): void { - if (!config.ctx) return; - config.ctx.waitUntil( - createAuditLog(config.db, { - actor: actorId(identity), - action, - cacheName, - detail, - ip, - status, - }).catch((err) => { - console.error("Failed to write audit log", { error: err instanceof Error ? err.message : String(err) }); - }), - ); -} - -async function handleHealthz(config: WorkerConfig): Promise { - try { - await findCacheByName(config.db, "__healthz_probe__"); - return json({ ok: true, service: "helios-cache" }); - } catch { - return json({ ok: false, service: "helios-cache" }, 503); - } -} - -// Reused across requests — avoids per-request allocation -const textEncoder = new TextEncoder(); - -// Pre-computed static nix-cache-info body — avoids per-request string allocation -const NIX_CACHE_INFO_BODY = "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 40\n"; - -function NIX_CACHE_INFO_RESPONSE(): Response { - return new Response(NIX_CACHE_INFO_BODY, { - headers: { "content-type": "text/x-nix-cache-info" }, - }); -} - -// Nix mass-queries every substituter for every store path in a build, so most -// narinfo lookups are misses. Cache 404s briefly at the edge to keep that -// negative-lookup storm off D1. Short TTL so newly published paths appear quickly. -const NARINFO_NEGATIVE_TTL_SECONDS = 60; - -async function handleNarinfo( - config: WorkerConfig, - cacheName: CacheName, - hashStr: string, - method: string, - request: Request, -): Promise { - const storePathHash = parseStorePathHash(hashStr); - if (typeof storePathHash === "object") { - return new Response("Not Found", { status: 404 }); - } - - // Normalize to GET for shared cache (HEAD and GET share the same cache entry) - const cacheKey = new Request(request.url, { method: "GET" }); - const cache = await caches.open("helios-narinfo"); - const cached = await cache.match(cacheKey); - if (cached) { - if (method === "HEAD") { - return new Response(null, { status: cached.status, headers: cached.headers }); - } - return cached; - } - - // Unknown cache and unpublished path both produce a (negatively cached) 404. - const cacheId = await resolveCacheId(config.db, cacheName); - const result = cacheId === undefined - ? undefined - : await findPublishedPathWithBlob(config.db, cacheId, storePathHash); - if (!result) { - const notFoundHeaders = { "cache-control": `public, max-age=${NARINFO_NEGATIVE_TTL_SECONDS}` }; - if (config.ctx) { - // Fresh Response instead of clone(): a tee'd body whose sibling is - // never consumed can stall cache.put. - config.ctx.waitUntil( - cache.put(cacheKey, new Response("Not Found", { status: 404, headers: notFoundHeaders })), - ); - } - if (method === "HEAD") { - return new Response(null, { status: 404, headers: notFoundHeaders }); - } - return new Response("Not Found", { status: 404, headers: notFoundHeaders }); - } - - const refs: string[] = JSON.parse(result.path.referencesJson); - const storedSigs: string[] = JSON.parse(result.path.signaturesJson); - const fingerprint = computeFingerprint(result.path, refs); - const sig = await signNarinfo(fingerprint, config.signingKeyName, config.signingPrivateKey); - const signatures = sig ? [sig] : []; - const body = renderNarinfo(result.path, result.blob, refs, storedSigs, signatures); - - const response = new Response(body, { - headers: { - "content-type": "text/x-nix-narinfo", - "content-length": String(textEncoder.encode(body).byteLength), - "cache-control": "public, max-age=31536000, immutable", - }, - }); - - // Cache at edge (non-blocking) - if (config.ctx) { - config.ctx.waitUntil(cache.put(cacheKey, response.clone())); - } - - if (method === "HEAD") { - return new Response(null, { - status: 200, - headers: response.headers, - }); - } - - return response; -} - -async function handleNarDownload( - config: WorkerConfig, - rawFileHash: string, - rawCompression: string, - request: Request, - method: string, -): Promise { - const fileHash = parseFileHash(rawFileHash); - if (typeof fileHash === "object") { - return new Response("Not Found", { status: 404 }); - } - - const compression = parseCompression(rawCompression); - if (typeof compression === "object") { - return new Response("Not Found", { status: 404 }); - } - - // NAR keys are content-addressed, so we can skip the D1 lookup - // and go straight to R2 using the key derived from the URL - const r2Key = `nars/sha256/${fileHash}/${compression}.nar`; - - // Presigned R2 redirect (bypasses Worker entirely) - if (config.r2AccessKeyId && config.r2SecretAccessKey && config.r2Endpoint) { - const url = await createPresignedUrl( - config.r2Endpoint, - config.r2AccessKeyId, - config.r2SecretAccessKey, - config.r2BucketName, - r2Key, - 3600, - ); - return new Response(null, { - status: 302, - headers: { - location: url, - "cache-control": "public, max-age=3600", - }, - }); - } - - // Edge cache check (use a GET key for shared cache) - const cacheUrl = new URL(request.url); - const cacheRequest = new Request(cacheUrl, { method: "GET" }); - const cache = await caches.open("helios-nar"); - const cached = await cache.match(cacheRequest); - if (cached) { - if (method === "HEAD") { - return new Response(null, { status: 200, headers: cached.headers }); - } - return cached; - } - - if (method === "HEAD") { - const head = await config.bucket.head(r2Key); - if (!head) { - return new Response("Not Found", { status: 404 }); - } - return new Response(null, { - status: 200, - headers: { - "content-type": "application/x-nix-nar", - "content-length": String(head.size), - "cache-control": "public, max-age=31536000, immutable", - }, - }); - } - - const object = await config.bucket.get(r2Key); - if (!object) { - return new Response("Not Found", { status: 404 }); - } - - const response = new Response(object.body, { - headers: { - "content-type": "application/x-nix-nar", - "content-length": String(object.size), - "cache-control": "public, max-age=31536000, immutable", - }, - }); - - // Store in CF edge cache (non-blocking) - if (config.ctx) { - config.ctx.waitUntil(cache.put(cacheRequest, response.clone())); - } - - return response; -} - -async function handleAdminApi( - request: Request, - config: WorkerConfig, - pathname: string, -): Promise { - const authResult = await authenticateAdmin(request, config); - if (authResult.kind === "error") { - return authResult.response; - } - - const ip = request.headers.get("cf-connecting-ip"); - - if (pathname === "/_api/v1/admin/tokens" && request.method === "POST") { - const response = await handleCreateToken(request, config); - writeAuditLog(config, authResult.identity, "token.create", null, response.status, "{}", ip); - return response; - } - - if (pathname === "/_api/v1/admin/tokens" && request.method === "GET") { - const response = await handleListTokens(config); - writeAuditLog(config, authResult.identity, "token.list", null, response.status, "{}", ip); - return response; - } - - const revokeMatch = pathname.match(RE_REVOKE); - if (revokeMatch && request.method === "POST") { - const response = await handleRevokeToken(request, config, revokeMatch[1]); - writeAuditLog(config, authResult.identity, "token.revoke", null, response.status, JSON.stringify({ jti: revokeMatch[1] }), ip); - return response; - } - - return new Response("Not Found", { status: 404 }); -} - -async function handleWriteApi( - request: Request, - config: WorkerConfig, - pathname: string, -): Promise { - const authResult = await authenticateWrite(request, config); - if (authResult.kind === "error") { - return authResult.response; - } - - const identity = authResult.identity; - const ip = request.headers.get("cf-connecting-ip"); - - // Check push permission for JWT tokens - if (identity.kind === "jwt" && !hasPermission(identity.claims, "push")) { - const response = errorResponse("Forbidden", 403); - writeAuditLog(config, identity, "push", null, 403, '{"reason":"missing push permission"}', ip); - return response; - } - - if (pathname === "/_api/v1/get-missing-paths") { - const response = await handleGetMissingPaths(request, config, identity); - writeAuditLog(config, identity, "push", null, response.status, '{"endpoint":"get-missing-paths"}', ip); - return response; - } - - const sessionCreate = pathname.match(RE_SESSION_CREATE); - if (sessionCreate) { - const cacheName = sessionCreate[1]; - if (identity.kind === "jwt" && !hasCacheAccess(identity.claims, cacheName)) { - const response = errorResponse("Forbidden", 403); - writeAuditLog(config, identity, "push", cacheName, 403, '{"reason":"cache access denied"}', ip); - return response; - } - const response = await handleCreateSession(request, config, cacheName); - writeAuditLog(config, identity, "push", cacheName, response.status, '{"endpoint":"create-session"}', ip); - return response; - } - - // Session-scoped upload routes — all share the same audit pattern - const uploadRoutes: ReadonlyArray<{ - readonly pattern: RegExp; - readonly endpoint: string; - readonly handle: (match: RegExpMatchArray) => Promise; - }> = [ - { - pattern: RE_MULTIPART, - endpoint: "multipart", - handle: (m) => handleMultipart(config, m[1], identity), - }, - { - pattern: RE_PART_UPLOAD, - endpoint: "upload-part", - handle: (m) => handleUploadPart(request, config, m[1], parseInt(m[2], 10), identity), - }, - { - pattern: RE_BLOB, - endpoint: "upload-blob", - handle: (m) => handleUploadBlob(request, config, m[1], identity), - }, - { - pattern: RE_COMPLETE, - endpoint: "complete", - handle: (m) => handleComplete(request, config, m[1], identity), - }, - { - pattern: RE_PUBLISH, - endpoint: "publish", - handle: (m) => handlePublish(config, m[1], identity), - }, - ]; - - for (const route of uploadRoutes) { - const match = pathname.match(route.pattern); - if (match) { - const response = await route.handle(match); - writeAuditLog(config, identity, "push", null, response.status, `{"endpoint":"${route.endpoint}"}`, ip); - return response; - } - } - - return new Response("Not Found", { status: 404 }); -} diff --git a/workers/cache/src/signing.ts b/workers/cache/src/signing.ts deleted file mode 100644 index 81dd8f3..0000000 --- a/workers/cache/src/signing.ts +++ /dev/null @@ -1,59 +0,0 @@ -import type { PublishedPath } from "./db/types.js"; - -const textEncoder = new TextEncoder(); - -export function computeFingerprint(path: PublishedPath, refs: readonly string[]): string { - const refPaths = refs.map(r => `/nix/store/${r}`); - return `1;${path.storePath};${path.narHash};${path.narSize};${refPaths.join(",")}`; -} - -const PKCS8_ED25519_PREFIX = new Uint8Array([ - 0x30, 0x2e, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, - 0x03, 0x2b, 0x65, 0x70, 0x04, 0x22, 0x04, 0x20, -]); - -let cachedKeyMaterial: string | undefined; -let cachedKey: CryptoKey | undefined; - -async function getSigningKey(privateKeyBase64: string): Promise { - if (cachedKey && cachedKeyMaterial === privateKeyBase64) { - return cachedKey; - } - - const keyBytes = Uint8Array.from(atob(privateKeyBase64), c => c.charCodeAt(0)); - const seed = keyBytes.slice(0, 32); - const pkcs8 = new Uint8Array(PKCS8_ED25519_PREFIX.length + 32); - pkcs8.set(PKCS8_ED25519_PREFIX); - pkcs8.set(seed, PKCS8_ED25519_PREFIX.length); - const pkcs8Buffer = new ArrayBuffer(pkcs8.byteLength); - new Uint8Array(pkcs8Buffer).set(pkcs8); - - const key = await crypto.subtle.importKey( - "pkcs8", - pkcs8Buffer, - { name: "Ed25519" }, - false, - ["sign"], - ); - - cachedKeyMaterial = privateKeyBase64; - cachedKey = key; - return key; -} - -export async function signNarinfo( - fingerprint: string, - keyName: string, - privateKeyBase64: string, -): Promise { - if (!privateKeyBase64) { - return ""; - } - - const key = await getSigningKey(privateKeyBase64); - const data = textEncoder.encode(fingerprint); - const signature = await crypto.subtle.sign("Ed25519", key, data); - const sigBase64 = btoa(String.fromCharCode(...new Uint8Array(signature))); - - return `${keyName}:${sigBase64}`; -} diff --git a/workers/cache/src/uploads.ts b/workers/cache/src/uploads.ts deleted file mode 100644 index a1546b8..0000000 --- a/workers/cache/src/uploads.ts +++ /dev/null @@ -1,432 +0,0 @@ -import type { WorkerConfig } from "./config.js"; -import type { AuthIdentity } from "./auth.js"; -import { hasCacheAccess } from "./jwt.js"; -import { - findCacheByName, - findCacheById, - createUploadSession, - findUploadSession, - transitionSessionStatus, - transitionToMultipart, - upsertUploadPart, - upsertUploadParts, - findUploadParts, - createBlobObject, - findBlobObject, - createPublishedPath, - findPublishedPath, - findPublishedHashes, -} from "./db/repository.js"; -import type { UploadSession } from "./db/types.js"; -import { buildR2ObjectKey, parseFileHash, parseCompression, parseStorePathHash } from "@helios/cache-domain"; -import { jsonResponse, errorResponse, parseJsonBody } from "./responses.js"; -import type { DrizzleD1Database } from "drizzle-orm/d1"; - -// Module-scope cache: cache ID → cache name. Cache rows are immutable today -// (no rename or delete endpoints), so entries never go stale. Avoids a D1 -// round trip on every session-scoped upload request made with a JWT. -const cacheIdToName = new Map(); - -async function resolveCacheName( - db: DrizzleD1Database, - cacheId: number, -): Promise { - const cached = cacheIdToName.get(cacheId); - if (cached !== undefined) return cached; - - const row = await findCacheById(db, cacheId); - if (!row) return undefined; - - cacheIdToName.set(cacheId, row.name); - return row.name; -} - -// ── Session Auth Helper ── - -/** - * Load an upload session and verify the caller has access to its cache. - * Bearer-token callers are trusted implicitly; JWT callers must hold - * access to the cache the session belongs to. - */ -async function loadSessionWithAuthCheck( - config: WorkerConfig, - sessionId: string, - identity: AuthIdentity, -): Promise<{ session: UploadSession; error?: never } | { session?: never; error: Response }> { - const session = await findUploadSession(config.db, sessionId); - if (!session) { - return { error: errorResponse("Not found", 404) }; - } - - // Enforce session expiry at request time, not just via GC - if (session.expiresAt <= new Date().toISOString()) { - return { error: errorResponse("Not found", 404) }; - } - - if (identity.kind === "jwt") { - const cacheName = await resolveCacheName(config.db, session.cacheId); - if (cacheName === undefined || !hasCacheAccess(identity.claims, cacheName)) { - return { error: errorResponse("Forbidden", 403) }; - } - } - - return { session }; -} - -// ── Create Upload Session ── - -export async function handleCreateSession( - request: Request, - config: WorkerConfig, - cacheName: string, -): Promise { - const cache = await findCacheByName(config.db, cacheName); - if (!cache) { - return errorResponse("Cache not found", 404); - } - // Seed the ID → name cache so session-scoped requests skip the D1 lookup. - cacheIdToName.set(cache.id, cache.name); - - const body = await parseJsonBody<{ - storePath: string; - storePathHash: string; - narHash: string; - narSize: number; - fileHash: string; - fileSize: number; - compression: string; - references?: string[]; - deriver?: string; - system?: string; - }>(request); - if (body instanceof Response) return body; - - if (!body.storePath || !body.storePathHash || !body.narHash || !body.fileHash || !body.compression) { - return errorResponse("Missing required fields", 400); - } - - if (typeof body.narSize !== "number" || typeof body.fileSize !== "number") { - return errorResponse("narSize and fileSize must be numbers", 400); - } - - const parsedHash = parseStorePathHash(body.storePathHash); - if (typeof parsedHash === "object") { - return errorResponse(parsedHash.message, 400); - } - - if (!Number.isFinite(body.narSize) || body.narSize <= 0) { - return errorResponse("narSize must be a positive number", 400); - } - if (!Number.isFinite(body.fileSize) || body.fileSize <= 0) { - return errorResponse("fileSize must be a positive number", 400); - } - - const fileHash = parseFileHash(body.fileHash); - if (typeof fileHash === "object") { - return errorResponse(fileHash.message, 400); - } - - const compression = parseCompression(body.compression); - if (typeof compression === "object") { - return errorResponse(compression.message, 400); - } - - const r2Key = buildR2ObjectKey(fileHash, compression); - const sessionId = crypto.randomUUID(); - const expiresAt = new Date(Date.now() + 3600_000).toISOString(); - - const session = await createUploadSession(config.db, { - id: sessionId, - cacheId: cache.id, - storePathHash: body.storePathHash, - storePath: body.storePath, - narHash: body.narHash, - narSize: body.narSize, - fileHash: body.fileHash, - fileSize: body.fileSize, - compression: body.compression, - referencesJson: JSON.stringify(body.references ?? []), - deriver: body.deriver ?? null, - system: body.system ?? null, - r2UploadKey: r2Key, - r2UploadId: null, - expiresAt, - }); - - return jsonResponse({ - sessionId: session.id, - r2Key, - uploadMethod: "direct", - expiresAt, - }, 201); -} - -// ── Initiate Multipart Upload ── - -export async function handleMultipart( - config: WorkerConfig, - sessionId: string, - identity: AuthIdentity, -): Promise { - const result = await loadSessionWithAuthCheck(config, sessionId, identity); - if (result.error) return result.error; - const session = result.session; - - if (session.status !== "pending") { - return errorResponse("Conflict", 409); - } - - if (!session.r2UploadKey) { - return errorResponse("Internal error", 500); - } - - const multipart = await config.bucket.createMultipartUpload(session.r2UploadKey); - - const transitioned = await transitionToMultipart(config.db, sessionId, multipart.uploadId); - if (!transitioned) { - try { await multipart.abort(); } catch { /* best effort */ } - return errorResponse("Conflict", 409); - } - - return jsonResponse({ - uploadId: multipart.uploadId, - r2Key: session.r2UploadKey, - }); -} - -// ── Upload Part (Multipart) ── - -export async function handleUploadPart( - request: Request, - config: WorkerConfig, - sessionId: string, - partNumber: number, - identity: AuthIdentity, -): Promise { - const result = await loadSessionWithAuthCheck(config, sessionId, identity); - if (result.error) return result.error; - const session = result.session; - - if (session.status !== "uploading") { - return errorResponse("Conflict", 409); - } - - if (!session.r2UploadKey || !session.r2UploadId) { - return errorResponse("Bad request", 400); - } - - if (!request.body) { - return errorResponse("Request body is empty", 400); - } - - const multipart = config.bucket.resumeMultipartUpload(session.r2UploadKey, session.r2UploadId); - const uploadedPart = await multipart.uploadPart(partNumber, request.body); - - await upsertUploadPart(config.db, { - sessionId, - partNumber, - etag: uploadedPart.etag, - size: 0, - }); - - return jsonResponse({ partNumber, etag: uploadedPart.etag }); -} - -// ── Upload Blob ── - -export async function handleUploadBlob( - request: Request, - config: WorkerConfig, - sessionId: string, - identity: AuthIdentity, -): Promise { - const result = await loadSessionWithAuthCheck(config, sessionId, identity); - if (result.error) return result.error; - const session = result.session; - - if (session.status !== "pending" && session.status !== "uploading") { - return errorResponse("Conflict", 409); - } - - if (!session.r2UploadKey) { - return errorResponse("Internal error", 500); - } - - if (!request.body) { - return errorResponse("Request body is empty", 400); - } - - // Blob upload is idempotent (PUT overwrites), so allow both pending and uploading states. - // Attempt pending→uploading; if already uploading the transition fails but that is acceptable. - if (session.status === "pending") { - const transitioned = await transitionSessionStatus(config.db, sessionId, "pending", "uploading"); - if (!transitioned) { - // Re-read to check if someone else moved it to uploading (retry-safe) - const current = await findUploadSession(config.db, sessionId); - if (!current || current.status !== "uploading") { - return errorResponse("Conflict", 409); - } - } - } - - await config.bucket.put(session.r2UploadKey, request.body, { - httpMetadata: { contentType: "application/x-nix-nar" }, - }); - - return jsonResponse({ uploaded: true, r2Key: session.r2UploadKey }); -} - -// ── Complete Upload ── - -export async function handleComplete( - request: Request, - config: WorkerConfig, - sessionId: string, - identity: AuthIdentity, -): Promise { - const result = await loadSessionWithAuthCheck(config, sessionId, identity); - if (result.error) return result.error; - const session = result.session; - - if (session.status !== "uploading") { - return errorResponse("Conflict", 409); - } - - if (!session.r2UploadKey) { - return errorResponse("Internal error", 500); - } - - const body = await parseJsonBody<{ - parts?: Array<{ partNumber: number; etag: string; size: number }>; - }>(request); - if (body instanceof Response) return body; - - if (body.parts && body.parts.length > 0) { - await upsertUploadParts(config.db, sessionId, body.parts); - } - - if (session.r2UploadId) { - const parts = await findUploadParts(config.db, sessionId); - const multipart = config.bucket.resumeMultipartUpload(session.r2UploadKey, session.r2UploadId); - await multipart.complete(parts.map(p => ({ partNumber: p.partNumber, etag: p.etag }))); - } - - const head = await config.bucket.head(session.r2UploadKey); - if (!head) { - return errorResponse("Upload incomplete", 400); - } - - if (head.size !== session.fileSize) { - return errorResponse("File size mismatch", 400); - } - - const transitioned = await transitionSessionStatus(config.db, sessionId, "uploading", "completed"); - if (!transitioned) { - return errorResponse("Conflict", 409); - } - - return jsonResponse({ status: "completed" }); -} - -// ── Publish Path ── - -export async function handlePublish( - config: WorkerConfig, - sessionId: string, - identity: AuthIdentity, -): Promise { - const result = await loadSessionWithAuthCheck(config, sessionId, identity); - if (result.error) return result.error; - const session = result.session; - - if (session.status !== "completed") { - return errorResponse("Conflict", 409); - } - - const existing = await findPublishedPath(config.db, session.cacheId, session.storePathHash); - if (existing) { - return jsonResponse({ published: true, storePathHash: session.storePathHash, alreadyExisted: true }); - } - - let blob = await findBlobObject(config.db, session.fileHash, session.compression); - if (!blob) { - if (!session.r2UploadKey) { - return errorResponse("Internal error", 500); - } - try { - blob = await createBlobObject(config.db, { - fileHash: session.fileHash, - fileSize: session.fileSize, - compression: session.compression, - r2Key: session.r2UploadKey, - }); - } catch { - blob = await findBlobObject(config.db, session.fileHash, session.compression); - if (!blob) { - return errorResponse("Internal error", 500); - } - } - } - - try { - await createPublishedPath(config.db, { - cacheId: session.cacheId, - storePathHash: session.storePathHash, - storePath: session.storePath, - narHash: session.narHash, - narSize: session.narSize, - blobObjectId: blob.id, - referencesJson: session.referencesJson, - deriver: session.deriver, - system: session.system, - signaturesJson: "[]", - }); - } catch { - const raced = await findPublishedPath(config.db, session.cacheId, session.storePathHash); - if (raced) { - return jsonResponse({ published: true, storePathHash: session.storePathHash, alreadyExisted: true }); - } - return errorResponse("Internal error", 500); - } - - return jsonResponse({ published: true, storePathHash: session.storePathHash }); -} - -// ── Get Missing Paths ── - -const MAX_MISSING_PATHS_BATCH = 1000; - -export async function handleGetMissingPaths( - request: Request, - config: WorkerConfig, - identity: AuthIdentity, -): Promise { - const body = await parseJsonBody<{ - cache: string; - storePathHashes: string[]; - }>(request); - if (body instanceof Response) return body; - - if (!body.cache || !Array.isArray(body.storePathHashes)) { - return errorResponse("Missing required fields: cache, storePathHashes", 400); - } - - if (body.storePathHashes.length > MAX_MISSING_PATHS_BATCH) { - return errorResponse(`storePathHashes exceeds maximum of ${MAX_MISSING_PATHS_BATCH}`, 400); - } - - const cache = await findCacheByName(config.db, body.cache); - if (!cache) { - return errorResponse("Cache not found", 404); - } - - if (identity.kind === "jwt" && !hasCacheAccess(identity.claims, body.cache)) { - return errorResponse("Forbidden", 403); - } - - const existingHashes = await findPublishedHashes(config.db, cache.id, body.storePathHashes); - const existingSet = new Set(existingHashes); - const missing = body.storePathHashes.filter((h) => !existingSet.has(h)); - - return jsonResponse({ missing }); -} diff --git a/workers/cache/test/apply-migrations.ts b/workers/cache/test/apply-migrations.ts deleted file mode 100644 index 5311171..0000000 --- a/workers/cache/test/apply-migrations.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { applyD1Migrations } from "cloudflare:test"; -import { env } from "cloudflare:test"; - -await applyD1Migrations(env.CACHE_DB, env.TEST_MIGRATIONS); diff --git a/workers/cache/test/auth.spec.ts b/workers/cache/test/auth.spec.ts deleted file mode 100644 index ac2fcc0..0000000 --- a/workers/cache/test/auth.spec.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { env, createExecutionContext, waitOnExecutionContext, SELF } from "cloudflare:test"; -import { describe, it, expect, beforeAll } from "vitest"; -import { drizzle } from "drizzle-orm/d1"; -import { createCache } from "../src/db/repository.js"; -import worker from "../src"; - -const AUTH_TOKEN = "test-secret-token"; -const CACHE_NAME = "auth-test"; - -function envWithAuth(): typeof env { - return { ...env, AUTH_TOKEN }; -} - -function postWithWorker( - path: string, - body: unknown, - headers: Record = {}, -): Promise { - const request = new Request(`http://example.com${path}`, { - method: "POST", - headers: { "content-type": "application/json", ...headers }, - body: JSON.stringify(body), - }); - const ctx = createExecutionContext(); - const response = worker.fetch(request, envWithAuth(), ctx); - waitOnExecutionContext(ctx); - return response; -} - -beforeAll(async () => { - const db = drizzle(env.CACHE_DB); - await createCache(db, CACHE_NAME); -}); - -describe("write auth", () => { - it("rejects write requests without authorization header", async () => { - const res = await postWithWorker( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - { storePath: "/nix/store/test", storePathHash: "a".repeat(32), narHash: "sha256:" + "1".repeat(52), narSize: 100, fileHash: "b".repeat(64), fileSize: 100, compression: "zstd" }, - ); - - expect(res.status).toBe(401); - const body = await res.json<{ error: string }>(); - expect(body.error).toBe("Unauthorized"); - }); - - it("rejects write requests with wrong bearer token", async () => { - const res = await postWithWorker( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - { storePath: "/nix/store/test", storePathHash: "a".repeat(32), narHash: "sha256:" + "1".repeat(52), narSize: 100, fileHash: "b".repeat(64), fileSize: 100, compression: "zstd" }, - { authorization: "Bearer wrong-token" }, - ); - - expect(res.status).toBe(403); - const body = await res.json<{ error: string }>(); - expect(body.error).toBe("Forbidden"); - }); - - it("accepts write requests with valid bearer token", async () => { - const res = await postWithWorker( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - { storePath: "/nix/store/test", storePathHash: "c".repeat(32), narHash: "sha256:" + "1".repeat(52), narSize: 100, fileHash: "d".repeat(64), fileSize: 100, compression: "zstd" }, - { authorization: `Bearer ${AUTH_TOKEN}` }, - ); - - expect(res.status).toBe(201); - }); - - it("rejects non-bearer auth schemes", async () => { - const res = await postWithWorker( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - { storePath: "/nix/store/test", storePathHash: "a".repeat(32), narHash: "sha256:" + "1".repeat(52), narSize: 100, fileHash: "b".repeat(64), fileSize: 100, compression: "zstd" }, - { authorization: `Basic ${AUTH_TOKEN}` }, - ); - - expect(res.status).toBe(401); - }); -}); - -describe("read endpoints remain public", () => { - it("GET nix-cache-info works without auth even when AUTH_TOKEN is set", async () => { - const request = new Request(`http://example.com/${CACHE_NAME}/nix-cache-info`); - const ctx = createExecutionContext(); - const res = await worker.fetch(request, envWithAuth(), ctx); - await waitOnExecutionContext(ctx); - - expect(res.status).toBe(200); - const body = await res.text(); - expect(body).toContain("StoreDir: /nix/store"); - }); - - it("read endpoints via SELF work without auth (AUTH_TOKEN empty in test env)", async () => { - const res = await SELF.fetch(`http://example.com/${CACHE_NAME}/nix-cache-info`); - - expect(res.status).toBe(200); - }); -}); diff --git a/workers/cache/test/db.spec.ts b/workers/cache/test/db.spec.ts deleted file mode 100644 index a3cf6c5..0000000 --- a/workers/cache/test/db.spec.ts +++ /dev/null @@ -1,390 +0,0 @@ -import { env } from "cloudflare:test"; -import { describe, it, expect } from "vitest"; -import { drizzle } from "drizzle-orm/d1"; -import { - createCache, - findCacheByName, - createBlobObject, - findBlobObject, - createPublishedPath, - findPublishedPath, - createUploadSession, - findUploadSession, - transitionSessionStatus, - findExpiredSessions, - upsertUploadPart, - findUploadParts, - createGcMark, - findGcMarks, - deleteGcMark, -} from "../src/db/repository.js"; - -function getDb() { - return drizzle(env.CACHE_DB); -} - -function makeSessionParams(overrides: { - readonly id: string; - readonly cacheId: number; - readonly expiresAt?: string; - readonly storePathHash?: string; -}) { - return { - id: overrides.id, - cacheId: overrides.cacheId, - storePathHash: overrides.storePathHash ?? crypto.randomUUID(), - storePath: `/nix/store/${crypto.randomUUID()}`, - narHash: `sha256:${crypto.randomUUID()}`, - narSize: 1024, - fileHash: `sha256:${crypto.randomUUID()}`, - fileSize: 512, - compression: "zstd", - referencesJson: "[]", - deriver: null, - system: null, - r2UploadKey: null, - r2UploadId: null, - expiresAt: overrides.expiresAt ?? "2099-01-01T00:00:00.000Z", - }; -} - -describe("caches", () => { - it("createCache creates a public cache with expected fields", async () => { - const name = `cache-create-${crypto.randomUUID()}`; - const row = await createCache(getDb(), name); - - expect(row.name).toBe(name); - expect(typeof row.id).toBe("number"); - expect(typeof row.createdAt).toBe("string"); - }); - - it("findCacheByName returns a matching cache", async () => { - const name = `cache-find-${crypto.randomUUID()}`; - const created = await createCache(getDb(), name); - const found = await findCacheByName(getDb(), name); - - expect(found).toBeDefined(); - expect(found!.id).toBe(created.id); - expect(found!.name).toBe(name); - }); - - it("findCacheByName returns undefined for non-existent name", async () => { - const found = await findCacheByName(getDb(), `no-such-cache-${crypto.randomUUID()}`); - expect(found).toBeUndefined(); - }); - - it("createCache with duplicate name throws", async () => { - const name = `cache-dup-${crypto.randomUUID()}`; - await createCache(getDb(), name); - - await expect(createCache(getDb(), name)).rejects.toThrow(); - }); -}); - -describe("blob_objects", () => { - it("createBlobObject creates a row with expected fields", async () => { - const fileHash = `sha256:${crypto.randomUUID()}`; - const row = await createBlobObject(getDb(), { - fileHash, - fileSize: 2048, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - - expect(row.fileHash).toBe(fileHash); - expect(row.fileSize).toBe(2048); - expect(row.compression).toBe("zstd"); - expect(typeof row.id).toBe("number"); - expect(typeof row.createdAt).toBe("string"); - }); - - it("findBlobObject returns a matching blob", async () => { - const fileHash = `sha256:${crypto.randomUUID()}`; - const created = await createBlobObject(getDb(), { - fileHash, - fileSize: 100, - compression: "none", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - const found = await findBlobObject(getDb(), fileHash, "none"); - - expect(found).toBeDefined(); - expect(found!.id).toBe(created.id); - }); - - it("findBlobObject returns undefined for non-existent", async () => { - const found = await findBlobObject(getDb(), `sha256:${crypto.randomUUID()}`, "zstd"); - expect(found).toBeUndefined(); - }); - - it("createBlobObject with same fileHash+compression throws", async () => { - const fileHash = `sha256:${crypto.randomUUID()}`; - await createBlobObject(getDb(), { - fileHash, - fileSize: 100, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - - await expect( - createBlobObject(getDb(), { - fileHash, - fileSize: 200, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }), - ).rejects.toThrow(); - }); -}); - -describe("published_paths", () => { - it("createPublishedPath creates a row with expected fields", async () => { - const cache = await createCache(getDb(), `pp-create-${crypto.randomUUID()}`); - const blob = await createBlobObject(getDb(), { - fileHash: `sha256:${crypto.randomUUID()}`, - fileSize: 500, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - const storePathHash = crypto.randomUUID(); - - const row = await createPublishedPath(getDb(), { - cacheId: cache.id, - storePathHash, - storePath: `/nix/store/${storePathHash}`, - narHash: `sha256:${crypto.randomUUID()}`, - narSize: 4096, - blobObjectId: blob.id, - referencesJson: "[]", - deriver: null, - system: "x86_64-linux", - signaturesJson: "[]", - }); - - expect(row.cacheId).toBe(cache.id); - expect(row.storePathHash).toBe(storePathHash); - expect(row.blobObjectId).toBe(blob.id); - expect(row.system).toBe("x86_64-linux"); - expect(typeof row.id).toBe("number"); - expect(typeof row.createdAt).toBe("string"); - }); - - it("findPublishedPath returns a matching path", async () => { - const cache = await createCache(getDb(), `pp-find-${crypto.randomUUID()}`); - const blob = await createBlobObject(getDb(), { - fileHash: `sha256:${crypto.randomUUID()}`, - fileSize: 500, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - const storePathHash = crypto.randomUUID(); - - const created = await createPublishedPath(getDb(), { - cacheId: cache.id, - storePathHash, - storePath: `/nix/store/${storePathHash}`, - narHash: `sha256:${crypto.randomUUID()}`, - narSize: 4096, - blobObjectId: blob.id, - referencesJson: "[]", - deriver: null, - system: null, - signaturesJson: "[]", - }); - const found = await findPublishedPath(getDb(), cache.id, storePathHash); - - expect(found).toBeDefined(); - expect(found!.id).toBe(created.id); - }); - - it("findPublishedPath returns undefined for non-existent", async () => { - const found = await findPublishedPath(getDb(), 999999, crypto.randomUUID()); - expect(found).toBeUndefined(); - }); - - it("createPublishedPath with same cache+storePathHash throws", async () => { - const cache = await createCache(getDb(), `pp-dup-${crypto.randomUUID()}`); - const blob = await createBlobObject(getDb(), { - fileHash: `sha256:${crypto.randomUUID()}`, - fileSize: 500, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - const storePathHash = crypto.randomUUID(); - const common = { - cacheId: cache.id, - storePathHash, - storePath: `/nix/store/${storePathHash}`, - narHash: `sha256:${crypto.randomUUID()}`, - narSize: 4096, - blobObjectId: blob.id, - referencesJson: "[]", - deriver: null, - system: null, - signaturesJson: "[]", - }; - - await createPublishedPath(getDb(), common); - await expect(createPublishedPath(getDb(), common)).rejects.toThrow(); - }); -}); - -describe("upload_sessions", () => { - it("createUploadSession creates a session with status pending", async () => { - const cache = await createCache(getDb(), `us-create-${crypto.randomUUID()}`); - const sessionId = crypto.randomUUID(); - const row = await createUploadSession(getDb(), makeSessionParams({ id: sessionId, cacheId: cache.id })); - - expect(row.id).toBe(sessionId); - expect(row.cacheId).toBe(cache.id); - expect(row.status).toBe("pending"); - expect(typeof row.createdAt).toBe("string"); - }); - - it("findUploadSession returns a matching session", async () => { - const cache = await createCache(getDb(), `us-find-${crypto.randomUUID()}`); - const sessionId = crypto.randomUUID(); - await createUploadSession(getDb(), makeSessionParams({ id: sessionId, cacheId: cache.id })); - const found = await findUploadSession(getDb(), sessionId); - - expect(found).toBeDefined(); - expect(found!.id).toBe(sessionId); - }); - - it("transitionSessionStatus updates and returns the session", async () => { - const cache = await createCache(getDb(), `us-update-${crypto.randomUUID()}`); - const sessionId = crypto.randomUUID(); - await createUploadSession(getDb(), makeSessionParams({ id: sessionId, cacheId: cache.id })); - - const updated = await transitionSessionStatus(getDb(), sessionId, "pending", "uploading"); - - expect(updated).toBeDefined(); - expect(updated!.status).toBe("uploading"); - expect(updated!.id).toBe(sessionId); - }); - - it("findExpiredSessions returns only expired non-completed sessions", async () => { - const cache = await createCache(getDb(), `us-expired-${crypto.randomUUID()}`); - - const expiredId = crypto.randomUUID(); - await createUploadSession( - getDb(), - makeSessionParams({ id: expiredId, cacheId: cache.id, expiresAt: "2020-01-01T00:00:00.000Z" }), - ); - - const freshId = crypto.randomUUID(); - await createUploadSession( - getDb(), - makeSessionParams({ id: freshId, cacheId: cache.id, expiresAt: "2099-01-01T00:00:00.000Z" }), - ); - - const expired = await findExpiredSessions(getDb(), new Date().toISOString()); - const expiredIds = expired.map((s) => s.id); - - expect(expiredIds).toContain(expiredId); - expect(expiredIds).not.toContain(freshId); - }); - - it("findExpiredSessions excludes completed sessions even if expired", async () => { - const cache = await createCache(getDb(), `us-exp-compl-${crypto.randomUUID()}`); - - const sessionId = crypto.randomUUID(); - await createUploadSession( - getDb(), - makeSessionParams({ id: sessionId, cacheId: cache.id, expiresAt: "2020-01-01T00:00:00.000Z" }), - ); - await transitionSessionStatus(getDb(), sessionId, "pending", "completed"); - - const expired = await findExpiredSessions(getDb(), new Date().toISOString()); - const expiredIds = expired.map((s) => s.id); - - expect(expiredIds).not.toContain(sessionId); - }); -}); - -describe("upload_parts", () => { - it("upsertUploadPart creates a part with expected fields", async () => { - const cache = await createCache(getDb(), `up-create-${crypto.randomUUID()}`); - const sessionId = crypto.randomUUID(); - await createUploadSession(getDb(), makeSessionParams({ id: sessionId, cacheId: cache.id })); - - const row = await upsertUploadPart(getDb(), { - sessionId, - partNumber: 1, - etag: "etag-abc", - size: 1024, - }); - - expect(row.sessionId).toBe(sessionId); - expect(row.partNumber).toBe(1); - expect(row.etag).toBe("etag-abc"); - expect(row.size).toBe(1024); - expect(typeof row.id).toBe("number"); - }); - - it("findUploadParts returns parts ordered by partNumber", async () => { - const cache = await createCache(getDb(), `up-find-${crypto.randomUUID()}`); - const sessionId = crypto.randomUUID(); - await createUploadSession(getDb(), makeSessionParams({ id: sessionId, cacheId: cache.id })); - - await upsertUploadPart(getDb(), { sessionId, partNumber: 3, etag: "e3", size: 300 }); - await upsertUploadPart(getDb(), { sessionId, partNumber: 1, etag: "e1", size: 100 }); - await upsertUploadPart(getDb(), { sessionId, partNumber: 2, etag: "e2", size: 200 }); - - const parts = await findUploadParts(getDb(), sessionId); - - expect(parts).toHaveLength(3); - expect(parts[0].partNumber).toBe(1); - expect(parts[1].partNumber).toBe(2); - expect(parts[2].partNumber).toBe(3); - }); -}); - -describe("gc_marks", () => { - it("createGcMark creates a mark with expected fields", async () => { - const targetId = crypto.randomUUID(); - const row = await createGcMark(getDb(), { - targetType: "blob", - targetId, - reason: "orphaned", - }); - - expect(row.targetType).toBe("blob"); - expect(row.targetId).toBe(targetId); - expect(row.reason).toBe("orphaned"); - expect(typeof row.id).toBe("number"); - expect(typeof row.markedAt).toBe("string"); - }); - - it("findGcMarks returns only marks matching targetType", async () => { - const blobId = crypto.randomUUID(); - const sessionId = crypto.randomUUID(); - - await createGcMark(getDb(), { targetType: "blob_find", targetId: blobId, reason: "orphaned" }); - await createGcMark(getDb(), { targetType: "session_find", targetId: sessionId, reason: "expired" }); - - const blobMarks = await findGcMarks(getDb(), "blob_find"); - - expect(blobMarks).toHaveLength(1); - expect(blobMarks[0].targetId).toBe(blobId); - }); - - it("deleteGcMark removes the mark", async () => { - const targetId = crypto.randomUUID(); - await createGcMark(getDb(), { targetType: "blob_del", targetId, reason: "orphaned" }); - - await deleteGcMark(getDb(), "blob_del", targetId); - - const marks = await findGcMarks(getDb(), "blob_del"); - expect(marks).toHaveLength(0); - }); - - it("createGcMark with same target_type+target_id throws", async () => { - const targetId = crypto.randomUUID(); - await createGcMark(getDb(), { targetType: "blob_idem", targetId, reason: "orphaned" }); - - await expect( - createGcMark(getDb(), { targetType: "blob_idem", targetId, reason: "orphaned again" }), - ).rejects.toThrow(); - }); -}); \ No newline at end of file diff --git a/workers/cache/test/env.d.ts b/workers/cache/test/env.d.ts deleted file mode 100644 index 3972994..0000000 --- a/workers/cache/test/env.d.ts +++ /dev/null @@ -1,7 +0,0 @@ -import type { D1Migration } from "@cloudflare/vitest-pool-workers"; - -declare module "cloudflare:test" { - interface ProvidedEnv extends Env { - TEST_MIGRATIONS: D1Migration[]; - } -} diff --git a/workers/cache/test/gc.spec.ts b/workers/cache/test/gc.spec.ts deleted file mode 100644 index 9b2d564..0000000 --- a/workers/cache/test/gc.spec.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { env } from "cloudflare:test"; -import { describe, it, expect } from "vitest"; -import { drizzle } from "drizzle-orm/d1"; -import { - createCache, - createUploadSession, - createBlobObject, - createPublishedPath, - findUploadSession, - findBlobObjectById, -} from "../src/db/repository.js"; -import { runGarbageCollection } from "../src/gc.js"; -import type { WorkerConfig } from "../src/config.js"; - -function getConfig(): WorkerConfig { - return { - bucket: env.CACHE_BUCKET, - db: drizzle(env.CACHE_DB), - }; -} - -function makeSessionParams(overrides: { - readonly id: string; - readonly cacheId: number; - readonly expiresAt?: string; - readonly r2UploadKey?: string | null; -}) { - return { - id: overrides.id, - cacheId: overrides.cacheId, - storePathHash: crypto.randomUUID(), - storePath: `/nix/store/${crypto.randomUUID()}`, - narHash: `sha256:${crypto.randomUUID()}`, - narSize: 1024, - fileHash: `sha256:${crypto.randomUUID()}`, - fileSize: 512, - compression: "zstd", - referencesJson: "[]", - deriver: null, - system: null, - r2UploadKey: overrides.r2UploadKey ?? null, - r2UploadId: null, - expiresAt: overrides.expiresAt ?? "2099-01-01T00:00:00.000Z", - }; -} - -describe("gc: expire abandoned sessions", () => { - it("expires and deletes sessions past their expiresAt", async () => { - const config = getConfig(); - const cache = await createCache(config.db, `gc-expire-${crypto.randomUUID()}`); - const sessionId = crypto.randomUUID(); - await createUploadSession( - config.db, - makeSessionParams({ id: sessionId, cacheId: cache.id, expiresAt: "2020-01-01T00:00:00.000Z" }), - ); - - const result = await runGarbageCollection(config); - - expect(result.expiredSessions).toBeGreaterThanOrEqual(1); - expect(result.errors).toHaveLength(0); - - const found = await findUploadSession(config.db, sessionId); - expect(found).toBeUndefined(); - }); -}); - -describe("gc: delete unreferenced blobs", () => { - it("deletes blob objects not referenced by any published path", async () => { - const config = getConfig(); - const blob = await createBlobObject(config.db, { - fileHash: `sha256:${crypto.randomUUID()}`, - fileSize: 2048, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - - const result = await runGarbageCollection(config); - - expect(result.deletedBlobs).toBeGreaterThanOrEqual(1); - expect(result.errors).toHaveLength(0); - - const found = await findBlobObjectById(config.db, blob.id); - expect(found).toBeUndefined(); - }); -}); - -describe("gc: keep referenced blobs", () => { - it("preserves blob objects that are referenced by a published path", async () => { - const config = getConfig(); - const cache = await createCache(config.db, `gc-keep-${crypto.randomUUID()}`); - const blob = await createBlobObject(config.db, { - fileHash: `sha256:${crypto.randomUUID()}`, - fileSize: 2048, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - const storePathHash = crypto.randomUUID(); - await createPublishedPath(config.db, { - cacheId: cache.id, - storePathHash, - storePath: `/nix/store/${storePathHash}`, - narHash: `sha256:${crypto.randomUUID()}`, - narSize: 4096, - blobObjectId: blob.id, - referencesJson: "[]", - deriver: null, - system: null, - signaturesJson: "[]", - }); - - const result = await runGarbageCollection(config); - - const found = await findBlobObjectById(config.db, blob.id); - expect(found).toBeDefined(); - }); -}); - -describe("gc: keep active sessions", () => { - it("does not expire sessions with future expiresAt", async () => { - const config = getConfig(); - const cache = await createCache(config.db, `gc-active-${crypto.randomUUID()}`); - const sessionId = crypto.randomUUID(); - await createUploadSession( - config.db, - makeSessionParams({ id: sessionId, cacheId: cache.id, expiresAt: "2099-01-01T00:00:00.000Z" }), - ); - - const result = await runGarbageCollection(config); - - expect(result.expiredSessions).toBe(0); - - const found = await findUploadSession(config.db, sessionId); - expect(found).toBeDefined(); - }); -}); diff --git a/workers/cache/test/index.spec.ts b/workers/cache/test/index.spec.ts deleted file mode 100644 index eea5bd0..0000000 --- a/workers/cache/test/index.spec.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { - env, - createExecutionContext, - waitOnExecutionContext, - SELF, -} from "cloudflare:test"; -import { describe, it, expect } from "vitest"; -import worker from "../src"; - -describe("cache worker", () => { - it("serves service info on /", async () => { - const request = new Request( - "http://example.com/" - ); - const ctx = createExecutionContext(); - const response = await worker.fetch(request, env, ctx); - - await waitOnExecutionContext(ctx); - - expect(response.status).toBe(200); - expect(await response.json()).toEqual({ - service: "helios-cache", - status: "ok", - }); - }); - - it("serves a health check on /healthz", async () => { - const request = new Request("http://example.com/healthz"); - const response = await SELF.fetch(request); - - expect(response.status).toBe(200); - const body = await response.json<{ ok: boolean; service: string }>(); - expect(body.service).toBe("helios-cache"); - expect(body.ok).toBe(true); - }); - - it("rejects unsupported methods on cache paths", async () => { - const request = new Request("http://example.com/mycache/test.narinfo", { method: "POST" }); - const response = await SELF.fetch(request); - - expect(response.status).toBe(405); - expect(response.headers.get("allow")).toBe("GET, HEAD"); - }); - - it("returns 404 for unknown paths", async () => { - const request = new Request("http://example.com/nonexistent"); - const response = await SELF.fetch(request); - - expect(response.status).toBe(404); - }); -}); diff --git a/workers/cache/test/read-path.spec.ts b/workers/cache/test/read-path.spec.ts deleted file mode 100644 index 9b8994f..0000000 --- a/workers/cache/test/read-path.spec.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { env, SELF } from "cloudflare:test"; -import { describe, it, expect, beforeAll } from "vitest"; -import { drizzle } from "drizzle-orm/d1"; -import { createCache, createBlobObject, createPublishedPath } from "../src/db/repository.js"; -import { handleRequest } from "../src/router.js"; -import { resolveConfig } from "../src/config.js"; - -const STORE_PATH_HASH = "0".repeat(32); -const FILE_HASH = "a".repeat(64); -const NAR_HASH = `sha256:${"1".repeat(52)}`; -const R2_KEY = `nars/sha256/${FILE_HASH}/zstd.nar`; -const R2_BODY = new Uint8Array([1, 2, 3, 4]); - -function getDb() { - return drizzle(env.CACHE_DB); -} - -beforeAll(async () => { - const db = getDb(); - - const cache = await createCache(db, "test-cache"); - - const blob = await createBlobObject(db, { - fileHash: FILE_HASH, - fileSize: 1024, - compression: "zstd", - r2Key: R2_KEY, - }); - - await createPublishedPath(db, { - cacheId: cache.id, - storePathHash: STORE_PATH_HASH, - storePath: `/nix/store/${STORE_PATH_HASH}-hello`, - narHash: NAR_HASH, - narSize: 2048, - blobObjectId: blob.id, - referencesJson: "[]", - deriver: null, - system: null, - signaturesJson: "[]", - }); - - await env.CACHE_BUCKET.put(R2_KEY, R2_BODY); -}); - -describe("nix-cache-info", () => { - it("returns cache info for an existing cache", async () => { - const res = await SELF.fetch("http://example.com/test-cache/nix-cache-info"); - - expect(res.status).toBe(200); - expect(res.headers.get("content-type")).toBe("text/x-nix-cache-info"); - const body = await res.text(); - expect(body).toContain("StoreDir: /nix/store"); - }); - - it("returns 200 for any cache path", async () => { - const res = await SELF.fetch("http://example.com/nonexistent/nix-cache-info"); - - expect(res.status).toBe(200); - }); -}); - -describe("narinfo", () => { - it("returns narinfo for a published path", async () => { - const res = await SELF.fetch(`http://example.com/test-cache/${STORE_PATH_HASH}.narinfo`); - - expect(res.status).toBe(200); - expect(res.headers.get("content-type")).toBe("text/x-nix-narinfo"); - const body = await res.text(); - expect(body).toContain("StorePath:"); - expect(body).toContain("NarHash:"); - expect(body).toContain("FileHash:"); - expect(body).toContain("Compression: zstd"); - expect(body).toContain("URL: nar/"); - }); - - it("HEAD returns 200 with headers but no body", async () => { - const request = new Request(`http://example.com/test-cache/${STORE_PATH_HASH}.narinfo`, { - method: "HEAD", - }); - const config = resolveConfig(env); - const res = await handleRequest(request, config); - - expect(res.status).toBe(200); - expect(res.headers.get("content-type")).toBe("text/x-nix-narinfo"); - expect(res.headers.get("content-length")).toBeTruthy(); - expect(res.headers.get("cache-control")).toContain("immutable"); - const body = await res.text(); - expect(body).toBe(""); - }); - - it("returns 404 for a valid hash that is not published", async () => { - const res = await SELF.fetch(`http://example.com/test-cache/${"a".repeat(32)}.narinfo`); - - expect(res.status).toBe(404); - }); - - it("returns 404 for an invalid hash format", async () => { - const res = await SELF.fetch("http://example.com/test-cache/INVALID.narinfo"); - - expect(res.status).toBe(404); - }); - - it("returns 404 when the cache does not exist", async () => { - const res = await SELF.fetch(`http://example.com/noexist/${STORE_PATH_HASH}.narinfo`); - - expect(res.status).toBe(404); - }); -}); - -describe("nar download", () => { - it("returns the nar blob for a valid request", async () => { - const res = await SELF.fetch(`http://example.com/test-cache/nar/${FILE_HASH}/zstd.nar`); - - expect(res.status).toBe(200); - expect(res.headers.get("content-type")).toBe("application/x-nix-nar"); - expect(res.headers.get("cache-control")).toContain("immutable"); - const body = new Uint8Array(await res.arrayBuffer()); - expect(body).toEqual(R2_BODY); - }); - - it("returns 404 for a non-existent blob", async () => { - const res = await SELF.fetch(`http://example.com/test-cache/nar/${"b".repeat(64)}/zstd.nar`); - - expect(res.status).toBe(404); - }); - - it("returns 404 for an invalid file hash", async () => { - const res = await SELF.fetch("http://example.com/test-cache/nar/invalid/zstd.nar"); - - expect(res.status).toBe(404); - }); - - it("returns 404 for an invalid compression", async () => { - const res = await SELF.fetch(`http://example.com/test-cache/nar/${FILE_HASH}/gzip.nar`); - - expect(res.status).toBe(404); - }); -}); diff --git a/workers/cache/test/tsconfig.json b/workers/cache/test/tsconfig.json deleted file mode 100644 index 978ecd8..0000000 --- a/workers/cache/test/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../tsconfig.json", - "compilerOptions": { - "types": ["@cloudflare/vitest-pool-workers"] - }, - "include": ["./**/*.ts", "../worker-configuration.d.ts"], - "exclude": [] -} diff --git a/workers/cache/test/upload-flow.spec.ts b/workers/cache/test/upload-flow.spec.ts deleted file mode 100644 index 775e6a8..0000000 --- a/workers/cache/test/upload-flow.spec.ts +++ /dev/null @@ -1,241 +0,0 @@ -import { env, SELF } from "cloudflare:test"; -import { describe, it, expect, beforeAll } from "vitest"; -import { drizzle } from "drizzle-orm/d1"; -import { createCache, findPublishedPath, findCacheByName } from "../src/db/repository.js"; - -const CACHE_NAME = "upload-test"; -const AUTH_TOKEN = "test-auth-token"; -const BLOB_DATA = new Uint8Array([1, 2, 3, 4]); - -function getDb() { - return drizzle(env.CACHE_DB); -} - -function post(path: string, body: unknown): Promise { - return SELF.fetch(`http://example.com${path}`, { - method: "POST", - headers: { - "content-type": "application/json", - authorization: `Bearer ${AUTH_TOKEN}`, - }, - body: JSON.stringify(body), - }); -} - -function sessionBody(overrides: { fileHash: string; storePathHash: string }) { - return { - storePath: `/nix/store/${overrides.storePathHash}-hello`, - storePathHash: overrides.storePathHash, - narHash: `sha256:${"1".repeat(52)}`, - narSize: 2048, - fileHash: overrides.fileHash, - fileSize: BLOB_DATA.byteLength, - compression: "zstd", - }; -} - -function uniqueFileHash() { - return crypto.randomUUID().replace(/-/g, "").slice(0, 64).padEnd(64, "0"); -} - -let storePathCounter = 1; -function uniqueStorePathHash() { - const c = String(storePathCounter++); - return c.repeat(32).slice(0, 32); -} - -async function createSessionAndUpload(fileHash: string, storePathHash: string) { - const createRes = await post( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - sessionBody({ fileHash, storePathHash }), - ); - expect(createRes.status).toBe(201); - const session = await createRes.json<{ sessionId: string; r2Key: string }>(); - - const blobRes = await SELF.fetch(`http://example.com/_api/v1/uploads/${session.sessionId}/blob`, { - method: "PUT", - headers: { authorization: `Bearer ${AUTH_TOKEN}` }, - body: BLOB_DATA, - }); - expect(blobRes.status).toBe(200); - - const completeRes = await post(`/_api/v1/uploads/${session.sessionId}/complete`, {}); - expect(completeRes.status).toBe(200); - - return session; -} - -beforeAll(async () => { - const db = getDb(); - await createCache(db, CACHE_NAME); -}); - -describe("create upload session", () => { - it("creates a session and returns 201 with expected fields", async () => { - const res = await post( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - sessionBody({ fileHash: uniqueFileHash(), storePathHash: uniqueStorePathHash() }), - ); - - expect(res.status).toBe(201); - const body = await res.json>(); - expect(body).toHaveProperty("sessionId"); - expect(body).toHaveProperty("r2Key"); - expect(body).toHaveProperty("uploadMethod", "direct"); - expect(body).toHaveProperty("expiresAt"); - }); - - it("returns 404 for non-existent cache", async () => { - const res = await post( - "/_api/v1/caches/no-such-cache/upload-sessions", - sessionBody({ fileHash: uniqueFileHash(), storePathHash: uniqueStorePathHash() }), - ); - - expect(res.status).toBe(404); - }); - - it("returns 400 for missing required fields", async () => { - const res = await post(`/_api/v1/caches/${CACHE_NAME}/upload-sessions`, {}); - - expect(res.status).toBe(400); - }); -}); - -describe("complete upload (direct)", () => { - it("completes a direct upload flow", async () => { - const fileHash = uniqueFileHash(); - const storePathHash = uniqueStorePathHash(); - - const createRes = await post( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - sessionBody({ fileHash, storePathHash }), - ); - expect(createRes.status).toBe(201); - const session = await createRes.json<{ sessionId: string; r2Key: string }>(); - - const blobRes = await SELF.fetch(`http://example.com/_api/v1/uploads/${session.sessionId}/blob`, { - method: "PUT", - headers: { authorization: `Bearer ${AUTH_TOKEN}` }, - body: BLOB_DATA, - }); - expect(blobRes.status).toBe(200); - - const completeRes = await post(`/_api/v1/uploads/${session.sessionId}/complete`, {}); - expect(completeRes.status).toBe(200); - const completeBody = await completeRes.json<{ status: string }>(); - expect(completeBody.status).toBe("completed"); - }); - - it("returns 404 for non-existent session", async () => { - const res = await post("/_api/v1/uploads/00000000-0000-0000-0000-000000000000/complete", {}); - - expect(res.status).toBe(404); - }); - - it("returns 409 for already-completed session", async () => { - const session = await createSessionAndUpload(uniqueFileHash(), uniqueStorePathHash()); - - const res = await post(`/_api/v1/uploads/${session.sessionId}/complete`, {}); - expect(res.status).toBe(409); - }); - - it("returns 409 when session has not uploaded a blob yet", async () => { - const createRes = await post( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - sessionBody({ fileHash: uniqueFileHash(), storePathHash: uniqueStorePathHash() }), - ); - expect(createRes.status).toBe(201); - const session = await createRes.json<{ sessionId: string }>(); - - const res = await post(`/_api/v1/uploads/${session.sessionId}/complete`, {}); - expect(res.status).toBe(409); - }); -}); - -describe("publish", () => { - it("publishes a completed upload and path is readable", async () => { - const fileHash = uniqueFileHash(); - const storePathHash = uniqueStorePathHash(); - const session = await createSessionAndUpload(fileHash, storePathHash); - - const publishRes = await post(`/_api/v1/uploads/${session.sessionId}/publish`, {}); - expect(publishRes.status).toBe(200); - const publishBody = await publishRes.json<{ published: boolean; storePathHash: string }>(); - expect(publishBody.published).toBe(true); - expect(publishBody.storePathHash).toBe(storePathHash); - - const db = getDb(); - const cache = await findCacheByName(db, CACHE_NAME); - expect(cache).toBeDefined(); - const published = await findPublishedPath(db, cache!.id, storePathHash); - expect(published).toBeDefined(); - }); - - it("returns 409 for uncompleted session", async () => { - const createRes = await post( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - sessionBody({ fileHash: uniqueFileHash(), storePathHash: uniqueStorePathHash() }), - ); - expect(createRes.status).toBe(201); - const session = await createRes.json<{ sessionId: string }>(); - - const res = await post(`/_api/v1/uploads/${session.sessionId}/publish`, {}); - expect(res.status).toBe(409); - }); - - it("concurrent duplicate publish does not create duplicate paths", async () => { - const session = await createSessionAndUpload(uniqueFileHash(), uniqueStorePathHash()); - - const [r1, r2] = await Promise.all([ - post(`/_api/v1/uploads/${session.sessionId}/publish`, {}), - post(`/_api/v1/uploads/${session.sessionId}/publish`, {}), - ]); - - expect(r1.status).toBe(200); - expect(r2.status).toBe(200); - const b1 = await r1.json<{ published: boolean; alreadyExisted?: boolean }>(); - const b2 = await r2.json<{ published: boolean; alreadyExisted?: boolean }>(); - expect(b1.published).toBe(true); - expect(b2.published).toBe(true); - }); - - it("duplicate publish is idempotent", async () => { - const session = await createSessionAndUpload(uniqueFileHash(), uniqueStorePathHash()); - - const first = await post(`/_api/v1/uploads/${session.sessionId}/publish`, {}); - expect(first.status).toBe(200); - - const second = await post(`/_api/v1/uploads/${session.sessionId}/publish`, {}); - expect(second.status).toBe(200); - const body = await second.json<{ alreadyExisted: boolean }>(); - expect(body.alreadyExisted).toBe(true); - }); -}); - -describe("get-missing-paths", () => { - it("returns only unpublished paths as missing", async () => { - const storePathHash = uniqueStorePathHash(); - const session = await createSessionAndUpload(uniqueFileHash(), storePathHash); - await post(`/_api/v1/uploads/${session.sessionId}/publish`, {}); - - const randomHash = "x".repeat(32).replace(/x/g, () => "abcdfg0123456789"[Math.floor(Math.random() * 16)]); - - const res = await post("/_api/v1/get-missing-paths", { - cache: CACHE_NAME, - storePathHashes: [storePathHash, randomHash], - }); - expect(res.status).toBe(200); - const body = await res.json<{ missing: string[] }>(); - expect(body.missing).not.toContain(storePathHash); - expect(body.missing).toContain(randomHash); - }); - - it("returns 404 for non-existent cache", async () => { - const res = await post("/_api/v1/get-missing-paths", { - cache: "nonexistent-cache", - storePathHashes: ["0".repeat(32)], - }); - - expect(res.status).toBe(404); - }); -}); diff --git a/workers/cache/tsconfig.json b/workers/cache/tsconfig.json deleted file mode 100644 index 9c0eaa5..0000000 --- a/workers/cache/tsconfig.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "compilerOptions": { - "target": "es2024", - "lib": ["es2024", "webworker"], - "module": "es2022", - "moduleResolution": "Bundler", - "resolveJsonModule": true, - "noEmit": true, - "isolatedModules": true, - "allowSyntheticDefaultImports": true, - "forceConsistentCasingInFileNames": true, - "strict": true, - "skipLibCheck": true, - "types": ["./worker-configuration.d.ts"] - }, - "exclude": ["test"], - "include": ["worker-configuration.d.ts", "src/**/*.ts"] -} diff --git a/workers/cache/vitest.config.mts b/workers/cache/vitest.config.mts deleted file mode 100644 index f3d3fe2..0000000 --- a/workers/cache/vitest.config.mts +++ /dev/null @@ -1,24 +0,0 @@ -import path from 'node:path'; -import { defineWorkersConfig, readD1Migrations } from '@cloudflare/vitest-pool-workers/config'; - -export default defineWorkersConfig(async () => { - const migrationsPath = path.join(import.meta.dirname, 'migrations'); - const migrations = await readD1Migrations(migrationsPath); - - return { - test: { - setupFiles: ['./test/apply-migrations.ts'], - poolOptions: { - workers: { - wrangler: { configPath: './wrangler.jsonc' }, - miniflare: { - bindings: { - TEST_MIGRATIONS: migrations, - AUTH_TOKEN: "test-auth-token", - }, - }, - }, - }, - }, - }; -}); diff --git a/workers/cache/worker-configuration.d.ts b/workers/cache/worker-configuration.d.ts deleted file mode 100644 index bd64b48..0000000 --- a/workers/cache/worker-configuration.d.ts +++ /dev/null @@ -1,13542 +0,0 @@ -/* eslint-disable */ -// Generated by Wrangler by running `wrangler types` (hash: b1675f5d714a0a73a72c63752a4938cd) -// Runtime types generated with workerd@1.20260421.1 2026-03-10 -declare namespace Cloudflare { - interface GlobalProps { - mainModule: typeof import("./src/index"); - } - interface Env { - CACHE_BUCKET: R2Bucket; - CACHE_DB: D1Database; - SIGNING_KEY_NAME: "helios-cache-1"; - } -} -interface Env extends Cloudflare.Env {} - -// Begin runtime types -/*! ***************************************************************************** -Copyright (c) Cloudflare. All rights reserved. -Copyright (c) Microsoft Corporation. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. You may obtain a copy of the -License at http://www.apache.org/licenses/LICENSE-2.0 -THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED -WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -MERCHANTABLITY OR NON-INFRINGEMENT. -See the Apache Version 2.0 License for specific language governing permissions -and limitations under the License. -***************************************************************************** */ -/* eslint-disable */ -// noinspection JSUnusedGlobalSymbols -declare var onmessage: never; -/** - * The **`DOMException`** interface represents an abnormal event (called an **exception**) that occurs as a result of calling a method or accessing a property of a web API. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException) - */ -declare class DOMException extends Error { - constructor(message?: string, name?: string); - /** - * The **`message`** read-only property of the a message or description associated with the given error name. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException/message) - */ - readonly message: string; - /** - * The **`name`** read-only property of the one of the strings associated with an error name. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException/name) - */ - readonly name: string; - /** - * The **`code`** read-only property of the DOMException interface returns one of the legacy error code constants, or `0` if none match. - * @deprecated - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException/code) - */ - readonly code: number; - static readonly INDEX_SIZE_ERR: number; - static readonly DOMSTRING_SIZE_ERR: number; - static readonly HIERARCHY_REQUEST_ERR: number; - static readonly WRONG_DOCUMENT_ERR: number; - static readonly INVALID_CHARACTER_ERR: number; - static readonly NO_DATA_ALLOWED_ERR: number; - static readonly NO_MODIFICATION_ALLOWED_ERR: number; - static readonly NOT_FOUND_ERR: number; - static readonly NOT_SUPPORTED_ERR: number; - static readonly INUSE_ATTRIBUTE_ERR: number; - static readonly INVALID_STATE_ERR: number; - static readonly SYNTAX_ERR: number; - static readonly INVALID_MODIFICATION_ERR: number; - static readonly NAMESPACE_ERR: number; - static readonly INVALID_ACCESS_ERR: number; - static readonly VALIDATION_ERR: number; - static readonly TYPE_MISMATCH_ERR: number; - static readonly SECURITY_ERR: number; - static readonly NETWORK_ERR: number; - static readonly ABORT_ERR: number; - static readonly URL_MISMATCH_ERR: number; - static readonly QUOTA_EXCEEDED_ERR: number; - static readonly TIMEOUT_ERR: number; - static readonly INVALID_NODE_TYPE_ERR: number; - static readonly DATA_CLONE_ERR: number; - get stack(): any; - set stack(value: any); -} -type WorkerGlobalScopeEventMap = { - fetch: FetchEvent; - scheduled: ScheduledEvent; - queue: QueueEvent; - unhandledrejection: PromiseRejectionEvent; - rejectionhandled: PromiseRejectionEvent; -}; -declare abstract class WorkerGlobalScope extends EventTarget { - EventTarget: typeof EventTarget; -} -/* The **`console`** object provides access to the debugging console (e.g., the Web console in Firefox). * - * The **`console`** object provides access to the debugging console (e.g., the Web console in Firefox). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console) - */ -interface Console { - "assert"(condition?: boolean, ...data: any[]): void; - /** - * The **`console.clear()`** static method clears the console if possible. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/clear_static) - */ - clear(): void; - /** - * The **`console.count()`** static method logs the number of times that this particular call to `count()` has been called. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/count_static) - */ - count(label?: string): void; - /** - * The **`console.countReset()`** static method resets counter used with console/count_static. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/countReset_static) - */ - countReset(label?: string): void; - /** - * The **`console.debug()`** static method outputs a message to the console at the 'debug' log level. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/debug_static) - */ - debug(...data: any[]): void; - /** - * The **`console.dir()`** static method displays a list of the properties of the specified JavaScript object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/dir_static) - */ - dir(item?: any, options?: any): void; - /** - * The **`console.dirxml()`** static method displays an interactive tree of the descendant elements of the specified XML/HTML element. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/dirxml_static) - */ - dirxml(...data: any[]): void; - /** - * The **`console.error()`** static method outputs a message to the console at the 'error' log level. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/error_static) - */ - error(...data: any[]): void; - /** - * The **`console.group()`** static method creates a new inline group in the Web console log, causing any subsequent console messages to be indented by an additional level, until console/groupEnd_static is called. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/group_static) - */ - group(...data: any[]): void; - /** - * The **`console.groupCollapsed()`** static method creates a new inline group in the console. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/groupCollapsed_static) - */ - groupCollapsed(...data: any[]): void; - /** - * The **`console.groupEnd()`** static method exits the current inline group in the console. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/groupEnd_static) - */ - groupEnd(): void; - /** - * The **`console.info()`** static method outputs a message to the console at the 'info' log level. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/info_static) - */ - info(...data: any[]): void; - /** - * The **`console.log()`** static method outputs a message to the console. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/log_static) - */ - log(...data: any[]): void; - /** - * The **`console.table()`** static method displays tabular data as a table. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/table_static) - */ - table(tabularData?: any, properties?: string[]): void; - /** - * The **`console.time()`** static method starts a timer you can use to track how long an operation takes. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/time_static) - */ - time(label?: string): void; - /** - * The **`console.timeEnd()`** static method stops a timer that was previously started by calling console/time_static. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/timeEnd_static) - */ - timeEnd(label?: string): void; - /** - * The **`console.timeLog()`** static method logs the current value of a timer that was previously started by calling console/time_static. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/timeLog_static) - */ - timeLog(label?: string, ...data: any[]): void; - timeStamp(label?: string): void; - /** - * The **`console.trace()`** static method outputs a stack trace to the console. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/trace_static) - */ - trace(...data: any[]): void; - /** - * The **`console.warn()`** static method outputs a warning message to the console at the 'warning' log level. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/warn_static) - */ - warn(...data: any[]): void; -} -declare const console: Console; -type BufferSource = ArrayBufferView | ArrayBuffer; -type TypedArray = Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | Float32Array | Float64Array | BigInt64Array | BigUint64Array; -declare namespace WebAssembly { - class CompileError extends Error { - constructor(message?: string); - } - class RuntimeError extends Error { - constructor(message?: string); - } - type ValueType = "anyfunc" | "externref" | "f32" | "f64" | "i32" | "i64" | "v128"; - interface GlobalDescriptor { - value: ValueType; - mutable?: boolean; - } - class Global { - constructor(descriptor: GlobalDescriptor, value?: any); - value: any; - valueOf(): any; - } - type ImportValue = ExportValue | number; - type ModuleImports = Record; - type Imports = Record; - type ExportValue = Function | Global | Memory | Table; - type Exports = Record; - class Instance { - constructor(module: Module, imports?: Imports); - readonly exports: Exports; - } - interface MemoryDescriptor { - initial: number; - maximum?: number; - shared?: boolean; - } - class Memory { - constructor(descriptor: MemoryDescriptor); - readonly buffer: ArrayBuffer; - grow(delta: number): number; - } - type ImportExportKind = "function" | "global" | "memory" | "table"; - interface ModuleExportDescriptor { - kind: ImportExportKind; - name: string; - } - interface ModuleImportDescriptor { - kind: ImportExportKind; - module: string; - name: string; - } - abstract class Module { - static customSections(module: Module, sectionName: string): ArrayBuffer[]; - static exports(module: Module): ModuleExportDescriptor[]; - static imports(module: Module): ModuleImportDescriptor[]; - } - type TableKind = "anyfunc" | "externref"; - interface TableDescriptor { - element: TableKind; - initial: number; - maximum?: number; - } - class Table { - constructor(descriptor: TableDescriptor, value?: any); - readonly length: number; - get(index: number): any; - grow(delta: number, value?: any): number; - set(index: number, value?: any): void; - } - function instantiate(module: Module, imports?: Imports): Promise; - function validate(bytes: BufferSource): boolean; -} -/** - * The **`ServiceWorkerGlobalScope`** interface of the Service Worker API represents the global execution context of a service worker. - * Available only in secure contexts. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ServiceWorkerGlobalScope) - */ -interface ServiceWorkerGlobalScope extends WorkerGlobalScope { - DOMException: typeof DOMException; - WorkerGlobalScope: typeof WorkerGlobalScope; - btoa(data: string): string; - atob(data: string): string; - setTimeout(callback: (...args: any[]) => void, msDelay?: number): number; - setTimeout(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; - clearTimeout(timeoutId: number | null): void; - setInterval(callback: (...args: any[]) => void, msDelay?: number): number; - setInterval(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; - clearInterval(timeoutId: number | null): void; - queueMicrotask(task: Function): void; - structuredClone(value: T, options?: StructuredSerializeOptions): T; - reportError(error: any): void; - fetch(input: RequestInfo | URL, init?: RequestInit): Promise; - self: ServiceWorkerGlobalScope; - crypto: Crypto; - caches: CacheStorage; - scheduler: Scheduler; - performance: Performance; - Cloudflare: Cloudflare; - readonly origin: string; - Event: typeof Event; - ExtendableEvent: typeof ExtendableEvent; - CustomEvent: typeof CustomEvent; - PromiseRejectionEvent: typeof PromiseRejectionEvent; - FetchEvent: typeof FetchEvent; - TailEvent: typeof TailEvent; - TraceEvent: typeof TailEvent; - ScheduledEvent: typeof ScheduledEvent; - MessageEvent: typeof MessageEvent; - CloseEvent: typeof CloseEvent; - ReadableStreamDefaultReader: typeof ReadableStreamDefaultReader; - ReadableStreamBYOBReader: typeof ReadableStreamBYOBReader; - ReadableStream: typeof ReadableStream; - WritableStream: typeof WritableStream; - WritableStreamDefaultWriter: typeof WritableStreamDefaultWriter; - TransformStream: typeof TransformStream; - ByteLengthQueuingStrategy: typeof ByteLengthQueuingStrategy; - CountQueuingStrategy: typeof CountQueuingStrategy; - ErrorEvent: typeof ErrorEvent; - MessageChannel: typeof MessageChannel; - MessagePort: typeof MessagePort; - EventSource: typeof EventSource; - ReadableStreamBYOBRequest: typeof ReadableStreamBYOBRequest; - ReadableStreamDefaultController: typeof ReadableStreamDefaultController; - ReadableByteStreamController: typeof ReadableByteStreamController; - WritableStreamDefaultController: typeof WritableStreamDefaultController; - TransformStreamDefaultController: typeof TransformStreamDefaultController; - CompressionStream: typeof CompressionStream; - DecompressionStream: typeof DecompressionStream; - TextEncoderStream: typeof TextEncoderStream; - TextDecoderStream: typeof TextDecoderStream; - Headers: typeof Headers; - Body: typeof Body; - Request: typeof Request; - Response: typeof Response; - WebSocket: typeof WebSocket; - WebSocketPair: typeof WebSocketPair; - WebSocketRequestResponsePair: typeof WebSocketRequestResponsePair; - AbortController: typeof AbortController; - AbortSignal: typeof AbortSignal; - TextDecoder: typeof TextDecoder; - TextEncoder: typeof TextEncoder; - navigator: Navigator; - Navigator: typeof Navigator; - URL: typeof URL; - URLSearchParams: typeof URLSearchParams; - URLPattern: typeof URLPattern; - Blob: typeof Blob; - File: typeof File; - FormData: typeof FormData; - Crypto: typeof Crypto; - SubtleCrypto: typeof SubtleCrypto; - CryptoKey: typeof CryptoKey; - CacheStorage: typeof CacheStorage; - Cache: typeof Cache; - FixedLengthStream: typeof FixedLengthStream; - IdentityTransformStream: typeof IdentityTransformStream; - HTMLRewriter: typeof HTMLRewriter; -} -declare function addEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetAddEventListenerOptions | boolean): void; -declare function removeEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetEventListenerOptions | boolean): void; -/** - * The **`dispatchEvent()`** method of the EventTarget sends an Event to the object, (synchronously) invoking the affected event listeners in the appropriate order. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/dispatchEvent) - */ -declare function dispatchEvent(event: WorkerGlobalScopeEventMap[keyof WorkerGlobalScopeEventMap]): boolean; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/btoa) */ -declare function btoa(data: string): string; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/atob) */ -declare function atob(data: string): string; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setTimeout) */ -declare function setTimeout(callback: (...args: any[]) => void, msDelay?: number): number; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setTimeout) */ -declare function setTimeout(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/clearTimeout) */ -declare function clearTimeout(timeoutId: number | null): void; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setInterval) */ -declare function setInterval(callback: (...args: any[]) => void, msDelay?: number): number; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setInterval) */ -declare function setInterval(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/clearInterval) */ -declare function clearInterval(timeoutId: number | null): void; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/queueMicrotask) */ -declare function queueMicrotask(task: Function): void; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/structuredClone) */ -declare function structuredClone(value: T, options?: StructuredSerializeOptions): T; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/reportError) */ -declare function reportError(error: any): void; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/fetch) */ -declare function fetch(input: RequestInfo | URL, init?: RequestInit): Promise; -declare const self: ServiceWorkerGlobalScope; -/** -* The Web Crypto API provides a set of low-level functions for common cryptographic tasks. -* The Workers runtime implements the full surface of this API, but with some differences in -* the [supported algorithms](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/#supported-algorithms) -* compared to those implemented in most browsers. -* -* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/) -*/ -declare const crypto: Crypto; -/** -* The Cache API allows fine grained control of reading and writing from the Cloudflare global network cache. -* -* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/) -*/ -declare const caches: CacheStorage; -declare const scheduler: Scheduler; -/** -* The Workers runtime supports a subset of the Performance API, used to measure timing and performance, -* as well as timing of subrequests and other operations. -* -* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/) -*/ -declare const performance: Performance; -declare const Cloudflare: Cloudflare; -declare const origin: string; -declare const navigator: Navigator; -interface TestController { -} -interface ExecutionContext { - waitUntil(promise: Promise): void; - passThroughOnException(): void; - readonly exports: Cloudflare.Exports; - readonly props: Props; - cache?: CacheContext; -} -type ExportedHandlerFetchHandler = (request: Request>, env: Env, ctx: ExecutionContext) => Response | Promise; -type ExportedHandlerConnectHandler = (socket: Socket, env: Env, ctx: ExecutionContext) => void | Promise; -type ExportedHandlerTailHandler = (events: TraceItem[], env: Env, ctx: ExecutionContext) => void | Promise; -type ExportedHandlerTraceHandler = (traces: TraceItem[], env: Env, ctx: ExecutionContext) => void | Promise; -type ExportedHandlerTailStreamHandler = (event: TailStream.TailEvent, env: Env, ctx: ExecutionContext) => TailStream.TailEventHandlerType | Promise; -type ExportedHandlerScheduledHandler = (controller: ScheduledController, env: Env, ctx: ExecutionContext) => void | Promise; -type ExportedHandlerQueueHandler = (batch: MessageBatch, env: Env, ctx: ExecutionContext) => void | Promise; -type ExportedHandlerTestHandler = (controller: TestController, env: Env, ctx: ExecutionContext) => void | Promise; -interface ExportedHandler { - fetch?: ExportedHandlerFetchHandler; - connect?: ExportedHandlerConnectHandler; - tail?: ExportedHandlerTailHandler; - trace?: ExportedHandlerTraceHandler; - tailStream?: ExportedHandlerTailStreamHandler; - scheduled?: ExportedHandlerScheduledHandler; - test?: ExportedHandlerTestHandler; - email?: EmailExportedHandler; - queue?: ExportedHandlerQueueHandler; -} -interface StructuredSerializeOptions { - transfer?: any[]; -} -declare abstract class Navigator { - sendBeacon(url: string, body?: BodyInit): boolean; - readonly userAgent: string; - readonly hardwareConcurrency: number; - readonly platform: string; - readonly language: string; - readonly languages: string[]; -} -interface AlarmInvocationInfo { - readonly isRetry: boolean; - readonly retryCount: number; - readonly scheduledTime: number; -} -interface Cloudflare { - readonly compatibilityFlags: Record; -} -interface CachePurgeError { - code: number; - message: string; -} -interface CachePurgeResult { - success: boolean; - errors: CachePurgeError[]; -} -interface CachePurgeOptions { - tags?: string[]; - pathPrefixes?: string[]; - purgeEverything?: boolean; -} -interface CacheContext { - purge(options: CachePurgeOptions): Promise; -} -declare abstract class ColoLocalActorNamespace { - get(actorId: string): Fetcher; -} -interface DurableObject { - fetch(request: Request): Response | Promise; - connect?(socket: Socket): void | Promise; - alarm?(alarmInfo?: AlarmInvocationInfo): void | Promise; - webSocketMessage?(ws: WebSocket, message: string | ArrayBuffer): void | Promise; - webSocketClose?(ws: WebSocket, code: number, reason: string, wasClean: boolean): void | Promise; - webSocketError?(ws: WebSocket, error: unknown): void | Promise; -} -type DurableObjectStub = Fetcher & { - readonly id: DurableObjectId; - readonly name?: string; -}; -interface DurableObjectId { - toString(): string; - equals(other: DurableObjectId): boolean; - readonly name?: string; - readonly jurisdiction?: string; -} -declare abstract class DurableObjectNamespace { - newUniqueId(options?: DurableObjectNamespaceNewUniqueIdOptions): DurableObjectId; - idFromName(name: string): DurableObjectId; - idFromString(id: string): DurableObjectId; - get(id: DurableObjectId, options?: DurableObjectNamespaceGetDurableObjectOptions): DurableObjectStub; - getByName(name: string, options?: DurableObjectNamespaceGetDurableObjectOptions): DurableObjectStub; - jurisdiction(jurisdiction: DurableObjectJurisdiction): DurableObjectNamespace; -} -type DurableObjectJurisdiction = "eu" | "fedramp" | "fedramp-high"; -interface DurableObjectNamespaceNewUniqueIdOptions { - jurisdiction?: DurableObjectJurisdiction; -} -type DurableObjectLocationHint = "wnam" | "enam" | "sam" | "weur" | "eeur" | "apac" | "oc" | "afr" | "me"; -type DurableObjectRoutingMode = "primary-only"; -interface DurableObjectNamespaceGetDurableObjectOptions { - locationHint?: DurableObjectLocationHint; - routingMode?: DurableObjectRoutingMode; -} -interface DurableObjectClass<_T extends Rpc.DurableObjectBranded | undefined = undefined> { -} -interface DurableObjectState { - waitUntil(promise: Promise): void; - readonly exports: Cloudflare.Exports; - readonly props: Props; - readonly id: DurableObjectId; - readonly storage: DurableObjectStorage; - container?: Container; - facets: DurableObjectFacets; - blockConcurrencyWhile(callback: () => Promise): Promise; - acceptWebSocket(ws: WebSocket, tags?: string[]): void; - getWebSockets(tag?: string): WebSocket[]; - setWebSocketAutoResponse(maybeReqResp?: WebSocketRequestResponsePair): void; - getWebSocketAutoResponse(): WebSocketRequestResponsePair | null; - getWebSocketAutoResponseTimestamp(ws: WebSocket): Date | null; - setHibernatableWebSocketEventTimeout(timeoutMs?: number): void; - getHibernatableWebSocketEventTimeout(): number | null; - getTags(ws: WebSocket): string[]; - abort(reason?: string): void; -} -interface DurableObjectTransaction { - get(key: string, options?: DurableObjectGetOptions): Promise; - get(keys: string[], options?: DurableObjectGetOptions): Promise>; - list(options?: DurableObjectListOptions): Promise>; - put(key: string, value: T, options?: DurableObjectPutOptions): Promise; - put(entries: Record, options?: DurableObjectPutOptions): Promise; - delete(key: string, options?: DurableObjectPutOptions): Promise; - delete(keys: string[], options?: DurableObjectPutOptions): Promise; - rollback(): void; - getAlarm(options?: DurableObjectGetAlarmOptions): Promise; - setAlarm(scheduledTime: number | Date, options?: DurableObjectSetAlarmOptions): Promise; - deleteAlarm(options?: DurableObjectSetAlarmOptions): Promise; -} -interface DurableObjectStorage { - get(key: string, options?: DurableObjectGetOptions): Promise; - get(keys: string[], options?: DurableObjectGetOptions): Promise>; - list(options?: DurableObjectListOptions): Promise>; - put(key: string, value: T, options?: DurableObjectPutOptions): Promise; - put(entries: Record, options?: DurableObjectPutOptions): Promise; - delete(key: string, options?: DurableObjectPutOptions): Promise; - delete(keys: string[], options?: DurableObjectPutOptions): Promise; - deleteAll(options?: DurableObjectPutOptions): Promise; - transaction(closure: (txn: DurableObjectTransaction) => Promise): Promise; - getAlarm(options?: DurableObjectGetAlarmOptions): Promise; - setAlarm(scheduledTime: number | Date, options?: DurableObjectSetAlarmOptions): Promise; - deleteAlarm(options?: DurableObjectSetAlarmOptions): Promise; - sync(): Promise; - sql: SqlStorage; - kv: SyncKvStorage; - transactionSync(closure: () => T): T; - getCurrentBookmark(): Promise; - getBookmarkForTime(timestamp: number | Date): Promise; - onNextSessionRestoreBookmark(bookmark: string): Promise; -} -interface DurableObjectListOptions { - start?: string; - startAfter?: string; - end?: string; - prefix?: string; - reverse?: boolean; - limit?: number; - allowConcurrency?: boolean; - noCache?: boolean; -} -interface DurableObjectGetOptions { - allowConcurrency?: boolean; - noCache?: boolean; -} -interface DurableObjectGetAlarmOptions { - allowConcurrency?: boolean; -} -interface DurableObjectPutOptions { - allowConcurrency?: boolean; - allowUnconfirmed?: boolean; - noCache?: boolean; -} -interface DurableObjectSetAlarmOptions { - allowConcurrency?: boolean; - allowUnconfirmed?: boolean; -} -declare class WebSocketRequestResponsePair { - constructor(request: string, response: string); - get request(): string; - get response(): string; -} -interface DurableObjectFacets { - get(name: string, getStartupOptions: () => FacetStartupOptions | Promise>): Fetcher; - abort(name: string, reason: any): void; - delete(name: string): void; -} -interface FacetStartupOptions { - id?: DurableObjectId | string; - class: DurableObjectClass; -} -interface AnalyticsEngineDataset { - writeDataPoint(event?: AnalyticsEngineDataPoint): void; -} -interface AnalyticsEngineDataPoint { - indexes?: ((ArrayBuffer | string) | null)[]; - doubles?: number[]; - blobs?: ((ArrayBuffer | string) | null)[]; -} -/** - * The **`Event`** interface represents an event which takes place on an `EventTarget`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event) - */ -declare class Event { - constructor(type: string, init?: EventInit); - /** - * The **`type`** read-only property of the Event interface returns a string containing the event's type. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/type) - */ - get type(): string; - /** - * The **`eventPhase`** read-only property of the being evaluated. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/eventPhase) - */ - get eventPhase(): number; - /** - * The read-only **`composed`** property of the or not the event will propagate across the shadow DOM boundary into the standard DOM. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/composed) - */ - get composed(): boolean; - /** - * The **`bubbles`** read-only property of the Event interface indicates whether the event bubbles up through the DOM tree or not. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/bubbles) - */ - get bubbles(): boolean; - /** - * The **`cancelable`** read-only property of the Event interface indicates whether the event can be canceled, and therefore prevented as if the event never happened. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/cancelable) - */ - get cancelable(): boolean; - /** - * The **`defaultPrevented`** read-only property of the Event interface returns a boolean value indicating whether or not the call to Event.preventDefault() canceled the event. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/defaultPrevented) - */ - get defaultPrevented(): boolean; - /** - * The Event property **`returnValue`** indicates whether the default action for this event has been prevented or not. - * @deprecated - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/returnValue) - */ - get returnValue(): boolean; - /** - * The **`currentTarget`** read-only property of the Event interface identifies the element to which the event handler has been attached. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/currentTarget) - */ - get currentTarget(): EventTarget | undefined; - /** - * The read-only **`target`** property of the dispatched. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/target) - */ - get target(): EventTarget | undefined; - /** - * The deprecated **`Event.srcElement`** is an alias for the Event.target property. - * @deprecated - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/srcElement) - */ - get srcElement(): EventTarget | undefined; - /** - * The **`timeStamp`** read-only property of the Event interface returns the time (in milliseconds) at which the event was created. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/timeStamp) - */ - get timeStamp(): number; - /** - * The **`isTrusted`** read-only property of the when the event was generated by the user agent (including via user actions and programmatic methods such as HTMLElement.focus()), and `false` when the event was dispatched via The only exception is the `click` event, which initializes the `isTrusted` property to `false` in user agents. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/isTrusted) - */ - get isTrusted(): boolean; - /** - * The **`cancelBubble`** property of the Event interface is deprecated. - * @deprecated - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/cancelBubble) - */ - get cancelBubble(): boolean; - /** - * The **`cancelBubble`** property of the Event interface is deprecated. - * @deprecated - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/cancelBubble) - */ - set cancelBubble(value: boolean); - /** - * The **`stopImmediatePropagation()`** method of the If several listeners are attached to the same element for the same event type, they are called in the order in which they were added. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/stopImmediatePropagation) - */ - stopImmediatePropagation(): void; - /** - * The **`preventDefault()`** method of the Event interface tells the user agent that if the event does not get explicitly handled, its default action should not be taken as it normally would be. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/preventDefault) - */ - preventDefault(): void; - /** - * The **`stopPropagation()`** method of the Event interface prevents further propagation of the current event in the capturing and bubbling phases. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/stopPropagation) - */ - stopPropagation(): void; - /** - * The **`composedPath()`** method of the Event interface returns the event's path which is an array of the objects on which listeners will be invoked. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/composedPath) - */ - composedPath(): EventTarget[]; - static readonly NONE: number; - static readonly CAPTURING_PHASE: number; - static readonly AT_TARGET: number; - static readonly BUBBLING_PHASE: number; -} -interface EventInit { - bubbles?: boolean; - cancelable?: boolean; - composed?: boolean; -} -type EventListener = (event: EventType) => void; -interface EventListenerObject { - handleEvent(event: EventType): void; -} -type EventListenerOrEventListenerObject = EventListener | EventListenerObject; -/** - * The **`EventTarget`** interface is implemented by objects that can receive events and may have listeners for them. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget) - */ -declare class EventTarget = Record> { - constructor(); - /** - * The **`addEventListener()`** method of the EventTarget interface sets up a function that will be called whenever the specified event is delivered to the target. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/addEventListener) - */ - addEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetAddEventListenerOptions | boolean): void; - /** - * The **`removeEventListener()`** method of the EventTarget interface removes an event listener previously registered with EventTarget.addEventListener() from the target. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/removeEventListener) - */ - removeEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetEventListenerOptions | boolean): void; - /** - * The **`dispatchEvent()`** method of the EventTarget sends an Event to the object, (synchronously) invoking the affected event listeners in the appropriate order. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/dispatchEvent) - */ - dispatchEvent(event: EventMap[keyof EventMap]): boolean; -} -interface EventTargetEventListenerOptions { - capture?: boolean; -} -interface EventTargetAddEventListenerOptions { - capture?: boolean; - passive?: boolean; - once?: boolean; - signal?: AbortSignal; -} -interface EventTargetHandlerObject { - handleEvent: (event: Event) => any | undefined; -} -/** - * The **`AbortController`** interface represents a controller object that allows you to abort one or more Web requests as and when desired. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortController) - */ -declare class AbortController { - constructor(); - /** - * The **`signal`** read-only property of the AbortController interface returns an AbortSignal object instance, which can be used to communicate with/abort an asynchronous operation as desired. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortController/signal) - */ - get signal(): AbortSignal; - /** - * The **`abort()`** method of the AbortController interface aborts an asynchronous operation before it has completed. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortController/abort) - */ - abort(reason?: any): void; -} -/** - * The **`AbortSignal`** interface represents a signal object that allows you to communicate with an asynchronous operation (such as a fetch request) and abort it if required via an AbortController object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal) - */ -declare abstract class AbortSignal extends EventTarget { - /** - * The **`AbortSignal.abort()`** static method returns an AbortSignal that is already set as aborted (and which does not trigger an AbortSignal/abort_event event). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/abort_static) - */ - static abort(reason?: any): AbortSignal; - /** - * The **`AbortSignal.timeout()`** static method returns an AbortSignal that will automatically abort after a specified time. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/timeout_static) - */ - static timeout(delay: number): AbortSignal; - /** - * The **`AbortSignal.any()`** static method takes an iterable of abort signals and returns an AbortSignal. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/any_static) - */ - static any(signals: AbortSignal[]): AbortSignal; - /** - * The **`aborted`** read-only property returns a value that indicates whether the asynchronous operations the signal is communicating with are aborted (`true`) or not (`false`). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/aborted) - */ - get aborted(): boolean; - /** - * The **`reason`** read-only property returns a JavaScript value that indicates the abort reason. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/reason) - */ - get reason(): any; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/abort_event) */ - get onabort(): any | null; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/abort_event) */ - set onabort(value: any | null); - /** - * The **`throwIfAborted()`** method throws the signal's abort AbortSignal.reason if the signal has been aborted; otherwise it does nothing. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/throwIfAborted) - */ - throwIfAborted(): void; -} -interface Scheduler { - wait(delay: number, maybeOptions?: SchedulerWaitOptions): Promise; -} -interface SchedulerWaitOptions { - signal?: AbortSignal; -} -/** - * The **`ExtendableEvent`** interface extends the lifetime of the `install` and `activate` events dispatched on the global scope as part of the service worker lifecycle. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ExtendableEvent) - */ -declare abstract class ExtendableEvent extends Event { - /** - * The **`ExtendableEvent.waitUntil()`** method tells the event dispatcher that work is ongoing. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ExtendableEvent/waitUntil) - */ - waitUntil(promise: Promise): void; -} -/** - * The **`CustomEvent`** interface represents events initialized by an application for any purpose. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CustomEvent) - */ -declare class CustomEvent extends Event { - constructor(type: string, init?: CustomEventCustomEventInit); - /** - * The read-only **`detail`** property of the CustomEvent interface returns any data passed when initializing the event. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CustomEvent/detail) - */ - get detail(): T; -} -interface CustomEventCustomEventInit { - bubbles?: boolean; - cancelable?: boolean; - composed?: boolean; - detail?: any; -} -/** - * The **`Blob`** interface represents a blob, which is a file-like object of immutable, raw data; they can be read as text or binary data, or converted into a ReadableStream so its methods can be used for processing the data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob) - */ -declare class Blob { - constructor(type?: ((ArrayBuffer | ArrayBufferView) | string | Blob)[], options?: BlobOptions); - /** - * The **`size`** read-only property of the Blob interface returns the size of the Blob or File in bytes. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/size) - */ - get size(): number; - /** - * The **`type`** read-only property of the Blob interface returns the MIME type of the file. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/type) - */ - get type(): string; - /** - * The **`slice()`** method of the Blob interface creates and returns a new `Blob` object which contains data from a subset of the blob on which it's called. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/slice) - */ - slice(start?: number, end?: number, type?: string): Blob; - /** - * The **`arrayBuffer()`** method of the Blob interface returns a Promise that resolves with the contents of the blob as binary data contained in an ArrayBuffer. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/arrayBuffer) - */ - arrayBuffer(): Promise; - /** - * The **`bytes()`** method of the Blob interface returns a Promise that resolves with a Uint8Array containing the contents of the blob as an array of bytes. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/bytes) - */ - bytes(): Promise; - /** - * The **`text()`** method of the string containing the contents of the blob, interpreted as UTF-8. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/text) - */ - text(): Promise; - /** - * The **`stream()`** method of the Blob interface returns a ReadableStream which upon reading returns the data contained within the `Blob`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/stream) - */ - stream(): ReadableStream; -} -interface BlobOptions { - type?: string; -} -/** - * The **`File`** interface provides information about files and allows JavaScript in a web page to access their content. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/File) - */ -declare class File extends Blob { - constructor(bits: ((ArrayBuffer | ArrayBufferView) | string | Blob)[] | undefined, name: string, options?: FileOptions); - /** - * The **`name`** read-only property of the File interface returns the name of the file represented by a File object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/name) - */ - get name(): string; - /** - * The **`lastModified`** read-only property of the File interface provides the last modified date of the file as the number of milliseconds since the Unix epoch (January 1, 1970 at midnight). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/lastModified) - */ - get lastModified(): number; -} -interface FileOptions { - type?: string; - lastModified?: number; -} -/** -* The Cache API allows fine grained control of reading and writing from the Cloudflare global network cache. -* -* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/) -*/ -declare abstract class CacheStorage { - /** - * The **`open()`** method of the the Cache object matching the `cacheName`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CacheStorage/open) - */ - open(cacheName: string): Promise; - readonly default: Cache; -} -/** -* The Cache API allows fine grained control of reading and writing from the Cloudflare global network cache. -* -* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/) -*/ -declare abstract class Cache { - /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/#delete) */ - delete(request: RequestInfo | URL, options?: CacheQueryOptions): Promise; - /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/#match) */ - match(request: RequestInfo | URL, options?: CacheQueryOptions): Promise; - /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/#put) */ - put(request: RequestInfo | URL, response: Response): Promise; -} -interface CacheQueryOptions { - ignoreMethod?: boolean; -} -/** -* The Web Crypto API provides a set of low-level functions for common cryptographic tasks. -* The Workers runtime implements the full surface of this API, but with some differences in -* the [supported algorithms](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/#supported-algorithms) -* compared to those implemented in most browsers. -* -* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/) -*/ -declare abstract class Crypto { - /** - * The **`Crypto.subtle`** read-only property returns a cryptographic operations. - * Available only in secure contexts. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/subtle) - */ - get subtle(): SubtleCrypto; - /** - * The **`Crypto.getRandomValues()`** method lets you get cryptographically strong random values. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/getRandomValues) - */ - getRandomValues(buffer: T): T; - /** - * The **`randomUUID()`** method of the Crypto interface is used to generate a v4 UUID using a cryptographically secure random number generator. - * Available only in secure contexts. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/randomUUID) - */ - randomUUID(): string; - DigestStream: typeof DigestStream; -} -/** - * The **`SubtleCrypto`** interface of the Web Crypto API provides a number of low-level cryptographic functions. - * Available only in secure contexts. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto) - */ -declare abstract class SubtleCrypto { - /** - * The **`encrypt()`** method of the SubtleCrypto interface encrypts data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/encrypt) - */ - encrypt(algorithm: string | SubtleCryptoEncryptAlgorithm, key: CryptoKey, plainText: ArrayBuffer | ArrayBufferView): Promise; - /** - * The **`decrypt()`** method of the SubtleCrypto interface decrypts some encrypted data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/decrypt) - */ - decrypt(algorithm: string | SubtleCryptoEncryptAlgorithm, key: CryptoKey, cipherText: ArrayBuffer | ArrayBufferView): Promise; - /** - * The **`sign()`** method of the SubtleCrypto interface generates a digital signature. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/sign) - */ - sign(algorithm: string | SubtleCryptoSignAlgorithm, key: CryptoKey, data: ArrayBuffer | ArrayBufferView): Promise; - /** - * The **`verify()`** method of the SubtleCrypto interface verifies a digital signature. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/verify) - */ - verify(algorithm: string | SubtleCryptoSignAlgorithm, key: CryptoKey, signature: ArrayBuffer | ArrayBufferView, data: ArrayBuffer | ArrayBufferView): Promise; - /** - * The **`digest()`** method of the SubtleCrypto interface generates a _digest_ of the given data, using the specified hash function. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/digest) - */ - digest(algorithm: string | SubtleCryptoHashAlgorithm, data: ArrayBuffer | ArrayBufferView): Promise; - /** - * The **`generateKey()`** method of the SubtleCrypto interface is used to generate a new key (for symmetric algorithms) or key pair (for public-key algorithms). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/generateKey) - */ - generateKey(algorithm: string | SubtleCryptoGenerateKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; - /** - * The **`deriveKey()`** method of the SubtleCrypto interface can be used to derive a secret key from a master key. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/deriveKey) - */ - deriveKey(algorithm: string | SubtleCryptoDeriveKeyAlgorithm, baseKey: CryptoKey, derivedKeyAlgorithm: string | SubtleCryptoImportKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; - /** - * The **`deriveBits()`** method of the key. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/deriveBits) - */ - deriveBits(algorithm: string | SubtleCryptoDeriveKeyAlgorithm, baseKey: CryptoKey, length?: number | null): Promise; - /** - * The **`importKey()`** method of the SubtleCrypto interface imports a key: that is, it takes as input a key in an external, portable format and gives you a CryptoKey object that you can use in the Web Crypto API. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/importKey) - */ - importKey(format: string, keyData: (ArrayBuffer | ArrayBufferView) | JsonWebKey, algorithm: string | SubtleCryptoImportKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; - /** - * The **`exportKey()`** method of the SubtleCrypto interface exports a key: that is, it takes as input a CryptoKey object and gives you the key in an external, portable format. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/exportKey) - */ - exportKey(format: string, key: CryptoKey): Promise; - /** - * The **`wrapKey()`** method of the SubtleCrypto interface 'wraps' a key. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/wrapKey) - */ - wrapKey(format: string, key: CryptoKey, wrappingKey: CryptoKey, wrapAlgorithm: string | SubtleCryptoEncryptAlgorithm): Promise; - /** - * The **`unwrapKey()`** method of the SubtleCrypto interface 'unwraps' a key. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/unwrapKey) - */ - unwrapKey(format: string, wrappedKey: ArrayBuffer | ArrayBufferView, unwrappingKey: CryptoKey, unwrapAlgorithm: string | SubtleCryptoEncryptAlgorithm, unwrappedKeyAlgorithm: string | SubtleCryptoImportKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; - timingSafeEqual(a: ArrayBuffer | ArrayBufferView, b: ArrayBuffer | ArrayBufferView): boolean; -} -/** - * The **`CryptoKey`** interface of the Web Crypto API represents a cryptographic key obtained from one of the SubtleCrypto methods SubtleCrypto.generateKey, SubtleCrypto.deriveKey, SubtleCrypto.importKey, or SubtleCrypto.unwrapKey. - * Available only in secure contexts. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey) - */ -declare abstract class CryptoKey { - /** - * The read-only **`type`** property of the CryptoKey interface indicates which kind of key is represented by the object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/type) - */ - readonly type: string; - /** - * The read-only **`extractable`** property of the CryptoKey interface indicates whether or not the key may be extracted using `SubtleCrypto.exportKey()` or `SubtleCrypto.wrapKey()`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/extractable) - */ - readonly extractable: boolean; - /** - * The read-only **`algorithm`** property of the CryptoKey interface returns an object describing the algorithm for which this key can be used, and any associated extra parameters. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/algorithm) - */ - readonly algorithm: CryptoKeyKeyAlgorithm | CryptoKeyAesKeyAlgorithm | CryptoKeyHmacKeyAlgorithm | CryptoKeyRsaKeyAlgorithm | CryptoKeyEllipticKeyAlgorithm | CryptoKeyArbitraryKeyAlgorithm; - /** - * The read-only **`usages`** property of the CryptoKey interface indicates what can be done with the key. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/usages) - */ - readonly usages: string[]; -} -interface CryptoKeyPair { - publicKey: CryptoKey; - privateKey: CryptoKey; -} -interface JsonWebKey { - kty: string; - use?: string; - key_ops?: string[]; - alg?: string; - ext?: boolean; - crv?: string; - x?: string; - y?: string; - d?: string; - n?: string; - e?: string; - p?: string; - q?: string; - dp?: string; - dq?: string; - qi?: string; - oth?: RsaOtherPrimesInfo[]; - k?: string; -} -interface RsaOtherPrimesInfo { - r?: string; - d?: string; - t?: string; -} -interface SubtleCryptoDeriveKeyAlgorithm { - name: string; - salt?: (ArrayBuffer | ArrayBufferView); - iterations?: number; - hash?: (string | SubtleCryptoHashAlgorithm); - $public?: CryptoKey; - info?: (ArrayBuffer | ArrayBufferView); -} -interface SubtleCryptoEncryptAlgorithm { - name: string; - iv?: (ArrayBuffer | ArrayBufferView); - additionalData?: (ArrayBuffer | ArrayBufferView); - tagLength?: number; - counter?: (ArrayBuffer | ArrayBufferView); - length?: number; - label?: (ArrayBuffer | ArrayBufferView); -} -interface SubtleCryptoGenerateKeyAlgorithm { - name: string; - hash?: (string | SubtleCryptoHashAlgorithm); - modulusLength?: number; - publicExponent?: (ArrayBuffer | ArrayBufferView); - length?: number; - namedCurve?: string; -} -interface SubtleCryptoHashAlgorithm { - name: string; -} -interface SubtleCryptoImportKeyAlgorithm { - name: string; - hash?: (string | SubtleCryptoHashAlgorithm); - length?: number; - namedCurve?: string; - compressed?: boolean; -} -interface SubtleCryptoSignAlgorithm { - name: string; - hash?: (string | SubtleCryptoHashAlgorithm); - dataLength?: number; - saltLength?: number; -} -interface CryptoKeyKeyAlgorithm { - name: string; -} -interface CryptoKeyAesKeyAlgorithm { - name: string; - length: number; -} -interface CryptoKeyHmacKeyAlgorithm { - name: string; - hash: CryptoKeyKeyAlgorithm; - length: number; -} -interface CryptoKeyRsaKeyAlgorithm { - name: string; - modulusLength: number; - publicExponent: ArrayBuffer | ArrayBufferView; - hash?: CryptoKeyKeyAlgorithm; -} -interface CryptoKeyEllipticKeyAlgorithm { - name: string; - namedCurve: string; -} -interface CryptoKeyArbitraryKeyAlgorithm { - name: string; - hash?: CryptoKeyKeyAlgorithm; - namedCurve?: string; - length?: number; -} -declare class DigestStream extends WritableStream { - constructor(algorithm: string | SubtleCryptoHashAlgorithm); - readonly digest: Promise; - get bytesWritten(): number | bigint; -} -/** - * The **`TextDecoder`** interface represents a decoder for a specific text encoding, such as `UTF-8`, `ISO-8859-2`, `KOI8-R`, `GBK`, etc. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextDecoder) - */ -declare class TextDecoder { - constructor(label?: string, options?: TextDecoderConstructorOptions); - /** - * The **`TextDecoder.decode()`** method returns a string containing text decoded from the buffer passed as a parameter. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextDecoder/decode) - */ - decode(input?: (ArrayBuffer | ArrayBufferView), options?: TextDecoderDecodeOptions): string; - get encoding(): string; - get fatal(): boolean; - get ignoreBOM(): boolean; -} -/** - * The **`TextEncoder`** interface takes a stream of code points as input and emits a stream of UTF-8 bytes. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoder) - */ -declare class TextEncoder { - constructor(); - /** - * The **`TextEncoder.encode()`** method takes a string as input, and returns a Global_Objects/Uint8Array containing the text given in parameters encoded with the specific method for that TextEncoder object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoder/encode) - */ - encode(input?: string): Uint8Array; - /** - * The **`TextEncoder.encodeInto()`** method takes a string to encode and a destination Uint8Array to put resulting UTF-8 encoded text into, and returns a dictionary object indicating the progress of the encoding. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoder/encodeInto) - */ - encodeInto(input: string, buffer: Uint8Array): TextEncoderEncodeIntoResult; - get encoding(): string; -} -interface TextDecoderConstructorOptions { - fatal: boolean; - ignoreBOM: boolean; -} -interface TextDecoderDecodeOptions { - stream: boolean; -} -interface TextEncoderEncodeIntoResult { - read: number; - written: number; -} -/** - * The **`ErrorEvent`** interface represents events providing information related to errors in scripts or in files. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent) - */ -declare class ErrorEvent extends Event { - constructor(type: string, init?: ErrorEventErrorEventInit); - /** - * The **`filename`** read-only property of the ErrorEvent interface returns a string containing the name of the script file in which the error occurred. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/filename) - */ - get filename(): string; - /** - * The **`message`** read-only property of the ErrorEvent interface returns a string containing a human-readable error message describing the problem. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/message) - */ - get message(): string; - /** - * The **`lineno`** read-only property of the ErrorEvent interface returns an integer containing the line number of the script file on which the error occurred. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/lineno) - */ - get lineno(): number; - /** - * The **`colno`** read-only property of the ErrorEvent interface returns an integer containing the column number of the script file on which the error occurred. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/colno) - */ - get colno(): number; - /** - * The **`error`** read-only property of the ErrorEvent interface returns a JavaScript value, such as an Error or DOMException, representing the error associated with this event. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/error) - */ - get error(): any; -} -interface ErrorEventErrorEventInit { - message?: string; - filename?: string; - lineno?: number; - colno?: number; - error?: any; -} -/** - * The **`MessageEvent`** interface represents a message received by a target object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent) - */ -declare class MessageEvent extends Event { - constructor(type: string, initializer: MessageEventInit); - /** - * The **`data`** read-only property of the The data sent by the message emitter; this can be any data type, depending on what originated this event. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/data) - */ - readonly data: any; - /** - * The **`origin`** read-only property of the origin of the message emitter. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/origin) - */ - readonly origin: string | null; - /** - * The **`lastEventId`** read-only property of the unique ID for the event. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/lastEventId) - */ - readonly lastEventId: string; - /** - * The **`source`** read-only property of the a WindowProxy, MessagePort, or a `MessageEventSource` (which can be a WindowProxy, message emitter. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/source) - */ - readonly source: MessagePort | null; - /** - * The **`ports`** read-only property of the containing all MessagePort objects sent with the message, in order. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/ports) - */ - readonly ports: MessagePort[]; -} -interface MessageEventInit { - data: ArrayBuffer | string; -} -/** - * The **`PromiseRejectionEvent`** interface represents events which are sent to the global script context when JavaScript Promises are rejected. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PromiseRejectionEvent) - */ -declare abstract class PromiseRejectionEvent extends Event { - /** - * The PromiseRejectionEvent interface's **`promise`** read-only property indicates the JavaScript rejected. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PromiseRejectionEvent/promise) - */ - readonly promise: Promise; - /** - * The PromiseRejectionEvent **`reason`** read-only property is any JavaScript value or Object which provides the reason passed into Promise.reject(). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PromiseRejectionEvent/reason) - */ - readonly reason: any; -} -/** - * The **`FormData`** interface provides a way to construct a set of key/value pairs representing form fields and their values, which can be sent using the Window/fetch, XMLHttpRequest.send() or navigator.sendBeacon() methods. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData) - */ -declare class FormData { - constructor(); - /** - * The **`append()`** method of the FormData interface appends a new value onto an existing key inside a `FormData` object, or adds the key if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/append) - */ - append(name: string, value: string | Blob): void; - /** - * The **`append()`** method of the FormData interface appends a new value onto an existing key inside a `FormData` object, or adds the key if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/append) - */ - append(name: string, value: string): void; - /** - * The **`append()`** method of the FormData interface appends a new value onto an existing key inside a `FormData` object, or adds the key if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/append) - */ - append(name: string, value: Blob, filename?: string): void; - /** - * The **`delete()`** method of the FormData interface deletes a key and its value(s) from a `FormData` object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/delete) - */ - delete(name: string): void; - /** - * The **`get()`** method of the FormData interface returns the first value associated with a given key from within a `FormData` object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/get) - */ - get(name: string): (File | string) | null; - /** - * The **`getAll()`** method of the FormData interface returns all the values associated with a given key from within a `FormData` object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/getAll) - */ - getAll(name: string): (File | string)[]; - /** - * The **`has()`** method of the FormData interface returns whether a `FormData` object contains a certain key. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/has) - */ - has(name: string): boolean; - /** - * The **`set()`** method of the FormData interface sets a new value for an existing key inside a `FormData` object, or adds the key/value if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/set) - */ - set(name: string, value: string | Blob): void; - /** - * The **`set()`** method of the FormData interface sets a new value for an existing key inside a `FormData` object, or adds the key/value if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/set) - */ - set(name: string, value: string): void; - /** - * The **`set()`** method of the FormData interface sets a new value for an existing key inside a `FormData` object, or adds the key/value if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/set) - */ - set(name: string, value: Blob, filename?: string): void; - /* Returns an array of key, value pairs for every entry in the list. */ - entries(): IterableIterator<[ - key: string, - value: File | string - ]>; - /* Returns a list of keys in the list. */ - keys(): IterableIterator; - /* Returns a list of values in the list. */ - values(): IterableIterator<(File | string)>; - forEach(callback: (this: This, value: File | string, key: string, parent: FormData) => void, thisArg?: This): void; - [Symbol.iterator](): IterableIterator<[ - key: string, - value: File | string - ]>; -} -interface ContentOptions { - html?: boolean; -} -declare class HTMLRewriter { - constructor(); - on(selector: string, handlers: HTMLRewriterElementContentHandlers): HTMLRewriter; - onDocument(handlers: HTMLRewriterDocumentContentHandlers): HTMLRewriter; - transform(response: Response): Response; -} -interface HTMLRewriterElementContentHandlers { - element?(element: Element): void | Promise; - comments?(comment: Comment): void | Promise; - text?(element: Text): void | Promise; -} -interface HTMLRewriterDocumentContentHandlers { - doctype?(doctype: Doctype): void | Promise; - comments?(comment: Comment): void | Promise; - text?(text: Text): void | Promise; - end?(end: DocumentEnd): void | Promise; -} -interface Doctype { - readonly name: string | null; - readonly publicId: string | null; - readonly systemId: string | null; -} -interface Element { - tagName: string; - readonly attributes: IterableIterator; - readonly removed: boolean; - readonly namespaceURI: string; - getAttribute(name: string): string | null; - hasAttribute(name: string): boolean; - setAttribute(name: string, value: string): Element; - removeAttribute(name: string): Element; - before(content: string | ReadableStream | Response, options?: ContentOptions): Element; - after(content: string | ReadableStream | Response, options?: ContentOptions): Element; - prepend(content: string | ReadableStream | Response, options?: ContentOptions): Element; - append(content: string | ReadableStream | Response, options?: ContentOptions): Element; - replace(content: string | ReadableStream | Response, options?: ContentOptions): Element; - remove(): Element; - removeAndKeepContent(): Element; - setInnerContent(content: string | ReadableStream | Response, options?: ContentOptions): Element; - onEndTag(handler: (tag: EndTag) => void | Promise): void; -} -interface EndTag { - name: string; - before(content: string | ReadableStream | Response, options?: ContentOptions): EndTag; - after(content: string | ReadableStream | Response, options?: ContentOptions): EndTag; - remove(): EndTag; -} -interface Comment { - text: string; - readonly removed: boolean; - before(content: string, options?: ContentOptions): Comment; - after(content: string, options?: ContentOptions): Comment; - replace(content: string, options?: ContentOptions): Comment; - remove(): Comment; -} -interface Text { - readonly text: string; - readonly lastInTextNode: boolean; - readonly removed: boolean; - before(content: string | ReadableStream | Response, options?: ContentOptions): Text; - after(content: string | ReadableStream | Response, options?: ContentOptions): Text; - replace(content: string | ReadableStream | Response, options?: ContentOptions): Text; - remove(): Text; -} -interface DocumentEnd { - append(content: string, options?: ContentOptions): DocumentEnd; -} -/** - * This is the event type for `fetch` events dispatched on the ServiceWorkerGlobalScope. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FetchEvent) - */ -declare abstract class FetchEvent extends ExtendableEvent { - /** - * The **`request`** read-only property of the the event handler. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FetchEvent/request) - */ - readonly request: Request; - /** - * The **`respondWith()`** method of allows you to provide a promise for a Response yourself. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FetchEvent/respondWith) - */ - respondWith(promise: Response | Promise): void; - passThroughOnException(): void; -} -type HeadersInit = Headers | Iterable> | Record; -/** - * The **`Headers`** interface of the Fetch API allows you to perform various actions on HTTP request and response headers. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers) - */ -declare class Headers { - constructor(init?: HeadersInit); - /** - * The **`get()`** method of the Headers interface returns a byte string of all the values of a header within a `Headers` object with a given name. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/get) - */ - get(name: string): string | null; - getAll(name: string): string[]; - /** - * The **`getSetCookie()`** method of the Headers interface returns an array containing the values of all Set-Cookie headers associated with a response. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/getSetCookie) - */ - getSetCookie(): string[]; - /** - * The **`has()`** method of the Headers interface returns a boolean stating whether a `Headers` object contains a certain header. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/has) - */ - has(name: string): boolean; - /** - * The **`set()`** method of the Headers interface sets a new value for an existing header inside a `Headers` object, or adds the header if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/set) - */ - set(name: string, value: string): void; - /** - * The **`append()`** method of the Headers interface appends a new value onto an existing header inside a `Headers` object, or adds the header if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/append) - */ - append(name: string, value: string): void; - /** - * The **`delete()`** method of the Headers interface deletes a header from the current `Headers` object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/delete) - */ - delete(name: string): void; - forEach(callback: (this: This, value: string, key: string, parent: Headers) => void, thisArg?: This): void; - /* Returns an iterator allowing to go through all key/value pairs contained in this object. */ - entries(): IterableIterator<[ - key: string, - value: string - ]>; - /* Returns an iterator allowing to go through all keys of the key/value pairs contained in this object. */ - keys(): IterableIterator; - /* Returns an iterator allowing to go through all values of the key/value pairs contained in this object. */ - values(): IterableIterator; - [Symbol.iterator](): IterableIterator<[ - key: string, - value: string - ]>; -} -type BodyInit = ReadableStream | string | ArrayBuffer | ArrayBufferView | Blob | URLSearchParams | FormData | Iterable | AsyncIterable; -declare abstract class Body { - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/body) */ - get body(): ReadableStream | null; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/bodyUsed) */ - get bodyUsed(): boolean; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/arrayBuffer) */ - arrayBuffer(): Promise; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/bytes) */ - bytes(): Promise; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/text) */ - text(): Promise; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/json) */ - json(): Promise; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/formData) */ - formData(): Promise; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/blob) */ - blob(): Promise; -} -/** - * The **`Response`** interface of the Fetch API represents the response to a request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response) - */ -declare var Response: { - prototype: Response; - new (body?: BodyInit | null, init?: ResponseInit): Response; - error(): Response; - redirect(url: string, status?: number): Response; - json(any: any, maybeInit?: (ResponseInit | Response)): Response; -}; -/** - * The **`Response`** interface of the Fetch API represents the response to a request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response) - */ -interface Response extends Body { - /** - * The **`clone()`** method of the Response interface creates a clone of a response object, identical in every way, but stored in a different variable. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/clone) - */ - clone(): Response; - /** - * The **`status`** read-only property of the Response interface contains the HTTP status codes of the response. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/status) - */ - status: number; - /** - * The **`statusText`** read-only property of the Response interface contains the status message corresponding to the HTTP status code in Response.status. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/statusText) - */ - statusText: string; - /** - * The **`headers`** read-only property of the with the response. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/headers) - */ - headers: Headers; - /** - * The **`ok`** read-only property of the Response interface contains a Boolean stating whether the response was successful (status in the range 200-299) or not. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/ok) - */ - ok: boolean; - /** - * The **`redirected`** read-only property of the Response interface indicates whether or not the response is the result of a request you made which was redirected. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/redirected) - */ - redirected: boolean; - /** - * The **`url`** read-only property of the Response interface contains the URL of the response. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/url) - */ - url: string; - webSocket: WebSocket | null; - cf: any | undefined; - /** - * The **`type`** read-only property of the Response interface contains the type of the response. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/type) - */ - type: "default" | "error"; -} -interface ResponseInit { - status?: number; - statusText?: string; - headers?: HeadersInit; - cf?: any; - webSocket?: (WebSocket | null); - encodeBody?: "automatic" | "manual"; -} -type RequestInfo> = Request | string; -/** - * The **`Request`** interface of the Fetch API represents a resource request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request) - */ -declare var Request: { - prototype: Request; - new >(input: RequestInfo | URL, init?: RequestInit): Request; -}; -/** - * The **`Request`** interface of the Fetch API represents a resource request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request) - */ -interface Request> extends Body { - /** - * The **`clone()`** method of the Request interface creates a copy of the current `Request` object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/clone) - */ - clone(): Request; - /** - * The **`method`** read-only property of the `POST`, etc.) A String indicating the method of the request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/method) - */ - method: string; - /** - * The **`url`** read-only property of the Request interface contains the URL of the request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/url) - */ - url: string; - /** - * The **`headers`** read-only property of the with the request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/headers) - */ - headers: Headers; - /** - * The **`redirect`** read-only property of the Request interface contains the mode for how redirects are handled. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/redirect) - */ - redirect: string; - fetcher: Fetcher | null; - /** - * The read-only **`signal`** property of the Request interface returns the AbortSignal associated with the request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/signal) - */ - signal: AbortSignal; - cf?: Cf; - /** - * The **`integrity`** read-only property of the Request interface contains the subresource integrity value of the request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/integrity) - */ - integrity: string; - /** - * The **`keepalive`** read-only property of the Request interface contains the request's `keepalive` setting (`true` or `false`), which indicates whether the browser will keep the associated request alive if the page that initiated it is unloaded before the request is complete. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/keepalive) - */ - keepalive: boolean; - /** - * The **`cache`** read-only property of the Request interface contains the cache mode of the request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/cache) - */ - cache?: "no-store" | "no-cache"; -} -interface RequestInit { - /* A string to set request's method. */ - method?: string; - /* A Headers object, an object literal, or an array of two-item arrays to set request's headers. */ - headers?: HeadersInit; - /* A BodyInit object or null to set request's body. */ - body?: BodyInit | null; - /* A string indicating whether request follows redirects, results in an error upon encountering a redirect, or returns the redirect (in an opaque fashion). Sets request's redirect. */ - redirect?: string; - fetcher?: (Fetcher | null); - cf?: Cf; - /* A string indicating how the request will interact with the browser's cache to set request's cache. */ - cache?: "no-store" | "no-cache"; - /* A cryptographic hash of the resource to be fetched by request. Sets request's integrity. */ - integrity?: string; - /* An AbortSignal to set request's signal. */ - signal?: (AbortSignal | null); - encodeResponseBody?: "automatic" | "manual"; -} -type Service Rpc.WorkerEntrypointBranded) | Rpc.WorkerEntrypointBranded | ExportedHandler | undefined = undefined> = T extends new (...args: any[]) => Rpc.WorkerEntrypointBranded ? Fetcher> : T extends Rpc.WorkerEntrypointBranded ? Fetcher : T extends Exclude ? never : Fetcher; -type Fetcher = (T extends Rpc.EntrypointBranded ? Rpc.Provider : unknown) & { - fetch(input: RequestInfo | URL, init?: RequestInit): Promise; - connect(address: SocketAddress | string, options?: SocketOptions): Socket; -}; -interface KVNamespaceListKey { - name: Key; - expiration?: number; - metadata?: Metadata; -} -type KVNamespaceListResult = { - list_complete: false; - keys: KVNamespaceListKey[]; - cursor: string; - cacheStatus: string | null; -} | { - list_complete: true; - keys: KVNamespaceListKey[]; - cacheStatus: string | null; -}; -interface KVNamespace { - get(key: Key, options?: Partial>): Promise; - get(key: Key, type: "text"): Promise; - get(key: Key, type: "json"): Promise; - get(key: Key, type: "arrayBuffer"): Promise; - get(key: Key, type: "stream"): Promise; - get(key: Key, options?: KVNamespaceGetOptions<"text">): Promise; - get(key: Key, options?: KVNamespaceGetOptions<"json">): Promise; - get(key: Key, options?: KVNamespaceGetOptions<"arrayBuffer">): Promise; - get(key: Key, options?: KVNamespaceGetOptions<"stream">): Promise; - get(key: Array, type: "text"): Promise>; - get(key: Array, type: "json"): Promise>; - get(key: Array, options?: Partial>): Promise>; - get(key: Array, options?: KVNamespaceGetOptions<"text">): Promise>; - get(key: Array, options?: KVNamespaceGetOptions<"json">): Promise>; - list(options?: KVNamespaceListOptions): Promise>; - put(key: Key, value: string | ArrayBuffer | ArrayBufferView | ReadableStream, options?: KVNamespacePutOptions): Promise; - getWithMetadata(key: Key, options?: Partial>): Promise>; - getWithMetadata(key: Key, type: "text"): Promise>; - getWithMetadata(key: Key, type: "json"): Promise>; - getWithMetadata(key: Key, type: "arrayBuffer"): Promise>; - getWithMetadata(key: Key, type: "stream"): Promise>; - getWithMetadata(key: Key, options: KVNamespaceGetOptions<"text">): Promise>; - getWithMetadata(key: Key, options: KVNamespaceGetOptions<"json">): Promise>; - getWithMetadata(key: Key, options: KVNamespaceGetOptions<"arrayBuffer">): Promise>; - getWithMetadata(key: Key, options: KVNamespaceGetOptions<"stream">): Promise>; - getWithMetadata(key: Array, type: "text"): Promise>>; - getWithMetadata(key: Array, type: "json"): Promise>>; - getWithMetadata(key: Array, options?: Partial>): Promise>>; - getWithMetadata(key: Array, options?: KVNamespaceGetOptions<"text">): Promise>>; - getWithMetadata(key: Array, options?: KVNamespaceGetOptions<"json">): Promise>>; - delete(key: Key): Promise; -} -interface KVNamespaceListOptions { - limit?: number; - prefix?: (string | null); - cursor?: (string | null); -} -interface KVNamespaceGetOptions { - type: Type; - cacheTtl?: number; -} -interface KVNamespacePutOptions { - expiration?: number; - expirationTtl?: number; - metadata?: (any | null); -} -interface KVNamespaceGetWithMetadataResult { - value: Value | null; - metadata: Metadata | null; - cacheStatus: string | null; -} -type QueueContentType = "text" | "bytes" | "json" | "v8"; -interface Queue { - metrics(): Promise; - send(message: Body, options?: QueueSendOptions): Promise; - sendBatch(messages: Iterable>, options?: QueueSendBatchOptions): Promise; -} -interface QueueSendMetrics { - backlogCount: number; - backlogBytes: number; - oldestMessageTimestamp?: Date; -} -interface QueueSendMetadata { - metrics: QueueSendMetrics; -} -interface QueueSendResponse { - metadata: QueueSendMetadata; -} -interface QueueSendBatchMetrics { - backlogCount: number; - backlogBytes: number; - oldestMessageTimestamp?: Date; -} -interface QueueSendBatchMetadata { - metrics: QueueSendBatchMetrics; -} -interface QueueSendBatchResponse { - metadata: QueueSendBatchMetadata; -} -interface QueueSendOptions { - contentType?: QueueContentType; - delaySeconds?: number; -} -interface QueueSendBatchOptions { - delaySeconds?: number; -} -interface MessageSendRequest { - body: Body; - contentType?: QueueContentType; - delaySeconds?: number; -} -interface QueueMetrics { - backlogCount: number; - backlogBytes: number; - oldestMessageTimestamp?: Date; -} -interface MessageBatchMetrics { - backlogCount: number; - backlogBytes: number; - oldestMessageTimestamp?: Date; -} -interface MessageBatchMetadata { - metrics: MessageBatchMetrics; -} -interface QueueRetryOptions { - delaySeconds?: number; -} -interface Message { - readonly id: string; - readonly timestamp: Date; - readonly body: Body; - readonly attempts: number; - retry(options?: QueueRetryOptions): void; - ack(): void; -} -interface QueueEvent extends ExtendableEvent { - readonly messages: readonly Message[]; - readonly queue: string; - readonly metadata: MessageBatchMetadata; - retryAll(options?: QueueRetryOptions): void; - ackAll(): void; -} -interface MessageBatch { - readonly messages: readonly Message[]; - readonly queue: string; - readonly metadata: MessageBatchMetadata; - retryAll(options?: QueueRetryOptions): void; - ackAll(): void; -} -interface R2Error extends Error { - readonly name: string; - readonly code: number; - readonly message: string; - readonly action: string; - readonly stack: any; -} -interface R2ListOptions { - limit?: number; - prefix?: string; - cursor?: string; - delimiter?: string; - startAfter?: string; - include?: ("httpMetadata" | "customMetadata")[]; -} -declare abstract class R2Bucket { - head(key: string): Promise; - get(key: string, options: R2GetOptions & { - onlyIf: R2Conditional | Headers; - }): Promise; - get(key: string, options?: R2GetOptions): Promise; - put(key: string, value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob, options?: R2PutOptions & { - onlyIf: R2Conditional | Headers; - }): Promise; - put(key: string, value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob, options?: R2PutOptions): Promise; - createMultipartUpload(key: string, options?: R2MultipartOptions): Promise; - resumeMultipartUpload(key: string, uploadId: string): R2MultipartUpload; - delete(keys: string | string[]): Promise; - list(options?: R2ListOptions): Promise; -} -interface R2MultipartUpload { - readonly key: string; - readonly uploadId: string; - uploadPart(partNumber: number, value: ReadableStream | (ArrayBuffer | ArrayBufferView) | string | Blob, options?: R2UploadPartOptions): Promise; - abort(): Promise; - complete(uploadedParts: R2UploadedPart[]): Promise; -} -interface R2UploadedPart { - partNumber: number; - etag: string; -} -declare abstract class R2Object { - readonly key: string; - readonly version: string; - readonly size: number; - readonly etag: string; - readonly httpEtag: string; - readonly checksums: R2Checksums; - readonly uploaded: Date; - readonly httpMetadata?: R2HTTPMetadata; - readonly customMetadata?: Record; - readonly range?: R2Range; - readonly storageClass: string; - readonly ssecKeyMd5?: string; - writeHttpMetadata(headers: Headers): void; -} -interface R2ObjectBody extends R2Object { - get body(): ReadableStream; - get bodyUsed(): boolean; - arrayBuffer(): Promise; - bytes(): Promise; - text(): Promise; - json(): Promise; - blob(): Promise; -} -type R2Range = { - offset: number; - length?: number; -} | { - offset?: number; - length: number; -} | { - suffix: number; -}; -interface R2Conditional { - etagMatches?: string; - etagDoesNotMatch?: string; - uploadedBefore?: Date; - uploadedAfter?: Date; - secondsGranularity?: boolean; -} -interface R2GetOptions { - onlyIf?: (R2Conditional | Headers); - range?: (R2Range | Headers); - ssecKey?: (ArrayBuffer | string); -} -interface R2PutOptions { - onlyIf?: (R2Conditional | Headers); - httpMetadata?: (R2HTTPMetadata | Headers); - customMetadata?: Record; - md5?: ((ArrayBuffer | ArrayBufferView) | string); - sha1?: ((ArrayBuffer | ArrayBufferView) | string); - sha256?: ((ArrayBuffer | ArrayBufferView) | string); - sha384?: ((ArrayBuffer | ArrayBufferView) | string); - sha512?: ((ArrayBuffer | ArrayBufferView) | string); - storageClass?: string; - ssecKey?: (ArrayBuffer | string); -} -interface R2MultipartOptions { - httpMetadata?: (R2HTTPMetadata | Headers); - customMetadata?: Record; - storageClass?: string; - ssecKey?: (ArrayBuffer | string); -} -interface R2Checksums { - readonly md5?: ArrayBuffer; - readonly sha1?: ArrayBuffer; - readonly sha256?: ArrayBuffer; - readonly sha384?: ArrayBuffer; - readonly sha512?: ArrayBuffer; - toJSON(): R2StringChecksums; -} -interface R2StringChecksums { - md5?: string; - sha1?: string; - sha256?: string; - sha384?: string; - sha512?: string; -} -interface R2HTTPMetadata { - contentType?: string; - contentLanguage?: string; - contentDisposition?: string; - contentEncoding?: string; - cacheControl?: string; - cacheExpiry?: Date; -} -type R2Objects = { - objects: R2Object[]; - delimitedPrefixes: string[]; -} & ({ - truncated: true; - cursor: string; -} | { - truncated: false; -}); -interface R2UploadPartOptions { - ssecKey?: (ArrayBuffer | string); -} -declare abstract class ScheduledEvent extends ExtendableEvent { - readonly scheduledTime: number; - readonly cron: string; - noRetry(): void; -} -interface ScheduledController { - readonly scheduledTime: number; - readonly cron: string; - noRetry(): void; -} -interface QueuingStrategy { - highWaterMark?: (number | bigint); - size?: (chunk: T) => number | bigint; -} -interface UnderlyingSink { - type?: string; - start?: (controller: WritableStreamDefaultController) => void | Promise; - write?: (chunk: W, controller: WritableStreamDefaultController) => void | Promise; - abort?: (reason: any) => void | Promise; - close?: () => void | Promise; -} -interface UnderlyingByteSource { - type: "bytes"; - autoAllocateChunkSize?: number; - start?: (controller: ReadableByteStreamController) => void | Promise; - pull?: (controller: ReadableByteStreamController) => void | Promise; - cancel?: (reason: any) => void | Promise; -} -interface UnderlyingSource { - type?: "" | undefined; - start?: (controller: ReadableStreamDefaultController) => void | Promise; - pull?: (controller: ReadableStreamDefaultController) => void | Promise; - cancel?: (reason: any) => void | Promise; - expectedLength?: (number | bigint); -} -interface Transformer { - readableType?: string; - writableType?: string; - start?: (controller: TransformStreamDefaultController) => void | Promise; - transform?: (chunk: I, controller: TransformStreamDefaultController) => void | Promise; - flush?: (controller: TransformStreamDefaultController) => void | Promise; - cancel?: (reason: any) => void | Promise; - expectedLength?: number; -} -interface StreamPipeOptions { - preventAbort?: boolean; - preventCancel?: boolean; - /** - * Pipes this readable stream to a given writable stream destination. The way in which the piping process behaves under various error conditions can be customized with a number of passed options. It returns a promise that fulfills when the piping process completes successfully, or rejects if any errors were encountered. - * - * Piping a stream will lock it for the duration of the pipe, preventing any other consumer from acquiring a reader. - * - * Errors and closures of the source and destination streams propagate as follows: - * - * An error in this source readable stream will abort destination, unless preventAbort is truthy. The returned promise will be rejected with the source's error, or with any error that occurs during aborting the destination. - * - * An error in destination will cancel this source readable stream, unless preventCancel is truthy. The returned promise will be rejected with the destination's error, or with any error that occurs during canceling the source. - * - * When this source readable stream closes, destination will be closed, unless preventClose is truthy. The returned promise will be fulfilled once this process completes, unless an error is encountered while closing the destination, in which case it will be rejected with that error. - * - * If destination starts out closed or closing, this source readable stream will be canceled, unless preventCancel is true. The returned promise will be rejected with an error indicating piping to a closed stream failed, or with any error that occurs during canceling the source. - * - * The signal option can be set to an AbortSignal to allow aborting an ongoing pipe operation via the corresponding AbortController. In this case, this source readable stream will be canceled, and destination aborted, unless the respective options preventCancel or preventAbort are set. - */ - preventClose?: boolean; - signal?: AbortSignal; -} -type ReadableStreamReadResult = { - done: false; - value: R; -} | { - done: true; - value?: undefined; -}; -/** - * The `ReadableStream` interface of the Streams API represents a readable stream of byte data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream) - */ -interface ReadableStream { - /** - * The **`locked`** read-only property of the ReadableStream interface returns whether or not the readable stream is locked to a reader. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/locked) - */ - get locked(): boolean; - /** - * The **`cancel()`** method of the ReadableStream interface returns a Promise that resolves when the stream is canceled. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/cancel) - */ - cancel(reason?: any): Promise; - /** - * The **`getReader()`** method of the ReadableStream interface creates a reader and locks the stream to it. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/getReader) - */ - getReader(): ReadableStreamDefaultReader; - /** - * The **`getReader()`** method of the ReadableStream interface creates a reader and locks the stream to it. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/getReader) - */ - getReader(options: ReadableStreamGetReaderOptions): ReadableStreamBYOBReader; - /** - * The **`pipeThrough()`** method of the ReadableStream interface provides a chainable way of piping the current stream through a transform stream or any other writable/readable pair. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/pipeThrough) - */ - pipeThrough(transform: ReadableWritablePair, options?: StreamPipeOptions): ReadableStream; - /** - * The **`pipeTo()`** method of the ReadableStream interface pipes the current `ReadableStream` to a given WritableStream and returns a Promise that fulfills when the piping process completes successfully, or rejects if any errors were encountered. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/pipeTo) - */ - pipeTo(destination: WritableStream, options?: StreamPipeOptions): Promise; - /** - * The **`tee()`** method of the two-element array containing the two resulting branches as new ReadableStream instances. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/tee) - */ - tee(): [ - ReadableStream, - ReadableStream - ]; - values(options?: ReadableStreamValuesOptions): AsyncIterableIterator; - [Symbol.asyncIterator](options?: ReadableStreamValuesOptions): AsyncIterableIterator; -} -/** - * The `ReadableStream` interface of the Streams API represents a readable stream of byte data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream) - */ -declare const ReadableStream: { - prototype: ReadableStream; - new (underlyingSource: UnderlyingByteSource, strategy?: QueuingStrategy): ReadableStream; - new (underlyingSource?: UnderlyingSource, strategy?: QueuingStrategy): ReadableStream; -}; -/** - * The **`ReadableStreamDefaultReader`** interface of the Streams API represents a default reader that can be used to read stream data supplied from a network (such as a fetch request). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultReader) - */ -declare class ReadableStreamDefaultReader { - constructor(stream: ReadableStream); - get closed(): Promise; - cancel(reason?: any): Promise; - /** - * The **`read()`** method of the ReadableStreamDefaultReader interface returns a Promise providing access to the next chunk in the stream's internal queue. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultReader/read) - */ - read(): Promise>; - /** - * The **`releaseLock()`** method of the ReadableStreamDefaultReader interface releases the reader's lock on the stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultReader/releaseLock) - */ - releaseLock(): void; -} -/** - * The `ReadableStreamBYOBReader` interface of the Streams API defines a reader for a ReadableStream that supports zero-copy reading from an underlying byte source. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBReader) - */ -declare class ReadableStreamBYOBReader { - constructor(stream: ReadableStream); - get closed(): Promise; - cancel(reason?: any): Promise; - /** - * The **`read()`** method of the ReadableStreamBYOBReader interface is used to read data into a view on a user-supplied buffer from an associated readable byte stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBReader/read) - */ - read(view: T): Promise>; - /** - * The **`releaseLock()`** method of the ReadableStreamBYOBReader interface releases the reader's lock on the stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBReader/releaseLock) - */ - releaseLock(): void; - readAtLeast(minElements: number, view: T): Promise>; -} -interface ReadableStreamBYOBReaderReadableStreamBYOBReaderReadOptions { - min?: number; -} -interface ReadableStreamGetReaderOptions { - /** - * Creates a ReadableStreamBYOBReader and locks the stream to the new reader. - * - * This call behaves the same way as the no-argument variant, except that it only works on readable byte streams, i.e. streams which were constructed specifically with the ability to handle "bring your own buffer" reading. The returned BYOB reader provides the ability to directly read individual chunks from the stream via its read() method, into developer-supplied buffers, allowing more precise control over allocation. - */ - mode: "byob"; -} -/** - * The **`ReadableStreamBYOBRequest`** interface of the Streams API represents a 'pull request' for data from an underlying source that will made as a zero-copy transfer to a consumer (bypassing the stream's internal queues). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest) - */ -declare abstract class ReadableStreamBYOBRequest { - /** - * The **`view`** getter property of the ReadableStreamBYOBRequest interface returns the current view. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest/view) - */ - get view(): Uint8Array | null; - /** - * The **`respond()`** method of the ReadableStreamBYOBRequest interface is used to signal to the associated readable byte stream that the specified number of bytes were written into the ReadableStreamBYOBRequest.view. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest/respond) - */ - respond(bytesWritten: number): void; - /** - * The **`respondWithNewView()`** method of the ReadableStreamBYOBRequest interface specifies a new view that the consumer of the associated readable byte stream should write to instead of ReadableStreamBYOBRequest.view. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest/respondWithNewView) - */ - respondWithNewView(view: ArrayBuffer | ArrayBufferView): void; - get atLeast(): number | null; -} -/** - * The **`ReadableStreamDefaultController`** interface of the Streams API represents a controller allowing control of a ReadableStream's state and internal queue. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController) - */ -declare abstract class ReadableStreamDefaultController { - /** - * The **`desiredSize`** read-only property of the required to fill the stream's internal queue. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/desiredSize) - */ - get desiredSize(): number | null; - /** - * The **`close()`** method of the ReadableStreamDefaultController interface closes the associated stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/close) - */ - close(): void; - /** - * The **`enqueue()`** method of the ```js-nolint enqueue(chunk) ``` - `chunk` - : The chunk to enqueue. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/enqueue) - */ - enqueue(chunk?: R): void; - /** - * The **`error()`** method of the with the associated stream to error. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/error) - */ - error(reason: any): void; -} -/** - * The **`ReadableByteStreamController`** interface of the Streams API represents a controller for a readable byte stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController) - */ -declare abstract class ReadableByteStreamController { - /** - * The **`byobRequest`** read-only property of the ReadableByteStreamController interface returns the current BYOB request, or `null` if there are no pending requests. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/byobRequest) - */ - get byobRequest(): ReadableStreamBYOBRequest | null; - /** - * The **`desiredSize`** read-only property of the ReadableByteStreamController interface returns the number of bytes required to fill the stream's internal queue to its 'desired size'. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/desiredSize) - */ - get desiredSize(): number | null; - /** - * The **`close()`** method of the ReadableByteStreamController interface closes the associated stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/close) - */ - close(): void; - /** - * The **`enqueue()`** method of the ReadableByteStreamController interface enqueues a given chunk on the associated readable byte stream (the chunk is copied into the stream's internal queues). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/enqueue) - */ - enqueue(chunk: ArrayBuffer | ArrayBufferView): void; - /** - * The **`error()`** method of the ReadableByteStreamController interface causes any future interactions with the associated stream to error with the specified reason. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/error) - */ - error(reason: any): void; -} -/** - * The **`WritableStreamDefaultController`** interface of the Streams API represents a controller allowing control of a WritableStream's state. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultController) - */ -declare abstract class WritableStreamDefaultController { - /** - * The read-only **`signal`** property of the WritableStreamDefaultController interface returns the AbortSignal associated with the controller. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultController/signal) - */ - get signal(): AbortSignal; - /** - * The **`error()`** method of the with the associated stream to error. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultController/error) - */ - error(reason?: any): void; -} -/** - * The **`TransformStreamDefaultController`** interface of the Streams API provides methods to manipulate the associated ReadableStream and WritableStream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController) - */ -declare abstract class TransformStreamDefaultController { - /** - * The **`desiredSize`** read-only property of the TransformStreamDefaultController interface returns the desired size to fill the queue of the associated ReadableStream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/desiredSize) - */ - get desiredSize(): number | null; - /** - * The **`enqueue()`** method of the TransformStreamDefaultController interface enqueues the given chunk in the readable side of the stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/enqueue) - */ - enqueue(chunk?: O): void; - /** - * The **`error()`** method of the TransformStreamDefaultController interface errors both sides of the stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/error) - */ - error(reason: any): void; - /** - * The **`terminate()`** method of the TransformStreamDefaultController interface closes the readable side and errors the writable side of the stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/terminate) - */ - terminate(): void; -} -interface ReadableWritablePair { - readable: ReadableStream; - /** - * Provides a convenient, chainable way of piping this readable stream through a transform stream (or any other { writable, readable } pair). It simply pipes the stream into the writable side of the supplied pair, and returns the readable side for further use. - * - * Piping a stream will lock it for the duration of the pipe, preventing any other consumer from acquiring a reader. - */ - writable: WritableStream; -} -/** - * The **`WritableStream`** interface of the Streams API provides a standard abstraction for writing streaming data to a destination, known as a sink. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream) - */ -declare class WritableStream { - constructor(underlyingSink?: UnderlyingSink, queuingStrategy?: QueuingStrategy); - /** - * The **`locked`** read-only property of the WritableStream interface returns a boolean indicating whether the `WritableStream` is locked to a writer. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/locked) - */ - get locked(): boolean; - /** - * The **`abort()`** method of the WritableStream interface aborts the stream, signaling that the producer can no longer successfully write to the stream and it is to be immediately moved to an error state, with any queued writes discarded. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/abort) - */ - abort(reason?: any): Promise; - /** - * The **`close()`** method of the WritableStream interface closes the associated stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/close) - */ - close(): Promise; - /** - * The **`getWriter()`** method of the WritableStream interface returns a new instance of WritableStreamDefaultWriter and locks the stream to that instance. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/getWriter) - */ - getWriter(): WritableStreamDefaultWriter; -} -/** - * The **`WritableStreamDefaultWriter`** interface of the Streams API is the object returned by WritableStream.getWriter() and once created locks the writer to the `WritableStream` ensuring that no other streams can write to the underlying sink. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter) - */ -declare class WritableStreamDefaultWriter { - constructor(stream: WritableStream); - /** - * The **`closed`** read-only property of the the stream errors or the writer's lock is released. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/closed) - */ - get closed(): Promise; - /** - * The **`ready`** read-only property of the that resolves when the desired size of the stream's internal queue transitions from non-positive to positive, signaling that it is no longer applying backpressure. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/ready) - */ - get ready(): Promise; - /** - * The **`desiredSize`** read-only property of the to fill the stream's internal queue. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/desiredSize) - */ - get desiredSize(): number | null; - /** - * The **`abort()`** method of the the producer can no longer successfully write to the stream and it is to be immediately moved to an error state, with any queued writes discarded. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/abort) - */ - abort(reason?: any): Promise; - /** - * The **`close()`** method of the stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/close) - */ - close(): Promise; - /** - * The **`write()`** method of the operation. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/write) - */ - write(chunk?: W): Promise; - /** - * The **`releaseLock()`** method of the corresponding stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/releaseLock) - */ - releaseLock(): void; -} -/** - * The **`TransformStream`** interface of the Streams API represents a concrete implementation of the pipe chain _transform stream_ concept. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStream) - */ -declare class TransformStream { - constructor(transformer?: Transformer, writableStrategy?: QueuingStrategy, readableStrategy?: QueuingStrategy); - /** - * The **`readable`** read-only property of the TransformStream interface returns the ReadableStream instance controlled by this `TransformStream`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStream/readable) - */ - get readable(): ReadableStream; - /** - * The **`writable`** read-only property of the TransformStream interface returns the WritableStream instance controlled by this `TransformStream`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStream/writable) - */ - get writable(): WritableStream; -} -declare class FixedLengthStream extends IdentityTransformStream { - constructor(expectedLength: number | bigint, queuingStrategy?: IdentityTransformStreamQueuingStrategy); -} -declare class IdentityTransformStream extends TransformStream { - constructor(queuingStrategy?: IdentityTransformStreamQueuingStrategy); -} -interface IdentityTransformStreamQueuingStrategy { - highWaterMark?: (number | bigint); -} -interface ReadableStreamValuesOptions { - preventCancel?: boolean; -} -/** - * The **`CompressionStream`** interface of the Compression Streams API is an API for compressing a stream of data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CompressionStream) - */ -declare class CompressionStream extends TransformStream { - constructor(format: "gzip" | "deflate" | "deflate-raw"); -} -/** - * The **`DecompressionStream`** interface of the Compression Streams API is an API for decompressing a stream of data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DecompressionStream) - */ -declare class DecompressionStream extends TransformStream { - constructor(format: "gzip" | "deflate" | "deflate-raw"); -} -/** - * The **`TextEncoderStream`** interface of the Encoding API converts a stream of strings into bytes in the UTF-8 encoding. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoderStream) - */ -declare class TextEncoderStream extends TransformStream { - constructor(); - get encoding(): string; -} -/** - * The **`TextDecoderStream`** interface of the Encoding API converts a stream of text in a binary encoding, such as UTF-8 etc., to a stream of strings. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextDecoderStream) - */ -declare class TextDecoderStream extends TransformStream { - constructor(label?: string, options?: TextDecoderStreamTextDecoderStreamInit); - get encoding(): string; - get fatal(): boolean; - get ignoreBOM(): boolean; -} -interface TextDecoderStreamTextDecoderStreamInit { - fatal?: boolean; - ignoreBOM?: boolean; -} -/** - * The **`ByteLengthQueuingStrategy`** interface of the Streams API provides a built-in byte length queuing strategy that can be used when constructing streams. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ByteLengthQueuingStrategy) - */ -declare class ByteLengthQueuingStrategy implements QueuingStrategy { - constructor(init: QueuingStrategyInit); - /** - * The read-only **`ByteLengthQueuingStrategy.highWaterMark`** property returns the total number of bytes that can be contained in the internal queue before backpressure is applied. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ByteLengthQueuingStrategy/highWaterMark) - */ - get highWaterMark(): number; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/ByteLengthQueuingStrategy/size) */ - get size(): (chunk?: any) => number; -} -/** - * The **`CountQueuingStrategy`** interface of the Streams API provides a built-in chunk counting queuing strategy that can be used when constructing streams. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CountQueuingStrategy) - */ -declare class CountQueuingStrategy implements QueuingStrategy { - constructor(init: QueuingStrategyInit); - /** - * The read-only **`CountQueuingStrategy.highWaterMark`** property returns the total number of chunks that can be contained in the internal queue before backpressure is applied. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CountQueuingStrategy/highWaterMark) - */ - get highWaterMark(): number; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/CountQueuingStrategy/size) */ - get size(): (chunk?: any) => number; -} -interface QueuingStrategyInit { - /** - * Creates a new ByteLengthQueuingStrategy with the provided high water mark. - * - * Note that the provided high water mark will not be validated ahead of time. Instead, if it is negative, NaN, or not a number, the resulting ByteLengthQueuingStrategy will cause the corresponding stream constructor to throw. - */ - highWaterMark: number; -} -interface TracePreviewInfo { - id: string; - slug: string; - name: string; -} -interface ScriptVersion { - id?: string; - tag?: string; - message?: string; -} -declare abstract class TailEvent extends ExtendableEvent { - readonly events: TraceItem[]; - readonly traces: TraceItem[]; -} -interface TraceItem { - readonly event: (TraceItemFetchEventInfo | TraceItemJsRpcEventInfo | TraceItemConnectEventInfo | TraceItemScheduledEventInfo | TraceItemAlarmEventInfo | TraceItemQueueEventInfo | TraceItemEmailEventInfo | TraceItemTailEventInfo | TraceItemCustomEventInfo | TraceItemHibernatableWebSocketEventInfo) | null; - readonly eventTimestamp: number | null; - readonly logs: TraceLog[]; - readonly exceptions: TraceException[]; - readonly diagnosticsChannelEvents: TraceDiagnosticChannelEvent[]; - readonly scriptName: string | null; - readonly entrypoint?: string; - readonly scriptVersion?: ScriptVersion; - readonly dispatchNamespace?: string; - readonly scriptTags?: string[]; - readonly tailAttributes?: Record; - readonly preview?: TracePreviewInfo; - readonly durableObjectId?: string; - readonly outcome: string; - readonly executionModel: string; - readonly truncated: boolean; - readonly cpuTime: number; - readonly wallTime: number; -} -interface TraceItemAlarmEventInfo { - readonly scheduledTime: Date; -} -interface TraceItemConnectEventInfo { -} -interface TraceItemCustomEventInfo { -} -interface TraceItemScheduledEventInfo { - readonly scheduledTime: number; - readonly cron: string; -} -interface TraceItemQueueEventInfo { - readonly queue: string; - readonly batchSize: number; -} -interface TraceItemEmailEventInfo { - readonly mailFrom: string; - readonly rcptTo: string; - readonly rawSize: number; -} -interface TraceItemTailEventInfo { - readonly consumedEvents: TraceItemTailEventInfoTailItem[]; -} -interface TraceItemTailEventInfoTailItem { - readonly scriptName: string | null; -} -interface TraceItemFetchEventInfo { - readonly response?: TraceItemFetchEventInfoResponse; - readonly request: TraceItemFetchEventInfoRequest; -} -interface TraceItemFetchEventInfoRequest { - readonly cf?: any; - readonly headers: Record; - readonly method: string; - readonly url: string; - getUnredacted(): TraceItemFetchEventInfoRequest; -} -interface TraceItemFetchEventInfoResponse { - readonly status: number; -} -interface TraceItemJsRpcEventInfo { - readonly rpcMethod: string; -} -interface TraceItemHibernatableWebSocketEventInfo { - readonly getWebSocketEvent: TraceItemHibernatableWebSocketEventInfoMessage | TraceItemHibernatableWebSocketEventInfoClose | TraceItemHibernatableWebSocketEventInfoError; -} -interface TraceItemHibernatableWebSocketEventInfoMessage { - readonly webSocketEventType: string; -} -interface TraceItemHibernatableWebSocketEventInfoClose { - readonly webSocketEventType: string; - readonly code: number; - readonly wasClean: boolean; -} -interface TraceItemHibernatableWebSocketEventInfoError { - readonly webSocketEventType: string; -} -interface TraceLog { - readonly timestamp: number; - readonly level: string; - readonly message: any; -} -interface TraceException { - readonly timestamp: number; - readonly message: string; - readonly name: string; - readonly stack?: string; -} -interface TraceDiagnosticChannelEvent { - readonly timestamp: number; - readonly channel: string; - readonly message: any; -} -interface TraceMetrics { - readonly cpuTime: number; - readonly wallTime: number; -} -interface UnsafeTraceMetrics { - fromTrace(item: TraceItem): TraceMetrics; -} -/** - * The **`URL`** interface is used to parse, construct, normalize, and encode URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL) - */ -declare class URL { - constructor(url: string | URL, base?: string | URL); - /** - * The **`origin`** read-only property of the URL interface returns a string containing the Unicode serialization of the origin of the represented URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/origin) - */ - get origin(): string; - /** - * The **`href`** property of the URL interface is a string containing the whole URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/href) - */ - get href(): string; - /** - * The **`href`** property of the URL interface is a string containing the whole URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/href) - */ - set href(value: string); - /** - * The **`protocol`** property of the URL interface is a string containing the protocol or scheme of the URL, including the final `':'`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/protocol) - */ - get protocol(): string; - /** - * The **`protocol`** property of the URL interface is a string containing the protocol or scheme of the URL, including the final `':'`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/protocol) - */ - set protocol(value: string); - /** - * The **`username`** property of the URL interface is a string containing the username component of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/username) - */ - get username(): string; - /** - * The **`username`** property of the URL interface is a string containing the username component of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/username) - */ - set username(value: string); - /** - * The **`password`** property of the URL interface is a string containing the password component of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/password) - */ - get password(): string; - /** - * The **`password`** property of the URL interface is a string containing the password component of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/password) - */ - set password(value: string); - /** - * The **`host`** property of the URL interface is a string containing the host, which is the URL.hostname, and then, if the port of the URL is nonempty, a `':'`, followed by the URL.port of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/host) - */ - get host(): string; - /** - * The **`host`** property of the URL interface is a string containing the host, which is the URL.hostname, and then, if the port of the URL is nonempty, a `':'`, followed by the URL.port of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/host) - */ - set host(value: string); - /** - * The **`hostname`** property of the URL interface is a string containing either the domain name or IP address of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hostname) - */ - get hostname(): string; - /** - * The **`hostname`** property of the URL interface is a string containing either the domain name or IP address of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hostname) - */ - set hostname(value: string); - /** - * The **`port`** property of the URL interface is a string containing the port number of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/port) - */ - get port(): string; - /** - * The **`port`** property of the URL interface is a string containing the port number of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/port) - */ - set port(value: string); - /** - * The **`pathname`** property of the URL interface represents a location in a hierarchical structure. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/pathname) - */ - get pathname(): string; - /** - * The **`pathname`** property of the URL interface represents a location in a hierarchical structure. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/pathname) - */ - set pathname(value: string); - /** - * The **`search`** property of the URL interface is a search string, also called a _query string_, that is a string containing a `'?'` followed by the parameters of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/search) - */ - get search(): string; - /** - * The **`search`** property of the URL interface is a search string, also called a _query string_, that is a string containing a `'?'` followed by the parameters of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/search) - */ - set search(value: string); - /** - * The **`hash`** property of the URL interface is a string containing a `'#'` followed by the fragment identifier of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hash) - */ - get hash(): string; - /** - * The **`hash`** property of the URL interface is a string containing a `'#'` followed by the fragment identifier of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hash) - */ - set hash(value: string); - /** - * The **`searchParams`** read-only property of the access to the [MISSING: httpmethod('GET')] decoded query arguments contained in the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/searchParams) - */ - get searchParams(): URLSearchParams; - /** - * The **`toJSON()`** method of the URL interface returns a string containing a serialized version of the URL, although in practice it seems to have the same effect as ```js-nolint toJSON() ``` None. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/toJSON) - */ - toJSON(): string; - /*function toString() { [native code] }*/ - toString(): string; - /** - * The **`URL.canParse()`** static method of the URL interface returns a boolean indicating whether or not an absolute URL, or a relative URL combined with a base URL, are parsable and valid. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/canParse_static) - */ - static canParse(url: string, base?: string): boolean; - /** - * The **`URL.parse()`** static method of the URL interface returns a newly created URL object representing the URL defined by the parameters. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/parse_static) - */ - static parse(url: string, base?: string): URL | null; - /** - * The **`createObjectURL()`** static method of the URL interface creates a string containing a URL representing the object given in the parameter. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/createObjectURL_static) - */ - static createObjectURL(object: File | Blob): string; - /** - * The **`revokeObjectURL()`** static method of the URL interface releases an existing object URL which was previously created by calling Call this method when you've finished using an object URL to let the browser know not to keep the reference to the file any longer. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/revokeObjectURL_static) - */ - static revokeObjectURL(object_url: string): void; -} -/** - * The **`URLSearchParams`** interface defines utility methods to work with the query string of a URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams) - */ -declare class URLSearchParams { - constructor(init?: (Iterable> | Record | string)); - /** - * The **`size`** read-only property of the URLSearchParams interface indicates the total number of search parameter entries. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/size) - */ - get size(): number; - /** - * The **`append()`** method of the URLSearchParams interface appends a specified key/value pair as a new search parameter. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/append) - */ - append(name: string, value: string): void; - /** - * The **`delete()`** method of the URLSearchParams interface deletes specified parameters and their associated value(s) from the list of all search parameters. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/delete) - */ - delete(name: string, value?: string): void; - /** - * The **`get()`** method of the URLSearchParams interface returns the first value associated to the given search parameter. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/get) - */ - get(name: string): string | null; - /** - * The **`getAll()`** method of the URLSearchParams interface returns all the values associated with a given search parameter as an array. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/getAll) - */ - getAll(name: string): string[]; - /** - * The **`has()`** method of the URLSearchParams interface returns a boolean value that indicates whether the specified parameter is in the search parameters. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/has) - */ - has(name: string, value?: string): boolean; - /** - * The **`set()`** method of the URLSearchParams interface sets the value associated with a given search parameter to the given value. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/set) - */ - set(name: string, value: string): void; - /** - * The **`URLSearchParams.sort()`** method sorts all key/value pairs contained in this object in place and returns `undefined`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/sort) - */ - sort(): void; - /* Returns an array of key, value pairs for every entry in the search params. */ - entries(): IterableIterator<[ - key: string, - value: string - ]>; - /* Returns a list of keys in the search params. */ - keys(): IterableIterator; - /* Returns a list of values in the search params. */ - values(): IterableIterator; - forEach(callback: (this: This, value: string, key: string, parent: URLSearchParams) => void, thisArg?: This): void; - /*function toString() { [native code] }*/ - toString(): string; - [Symbol.iterator](): IterableIterator<[ - key: string, - value: string - ]>; -} -declare class URLPattern { - constructor(input?: (string | URLPatternInit), baseURL?: (string | URLPatternOptions), patternOptions?: URLPatternOptions); - get protocol(): string; - get username(): string; - get password(): string; - get hostname(): string; - get port(): string; - get pathname(): string; - get search(): string; - get hash(): string; - get hasRegExpGroups(): boolean; - test(input?: (string | URLPatternInit), baseURL?: string): boolean; - exec(input?: (string | URLPatternInit), baseURL?: string): URLPatternResult | null; -} -interface URLPatternInit { - protocol?: string; - username?: string; - password?: string; - hostname?: string; - port?: string; - pathname?: string; - search?: string; - hash?: string; - baseURL?: string; -} -interface URLPatternComponentResult { - input: string; - groups: Record; -} -interface URLPatternResult { - inputs: (string | URLPatternInit)[]; - protocol: URLPatternComponentResult; - username: URLPatternComponentResult; - password: URLPatternComponentResult; - hostname: URLPatternComponentResult; - port: URLPatternComponentResult; - pathname: URLPatternComponentResult; - search: URLPatternComponentResult; - hash: URLPatternComponentResult; -} -interface URLPatternOptions { - ignoreCase?: boolean; -} -/** - * A `CloseEvent` is sent to clients using WebSockets when the connection is closed. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent) - */ -declare class CloseEvent extends Event { - constructor(type: string, initializer?: CloseEventInit); - /** - * The **`code`** read-only property of the CloseEvent interface returns a WebSocket connection close code indicating the reason the connection was closed. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent/code) - */ - readonly code: number; - /** - * The **`reason`** read-only property of the CloseEvent interface returns the WebSocket connection close reason the server gave for closing the connection; that is, a concise human-readable prose explanation for the closure. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent/reason) - */ - readonly reason: string; - /** - * The **`wasClean`** read-only property of the CloseEvent interface returns `true` if the connection closed cleanly. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent/wasClean) - */ - readonly wasClean: boolean; -} -interface CloseEventInit { - code?: number; - reason?: string; - wasClean?: boolean; -} -type WebSocketEventMap = { - close: CloseEvent; - message: MessageEvent; - open: Event; - error: ErrorEvent; -}; -/** - * The `WebSocket` object provides the API for creating and managing a WebSocket connection to a server, as well as for sending and receiving data on the connection. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket) - */ -declare var WebSocket: { - prototype: WebSocket; - new (url: string, protocols?: (string[] | string)): WebSocket; - readonly READY_STATE_CONNECTING: number; - readonly CONNECTING: number; - readonly READY_STATE_OPEN: number; - readonly OPEN: number; - readonly READY_STATE_CLOSING: number; - readonly CLOSING: number; - readonly READY_STATE_CLOSED: number; - readonly CLOSED: number; -}; -/** - * The `WebSocket` object provides the API for creating and managing a WebSocket connection to a server, as well as for sending and receiving data on the connection. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket) - */ -interface WebSocket extends EventTarget { - accept(options?: WebSocketAcceptOptions): void; - /** - * The **`WebSocket.send()`** method enqueues the specified data to be transmitted to the server over the WebSocket connection, increasing the value of `bufferedAmount` by the number of bytes needed to contain the data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/send) - */ - send(message: (ArrayBuffer | ArrayBufferView) | string): void; - /** - * The **`WebSocket.close()`** method closes the already `CLOSED`, this method does nothing. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/close) - */ - close(code?: number, reason?: string): void; - serializeAttachment(attachment: any): void; - deserializeAttachment(): any | null; - /** - * The **`WebSocket.readyState`** read-only property returns the current state of the WebSocket connection. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/readyState) - */ - readyState: number; - /** - * The **`WebSocket.url`** read-only property returns the absolute URL of the WebSocket as resolved by the constructor. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/url) - */ - url: string | null; - /** - * The **`WebSocket.protocol`** read-only property returns the name of the sub-protocol the server selected; this will be one of the strings specified in the `protocols` parameter when creating the WebSocket object, or the empty string if no connection is established. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/protocol) - */ - protocol: string | null; - /** - * The **`WebSocket.extensions`** read-only property returns the extensions selected by the server. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/extensions) - */ - extensions: string | null; - /** - * The **`WebSocket.binaryType`** property controls the type of binary data being received over the WebSocket connection. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/binaryType) - */ - binaryType: "blob" | "arraybuffer"; -} -interface WebSocketAcceptOptions { - /** - * When set to `true`, receiving a server-initiated WebSocket Close frame will not - * automatically send a reciprocal Close frame, leaving the connection in a half-open - * state. This is useful for proxying scenarios where you need to coordinate closing - * both sides independently. Defaults to `false` when the - * `no_web_socket_half_open_by_default` compatibility flag is enabled. - */ - allowHalfOpen?: boolean; -} -declare const WebSocketPair: { - new (): { - 0: WebSocket; - 1: WebSocket; - }; -}; -interface SqlStorage { - exec>(query: string, ...bindings: any[]): SqlStorageCursor; - get databaseSize(): number; - Cursor: typeof SqlStorageCursor; - Statement: typeof SqlStorageStatement; -} -declare abstract class SqlStorageStatement { -} -type SqlStorageValue = ArrayBuffer | string | number | null; -declare abstract class SqlStorageCursor> { - next(): { - done?: false; - value: T; - } | { - done: true; - value?: never; - }; - toArray(): T[]; - one(): T; - raw(): IterableIterator; - columnNames: string[]; - get rowsRead(): number; - get rowsWritten(): number; - [Symbol.iterator](): IterableIterator; -} -interface Socket { - get readable(): ReadableStream; - get writable(): WritableStream; - get closed(): Promise; - get opened(): Promise; - get upgraded(): boolean; - get secureTransport(): "on" | "off" | "starttls"; - close(): Promise; - startTls(options?: TlsOptions): Socket; -} -interface SocketOptions { - secureTransport?: string; - allowHalfOpen: boolean; - highWaterMark?: (number | bigint); -} -interface SocketAddress { - hostname: string; - port: number; -} -interface TlsOptions { - expectedServerHostname?: string; -} -interface SocketInfo { - remoteAddress?: string; - localAddress?: string; -} -/** - * The **`EventSource`** interface is web content's interface to server-sent events. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource) - */ -declare class EventSource extends EventTarget { - constructor(url: string, init?: EventSourceEventSourceInit); - /** - * The **`close()`** method of the EventSource interface closes the connection, if one is made, and sets the ```js-nolint close() ``` None. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/close) - */ - close(): void; - /** - * The **`url`** read-only property of the URL of the source. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/url) - */ - get url(): string; - /** - * The **`withCredentials`** read-only property of the the `EventSource` object was instantiated with CORS credentials set. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/withCredentials) - */ - get withCredentials(): boolean; - /** - * The **`readyState`** read-only property of the connection. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/readyState) - */ - get readyState(): number; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/open_event) */ - get onopen(): any | null; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/open_event) */ - set onopen(value: any | null); - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/message_event) */ - get onmessage(): any | null; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/message_event) */ - set onmessage(value: any | null); - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/error_event) */ - get onerror(): any | null; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/error_event) */ - set onerror(value: any | null); - static readonly CONNECTING: number; - static readonly OPEN: number; - static readonly CLOSED: number; - static from(stream: ReadableStream): EventSource; -} -interface EventSourceEventSourceInit { - withCredentials?: boolean; - fetcher?: Fetcher; -} -interface Container { - get running(): boolean; - start(options?: ContainerStartupOptions): void; - monitor(): Promise; - destroy(error?: any): Promise; - signal(signo: number): void; - getTcpPort(port: number): Fetcher; - setInactivityTimeout(durationMs: number | bigint): Promise; - interceptOutboundHttp(addr: string, binding: Fetcher): Promise; - interceptAllOutboundHttp(binding: Fetcher): Promise; - snapshotDirectory(options: ContainerDirectorySnapshotOptions): Promise; - snapshotContainer(options: ContainerSnapshotOptions): Promise; - interceptOutboundHttps(addr: string, binding: Fetcher): Promise; -} -interface ContainerDirectorySnapshot { - id: string; - size: number; - dir: string; - name?: string; -} -interface ContainerDirectorySnapshotOptions { - dir: string; - name?: string; -} -interface ContainerDirectorySnapshotRestoreParams { - snapshot: ContainerDirectorySnapshot; - mountPoint?: string; -} -interface ContainerSnapshot { - id: string; - size: number; - name?: string; -} -interface ContainerSnapshotOptions { - name?: string; -} -interface ContainerStartupOptions { - entrypoint?: string[]; - enableInternet: boolean; - env?: Record; - labels?: Record; - directorySnapshots?: ContainerDirectorySnapshotRestoreParams[]; - containerSnapshot?: ContainerSnapshot; -} -/** - * The **`MessagePort`** interface of the Channel Messaging API represents one of the two ports of a MessageChannel, allowing messages to be sent from one port and listening out for them arriving at the other. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort) - */ -declare abstract class MessagePort extends EventTarget { - /** - * The **`postMessage()`** method of the transfers ownership of objects to other browsing contexts. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort/postMessage) - */ - postMessage(data?: any, options?: (any[] | MessagePortPostMessageOptions)): void; - /** - * The **`close()`** method of the MessagePort interface disconnects the port, so it is no longer active. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort/close) - */ - close(): void; - /** - * The **`start()`** method of the MessagePort interface starts the sending of messages queued on the port. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort/start) - */ - start(): void; - get onmessage(): any | null; - set onmessage(value: any | null); -} -/** - * The **`MessageChannel`** interface of the Channel Messaging API allows us to create a new message channel and send data through it via its two MessagePort properties. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageChannel) - */ -declare class MessageChannel { - constructor(); - /** - * The **`port1`** read-only property of the the port attached to the context that originated the channel. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageChannel/port1) - */ - readonly port1: MessagePort; - /** - * The **`port2`** read-only property of the the port attached to the context at the other end of the channel, which the message is initially sent to. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageChannel/port2) - */ - readonly port2: MessagePort; -} -interface MessagePortPostMessageOptions { - transfer?: any[]; -} -type LoopbackForExport Rpc.EntrypointBranded) | ExportedHandler | undefined = undefined> = T extends new (...args: any[]) => Rpc.WorkerEntrypointBranded ? LoopbackServiceStub> : T extends new (...args: any[]) => Rpc.DurableObjectBranded ? LoopbackDurableObjectClass> : T extends ExportedHandler ? LoopbackServiceStub : undefined; -type LoopbackServiceStub = Fetcher & (T extends CloudflareWorkersModule.WorkerEntrypoint ? (opts: { - props?: Props; -}) => Fetcher : (opts: { - props?: any; -}) => Fetcher); -type LoopbackDurableObjectClass = DurableObjectClass & (T extends CloudflareWorkersModule.DurableObject ? (opts: { - props?: Props; -}) => DurableObjectClass : (opts: { - props?: any; -}) => DurableObjectClass); -interface LoopbackDurableObjectNamespace extends DurableObjectNamespace { -} -interface LoopbackColoLocalActorNamespace extends ColoLocalActorNamespace { -} -interface SyncKvStorage { - get(key: string): T | undefined; - list(options?: SyncKvListOptions): Iterable<[ - string, - T - ]>; - put(key: string, value: T): void; - delete(key: string): boolean; -} -interface SyncKvListOptions { - start?: string; - startAfter?: string; - end?: string; - prefix?: string; - reverse?: boolean; - limit?: number; -} -interface WorkerStub { - getEntrypoint(name?: string, options?: WorkerStubEntrypointOptions): Fetcher; - getDurableObjectClass(name?: string, options?: WorkerStubEntrypointOptions): DurableObjectClass; -} -interface WorkerStubEntrypointOptions { - props?: any; - limits?: workerdResourceLimits; -} -interface WorkerLoader { - get(name: string | null, getCode: () => WorkerLoaderWorkerCode | Promise): WorkerStub; - load(code: WorkerLoaderWorkerCode): WorkerStub; -} -interface WorkerLoaderModule { - js?: string; - cjs?: string; - text?: string; - data?: ArrayBuffer; - json?: any; - py?: string; - wasm?: ArrayBuffer; -} -interface WorkerLoaderWorkerCode { - compatibilityDate: string; - compatibilityFlags?: string[]; - allowExperimental?: boolean; - limits?: workerdResourceLimits; - mainModule: string; - modules: Record; - env?: any; - globalOutbound?: (Fetcher | null); - tails?: Fetcher[]; - streamingTails?: Fetcher[]; -} -interface workerdResourceLimits { - cpuMs?: number; - subRequests?: number; -} -/** -* The Workers runtime supports a subset of the Performance API, used to measure timing and performance, -* as well as timing of subrequests and other operations. -* -* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/) -*/ -declare abstract class Performance { - /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/#performancetimeorigin) */ - get timeOrigin(): number; - /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/#performancenow) */ - now(): number; - /** - * The **`toJSON()`** method of the Performance interface is a Serialization; it returns a JSON representation of the Performance object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/toJSON) - */ - toJSON(): object; -} -// ============ AI Search Error Interfaces ============ -interface AiSearchInternalError extends Error { -} -interface AiSearchNotFoundError extends Error { -} -// ============ AI Search Common Types ============ -/** A single message in a conversation-style search or chat request. */ -type AiSearchMessage = { - role: 'system' | 'developer' | 'user' | 'assistant' | 'tool'; - content: string | null; -}; -/** - * Common shape for `ai_search_options` used by both single-instance and multi-instance requests. - * Contains retrieval, query rewrite, reranking, and cache sub-options. - */ -type AiSearchOptions = { - retrieval?: { - /** Which retrieval backend to use. Defaults to the instance's configured index_method. */ - retrieval_type?: 'vector' | 'keyword' | 'hybrid'; - /** Fusion method for combining vector + keyword results. */ - fusion_method?: 'max' | 'rrf'; - /** How keyword terms are combined: "and" = all terms must match, "or" = any term matches. */ - keyword_match_mode?: 'and' | 'or'; - /** Minimum similarity score (0-1) for a result to be included. Default 0.4. */ - match_threshold?: number; - /** Maximum number of results to return (1-50). Default 10. */ - max_num_results?: number; - /** Vectorize metadata filters applied to the search. */ - filters?: VectorizeVectorMetadataFilter; - /** Number of surrounding chunks to include for context (0-3). Default 0. */ - context_expansion?: number; - /** If true, return only item metadata without chunk text. */ - metadata_only?: boolean; - /** If true (default), return empty results on retrieval failure instead of throwing. */ - return_on_failure?: boolean; - /** Boost results by metadata field values. Max 3 entries. */ - boost_by?: Array<{ - field: string; - direction?: 'asc' | 'desc' | 'exists' | 'not_exists'; - }>; - [key: string]: unknown; - }; - query_rewrite?: { - enabled?: boolean; - model?: string; - rewrite_prompt?: string; - [key: string]: unknown; - }; - reranking?: { - enabled?: boolean; - model?: string; - /** Match threshold (0-1, default 0.4) */ - match_threshold?: number; - [key: string]: unknown; - }; - cache?: { - enabled?: boolean; - cache_threshold?: 'super_strict_match' | 'close_enough' | 'flexible_friend' | 'anything_goes'; - }; - [key: string]: unknown; -}; -// ============ AI Search Request Types ============ -/** - * Request body for single-instance search. - * Exactly one of `query` or `messages` must be provided. - */ -type AiSearchSearchRequest = { - /** Simple query string. */ - query: string; - messages?: never; - ai_search_options?: AiSearchOptions; -} | { - query?: never; - /** Conversation-style input. At least one user message with non-empty content is required. */ - messages: AiSearchMessage[]; - ai_search_options?: AiSearchOptions; -}; -type AiSearchChatCompletionsRequest = { - messages: AiSearchMessage[]; - model?: string; - stream?: boolean; - ai_search_options?: AiSearchOptions; - [key: string]: unknown; -}; -// ============ AI Search Multi-Instance Types (Namespace-Scoped) ============ -/** `ai_search_options` shape for multi-instance requests — requires `instance_ids`. */ -type AiSearchMultiSearchOptions = AiSearchOptions & { - /** Instance IDs to search across (1-10). */ - instance_ids: string[]; -}; -/** - * Request for searching across multiple instances within a namespace. - * `ai_search_options` is required and must include `instance_ids`. - * Exactly one of `query` or `messages` must be provided. - */ -type AiSearchMultiSearchRequest = { - /** Simple query string. */ - query: string; - messages?: never; - ai_search_options: AiSearchMultiSearchOptions; -} | { - query?: never; - /** Conversation-style input. */ - messages: AiSearchMessage[]; - ai_search_options: AiSearchMultiSearchOptions; -}; -/** A search result chunk tagged with the instance it originated from. */ -type AiSearchMultiSearchChunk = AiSearchSearchResponse['chunks'][number] & { - instance_id: string; -}; -/** Describes a per-instance error during a multi-instance operation. */ -type AiSearchMultiSearchError = { - instance_id: string; - message: string; -}; -/** Response from a multi-instance search, with chunks tagged by instance and optional partial-failure errors. */ -type AiSearchMultiSearchResponse = { - search_query: string; - chunks: AiSearchMultiSearchChunk[]; - errors?: AiSearchMultiSearchError[]; -}; -/** Request for chat completions across multiple instances within a namespace. `ai_search_options` is required and must include `instance_ids`. */ -type AiSearchMultiChatCompletionsRequest = Omit & { - ai_search_options: AiSearchMultiSearchOptions; -}; -/** Response from multi-instance chat completions, with chunks tagged by instance and optional partial-failure errors. */ -type AiSearchMultiChatCompletionsResponse = Omit & { - chunks: AiSearchMultiSearchChunk[]; - errors?: AiSearchMultiSearchError[]; -}; -// ============ AI Search Response Types ============ -type AiSearchSearchResponse = { - search_query: string; - chunks: Array<{ - id: string; - type: string; - /** Match score (0-1) */ - score: number; - text: string; - item: { - timestamp?: number; - key: string; - metadata?: Record; - }; - scoring_details?: { - /** Keyword match score (0-1) */ - keyword_score?: number; - /** Vector similarity score (0-1) */ - vector_score?: number; - /** Keyword rank position */ - keyword_rank?: number; - /** Vector rank position */ - vector_rank?: number; - /** Reranking model score */ - reranking_score?: number; - /** Fusion method used to combine results */ - fusion_method?: 'rrf' | 'max'; - [key: string]: unknown; - }; - }>; -}; -type AiSearchChatCompletionsResponse = { - id?: string; - object?: string; - model?: string; - choices: Array<{ - index?: number; - message: { - role: 'system' | 'developer' | 'user' | 'assistant' | 'tool'; - content: string | null; - [key: string]: unknown; - }; - [key: string]: unknown; - }>; - chunks: AiSearchSearchResponse['chunks']; - [key: string]: unknown; -}; -type AiSearchStatsResponse = { - queued?: number; - running?: number; - completed?: number; - error?: number; - skipped?: number; - outdated?: number; - last_activity?: string; - /** Storage engine statistics. */ - engine?: { - vectorize?: { - vectorsCount: number; - dimensions: number; - }; - r2?: { - payloadSizeBytes: number; - metadataSizeBytes: number; - objectCount: number; - }; - }; -}; -// ============ AI Search Instance Info Types ============ -type AiSearchInstanceInfo = { - id: string; - type?: 'r2' | 'web-crawler' | string; - source?: string; - source_params?: unknown; - paused?: boolean; - status?: string; - namespace?: string; - created_at?: string; - modified_at?: string; - token_id?: string; - ai_gateway_id?: string; - rewrite_query?: boolean; - reranking?: boolean; - embedding_model?: string; - ai_search_model?: string; - rewrite_model?: string; - reranking_model?: string; - /** @deprecated Use index_method instead. */ - hybrid_search_enabled?: boolean; - /** Controls which storage backends are active. */ - index_method?: { - vector?: boolean; - keyword?: boolean; - }; - /** Fusion method for combining vector and keyword results. */ - fusion_method?: 'max' | 'rrf'; - indexing_options?: { - keyword_tokenizer?: 'porter' | 'trigram'; - } | null; - retrieval_options?: { - keyword_match_mode?: 'and' | 'or'; - boost_by?: Array<{ - field: string; - direction?: 'asc' | 'desc' | 'exists' | 'not_exists'; - }>; - } | null; - chunk?: boolean; - chunk_size?: number; - chunk_overlap?: number; - score_threshold?: number; - max_num_results?: number; - cache?: boolean; - cache_threshold?: 'super_strict_match' | 'close_enough' | 'flexible_friend' | 'anything_goes'; - custom_metadata?: Array<{ - field_name: string; - data_type: 'text' | 'number' | 'boolean' | 'datetime'; - }>; - /** Sync interval in seconds. */ - sync_interval?: 3600 | 7200 | 14400 | 21600 | 43200 | 86400; - metadata?: Record; - [key: string]: unknown; -}; -/** Pagination, search, and ordering parameters for listing instances within a namespace. */ -type AiSearchListInstancesParams = { - page?: number; - per_page?: number; - /** Search instances by ID. */ - search?: string; - /** Field to sort by. */ - order_by?: 'created_at'; - /** Sort direction. */ - order_by_direction?: 'asc' | 'desc'; -}; -type AiSearchListResponse = { - result: AiSearchInstanceInfo[]; - result_info?: { - count: number; - page: number; - per_page: number; - total_count: number; - }; -}; -// ============ AI Search Config Types ============ -type AiSearchConfig = { - /** Instance ID (1-32 chars, pattern: ^[a-z0-9_]+(?:-[a-z0-9_]+)*$) */ - id: string; - /** Instance type. Omit to create with built-in storage. */ - type?: 'r2' | 'web-crawler' | string; - /** Source URL (required for web-crawler type). */ - source?: string; - source_params?: unknown; - /** Token ID (UUID format) */ - token_id?: string; - ai_gateway_id?: string; - /** Enable query rewriting (default false) */ - rewrite_query?: boolean; - /** Enable reranking (default false) */ - reranking?: boolean; - embedding_model?: string; - ai_search_model?: string; - rewrite_model?: string; - reranking_model?: string; - /** @deprecated Use index_method instead. */ - hybrid_search_enabled?: boolean; - /** Controls which storage backends are used during indexing. Defaults to vector-only. */ - index_method?: { - vector?: boolean; - keyword?: boolean; - }; - /** Fusion method for combining vector and keyword results. "rrf" = reciprocal rank fusion (default), "max" = maximum score. */ - fusion_method?: 'max' | 'rrf'; - indexing_options?: { - keyword_tokenizer?: 'porter' | 'trigram'; - } | null; - retrieval_options?: { - keyword_match_mode?: 'and' | 'or'; - boost_by?: Array<{ - field: string; - direction?: 'asc' | 'desc' | 'exists' | 'not_exists'; - }>; - } | null; - chunk?: boolean; - chunk_size?: number; - chunk_overlap?: number; - /** Minimum similarity score (0-1) for a result to be included. */ - score_threshold?: number; - max_num_results?: number; - cache?: boolean; - /** Similarity threshold for cache hits. Stricter = fewer cache hits but higher relevance. */ - cache_threshold?: 'super_strict_match' | 'close_enough' | 'flexible_friend' | 'anything_goes'; - custom_metadata?: Array<{ - field_name: string; - data_type: 'text' | 'number' | 'boolean' | 'datetime'; - }>; - namespace?: string; - /** Sync interval in seconds. 3600=1h, 7200=2h, 14400=4h, 21600=6h, 43200=12h, 86400=24h. */ - sync_interval?: 3600 | 7200 | 14400 | 21600 | 43200 | 86400; - metadata?: Record; - [key: string]: unknown; -}; -// ============ AI Search Item Types ============ -type AiSearchItemInfo = { - id: string; - key: string; - status: 'completed' | 'error' | 'skipped' | 'queued' | 'running' | 'outdated'; - next_action?: 'INDEX' | 'DELETE' | null; - error?: string; - checksum?: string; - namespace?: string; - chunks_count?: number | null; - file_size?: number | null; - source_id?: string | null; - last_seen_at?: string; - created_at?: string; - metadata?: Record; - [key: string]: unknown; -}; -type AiSearchItemContentResult = { - body: ReadableStream; - contentType: string; - filename: string; - size: number; -}; -type AiSearchUploadItemOptions = { - metadata?: Record; -}; -type AiSearchListItemsParams = { - page?: number; - per_page?: number; - /** Search items by key name. */ - search?: string; - /** Sort order for results. */ - sort_by?: 'status' | 'modified_at'; - /** Filter items by processing status. */ - status?: 'queued' | 'running' | 'completed' | 'error' | 'skipped' | 'outdated'; - /** Filter items by source (e.g. "builtin" or "web-crawler:https://example.com"). */ - source?: string; - /** JSON-encoded Vectorize filter for metadata filtering. */ - metadata_filter?: string; -}; -type AiSearchListItemsResponse = { - result: AiSearchItemInfo[]; - result_info?: { - count: number; - page: number; - per_page: number; - total_count: number; - }; -}; -// ============ AI Search Item Logs Types ============ -type AiSearchItemLogsParams = { - /** Maximum number of log entries to return (1-100, default 50). */ - limit?: number; - /** Opaque cursor for pagination. Pass the `cursor` value from a previous response. */ - cursor?: string; -}; -type AiSearchItemLog = { - timestamp: string; - action: string; - message: string; - fileKey?: string; - chunkCount?: number; - processingTimeMs?: number; - errorType?: string; -}; -/** Paginated response for item processing logs (cursor-based). */ -type AiSearchItemLogsResponse = { - result: AiSearchItemLog[]; - result_info: { - count: number; - per_page: number; - cursor: string | null; - truncated: boolean; - }; -}; -// ============ AI Search Item Chunks Types ============ -type AiSearchItemChunksParams = { - /** Maximum number of chunks to return (1-100, default 20). */ - limit?: number; - /** Offset into the chunks list (default 0). */ - offset?: number; -}; -/** A single indexed chunk belonging to an item, including its text content and byte range. */ -type AiSearchItemChunk = { - id: string; - text: string; - start_byte: number; - end_byte: number; - item?: { - timestamp?: number; - key: string; - metadata?: Record; - }; -}; -/** Paginated response for item chunks (offset-based). */ -type AiSearchItemChunksResponse = { - result: AiSearchItemChunk[]; - result_info: { - count: number; - total: number; - limit: number; - offset: number; - }; -}; -// ============ AI Search Job Types ============ -type AiSearchJobInfo = { - id: string; - source: 'user' | 'schedule'; - description?: string; - last_seen_at?: string; - started_at?: string; - ended_at?: string; - end_reason?: string; -}; -type AiSearchJobLog = { - id: number; - message: string; - message_type: number; - created_at: number; -}; -type AiSearchCreateJobParams = { - description?: string; -}; -type AiSearchListJobsParams = { - page?: number; - per_page?: number; -}; -type AiSearchListJobsResponse = { - result: AiSearchJobInfo[]; - result_info?: { - count: number; - page: number; - per_page: number; - total_count: number; - }; -}; -type AiSearchJobLogsParams = { - page?: number; - per_page?: number; -}; -type AiSearchJobLogsResponse = { - result: AiSearchJobLog[]; - result_info?: { - count: number; - page: number; - per_page: number; - total_count: number; - }; -}; -// ============ AI Search Sub-Service Classes ============ -/** - * Single item service for an AI Search instance. - * Provides info, download, sync, logs, and chunks operations on a specific item. - */ -declare abstract class AiSearchItem { - /** Get metadata about this item. */ - info(): Promise; - /** - * Download the item's content. - * @returns Object with body stream, content type, filename, and size. - */ - download(): Promise; - /** - * Trigger re-indexing of this item. - * @returns The updated item info. - */ - sync(): Promise; - /** - * Retrieve processing logs for this item (cursor-based pagination). - * @param params Optional pagination parameters (limit, cursor). - * @returns Paginated log entries for this item. - */ - logs(params?: AiSearchItemLogsParams): Promise; - /** - * List indexed chunks for this item (offset-based pagination). - * @param params Optional pagination parameters (limit, offset). - * @returns Paginated chunk entries for this item. - */ - chunks(params?: AiSearchItemChunksParams): Promise; -} -/** - * Items collection service for an AI Search instance. - * Provides list, upload, and access to individual items. - */ -declare abstract class AiSearchItems { - /** List items in this instance. */ - list(params?: AiSearchListItemsParams): Promise; - /** - * Upload a file as an item. Behaves as an upsert: if an item with the same - * filename already exists, it is overwritten and re-indexed. - * @param name Filename for the uploaded item. - * @param content File content as a ReadableStream, Blob, or string. - * @param options Optional metadata to attach to the item. - * @returns The created item info. - */ - upload(name: string, content: ReadableStream | Blob | string, options?: AiSearchUploadItemOptions): Promise; - /** - * Upload a file and poll until processing completes. - * Behaves as an upsert: if an item with the same filename already exists, - * it is overwritten and re-indexed. - * @param name Filename for the uploaded item. - * @param content File content as a ReadableStream, Blob, or string. - * @param options Optional metadata and polling configuration. - * @returns The item info after processing completes (or timeout). - */ - uploadAndPoll(name: string, content: ReadableStream | Blob | string, options?: AiSearchUploadItemOptions & { - /** Polling interval in milliseconds (default 1000). */ - pollIntervalMs?: number; - /** Maximum time to wait in milliseconds (default 30000). */ - timeoutMs?: number; - }): Promise; - /** - * Get an item by ID. - * @param itemId The item identifier. - * @returns Item service for info, download, sync, logs, and chunks operations. - */ - get(itemId: string): AiSearchItem; - /** - * Delete an item from the instance. - * @param itemId The item identifier. - */ - delete(itemId: string): Promise; -} -/** - * Single job service for an AI Search instance. - * Provides info, logs, and cancel operations for a specific job. - */ -declare abstract class AiSearchJob { - /** Get metadata about this job. */ - info(): Promise; - /** Get logs for this job. */ - logs(params?: AiSearchJobLogsParams): Promise; - /** - * Cancel a running job. - * @returns The updated job info. - * @throws AiSearchNotFoundError if the job does not exist. - */ - cancel(): Promise; -} -/** - * Jobs collection service for an AI Search instance. - * Provides list, create, and access to individual jobs. - */ -declare abstract class AiSearchJobs { - /** List jobs for this instance. */ - list(params?: AiSearchListJobsParams): Promise; - /** - * Create a new indexing job. - * @param params Optional job parameters. - * @returns The created job info. - */ - create(params?: AiSearchCreateJobParams): Promise; - /** - * Get a job by ID. - * @param jobId The job identifier. - * @returns Job service for info, logs, and cancel operations. - */ - get(jobId: string): AiSearchJob; -} -// ============ AI Search Binding Classes ============ -/** - * Instance-level AI Search service. - * - * Used as: - * - The return type of `AiSearchNamespace.get(name)` (namespace binding) - * - The type of `env.BLOG_SEARCH` (single instance binding via `ai_search`) - * - * Provides search, chat, update, stats, items, and jobs operations. - * - * @example - * ```ts - * // Via namespace binding - * const instance = env.AI_SEARCH.get("blog"); - * const results = await instance.search({ - * query: "How does caching work?", - * }); - * - * // Via single instance binding - * const results = await env.BLOG_SEARCH.search({ - * messages: [{ role: "user", content: "How does caching work?" }], - * }); - * ``` - */ -declare abstract class AiSearchInstance { - /** - * Search the AI Search instance for relevant chunks. - * @param params Search request with query or messages and optional AI search options. - * @returns Search response with matching chunks and search query. - */ - search(params: AiSearchSearchRequest): Promise; - /** - * Generate chat completions with AI Search context (streaming). - * @param params Chat completions request with stream: true. - * @returns ReadableStream of server-sent events. - */ - chatCompletions(params: AiSearchChatCompletionsRequest & { - stream: true; - }): Promise; - /** - * Generate chat completions with AI Search context. - * @param params Chat completions request. - * @returns Chat completion response with choices and RAG chunks. - */ - chatCompletions(params: AiSearchChatCompletionsRequest): Promise; - /** - * Update the instance configuration. - * @param config Partial configuration to update. - * @returns Updated instance info. - */ - update(config: Partial): Promise; - /** Get metadata about this instance. */ - info(): Promise; - /** - * Get instance statistics (item count, indexing status, etc.). - * @returns Statistics with counts per status, last activity time, and engine details. - */ - stats(): Promise; - /** Items collection — list, upload, and manage items in this instance. */ - get items(): AiSearchItems; - /** Jobs collection — list, create, and inspect indexing jobs. */ - get jobs(): AiSearchJobs; -} -/** - * Namespace-level AI Search service. - * - * Used as the type of `env.AI_SEARCH` (namespace binding via `ai_search_namespaces`). - * Scoped to a single namespace. Provides dynamic instance access, creation, deletion, - * and multi-instance search/chat operations. - * - * @example - * ```ts - * // Access an instance within the namespace - * const blog = env.AI_SEARCH.get("blog"); - * const results = await blog.search({ query: "How does caching work?" }); - * - * // List all instances in the namespace - * const instances = await env.AI_SEARCH.list(); - * - * // Create a new instance with built-in storage - * const tenant = await env.AI_SEARCH.create({ id: "tenant-123" }); - * - * // Upload items into the instance - * await tenant.items.upload("doc.pdf", fileContent); - * - * // Search across multiple instances - * const multi = await env.AI_SEARCH.search({ - * query: "caching", - * ai_search_options: { instance_ids: ["blog", "docs"] }, - * }); - * - * // Delete an instance - * await env.AI_SEARCH.delete("tenant-123"); - * ``` - */ -declare abstract class AiSearchNamespace { - /** - * Get an instance by name within the bound namespace. - * @param name Instance name. - * @returns Instance service for search, chat, update, stats, items, and jobs. - */ - get(name: string): AiSearchInstance; - /** - * List instances in the bound namespace. - * @param params Optional pagination, search, and ordering parameters. - * @returns Array of instance metadata with pagination info. - */ - list(params?: AiSearchListInstancesParams): Promise; - /** - * Create a new instance within the bound namespace. - * @param config Instance configuration. Only `id` is required — omit `type` and `source` to create with built-in storage. - * @returns Instance service for the newly created instance. - * - * @example - * ```ts - * // Create with built-in storage (upload items manually) - * const instance = await env.AI_SEARCH.create({ id: "my-search" }); - * - * // Create with web crawler source - * const instance = await env.AI_SEARCH.create({ - * id: "docs-search", - * type: "web-crawler", - * source: "https://developers.cloudflare.com", - * }); - * ``` - */ - create(config: AiSearchConfig): Promise; - /** - * Delete an instance from the bound namespace. - * @param name Instance name to delete. - */ - delete(name: string): Promise; - /** - * Search across multiple instances within the bound namespace. - * Fans out to the specified instance_ids and merges results. - * @param params Search request with required `ai_search_options.instance_ids`. - * @returns Search response with chunks tagged by instance_id and optional partial-failure errors. - */ - search(params: AiSearchMultiSearchRequest): Promise; - /** - * Generate chat completions across multiple instances within the bound namespace (streaming). - * Fans out to the specified instance_ids, merges context, and generates a response. - * @param params Chat completions request with stream: true and required `ai_search_options.instance_ids`. - * @returns ReadableStream of server-sent events. - */ - chatCompletions(params: AiSearchMultiChatCompletionsRequest & { - stream: true; - }): Promise; - /** - * Generate chat completions across multiple instances within the bound namespace. - * Fans out to the specified instance_ids, merges context, and generates a response. - * @param params Chat completions request with required `ai_search_options.instance_ids`. - * @returns Chat completion response with choices, chunks tagged by instance_id, and optional partial-failure errors. - */ - chatCompletions(params: AiSearchMultiChatCompletionsRequest): Promise; -} -type AiImageClassificationInput = { - image: number[]; -}; -type AiImageClassificationOutput = { - score?: number; - label?: string; -}[]; -declare abstract class BaseAiImageClassification { - inputs: AiImageClassificationInput; - postProcessedOutputs: AiImageClassificationOutput; -} -type AiImageToTextInput = { - image: number[]; - prompt?: string; - max_tokens?: number; - temperature?: number; - top_p?: number; - top_k?: number; - seed?: number; - repetition_penalty?: number; - frequency_penalty?: number; - presence_penalty?: number; - raw?: boolean; - messages?: RoleScopedChatInput[]; -}; -type AiImageToTextOutput = { - description: string; -}; -declare abstract class BaseAiImageToText { - inputs: AiImageToTextInput; - postProcessedOutputs: AiImageToTextOutput; -} -type AiImageTextToTextInput = { - image: string; - prompt?: string; - max_tokens?: number; - temperature?: number; - ignore_eos?: boolean; - top_p?: number; - top_k?: number; - seed?: number; - repetition_penalty?: number; - frequency_penalty?: number; - presence_penalty?: number; - raw?: boolean; - messages?: RoleScopedChatInput[]; -}; -type AiImageTextToTextOutput = { - description: string; -}; -declare abstract class BaseAiImageTextToText { - inputs: AiImageTextToTextInput; - postProcessedOutputs: AiImageTextToTextOutput; -} -type AiMultimodalEmbeddingsInput = { - image: string; - text: string[]; -}; -type AiIMultimodalEmbeddingsOutput = { - data: number[][]; - shape: number[]; -}; -declare abstract class BaseAiMultimodalEmbeddings { - inputs: AiImageTextToTextInput; - postProcessedOutputs: AiImageTextToTextOutput; -} -type AiObjectDetectionInput = { - image: number[]; -}; -type AiObjectDetectionOutput = { - score?: number; - label?: string; -}[]; -declare abstract class BaseAiObjectDetection { - inputs: AiObjectDetectionInput; - postProcessedOutputs: AiObjectDetectionOutput; -} -type AiSentenceSimilarityInput = { - source: string; - sentences: string[]; -}; -type AiSentenceSimilarityOutput = number[]; -declare abstract class BaseAiSentenceSimilarity { - inputs: AiSentenceSimilarityInput; - postProcessedOutputs: AiSentenceSimilarityOutput; -} -type AiAutomaticSpeechRecognitionInput = { - audio: number[]; -}; -type AiAutomaticSpeechRecognitionOutput = { - text?: string; - words?: { - word: string; - start: number; - end: number; - }[]; - vtt?: string; -}; -declare abstract class BaseAiAutomaticSpeechRecognition { - inputs: AiAutomaticSpeechRecognitionInput; - postProcessedOutputs: AiAutomaticSpeechRecognitionOutput; -} -type AiSummarizationInput = { - input_text: string; - max_length?: number; -}; -type AiSummarizationOutput = { - summary: string; -}; -declare abstract class BaseAiSummarization { - inputs: AiSummarizationInput; - postProcessedOutputs: AiSummarizationOutput; -} -type AiTextClassificationInput = { - text: string; -}; -type AiTextClassificationOutput = { - score?: number; - label?: string; -}[]; -declare abstract class BaseAiTextClassification { - inputs: AiTextClassificationInput; - postProcessedOutputs: AiTextClassificationOutput; -} -type AiTextEmbeddingsInput = { - text: string | string[]; -}; -type AiTextEmbeddingsOutput = { - shape: number[]; - data: number[][]; -}; -declare abstract class BaseAiTextEmbeddings { - inputs: AiTextEmbeddingsInput; - postProcessedOutputs: AiTextEmbeddingsOutput; -} -type RoleScopedChatInput = { - role: "user" | "assistant" | "system" | "tool" | (string & NonNullable); - content: string; - name?: string; -}; -type AiTextGenerationToolLegacyInput = { - name: string; - description: string; - parameters?: { - type: "object" | (string & NonNullable); - properties: { - [key: string]: { - type: string; - description?: string; - }; - }; - required: string[]; - }; -}; -type AiTextGenerationToolInput = { - type: "function" | (string & NonNullable); - function: { - name: string; - description: string; - parameters?: { - type: "object" | (string & NonNullable); - properties: { - [key: string]: { - type: string; - description?: string; - }; - }; - required: string[]; - }; - }; -}; -type AiTextGenerationFunctionsInput = { - name: string; - code: string; -}; -type AiTextGenerationResponseFormat = { - type: string; - json_schema?: any; -}; -type AiTextGenerationInput = { - prompt?: string; - raw?: boolean; - stream?: boolean; - max_tokens?: number; - temperature?: number; - top_p?: number; - top_k?: number; - seed?: number; - repetition_penalty?: number; - frequency_penalty?: number; - presence_penalty?: number; - messages?: RoleScopedChatInput[]; - response_format?: AiTextGenerationResponseFormat; - tools?: AiTextGenerationToolInput[] | AiTextGenerationToolLegacyInput[] | (object & NonNullable); - functions?: AiTextGenerationFunctionsInput[]; -}; -type AiTextGenerationToolLegacyOutput = { - name: string; - arguments: unknown; -}; -type AiTextGenerationToolOutput = { - id: string; - type: "function"; - function: { - name: string; - arguments: string; - }; -}; -type UsageTags = { - prompt_tokens: number; - completion_tokens: number; - total_tokens: number; -}; -type AiTextGenerationOutput = { - response?: string; - tool_calls?: AiTextGenerationToolLegacyOutput[] & AiTextGenerationToolOutput[]; - usage?: UsageTags; -}; -declare abstract class BaseAiTextGeneration { - inputs: AiTextGenerationInput; - postProcessedOutputs: AiTextGenerationOutput; -} -type AiTextToSpeechInput = { - prompt: string; - lang?: string; -}; -type AiTextToSpeechOutput = Uint8Array | { - audio: string; -}; -declare abstract class BaseAiTextToSpeech { - inputs: AiTextToSpeechInput; - postProcessedOutputs: AiTextToSpeechOutput; -} -type AiTextToImageInput = { - prompt: string; - negative_prompt?: string; - height?: number; - width?: number; - image?: number[]; - image_b64?: string; - mask?: number[]; - num_steps?: number; - strength?: number; - guidance?: number; - seed?: number; -}; -type AiTextToImageOutput = ReadableStream; -declare abstract class BaseAiTextToImage { - inputs: AiTextToImageInput; - postProcessedOutputs: AiTextToImageOutput; -} -type AiTranslationInput = { - text: string; - target_lang: string; - source_lang?: string; -}; -type AiTranslationOutput = { - translated_text?: string; -}; -declare abstract class BaseAiTranslation { - inputs: AiTranslationInput; - postProcessedOutputs: AiTranslationOutput; -} -/** - * Workers AI support for OpenAI's Chat Completions API - */ -type ChatCompletionContentPartText = { - type: "text"; - text: string; -}; -type ChatCompletionContentPartImage = { - type: "image_url"; - image_url: { - url: string; - detail?: "auto" | "low" | "high"; - }; -}; -type ChatCompletionContentPartInputAudio = { - type: "input_audio"; - input_audio: { - /** Base64 encoded audio data. */ - data: string; - format: "wav" | "mp3"; - }; -}; -type ChatCompletionContentPartFile = { - type: "file"; - file: { - /** Base64 encoded file data. */ - file_data?: string; - /** The ID of an uploaded file. */ - file_id?: string; - filename?: string; - }; -}; -type ChatCompletionContentPartRefusal = { - type: "refusal"; - refusal: string; -}; -type ChatCompletionContentPart = ChatCompletionContentPartText | ChatCompletionContentPartImage | ChatCompletionContentPartInputAudio | ChatCompletionContentPartFile; -type FunctionDefinition = { - name: string; - description?: string; - parameters?: Record; - strict?: boolean | null; -}; -type ChatCompletionFunctionTool = { - type: "function"; - function: FunctionDefinition; -}; -type ChatCompletionCustomToolGrammarFormat = { - type: "grammar"; - grammar: { - definition: string; - syntax: "lark" | "regex"; - }; -}; -type ChatCompletionCustomToolTextFormat = { - type: "text"; -}; -type ChatCompletionCustomToolFormat = ChatCompletionCustomToolTextFormat | ChatCompletionCustomToolGrammarFormat; -type ChatCompletionCustomTool = { - type: "custom"; - custom: { - name: string; - description?: string; - format?: ChatCompletionCustomToolFormat; - }; -}; -type ChatCompletionTool = ChatCompletionFunctionTool | ChatCompletionCustomTool; -type ChatCompletionMessageFunctionToolCall = { - id: string; - type: "function"; - function: { - name: string; - /** JSON-encoded arguments string. */ - arguments: string; - }; -}; -type ChatCompletionMessageCustomToolCall = { - id: string; - type: "custom"; - custom: { - name: string; - input: string; - }; -}; -type ChatCompletionMessageToolCall = ChatCompletionMessageFunctionToolCall | ChatCompletionMessageCustomToolCall; -type ChatCompletionToolChoiceFunction = { - type: "function"; - function: { - name: string; - }; -}; -type ChatCompletionToolChoiceCustom = { - type: "custom"; - custom: { - name: string; - }; -}; -type ChatCompletionToolChoiceAllowedTools = { - type: "allowed_tools"; - allowed_tools: { - mode: "auto" | "required"; - tools: Array>; - }; -}; -type ChatCompletionToolChoiceOption = "none" | "auto" | "required" | ChatCompletionToolChoiceFunction | ChatCompletionToolChoiceCustom | ChatCompletionToolChoiceAllowedTools; -type DeveloperMessage = { - role: "developer"; - content: string | Array<{ - type: "text"; - text: string; - }>; - name?: string; -}; -type SystemMessage = { - role: "system"; - content: string | Array<{ - type: "text"; - text: string; - }>; - name?: string; -}; -/** - * Permissive merged content part used inside UserMessage arrays. - * - * Cabidela has a limitation where anyOf/oneOf with enum-based discrimination - * inside nested array items does not correctly match different branches for - * different array elements, so the schema uses a single merged object. - */ -type UserMessageContentPart = { - type: "text" | "image_url" | "input_audio" | "file"; - text?: string; - image_url?: { - url?: string; - detail?: "auto" | "low" | "high"; - }; - input_audio?: { - data?: string; - format?: "wav" | "mp3"; - }; - file?: { - file_data?: string; - file_id?: string; - filename?: string; - }; -}; -type UserMessage = { - role: "user"; - content: string | Array; - name?: string; -}; -type AssistantMessageContentPart = { - type: "text" | "refusal"; - text?: string; - refusal?: string; -}; -type AssistantMessage = { - role: "assistant"; - content?: string | null | Array; - refusal?: string | null; - name?: string; - audio?: { - id: string; - }; - tool_calls?: Array; - function_call?: { - name: string; - arguments: string; - }; -}; -type ToolMessage = { - role: "tool"; - content: string | Array<{ - type: "text"; - text: string; - }>; - tool_call_id: string; -}; -type FunctionMessage = { - role: "function"; - content: string; - name: string; -}; -type ChatCompletionMessageParam = DeveloperMessage | SystemMessage | UserMessage | AssistantMessage | ToolMessage | FunctionMessage; -type ChatCompletionsResponseFormatText = { - type: "text"; -}; -type ChatCompletionsResponseFormatJSONObject = { - type: "json_object"; -}; -type ResponseFormatJSONSchema = { - type: "json_schema"; - json_schema: { - name: string; - description?: string; - schema?: Record; - strict?: boolean | null; - }; -}; -type ResponseFormat = ChatCompletionsResponseFormatText | ChatCompletionsResponseFormatJSONObject | ResponseFormatJSONSchema; -type ChatCompletionsStreamOptions = { - include_usage?: boolean; - include_obfuscation?: boolean; -}; -type PredictionContent = { - type: "content"; - content: string | Array<{ - type: "text"; - text: string; - }>; -}; -type AudioParams = { - voice: string | { - id: string; - }; - format: "wav" | "aac" | "mp3" | "flac" | "opus" | "pcm16"; -}; -type WebSearchUserLocation = { - type: "approximate"; - approximate: { - city?: string; - country?: string; - region?: string; - timezone?: string; - }; -}; -type WebSearchOptions = { - search_context_size?: "low" | "medium" | "high"; - user_location?: WebSearchUserLocation; -}; -type ChatTemplateKwargs = { - /** Whether to enable reasoning, enabled by default. */ - enable_thinking?: boolean; - /** If false, preserves reasoning context between turns. */ - clear_thinking?: boolean; -}; -/** Shared optional properties used by both Prompt and Messages input branches. */ -type ChatCompletionsCommonOptions = { - model?: string; - audio?: AudioParams; - frequency_penalty?: number | null; - logit_bias?: Record | null; - logprobs?: boolean | null; - top_logprobs?: number | null; - max_tokens?: number | null; - max_completion_tokens?: number | null; - metadata?: Record | null; - modalities?: Array<"text" | "audio"> | null; - n?: number | null; - parallel_tool_calls?: boolean; - prediction?: PredictionContent; - presence_penalty?: number | null; - reasoning_effort?: "low" | "medium" | "high" | null; - chat_template_kwargs?: ChatTemplateKwargs; - response_format?: ResponseFormat; - seed?: number | null; - service_tier?: "auto" | "default" | "flex" | "scale" | "priority" | null; - stop?: string | Array | null; - store?: boolean | null; - stream?: boolean | null; - stream_options?: ChatCompletionsStreamOptions; - temperature?: number | null; - tool_choice?: ChatCompletionToolChoiceOption; - tools?: Array; - top_p?: number | null; - user?: string; - web_search_options?: WebSearchOptions; - function_call?: "none" | "auto" | { - name: string; - }; - functions?: Array; -}; -type PromptTokensDetails = { - cached_tokens?: number; - audio_tokens?: number; -}; -type CompletionTokensDetails = { - reasoning_tokens?: number; - audio_tokens?: number; - accepted_prediction_tokens?: number; - rejected_prediction_tokens?: number; -}; -type CompletionUsage = { - prompt_tokens: number; - completion_tokens: number; - total_tokens: number; - prompt_tokens_details?: PromptTokensDetails; - completion_tokens_details?: CompletionTokensDetails; -}; -type ChatCompletionTopLogprob = { - token: string; - logprob: number; - bytes: Array | null; -}; -type ChatCompletionTokenLogprob = { - token: string; - logprob: number; - bytes: Array | null; - top_logprobs: Array; -}; -type ChatCompletionAudio = { - id: string; - /** Base64 encoded audio bytes. */ - data: string; - expires_at: number; - transcript: string; -}; -type ChatCompletionUrlCitation = { - type: "url_citation"; - url_citation: { - url: string; - title: string; - start_index: number; - end_index: number; - }; -}; -type ChatCompletionResponseMessage = { - role: "assistant"; - content: string | null; - refusal: string | null; - annotations?: Array; - audio?: ChatCompletionAudio; - tool_calls?: Array; - function_call?: { - name: string; - arguments: string; - } | null; -}; -type ChatCompletionLogprobs = { - content: Array | null; - refusal?: Array | null; -}; -type ChatCompletionChoice = { - index: number; - message: ChatCompletionResponseMessage; - finish_reason: "stop" | "length" | "tool_calls" | "content_filter" | "function_call"; - logprobs: ChatCompletionLogprobs | null; -}; -type ChatCompletionsPromptInput = { - prompt: string; -} & ChatCompletionsCommonOptions; -type ChatCompletionsMessagesInput = { - messages: Array; -} & ChatCompletionsCommonOptions; -type ChatCompletionsOutput = { - id: string; - object: string; - created: number; - model: string; - choices: Array; - usage?: CompletionUsage; - system_fingerprint?: string | null; - service_tier?: "auto" | "default" | "flex" | "scale" | "priority" | null; -}; -/** - * Workers AI support for OpenAI's Responses API - * Reference: https://github.com/openai/openai-node/blob/master/src/resources/responses/responses.ts - * - * It's a stripped down version from its source. - * It currently supports basic function calling, json mode and accepts images as input. - * - * It does not include types for WebSearch, CodeInterpreter, FileInputs, MCP, CustomTools. - * We plan to add those incrementally as model + platform capabilities evolve. - */ -type ResponsesInput = { - background?: boolean | null; - conversation?: string | ResponseConversationParam | null; - include?: Array | null; - input?: string | ResponseInput; - instructions?: string | null; - max_output_tokens?: number | null; - parallel_tool_calls?: boolean | null; - previous_response_id?: string | null; - prompt_cache_key?: string; - reasoning?: Reasoning | null; - safety_identifier?: string; - service_tier?: "auto" | "default" | "flex" | "scale" | "priority" | null; - stream?: boolean | null; - stream_options?: StreamOptions | null; - temperature?: number | null; - text?: ResponseTextConfig; - tool_choice?: ToolChoiceOptions | ToolChoiceFunction; - tools?: Array; - top_p?: number | null; - truncation?: "auto" | "disabled" | null; -}; -type ResponsesOutput = { - id?: string; - created_at?: number; - output_text?: string; - error?: ResponseError | null; - incomplete_details?: ResponseIncompleteDetails | null; - instructions?: string | Array | null; - object?: "response"; - output?: Array; - parallel_tool_calls?: boolean; - temperature?: number | null; - tool_choice?: ToolChoiceOptions | ToolChoiceFunction; - tools?: Array; - top_p?: number | null; - max_output_tokens?: number | null; - previous_response_id?: string | null; - prompt?: ResponsePrompt | null; - reasoning?: Reasoning | null; - safety_identifier?: string; - service_tier?: "auto" | "default" | "flex" | "scale" | "priority" | null; - status?: ResponseStatus; - text?: ResponseTextConfig; - truncation?: "auto" | "disabled" | null; - usage?: ResponseUsage; -}; -type EasyInputMessage = { - content: string | ResponseInputMessageContentList; - role: "user" | "assistant" | "system" | "developer"; - type?: "message"; -}; -type ResponsesFunctionTool = { - name: string; - parameters: { - [key: string]: unknown; - } | null; - strict: boolean | null; - type: "function"; - description?: string | null; -}; -type ResponseIncompleteDetails = { - reason?: "max_output_tokens" | "content_filter"; -}; -type ResponsePrompt = { - id: string; - variables?: { - [key: string]: string | ResponseInputText | ResponseInputImage; - } | null; - version?: string | null; -}; -type Reasoning = { - effort?: ReasoningEffort | null; - generate_summary?: "auto" | "concise" | "detailed" | null; - summary?: "auto" | "concise" | "detailed" | null; -}; -type ResponseContent = ResponseInputText | ResponseInputImage | ResponseOutputText | ResponseOutputRefusal | ResponseContentReasoningText; -type ResponseContentReasoningText = { - text: string; - type: "reasoning_text"; -}; -type ResponseConversationParam = { - id: string; -}; -type ResponseCreatedEvent = { - response: Response; - sequence_number: number; - type: "response.created"; -}; -type ResponseCustomToolCallOutput = { - call_id: string; - output: string | Array; - type: "custom_tool_call_output"; - id?: string; -}; -type ResponseError = { - code: "server_error" | "rate_limit_exceeded" | "invalid_prompt" | "vector_store_timeout" | "invalid_image" | "invalid_image_format" | "invalid_base64_image" | "invalid_image_url" | "image_too_large" | "image_too_small" | "image_parse_error" | "image_content_policy_violation" | "invalid_image_mode" | "image_file_too_large" | "unsupported_image_media_type" | "empty_image_file" | "failed_to_download_image" | "image_file_not_found"; - message: string; -}; -type ResponseErrorEvent = { - code: string | null; - message: string; - param: string | null; - sequence_number: number; - type: "error"; -}; -type ResponseFailedEvent = { - response: Response; - sequence_number: number; - type: "response.failed"; -}; -type ResponseFormatText = { - type: "text"; -}; -type ResponseFormatJSONObject = { - type: "json_object"; -}; -type ResponseFormatTextConfig = ResponseFormatText | ResponseFormatTextJSONSchemaConfig | ResponseFormatJSONObject; -type ResponseFormatTextJSONSchemaConfig = { - name: string; - schema: { - [key: string]: unknown; - }; - type: "json_schema"; - description?: string; - strict?: boolean | null; -}; -type ResponseFunctionCallArgumentsDeltaEvent = { - delta: string; - item_id: string; - output_index: number; - sequence_number: number; - type: "response.function_call_arguments.delta"; -}; -type ResponseFunctionCallArgumentsDoneEvent = { - arguments: string; - item_id: string; - name: string; - output_index: number; - sequence_number: number; - type: "response.function_call_arguments.done"; -}; -type ResponseFunctionCallOutputItem = ResponseInputTextContent | ResponseInputImageContent; -type ResponseFunctionCallOutputItemList = Array; -type ResponseFunctionToolCall = { - arguments: string; - call_id: string; - name: string; - type: "function_call"; - id?: string; - status?: "in_progress" | "completed" | "incomplete"; -}; -interface ResponseFunctionToolCallItem extends ResponseFunctionToolCall { - id: string; -} -type ResponseFunctionToolCallOutputItem = { - id: string; - call_id: string; - output: string | Array; - type: "function_call_output"; - status?: "in_progress" | "completed" | "incomplete"; -}; -type ResponseIncludable = "message.input_image.image_url" | "message.output_text.logprobs"; -type ResponseIncompleteEvent = { - response: Response; - sequence_number: number; - type: "response.incomplete"; -}; -type ResponseInput = Array; -type ResponseInputContent = ResponseInputText | ResponseInputImage; -type ResponseInputImage = { - detail: "low" | "high" | "auto"; - type: "input_image"; - /** - * Base64 encoded image - */ - image_url?: string | null; -}; -type ResponseInputImageContent = { - type: "input_image"; - detail?: "low" | "high" | "auto" | null; - /** - * Base64 encoded image - */ - image_url?: string | null; -}; -type ResponseInputItem = EasyInputMessage | ResponseInputItemMessage | ResponseOutputMessage | ResponseFunctionToolCall | ResponseInputItemFunctionCallOutput | ResponseReasoningItem; -type ResponseInputItemFunctionCallOutput = { - call_id: string; - output: string | ResponseFunctionCallOutputItemList; - type: "function_call_output"; - id?: string | null; - status?: "in_progress" | "completed" | "incomplete" | null; -}; -type ResponseInputItemMessage = { - content: ResponseInputMessageContentList; - role: "user" | "system" | "developer"; - status?: "in_progress" | "completed" | "incomplete"; - type?: "message"; -}; -type ResponseInputMessageContentList = Array; -type ResponseInputMessageItem = { - id: string; - content: ResponseInputMessageContentList; - role: "user" | "system" | "developer"; - status?: "in_progress" | "completed" | "incomplete"; - type?: "message"; -}; -type ResponseInputText = { - text: string; - type: "input_text"; -}; -type ResponseInputTextContent = { - text: string; - type: "input_text"; -}; -type ResponseItem = ResponseInputMessageItem | ResponseOutputMessage | ResponseFunctionToolCallItem | ResponseFunctionToolCallOutputItem; -type ResponseOutputItem = ResponseOutputMessage | ResponseFunctionToolCall | ResponseReasoningItem; -type ResponseOutputItemAddedEvent = { - item: ResponseOutputItem; - output_index: number; - sequence_number: number; - type: "response.output_item.added"; -}; -type ResponseOutputItemDoneEvent = { - item: ResponseOutputItem; - output_index: number; - sequence_number: number; - type: "response.output_item.done"; -}; -type ResponseOutputMessage = { - id: string; - content: Array; - role: "assistant"; - status: "in_progress" | "completed" | "incomplete"; - type: "message"; -}; -type ResponseOutputRefusal = { - refusal: string; - type: "refusal"; -}; -type ResponseOutputText = { - text: string; - type: "output_text"; - logprobs?: Array; -}; -type ResponseReasoningItem = { - id: string; - summary: Array; - type: "reasoning"; - content?: Array; - encrypted_content?: string | null; - status?: "in_progress" | "completed" | "incomplete"; -}; -type ResponseReasoningSummaryItem = { - text: string; - type: "summary_text"; -}; -type ResponseReasoningContentItem = { - text: string; - type: "reasoning_text"; -}; -type ResponseReasoningTextDeltaEvent = { - content_index: number; - delta: string; - item_id: string; - output_index: number; - sequence_number: number; - type: "response.reasoning_text.delta"; -}; -type ResponseReasoningTextDoneEvent = { - content_index: number; - item_id: string; - output_index: number; - sequence_number: number; - text: string; - type: "response.reasoning_text.done"; -}; -type ResponseRefusalDeltaEvent = { - content_index: number; - delta: string; - item_id: string; - output_index: number; - sequence_number: number; - type: "response.refusal.delta"; -}; -type ResponseRefusalDoneEvent = { - content_index: number; - item_id: string; - output_index: number; - refusal: string; - sequence_number: number; - type: "response.refusal.done"; -}; -type ResponseStatus = "completed" | "failed" | "in_progress" | "cancelled" | "queued" | "incomplete"; -type ResponseStreamEvent = ResponseCompletedEvent | ResponseCreatedEvent | ResponseErrorEvent | ResponseFunctionCallArgumentsDeltaEvent | ResponseFunctionCallArgumentsDoneEvent | ResponseFailedEvent | ResponseIncompleteEvent | ResponseOutputItemAddedEvent | ResponseOutputItemDoneEvent | ResponseReasoningTextDeltaEvent | ResponseReasoningTextDoneEvent | ResponseRefusalDeltaEvent | ResponseRefusalDoneEvent | ResponseTextDeltaEvent | ResponseTextDoneEvent; -type ResponseCompletedEvent = { - response: Response; - sequence_number: number; - type: "response.completed"; -}; -type ResponseTextConfig = { - format?: ResponseFormatTextConfig; - verbosity?: "low" | "medium" | "high" | null; -}; -type ResponseTextDeltaEvent = { - content_index: number; - delta: string; - item_id: string; - logprobs: Array; - output_index: number; - sequence_number: number; - type: "response.output_text.delta"; -}; -type ResponseTextDoneEvent = { - content_index: number; - item_id: string; - logprobs: Array; - output_index: number; - sequence_number: number; - text: string; - type: "response.output_text.done"; -}; -type Logprob = { - token: string; - logprob: number; - top_logprobs?: Array; -}; -type TopLogprob = { - token?: string; - logprob?: number; -}; -type ResponseUsage = { - input_tokens: number; - output_tokens: number; - total_tokens: number; -}; -type Tool = ResponsesFunctionTool; -type ToolChoiceFunction = { - name: string; - type: "function"; -}; -type ToolChoiceOptions = "none"; -type ReasoningEffort = "minimal" | "low" | "medium" | "high" | null; -type StreamOptions = { - include_obfuscation?: boolean; -}; -/** Marks keys from T that aren't in U as optional never */ -type Without = { - [P in Exclude]?: never; -}; -/** Either T or U, but not both (mutually exclusive) */ -type XOR = (T & Without) | (U & Without); -type Ai_Cf_Baai_Bge_Base_En_V1_5_Input = { - text: string | string[]; - /** - * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. - */ - pooling?: "mean" | "cls"; -} | { - /** - * Batch of the embeddings requests to run using async-queue - */ - requests: { - text: string | string[]; - /** - * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. - */ - pooling?: "mean" | "cls"; - }[]; -}; -type Ai_Cf_Baai_Bge_Base_En_V1_5_Output = { - shape?: number[]; - /** - * Embeddings of the requested text values - */ - data?: number[][]; - /** - * The pooling method used in the embedding process. - */ - pooling?: "mean" | "cls"; -} | Ai_Cf_Baai_Bge_Base_En_V1_5_AsyncResponse; -interface Ai_Cf_Baai_Bge_Base_En_V1_5_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Baai_Bge_Base_En_V1_5 { - inputs: Ai_Cf_Baai_Bge_Base_En_V1_5_Input; - postProcessedOutputs: Ai_Cf_Baai_Bge_Base_En_V1_5_Output; -} -type Ai_Cf_Openai_Whisper_Input = string | { - /** - * An array of integers that represent the audio data constrained to 8-bit unsigned integer values - */ - audio: number[]; -}; -interface Ai_Cf_Openai_Whisper_Output { - /** - * The transcription - */ - text: string; - word_count?: number; - words?: { - word?: string; - /** - * The second this word begins in the recording - */ - start?: number; - /** - * The ending second when the word completes - */ - end?: number; - }[]; - vtt?: string; -} -declare abstract class Base_Ai_Cf_Openai_Whisper { - inputs: Ai_Cf_Openai_Whisper_Input; - postProcessedOutputs: Ai_Cf_Openai_Whisper_Output; -} -type Ai_Cf_Meta_M2M100_1_2B_Input = { - /** - * The text to be translated - */ - text: string; - /** - * The language code of the source text (e.g., 'en' for English). Defaults to 'en' if not specified - */ - source_lang?: string; - /** - * The language code to translate the text into (e.g., 'es' for Spanish) - */ - target_lang: string; -} | { - /** - * Batch of the embeddings requests to run using async-queue - */ - requests: { - /** - * The text to be translated - */ - text: string; - /** - * The language code of the source text (e.g., 'en' for English). Defaults to 'en' if not specified - */ - source_lang?: string; - /** - * The language code to translate the text into (e.g., 'es' for Spanish) - */ - target_lang: string; - }[]; -}; -type Ai_Cf_Meta_M2M100_1_2B_Output = { - /** - * The translated text in the target language - */ - translated_text?: string; -} | Ai_Cf_Meta_M2M100_1_2B_AsyncResponse; -interface Ai_Cf_Meta_M2M100_1_2B_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Meta_M2M100_1_2B { - inputs: Ai_Cf_Meta_M2M100_1_2B_Input; - postProcessedOutputs: Ai_Cf_Meta_M2M100_1_2B_Output; -} -type Ai_Cf_Baai_Bge_Small_En_V1_5_Input = { - text: string | string[]; - /** - * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. - */ - pooling?: "mean" | "cls"; -} | { - /** - * Batch of the embeddings requests to run using async-queue - */ - requests: { - text: string | string[]; - /** - * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. - */ - pooling?: "mean" | "cls"; - }[]; -}; -type Ai_Cf_Baai_Bge_Small_En_V1_5_Output = { - shape?: number[]; - /** - * Embeddings of the requested text values - */ - data?: number[][]; - /** - * The pooling method used in the embedding process. - */ - pooling?: "mean" | "cls"; -} | Ai_Cf_Baai_Bge_Small_En_V1_5_AsyncResponse; -interface Ai_Cf_Baai_Bge_Small_En_V1_5_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Baai_Bge_Small_En_V1_5 { - inputs: Ai_Cf_Baai_Bge_Small_En_V1_5_Input; - postProcessedOutputs: Ai_Cf_Baai_Bge_Small_En_V1_5_Output; -} -type Ai_Cf_Baai_Bge_Large_En_V1_5_Input = { - text: string | string[]; - /** - * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. - */ - pooling?: "mean" | "cls"; -} | { - /** - * Batch of the embeddings requests to run using async-queue - */ - requests: { - text: string | string[]; - /** - * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. - */ - pooling?: "mean" | "cls"; - }[]; -}; -type Ai_Cf_Baai_Bge_Large_En_V1_5_Output = { - shape?: number[]; - /** - * Embeddings of the requested text values - */ - data?: number[][]; - /** - * The pooling method used in the embedding process. - */ - pooling?: "mean" | "cls"; -} | Ai_Cf_Baai_Bge_Large_En_V1_5_AsyncResponse; -interface Ai_Cf_Baai_Bge_Large_En_V1_5_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Baai_Bge_Large_En_V1_5 { - inputs: Ai_Cf_Baai_Bge_Large_En_V1_5_Input; - postProcessedOutputs: Ai_Cf_Baai_Bge_Large_En_V1_5_Output; -} -type Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Input = string | { - /** - * The input text prompt for the model to generate a response. - */ - prompt?: string; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * Controls the creativity of the AI's responses by adjusting how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; - image: number[] | (string & NonNullable); - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; -}; -interface Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Output { - description?: string; -} -declare abstract class Base_Ai_Cf_Unum_Uform_Gen2_Qwen_500M { - inputs: Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Input; - postProcessedOutputs: Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Output; -} -type Ai_Cf_Openai_Whisper_Tiny_En_Input = string | { - /** - * An array of integers that represent the audio data constrained to 8-bit unsigned integer values - */ - audio: number[]; -}; -interface Ai_Cf_Openai_Whisper_Tiny_En_Output { - /** - * The transcription - */ - text: string; - word_count?: number; - words?: { - word?: string; - /** - * The second this word begins in the recording - */ - start?: number; - /** - * The ending second when the word completes - */ - end?: number; - }[]; - vtt?: string; -} -declare abstract class Base_Ai_Cf_Openai_Whisper_Tiny_En { - inputs: Ai_Cf_Openai_Whisper_Tiny_En_Input; - postProcessedOutputs: Ai_Cf_Openai_Whisper_Tiny_En_Output; -} -interface Ai_Cf_Openai_Whisper_Large_V3_Turbo_Input { - audio: string | { - body?: object; - contentType?: string; - }; - /** - * Supported tasks are 'translate' or 'transcribe'. - */ - task?: string; - /** - * The language of the audio being transcribed or translated. - */ - language?: string; - /** - * Preprocess the audio with a voice activity detection model. - */ - vad_filter?: boolean; - /** - * A text prompt to help provide context to the model on the contents of the audio. - */ - initial_prompt?: string; - /** - * The prefix appended to the beginning of the output of the transcription and can guide the transcription result. - */ - prefix?: string; - /** - * The number of beams to use in beam search decoding. Higher values may improve accuracy at the cost of speed. - */ - beam_size?: number; - /** - * Whether to condition on previous text during transcription. Setting to false may help prevent hallucination loops. - */ - condition_on_previous_text?: boolean; - /** - * Threshold for detecting no-speech segments. Segments with no-speech probability above this value are skipped. - */ - no_speech_threshold?: number; - /** - * Threshold for filtering out segments with high compression ratio, which often indicate repetitive or hallucinated text. - */ - compression_ratio_threshold?: number; - /** - * Threshold for filtering out segments with low average log probability, indicating low confidence. - */ - log_prob_threshold?: number; - /** - * Optional threshold (in seconds) to skip silent periods that may cause hallucinations. - */ - hallucination_silence_threshold?: number; -} -interface Ai_Cf_Openai_Whisper_Large_V3_Turbo_Output { - transcription_info?: { - /** - * The language of the audio being transcribed or translated. - */ - language?: string; - /** - * The confidence level or probability of the detected language being accurate, represented as a decimal between 0 and 1. - */ - language_probability?: number; - /** - * The total duration of the original audio file, in seconds. - */ - duration?: number; - /** - * The duration of the audio after applying Voice Activity Detection (VAD) to remove silent or irrelevant sections, in seconds. - */ - duration_after_vad?: number; - }; - /** - * The complete transcription of the audio. - */ - text: string; - /** - * The total number of words in the transcription. - */ - word_count?: number; - segments?: { - /** - * The starting time of the segment within the audio, in seconds. - */ - start?: number; - /** - * The ending time of the segment within the audio, in seconds. - */ - end?: number; - /** - * The transcription of the segment. - */ - text?: string; - /** - * The temperature used in the decoding process, controlling randomness in predictions. Lower values result in more deterministic outputs. - */ - temperature?: number; - /** - * The average log probability of the predictions for the words in this segment, indicating overall confidence. - */ - avg_logprob?: number; - /** - * The compression ratio of the input to the output, measuring how much the text was compressed during the transcription process. - */ - compression_ratio?: number; - /** - * The probability that the segment contains no speech, represented as a decimal between 0 and 1. - */ - no_speech_prob?: number; - words?: { - /** - * The individual word transcribed from the audio. - */ - word?: string; - /** - * The starting time of the word within the audio, in seconds. - */ - start?: number; - /** - * The ending time of the word within the audio, in seconds. - */ - end?: number; - }[]; - }[]; - /** - * The transcription in WebVTT format, which includes timing and text information for use in subtitles. - */ - vtt?: string; -} -declare abstract class Base_Ai_Cf_Openai_Whisper_Large_V3_Turbo { - inputs: Ai_Cf_Openai_Whisper_Large_V3_Turbo_Input; - postProcessedOutputs: Ai_Cf_Openai_Whisper_Large_V3_Turbo_Output; -} -type Ai_Cf_Baai_Bge_M3_Input = Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts | Ai_Cf_Baai_Bge_M3_Input_Embedding | { - /** - * Batch of the embeddings requests to run using async-queue - */ - requests: (Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts_1 | Ai_Cf_Baai_Bge_M3_Input_Embedding_1)[]; -}; -interface Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts { - /** - * A query you wish to perform against the provided contexts. If no query is provided the model with respond with embeddings for contexts - */ - query?: string; - /** - * List of provided contexts. Note that the index in this array is important, as the response will refer to it. - */ - contexts: { - /** - * One of the provided context content - */ - text?: string; - }[]; - /** - * When provided with too long context should the model error out or truncate the context to fit? - */ - truncate_inputs?: boolean; -} -interface Ai_Cf_Baai_Bge_M3_Input_Embedding { - text: string | string[]; - /** - * When provided with too long context should the model error out or truncate the context to fit? - */ - truncate_inputs?: boolean; -} -interface Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts_1 { - /** - * A query you wish to perform against the provided contexts. If no query is provided the model with respond with embeddings for contexts - */ - query?: string; - /** - * List of provided contexts. Note that the index in this array is important, as the response will refer to it. - */ - contexts: { - /** - * One of the provided context content - */ - text?: string; - }[]; - /** - * When provided with too long context should the model error out or truncate the context to fit? - */ - truncate_inputs?: boolean; -} -interface Ai_Cf_Baai_Bge_M3_Input_Embedding_1 { - text: string | string[]; - /** - * When provided with too long context should the model error out or truncate the context to fit? - */ - truncate_inputs?: boolean; -} -type Ai_Cf_Baai_Bge_M3_Output = Ai_Cf_Baai_Bge_M3_Output_Query | Ai_Cf_Baai_Bge_M3_Output_EmbeddingFor_Contexts | Ai_Cf_Baai_Bge_M3_Output_Embedding | Ai_Cf_Baai_Bge_M3_AsyncResponse; -interface Ai_Cf_Baai_Bge_M3_Output_Query { - response?: { - /** - * Index of the context in the request - */ - id?: number; - /** - * Score of the context under the index. - */ - score?: number; - }[]; -} -interface Ai_Cf_Baai_Bge_M3_Output_EmbeddingFor_Contexts { - response?: number[][]; - shape?: number[]; - /** - * The pooling method used in the embedding process. - */ - pooling?: "mean" | "cls"; -} -interface Ai_Cf_Baai_Bge_M3_Output_Embedding { - shape?: number[]; - /** - * Embeddings of the requested text values - */ - data?: number[][]; - /** - * The pooling method used in the embedding process. - */ - pooling?: "mean" | "cls"; -} -interface Ai_Cf_Baai_Bge_M3_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Baai_Bge_M3 { - inputs: Ai_Cf_Baai_Bge_M3_Input; - postProcessedOutputs: Ai_Cf_Baai_Bge_M3_Output; -} -interface Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Input { - /** - * A text description of the image you want to generate. - */ - prompt: string; - /** - * The number of diffusion steps; higher values can improve quality but take longer. - */ - steps?: number; -} -interface Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Output { - /** - * The generated image in Base64 format. - */ - image?: string; -} -declare abstract class Base_Ai_Cf_Black_Forest_Labs_Flux_1_Schnell { - inputs: Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Input; - postProcessedOutputs: Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Output; -} -type Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Input = Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Prompt | Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Messages; -interface Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - image?: number[] | (string & NonNullable); - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; - /** - * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. - */ - lora?: string; -} -interface Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role?: string; - /** - * The tool call id. If you don't know what to put here you can fall back to 000000001 - */ - tool_call_id?: string; - content?: string | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }[] | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }; - }[]; - image?: number[] | (string & NonNullable); - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - /** - * If true, the response will be streamed back incrementally. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Controls the creativity of the AI's responses by adjusting how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -type Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Output = { - /** - * The generated text response from the model - */ - response?: string; - /** - * An array of tool calls requests made during the response generation - */ - tool_calls?: { - /** - * The arguments passed to be passed to the tool call request - */ - arguments?: object; - /** - * The name of the tool to be called - */ - name?: string; - }[]; -}; -declare abstract class Base_Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct { - inputs: Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Input; - postProcessedOutputs: Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Output; -} -type Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Input = Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Prompt | Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Messages | Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Async_Batch; -interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. - */ - lora?: string; - response_format?: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role: string; - content: string | { - /** - * Type of the content (text) - */ - type?: string; - /** - * Text content - */ - text?: string; - }[]; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_1; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_1 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Async_Batch { - requests?: { - /** - * User-supplied reference. This field will be present in the response as well it can be used to reference the request and response. It's NOT validated to be unique. - */ - external_reference?: string; - /** - * Prompt for the text generation model - */ - prompt?: string; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; - response_format?: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_2; - }[]; -} -interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_2 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -type Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Output = { - /** - * The generated text response from the model - */ - response: string; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * An array of tool calls requests made during the response generation - */ - tool_calls?: { - /** - * The arguments passed to be passed to the tool call request - */ - arguments?: object; - /** - * The name of the tool to be called - */ - name?: string; - }[]; -} | string | Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_AsyncResponse; -interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast { - inputs: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Input; - postProcessedOutputs: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Output; -} -interface Ai_Cf_Meta_Llama_Guard_3_8B_Input { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender must alternate between 'user' and 'assistant'. - */ - role: "user" | "assistant"; - /** - * The content of the message as a string. - */ - content: string; - }[]; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Dictate the output format of the generated response. - */ - response_format?: { - /** - * Set to json_object to process and output generated text as JSON. - */ - type?: string; - }; -} -interface Ai_Cf_Meta_Llama_Guard_3_8B_Output { - response?: string | { - /** - * Whether the conversation is safe or not. - */ - safe?: boolean; - /** - * A list of what hazard categories predicted for the conversation, if the conversation is deemed unsafe. - */ - categories?: string[]; - }; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; -} -declare abstract class Base_Ai_Cf_Meta_Llama_Guard_3_8B { - inputs: Ai_Cf_Meta_Llama_Guard_3_8B_Input; - postProcessedOutputs: Ai_Cf_Meta_Llama_Guard_3_8B_Output; -} -interface Ai_Cf_Baai_Bge_Reranker_Base_Input { - /** - * A query you wish to perform against the provided contexts. - */ - /** - * Number of returned results starting with the best score. - */ - top_k?: number; - /** - * List of provided contexts. Note that the index in this array is important, as the response will refer to it. - */ - contexts: { - /** - * One of the provided context content - */ - text?: string; - }[]; -} -interface Ai_Cf_Baai_Bge_Reranker_Base_Output { - response?: { - /** - * Index of the context in the request - */ - id?: number; - /** - * Score of the context under the index. - */ - score?: number; - }[]; -} -declare abstract class Base_Ai_Cf_Baai_Bge_Reranker_Base { - inputs: Ai_Cf_Baai_Bge_Reranker_Base_Input; - postProcessedOutputs: Ai_Cf_Baai_Bge_Reranker_Base_Output; -} -type Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Input = Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Prompt | Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Messages; -interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. - */ - lora?: string; - response_format?: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role: string; - /** - * The content of the message as a string. - */ - content: string; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode_1; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode_1 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -type Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Output = { - /** - * The generated text response from the model - */ - response: string; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * An array of tool calls requests made during the response generation - */ - tool_calls?: { - /** - * The arguments passed to be passed to the tool call request - */ - arguments?: object; - /** - * The name of the tool to be called - */ - name?: string; - }[]; -}; -declare abstract class Base_Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct { - inputs: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Input; - postProcessedOutputs: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Output; -} -type Ai_Cf_Qwen_Qwq_32B_Input = Ai_Cf_Qwen_Qwq_32B_Prompt | Ai_Cf_Qwen_Qwq_32B_Messages; -interface Ai_Cf_Qwen_Qwq_32B_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * JSON schema that should be fulfilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Qwen_Qwq_32B_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role?: string; - /** - * The tool call id. If you don't know what to put here you can fall back to 000000001 - */ - tool_call_id?: string; - content?: string | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }[] | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - /** - * JSON schema that should be fufilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -type Ai_Cf_Qwen_Qwq_32B_Output = { - /** - * The generated text response from the model - */ - response: string; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * An array of tool calls requests made during the response generation - */ - tool_calls?: { - /** - * The arguments passed to be passed to the tool call request - */ - arguments?: object; - /** - * The name of the tool to be called - */ - name?: string; - }[]; -}; -declare abstract class Base_Ai_Cf_Qwen_Qwq_32B { - inputs: Ai_Cf_Qwen_Qwq_32B_Input; - postProcessedOutputs: Ai_Cf_Qwen_Qwq_32B_Output; -} -type Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Input = Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Prompt | Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Messages; -interface Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * JSON schema that should be fulfilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role?: string; - /** - * The tool call id. Must be supplied for tool calls for Mistral-3. If you don't know what to put here you can fall back to 000000001 - */ - tool_call_id?: string; - content?: string | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }[] | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - /** - * JSON schema that should be fufilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -type Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Output = { - /** - * The generated text response from the model - */ - response: string; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * An array of tool calls requests made during the response generation - */ - tool_calls?: { - /** - * The arguments passed to be passed to the tool call request - */ - arguments?: object; - /** - * The name of the tool to be called - */ - name?: string; - }[]; -}; -declare abstract class Base_Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct { - inputs: Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Input; - postProcessedOutputs: Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Output; -} -type Ai_Cf_Google_Gemma_3_12B_It_Input = Ai_Cf_Google_Gemma_3_12B_It_Prompt | Ai_Cf_Google_Gemma_3_12B_It_Messages; -interface Ai_Cf_Google_Gemma_3_12B_It_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * JSON schema that should be fufilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Google_Gemma_3_12B_It_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role?: string; - content?: string | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }[]; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - /** - * JSON schema that should be fufilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -type Ai_Cf_Google_Gemma_3_12B_It_Output = { - /** - * The generated text response from the model - */ - response: string; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * An array of tool calls requests made during the response generation - */ - tool_calls?: { - /** - * The arguments passed to be passed to the tool call request - */ - arguments?: object; - /** - * The name of the tool to be called - */ - name?: string; - }[]; -}; -declare abstract class Base_Ai_Cf_Google_Gemma_3_12B_It { - inputs: Ai_Cf_Google_Gemma_3_12B_It_Input; - postProcessedOutputs: Ai_Cf_Google_Gemma_3_12B_It_Output; -} -type Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Input = Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt | Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages | Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Async_Batch; -interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * JSON schema that should be fulfilled for the response. - */ - guided_json?: object; - response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role?: string; - /** - * The tool call id. If you don't know what to put here you can fall back to 000000001 - */ - tool_call_id?: string; - content?: string | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }[] | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; - /** - * JSON schema that should be fufilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Async_Batch { - requests: (Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt_Inner | Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages_Inner)[]; -} -interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt_Inner { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * JSON schema that should be fulfilled for the response. - */ - guided_json?: object; - response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages_Inner { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role?: string; - /** - * The tool call id. If you don't know what to put here you can fall back to 000000001 - */ - tool_call_id?: string; - content?: string | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }[] | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; - /** - * JSON schema that should be fufilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -type Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Output = { - /** - * The generated text response from the model - */ - response: string; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * An array of tool calls requests made during the response generation - */ - tool_calls?: { - /** - * The tool call id. - */ - id?: string; - /** - * Specifies the type of tool (e.g., 'function'). - */ - type?: string; - /** - * Details of the function tool. - */ - function?: { - /** - * The name of the tool to be called - */ - name?: string; - /** - * The arguments passed to be passed to the tool call request - */ - arguments?: object; - }; - }[]; -}; -declare abstract class Base_Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct { - inputs: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Input; - postProcessedOutputs: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Output; -} -type Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Input = Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Async_Batch; -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. - */ - lora?: string; - response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role: string; - content: string | { - /** - * Type of the content (text) - */ - type?: string; - /** - * Text content - */ - text?: string; - }[]; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_1; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_1 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Async_Batch { - requests: (Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt_1 | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages_1)[]; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt_1 { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. - */ - lora?: string; - response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_2; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_2 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages_1 { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role: string; - content: string | { - /** - * Type of the content (text) - */ - type?: string; - /** - * Text content - */ - text?: string; - }[]; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_3; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_3 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -type Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Output = Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Chat_Completion_Response | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Text_Completion_Response | string | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_AsyncResponse; -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Chat_Completion_Response { - /** - * Unique identifier for the completion - */ - id?: string; - /** - * Object type identifier - */ - object?: "chat.completion"; - /** - * Unix timestamp of when the completion was created - */ - created?: number; - /** - * Model used for the completion - */ - model?: string; - /** - * List of completion choices - */ - choices?: { - /** - * Index of the choice in the list - */ - index?: number; - /** - * The message generated by the model - */ - message?: { - /** - * Role of the message author - */ - role: string; - /** - * The content of the message - */ - content: string; - /** - * Internal reasoning content (if available) - */ - reasoning_content?: string; - /** - * Tool calls made by the assistant - */ - tool_calls?: { - /** - * Unique identifier for the tool call - */ - id: string; - /** - * Type of tool call - */ - type: "function"; - function: { - /** - * Name of the function to call - */ - name: string; - /** - * JSON string of arguments for the function - */ - arguments: string; - }; - }[]; - }; - /** - * Reason why the model stopped generating - */ - finish_reason?: string; - /** - * Stop reason (may be null) - */ - stop_reason?: string | null; - /** - * Log probabilities (if requested) - */ - logprobs?: {} | null; - }[]; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * Log probabilities for the prompt (if requested) - */ - prompt_logprobs?: {} | null; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Text_Completion_Response { - /** - * Unique identifier for the completion - */ - id?: string; - /** - * Object type identifier - */ - object?: "text_completion"; - /** - * Unix timestamp of when the completion was created - */ - created?: number; - /** - * Model used for the completion - */ - model?: string; - /** - * List of completion choices - */ - choices?: { - /** - * Index of the choice in the list - */ - index: number; - /** - * The generated text completion - */ - text: string; - /** - * Reason why the model stopped generating - */ - finish_reason: string; - /** - * Stop reason (may be null) - */ - stop_reason?: string | null; - /** - * Log probabilities (if requested) - */ - logprobs?: {} | null; - /** - * Log probabilities for the prompt (if requested) - */ - prompt_logprobs?: {} | null; - }[]; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8 { - inputs: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Input; - postProcessedOutputs: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Output; -} -interface Ai_Cf_Deepgram_Nova_3_Input { - audio: { - body: object; - contentType: string; - }; - /** - * Sets how the model will interpret strings submitted to the custom_topic param. When strict, the model will only return topics submitted using the custom_topic param. When extended, the model will return its own detected topics in addition to those submitted using the custom_topic param. - */ - custom_topic_mode?: "extended" | "strict"; - /** - * Custom topics you want the model to detect within your input audio or text if present Submit up to 100 - */ - custom_topic?: string; - /** - * Sets how the model will interpret intents submitted to the custom_intent param. When strict, the model will only return intents submitted using the custom_intent param. When extended, the model will return its own detected intents in addition those submitted using the custom_intents param - */ - custom_intent_mode?: "extended" | "strict"; - /** - * Custom intents you want the model to detect within your input audio if present - */ - custom_intent?: string; - /** - * Identifies and extracts key entities from content in submitted audio - */ - detect_entities?: boolean; - /** - * Identifies the dominant language spoken in submitted audio - */ - detect_language?: boolean; - /** - * Recognize speaker changes. Each word in the transcript will be assigned a speaker number starting at 0 - */ - diarize?: boolean; - /** - * Identify and extract key entities from content in submitted audio - */ - dictation?: boolean; - /** - * Specify the expected encoding of your submitted audio - */ - encoding?: "linear16" | "flac" | "mulaw" | "amr-nb" | "amr-wb" | "opus" | "speex" | "g729"; - /** - * Arbitrary key-value pairs that are attached to the API response for usage in downstream processing - */ - extra?: string; - /** - * Filler Words can help transcribe interruptions in your audio, like 'uh' and 'um' - */ - filler_words?: boolean; - /** - * Key term prompting can boost or suppress specialized terminology and brands. - */ - keyterm?: string; - /** - * Keywords can boost or suppress specialized terminology and brands. - */ - keywords?: string; - /** - * The BCP-47 language tag that hints at the primary spoken language. Depending on the Model and API endpoint you choose only certain languages are available. - */ - language?: string; - /** - * Spoken measurements will be converted to their corresponding abbreviations. - */ - measurements?: boolean; - /** - * Opts out requests from the Deepgram Model Improvement Program. Refer to our Docs for pricing impacts before setting this to true. https://dpgr.am/deepgram-mip. - */ - mip_opt_out?: boolean; - /** - * Mode of operation for the model representing broad area of topic that will be talked about in the supplied audio - */ - mode?: "general" | "medical" | "finance"; - /** - * Transcribe each audio channel independently. - */ - multichannel?: boolean; - /** - * Numerals converts numbers from written format to numerical format. - */ - numerals?: boolean; - /** - * Splits audio into paragraphs to improve transcript readability. - */ - paragraphs?: boolean; - /** - * Profanity Filter looks for recognized profanity and converts it to the nearest recognized non-profane word or removes it from the transcript completely. - */ - profanity_filter?: boolean; - /** - * Add punctuation and capitalization to the transcript. - */ - punctuate?: boolean; - /** - * Redaction removes sensitive information from your transcripts. - */ - redact?: string; - /** - * Search for terms or phrases in submitted audio and replaces them. - */ - replace?: string; - /** - * Search for terms or phrases in submitted audio. - */ - search?: string; - /** - * Recognizes the sentiment throughout a transcript or text. - */ - sentiment?: boolean; - /** - * Apply formatting to transcript output. When set to true, additional formatting will be applied to transcripts to improve readability. - */ - smart_format?: boolean; - /** - * Detect topics throughout a transcript or text. - */ - topics?: boolean; - /** - * Segments speech into meaningful semantic units. - */ - utterances?: boolean; - /** - * Seconds to wait before detecting a pause between words in submitted audio. - */ - utt_split?: number; - /** - * The number of channels in the submitted audio - */ - channels?: number; - /** - * Specifies whether the streaming endpoint should provide ongoing transcription updates as more audio is received. When set to true, the endpoint sends continuous updates, meaning transcription results may evolve over time. Note: Supported only for webosockets. - */ - interim_results?: boolean; - /** - * Indicates how long model will wait to detect whether a speaker has finished speaking or pauses for a significant period of time. When set to a value, the streaming endpoint immediately finalizes the transcription for the processed time range and returns the transcript with a speech_final parameter set to true. Can also be set to false to disable endpointing - */ - endpointing?: string; - /** - * Indicates that speech has started. You'll begin receiving Speech Started messages upon speech starting. Note: Supported only for webosockets. - */ - vad_events?: boolean; - /** - * Indicates how long model will wait to send an UtteranceEnd message after a word has been transcribed. Use with interim_results. Note: Supported only for webosockets. - */ - utterance_end_ms?: boolean; -} -interface Ai_Cf_Deepgram_Nova_3_Output { - results?: { - channels?: { - alternatives?: { - confidence?: number; - transcript?: string; - words?: { - confidence?: number; - end?: number; - start?: number; - word?: string; - }[]; - }[]; - }[]; - summary?: { - result?: string; - short?: string; - }; - sentiments?: { - segments?: { - text?: string; - start_word?: number; - end_word?: number; - sentiment?: string; - sentiment_score?: number; - }[]; - average?: { - sentiment?: string; - sentiment_score?: number; - }; - }; - }; -} -declare abstract class Base_Ai_Cf_Deepgram_Nova_3 { - inputs: Ai_Cf_Deepgram_Nova_3_Input; - postProcessedOutputs: Ai_Cf_Deepgram_Nova_3_Output; -} -interface Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Input { - queries?: string | string[]; - /** - * Optional instruction for the task - */ - instruction?: string; - documents?: string | string[]; - text?: string | string[]; -} -interface Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Output { - data?: number[][]; - shape?: number[]; -} -declare abstract class Base_Ai_Cf_Qwen_Qwen3_Embedding_0_6B { - inputs: Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Input; - postProcessedOutputs: Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Output; -} -type Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Input = { - /** - * readable stream with audio data and content-type specified for that data - */ - audio: { - body: object; - contentType: string; - }; - /** - * type of data PCM data that's sent to the inference server as raw array - */ - dtype?: "uint8" | "float32" | "float64"; -} | { - /** - * base64 encoded audio data - */ - audio: string; - /** - * type of data PCM data that's sent to the inference server as raw array - */ - dtype?: "uint8" | "float32" | "float64"; -}; -interface Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Output { - /** - * if true, end-of-turn was detected - */ - is_complete?: boolean; - /** - * probability of the end-of-turn detection - */ - probability?: number; -} -declare abstract class Base_Ai_Cf_Pipecat_Ai_Smart_Turn_V2 { - inputs: Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Input; - postProcessedOutputs: Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Output; -} -declare abstract class Base_Ai_Cf_Openai_Gpt_Oss_120B { - inputs: XOR; - postProcessedOutputs: XOR; -} -declare abstract class Base_Ai_Cf_Openai_Gpt_Oss_20B { - inputs: XOR; - postProcessedOutputs: XOR; -} -interface Ai_Cf_Leonardo_Phoenix_1_0_Input { - /** - * A text description of the image you want to generate. - */ - prompt: string; - /** - * Controls how closely the generated image should adhere to the prompt; higher values make the image more aligned with the prompt - */ - guidance?: number; - /** - * Random seed for reproducibility of the image generation - */ - seed?: number; - /** - * The height of the generated image in pixels - */ - height?: number; - /** - * The width of the generated image in pixels - */ - width?: number; - /** - * The number of diffusion steps; higher values can improve quality but take longer - */ - num_steps?: number; - /** - * Specify what to exclude from the generated images - */ - negative_prompt?: string; -} -/** - * The generated image in JPEG format - */ -type Ai_Cf_Leonardo_Phoenix_1_0_Output = string; -declare abstract class Base_Ai_Cf_Leonardo_Phoenix_1_0 { - inputs: Ai_Cf_Leonardo_Phoenix_1_0_Input; - postProcessedOutputs: Ai_Cf_Leonardo_Phoenix_1_0_Output; -} -interface Ai_Cf_Leonardo_Lucid_Origin_Input { - /** - * A text description of the image you want to generate. - */ - prompt: string; - /** - * Controls how closely the generated image should adhere to the prompt; higher values make the image more aligned with the prompt - */ - guidance?: number; - /** - * Random seed for reproducibility of the image generation - */ - seed?: number; - /** - * The height of the generated image in pixels - */ - height?: number; - /** - * The width of the generated image in pixels - */ - width?: number; - /** - * The number of diffusion steps; higher values can improve quality but take longer - */ - num_steps?: number; - /** - * The number of diffusion steps; higher values can improve quality but take longer - */ - steps?: number; -} -interface Ai_Cf_Leonardo_Lucid_Origin_Output { - /** - * The generated image in Base64 format. - */ - image?: string; -} -declare abstract class Base_Ai_Cf_Leonardo_Lucid_Origin { - inputs: Ai_Cf_Leonardo_Lucid_Origin_Input; - postProcessedOutputs: Ai_Cf_Leonardo_Lucid_Origin_Output; -} -interface Ai_Cf_Deepgram_Aura_1_Input { - /** - * Speaker used to produce the audio. - */ - speaker?: "angus" | "asteria" | "arcas" | "orion" | "orpheus" | "athena" | "luna" | "zeus" | "perseus" | "helios" | "hera" | "stella"; - /** - * Encoding of the output audio. - */ - encoding?: "linear16" | "flac" | "mulaw" | "alaw" | "mp3" | "opus" | "aac"; - /** - * Container specifies the file format wrapper for the output audio. The available options depend on the encoding type.. - */ - container?: "none" | "wav" | "ogg"; - /** - * The text content to be converted to speech - */ - text: string; - /** - * Sample Rate specifies the sample rate for the output audio. Based on the encoding, different sample rates are supported. For some encodings, the sample rate is not configurable - */ - sample_rate?: number; - /** - * The bitrate of the audio in bits per second. Choose from predefined ranges or specific values based on the encoding type. - */ - bit_rate?: number; -} -/** - * The generated audio in MP3 format - */ -type Ai_Cf_Deepgram_Aura_1_Output = string; -declare abstract class Base_Ai_Cf_Deepgram_Aura_1 { - inputs: Ai_Cf_Deepgram_Aura_1_Input; - postProcessedOutputs: Ai_Cf_Deepgram_Aura_1_Output; -} -interface Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Input { - /** - * Input text to translate. Can be a single string or a list of strings. - */ - text: string | string[]; - /** - * Target langauge to translate to - */ - target_language: "asm_Beng" | "awa_Deva" | "ben_Beng" | "bho_Deva" | "brx_Deva" | "doi_Deva" | "eng_Latn" | "gom_Deva" | "gon_Deva" | "guj_Gujr" | "hin_Deva" | "hne_Deva" | "kan_Knda" | "kas_Arab" | "kas_Deva" | "kha_Latn" | "lus_Latn" | "mag_Deva" | "mai_Deva" | "mal_Mlym" | "mar_Deva" | "mni_Beng" | "mni_Mtei" | "npi_Deva" | "ory_Orya" | "pan_Guru" | "san_Deva" | "sat_Olck" | "snd_Arab" | "snd_Deva" | "tam_Taml" | "tel_Telu" | "urd_Arab" | "unr_Deva"; -} -interface Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Output { - /** - * Translated texts - */ - translations: string[]; -} -declare abstract class Base_Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B { - inputs: Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Input; - postProcessedOutputs: Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Output; -} -type Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Input = Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Async_Batch; -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. - */ - lora?: string; - response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role: string; - content: string | { - /** - * Type of the content (text) - */ - type?: string; - /** - * Text content - */ - text?: string; - }[]; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_1; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_1 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Async_Batch { - requests: (Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt_1 | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages_1)[]; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt_1 { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. - */ - lora?: string; - response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_2; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_2 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages_1 { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role: string; - content: string | { - /** - * Type of the content (text) - */ - type?: string; - /** - * Text content - */ - text?: string; - }[]; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_3; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_3 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -type Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Output = Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Chat_Completion_Response | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Text_Completion_Response | string | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_AsyncResponse; -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Chat_Completion_Response { - /** - * Unique identifier for the completion - */ - id?: string; - /** - * Object type identifier - */ - object?: "chat.completion"; - /** - * Unix timestamp of when the completion was created - */ - created?: number; - /** - * Model used for the completion - */ - model?: string; - /** - * List of completion choices - */ - choices?: { - /** - * Index of the choice in the list - */ - index?: number; - /** - * The message generated by the model - */ - message?: { - /** - * Role of the message author - */ - role: string; - /** - * The content of the message - */ - content: string; - /** - * Internal reasoning content (if available) - */ - reasoning_content?: string; - /** - * Tool calls made by the assistant - */ - tool_calls?: { - /** - * Unique identifier for the tool call - */ - id: string; - /** - * Type of tool call - */ - type: "function"; - function: { - /** - * Name of the function to call - */ - name: string; - /** - * JSON string of arguments for the function - */ - arguments: string; - }; - }[]; - }; - /** - * Reason why the model stopped generating - */ - finish_reason?: string; - /** - * Stop reason (may be null) - */ - stop_reason?: string | null; - /** - * Log probabilities (if requested) - */ - logprobs?: {} | null; - }[]; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * Log probabilities for the prompt (if requested) - */ - prompt_logprobs?: {} | null; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Text_Completion_Response { - /** - * Unique identifier for the completion - */ - id?: string; - /** - * Object type identifier - */ - object?: "text_completion"; - /** - * Unix timestamp of when the completion was created - */ - created?: number; - /** - * Model used for the completion - */ - model?: string; - /** - * List of completion choices - */ - choices?: { - /** - * Index of the choice in the list - */ - index: number; - /** - * The generated text completion - */ - text: string; - /** - * Reason why the model stopped generating - */ - finish_reason: string; - /** - * Stop reason (may be null) - */ - stop_reason?: string | null; - /** - * Log probabilities (if requested) - */ - logprobs?: {} | null; - /** - * Log probabilities for the prompt (if requested) - */ - prompt_logprobs?: {} | null; - }[]; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It { - inputs: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Input; - postProcessedOutputs: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Output; -} -interface Ai_Cf_Pfnet_Plamo_Embedding_1B_Input { - /** - * Input text to embed. Can be a single string or a list of strings. - */ - text: string | string[]; -} -interface Ai_Cf_Pfnet_Plamo_Embedding_1B_Output { - /** - * Embedding vectors, where each vector is a list of floats. - */ - data: number[][]; - /** - * Shape of the embedding data as [number_of_embeddings, embedding_dimension]. - * - * @minItems 2 - * @maxItems 2 - */ - shape: [ - number, - number - ]; -} -declare abstract class Base_Ai_Cf_Pfnet_Plamo_Embedding_1B { - inputs: Ai_Cf_Pfnet_Plamo_Embedding_1B_Input; - postProcessedOutputs: Ai_Cf_Pfnet_Plamo_Embedding_1B_Output; -} -interface Ai_Cf_Deepgram_Flux_Input { - /** - * Encoding of the audio stream. Currently only supports raw signed little-endian 16-bit PCM. - */ - encoding: "linear16"; - /** - * Sample rate of the audio stream in Hz. - */ - sample_rate: string; - /** - * End-of-turn confidence required to fire an eager end-of-turn event. When set, enables EagerEndOfTurn and TurnResumed events. Valid Values 0.3 - 0.9. - */ - eager_eot_threshold?: string; - /** - * End-of-turn confidence required to finish a turn. Valid Values 0.5 - 0.9. - */ - eot_threshold?: string; - /** - * A turn will be finished when this much time has passed after speech, regardless of EOT confidence. - */ - eot_timeout_ms?: string; - /** - * Keyterm prompting can improve recognition of specialized terminology. Pass multiple keyterm query parameters to boost multiple keyterms. - */ - keyterm?: string; - /** - * Opts out requests from the Deepgram Model Improvement Program. Refer to Deepgram Docs for pricing impacts before setting this to true. https://dpgr.am/deepgram-mip - */ - mip_opt_out?: "true" | "false"; - /** - * Label your requests for the purpose of identification during usage reporting - */ - tag?: string; -} -/** - * Output will be returned as websocket messages. - */ -interface Ai_Cf_Deepgram_Flux_Output { - /** - * The unique identifier of the request (uuid) - */ - request_id?: string; - /** - * Starts at 0 and increments for each message the server sends to the client. - */ - sequence_id?: number; - /** - * The type of event being reported. - */ - event?: "Update" | "StartOfTurn" | "EagerEndOfTurn" | "TurnResumed" | "EndOfTurn"; - /** - * The index of the current turn - */ - turn_index?: number; - /** - * Start time in seconds of the audio range that was transcribed - */ - audio_window_start?: number; - /** - * End time in seconds of the audio range that was transcribed - */ - audio_window_end?: number; - /** - * Text that was said over the course of the current turn - */ - transcript?: string; - /** - * The words in the transcript - */ - words?: { - /** - * The individual punctuated, properly-cased word from the transcript - */ - word: string; - /** - * Confidence that this word was transcribed correctly - */ - confidence: number; - }[]; - /** - * Confidence that no more speech is coming in this turn - */ - end_of_turn_confidence?: number; -} -declare abstract class Base_Ai_Cf_Deepgram_Flux { - inputs: Ai_Cf_Deepgram_Flux_Input; - postProcessedOutputs: Ai_Cf_Deepgram_Flux_Output; -} -interface Ai_Cf_Deepgram_Aura_2_En_Input { - /** - * Speaker used to produce the audio. - */ - speaker?: "amalthea" | "andromeda" | "apollo" | "arcas" | "aries" | "asteria" | "athena" | "atlas" | "aurora" | "callista" | "cora" | "cordelia" | "delia" | "draco" | "electra" | "harmonia" | "helena" | "hera" | "hermes" | "hyperion" | "iris" | "janus" | "juno" | "jupiter" | "luna" | "mars" | "minerva" | "neptune" | "odysseus" | "ophelia" | "orion" | "orpheus" | "pandora" | "phoebe" | "pluto" | "saturn" | "thalia" | "theia" | "vesta" | "zeus"; - /** - * Encoding of the output audio. - */ - encoding?: "linear16" | "flac" | "mulaw" | "alaw" | "mp3" | "opus" | "aac"; - /** - * Container specifies the file format wrapper for the output audio. The available options depend on the encoding type.. - */ - container?: "none" | "wav" | "ogg"; - /** - * The text content to be converted to speech - */ - text: string; - /** - * Sample Rate specifies the sample rate for the output audio. Based on the encoding, different sample rates are supported. For some encodings, the sample rate is not configurable - */ - sample_rate?: number; - /** - * The bitrate of the audio in bits per second. Choose from predefined ranges or specific values based on the encoding type. - */ - bit_rate?: number; -} -/** - * The generated audio in MP3 format - */ -type Ai_Cf_Deepgram_Aura_2_En_Output = string; -declare abstract class Base_Ai_Cf_Deepgram_Aura_2_En { - inputs: Ai_Cf_Deepgram_Aura_2_En_Input; - postProcessedOutputs: Ai_Cf_Deepgram_Aura_2_En_Output; -} -interface Ai_Cf_Deepgram_Aura_2_Es_Input { - /** - * Speaker used to produce the audio. - */ - speaker?: "sirio" | "nestor" | "carina" | "celeste" | "alvaro" | "diana" | "aquila" | "selena" | "estrella" | "javier"; - /** - * Encoding of the output audio. - */ - encoding?: "linear16" | "flac" | "mulaw" | "alaw" | "mp3" | "opus" | "aac"; - /** - * Container specifies the file format wrapper for the output audio. The available options depend on the encoding type.. - */ - container?: "none" | "wav" | "ogg"; - /** - * The text content to be converted to speech - */ - text: string; - /** - * Sample Rate specifies the sample rate for the output audio. Based on the encoding, different sample rates are supported. For some encodings, the sample rate is not configurable - */ - sample_rate?: number; - /** - * The bitrate of the audio in bits per second. Choose from predefined ranges or specific values based on the encoding type. - */ - bit_rate?: number; -} -/** - * The generated audio in MP3 format - */ -type Ai_Cf_Deepgram_Aura_2_Es_Output = string; -declare abstract class Base_Ai_Cf_Deepgram_Aura_2_Es { - inputs: Ai_Cf_Deepgram_Aura_2_Es_Input; - postProcessedOutputs: Ai_Cf_Deepgram_Aura_2_Es_Output; -} -interface Ai_Cf_Black_Forest_Labs_Flux_2_Dev_Input { - multipart: { - body?: object; - contentType?: string; - }; -} -interface Ai_Cf_Black_Forest_Labs_Flux_2_Dev_Output { - /** - * Generated image as Base64 string. - */ - image?: string; -} -declare abstract class Base_Ai_Cf_Black_Forest_Labs_Flux_2_Dev { - inputs: Ai_Cf_Black_Forest_Labs_Flux_2_Dev_Input; - postProcessedOutputs: Ai_Cf_Black_Forest_Labs_Flux_2_Dev_Output; -} -interface Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B_Input { - multipart: { - body?: object; - contentType?: string; - }; -} -interface Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B_Output { - /** - * Generated image as Base64 string. - */ - image?: string; -} -declare abstract class Base_Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B { - inputs: Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B_Input; - postProcessedOutputs: Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B_Output; -} -interface Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B_Input { - multipart: { - body?: object; - contentType?: string; - }; -} -interface Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B_Output { - /** - * Generated image as Base64 string. - */ - image?: string; -} -declare abstract class Base_Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B { - inputs: Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B_Input; - postProcessedOutputs: Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B_Output; -} -declare abstract class Base_Ai_Cf_Zai_Org_Glm_4_7_Flash { - inputs: ChatCompletionsInput; - postProcessedOutputs: ChatCompletionsOutput; -} -declare abstract class Base_Ai_Cf_Moonshotai_Kimi_K2_5 { - inputs: ChatCompletionsInput; - postProcessedOutputs: ChatCompletionsOutput; -} -declare abstract class Base_Ai_Cf_Nvidia_Nemotron_3_120B_A12B { - inputs: ChatCompletionsInput; - postProcessedOutputs: ChatCompletionsOutput; -} -declare abstract class Base_Ai_Cf_Google_Gemma_4_26B_A4B_IT { - inputs: ChatCompletionsInput; - postProcessedOutputs: ChatCompletionsOutput; -} -interface AiModels { - "@cf/huggingface/distilbert-sst-2-int8": BaseAiTextClassification; - "@cf/stabilityai/stable-diffusion-xl-base-1.0": BaseAiTextToImage; - "@cf/runwayml/stable-diffusion-v1-5-inpainting": BaseAiTextToImage; - "@cf/runwayml/stable-diffusion-v1-5-img2img": BaseAiTextToImage; - "@cf/lykon/dreamshaper-8-lcm": BaseAiTextToImage; - "@cf/bytedance/stable-diffusion-xl-lightning": BaseAiTextToImage; - "@cf/myshell-ai/melotts": BaseAiTextToSpeech; - "@cf/google/embeddinggemma-300m": BaseAiTextEmbeddings; - "@cf/microsoft/resnet-50": BaseAiImageClassification; - "@cf/meta/llama-2-7b-chat-int8": BaseAiTextGeneration; - "@cf/mistral/mistral-7b-instruct-v0.1": BaseAiTextGeneration; - "@cf/meta/llama-2-7b-chat-fp16": BaseAiTextGeneration; - "@hf/thebloke/llama-2-13b-chat-awq": BaseAiTextGeneration; - "@hf/thebloke/mistral-7b-instruct-v0.1-awq": BaseAiTextGeneration; - "@hf/thebloke/zephyr-7b-beta-awq": BaseAiTextGeneration; - "@hf/thebloke/openhermes-2.5-mistral-7b-awq": BaseAiTextGeneration; - "@hf/thebloke/neural-chat-7b-v3-1-awq": BaseAiTextGeneration; - "@hf/thebloke/deepseek-coder-6.7b-base-awq": BaseAiTextGeneration; - "@hf/thebloke/deepseek-coder-6.7b-instruct-awq": BaseAiTextGeneration; - "@cf/deepseek-ai/deepseek-math-7b-instruct": BaseAiTextGeneration; - "@cf/defog/sqlcoder-7b-2": BaseAiTextGeneration; - "@cf/openchat/openchat-3.5-0106": BaseAiTextGeneration; - "@cf/tiiuae/falcon-7b-instruct": BaseAiTextGeneration; - "@cf/thebloke/discolm-german-7b-v1-awq": BaseAiTextGeneration; - "@cf/qwen/qwen1.5-0.5b-chat": BaseAiTextGeneration; - "@cf/qwen/qwen1.5-7b-chat-awq": BaseAiTextGeneration; - "@cf/qwen/qwen1.5-14b-chat-awq": BaseAiTextGeneration; - "@cf/tinyllama/tinyllama-1.1b-chat-v1.0": BaseAiTextGeneration; - "@cf/microsoft/phi-2": BaseAiTextGeneration; - "@cf/qwen/qwen1.5-1.8b-chat": BaseAiTextGeneration; - "@cf/mistral/mistral-7b-instruct-v0.2-lora": BaseAiTextGeneration; - "@hf/nousresearch/hermes-2-pro-mistral-7b": BaseAiTextGeneration; - "@hf/nexusflow/starling-lm-7b-beta": BaseAiTextGeneration; - "@hf/google/gemma-7b-it": BaseAiTextGeneration; - "@cf/meta-llama/llama-2-7b-chat-hf-lora": BaseAiTextGeneration; - "@cf/google/gemma-2b-it-lora": BaseAiTextGeneration; - "@cf/google/gemma-7b-it-lora": BaseAiTextGeneration; - "@hf/mistral/mistral-7b-instruct-v0.2": BaseAiTextGeneration; - "@cf/meta/llama-3-8b-instruct": BaseAiTextGeneration; - "@cf/fblgit/una-cybertron-7b-v2-bf16": BaseAiTextGeneration; - "@cf/meta/llama-3-8b-instruct-awq": BaseAiTextGeneration; - "@cf/meta/llama-3.1-8b-instruct-fp8": BaseAiTextGeneration; - "@cf/meta/llama-3.1-8b-instruct-awq": BaseAiTextGeneration; - "@cf/meta/llama-3.2-3b-instruct": BaseAiTextGeneration; - "@cf/meta/llama-3.2-1b-instruct": BaseAiTextGeneration; - "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b": BaseAiTextGeneration; - "@cf/ibm-granite/granite-4.0-h-micro": BaseAiTextGeneration; - "@cf/facebook/bart-large-cnn": BaseAiSummarization; - "@cf/llava-hf/llava-1.5-7b-hf": BaseAiImageToText; - "@cf/baai/bge-base-en-v1.5": Base_Ai_Cf_Baai_Bge_Base_En_V1_5; - "@cf/openai/whisper": Base_Ai_Cf_Openai_Whisper; - "@cf/meta/m2m100-1.2b": Base_Ai_Cf_Meta_M2M100_1_2B; - "@cf/baai/bge-small-en-v1.5": Base_Ai_Cf_Baai_Bge_Small_En_V1_5; - "@cf/baai/bge-large-en-v1.5": Base_Ai_Cf_Baai_Bge_Large_En_V1_5; - "@cf/unum/uform-gen2-qwen-500m": Base_Ai_Cf_Unum_Uform_Gen2_Qwen_500M; - "@cf/openai/whisper-tiny-en": Base_Ai_Cf_Openai_Whisper_Tiny_En; - "@cf/openai/whisper-large-v3-turbo": Base_Ai_Cf_Openai_Whisper_Large_V3_Turbo; - "@cf/baai/bge-m3": Base_Ai_Cf_Baai_Bge_M3; - "@cf/black-forest-labs/flux-1-schnell": Base_Ai_Cf_Black_Forest_Labs_Flux_1_Schnell; - "@cf/meta/llama-3.2-11b-vision-instruct": Base_Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct; - "@cf/meta/llama-3.3-70b-instruct-fp8-fast": Base_Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast; - "@cf/meta/llama-guard-3-8b": Base_Ai_Cf_Meta_Llama_Guard_3_8B; - "@cf/baai/bge-reranker-base": Base_Ai_Cf_Baai_Bge_Reranker_Base; - "@cf/qwen/qwen2.5-coder-32b-instruct": Base_Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct; - "@cf/qwen/qwq-32b": Base_Ai_Cf_Qwen_Qwq_32B; - "@cf/mistralai/mistral-small-3.1-24b-instruct": Base_Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct; - "@cf/google/gemma-3-12b-it": Base_Ai_Cf_Google_Gemma_3_12B_It; - "@cf/meta/llama-4-scout-17b-16e-instruct": Base_Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct; - "@cf/qwen/qwen3-30b-a3b-fp8": Base_Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8; - "@cf/deepgram/nova-3": Base_Ai_Cf_Deepgram_Nova_3; - "@cf/qwen/qwen3-embedding-0.6b": Base_Ai_Cf_Qwen_Qwen3_Embedding_0_6B; - "@cf/pipecat-ai/smart-turn-v2": Base_Ai_Cf_Pipecat_Ai_Smart_Turn_V2; - "@cf/openai/gpt-oss-120b": Base_Ai_Cf_Openai_Gpt_Oss_120B; - "@cf/openai/gpt-oss-20b": Base_Ai_Cf_Openai_Gpt_Oss_20B; - "@cf/leonardo/phoenix-1.0": Base_Ai_Cf_Leonardo_Phoenix_1_0; - "@cf/leonardo/lucid-origin": Base_Ai_Cf_Leonardo_Lucid_Origin; - "@cf/deepgram/aura-1": Base_Ai_Cf_Deepgram_Aura_1; - "@cf/ai4bharat/indictrans2-en-indic-1B": Base_Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B; - "@cf/aisingapore/gemma-sea-lion-v4-27b-it": Base_Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It; - "@cf/pfnet/plamo-embedding-1b": Base_Ai_Cf_Pfnet_Plamo_Embedding_1B; - "@cf/deepgram/flux": Base_Ai_Cf_Deepgram_Flux; - "@cf/deepgram/aura-2-en": Base_Ai_Cf_Deepgram_Aura_2_En; - "@cf/deepgram/aura-2-es": Base_Ai_Cf_Deepgram_Aura_2_Es; - "@cf/black-forest-labs/flux-2-dev": Base_Ai_Cf_Black_Forest_Labs_Flux_2_Dev; - "@cf/black-forest-labs/flux-2-klein-4b": Base_Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B; - "@cf/black-forest-labs/flux-2-klein-9b": Base_Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B; - "@cf/zai-org/glm-4.7-flash": Base_Ai_Cf_Zai_Org_Glm_4_7_Flash; - "@cf/moonshotai/kimi-k2.5": Base_Ai_Cf_Moonshotai_Kimi_K2_5; - "@cf/nvidia/nemotron-3-120b-a12b": Base_Ai_Cf_Nvidia_Nemotron_3_120B_A12B; -} -type AiOptions = { - /** - * Send requests as an asynchronous batch job, only works for supported models - * https://developers.cloudflare.com/workers-ai/features/batch-api - */ - queueRequest?: boolean; - /** - * Establish websocket connections, only works for supported models - */ - websocket?: boolean; - /** - * Tag your requests to group and view them in Cloudflare dashboard. - * - * Rules: - * Tags must only contain letters, numbers, and the symbols: : - . / @ - * Each tag can have maximum 50 characters. - * Maximum 5 tags are allowed each request. - * Duplicate tags will removed. - */ - tags?: string[]; - gateway?: GatewayOptions; - returnRawResponse?: boolean; - prefix?: string; - extraHeaders?: object; - signal?: AbortSignal; -}; -type AiModelsSearchParams = { - author?: string; - hide_experimental?: boolean; - page?: number; - per_page?: number; - search?: string; - source?: number; - task?: string; -}; -type AiModelsSearchObject = { - id: string; - source: number; - name: string; - description: string; - task: { - id: string; - name: string; - description: string; - }; - tags: string[]; - properties: { - property_id: string; - value: string; - }[]; -}; -type ChatCompletionsBase = XOR; -type ChatCompletionsInput = XOR; -interface InferenceUpstreamError extends Error { -} -interface AiInternalError extends Error { -} -type AiModelListType = Record; -type AiAsyncBatchResponse = { - request_id: string; -}; -declare abstract class Ai { - aiGatewayLogId: string | null; - gateway(gatewayId: string): AiGateway; - /** - * @deprecated Use the standalone `ai_search_namespaces` or `ai_search` Workers bindings instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ - aiSearch(): AiSearchNamespace; - /** - * @deprecated AutoRAG has been replaced by AI Search. - * Use the standalone `ai_search_namespaces` or `ai_search` Workers bindings instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - * - * @param autoragId Instance ID - */ - autorag(autoragId: string): AutoRAG; - // Batch request - run(model: Name, inputs: { - requests: AiModelList[Name]['inputs'][]; - }, options: AiOptions & { - queueRequest: true; - }): Promise; - // Raw response - run(model: Name, inputs: AiModelList[Name]['inputs'], options: AiOptions & { - returnRawResponse: true; - }): Promise; - // WebSocket - run(model: Name, inputs: AiModelList[Name]['inputs'], options: AiOptions & { - websocket: true; - }): Promise; - // Streaming - run(model: Name, inputs: AiModelList[Name]['inputs'] & { - stream: true; - }, options?: AiOptions): Promise; - // Normal (default) - known model - run(model: Name, inputs: AiModelList[Name]['inputs'], options?: AiOptions): Promise; - // Unknown model (gateway fallback) - run(model: string & {}, inputs: Record, options?: AiOptions): Promise>; - models(params?: AiModelsSearchParams): Promise; - toMarkdown(): ToMarkdownService; - toMarkdown(files: MarkdownDocument[], options?: ConversionRequestOptions): Promise; - toMarkdown(files: MarkdownDocument, options?: ConversionRequestOptions): Promise; -} -type GatewayRetries = { - maxAttempts?: 1 | 2 | 3 | 4 | 5; - retryDelayMs?: number; - backoff?: 'constant' | 'linear' | 'exponential'; -}; -type GatewayOptions = { - id: string; - cacheKey?: string; - cacheTtl?: number; - skipCache?: boolean; - metadata?: Record; - collectLog?: boolean; - eventId?: string; - requestTimeoutMs?: number; - retries?: GatewayRetries; -}; -type UniversalGatewayOptions = Exclude & { - /** - ** @deprecated - */ - id?: string; -}; -type AiGatewayPatchLog = { - score?: number | null; - feedback?: -1 | 1 | null; - metadata?: Record | null; -}; -type AiGatewayLog = { - id: string; - provider: string; - model: string; - model_type?: string; - path: string; - duration: number; - request_type?: string; - request_content_type?: string; - status_code: number; - response_content_type?: string; - success: boolean; - cached: boolean; - tokens_in?: number; - tokens_out?: number; - metadata?: Record; - step?: number; - cost?: number; - custom_cost?: boolean; - request_size: number; - request_head?: string; - request_head_complete: boolean; - response_size: number; - response_head?: string; - response_head_complete: boolean; - created_at: Date; -}; -type AIGatewayProviders = 'workers-ai' | 'anthropic' | 'aws-bedrock' | 'azure-openai' | 'google-vertex-ai' | 'huggingface' | 'openai' | 'perplexity-ai' | 'replicate' | 'groq' | 'cohere' | 'google-ai-studio' | 'mistral' | 'grok' | 'openrouter' | 'deepseek' | 'cerebras' | 'cartesia' | 'elevenlabs' | 'adobe-firefly'; -type AIGatewayHeaders = { - 'cf-aig-metadata': Record | string; - 'cf-aig-custom-cost': { - per_token_in?: number; - per_token_out?: number; - } | { - total_cost?: number; - } | string; - 'cf-aig-cache-ttl': number | string; - 'cf-aig-skip-cache': boolean | string; - 'cf-aig-cache-key': string; - 'cf-aig-event-id': string; - 'cf-aig-request-timeout': number | string; - 'cf-aig-max-attempts': number | string; - 'cf-aig-retry-delay': number | string; - 'cf-aig-backoff': string; - 'cf-aig-collect-log': boolean | string; - Authorization: string; - 'Content-Type': string; - [key: string]: string | number | boolean | object; -}; -type AIGatewayUniversalRequest = { - provider: AIGatewayProviders | string; // eslint-disable-line - endpoint: string; - headers: Partial; - query: unknown; -}; -interface AiGatewayInternalError extends Error { -} -interface AiGatewayLogNotFound extends Error { -} -declare abstract class AiGateway { - patchLog(logId: string, data: AiGatewayPatchLog): Promise; - getLog(logId: string): Promise; - run(data: AIGatewayUniversalRequest | AIGatewayUniversalRequest[], options?: { - gateway?: UniversalGatewayOptions; - extraHeaders?: object; - signal?: AbortSignal; - }): Promise; - getUrl(provider?: AIGatewayProviders | string): Promise; // eslint-disable-line -} -// Copyright (c) 2022-2025 Cloudflare, Inc. -// Licensed under the Apache 2.0 license found in the LICENSE file or at: -// https://opensource.org/licenses/Apache-2.0 -/** - * Artifacts — Git-compatible file storage on Cloudflare Workers. - * - * Provides programmatic access to create, manage, and fork repositories, - * and to issue and revoke scoped access tokens. - */ -/** Information about a repository. */ -interface ArtifactsRepoInfo { - /** Unique repository ID. */ - id: string; - /** Repository name. */ - name: string; - /** Repository description, or null if not set. */ - description: string | null; - /** Default branch name (e.g. "main"). */ - defaultBranch: string; - /** ISO 8601 creation timestamp. */ - createdAt: string; - /** ISO 8601 last-updated timestamp. */ - updatedAt: string; - /** ISO 8601 timestamp of the last push, or null if never pushed. */ - lastPushAt: string | null; - /** Fork source (e.g. "github:owner/repo", "artifacts:namespace/repo"), or null if not a fork. */ - source: string | null; - /** Whether the repository is read-only. */ - readOnly: boolean; - /** HTTPS git remote URL. */ - remote: string; -} -/** Result of creating a repository — includes the initial access token. */ -interface ArtifactsCreateRepoResult { - /** Unique repository ID. */ - id: string; - /** Repository name. */ - name: string; - /** Repository description, or null if not set. */ - description: string | null; - /** Default branch name. */ - defaultBranch: string; - /** HTTPS git remote URL. */ - remote: string; - /** Plaintext access token (only returned at creation time). */ - token: string; - /** ISO 8601 token expiry timestamp. */ - tokenExpiresAt: string; -} -/** Paginated list of repositories. */ -interface ArtifactsRepoListResult { - /** Repositories in this page (without the `remote` field). */ - repos: Omit[]; - /** Total number of repositories in the namespace. */ - total: number; - /** Cursor for the next page, if there are more results. */ - cursor?: string; -} -/** Result of creating an access token. */ -interface ArtifactsCreateTokenResult { - /** Unique token ID. */ - id: string; - /** Plaintext token (only returned at creation time). */ - plaintext: string; - /** Token scope: "read" or "write". */ - scope: 'read' | 'write'; - /** ISO 8601 token expiry timestamp. */ - expiresAt: string; -} -/** Token metadata (no plaintext). */ -interface ArtifactsTokenInfo { - /** Unique token ID. */ - id: string; - /** Token scope: "read" or "write". */ - scope: 'read' | 'write'; - /** Token state: "active", "expired", or "revoked". */ - state: 'active' | 'expired' | 'revoked'; - /** ISO 8601 creation timestamp. */ - createdAt: string; - /** ISO 8601 expiry timestamp. */ - expiresAt: string; -} -/** Paginated list of tokens for a repository. */ -interface ArtifactsTokenListResult { - /** Tokens in this page. */ - tokens: ArtifactsTokenInfo[]; - /** Total number of tokens for the repository. */ - total: number; -} -/** Handle for a single repository. Returned by Artifacts.get(). */ -interface ArtifactsRepo extends ArtifactsRepoInfo { - /** - * Create an access token for this repo. - * @param scope Token scope: "write" (default) or "read". - * @param ttl Time-to-live in seconds (default 86400, min 60, max 31536000). - */ - createToken(scope?: 'write' | 'read', ttl?: number): Promise; - /** List tokens for this repo (metadata only, no plaintext). */ - listTokens(): Promise; - /** - * Revoke a token by plaintext or ID. - * @param tokenOrId Plaintext token or token ID. - * @returns true if revoked, false if not found. - */ - revokeToken(tokenOrId: string): Promise; - // ── Fork ── - /** - * Fork this repo to a new repo. - * @param name Target repository name. - * @param opts Optional: description, readOnly flag, defaultBranchOnly (default true). - */ - fork(name: string, opts?: { - description?: string; - readOnly?: boolean; - defaultBranchOnly?: boolean; - }): Promise; -} -/** Artifacts binding — namespace-level operations. */ -interface Artifacts { - /** - * Create a new repository with an initial access token. - * @param name Repository name (alphanumeric, dots, hyphens, underscores). - * @param opts Optional: readOnly flag, description, default branch name. - * @returns Repo metadata with initial token. - */ - create(name: string, opts?: { - readOnly?: boolean; - description?: string; - setDefaultBranch?: string; - }): Promise; - /** - * Get a handle to an existing repository. - * @param name Repository name. - * @returns Repo handle. - */ - get(name: string): Promise; - /** - * Import a repository from an external git remote. - * @param params Source URL and optional branch/depth, plus target name and options. - * @returns Repo metadata with initial token. - */ - import(params: { - source: { - url: string; - branch?: string; - depth?: number; - }; - target: { - name: string; - opts?: { - description?: string; - readOnly?: boolean; - }; - }; - }): Promise; - /** - * List repositories with cursor-based pagination. - * @param opts Optional: limit (1–200, default 50), cursor for next page. - */ - list(opts?: { - limit?: number; - cursor?: string; - }): Promise; - /** - * Delete a repository and all associated tokens. - * @param name Repository name. - * @returns true if deleted, false if not found. - */ - delete(name: string): Promise; -} -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -interface AutoRAGInternalError extends Error { -} -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -interface AutoRAGNotFoundError extends Error { -} -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -interface AutoRAGUnauthorizedError extends Error { -} -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -interface AutoRAGNameNotSetError extends Error { -} -type ComparisonFilter = { - key: string; - type: 'eq' | 'ne' | 'gt' | 'gte' | 'lt' | 'lte'; - value: string | number | boolean; -}; -type CompoundFilter = { - type: 'and' | 'or'; - filters: ComparisonFilter[]; -}; -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -type AutoRagSearchRequest = { - query: string; - filters?: CompoundFilter | ComparisonFilter; - max_num_results?: number; - ranking_options?: { - ranker?: string; - score_threshold?: number; - }; - reranking?: { - enabled?: boolean; - model?: string; - }; - rewrite_query?: boolean; -}; -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -type AutoRagAiSearchRequest = AutoRagSearchRequest & { - stream?: boolean; - system_prompt?: string; -}; -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -type AutoRagAiSearchRequestStreaming = Omit & { - stream: true; -}; -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -type AutoRagSearchResponse = { - object: 'vector_store.search_results.page'; - search_query: string; - data: { - file_id: string; - filename: string; - score: number; - attributes: Record; - content: { - type: 'text'; - text: string; - }[]; - }[]; - has_more: boolean; - next_page: string | null; -}; -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -type AutoRagListResponse = { - id: string; - enable: boolean; - type: string; - source: string; - vectorize_name: string; - paused: boolean; - status: string; -}[]; -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -type AutoRagAiSearchResponse = AutoRagSearchResponse & { - response: string; -}; -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -declare abstract class AutoRAG { - /** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ - list(): Promise; - /** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ - search(params: AutoRagSearchRequest): Promise; - /** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ - aiSearch(params: AutoRagAiSearchRequestStreaming): Promise; - /** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ - aiSearch(params: AutoRagAiSearchRequest): Promise; - /** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ - aiSearch(params: AutoRagAiSearchRequest): Promise; -} -interface BasicImageTransformations { - /** - * Maximum width in image pixels. The value must be an integer. - */ - width?: number; - /** - * Maximum height in image pixels. The value must be an integer. - */ - height?: number; - /** - * Resizing mode as a string. It affects interpretation of width and height - * options: - * - scale-down: Similar to contain, but the image is never enlarged. If - * the image is larger than given width or height, it will be resized. - * Otherwise its original size will be kept. - * - contain: Resizes to maximum size that fits within the given width and - * height. If only a single dimension is given (e.g. only width), the - * image will be shrunk or enlarged to exactly match that dimension. - * Aspect ratio is always preserved. - * - cover: Resizes (shrinks or enlarges) to fill the entire area of width - * and height. If the image has an aspect ratio different from the ratio - * of width and height, it will be cropped to fit. - * - crop: The image will be shrunk and cropped to fit within the area - * specified by width and height. The image will not be enlarged. For images - * smaller than the given dimensions it's the same as scale-down. For - * images larger than the given dimensions, it's the same as cover. - * See also trim. - * - pad: Resizes to the maximum size that fits within the given width and - * height, and then fills the remaining area with a background color - * (white by default). Use of this mode is not recommended, as the same - * effect can be more efficiently achieved with the contain mode and the - * CSS object-fit: contain property. - * - squeeze: Stretches and deforms to the width and height given, even if it - * breaks aspect ratio - */ - fit?: "scale-down" | "contain" | "cover" | "crop" | "pad" | "squeeze"; - /** - * Image segmentation using artificial intelligence models. Sets pixels not - * within selected segment area to transparent e.g "foreground" sets every - * background pixel as transparent. - */ - segment?: "foreground"; - /** - * When cropping with fit: "cover", this defines the side or point that should - * be left uncropped. The value is either a string - * "left", "right", "top", "bottom", "auto", or "center" (the default), - * or an object {x, y} containing focal point coordinates in the original - * image expressed as fractions ranging from 0.0 (top or left) to 1.0 - * (bottom or right), 0.5 being the center. {fit: "cover", gravity: "top"} will - * crop bottom or left and right sides as necessary, but won’t crop anything - * from the top. {fit: "cover", gravity: {x:0.5, y:0.2}} will crop each side to - * preserve as much as possible around a point at 20% of the height of the - * source image. - */ - gravity?: 'face' | 'left' | 'right' | 'top' | 'bottom' | 'center' | 'auto' | 'entropy' | BasicImageTransformationsGravityCoordinates; - /** - * Background color to add underneath the image. Applies only to images with - * transparency (such as PNG). Accepts any CSS color (#RRGGBB, rgba(…), - * hsl(…), etc.) - */ - background?: string; - /** - * Number of degrees (90, 180, 270) to rotate the image by. width and height - * options refer to axes after rotation. - */ - rotate?: 0 | 90 | 180 | 270 | 360; -} -interface BasicImageTransformationsGravityCoordinates { - x?: number; - y?: number; - mode?: 'remainder' | 'box-center'; -} -/** - * In addition to the properties you can set in the RequestInit dict - * that you pass as an argument to the Request constructor, you can - * set certain properties of a `cf` object to control how Cloudflare - * features are applied to that new Request. - * - * Note: Currently, these properties cannot be tested in the - * playground. - */ -interface RequestInitCfProperties extends Record { - cacheEverything?: boolean; - /** - * A request's cache key is what determines if two requests are - * "the same" for caching purposes. If a request has the same cache key - * as some previous request, then we can serve the same cached response for - * both. (e.g. 'some-key') - * - * Only available for Enterprise customers. - */ - cacheKey?: string; - /** - * This allows you to append additional Cache-Tag response headers - * to the origin response without modifications to the origin server. - * This will allow for greater control over the Purge by Cache Tag feature - * utilizing changes only in the Workers process. - * - * Only available for Enterprise customers. - */ - cacheTags?: string[]; - /** - * Force response to be cached for a given number of seconds. (e.g. 300) - */ - cacheTtl?: number; - /** - * Force response to be cached for a given number of seconds based on the Origin status code. - * (e.g. { '200-299': 86400, '404': 1, '500-599': 0 }) - */ - cacheTtlByStatus?: Record; - /** - * Explicit Cache-Control header value to set on the response stored in cache. - * This gives full control over cache directives (e.g. 'public, max-age=3600, s-maxage=86400'). - * - * Cannot be used together with `cacheTtl` or the `cache` request option (`no-store`/`no-cache`), - * as these are mutually exclusive cache control mechanisms. Setting both will throw a TypeError. - * - * Can be used together with `cacheTtlByStatus`. - */ - cacheControl?: string; - /** - * Whether the response should be eligible for Cache Reserve storage. - */ - cacheReserveEligible?: boolean; - /** - * Whether to respect strong ETags (as opposed to weak ETags) from the origin. - */ - respectStrongEtag?: boolean; - /** - * Whether to strip ETag headers from the origin response before caching. - */ - stripEtags?: boolean; - /** - * Whether to strip Last-Modified headers from the origin response before caching. - */ - stripLastModified?: boolean; - /** - * Whether to enable Cache Deception Armor, which protects against web cache - * deception attacks by verifying the Content-Type matches the URL extension. - */ - cacheDeceptionArmor?: boolean; - /** - * Minimum file size in bytes for a response to be eligible for Cache Reserve storage. - */ - cacheReserveMinimumFileSize?: number; - scrapeShield?: boolean; - apps?: boolean; - image?: RequestInitCfPropertiesImage; - minify?: RequestInitCfPropertiesImageMinify; - mirage?: boolean; - polish?: "lossy" | "lossless" | "off"; - r2?: RequestInitCfPropertiesR2; - /** - * Redirects the request to an alternate origin server. You can use this, - * for example, to implement load balancing across several origins. - * (e.g.us-east.example.com) - * - * Note - For security reasons, the hostname set in resolveOverride must - * be proxied on the same Cloudflare zone of the incoming request. - * Otherwise, the setting is ignored. CNAME hosts are allowed, so to - * resolve to a host under a different domain or a DNS only domain first - * declare a CNAME record within your own zone’s DNS mapping to the - * external hostname, set proxy on Cloudflare, then set resolveOverride - * to point to that CNAME record. - */ - resolveOverride?: string; -} -interface RequestInitCfPropertiesImageDraw extends BasicImageTransformations { - /** - * Absolute URL of the image file to use for the drawing. It can be any of - * the supported file formats. For drawing of watermarks or non-rectangular - * overlays we recommend using PNG or WebP images. - */ - url: string; - /** - * Floating-point number between 0 (transparent) and 1 (opaque). - * For example, opacity: 0.5 makes overlay semitransparent. - */ - opacity?: number; - /** - * - If set to true, the overlay image will be tiled to cover the entire - * area. This is useful for stock-photo-like watermarks. - * - If set to "x", the overlay image will be tiled horizontally only - * (form a line). - * - If set to "y", the overlay image will be tiled vertically only - * (form a line). - */ - repeat?: true | "x" | "y"; - /** - * Position of the overlay image relative to a given edge. Each property is - * an offset in pixels. 0 aligns exactly to the edge. For example, left: 10 - * positions left side of the overlay 10 pixels from the left edge of the - * image it's drawn over. bottom: 0 aligns bottom of the overlay with bottom - * of the background image. - * - * Setting both left & right, or both top & bottom is an error. - * - * If no position is specified, the image will be centered. - */ - top?: number; - left?: number; - bottom?: number; - right?: number; -} -interface RequestInitCfPropertiesImage extends BasicImageTransformations { - /** - * Device Pixel Ratio. Default 1. Multiplier for width/height that makes it - * easier to specify higher-DPI sizes in . - */ - dpr?: number; - /** - * Allows you to trim your image. Takes dpr into account and is performed before - * resizing or rotation. - * - * It can be used as: - * - left, top, right, bottom - it will specify the number of pixels to cut - * off each side - * - width, height - the width/height you'd like to end up with - can be used - * in combination with the properties above - * - border - this will automatically trim the surroundings of an image based on - * it's color. It consists of three properties: - * - color: rgb or hex representation of the color you wish to trim (todo: verify the rgba bit) - * - tolerance: difference from color to treat as color - * - keep: the number of pixels of border to keep - */ - trim?: "border" | { - top?: number; - bottom?: number; - left?: number; - right?: number; - width?: number; - height?: number; - border?: boolean | { - color?: string; - tolerance?: number; - keep?: number; - }; - }; - /** - * Quality setting from 1-100 (useful values are in 60-90 range). Lower values - * make images look worse, but load faster. The default is 85. It applies only - * to JPEG and WebP images. It doesn’t have any effect on PNG. - */ - quality?: number | "low" | "medium-low" | "medium-high" | "high"; - /** - * Output format to generate. It can be: - * - avif: generate images in AVIF format. - * - webp: generate images in Google WebP format. Set quality to 100 to get - * the WebP-lossless format. - * - json: instead of generating an image, outputs information about the - * image, in JSON format. The JSON object will contain image size - * (before and after resizing), source image’s MIME type, file size, etc. - * - jpeg: generate images in JPEG format. - * - png: generate images in PNG format. - */ - format?: "avif" | "webp" | "json" | "jpeg" | "png" | "baseline-jpeg" | "png-force" | "svg"; - /** - * Whether to preserve animation frames from input files. Default is true. - * Setting it to false reduces animations to still images. This setting is - * recommended when enlarging images or processing arbitrary user content, - * because large GIF animations can weigh tens or even hundreds of megabytes. - * It is also useful to set anim:false when using format:"json" to get the - * response quicker without the number of frames. - */ - anim?: boolean; - /** - * What EXIF data should be preserved in the output image. Note that EXIF - * rotation and embedded color profiles are always applied ("baked in" into - * the image), and aren't affected by this option. Note that if the Polish - * feature is enabled, all metadata may have been removed already and this - * option may have no effect. - * - keep: Preserve most of EXIF metadata, including GPS location if there's - * any. - * - copyright: Only keep the copyright tag, and discard everything else. - * This is the default behavior for JPEG files. - * - none: Discard all invisible EXIF metadata. Currently WebP and PNG - * output formats always discard metadata. - */ - metadata?: "keep" | "copyright" | "none"; - /** - * Strength of sharpening filter to apply to the image. Floating-point - * number between 0 (no sharpening, default) and 10 (maximum). 1.0 is a - * recommended value for downscaled images. - */ - sharpen?: number; - /** - * Radius of a blur filter (approximate gaussian). Maximum supported radius - * is 250. - */ - blur?: number; - /** - * Overlays are drawn in the order they appear in the array (last array - * entry is the topmost layer). - */ - draw?: RequestInitCfPropertiesImageDraw[]; - /** - * Fetching image from authenticated origin. Setting this property will - * pass authentication headers (Authorization, Cookie, etc.) through to - * the origin. - */ - "origin-auth"?: "share-publicly"; - /** - * Adds a border around the image. The border is added after resizing. Border - * width takes dpr into account, and can be specified either using a single - * width property, or individually for each side. - */ - border?: { - color: string; - width: number; - } | { - color: string; - top: number; - right: number; - bottom: number; - left: number; - }; - /** - * Increase brightness by a factor. A value of 1.0 equals no change, a value - * of 0.5 equals half brightness, and a value of 2.0 equals twice as bright. - * 0 is ignored. - */ - brightness?: number; - /** - * Increase contrast by a factor. A value of 1.0 equals no change, a value of - * 0.5 equals low contrast, and a value of 2.0 equals high contrast. 0 is - * ignored. - */ - contrast?: number; - /** - * Increase exposure by a factor. A value of 1.0 equals no change, a value of - * 0.5 darkens the image, and a value of 2.0 lightens the image. 0 is ignored. - */ - gamma?: number; - /** - * Increase contrast by a factor. A value of 1.0 equals no change, a value of - * 0.5 equals low contrast, and a value of 2.0 equals high contrast. 0 is - * ignored. - */ - saturation?: number; - /** - * Flips the images horizontally, vertically, or both. Flipping is applied before - * rotation, so if you apply flip=h,rotate=90 then the image will be flipped - * horizontally, then rotated by 90 degrees. - */ - flip?: 'h' | 'v' | 'hv'; - /** - * Slightly reduces latency on a cache miss by selecting a - * quickest-to-compress file format, at a cost of increased file size and - * lower image quality. It will usually override the format option and choose - * JPEG over WebP or AVIF. We do not recommend using this option, except in - * unusual circumstances like resizing uncacheable dynamically-generated - * images. - */ - compression?: "fast"; -} -interface RequestInitCfPropertiesImageMinify { - javascript?: boolean; - css?: boolean; - html?: boolean; -} -interface RequestInitCfPropertiesR2 { - /** - * Colo id of bucket that an object is stored in - */ - bucketColoId?: number; -} -/** - * Request metadata provided by Cloudflare's edge. - */ -type IncomingRequestCfProperties = IncomingRequestCfPropertiesBase & IncomingRequestCfPropertiesBotManagementEnterprise & IncomingRequestCfPropertiesCloudflareForSaaSEnterprise & IncomingRequestCfPropertiesGeographicInformation & IncomingRequestCfPropertiesCloudflareAccessOrApiShield; -interface IncomingRequestCfPropertiesBase extends Record { - /** - * [ASN](https://www.iana.org/assignments/as-numbers/as-numbers.xhtml) of the incoming request. - * - * @example 395747 - */ - asn?: number; - /** - * The organization which owns the ASN of the incoming request. - * - * @example "Google Cloud" - */ - asOrganization?: string; - /** - * The original value of the `Accept-Encoding` header if Cloudflare modified it. - * - * @example "gzip, deflate, br" - */ - clientAcceptEncoding?: string; - /** - * The number of milliseconds it took for the request to reach your worker. - * - * @example 22 - */ - clientTcpRtt?: number; - /** - * The three-letter [IATA](https://en.wikipedia.org/wiki/IATA_airport_code) - * airport code of the data center that the request hit. - * - * @example "DFW" - */ - colo: string; - /** - * Represents the upstream's response to a - * [TCP `keepalive` message](https://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html) - * from cloudflare. - * - * For workers with no upstream, this will always be `1`. - * - * @example 3 - */ - edgeRequestKeepAliveStatus: IncomingRequestCfPropertiesEdgeRequestKeepAliveStatus; - /** - * The HTTP Protocol the request used. - * - * @example "HTTP/2" - */ - httpProtocol: string; - /** - * The browser-requested prioritization information in the request object. - * - * If no information was set, defaults to the empty string `""` - * - * @example "weight=192;exclusive=0;group=3;group-weight=127" - * @default "" - */ - requestPriority: string; - /** - * The TLS version of the connection to Cloudflare. - * In requests served over plaintext (without TLS), this property is the empty string `""`. - * - * @example "TLSv1.3" - */ - tlsVersion: string; - /** - * The cipher for the connection to Cloudflare. - * In requests served over plaintext (without TLS), this property is the empty string `""`. - * - * @example "AEAD-AES128-GCM-SHA256" - */ - tlsCipher: string; - /** - * Metadata containing the [`HELLO`](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.1.2) and [`FINISHED`](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.9) messages from this request's TLS handshake. - * - * If the incoming request was served over plaintext (without TLS) this field is undefined. - */ - tlsExportedAuthenticator?: IncomingRequestCfPropertiesExportedAuthenticatorMetadata; -} -interface IncomingRequestCfPropertiesBotManagementBase { - /** - * Cloudflare’s [level of certainty](https://developers.cloudflare.com/bots/concepts/bot-score/) that a request comes from a bot, - * represented as an integer percentage between `1` (almost certainly a bot) and `99` (almost certainly human). - * - * @example 54 - */ - score: number; - /** - * A boolean value that is true if the request comes from a good bot, like Google or Bing. - * Most customers choose to allow this traffic. For more details, see [Traffic from known bots](https://developers.cloudflare.com/firewall/known-issues-and-faq/#how-does-firewall-rules-handle-traffic-from-known-bots). - */ - verifiedBot: boolean; - /** - * A boolean value that is true if the request originates from a - * Cloudflare-verified proxy service. - */ - corporateProxy: boolean; - /** - * A boolean value that's true if the request matches [file extensions](https://developers.cloudflare.com/bots/reference/static-resources/) for many types of static resources. - */ - staticResource: boolean; - /** - * List of IDs that correlate to the Bot Management heuristic detections made on a request (you can have multiple heuristic detections on the same request). - */ - detectionIds: number[]; -} -interface IncomingRequestCfPropertiesBotManagement { - /** - * Results of Cloudflare's Bot Management analysis - */ - botManagement: IncomingRequestCfPropertiesBotManagementBase; - /** - * Duplicate of `botManagement.score`. - * - * @deprecated - */ - clientTrustScore: number; -} -interface IncomingRequestCfPropertiesBotManagementEnterprise extends IncomingRequestCfPropertiesBotManagement { - /** - * Results of Cloudflare's Bot Management analysis - */ - botManagement: IncomingRequestCfPropertiesBotManagementBase & { - /** - * A [JA3 Fingerprint](https://developers.cloudflare.com/bots/concepts/ja3-fingerprint/) to help profile specific SSL/TLS clients - * across different destination IPs, Ports, and X509 certificates. - */ - ja3Hash: string; - }; -} -interface IncomingRequestCfPropertiesCloudflareForSaaSEnterprise { - /** - * Custom metadata set per-host in [Cloudflare for SaaS](https://developers.cloudflare.com/cloudflare-for-platforms/cloudflare-for-saas/). - * - * This field is only present if you have Cloudflare for SaaS enabled on your account - * and you have followed the [required steps to enable it]((https://developers.cloudflare.com/cloudflare-for-platforms/cloudflare-for-saas/domain-support/custom-metadata/)). - */ - hostMetadata?: HostMetadata; -} -interface IncomingRequestCfPropertiesCloudflareAccessOrApiShield { - /** - * Information about the client certificate presented to Cloudflare. - * - * This is populated when the incoming request is served over TLS using - * either Cloudflare Access or API Shield (mTLS) - * and the presented SSL certificate has a valid - * [Certificate Serial Number](https://ldapwiki.com/wiki/Certificate%20Serial%20Number) - * (i.e., not `null` or `""`). - * - * Otherwise, a set of placeholder values are used. - * - * The property `certPresented` will be set to `"1"` when - * the object is populated (i.e. the above conditions were met). - */ - tlsClientAuth: IncomingRequestCfPropertiesTLSClientAuth | IncomingRequestCfPropertiesTLSClientAuthPlaceholder; -} -/** - * Metadata about the request's TLS handshake - */ -interface IncomingRequestCfPropertiesExportedAuthenticatorMetadata { - /** - * The client's [`HELLO` message](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.1.2), encoded in hexadecimal - * - * @example "44372ba35fa1270921d318f34c12f155dc87b682cf36a790cfaa3ba8737a1b5d" - */ - clientHandshake: string; - /** - * The server's [`HELLO` message](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.1.2), encoded in hexadecimal - * - * @example "44372ba35fa1270921d318f34c12f155dc87b682cf36a790cfaa3ba8737a1b5d" - */ - serverHandshake: string; - /** - * The client's [`FINISHED` message](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.9), encoded in hexadecimal - * - * @example "084ee802fe1348f688220e2a6040a05b2199a761f33cf753abb1b006792d3f8b" - */ - clientFinished: string; - /** - * The server's [`FINISHED` message](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.9), encoded in hexadecimal - * - * @example "084ee802fe1348f688220e2a6040a05b2199a761f33cf753abb1b006792d3f8b" - */ - serverFinished: string; -} -/** - * Geographic data about the request's origin. - */ -interface IncomingRequestCfPropertiesGeographicInformation { - /** - * The [ISO 3166-1 Alpha 2](https://www.iso.org/iso-3166-country-codes.html) country code the request originated from. - * - * If your worker is [configured to accept TOR connections](https://support.cloudflare.com/hc/en-us/articles/203306930-Understanding-Cloudflare-Tor-support-and-Onion-Routing), this may also be `"T1"`, indicating a request that originated over TOR. - * - * If Cloudflare is unable to determine where the request originated this property is omitted. - * - * The country code `"T1"` is used for requests originating on TOR. - * - * @example "GB" - */ - country?: Iso3166Alpha2Code | "T1"; - /** - * If present, this property indicates that the request originated in the EU - * - * @example "1" - */ - isEUCountry?: "1"; - /** - * A two-letter code indicating the continent the request originated from. - * - * @example "AN" - */ - continent?: ContinentCode; - /** - * The city the request originated from - * - * @example "Austin" - */ - city?: string; - /** - * Postal code of the incoming request - * - * @example "78701" - */ - postalCode?: string; - /** - * Latitude of the incoming request - * - * @example "30.27130" - */ - latitude?: string; - /** - * Longitude of the incoming request - * - * @example "-97.74260" - */ - longitude?: string; - /** - * Timezone of the incoming request - * - * @example "America/Chicago" - */ - timezone?: string; - /** - * If known, the ISO 3166-2 name for the first level region associated with - * the IP address of the incoming request - * - * @example "Texas" - */ - region?: string; - /** - * If known, the ISO 3166-2 code for the first-level region associated with - * the IP address of the incoming request - * - * @example "TX" - */ - regionCode?: string; - /** - * Metro code (DMA) of the incoming request - * - * @example "635" - */ - metroCode?: string; -} -/** Data about the incoming request's TLS certificate */ -interface IncomingRequestCfPropertiesTLSClientAuth { - /** Always `"1"`, indicating that the certificate was presented */ - certPresented: "1"; - /** - * Result of certificate verification. - * - * @example "FAILED:self signed certificate" - */ - certVerified: Exclude; - /** The presented certificate's revokation status. - * - * - A value of `"1"` indicates the certificate has been revoked - * - A value of `"0"` indicates the certificate has not been revoked - */ - certRevoked: "1" | "0"; - /** - * The certificate issuer's [distinguished name](https://knowledge.digicert.com/generalinformation/INFO1745.html) - * - * @example "CN=cloudflareaccess.com, C=US, ST=Texas, L=Austin, O=Cloudflare" - */ - certIssuerDN: string; - /** - * The certificate subject's [distinguished name](https://knowledge.digicert.com/generalinformation/INFO1745.html) - * - * @example "CN=*.cloudflareaccess.com, C=US, ST=Texas, L=Austin, O=Cloudflare" - */ - certSubjectDN: string; - /** - * The certificate issuer's [distinguished name](https://knowledge.digicert.com/generalinformation/INFO1745.html) ([RFC 2253](https://www.rfc-editor.org/rfc/rfc2253.html) formatted) - * - * @example "CN=cloudflareaccess.com, C=US, ST=Texas, L=Austin, O=Cloudflare" - */ - certIssuerDNRFC2253: string; - /** - * The certificate subject's [distinguished name](https://knowledge.digicert.com/generalinformation/INFO1745.html) ([RFC 2253](https://www.rfc-editor.org/rfc/rfc2253.html) formatted) - * - * @example "CN=*.cloudflareaccess.com, C=US, ST=Texas, L=Austin, O=Cloudflare" - */ - certSubjectDNRFC2253: string; - /** The certificate issuer's distinguished name (legacy policies) */ - certIssuerDNLegacy: string; - /** The certificate subject's distinguished name (legacy policies) */ - certSubjectDNLegacy: string; - /** - * The certificate's serial number - * - * @example "00936EACBE07F201DF" - */ - certSerial: string; - /** - * The certificate issuer's serial number - * - * @example "2489002934BDFEA34" - */ - certIssuerSerial: string; - /** - * The certificate's Subject Key Identifier - * - * @example "BB:AF:7E:02:3D:FA:A6:F1:3C:84:8E:AD:EE:38:98:EC:D9:32:32:D4" - */ - certSKI: string; - /** - * The certificate issuer's Subject Key Identifier - * - * @example "BB:AF:7E:02:3D:FA:A6:F1:3C:84:8E:AD:EE:38:98:EC:D9:32:32:D4" - */ - certIssuerSKI: string; - /** - * The certificate's SHA-1 fingerprint - * - * @example "6b9109f323999e52259cda7373ff0b4d26bd232e" - */ - certFingerprintSHA1: string; - /** - * The certificate's SHA-256 fingerprint - * - * @example "acf77cf37b4156a2708e34c4eb755f9b5dbbe5ebb55adfec8f11493438d19e6ad3f157f81fa3b98278453d5652b0c1fd1d71e5695ae4d709803a4d3f39de9dea" - */ - certFingerprintSHA256: string; - /** - * The effective starting date of the certificate - * - * @example "Dec 22 19:39:00 2018 GMT" - */ - certNotBefore: string; - /** - * The effective expiration date of the certificate - * - * @example "Dec 22 19:39:00 2018 GMT" - */ - certNotAfter: string; -} -/** Placeholder values for TLS Client Authorization */ -interface IncomingRequestCfPropertiesTLSClientAuthPlaceholder { - certPresented: "0"; - certVerified: "NONE"; - certRevoked: "0"; - certIssuerDN: ""; - certSubjectDN: ""; - certIssuerDNRFC2253: ""; - certSubjectDNRFC2253: ""; - certIssuerDNLegacy: ""; - certSubjectDNLegacy: ""; - certSerial: ""; - certIssuerSerial: ""; - certSKI: ""; - certIssuerSKI: ""; - certFingerprintSHA1: ""; - certFingerprintSHA256: ""; - certNotBefore: ""; - certNotAfter: ""; -} -/** Possible outcomes of TLS verification */ -declare type CertVerificationStatus = -/** Authentication succeeded */ -"SUCCESS" -/** No certificate was presented */ - | "NONE" -/** Failed because the certificate was self-signed */ - | "FAILED:self signed certificate" -/** Failed because the certificate failed a trust chain check */ - | "FAILED:unable to verify the first certificate" -/** Failed because the certificate not yet valid */ - | "FAILED:certificate is not yet valid" -/** Failed because the certificate is expired */ - | "FAILED:certificate has expired" -/** Failed for another unspecified reason */ - | "FAILED"; -/** - * An upstream endpoint's response to a TCP `keepalive` message from Cloudflare. - */ -declare type IncomingRequestCfPropertiesEdgeRequestKeepAliveStatus = 0 /** Unknown */ | 1 /** no keepalives (not found) */ | 2 /** no connection re-use, opening keepalive connection failed */ | 3 /** no connection re-use, keepalive accepted and saved */ | 4 /** connection re-use, refused by the origin server (`TCP FIN`) */ | 5; /** connection re-use, accepted by the origin server */ -/** ISO 3166-1 Alpha-2 codes */ -declare type Iso3166Alpha2Code = "AD" | "AE" | "AF" | "AG" | "AI" | "AL" | "AM" | "AO" | "AQ" | "AR" | "AS" | "AT" | "AU" | "AW" | "AX" | "AZ" | "BA" | "BB" | "BD" | "BE" | "BF" | "BG" | "BH" | "BI" | "BJ" | "BL" | "BM" | "BN" | "BO" | "BQ" | "BR" | "BS" | "BT" | "BV" | "BW" | "BY" | "BZ" | "CA" | "CC" | "CD" | "CF" | "CG" | "CH" | "CI" | "CK" | "CL" | "CM" | "CN" | "CO" | "CR" | "CU" | "CV" | "CW" | "CX" | "CY" | "CZ" | "DE" | "DJ" | "DK" | "DM" | "DO" | "DZ" | "EC" | "EE" | "EG" | "EH" | "ER" | "ES" | "ET" | "FI" | "FJ" | "FK" | "FM" | "FO" | "FR" | "GA" | "GB" | "GD" | "GE" | "GF" | "GG" | "GH" | "GI" | "GL" | "GM" | "GN" | "GP" | "GQ" | "GR" | "GS" | "GT" | "GU" | "GW" | "GY" | "HK" | "HM" | "HN" | "HR" | "HT" | "HU" | "ID" | "IE" | "IL" | "IM" | "IN" | "IO" | "IQ" | "IR" | "IS" | "IT" | "JE" | "JM" | "JO" | "JP" | "KE" | "KG" | "KH" | "KI" | "KM" | "KN" | "KP" | "KR" | "KW" | "KY" | "KZ" | "LA" | "LB" | "LC" | "LI" | "LK" | "LR" | "LS" | "LT" | "LU" | "LV" | "LY" | "MA" | "MC" | "MD" | "ME" | "MF" | "MG" | "MH" | "MK" | "ML" | "MM" | "MN" | "MO" | "MP" | "MQ" | "MR" | "MS" | "MT" | "MU" | "MV" | "MW" | "MX" | "MY" | "MZ" | "NA" | "NC" | "NE" | "NF" | "NG" | "NI" | "NL" | "NO" | "NP" | "NR" | "NU" | "NZ" | "OM" | "PA" | "PE" | "PF" | "PG" | "PH" | "PK" | "PL" | "PM" | "PN" | "PR" | "PS" | "PT" | "PW" | "PY" | "QA" | "RE" | "RO" | "RS" | "RU" | "RW" | "SA" | "SB" | "SC" | "SD" | "SE" | "SG" | "SH" | "SI" | "SJ" | "SK" | "SL" | "SM" | "SN" | "SO" | "SR" | "SS" | "ST" | "SV" | "SX" | "SY" | "SZ" | "TC" | "TD" | "TF" | "TG" | "TH" | "TJ" | "TK" | "TL" | "TM" | "TN" | "TO" | "TR" | "TT" | "TV" | "TW" | "TZ" | "UA" | "UG" | "UM" | "US" | "UY" | "UZ" | "VA" | "VC" | "VE" | "VG" | "VI" | "VN" | "VU" | "WF" | "WS" | "YE" | "YT" | "ZA" | "ZM" | "ZW"; -/** The 2-letter continent codes Cloudflare uses */ -declare type ContinentCode = "AF" | "AN" | "AS" | "EU" | "NA" | "OC" | "SA"; -type CfProperties = IncomingRequestCfProperties | RequestInitCfProperties; -interface D1Meta { - duration: number; - size_after: number; - rows_read: number; - rows_written: number; - last_row_id: number; - changed_db: boolean; - changes: number; - /** - * The region of the database instance that executed the query. - */ - served_by_region?: string; - /** - * The three letters airport code of the colo that executed the query. - */ - served_by_colo?: string; - /** - * True if-and-only-if the database instance that executed the query was the primary. - */ - served_by_primary?: boolean; - timings?: { - /** - * The duration of the SQL query execution by the database instance. It doesn't include any network time. - */ - sql_duration_ms: number; - }; - /** - * Number of total attempts to execute the query, due to automatic retries. - * Note: All other fields in the response like `timings` only apply to the last attempt. - */ - total_attempts?: number; -} -interface D1Response { - success: true; - meta: D1Meta & Record; - error?: never; -} -type D1Result = D1Response & { - results: T[]; -}; -interface D1ExecResult { - count: number; - duration: number; -} -type D1SessionConstraint = -// Indicates that the first query should go to the primary, and the rest queries -// using the same D1DatabaseSession will go to any replica that is consistent with -// the bookmark maintained by the session (returned by the first query). -'first-primary' -// Indicates that the first query can go anywhere (primary or replica), and the rest queries -// using the same D1DatabaseSession will go to any replica that is consistent with -// the bookmark maintained by the session (returned by the first query). - | 'first-unconstrained'; -type D1SessionBookmark = string; -declare abstract class D1Database { - prepare(query: string): D1PreparedStatement; - batch(statements: D1PreparedStatement[]): Promise[]>; - exec(query: string): Promise; - /** - * Creates a new D1 Session anchored at the given constraint or the bookmark. - * All queries executed using the created session will have sequential consistency, - * meaning that all writes done through the session will be visible in subsequent reads. - * - * @param constraintOrBookmark Either the session constraint or the explicit bookmark to anchor the created session. - */ - withSession(constraintOrBookmark?: D1SessionBookmark | D1SessionConstraint): D1DatabaseSession; - /** - * @deprecated dump() will be removed soon, only applies to deprecated alpha v1 databases. - */ - dump(): Promise; -} -declare abstract class D1DatabaseSession { - prepare(query: string): D1PreparedStatement; - batch(statements: D1PreparedStatement[]): Promise[]>; - /** - * @returns The latest session bookmark across all executed queries on the session. - * If no query has been executed yet, `null` is returned. - */ - getBookmark(): D1SessionBookmark | null; -} -declare abstract class D1PreparedStatement { - bind(...values: unknown[]): D1PreparedStatement; - first(colName: string): Promise; - first>(): Promise; - run>(): Promise>; - all>(): Promise>; - raw(options: { - columnNames: true; - }): Promise<[ - string[], - ...T[] - ]>; - raw(options?: { - columnNames?: false; - }): Promise; -} -// `Disposable` was added to TypeScript's standard lib types in version 5.2. -// To support older TypeScript versions, define an empty `Disposable` interface. -// Users won't be able to use `using`/`Symbol.dispose` without upgrading to 5.2, -// but this will ensure type checking on older versions still passes. -// TypeScript's interface merging will ensure our empty interface is effectively -// ignored when `Disposable` is included in the standard lib. -interface Disposable { -} -/** - * The returned data after sending an email - */ -interface EmailSendResult { - /** - * The Email Message ID - */ - messageId: string; -} -/** - * An email message that can be sent from a Worker. - */ -interface EmailMessage { - /** - * Envelope From attribute of the email message. - */ - readonly from: string; - /** - * Envelope To attribute of the email message. - */ - readonly to: string; -} -/** - * An email message that is sent to a consumer Worker and can be rejected/forwarded. - */ -interface ForwardableEmailMessage extends EmailMessage { - /** - * Stream of the email message content. - */ - readonly raw: ReadableStream; - /** - * An [Headers object](https://developer.mozilla.org/en-US/docs/Web/API/Headers). - */ - readonly headers: Headers; - /** - * Size of the email message content. - */ - readonly rawSize: number; - /** - * Reject this email message by returning a permanent SMTP error back to the connecting client including the given reason. - * @param reason The reject reason. - * @returns void - */ - setReject(reason: string): void; - /** - * Forward this email message to a verified destination address of the account. - * @param rcptTo Verified destination address. - * @param headers A [Headers object](https://developer.mozilla.org/en-US/docs/Web/API/Headers). - * @returns A promise that resolves when the email message is forwarded. - */ - forward(rcptTo: string, headers?: Headers): Promise; - /** - * Reply to the sender of this email message with a new EmailMessage object. - * @param message The reply message. - * @returns A promise that resolves when the email message is replied. - */ - reply(message: EmailMessage): Promise; -} -/** A file attachment for an email message */ -type EmailAttachment = { - disposition: 'inline'; - contentId: string; - filename: string; - type: string; - content: string | ArrayBuffer | ArrayBufferView; -} | { - disposition: 'attachment'; - contentId?: undefined; - filename: string; - type: string; - content: string | ArrayBuffer | ArrayBufferView; -}; -/** An Email Address */ -interface EmailAddress { - name: string; - email: string; -} -/** - * A binding that allows a Worker to send email messages. - */ -interface SendEmail { - send(message: EmailMessage): Promise; - send(builder: { - from: string | EmailAddress; - to: string | string[]; - subject: string; - replyTo?: string | EmailAddress; - cc?: string | string[]; - bcc?: string | string[]; - headers?: Record; - text?: string; - html?: string; - attachments?: EmailAttachment[]; - }): Promise; -} -declare abstract class EmailEvent extends ExtendableEvent { - readonly message: ForwardableEmailMessage; -} -declare type EmailExportedHandler = (message: ForwardableEmailMessage, env: Env, ctx: ExecutionContext) => void | Promise; -declare module "cloudflare:email" { - let _EmailMessage: { - prototype: EmailMessage; - new (from: string, to: string, raw: ReadableStream | string): EmailMessage; - }; - export { _EmailMessage as EmailMessage }; -} -/** - * Evaluation context for targeting rules. - * Keys are attribute names (e.g. "userId", "country"), values are the attribute values. - */ -type FlagshipEvaluationContext = Record; -interface FlagshipEvaluationDetails { - flagKey: string; - value: T; - variant?: string | undefined; - reason?: string | undefined; - errorCode?: string | undefined; - errorMessage?: string | undefined; -} -interface FlagshipEvaluationError extends Error { -} -/** - * Feature flags binding for evaluating feature flags from a Cloudflare Workers script. - * - * @example - * ```typescript - * // Get a boolean flag value with a default - * const enabled = await env.FLAGS.getBooleanValue('my-feature', false); - * - * // Get a flag value with evaluation context for targeting - * const variant = await env.FLAGS.getStringValue('experiment', 'control', { - * userId: 'user-123', - * country: 'US', - * }); - * - * // Get full evaluation details including variant and reason - * const details = await env.FLAGS.getBooleanDetails('my-feature', false); - * console.log(details.variant, details.reason); - * ``` - */ -declare abstract class Flagship { - /** - * Get a flag value without type checking. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Optional default value returned when evaluation fails. - * @param context Optional evaluation context for targeting rules. - */ - get(flagKey: string, defaultValue?: unknown, context?: FlagshipEvaluationContext): Promise; - /** - * Get a boolean flag value. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getBooleanValue(flagKey: string, defaultValue: boolean, context?: FlagshipEvaluationContext): Promise; - /** - * Get a string flag value. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getStringValue(flagKey: string, defaultValue: string, context?: FlagshipEvaluationContext): Promise; - /** - * Get a number flag value. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getNumberValue(flagKey: string, defaultValue: number, context?: FlagshipEvaluationContext): Promise; - /** - * Get an object flag value. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getObjectValue(flagKey: string, defaultValue: T, context?: FlagshipEvaluationContext): Promise; - /** - * Get a boolean flag value with full evaluation details. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getBooleanDetails(flagKey: string, defaultValue: boolean, context?: FlagshipEvaluationContext): Promise>; - /** - * Get a string flag value with full evaluation details. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getStringDetails(flagKey: string, defaultValue: string, context?: FlagshipEvaluationContext): Promise>; - /** - * Get a number flag value with full evaluation details. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getNumberDetails(flagKey: string, defaultValue: number, context?: FlagshipEvaluationContext): Promise>; - /** - * Get an object flag value with full evaluation details. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getObjectDetails(flagKey: string, defaultValue: T, context?: FlagshipEvaluationContext): Promise>; -} -/** - * Hello World binding to serve as an explanatory example. DO NOT USE - */ -interface HelloWorldBinding { - /** - * Retrieve the current stored value - */ - get(): Promise<{ - value: string; - ms?: number; - }>; - /** - * Set a new stored value - */ - set(value: string): Promise; -} -interface Hyperdrive { - /** - * Connect directly to Hyperdrive as if it's your database, returning a TCP socket. - * - * Calling this method returns an identical socket to if you call - * `connect("host:port")` using the `host` and `port` fields from this object. - * Pick whichever approach works better with your preferred DB client library. - * - * Note that this socket is not yet authenticated -- it's expected that your - * code (or preferably, the client library of your choice) will authenticate - * using the information in this class's readonly fields. - */ - connect(): Socket; - /** - * A valid DB connection string that can be passed straight into the typical - * client library/driver/ORM. This will typically be the easiest way to use - * Hyperdrive. - */ - readonly connectionString: string; - /* - * A randomly generated hostname that is only valid within the context of the - * currently running Worker which, when passed into `connect()` function from - * the "cloudflare:sockets" module, will connect to the Hyperdrive instance - * for your database. - */ - readonly host: string; - /* - * The port that must be paired the the host field when connecting. - */ - readonly port: number; - /* - * The username to use when authenticating to your database via Hyperdrive. - * Unlike the host and password, this will be the same every time - */ - readonly user: string; - /* - * The randomly generated password to use when authenticating to your - * database via Hyperdrive. Like the host field, this password is only valid - * within the context of the currently running Worker instance from which - * it's read. - */ - readonly password: string; - /* - * The name of the database to connect to. - */ - readonly database: string; -} -// Copyright (c) 2024 Cloudflare, Inc. -// Licensed under the Apache 2.0 license found in the LICENSE file or at: -// https://opensource.org/licenses/Apache-2.0 -type ImageInfoResponse = { - format: 'image/svg+xml'; -} | { - format: string; - fileSize: number; - width: number; - height: number; -}; -type ImageTransform = { - width?: number; - height?: number; - background?: string; - blur?: number; - border?: { - color?: string; - width?: number; - } | { - top?: number; - bottom?: number; - left?: number; - right?: number; - }; - brightness?: number; - contrast?: number; - fit?: 'scale-down' | 'contain' | 'pad' | 'squeeze' | 'cover' | 'crop'; - flip?: 'h' | 'v' | 'hv'; - gamma?: number; - segment?: 'foreground'; - gravity?: 'face' | 'left' | 'right' | 'top' | 'bottom' | 'center' | 'auto' | 'entropy' | { - x?: number; - y?: number; - mode: 'remainder' | 'box-center'; - }; - rotate?: 0 | 90 | 180 | 270; - saturation?: number; - sharpen?: number; - trim?: 'border' | { - top?: number; - bottom?: number; - left?: number; - right?: number; - width?: number; - height?: number; - border?: boolean | { - color?: string; - tolerance?: number; - keep?: number; - }; - }; -}; -type ImageDrawOptions = { - opacity?: number; - repeat?: boolean | string; - top?: number; - left?: number; - bottom?: number; - right?: number; -}; -type ImageInputOptions = { - encoding?: 'base64'; -}; -type ImageOutputOptions = { - format: 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp' | 'image/avif' | 'rgb' | 'rgba'; - quality?: number; - background?: string; - anim?: boolean; -}; -interface ImageMetadata { - id: string; - filename?: string; - uploaded?: string; - requireSignedURLs: boolean; - meta?: Record; - variants: string[]; - draft?: boolean; - creator?: string; -} -interface ImageUploadOptions { - id?: string; - filename?: string; - requireSignedURLs?: boolean; - metadata?: Record; - creator?: string; - encoding?: 'base64'; -} -interface ImageUpdateOptions { - requireSignedURLs?: boolean; - metadata?: Record; - creator?: string; -} -interface ImageListOptions { - limit?: number; - cursor?: string; - sortOrder?: 'asc' | 'desc'; - creator?: string; -} -interface ImageList { - images: ImageMetadata[]; - cursor?: string; - listComplete: boolean; -} -interface ImageHandle { - /** - * Get metadata for a hosted image - * @returns Image metadata, or null if not found - */ - details(): Promise; - /** - * Get the raw image data for a hosted image - * @returns ReadableStream of image bytes, or null if not found - */ - bytes(): Promise | null>; - /** - * Update hosted image metadata - * @param options Properties to update - * @returns Updated image metadata - * @throws {@link ImagesError} if update fails - */ - update(options: ImageUpdateOptions): Promise; - /** - * Delete a hosted image - * @returns True if deleted, false if not found - */ - delete(): Promise; -} -interface HostedImagesBinding { - /** - * Get a handle for a hosted image - * @param imageId The ID of the image (UUID or custom ID) - * @returns A handle for per-image operations - */ - image(imageId: string): ImageHandle; - /** - * Upload a new hosted image - * @param image The image file to upload - * @param options Upload configuration - * @returns Metadata for the uploaded image - * @throws {@link ImagesError} if upload fails - */ - upload(image: ReadableStream | ArrayBuffer, options?: ImageUploadOptions): Promise; - /** - * List hosted images with pagination - * @param options List configuration - * @returns List of images with pagination info - * @throws {@link ImagesError} if list fails - */ - list(options?: ImageListOptions): Promise; -} -interface ImagesBinding { - /** - * Get image metadata (type, width and height) - * @throws {@link ImagesError} with code 9412 if input is not an image - * @param stream The image bytes - */ - info(stream: ReadableStream, options?: ImageInputOptions): Promise; - /** - * Begin applying a series of transformations to an image - * @param stream The image bytes - * @returns A transform handle - */ - input(stream: ReadableStream, options?: ImageInputOptions): ImageTransformer; - /** - * Access hosted images CRUD operations - */ - readonly hosted: HostedImagesBinding; -} -interface ImageTransformer { - /** - * Apply transform next, returning a transform handle. - * You can then apply more transformations, draw, or retrieve the output. - * @param transform - */ - transform(transform: ImageTransform): ImageTransformer; - /** - * Draw an image on this transformer, returning a transform handle. - * You can then apply more transformations, draw, or retrieve the output. - * @param image The image (or transformer that will give the image) to draw - * @param options The options configuring how to draw the image - */ - draw(image: ReadableStream | ImageTransformer, options?: ImageDrawOptions): ImageTransformer; - /** - * Retrieve the image that results from applying the transforms to the - * provided input - * @param options Options that apply to the output e.g. output format - */ - output(options: ImageOutputOptions): Promise; -} -type ImageTransformationOutputOptions = { - encoding?: 'base64'; -}; -interface ImageTransformationResult { - /** - * The image as a response, ready to store in cache or return to users - */ - response(): Response; - /** - * The content type of the returned image - */ - contentType(): string; - /** - * The bytes of the response - */ - image(options?: ImageTransformationOutputOptions): ReadableStream; -} -interface ImagesError extends Error { - readonly code: number; - readonly message: string; - readonly stack?: string; -} -/** - * Media binding for transforming media streams. - * Provides the entry point for media transformation operations. - */ -interface MediaBinding { - /** - * Creates a media transformer from an input stream. - * @param media - The input media bytes - * @returns A MediaTransformer instance for applying transformations - */ - input(media: ReadableStream): MediaTransformer; -} -/** - * Media transformer for applying transformation operations to media content. - * Handles sizing, fitting, and other input transformation parameters. - */ -interface MediaTransformer { - /** - * Applies transformation options to the media content. - * @param transform - Configuration for how the media should be transformed - * @returns A generator for producing the transformed media output - */ - transform(transform?: MediaTransformationInputOptions): MediaTransformationGenerator; - /** - * Generates the final media output with specified options. - * @param output - Configuration for the output format and parameters - * @returns The final transformation result containing the transformed media - */ - output(output?: MediaTransformationOutputOptions): MediaTransformationResult; -} -/** - * Generator for producing media transformation results. - * Configures the output format and parameters for the transformed media. - */ -interface MediaTransformationGenerator { - /** - * Generates the final media output with specified options. - * @param output - Configuration for the output format and parameters - * @returns The final transformation result containing the transformed media - */ - output(output?: MediaTransformationOutputOptions): MediaTransformationResult; -} -/** - * Result of a media transformation operation. - * Provides multiple ways to access the transformed media content. - */ -interface MediaTransformationResult { - /** - * Returns the transformed media as a readable stream of bytes. - * @returns A promise containing a readable stream with the transformed media - */ - media(): Promise>; - /** - * Returns the transformed media as an HTTP response object. - * @returns The transformed media as a Promise, ready to store in cache or return to users - */ - response(): Promise; - /** - * Returns the MIME type of the transformed media. - * @returns A promise containing the content type string (e.g., 'image/jpeg', 'video/mp4') - */ - contentType(): Promise; -} -/** - * Configuration options for transforming media input. - * Controls how the media should be resized and fitted. - */ -type MediaTransformationInputOptions = { - /** How the media should be resized to fit the specified dimensions */ - fit?: 'contain' | 'cover' | 'scale-down'; - /** Target width in pixels */ - width?: number; - /** Target height in pixels */ - height?: number; -}; -/** - * Configuration options for Media Transformations output. - * Controls the format, timing, and type of the generated output. - */ -type MediaTransformationOutputOptions = { - /** - * Output mode determining the type of media to generate - */ - mode?: 'video' | 'spritesheet' | 'frame' | 'audio'; - /** Whether to include audio in the output */ - audio?: boolean; - /** - * Starting timestamp for frame extraction or start time for clips. (e.g. '2s'). - */ - time?: string; - /** - * Duration for video clips, audio extraction, and spritesheet generation (e.g. '5s'). - */ - duration?: string; - /** - * Number of frames in the spritesheet. - */ - imageCount?: number; - /** - * Output format for the generated media. - */ - format?: 'jpg' | 'png' | 'm4a'; -}; -/** - * Error object for media transformation operations. - * Extends the standard Error interface with additional media-specific information. - */ -interface MediaError extends Error { - readonly code: number; - readonly message: string; - readonly stack?: string; -} -declare module 'cloudflare:node' { - interface NodeStyleServer { - listen(...args: unknown[]): this; - address(): { - port?: number | null | undefined; - }; - } - export function httpServerHandler(port: number): ExportedHandler; - export function httpServerHandler(options: { - port: number; - }): ExportedHandler; - export function httpServerHandler(server: NodeStyleServer): ExportedHandler; -} -type Params

= Record; -type EventContext = { - request: Request>; - functionPath: string; - waitUntil: (promise: Promise) => void; - passThroughOnException: () => void; - next: (input?: Request | string, init?: RequestInit) => Promise; - env: Env & { - ASSETS: { - fetch: typeof fetch; - }; - }; - params: Params

; - data: Data; -}; -type PagesFunction = Record> = (context: EventContext) => Response | Promise; -type EventPluginContext = { - request: Request>; - functionPath: string; - waitUntil: (promise: Promise) => void; - passThroughOnException: () => void; - next: (input?: Request | string, init?: RequestInit) => Promise; - env: Env & { - ASSETS: { - fetch: typeof fetch; - }; - }; - params: Params

; - data: Data; - pluginArgs: PluginArgs; -}; -type PagesPluginFunction = Record, PluginArgs = unknown> = (context: EventPluginContext) => Response | Promise; -declare module "assets:*" { - export const onRequest: PagesFunction; -} -// Copyright (c) 2022-2023 Cloudflare, Inc. -// Licensed under the Apache 2.0 license found in the LICENSE file or at: -// https://opensource.org/licenses/Apache-2.0 -declare module "cloudflare:pipelines" { - export abstract class PipelineTransformationEntrypoint { - protected env: Env; - protected ctx: ExecutionContext; - constructor(ctx: ExecutionContext, env: Env); - /** - * run receives an array of PipelineRecord which can be - * transformed and returned to the pipeline - * @param records Incoming records from the pipeline to be transformed - * @param metadata Information about the specific pipeline calling the transformation entrypoint - * @returns A promise containing the transformed PipelineRecord array - */ - public run(records: I[], metadata: PipelineBatchMetadata): Promise; - } - export type PipelineRecord = Record; - export type PipelineBatchMetadata = { - pipelineId: string; - pipelineName: string; - }; - export interface Pipeline { - /** - * The Pipeline interface represents the type of a binding to a Pipeline - * - * @param records The records to send to the pipeline - */ - send(records: T[]): Promise; - } -} -// PubSubMessage represents an incoming PubSub message. -// The message includes metadata about the broker, the client, and the payload -// itself. -// https://developers.cloudflare.com/pub-sub/ -interface PubSubMessage { - // Message ID - readonly mid: number; - // MQTT broker FQDN in the form mqtts://BROKER.NAMESPACE.cloudflarepubsub.com:PORT - readonly broker: string; - // The MQTT topic the message was sent on. - readonly topic: string; - // The client ID of the client that published this message. - readonly clientId: string; - // The unique identifier (JWT ID) used by the client to authenticate, if token - // auth was used. - readonly jti?: string; - // A Unix timestamp (seconds from Jan 1, 1970), set when the Pub/Sub Broker - // received the message from the client. - readonly receivedAt: number; - // An (optional) string with the MIME type of the payload, if set by the - // client. - readonly contentType: string; - // Set to 1 when the payload is a UTF-8 string - // https://docs.oasis-open.org/mqtt/mqtt/v5.0/os/mqtt-v5.0-os.html#_Toc3901063 - readonly payloadFormatIndicator: number; - // Pub/Sub (MQTT) payloads can be UTF-8 strings, or byte arrays. - // You can use payloadFormatIndicator to inspect this before decoding. - payload: string | Uint8Array; -} -// JsonWebKey extended by kid parameter -interface JsonWebKeyWithKid extends JsonWebKey { - // Key Identifier of the JWK - readonly kid: string; -} -interface RateLimitOptions { - key: string; -} -interface RateLimitOutcome { - success: boolean; -} -interface RateLimit { - /** - * Rate limit a request based on the provided options. - * @see https://developers.cloudflare.com/workers/runtime-apis/bindings/rate-limit/ - * @returns A promise that resolves with the outcome of the rate limit. - */ - limit(options: RateLimitOptions): Promise; -} -// Namespace for RPC utility types. Unfortunately, we can't use a `module` here as these types need -// to referenced by `Fetcher`. This is included in the "importable" version of the types which -// strips all `module` blocks. -declare namespace Rpc { - // Branded types for identifying `WorkerEntrypoint`/`DurableObject`/`Target`s. - // TypeScript uses *structural* typing meaning anything with the same shape as type `T` is a `T`. - // For the classes exported by `cloudflare:workers` we want *nominal* typing (i.e. we only want to - // accept `WorkerEntrypoint` from `cloudflare:workers`, not any other class with the same shape) - export const __RPC_STUB_BRAND: '__RPC_STUB_BRAND'; - export const __RPC_TARGET_BRAND: '__RPC_TARGET_BRAND'; - export const __WORKER_ENTRYPOINT_BRAND: '__WORKER_ENTRYPOINT_BRAND'; - export const __DURABLE_OBJECT_BRAND: '__DURABLE_OBJECT_BRAND'; - export const __WORKFLOW_ENTRYPOINT_BRAND: '__WORKFLOW_ENTRYPOINT_BRAND'; - export interface RpcTargetBranded { - [__RPC_TARGET_BRAND]: never; - } - export interface WorkerEntrypointBranded { - [__WORKER_ENTRYPOINT_BRAND]: never; - } - export interface DurableObjectBranded { - [__DURABLE_OBJECT_BRAND]: never; - } - export interface WorkflowEntrypointBranded { - [__WORKFLOW_ENTRYPOINT_BRAND]: never; - } - export type EntrypointBranded = WorkerEntrypointBranded | DurableObjectBranded | WorkflowEntrypointBranded; - // Types that can be used through `Stub`s - export type Stubable = RpcTargetBranded | ((...args: any[]) => any); - // Types that can be passed over RPC - // The reason for using a generic type here is to build a serializable subset of structured - // cloneable composite types. This allows types defined with the "interface" keyword to pass the - // serializable check as well. Otherwise, only types defined with the "type" keyword would pass. - type Serializable = - // Structured cloneables - BaseType - // Structured cloneable composites - | Map ? Serializable : never, T extends Map ? Serializable : never> | Set ? Serializable : never> | ReadonlyArray ? Serializable : never> | { - [K in keyof T]: K extends number | string ? Serializable : never; - } - // Special types - | Stub - // Serialized as stubs, see `Stubify` - | Stubable; - // Base type for all RPC stubs, including common memory management methods. - // `T` is used as a marker type for unwrapping `Stub`s later. - interface StubBase extends Disposable { - [__RPC_STUB_BRAND]: T; - dup(): this; - } - export type Stub = Provider & StubBase; - // This represents all the types that can be sent as-is over an RPC boundary - type BaseType = void | undefined | null | boolean | number | bigint | string | TypedArray | ArrayBuffer | DataView | Date | Error | RegExp | ReadableStream | WritableStream | Request | Response | Headers; - // Recursively rewrite all `Stubable` types with `Stub`s - // prettier-ignore - type Stubify = T extends Stubable ? Stub : T extends Map ? Map, Stubify> : T extends Set ? Set> : T extends Array ? Array> : T extends ReadonlyArray ? ReadonlyArray> : T extends BaseType ? T : T extends { - [key: string | number]: any; - } ? { - [K in keyof T]: Stubify; - } : T; - // Recursively rewrite all `Stub`s with the corresponding `T`s. - // Note we use `StubBase` instead of `Stub` here to avoid circular dependencies: - // `Stub` depends on `Provider`, which depends on `Unstubify`, which would depend on `Stub`. - // prettier-ignore - type Unstubify = T extends StubBase ? V : T extends Map ? Map, Unstubify> : T extends Set ? Set> : T extends Array ? Array> : T extends ReadonlyArray ? ReadonlyArray> : T extends BaseType ? T : T extends { - [key: string | number]: unknown; - } ? { - [K in keyof T]: Unstubify; - } : T; - type UnstubifyAll = { - [I in keyof A]: Unstubify; - }; - // Utility type for adding `Provider`/`Disposable`s to `object` types only. - // Note `unknown & T` is equivalent to `T`. - type MaybeProvider = T extends object ? Provider : unknown; - type MaybeDisposable = T extends object ? Disposable : unknown; - // Type for method return or property on an RPC interface. - // - Stubable types are replaced by stubs. - // - Serializable types are passed by value, with stubable types replaced by stubs - // and a top-level `Disposer`. - // Everything else can't be passed over PRC. - // Technically, we use custom thenables here, but they quack like `Promise`s. - // Intersecting with `(Maybe)Provider` allows pipelining. - // prettier-ignore - type Result = R extends Stubable ? Promise> & Provider : R extends Serializable ? Promise & MaybeDisposable> & MaybeProvider : never; - // Type for method or property on an RPC interface. - // For methods, unwrap `Stub`s in parameters, and rewrite returns to be `Result`s. - // Unwrapping `Stub`s allows calling with `Stubable` arguments. - // For properties, rewrite types to be `Result`s. - // In each case, unwrap `Promise`s. - type MethodOrProperty = V extends (...args: infer P) => infer R ? (...args: UnstubifyAll

) => Result> : Result>; - // Type for the callable part of an `Provider` if `T` is callable. - // This is intersected with methods/properties. - type MaybeCallableProvider = T extends (...args: any[]) => any ? MethodOrProperty : unknown; - // Base type for all other types providing RPC-like interfaces. - // Rewrites all methods/properties to be `MethodOrProperty`s, while preserving callable types. - // `Reserved` names (e.g. stub method names like `dup()`) and symbols can't be accessed over RPC. - export type Provider = MaybeCallableProvider & Pick<{ - [K in keyof T]: MethodOrProperty; - }, Exclude>>; -} -declare namespace Cloudflare { - // Type of `env`. - // - // The specific project can extend `Env` by redeclaring it in project-specific files. Typescript - // will merge all declarations. - // - // You can use `wrangler types` to generate the `Env` type automatically. - interface Env { - } - // Project-specific parameters used to inform types. - // - // This interface is, again, intended to be declared in project-specific files, and then that - // declaration will be merged with this one. - // - // A project should have a declaration like this: - // - // interface GlobalProps { - // // Declares the main module's exports. Used to populate Cloudflare.Exports aka the type - // // of `ctx.exports`. - // mainModule: typeof import("my-main-module"); - // - // // Declares which of the main module's exports are configured with durable storage, and - // // thus should behave as Durable Object namsepace bindings. - // durableNamespaces: "MyDurableObject" | "AnotherDurableObject"; - // } - // - // You can use `wrangler types` to generate `GlobalProps` automatically. - interface GlobalProps { - } - // Evaluates to the type of a property in GlobalProps, defaulting to `Default` if it is not - // present. - type GlobalProp = K extends keyof GlobalProps ? GlobalProps[K] : Default; - // The type of the program's main module exports, if known. Requires `GlobalProps` to declare the - // `mainModule` property. - type MainModule = GlobalProp<"mainModule", {}>; - // The type of ctx.exports, which contains loopback bindings for all top-level exports. - type Exports = { - [K in keyof MainModule]: LoopbackForExport - // If the export is listed in `durableNamespaces`, then it is also a - // DurableObjectNamespace. - & (K extends GlobalProp<"durableNamespaces", never> ? MainModule[K] extends new (...args: any[]) => infer DoInstance ? DoInstance extends Rpc.DurableObjectBranded ? DurableObjectNamespace : DurableObjectNamespace : DurableObjectNamespace : {}); - }; -} -declare namespace CloudflareWorkersModule { - export type RpcStub = Rpc.Stub; - export const RpcStub: { - new (value: T): Rpc.Stub; - }; - export abstract class RpcTarget implements Rpc.RpcTargetBranded { - [Rpc.__RPC_TARGET_BRAND]: never; - } - // `protected` fields don't appear in `keyof`s, so can't be accessed over RPC - export abstract class WorkerEntrypoint implements Rpc.WorkerEntrypointBranded { - [Rpc.__WORKER_ENTRYPOINT_BRAND]: never; - protected ctx: ExecutionContext; - protected env: Env; - constructor(ctx: ExecutionContext, env: Env); - email?(message: ForwardableEmailMessage): void | Promise; - fetch?(request: Request): Response | Promise; - connect?(socket: Socket): void | Promise; - queue?(batch: MessageBatch): void | Promise; - scheduled?(controller: ScheduledController): void | Promise; - tail?(events: TraceItem[]): void | Promise; - tailStream?(event: TailStream.TailEvent): TailStream.TailEventHandlerType | Promise; - test?(controller: TestController): void | Promise; - trace?(traces: TraceItem[]): void | Promise; - } - export abstract class DurableObject implements Rpc.DurableObjectBranded { - [Rpc.__DURABLE_OBJECT_BRAND]: never; - protected ctx: DurableObjectState; - protected env: Env; - constructor(ctx: DurableObjectState, env: Env); - alarm?(alarmInfo?: AlarmInvocationInfo): void | Promise; - fetch?(request: Request): Response | Promise; - connect?(socket: Socket): void | Promise; - webSocketMessage?(ws: WebSocket, message: string | ArrayBuffer): void | Promise; - webSocketClose?(ws: WebSocket, code: number, reason: string, wasClean: boolean): void | Promise; - webSocketError?(ws: WebSocket, error: unknown): void | Promise; - } - export type WorkflowDurationLabel = 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'year'; - export type WorkflowSleepDuration = `${number} ${WorkflowDurationLabel}${'s' | ''}` | number; - export type WorkflowDelayDuration = WorkflowSleepDuration; - export type WorkflowTimeoutDuration = WorkflowSleepDuration; - export type WorkflowRetentionDuration = WorkflowSleepDuration; - export type WorkflowBackoff = 'constant' | 'linear' | 'exponential'; - export type WorkflowStepConfig = { - retries?: { - limit: number; - delay: WorkflowDelayDuration | number; - backoff?: WorkflowBackoff; - }; - timeout?: WorkflowTimeoutDuration | number; - }; - export type WorkflowEvent = { - payload: Readonly; - timestamp: Date; - instanceId: string; - }; - export type WorkflowStepEvent = { - payload: Readonly; - timestamp: Date; - type: string; - }; - export type WorkflowStepContext = { - step: { - name: string; - count: number; - }; - attempt: number; - config: WorkflowStepConfig; - }; - export abstract class WorkflowStep { - do>(name: string, callback: (ctx: WorkflowStepContext) => Promise): Promise; - do>(name: string, config: WorkflowStepConfig, callback: (ctx: WorkflowStepContext) => Promise): Promise; - sleep: (name: string, duration: WorkflowSleepDuration) => Promise; - sleepUntil: (name: string, timestamp: Date | number) => Promise; - waitForEvent>(name: string, options: { - type: string; - timeout?: WorkflowTimeoutDuration | number; - }): Promise>; - } - export type WorkflowInstanceStatus = 'queued' | 'running' | 'paused' | 'errored' | 'terminated' | 'complete' | 'waiting' | 'waitingForPause' | 'unknown'; - export abstract class WorkflowEntrypoint | unknown = unknown> implements Rpc.WorkflowEntrypointBranded { - [Rpc.__WORKFLOW_ENTRYPOINT_BRAND]: never; - protected ctx: ExecutionContext; - protected env: Env; - constructor(ctx: ExecutionContext, env: Env); - run(event: Readonly>, step: WorkflowStep): Promise; - } - export function waitUntil(promise: Promise): void; - export function withEnv(newEnv: unknown, fn: () => unknown): unknown; - export function withExports(newExports: unknown, fn: () => unknown): unknown; - export function withEnvAndExports(newEnv: unknown, newExports: unknown, fn: () => unknown): unknown; - export const env: Cloudflare.Env; - export const exports: Cloudflare.Exports; -} -declare module 'cloudflare:workers' { - export = CloudflareWorkersModule; -} -interface SecretsStoreSecret { - /** - * Get a secret from the Secrets Store, returning a string of the secret value - * if it exists, or throws an error if it does not exist - */ - get(): Promise; -} -declare module "cloudflare:sockets" { - function _connect(address: string | SocketAddress, options?: SocketOptions): Socket; - export { _connect as connect }; -} -/** - * Binding entrypoint for Cloudflare Stream. - * - * Usage: - * - Binding-level operations: - * `await env.STREAM.videos.upload` - * `await env.STREAM.videos.createDirectUpload` - * `await env.STREAM.videos.*` - * `await env.STREAM.watermarks.*` - * - Per-video operations: - * `await env.STREAM.video(id).downloads.*` - * `await env.STREAM.video(id).captions.*` - * - * Example usage: - * ```ts - * await env.STREAM.video(id).downloads.generate(); - * - * const video = env.STREAM.video(id) - * const captions = video.captions.list(); - * const videoDetails = video.details() - * ``` - */ -interface StreamBinding { - /** - * Returns a handle scoped to a single video for per-video operations. - * @param id The unique identifier for the video. - * @returns A handle for per-video operations. - */ - video(id: string): StreamVideoHandle; - /** - * Uploads a new video from a provided URL. - * @param url The URL to upload from. - * @param params Optional upload parameters. - * @returns The uploaded video details. - * @throws {BadRequestError} if the upload parameter is invalid or the URL is invalid - * @throws {QuotaReachedError} if the account storage capacity is exceeded - * @throws {MaxFileSizeError} if the file size is too large - * @throws {RateLimitedError} if the server received too many requests - * @throws {AlreadyUploadedError} if a video was already uploaded to this URL - * @throws {InternalError} if an unexpected error occurs - */ - upload(url: string, params?: StreamUrlUploadParams): Promise; - /** - * Creates a direct upload that allows video uploads without an API key. - * @param params Parameters for the direct upload - * @returns The direct upload details. - * @throws {BadRequestError} if the parameters are invalid - * @throws {RateLimitedError} if the server received too many requests - * @throws {InternalError} if an unexpected error occurs - */ - createDirectUpload(params: StreamDirectUploadCreateParams): Promise; - videos: StreamVideos; - watermarks: StreamWatermarks; -} -/** - * Handle for operations scoped to a single Stream video. - */ -interface StreamVideoHandle { - /** - * The unique identifier for the video. - */ - id: string; - /** - * Get a full videos details - * @returns The full video details. - * @throws {NotFoundError} if the video is not found - * @throws {InternalError} if an unexpected error occurs - */ - details(): Promise; - /** - * Update details for a single video. - * @param params The fields to update for the video. - * @returns The updated video details. - * @throws {NotFoundError} if the video is not found - * @throws {BadRequestError} if the parameters are invalid - * @throws {InternalError} if an unexpected error occurs - */ - update(params: StreamUpdateVideoParams): Promise; - /** - * Deletes a video and its copies from Cloudflare Stream. - * @returns A promise that resolves when deletion completes. - * @throws {NotFoundError} if the video is not found - * @throws {InternalError} if an unexpected error occurs - */ - delete(): Promise; - /** - * Creates a signed URL token for a video. - * @returns The signed token that was created. - * @throws {InternalError} if the signing key cannot be retrieved or the token cannot be signed - */ - generateToken(): Promise; - downloads: StreamScopedDownloads; - captions: StreamScopedCaptions; -} -interface StreamVideo { - /** - * The unique identifier for the video. - */ - id: string; - /** - * A user-defined identifier for the media creator. - */ - creator: string | null; - /** - * The thumbnail URL for the video. - */ - thumbnail: string; - /** - * The thumbnail timestamp percentage. - */ - thumbnailTimestampPct: number; - /** - * Indicates whether the video is ready to stream. - */ - readyToStream: boolean; - /** - * The date and time the video became ready to stream. - */ - readyToStreamAt: string | null; - /** - * Processing status information. - */ - status: StreamVideoStatus; - /** - * A user modifiable key-value store. - */ - meta: Record; - /** - * The date and time the video was created. - */ - created: string; - /** - * The date and time the video was last modified. - */ - modified: string; - /** - * The date and time at which the video will be deleted. - */ - scheduledDeletion: string | null; - /** - * The size of the video in bytes. - */ - size: number; - /** - * The preview URL for the video. - */ - preview?: string; - /** - * Origins allowed to display the video. - */ - allowedOrigins: Array; - /** - * Indicates whether signed URLs are required. - */ - requireSignedURLs: boolean | null; - /** - * The date and time the video was uploaded. - */ - uploaded: string | null; - /** - * The date and time when the upload URL expires. - */ - uploadExpiry: string | null; - /** - * The maximum size in bytes for direct uploads. - */ - maxSizeBytes: number | null; - /** - * The maximum duration in seconds for direct uploads. - */ - maxDurationSeconds: number | null; - /** - * The video duration in seconds. -1 indicates unknown. - */ - duration: number; - /** - * Input metadata for the original upload. - */ - input: StreamVideoInput; - /** - * Playback URLs for the video. - */ - hlsPlaybackUrl: string; - dashPlaybackUrl: string; - /** - * The watermark applied to the video, if any. - */ - watermark: StreamWatermark | null; - /** - * The live input id associated with the video, if any. - */ - liveInputId?: string | null; - /** - * The source video id if this is a clip. - */ - clippedFromId: string | null; - /** - * Public details associated with the video. - */ - publicDetails: StreamPublicDetails | null; -} -type StreamVideoStatus = { - /** - * The current processing state. - */ - state: string; - /** - * The current processing step. - */ - step?: string; - /** - * The percent complete as a string. - */ - pctComplete?: string; - /** - * An error reason code, if applicable. - */ - errorReasonCode: string; - /** - * An error reason text, if applicable. - */ - errorReasonText: string; -}; -type StreamVideoInput = { - /** - * The input width in pixels. - */ - width: number; - /** - * The input height in pixels. - */ - height: number; -}; -type StreamPublicDetails = { - /** - * The public title for the video. - */ - title: string | null; - /** - * The public share link. - */ - share_link: string | null; - /** - * The public channel link. - */ - channel_link: string | null; - /** - * The public logo URL. - */ - logo: string | null; -}; -type StreamDirectUpload = { - /** - * The URL an unauthenticated upload can use for a single multipart request. - */ - uploadURL: string; - /** - * A Cloudflare-generated unique identifier for a media item. - */ - id: string; - /** - * The watermark profile applied to the upload. - */ - watermark: StreamWatermark | null; - /** - * The scheduled deletion time, if any. - */ - scheduledDeletion: string | null; -}; -type StreamDirectUploadCreateParams = { - /** - * The maximum duration in seconds for a video upload. - */ - maxDurationSeconds: number; - /** - * The date and time after upload when videos will not be accepted. - */ - expiry?: string; - /** - * A user-defined identifier for the media creator. - */ - creator?: string; - /** - * A user modifiable key-value store used to reference other systems of record for - * managing videos. - */ - meta?: Record; - /** - * Lists the origins allowed to display the video. - */ - allowedOrigins?: Array; - /** - * Indicates whether the video can be accessed using the id. When set to `true`, - * a signed token must be generated with a signing key to view the video. - */ - requireSignedURLs?: boolean; - /** - * The thumbnail timestamp percentage. - */ - thumbnailTimestampPct?: number; - /** - * The date and time at which the video will be deleted. Include `null` to remove - * a scheduled deletion. - */ - scheduledDeletion?: string | null; - /** - * The watermark profile to apply. - */ - watermark?: StreamDirectUploadWatermark; -}; -type StreamDirectUploadWatermark = { - /** - * The unique identifier for the watermark profile. - */ - id: string; -}; -type StreamUrlUploadParams = { - /** - * Lists the origins allowed to display the video. Enter allowed origin - * domains in an array and use `*` for wildcard subdomains. Empty arrays allow the - * video to be viewed on any origin. - */ - allowedOrigins?: Array; - /** - * A user-defined identifier for the media creator. - */ - creator?: string; - /** - * A user modifiable key-value store used to reference other systems of - * record for managing videos. - */ - meta?: Record; - /** - * Indicates whether the video can be a accessed using the id. When - * set to `true`, a signed token must be generated with a signing key to view the - * video. - */ - requireSignedURLs?: boolean; - /** - * Indicates the date and time at which the video will be deleted. Omit - * the field to indicate no change, or include with a `null` value to remove an - * existing scheduled deletion. If specified, must be at least 30 days from upload - * time. - */ - scheduledDeletion?: string | null; - /** - * The timestamp for a thumbnail image calculated as a percentage value - * of the video's duration. To convert from a second-wise timestamp to a - * percentage, divide the desired timestamp by the total duration of the video. If - * this value is not set, the default thumbnail image is taken from 0s of the - * video. - */ - thumbnailTimestampPct?: number; - /** - * The identifier for the watermark profile - */ - watermarkId?: string; -}; -interface StreamScopedCaptions { - /** - * Uploads the caption or subtitle file to the endpoint for a specific BCP47 language. - * One caption or subtitle file per language is allowed. - * @param language The BCP 47 language tag for the caption or subtitle. - * @param input The caption or subtitle stream to upload. - * @returns The created caption entry. - * @throws {NotFoundError} if the video is not found - * @throws {BadRequestError} if the language or file is invalid - * @throws {InternalError} if an unexpected error occurs - */ - upload(language: string, input: ReadableStream): Promise; - /** - * Generate captions or subtitles for the provided language via AI. - * @param language The BCP 47 language tag to generate. - * @returns The generated caption entry. - * @throws {NotFoundError} if the video is not found - * @throws {BadRequestError} if the language is invalid - * @throws {StreamError} if a generated caption already exists - * @throws {StreamError} if the video duration is too long - * @throws {StreamError} if the video is missing audio - * @throws {StreamError} if the requested language is not supported - * @throws {InternalError} if an unexpected error occurs - */ - generate(language: string): Promise; - /** - * Lists the captions or subtitles. - * Use the language parameter to filter by a specific language. - * @param language The optional BCP 47 language tag to filter by. - * @returns The list of captions or subtitles. - * @throws {NotFoundError} if the video or caption is not found - * @throws {InternalError} if an unexpected error occurs - */ - list(language?: string): Promise; - /** - * Removes the captions or subtitles from a video. - * @param language The BCP 47 language tag to remove. - * @returns A promise that resolves when deletion completes. - * @throws {NotFoundError} if the video or caption is not found - * @throws {InternalError} if an unexpected error occurs - */ - delete(language: string): Promise; -} -interface StreamScopedDownloads { - /** - * Generates a download for a video when a video is ready to view. Available - * types are `default` and `audio`. Defaults to `default` when omitted. - * @param downloadType The download type to create. - * @returns The current downloads for the video. - * @throws {NotFoundError} if the video is not found - * @throws {BadRequestError} if the download type is invalid - * @throws {StreamError} if the video duration is too long to generate a download - * @throws {StreamError} if the video is not ready to stream - * @throws {InternalError} if an unexpected error occurs - */ - generate(downloadType?: StreamDownloadType): Promise; - /** - * Lists the downloads created for a video. - * @returns The current downloads for the video. - * @throws {NotFoundError} if the video or downloads are not found - * @throws {InternalError} if an unexpected error occurs - */ - get(): Promise; - /** - * Delete the downloads for a video. Available types are `default` and `audio`. - * Defaults to `default` when omitted. - * @param downloadType The download type to delete. - * @returns A promise that resolves when deletion completes. - * @throws {NotFoundError} if the video or downloads are not found - * @throws {InternalError} if an unexpected error occurs - */ - delete(downloadType?: StreamDownloadType): Promise; -} -interface StreamVideos { - /** - * Lists all videos in a users account. - * @returns The list of videos. - * @throws {BadRequestError} if the parameters are invalid - * @throws {InternalError} if an unexpected error occurs - */ - list(params?: StreamVideosListParams): Promise; -} -interface StreamWatermarks { - /** - * Generate a new watermark profile - * @param input The image stream to upload - * @param params The watermark creation parameters. - * @returns The created watermark profile. - * @throws {BadRequestError} if the parameters are invalid - * @throws {InvalidURLError} if the URL is invalid - * @throws {TooManyWatermarksError} if the number of allowed watermarks is reached - * @throws {InternalError} if an unexpected error occurs - */ - generate(input: ReadableStream, params: StreamWatermarkCreateParams): Promise; - /** - * Generate a new watermark profile - * @param url The image url to upload - * @param params The watermark creation parameters. - * @returns The created watermark profile. - * @throws {BadRequestError} if the parameters are invalid - * @throws {InvalidURLError} if the URL is invalid - * @throws {TooManyWatermarksError} if the number of allowed watermarks is reached - * @throws {InternalError} if an unexpected error occurs - */ - generate(url: string, params: StreamWatermarkCreateParams): Promise; - /** - * Lists all watermark profiles for an account. - * @returns The list of watermark profiles. - * @throws {InternalError} if an unexpected error occurs - */ - list(): Promise; - /** - * Retrieves details for a single watermark profile. - * @param watermarkId The watermark profile identifier. - * @returns The watermark profile details. - * @throws {NotFoundError} if the watermark is not found - * @throws {InternalError} if an unexpected error occurs - */ - get(watermarkId: string): Promise; - /** - * Deletes a watermark profile. - * @param watermarkId The watermark profile identifier. - * @returns A promise that resolves when deletion completes. - * @throws {NotFoundError} if the watermark is not found - * @throws {InternalError} if an unexpected error occurs - */ - delete(watermarkId: string): Promise; -} -type StreamUpdateVideoParams = { - /** - * Lists the origins allowed to display the video. Enter allowed origin - * domains in an array and use `*` for wildcard subdomains. Empty arrays allow the - * video to be viewed on any origin. - */ - allowedOrigins?: Array; - /** - * A user-defined identifier for the media creator. - */ - creator?: string; - /** - * The maximum duration in seconds for a video upload. Can be set for a - * video that is not yet uploaded to limit its duration. Uploads that exceed the - * specified duration will fail during processing. A value of `-1` means the value - * is unknown. - */ - maxDurationSeconds?: number; - /** - * A user modifiable key-value store used to reference other systems of - * record for managing videos. - */ - meta?: Record; - /** - * Indicates whether the video can be a accessed using the id. When - * set to `true`, a signed token must be generated with a signing key to view the - * video. - */ - requireSignedURLs?: boolean; - /** - * Indicates the date and time at which the video will be deleted. Omit - * the field to indicate no change, or include with a `null` value to remove an - * existing scheduled deletion. If specified, must be at least 30 days from upload - * time. - */ - scheduledDeletion?: string | null; - /** - * The timestamp for a thumbnail image calculated as a percentage value - * of the video's duration. To convert from a second-wise timestamp to a - * percentage, divide the desired timestamp by the total duration of the video. If - * this value is not set, the default thumbnail image is taken from 0s of the - * video. - */ - thumbnailTimestampPct?: number; -}; -type StreamCaption = { - /** - * Whether the caption was generated via AI. - */ - generated?: boolean; - /** - * The language label displayed in the native language to users. - */ - label: string; - /** - * The language tag in BCP 47 format. - */ - language: string; - /** - * The status of a generated caption. - */ - status?: 'ready' | 'inprogress' | 'error'; -}; -type StreamDownloadStatus = 'ready' | 'inprogress' | 'error'; -type StreamDownloadType = 'default' | 'audio'; -type StreamDownload = { - /** - * Indicates the progress as a percentage between 0 and 100. - */ - percentComplete: number; - /** - * The status of a generated download. - */ - status: StreamDownloadStatus; - /** - * The URL to access the generated download. - */ - url?: string; -}; -/** - * An object with download type keys. Each key is optional and only present if that - * download type has been created. - */ -type StreamDownloadGetResponse = { - /** - * The audio-only download. Only present if this download type has been created. - */ - audio?: StreamDownload; - /** - * The default video download. Only present if this download type has been created. - */ - default?: StreamDownload; -}; -type StreamWatermarkPosition = 'upperRight' | 'upperLeft' | 'lowerLeft' | 'lowerRight' | 'center'; -type StreamWatermark = { - /** - * The unique identifier for a watermark profile. - */ - id: string; - /** - * The size of the image in bytes. - */ - size: number; - /** - * The height of the image in pixels. - */ - height: number; - /** - * The width of the image in pixels. - */ - width: number; - /** - * The date and a time a watermark profile was created. - */ - created: string; - /** - * The source URL for a downloaded image. If the watermark profile was created via - * direct upload, this field is null. - */ - downloadedFrom: string | null; - /** - * A short description of the watermark profile. - */ - name: string; - /** - * The translucency of the image. A value of `0.0` makes the image completely - * transparent, and `1.0` makes the image completely opaque. Note that if the image - * is already semi-transparent, setting this to `1.0` will not make the image - * completely opaque. - */ - opacity: number; - /** - * The whitespace between the adjacent edges (determined by position) of the video - * and the image. `0.0` indicates no padding, and `1.0` indicates a fully padded - * video width or length, as determined by the algorithm. - */ - padding: number; - /** - * The size of the image relative to the overall size of the video. This parameter - * will adapt to horizontal and vertical videos automatically. `0.0` indicates no - * scaling (use the size of the image as-is), and `1.0 `fills the entire video. - */ - scale: number; - /** - * The location of the image. Valid positions are: `upperRight`, `upperLeft`, - * `lowerLeft`, `lowerRight`, and `center`. Note that `center` ignores the - * `padding` parameter. - */ - position: StreamWatermarkPosition; -}; -type StreamWatermarkCreateParams = { - /** - * A short description of the watermark profile. - */ - name?: string; - /** - * The translucency of the image. A value of `0.0` makes the image completely - * transparent, and `1.0` makes the image completely opaque. Note that if the - * image is already semi-transparent, setting this to `1.0` will not make the - * image completely opaque. - */ - opacity?: number; - /** - * The whitespace between the adjacent edges (determined by position) of the - * video and the image. `0.0` indicates no padding, and `1.0` indicates a fully - * padded video width or length, as determined by the algorithm. - */ - padding?: number; - /** - * The size of the image relative to the overall size of the video. This - * parameter will adapt to horizontal and vertical videos automatically. `0.0` - * indicates no scaling (use the size of the image as-is), and `1.0 `fills the - * entire video. - */ - scale?: number; - /** - * The location of the image. - */ - position?: StreamWatermarkPosition; -}; -type StreamVideosListParams = { - /** - * The maximum number of videos to return. - */ - limit?: number; - /** - * Return videos created before this timestamp. - * (RFC3339/RFC3339Nano) - */ - before?: string; - /** - * Comparison operator for the `before` field. - * @default 'lt' - */ - beforeComp?: StreamPaginationComparison; - /** - * Return videos created after this timestamp. - * (RFC3339/RFC3339Nano) - */ - after?: string; - /** - * Comparison operator for the `after` field. - * @default 'gte' - */ - afterComp?: StreamPaginationComparison; -}; -type StreamPaginationComparison = 'eq' | 'gt' | 'gte' | 'lt' | 'lte'; -/** - * Error object for Stream binding operations. - */ -interface StreamError extends Error { - readonly code: number; - readonly statusCode: number; - readonly message: string; - readonly stack?: string; -} -interface InternalError extends StreamError { - name: 'InternalError'; -} -interface BadRequestError extends StreamError { - name: 'BadRequestError'; -} -interface NotFoundError extends StreamError { - name: 'NotFoundError'; -} -interface ForbiddenError extends StreamError { - name: 'ForbiddenError'; -} -interface RateLimitedError extends StreamError { - name: 'RateLimitedError'; -} -interface QuotaReachedError extends StreamError { - name: 'QuotaReachedError'; -} -interface MaxFileSizeError extends StreamError { - name: 'MaxFileSizeError'; -} -interface InvalidURLError extends StreamError { - name: 'InvalidURLError'; -} -interface AlreadyUploadedError extends StreamError { - name: 'AlreadyUploadedError'; -} -interface TooManyWatermarksError extends StreamError { - name: 'TooManyWatermarksError'; -} -type MarkdownDocument = { - name: string; - blob: Blob; -}; -type ConversionResponse = { - id: string; - name: string; - mimeType: string; - format: 'markdown'; - tokens: number; - data: string; -} | { - id: string; - name: string; - mimeType: string; - format: 'error'; - error: string; -}; -type ImageConversionOptions = { - descriptionLanguage?: 'en' | 'es' | 'fr' | 'it' | 'pt' | 'de'; -}; -type EmbeddedImageConversionOptions = ImageConversionOptions & { - convert?: boolean; - maxConvertedImages?: number; -}; -type ConversionOptions = { - html?: { - images?: EmbeddedImageConversionOptions & { - convertOGImage?: boolean; - }; - hostname?: string; - cssSelector?: string; - }; - docx?: { - images?: EmbeddedImageConversionOptions; - }; - image?: ImageConversionOptions; - pdf?: { - images?: EmbeddedImageConversionOptions; - metadata?: boolean; - }; -}; -type ConversionRequestOptions = { - gateway?: GatewayOptions; - extraHeaders?: object; - conversionOptions?: ConversionOptions; -}; -type SupportedFileFormat = { - mimeType: string; - extension: string; -}; -declare abstract class ToMarkdownService { - transform(files: MarkdownDocument[], options?: ConversionRequestOptions): Promise; - transform(files: MarkdownDocument, options?: ConversionRequestOptions): Promise; - supported(): Promise; -} -declare namespace TailStream { - interface Header { - readonly name: string; - readonly value: string; - } - interface FetchEventInfo { - readonly type: "fetch"; - readonly method: string; - readonly url: string; - readonly cfJson?: object; - readonly headers: Header[]; - } - interface JsRpcEventInfo { - readonly type: "jsrpc"; - } - interface ScheduledEventInfo { - readonly type: "scheduled"; - readonly scheduledTime: Date; - readonly cron: string; - } - interface AlarmEventInfo { - readonly type: "alarm"; - readonly scheduledTime: Date; - } - interface QueueEventInfo { - readonly type: "queue"; - readonly queueName: string; - readonly batchSize: number; - } - interface EmailEventInfo { - readonly type: "email"; - readonly mailFrom: string; - readonly rcptTo: string; - readonly rawSize: number; - } - interface TraceEventInfo { - readonly type: "trace"; - readonly traces: (string | null)[]; - } - interface HibernatableWebSocketEventInfoMessage { - readonly type: "message"; - } - interface HibernatableWebSocketEventInfoError { - readonly type: "error"; - } - interface HibernatableWebSocketEventInfoClose { - readonly type: "close"; - readonly code: number; - readonly wasClean: boolean; - } - interface HibernatableWebSocketEventInfo { - readonly type: "hibernatableWebSocket"; - readonly info: HibernatableWebSocketEventInfoClose | HibernatableWebSocketEventInfoError | HibernatableWebSocketEventInfoMessage; - } - interface CustomEventInfo { - readonly type: "custom"; - } - interface FetchResponseInfo { - readonly type: "fetch"; - readonly statusCode: number; - } - interface ConnectEventInfo { - readonly type: "connect"; - } - type EventOutcome = "ok" | "canceled" | "exception" | "unknown" | "killSwitch" | "daemonDown" | "exceededCpu" | "exceededMemory" | "loadShed" | "responseStreamDisconnected" | "scriptNotFound"; - interface ScriptVersion { - readonly id: string; - readonly tag?: string; - readonly message?: string; - } - interface TracePreviewInfo { - readonly id: string; - readonly slug: string; - readonly name: string; - } - interface Onset { - readonly type: "onset"; - readonly attributes: Attribute[]; - // id for the span being opened by this Onset event. - readonly spanId: string; - readonly dispatchNamespace?: string; - readonly entrypoint?: string; - readonly executionModel: string; - readonly scriptName?: string; - readonly scriptTags?: string[]; - readonly scriptVersion?: ScriptVersion; - readonly preview?: TracePreviewInfo; - readonly info: FetchEventInfo | ConnectEventInfo | JsRpcEventInfo | ScheduledEventInfo | AlarmEventInfo | QueueEventInfo | EmailEventInfo | TraceEventInfo | HibernatableWebSocketEventInfo | CustomEventInfo; - } - interface Outcome { - readonly type: "outcome"; - readonly outcome: EventOutcome; - readonly cpuTime: number; - readonly wallTime: number; - } - interface SpanOpen { - readonly type: "spanOpen"; - readonly name: string; - // id for the span being opened by this SpanOpen event. - readonly spanId: string; - readonly info?: FetchEventInfo | JsRpcEventInfo | Attributes; - } - interface SpanClose { - readonly type: "spanClose"; - readonly outcome: EventOutcome; - } - interface DiagnosticChannelEvent { - readonly type: "diagnosticChannel"; - readonly channel: string; - readonly message: any; - } - interface Exception { - readonly type: "exception"; - readonly name: string; - readonly message: string; - readonly stack?: string; - } - interface Log { - readonly type: "log"; - readonly level: "debug" | "error" | "info" | "log" | "warn"; - readonly message: object; - } - interface DroppedEventsDiagnostic { - readonly diagnosticsType: "droppedEvents"; - readonly count: number; - } - interface StreamDiagnostic { - readonly type: 'streamDiagnostic'; - // To add new diagnostic types, define a new interface and add it to this union type. - readonly diagnostic: DroppedEventsDiagnostic; - } - // This marks the worker handler return information. - // This is separate from Outcome because the worker invocation can live for a long time after - // returning. For example - Websockets that return an http upgrade response but then continue - // streaming information or SSE http connections. - interface Return { - readonly type: "return"; - readonly info?: FetchResponseInfo; - } - interface Attribute { - readonly name: string; - readonly value: string | string[] | boolean | boolean[] | number | number[] | bigint | bigint[]; - } - interface Attributes { - readonly type: "attributes"; - readonly info: Attribute[]; - } - type EventType = Onset | Outcome | SpanOpen | SpanClose | DiagnosticChannelEvent | Exception | Log | StreamDiagnostic | Return | Attributes; - // Context in which this trace event lives. - interface SpanContext { - // Single id for the entire top-level invocation - // This should be a new traceId for the first worker stage invoked in the eyeball request and then - // same-account service-bindings should reuse the same traceId but cross-account service-bindings - // should use a new traceId. - readonly traceId: string; - // spanId in which this event is handled - // for Onset and SpanOpen events this would be the parent span id - // for Outcome and SpanClose these this would be the span id of the opening Onset and SpanOpen events - // For Hibernate and Mark this would be the span under which they were emitted. - // spanId is not set ONLY if: - // 1. This is an Onset event - // 2. We are not inheriting any SpanContext. (e.g. this is a cross-account service binding or a new top-level invocation) - readonly spanId?: string; - } - interface TailEvent { - // invocation id of the currently invoked worker stage. - // invocation id will always be unique to every Onset event and will be the same until the Outcome event. - readonly invocationId: string; - // Inherited spanContext for this event. - readonly spanContext: SpanContext; - readonly timestamp: Date; - readonly sequence: number; - readonly event: Event; - } - type TailEventHandler = (event: TailEvent) => void | Promise; - type TailEventHandlerObject = { - outcome?: TailEventHandler; - spanOpen?: TailEventHandler; - spanClose?: TailEventHandler; - diagnosticChannel?: TailEventHandler; - exception?: TailEventHandler; - log?: TailEventHandler; - return?: TailEventHandler; - attributes?: TailEventHandler; - }; - type TailEventHandlerType = TailEventHandler | TailEventHandlerObject; -} -// Copyright (c) 2022-2023 Cloudflare, Inc. -// Licensed under the Apache 2.0 license found in the LICENSE file or at: -// https://opensource.org/licenses/Apache-2.0 -/** - * Data types supported for holding vector metadata. - */ -type VectorizeVectorMetadataValue = string | number | boolean | string[]; -/** - * Additional information to associate with a vector. - */ -type VectorizeVectorMetadata = VectorizeVectorMetadataValue | Record; -type VectorFloatArray = Float32Array | Float64Array; -interface VectorizeError { - code?: number; - error: string; -} -/** - * Comparison logic/operation to use for metadata filtering. - * - * This list is expected to grow as support for more operations are released. - */ -type VectorizeVectorMetadataFilterOp = '$eq' | '$ne' | '$lt' | '$lte' | '$gt' | '$gte'; -type VectorizeVectorMetadataFilterCollectionOp = '$in' | '$nin'; -/** - * Filter criteria for vector metadata used to limit the retrieved query result set. - */ -type VectorizeVectorMetadataFilter = { - [field: string]: Exclude | null | { - [Op in VectorizeVectorMetadataFilterOp]?: Exclude | null; - } | { - [Op in VectorizeVectorMetadataFilterCollectionOp]?: Exclude[]; - }; -}; -/** - * Supported distance metrics for an index. - * Distance metrics determine how other "similar" vectors are determined. - */ -type VectorizeDistanceMetric = "euclidean" | "cosine" | "dot-product"; -/** - * Metadata return levels for a Vectorize query. - * - * Default to "none". - * - * @property all Full metadata for the vector return set, including all fields (including those un-indexed) without truncation. This is a more expensive retrieval, as it requires additional fetching & reading of un-indexed data. - * @property indexed Return all metadata fields configured for indexing in the vector return set. This level of retrieval is "free" in that no additional overhead is incurred returning this data. However, note that indexed metadata is subject to truncation (especially for larger strings). - * @property none No indexed metadata will be returned. - */ -type VectorizeMetadataRetrievalLevel = "all" | "indexed" | "none"; -interface VectorizeQueryOptions { - topK?: number; - namespace?: string; - returnValues?: boolean; - returnMetadata?: boolean | VectorizeMetadataRetrievalLevel; - filter?: VectorizeVectorMetadataFilter; -} -/** - * Information about the configuration of an index. - */ -type VectorizeIndexConfig = { - dimensions: number; - metric: VectorizeDistanceMetric; -} | { - preset: string; // keep this generic, as we'll be adding more presets in the future and this is only in a read capacity -}; -/** - * Metadata about an existing index. - * - * This type is exclusively for the Vectorize **beta** and will be deprecated once Vectorize RC is released. - * See {@link VectorizeIndexInfo} for its post-beta equivalent. - */ -interface VectorizeIndexDetails { - /** The unique ID of the index */ - readonly id: string; - /** The name of the index. */ - name: string; - /** (optional) A human readable description for the index. */ - description?: string; - /** The index configuration, including the dimension size and distance metric. */ - config: VectorizeIndexConfig; - /** The number of records containing vectors within the index. */ - vectorsCount: number; -} -/** - * Metadata about an existing index. - */ -interface VectorizeIndexInfo { - /** The number of records containing vectors within the index. */ - vectorCount: number; - /** Number of dimensions the index has been configured for. */ - dimensions: number; - /** ISO 8601 datetime of the last processed mutation on in the index. All changes before this mutation will be reflected in the index state. */ - processedUpToDatetime: number; - /** UUIDv4 of the last mutation processed by the index. All changes before this mutation will be reflected in the index state. */ - processedUpToMutation: number; -} -/** - * Represents a single vector value set along with its associated metadata. - */ -interface VectorizeVector { - /** The ID for the vector. This can be user-defined, and must be unique. It should uniquely identify the object, and is best set based on the ID of what the vector represents. */ - id: string; - /** The vector values */ - values: VectorFloatArray | number[]; - /** The namespace this vector belongs to. */ - namespace?: string; - /** Metadata associated with the vector. Includes the values of other fields and potentially additional details. */ - metadata?: Record; -} -/** - * Represents a matched vector for a query along with its score and (if specified) the matching vector information. - */ -type VectorizeMatch = Pick, "values"> & Omit & { - /** The score or rank for similarity, when returned as a result */ - score: number; -}; -/** - * A set of matching {@link VectorizeMatch} for a particular query. - */ -interface VectorizeMatches { - matches: VectorizeMatch[]; - count: number; -} -/** - * Results of an operation that performed a mutation on a set of vectors. - * Here, `ids` is a list of vectors that were successfully processed. - * - * This type is exclusively for the Vectorize **beta** and will be deprecated once Vectorize RC is released. - * See {@link VectorizeAsyncMutation} for its post-beta equivalent. - */ -interface VectorizeVectorMutation { - /* List of ids of vectors that were successfully processed. */ - ids: string[]; - /* Total count of the number of processed vectors. */ - count: number; -} -/** - * Result type indicating a mutation on the Vectorize Index. - * Actual mutations are processed async where the `mutationId` is the unique identifier for the operation. - */ -interface VectorizeAsyncMutation { - /** The unique identifier for the async mutation operation containing the changeset. */ - mutationId: string; -} -/** - * A Vectorize Vector Search Index for querying vectors/embeddings. - * - * This type is exclusively for the Vectorize **beta** and will be deprecated once Vectorize RC is released. - * See {@link Vectorize} for its new implementation. - */ -declare abstract class VectorizeIndex { - /** - * Get information about the currently bound index. - * @returns A promise that resolves with information about the current index. - */ - public describe(): Promise; - /** - * Use the provided vector to perform a similarity search across the index. - * @param vector Input vector that will be used to drive the similarity search. - * @param options Configuration options to massage the returned data. - * @returns A promise that resolves with matched and scored vectors. - */ - public query(vector: VectorFloatArray | number[], options?: VectorizeQueryOptions): Promise; - /** - * Insert a list of vectors into the index dataset. If a provided id exists, an error will be thrown. - * @param vectors List of vectors that will be inserted. - * @returns A promise that resolves with the ids & count of records that were successfully processed. - */ - public insert(vectors: VectorizeVector[]): Promise; - /** - * Upsert a list of vectors into the index dataset. If a provided id exists, it will be replaced with the new values. - * @param vectors List of vectors that will be upserted. - * @returns A promise that resolves with the ids & count of records that were successfully processed. - */ - public upsert(vectors: VectorizeVector[]): Promise; - /** - * Delete a list of vectors with a matching id. - * @param ids List of vector ids that should be deleted. - * @returns A promise that resolves with the ids & count of records that were successfully processed (and thus deleted). - */ - public deleteByIds(ids: string[]): Promise; - /** - * Get a list of vectors with a matching id. - * @param ids List of vector ids that should be returned. - * @returns A promise that resolves with the raw unscored vectors matching the id set. - */ - public getByIds(ids: string[]): Promise; -} -/** - * A Vectorize Vector Search Index for querying vectors/embeddings. - * - * Mutations in this version are async, returning a mutation id. - */ -declare abstract class Vectorize { - /** - * Get information about the currently bound index. - * @returns A promise that resolves with information about the current index. - */ - public describe(): Promise; - /** - * Use the provided vector to perform a similarity search across the index. - * @param vector Input vector that will be used to drive the similarity search. - * @param options Configuration options to massage the returned data. - * @returns A promise that resolves with matched and scored vectors. - */ - public query(vector: VectorFloatArray | number[], options?: VectorizeQueryOptions): Promise; - /** - * Use the provided vector-id to perform a similarity search across the index. - * @param vectorId Id for a vector in the index against which the index should be queried. - * @param options Configuration options to massage the returned data. - * @returns A promise that resolves with matched and scored vectors. - */ - public queryById(vectorId: string, options?: VectorizeQueryOptions): Promise; - /** - * Insert a list of vectors into the index dataset. If a provided id exists, an error will be thrown. - * @param vectors List of vectors that will be inserted. - * @returns A promise that resolves with a unique identifier of a mutation containing the insert changeset. - */ - public insert(vectors: VectorizeVector[]): Promise; - /** - * Upsert a list of vectors into the index dataset. If a provided id exists, it will be replaced with the new values. - * @param vectors List of vectors that will be upserted. - * @returns A promise that resolves with a unique identifier of a mutation containing the upsert changeset. - */ - public upsert(vectors: VectorizeVector[]): Promise; - /** - * Delete a list of vectors with a matching id. - * @param ids List of vector ids that should be deleted. - * @returns A promise that resolves with a unique identifier of a mutation containing the delete changeset. - */ - public deleteByIds(ids: string[]): Promise; - /** - * Get a list of vectors with a matching id. - * @param ids List of vector ids that should be returned. - * @returns A promise that resolves with the raw unscored vectors matching the id set. - */ - public getByIds(ids: string[]): Promise; -} -/** - * The interface for "version_metadata" binding - * providing metadata about the Worker Version using this binding. - */ -type WorkerVersionMetadata = { - /** The ID of the Worker Version using this binding */ - id: string; - /** The tag of the Worker Version using this binding */ - tag: string; - /** The timestamp of when the Worker Version was uploaded */ - timestamp: string; -}; -interface DynamicDispatchLimits { - /** - * Limit CPU time in milliseconds. - */ - cpuMs?: number; - /** - * Limit number of subrequests. - */ - subRequests?: number; -} -interface DynamicDispatchOptions { - /** - * Limit resources of invoked Worker script. - */ - limits?: DynamicDispatchLimits; - /** - * Arguments for outbound Worker script, if configured. - */ - outbound?: { - [key: string]: any; - }; -} -interface DispatchNamespace { - /** - * @param name Name of the Worker script. - * @param args Arguments to Worker script. - * @param options Options for Dynamic Dispatch invocation. - * @returns A Fetcher object that allows you to send requests to the Worker script. - * @throws If the Worker script does not exist in this dispatch namespace, an error will be thrown. - */ - get(name: string, args?: { - [key: string]: any; - }, options?: DynamicDispatchOptions): Fetcher; -} -declare module 'cloudflare:workflows' { - /** - * NonRetryableError allows for a user to throw a fatal error - * that makes a Workflow instance fail immediately without triggering a retry - */ - export class NonRetryableError extends Error { - public constructor(message: string, name?: string); - } -} -declare abstract class Workflow { - /** - * Get a handle to an existing instance of the Workflow. - * @param id Id for the instance of this Workflow - * @returns A promise that resolves with a handle for the Instance - */ - public get(id: string): Promise; - /** - * Create a new instance and return a handle to it. If a provided id exists, an error will be thrown. - * @param options Options when creating an instance including id and params - * @returns A promise that resolves with a handle for the Instance - */ - public create(options?: WorkflowInstanceCreateOptions): Promise; - /** - * Create a batch of instances and return handle for all of them. If a provided id exists, an error will be thrown. - * `createBatch` is limited at 100 instances at a time or when the RPC limit for the batch (1MiB) is reached. - * @param batch List of Options when creating an instance including name and params - * @returns A promise that resolves with a list of handles for the created instances. - */ - public createBatch(batch: WorkflowInstanceCreateOptions[]): Promise; -} -type WorkflowDurationLabel = 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'year'; -type WorkflowSleepDuration = `${number} ${WorkflowDurationLabel}${'s' | ''}` | number; -type WorkflowRetentionDuration = WorkflowSleepDuration; -interface WorkflowInstanceCreateOptions { - /** - * An id for your Workflow instance. Must be unique within the Workflow. - */ - id?: string; - /** - * The event payload the Workflow instance is triggered with - */ - params?: PARAMS; - /** - * The retention policy for Workflow instance. - * Defaults to the maximum retention period available for the owner's account. - */ - retention?: { - successRetention?: WorkflowRetentionDuration; - errorRetention?: WorkflowRetentionDuration; - }; -} -type InstanceStatus = { - status: 'queued' // means that instance is waiting to be started (see concurrency limits) - | 'running' | 'paused' | 'errored' | 'terminated' // user terminated the instance while it was running - | 'complete' | 'waiting' // instance is hibernating and waiting for sleep or event to finish - | 'waitingForPause' // instance is finishing the current work to pause - | 'unknown'; - error?: { - name: string; - message: string; - }; - output?: unknown; -}; -interface WorkflowError { - code?: number; - message: string; -} -declare abstract class WorkflowInstance { - public id: string; - /** - * Pause the instance. - */ - public pause(): Promise; - /** - * Resume the instance. If it is already running, an error will be thrown. - */ - public resume(): Promise; - /** - * Terminate the instance. If it is errored, terminated or complete, an error will be thrown. - */ - public terminate(): Promise; - /** - * Restart the instance. - */ - public restart(): Promise; - /** - * Returns the current status of the instance. - */ - public status(): Promise; - /** - * Send an event to this instance. - */ - public sendEvent({ type, payload, }: { - type: string; - payload: unknown; - }): Promise; -} diff --git a/workers/cache/wrangler.jsonc b/workers/cache/wrangler.jsonc deleted file mode 100644 index 0fa8c8c..0000000 --- a/workers/cache/wrangler.jsonc +++ /dev/null @@ -1,31 +0,0 @@ -{ - "$schema": "node_modules/wrangler/config-schema.json", - "name": "helios-cache", - "main": "src/index.ts", - "compatibility_date": "2026-03-10", - "observability": { - "enabled": true - }, - "upload_source_maps": true, - "workers_dev": true, - "vars": { - "SIGNING_KEY_NAME": "helios-cache-1" - }, - "r2_buckets": [ - { - "binding": "CACHE_BUCKET", - "bucket_name": "helios-cache" - } - ], - "d1_databases": [ - { - "binding": "CACHE_DB", - "database_name": "helios-cache", - "database_id": "", - "migrations_dir": "migrations" - } - ], - "triggers": { - "crons": ["0 */6 * * *"] - } -} diff --git a/wrangler.jsonc b/wrangler.jsonc deleted file mode 100644 index 1bdd92f..0000000 --- a/wrangler.jsonc +++ /dev/null @@ -1,31 +0,0 @@ -{ - "$schema": "node_modules/wrangler/config-schema.json", - "name": "helios-cache", - "main": "workers/cache/src/index.ts", - "compatibility_date": "2026-03-10", - "observability": { - "enabled": true - }, - "upload_source_maps": true, - "workers_dev": true, - "vars": { - "SIGNING_KEY_NAME": "helios-cache-1" - }, - "r2_buckets": [ - { - "binding": "CACHE_BUCKET", - "bucket_name": "helios-cache" - } - ], - "d1_databases": [ - { - "binding": "CACHE_DB", - "database_name": "helios-cache", - "database_id": "", - "migrations_dir": "workers/cache/migrations" - } - ], - "triggers": { - "crons": ["0 */6 * * *"] - } -}