From e1eed9b8baf7652e9eb31d2fe380a0354911606b Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 12:23:53 +0200 Subject: [PATCH 01/56] pkg: add zig packages for nix formats Add standalone Zig packages for the Nix formats a binary cache needs: nix-base32, nix-store-path, nix-archive, nix-narinfo, nix-derivation and nix-flake-lock. Each builds and tests on its own with zig build test, and scripts/test-zig.sh runs them all. nix-archive serialises store paths to NAR straight from the kernel with openat, getdents64 and statx, mmapping large files, and hashes and zstd-compresses the stream in one pass. Its SHA-256 keeps the SHA-NI state in registers across blocks, which matches OpenSSL where the std implementation, reloading state for every block, is 12% slower. The same package verifies uploads by decompressing and hashing them as they stream in. nix-narinfo validates every field before rendering, so no input can inject narinfo lines, and signs with a fixed-base comb that needs a quarter of the point operations of std's Ed25519 base multiplication. The scanning and validation loops in base32, store paths, narinfo, derivations and flake.lock use @Vector, each with a scalar reference and a test that the two agree. nix-flake-lock produces output byte-identical to the nix-flake-lock crate. --- pkg/nix-archive/build.zig | 17 + pkg/nix-archive/build.zig.zon | 9 + pkg/nix-archive/src/root.zig | 31 + pkg/nix-archive/src/sha256.zig | 173 ++++ pkg/nix-archive/src/sink.zig | 161 ++++ pkg/nix-archive/src/tests.zig | 88 ++ pkg/nix-archive/src/verify.zig | 79 ++ pkg/nix-archive/src/walk.zig | 202 +++++ pkg/nix-archive/src/zstd.zig | 42 + pkg/nix-base32/build.zig | 15 + pkg/nix-base32/build.zig.zon | 9 + pkg/nix-base32/src/root.zig | 258 ++++++ pkg/nix-derivation/build.zig | 15 + pkg/nix-derivation/build.zig.zon | 9 + pkg/nix-derivation/src/root.zig | 208 +++++ pkg/nix-flake-lock/build.zig | 15 + pkg/nix-flake-lock/build.zig.zon | 9 + pkg/nix-flake-lock/src/bump.zig | 104 +++ pkg/nix-flake-lock/src/root.zig | 1355 ++++++++++++++++++++++++++++++ pkg/nix-narinfo/build.zig | 17 + pkg/nix-narinfo/build.zig.zon | 11 + pkg/nix-narinfo/src/basemul.zig | 116 +++ pkg/nix-narinfo/src/parse.zig | 157 ++++ pkg/nix-narinfo/src/render.zig | 283 +++++++ pkg/nix-narinfo/src/root.zig | 19 + pkg/nix-store-path/build.zig | 16 + pkg/nix-store-path/build.zig.zon | 10 + pkg/nix-store-path/src/root.zig | 96 +++ scripts/test-zig.sh | 10 + 29 files changed, 3534 insertions(+) create mode 100644 pkg/nix-archive/build.zig create mode 100644 pkg/nix-archive/build.zig.zon create mode 100644 pkg/nix-archive/src/root.zig create mode 100644 pkg/nix-archive/src/sha256.zig create mode 100644 pkg/nix-archive/src/sink.zig create mode 100644 pkg/nix-archive/src/tests.zig create mode 100644 pkg/nix-archive/src/verify.zig create mode 100644 pkg/nix-archive/src/walk.zig create mode 100644 pkg/nix-archive/src/zstd.zig create mode 100644 pkg/nix-base32/build.zig create mode 100644 pkg/nix-base32/build.zig.zon create mode 100644 pkg/nix-base32/src/root.zig create mode 100644 pkg/nix-derivation/build.zig create mode 100644 pkg/nix-derivation/build.zig.zon create mode 100644 pkg/nix-derivation/src/root.zig create mode 100644 pkg/nix-flake-lock/build.zig create mode 100644 pkg/nix-flake-lock/build.zig.zon create mode 100644 pkg/nix-flake-lock/src/bump.zig create mode 100644 pkg/nix-flake-lock/src/root.zig create mode 100644 pkg/nix-narinfo/build.zig create mode 100644 pkg/nix-narinfo/build.zig.zon create mode 100644 pkg/nix-narinfo/src/basemul.zig create mode 100644 pkg/nix-narinfo/src/parse.zig create mode 100644 pkg/nix-narinfo/src/render.zig create mode 100644 pkg/nix-narinfo/src/root.zig create mode 100644 pkg/nix-store-path/build.zig create mode 100644 pkg/nix-store-path/build.zig.zon create mode 100644 pkg/nix-store-path/src/root.zig create mode 100755 scripts/test-zig.sh diff --git a/pkg/nix-archive/build.zig b/pkg/nix-archive/build.zig new file mode 100644 index 0000000..2da35ef --- /dev/null +++ b/pkg/nix-archive/build.zig @@ -0,0 +1,17 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mod = b.addModule("nix-archive", .{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + }); + mod.link_libc = true; + + const tests = b.addTest(.{ .root_module = mod }); + tests.root_module.linkSystemLibrary("zstd", .{}); + b.step("test", "Run unit tests").dependOn(&b.addRunArtifact(tests).step); +} diff --git a/pkg/nix-archive/build.zig.zon b/pkg/nix-archive/build.zig.zon new file mode 100644 index 0000000..440b90e --- /dev/null +++ b/pkg/nix-archive/build.zig.zon @@ -0,0 +1,9 @@ +.{ + .name = .nix_archive, + .version = "0.1.0", + .fingerprint = 0xcb3a212dbdaff4ec, + .minimum_zig_version = "0.16.0", + .dependencies = .{ + }, + .paths = .{ "build.zig", "build.zig.zon", "src" }, +} diff --git a/pkg/nix-archive/src/root.zig b/pkg/nix-archive/src/root.zig new file mode 100644 index 0000000..d6a8f86 --- /dev/null +++ b/pkg/nix-archive/src/root.zig @@ -0,0 +1,31 @@ +//! NAR (Nix ARchive) production and verification. +//! +//! - `dump`: serialise a filesystem tree straight from the kernel (Linux). +//! - `Sink`: hash, count and zstd-compress a NAR stream in one pass. +//! - `Verifier`: hash and decompress an uploaded stream, checking it is a NAR. + +const walk = @import("walk.zig"); +const sink = @import("sink.zig"); +const verify = @import("verify.zig"); + +pub const Sink = sink.Sink; +pub const Digest = sink.Digest; +pub const WriteFn = sink.WriteFn; +pub const Options = sink.Options; +pub const SinkError = sink.Error; + +pub const dump = walk.dump; +pub const supported = walk.supported; +pub const DumpError = walk.Error; +pub fn lastErrno() i32 { + return walk.last_errno; +} + +pub const Verifier = verify.Verifier; +pub const Compression = verify.Compression; +pub const Sha256 = @import("sha256.zig").Sha256; + +test { + _ = @import("tests.zig"); + _ = @import("sha256.zig"); +} diff --git a/pkg/nix-archive/src/sha256.zig b/pkg/nix-archive/src/sha256.zig new file mode 100644 index 0000000..d0732db --- /dev/null +++ b/pkg/nix-archive/src/sha256.zig @@ -0,0 +1,173 @@ +//! Multi-block SHA-256 for x86-64 SHA-NI. +//! +//! std's SHA-NI path compresses one block per call: it loads and shuffles +//! the state from memory, runs the rounds, and stores it back, every 64 +//! bytes. Here the state stays in two xmm registers (ABEF / CDGH) for the +//! whole buffer, and message words are byte-swapped with one shuffle per +//! 16 bytes. The per-block instruction sequence is std's. Without SHA-NI +//! this is std's implementation. + +const std = @import("std"); +const builtin = @import("builtin"); + +pub const accelerated = builtin.cpu.arch == .x86_64 and + builtin.zig_backend != .stage2_c and + builtin.cpu.hasAll(.x86, &.{ .sha, .avx2 }); + +pub const Sha256 = if (accelerated) Fast else std.crypto.hash.sha2.Sha256; + +const V4u32 = @Vector(4, u32); + +const K = [64]u32{ + 0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5, 0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5, + 0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3, 0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174, + 0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC, 0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA, + 0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7, 0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967, + 0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13, 0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85, + 0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3, 0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070, + 0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5, 0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3, + 0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208, 0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2, +}; + +const iv = [8]u32{ 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19 }; + +inline fn loadBe(bytes: *const [16]u8) V4u32 { + const v: @Vector(16, u8) = bytes.*; + const swapped = @shuffle(u8, v, undefined, [16]i32{ 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12 }); + return @bitCast(swapped); +} + +/// Compresses `blocks` (a multiple of 64 bytes) into `state`. +fn compress(state: *[8]u32, blocks: []const u8) void { + var x: V4u32 = .{ state[5], state[4], state[1], state[0] }; + var y: V4u32 = .{ state[7], state[6], state[3], state[2] }; + var off: usize = 0; + while (off < blocks.len) : (off += 64) { + const x0 = x; + const y0 = y; + var m: [16]V4u32 = undefined; + inline for (0..4) |i| m[i] = loadBe(blocks[off + 16 * i ..][0..16]); + + inline for (0..16) |k| { + if (k < 12) { + var tmp = m[k]; + m[k + 4] = asm ( + \\ sha256msg1 %[w4_7], %[tmp] + \\ vpalignr $0x4, %[w8_11], %[w12_15], %[result] + \\ paddd %[tmp], %[result] + \\ sha256msg2 %[w12_15], %[result] + : [tmp] "=&x" (tmp), + [result] "=&x" (-> V4u32), + : [_] "0" (tmp), + [w4_7] "x" (m[k + 1]), + [w8_11] "x" (m[k + 2]), + [w12_15] "x" (m[k + 3]), + ); + } + const w: V4u32 = m[k] +% @as(V4u32, K[4 * k ..][0..4].*); + y = asm ("sha256rnds2 %[x], %[y]" + : [y] "=x" (-> V4u32), + : [_] "0" (y), + [x] "x" (x), + [_] "{xmm0}" (w), + ); + x = asm ("sha256rnds2 %[y], %[x]" + : [x] "=x" (-> V4u32), + : [_] "0" (x), + [y] "x" (y), + [_] "{xmm0}" (@as(V4u32, @bitCast(@as(u128, @bitCast(w)) >> 64))), + ); + } + x +%= x0; + y +%= y0; + } + state[0] = x[3]; + state[1] = x[2]; + state[4] = x[1]; + state[5] = x[0]; + state[2] = y[3]; + state[3] = y[2]; + state[6] = y[1]; + state[7] = y[0]; +} + +pub const Fast = struct { + pub const digest_length = 32; + pub const Options = struct {}; + + s: [8]u32 = iv, + buf: [64]u8 = undefined, + buf_len: u8 = 0, + total_len: u64 = 0, + + pub fn init(_: Options) Fast { + return .{}; + } + + pub fn hash(b: []const u8, out: *[digest_length]u8, options: Options) void { + var d = Fast.init(options); + d.update(b); + d.final(out); + } + + pub fn update(d: *Fast, b: []const u8) void { + var off: usize = 0; + if (d.buf_len != 0) { + const take = @min(64 - d.buf_len, b.len); + @memcpy(d.buf[d.buf_len..][0..take], b[0..take]); + d.buf_len += @intCast(take); + off = take; + if (d.buf_len < 64) { + d.total_len += b.len; + return; + } + compress(&d.s, &d.buf); + d.buf_len = 0; + } + const full = (b.len - off) / 64 * 64; + if (full > 0) compress(&d.s, b[off..][0..full]); + off += full; + const rest = b[off..]; + @memcpy(d.buf[0..rest.len], rest); + d.buf_len = @intCast(rest.len); + d.total_len += b.len; + } + + pub fn final(d: *Fast, out: *[digest_length]u8) void { + const bits = d.total_len * 8; + d.buf[d.buf_len] = 0x80; + @memset(d.buf[d.buf_len + 1 ..], 0); + if (d.buf_len >= 56) { + compress(&d.s, &d.buf); + @memset(&d.buf, 0); + } + std.mem.writeInt(u64, d.buf[56..64], bits, .big); + compress(&d.s, &d.buf); + for (d.s, 0..) |word, i| std.mem.writeInt(u32, out[4 * i ..][0..4], word, .big); + } +}; + +test "matches std SHA-256 for every length and split" { + const Std = std.crypto.hash.sha2.Sha256; + var prng: std.Random.DefaultPrng = .init(9); + const r = prng.random(); + var data: [1000]u8 = undefined; + r.bytes(&data); + for (0..data.len) |n| { + var want: [32]u8 = undefined; + Std.hash(data[0..n], &want, .{}); + var got: [32]u8 = undefined; + Sha256.hash(data[0..n], &got, .{}); + try std.testing.expectEqualSlices(u8, &want, &got); + + var h = Sha256.init(.{}); + var off: usize = 0; + while (off < n) { + const step = @min(n - off, r.uintAtMost(usize, 150)); + h.update(data[off..][0..step]); + off += step; + } + h.final(&got); + try std.testing.expectEqualSlices(u8, &want, &got); + } +} diff --git a/pkg/nix-archive/src/sink.zig b/pkg/nix-archive/src/sink.zig new file mode 100644 index 0000000..b753868 --- /dev/null +++ b/pkg/nix-archive/src/sink.zig @@ -0,0 +1,161 @@ +//! The compressing sink: NAR bytes go in, get hashed and counted, are +//! zstd-compressed, and the compressed bytes are hashed, counted and +//! handed to a caller-supplied write callback. Everything happens in a +//! single pass; zstd worker threads do the compression when enabled. + +const std = @import("std"); +const zstd = @import("zstd.zig"); +const Sha256 = @import("sha256.zig").Sha256; + +pub const WriteFn = *const fn (ctx: ?*anyopaque, buf: [*]const u8, len: usize) callconv(.c) c_int; + +pub const Digest = extern struct { + file_hash: [32]u8, + file_size: u64, + nar_hash: [32]u8, + nar_size: u64, +}; + +pub const Error = error{ Zstd, Aborted, OutOfMemory }; + +pub const Options = struct { + /// 0 disables compression (the output is the raw NAR). + level: c_int = 3, + /// zstd worker threads; 0 compresses on the calling thread. + threads: c_int = 0, + /// Expected NAR size, 0 when unknown. Lets zstd size its tables. + size_hint: u64 = 0, +}; + +const in_capacity = 1 << 20; + +/// Writes at least this large bypass the input buffer. +pub const direct_threshold = in_capacity; + +/// NAR streams below this size are compressed on the calling thread: +/// spinning up zstd workers costs more than it saves. +const multithread_threshold = 4 << 20; + +pub const Sink = struct { + nar_hasher: Sha256 = .init(.{}), + nar_size: u64 = 0, + file_hasher: Sha256 = .init(.{}), + file_size: u64 = 0, + cctx: ?*zstd.CCtx = null, + in_buf: []u8, + in_len: usize = 0, + out_buf: []u8, + write_fn: WriteFn, + write_ctx: ?*anyopaque, + + pub fn init(allocator: std.mem.Allocator, options: Options, write_fn: WriteFn, write_ctx: ?*anyopaque) Error!Sink { + const in_buf = try allocator.alloc(u8, in_capacity); + errdefer allocator.free(in_buf); + + var cctx: ?*zstd.CCtx = null; + var out_len: usize = 0; + if (options.level != 0) { + cctx = zstd.ZSTD_createCCtx() orelse return error.OutOfMemory; + errdefer _ = zstd.ZSTD_freeCCtx(cctx); + if (zstd.isError(zstd.ZSTD_CCtx_setParameter(cctx.?, zstd.c_compressionLevel, options.level))) return error.Zstd; + if (options.threads > 0 and (options.size_hint == 0 or options.size_hint >= multithread_threshold)) { + // Fails harmlessly on a single-threaded libzstd build. + _ = zstd.ZSTD_CCtx_setParameter(cctx.?, zstd.c_nbWorkers, options.threads); + } + if (options.size_hint > 0 and options.size_hint <= std.math.maxInt(c_int)) { + _ = zstd.ZSTD_CCtx_setParameter(cctx.?, zstd.c_srcSizeHint, @intCast(options.size_hint)); + } + out_len = zstd.ZSTD_CStreamOutSize(); + } + const out_buf = try allocator.alloc(u8, out_len); + + return .{ + .cctx = cctx, + .in_buf = in_buf, + .out_buf = out_buf, + .write_fn = write_fn, + .write_ctx = write_ctx, + }; + } + + pub fn deinit(self: *Sink, allocator: std.mem.Allocator) void { + if (self.cctx) |cctx| _ = zstd.ZSTD_freeCCtx(cctx); + allocator.free(self.in_buf); + allocator.free(self.out_buf); + self.* = undefined; + } + + /// Buffered write of NAR bytes. + pub fn write(self: *Sink, bytes: []const u8) Error!void { + if (self.in_len + bytes.len <= self.in_buf.len) { + @memcpy(self.in_buf[self.in_len..][0..bytes.len], bytes); + self.in_len += bytes.len; + return; + } + try self.flush(); + if (bytes.len >= self.in_buf.len) { + try self.feed(bytes); + } else { + @memcpy(self.in_buf[0..bytes.len], bytes); + self.in_len = bytes.len; + } + } + + /// Free tail of the input buffer, for callers that read straight into it. + pub fn spare(self: *Sink) []u8 { + return self.in_buf[self.in_len..]; + } + + pub fn commit(self: *Sink, n: usize) Error!void { + self.in_len += n; + if (self.in_len == self.in_buf.len) try self.flush(); + } + + pub fn flush(self: *Sink) Error!void { + if (self.in_len == 0) return; + const pending = self.in_buf[0..self.in_len]; + self.in_len = 0; + try self.feed(pending); + } + + fn feed(self: *Sink, data: []const u8) Error!void { + self.nar_hasher.update(data); + self.nar_size += data.len; + const cctx = self.cctx orelse return self.emit(data); + var input: zstd.InBuffer = .{ .src = data.ptr, .size = data.len, .pos = 0 }; + while (input.pos < input.size) { + var output: zstd.OutBuffer = .{ .dst = self.out_buf.ptr, .size = self.out_buf.len, .pos = 0 }; + if (zstd.isError(zstd.ZSTD_compressStream2(cctx, &output, &input, zstd.e_continue))) return error.Zstd; + if (output.pos > 0) try self.emit(self.out_buf[0..output.pos]); + } + } + + fn emit(self: *Sink, data: []const u8) Error!void { + // Uncompressed output is the NAR itself: hash it once, not twice. + if (self.cctx != null) self.file_hasher.update(data); + self.file_size += data.len; + if (self.write_fn(self.write_ctx, data.ptr, data.len) != 0) return error.Aborted; + } + + pub fn finish(self: *Sink) Error!Digest { + try self.flush(); + if (self.cctx) |cctx| { + var input: zstd.InBuffer = .{ .src = null, .size = 0, .pos = 0 }; + while (true) { + var output: zstd.OutBuffer = .{ .dst = self.out_buf.ptr, .size = self.out_buf.len, .pos = 0 }; + const remaining = zstd.ZSTD_compressStream2(cctx, &output, &input, zstd.e_end); + if (zstd.isError(remaining)) return error.Zstd; + if (output.pos > 0) try self.emit(self.out_buf[0..output.pos]); + if (remaining == 0) break; + } + } + var digest: Digest = undefined; + self.nar_hasher.final(&digest.nar_hash); + if (self.cctx != null) self.file_hasher.final(&digest.file_hash) else { + digest.file_hash = digest.nar_hash; + } + digest.nar_size = self.nar_size; + digest.file_size = self.file_size; + return digest; + } +}; diff --git a/pkg/nix-archive/src/tests.zig b/pkg/nix-archive/src/tests.zig new file mode 100644 index 0000000..0f6040a --- /dev/null +++ b/pkg/nix-archive/src/tests.zig @@ -0,0 +1,88 @@ +const std = @import("std"); +const Sink = @import("sink.zig").Sink; +const Verifier = @import("verify.zig").Verifier; + +const Collect = struct { + list: std.ArrayList(u8) = .empty, + + fn write(ctx: ?*anyopaque, buf: [*]const u8, len: usize) callconv(.c) c_int { + const self: *Collect = @ptrCast(@alignCast(ctx.?)); + self.list.appendSlice(std.testing.allocator, buf[0..len]) catch return 1; + return 0; + } +}; + +fn fakeNar(a: std.mem.Allocator, body_len: usize) ![]u8 { + var nar: std.ArrayList(u8) = .empty; + try nar.appendSlice(a, "\x0d\x00\x00\x00\x00\x00\x00\x00nix-archive-1\x00\x00\x00"); + var prng: std.Random.DefaultPrng = .init(42); + for (0..body_len) |i| try nar.append(a, if (i % 7 == 0) prng.random().int(u8) else 'x'); + return nar.toOwnedSlice(a); +} + +fn roundTrip(level: c_int, threads: c_int, body_len: usize) !void { + const a = std.testing.allocator; + const nar = try fakeNar(a, body_len); + defer a.free(nar); + + var out: Collect = .{}; + defer out.list.deinit(a); + var sink = try Sink.init(a, .{ .level = level, .threads = threads }, Collect.write, &out); + defer sink.deinit(a); + // Uneven writes exercise the buffering paths. + var off: usize = 0; + var step: usize = 1; + while (off < nar.len) : (step = step * 3 + 1) { + const n = @min(step, nar.len - off); + try sink.write(nar[off..][0..n]); + off += n; + } + const produced = try sink.finish(); + + var expected_nar: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(nar, &expected_nar, .{}); + try std.testing.expectEqualSlices(u8, &expected_nar, &produced.nar_hash); + try std.testing.expectEqual(@as(u64, nar.len), produced.nar_size); + try std.testing.expectEqual(@as(u64, out.list.items.len), produced.file_size); + + var v = try Verifier.init(a, if (level == 0) .none else .zstd); + defer v.deinit(a); + var i: usize = 0; + while (i < out.list.items.len) : (i += 4093) { + try v.update(out.list.items[i..@min(i + 4093, out.list.items.len)]); + } + const verified = try v.finish(); + try std.testing.expectEqualSlices(u8, &produced.nar_hash, &verified.nar_hash); + try std.testing.expectEqualSlices(u8, &produced.file_hash, &verified.file_hash); + try std.testing.expectEqual(produced.nar_size, verified.nar_size); + try std.testing.expectEqual(produced.file_size, verified.file_size); +} + +test "compress then verify round-trips" { + try roundTrip(3, 0, 1000); + try roundTrip(3, 0, 3 << 20); + try roundTrip(1, 4, 9 << 20); + try roundTrip(0, 0, 5000); +} + +test "verifier rejects truncated and non-NAR input" { + const a = std.testing.allocator; + const nar = try fakeNar(a, 100_000); + defer a.free(nar); + var out: Collect = .{}; + defer out.list.deinit(a); + var sink = try Sink.init(a, .{}, Collect.write, &out); + defer sink.deinit(a); + try sink.write(nar); + _ = try sink.finish(); + + var v = try Verifier.init(a, .zstd); + defer v.deinit(a); + try v.update(out.list.items[0 .. out.list.items.len / 2]); + try std.testing.expectError(error.Truncated, v.finish()); + + var raw = try Verifier.init(a, .none); + defer raw.deinit(a); + try raw.update("definitely not a nar archive...."); + try std.testing.expectError(error.NotNar, raw.finish()); +} diff --git a/pkg/nix-archive/src/verify.zig b/pkg/nix-archive/src/verify.zig new file mode 100644 index 0000000..0976ae9 --- /dev/null +++ b/pkg/nix-archive/src/verify.zig @@ -0,0 +1,79 @@ +//! Server-side upload verification: feed the compressed NAR as it arrives +//! and get back the hash/size of both the compressed file and the NAR it +//! decompresses to. The server signs narinfo from these values rather than +//! trusting what the client claims. + +const std = @import("std"); +const zstd = @import("zstd.zig"); +const Digest = @import("sink.zig").Digest; +const Sha256 = @import("sha256.zig").Sha256; + +pub const Compression = enum(c_int) { none = 0, zstd = 1 }; + +pub const Error = error{ Zstd, Truncated, NotNar, OutOfMemory }; + +const nar_magic = "\x0d\x00\x00\x00\x00\x00\x00\x00nix-archive-1\x00\x00\x00"; + +pub const Verifier = struct { + file_hasher: Sha256 = .init(.{}), + file_size: u64 = 0, + nar_hasher: Sha256 = .init(.{}), + nar_size: u64 = 0, + dctx: ?*zstd.DCtx = null, + out_buf: []u8 = &.{}, + /// Last ZSTD_decompressStream result; 0 means a frame just ended. + frame_state: usize = 0, + head: [nar_magic.len]u8 = undefined, + + pub fn init(allocator: std.mem.Allocator, compression: Compression) Error!Verifier { + var v: Verifier = .{}; + if (compression == .zstd) { + v.dctx = zstd.ZSTD_createDCtx() orelse return error.OutOfMemory; + errdefer _ = zstd.ZSTD_freeDCtx(v.dctx); + v.out_buf = try allocator.alloc(u8, zstd.ZSTD_DStreamOutSize() * 4); + } + return v; + } + + pub fn deinit(self: *Verifier, allocator: std.mem.Allocator) void { + if (self.dctx) |dctx| _ = zstd.ZSTD_freeDCtx(dctx); + allocator.free(self.out_buf); + self.* = undefined; + } + + fn nar(self: *Verifier, data: []const u8) void { + if (self.nar_size < self.head.len) { + const n = @min(data.len, self.head.len - self.nar_size); + @memcpy(self.head[self.nar_size..][0..n], data[0..n]); + } + self.nar_hasher.update(data); + self.nar_size += data.len; + } + + pub fn update(self: *Verifier, data: []const u8) Error!void { + self.file_hasher.update(data); + self.file_size += data.len; + const dctx = self.dctx orelse return self.nar(data); + var input: zstd.InBuffer = .{ .src = data.ptr, .size = data.len, .pos = 0 }; + while (true) { + var output: zstd.OutBuffer = .{ .dst = self.out_buf.ptr, .size = self.out_buf.len, .pos = 0 }; + const rc = zstd.ZSTD_decompressStream(dctx, &output, &input); + if (zstd.isError(rc)) return error.Zstd; + self.frame_state = rc; + if (output.pos > 0) self.nar(self.out_buf[0..output.pos]); + // Done once the input is consumed and zstd has no more buffered output. + if (input.pos == input.size and output.pos < output.size) break; + } + } + + pub fn finish(self: *Verifier) Error!Digest { + if (self.dctx != null and self.frame_state != 0) return error.Truncated; + if (self.nar_size < self.head.len or !std.mem.eql(u8, &self.head, nar_magic)) return error.NotNar; + var digest: Digest = undefined; + self.file_hasher.final(&digest.file_hash); + self.nar_hasher.final(&digest.nar_hash); + digest.file_size = self.file_size; + digest.nar_size = self.nar_size; + return digest; + } +}; diff --git a/pkg/nix-archive/src/walk.zig b/pkg/nix-archive/src/walk.zig new file mode 100644 index 0000000..d47e73f --- /dev/null +++ b/pkg/nix-archive/src/walk.zig @@ -0,0 +1,202 @@ +//! NAR serialisation straight from the filesystem, without spawning +//! `nix store dump-path`. Walks with openat/getdents64/statx relative to +//! directory fds, and reads file contents directly into the sink's input +//! buffer so file data is copied exactly once out of the kernel. + +const std = @import("std"); +const builtin = @import("builtin"); +const Sink = @import("sink.zig").Sink; + +pub const supported = builtin.os.tag == .linux; + +pub const Error = error{ + Io, + UnsupportedFileType, + FileChanged, + NameTooLong, +} || @import("sink.zig").Error; + +/// errno of the last failed syscall on this thread, for diagnostics. +pub threadlocal var last_errno: i32 = 0; + +const linux = std.os.linux; + +fn check(rc: usize) Error!usize { + const e = linux.errno(rc); + if (e == .SUCCESS) return rc; + last_errno = @intFromEnum(e); + return error.Io; +} + +fn closeFd(fd: i32) void { + _ = linux.close(fd); +} + +/// Files at least this large are mmapped; sink.write feeds them unbuffered. +const mmap_threshold = @import("sink.zig").direct_threshold; + +const Kind = enum { regular, executable, symlink, directory }; + +const Node = struct { + kind: Kind, + size: u64, +}; + +fn statAt(dirfd: i32, name: [*:0]const u8) Error!Node { + var stx: linux.Statx = undefined; + _ = try check(linux.statx(dirfd, name, linux.AT.SYMLINK_NOFOLLOW, .{ .TYPE = true, .MODE = true, .SIZE = true }, &stx)); + const mode: u32 = stx.mode; + return switch (mode & linux.S.IFMT) { + linux.S.IFREG => .{ .kind = if (mode & 0o100 != 0) .executable else .regular, .size = stx.size }, + linux.S.IFLNK => .{ .kind = .symlink, .size = stx.size }, + linux.S.IFDIR => .{ .kind = .directory, .size = 0 }, + else => error.UnsupportedFileType, + }; +} + +const Writer = struct { + sink: *Sink, + arena: std.mem.Allocator, + /// getdents64 buffer, shared: a directory is fully listed before recursing. + dents: []u8, + + fn str(self: *Writer, s: []const u8) Error!void { + try self.len(s.len); + try self.sink.write(s); + try self.pad(s.len); + } + + fn len(self: *Writer, n: u64) Error!void { + var buf: [8]u8 = undefined; + std.mem.writeInt(u64, &buf, n, .little); + try self.sink.write(&buf); + } + + fn pad(self: *Writer, n: u64) Error!void { + const zeros = [_]u8{0} ** 8; + const rem: usize = @intCast(n % 8); + if (rem != 0) try self.sink.write(zeros[0 .. 8 - rem]); + } + + fn node(self: *Writer, dirfd: i32, name: [*:0]const u8, n: Node) Error!void { + try self.str("("); + try self.str("type"); + switch (n.kind) { + .regular, .executable => { + try self.str("regular"); + if (n.kind == .executable) { + try self.str("executable"); + try self.str(""); + } + try self.str("contents"); + try self.contents(dirfd, name, n.size); + }, + .symlink => { + try self.str("symlink"); + try self.str("target"); + var buf: [4096]u8 = undefined; + const got = try check(linux.readlinkat(dirfd, name, &buf, buf.len)); + if (got >= buf.len) return error.NameTooLong; + try self.str(buf[0..got]); + }, + .directory => { + try self.str("directory"); + try self.directory(dirfd, name); + }, + } + try self.str(")"); + } + + fn contents(self: *Writer, dirfd: i32, name: [*:0]const u8, size: u64) Error!void { + const fd: i32 = @intCast(try check(linux.openat(dirfd, name, .{ .ACCMODE = .RDONLY, .NOFOLLOW = true, .CLOEXEC = true }, 0))); + defer closeFd(fd); + try self.len(size); + // Large files are hashed straight out of the page cache: no + // kernel-to-user copy, and the sink feeds the mapping without + // buffering it. Store paths are immutable, so the mapping is stable. + if (size >= mmap_threshold) { + const map_len: usize = @intCast(size); + const rc = linux.mmap(null, map_len, .{ .READ = true }, .{ .TYPE = .PRIVATE, .POPULATE = true }, fd, 0); + if (linux.errno(rc) == .SUCCESS) { + const ptr: [*]u8 = @ptrFromInt(rc); + defer _ = linux.munmap(ptr, map_len); + _ = linux.madvise(ptr, map_len, linux.MADV.SEQUENTIAL); + try self.sink.write(ptr[0..map_len]); + return self.pad(size); + } + } + var remaining = size; + while (remaining > 0) { + var dst = self.sink.spare(); + if (dst.len == 0) { + try self.sink.flush(); + dst = self.sink.spare(); + } + const want: usize = @intCast(@min(remaining, dst.len)); + const rc = linux.read(fd, dst.ptr, want); + const e = linux.errno(rc); + if (e == .INTR) continue; + const got = try check(rc); + if (got == 0) return error.FileChanged; + remaining -= got; + try self.sink.commit(got); + } + try self.pad(size); + } + + fn directory(self: *Writer, parent: i32, name: [*:0]const u8) Error!void { + const fd: i32 = @intCast(try check(linux.openat(parent, name, .{ .ACCMODE = .RDONLY, .DIRECTORY = true, .NOFOLLOW = true, .CLOEXEC = true }, 0))); + defer closeFd(fd); + + const Entry = struct { name: [:0]const u8, dtype: u8 }; + var entries: std.ArrayList(Entry) = .empty; + const buf = self.dents; + while (true) { + const n = try check(linux.getdents64(fd, buf.ptr, buf.len)); + if (n == 0) break; + var off: usize = 0; + while (off < n) { + const reclen = std.mem.readInt(u16, buf[off + 16 ..][0..2], builtin.cpu.arch.endian()); + const dtype = buf[off + 18]; + const entry_name = std.mem.sliceTo(@as([*:0]const u8, @ptrCast(&buf[off + 19])), 0); + off += reclen; + if (std.mem.eql(u8, entry_name, ".") or std.mem.eql(u8, entry_name, "..")) continue; + try entries.append(self.arena, .{ .name = try self.arena.dupeZ(u8, entry_name), .dtype = dtype }); + } + } + + // NAR entries are ordered bytewise, like Nix's std::map. + std.mem.sort(Entry, entries.items, {}, struct { + fn lessThan(_: void, a: Entry, b: Entry) bool { + return std.mem.lessThan(u8, a.name, b.name); + } + }.lessThan); + + for (entries.items) |entry| { + const child: Node = switch (entry.dtype) { + linux.DT.DIR => .{ .kind = .directory, .size = 0 }, + linux.DT.LNK => .{ .kind = .symlink, .size = 0 }, + else => try statAt(fd, entry.name.ptr), + }; + try self.str("entry"); + try self.str("("); + try self.str("name"); + try self.str(entry.name); + try self.str("node"); + try self.node(fd, entry.name.ptr, child); + try self.str(")"); + } + } +}; + +/// Serialise `path` as a NAR into `sink`. +pub fn dump(sink: *Sink, path: [*:0]const u8) Error!void { + if (!supported) @compileError("NAR dumping is only implemented for Linux"); + var arena_state: std.heap.ArenaAllocator = .init(std.heap.c_allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var w: Writer = .{ .sink = sink, .arena = arena, .dents = try arena.alloc(u8, 64 * 1024) }; + const root = try statAt(linux.AT.FDCWD, path); + try w.str("nix-archive-1"); + try w.node(linux.AT.FDCWD, path, root); +} diff --git a/pkg/nix-archive/src/zstd.zig b/pkg/nix-archive/src/zstd.zig new file mode 100644 index 0000000..cdf9e73 --- /dev/null +++ b/pkg/nix-archive/src/zstd.zig @@ -0,0 +1,42 @@ +//! Minimal hand-written bindings for the libzstd streaming API. + +pub const CCtx = opaque {}; +pub const DCtx = opaque {}; + +pub const InBuffer = extern struct { + src: ?*const anyopaque, + size: usize, + pos: usize, +}; + +pub const OutBuffer = extern struct { + dst: ?*anyopaque, + size: usize, + pos: usize, +}; + +pub const c_compressionLevel: c_int = 100; +pub const c_checksumFlag: c_int = 201; +pub const c_nbWorkers: c_int = 400; +/// ZSTD_c_srcSizeHint (experimental, stable since 1.4.x). +pub const c_srcSizeHint: c_int = 1004; + +pub const e_continue: c_int = 0; +pub const e_end: c_int = 2; + +pub extern "c" fn ZSTD_createCCtx() ?*CCtx; +pub extern "c" fn ZSTD_freeCCtx(cctx: ?*CCtx) usize; +pub extern "c" fn ZSTD_CCtx_setParameter(cctx: *CCtx, param: c_int, value: c_int) usize; +pub extern "c" fn ZSTD_compressStream2(cctx: *CCtx, output: *OutBuffer, input: *InBuffer, end_op: c_int) usize; +pub extern "c" fn ZSTD_CStreamOutSize() usize; + +pub extern "c" fn ZSTD_createDCtx() ?*DCtx; +pub extern "c" fn ZSTD_freeDCtx(dctx: ?*DCtx) usize; +pub extern "c" fn ZSTD_decompressStream(dctx: *DCtx, output: *OutBuffer, input: *InBuffer) usize; +pub extern "c" fn ZSTD_DStreamOutSize() usize; + +pub extern "c" fn ZSTD_isError(code: usize) c_uint; + +pub fn isError(code: usize) bool { + return ZSTD_isError(code) != 0; +} diff --git a/pkg/nix-base32/build.zig b/pkg/nix-base32/build.zig new file mode 100644 index 0000000..1fe3eb0 --- /dev/null +++ b/pkg/nix-base32/build.zig @@ -0,0 +1,15 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mod = b.addModule("nix-base32", .{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + }); + + const tests = b.addTest(.{ .root_module = mod }); + b.step("test", "Run unit tests").dependOn(&b.addRunArtifact(tests).step); +} diff --git a/pkg/nix-base32/build.zig.zon b/pkg/nix-base32/build.zig.zon new file mode 100644 index 0000000..938d1e5 --- /dev/null +++ b/pkg/nix-base32/build.zig.zon @@ -0,0 +1,9 @@ +.{ + .name = .nix_base32, + .version = "0.1.0", + .fingerprint = 0x9c08ba94cedadc5b, + .minimum_zig_version = "0.16.0", + .dependencies = .{ + }, + .paths = .{ "build.zig", "build.zig.zon", "src" }, +} diff --git a/pkg/nix-base32/src/root.zig b/pkg/nix-base32/src/root.zig new file mode 100644 index 0000000..fad33c3 --- /dev/null +++ b/pkg/nix-base32/src/root.zig @@ -0,0 +1,258 @@ +//! Nix's base-32 encoding: a custom alphabet (no e, o, t, u) with the +//! digits emitted from the most significant 5-bit group down. +//! +//! Character classification is vectorised (Zig @Vector): a whole store path +//! hash (32 chars) is validated and mapped to 5-bit digits in one pass. +//! The scalar table path handles tails and is kept as the reference. + +const std = @import("std"); + +pub const alphabet = "0123456789abcdfghijklmnpqrsvwxyz"; + +const invalid: u8 = 0xff; + +const reverse: [256]u8 = blk: { + var table = [_]u8{invalid} ** 256; + for (alphabet, 0..) |c, i| table[c] = @intCast(i); + break :blk table; +}; + +/// Lanes per vector step. 32 covers a store path hash exactly and lowers to +/// one AVX2 register (two SSE registers, one half of an AVX-512 one). +pub const lanes = 32; +const V = @Vector(lanes, u8); +const Mask = std.meta.Int(.unsigned, lanes); + +fn splat(c: u8) V { + return @splat(c); +} + +fn bits(b: @Vector(lanes, bool)) Mask { + return @bitCast(b); +} + +/// Maps 32 characters to their 5-bit digit values. Returns a bitmask with a +/// set bit for every lane that is not in the alphabet. +/// +/// Digits map to c - '0'. Letters map to c - 'a' + 10 minus one for every +/// skipped letter (e, o, t, u) below c. +inline fn classify(v: V, out: *V) Mask { + const is_digit = bits(v >= splat('0')) & bits(v <= splat('9')); + const is_lower = bits(v >= splat('a')) & bits(v <= splat('z')); + const skipped = bits(v == splat('e')) | bits(v == splat('o')) | bits(v == splat('t')) | bits(v == splat('u')); + const valid = is_digit | (is_lower & ~skipped); + + const one = splat(1); + const zero = splat(0); + const gaps = @select(u8, v > splat('e'), one, zero) + + @select(u8, v > splat('o'), one, zero) + + @select(u8, v > splat('t'), one, zero) + + @select(u8, v > splat('u'), one, zero); + const letter = v -% splat('a' - 10) -% gaps; + const digit = v -% splat('0'); + out.* = @select(u8, @as(@Vector(lanes, bool), @bitCast(is_digit)), digit, letter); + return ~valid; +} + +pub fn encodedLen(byte_len: usize) usize { + if (byte_len == 0) return 0; + return (byte_len * 8 - 1) / 5 + 1; +} + +/// Writes exactly `encodedLen(bytes.len)` characters into `out`. +/// +/// 5 bytes hold exactly 8 digits, so full groups are unpacked from one u64 +/// without cross-byte carries; only the last partial group goes bit by bit. +pub fn encode(out: []u8, bytes: []const u8) void { + const len = encodedLen(bytes.len); + std.debug.assert(out.len >= len); + var k: usize = 0; + while (5 * k + 5 <= bytes.len) : (k += 1) { + const v = std.mem.readInt(u40, bytes[5 * k ..][0..5], .little); + inline for (0..8) |i| out[len - 1 - (8 * k + i)] = alphabet[@as(u5, @truncate(v >> (5 * i)))]; + } + for (8 * k..len) |n| out[len - 1 - n] = alphabet[digitAt(bytes, n)]; +} + +fn digitAt(bytes: []const u8, n: usize) u8 { + const b = n * 5; + const i = b / 8; + const j: u3 = @intCast(b % 8); + var c: u16 = bytes[i] >> j; + if (i + 1 < bytes.len) c |= @as(u16, bytes[i + 1]) << (@as(u4, 8) - j); + return @intCast(c & 0x1f); +} + +pub fn encodeScalar(out: []u8, bytes: []const u8) void { + const len = encodedLen(bytes.len); + for (0..len) |n| out[len - 1 - n] = alphabet[digitAt(bytes, n)]; +} + +pub const DecodeError = error{InvalidNix32}; + +/// Converts characters to digit values in place order, vector-at-a-time. +fn digits(dst: []u8, in: []const u8) DecodeError!void { + var i: usize = 0; + while (i + lanes <= in.len) : (i += lanes) { + var d: V = undefined; + if (classify(in[i..][0..lanes].*, &d) != 0) return error.InvalidNix32; + dst[i..][0..lanes].* = d; + } + for (in[i..], dst[i..in.len]) |c, *d| { + d.* = reverse[c]; + if (d.* == invalid) return error.InvalidNix32; + } +} + +/// Packs digits (most significant first) into little-endian bytes: full +/// 8-digit groups become 5 bytes via one u64, the tail goes bit by bit. +fn pack(out: []u8, d: []const u8) DecodeError!void { + const len = d.len; + var k: usize = 0; + while (8 * k + 8 <= len and 5 * k + 5 <= out.len) : (k += 1) { + var v: u40 = 0; + inline for (0..8) |i| v |= @as(u40, d[len - 1 - (8 * k + i)]) << (5 * i); + std.mem.writeInt(u40, out[5 * k ..][0..5], v, .little); + } + @memset(out[5 * k ..], 0); + return packBits(out, d, 8 * k); +} + +fn packBits(out: []u8, d: []const u8, from: usize) DecodeError!void { + for (from..d.len) |n| { + const digit = d[d.len - n - 1]; + const b = n * 5; + const i = b / 8; + const j: u3 = @intCast(b % 8); + out[i] |= digit << j; + const carry: u8 = @intCast(@as(u16, digit) >> (@as(u4, 8) - j)); + if (i + 1 < out.len) { + out[i + 1] |= carry; + } else if (carry != 0) { + return error.InvalidNix32; + } + } +} + +/// Decodes `in` into exactly `out.len` bytes. `in.len` must equal +/// `encodedLen(out.len)`, and no bits may overflow the output. +pub fn decode(out: []u8, in: []const u8) DecodeError!void { + if (in.len != encodedLen(out.len)) return error.InvalidNix32; + var buf: [128]u8 = undefined; + if (in.len > buf.len) return decodeScalar(out, in); + try digits(buf[0..in.len], in); + try pack(out, buf[0..in.len]); +} + +/// Table-driven reference implementation. +pub fn decodeScalar(out: []u8, in: []const u8) DecodeError!void { + if (in.len != encodedLen(out.len)) return error.InvalidNix32; + var buf: [256]u8 = undefined; + if (in.len > buf.len) return error.InvalidNix32; + for (in, buf[0..in.len]) |c, *d| { + d.* = reverse[c]; + if (d.* == invalid) return error.InvalidNix32; + } + @memset(out, 0); + try packBits(out, buf[0..in.len], 0); +} + +pub fn isValid(in: []const u8) bool { + var i: usize = 0; + var scratch: V = undefined; + while (i + lanes <= in.len) : (i += lanes) { + if (classify(in[i..][0..lanes].*, &scratch) != 0) return false; + } + for (in[i..]) |c| if (reverse[c] == invalid) return false; + return true; +} + +pub fn isValidScalar(in: []const u8) bool { + for (in) |c| if (reverse[c] == invalid) return false; + return true; +} + +test "sha256 of empty string round-trips" { + var digest: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash("", &digest, .{}); + var text: [52]u8 = undefined; + encode(&text, &digest); + try std.testing.expectEqualStrings("0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73", &text); + var back: [32]u8 = undefined; + try decode(&back, &text); + try std.testing.expectEqualSlices(u8, &digest, &back); +} + +test "store path hash length" { + try std.testing.expectEqual(@as(usize, 32), encodedLen(20)); + try std.testing.expectEqual(@as(usize, 52), encodedLen(32)); +} + +test "rejects invalid characters and overflow" { + var out: [20]u8 = undefined; + try std.testing.expectError(error.InvalidNix32, decode(&out, "e" ** 32)); + // 52 chars carry 260 bits; the top 4 must be zero for a 32-byte digest. + var digest: [32]u8 = undefined; + try std.testing.expectError(error.InvalidNix32, decode(&digest, "z" ** 52)); +} + +test "vector classifier agrees with the table for every byte in every lane" { + for (0..256) |c| { + for (0..lanes) |lane| { + var in = [_]u8{'0'} ** lanes; + in[lane] = @intCast(c); + try std.testing.expectEqual(isValidScalar(&in), isValid(&in)); + var d: V = undefined; + const bad = classify(in, &d); + if (bad == 0) { + const arr: [lanes]u8 = d; + try std.testing.expectEqual(reverse[c], arr[lane]); + } + } + } +} + +test "vector and scalar decode agree on random inputs" { + var prng: std.Random.DefaultPrng = .init(7); + const r = prng.random(); + for (0..10_000) |_| { + var bytes: [20]u8 = undefined; + r.bytes(&bytes); + var text: [32]u8 = undefined; + encode(&text, &bytes); + if (r.boolean()) text[r.uintLessThan(usize, 32)] = r.int(u8); + var a: [20]u8 = undefined; + var b: [20]u8 = undefined; + const ra = decode(&a, &text); + const rb = decodeScalar(&b, &text); + if (rb) |_| { + try ra; + try std.testing.expectEqualSlices(u8, &b, &a); + } else |_| { + try std.testing.expectError(error.InvalidNix32, ra); + } + } +} + +test "grouped encode/decode agree with the bitwise reference at every length" { + var prng: std.Random.DefaultPrng = .init(11); + const r = prng.random(); + for (1..64) |n| { + for (0..200) |_| { + var bytes: [64]u8 = undefined; + r.bytes(bytes[0..n]); + var fast: [128]u8 = undefined; + var slow: [128]u8 = undefined; + const len = encodedLen(n); + encode(fast[0..len], bytes[0..n]); + encodeScalar(slow[0..len], bytes[0..n]); + try std.testing.expectEqualStrings(slow[0..len], fast[0..len]); + var back: [64]u8 = undefined; + try decode(back[0..n], fast[0..len]); + try std.testing.expectEqualSlices(u8, bytes[0..n], back[0..n]); + var back_slow: [64]u8 = undefined; + try decodeScalar(back_slow[0..n], fast[0..len]); + try std.testing.expectEqualSlices(u8, bytes[0..n], back_slow[0..n]); + } + } +} diff --git a/pkg/nix-derivation/build.zig b/pkg/nix-derivation/build.zig new file mode 100644 index 0000000..570f659 --- /dev/null +++ b/pkg/nix-derivation/build.zig @@ -0,0 +1,15 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mod = b.addModule("nix-derivation", .{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + }); + + const tests = b.addTest(.{ .root_module = mod }); + b.step("test", "Run unit tests").dependOn(&b.addRunArtifact(tests).step); +} diff --git a/pkg/nix-derivation/build.zig.zon b/pkg/nix-derivation/build.zig.zon new file mode 100644 index 0000000..12cfd89 --- /dev/null +++ b/pkg/nix-derivation/build.zig.zon @@ -0,0 +1,9 @@ +.{ + .name = .nix_derivation, + .version = "0.1.0", + .fingerprint = 0x7a251e78739a22ca, + .minimum_zig_version = "0.16.0", + .dependencies = .{ + }, + .paths = .{ "build.zig", "build.zig.zon", "src" }, +} diff --git a/pkg/nix-derivation/src/root.zig b/pkg/nix-derivation/src/root.zig new file mode 100644 index 0000000..57e8016 --- /dev/null +++ b/pkg/nix-derivation/src/root.zig @@ -0,0 +1,208 @@ +//! Parser for Nix store derivations (`.drv`, ATerm syntax): +//! +//! Derive([outputs],[input drvs],[input srcs],"system","builder",[args],[env]) +//! +//! Strings without escapes are returned as slices of the input (zero-copy); +//! escaped strings are unescaped into the arena. The string scanner looks +//! for '"' or '\\' 32 bytes at a time. + +const std = @import("std"); + +pub const Error = error{ Syntax, OutOfMemory }; + +pub const Output = struct { + name: []const u8, + path: []const u8, + hash_algo: []const u8, + hash: []const u8, +}; + +pub const InputDrv = struct { + path: []const u8, + outputs: []const []const u8, +}; + +pub const EnvVar = struct { + name: []const u8, + value: []const u8, +}; + +pub const Derivation = struct { + outputs: []const Output, + input_drvs: []const InputDrv, + input_srcs: []const []const u8, + platform: []const u8, + builder: []const u8, + args: []const []const u8, + env: []const EnvVar, +}; + +const lanes = 32; +const V = @Vector(lanes, u8); +const Mask = std.meta.Int(.unsigned, lanes); + +/// Index of the first '"' or '\\' at or after `start`. +pub fn indexOfQuoteOrEscape(s: []const u8, start: usize) ?usize { + var i = start; + const quote: V = @splat('"'); + const backslash: V = @splat('\\'); + while (i + lanes <= s.len) : (i += lanes) { + const v: V = s[i..][0..lanes].*; + const m: Mask = @as(Mask, @bitCast(v == quote)) | @as(Mask, @bitCast(v == backslash)); + if (m != 0) return i + @ctz(m); + } + while (i < s.len) : (i += 1) if (s[i] == '"' or s[i] == '\\') return i; + return null; +} + +const Parser = struct { + s: []const u8, + pos: usize = 0, + arena: std.mem.Allocator, + + fn expect(p: *Parser, comptime lit: []const u8) Error!void { + if (!std.mem.startsWith(u8, p.s[p.pos..], lit)) return error.Syntax; + p.pos += lit.len; + } + + fn peek(p: *Parser) Error!u8 { + if (p.pos >= p.s.len) return error.Syntax; + return p.s[p.pos]; + } + + fn string(p: *Parser) Error![]const u8 { + try p.expect("\""); + const start = p.pos; + var i = indexOfQuoteOrEscape(p.s, start) orelse return error.Syntax; + if (p.s[i] == '"') { + p.pos = i + 1; + return p.s[start..i]; + } + // Escaped: copy the clean prefix, then unescape the rest. + var out: std.ArrayList(u8) = .empty; + try out.appendSlice(p.arena, p.s[start..i]); + while (true) { + if (p.s[i] == '"') { + p.pos = i + 1; + return out.toOwnedSlice(p.arena); + } + // Backslash escape. + if (i + 1 >= p.s.len) return error.Syntax; + try out.append(p.arena, switch (p.s[i + 1]) { + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + else => |c| c, + }); + const next = indexOfQuoteOrEscape(p.s, i + 2) orelse return error.Syntax; + try out.appendSlice(p.arena, p.s[i + 2 .. next]); + i = next; + } + } + + fn list(p: *Parser, comptime T: type, comptime item: fn (*Parser) Error!T) Error![]const T { + try p.expect("["); + var items: std.ArrayList(T) = .empty; + if (try p.peek() == ']') { + p.pos += 1; + return &.{}; + } + while (true) { + try items.append(p.arena, try item(p)); + switch (try p.peek()) { + ',' => p.pos += 1, + ']' => { + p.pos += 1; + return items.toOwnedSlice(p.arena); + }, + else => return error.Syntax, + } + } + } + + fn output(p: *Parser) Error!Output { + try p.expect("("); + const name = try p.string(); + try p.expect(","); + const path = try p.string(); + try p.expect(","); + const algo = try p.string(); + try p.expect(","); + const hash = try p.string(); + try p.expect(")"); + return .{ .name = name, .path = path, .hash_algo = algo, .hash = hash }; + } + + fn inputDrv(p: *Parser) Error!InputDrv { + try p.expect("("); + const path = try p.string(); + try p.expect(","); + const outs = try p.list([]const u8, string); + try p.expect(")"); + return .{ .path = path, .outputs = outs }; + } + + fn envVar(p: *Parser) Error!EnvVar { + try p.expect("("); + const name = try p.string(); + try p.expect(","); + const value = try p.string(); + try p.expect(")"); + return .{ .name = name, .value = value }; + } +}; + +/// Parses a derivation. All allocations go to `arena`; free it as a whole. +pub fn parse(arena: std.mem.Allocator, text: []const u8) Error!Derivation { + var p: Parser = .{ .s = text, .arena = arena }; + try p.expect("Derive("); + const outputs = try p.list(Output, Parser.output); + try p.expect(","); + const input_drvs = try p.list(InputDrv, Parser.inputDrv); + try p.expect(","); + const input_srcs = try p.list([]const u8, Parser.string); + try p.expect(","); + const platform = try p.string(); + try p.expect(","); + const builder = try p.string(); + try p.expect(","); + const args = try p.list([]const u8, Parser.string); + try p.expect(","); + const env = try p.list(EnvVar, Parser.envVar); + try p.expect(")"); + if (p.pos != text.len) return error.Syntax; + return .{ + .outputs = outputs, + .input_drvs = input_drvs, + .input_srcs = input_srcs, + .platform = platform, + .builder = builder, + .args = args, + .env = env, + }; +} + +test "parses a derivation with escapes" { + var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); + defer arena.deinit(); + const text = + \\Derive([("out","/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-hello","","")],[("/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-bash.drv",["out"])],["/nix/store/cccccccccccccccccccccccccccccccc-builder.sh"],"x86_64-linux","/bin/sh",["-e","/builder.sh"],[("buildPhase","echo \"hi\"\nmake -j\\$N"),("name","hello")]) + ; + const d = try parse(arena.allocator(), text); + try std.testing.expectEqual(@as(usize, 1), d.outputs.len); + try std.testing.expectEqualStrings("out", d.input_drvs[0].outputs[0]); + try std.testing.expectEqualStrings("echo \"hi\"\nmake -j\\$N", d.env[0].value); + try std.testing.expectEqualStrings("x86_64-linux", d.platform); + try std.testing.expectError(error.Syntax, parse(arena.allocator(), "Derive([],")); +} + +test "vector quote search matches scalar" { + var buf = [_]u8{'x'} ** 100; + for (0..buf.len) |i| { + for ([_]u8{ '"', '\\' }) |c| { + buf[i] = c; + for (0..i + 1) |start| try std.testing.expectEqual(@as(?usize, i), indexOfQuoteOrEscape(&buf, start)); + buf[i] = 'x'; + } + } +} diff --git a/pkg/nix-flake-lock/build.zig b/pkg/nix-flake-lock/build.zig new file mode 100644 index 0000000..7987c9c --- /dev/null +++ b/pkg/nix-flake-lock/build.zig @@ -0,0 +1,15 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mod = b.addModule("nix-flake-lock", .{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + }); + + const tests = b.addTest(.{ .root_module = mod }); + b.step("test", "Run unit tests").dependOn(&b.addRunArtifact(tests).step); +} diff --git a/pkg/nix-flake-lock/build.zig.zon b/pkg/nix-flake-lock/build.zig.zon new file mode 100644 index 0000000..1a4fa42 --- /dev/null +++ b/pkg/nix-flake-lock/build.zig.zon @@ -0,0 +1,9 @@ +.{ + .name = .nix_flake_lock, + .version = "0.1.0", + .fingerprint = 0x6174916c0100c5fd, + .minimum_zig_version = "0.16.0", + .dependencies = .{ + }, + .paths = .{ "build.zig", "build.zig.zon", "src" }, +} diff --git a/pkg/nix-flake-lock/src/bump.zig b/pkg/nix-flake-lock/src/bump.zig new file mode 100644 index 0000000..d09f75a --- /dev/null +++ b/pkg/nix-flake-lock/src/bump.zig @@ -0,0 +1,104 @@ +//! Single-threaded bump allocator. std.heap.ArenaAllocator is lock-free and +//! thread-safe in Zig 0.16, which puts atomics on every allocation; parsing +//! allocates from one thread only, so plain pointer bumping is enough. +//! Chunks are chained through a header at their start, so a document that +//! fits in its first chunk costs exactly one malloc. + +const std = @import("std"); +const Allocator = std.mem.Allocator; + +const Header = struct { next: ?*Header, len: usize }; + +pub const Bump = struct { + gpa: Allocator, + head: ?*Header = null, + cur: usize = 0, + end: usize = 0, + + pub fn init(gpa: Allocator, first_chunk: usize) Allocator.Error!Bump { + var b: Bump = .{ .gpa = gpa }; + try b.grow(first_chunk); + return b; + } + + pub fn deinit(b: *Bump) void { + var h = b.head; + while (h) |chunk| { + h = chunk.next; + const bytes: [*]align(@alignOf(Header)) u8 = @ptrCast(chunk); + b.gpa.free(bytes[0..chunk.len]); + } + b.* = undefined; + } + + fn grow(b: *Bump, need: usize) Allocator.Error!void { + const last = if (b.head) |h| h.len else 0; + const size = @max(need + @sizeOf(Header) + 64, last * 2, 4096); + const mem = try b.gpa.alignedAlloc(u8, .of(Header), size); + const h: *Header = @ptrCast(mem.ptr); + h.* = .{ .next = b.head, .len = size }; + b.head = h; + b.cur = @intFromPtr(mem.ptr) + @sizeOf(Header); + b.end = @intFromPtr(mem.ptr) + size; + } + + inline fn raw(b: *Bump, len: usize, alignment: usize) Allocator.Error![*]u8 { + var start = std.mem.alignForward(usize, b.cur, alignment); + if (start + len > b.end) { + @branchHint(.unlikely); + try b.grow(len + alignment); + start = std.mem.alignForward(usize, b.cur, alignment); + } + b.cur = start + len; + return @ptrFromInt(start); + } + + pub inline fn alloc(b: *Bump, comptime T: type, n: usize) Allocator.Error![]T { + const p: [*]T = @ptrCast(@alignCast(try b.raw(n * @sizeOf(T), @alignOf(T)))); + return p[0..n]; + } + + pub inline fn dupe(b: *Bump, comptime T: type, items: []const T) Allocator.Error![]T { + const out = try b.alloc(T, items.len); + @memcpy(out, items); + return out; + } + + pub fn allocator(b: *Bump) Allocator { + return .{ .ptr = b, .vtable = &.{ .alloc = vAlloc, .resize = vResize, .remap = vRemap, .free = vFree } }; + } + + fn vAlloc(ctx: *anyopaque, len: usize, alignment: std.mem.Alignment, _: usize) ?[*]u8 { + const b: *Bump = @ptrCast(@alignCast(ctx)); + return b.raw(len, alignment.toByteUnits()) catch null; + } + + fn vResize(ctx: *anyopaque, memory: []u8, _: std.mem.Alignment, new_len: usize, _: usize) bool { + const b: *Bump = @ptrCast(@alignCast(ctx)); + // Only the most recent allocation can grow or shrink in place. + if (@intFromPtr(memory.ptr) + memory.len != b.cur) return new_len <= memory.len; + if (@intFromPtr(memory.ptr) + new_len > b.end) return false; + b.cur = @intFromPtr(memory.ptr) + new_len; + return true; + } + + fn vRemap(ctx: *anyopaque, memory: []u8, alignment: std.mem.Alignment, new_len: usize, ra: usize) ?[*]u8 { + return if (vResize(ctx, memory, alignment, new_len, ra)) memory.ptr else null; + } + + fn vFree(_: *anyopaque, _: []u8, _: std.mem.Alignment, _: usize) void {} +}; + +test "bump allocations are aligned and grow across chunks" { + var b = try Bump.init(std.testing.allocator, 16); + defer b.deinit(); + for (0..1000) |i| { + const s = try b.alloc(u64, i % 7 + 1); + try std.testing.expect(@intFromPtr(s.ptr) % 8 == 0); + s[0] = i; + _ = try b.alloc(u8, 3); + } + var list: std.ArrayList(u32) = .empty; + for (0..10_000) |i| try list.append(b.allocator(), @intCast(i)); + try std.testing.expectEqual(@as(u32, 9999), list.items[9999]); +} diff --git a/pkg/nix-flake-lock/src/root.zig b/pkg/nix-flake-lock/src/root.zig new file mode 100644 index 0000000..08ad8ff --- /dev/null +++ b/pkg/nix-flake-lock/src/root.zig @@ -0,0 +1,1355 @@ +//! Nix `flake.lock` parsing (versions 5-7), `follows` validation and +//! resolution, and canonical version 7 serialisation, without Nix. +//! +//! Semantics follow cachix/nix-flake-lock (Rust): last duplicate key wins, +//! version 5 `info` is overlaid onto `locked`, unreachable nodes are +//! dropped, and output is byte-identical to Nix's canonical writer. +//! +//! Speed comes from single-pass scanning: strings are scanned 32 bytes at a +//! time for '"', '\\', control and non-ASCII bytes at once (UTF-8 is only +//! validated where it may legally occur, inside strings), whitespace runs are +//! skipped with a vector compare, lists are collected in reusable scratch +//! stacks and copied once into an arena, and canonical names borrow from +//! the input instead of allocating. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Bump = @import("bump.zig").Bump; + +pub const Limits = struct { + max_bytes: usize = 16 * 1024 * 1024, + max_nodes: usize = 1_000_000, + max_depth: usize = 256, + max_path_components: usize = 65_536, +}; + +pub const ParseError = error{ + InputTooLarge, + InvalidUtf8, + UnexpectedEof, + Syntax, + TrailingCharacters, + ControlCharacter, + InvalidEscape, + InvalidUnicodeEscape, + InvalidNumber, + UnsupportedVersion, + MissingField, + MissingNode, + TooManyNodes, + NestingTooDeep, + PathTooLong, + UnsupportedInputAttributeType, + CycleToRoot, + OutOfMemory, +}; + +pub const ValidationError = error{ FollowsCycle, MissingFollowsTarget, OutOfMemory }; + +/// Where a parse error happened. +pub const Diagnostic = struct { offset: usize = 0 }; + +pub const Value = union(enum) { + string: []const u8, + integer: u64, + boolean: bool, +}; + +pub const Attr = struct { name: []const u8, value: Value }; + +pub const Path = []const []const u8; + +pub const Edge = union(enum) { + node: u32, + follows: Path, +}; + +pub const Input = struct { name: []const u8, edge: Edge }; + +pub const Locked = struct { + original: []const Attr, + locked: []const Attr, + is_flake: bool, + parent: ?Path, + + pub fn get(attrs: []const Attr, name: []const u8) ?Value { + const i = findSorted(Attr, attrs, name) orelse return null; + return attrs[i].value; + } +}; + +pub const Node = struct { + /// Key in the source document (not graph identity). + key: []const u8, + /// Sorted by name. + inputs: []const Input, + /// Null for the root node. + locked: ?Locked, + + pub fn input(self: *const Node, name: []const u8) ?*const Input { + const i = findSorted(Input, self.inputs, name) orelse return null; + return &self.inputs[i]; + } +}; + +pub const root_id: u32 = 0; + +pub const LockFile = struct { + source_version: u8, + /// Reachable nodes; the root is at index 0. + nodes: []const Node, + + pub fn resolve(self: *const LockFile, gpa: Allocator, path: Path) ValidationError!?u32 { + var r: Resolver = .init(gpa, self); + defer r.deinit(); + return r.resolve(path, false); + } + + fn hasFollows(self: *const LockFile) bool { + for (self.nodes) |n| for (n.inputs) |in| switch (in.edge) { + .follows => |p| if (p.len > 0) return true, + .node => {}, + }; + return false; + } + + /// Checks that every `follows` target exists and no chain is cyclic. + pub fn validate(self: *const LockFile, gpa: Allocator) ValidationError!void { + // Only `follows` edges can fail, and every parsed node is reachable, + // so a graph without non-empty follows edges is valid as is. + if (!self.hasFollows()) return; + var fallback = std.heap.stackFallback(4096, gpa); + var bump = try Bump.init(fallback.get(), 2 * 1024); + defer bump.deinit(); + var r: Resolver = .init(bump.allocator(), self); + const visited = try bump.alloc(bool, self.nodes.len); + @memset(visited, false); + visited[root_id] = true; + // Each node is pushed at most once. + const stack_buf = try bump.alloc(u32, self.nodes.len); + var stack: std.ArrayList(u32) = .initBuffer(stack_buf); + stack.appendAssumeCapacity(root_id); + while (stack.pop()) |id| { + const node = &self.nodes[id]; + var i = node.inputs.len; + while (i > 0) { + i -= 1; + switch (node.inputs[i].edge) { + .node => |child| if (!visited[child]) { + visited[child] = true; + stack.appendAssumeCapacity(child); + }, + .follows => |target| if (target.len > 0) { + if (try r.resolve(target, true) == null) return error.MissingFollowsTarget; + }, + } + } + } + } + + /// Appends canonical, two-space-indented version 7 JSON (with the + /// trailing newline Nix writes). + pub fn writeNix(self: *const LockFile, gpa: Allocator, out: *std.ArrayList(u8)) Allocator.Error!void { + var w: Writer = .{ .gpa = gpa, .out = out, .lock = self }; + return w.write(); + } +}; + +fn findSorted(comptime T: type, items: []const T, name: []const u8) ?usize { + var lo: usize = 0; + var hi: usize = items.len; + while (lo < hi) { + const mid = lo + (hi - lo) / 2; + switch (std.mem.order(u8, items[mid].name, name)) { + .eq => return mid, + .lt => lo = mid + 1, + .gt => hi = mid, + } + } + return null; +} + +// ── SIMD scanning ── + +const lanes = 32; +const V = @Vector(lanes, u8); +const Mask = std.meta.Int(.unsigned, lanes); + +inline fn splat(c: u8) V { + return @splat(c); +} + +inline fn bits(b: @Vector(lanes, bool)) Mask { + return @bitCast(b); +} + +/// First byte at or after `i` that ends or interrupts a plain string run: +/// '"', '\\', a control byte, or a non-ASCII byte. +fn scanString(s: []const u8, start: usize) ?usize { + var i = start; + while (i + lanes <= s.len) : (i += lanes) { + const v: V = s[i..][0..lanes].*; + const m = bits(v == splat('"')) | bits(v == splat('\\')) | bits(v < splat(0x20)) | bits(v >= splat(0x80)); + if (m != 0) return i + @ctz(m); + } + while (i < s.len) : (i += 1) { + const c = s[i]; + if (c == '"' or c == '\\' or c < 0x20 or c >= 0x80) return i; + } + return null; +} + +fn isSpace(c: u8) bool { + return c == ' ' or c == '\n' or c == '\r' or c == '\t'; +} + +/// First non-whitespace byte at or after `i`. +fn skipSpace(s: []const u8, start: usize) usize { + var i = start; + if (i >= s.len or !isSpace(s[i])) return i; + while (i + lanes <= s.len) : (i += lanes) { + const v: V = s[i..][0..lanes].*; + const ws = bits(v == splat(' ')) | bits(v == splat('\n')) | bits(v == splat('\r')) | bits(v == splat('\t')); + if (ws != ~@as(Mask, 0)) return i + @ctz(~ws); + } + while (i < s.len and isSpace(s[i])) i += 1; + return i; +} + +/// Bytes that must be escaped in JSON output. +fn scanEscape(s: []const u8, start: usize) usize { + var i = start; + while (i + lanes <= s.len) : (i += lanes) { + const v: V = s[i..][0..lanes].*; + const m = bits(v == splat('"')) | bits(v == splat('\\')) | bits(v < splat(0x20)); + if (m != 0) return i + @ctz(m); + } + while (i < s.len) : (i += 1) { + const c = s[i]; + if (c == '"' or c == '\\' or c < 0x20) return i; + } + return s.len; +} + +// ── Parser ── + +const WireEdge = union(enum) { node: []const u8, follows: Path }; + +const WireInput = struct { + offset: usize, + name: []const u8, + edge: WireEdge, + target: u32 = 0, +}; + +const WireNode = struct { + offset: usize, + key: []const u8, + inputs: []WireInput = &.{}, + original: ?[]Attr = null, + locked: ?[]Attr = null, + info: ?[]Attr = null, + is_flake: bool = true, + parent: ?Path = null, +}; + +fn Keyed(comptime T: type) type { + return struct { offset: usize, item: T }; +} + +/// Sort by name and keep the last occurrence of each duplicate, like Nix. +fn sortDedup(comptime T: type, items: []Keyed(T), nameOf: fn (*const T) []const u8) usize { + const Ctx = struct { + fn lessThan(_: void, a: Keyed(T), b: Keyed(T)) bool { + return switch (std.mem.order(u8, nameOf(&a.item), nameOf(&b.item))) { + .lt => true, + .gt => false, + .eq => a.offset > b.offset, + }; + } + }; + if (items.len < 2) return items.len; + // Inputs are usually already sorted: skip the sort when they are. + var sorted = true; + for (items[1..], items[0 .. items.len - 1]) |b, a| { + if (std.mem.order(u8, nameOf(&a.item), nameOf(&b.item)) != .lt) { + sorted = false; + break; + } + } + if (sorted) return items.len; + // Offsets are unique, so an unstable sort gives the same order. + std.mem.sortUnstable(Keyed(T), items, {}, Ctx.lessThan); + var w: usize = 1; + for (items[1..]) |it| { + if (!std.mem.eql(u8, nameOf(&it.item), nameOf(&items[w - 1].item))) { + items[w] = it; + w += 1; + } + } + return w; +} + +fn attrName(a: *const Attr) []const u8 { + return a.name; +} +fn inputName(i: *const WireInput) []const u8 { + return i.name; +} +fn nodeKey(n: *const WireNode) []const u8 { + return n.key; +} + +const Parser = struct { + s: []const u8, + pos: usize = 0, + depth: usize = 0, + limits: Limits, + bump: *Bump, + /// The bump allocator as a std Allocator, for growable lists. + arena: Allocator, + diag: ?*Diagnostic, + attr_scratch: std.ArrayList(Keyed(Attr)) = .empty, + input_scratch: std.ArrayList(Keyed(WireInput)) = .empty, + path_scratch: std.ArrayList([]const u8) = .empty, + + fn fail(p: *Parser, err: ParseError, at: usize) ParseError { + if (p.diag) |d| d.offset = at; + return err; + } + + inline fn ws(p: *Parser) void { + p.pos = skipSpace(p.s, p.pos); + } + + inline fn peek(p: *Parser) ?u8 { + return if (p.pos < p.s.len) p.s[p.pos] else null; + } + + inline fn consume(p: *Parser, c: u8) bool { + if (p.pos < p.s.len and p.s[p.pos] == c) { + p.pos += 1; + return true; + } + return false; + } + + fn expect(p: *Parser, c: u8) ParseError!void { + if (p.consume(c)) return; + return p.fail(if (p.pos >= p.s.len) error.UnexpectedEof else error.Syntax, p.pos); + } + + fn enter(p: *Parser) ParseError!void { + p.depth += 1; + if (p.depth > p.limits.max_depth) return p.fail(error.NestingTooDeep, p.pos); + } + + fn leave(p: *Parser) void { + p.depth -= 1; + } + + /// After a member: true when the object/array closed, false after ','. + fn next(p: *Parser, close: u8) ParseError!bool { + p.ws(); + if (p.consume(close)) { + p.leave(); + return true; + } + try p.expect(','); + p.ws(); + return false; + } + + fn startsWith(p: *Parser, lit: []const u8) bool { + return std.mem.startsWith(u8, p.s[p.pos..], lit); + } + + fn string(p: *Parser) ParseError![]const u8 { + try p.expect('"'); + const start = p.pos; + var decoded: ?std.ArrayList(u8) = null; + var seg = start; + while (true) { + const i = scanString(p.s, p.pos) orelse return p.fail(error.UnexpectedEof, p.s.len); + const c = p.s[i]; + switch (c) { + '"' => { + p.pos = i + 1; + if (decoded) |*d| { + try d.appendSlice(p.arena, p.s[seg..i]); + return d.items; + } + return p.s[start..i]; + }, + '\\' => { + if (decoded == null) decoded = .empty; + try decoded.?.appendSlice(p.arena, p.s[seg..i]); + p.pos = i + 1; + try p.escape(&decoded.?); + seg = p.pos; + }, + 0...0x1f => return p.fail(error.ControlCharacter, i), + else => { + const len = std.unicode.utf8ByteSequenceLength(c) catch return p.fail(error.InvalidUtf8, i); + if (i + len > p.s.len) return p.fail(error.InvalidUtf8, i); + _ = std.unicode.utf8Decode(p.s[i..][0..len]) catch return p.fail(error.InvalidUtf8, i); + p.pos = i + len; + }, + } + } + } + + fn escape(p: *Parser, out: *std.ArrayList(u8)) ParseError!void { + if (p.pos >= p.s.len) return p.fail(error.UnexpectedEof, p.pos); + const e = p.s[p.pos]; + p.pos += 1; + const simple: ?u8 = switch (e) { + '"' => '"', + '\\' => '\\', + '/' => '/', + 'b' => 0x08, + 'f' => 0x0c, + 'n' => '\n', + 'r' => '\r', + 't' => '\t', + 'u' => null, + else => return p.fail(error.InvalidEscape, p.pos), + }; + if (simple) |c| return out.append(p.arena, c); + const first = try p.hexQuad(); + var cp: u21 = first; + if (first >= 0xd800 and first <= 0xdbff) { + if (!p.consume('\\') or !p.consume('u')) return p.fail(error.InvalidUnicodeEscape, p.pos); + const second = try p.hexQuad(); + if (second < 0xdc00 or second > 0xdfff) return p.fail(error.InvalidUnicodeEscape, p.pos); + cp = 0x10000 + ((@as(u21, first) - 0xd800) << 10) + (second - 0xdc00); + } else if (first >= 0xdc00 and first <= 0xdfff) { + return p.fail(error.InvalidUnicodeEscape, p.pos); + } + var buf: [4]u8 = undefined; + const n = std.unicode.utf8Encode(cp, &buf) catch return p.fail(error.InvalidUnicodeEscape, p.pos); + try out.appendSlice(p.arena, buf[0..n]); + } + + fn hexQuad(p: *Parser) ParseError!u16 { + if (p.pos + 4 > p.s.len) return p.fail(error.UnexpectedEof, p.pos); + var v: u16 = 0; + for (p.s[p.pos..][0..4]) |c| { + const d: u16 = switch (c) { + '0'...'9' => c - '0', + 'a'...'f' => c - 'a' + 10, + 'A'...'F' => c - 'A' + 10, + else => return p.fail(error.InvalidUnicodeEscape, p.pos), + }; + v = (v << 4) | d; + p.pos += 1; + } + return v; + } + + fn unsigned(p: *Parser) ParseError!u64 { + const start = p.pos; + const first = p.peek() orelse return p.fail(error.UnexpectedEof, p.pos); + if (first < '0' or first > '9') return p.fail(error.InvalidNumber, start); + p.pos += 1; + if (first == '0' and p.peek() != null and std.ascii.isDigit(p.peek().?)) return p.fail(error.InvalidNumber, start); + var v: u64 = first - '0'; + while (p.peek()) |c| { + if (c < '0' or c > '9') break; + p.pos += 1; + v = std.math.mul(u64, v, 10) catch return p.fail(error.InvalidNumber, start); + v = std.math.add(u64, v, c - '0') catch return p.fail(error.InvalidNumber, start); + } + if (p.peek()) |c| if (c == '.' or c == 'e' or c == 'E') return p.fail(error.InvalidNumber, start); + return v; + } + + fn boolean(p: *Parser) ParseError!bool { + if (p.startsWith("true")) { + p.pos += 4; + return true; + } + if (p.startsWith("false")) { + p.pos += 5; + return false; + } + return p.fail(error.Syntax, p.pos); + } + + fn null_(p: *Parser) ParseError!void { + if (!p.startsWith("null")) return p.fail(error.Syntax, p.pos); + p.pos += 4; + } + + fn skipValue(p: *Parser) ParseError!void { + p.ws(); + const c = p.peek() orelse return p.fail(error.UnexpectedEof, p.pos); + switch (c) { + '"' => _ = try p.string(), + '{' => { + p.pos += 1; + try p.enter(); + p.ws(); + if (p.consume('}')) return p.leave(); + while (true) { + _ = try p.string(); + p.ws(); + try p.expect(':'); + try p.skipValue(); + if (try p.next('}')) return; + } + }, + '[' => { + p.pos += 1; + try p.enter(); + p.ws(); + if (p.consume(']')) return p.leave(); + while (true) { + try p.skipValue(); + if (try p.next(']')) return; + } + }, + 't', 'f' => _ = try p.boolean(), + 'n' => try p.null_(), + '-', '0'...'9' => try p.skipNumber(), + else => return p.fail(error.Syntax, p.pos), + } + } + + fn skipNumber(p: *Parser) ParseError!void { + const start = p.pos; + _ = p.consume('-'); + const d = p.peek() orelse return p.fail(error.InvalidNumber, start); + p.pos += 1; + if (d == '0') { + if (p.peek() != null and std.ascii.isDigit(p.peek().?)) return p.fail(error.InvalidNumber, start); + } else if (d >= '1' and d <= '9') { + while (p.peek() != null and std.ascii.isDigit(p.peek().?)) p.pos += 1; + } else return p.fail(error.InvalidNumber, start); + if (p.consume('.')) { + if (p.peek() == null or !std.ascii.isDigit(p.peek().?)) return p.fail(error.InvalidNumber, start); + while (p.peek() != null and std.ascii.isDigit(p.peek().?)) p.pos += 1; + } + if (p.peek()) |e| if (e == 'e' or e == 'E') { + p.pos += 1; + if (p.peek()) |sign| if (sign == '+' or sign == '-') { + p.pos += 1; + }; + if (p.peek() == null or !std.ascii.isDigit(p.peek().?)) return p.fail(error.InvalidNumber, start); + while (p.peek() != null and std.ascii.isDigit(p.peek().?)) p.pos += 1; + }; + } + + fn key(p: *Parser) ParseError![]const u8 { + const k = try p.string(); + p.ws(); + try p.expect(':'); + p.ws(); + return k; + } + + fn path(p: *Parser) ParseError!Path { + try p.expect('['); + try p.enter(); + p.ws(); + if (p.consume(']')) { + p.leave(); + return &.{}; + } + const mark = p.path_scratch.items.len; + defer p.path_scratch.shrinkRetainingCapacity(mark); + while (true) { + try p.path_scratch.append(p.arena, try p.string()); + if (p.path_scratch.items.len - mark > p.limits.max_path_components) return p.fail(error.PathTooLong, p.pos); + if (try p.next(']')) break; + } + return p.bump.dupe([]const u8, p.path_scratch.items[mark..]); + } + + fn attrs(p: *Parser) ParseError![]Attr { + try p.expect('{'); + try p.enter(); + p.ws(); + if (p.consume('}')) { + p.leave(); + return &.{}; + } + const mark = p.attr_scratch.items.len; + defer p.attr_scratch.shrinkRetainingCapacity(mark); + while (true) { + const offset = p.pos; + const name = try p.key(); + const c = p.peek() orelse return p.fail(error.UnexpectedEof, p.pos); + const value: Value = switch (c) { + '"' => .{ .string = try p.string() }, + 't', 'f' => .{ .boolean = try p.boolean() }, + '0'...'9' => .{ .integer = try p.unsigned() }, + else => return p.fail(error.UnsupportedInputAttributeType, p.pos), + }; + try p.attr_scratch.append(p.arena, .{ .offset = offset, .item = .{ .name = name, .value = value } }); + if (try p.next('}')) break; + } + const items = p.attr_scratch.items[mark..]; + const n = sortDedup(Attr, items, attrName); + const out = try p.bump.alloc(Attr, n); + for (items[0..n], out) |k, *o| o.* = k.item; + return out; + } + + fn inputs(p: *Parser) ParseError![]WireInput { + try p.expect('{'); + try p.enter(); + p.ws(); + if (p.consume('}')) { + p.leave(); + return &.{}; + } + const mark = p.input_scratch.items.len; + defer p.input_scratch.shrinkRetainingCapacity(mark); + while (true) { + const offset = p.pos; + const name = try p.key(); + const edge: WireEdge = if (p.peek() == '[') .{ .follows = try p.path() } else .{ .node = try p.string() }; + try p.input_scratch.append(p.arena, .{ .offset = offset, .item = .{ .offset = offset, .name = name, .edge = edge } }); + if (try p.next('}')) break; + } + const items = p.input_scratch.items[mark..]; + const n = sortDedup(WireInput, items, inputName); + const out = try p.bump.alloc(WireInput, n); + for (items[0..n], out) |k, *o| o.* = k.item; + return out; + } + + fn node(p: *Parser, offset: usize, k: []const u8) ParseError!WireNode { + try p.expect('{'); + try p.enter(); + var n: WireNode = .{ .offset = offset, .key = k }; + p.ws(); + if (p.consume('}')) { + p.leave(); + return n; + } + while (true) { + const field = try p.key(); + if (eq(field, "inputs")) { + n.inputs = try p.inputs(); + } else if (eq(field, "original")) { + n.original = try p.attrs(); + } else if (eq(field, "locked")) { + n.locked = try p.attrs(); + } else if (eq(field, "info")) { + n.info = try p.attrs(); + } else if (eq(field, "flake")) { + n.is_flake = try p.boolean(); + } else if (eq(field, "parent")) { + if (p.startsWith("null")) { + try p.null_(); + n.parent = null; + } else n.parent = try p.path(); + } else try p.skipValue(); + if (try p.next('}')) return n; + } + } + + fn nodes(p: *Parser) ParseError![]WireNode { + try p.expect('{'); + try p.enter(); + var list: std.ArrayList(Keyed(WireNode)) = .empty; + defer list.deinit(p.arena); + p.ws(); + if (p.consume('}')) { + p.leave(); + } else while (true) { + const offset = p.pos; + const k = try p.key(); + try list.append(p.arena, .{ .offset = offset, .item = try p.node(offset, k) }); + if (try p.next('}')) break; + } + const n = sortDedup(WireNode, list.items, nodeKey); + if (n > p.limits.max_nodes) return p.fail(error.TooManyNodes, p.pos); + const out = try p.bump.alloc(WireNode, n); + for (list.items[0..n], out) |k, *o| o.* = k.item; + return out; + } +}; + +fn eq(a: []const u8, comptime b: []const u8) bool { + return a.len == b.len and std.mem.eql(u8, a, b); +} + +/// Overlays `newer` onto `older` (both sorted by name); `newer` wins. +fn overlay(arena: Allocator, older: []const Attr, newer: []const Attr) Allocator.Error![]const Attr { + var out: std.ArrayList(Attr) = try .initCapacity(arena, older.len + newer.len); + var i: usize = 0; + var j: usize = 0; + while (i < older.len and j < newer.len) { + switch (std.mem.order(u8, older[i].name, newer[j].name)) { + .lt => { + out.appendAssumeCapacity(older[i]); + i += 1; + }, + .eq => { + out.appendAssumeCapacity(newer[j]); + i += 1; + j += 1; + }, + .gt => { + out.appendAssumeCapacity(newer[j]); + j += 1; + }, + } + } + out.appendSliceAssumeCapacity(older[i..]); + out.appendSliceAssumeCapacity(newer[j..]); + return out.items; +} + +fn findNode(nodes: []const WireNode, k: []const u8) ?usize { + var lo: usize = 0; + var hi: usize = nodes.len; + while (lo < hi) { + const mid = lo + (hi - lo) / 2; + switch (std.mem.order(u8, nodes[mid].key, k)) { + .eq => return mid, + .lt => lo = mid + 1, + .gt => hi = mid, + } + } + return null; +} + +/// A parsed lock file and the memory backing it. Unescaped strings borrow +/// from the input, which must outlive the document. +pub const Document = struct { + lock: LockFile, + bump: Bump, + + pub fn deinit(self: *Document) void { + self.bump.deinit(); + self.* = undefined; + } +}; + +/// Parses a lock file. All allocations come from one bump allocator owned +/// by the returned document; a typical file needs a single malloc. +pub fn parse(gpa: Allocator, bytes: []const u8, limits: Limits, diag: ?*Diagnostic) ParseError!Document { + if (bytes.len > limits.max_bytes) return error.InputTooLarge; + var bump = try Bump.init(gpa, bytes.len * 2 + 4096); + errdefer bump.deinit(); + var p: Parser = .{ .s = bytes, .limits = limits, .bump = &bump, .arena = bump.allocator(), .diag = diag }; + + var version: ?u64 = null; + var root_key: ?[]const u8 = null; + var wire: ?[]WireNode = null; + p.ws(); + try p.expect('{'); + try p.enter(); + p.ws(); + if (p.consume('}')) { + p.leave(); + } else while (true) { + const k = try p.key(); + if (eq(k, "version")) { + version = try p.unsigned(); + } else if (eq(k, "root")) { + root_key = try p.string(); + } else if (eq(k, "nodes")) { + wire = try p.nodes(); + } else try p.skipValue(); + if (try p.next('}')) break; + } + const v = version orelse return p.fail(error.MissingField, p.pos); + if (v < 5 or v > 7) return p.fail(error.UnsupportedVersion, p.pos); + const rk = root_key orelse return p.fail(error.MissingField, p.pos); + const wn = wire orelse return p.fail(error.MissingField, p.pos); + p.ws(); + if (p.pos != bytes.len) return p.fail(error.TrailingCharacters, p.pos); + const lock = try build(&p, @intCast(v), rk, wn); + return .{ .lock = lock, .bump = bump }; +} + +fn build(p: *Parser, version: u8, root_key: []const u8, wire: []WireNode) ParseError!LockFile { + const gpa = p.arena; + const root_old = findNode(wire, root_key) orelse return p.fail(error.MissingNode, 0); + + const new_id = try gpa.alloc(u32, wire.len); + defer gpa.free(new_id); + const unseen = std.math.maxInt(u32); + @memset(new_id, unseen); + var stack: std.ArrayList(u32) = .empty; + defer stack.deinit(gpa); + try stack.append(gpa, @intCast(root_old)); + var reachable: usize = 0; + // Mark reachability first (0 = seen), assign dense ids afterwards so the + // arena order matches source key order, like the reference. + while (stack.pop()) |idx| { + if (new_id[idx] != unseen) continue; + new_id[idx] = 0; + reachable += 1; + for (wire[idx].inputs) |*in| switch (in.edge) { + .node => |target| { + const t = findNode(wire, target) orelse return p.fail(error.MissingNode, in.offset); + if (t == root_old) return p.fail(error.CycleToRoot, in.offset); + in.target = @intCast(t); + try stack.append(gpa, @intCast(t)); + }, + .follows => {}, + }; + } + if (reachable > p.limits.max_nodes or reachable > std.math.maxInt(u32)) return p.fail(error.TooManyNodes, 0); + var next_id: u32 = 1; + for (new_id, 0..) |*id, old| { + if (id.* == unseen) continue; + if (old == root_old) { + id.* = root_id; + } else { + id.* = next_id; + next_id += 1; + } + } + + const nodes = try p.bump.alloc(Node, reachable); + for (wire, 0..) |*w, old| { + const id = new_id[old]; + if (id == unseen) continue; + const ins = try p.bump.alloc(Input, w.inputs.len); + for (w.inputs, ins) |in, *o| o.* = .{ + .name = in.name, + .edge = switch (in.edge) { + .node => .{ .node = new_id[in.target] }, + .follows => |f| .{ .follows = f }, + }, + }; + var locked: ?Locked = null; + if (old != root_old) { + const l = w.locked orelse return p.fail(error.MissingField, w.offset); + const original = w.original orelse return p.fail(error.MissingField, w.offset); + const merged: []const Attr = if (w.info) |info| try overlay(p.arena, l, info) else l; + locked = .{ .original = original, .locked = merged, .is_flake = w.is_flake, .parent = w.parent }; + } + nodes[id] = .{ .key = w.key, .inputs = ins, .locked = locked }; + } + return .{ .source_version = version, .nodes = nodes }; +} + +// ── follows resolution ── + +const EdgeKey = struct { node: u32, input: u32 }; + +const Resolver = struct { + gpa: Allocator, + lock: *const LockFile, + /// Resolved follows targets, keyed by the path slice's address. + cache: std.AutoHashMapUnmanaged(usize, ?u32) = .empty, + active: std.AutoHashMapUnmanaged(EdgeKey, void) = .empty, + + const Step = union(enum) { component: []const u8, finish: struct { edge: EdgeKey, target: usize } }; + + fn init(gpa: Allocator, lock: *const LockFile) Resolver { + return .{ .gpa = gpa, .lock = lock }; + } + + fn deinit(r: *Resolver) void { + r.cache.deinit(r.gpa); + r.active.deinit(r.gpa); + } + + fn resolve(r: *Resolver, path: Path, use_cache: bool) ValidationError!?u32 { + var steps: std.ArrayList(Step) = .empty; + defer steps.deinit(r.gpa); + var i = path.len; + while (i > 0) { + i -= 1; + try steps.append(r.gpa, .{ .component = path[i] }); + } + var current: u32 = root_id; + r.active.clearRetainingCapacity(); + while (steps.pop()) |step| switch (step) { + .component => |name| { + const node = &r.lock.nodes[current]; + const idx = findSorted(Input, node.inputs, name) orelse return null; + switch (node.inputs[idx].edge) { + .node => |child| current = child, + .follows => |target| { + const addr = @intFromPtr(target.ptr); + if (use_cache) if (r.cache.get(addr)) |cached| { + current = cached orelse return null; + continue; + }; + const edge: EdgeKey = .{ .node = current, .input = @intCast(idx) }; + const gop = try r.active.getOrPut(r.gpa, edge); + if (gop.found_existing) return error.FollowsCycle; + try steps.append(r.gpa, .{ .finish = .{ .edge = edge, .target = addr } }); + var j = target.len; + while (j > 0) { + j -= 1; + try steps.append(r.gpa, .{ .component = target[j] }); + } + current = root_id; + }, + } + }, + .finish => |f| { + _ = r.active.remove(f.edge); + if (use_cache) try r.cache.put(r.gpa, f.target, current); + }, + }; + return current; + } +}; + +// ── Canonical writer ── + +const Writer = struct { + gpa: Allocator, + out: *std.ArrayList(u8), + lock: *const LockFile, + + // Capacity is reserved per node (see `reserveNode`), so every write + // below is an unchecked copy. + inline fn raw(w: *Writer, s: []const u8) void { + w.out.appendSliceAssumeCapacity(s); + } + + inline fn byte(w: *Writer, c: u8) void { + w.out.appendAssumeCapacity(c); + } + + inline fn indent(w: *Writer, level: usize) void { + w.out.appendNTimesAssumeCapacity(' ', 2 * level); + } + + fn str(w: *Writer, s: []const u8) void { + const hex = "0123456789abcdef"; + w.byte('"'); + var start: usize = 0; + while (true) { + const i = scanEscape(s, start); + w.raw(s[start..i]); + if (i == s.len) break; + const c = s[i]; + switch (c) { + '"' => w.raw("\\\""), + '\\' => w.raw("\\\\"), + 0x08 => w.raw("\\b"), + 0x0c => w.raw("\\f"), + '\n' => w.raw("\\n"), + '\r' => w.raw("\\r"), + '\t' => w.raw("\\t"), + else => w.raw(&.{ '\\', 'u', '0', '0', hex[c >> 4], hex[c & 15] }), + } + start = i + 1; + } + w.byte('"'); + } + + fn int(w: *Writer, value: u64) void { + var buf: [20]u8 = undefined; + var i: usize = buf.len; + var v = value; + while (true) { + i -= 1; + buf[i] = @intCast('0' + v % 10); + v /= 10; + if (v == 0) break; + } + w.raw(buf[i..]); + } + + fn field(w: *Writer, level: usize, comptime name: []const u8) void { + w.indent(level); + w.raw("\"" ++ name ++ "\": "); + } + + fn sep(w: *Writer, written: usize, total: usize) void { + if (written != total) w.byte(','); + w.byte('\n'); + } + + fn path(w: *Writer, p: Path, level: usize) void { + if (p.len == 0) return w.raw("[]"); + w.raw("[\n"); + for (p, 0..) |c, i| { + w.indent(level + 1); + w.str(c); + if (i + 1 != p.len) w.byte(','); + w.byte('\n'); + } + w.indent(level); + w.byte(']'); + } + + fn attrs(w: *Writer, a: []const Attr, level: usize) void { + var len: usize = 0; + for (a) |x| len += @intFromBool(!eq(x.name, "__final")); + if (len == 0) return w.raw("{}"); + w.raw("{\n"); + var i: usize = 0; + for (a) |x| { + if (eq(x.name, "__final")) continue; + w.indent(level + 1); + w.str(x.name); + w.raw(": "); + switch (x.value) { + .string => |s| w.str(s), + .integer => |n| w.int(n), + .boolean => |b| w.raw(if (b) "true" else "false"), + } + i += 1; + if (i != len) w.byte(','); + w.byte('\n'); + } + w.indent(level); + w.byte('}'); + } + + /// Upper bound on the bytes `node` writes: escaping expands a byte to at + /// most 6, and a line has at most 2 * (level + 3) indentation plus a few + /// bytes of punctuation. + fn reserveNode(w: *Writer, n: *const Node, nm: []const []const u8, name: []const u8) Allocator.Error!void { + const line = 2 * 6 + 16; + var bound: usize = 6 * name.len + 4 * line; + for (n.inputs) |in| { + bound += 6 * in.name.len + line; + switch (in.edge) { + .node => |id| bound += 6 * nm[id].len, + .follows => |p| for (p) |c| { + bound += 6 * c.len + line; + }, + } + } + if (n.locked) |l| { + for ([_][]const Attr{ l.locked, l.original }) |a| for (a) |x| { + bound += 6 * x.name.len + line + switch (x.value) { + .string => |s| 6 * s.len, + else => 20, + }; + }; + if (l.parent) |p| for (p) |c| { + bound += 6 * c.len + line; + }; + } + try w.out.ensureUnusedCapacity(w.gpa, bound); + } + + fn node(w: *Writer, n: *const Node, nm: []const []const u8, level: usize) void { + const has_flake = if (n.locked) |l| !l.is_flake else false; + const has_inputs = n.inputs.len > 0; + const has_parent = if (n.locked) |l| l.parent != null else false; + const total = @as(usize, @intFromBool(has_flake)) + @intFromBool(has_inputs) + @as(usize, if (n.locked != null) 2 else 0) + @intFromBool(has_parent); + if (total == 0) return w.raw("{}"); + w.raw("{\n"); + var written: usize = 0; + if (has_flake) { + w.field(level + 1, "flake"); + w.raw("false"); + written += 1; + w.sep(written, total); + } + if (has_inputs) { + w.field(level + 1, "inputs"); + w.raw("{\n"); + for (n.inputs, 0..) |in, i| { + w.indent(level + 2); + w.str(in.name); + w.raw(": "); + switch (in.edge) { + .node => |id| w.str(nm[id]), + .follows => |p| w.path(p, level + 2), + } + if (i + 1 != n.inputs.len) w.byte(','); + w.byte('\n'); + } + w.indent(level + 1); + w.byte('}'); + written += 1; + w.sep(written, total); + } + if (n.locked) |l| { + w.field(level + 1, "locked"); + w.attrs(l.locked, level + 1); + written += 1; + w.sep(written, total); + w.field(level + 1, "original"); + w.attrs(l.original, level + 1); + written += 1; + w.sep(written, total); + if (l.parent) |p| { + w.field(level + 1, "parent"); + w.path(p, level + 1); + written += 1; + w.sep(written, total); + } + } + w.indent(level); + w.byte('}'); + } + + /// Nix's naming: depth-first from the root, each node named after the + /// first input that reaches it, with `_2`, `_3`, ... on collisions. + fn names(w: *Writer, tmp: Allocator) Allocator.Error![]const []const u8 { + const lock = w.lock; + const out = try tmp.alloc([]const u8, lock.nodes.len); + const named = try tmp.alloc(bool, lock.nodes.len); + @memset(named, false); + out[root_id] = "root"; + named[root_id] = true; + // Small graphs check collisions by scanning assigned names; a hash + // set only pays off beyond a few dozen nodes. + const small = lock.nodes.len <= 48; + var assigned: std.ArrayList([]const u8) = .empty; + var used: std.StringHashMapUnmanaged(void) = .empty; + if (small) { + try assigned.ensureTotalCapacity(tmp, lock.nodes.len); + assigned.appendAssumeCapacity("root"); + } else { + try used.ensureTotalCapacity(tmp, @intCast(lock.nodes.len)); + used.putAssumeCapacity("root", {}); + } + var suffix: std.StringHashMapUnmanaged(u64) = .empty; + const Frame = struct { node: u32, next: u32 }; + var stack: std.ArrayList(Frame) = .empty; + try stack.append(tmp, .{ .node = root_id, .next = 0 }); + while (stack.items.len > 0) { + const top = &stack.items[stack.items.len - 1]; + const n = &lock.nodes[top.node]; + if (top.next >= n.inputs.len) { + _ = stack.pop(); + continue; + } + const in = n.inputs[top.next]; + top.next += 1; + const child = switch (in.edge) { + .node => |c| c, + .follows => continue, + }; + if (named[child]) continue; + const taken = struct { + fn check(s: bool, a: []const []const u8, u: *const std.StringHashMapUnmanaged(void), name: []const u8) bool { + if (!s) return u.contains(name); + for (a) |x| if (std.mem.eql(u8, x, name)) return true; + return false; + } + }.check; + var name = in.name; + if (taken(small, assigned.items, &used, name)) { + const gop = try suffix.getOrPut(tmp, in.name); + if (!gop.found_existing) gop.value_ptr.* = 2; + while (true) { + const candidate = try std.fmt.allocPrint(tmp, "{s}_{d}", .{ in.name, gop.value_ptr.* }); + gop.value_ptr.* += 1; + if (!taken(small, assigned.items, &used, candidate)) { + name = candidate; + break; + } + } + } + if (small) assigned.appendAssumeCapacity(name) else try used.put(tmp, name, {}); + out[child] = name; + named[child] = true; + try stack.append(tmp, .{ .node = child, .next = 0 }); + } + return out; + } + + fn write(w: *Writer) Allocator.Error!void { + var fallback = std.heap.stackFallback(8192, w.gpa); + var bump = try Bump.init(fallback.get(), 4096); + defer bump.deinit(); + const tmp = bump.allocator(); + const nm = try w.names(tmp); + const order = try bump.alloc(u32, nm.len); + for (order, 0..) |*o, i| o.* = @intCast(i); + std.mem.sortUnstable(u32, order, nm, struct { + fn lessThan(ctx: []const []const u8, a: u32, b: u32) bool { + return std.mem.lessThan(u8, ctx[a], ctx[b]); + } + }.lessThan); + try w.out.ensureUnusedCapacity(w.gpa, 64); + w.raw("{\n \"nodes\": {"); + for (order, 0..) |id, i| { + const n = &w.lock.nodes[id]; + try w.reserveNode(n, nm, nm[id]); + w.raw(if (i == 0) "\n" else ",\n"); + w.indent(2); + w.str(nm[id]); + w.raw(": "); + w.node(n, nm, 2); + } + try w.out.ensureUnusedCapacity(w.gpa, 64); + w.raw("\n },\n \"root\": \"root\",\n \"version\": 7\n}\n"); + } +}; + +// ── Tests ── + +const testing = std.testing; + +const typical = + \\{ + \\ "nodes": { + \\ "nixpkgs": { + \\ "locked": { + \\ "lastModified": 1750000000, + \\ "narHash": "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + \\ "owner": "NixOS", + \\ "repo": "nixpkgs", + \\ "rev": "0123456789abcdef0123456789abcdef01234567", + \\ "type": "github" + \\ }, + \\ "original": { + \\ "owner": "NixOS", + \\ "ref": "nixos-unstable", + \\ "repo": "nixpkgs", + \\ "type": "github" + \\ } + \\ }, + \\ "root": { + \\ "inputs": { + \\ "nixpkgs": "nixpkgs", + \\ "tools": "tools" + \\ } + \\ }, + \\ "tools": { + \\ "inputs": { + \\ "nixpkgs": [ + \\ "nixpkgs" + \\ ] + \\ }, + \\ "locked": { + \\ "lastModified": 1750000001, + \\ "narHash": "sha256-BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB=", + \\ "owner": "example", + \\ "repo": "tools", + \\ "rev": "89abcdef0123456789abcdef0123456789abcdef", + \\ "type": "github" + \\ }, + \\ "original": { + \\ "owner": "example", + \\ "repo": "tools", + \\ "type": "github" + \\ } + \\ } + \\ }, + \\ "root": "root", + \\ "version": 7 + \\} + \\ +; + +test "canonical documents round-trip byte for byte" { + var doc = try parse(testing.allocator, typical, .{}, null); + defer doc.deinit(); + try doc.lock.validate(testing.allocator); + var out: std.ArrayList(u8) = .empty; + defer out.deinit(testing.allocator); + try doc.lock.writeNix(testing.allocator, &out); + try testing.expectEqualStrings(typical, out.items); + try testing.expectEqual(@as(?u32, doc.lock.nodes[root_id].input("nixpkgs").?.edge.node), try doc.lock.resolve(testing.allocator, &.{ "tools", "nixpkgs" })); +} + +test "version 5 info overlay, duplicate keys and renaming" { + const src = + \\{"version":5,"root":"r","nodes":{"r":{"inputs":{"a":"x","b":"y"}}, + \\"x":{"locked":{"type":"path","path":"1"},"original":{"type":"path"},"info":{"lastModified":7}}, + \\"y":{"inputs":{"a":"z"},"locked":{"type":"path"},"original":{"type":"path"},"flake":false}, + \\"z":{"locked":{"k":"old","k":"new"},"original":{}}, "unused":{"broken":true}}} + ; + var doc = try parse(testing.allocator, src, .{}, null); + defer doc.deinit(); + try testing.expectEqual(@as(usize, 4), doc.lock.nodes.len); + var out: std.ArrayList(u8) = .empty; + defer out.deinit(testing.allocator); + try doc.lock.writeNix(testing.allocator, &out); + try testing.expect(std.mem.indexOf(u8, out.items, "\"lastModified\": 7,") != null); + try testing.expect(std.mem.indexOf(u8, out.items, "\"a_2\": {") != null); + try testing.expect(std.mem.indexOf(u8, out.items, "\"k\": \"new\"") != null); + try testing.expect(std.mem.indexOf(u8, out.items, "\"flake\": false,") != null); + // Re-parsing canonical output is a fixed point. + var doc2 = try parse(testing.allocator, out.items, .{}, null); + defer doc2.deinit(); + var again: std.ArrayList(u8) = .empty; + defer again.deinit(testing.allocator); + try doc2.lock.writeNix(testing.allocator, &again); + try testing.expectEqualStrings(out.items, again.items); +} + +test "large graphs use the hashed collision path" { + var src: std.ArrayList(u8) = .empty; + defer src.deinit(testing.allocator); + try src.appendSlice(testing.allocator, "{\"version\":7,\"root\":\"root\",\"nodes\":{\"root\":{\"inputs\":{"); + for (0..100) |i| try src.print(testing.allocator, "{s}\"d{d}\":\"d{d}\"", .{ if (i == 0) "" else ",", i, i }); + try src.appendSlice(testing.allocator, "}}"); + for (0..100) |i| try src.print(testing.allocator, ",\"d{d}\":{{\"inputs\":{{\"s\":\"s{d}\"}},\"locked\":{{}},\"original\":{{}}}},\"s{d}\":{{\"locked\":{{}},\"original\":{{}}}}", .{ i, i, i }); + try src.appendSlice(testing.allocator, "}}"); + var doc = try parse(testing.allocator, src.items, .{}, null); + defer doc.deinit(); + var out: std.ArrayList(u8) = .empty; + defer out.deinit(testing.allocator); + try doc.lock.writeNix(testing.allocator, &out); + try testing.expect(std.mem.indexOf(u8, out.items, "\"s_100\": {") != null); + try testing.expect(std.mem.indexOf(u8, out.items, "\"s_101\"") == null); +} + +test "rejects malformed and adversarial inputs" { + const a = testing.allocator; + const cases = .{ + .{ "{\"version\":4,\"root\":\"r\",\"nodes\":{\"r\":{}}}", error.UnsupportedVersion }, + .{ "{\"version\":7,\"root\":\"r\",\"nodes\":{\"r\":{\"inputs\":{\"a\":\"r\"}}}}", error.CycleToRoot }, + .{ "{\"version\":7,\"root\":\"r\",\"nodes\":{\"r\":{\"inputs\":{\"a\":\"q\"}}}}", error.MissingNode }, + .{ "{\"version\":7,\"root\":\"r\",\"nodes\":{\"r\":{}}} x", error.TrailingCharacters }, + .{ "{\"version\":07,\"root\":\"r\",\"nodes\":{}}", error.InvalidNumber }, + .{ "{\"version\":7,\"root\":\"r\\x\",\"nodes\":{}}", error.InvalidEscape }, + .{ "{\"version\":7,\"root\":\"\xff\",\"nodes\":{}}", error.InvalidUtf8 }, + .{ "{\"version\":7,\"root\":\"a\x01\",\"nodes\":{}}", error.ControlCharacter }, + .{ "{\"version\":7,\"root\":\"r\",\"nodes\":{\"r\":{\"inputs\":{\"a\":\"b\"}},\"b\":{\"locked\":{}}}}", error.MissingField }, + .{ "{\"version\":7,\"root\":\"r\",\"nodes\":{\"r\":{\"inputs\":{\"a\":\"b\"}},\"b\":{\"locked\":{\"x\":null},\"original\":{}}}}", error.UnsupportedInputAttributeType }, + }; + inline for (cases) |c| try testing.expectError(c[1], parse(a, c[0], .{}, null)); + try testing.expectError(error.NestingTooDeep, parse(a, "{\"x\":" ++ "[" ** 300 ++ "]" ** 300 ++ "}", .{}, null)); +} + +test "follows validation detects cycles and missing targets" { + var cyc = try parse(testing.allocator, + \\{"version":7,"root":"r","nodes":{"r":{"inputs":{"a":["b"],"b":["a"]}}}} + , .{}, null); + defer cyc.deinit(); + try testing.expectError(error.FollowsCycle, cyc.lock.validate(testing.allocator)); + var missing = try parse(testing.allocator, + \\{"version":7,"root":"r","nodes":{"r":{"inputs":{"a":["nope"]}}}} + , .{}, null); + defer missing.deinit(); + try testing.expectError(error.MissingFollowsTarget, missing.lock.validate(testing.allocator)); +} + +test "unicode escapes and string escaping on output" { + var doc = try parse(testing.allocator, + \\{"version":7,"root":"r","nodes":{"r":{"inputs":{"a":"b"}},"b":{"locked":{"s":"\u00e9\ud83d\ude00 \"q\" \t é"},"original":{}}}} + , .{}, null); + defer doc.deinit(); + const s = Locked.get(doc.lock.nodes[1].locked.?.locked, "s").?.string; + try testing.expectEqualStrings("é😀 \"q\" \t é", s); + var out: std.ArrayList(u8) = .empty; + defer out.deinit(testing.allocator); + try doc.lock.writeNix(testing.allocator, &out); + try testing.expect(std.mem.indexOf(u8, out.items, "\"é😀 \\\"q\\\" \\t é\"") != null); +} + +test "vector scanners agree with scalar definitions" { + var buf = [_]u8{'a'} ** 80; + for (0..buf.len) |i| { + for ([_]u8{ '"', '\\', 0x01, 0x80, 0xff }) |c| { + buf[i] = c; + try testing.expectEqual(@as(?usize, i), scanString(&buf, 0)); + if (c < 0x80) try testing.expectEqual(i, scanEscape(&buf, 0)); + buf[i] = 'a'; + } + var sp = [_]u8{' '} ** 80; + sp[i] = 'x'; + try testing.expectEqual(i, skipSpace(&sp, 0)); + } +} + +test { + _ = @import("bump.zig"); +} diff --git a/pkg/nix-narinfo/build.zig b/pkg/nix-narinfo/build.zig new file mode 100644 index 0000000..8fe6d66 --- /dev/null +++ b/pkg/nix-narinfo/build.zig @@ -0,0 +1,17 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mod = b.addModule("nix-narinfo", .{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + }); + mod.addImport("nix-base32", b.dependency("nix_base32", .{ .target = target, .optimize = optimize }).module("nix-base32")); + mod.addImport("nix-store-path", b.dependency("nix_store_path", .{ .target = target, .optimize = optimize }).module("nix-store-path")); + + const tests = b.addTest(.{ .root_module = mod }); + b.step("test", "Run unit tests").dependOn(&b.addRunArtifact(tests).step); +} diff --git a/pkg/nix-narinfo/build.zig.zon b/pkg/nix-narinfo/build.zig.zon new file mode 100644 index 0000000..cf7be98 --- /dev/null +++ b/pkg/nix-narinfo/build.zig.zon @@ -0,0 +1,11 @@ +.{ + .name = .nix_narinfo, + .version = "0.1.0", + .fingerprint = 0xf254156c7cc4c9ba, + .minimum_zig_version = "0.16.0", + .dependencies = .{ + .nix_base32 = .{ .path = "../nix-base32" }, + .nix_store_path = .{ .path = "../nix-store-path" }, + }, + .paths = .{ "build.zig", "build.zig.zon", "src" }, +} diff --git a/pkg/nix-narinfo/src/basemul.zig b/pkg/nix-narinfo/src/basemul.zig new file mode 100644 index 0000000..563418e --- /dev/null +++ b/pkg/nix-narinfo/src/basemul.zig @@ -0,0 +1,116 @@ +//! Constant-time fixed-base scalar multiplication for Ed25519 signing, using +//! the comb method from ref10 (SUPERCOP's ge_scalarmult_base): +//! +//! s = sum e_i 16^i, e_i in [-8, 8] +//! s*B = sum_{odd i} e_i 16^i B * 16 + sum_{even i} e_i 16^i B +//! +//! with a table of j * 256^k * B (j = 1..8, k = 0..31) in affine "Niels" +//! form. That is 64 mixed additions and 4 doublings, against 252 doublings +//! and 64 additions for std's 4-bit window. Table entries are selected with +//! conditional moves over all 8 candidates, so memory access and timing do +//! not depend on the (secret) scalar. + +const std = @import("std"); +const Curve = std.crypto.ecc.Edwards25519; +const Fe = Curve.Fe; + +const Niels = struct { + yplusx: Fe, + yminusx: Fe, + xy2d: Fe, + + const identity: Niels = .{ .yplusx = Fe.one, .yminusx = Fe.one, .xy2d = Fe.zero }; + + fn fromPoint(p: Curve) Niels { + const zinv = p.z.invert(); + const x = p.x.mul(zinv); + const y = p.y.mul(zinv); + return .{ .yplusx = y.add(x), .yminusx = y.sub(x), .xy2d = x.mul(y).mul(Fe.edwards25519d2) }; + } + + fn cMov(self: *Niels, other: Niels, c: u64) void { + self.yplusx.cMov(other.yplusx, c); + self.yminusx.cMov(other.yminusx, c); + self.xy2d.cMov(other.xy2d, c); + } +}; + +pub const Table = [32][8]Niels; + +/// Builds the comb table for the Ed25519 base point (~1 ms; do it once). +pub fn table() Table { + var t: Table = undefined; + var base = Curve.basePoint; + for (0..32) |k| { + var acc = base; + for (0..8) |j| { + t[k][j] = Niels.fromPoint(acc); + acc = acc.add(base); + } + for (0..8) |_| base = base.dbl(); + } + return t; +} + +/// Mixed addition: p + q with q affine (ref10 ge_madd), 7 multiplications. +fn madd(p: Curve, q: Niels) Curve { + const a = p.y.sub(p.x).mul(q.yminusx); + const b = p.y.add(p.x).mul(q.yplusx); + const c = p.t.mul(q.xy2d); + const d = p.z.add(p.z); + const x = b.sub(a); + const y = b.add(a); + const z = d.add(c); + const t = d.sub(c); + return .{ .x = x.mul(t), .y = y.mul(z), .z = z.mul(t), .t = x.mul(y) }; +} + +fn select(row: *const [8]Niels, digit: i8) Niels { + const negative: u8 = @intFromBool(digit < 0); + const abs: u8 = @intCast(@as(i16, digit) * (1 - 2 * @as(i16, negative))); + var out = Niels.identity; + for (row, 1..) |entry, j| { + const eq: u64 = ((@as(u64, abs ^ @as(u8, @intCast(j))) -% 1) >> 63) & 1; + out.cMov(entry, eq); + } + // -(x, y) = (-x, y): swap y+x with y-x and negate 2dxy. + const neg: Niels = .{ .yplusx = out.yminusx, .yminusx = out.yplusx, .xy2d = out.xy2d.neg() }; + out.cMov(neg, negative); + return out; +} + +/// s * B for a scalar `s` < 2^255 (already reduced mod L). +pub fn mulBase(t: *const Table, s: [32]u8) Curve { + var e: [64]i8 = undefined; + for (s, 0..) |byte, i| { + e[2 * i] = @intCast(byte & 15); + e[2 * i + 1] = @intCast(byte >> 4); + } + var carry: i8 = 0; + for (e[0..63]) |*d| { + d.* += carry; + carry = (d.* + 8) >> 4; + d.* -= carry << 4; + } + e[63] += carry; + + var h = Curve.identityElement; + var i: usize = 1; + while (i < 64) : (i += 2) h = madd(h, select(&t[i / 2], e[i])); + h = h.dbl().dbl().dbl().dbl(); + i = 0; + while (i < 64) : (i += 2) h = madd(h, select(&t[i / 2], e[i])); + return h; +} + +test "comb multiplication matches std for random scalars" { + const t = table(); + var prng: std.Random.DefaultPrng = .init(5); + for (0..300) |_| { + var wide: [64]u8 = undefined; + prng.random().bytes(&wide); + const s = Curve.scalar.reduce64(wide); + const expected = try Curve.basePoint.mul(s); + try std.testing.expectEqualSlices(u8, &expected.toBytes(), &mulBase(&t, s).toBytes()); + } +} diff --git a/pkg/nix-narinfo/src/parse.zig b/pkg/nix-narinfo/src/parse.zig new file mode 100644 index 0000000..c32e4e0 --- /dev/null +++ b/pkg/nix-narinfo/src/parse.zig @@ -0,0 +1,157 @@ +//! Zero-copy narinfo parser. Fields are slices into the input. +//! +//! Lines are split with a vector newline search (compare 32 bytes against +//! '\n', @ctz the mask); store paths are validated with the vectorised +//! checks in nix-store-path. + +const std = @import("std"); +const store_path = @import("nix-store-path"); + +pub const max_sigs = 8; + +pub const Error = error{ MissingField, InvalidLine, InvalidNumber, InvalidStorePath, InvalidReference, TooManySignatures }; + +pub const NarInfo = struct { + store_path: []const u8 = "", + url: []const u8 = "", + compression: []const u8 = "", + file_hash: []const u8 = "", + file_size: ?u64 = null, + nar_hash: []const u8 = "", + nar_size: u64 = 0, + /// Space-separated basenames, validated. + references: []const u8 = "", + deriver: []const u8 = "", + system: []const u8 = "", + ca: []const u8 = "", + sig_buf: [max_sigs][]const u8 = undefined, + sig_count: u8 = 0, + + pub fn sigs(self: *const NarInfo) []const []const u8 { + return self.sig_buf[0..self.sig_count]; + } + + pub fn referenceIterator(self: *const NarInfo) std.mem.TokenIterator(u8, .scalar) { + return std.mem.tokenizeScalar(u8, self.references, ' '); + } +}; + +const lanes = 32; +const V = @Vector(lanes, u8); +const Mask = std.meta.Int(.unsigned, lanes); + +pub fn indexOfNewline(s: []const u8, start: usize) ?usize { + var i = start; + const nl: V = @splat('\n'); + while (i + lanes <= s.len) : (i += lanes) { + const v: V = s[i..][0..lanes].*; + const m: Mask = @bitCast(v == nl); + if (m != 0) return i + @ctz(m); + } + while (i < s.len) : (i += 1) if (s[i] == '\n') return i; + return null; +} + +fn number(s: []const u8) Error!u64 { + return std.fmt.parseInt(u64, s, 10) catch error.InvalidNumber; +} + +fn eq(a: []const u8, comptime b: []const u8) bool { + return a.len == b.len and std.mem.eql(u8, a, b); +} + +pub fn parse(text: []const u8) Error!NarInfo { + var info: NarInfo = .{}; + var have_size = false; + var pos: usize = 0; + while (pos < text.len) { + const end = indexOfNewline(text, pos) orelse text.len; + const line = text[pos..end]; + pos = end + 1; + if (line.len == 0) continue; + const colon = std.mem.indexOfScalar(u8, line, ':') orelse return error.InvalidLine; + const key = line[0..colon]; + // Nix skips exactly ": "; tolerate a bare ":" before end of line. + const value = if (colon + 2 <= line.len) line[colon + 2 ..] else ""; + switch (key.len) { + 2 => if (eq(key, "CA")) { + info.ca = value; + }, + 3 => if (eq(key, "URL")) { + info.url = value; + } else if (eq(key, "Sig")) { + if (info.sig_count == max_sigs) return error.TooManySignatures; + info.sig_buf[info.sig_count] = value; + info.sig_count += 1; + }, + 6 => if (eq(key, "System")) { + info.system = value; + }, + 7 => if (eq(key, "NarHash")) { + info.nar_hash = value; + } else if (eq(key, "NarSize")) { + info.nar_size = try number(value); + have_size = true; + } else if (eq(key, "Deriver")) { + if (value.len > 0 and !eq(value, "unknown-deriver") and !store_path.isValidBaseName(value)) return error.InvalidStorePath; + info.deriver = value; + }, + 8 => if (eq(key, "FileHash")) { + info.file_hash = value; + } else if (eq(key, "FileSize")) { + info.file_size = try number(value); + }, + 9 => if (eq(key, "StorePath")) { + if (store_path.baseName(value) == null) return error.InvalidStorePath; + info.store_path = value; + }, + 10 => if (eq(key, "References")) { + var it = std.mem.tokenizeScalar(u8, value, ' '); + while (it.next()) |r| if (!store_path.isValidBaseName(r)) return error.InvalidReference; + info.references = value; + }, + 11 => if (eq(key, "Compression")) { + info.compression = value; + }, + else => {}, + } + } + if (info.store_path.len == 0 or info.url.len == 0 or info.nar_hash.len == 0 or !have_size) return error.MissingField; + if (info.compression.len == 0) info.compression = "bzip2"; // Nix's historical default + return info; +} + +test "parses a narinfo" { + const text = + \\StorePath: /nix/store/p7jg95rzvfalb95k3mskk0jqxc9d724n-libunistring-1.4.1 + \\URL: nar/0k1zvrlb2c66njmrbc972fpz7a4ncqr1w4rillf0qwa4cjqa1m6c.nar.zst + \\Compression: zstd + \\FileHash: sha256:0k1zvrlb2c66njmrbc972fpz7a4ncqr1w4rillf0qwa4cjqa1m6c + \\FileSize: 796947 + \\NarHash: sha256:162jnvprzbbglkdxvh3vgcy35j34wva4ni1zj71826vw25ycbdf0 + \\NarSize: 2078944 + \\References: p7jg95rzvfalb95k3mskk0jqxc9d724n-libunistring-1.4.1 + \\Deriver: b3bvrzp605zk7v12ar918r07500gnayv-libunistring-1.4.1.drv + \\Sig: t-1:vkWbgswaWiUPIgtNgxNf9AYTerK7QJU2O4TfykgO9F8N6akQpbpnF5fNvsWw2ENiX0vqo6zBB4XzdAcT4uAkDQ== + \\ + ; + const info = try parse(text); + try std.testing.expectEqualStrings("zstd", info.compression); + try std.testing.expectEqual(@as(u64, 2078944), info.nar_size); + try std.testing.expectEqual(@as(?u64, 796947), info.file_size); + try std.testing.expectEqual(@as(usize, 1), info.sigs().len); + var it = info.referenceIterator(); + try std.testing.expectEqualStrings("p7jg95rzvfalb95k3mskk0jqxc9d724n-libunistring-1.4.1", it.next().?); + try std.testing.expectError(error.MissingField, parse("StorePath: /nix/store/p7jg95rzvfalb95k3mskk0jqxc9d724n-x\n")); + try std.testing.expectError(error.InvalidStorePath, parse("StorePath: /nix/store/bad\n")); +} + +test "vector newline search matches scalar" { + var buf = [_]u8{'x'} ** 100; + for (0..buf.len) |i| { + buf[i] = '\n'; + for (0..i + 1) |start| try std.testing.expectEqual(@as(?usize, i), indexOfNewline(&buf, start)); + buf[i] = 'x'; + } + try std.testing.expectEqual(@as(?usize, null), indexOfNewline(&buf, 0)); +} diff --git a/pkg/nix-narinfo/src/render.zig b/pkg/nix-narinfo/src/render.zig new file mode 100644 index 0000000..22f521b --- /dev/null +++ b/pkg/nix-narinfo/src/render.zig @@ -0,0 +1,283 @@ +//! Narinfo rendering and Ed25519 fingerprint signing. Every field that ends +//! up in a narinfo line is validated here, so no caller-supplied string can +//! inject extra lines. + +const std = @import("std"); +const nix32 = @import("nix-base32"); +const store_path = @import("nix-store-path"); +const Ed25519 = std.crypto.sign.Ed25519; +const Curve = std.crypto.ecc.Edwards25519; +const basemul = @import("basemul.zig"); + +const store_dir = store_path.store_dir; +const baseName = store_path.baseName; +const isValidBaseName = store_path.isValidBaseName; + +pub const Error = error{ InvalidStorePath, InvalidReference, InvalidDeriver, InvalidSystem, InvalidCompression, InvalidKey, OutOfMemory }; + +fn isValidSystem(s: []const u8) bool { + if (s.len == 0 or s.len > 64) return false; + for (s) |c| if (!(std.ascii.isAlphanumeric(c) or c == '_' or c == '-' or c == '.')) return false; + return true; +} + +pub const Signer = struct { + name: []u8, + key_pair: Ed25519.KeyPair, + /// RFC 8032 expanded secret, derived once. std's KeyPair.sign re-derives + /// the public key (a full scalar multiplication) on every call to check + /// for a mismatch; our key pair is validated once at load instead. + scalar: [32]u8, + prefix: [32]u8, + public: [32]u8, + /// Comb table for the base point: signing does 64 additions, not 252 doublings. + table: basemul.Table, + + /// Parses a Nix secret key: `:`. + pub fn parse(allocator: std.mem.Allocator, text: []const u8) Error!*Signer { + const trimmed = std.mem.trim(u8, text, " \t\r\n"); + const colon = std.mem.indexOfScalar(u8, trimmed, ':') orelse return error.InvalidKey; + const name = trimmed[0..colon]; + if (name.len == 0) return error.InvalidKey; + for (name) |c| if (c == '\n' or c == ' ') return error.InvalidKey; + var raw: [Ed25519.SecretKey.encoded_length]u8 = undefined; + const decoder = std.base64.standard.Decoder; + const b64 = trimmed[colon + 1 ..]; + const n = decoder.calcSizeForSlice(b64) catch return error.InvalidKey; + if (n != raw.len) return error.InvalidKey; + decoder.decode(&raw, b64) catch return error.InvalidKey; + const secret = Ed25519.SecretKey.fromBytes(raw) catch return error.InvalidKey; + const key_pair = Ed25519.KeyPair.fromSecretKey(secret) catch return error.InvalidKey; + + var expanded: [64]u8 = undefined; + std.crypto.hash.sha2.Sha512.hash(raw[0..32], &expanded, .{}); + var scalar = expanded[0..32].*; + Curve.scalar.clamp(&scalar); + + const self = try allocator.create(Signer); + errdefer allocator.destroy(self); + self.* = .{ + .name = try allocator.dupe(u8, name), + .key_pair = key_pair, + .scalar = scalar, + .prefix = expanded[32..64].*, + .public = key_pair.public_key.toBytes(), + .table = basemul.table(), + }; + return self; + } + + pub fn destroy(self: *Signer, allocator: std.mem.Allocator) void { + allocator.free(self.name); + allocator.destroy(self); + } + + /// Deterministic Ed25519 (RFC 8032 §5.1.6): one base-point multiplication. + pub fn signRaw(self: *const Signer, msg: []const u8) Error![64]u8 { + const Sha512 = std.crypto.hash.sha2.Sha512; + var h = Sha512.init(.{}); + h.update(&self.prefix); + h.update(msg); + var r64: [64]u8 = undefined; + h.final(&r64); + const r = Curve.scalar.reduce64(r64); + const big_r = basemul.mulBase(&self.table, r).toBytes(); + + h = Sha512.init(.{}); + h.update(&big_r); + h.update(&self.public); + h.update(msg); + var k64: [64]u8 = undefined; + h.final(&k64); + const k = Curve.scalar.reduce64(k64); + + var sig: [64]u8 = undefined; + sig[0..32].* = big_r; + sig[32..64].* = Curve.scalar.mulAdd(k, self.scalar, r); + return sig; + } + + /// `:`. + pub fn sign(self: *const Signer, out: *std.ArrayList(u8), allocator: std.mem.Allocator, msg: []const u8) Error!void { + const bytes = try self.signRaw(msg); + var b64: [std.base64.standard.Encoder.calcSize(bytes.len)]u8 = undefined; + _ = std.base64.standard.Encoder.encode(&b64, &bytes); + try out.appendSlice(allocator, self.name); + try out.append(allocator, ':'); + try out.appendSlice(allocator, &b64); + } + + /// `:`, the value for `trusted-public-keys`. + pub fn publicKey(self: *const Signer, out: *std.ArrayList(u8), allocator: std.mem.Allocator) Error!void { + const bytes = self.key_pair.public_key.toBytes(); + var b64: [std.base64.standard.Encoder.calcSize(bytes.len)]u8 = undefined; + _ = std.base64.standard.Encoder.encode(&b64, &bytes); + try out.appendSlice(allocator, self.name); + try out.append(allocator, ':'); + try out.appendSlice(allocator, &b64); + } +}; + +pub const Input = struct { + store_path: []const u8, + nar_hash: *const [32]u8, + nar_size: u64, + file_hash: *const [32]u8, + file_size: u64, + compression: []const u8, + /// Space-separated store path basenames, any order. + references: []const u8, + /// Deriver basename, or empty. + deriver: []const u8, + /// System, or empty. + system: []const u8, +}; + +fn fileExtension(compression: []const u8) Error![]const u8 { + if (std.mem.eql(u8, compression, "zstd")) return ".nar.zst"; + if (std.mem.eql(u8, compression, "none")) return ".nar"; + if (std.mem.eql(u8, compression, "xz")) return ".nar.xz"; + if (std.mem.eql(u8, compression, "bzip2")) return ".nar.bz2"; + return error.InvalidCompression; +} + +fn lessThanStr(_: void, a: []const u8, b: []const u8) bool { + return std.mem.lessThan(u8, a, b); +} + +/// Renders a complete narinfo. `signer` may be null for an unsigned cache. +pub fn render(allocator: std.mem.Allocator, in: Input, signer: ?*const Signer) Error![]u8 { + _ = baseName(in.store_path) orelse return error.InvalidStorePath; + const ext = try fileExtension(in.compression); + if (in.deriver.len > 0 and !(isValidBaseName(in.deriver) and std.mem.endsWith(u8, in.deriver, ".drv"))) return error.InvalidDeriver; + if (in.system.len > 0 and !isValidSystem(in.system)) return error.InvalidSystem; + + var refs: std.ArrayList([]const u8) = .empty; + defer refs.deinit(allocator); + var it = std.mem.tokenizeScalar(u8, in.references, ' '); + while (it.next()) |r| { + if (!isValidBaseName(r)) return error.InvalidReference; + try refs.append(allocator, r); + } + std.mem.sort([]const u8, refs.items, {}, lessThanStr); + + var nar_hash: [52]u8 = undefined; + nix32.encode(&nar_hash, in.nar_hash); + var file_hash: [52]u8 = undefined; + nix32.encode(&file_hash, in.file_hash); + + var out: std.ArrayList(u8) = .empty; + errdefer out.deinit(allocator); + const w = struct { + fn line(o: *std.ArrayList(u8), a: std.mem.Allocator, parts: []const []const u8) Error!void { + for (parts) |p| try o.appendSlice(a, p); + try o.append(a, '\n'); + } + }.line; + var num_buf: [2][20]u8 = undefined; + const nar_size = std.fmt.bufPrint(&num_buf[0], "{d}", .{in.nar_size}) catch unreachable; + const file_size = std.fmt.bufPrint(&num_buf[1], "{d}", .{in.file_size}) catch unreachable; + + try w(&out, allocator, &.{ "StorePath: ", in.store_path }); + try w(&out, allocator, &.{ "URL: nar/", &file_hash, ext }); + try w(&out, allocator, &.{ "Compression: ", in.compression }); + try w(&out, allocator, &.{ "FileHash: sha256:", &file_hash }); + try w(&out, allocator, &.{ "FileSize: ", file_size }); + try w(&out, allocator, &.{ "NarHash: sha256:", &nar_hash }); + try w(&out, allocator, &.{ "NarSize: ", nar_size }); + // Always "References: " with the space, even when empty: Nix's parser + // skips two bytes after the colon and would otherwise eat the newline. + try out.appendSlice(allocator, "References: "); + for (refs.items, 0..) |r, i| { + if (i > 0) try out.append(allocator, ' '); + try out.appendSlice(allocator, r); + } + try out.append(allocator, '\n'); + if (in.deriver.len > 0) try w(&out, allocator, &.{ "Deriver: ", in.deriver }); + if (in.system.len > 0) try w(&out, allocator, &.{ "System: ", in.system }); + + if (signer) |s| { + // 1;;sha256:;; + var fp: std.ArrayList(u8) = .empty; + defer fp.deinit(allocator); + try fp.appendSlice(allocator, "1;"); + try fp.appendSlice(allocator, in.store_path); + try fp.appendSlice(allocator, ";sha256:"); + try fp.appendSlice(allocator, &nar_hash); + try fp.append(allocator, ';'); + try fp.appendSlice(allocator, nar_size); + try fp.append(allocator, ';'); + for (refs.items, 0..) |r, i| { + if (i > 0) try fp.append(allocator, ','); + try fp.appendSlice(allocator, store_dir); + try fp.appendSlice(allocator, r); + } + try out.appendSlice(allocator, "Sig: "); + try s.sign(&out, allocator, fp.items); + try out.append(allocator, '\n'); + } + + return out.toOwnedSlice(allocator); +} + + +test "renders and signs a narinfo" { + const a = std.testing.allocator; + // Secret key from `nix key generate-secret --key-name test-1`. + const signer = try Signer.parse(a, "test-1:suJsWimvBNFUIc0JJE18OfOMygH/f2GuTC2/XwD6rPKkV+1VKilMIkXl6Ax9hqeKpUF/BSOH+Fqng4tqZlirhw=="); + defer signer.destroy(a); + const nar = [_]u8{1} ** 32; + const file = [_]u8{2} ** 32; + const text = try render(a, .{ + .store_path = "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello", + .nar_hash = &nar, + .nar_size = 1234, + .file_hash = &file, + .file_size = 99, + .compression = "zstd", + .references = "1mdqa9w1p6cmli6976v4wi0sw9r4p5pr-b 0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello", + .deriver = "", + .system = "x86_64-linux", + }, signer); + defer a.free(text); + try std.testing.expect(std.mem.indexOf(u8, text, "References: 0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello 1mdqa9w1p6cmli6976v4wi0sw9r4p5pr-b\n") != null); + try std.testing.expect(std.mem.indexOf(u8, text, "\nSig: test-1:") != null); + const empty = try render(a, .{ + .store_path = "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello", + .nar_hash = &nar, + .nar_size = 1, + .file_hash = &file, + .file_size = 1, + .compression = "zstd", + .references = "", + .deriver = "0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello.drv", + .system = "", + }, null); + defer a.free(empty); + try std.testing.expect(std.mem.indexOf(u8, empty, "\nReferences: \nDeriver: ") != null); + try std.testing.expectError(error.InvalidSystem, render(a, .{ + .store_path = "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello", + .nar_hash = &nar, + .nar_size = 1, + .file_hash = &file, + .file_size = 1, + .compression = "zstd", + .references = "", + .deriver = "", + .system = "x86\nSig: evil", + }, null)); +} + +test "fast signing is byte-identical to std Ed25519" { + const a = std.testing.allocator; + const signer = try Signer.parse(a, "test-1:suJsWimvBNFUIc0JJE18OfOMygH/f2GuTC2/XwD6rPKkV+1VKilMIkXl6Ax9hqeKpUF/BSOH+Fqng4tqZlirhw=="); + defer signer.destroy(a); + var prng: std.Random.DefaultPrng = .init(3); + var msg: [300]u8 = undefined; + for (0..200) |i| { + prng.random().bytes(msg[0 .. i + 1]); + const ours = try signer.signRaw(msg[0 .. i + 1]); + const theirs = (try signer.key_pair.sign(msg[0 .. i + 1], null)).toBytes(); + try std.testing.expectEqualSlices(u8, &theirs, &ours); + } +} diff --git a/pkg/nix-narinfo/src/root.zig b/pkg/nix-narinfo/src/root.zig new file mode 100644 index 0000000..a5df7fb --- /dev/null +++ b/pkg/nix-narinfo/src/root.zig @@ -0,0 +1,19 @@ +//! Narinfo: parsing, validated rendering, and fingerprint signing. + +const render_mod = @import("render.zig"); +const parse_mod = @import("parse.zig"); + +pub const Signer = render_mod.Signer; +pub const Input = render_mod.Input; +pub const RenderError = render_mod.Error; +pub const render = render_mod.render; + +pub const NarInfo = parse_mod.NarInfo; +pub const ParseError = parse_mod.Error; +pub const parse = parse_mod.parse; + +test { + _ = render_mod; + _ = parse_mod; + _ = @import("basemul.zig"); +} diff --git a/pkg/nix-store-path/build.zig b/pkg/nix-store-path/build.zig new file mode 100644 index 0000000..44b7bcf --- /dev/null +++ b/pkg/nix-store-path/build.zig @@ -0,0 +1,16 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mod = b.addModule("nix-store-path", .{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + }); + mod.addImport("nix-base32", b.dependency("nix_base32", .{ .target = target, .optimize = optimize }).module("nix-base32")); + + const tests = b.addTest(.{ .root_module = mod }); + b.step("test", "Run unit tests").dependOn(&b.addRunArtifact(tests).step); +} diff --git a/pkg/nix-store-path/build.zig.zon b/pkg/nix-store-path/build.zig.zon new file mode 100644 index 0000000..55b9cd1 --- /dev/null +++ b/pkg/nix-store-path/build.zig.zon @@ -0,0 +1,10 @@ +.{ + .name = .nix_store_path, + .version = "0.1.0", + .fingerprint = 0xdcfce087c7a4494d, + .minimum_zig_version = "0.16.0", + .dependencies = .{ + .nix_base32 = .{ .path = "../nix-base32" }, + }, + .paths = .{ "build.zig", "build.zig.zon", "src" }, +} diff --git a/pkg/nix-store-path/src/root.zig b/pkg/nix-store-path/src/root.zig new file mode 100644 index 0000000..9e571d1 --- /dev/null +++ b/pkg/nix-store-path/src/root.zig @@ -0,0 +1,96 @@ +//! Store path parsing and validation, matching what Nix's StorePath accepts: +//! `/nix/store/<32 nix32 chars>-`, name 1-211 chars of +//! `[A-Za-z0-9+-._?=]`, not starting with '.'. +//! +//! The name check is vectorised: 32 characters are classified per step. + +const std = @import("std"); +const base32 = @import("nix-base32"); + +pub const store_dir = "/nix/store/"; +pub const hash_len = 32; +pub const max_name_len = 211; + +const lanes = 32; +const V = @Vector(lanes, u8); +const Mask = std.meta.Int(.unsigned, lanes); + +fn splat(c: u8) V { + return @splat(c); +} + +fn bits(b: @Vector(lanes, bool)) Mask { + return @bitCast(b); +} + +inline fn invalidNameLanes(v: V) Mask { + const lower = v | splat(0x20); // folds A-Z onto a-z; no other byte lands in a-z from outside letters + const alpha = bits(lower >= splat('a')) & bits(lower <= splat('z')); + const digit = bits(v >= splat('0')) & bits(v <= splat('9')); + const special = bits(v == splat('+')) | bits(v == splat('-')) | bits(v == splat('.')) | + bits(v == splat('_')) | bits(v == splat('?')) | bits(v == splat('=')); + return ~(alpha | digit | special); +} + +fn isNameChar(c: u8) bool { + return std.ascii.isAlphanumeric(c) or switch (c) { + '+', '-', '.', '_', '?', '=' => true, + else => false, + }; +} + +pub fn isValidName(name: []const u8) bool { + if (name.len == 0 or name.len > max_name_len or name[0] == '.') return false; + var i: usize = 0; + while (i + lanes <= name.len) : (i += lanes) { + if (invalidNameLanes(name[i..][0..lanes].*) != 0) return false; + } + for (name[i..]) |c| if (!isNameChar(c)) return false; + return true; +} + +pub fn isValidNameScalar(name: []const u8) bool { + if (name.len == 0 or name.len > max_name_len or name[0] == '.') return false; + for (name) |c| if (!isNameChar(c)) return false; + return true; +} + +/// `<32 nix32 chars>-`. +pub fn isValidBaseName(base: []const u8) bool { + if (base.len < hash_len + 2) return false; + return base[hash_len] == '-' and base32.isValid(base[0..hash_len]) and isValidName(base[hash_len + 1 ..]); +} + +/// The basename of a full store path, or null if it is not a valid one. +pub fn baseName(store_path: []const u8) ?[]const u8 { + if (!std.mem.startsWith(u8, store_path, store_dir)) return null; + const base = store_path[store_dir.len..]; + return if (isValidBaseName(base)) base else null; +} + +/// The decoded 20-byte hash of a valid basename. +pub fn hashOf(base: []const u8) ?[20]u8 { + if (base.len < hash_len) return null; + var out: [20]u8 = undefined; + base32.decode(&out, base[0..hash_len]) catch return null; + return out; +} + +test "validates base names" { + try std.testing.expect(isValidBaseName("0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello-2.12")); + try std.testing.expect(!isValidBaseName("0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello\nSig: x")); + try std.testing.expect(!isValidBaseName("0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-.hidden")); + try std.testing.expect(!isValidBaseName("emdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello")); + try std.testing.expect(baseName("/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello") != null); + try std.testing.expect(baseName("/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello/bin") == null); +} + +test "vector name check agrees with scalar for every byte in every lane" { + for (0..256) |c| { + for (0..lanes + 3) |pos| { + var name = [_]u8{'a'} ** (lanes + 3); + name[pos] = @intCast(c); + try std.testing.expectEqual(isValidNameScalar(&name), isValidName(&name)); + } + } +} diff --git a/scripts/test-zig.sh b/scripts/test-zig.sh new file mode 100755 index 0000000..4ebf9a3 --- /dev/null +++ b/scripts/test-zig.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Runs the unit tests of every Zig package, then builds libhelios. +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +for pkg in "$ROOT"/pkg/*/; do + echo "== $(basename "$pkg")" + (cd "$pkg" && zig build test -Dcpu=native --summary failures) +done +echo "== libhelios" +(cd "$ROOT/core" && zig build -Doptimize=ReleaseFast -Dcpu=native) From 96f72b8785ee0359d9153a5382e4b3f3d1914dfb Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 12:23:53 +0200 Subject: [PATCH 02/56] core: add libhelios c abi and rust bindings libhelios exposes the pkg packages through a C ABI, declared in core/include/helios.h, and builds as a static library. crates/helios-core runs zig build from its build script, links the result together with the system libzstd and wraps it in a safe Rust API. Calls across the boundary are coarse: a whole NAR dump, upload or narinfo per call, with output chunks handed back through a callback. A test checks that NAR hashes match the Nix database for a real closure. --- Cargo.lock | 121 +++++++++ Cargo.toml | 14 ++ core/build.zig | 36 +++ core/build.zig.zon | 13 + core/include/helios.h | 113 +++++++++ core/src/root.zig | 221 +++++++++++++++++ crates/helios-core/Cargo.toml | 16 ++ crates/helios-core/build.rs | 62 +++++ crates/helios-core/src/lib.rs | 445 ++++++++++++++++++++++++++++++++++ 9 files changed, 1041 insertions(+) create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 core/build.zig create mode 100644 core/build.zig.zon create mode 100644 core/include/helios.h create mode 100644 core/src/root.zig create mode 100644 crates/helios-core/Cargo.toml create mode 100644 crates/helios-core/build.rs create mode 100644 crates/helios-core/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..8de586e --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,121 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "helios-core" +version = "0.1.0" +dependencies = [ + "pkg-config", + "tempfile", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..afbb718 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,14 @@ +[workspace] +resolver = "3" +members = ["crates/*"] + +[workspace.package] +version = "0.1.0" +edition = "2024" +license-file = "LICENSE" +publish = false + +[profile.release] +lto = "fat" +codegen-units = 1 +panic = "abort" diff --git a/core/build.zig b/core/build.zig new file mode 100644 index 0000000..104b5ea --- /dev/null +++ b/core/build.zig @@ -0,0 +1,36 @@ +//! libhelios: the C ABI over the pkg/* Nix packages, linked into the Rust +//! server and CLI as a static library. + +const std = @import("std"); + +const packages = [_][]const u8{ "nix-base32", "nix-store-path", "nix-archive", "nix-narinfo" }; + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + + const mod = b.createModule(.{ + .root_source_file = b.path("src/root.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + // Linked into Rust binaries, which are PIE. + .pic = true, + }); + inline for (packages) |name| { + const dep = b.dependency(comptime underscored(name), .{ .target = target, .optimize = optimize }); + mod.addImport(name, dep.module(name)); + } + + const lib = b.addLibrary(.{ .name = "helios", .linkage = .static, .root_module = mod }); + lib.bundle_compiler_rt = true; + lib.installHeader(b.path("include/helios.h"), "helios.h"); + b.installArtifact(lib); +} + +fn underscored(comptime name: []const u8) []const u8 { + comptime var out: [name.len]u8 = undefined; + for (name, 0..) |c, i| out[i] = if (c == '-') '_' else c; + const final = out; + return &final; +} diff --git a/core/build.zig.zon b/core/build.zig.zon new file mode 100644 index 0000000..07254df --- /dev/null +++ b/core/build.zig.zon @@ -0,0 +1,13 @@ +.{ + .name = .libhelios, + .version = "0.1.0", + .fingerprint = 0x27391ab28c1ecfe2, + .minimum_zig_version = "0.16.0", + .dependencies = .{ + .nix_base32 = .{ .path = "../pkg/nix-base32" }, + .nix_store_path = .{ .path = "../pkg/nix-store-path" }, + .nix_archive = .{ .path = "../pkg/nix-archive" }, + .nix_narinfo = .{ .path = "../pkg/nix-narinfo" }, + }, + .paths = .{ "build.zig", "build.zig.zon", "src", "include" }, +} diff --git a/core/include/helios.h b/core/include/helios.h new file mode 100644 index 0000000..7baa783 --- /dev/null +++ b/core/include/helios.h @@ -0,0 +1,113 @@ +/* libhelios: NAR pipeline and Nix binary cache formats. */ +#ifndef HELIOS_H +#define HELIOS_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define HL_OK 0 +#define HL_E_IO (-1) +#define HL_E_UNSUPPORTED_FILE (-2) +#define HL_E_ZSTD (-3) +#define HL_E_ABORTED (-4) +#define HL_E_INVALID (-5) +#define HL_E_NOMEM (-6) +#define HL_E_CHANGED (-7) +#define HL_E_UNSUPPORTED_OS (-8) +#define HL_E_TRUNCATED (-9) +#define HL_E_NOT_NAR (-10) +#define HL_E_BUFFER (-11) + +const char *hl_strerror(int rc); +/* errno of the last failed syscall on the calling thread. */ +int hl_last_errno(void); + +/* ── nix32 ── */ + +/* Writes ceil(len * 8 / 5) chars (32 for 20 bytes, 52 for 32 bytes). */ +size_t hl_nix32_encode(const uint8_t *bytes, size_t len, char *out); +int hl_nix32_decode(const char *text, size_t text_len, uint8_t *out, size_t out_len); +bool hl_store_basename_valid(const char *base, size_t len); + +/* ── Compressing NAR pipeline ── */ + +typedef struct { + uint8_t file_hash[32]; /* sha256 of the compressed stream */ + uint64_t file_size; + uint8_t nar_hash[32]; /* sha256 of the uncompressed NAR */ + uint64_t nar_size; +} hl_digest; + +typedef struct { + int level; /* zstd level; 0 disables compression */ + int threads; /* zstd worker threads; 0 = calling thread */ + uint64_t size_hint; /* expected NAR size, 0 if unknown */ +} hl_dump_options; + +/* Return non-zero to abort the pipeline with HL_E_ABORTED. */ +typedef int (*hl_write_fn)(void *ctx, const uint8_t *buf, size_t len); + +/* Serialise `path` as a NAR, compress it, and stream it to `write`. + * Linux only; returns HL_E_UNSUPPORTED_OS elsewhere. */ +int hl_nar_dump(const char *path, const hl_dump_options *opts, hl_write_fn write, void *ctx, hl_digest *out); + +/* Same pipeline, fed with NAR bytes from elsewhere (e.g. nix store dump-path). */ +typedef struct hl_compressor hl_compressor; +hl_compressor *hl_compressor_new(const hl_dump_options *opts, hl_write_fn write, void *ctx); +int hl_compressor_update(hl_compressor *c, const uint8_t *data, size_t len); +int hl_compressor_finish(hl_compressor *c, hl_digest *out); +void hl_compressor_free(hl_compressor *c); + +/* ── Upload verifier ── */ + +#define HL_COMPRESSION_NONE 0 +#define HL_COMPRESSION_ZSTD 1 + +typedef struct hl_verifier hl_verifier; +hl_verifier *hl_verifier_new(int compression); +int hl_verifier_update(hl_verifier *v, const uint8_t *data, size_t len); +/* Fails with HL_E_TRUNCATED or HL_E_NOT_NAR on bad input. */ +int hl_verifier_finish(hl_verifier *v, hl_digest *out); +void hl_verifier_free(hl_verifier *v); + +/* ── Signing and narinfo ── */ + +typedef struct hl_signer hl_signer; +/* Parses a Nix secret key: ":". */ +hl_signer *hl_signer_new(const char *key, size_t len); +void hl_signer_free(hl_signer *s); +/* Writes ":". Sets *out_len even on HL_E_BUFFER. */ +int hl_signer_public_key(const hl_signer *s, char *out, size_t cap, size_t *out_len); + +typedef struct { + const char *ptr; + size_t len; +} hl_str; + +typedef struct { + hl_str store_path; /* /nix/store/- */ + const uint8_t *nar_hash; /* 32 bytes */ + uint64_t nar_size; + const uint8_t *file_hash; /* 32 bytes */ + uint64_t file_size; + hl_str compression; /* zstd | none | xz | bzip2 */ + hl_str references; /* space-separated basenames */ + hl_str deriver; /* basename or empty */ + hl_str system; /* or empty */ +} hl_narinfo_input; + +/* Validates every field and renders a (signed, if signer != NULL) narinfo + * into a heap buffer that must be released with hl_free. */ +int hl_narinfo_render(const hl_narinfo_input *in, const hl_signer *signer, char **out, size_t *out_len); +void hl_free(char *ptr, size_t len); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/core/src/root.zig b/core/src/root.zig new file mode 100644 index 0000000..db96907 --- /dev/null +++ b/core/src/root.zig @@ -0,0 +1,221 @@ +//! C ABI for libhelios. See include/helios.h for the contract. + +const std = @import("std"); +const nix32 = @import("nix-base32"); +const store_path = @import("nix-store-path"); +const archive = @import("nix-archive"); + +const narinfo = @import("nix-narinfo"); + +const Sink = archive.Sink; +const Digest = archive.Digest; +const WriteFn = archive.WriteFn; +const gpa = std.heap.c_allocator; + +pub const HL_OK: c_int = 0; +pub const HL_E_IO: c_int = -1; +pub const HL_E_UNSUPPORTED_FILE: c_int = -2; +pub const HL_E_ZSTD: c_int = -3; +pub const HL_E_ABORTED: c_int = -4; +pub const HL_E_INVALID: c_int = -5; +pub const HL_E_NOMEM: c_int = -6; +pub const HL_E_CHANGED: c_int = -7; +pub const HL_E_UNSUPPORTED_OS: c_int = -8; +pub const HL_E_TRUNCATED: c_int = -9; +pub const HL_E_NOT_NAR: c_int = -10; +pub const HL_E_BUFFER: c_int = -11; + +fn code(err: anyerror) c_int { + return switch (err) { + error.Io, error.NameTooLong => HL_E_IO, + error.UnsupportedFileType => HL_E_UNSUPPORTED_FILE, + error.Zstd => HL_E_ZSTD, + error.Aborted => HL_E_ABORTED, + error.OutOfMemory => HL_E_NOMEM, + error.FileChanged => HL_E_CHANGED, + error.Truncated => HL_E_TRUNCATED, + error.NotNar => HL_E_NOT_NAR, + else => HL_E_INVALID, + }; +} + +export fn hl_strerror(rc: c_int) [*:0]const u8 { + return switch (rc) { + HL_OK => "ok", + HL_E_IO => "I/O error", + HL_E_UNSUPPORTED_FILE => "unsupported file type in store path", + HL_E_ZSTD => "zstd error", + HL_E_ABORTED => "aborted by write callback", + HL_E_INVALID => "invalid input", + HL_E_NOMEM => "out of memory", + HL_E_CHANGED => "file changed while reading", + HL_E_UNSUPPORTED_OS => "NAR dumping is not supported on this OS", + HL_E_TRUNCATED => "compressed stream is truncated", + HL_E_NOT_NAR => "not a NAR archive", + HL_E_BUFFER => "output buffer too small", + else => "unknown error", + }; +} + +export fn hl_last_errno() c_int { + return archive.lastErrno(); +} + +// ── nix32 ── + +export fn hl_nix32_encode(bytes: [*]const u8, len: usize, out: [*]u8) usize { + const n = nix32.encodedLen(len); + nix32.encode(out[0..n], bytes[0..len]); + return n; +} + +export fn hl_nix32_decode(text: [*]const u8, text_len: usize, out: [*]u8, out_len: usize) c_int { + nix32.decode(out[0..out_len], text[0..text_len]) catch return HL_E_INVALID; + return HL_OK; +} + +export fn hl_store_basename_valid(base: [*]const u8, len: usize) bool { + return store_path.isValidBaseName(base[0..len]); +} + +// ── Compressing NAR pipeline ── + +pub const DumpOptions = extern struct { + level: c_int, + threads: c_int, + size_hint: u64, +}; + +fn sinkOptions(opts: *const DumpOptions) archive.Options { + return .{ .level = opts.level, .threads = opts.threads, .size_hint = opts.size_hint }; +} + +export fn hl_nar_dump(path: [*:0]const u8, opts: *const DumpOptions, write: WriteFn, ctx: ?*anyopaque, out: *Digest) c_int { + if (comptime !archive.supported) return HL_E_UNSUPPORTED_OS; + var sink = Sink.init(gpa, sinkOptions(opts), write, ctx) catch |e| return code(e); + defer sink.deinit(gpa); + archive.dump(&sink, path) catch |e| return code(e); + out.* = sink.finish() catch |e| return code(e); + return HL_OK; +} + +export fn hl_compressor_new(opts: *const DumpOptions, write: WriteFn, ctx: ?*anyopaque) ?*Sink { + const sink = gpa.create(Sink) catch return null; + sink.* = Sink.init(gpa, sinkOptions(opts), write, ctx) catch { + gpa.destroy(sink); + return null; + }; + return sink; +} + +export fn hl_compressor_update(sink: *Sink, data: [*]const u8, len: usize) c_int { + sink.write(data[0..len]) catch |e| return code(e); + return HL_OK; +} + +export fn hl_compressor_finish(sink: *Sink, out: *Digest) c_int { + out.* = sink.finish() catch |e| return code(e); + return HL_OK; +} + +export fn hl_compressor_free(sink: ?*Sink) void { + const s = sink orelse return; + s.deinit(gpa); + gpa.destroy(s); +} + +// ── Upload verifier ── + +export fn hl_verifier_new(compression: c_int) ?*archive.Verifier { + const kind = std.enums.fromInt(archive.Compression, compression) orelse return null; + const v = gpa.create(archive.Verifier) catch return null; + v.* = archive.Verifier.init(gpa, kind) catch { + gpa.destroy(v); + return null; + }; + return v; +} + +export fn hl_verifier_update(v: *archive.Verifier, data: [*]const u8, len: usize) c_int { + v.update(data[0..len]) catch |e| return code(e); + return HL_OK; +} + +export fn hl_verifier_finish(v: *archive.Verifier, out: *Digest) c_int { + out.* = v.finish() catch |e| return code(e); + return HL_OK; +} + +export fn hl_verifier_free(v: ?*archive.Verifier) void { + const p = v orelse return; + p.deinit(gpa); + gpa.destroy(p); +} + +// ── Signing and narinfo ── + +export fn hl_signer_new(key: [*]const u8, len: usize) ?*narinfo.Signer { + return narinfo.Signer.parse(gpa, key[0..len]) catch null; +} + +export fn hl_signer_free(s: ?*narinfo.Signer) void { + if (s) |p| p.destroy(gpa); +} + +fn copyOut(bytes: []const u8, out: ?[*]u8, cap: usize, out_len: *usize) c_int { + out_len.* = bytes.len; + if (bytes.len > cap or out == null) return HL_E_BUFFER; + @memcpy(out.?[0..bytes.len], bytes); + return HL_OK; +} + +export fn hl_signer_public_key(s: *const narinfo.Signer, out: ?[*]u8, cap: usize, out_len: *usize) c_int { + var list: std.ArrayList(u8) = .empty; + defer list.deinit(gpa); + s.publicKey(&list, gpa) catch |e| return code(e); + return copyOut(list.items, out, cap, out_len); +} + +pub const Str = extern struct { + ptr: ?[*]const u8, + len: usize, + + fn slice(self: Str) []const u8 { + return if (self.ptr) |p| p[0..self.len] else ""; + } +}; + +pub const NarinfoInput = extern struct { + store_path: Str, + nar_hash: *const [32]u8, + nar_size: u64, + file_hash: *const [32]u8, + file_size: u64, + compression: Str, + references: Str, + deriver: Str, + system: Str, +}; + +/// Renders into a heap buffer released with hl_free. +export fn hl_narinfo_render(in: *const NarinfoInput, signer: ?*const narinfo.Signer, out: *?[*]u8, out_len: *usize) c_int { + const text = narinfo.render(gpa, .{ + .store_path = in.store_path.slice(), + .nar_hash = in.nar_hash, + .nar_size = in.nar_size, + .file_hash = in.file_hash, + .file_size = in.file_size, + .compression = in.compression.slice(), + .references = in.references.slice(), + .deriver = in.deriver.slice(), + .system = in.system.slice(), + }, signer) catch |e| return code(e); + out.* = text.ptr; + out_len.* = text.len; + return HL_OK; +} + +export fn hl_free(ptr: ?[*]u8, len: usize) void { + if (ptr) |p| gpa.free(p[0..len]); +} + diff --git a/crates/helios-core/Cargo.toml b/crates/helios-core/Cargo.toml new file mode 100644 index 0000000..de80468 --- /dev/null +++ b/crates/helios-core/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "helios-core" +description = "Safe Rust bindings to libhelios (Zig): NAR pipeline, narinfo, signing" +version.workspace = true +edition.workspace = true +license-file.workspace = true +publish.workspace = true +links = "helios" + +[dependencies] + +[build-dependencies] +pkg-config = "0.3.34" + +[dev-dependencies] +tempfile = "3.27.0" diff --git a/crates/helios-core/build.rs b/crates/helios-core/build.rs new file mode 100644 index 0000000..6859ff9 --- /dev/null +++ b/crates/helios-core/build.rs @@ -0,0 +1,62 @@ +//! Builds libhelios (Zig) as a static library and links it with libzstd. +//! +//! Environment: +//! - `ZIG`: zig executable (default `zig`) +//! - `HELIOS_ZIG_CPU`: `-Dcpu` value (default `native`; use `baseline` for +//! portable binaries, which loses SHA-NI accelerated hashing) +//! - `HELIOS_ZIG_TARGET`: `-Dtarget` value when cross-compiling + +use std::env; +use std::path::PathBuf; +use std::process::Command; + +fn main() { + let manifest = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let core = manifest.join("../../core").canonicalize().expect("core/ directory"); + let out = PathBuf::from(env::var("OUT_DIR").unwrap()); + let prefix = out.join("zig-out"); + + for path in ["build.zig", "build.zig.zon", "src", "include"] { + println!("cargo:rerun-if-changed={}", core.join(path).display()); + } + for pkg in std::fs::read_dir(core.join("../pkg")).expect("pkg/ directory").flatten() { + println!("cargo:rerun-if-changed={}", pkg.path().display()); + } + for var in ["ZIG", "HELIOS_ZIG_CPU", "HELIOS_ZIG_TARGET"] { + println!("cargo:rerun-if-env-changed={var}"); + } + + let zig = env::var("ZIG").unwrap_or_else(|_| "zig".into()); + let cpu = env::var("HELIOS_ZIG_CPU").unwrap_or_else(|_| "native".into()); + let mut cmd = Command::new(&zig); + cmd.current_dir(&core) + .arg("build") + .arg("-Doptimize=ReleaseFast") + .arg(format!("-Dcpu={cpu}")) + .arg("--prefix") + .arg(&prefix) + .arg("--cache-dir") + .arg(out.join("zig-cache")); + if let Ok(target) = env::var("HELIOS_ZIG_TARGET") { + cmd.arg(format!("-Dtarget={target}")); + } + if env::var_os("ZIG_GLOBAL_CACHE_DIR").is_none() { + cmd.env("ZIG_GLOBAL_CACHE_DIR", out.join("zig-global-cache")); + } + let status = cmd + .status() + .unwrap_or_else(|e| panic!("failed to run `{zig} build` (is zig 0.16 on PATH?): {e}")); + assert!(status.success(), "zig build failed for libhelios"); + + println!("cargo:rustc-link-search=native={}", prefix.join("lib").display()); + println!("cargo:rustc-link-lib=static=helios"); + println!("cargo:include={}", prefix.join("include").display()); + + if pkg_config::Config::new() + .atleast_version("1.5") + .probe("libzstd") + .is_err() + { + println!("cargo:rustc-link-lib=zstd"); + } +} diff --git a/crates/helios-core/src/lib.rs b/crates/helios-core/src/lib.rs new file mode 100644 index 0000000..2bb1fa1 --- /dev/null +++ b/crates/helios-core/src/lib.rs @@ -0,0 +1,445 @@ +//! Safe bindings to libhelios, the Zig core: the compressing NAR pipeline, +//! the upload verifier, nix32, narinfo rendering and Ed25519 signing. + +use std::ffi::{CStr, CString, c_int, c_void}; +use std::fmt; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; + +mod sys { + use std::ffi::{c_char, c_int, c_void}; + + #[repr(C)] + #[derive(Clone, Copy, Default)] + pub struct Digest { + pub file_hash: [u8; 32], + pub file_size: u64, + pub nar_hash: [u8; 32], + pub nar_size: u64, + } + + #[repr(C)] + pub struct DumpOptions { + pub level: c_int, + pub threads: c_int, + pub size_hint: u64, + } + + #[repr(C)] + pub struct Str { + pub ptr: *const u8, + pub len: usize, + } + + #[repr(C)] + pub struct NarinfoInput { + pub store_path: Str, + pub nar_hash: *const [u8; 32], + pub nar_size: u64, + pub file_hash: *const [u8; 32], + pub file_size: u64, + pub compression: Str, + pub references: Str, + pub deriver: Str, + pub system: Str, + } + + pub type WriteFn = unsafe extern "C" fn(ctx: *mut c_void, buf: *const u8, len: usize) -> c_int; + + pub enum Compressor {} + pub enum Verifier {} + pub enum Signer {} + + unsafe extern "C" { + pub fn hl_strerror(rc: c_int) -> *const c_char; + pub fn hl_last_errno() -> c_int; + pub fn hl_nix32_encode(bytes: *const u8, len: usize, out: *mut u8) -> usize; + pub fn hl_nix32_decode(text: *const u8, text_len: usize, out: *mut u8, out_len: usize) -> c_int; + pub fn hl_store_basename_valid(base: *const u8, len: usize) -> bool; + pub fn hl_nar_dump(path: *const c_char, opts: *const DumpOptions, write: WriteFn, ctx: *mut c_void, out: *mut Digest) -> c_int; + pub fn hl_compressor_new(opts: *const DumpOptions, write: WriteFn, ctx: *mut c_void) -> *mut Compressor; + pub fn hl_compressor_update(c: *mut Compressor, data: *const u8, len: usize) -> c_int; + pub fn hl_compressor_finish(c: *mut Compressor, out: *mut Digest) -> c_int; + pub fn hl_compressor_free(c: *mut Compressor); + pub fn hl_verifier_new(compression: c_int) -> *mut Verifier; + pub fn hl_verifier_update(v: *mut Verifier, data: *const u8, len: usize) -> c_int; + pub fn hl_verifier_finish(v: *mut Verifier, out: *mut Digest) -> c_int; + pub fn hl_verifier_free(v: *mut Verifier); + pub fn hl_signer_new(key: *const u8, len: usize) -> *mut Signer; + pub fn hl_signer_free(s: *mut Signer); + pub fn hl_signer_public_key(s: *const Signer, out: *mut u8, cap: usize, out_len: *mut usize) -> c_int; + pub fn hl_narinfo_render(input: *const NarinfoInput, signer: *const Signer, out: *mut *mut u8, out_len: *mut usize) -> c_int; + pub fn hl_free(ptr: *mut u8, len: usize); + } +} + +pub const HL_E_IO: i32 = -1; +pub const HL_E_ABORTED: i32 = -4; +pub const HL_E_INVALID: i32 = -5; +pub const HL_E_UNSUPPORTED_OS: i32 = -8; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Error { + pub code: i32, + /// OS errno for I/O failures, 0 otherwise. + pub errno: i32, +} + +impl Error { + fn from_rc(code: c_int) -> Self { + let errno = if code == HL_E_IO { unsafe { sys::hl_last_errno() } } else { 0 }; + Self { code, errno } + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let msg = unsafe { CStr::from_ptr(sys::hl_strerror(self.code)) }; + write!(f, "{}", msg.to_string_lossy())?; + if self.errno != 0 { + write!(f, ": {}", std::io::Error::from_raw_os_error(self.errno))?; + } + Ok(()) + } +} + +impl std::error::Error for Error {} + +fn check(rc: c_int) -> Result<(), Error> { + if rc == 0 { Ok(()) } else { Err(Error::from_rc(rc)) } +} + +// ── nix32 ── + +pub fn nix32_encode(bytes: &[u8]) -> String { + let len = if bytes.is_empty() { 0 } else { (bytes.len() * 8 - 1) / 5 + 1 }; + let mut out = vec![0u8; len]; + unsafe { sys::hl_nix32_encode(bytes.as_ptr(), bytes.len(), out.as_mut_ptr()) }; + // The nix32 alphabet is ASCII. + unsafe { String::from_utf8_unchecked(out) } +} + +pub fn nix32_decode(text: &str) -> Option<[u8; N]> { + let mut out = [0u8; N]; + let rc = unsafe { sys::hl_nix32_decode(text.as_ptr(), text.len(), out.as_mut_ptr(), N) }; + (rc == 0).then_some(out) +} + +/// `<32 nix32 chars>-` with only the characters Nix allows. +pub fn store_basename_valid(base: &str) -> bool { + unsafe { sys::hl_store_basename_valid(base.as_ptr(), base.len()) } +} + +// ── Pipeline ── + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Digest { + pub file_hash: [u8; 32], + pub file_size: u64, + pub nar_hash: [u8; 32], + pub nar_size: u64, +} + +impl From for Digest { + fn from(d: sys::Digest) -> Self { + Self { file_hash: d.file_hash, file_size: d.file_size, nar_hash: d.nar_hash, nar_size: d.nar_size } + } +} + +#[derive(Clone, Copy, Debug)] +pub struct DumpOptions { + /// zstd level; 0 disables compression. + pub level: i32, + /// zstd worker threads; 0 compresses on the calling thread. + pub threads: i32, + /// Expected NAR size, 0 if unknown. + pub size_hint: u64, +} + +impl Default for DumpOptions { + fn default() -> Self { + Self { level: 3, threads: 0, size_hint: 0 } + } +} + +impl DumpOptions { + fn raw(&self) -> sys::DumpOptions { + sys::DumpOptions { level: self.level, threads: self.threads, size_hint: self.size_hint } + } +} + +unsafe extern "C" fn trampoline bool>(ctx: *mut c_void, buf: *const u8, len: usize) -> c_int { + let f = unsafe { &mut *(ctx as *mut F) }; + let chunk = unsafe { std::slice::from_raw_parts(buf, len) }; + // Never unwind into Zig. + match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| f(chunk))) { + Ok(true) => 0, + _ => 1, + } +} + +/// Serialise `path` as a NAR, compress it and stream the output to `write`. +/// Return `false` from `write` to abort. Blocking; Linux only. +pub fn dump_nar bool>(path: &Path, opts: &DumpOptions, mut write: F) -> Result { + let cpath = CString::new(path.as_os_str().as_bytes()).map_err(|_| Error { code: HL_E_INVALID, errno: 0 })?; + let mut out = sys::Digest::default(); + let rc = unsafe { + sys::hl_nar_dump(cpath.as_ptr(), &opts.raw(), trampoline::, &mut write as *mut F as *mut c_void, &mut out) + }; + check(rc)?; + Ok(out.into()) +} + +/// The same pipeline, fed with NAR bytes produced elsewhere. +pub struct Compressor bool> { + raw: *mut sys::Compressor, + // Boxed so the pointer handed to Zig stays stable. + _write: Box, +} + +impl bool> Compressor { + pub fn new(opts: &DumpOptions, write: F) -> Result { + let mut write = Box::new(write); + let ctx = &mut *write as *mut F as *mut c_void; + let raw = unsafe { sys::hl_compressor_new(&opts.raw(), trampoline::, ctx) }; + if raw.is_null() { + return Err(Error { code: -6, errno: 0 }); + } + Ok(Self { raw, _write: write }) + } + + pub fn update(&mut self, data: &[u8]) -> Result<(), Error> { + check(unsafe { sys::hl_compressor_update(self.raw, data.as_ptr(), data.len()) }) + } + + pub fn finish(self) -> Result { + let mut out = sys::Digest::default(); + check(unsafe { sys::hl_compressor_finish(self.raw, &mut out) })?; + Ok(out.into()) + } +} + +impl bool> Drop for Compressor { + fn drop(&mut self) { + unsafe { sys::hl_compressor_free(self.raw) } + } +} + +// ── Verifier ── + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Compression { + None, + Zstd, +} + +impl Compression { + pub fn parse(s: &str) -> Option { + match s { + "zstd" => Some(Self::Zstd), + "none" => Some(Self::None), + _ => None, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::None => "none", + Self::Zstd => "zstd", + } + } + + pub fn extension(self) -> &'static str { + match self { + Self::None => ".nar", + Self::Zstd => ".nar.zst", + } + } +} + +/// Hashes an uploaded compressed NAR and the NAR it decompresses to. +pub struct Verifier { + raw: *mut sys::Verifier, +} + +// The verifier owns its state exclusively; it is only ever used from one thread at a time. +unsafe impl Send for Verifier {} + +impl Verifier { + pub fn new(compression: Compression) -> Result { + let kind = match compression { + Compression::None => 0, + Compression::Zstd => 1, + }; + let raw = unsafe { sys::hl_verifier_new(kind) }; + if raw.is_null() { + return Err(Error { code: -6, errno: 0 }); + } + Ok(Self { raw }) + } + + pub fn update(&mut self, data: &[u8]) -> Result<(), Error> { + check(unsafe { sys::hl_verifier_update(self.raw, data.as_ptr(), data.len()) }) + } + + pub fn finish(self) -> Result { + let mut out = sys::Digest::default(); + check(unsafe { sys::hl_verifier_finish(self.raw, &mut out) })?; + Ok(out.into()) + } +} + +impl Drop for Verifier { + fn drop(&mut self) { + unsafe { sys::hl_verifier_free(self.raw) } + } +} + +// ── Signing and narinfo ── + +pub struct Signer { + raw: *mut sys::Signer, +} + +// Immutable after construction; signing only reads the key pair. +unsafe impl Send for Signer {} +unsafe impl Sync for Signer {} + +impl Signer { + /// Parses a Nix secret key (`nix key generate-secret` output). + pub fn new(secret_key: &str) -> Option { + let raw = unsafe { sys::hl_signer_new(secret_key.as_ptr(), secret_key.len()) }; + (!raw.is_null()).then_some(Self { raw }) + } + + /// `:`, the value for `trusted-public-keys`. + pub fn public_key(&self) -> String { + let mut buf = vec![0u8; 256]; + let mut len = 0usize; + let rc = unsafe { sys::hl_signer_public_key(self.raw, buf.as_mut_ptr(), buf.len(), &mut len) }; + assert_eq!(rc, 0, "public key fits in 256 bytes"); + buf.truncate(len); + String::from_utf8(buf).expect("key name was valid UTF-8") + } +} + +impl Drop for Signer { + fn drop(&mut self) { + unsafe { sys::hl_signer_free(self.raw) } + } +} + +pub struct NarinfoInput<'a> { + pub store_path: &'a str, + pub nar_hash: &'a [u8; 32], + pub nar_size: u64, + pub file_hash: &'a [u8; 32], + pub file_size: u64, + pub compression: &'a str, + /// Space-separated basenames. + pub references: &'a str, + pub deriver: &'a str, + pub system: &'a str, +} + +fn s(v: &str) -> sys::Str { + sys::Str { ptr: v.as_ptr(), len: v.len() } +} + +/// Validates every field and renders the narinfo, signed if `signer` is set. +pub fn render_narinfo(input: &NarinfoInput<'_>, signer: Option<&Signer>) -> Result, Error> { + let raw = sys::NarinfoInput { + store_path: s(input.store_path), + nar_hash: input.nar_hash, + nar_size: input.nar_size, + file_hash: input.file_hash, + file_size: input.file_size, + compression: s(input.compression), + references: s(input.references), + deriver: s(input.deriver), + system: s(input.system), + }; + let mut ptr: *mut u8 = std::ptr::null_mut(); + let mut len = 0usize; + let signer_ptr = signer.map_or(std::ptr::null(), |s| s.raw as *const sys::Signer); + check(unsafe { sys::hl_narinfo_render(&raw, signer_ptr, &mut ptr, &mut len) })?; + let text = unsafe { std::slice::from_raw_parts(ptr, len) }.to_vec(); + unsafe { sys::hl_free(ptr, len) }; + Ok(text) +} + +#[cfg(test)] +mod tests { + use super::*; + + const KEY: &str = "test-1:suJsWimvBNFUIc0JJE18OfOMygH/f2GuTC2/XwD6rPKkV+1VKilMIkXl6Ax9hqeKpUF/BSOH+Fqng4tqZlirhw=="; + + #[test] + fn nix32_round_trip() { + let bytes = [7u8; 20]; + let text = nix32_encode(&bytes); + assert_eq!(text.len(), 32); + assert_eq!(nix32_decode::<20>(&text), Some(bytes)); + assert_eq!(nix32_decode::<20>("not-valid"), None); + } + + #[test] + fn signer_public_key() { + let signer = Signer::new(KEY).unwrap(); + assert!(signer.public_key().starts_with("test-1:")); + assert!(Signer::new("garbage").is_none()); + } + + #[test] + fn dump_matches_compressor_and_verifier() { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().join("pkg"); + std::fs::create_dir_all(root.join("bin")).unwrap(); + std::fs::write(root.join("bin/hello"), b"#!/bin/sh\necho hi\n").unwrap(); + std::fs::set_permissions(root.join("bin/hello"), std::os::unix::fs::PermissionsExt::from_mode(0o755)).unwrap(); + std::fs::write(root.join("big"), vec![b'a'; 5 << 20]).unwrap(); + std::os::unix::fs::symlink("bin/hello", root.join("link")).unwrap(); + + let mut compressed = Vec::new(); + let digest = dump_nar(&root, &DumpOptions { level: 3, threads: 2, size_hint: 0 }, |c| { + compressed.extend_from_slice(c); + true + }) + .unwrap(); + assert_eq!(digest.file_size, compressed.len() as u64); + + let mut v = Verifier::new(Compression::Zstd).unwrap(); + for chunk in compressed.chunks(65536) { + v.update(chunk).unwrap(); + } + let verified = v.finish().unwrap(); + assert_eq!(verified, digest); + + let aborted = dump_nar(&root, &DumpOptions::default(), |_| false); + assert_eq!(aborted.unwrap_err().code, HL_E_ABORTED); + } + + /// Our NAR serialiser must agree byte-for-byte with Nix: compare NAR + /// hashes against the Nix database for a real closure. + #[test] + fn nar_hash_matches_nix_store() { + use std::process::Command; + let Ok(out) = Command::new("nix-store").args(["-qR", "/run/current-system/sw/bin/nix"]).output() else { + eprintln!("nix-store not available; skipping"); + return; + }; + if !out.status.success() { + eprintln!("no local system closure; skipping"); + return; + } + let paths: Vec = String::from_utf8(out.stdout).unwrap().lines().map(str::to_owned).collect(); + assert!(!paths.is_empty()); + let hashes = Command::new("nix-store").args(["-q", "--hash"]).args(&paths).output().unwrap(); + let hashes = String::from_utf8(hashes.stdout).unwrap(); + for (path, expected) in paths.iter().zip(hashes.lines()) { + let digest = dump_nar(Path::new(path), &DumpOptions { level: 1, threads: 0, size_hint: 0 }, |_| true) + .unwrap_or_else(|e| panic!("{path}: {e}")); + assert_eq!(format!("sha256:{}", nix32_encode(&digest.nar_hash)), expected, "{path}"); + } + } +} From 6a1317a850b6c37f6404d8a2db2a6cea9f51f284 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 12:23:53 +0200 Subject: [PATCH 03/56] server: add helios-server helios-server replaces the Cloudflare worker. It keeps metadata in SQLite in WAL mode and NARs as content-addressed files on local disk, and serves the substituter protocol and a push API over HTTP. Narinfo misses are answered from an in-memory index of every published path, so the common case of Nix probing a substituter never touches SQLite; hits come from an LRU of narinfo rendered and signed once at publish time. Uploads stream to disk while libhelios decompresses and hashes them, and paths are only published against a NAR hash the server computed itself. This fixes several problems the worker had. Garbage collection could delete a content-addressed NAR that another path still referenced, uploads were never checked against their hash, unvalidated fields could inject narinfo lines, and private caches and pull tokens were not enforced. Token revocation now takes effect immediately instead of after a cache expiry. --- Cargo.lock | 2299 +++++++++++++++++++++++++++- crates/helios-server/Cargo.toml | 33 + crates/helios-server/src/admin.rs | 215 +++ crates/helios-server/src/audit.rs | 94 ++ crates/helios-server/src/auth.rs | 203 +++ crates/helios-server/src/config.rs | 97 ++ crates/helios-server/src/db.rs | 129 ++ crates/helios-server/src/error.rs | 72 + crates/helios-server/src/gc.rs | 102 ++ crates/helios-server/src/main.rs | 102 ++ crates/helios-server/src/push.rs | 490 ++++++ crates/helios-server/src/read.rs | 160 ++ crates/helios-server/src/state.rs | 162 ++ 13 files changed, 4111 insertions(+), 47 deletions(-) create mode 100644 crates/helios-server/Cargo.toml create mode 100644 crates/helios-server/src/admin.rs create mode 100644 crates/helios-server/src/audit.rs create mode 100644 crates/helios-server/src/auth.rs create mode 100644 crates/helios-server/src/config.rs create mode 100644 crates/helios-server/src/db.rs create mode 100644 crates/helios-server/src/error.rs create mode 100644 crates/helios-server/src/gc.rs create mode 100644 crates/helios-server/src/main.rs create mode 100644 crates/helios-server/src/push.rs create mode 100644 crates/helios-server/src/read.rs create mode 100644 crates/helios-server/src/state.rs diff --git a/Cargo.lock b/Cargo.lock index 8de586e..b397669 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,18 +2,373 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "aws-lc-rs" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "axum" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" +dependencies = [ + "axum-core", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bitflags" version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", + "ctutils", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.14" @@ -21,9 +376,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.5.0" @@ -31,91 +398,1929 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] -name = "getrandom" -version = "0.4.3" +name = "find-msvc-tools" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ - "cfg-if", - "libc", - "r-efi", + "percent-encoding", ] [[package]] -name = "helios-core" -version = "0.1.0" +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ - "pkg-config", - "tempfile", + "futures-core", ] [[package]] -name = "libc" -version = "0.2.189" +name = "futures-core" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] -name = "linux-raw-sys" -version = "0.12.1" +name = "futures-io" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] -name = "once_cell" -version = "1.21.4" +name = "futures-macro" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] [[package]] -name = "pkg-config" +name = "futures-sink" version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] -name = "r-efi" -version = "6.0.0" +name = "futures-task" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] -name = "rustix" -version = "1.1.5" +name = "futures-util" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ - "bitflags", - "errno", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", "libc", - "linux-raw-sys", - "windows-sys", + "wasi", + "wasm-bindgen", ] [[package]] -name = "tempfile" -version = "3.27.0" +name = "getrandom" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ - "fastrand", - "getrandom", - "once_cell", - "rustix", - "windows-sys", + "cfg-if", + "js-sys", + "libc", + "r-efi", + "rand_core", + "wasm-bindgen", ] [[package]] -name = "windows-link" -version = "0.2.1" +name = "h2" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] [[package]] -name = "windows-sys" -version = "0.61.2" +name = "hashbrown" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ - "windows-link", + "foldhash", ] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashlink" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" +dependencies = [ + "hashbrown 0.17.1", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "helios-core" +version = "0.1.0" +dependencies = [ + "pkg-config", + "tempfile", +] + +[[package]] +name = "helios-server" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "base64", + "bytes", + "clap", + "futures-util", + "helios-core", + "hmac", + "mimalloc", + "parking_lot", + "quick_cache", + "reqwest", + "rusqlite", + "serde", + "serde_json", + "sha2", + "subtle", + "tempfile", + "tokio", + "tokio-util", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "httparse", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "ipnet" +version = "2.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libmimalloc-sys" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" +dependencies = [ + "cc", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mimalloc" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" +dependencies = [ + "libmimalloc-sys", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick_cache" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403c1a912fec895cafb223201e368234842acb9220aaf08ab042ae89ba5f135c" +dependencies = [ + "equivalent", + "foldhash", + "hashbrown 0.17.1", + "parking_lot", +] + +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rsqlite-vfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" +dependencies = [ + "hashbrown 0.16.1", + "thiserror", +] + +[[package]] +name = "rusqlite" +version = "0.40.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", + "sqlite-wasm-rs", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + +[[package]] +name = "rustls-webpki" +version = "0.103.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "sqlite-wasm-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" +dependencies = [ + "cc", + "js-sys", + "rsqlite-vfs", + "wasm-bindgen", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 3.0.6", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/crates/helios-server/Cargo.toml b/crates/helios-server/Cargo.toml new file mode 100644 index 0000000..206d20a --- /dev/null +++ b/crates/helios-server/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "helios-server" +version.workspace = true +edition.workspace = true +license-file.workspace = true +publish.workspace = true + +[dependencies] +anyhow = "1.0.104" +axum = { version = "0.8.9", default-features = false, features = ["http1", "http2", "tokio", "query", "json"] } +base64 = "0.23.1" +bytes = "1.12.1" +clap = { version = "4.6.7", features = ["derive", "env"] } +futures-util = "0.3.34" +helios-core = { path = "../helios-core" } +hmac = "0.13.0" +mimalloc = "0.1.52" +parking_lot = "0.12.5" +quick_cache = "0.7.0" +rusqlite = { version = "0.40.2", features = ["bundled", "array", "fallible_uint"] } +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.151" +sha2 = "0.11.0" +subtle = "2.6.1" +tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "net", "fs", "signal", "sync", "time", "io-util"] } +tokio-util = { version = "0.7.19", features = ["io"] } +tracing = "0.1.44" +tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } +uuid = { version = "1.26.1", features = ["v4"] } + +[dev-dependencies] +reqwest = { version = "0.13.5", default-features = false, features = ["json", "stream", "rustls"] } +tempfile = "3.27.0" diff --git a/crates/helios-server/src/admin.rs b/crates/helios-server/src/admin.rs new file mode 100644 index 0000000..5445cc8 --- /dev/null +++ b/crates/helios-server/src/admin.rs @@ -0,0 +1,215 @@ +//! Admin API: caches and API tokens. Authenticated with the admin secret. + +use axum::Json; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use rusqlite::params; +use serde::Deserialize; +use serde_json::json; + +use crate::audit::Audit; +use crate::auth::{self, AUDIENCE, Claims, ISSUER, MAX_CACHES_PER_TOKEN, MAX_PERMS_PER_TOKEN, Perm}; +use crate::error::{ApiError, ApiResult}; +use crate::state::{Shared, TokenState}; + +const MAX_TOKEN_LIFETIME_DAYS: i64 = 365; +const DEFAULT_TOKEN_LIFETIME_DAYS: i64 = 90; +const MAX_SUBJECT_LEN: usize = 256; +const MAX_REASON_LEN: usize = 512; + +fn valid_cache_name(name: &str) -> bool { + let b = name.as_bytes(); + (1..=64).contains(&b.len()) + && b.iter().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == b'-') + && b[0] != b'-' + && b[b.len() - 1] != b'-' +} + +#[derive(Deserialize)] +pub struct CreateCacheReq { + name: String, + #[serde(default = "default_true")] + public: bool, +} + +fn default_true() -> bool { + true +} + +pub async fn create_cache( + State(st): State, + audit: Audit, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult<(StatusCode, Json)> { + let who = st.authorize_admin(&headers)?; + if !valid_cache_name(&req.name) { + return Err(ApiError::bad_request("cache name must be 1-64 lowercase alphanumerics or hyphens, no leading/trailing hyphen")); + } + let st2 = st.clone(); + let name = req.name.clone(); + let created = tokio::task::spawn_blocking(move || st2.insert_cache(&name, req.public)).await??; + let status = if created.is_some() { StatusCode::CREATED } else { StatusCode::CONFLICT }; + audit.log(&st, &who, "cache.create", Some(&req.name), status, json!({ "public": req.public })); + match created { + Some(info) => Ok((status, Json(json!({ "name": req.name, "public": info.public })))), + None => Err(ApiError::new(status, "cache already exists")), + } +} + +pub async fn list_caches(State(st): State, headers: HeaderMap) -> ApiResult> { + st.authorize_admin(&headers)?; + let mut caches: Vec<_> = st.caches.read().iter().map(|(name, c)| json!({ "name": name, "public": c.public })).collect(); + caches.sort_by(|a, b| a["name"].as_str().cmp(&b["name"].as_str())); + Ok(Json(json!({ "caches": caches }))) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CreateTokenReq { + subject: String, + caches: Vec, + perms: Vec, + expires_in_days: Option, +} + +pub async fn create_token( + State(st): State, + audit: Audit, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult<(StatusCode, Json)> { + let who = st.authorize_admin(&headers)?; + let secret = st.cfg.jwt_secret.clone().ok_or_else(|| ApiError::new(StatusCode::SERVICE_UNAVAILABLE, "token signing is not configured"))?; + if req.subject.is_empty() || req.subject.len() > MAX_SUBJECT_LEN { + return Err(ApiError::bad_request("subject must be 1-256 characters")); + } + let mut caches = req.caches; + caches.sort(); + caches.dedup(); + if caches.is_empty() || caches.len() > MAX_CACHES_PER_TOKEN || caches.iter().any(|c| c != "*" && !valid_cache_name(c)) { + return Err(ApiError::bad_request(format!("caches must be 1-{MAX_CACHES_PER_TOKEN} cache names or \"*\""))); + } + let mut perms = req.perms; + perms.sort(); + perms.dedup(); + if perms.is_empty() || perms.len() > MAX_PERMS_PER_TOKEN { + return Err(ApiError::bad_request("perms must be a non-empty list of \"pull\" and/or \"push\"")); + } + let days = req.expires_in_days.unwrap_or(DEFAULT_TOKEN_LIFETIME_DAYS); + if !(1..=MAX_TOKEN_LIFETIME_DAYS).contains(&days) { + return Err(ApiError::bad_request(format!("expiresInDays must be between 1 and {MAX_TOKEN_LIFETIME_DAYS}"))); + } + + let now = crate::db::now(); + let claims = Claims { + jti: uuid::Uuid::new_v4().to_string(), + sub: req.subject, + iss: ISSUER.into(), + aud: AUDIENCE.into(), + caches, + perms, + iat: now, + exp: now + days * 86400, + }; + let token = auth::sign(&claims, &secret); + let db = st.db.clone(); + let row = claims.clone(); + tokio::task::spawn_blocking(move || { + db.write(|conn| { + conn.execute( + "INSERT INTO tokens (jti, subject, caches, perms, created_at, expires_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + row.jti, + row.sub, + serde_json::to_string(&row.caches).unwrap_or_default(), + serde_json::to_string(&row.perms).unwrap_or_default(), + row.iat, + row.exp + ], + ) + }) + }) + .await??; + st.tokens.write().insert(claims.jti.as_str().into(), TokenState { expires_at: claims.exp, revoked: false }); + audit.log(&st, &who, "token.create", None, StatusCode::CREATED, json!({ "jti": claims.jti, "subject": claims.sub })); + Ok(( + StatusCode::CREATED, + Json(json!({ + "token": token, + "jti": claims.jti, + "subject": claims.sub, + "caches": claims.caches, + "perms": claims.perms, + "expiresAt": claims.exp, + })), + )) +} + +pub async fn list_tokens(State(st): State, headers: HeaderMap) -> ApiResult> { + st.authorize_admin(&headers)?; + let db = st.db.clone(); + let tokens = tokio::task::spawn_blocking(move || { + db.read(|conn| { + let mut stmt = conn.prepare( + "SELECT jti, subject, caches, perms, created_at, expires_at, revoked_at, revoked_by, revocation_reason + FROM tokens ORDER BY created_at", + )?; + let rows = stmt.query_map([], |r| { + let caches: String = r.get(2)?; + let perms: String = r.get(3)?; + Ok(json!({ + "jti": r.get::<_, String>(0)?, + "subject": r.get::<_, String>(1)?, + "caches": serde_json::from_str::(&caches).unwrap_or_default(), + "perms": serde_json::from_str::(&perms).unwrap_or_default(), + "createdAt": r.get::<_, i64>(4)?, + "expiresAt": r.get::<_, i64>(5)?, + "revokedAt": r.get::<_, Option>(6)?, + "revokedBy": r.get::<_, Option>(7)?, + "revocationReason": r.get::<_, Option>(8)?, + })) + })?; + rows.collect::>>() + }) + }) + .await??; + Ok(Json(json!({ "tokens": tokens }))) +} + +#[derive(Deserialize)] +pub struct RevokeReq { + reason: String, +} + +pub async fn revoke_token( + State(st): State, + Path(jti): Path, + audit: Audit, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult> { + let who = st.authorize_admin(&headers)?; + if req.reason.is_empty() || req.reason.len() > MAX_REASON_LEN { + return Err(ApiError::bad_request("reason must be 1-512 characters")); + } + let db = st.db.clone(); + let id = jti.clone(); + let changed = tokio::task::spawn_blocking(move || { + db.write(|conn| { + conn.execute( + "UPDATE tokens SET revoked_at = ?2, revoked_by = 'admin', revocation_reason = ?3 WHERE jti = ?1 AND revoked_at IS NULL", + params![id, crate::db::now(), req.reason], + ) + }) + }) + .await??; + if changed == 0 { + return Err(ApiError::new(StatusCode::NOT_FOUND, "token not found or already revoked")); + } + if let Some(t) = st.tokens.write().get_mut(jti.as_str()) { + t.revoked = true; + } + audit.log(&st, &who, "token.revoke", None, StatusCode::OK, json!({ "jti": jti })); + Ok(Json(json!({ "revoked": true, "jti": jti }))) +} diff --git a/crates/helios-server/src/audit.rs b/crates/helios-server/src/audit.rs new file mode 100644 index 0000000..14cbf41 --- /dev/null +++ b/crates/helios-server/src/audit.rs @@ -0,0 +1,94 @@ +//! Audit logging off the request path: events go through a channel and are +//! inserted in batched transactions. + +use std::net::{IpAddr, SocketAddr}; +use std::time::Duration; + +use axum::extract::{ConnectInfo, FromRequestParts}; +use axum::http::StatusCode; +use axum::http::request::Parts; +use rusqlite::params; +use tokio::sync::mpsc; + +use crate::auth::Identity; +use crate::db::Db; +use crate::state::Shared; + +pub struct AuditEvent { + ts: i64, + actor: String, + action: &'static str, + cache: Option, + detail: String, + ip: Option, + status: u16, +} + +/// Request extractor carrying the client IP for audit events. +pub struct Audit { + ip: Option, +} + +impl FromRequestParts for Audit { + type Rejection = std::convert::Infallible; + + async fn from_request_parts(parts: &mut Parts, st: &Shared) -> Result { + let forwarded = st + .cfg + .trust_proxy + .then(|| parts.headers.get("x-forwarded-for")?.to_str().ok()?.split(',').next()?.trim().parse().ok()) + .flatten(); + let peer = parts.extensions.get::>().map(|c| c.0.ip()); + Ok(Self { ip: forwarded.or(peer) }) + } +} + +impl Audit { + pub fn log(&self, st: &Shared, who: &Identity, action: &'static str, cache: Option<&str>, status: StatusCode, detail: serde_json::Value) { + let _ = st.audit.send(AuditEvent { + ts: crate::db::now(), + actor: who.actor().to_owned(), + action, + cache: cache.map(str::to_owned), + detail: detail.to_string(), + ip: self.ip.map(|ip| ip.to_string()), + status: status.as_u16(), + }); + } +} + +pub async fn run(db: Db, mut rx: mpsc::UnboundedReceiver) { + let mut batch = Vec::with_capacity(256); + loop { + if rx.recv_many(&mut batch, 1024).await == 0 { + return; + } + // Coalesce bursts into one transaction. + tokio::time::sleep(Duration::from_millis(200)).await; + while let Ok(ev) = rx.try_recv() { + batch.push(ev); + } + let events = std::mem::take(&mut batch); + let db = db.clone(); + let result = tokio::task::spawn_blocking(move || { + db.write(|conn| -> rusqlite::Result<()> { + let tx = conn.transaction()?; + { + let mut stmt = tx.prepare_cached( + "INSERT INTO audit_log (ts, actor, action, cache, detail, ip, status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + )?; + for e in &events { + stmt.execute(params![e.ts, e.actor, e.action, e.cache, e.detail, e.ip, e.status])?; + } + } + tx.commit() + }) + }) + .await; + match result { + Ok(Ok(())) => {} + Ok(Err(e)) => tracing::error!(error = %e, "writing audit log"), + Err(e) => tracing::error!(error = %e, "audit writer panicked"), + } + } +} diff --git a/crates/helios-server/src/auth.rs b/crates/helios-server/src/auth.rs new file mode 100644 index 0000000..8853d3f --- /dev/null +++ b/crates/helios-server/src/auth.rs @@ -0,0 +1,203 @@ +//! HS256 API tokens and the admin bearer secret. Revocation is checked +//! against the in-memory token table, so it takes effect immediately. + +use axum::http::{HeaderMap, header}; +use base64::Engine; +use base64::engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD}; +use hmac::{KeyInit, Mac}; +use serde::{Deserialize, Serialize}; +use subtle::ConstantTimeEq; + +use crate::error::ApiError; +use crate::state::{AppState, CacheInfo}; + +type HmacSha256 = hmac::Hmac; + +pub const ISSUER: &str = "helios-cache"; +pub const AUDIENCE: &str = "helios-cache"; +const HEADER_B64: &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"; // {"alg":"HS256","typ":"JWT"} +const MAX_TOKEN_BYTES: usize = 4096; +const MAX_CLOCK_SKEW: i64 = 60; +pub const MAX_CACHES_PER_TOKEN: usize = 50; +pub const MAX_PERMS_PER_TOKEN: usize = 10; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Perm { + Pull, + Push, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Claims { + pub jti: String, + pub sub: String, + pub iss: String, + pub aud: String, + pub caches: Vec, + pub perms: Vec, + pub iat: i64, + pub exp: i64, +} + +impl Claims { + pub fn allows(&self, cache: &str, perm: Perm) -> bool { + self.perms.contains(&perm) && self.caches.iter().any(|c| c == "*" || c == cache) + } +} + +pub enum Identity { + Token(Claims), + Admin, +} + +impl Identity { + pub fn actor(&self) -> &str { + match self { + Identity::Token(c) => &c.jti, + Identity::Admin => "admin", + } + } +} + +pub fn sign(claims: &Claims, secret: &[u8]) -> String { + let payload = URL_SAFE_NO_PAD.encode(serde_json::to_vec(claims).expect("claims serialise")); + let input = format!("{HEADER_B64}.{payload}"); + let mut mac = HmacSha256::new_from_slice(secret).expect("HMAC accepts any key length"); + mac.update(input.as_bytes()); + let sig = URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()); + format!("{input}.{sig}") +} + +fn verify(token: &str, secret: &[u8], now: i64) -> Option { + if token.len() > MAX_TOKEN_BYTES { + return None; + } + let (signed, sig_b64) = token.rsplit_once('.')?; + let (header, payload_b64) = signed.split_once('.')?; + if header != HEADER_B64 || payload_b64.contains('.') { + return None; + } + let sig = URL_SAFE_NO_PAD.decode(sig_b64).ok()?; + let mut mac = HmacSha256::new_from_slice(secret).ok()?; + mac.update(signed.as_bytes()); + mac.verify_slice(&sig).ok()?; + + let claims: Claims = serde_json::from_slice(&URL_SAFE_NO_PAD.decode(payload_b64).ok()?).ok()?; + let valid = !claims.jti.is_empty() + && !claims.sub.is_empty() + && claims.iss == ISSUER + && claims.aud == AUDIENCE + && claims.iat <= now + MAX_CLOCK_SKEW + && claims.exp > claims.iat + && claims.exp > now - MAX_CLOCK_SKEW + && (1..=MAX_CACHES_PER_TOKEN).contains(&claims.caches.len()) + && (1..=MAX_PERMS_PER_TOKEN).contains(&claims.perms.len()) + && claims.caches.iter().all(|c| !c.is_empty()); + valid.then_some(claims) +} + +/// Bearer token, or the password of HTTP Basic auth (what Nix sends from netrc). +fn credential(headers: &HeaderMap) -> Option { + let value = headers.get(header::AUTHORIZATION)?.to_str().ok()?; + let (scheme, rest) = value.split_once(' ')?; + let rest = rest.trim(); + if scheme.eq_ignore_ascii_case("bearer") { + return Some(rest.to_owned()); + } + if scheme.eq_ignore_ascii_case("basic") { + let decoded = String::from_utf8(STANDARD.decode(rest).ok()?).ok()?; + let (_, password) = decoded.split_once(':')?; + return Some(password.to_owned()); + } + None +} + +fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { + use sha2::Digest; + // Hash first so the comparison does not leak the secret's length. + let (da, db) = (sha2::Sha256::digest(a), sha2::Sha256::digest(b)); + da.as_slice().ct_eq(db.as_slice()).into() +} + +impl AppState { + fn token_claims(&self, headers: &HeaderMap) -> Result { + let secret = self.cfg.jwt_secret.as_deref().ok_or_else(ApiError::forbidden)?; + let token = credential(headers).ok_or_else(ApiError::unauthorized)?; + let now = crate::db::now(); + let claims = verify(&token, secret, now).ok_or_else(ApiError::unauthorized)?; + match self.tokens.read().get(claims.jti.as_str()) { + Some(t) if !t.revoked && t.expires_at > now => Ok(claims), + _ => Err(ApiError::unauthorized()), + } + } + + /// Requires a token granting `perm` on `cache`. + pub fn authorize(&self, headers: &HeaderMap, cache: &str, perm: Perm) -> Result { + let claims = self.token_claims(headers)?; + if !claims.allows(cache, perm) { + return Err(ApiError::forbidden()); + } + Ok(Identity::Token(claims)) + } + + /// Public caches are readable by anyone; private ones need `pull`. + pub fn authorize_read(&self, headers: &HeaderMap, name: &str, cache: CacheInfo) -> Result<(), ApiError> { + if cache.public { + return Ok(()); + } + self.authorize(headers, name, Perm::Pull).map(|_| ()) + } + + pub fn authorize_admin(&self, headers: &HeaderMap) -> Result { + let secret = self.cfg.admin_secret.as_deref().ok_or_else(ApiError::forbidden)?; + let given = credential(headers).ok_or_else(ApiError::unauthorized)?; + if !constant_time_eq(given.as_bytes(), secret) { + return Err(ApiError::forbidden()); + } + Ok(Identity::Admin) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn claims(now: i64) -> Claims { + Claims { + jti: "id".into(), + sub: "ci".into(), + iss: ISSUER.into(), + aud: AUDIENCE.into(), + caches: vec!["main".into()], + perms: vec![Perm::Push], + iat: now, + exp: now + 3600, + } + } + + #[test] + fn round_trip_and_tamper() { + let now = 1_700_000_000; + let token = sign(&claims(now), b"0123456789abcdef"); + assert!(verify(&token, b"0123456789abcdef", now).is_some()); + assert!(verify(&token, b"0123456789abcdeX", now).is_none()); + assert!(verify(&token, b"0123456789abcdef", now + 7200).is_none()); + let mut tampered = token.clone(); + tampered.insert(token.find('.').unwrap() + 3, 'x'); + assert!(verify(&tampered, b"0123456789abcdef", now).is_none()); + } + + #[test] + fn header_constant_matches() { + assert_eq!(URL_SAFE_NO_PAD.encode(br#"{"alg":"HS256","typ":"JWT"}"#), HEADER_B64); + } + + #[test] + fn scope_checks() { + let c = claims(0); + assert!(c.allows("main", Perm::Push)); + assert!(!c.allows("main", Perm::Pull)); + assert!(!c.allows("other", Perm::Push)); + } +} diff --git a/crates/helios-server/src/config.rs b/crates/helios-server/src/config.rs new file mode 100644 index 0000000..e8d8905 --- /dev/null +++ b/crates/helios-server/src/config.rs @@ -0,0 +1,97 @@ +use std::net::SocketAddr; +use std::path::PathBuf; +use std::time::Duration; + +use clap::Parser; + +/// Helios Nix binary cache server. +#[derive(Parser, Debug, Clone)] +#[command(version)] +pub struct Args { + /// Address to listen on. + #[arg(long, env = "HELIOS_LISTEN", default_value = "127.0.0.1:8080")] + pub listen: SocketAddr, + + /// Directory for the database and NAR files. + #[arg(long, env = "HELIOS_DATA_DIR", default_value = "/var/lib/helios")] + pub data_dir: PathBuf, + + /// Nix secret key file (`nix key generate-secret`) used to sign narinfo. + #[arg(long, env = "HELIOS_SIGNING_KEY_FILE")] + pub signing_key_file: Option, + + /// File holding the HMAC secret for API tokens (JWT HS256). + #[arg(long, env = "HELIOS_JWT_SECRET_FILE")] + pub jwt_secret_file: Option, + + /// File holding the bearer secret for the admin API. + #[arg(long, env = "HELIOS_ADMIN_SECRET_FILE")] + pub admin_secret_file: Option, + + /// Serve NARs by delegating to a reverse proxy (nginx X-Accel-Redirect) + /// under this internal location, e.g. `/_nar`. The location must alias + /// `/nar/`. Lets the proxy use sendfile for zero-copy downloads. + #[arg(long, env = "HELIOS_ACCEL_REDIRECT")] + pub accel_redirect: Option, + + /// Trust X-Forwarded-For for audit log client IPs. + #[arg(long, env = "HELIOS_TRUST_PROXY", default_value_t = false)] + pub trust_proxy: bool, + + /// Rendered narinfo bodies kept in memory. + #[arg(long, env = "HELIOS_NARINFO_CACHE_ENTRIES", default_value_t = 262_144)] + pub narinfo_cache_entries: usize, + + /// Largest accepted compressed NAR upload, in bytes. + #[arg(long, env = "HELIOS_MAX_UPLOAD_BYTES", default_value_t = 64 << 30)] + pub max_upload_bytes: u64, + + /// Hours between garbage collection runs. + #[arg(long, env = "HELIOS_GC_INTERVAL_HOURS", default_value_t = 6)] + pub gc_interval_hours: u64, + + /// Days to keep audit log entries. + #[arg(long, env = "HELIOS_AUDIT_RETENTION_DAYS", default_value_t = 30)] + pub audit_retention_days: u64, + + /// Print the public key for `trusted-public-keys` and exit. + #[arg(long)] + pub print_public_key: bool, +} + +#[derive(Debug, Clone)] +pub struct Config { + pub data_dir: PathBuf, + pub jwt_secret: Option>, + pub admin_secret: Option>, + pub accel_redirect: Option, + pub trust_proxy: bool, + pub narinfo_cache_entries: usize, + pub max_upload_bytes: u64, + pub gc_interval: Duration, + pub audit_retention: Duration, +} + +pub fn read_secret(path: &Option) -> anyhow::Result>> { + let Some(path) = path else { return Ok(None) }; + let raw = std::fs::read(path).map_err(|e| anyhow::anyhow!("reading {}: {e}", path.display()))?; + let trimmed = raw.trim_ascii().to_vec(); + anyhow::ensure!(trimmed.len() >= 16, "{} must hold at least 16 bytes", path.display()); + Ok(Some(trimmed)) +} + +impl Config { + pub fn from_args(args: &Args) -> anyhow::Result { + Ok(Self { + data_dir: args.data_dir.clone(), + jwt_secret: read_secret(&args.jwt_secret_file)?, + admin_secret: read_secret(&args.admin_secret_file)?, + accel_redirect: args.accel_redirect.as_ref().map(|s| s.trim_end_matches('/').to_owned()), + trust_proxy: args.trust_proxy, + narinfo_cache_entries: args.narinfo_cache_entries.max(16), + max_upload_bytes: args.max_upload_bytes, + gc_interval: Duration::from_secs(args.gc_interval_hours.max(1) * 3600), + audit_retention: Duration::from_secs(args.audit_retention_days * 86400), + }) + } +} diff --git a/crates/helios-server/src/db.rs b/crates/helios-server/src/db.rs new file mode 100644 index 0000000..3b8d405 --- /dev/null +++ b/crates/helios-server/src/db.rs @@ -0,0 +1,129 @@ +//! SQLite in WAL mode: one writer connection behind a mutex, and one +//! read-only connection per blocking-pool thread. + +use std::cell::RefCell; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use parking_lot::Mutex; +use rusqlite::{Connection, OpenFlags}; + +const MIGRATIONS: &[&str] = &[r#" +CREATE TABLE caches ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + is_public INTEGER NOT NULL DEFAULT 1, + created_at INTEGER NOT NULL +); +CREATE TABLE blobs ( + id INTEGER PRIMARY KEY, + file_hash BLOB NOT NULL UNIQUE, + file_size INTEGER NOT NULL, + compression TEXT NOT NULL, + nar_hash BLOB NOT NULL, + nar_size INTEGER NOT NULL, + created_at INTEGER NOT NULL +); +CREATE INDEX blobs_nar_hash ON blobs (nar_hash); +CREATE TABLE paths ( + id INTEGER PRIMARY KEY, + cache_id INTEGER NOT NULL REFERENCES caches (id), + hash BLOB NOT NULL, + blob_id INTEGER NOT NULL REFERENCES blobs (id), + store_path TEXT NOT NULL, + narinfo BLOB NOT NULL, + created_at INTEGER NOT NULL +); +CREATE UNIQUE INDEX paths_cache_hash ON paths (cache_id, hash); +CREATE INDEX paths_blob ON paths (blob_id, cache_id); +CREATE TABLE tokens ( + jti TEXT PRIMARY KEY, + subject TEXT NOT NULL, + caches TEXT NOT NULL, + perms TEXT NOT NULL, + created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + revoked_at INTEGER, + revoked_by TEXT, + revocation_reason TEXT +); +CREATE TABLE audit_log ( + id INTEGER PRIMARY KEY, + ts INTEGER NOT NULL, + actor TEXT NOT NULL, + action TEXT NOT NULL, + cache TEXT, + detail TEXT NOT NULL, + ip TEXT, + status INTEGER NOT NULL +); +CREATE INDEX audit_log_ts ON audit_log (ts); +"#]; + +const PRAGMAS: &str = " +PRAGMA busy_timeout = 5000; +PRAGMA synchronous = NORMAL; +PRAGMA temp_store = MEMORY; +PRAGMA mmap_size = 4294967296; +PRAGMA cache_size = -65536; +"; + +#[derive(Clone)] +pub struct Db { + path: Arc, + writer: Arc>, +} + +thread_local! { + static READER: RefCell, Connection)>> = const { RefCell::new(None) }; +} + +impl Db { + pub fn open(path: &Path) -> anyhow::Result { + let mut conn = Connection::open(path)?; + conn.pragma_update(None, "journal_mode", "WAL")?; + conn.execute_batch(PRAGMAS)?; + migrate(&mut conn)?; + Ok(Self { path: Arc::new(path.to_owned()), writer: Arc::new(Mutex::new(conn)) }) + } + + /// Runs `f` on the writer connection. Blocking: call from spawn_blocking. + pub fn write(&self, f: impl FnOnce(&mut Connection) -> T) -> T { + f(&mut self.writer.lock()) + } + + /// Runs `f` on this thread's read-only connection. Blocking. + pub fn read(&self, f: impl FnOnce(&Connection) -> rusqlite::Result) -> rusqlite::Result { + READER.with(|cell| { + let mut slot = cell.borrow_mut(); + let stale = !matches!(&*slot, Some((p, _)) if Arc::ptr_eq(p, &self.path)); + if stale { + let conn = Connection::open_with_flags( + &*self.path, + OpenFlags::SQLITE_OPEN_READ_ONLY | OpenFlags::SQLITE_OPEN_NO_MUTEX | OpenFlags::SQLITE_OPEN_URI, + )?; + conn.execute_batch(PRAGMAS)?; + rusqlite::vtab::array::load_module(&conn)?; + *slot = Some((self.path.clone(), conn)); + } + f(&slot.as_ref().expect("reader initialised").1) + }) + } +} + +fn migrate(conn: &mut Connection) -> rusqlite::Result<()> { + let version: usize = conn.pragma_query_value(None, "user_version", |r| r.get(0))?; + for (i, sql) in MIGRATIONS.iter().enumerate().skip(version) { + let tx = conn.transaction()?; + tx.execute_batch(sql)?; + tx.pragma_update(None, "user_version", i + 1)?; + tx.commit()?; + } + Ok(()) +} + +pub fn now() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_secs() as i64) +} diff --git a/crates/helios-server/src/error.rs b/crates/helios-server/src/error.rs new file mode 100644 index 0000000..2774a4f --- /dev/null +++ b/crates/helios-server/src/error.rs @@ -0,0 +1,72 @@ +use axum::http::{HeaderValue, StatusCode, header}; +use axum::response::{IntoResponse, Response}; +use serde_json::json; + +#[derive(Debug)] +pub struct ApiError { + pub status: StatusCode, + pub message: String, + pub extra: Option, +} + +impl ApiError { + pub fn new(status: StatusCode, message: impl Into) -> Self { + Self { status, message: message.into(), extra: None } + } + + pub fn bad_request(message: impl Into) -> Self { + Self::new(StatusCode::BAD_REQUEST, message) + } + + pub fn not_found() -> Self { + Self::new(StatusCode::NOT_FOUND, "not found") + } + + pub fn forbidden() -> Self { + Self::new(StatusCode::FORBIDDEN, "forbidden") + } + + pub fn unauthorized() -> Self { + Self::new(StatusCode::UNAUTHORIZED, "unauthorized") + } + + pub fn internal(err: impl std::fmt::Display) -> Self { + tracing::error!(error = %err, "internal error"); + Self::new(StatusCode::INTERNAL_SERVER_ERROR, "internal error") + } + + pub fn with(mut self, extra: serde_json::Value) -> Self { + self.extra = Some(extra); + self + } +} + +impl From for ApiError { + fn from(err: rusqlite::Error) -> Self { + Self::internal(err) + } +} + +impl From for ApiError { + fn from(err: tokio::task::JoinError) -> Self { + Self::internal(err) + } +} + +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + let mut body = json!({ "error": self.message }); + if let (Some(serde_json::Value::Object(extra)), serde_json::Value::Object(map)) = (self.extra, &mut body) { + map.extend(extra); + } + let mut resp = (self.status, axum::Json(body)).into_response(); + let headers = resp.headers_mut(); + headers.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store")); + if self.status == StatusCode::UNAUTHORIZED { + headers.insert(header::WWW_AUTHENTICATE, HeaderValue::from_static("Basic realm=\"helios\"")); + } + resp + } +} + +pub type ApiResult = Result; diff --git a/crates/helios-server/src/gc.rs b/crates/helios-server/src/gc.rs new file mode 100644 index 0000000..f26a8ae --- /dev/null +++ b/crates/helios-server/src/gc.rs @@ -0,0 +1,102 @@ +//! Periodic garbage collection: stale temp uploads, unreferenced blobs past +//! a grace period, expired tokens, and old audit log entries. + +use std::time::{Duration, SystemTime}; + +use helios_core::Compression; +use rusqlite::params; + +use crate::state::Shared; + +/// Blobs uploaded but not yet published are kept this long, so a push that +/// uploads first and publishes afterwards is never raced by GC. +const BLOB_GRACE: Duration = Duration::from_secs(3600); +const TMP_MAX_AGE: Duration = Duration::from_secs(3600); +const BLOB_BATCH: usize = 1000; + +#[derive(Debug, Default)] +pub struct GcStats { + pub tmp_files: usize, + pub blobs: usize, + pub bytes: u64, + pub tokens: usize, + pub audit_rows: usize, +} + +pub async fn run_forever(st: Shared) { + tokio::time::sleep(Duration::from_secs(60)).await; + let mut interval = tokio::time::interval(st.cfg.gc_interval); + loop { + interval.tick().await; + let st2 = st.clone(); + match tokio::task::spawn_blocking(move || collect(&st2)).await { + Ok(Ok(stats)) => tracing::info!(?stats, "gc finished"), + Ok(Err(e)) => tracing::error!(error = %e, "gc failed"), + Err(e) => tracing::error!(error = %e, "gc panicked"), + } + } +} + +pub fn collect(st: &Shared) -> anyhow::Result { + let mut stats = GcStats::default(); + let now = crate::db::now(); + + if let Ok(entries) = std::fs::read_dir(st.tmp_dir()) { + for entry in entries.flatten() { + let stale = entry + .metadata() + .and_then(|m| m.modified()) + .map(|t| SystemTime::now().duration_since(t).unwrap_or_default() > TMP_MAX_AGE) + .unwrap_or(false); + if stale && std::fs::remove_file(entry.path()).is_ok() { + stats.tmp_files += 1; + } + } + } + + let cutoff = now - BLOB_GRACE.as_secs() as i64; + loop { + // Row delete and unlink share the writer lock with upload's + // rename+upsert, so a re-uploaded blob can never lose its file. + let n = st.db.write(|conn| -> anyhow::Result { + let tx = conn.transaction()?; + let victims: Vec<(i64, Vec, String, u64)> = { + let mut stmt = tx.prepare_cached( + "SELECT id, file_hash, compression, file_size FROM blobs b WHERE created_at < ?1 + AND NOT EXISTS (SELECT 1 FROM paths p WHERE p.blob_id = b.id) LIMIT ?2", + )?; + stmt.query_map(params![cutoff, BLOB_BATCH], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))? + .collect::>()? + }; + { + let mut del = tx.prepare_cached("DELETE FROM blobs WHERE id = ?1")?; + for (id, ..) in &victims { + del.execute([id])?; + } + } + tx.commit()?; + for (_, hash, compression, size) in &victims { + let (Ok(hash), Some(compression)) = (<[u8; 32]>::try_from(hash.as_slice()), Compression::parse(compression)) else { + continue; + }; + match std::fs::remove_file(st.nar_path(&hash, compression)) { + Ok(()) => stats.bytes += size, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => tracing::warn!(error = %e, "removing blob file"), + } + } + Ok(victims.len()) + })?; + stats.blobs += n; + if n < BLOB_BATCH { + break; + } + } + + stats.tokens = st.db.write(|conn| conn.execute("DELETE FROM tokens WHERE expires_at < ?1", [now]))?; + st.tokens.write().retain(|_, t| t.expires_at >= now); + + let audit_cutoff = now - st.cfg.audit_retention.as_secs() as i64; + stats.audit_rows = st.db.write(|conn| conn.execute("DELETE FROM audit_log WHERE ts < ?1", [audit_cutoff]))?; + Ok(stats) +} diff --git a/crates/helios-server/src/main.rs b/crates/helios-server/src/main.rs new file mode 100644 index 0000000..85ed52d --- /dev/null +++ b/crates/helios-server/src/main.rs @@ -0,0 +1,102 @@ +mod admin; +mod audit; +mod auth; +mod config; +mod db; +mod error; +mod gc; +mod push; +mod read; +mod state; + +use std::net::SocketAddr; +use std::sync::Arc; + +use axum::Router; +use axum::extract::{DefaultBodyLimit, State}; +use axum::http::StatusCode; +use axum::routing::{get, post, put}; +use clap::Parser; +use serde_json::json; + +use crate::config::{Args, Config}; +use crate::state::{AppState, Shared}; + +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + +fn load_signer(args: &Args) -> anyhow::Result> { + let Some(path) = &args.signing_key_file else { return Ok(None) }; + let text = std::fs::read_to_string(path).map_err(|e| anyhow::anyhow!("reading {}: {e}", path.display()))?; + let signer = helios_core::Signer::new(&text).ok_or_else(|| anyhow::anyhow!("{} is not a Nix secret key", path.display()))?; + Ok(Some(signer)) +} + +pub fn router(st: Shared) -> Router { + let api = Router::new() + .route("/caches/{cache}/missing", post(push::missing)) + .route("/caches/{cache}/nars/known", post(push::known)) + .route("/caches/{cache}/nar", put(push::upload).layer(DefaultBodyLimit::disable())) + .route("/caches/{cache}/paths", post(push::publish)) + .route("/admin/caches", post(admin::create_cache).get(admin::list_caches)) + .route("/admin/tokens", post(admin::create_token).get(admin::list_tokens)) + .route("/admin/tokens/{jti}/revoke", post(admin::revoke_token)) + .layer(DefaultBodyLimit::max(64 << 20)); + + Router::new() + .route("/", get(|| async { axum::Json(json!({ "service": "helios", "status": "ok" })) })) + .route("/healthz", get(healthz)) + .nest("/_api/v2", api) + .fallback(read::handle) + .with_state(st) +} + +async fn healthz(State(st): State) -> (StatusCode, axum::Json) { + let db = st.db.clone(); + let ok = tokio::task::spawn_blocking(move || db.read(|c| c.query_row("SELECT 1", [], |_| Ok(()))).is_ok()) + .await + .unwrap_or(false); + let status = if ok { StatusCode::OK } else { StatusCode::SERVICE_UNAVAILABLE }; + (status, axum::Json(json!({ "ok": ok, "service": "helios" }))) +} + +pub async fn build_state(args: &Args) -> anyhow::Result { + let cfg = Config::from_args(args)?; + std::fs::create_dir_all(cfg.data_dir.join("nar"))?; + std::fs::create_dir_all(cfg.data_dir.join("tmp"))?; + let db = db::Db::open(&cfg.data_dir.join("helios.db"))?; + let signer = load_signer(args)?; + if signer.is_none() { + tracing::warn!("no signing key configured; narinfo will be unsigned"); + } + let (audit_tx, audit_rx) = tokio::sync::mpsc::unbounded_channel(); + tokio::spawn(audit::run(db.clone(), audit_rx)); + let st = Arc::new(AppState::load(cfg, db, signer, audit_tx)?); + tokio::spawn(gc::run_forever(st.clone())); + Ok(st) +} + +fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "helios_server=info".into())) + .init(); + let args = Args::parse(); + + if args.print_public_key { + let signer = load_signer(&args)?.ok_or_else(|| anyhow::anyhow!("--signing-key-file is required"))?; + println!("{}", signer.public_key()); + return Ok(()); + } + + tokio::runtime::Builder::new_multi_thread().enable_all().build()?.block_on(async move { + let st = build_state(&args).await?; + let listener = tokio::net::TcpListener::bind(args.listen).await?; + tracing::info!(listen = %args.listen, "helios listening"); + axum::serve(listener, router(st).into_make_service_with_connect_info::()) + .with_graceful_shutdown(async { + let _ = tokio::signal::ctrl_c().await; + }) + .await?; + Ok(()) + }) +} diff --git a/crates/helios-server/src/push.rs b/crates/helios-server/src/push.rs new file mode 100644 index 0000000..18f72eb --- /dev/null +++ b/crates/helios-server/src/push.rs @@ -0,0 +1,490 @@ +//! The push API. +//! +//! 1. `POST /_api/v2/caches/{cache}/missing` → which store paths are absent (memory only). +//! 2. `POST /_api/v2/caches/{cache}/nars/known` → which NAR hashes already have a blob. +//! 3. `PUT /_api/v2/caches/{cache}/nar?compression=zstd` → stream a compressed NAR. +//! The server hashes and decompresses it on the fly and records the verified +//! NAR hash; clients cannot claim content they did not upload. +//! 4. `POST /_api/v2/caches/{cache}/paths` → publish a batch in one transaction. + +use std::io::Write; +use std::rc::Rc; + +use axum::Json; +use axum::body::Body; +use axum::extract::{Path, Query, State}; +use axum::http::{HeaderMap, StatusCode}; +use base64::Engine; +use bytes::Bytes; +use futures_util::StreamExt; +use helios_core::{Compression, NarinfoInput, Verifier}; +use rusqlite::types::Value; +use rusqlite::{OptionalExtension, params}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +use crate::audit::Audit; +use crate::auth::Perm; +use crate::error::{ApiError, ApiResult}; +use crate::state::{CacheInfo, PathKey, Shared}; + +const MAX_MISSING_BATCH: usize = 100_000; +const MAX_PUBLISH_BATCH: usize = 5_000; +const STORE_DIR: &str = "/nix/store/"; + +fn cache_for(st: &Shared, name: &str) -> ApiResult { + st.cache(name).ok_or_else(|| ApiError::new(StatusCode::NOT_FOUND, "cache not found")) +} + +// ── Missing paths ── + +#[derive(Deserialize)] +pub struct MissingReq { + hashes: Vec, +} + +pub async fn missing( + State(st): State, + Path(name): Path, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult> { + let cache = cache_for(&st, &name)?; + st.authorize(&headers, &name, Perm::Push)?; + if req.hashes.len() > MAX_MISSING_BATCH { + return Err(ApiError::bad_request(format!("at most {MAX_MISSING_BATCH} hashes per request"))); + } + let index = st.index.read(); + let missing: Vec<&String> = req + .hashes + .iter() + .filter(|h| match helios_core::nix32_decode::<20>(h) { + Some(hash) => !index.contains(&PathKey { cache: cache.id, hash }), + None => true, + }) + .collect(); + Ok(Json(json!({ "missing": missing }))) +} + +// ── Known NARs ── + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct KnownReq { + nar_hashes: Vec, +} + +pub async fn known( + State(st): State, + Path(name): Path, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult> { + cache_for(&st, &name)?; + st.authorize(&headers, &name, Perm::Push)?; + if req.nar_hashes.len() > MAX_MISSING_BATCH { + return Err(ApiError::bad_request(format!("at most {MAX_MISSING_BATCH} hashes per request"))); + } + let parsed: Vec> = req.nar_hashes.iter().map(|h| parse_sha256(h)).collect(); + let db = st.db.clone(); + let values: Vec = parsed.iter().flatten().map(|h| Value::Blob(h.to_vec())).collect(); + let found = tokio::task::spawn_blocking(move || { + db.read(|conn| { + let mut stmt = conn.prepare_cached("SELECT DISTINCT nar_hash FROM blobs WHERE nar_hash IN rarray(?1)")?; + let rows = stmt.query_map([Rc::new(values)], |r| r.get::<_, Vec>(0))?; + rows.collect::>>>() + }) + }) + .await??; + let known: Vec<&String> = req + .nar_hashes + .iter() + .zip(&parsed) + .filter(|(_, h)| h.is_some_and(|h| found.contains(&h[..]))) + .map(|(s, _)| s) + .collect(); + Ok(Json(json!({ "known": known }))) +} + +// ── Upload ── + +#[derive(Deserialize)] +pub struct UploadQuery { + compression: Option, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct UploadResp { + file_hash: String, + file_size: u64, + nar_hash: String, + nar_size: u64, +} + +pub async fn upload( + State(st): State, + Path(name): Path, + Query(q): Query, + audit: Audit, + headers: HeaderMap, + body: Body, +) -> ApiResult<(StatusCode, Json)> { + cache_for(&st, &name)?; + let identity = st.authorize(&headers, &name, Perm::Push)?; + let compression = Compression::parse(q.compression.as_deref().unwrap_or("zstd")) + .ok_or_else(|| ApiError::bad_request("compression must be zstd or none"))?; + + let tmp = st.tmp_dir().join(uuid::Uuid::new_v4().to_string()); + let (tx, mut rx) = tokio::sync::mpsc::channel::(32); + let worker_tmp = tmp.clone(); + // Disk writes, hashing and decompression happen off the async runtime. + let worker = tokio::task::spawn_blocking(move || -> Result { + let file = std::fs::File::create_new(&worker_tmp).map_err(|e| e.to_string())?; + let mut out = std::io::BufWriter::with_capacity(1 << 20, file); + let mut verifier = Verifier::new(compression).map_err(|e| e.to_string())?; + while let Some(chunk) = rx.blocking_recv() { + verifier.update(&chunk).map_err(|e| format!("invalid NAR stream: {e}"))?; + out.write_all(&chunk).map_err(|e| e.to_string())?; + } + let file = out.into_inner().map_err(|e| e.to_string())?; + file.sync_data().map_err(|e| e.to_string())?; + verifier.finish().map_err(|e| format!("invalid NAR stream: {e}")) + }); + + let mut stream = body.into_data_stream(); + let mut received: u64 = 0; + let mut failure: Option = None; + while let Some(chunk) = stream.next().await { + let chunk = match chunk { + Ok(c) => c, + Err(e) => { + failure = Some(ApiError::bad_request(format!("reading body: {e}"))); + break; + } + }; + received += chunk.len() as u64; + if received > st.cfg.max_upload_bytes { + failure = Some(ApiError::new(StatusCode::PAYLOAD_TOO_LARGE, "upload too large")); + break; + } + if tx.send(chunk).await.is_err() { + break; // Worker failed; its error is reported below. + } + } + drop(tx); + let result = worker.await?; + + let digest = match (failure, result) { + (Some(err), _) => { + let _ = tokio::fs::remove_file(&tmp).await; + audit.log(&st, &identity, "nar.upload", Some(&name), err.status, json!({ "error": err.message })); + return Err(err); + } + (None, Err(msg)) => { + let _ = tokio::fs::remove_file(&tmp).await; + let err = ApiError::bad_request(msg); + audit.log(&st, &identity, "nar.upload", Some(&name), err.status, json!({ "error": err.message })); + return Err(err); + } + (None, Ok(d)) => d, + }; + + let dest = st.nar_path(&digest.file_hash, compression); + let st2 = st.clone(); + tokio::task::spawn_blocking(move || -> ApiResult<()> { + if let Some(parent) = dest.parent() { + std::fs::create_dir_all(parent).map_err(ApiError::internal)?; + } + // Rename and row upsert happen under the writer lock so GC can never + // delete the file between them. Renaming over an existing blob is + // harmless: the content is identical by hash. + st2.db.write(|conn| -> ApiResult<()> { + std::fs::rename(&tmp, &dest).map_err(ApiError::internal)?; + conn.prepare_cached( + "INSERT INTO blobs (file_hash, file_size, compression, nar_hash, nar_size, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT (file_hash) DO UPDATE SET created_at = excluded.created_at", + )? + .execute(params![ + &digest.file_hash[..], + digest.file_size, + compression.as_str(), + &digest.nar_hash[..], + digest.nar_size, + crate::db::now() + ])?; + Ok(()) + })?; + Ok(()) + }) + .await??; + + let resp = UploadResp { + file_hash: helios_core::nix32_encode(&digest.file_hash), + file_size: digest.file_size, + nar_hash: format!("sha256:{}", helios_core::nix32_encode(&digest.nar_hash)), + nar_size: digest.nar_size, + }; + audit.log(&st, &identity, "nar.upload", Some(&name), StatusCode::CREATED, json!({ "fileHash": resp.file_hash, "size": resp.file_size })); + Ok((StatusCode::CREATED, Json(resp))) +} + +// ── Publish ── + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PathSpec { + store_path: String, + nar_hash: String, + nar_size: u64, + #[serde(default)] + references: Vec, + deriver: Option, + system: Option, + /// Pin a specific uploaded blob (nix32 file hash); otherwise any blob + /// with a matching verified NAR hash is used. + file_hash: Option, +} + +#[derive(Deserialize)] +pub struct PublishReq { + paths: Vec, +} + +/// Accepts `sha256:`, `sha256:` and SRI `sha256-`. +pub fn parse_sha256(s: &str) -> Option<[u8; 32]> { + if let Some(b64) = s.strip_prefix("sha256-") { + return base64::engine::general_purpose::STANDARD.decode(b64).ok()?.try_into().ok(); + } + let h = s.strip_prefix("sha256:").unwrap_or(s); + match h.len() { + 52 => helios_core::nix32_decode::<32>(h), + 64 => { + let mut out = [0u8; 32]; + for (i, byte) in out.iter_mut().enumerate() { + *byte = u8::from_str_radix(h.get(i * 2..i * 2 + 2)?, 16).ok()?; + } + Some(out) + } + _ => None, + } +} + +fn basename(s: &str) -> &str { + s.strip_prefix(STORE_DIR).unwrap_or(s) +} + +struct Prepared { + key: PathKey, + store_path: String, + nar_hash: [u8; 32], + nar_size: u64, + file_hash: Option<[u8; 32]>, + references: String, + deriver: String, + system: String, +} + +struct Rendered { + key: PathKey, + store_path: String, + blob_id: i64, + narinfo: Vec, +} + +fn prepare(cache: CacheInfo, spec: PathSpec) -> Result { + let base = spec + .store_path + .strip_prefix(STORE_DIR) + .filter(|b| helios_core::store_basename_valid(b)) + .ok_or_else(|| format!("invalid store path: {}", spec.store_path))?; + let hash = helios_core::nix32_decode::<20>(&base[..32]).ok_or("invalid store path hash")?; + let nar_hash = parse_sha256(&spec.nar_hash).ok_or_else(|| format!("{}: invalid narHash", spec.store_path))?; + let file_hash = match &spec.file_hash { + Some(h) => Some(parse_sha256(h).ok_or_else(|| format!("{}: invalid fileHash", spec.store_path))?), + None => None, + }; + let mut references = String::new(); + for r in &spec.references { + let r = basename(r); + if !helios_core::store_basename_valid(r) { + return Err(format!("{}: invalid reference {r}", spec.store_path)); + } + if !references.is_empty() { + references.push(' '); + } + references.push_str(r); + } + Ok(Prepared { + key: PathKey { cache: cache.id, hash }, + store_path: spec.store_path, + nar_hash, + nar_size: spec.nar_size, + file_hash, + references, + deriver: spec.deriver.as_deref().map(basename).unwrap_or_default().to_owned(), + system: spec.system.unwrap_or_default(), + }) +} + +pub async fn publish( + State(st): State, + Path(name): Path, + audit: Audit, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult<(StatusCode, Json)> { + let cache = cache_for(&st, &name)?; + let identity = st.authorize(&headers, &name, Perm::Push)?; + if req.paths.len() > MAX_PUBLISH_BATCH { + return Err(ApiError::bad_request(format!("at most {MAX_PUBLISH_BATCH} paths per request"))); + } + + let mut prepared = Vec::with_capacity(req.paths.len()); + let mut already = 0usize; + { + let index = st.index.read(); + for spec in req.paths { + let p = prepare(cache, spec).map_err(ApiError::bad_request)?; + if index.contains(&p.key) { + already += 1; + } else { + prepared.push(p); + } + } + } + + let st2 = st.clone(); + let outcome = tokio::task::spawn_blocking(move || render_and_insert(&st2, prepared)).await??; + let (rendered_keys, missing) = match outcome { + Outcome::Published(keys) => (keys, Vec::new()), + Outcome::MissingNars(missing) => (Vec::new(), missing), + }; + if !missing.is_empty() { + let err = ApiError::new(StatusCode::CONFLICT, "nar_required").with(json!({ "missing": missing })); + audit.log(&st, &identity, "path.publish", Some(&name), err.status, json!({ "missingNars": missing.len() })); + return Err(err); + } + + let published = rendered_keys.len(); + { + let mut index = st.index.write(); + for key in rendered_keys { + index.insert(key); + } + } + audit.log(&st, &identity, "path.publish", Some(&name), StatusCode::CREATED, json!({ "published": published, "alreadyExisted": already })); + Ok((StatusCode::CREATED, Json(json!({ "published": published, "alreadyExisted": already })))) +} + +enum Outcome { + Published(Vec), + MissingNars(Vec), +} + +struct Blob { + id: i64, + file_hash: [u8; 32], + file_size: u64, + compression: String, + nar_size: u64, +} + +fn render_and_insert(st: &Shared, prepared: Vec) -> ApiResult { + // Resolve blobs on a read connection, outside the writer lock. + let blobs: Vec> = st.db.read(|conn| { + let mut by_nar = conn.prepare_cached( + "SELECT id, file_hash, file_size, compression, nar_size FROM blobs + WHERE nar_hash = ?1 ORDER BY compression = 'zstd' DESC, id LIMIT 1", + )?; + let mut by_file = conn.prepare_cached( + "SELECT id, file_hash, file_size, compression, nar_size FROM blobs WHERE file_hash = ?1 AND nar_hash = ?2", + )?; + let row = |r: &rusqlite::Row<'_>| { + Ok(Blob { + id: r.get(0)?, + file_hash: r.get::<_, Vec>(1)?.try_into().unwrap_or([0; 32]), + file_size: r.get(2)?, + compression: r.get(3)?, + nar_size: r.get(4)?, + }) + }; + prepared + .iter() + .map(|p| match &p.file_hash { + Some(fh) => by_file.query_row(params![&fh[..], &p.nar_hash[..]], row).optional(), + None => by_nar.query_row(params![&p.nar_hash[..]], row).optional(), + }) + .collect() + })?; + + let missing: Vec = prepared.iter().zip(&blobs).filter(|(_, b)| b.is_none()).map(|(p, _)| p.store_path.clone()).collect(); + if !missing.is_empty() { + return Ok(Outcome::MissingNars(missing)); + } + + // Render and sign in parallel; Ed25519 dominates the cost of a publish. + let work: Vec<(Prepared, Blob)> = prepared.into_iter().zip(blobs.into_iter().map(Option::unwrap)).collect(); + let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(work.len().div_ceil(64).max(1)); + let chunk = work.len().div_ceil(threads).max(1); + let rendered: Vec> = std::thread::scope(|scope| { + let handles: Vec<_> = work + .chunks(chunk) + .map(|part| scope.spawn(move || part.iter().map(|(p, b)| render(st, p, b)).collect::>())) + .collect(); + handles.into_iter().flat_map(|h| h.join().expect("render thread panicked")).collect() + }); + let rendered: Vec = rendered.into_iter().collect::>()?; + + let now = crate::db::now(); + let keys = st.db.write(|conn| { + let tx = conn.transaction()?; + let mut keys = Vec::with_capacity(rendered.len()); + { + let mut blob_exists = tx.prepare_cached("SELECT 1 FROM blobs WHERE id = ?1")?; + let mut insert = tx.prepare_cached( + "INSERT INTO paths (cache_id, hash, blob_id, store_path, narinfo, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) ON CONFLICT (cache_id, hash) DO NOTHING", + )?; + for r in &rendered { + // Re-checked under the writer lock: GC may have collected the blob. + if !blob_exists.exists([r.blob_id])? { + return Err(rusqlite::Error::QueryReturnedNoRows); + } + if insert.execute(params![r.key.cache, &r.key.hash[..], r.blob_id, r.store_path, r.narinfo, now])? > 0 { + keys.push(r.key); + } + } + } + tx.commit()?; + Ok(keys) + }); + match keys { + Ok(keys) => Ok(Outcome::Published(keys)), + Err(rusqlite::Error::QueryReturnedNoRows) => Err(ApiError::new(StatusCode::CONFLICT, "blob was garbage collected; retry")), + Err(e) => Err(e.into()), + } +} + +fn render(st: &Shared, p: &Prepared, b: &Blob) -> ApiResult { + if b.nar_size != p.nar_size { + return Err(ApiError::bad_request(format!("{}: narSize {} does not match uploaded NAR ({})", p.store_path, p.nar_size, b.nar_size))); + } + let narinfo = helios_core::render_narinfo( + &NarinfoInput { + store_path: &p.store_path, + nar_hash: &p.nar_hash, + nar_size: b.nar_size, + file_hash: &b.file_hash, + file_size: b.file_size, + compression: &b.compression, + references: &p.references, + deriver: &p.deriver, + system: &p.system, + }, + st.signer.as_ref(), + ) + .map_err(|e| ApiError::bad_request(format!("{}: {e}", p.store_path)))?; + Ok(Rendered { key: p.key, store_path: p.store_path.clone(), blob_id: b.id, narinfo }) +} diff --git a/crates/helios-server/src/read.rs b/crates/helios-server/src/read.rs new file mode 100644 index 0000000..9c0e6e1 --- /dev/null +++ b/crates/helios-server/src/read.rs @@ -0,0 +1,160 @@ +//! The substituter read path: `//nix-cache-info`, +//! `//.narinfo` and `//nar/.nar[.zst]`. +//! URLs are parsed by hand, without allocating. + +use axum::body::Body; +use axum::extract::State; +use axum::http::{HeaderMap, HeaderValue, Method, StatusCode, Uri, header}; +use axum::response::{IntoResponse, Response}; +use bytes::Bytes; +use helios_core::Compression; +use rusqlite::{OptionalExtension, params}; + +use crate::error::{ApiError, ApiResult}; +use crate::state::{PathKey, Shared}; + +const NIX_CACHE_INFO: &str = "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 40\n"; +const IMMUTABLE: HeaderValue = HeaderValue::from_static("public, max-age=31536000, immutable"); +const PRIVATE: HeaderValue = HeaderValue::from_static("private, max-age=3600"); +/// Most narinfo requests are misses (Nix asks every substituter about every +/// path). Let proxies cache them briefly; new paths still appear quickly. +const NEGATIVE: HeaderValue = HeaderValue::from_static("public, max-age=60"); + +enum Route<'a> { + CacheInfo, + Narinfo(&'a str), + Nar(&'a str, Compression), +} + +fn route(rest: &str) -> Option> { + if rest == "nix-cache-info" { + return Some(Route::CacheInfo); + } + if let Some(hash) = rest.strip_suffix(".narinfo") { + return (hash.len() == 32).then_some(Route::Narinfo(hash)); + } + let file = rest.strip_prefix("nar/")?; + if let Some(hash) = file.strip_suffix(".nar.zst") { + return Some(Route::Nar(hash, Compression::Zstd)); + } + file.strip_suffix(".nar").map(|hash| Route::Nar(hash, Compression::None)) +} + +fn not_found(cache_control: HeaderValue) -> Response { + (StatusCode::NOT_FOUND, [(header::CACHE_CONTROL, cache_control)], "not found").into_response() +} + +pub async fn handle(State(st): State, method: Method, uri: Uri, headers: HeaderMap) -> Response { + match serve(st, method, uri, headers).await { + Ok(resp) => resp, + Err(err) => err.into_response(), + } +} + +async fn serve(st: Shared, method: Method, uri: Uri, headers: HeaderMap) -> ApiResult { + if method != Method::GET && method != Method::HEAD { + return Ok((StatusCode::METHOD_NOT_ALLOWED, [(header::ALLOW, "GET, HEAD")]).into_response()); + } + let path = uri.path().trim_start_matches('/'); + let Some((name, rest)) = path.split_once('/') else { + return Ok(not_found(NEGATIVE)); + }; + let Some(route) = route(rest) else { + return Ok(not_found(NEGATIVE)); + }; + let Some(cache) = st.cache(name) else { + return Ok(not_found(NEGATIVE)); + }; + st.authorize_read(&headers, name, cache)?; + let cache_control = if cache.public { IMMUTABLE } else { PRIVATE }; + + match route { + Route::CacheInfo => Ok(([(header::CONTENT_TYPE, "text/x-nix-cache-info")], NIX_CACHE_INFO).into_response()), + Route::Narinfo(hash) => { + let Some(hash) = helios_core::nix32_decode::<20>(hash) else { + return Ok(not_found(NEGATIVE)); + }; + let key = PathKey { cache: cache.id, hash }; + if !st.index.read().contains(&key) { + return Ok(not_found(if cache.public { NEGATIVE } else { PRIVATE })); + } + let body = match st.narinfo.get(&key) { + Some(body) => body, + None => { + let db = st.db.clone(); + let loaded = tokio::task::spawn_blocking(move || { + db.read(|conn| { + conn.prepare_cached("SELECT narinfo FROM paths WHERE cache_id = ?1 AND hash = ?2")? + .query_row(params![key.cache, &key.hash[..]], |r| r.get::<_, Vec>(0)) + .optional() + }) + }) + .await??; + let Some(text) = loaded else { + return Ok(not_found(NEGATIVE)); + }; + let body = Bytes::from(text); + st.narinfo.insert(key, body.clone()); + body + } + }; + Ok(([(header::CONTENT_TYPE, HeaderValue::from_static("text/x-nix-narinfo")), (header::CACHE_CONTROL, cache_control)], body) + .into_response()) + } + Route::Nar(hash, compression) => { + let Some(file_hash) = helios_core::nix32_decode::<32>(hash) else { + return Ok(not_found(NEGATIVE)); + }; + let db = st.db.clone(); + let cache_id = cache.id; + // Content-addressed, but only served from caches that publish it. + let size = tokio::task::spawn_blocking(move || { + db.read(|conn| { + conn.prepare_cached( + "SELECT b.file_size FROM blobs b WHERE b.file_hash = ?1 AND b.compression = ?2 + AND EXISTS (SELECT 1 FROM paths p WHERE p.blob_id = b.id AND p.cache_id = ?3)", + )? + .query_row(params![&file_hash[..], compression.as_str(), cache_id], |r| r.get::<_, u64>(0)) + .optional() + }) + }) + .await??; + let Some(size) = size else { + return Ok(not_found(NEGATIVE)); + }; + nar_response(&st, &file_hash, compression, size, &method, cache_control).await + } + } +} + +async fn nar_response( + st: &Shared, + file_hash: &[u8; 32], + compression: Compression, + size: u64, + method: &Method, + cache_control: HeaderValue, +) -> ApiResult { + let mut builder = Response::builder() + .header(header::CONTENT_TYPE, "application/x-nix-nar") + .header(header::CACHE_CONTROL, cache_control); + + if let Some(prefix) = &st.cfg.accel_redirect { + let name = helios_core::nix32_encode(file_hash); + let location = format!("{prefix}/{}/{name}{}", &name[..2], compression.extension()); + builder = builder.header("x-accel-redirect", location); + return builder.body(Body::empty()).map_err(ApiError::internal); + } + + builder = builder.header(header::CONTENT_LENGTH, size); + if method == Method::HEAD { + return builder.body(Body::empty()).map_err(ApiError::internal); + } + let file = match tokio::fs::File::open(st.nar_path(file_hash, compression)).await { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(not_found(NEGATIVE)), + Err(e) => return Err(ApiError::internal(e)), + }; + let stream = tokio_util::io::ReaderStream::with_capacity(file, 512 * 1024); + builder.body(Body::from_stream(stream)).map_err(ApiError::internal) +} diff --git a/crates/helios-server/src/state.rs b/crates/helios-server/src/state.rs new file mode 100644 index 0000000..4efb8cd --- /dev/null +++ b/crates/helios-server/src/state.rs @@ -0,0 +1,162 @@ +//! Shared server state. The hot read path (narinfo lookups) is served from +//! memory: a presence index answers misses without touching SQLite, and an +//! LRU holds rendered narinfo bodies for hits. + +use std::collections::{HashMap, HashSet}; +use std::hash::{BuildHasherDefault, Hash, Hasher}; +use std::path::PathBuf; +use std::sync::Arc; + +use bytes::Bytes; +use helios_core::{Compression, Signer}; +use parking_lot::RwLock; +use rusqlite::params; +use tokio::sync::mpsc; + +use crate::audit::AuditEvent; +use crate::config::Config; +use crate::db::Db; + +/// A published path: cache id plus the 20-byte decoded store path hash. +#[derive(Clone, Copy, PartialEq, Eq)] +pub struct PathKey { + pub cache: u32, + pub hash: [u8; 20], +} + +impl Hash for PathKey { + fn hash(&self, state: &mut H) { + // Store path hashes are uniformly distributed already. + let mut word = [0u8; 8]; + word.copy_from_slice(&self.hash[..8]); + state.write_u64(u64::from_ne_bytes(word) ^ u64::from(self.cache)); + } +} + +#[derive(Default)] +pub struct PassThroughHasher(u64); + +impl Hasher for PassThroughHasher { + fn finish(&self) -> u64 { + self.0 + } + fn write(&mut self, bytes: &[u8]) { + for &b in bytes { + self.0 = self.0.rotate_left(8) ^ u64::from(b); + } + } + fn write_u64(&mut self, n: u64) { + self.0 = n; + } +} + +pub type PathIndex = HashSet>; + +#[derive(Clone, Copy)] +pub struct CacheInfo { + pub id: u32, + pub public: bool, +} + +#[derive(Clone, Copy)] +pub struct TokenState { + pub expires_at: i64, + pub revoked: bool, +} + +pub struct AppState { + pub cfg: Config, + pub db: Db, + pub signer: Option, + pub caches: RwLock, CacheInfo>>, + pub index: RwLock, + pub narinfo: quick_cache::sync::Cache>, + pub tokens: RwLock, TokenState>>, + pub audit: mpsc::UnboundedSender, +} + +pub type Shared = Arc; + +impl AppState { + pub fn load(cfg: Config, db: Db, signer: Option, audit: mpsc::UnboundedSender) -> anyhow::Result { + let (caches, index, tokens) = db.read(|conn| { + let mut caches = HashMap::new(); + let mut stmt = conn.prepare("SELECT id, name, is_public FROM caches")?; + let mut rows = stmt.query([])?; + while let Some(r) = rows.next()? { + let name: String = r.get(1)?; + caches.insert(name.into_boxed_str(), CacheInfo { id: r.get(0)?, public: r.get::<_, i64>(2)? != 0 }); + } + + let count: usize = conn.query_row("SELECT count(*) FROM paths", [], |r| r.get(0))?; + let mut index = PathIndex::with_capacity_and_hasher(count + count / 4, Default::default()); + let mut stmt = conn.prepare("SELECT cache_id, hash FROM paths")?; + let mut rows = stmt.query([])?; + while let Some(r) = rows.next()? { + let hash = r.get_ref(1)?.as_blob()?; + if let Ok(hash) = <[u8; 20]>::try_from(hash) { + index.insert(PathKey { cache: r.get(0)?, hash }); + } + } + + let mut tokens = HashMap::new(); + let mut stmt = conn.prepare("SELECT jti, expires_at, revoked_at IS NOT NULL FROM tokens")?; + let mut rows = stmt.query([])?; + while let Some(r) = rows.next()? { + let jti: String = r.get(0)?; + tokens.insert(jti.into_boxed_str(), TokenState { expires_at: r.get(1)?, revoked: r.get(2)? }); + } + Ok((caches, index, tokens)) + })?; + + tracing::info!(caches = caches.len(), paths = index.len(), tokens = tokens.len(), "loaded state"); + let narinfo = quick_cache::sync::Cache::with( + cfg.narinfo_cache_entries, + cfg.narinfo_cache_entries as u64, + quick_cache::UnitWeighter, + Default::default(), + quick_cache::sync::DefaultLifecycle::default(), + ); + Ok(Self { + cfg, + db, + signer, + caches: RwLock::new(caches), + index: RwLock::new(index), + narinfo, + tokens: RwLock::new(tokens), + audit, + }) + } + + pub fn cache(&self, name: &str) -> Option { + self.caches.read().get(name).copied() + } + + pub fn nar_path(&self, file_hash: &[u8; 32], compression: Compression) -> PathBuf { + let name = helios_core::nix32_encode(file_hash); + self.cfg.data_dir.join("nar").join(&name[..2]).join(format!("{name}{}", compression.extension())) + } + + pub fn tmp_dir(&self) -> PathBuf { + self.cfg.data_dir.join("tmp") + } + + pub fn insert_cache(&self, name: &str, public: bool) -> rusqlite::Result> { + let id = self.db.write(|conn| -> rusqlite::Result> { + conn.execute( + "INSERT INTO caches (name, is_public, created_at) VALUES (?1, ?2, ?3) ON CONFLICT (name) DO NOTHING", + params![name, public, crate::db::now()], + )?; + if conn.changes() == 0 { + return Ok(None); + } + Ok(Some(conn.last_insert_rowid() as u32)) + })?; + Ok(id.map(|id| { + let info = CacheInfo { id, public }; + self.caches.write().insert(name.into(), info); + info + })) + } +} From 4fb8dd98783eb6cc17d11b2ae0fa9fa92f5db6a2 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 12:23:53 +0200 Subject: [PATCH 04/56] cli: add helios cli and end-to-end test The CLI pushes store paths and closures and manages caches and tokens. Each NAR is serialised, compressed and hashed by libhelios and streamed straight into the upload request; the TypeScript CLI spawned nix store dump-path and zstd for every path and wrote a temporary file. Path metadata comes from a single nix path-info call, every NAR hash is checked against the Nix database, and paths are published dependencies first. A 1,350-path system closure pushes in 3.3 seconds. scripts/e2e.sh starts a server, pushes a closure, and has Nix substitute it back into a fresh store with signatures required. --- Cargo.lock | 59 ++++++--- crates/helios-cli/Cargo.toml | 24 ++++ crates/helios-cli/src/api.rs | 133 +++++++++++++++++++++ crates/helios-cli/src/config.rs | 61 ++++++++++ crates/helios-cli/src/main.rs | 125 +++++++++++++++++++ crates/helios-cli/src/nix.rs | 169 ++++++++++++++++++++++++++ crates/helios-cli/src/push.rs | 204 ++++++++++++++++++++++++++++++++ scripts/e2e.sh | 107 +++++++++++++++++ 8 files changed, 868 insertions(+), 14 deletions(-) create mode 100644 crates/helios-cli/Cargo.toml create mode 100644 crates/helios-cli/src/api.rs create mode 100644 crates/helios-cli/src/config.rs create mode 100644 crates/helios-cli/src/main.rs create mode 100644 crates/helios-cli/src/nix.rs create mode 100644 crates/helios-cli/src/push.rs create mode 100755 scripts/e2e.sh diff --git a/Cargo.lock b/Cargo.lock index b397669..0451ef0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -569,6 +569,24 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "helios-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64", + "bytes", + "clap", + "futures-util", + "helios-core", + "mimalloc", + "reqwest", + "serde", + "serde_json", + "tokio", + "tokio-stream", +] + [[package]] name = "helios-core" version = "0.1.0" @@ -923,9 +941,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.105" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ "cfg-if", "futures-util", @@ -1276,6 +1294,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", + "h2", "http", "http-body", "http-body-util", @@ -1820,6 +1839,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.19" @@ -2038,9 +2068,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasm-bindgen" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", @@ -2051,19 +2081,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.78" +version = "0.4.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef4c5d3d2cdf5c54f4231181768f5510842e350db025faf1f7163b1030ed928" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" dependencies = [ "js-sys", + "tokio", "wasm-bindgen", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2071,9 +2102,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ "bumpalo", "proc-macro2", @@ -2084,9 +2115,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.128" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] @@ -2106,9 +2137,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.105" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbddc4a036f00ec4f18c83445bd3115cb306a91da554919a099d9222fe4a7f8" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/crates/helios-cli/Cargo.toml b/crates/helios-cli/Cargo.toml new file mode 100644 index 0000000..e4742a0 --- /dev/null +++ b/crates/helios-cli/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "helios-cli" +version.workspace = true +edition.workspace = true +license-file.workspace = true +publish.workspace = true + +[dependencies] +anyhow = "1.0.104" +base64 = "0.23.1" +bytes = "1.12.1" +clap = { version = "4.6.7", features = ["derive", "env"] } +futures-util = "0.3.34" +helios-core = { path = "../helios-core" } +mimalloc = "0.1.52" +reqwest = { version = "0.13.5", default-features = false, features = ["json", "stream", "rustls", "http2"] } +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.151" +tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "sync", "process", "io-util"] } +tokio-stream = "0.1.19" + +[[bin]] +name = "helios" +path = "src/main.rs" diff --git a/crates/helios-cli/src/api.rs b/crates/helios-cli/src/api.rs new file mode 100644 index 0000000..d4af4d9 --- /dev/null +++ b/crates/helios-cli/src/api.rs @@ -0,0 +1,133 @@ +use anyhow::{Context, bail}; +use reqwest::{Body, Response, StatusCode}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::config::Server; + +#[derive(Clone)] +pub struct Client { + http: reqwest::Client, + base: String, + token: String, +} + +#[derive(Serialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct PathSpec { + pub store_path: String, + pub nar_hash: String, + pub nar_size: u64, + pub references: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub deriver: Option, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +pub struct Uploaded { + pub file_hash: String, + pub file_size: u64, + pub nar_hash: String, + pub nar_size: u64, +} + +pub enum Publish { + Done { published: u64 }, + MissingNars(Vec), +} + +async fn error_text(resp: Response) -> String { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + let msg = serde_json::from_str::(&body) + .ok() + .and_then(|v| v.get("error").and_then(Value::as_str).map(str::to_owned)) + .unwrap_or(body); + format!("{status}: {msg}") +} + +impl Client { + pub fn new(server: &Server) -> anyhow::Result { + let http = reqwest::Client::builder() + .user_agent(concat!("helios-cli/", env!("CARGO_PKG_VERSION"))) + .tcp_nodelay(true) + .pool_max_idle_per_host(64) + .build()?; + Ok(Self { http, base: server.server.trim_end_matches('/').to_owned(), token: server.token.clone() }) + } + + fn url(&self, path: &str) -> String { + format!("{}/_api/v2{path}", self.base) + } + + async fn json(&self, req: reqwest::RequestBuilder) -> anyhow::Result { + let resp = req.bearer_auth(&self.token).send().await?; + if !resp.status().is_success() { + bail!("{}", error_text(resp).await); + } + Ok(resp.json().await?) + } + + pub async fn missing(&self, cache: &str, hashes: &[&str]) -> anyhow::Result> { + #[derive(Deserialize)] + struct R { + missing: Vec, + } + let r: R = self.json(self.http.post(self.url(&format!("/caches/{cache}/missing"))).json(&json!({ "hashes": hashes }))).await?; + Ok(r.missing) + } + + pub async fn known(&self, cache: &str, nar_hashes: &[&str]) -> anyhow::Result> { + #[derive(Deserialize)] + struct R { + known: Vec, + } + let r: R = self + .json(self.http.post(self.url(&format!("/caches/{cache}/nars/known"))).json(&json!({ "narHashes": nar_hashes }))) + .await?; + Ok(r.known) + } + + pub async fn upload(&self, cache: &str, body: Body) -> anyhow::Result { + self.json( + self.http + .put(self.url(&format!("/caches/{cache}/nar?compression=zstd"))) + .header("content-type", "application/x-nix-nar") + .body(body), + ) + .await + } + + pub async fn publish(&self, cache: &str, paths: &[PathSpec]) -> anyhow::Result { + let resp = self + .http + .post(self.url(&format!("/caches/{cache}/paths"))) + .bearer_auth(&self.token) + .json(&json!({ "paths": paths })) + .send() + .await?; + if resp.status() == StatusCode::CONFLICT { + let v: Value = resp.json().await?; + if v["error"] == "nar_required" { + let missing = v["missing"].as_array().context("missing list")?.iter().filter_map(|m| m.as_str().map(str::to_owned)).collect(); + return Ok(Publish::MissingNars(missing)); + } + bail!("publish conflict: {}", v["error"]); + } + if !resp.status().is_success() { + bail!("{}", error_text(resp).await); + } + let v: Value = resp.json().await?; + Ok(Publish::Done { published: v["published"].as_u64().unwrap_or(0) }) + } + + pub async fn admin_get(&self, path: &str) -> anyhow::Result { + self.json(self.http.get(self.url(path))).await + } + + pub async fn admin_post(&self, path: &str, body: Value) -> anyhow::Result { + self.json(self.http.post(self.url(path)).json(&body)).await + } +} diff --git a/crates/helios-cli/src/config.rs b/crates/helios-cli/src/config.rs new file mode 100644 index 0000000..41e9b4b --- /dev/null +++ b/crates/helios-cli/src/config.rs @@ -0,0 +1,61 @@ +//! `~/.config/helios/config.json`. + +use std::collections::BTreeMap; +use std::path::PathBuf; + +use anyhow::{Context, bail}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Clone)] +pub struct Server { + pub server: String, + pub token: String, +} + +#[derive(Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub struct Config { + #[serde(skip_serializing_if = "Option::is_none")] + pub default_server: Option, + #[serde(default)] + pub servers: BTreeMap, +} + +fn path() -> anyhow::Result { + let base = match std::env::var_os("XDG_CONFIG_HOME") { + Some(dir) if !dir.is_empty() => PathBuf::from(dir), + _ => PathBuf::from(std::env::var_os("HOME").context("HOME is not set")?).join(".config"), + }; + Ok(base.join("helios").join("config.json")) +} + +pub fn load() -> anyhow::Result { + match std::fs::read(path()?) { + Ok(raw) => Ok(serde_json::from_slice(&raw).context("parsing helios config")?), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Config::default()), + Err(e) => Err(e.into()), + } +} + +pub fn login(name: &str, server: &str, token: &str) -> anyhow::Result<()> { + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; + + let mut cfg = load()?; + cfg.servers.insert(name.into(), Server { server: server.trim_end_matches('/').into(), token: token.into() }); + cfg.default_server = Some(name.into()); + let path = path()?; + std::fs::create_dir_all(path.parent().expect("config path has a parent"))?; + let mut file = std::fs::OpenOptions::new().write(true).create(true).truncate(true).mode(0o600).open(&path)?; + file.write_all(&serde_json::to_vec_pretty(&cfg)?)?; + file.write_all(b"\n")?; + Ok(()) +} + +pub fn server(name: Option<&str>) -> anyhow::Result { + let cfg = load()?; + let Some(key) = name.map(str::to_owned).or(cfg.default_server) else { + bail!("no server configured; run: helios login "); + }; + cfg.servers.get(&key).cloned().with_context(|| format!("server '{key}' not found; run: helios login {key} ")) +} diff --git a/crates/helios-cli/src/main.rs b/crates/helios-cli/src/main.rs new file mode 100644 index 0000000..4c7aa9e --- /dev/null +++ b/crates/helios-cli/src/main.rs @@ -0,0 +1,125 @@ +mod api; +mod config; +mod nix; +mod push; + +use clap::{Parser, Subcommand}; +use serde_json::{Value, json}; + +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + +/// Helios Nix binary cache CLI. +#[derive(Parser)] +#[command(version)] +struct Cli { + /// Server name from `helios login` (default: the last one logged in). + #[arg(long, global = true, env = "HELIOS_SERVER")] + server: Option, + + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Save server credentials (a push token, or the admin secret for admin commands). + Login { name: String, url: String, token: String }, + /// Push store paths (or installables like `.#foo`) to a cache. + Push { + cache: String, + #[arg(required = true)] + installables: Vec, + /// Push the full closure. + #[arg(long, short = 'r')] + closure: bool, + /// Parallel uploads. + #[arg(long, short = 'j', default_value_t = 8)] + jobs: usize, + /// zstd compression level (1-19). + #[arg(long, default_value_t = 3, value_parser = clap::value_parser!(i32).range(1..=19))] + level: i32, + }, + /// Manage caches (admin). + #[command(subcommand)] + Cache(CacheCmd), + /// Manage API tokens (admin). + #[command(subcommand)] + Token(TokenCmd), +} + +#[derive(Subcommand)] +enum CacheCmd { + Create { + name: String, + /// Require a pull token to read. + #[arg(long)] + private: bool, + }, + List, +} + +#[derive(Subcommand)] +enum TokenCmd { + Create { + subject: String, + /// Comma-separated cache names, or "*". + #[arg(long, default_value = "*")] + caches: String, + /// Comma-separated: push, pull. + #[arg(long, default_value = "push")] + perms: String, + /// Lifetime in days (1-365). + #[arg(long, default_value_t = 90)] + expires: i64, + }, + List, + Revoke { jti: String, reason: String }, +} + +fn split(s: &str) -> Vec { + s.split(',').map(str::trim).filter(|s| !s.is_empty()).map(str::to_owned).collect() +} + +fn print(v: &Value) { + println!("{}", serde_json::to_string_pretty(v).unwrap_or_default()); +} + +async fn run(cli: Cli) -> anyhow::Result<()> { + if let Command::Login { name, url, token } = &cli.command { + config::login(name, url, token)?; + println!("logged in to '{name}' at {url}"); + return Ok(()); + } + let client = api::Client::new(&config::server(cli.server.as_deref())?)?; + match cli.command { + Command::Login { .. } => unreachable!(), + Command::Push { cache, installables, closure, jobs, level } => { + push::push(&client, &cache, &installables, push::Options { jobs, level, closure }).await?; + } + Command::Cache(CacheCmd::Create { name, private }) => { + print(&client.admin_post("/admin/caches", json!({ "name": name, "public": !private })).await?); + } + Command::Cache(CacheCmd::List) => print(&client.admin_get("/admin/caches").await?), + Command::Token(TokenCmd::Create { subject, caches, perms, expires }) => { + let v = client + .admin_post("/admin/tokens", json!({ "subject": subject, "caches": split(&caches), "perms": split(&perms), "expiresInDays": expires })) + .await?; + eprintln!("store this token now; it is not shown again"); + print(&v); + } + Command::Token(TokenCmd::List) => print(&client.admin_get("/admin/tokens").await?), + Command::Token(TokenCmd::Revoke { jti, reason }) => { + print(&client.admin_post(&format!("/admin/tokens/{jti}/revoke"), json!({ "reason": reason })).await?); + } + } + Ok(()) +} + +#[tokio::main] +async fn main() { + if let Err(e) = run(Cli::parse()).await { + eprintln!("error: {e:#}"); + std::process::exit(1); + } +} diff --git a/crates/helios-cli/src/nix.rs b/crates/helios-cli/src/nix.rs new file mode 100644 index 0000000..553ec4a --- /dev/null +++ b/crates/helios-cli/src/nix.rs @@ -0,0 +1,169 @@ +//! Store path metadata from `nix path-info`: one process for the whole set. + +use std::collections::BTreeMap; + +use anyhow::{Context, bail}; +use serde::Deserialize; +use tokio::process::Command; + +#[derive(Debug, Clone)] +pub struct PathInfo { + pub path: String, + pub hash: String, + pub nar_hash: String, + pub nar_size: u64, + /// Full store paths. + pub references: Vec, + pub deriver: Option, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct RawInfo { + #[serde(default)] + path: Option, + nar_hash: String, + nar_size: u64, + #[serde(default)] + references: Vec, + #[serde(default)] + deriver: Option, +} + +const STORE_DIR: &str = "/nix/store/"; + +fn full(p: &str) -> String { + if p.starts_with('/') { p.to_owned() } else { format!("{STORE_DIR}{p}") } +} + +async fn run(args: &[&str], installables: &[String]) -> anyhow::Result { + Ok(Command::new("nix") + .args(["--extra-experimental-features", "nix-command flakes", "path-info"]) + .args(args) + .arg("--") + .args(installables) + .stderr(std::process::Stdio::piped()) + .output() + .await + .context("running nix path-info (is nix on PATH?)")?) +} + +pub async fn path_infos(installables: &[String], closure: bool) -> anyhow::Result> { + let mut args = vec!["--json", "--json-format", "1"]; + if closure { + args.push("--recursive"); + } + let mut out = run(&args, installables).await?; + if !out.status.success() && String::from_utf8_lossy(&out.stderr).contains("json-format") { + // Older Nix without --json-format. + args.retain(|a| *a != "--json-format" && *a != "1"); + out = run(&args, installables).await?; + } + if !out.status.success() { + bail!("nix path-info failed: {}", String::from_utf8_lossy(&out.stderr).trim()); + } + parse(&out.stdout) +} + +fn parse(stdout: &[u8]) -> anyhow::Result> { + let value: serde_json::Value = serde_json::from_slice(stdout).context("parsing nix path-info output")?; + let mut raw: BTreeMap = BTreeMap::new(); + match value { + serde_json::Value::Object(map) => { + for (path, info) in map { + if info.is_null() { + bail!("{path} is not a valid store path"); + } + raw.insert(full(&path), serde_json::from_value(info)?); + } + } + serde_json::Value::Array(items) => { + for item in items { + let info: RawInfo = serde_json::from_value(item)?; + let path = info.path.clone().context("path-info entry without a path")?; + raw.insert(full(&path), info); + } + } + _ => bail!("unexpected nix path-info output"), + } + raw.into_iter() + .map(|(path, info)| { + let base = path.strip_prefix(STORE_DIR).with_context(|| format!("{path} is not in /nix/store"))?; + let hash = base.get(..32).with_context(|| format!("{path} has no store hash"))?.to_owned(); + Ok(PathInfo { + hash, + nar_hash: info.nar_hash, + nar_size: info.nar_size, + references: info.references.iter().map(|r| full(r)).collect(), + deriver: info.deriver.map(|d| full(&d)), + path, + }) + }) + .collect() +} + +/// Dependencies before dependents, so a partially pushed closure never +/// exposes a path whose references are missing. +pub fn topo_order(infos: Vec) -> Vec { + use std::collections::HashMap; + let index: HashMap = infos.iter().enumerate().map(|(i, p)| (p.path.clone(), i)).collect(); + let mut pending: Vec = vec![0; infos.len()]; + let mut dependents: Vec> = vec![Vec::new(); infos.len()]; + for (i, info) in infos.iter().enumerate() { + for r in &info.references { + if let Some(&j) = index.get(r) + && j != i + { + pending[i] += 1; + dependents[j].push(i); + } + } + } + let mut ready: Vec = (0..infos.len()).filter(|&i| pending[i] == 0).collect(); + let mut order = Vec::with_capacity(infos.len()); + while let Some(i) = ready.pop() { + order.push(i); + for &d in &dependents[i] { + pending[d] -= 1; + if pending[d] == 0 { + ready.push(d); + } + } + } + // Cycles cannot occur in a valid store; keep anything left over anyway. + let placed: std::collections::HashSet = order.iter().copied().collect(); + order.extend((0..infos.len()).filter(|i| !placed.contains(i))); + let mut slots: Vec> = infos.into_iter().map(Some).collect(); + order.into_iter().map(|i| slots[i].take().expect("each index placed once")).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn info(name: &str, refs: &[&str]) -> PathInfo { + PathInfo { + path: format!("{STORE_DIR}{name}"), + hash: String::new(), + nar_hash: String::new(), + nar_size: 0, + references: refs.iter().map(|r| format!("{STORE_DIR}{r}")).collect(), + deriver: None, + } + } + + #[test] + fn dependencies_come_first() { + let order = topo_order(vec![info("app", &["lib", "app"]), info("lib", &["libc"]), info("libc", &[])]); + let names: Vec<_> = order.iter().map(|p| p.path.trim_start_matches(STORE_DIR)).collect(); + assert_eq!(names, ["libc", "lib", "app"]); + } + + #[test] + fn parses_both_json_formats() { + let obj = br#"{"/nix/store/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-x":{"narHash":"sha256-AAAA","narSize":8,"references":[],"deriver":null}}"#; + assert_eq!(parse(obj).unwrap()[0].hash, "a".repeat(32)); + let arr = br#"[{"path":"/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-y","narHash":"sha256:x","narSize":8,"references":["/nix/store/bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb-y"]}]"#; + assert_eq!(parse(arr).unwrap()[0].references.len(), 1); + } +} diff --git a/crates/helios-cli/src/push.rs b/crates/helios-cli/src/push.rs new file mode 100644 index 0000000..1a6e008 --- /dev/null +++ b/crates/helios-cli/src/push.rs @@ -0,0 +1,204 @@ +//! `helios push`: NARs are serialised, compressed and hashed by libhelios +//! and streamed straight into the upload request, with no temp files and no +//! `nix store dump-path | zstd` subprocesses. + +use std::collections::HashSet; +use std::path::Path; +use std::time::Instant; + +use anyhow::{Context, bail}; +use base64::Engine; +use bytes::Bytes; +use futures_util::StreamExt; +use helios_core::{DumpOptions, HL_E_UNSUPPORTED_OS}; +use tokio::sync::mpsc; +use tokio_stream::wrappers::ReceiverStream; + +use crate::api::{Client, PathSpec, Publish}; +use crate::nix::{self, PathInfo}; + +const QUERY_BATCH: usize = 50_000; +const PUBLISH_BATCH: usize = 1_000; + +pub struct Options { + pub jobs: usize, + pub level: i32, + pub closure: bool, +} + +fn parse_sha256(s: &str) -> Option<[u8; 32]> { + if let Some(b64) = s.strip_prefix("sha256-") { + return base64::engine::general_purpose::STANDARD.decode(b64).ok()?.try_into().ok(); + } + helios_core::nix32_decode::<32>(s.strip_prefix("sha256:")?) +} + +fn name(path: &str) -> &str { + path.strip_prefix("/nix/store/").and_then(|b| b.get(33..)).unwrap_or(path) +} + +fn human(bytes: u64) -> String { + const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; + let mut v = bytes as f64; + let mut unit = 0; + while v >= 1024.0 && unit < UNITS.len() - 1 { + v /= 1024.0; + unit += 1; + } + if unit == 0 { format!("{bytes} B") } else { format!("{v:.1} {}", UNITS[unit]) } +} + +/// Produce the compressed NAR on a blocking thread, feeding `tx`. +fn produce(path: &str, opts: DumpOptions, tx: mpsc::Sender>) -> anyhow::Result { + let send = |chunk: &[u8]| tx.blocking_send(Ok(Bytes::copy_from_slice(chunk))).is_ok(); + match helios_core::dump_nar(Path::new(path), &opts, send) { + Err(e) if e.code == HL_E_UNSUPPORTED_OS => {} + other => return other.with_context(|| format!("serialising {path}")), + } + // Non-Linux: let Nix serialise, libhelios compresses and hashes. + use std::io::Read; + let mut child = std::process::Command::new("nix-store") + .args(["--dump", path]) + .stdout(std::process::Stdio::piped()) + .spawn() + .context("running nix-store --dump")?; + let mut stdout = child.stdout.take().context("nix-store stdout")?; + let mut compressor = helios_core::Compressor::new(&opts, send)?; + let mut buf = vec![0u8; 1 << 20]; + loop { + let n = stdout.read(&mut buf)?; + if n == 0 { + break; + } + compressor.update(&buf[..n])?; + } + if !child.wait()?.success() { + bail!("nix-store --dump {path} failed"); + } + Ok(compressor.finish()?) +} + +async fn upload_one(client: &Client, cache: &str, info: &PathInfo, opts: DumpOptions) -> anyhow::Result { + let (tx, rx) = mpsc::channel(8); + let path = info.path.clone(); + let producer = tokio::task::spawn_blocking(move || produce(&path, opts, tx)); + let body = reqwest::Body::wrap_stream(ReceiverStream::new(rx)); + let uploaded = client.upload(cache, body).await; + let digest = producer.await?; + // An upload failure aborts the producer; report the upload's error. + let uploaded = uploaded?; + let digest = digest?; + + let ours = format!("sha256:{}", helios_core::nix32_encode(&digest.nar_hash)); + if parse_sha256(&info.nar_hash) != Some(digest.nar_hash) { + bail!("{}: NAR hash {ours} differs from the Nix database ({})", info.path, info.nar_hash); + } + if uploaded.nar_hash != ours || uploaded.file_hash != helios_core::nix32_encode(&digest.file_hash) || uploaded.nar_size != digest.nar_size { + bail!("{}: server verified different content than was sent", info.path); + } + Ok(uploaded.file_size) +} + +async fn upload_all(client: &Client, cache: &str, paths: &[&PathInfo], opts: &Options) -> anyhow::Result { + let total = paths.len(); + let threads = (std::thread::available_parallelism().map_or(4, |n| n.get()) / opts.jobs.max(1)).max(1) as i32; + let mut done = 0usize; + let mut bytes = 0u64; + let mut failures = Vec::new(); + let mut results = futures_util::stream::iter(paths.iter().map(|info| async move { + let dump = DumpOptions { level: opts.level, threads, size_hint: info.nar_size }; + let started = Instant::now(); + (info, upload_one(client, cache, info, dump).await, started.elapsed()) + })) + .buffer_unordered(opts.jobs.max(1)); + while let Some((info, result, took)) = results.next().await { + done += 1; + match result { + Ok(size) => { + bytes += size; + eprintln!("[{done}/{total}] {} {} → {} ({:.2}s)", name(&info.path), human(info.nar_size), human(size), took.as_secs_f64()); + } + Err(e) => { + eprintln!("[{done}/{total}] {} FAILED: {e:#}", name(&info.path)); + failures.push(info.path.clone()); + } + } + } + if !failures.is_empty() { + bail!("{} of {total} uploads failed", failures.len()); + } + Ok(bytes) +} + +pub async fn push(client: &Client, cache: &str, installables: &[String], opts: Options) -> anyhow::Result<()> { + let started = Instant::now(); + let infos = nix::path_infos(installables, opts.closure).await?; + let total = infos.len(); + + let mut missing: HashSet = HashSet::new(); + let hashes: Vec<&str> = infos.iter().map(|i| i.hash.as_str()).collect(); + for chunk in hashes.chunks(QUERY_BATCH) { + missing.extend(client.missing(cache, chunk).await?); + } + let todo: Vec = infos.into_iter().filter(|i| missing.contains(&i.hash)).collect(); + if todo.is_empty() { + eprintln!("all {total} paths already in '{cache}'"); + return Ok(()); + } + + // Paths whose NAR the server already holds (from any cache) skip the upload. + let mut known: HashSet = HashSet::new(); + let nar_hashes: Vec<&str> = todo.iter().map(|i| i.nar_hash.as_str()).collect(); + for chunk in nar_hashes.chunks(QUERY_BATCH) { + known.extend(client.known(cache, chunk).await?); + } + let uploads: Vec<&PathInfo> = todo.iter().filter(|i| !known.contains(&i.nar_hash)).collect(); + eprintln!( + "{total} paths, {} missing from '{cache}': uploading {}, reusing {} NARs", + todo.len(), + uploads.len(), + todo.len() - uploads.len() + ); + let mut uploaded_bytes = upload_all(client, cache, &uploads, &opts).await?; + + let ordered = nix::topo_order(todo); + let mut published = 0u64; + for batch in ordered.chunks(PUBLISH_BATCH) { + let specs: Vec = batch + .iter() + .map(|i| PathSpec { + store_path: i.path.clone(), + nar_hash: i.nar_hash.clone(), + nar_size: i.nar_size, + references: i.references.clone(), + deriver: i.deriver.clone(), + }) + .collect(); + let mut retried = false; + loop { + match client.publish(cache, &specs).await? { + Publish::Done { published: n } => { + published += n; + break; + } + // A NAR vanished between the check and the publish (e.g. GC); upload and retry once. + Publish::MissingNars(paths) if !retried => { + retried = true; + let wanted: HashSet<&str> = paths.iter().map(String::as_str).collect(); + let redo: Vec<&PathInfo> = batch.iter().filter(|i| wanted.contains(i.path.as_str())).collect(); + uploaded_bytes += upload_all(client, cache, &redo, &opts).await?; + } + Publish::MissingNars(paths) => bail!("server still lacks NARs for {} paths", paths.len()), + } + } + } + + let secs = started.elapsed().as_secs_f64(); + eprintln!( + "published {published} paths to '{cache}', uploaded {} in {secs:.2}s ({}/s)", + human(uploaded_bytes), + human((uploaded_bytes as f64 / secs.max(1e-3)) as u64) + ); + Ok(()) +} + diff --git a/scripts/e2e.sh b/scripts/e2e.sh new file mode 100755 index 0000000..0d78223 --- /dev/null +++ b/scripts/e2e.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# End-to-end test: start helios-server on a scratch data dir, push a real +# closure with the CLI, then have Nix substitute it back into a fresh store +# with signature checking on. +# +# Usage: scripts/e2e.sh [store-path] (run inside `nix develop`, after `cargo build --release`) +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +BIN="${BIN:-$ROOT/target/release}" +PORT="${PORT:-18080}" +URL="http://127.0.0.1:$PORT" +TARGET="${1:-$(readlink -f "$(command -v zstd)" | cut -d/ -f1-4)}" + +WORK="$(mktemp -d)" +cleanup() { + [ -n "${SERVER_PID:-}" ] && kill "$SERVER_PID" 2>/dev/null || true + chmod -R u+w "$WORK" 2>/dev/null || true + rm -rf "$WORK" +} +trap cleanup EXIT + +PASS=0 +FAIL=0 +check() { + if [ "$2" = "$3" ]; then + echo " ok: $1" + PASS=$((PASS + 1)) + else + echo " FAIL: $1 (expected=$2 got=$3)" + FAIL=$((FAIL + 1)) + fi +} +status() { curl -s -o /dev/null -w '%{http_code}' "$@"; } + +nix key generate-secret --key-name e2e-1 >"$WORK/key" +head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' >"$WORK/jwt" +head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' >"$WORK/admin" +PUBKEY="$("$BIN/helios-server" --print-public-key --signing-key-file "$WORK/key")" + +RUST_LOG=helios_server=warn "$BIN/helios-server" \ + --listen "127.0.0.1:$PORT" \ + --data-dir "$WORK/data" \ + --signing-key-file "$WORK/key" \ + --jwt-secret-file "$WORK/jwt" \ + --admin-secret-file "$WORK/admin" & +SERVER_PID=$! +for _ in $(seq 50); do curl -sf "$URL/healthz" >/dev/null && break; sleep 0.1; done + +export XDG_CONFIG_HOME="$WORK/config" +# Nix caches narinfo per substituter URL; keep runs (and their keys) apart. +export XDG_CACHE_HOME="$WORK/cache" +helios() { "$BIN/helios" "$@"; } + +echo "=== admin" +helios login admin "$URL" "$(cat "$WORK/admin")" >/dev/null +helios cache create main >/dev/null +helios cache create secret --private >/dev/null +PUSH_TOKEN="$(helios token create ci --caches main,secret --perms push,pull 2>/dev/null | jq -r .token)" +PULL_ONLY="$(helios token create reader --caches secret --perms pull 2>/dev/null | jq -r .token)" +check "token issued" true "$([ -n "$PUSH_TOKEN" ] && echo true)" +check "admin rejects push token" 403 "$(status -H "authorization: Bearer $PUSH_TOKEN" "$URL/_api/v2/admin/tokens")" + +echo "=== push $TARGET" +helios login ci "$URL" "$PUSH_TOKEN" >/dev/null +helios push main --closure "$TARGET" +CLOSURE_SIZE="$(nix-store -qR "$TARGET" | wc -l)" +AGAIN="$(helios push main --closure "$TARGET" 2>&1)" +check "second push is a no-op" "all $CLOSURE_SIZE paths already in 'main'" "$AGAIN" + +echo "=== read path" +HASH="$(basename "$TARGET" | cut -c1-32)" +check "nix-cache-info" 200 "$(status "$URL/main/nix-cache-info")" +check "narinfo hit" 200 "$(status "$URL/main/$HASH.narinfo")" +check "narinfo HEAD" 200 "$(status -I "$URL/main/$HASH.narinfo")" +check "narinfo miss" 404 "$(status "$URL/main/00000000000000000000000000000000.narinfo")" +check "unknown cache" 404 "$(status "$URL/nope/$HASH.narinfo")" +NAR_URL="$(curl -s "$URL/main/$HASH.narinfo" | sed -n 's/^URL: //p')" +check "nar download" 200 "$(status "$URL/main/$NAR_URL")" +check "nar not served from a cache that lacks it" 404 "$(status -H "authorization: Bearer $PUSH_TOKEN" "$URL/secret/$NAR_URL")" + +echo "=== substitute into a fresh store (signatures required)" +NIX_CONFIG="trusted-public-keys = $PUBKEY +require-sigs = true" nix copy --from "$URL/main" --to "$WORK/store" "$TARGET" 2>&1 | tail -3 +check "substituted closure is valid" "$CLOSURE_SIZE" "$(nix-store --store "$WORK/store" -qR "$TARGET" | wc -l)" +nix store verify --store "$WORK/store" --trusted-public-keys "$PUBKEY" -r "$TARGET" && check "nix store verify" 0 0 + +echo "=== private cache" +helios push secret "$TARGET" >/dev/null 2>&1 +check "private narinfo without auth" 401 "$(status "$URL/secret/$HASH.narinfo")" +check "private narinfo with netrc-style basic auth" 200 "$(status -u "reader:$PULL_ONLY" "$URL/secret/$HASH.narinfo")" +check "pull-only token cannot push" 403 "$(status -X POST -H "authorization: Bearer $PULL_ONLY" -H 'content-type: application/json' -d '{"hashes":[]}' "$URL/_api/v2/caches/secret/missing")" + +echo "=== abuse" +check "garbage upload rejected" 400 "$(status -X PUT -H "authorization: Bearer $PUSH_TOKEN" --data-binary 'not a zstd stream' "$URL/_api/v2/caches/main/nar")" +check "raw non-NAR upload rejected" 400 "$(status -X PUT -H "authorization: Bearer $PUSH_TOKEN" --data-binary 'hello' "$URL/_api/v2/caches/main/nar?compression=none")" +INJECT='{"paths":[{"storePath":"/nix/store/'"$HASH"'-x\nSig: evil","narHash":"sha256:'"$(printf 0%.0s $(seq 52))"'","narSize":1}]}' +check "narinfo injection rejected" 400 "$(status -X POST -H "authorization: Bearer $PUSH_TOKEN" -H 'content-type: application/json' -d "$INJECT" "$URL/_api/v2/caches/main/paths")" +FAKE='{"paths":[{"storePath":"/nix/store/00000000000000000000000000000000-fake","narHash":"sha256:'"$(printf 0%.0s $(seq 52))"'","narSize":1}]}' +check "publish without uploaded NAR" 409 "$(status -X POST -H "authorization: Bearer $PUSH_TOKEN" -H 'content-type: application/json' -d "$FAKE" "$URL/_api/v2/caches/main/paths")" +JTI="$(helios --server admin token list | jq -r '.tokens[] | select(.subject=="reader") | .jti')" +helios --server admin token revoke "$JTI" "e2e" >/dev/null +check "revocation is immediate" 401 "$(status -u "reader:$PULL_ONLY" "$URL/secret/$HASH.narinfo")" + +echo +echo "passed=$PASS failed=$FAIL" +[ "$FAIL" -eq 0 ] From 81aecb5c0c3fbd1bff2e4ffcbf093ff56b0c554f Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 12:23:53 +0200 Subject: [PATCH 05/56] bench: compare pkg against reference libraries and nix bench/run.sh times the pkg packages against the Haskell Nix libraries (nix-narinfo, nix-derivation, hnix-store-core and hnix-store-nar) and against Nix's C++ NAR serialiser, on a corpus built from the local store. bench/flake-lock.sh first checks that pkg/nix-flake-lock produces the same canonical output and validation results as the nix-flake-lock crate on fixtures, real lock files and mutated documents, then times both. All comparisons run single-threaded. --- bench/flake-lock-rs/Cargo.lock | 31 ++++ bench/flake-lock-rs/Cargo.toml | 15 ++ bench/flake-lock-rs/src/main.rs | 180 ++++++++++++++++++++ bench/flake-lock.sh | 55 ++++++ bench/haskell/Bench.hs | 128 ++++++++++++++ bench/haskell/ghc.nix | 12 ++ bench/run.sh | 59 +++++++ bench/zig/build.zig | 32 ++++ bench/zig/build.zig.zon | 15 ++ bench/zig/src/flake_lock.zig | 113 +++++++++++++ bench/zig/src/main.zig | 287 ++++++++++++++++++++++++++++++++ 11 files changed, 927 insertions(+) create mode 100644 bench/flake-lock-rs/Cargo.lock create mode 100644 bench/flake-lock-rs/Cargo.toml create mode 100644 bench/flake-lock-rs/src/main.rs create mode 100755 bench/flake-lock.sh create mode 100644 bench/haskell/Bench.hs create mode 100644 bench/haskell/ghc.nix create mode 100755 bench/run.sh create mode 100644 bench/zig/build.zig create mode 100644 bench/zig/build.zig.zon create mode 100644 bench/zig/src/flake_lock.zig create mode 100644 bench/zig/src/main.zig diff --git a/bench/flake-lock-rs/Cargo.lock b/bench/flake-lock-rs/Cargo.lock new file mode 100644 index 0000000..3e6dc2a --- /dev/null +++ b/bench/flake-lock-rs/Cargo.lock @@ -0,0 +1,31 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "flake-lock-rs" +version = "0.0.0" +dependencies = [ + "nix-flake-lock", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "nix-flake-lock" +version = "0.1.0" +source = "git+https://github.com/cachix/nix-flake-lock?rev=61c13ac#61c13ac3b0c13dd394dcad2c93f76c89c6b541bd" +dependencies = [ + "itoa", + "memchr", +] diff --git a/bench/flake-lock-rs/Cargo.toml b/bench/flake-lock-rs/Cargo.toml new file mode 100644 index 0000000..b9bd2de --- /dev/null +++ b/bench/flake-lock-rs/Cargo.toml @@ -0,0 +1,15 @@ +# Reference harness: cachix/nix-flake-lock, pinned. Not part of the workspace. +[package] +name = "flake-lock-rs" +version = "0.0.0" +edition = "2024" +publish = false + +[dependencies] +nix-flake-lock = { git = "https://github.com/cachix/nix-flake-lock", rev = "61c13ac" } + +[profile.release] +lto = "fat" +codegen-units = 1 + +[workspace] diff --git a/bench/flake-lock-rs/src/main.rs b/bench/flake-lock-rs/src/main.rs new file mode 100644 index 0000000..5de1dc7 --- /dev/null +++ b/bench/flake-lock-rs/src/main.rs @@ -0,0 +1,180 @@ +//! Reference side of bench/flake-lock.sh. +//! +//! - `canon ...`: for each file print `== ` then either the +//! canonical bytes plus `-- valid` / `-- invalid`, or `-- parse error`. +//! - `bench ...`: `: ` lines. +//! - `gen `: write the large / adversarial fixtures from upstream's benches. +//! - `fuzz ...`: write `n` mutated copies of the inputs. + +use std::fmt::Write as _; +use std::hint::black_box; +use std::io::Write as _; +use std::time::Instant; + +use nix_flake_lock::LockFile; + +fn read(path: &str) -> Vec { + std::fs::read(path).unwrap_or_else(|e| panic!("{path}: {e}")) +} + +pub fn iterations(len: usize) -> usize { + (4_000_000 / len.max(1)).clamp(3, 200_000) +} + +fn best(iters: usize, mut f: F) -> u128 { + for _ in 0..iters.min(1000) { + f(); + } + (0..7) + .map(|_| { + let t = Instant::now(); + for _ in 0..iters { + f(); + } + t.elapsed().as_nanos() + }) + .min() + .unwrap() +} + +fn large(nodes: usize, reverse: bool) -> String { + let idx = |p: usize| if reverse { nodes - p - 1 } else { p }; + let mut json = String::from("{\"nodes\":{\"root\":{\"inputs\":{"); + for p in 0..nodes { + if p != 0 { + json.push(','); + } + let i = idx(p); + write!(json, "\"n{i:06}\":\"n{i:06}\"").unwrap(); + } + json.push_str("}}"); + for p in 0..nodes { + let i = idx(p); + write!(json, ",\"n{i:06}\":{{\"locked\":{{\"path\":\"x\",\"type\":\"path\"}},\"original\":{{\"path\":\"x\",\"type\":\"path\"}}}}").unwrap(); + } + json.push_str("},\"root\":\"root\",\"version\":7}"); + json +} + +fn colliding(n: usize) -> String { + let mut json = String::from("{\"nodes\":{\"root\":{\"inputs\":{"); + for i in 0..n { + if i != 0 { + json.push(','); + } + write!(json, "\"dep{i:06}\":\"dep{i:06}\"").unwrap(); + } + json.push_str("}}"); + for i in 0..n { + write!(json, ",\"dep{i:06}\":{{\"inputs\":{{\"systems\":\"systems{i:06}\"}},\"locked\":{{\"path\":\"x\",\"type\":\"path\"}},\"original\":{{\"path\":\"x\",\"type\":\"path\"}}}},\"systems{i:06}\":{{\"locked\":{{\"path\":\"x\",\"type\":\"path\"}},\"original\":{{\"path\":\"x\",\"type\":\"path\"}}}}").unwrap(); + } + json.push_str("},\"root\":\"root\",\"version\":7}"); + json +} + +/// xorshift64*: deterministic mutations shared by nothing but this tool. +struct Rng(u64); +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + self.0.wrapping_mul(0x2545F4914F6CDD1D) + } + fn below(&mut self, n: usize) -> usize { + (self.next() % n.max(1) as u64) as usize + } +} + +fn mutate(rng: &mut Rng, mut doc: Vec) -> Vec { + const TOKENS: &[&[u8]] = &[ + b"\"", b"\\", b"\\u00e9", b"\\ud83d\\ude00", b"\\ud800", b"{", b"}", b"[", b"]", b",", b":", + b"null", b"true", b"false", b"0", b"01", b"1.5", b"-1", b"18446744073709551616", b"\"root\"", + b"\"inputs\"", b"\"follows\"", b"[\"a\"]", b"\"version\"", b"\xc3\xa9", b"\xff", b"\x01", b" ", b"\n", + ]; + for _ in 0..1 + rng.below(3) { + let at = rng.below(doc.len() + 1); + match rng.below(4) { + 0 => { + let t = TOKENS[rng.below(TOKENS.len())]; + doc.splice(at..at, t.iter().copied()); + } + 1 if !doc.is_empty() => { + let end = (at + 1 + rng.below(8)).min(doc.len()); + doc.drain(at.min(doc.len())..end); + } + 2 if at < doc.len() => doc[at] = TOKENS[rng.below(TOKENS.len())][0], + _ => { + // Duplicate a slice (duplicate keys, repeated members). + if doc.len() > 2 { + let a = rng.below(doc.len()); + let b = (a + rng.below(64)).min(doc.len()); + let piece = doc[a..b].to_vec(); + doc.splice(at..at, piece); + } + } + } + } + doc +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let out = std::io::stdout(); + let mut out = std::io::BufWriter::new(out.lock()); + match args.first().map(String::as_str) { + Some("canon") => { + for f in &args[1..] { + let bytes = read(f); + writeln!(out, "== {f}").unwrap(); + match LockFile::parse(&bytes) { + Ok(lock) => { + out.write_all(&lock.to_nix_bytes()).unwrap(); + writeln!(out, "-- {}", if lock.validate().is_ok() { "valid" } else { "invalid" }).unwrap(); + } + Err(_) => writeln!(out, "-- parse error").unwrap(), + } + } + } + Some("bench") => { + for f in &args[1..] { + let bytes = read(f); + let name = std::path::Path::new(f).file_name().unwrap().to_string_lossy().into_owned(); + let iters = iterations(bytes.len()); + let lock = LockFile::parse(&bytes).expect("bench inputs parse"); + let ns = best(iters, || { + black_box(LockFile::parse(black_box(&bytes)).unwrap()); + }); + writeln!(out, "parse:{name} {iters} {ns}").unwrap(); + let ns = best(iters, || { + black_box(black_box(&lock).to_nix_bytes()); + }); + writeln!(out, "serialize:{name} {iters} {ns}").unwrap(); + let ns = best(iters, || { + black_box(black_box(&lock).validate()).ok(); + }); + writeln!(out, "validate:{name} {iters} {ns}").unwrap(); + } + } + Some("gen") => { + let dir = &args[1]; + std::fs::write(format!("{dir}/large-sorted-8000.lock"), large(8000, false)).unwrap(); + std::fs::write(format!("{dir}/large-reverse-8000.lock"), large(8000, true)).unwrap(); + std::fs::write(format!("{dir}/colliding-4000.lock"), colliding(4000)).unwrap(); + } + Some("fuzz") => { + let dir = &args[1]; + let n: usize = args[2].parse().unwrap(); + let mut rng = Rng(args[3].parse::().unwrap() | 1); + let seeds: Vec> = args[4..].iter().map(|f| read(f)).collect(); + for i in 0..n { + let seed = seeds[rng.below(seeds.len())].clone(); + std::fs::write(format!("{dir}/fuzz-{i:05}.lock"), mutate(&mut rng, seed)).unwrap(); + } + } + _ => { + eprintln!("usage: flake-lock-rs canon|bench|gen|fuzz ..."); + std::process::exit(2); + } + } +} diff --git a/bench/flake-lock.sh b/bench/flake-lock.sh new file mode 100755 index 0000000..949b7db --- /dev/null +++ b/bench/flake-lock.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# pkg/nix-flake-lock (Zig) vs cachix/nix-flake-lock (Rust, pinned in +# bench/flake-lock-rs): a differential check that both produce identical +# canonical output and validation results, then a single-threaded benchmark. +# +# Corpus: upstream's fixtures and generated benchmark inputs, every +# flake.lock under $FLAKE_LOCK_SEARCH (default: $HOME), and FUZZ mutated +# documents. Usage (inside `nix develop`): bench/flake-lock.sh +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SEARCH="${FLAKE_LOCK_SEARCH:-$HOME}" +FUZZ="${FUZZ:-20000}" +W="$(mktemp -d)" +trap 'rm -rf "$W"' EXIT + +cargo build -q --release --manifest-path "$ROOT/bench/flake-lock-rs/Cargo.toml" +(cd "$ROOT/bench/zig" && zig build -Doptimize=ReleaseFast -Dcpu=native) +RS="$ROOT/bench/flake-lock-rs/target/release/flake-lock-rs" +ZIG="$ROOT/bench/zig/zig-out/bin/flake-lock" + +mkdir -p "$W/fixtures" "$W/real" "$W/fuzz" +UPSTREAM="$(find "${CARGO_HOME:-$HOME/.cargo}/git/checkouts" -path '*nix-flake-lock*/tests/fixtures' -type d | head -1)" +cp "$UPSTREAM"/*.lock "$W/fixtures/" +"$RS" gen "$W/fixtures" +i=0 +while IFS= read -r -d '' f; do + cp "$f" "$W/real/$(printf '%04d' $i).lock"; i=$((i + 1)) +done < <(find "$SEARCH" -name flake.lock -not -path '*/.cache/*' -not -path '*/node_modules/*' -size -16M -print0 2>/dev/null) +mapfile -t SEEDS < <(ls "$W"/real/*.lock 2>/dev/null | head -50 || true) +"$RS" fuzz "$W/fuzz" "$FUZZ" 42 "$W"/fixtures/{tiny,typical,legacy-v5,legacy-v6}.lock "${SEEDS[@]}" + +echo "== differential: $(ls "$W/fixtures" | wc -l) fixtures, $i real-world lock files, $FUZZ mutations" +find "$W/fixtures" "$W/real" "$W/fuzz" -name '*.lock' | sort >"$W/all" +xargs "$RS" canon <"$W/all" >"$W/rs.out" +FL_MODE=canon FL_LIST="$W/all" "$ZIG" >"$W/zig.out" +if cmp -s "$W/rs.out" "$W/zig.out"; then + echo "identical output on $(wc -l <"$W/all") documents" \ + "($(grep -c -- '-- valid' "$W/rs.out") valid, $(grep -c -- '-- invalid' "$W/rs.out") invalid follows, $(grep -c -- '-- parse error' "$W/rs.out") rejected)" +else + echo "MISMATCH"; diff <(cat "$W/rs.out") <(cat "$W/zig.out") | head -40; exit 1 +fi + +echo +echo "== benchmark (best of 7, ns per operation)" +mapfile -t BIGGEST < <(ls -S "$W"/real/*.lock 2>/dev/null | head -3 || true) +for f in "${BIGGEST[@]}"; do cp "$f" "$W/fixtures/real-$(basename "$f")"; done +find "$W/fixtures" -name '*.lock' | sort >"$W/bench" +xargs "$RS" bench <"$W/bench" | sort >"$W/rs.bench" +FL_MODE=bench FL_LIST="$W/bench" "$ZIG" | sort >"$W/zig.bench" +printf '%-38s %9s %12s %12s %8s\n' operation bytes "zig ns" "rust ns" speedup +join "$W/zig.bench" "$W/rs.bench" | while read -r key zi zn ri rn; do + file="${key#*:}"; bytes=$(stat -c %s "$W/fixtures/$file") + awk -v k="$key" -v b="$bytes" -v z="$zn" -v zi="$zi" -v r="$rn" -v ri="$ri" \ + 'BEGIN { zz = z / zi; rr = r / ri; printf "%-38s %9d %12.0f %12.0f %7.2fx\n", k, b, zz, rr, rr / zz }' +done diff --git a/bench/haskell/Bench.hs b/bench/haskell/Bench.hs new file mode 100644 index 0000000..98690ce --- /dev/null +++ b/bench/haskell/Bench.hs @@ -0,0 +1,128 @@ +{-# LANGUAGE BangPatterns #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE PackageImports #-} + +-- Benchmarks for the Haskell Nix libraries in Cachix's dependency tree +-- (nix-narinfo, nix-derivation, hnix-store-core, hnix-store-nar, ed25519). +-- Mirrors bench/zig/src/main.zig: same corpus, same output format +-- (` `), best of 7 rounds after a warm-up. +-- +-- Inputs are pre-decoded to Text outside the timed region, which favours +-- these libraries (our Zig parsers work on raw bytes). + +module Main (main) where + +import Control.Exception (evaluate) +import Control.Monad (forM, replicateM, when) +import qualified "crypton" Crypto.Hash as H +import qualified Crypto.Sign.Ed25519 as Ed +import qualified Data.Attoparsec.Text as A +import qualified Data.ByteString as BS +import qualified Data.ByteString.Base64 as B64 +import qualified Data.ByteString.Char8 as BC +import Data.IORef +import qualified Data.Map.Strict as Map +import qualified Data.Set as Set +import qualified Data.Text as T +import qualified Data.Text.Encoding as TE +import qualified Data.Text.Lazy as TL +import qualified Data.Text.Lazy.Builder as TB +import Data.Word (Word64) +import GHC.Clock (getMonotonicTimeNSec) +import qualified Nix.Derivation as D +import Nix.NarInfo (NarInfo (..)) +import qualified Nix.NarInfo as NI +import qualified "hnix-store-core" System.Nix.Base32 as B32 +import qualified "hnix-store-nar" System.Nix.Nar.Streamer as Nar +import qualified "hnix-store-core" System.Nix.StorePath as SP +import System.Environment (getEnv, lookupEnv) + +rounds :: Int +rounds = 7 + +records :: FilePath -> IO [BS.ByteString] +records path = filter (not . BS.null) . BS.split 0 <$> BS.readFile path + +report :: String -> Int -> Word64 -> IO () +report name ops ns = putStrLn (name ++ " " ++ show ops ++ " " ++ show ns) + +-- Applies `f` to `x` afresh every round. NOINLINE plus -fno-full-laziness +-- keep GHC from sharing one evaluation across rounds. +{-# NOINLINE bench #-} +bench :: String -> Int -> (a -> Int) -> a -> IO () +bench name ops f x = do + _ <- evaluate (f x) + times <- replicateM rounds $ do + t0 <- getMonotonicTimeNSec + !_ <- evaluate (f x) + t1 <- getMonotonicTimeNSec + pure (t1 - t0) + report name ops (minimum times) + +forceNarInfo :: NI.SimpleNarInfo -> Int +forceNarInfo ni = + length (storePath ni) + T.length (url ni) + T.length (compression ni) + T.length (fileHash ni) + + fromIntegral (fileSize ni) + T.length (narHash ni) + fromIntegral (narSize ni) + + sum (map length (Set.toList (references ni))) + + maybe 0 T.length (deriver ni) + maybe 0 T.length (system ni) + maybe 0 T.length (sig ni) + +forceDrv :: D.Derivation FilePath T.Text -> Int +forceDrv d = + sum [length (D.path o) + T.length (D.hashAlgo o) + T.length (D.hash o) + T.length k | (k, o) <- Map.toList (D.outputs d)] + + sum [length p + sum (map T.length (Set.toList os)) | (p, os) <- Map.toList (D.inputDrvs d)] + + sum (map length (Set.toList (D.inputSrcs d))) + + T.length (D.platform d) + T.length (D.builder d) + sum (fmap T.length (D.args d)) + + sum [T.length k + T.length v | (k, v) <- Map.toList (D.env d)] + +parseText :: A.Parser a -> T.Text -> a +parseText p t = either error id (A.parseOnly p t) + +-- Nix secret key file: ":". +signer :: BS.ByteString -> (BS.ByteString, Ed.SecretKey) +signer key = + let (name, rest) = BC.break (== ':') (BC.strip key) + in (name, Ed.SecretKey (either error id (B64.decode (BS.drop 1 rest)))) + +main :: IO () +main = do + dir <- getEnv "BENCH_DIR" + only <- maybe "" id <$> lookupEnv "BENCH_ONLY" + paths <- records (dir ++ "/paths.bin") + + when (only == "nar") $ do + t0 <- getMonotonicTimeNSec + total <- fmap sum . forM paths $ \p -> do + ref <- newIORef (H.hashInit :: H.Context H.SHA256) + size <- newIORef (0 :: Int) + Nar.dumpPath (BC.unpack p) $ \chunk -> do + modifyIORef' ref (`H.hashUpdate` chunk) + modifyIORef' size (+ BS.length chunk) + digest <- H.hashFinalize <$> readIORef ref + _ <- evaluate (length (show digest)) + readIORef size + t1 <- getMonotonicTimeNSec + report "nar-dump-sha256" total (t1 - t0) + + when (only /= "nar") $ do + narinfos <- map TE.decodeUtf8 <$> records (dir ++ "/narinfo.bin") + drvs <- map TE.decodeUtf8 <$> records (dir ++ "/drv.bin") + let parsed = map (parseText NI.parseNarInfo) narinfos + _ <- evaluate (sum (map forceNarInfo parsed)) + let hashes32 = [T.take 32 (T.drop 11 (T.pack (storePath ni))) | ni <- parsed] + hashes52 = [T.drop 7 (narHash ni) | ni <- parsed] + raw20 = [either error id (B32.decode h) | h <- hashes32] + fingerprints = + [ TE.encodeUtf8 (T.intercalate ";" ["1", T.pack (storePath ni), narHash ni, T.pack (show (narSize ni)), T.unwords (map T.pack (Set.toList (references ni)))]) + | ni <- parsed + ] + _ <- evaluate (sum (map BS.length raw20) + sum (map BS.length fingerprints) + sum (map T.length hashes52)) + (keyName, sk) <- signer <$> BS.readFile (dir ++ "/key") + + bench "narinfo-parse" (length narinfos) (sum . map (forceNarInfo . parseText NI.parseNarInfo)) narinfos + bench "narinfo-render" (length parsed) (sum . map (fromIntegral . TL.length . TB.toLazyText . NI.buildNarInfo)) parsed + bench "ed25519-sign" (length fingerprints) (\fps -> sum [BS.length (keyName <> ":" <> B64.encode (Ed.unSignature (Ed.dsign sk fp))) | fp <- fps]) fingerprints + bench "drv-parse" (length drvs) (sum . map (forceDrv . parseText D.parseDerivation)) drvs + bench "base32-decode-20" (length hashes32) (\hs -> sum [either error BS.length (B32.decode h) | h <- hs]) hashes32 + bench "base32-decode-32" (length hashes52) (\hs -> sum [either error BS.length (B32.decode h) | h <- hs]) hashes52 + bench "base32-encode-20" (length raw20) (sum . map (T.length . B32.encode)) raw20 + bench "storepath-parse" (length paths) (\ps -> sum [either error (\sp -> SP.storePathHash sp `seq` SP.storePathName sp `seq` 1) (SP.parsePath "/nix/store" p) | p <- ps]) paths diff --git a/bench/haskell/ghc.nix b/bench/haskell/ghc.nix new file mode 100644 index 0000000..91552b4 --- /dev/null +++ b/bench/haskell/ghc.nix @@ -0,0 +1,12 @@ +# GHC with the Haskell Nix libraries Cachix builds on. +{ pkgs ? import { } }: +pkgs.haskellPackages.ghcWithPackages (p: [ + p.nix-narinfo + p.nix-derivation + p.hnix-store-core + p.hnix-store-nar + p.ed25519 + p.crypton + p.base64-bytestring + p.attoparsec +]) diff --git a/bench/run.sh b/bench/run.sh new file mode 100755 index 0000000..4748a7f --- /dev/null +++ b/bench/run.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Benchmarks pkg/* (Zig) against the Haskell libraries Cachix uses, plus +# Nix's own C++ NAR serialiser. Single-threaded, same corpus for both. +# +# Usage: bench/run.sh [store-path] (inside `nix develop`) +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +TARGET="${1:-/run/current-system}" +WORK="$(mktemp -d)" +trap 'kill ${SERVER_PID:-} 2>/dev/null || true; rm -rf "$WORK"' EXIT + +echo "building..." >&2 +(cd "$ROOT" && cargo build --release -q) +(cd "$ROOT/bench/zig" && zig build -Doptimize=ReleaseFast -Dcpu=native) +GHC="$(nix build --no-link --print-out-paths --impure --expr "import $ROOT/bench/haskell/ghc.nix {}")/bin/ghc" +"$GHC" -O2 -fno-full-laziness -rtsopts -outputdir "$WORK/hs" -o "$WORK/bench-hs" "$ROOT/bench/haskell/Bench.hs" >/dev/null + +echo "building corpus from $TARGET..." >&2 +C="$WORK/corpus"; mkdir -p "$C" +nix key generate-secret --key-name bench-1 >"$C/key" +head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' >"$WORK/secret" +"$ROOT/target/release/helios-server" --listen 127.0.0.1:18099 --data-dir "$WORK/data" \ + --signing-key-file "$C/key" --jwt-secret-file "$WORK/secret" --admin-secret-file "$WORK/secret" 2>/dev/null & +SERVER_PID=$! +for _ in $(seq 50); do curl -sf http://127.0.0.1:18099/healthz >/dev/null && break; sleep 0.1; done +export XDG_CONFIG_HOME="$WORK/config" +H="$ROOT/target/release/helios" +"$H" login a http://127.0.0.1:18099 "$(cat "$WORK/secret")" >/dev/null +"$H" cache create main >/dev/null +"$H" login ci http://127.0.0.1:18099 "$("$H" --server a token create bench --caches main 2>/dev/null | jq -r .token)" >/dev/null +"$H" push main -r "$TARGET" >/dev/null 2>&1 +nix-store -qR "$TARGET" >"$WORK/paths" +while read -r p; do printf '%s\0' "$p"; done <"$WORK/paths" >"$C/paths.bin" +while read -r p; do curl -s "http://127.0.0.1:18099/main/$(basename "$p" | cut -c1-32).narinfo"; printf '\0'; done <"$WORK/paths" >"$C/narinfo.bin" +mapfile -t DRVS < <(find /nix/store -maxdepth 1 -name '*.drv' -print 2>/dev/null | head -3000 || true) +for d in "${DRVS[@]}"; do cat "$d"; printf '\0'; done >"$C/drv.bin" +kill "$SERVER_PID"; SERVER_PID= + +export BENCH_DIR="$C" +ZIG="$ROOT/bench/zig/zig-out/bin/bench" +"$ZIG" 2>&1 | sort >"$WORK/zig.txt" +"$WORK/bench-hs" +RTS -A64m -RTS | sort >"$WORK/hs.txt" + +# NAR: warm the page cache once, then time each implementation on the closure. +BENCH_ONLY=nar "$ZIG" >/dev/null 2>&1 +BENCH_ONLY=nar "$ZIG" 2>&1 >/dev/null | sed 's/^/zig /' >"$WORK/nar.txt" +BENCH_ONLY=nar "$WORK/bench-hs" +RTS -A64m -RTS | sed 's/^/hs /' >>"$WORK/nar.txt" +t0=$(date +%s%N); xargs nix hash path --type sha256 <"$WORK/paths" >/dev/null; t1=$(date +%s%N) +echo "cpp nar-dump-sha256 $(awk '{print $3}' "$WORK/nar.txt" | head -1) $((t1 - t0))" >>"$WORK/nar.txt" + +echo +echo "corpus: $(wc -l <"$WORK/paths") store paths, $(tr -cd '\0' <"$C/drv.bin" | wc -c) .drv files" +echo +printf '%-26s %14s %14s %9s\n' benchmark "helios ns/op" "haskell ns/op" speedup +join "$WORK/zig.txt" "$WORK/hs.txt" | awk '{z=$3/$2; h=$5/$4; printf "%-26s %14.1f %14.1f %8.1fx\n", $1, z, h, h/z}' +join -v1 "$WORK/zig.txt" "$WORK/hs.txt" | awk '{printf "%-26s %14.1f %14s\n", $1, $3/$2, "-"}' +echo +printf '%-30s %12s\n' "NAR serialise + SHA-256" "GiB/s (1 core)" +awk '{printf "%-30s %12.2f\n", ($1=="zig"?"helios (pkg/nix-archive)":($1=="hs"?"hnix-store-nar + crypton":"nix hash path (Nix C++)")), $3/$4*1e9/1073741824}' "$WORK/nar.txt" diff --git a/bench/zig/build.zig b/bench/zig/build.zig new file mode 100644 index 0000000..19a9224 --- /dev/null +++ b/bench/zig/build.zig @@ -0,0 +1,32 @@ +const std = @import("std"); + +const packages = [_][2][]const u8{ + .{ "nix-base32", "nix_base32" }, + .{ "nix-store-path", "nix_store_path" }, + .{ "nix-narinfo", "nix_narinfo" }, + .{ "nix-derivation", "nix_derivation" }, + .{ "nix-archive", "nix_archive" }, +}; + +pub fn build(b: *std.Build) void { + const target = b.standardTargetOptions(.{}); + const optimize = b.standardOptimizeOption(.{}); + const mod = b.createModule(.{ + .root_source_file = b.path("src/main.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + }); + inline for (packages) |p| mod.addImport(p[0], b.dependency(p[1], .{ .target = target, .optimize = optimize }).module(p[0])); + mod.linkSystemLibrary("zstd", .{}); + b.installArtifact(b.addExecutable(.{ .name = "bench", .root_module = mod })); + + const fl_mod = b.createModule(.{ + .root_source_file = b.path("src/flake_lock.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + }); + fl_mod.addImport("nix-flake-lock", b.dependency("nix_flake_lock", .{ .target = target, .optimize = optimize }).module("nix-flake-lock")); + b.installArtifact(b.addExecutable(.{ .name = "flake-lock", .root_module = fl_mod })); +} diff --git a/bench/zig/build.zig.zon b/bench/zig/build.zig.zon new file mode 100644 index 0000000..5cdca22 --- /dev/null +++ b/bench/zig/build.zig.zon @@ -0,0 +1,15 @@ +.{ + .name = .helios_bench, + .version = "0.1.0", + .fingerprint = 0xc67ae359dd92ff1e, + .minimum_zig_version = "0.16.0", + .dependencies = .{ + .nix_base32 = .{ .path = "../../pkg/nix-base32" }, + .nix_store_path = .{ .path = "../../pkg/nix-store-path" }, + .nix_narinfo = .{ .path = "../../pkg/nix-narinfo" }, + .nix_derivation = .{ .path = "../../pkg/nix-derivation" }, + .nix_archive = .{ .path = "../../pkg/nix-archive" }, + .nix_flake_lock = .{ .path = "../../pkg/nix-flake-lock" }, + }, + .paths = .{ "build.zig", "build.zig.zon", "src" }, +} diff --git a/bench/zig/src/flake_lock.zig b/bench/zig/src/flake_lock.zig new file mode 100644 index 0000000..89f9cf5 --- /dev/null +++ b/bench/zig/src/flake_lock.zig @@ -0,0 +1,113 @@ +//! Zig side of bench/flake-lock.sh, mirroring bench/flake-lock-rs. +//! FL_MODE=canon|bench, FL_LIST=. Output on stdout. + +const std = @import("std"); +const linux = std.os.linux; +const fl = @import("nix-flake-lock"); + +const gpa = std.heap.c_allocator; + +fn now() u64 { + var ts: linux.timespec = undefined; + _ = linux.clock_gettime(.MONOTONIC, &ts); + return @as(u64, @intCast(ts.sec)) * std.time.ns_per_s + @as(u64, @intCast(ts.nsec)); +} + +fn readFile(path: []const u8) ![]u8 { + const z = try gpa.dupeZ(u8, path); + defer gpa.free(z); + const fd: i32 = @intCast(@as(isize, @bitCast(linux.openat(linux.AT.FDCWD, z, .{ .ACCMODE = .RDONLY }, 0)))); + if (fd < 0) return error.Open; + defer _ = linux.close(fd); + var list: std.ArrayList(u8) = .empty; + var buf: [1 << 16]u8 = undefined; + while (true) { + const n = linux.read(fd, &buf, buf.len); + if (linux.errno(n) != .SUCCESS) return error.Read; + if (n == 0) break; + try list.appendSlice(gpa, buf[0..n]); + } + return list.toOwnedSlice(gpa); +} + +fn flush(out: *std.ArrayList(u8)) void { + var off: usize = 0; + while (off < out.items.len) { + const n = linux.write(1, out.items[off..].ptr, out.items.len - off); + if (linux.errno(n) != .SUCCESS) return; + off += n; + } + out.clearRetainingCapacity(); +} + +fn iterations(len: usize) usize { + return std.math.clamp(4_000_000 / @max(len, 1), 3, 200_000); +} + +fn best(iters: usize, ctx: anytype, comptime f: fn (@TypeOf(ctx)) void) u64 { + for (0..@min(iters, 1000)) |_| f(ctx); + var b: u64 = std.math.maxInt(u64); + for (0..7) |_| { + const t0 = now(); + for (0..iters) |_| f(ctx); + b = @min(b, now() - t0); + } + return b; +} + +const ParseCtx = struct { bytes: []const u8 }; + +fn parseFresh(c: ParseCtx) void { + // Parse and free each time: the same malloc traffic the Rust API incurs. + var doc = fl.parse(gpa, c.bytes, .{}, null) catch unreachable; + std.mem.doNotOptimizeAway(doc.lock.nodes.ptr); + doc.deinit(); +} + +const LockCtx = struct { lock: *const fl.LockFile }; + +fn serialize(c: LockCtx) void { + var out: std.ArrayList(u8) = .empty; + defer out.deinit(gpa); + c.lock.writeNix(gpa, &out) catch unreachable; + std.mem.doNotOptimizeAway(out.items.ptr); +} + +fn validate(c: LockCtx) void { + c.lock.validate(gpa) catch {}; +} + +pub fn main() !void { + const mode = std.mem.span(std.c.getenv("FL_MODE") orelse return error.NoMode); + const list = try readFile(std.mem.span(std.c.getenv("FL_LIST") orelse return error.NoList)); + var out: std.ArrayList(u8) = .empty; + var it = std.mem.tokenizeScalar(u8, list, '\n'); + while (it.next()) |path| { + const bytes = try readFile(path); + defer gpa.free(bytes); + if (std.mem.eql(u8, mode, "canon")) { + try out.print(gpa, "== {s}\n", .{path}); + if (fl.parse(gpa, bytes, .{}, null)) |doc_| { + var doc = doc_; + defer doc.deinit(); + const lock = &doc.lock; + try lock.writeNix(gpa, &out); + const ok = if (lock.validate(gpa)) true else |_| false; + try out.print(gpa, "-- {s}\n", .{if (ok) "valid" else "invalid"}); + } else |_| { + try out.appendSlice(gpa, "-- parse error\n"); + } + if (out.items.len > 1 << 20) flush(&out); + continue; + } + const name = std.fs.path.basename(path); + const iters = iterations(bytes.len); + var doc = try fl.parse(gpa, bytes, .{}, null); + defer doc.deinit(); + const lock = &doc.lock; + try out.print(gpa, "parse:{s} {d} {d}\n", .{ name, iters, best(iters, ParseCtx{ .bytes = bytes }, parseFresh) }); + try out.print(gpa, "serialize:{s} {d} {d}\n", .{ name, iters, best(iters, LockCtx{ .lock = lock }, serialize) }); + try out.print(gpa, "validate:{s} {d} {d}\n", .{ name, iters, best(iters, LockCtx{ .lock = lock }, validate) }); + } + flush(&out); +} diff --git a/bench/zig/src/main.zig b/bench/zig/src/main.zig new file mode 100644 index 0000000..92268d2 --- /dev/null +++ b/bench/zig/src/main.zig @@ -0,0 +1,287 @@ +//! Benchmarks for the pkg/* Nix libraries. Mirrors bench/haskell/Bench.hs: +//! both read the same NUL-separated corpus files from $BENCH_DIR and print +//! ` ` lines. + +const std = @import("std"); +const linux = std.os.linux; +const base32 = @import("nix-base32"); +const store_path = @import("nix-store-path"); +const narinfo = @import("nix-narinfo"); +const derivation = @import("nix-derivation"); +const archive = @import("nix-archive"); + +const gpa = std.heap.c_allocator; +const rounds = 7; + +fn now() u64 { + var ts: linux.timespec = undefined; + _ = linux.clock_gettime(.MONOTONIC, &ts); + return @as(u64, @intCast(ts.sec)) * std.time.ns_per_s + @as(u64, @intCast(ts.nsec)); +} + +fn readFile(path: [:0]const u8) ![]u8 { + const fd: i32 = @intCast(linux.openat(linux.AT.FDCWD, path, .{ .ACCMODE = .RDONLY }, 0)); + if (fd < 0) return error.Open; + defer _ = linux.close(fd); + var list: std.ArrayList(u8) = .empty; + var buf: [1 << 16]u8 = undefined; + while (true) { + const n = linux.read(fd, &buf, buf.len); + if (linux.errno(n) != .SUCCESS) return error.Read; + if (n == 0) break; + try list.appendSlice(gpa, buf[0..n]); + } + return list.toOwnedSlice(gpa); +} + +fn records(dir: []const u8, name: []const u8) ![]const []const u8 { + const path = try std.fmt.allocPrintSentinel(gpa, "{s}/{s}", .{ dir, name }, 0); + const data = try readFile(path); + var out: std.ArrayList([]const u8) = .empty; + var it = std.mem.splitScalar(u8, data, 0); + while (it.next()) |r| if (r.len > 0) try out.append(gpa, r); + return out.toOwnedSlice(gpa); +} + +fn report(name: []const u8, ops: usize, ns: u64) void { + std.debug.print("{s} {d} {d}\n", .{ name, ops, ns }); +} + +/// Runs `body` `rounds` times and reports the fastest round. +fn bench(name: []const u8, ops: usize, ctx: anytype, comptime body: fn (@TypeOf(ctx)) anyerror!void) !void { + try body(ctx); // warm up + var best: u64 = std.math.maxInt(u64); + for (0..rounds) |_| { + const t0 = now(); + try body(ctx); + best = @min(best, now() - t0); + } + report(name, ops, best); +} + +const Ctx = struct { + narinfos: []const []const u8, + parsed: []narinfo.NarInfo, + drvs: []const []const u8, + paths: []const []const u8, + hashes32: []const []const u8, + hashes52: []const []const u8, + raw20: [][20]u8, + fingerprints: []const []const u8, + signer: *narinfo.Signer, +}; + +fn narinfoParse(c: *const Ctx) !void { + for (c.narinfos) |t| std.mem.doNotOptimizeAway(try narinfo.parse(t)); +} + +fn narinfoRender(c: *const Ctx) !void { + for (c.parsed) |p| { + var nar: [32]u8 = undefined; + var file: [32]u8 = undefined; + try base32.decode(&nar, p.nar_hash["sha256:".len..]); + try base32.decode(&file, p.file_hash["sha256:".len..]); + const text = try narinfo.render(gpa, .{ + .store_path = p.store_path, + .nar_hash = &nar, + .nar_size = p.nar_size, + .file_hash = &file, + .file_size = p.file_size orelse 0, + .compression = p.compression, + .references = p.references, + .deriver = p.deriver, + .system = p.system, + }, null); + std.mem.doNotOptimizeAway(text.ptr); + gpa.free(text); + } +} + +fn sign(c: *const Ctx) !void { + var out: std.ArrayList(u8) = .empty; + defer out.deinit(gpa); + for (c.fingerprints) |fp| { + out.clearRetainingCapacity(); + try c.signer.sign(&out, gpa, fp); + std.mem.doNotOptimizeAway(out.items.ptr); + } +} + +fn drvParse(c: *const Ctx) !void { + var arena: std.heap.ArenaAllocator = .init(gpa); + defer arena.deinit(); + for (c.drvs) |d| { + std.mem.doNotOptimizeAway(try derivation.parse(arena.allocator(), d)); + _ = arena.reset(.retain_capacity); + } +} + +fn decode32(c: *const Ctx) !void { + for (c.hashes32) |h| { + var out: [20]u8 = undefined; + try base32.decode(&out, h); + std.mem.doNotOptimizeAway(out); + } +} + +fn decode32Scalar(c: *const Ctx) !void { + for (c.hashes32) |h| { + var out: [20]u8 = undefined; + try base32.decodeScalar(&out, h); + std.mem.doNotOptimizeAway(out); + } +} + +fn decode52(c: *const Ctx) !void { + for (c.hashes52) |h| { + var out: [32]u8 = undefined; + try base32.decode(&out, h); + std.mem.doNotOptimizeAway(out); + } +} + +fn encode20(c: *const Ctx) !void { + for (c.raw20) |*r| { + var out: [32]u8 = undefined; + base32.encode(&out, r); + std.mem.doNotOptimizeAway(out); + } +} + +fn storePathParse(c: *const Ctx) !void { + for (c.paths) |p| { + const base = store_path.baseName(p) orelse return error.Invalid; + std.mem.doNotOptimizeAway(store_path.hashOf(base)); + } +} + +fn storePathValid(c: *const Ctx) !void { + for (c.paths) |p| { + if (!store_path.isValidBaseName(p[store_path.store_dir.len..])) return error.Invalid; + } +} + +fn encode20Scalar(c: *const Ctx) !void { + for (c.raw20) |*r| { + var out: [32]u8 = undefined; + base32.encodeScalar(&out, r); + std.mem.doNotOptimizeAway(out); + } +} + +fn storePathValidScalar(c: *const Ctx) !void { + for (c.paths) |p| { + const base = p[store_path.store_dir.len..]; + if (!(base32.isValidScalar(base[0..32]) and store_path.isValidNameScalar(base[33..]))) return error.Invalid; + } +} + +fn discard(_: ?*anyopaque, _: [*]const u8, _: usize) callconv(.c) c_int { + return 0; +} + +fn narHash(c: *const Ctx) !void { + for (c.paths) |p| { + var sink = try archive.Sink.init(gpa, .{ .level = 0 }, discard, null); + defer sink.deinit(gpa); + const z = try gpa.dupeZ(u8, p); + defer gpa.free(z); + try archive.dump(&sink, z); + std.mem.doNotOptimizeAway(try sink.finish()); + } +} + +pub fn main() !void { + const dir = std.mem.span(std.c.getenv("BENCH_DIR") orelse return error.NoBenchDir); + const only = if (std.c.getenv("BENCH_ONLY")) |s| std.mem.span(s) else ""; + if (std.mem.eql(u8, only, "sha")) return shaThroughput(); + + const narinfos = try records(dir, "narinfo.bin"); + const parsed = try gpa.alloc(narinfo.NarInfo, narinfos.len); + for (narinfos, parsed) |t, *p| p.* = try narinfo.parse(t); + + const raw20 = try gpa.alloc([20]u8, parsed.len); + const hashes32 = try gpa.alloc([]const u8, parsed.len); + const hashes52 = try gpa.alloc([]const u8, parsed.len); + const fingerprints = try gpa.alloc([]const u8, parsed.len); + for (parsed, 0..) |p, i| { + hashes32[i] = p.store_path[store_path.store_dir.len..][0..32]; + hashes52[i] = p.nar_hash["sha256:".len..]; + try base32.decode(&raw20[i], hashes32[i]); + fingerprints[i] = try std.fmt.allocPrint(gpa, "1;{s};{s};{d};{s}", .{ p.store_path, p.nar_hash, p.nar_size, p.references }); + } + const key = try readFile(try std.fmt.allocPrintSentinel(gpa, "{s}/key", .{dir}, 0)); + const ctx: Ctx = .{ + .narinfos = narinfos, + .parsed = parsed, + .drvs = try records(dir, "drv.bin"), + .paths = try records(dir, "paths.bin"), + .hashes32 = hashes32, + .hashes52 = hashes52, + .raw20 = raw20, + .fingerprints = fingerprints, + .signer = try narinfo.Signer.parse(gpa, key), + }; + + if (std.mem.eql(u8, only, "nar")) { + var bytes: u64 = 0; + for (ctx.paths) |p| { + var sink = try archive.Sink.init(gpa, .{ .level = 0 }, discard, null); + defer sink.deinit(gpa); + try archive.dump(&sink, try gpa.dupeZ(u8, p)); + bytes += (try sink.finish()).nar_size; + } + const t0 = now(); + try narHash(&ctx); + report("nar-dump-sha256", @intCast(bytes), now() - t0); + return; + } + + try bench("narinfo-parse", narinfos.len, &ctx, narinfoParse); + try bench("narinfo-render", parsed.len, &ctx, narinfoRender); + try bench("ed25519-sign", fingerprints.len, &ctx, sign); + try bench("drv-parse", ctx.drvs.len, &ctx, drvParse); + try bench("base32-decode-20", hashes32.len, &ctx, decode32); + try bench("base32-decode-20-scalar", hashes32.len, &ctx, decode32Scalar); + try bench("base32-decode-32", hashes52.len, &ctx, decode52); + try bench("base32-encode-20", raw20.len, &ctx, encode20); + try bench("storepath-parse", ctx.paths.len, &ctx, storePathParse); + try bench("storepath-validate-simd", ctx.paths.len, &ctx, storePathValid); + try bench("storepath-validate-scalar", ctx.paths.len, &ctx, storePathValidScalar); + try bench("base32-encode-20-scalar", raw20.len, &ctx, encode20Scalar); +} + +/// Raw SHA-256 throughput over a 64 MiB buffer, for comparison with +/// `openssl speed -evp sha256`. +fn shaOnce(comptime H: type, name: []const u8, buf: []const u8) void { + var best: u64 = std.math.maxInt(u64); + for (0..5) |_| { + const t0 = now(); + var d: [32]u8 = undefined; + H.hash(buf, &d, .{}); + std.mem.doNotOptimizeAway(d); + best = @min(best, now() - t0); + } + report(name, buf.len, best); +} + +pub fn shaThroughput() void { + const big = gpa.alloc(u8, 64 << 20) catch unreachable; + @memset(big, 0xab); + shaOnce(std.crypto.hash.sha2.Sha256, "sha256-std", big); + shaOnce(archive.Sha256, "sha256-multiblock", big); +} + +pub fn shaThroughputOld() void { + const buf = gpa.alloc(u8, 64 << 20) catch unreachable; + @memset(buf, 0xab); + var best: u64 = std.math.maxInt(u64); + for (0..5) |_| { + const t0 = now(); + var d: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(buf, &d, .{}); + std.mem.doNotOptimizeAway(d); + best = @min(best, now() - t0); + } + report("sha256-64MiB", buf.len, best); +} From 8fed9f9d33bdbae7f05f588f1e4b185293a4d84d Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 12:23:54 +0200 Subject: [PATCH 06/56] build: replace the worker and pnpm workspace with cargo Remove the Cloudflare worker, its D1 migrations, the TypeScript CLI and the shared domain package, along with the pnpm workspace and Wrangler configuration. The flake now builds the Rust server and CLI, and its dev shell provides Zig, Rust, zstd and pkg-config. The package builds libhelios for the baseline CPU so it runs on any machine, which gives up SHA-NI hashing; building on the target host with HELIOS_ZIG_CPU=native keeps it. --- .gitignore | 16 +- apps/.gitkeep | 0 apps/cli/package.json | 20 - apps/cli/src/api.ts | 278 - apps/cli/src/config.ts | 57 - apps/cli/src/main.ts | 219 - apps/cli/src/nix.ts | 176 - apps/cli/src/pool.ts | 26 - apps/cli/src/push.ts | 147 - apps/cli/tsconfig.json | 17 - flake.nix | 110 +- package.json | 27 - packages/cache-domain/package.json | 17 - packages/cache-domain/src/index.ts | 20 - packages/cache-domain/src/parsers.ts | 70 - packages/cache-domain/src/types.ts | 27 - packages/cache-domain/test/parsers.test.ts | 145 - packages/cache-domain/tsconfig.json | 17 - packages/cache-domain/vitest.config.mts | 7 - pnpm-lock.yaml | 2723 ---- pnpm-workspace.yaml | 8 - scripts/smoke-test.sh | 202 - workers/cache/.editorconfig | 12 - workers/cache/.gitignore | 167 - workers/cache/.prettierrc | 6 - workers/cache/drizzle.config.ts | 7 - .../migrations/0000_cultured_night_nurse.sql | 79 - .../migrations/0001_overjoyed_exiles.sql | 2 - .../cache/migrations/0002_ordinary_wiccan.sql | 27 - .../cache/migrations/meta/0000_snapshot.json | 554 - .../cache/migrations/meta/0001_snapshot.json | 553 - .../cache/migrations/meta/0002_snapshot.json | 724 - workers/cache/migrations/meta/_journal.json | 27 - workers/cache/package.json | 25 - workers/cache/src/admin.ts | 162 - workers/cache/src/auth.ts | 166 - workers/cache/src/config.ts | 39 - workers/cache/src/db/index.ts | 58 - workers/cache/src/db/repository.ts | 507 - workers/cache/src/db/schema.ts | 111 - workers/cache/src/db/types.ts | 31 - workers/cache/src/env.d.ts | 13 - workers/cache/src/gc.ts | 112 - workers/cache/src/index.ts | 49 - workers/cache/src/jwt.ts | 248 - workers/cache/src/logging.ts | 15 - workers/cache/src/narinfo.ts | 40 - workers/cache/src/presign.ts | 38 - workers/cache/src/responses.ts | 23 - workers/cache/src/router.ts | 452 - workers/cache/src/signing.ts | 59 - workers/cache/src/uploads.ts | 432 - workers/cache/test/apply-migrations.ts | 4 - workers/cache/test/auth.spec.ts | 97 - workers/cache/test/db.spec.ts | 390 - workers/cache/test/env.d.ts | 7 - workers/cache/test/gc.spec.ts | 135 - workers/cache/test/index.spec.ts | 51 - workers/cache/test/read-path.spec.ts | 139 - workers/cache/test/tsconfig.json | 8 - workers/cache/test/upload-flow.spec.ts | 241 - workers/cache/tsconfig.json | 18 - workers/cache/vitest.config.mts | 24 - workers/cache/worker-configuration.d.ts | 13542 ---------------- workers/cache/wrangler.jsonc | 31 - wrangler.jsonc | 31 - 66 files changed, 52 insertions(+), 23733 deletions(-) delete mode 100644 apps/.gitkeep delete mode 100644 apps/cli/package.json delete mode 100644 apps/cli/src/api.ts delete mode 100644 apps/cli/src/config.ts delete mode 100644 apps/cli/src/main.ts delete mode 100644 apps/cli/src/nix.ts delete mode 100644 apps/cli/src/pool.ts delete mode 100644 apps/cli/src/push.ts delete mode 100644 apps/cli/tsconfig.json delete mode 100644 package.json delete mode 100644 packages/cache-domain/package.json delete mode 100644 packages/cache-domain/src/index.ts delete mode 100644 packages/cache-domain/src/parsers.ts delete mode 100644 packages/cache-domain/src/types.ts delete mode 100644 packages/cache-domain/test/parsers.test.ts delete mode 100644 packages/cache-domain/tsconfig.json delete mode 100644 packages/cache-domain/vitest.config.mts delete mode 100644 pnpm-lock.yaml delete mode 100644 pnpm-workspace.yaml delete mode 100755 scripts/smoke-test.sh delete mode 100644 workers/cache/.editorconfig delete mode 100644 workers/cache/.gitignore delete mode 100644 workers/cache/.prettierrc delete mode 100644 workers/cache/drizzle.config.ts delete mode 100644 workers/cache/migrations/0000_cultured_night_nurse.sql delete mode 100644 workers/cache/migrations/0001_overjoyed_exiles.sql delete mode 100644 workers/cache/migrations/0002_ordinary_wiccan.sql delete mode 100644 workers/cache/migrations/meta/0000_snapshot.json delete mode 100644 workers/cache/migrations/meta/0001_snapshot.json delete mode 100644 workers/cache/migrations/meta/0002_snapshot.json delete mode 100644 workers/cache/migrations/meta/_journal.json delete mode 100644 workers/cache/package.json delete mode 100644 workers/cache/src/admin.ts delete mode 100644 workers/cache/src/auth.ts delete mode 100644 workers/cache/src/config.ts delete mode 100644 workers/cache/src/db/index.ts delete mode 100644 workers/cache/src/db/repository.ts delete mode 100644 workers/cache/src/db/schema.ts delete mode 100644 workers/cache/src/db/types.ts delete mode 100644 workers/cache/src/env.d.ts delete mode 100644 workers/cache/src/gc.ts delete mode 100644 workers/cache/src/index.ts delete mode 100644 workers/cache/src/jwt.ts delete mode 100644 workers/cache/src/logging.ts delete mode 100644 workers/cache/src/narinfo.ts delete mode 100644 workers/cache/src/presign.ts delete mode 100644 workers/cache/src/responses.ts delete mode 100644 workers/cache/src/router.ts delete mode 100644 workers/cache/src/signing.ts delete mode 100644 workers/cache/src/uploads.ts delete mode 100644 workers/cache/test/apply-migrations.ts delete mode 100644 workers/cache/test/auth.spec.ts delete mode 100644 workers/cache/test/db.spec.ts delete mode 100644 workers/cache/test/env.d.ts delete mode 100644 workers/cache/test/gc.spec.ts delete mode 100644 workers/cache/test/index.spec.ts delete mode 100644 workers/cache/test/read-path.spec.ts delete mode 100644 workers/cache/test/tsconfig.json delete mode 100644 workers/cache/test/upload-flow.spec.ts delete mode 100644 workers/cache/tsconfig.json delete mode 100644 workers/cache/vitest.config.mts delete mode 100644 workers/cache/worker-configuration.d.ts delete mode 100644 workers/cache/wrangler.jsonc delete mode 100644 wrangler.jsonc diff --git a/.gitignore b/.gitignore index f4e6192..ef21e25 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,12 @@ -node_modules/ -.pnpm-store/ -.wrangler/ -coverage/ -dist/ -*.log .DS_Store +*.log result +result-* + +# Rust +/target +/bench/flake-lock-rs/target + +# Zig +.zig-cache/ +zig-out/ diff --git a/apps/.gitkeep b/apps/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/apps/cli/package.json b/apps/cli/package.json deleted file mode 100644 index a48b7cf..0000000 --- a/apps/cli/package.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "@helios/cli", - "version": "0.0.0", - "private": true, - "type": "module", - "bin": { - "helios": "./dist/main.js" - }, - "scripts": { - "build": "tsc", - "check": "tsc --noEmit" - }, - "dependencies": { - "@helios/cache-domain": "workspace:*" - }, - "devDependencies": { - "@types/node": "^26.1.1", - "typescript": "^7.0.2" - } -} diff --git a/apps/cli/src/api.ts b/apps/cli/src/api.ts deleted file mode 100644 index 4f75b19..0000000 --- a/apps/cli/src/api.ts +++ /dev/null @@ -1,278 +0,0 @@ -import { createReadStream } from "node:fs"; -import { stat } from "node:fs/promises"; -import { runPool } from "./pool.js"; - -export interface ApiClient { - readonly server: string; - readonly token: string; -} - -export function createClient(server: string, token: string): ApiClient { - return { server: server.replace(/\/$/, ""), token }; -} - -async function request( - client: ApiClient, - method: string, - path: string, - body?: unknown, - headers?: Record, -): Promise { - const url = `${client.server}${path}`; - const reqHeaders: Record = { - authorization: `Bearer ${client.token}`, - ...headers, - }; - - const init: RequestInit = { method, headers: reqHeaders }; - - if (body !== undefined && typeof body === "object" && !(body instanceof ReadableStream)) { - reqHeaders["content-type"] = "application/json"; - init.body = JSON.stringify(body); - } else if (body instanceof ReadableStream) { - init.body = body; - init.duplex = "half"; - } - - return fetch(url, init); -} - -// System boundary: Response.json() returns Promise in Node's lib types. -// We trust the shape of our own API responses at this boundary. -async function jsonBody(resp: Response): Promise { - return resp.json() as Promise; -} - -export interface SessionResponse { - readonly sessionId: string; - readonly r2Key: string; - readonly uploadMethod: string; - readonly expiresAt: string; -} - -export async function getMissingPaths( - client: ApiClient, - cache: string, - storePathHashes: string[], -): Promise { - const resp = await request(client, "POST", "/_api/v1/get-missing-paths", { - cache, - storePathHashes, - }); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`get-missing-paths failed (${resp.status}): ${text}`); - } - const data = await jsonBody<{ missing: string[] }>(resp); - return data.missing; -} - -export async function createUploadSession( - client: ApiClient, - cache: string, - params: { - storePath: string; - storePathHash: string; - narHash: string; - narSize: number; - fileHash: string; - fileSize: number; - compression: string; - references: string[]; - deriver?: string; - system?: string; - }, -): Promise { - const resp = await request( - client, - "POST", - `/_api/v1/caches/${cache}/upload-sessions`, - params, - ); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`create-session failed (${resp.status}): ${text}`); - } - return jsonBody(resp); -} - -export async function uploadBlob( - client: ApiClient, - sessionId: string, - filePath: string, -): Promise { - const fileInfo = await stat(filePath); - const stream = createReadStream(filePath); - const webStream = ReadableStream.from(stream); - - const resp = await request(client, "PUT", `/_api/v1/uploads/${sessionId}/blob`, webStream, { - "content-type": "application/octet-stream", - "content-length": String(fileInfo.size), - }); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`blob upload failed (${resp.status}): ${text}`); - } -} - -export async function completeUpload( - client: ApiClient, - sessionId: string, -): Promise { - const resp = await request(client, "POST", `/_api/v1/uploads/${sessionId}/complete`, {}); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`complete failed (${resp.status}): ${text}`); - } -} - -const MULTIPART_THRESHOLD = 90 * 1024 * 1024; // 90MB - -export function shouldUseMultipart(fileSize: number): boolean { - return fileSize > MULTIPART_THRESHOLD; -} - -export async function initiateMultipart( - client: ApiClient, - sessionId: string, -): Promise<{ uploadId: string; r2Key: string }> { - const resp = await request(client, "POST", `/_api/v1/uploads/${sessionId}/multipart`, {}); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`multipart initiation failed (${resp.status}): ${text}`); - } - return jsonBody<{ uploadId: string; r2Key: string }>(resp); -} - -export async function uploadPart( - client: ApiClient, - sessionId: string, - partNumber: number, - body: ReadableStream, - contentLength: number, -): Promise<{ partNumber: number; etag: string }> { - const resp = await request( - client, - "PUT", - `/_api/v1/uploads/${sessionId}/part/${String(partNumber)}`, - body, - { - "content-type": "application/octet-stream", - "content-length": String(contentLength), - }, - ); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`part upload failed (${resp.status}): ${text}`); - } - return jsonBody<{ partNumber: number; etag: string }>(resp); -} - -// Parts of a single blob upload in parallel. Kept modest because pushPaths -// already uploads several blobs concurrently. -const PART_UPLOAD_CONCURRENCY = 4; - -export async function uploadBlobMultipart( - client: ApiClient, - sessionId: string, - filePath: string, -): Promise { - const fileInfo = await stat(filePath); - const fileSize = Number(fileInfo.size); - const partSize = 90 * 1024 * 1024; // 90MB parts - - await initiateMultipart(client, sessionId); - - const totalParts = Math.ceil(fileSize / partSize); - const partNumbers = Array.from({ length: totalParts }, (_, i) => i + 1); - - // Parts are independent byte ranges of the same file, so they can be - // read and uploaded concurrently. - await runPool(partNumbers, PART_UPLOAD_CONCURRENCY, async (partNumber) => { - const start = (partNumber - 1) * partSize; - const end = Math.min(start + partSize, fileSize); - const length = end - start; - - const stream = createReadStream(filePath, { start, end: end - 1 }); - const webStream = ReadableStream.from(stream); - - await uploadPart(client, sessionId, partNumber, webStream, length); - }); -} - -export async function publishPath( - client: ApiClient, - sessionId: string, -): Promise<{ published: boolean; alreadyExisted?: boolean }> { - const resp = await request(client, "POST", `/_api/v1/uploads/${sessionId}/publish`, {}); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`publish failed (${resp.status}): ${text}`); - } - return jsonBody<{ published: boolean; alreadyExisted?: boolean }>(resp); -} - -// ── Token Management (Admin API) ── - -export interface TokenInfo { - readonly jti: string; - readonly subject: string; - readonly caches: string[]; - readonly perms: string[]; - readonly createdAt: string; - readonly expiresAt: string; - readonly createdBy: string; - readonly revokedAt: string | null; - readonly revokedBy: string | null; - readonly revocationReason: string | null; -} - -export interface CreateTokenResponse { - readonly token: string; - readonly jti: string; - readonly subject: string; - readonly caches: string[]; - readonly perms: string[]; - readonly expiresAt: string; -} - -export async function createToken( - client: ApiClient, - params: { - subject: string; - caches: string[]; - perms: string[]; - expiresInDays?: number; - }, -): Promise { - const resp = await request(client, "POST", "/_api/v1/admin/tokens", params); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`token create failed (${resp.status}): ${text}`); - } - return jsonBody(resp); -} - -export async function listTokens( - client: ApiClient, -): Promise { - const resp = await request(client, "GET", "/_api/v1/admin/tokens"); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`token list failed (${resp.status}): ${text}`); - } - const data = await jsonBody<{ tokens: TokenInfo[] }>(resp); - return data.tokens; -} - -export async function revokeToken( - client: ApiClient, - jti: string, - reason: string, -): Promise { - const resp = await request(client, "POST", `/_api/v1/admin/tokens/${jti}/revoke`, { reason }); - if (!resp.ok) { - const text = await resp.text(); - throw new Error(`token revoke failed (${resp.status}): ${text}`); - } -} diff --git a/apps/cli/src/config.ts b/apps/cli/src/config.ts deleted file mode 100644 index bbe8346..0000000 --- a/apps/cli/src/config.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { readFile, writeFile, mkdir } from "node:fs/promises"; -import { homedir } from "node:os"; -import { join } from "node:path"; - -interface ServerConfig { - readonly server: string; - readonly token: string; -} - -interface HeliosConfig { - readonly defaultServer?: string; - readonly servers: Record; -} - -const CONFIG_DIR = join(homedir(), ".config", "helios"); -const CONFIG_PATH = join(CONFIG_DIR, "config.json"); - -export async function loadConfig(): Promise { - try { - const raw = await readFile(CONFIG_PATH, "utf-8"); - return JSON.parse(raw); - } catch { - return { servers: {} }; - } -} - -export async function saveConfig(config: HeliosConfig): Promise { - await mkdir(CONFIG_DIR, { recursive: true }); - await writeFile(CONFIG_PATH, JSON.stringify(config, null, 2) + "\n", { - mode: 0o600, - }); -} - -export async function login( - name: string, - server: string, - token: string, -): Promise { - const config = await loadConfig(); - const servers = { ...config.servers, [name]: { server, token } }; - await saveConfig({ ...config, defaultServer: name, servers }); -} - -export async function getServer( - name?: string, -): Promise { - const config = await loadConfig(); - const key = name ?? config.defaultServer; - if (!key) { - throw new Error("No server specified and no default configured. Run: helios login "); - } - const entry = config.servers[key]; - if (!entry) { - throw new Error(`Server '${key}' not found in config. Run: helios login `); - } - return entry; -} diff --git a/apps/cli/src/main.ts b/apps/cli/src/main.ts deleted file mode 100644 index f596de4..0000000 --- a/apps/cli/src/main.ts +++ /dev/null @@ -1,219 +0,0 @@ -#!/usr/bin/env node - -import { login, getServer } from "./config.js"; -import { createClient, createToken, listTokens, revokeToken } from "./api.js"; -import { pushPaths } from "./push.js"; -import { getClosurePaths } from "./nix.js"; - -const USAGE = `helios - Nix binary cache CLI - -Commands: - helios login Save server credentials - helios push Push store paths to a cache - helios push --closure Push a store path and its closure - helios token create Create a new API token - helios token list List all API tokens - helios token revoke Revoke an API token - -Options: - --server Use a specific server (default: last logged-in) - --jobs Number of parallel uploads (default: 8) - --help Show this help - -Token create options: - --caches Comma-separated cache names or "*" (default: *) - --perms Comma-separated permissions (default: push) - --expires Token lifetime in days (default: 90) - -Examples: - helios login prod https://cache.example.com my-admin-secret - helios push main /nix/store/abc...-hello - helios push main --closure /run/current-system - helios token create ci-runner --caches main --perms push --expires 90 - helios token list - helios token revoke a1b2c3d4-... "employee offboarded" -`; - -async function main(): Promise { - const args = process.argv.slice(2); - - if (args.length === 0 || args.includes("--help") || args.includes("-h")) { - console.log(USAGE); - process.exit(0); - } - - const command = args[0]; - - if (command === "login") { - if (args.length < 4) { - console.error("Usage: helios login "); - process.exit(1); - } - await login(args[1], args[2], args[3]); - console.log(`Logged in to '${args[1]}' at ${args[2]}`); - return; - } - - if (command === "push") { - if (args.length < 3) { - console.error("Usage: helios push | helios push --closure "); - process.exit(1); - } - - const filteredArgs = [...args]; - - // Extract --server flag - let serverName: string | undefined; - const serverIdx = filteredArgs.indexOf("--server"); - if (serverIdx !== -1 && serverIdx + 1 < filteredArgs.length) { - serverName = filteredArgs[serverIdx + 1]; - filteredArgs.splice(serverIdx, 2); - } - - // Extract --jobs flag - let concurrency: number | undefined; - const jobsIdx = filteredArgs.indexOf("--jobs"); - if (jobsIdx !== -1 && jobsIdx + 1 < filteredArgs.length) { - concurrency = parseInt(filteredArgs[jobsIdx + 1], 10); - filteredArgs.splice(jobsIdx, 2); - } - - if (concurrency !== undefined && (!Number.isInteger(concurrency) || concurrency < 1)) { - console.error("--jobs must be a positive integer"); - process.exit(1); - } - - const serverConfig = await getServer(serverName); - const client = createClient(serverConfig.server, serverConfig.token); - const cache = filteredArgs[1]; - - const progress = (current: number, total: number, name: string, status: string) => { - console.log(`[${String(current)}/${String(total)}] ${name} ${status}`); - }; - - const pushOpts = { concurrency, onProgress: progress }; - - const closureIdx = filteredArgs.indexOf("--closure"); - let result; - if (closureIdx !== -1) { - const rootPath = filteredArgs[closureIdx + 1]; - if (!rootPath) { - console.error("--closure requires a store path or installable"); - process.exit(1); - } - console.log(`Resolving closure for ${rootPath}...`); - const paths = await getClosurePaths(rootPath); - console.log(`Found ${String(paths.length)} paths in closure`); - result = await pushPaths(client, cache, paths, pushOpts); - } else { - const paths = filteredArgs.slice(2); - if (paths.length === 0) { - console.error("No paths specified"); - process.exit(1); - } - result = await pushPaths(client, cache, paths, pushOpts); - } - - console.log(""); - console.log(`Done: ${String(result.pushed)} pushed, ${String(result.skipped)} skipped, ${String(result.failed)} failed`); - - if (result.failed > 0) { - process.exit(1); - } - return; - } - - if (command === "token") { - const subcommand = args[1]; - - // Extract --server flag - let serverName: string | undefined; - const serverIdx = args.indexOf("--server"); - if (serverIdx !== -1 && serverIdx + 1 < args.length) { - serverName = args[serverIdx + 1]; - } - - const serverConfig = await getServer(serverName); - const client = createClient(serverConfig.server, serverConfig.token); - - if (subcommand === "create") { - const subject = args[2]; - if (!subject) { - console.error("Usage: helios token create [--caches ...] [--perms ...] [--expires ]"); - process.exit(1); - } - - let cachesStr = "*"; - const cachesIdx = args.indexOf("--caches"); - if (cachesIdx !== -1 && cachesIdx + 1 < args.length) { - cachesStr = args[cachesIdx + 1]; - } - - let permsStr = "push"; - const permsIdx = args.indexOf("--perms"); - if (permsIdx !== -1 && permsIdx + 1 < args.length) { - permsStr = args[permsIdx + 1]; - } - - let expiresInDays: number | undefined; - const expiresIdx = args.indexOf("--expires"); - if (expiresIdx !== -1 && expiresIdx + 1 < args.length) { - expiresInDays = parseInt(args[expiresIdx + 1], 10); - if (!Number.isInteger(expiresInDays) || expiresInDays < 1) { - console.error("--expires must be a positive integer"); - process.exit(1); - } - } - - const caches = cachesStr.split(",").map(s => s.trim()).filter(Boolean); - const perms = permsStr.split(",").map(s => s.trim()).filter(Boolean); - - const result = await createToken(client, { subject, caches, perms, expiresInDays }); - console.log(`Token created for '${result.subject}'`); - console.log(` JTI: ${result.jti}`); - console.log(` Caches: ${result.caches.join(", ")}`); - console.log(` Perms: ${result.perms.join(", ")}`); - console.log(` Expires: ${result.expiresAt}`); - console.log(""); - console.log(result.token); - return; - } - - if (subcommand === "list") { - const tokens = await listTokens(client); - if (tokens.length === 0) { - console.log("No tokens found"); - return; - } - for (const t of tokens) { - const status = t.revokedAt ? `revoked (${t.revokedAt})` : "active"; - console.log(`${t.jti} ${t.subject} [${t.perms.join(",")}] caches=[${t.caches.join(",")}] ${status} expires=${t.expiresAt}`); - } - return; - } - - if (subcommand === "revoke") { - const jti = args[2]; - const reason = args.slice(3).join(" "); - if (!jti || !reason) { - console.error("Usage: helios token revoke "); - process.exit(1); - } - await revokeToken(client, jti, reason); - console.log(`Token ${jti} revoked`); - return; - } - - console.error("Usage: helios token "); - process.exit(1); - } - - console.error(`Unknown command: ${command}`); - console.error("Run 'helios --help' for usage"); - process.exit(1); -} - -main().catch((err: unknown) => { - console.error(err instanceof Error ? err.message : String(err)); - process.exit(1); -}); diff --git a/apps/cli/src/nix.ts b/apps/cli/src/nix.ts deleted file mode 100644 index d5e74e7..0000000 --- a/apps/cli/src/nix.ts +++ /dev/null @@ -1,176 +0,0 @@ -import { execFile, spawn } from "node:child_process"; -import { createHash } from "node:crypto"; -import { createWriteStream } from "node:fs"; -import { promisify } from "node:util"; - -const exec = promisify(execFile); - -export interface PathInfo { - readonly storePath: string; - readonly storePathHash: string; - readonly narHash: string; - readonly narSize: number; - readonly references: string[]; - readonly deriver?: string; -} - -interface NixPathInfoJson { - readonly narHash: string; - readonly narSize: number; - readonly references: string[]; - readonly deriver?: string; -} - -// nix path-info output for a large batch (references lists included) can far -// exceed execFile's default 1 MiB stdout buffer. -const EXEC_MAX_BUFFER_BYTES = 64 * 1024 * 1024; - -// Bound argv size per nix invocation; store paths are ~90 bytes each. -const PATH_INFO_BATCH_SIZE = 500; - -function toPathInfo(storePath: string, entry: NixPathInfoJson): PathInfo { - const basename = storePath.split("/").pop(); - if (!basename) { - throw new Error(`Invalid store path: ${storePath}`); - } - const storePathHash = basename.split("-")[0]; - - const refs = entry.references.map((r: string) => { - const refBase = r.split("/").pop(); - if (!refBase) { - throw new Error(`Invalid reference path: ${r}`); - } - return refBase.split("-")[0]; - }); - - return { - storePath, - storePathHash, - narHash: entry.narHash, - narSize: entry.narSize, - references: refs, - deriver: entry.deriver, - }; -} - -/** - * Query path info for many store paths with a bounded number of nix - * invocations, instead of one process spawn per path. - */ -export async function getPathInfos( - storePaths: readonly string[], -): Promise> { - const infos = new Map(); - - for (let i = 0; i < storePaths.length; i += PATH_INFO_BATCH_SIZE) { - const chunk = storePaths.slice(i, i + PATH_INFO_BATCH_SIZE); - const { stdout } = await exec("nix", ["path-info", "--json", ...chunk], { - maxBuffer: EXEC_MAX_BUFFER_BYTES, - }); - const parsed: Record = JSON.parse(stdout); - - for (const storePath of chunk) { - const entry = parsed[storePath]; - if (!entry) { - throw new Error(`nix path-info returned no data for ${storePath}`); - } - infos.set(storePath, toPathInfo(storePath, entry)); - } - } - - return infos; -} - -export async function getClosurePaths(storePath: string): Promise { - const { stdout } = await exec("nix", ["path-info", "-r", storePath], { - maxBuffer: EXEC_MAX_BUFFER_BYTES, - }); - return stdout.trim().split("\n").filter(Boolean); -} - -export interface CompressedNar { - readonly fileHash: string; - readonly fileSize: number; -} - -/** - * Dump a store path as a NAR, compress it with zstd, and write it to - * outputPath. The compressed stream is hashed and counted while it is - * written, so callers do not need to re-read the file to compute the - * upload hash and size. - */ -export async function dumpAndCompress( - storePath: string, - outputPath: string, -): Promise { - const nix = spawn("nix", ["store", "dump-path", storePath], { - stdio: ["ignore", "pipe", "ignore"], - }); - const zstd = spawn("zstd", ["-q", "-c"], { - stdio: ["pipe", "pipe", "ignore"], - }); - - nix.stdout.pipe(zstd.stdin); - - // Observe the piped chunks to hash and count them; pipe() still owns - // backpressure to the file stream. - const hash = createHash("sha256"); - let fileSize = 0; - zstd.stdout.on("data", (chunk: Buffer) => { - hash.update(chunk); - fileSize += chunk.length; - }); - - const out = createWriteStream(outputPath); - zstd.stdout.pipe(out); - - await new Promise((resolve, reject) => { - let nixExited = false; - let zstdExited = false; - let fileFlushed = false; - let settled = false; - - const fail = (err: Error) => { - if (!settled) { - settled = true; - reject(err); - } - }; - - const tryResolve = () => { - if (nixExited && zstdExited && fileFlushed && !settled) { - settled = true; - resolve(); - } - }; - - nix.on("error", fail); - zstd.on("error", fail); - out.on("error", fail); - - nix.on("close", (code) => { - nixExited = true; - if (code !== 0) { - fail(new Error(`nix store dump-path exited with ${String(code)}`)); - return; - } - tryResolve(); - }); - - zstd.on("close", (code) => { - zstdExited = true; - if (code !== 0) { - fail(new Error(`zstd exited with ${String(code)}`)); - return; - } - tryResolve(); - }); - - out.on("finish", () => { - fileFlushed = true; - tryResolve(); - }); - }); - - return { fileHash: hash.digest("hex"), fileSize }; -} diff --git a/apps/cli/src/pool.ts b/apps/cli/src/pool.ts deleted file mode 100644 index a42cd03..0000000 --- a/apps/cli/src/pool.ts +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Run fn over items with bounded concurrency. Items are claimed in order; - * the returned promise settles after every item has been processed. - * Rejections propagate to the caller. - */ -export async function runPool( - items: readonly T[], - concurrency: number, - fn: (item: T, index: number) => Promise, -): Promise { - let next = 0; - - async function worker(): Promise { - while (next < items.length) { - const idx = next; - next++; - await fn(items[idx], idx); - } - } - - const workers: Promise[] = []; - for (let i = 0; i < Math.min(concurrency, items.length); i++) { - workers.push(worker()); - } - await Promise.all(workers); -} diff --git a/apps/cli/src/push.ts b/apps/cli/src/push.ts deleted file mode 100644 index e4cba5e..0000000 --- a/apps/cli/src/push.ts +++ /dev/null @@ -1,147 +0,0 @@ -import { mkdtemp, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import type { ApiClient } from "./api.js"; -import { - getMissingPaths, - createUploadSession, - uploadBlob, - shouldUseMultipart, - uploadBlobMultipart, - completeUpload, - publishPath, -} from "./api.js"; -import { getPathInfos, dumpAndCompress } from "./nix.js"; -import type { PathInfo } from "./nix.js"; -import { runPool } from "./pool.js"; - -const DEFAULT_CONCURRENCY = 8; - -export interface PushResult { - readonly pushed: number; - readonly skipped: number; - readonly failed: number; - readonly errors: string[]; -} - -interface PushOptions { - readonly concurrency?: number; - readonly onProgress?: (current: number, total: number, name: string, status: string) => void; -} - -async function pushSinglePath( - client: ApiClient, - cache: string, - storePath: string, - info: PathInfo, -): Promise<{ sizeKb: number }> { - const narDir = await mkdtemp(join(tmpdir(), "helios-nar-")); - const narFile = join(narDir, "nar.zst"); - - try { - // Hash and size are computed while the compressed NAR is written, - // avoiding a second full read of the file. - const { fileHash, fileSize } = await dumpAndCompress(storePath, narFile); - - const session = await createUploadSession(client, cache, { - storePath: info.storePath, - storePathHash: info.storePathHash, - narHash: info.narHash, - narSize: info.narSize, - fileHash, - fileSize, - compression: "zstd", - references: info.references, - }); - - if (shouldUseMultipart(fileSize)) { - await uploadBlobMultipart(client, session.sessionId, narFile); - } else { - await uploadBlob(client, session.sessionId, narFile); - } - await completeUpload(client, session.sessionId); - await publishPath(client, session.sessionId); - - return { sizeKb: Math.round(fileSize / 1024) }; - } finally { - await rm(narDir, { recursive: true, force: true }); - } -} - -export async function pushPaths( - client: ApiClient, - cache: string, - storePaths: string[], - options?: PushOptions, -): Promise { - const concurrency = options?.concurrency ?? DEFAULT_CONCURRENCY; - const onProgress = options?.onProgress; - - const hashToPath = new Map(); - const hashes: string[] = []; - for (const p of storePaths) { - const basename = p.split("/").pop(); - if (!basename) continue; - const hash = basename.split("-")[0]; - hashToPath.set(hash, p); - hashes.push(hash); - } - - // Check missing in sequential batches (server chunks internally for D1 limits) - const missingSet = new Set(); - for (let i = 0; i < hashes.length; i += 500) { - const chunk = hashes.slice(i, i + 500); - const missing = await getMissingPaths(client, cache, chunk); - for (const h of missing) { - missingSet.add(h); - } - } - - const skipped = storePaths.length - missingSet.size; - let pushed = 0; - let failed = 0; - const errors: string[] = []; - let completed = 0; - - const missingHashes = hashes.filter((h) => missingSet.has(h)); - const total = missingHashes.length; - - if (total === 0) { - return { pushed: 0, skipped, failed: 0, errors: [] }; - } - - // One batched nix path-info call for all missing paths, instead of one - // process spawn per path inside the upload pool. - const missingPaths: string[] = []; - for (const hash of missingHashes) { - const storePath = hashToPath.get(hash); - if (storePath) missingPaths.push(storePath); - } - const pathInfos = await getPathInfos(missingPaths); - - await runPool(missingHashes, concurrency, async (hash) => { - const storePath = hashToPath.get(hash); - if (!storePath) return; - const name = storePath.split("/").pop() ?? hash; - - try { - const info = pathInfos.get(storePath); - if (!info) { - throw new Error(`missing path info for ${storePath}`); - } - const result = await pushSinglePath(client, cache, storePath, info); - completed++; - onProgress?.(completed, total, name, `ok (${String(result.sizeKb)}KB)`); - pushed++; - } catch (err) { - completed++; - const msg = err instanceof Error ? err.message : String(err); - errors.push(`${name}: ${msg}`); - onProgress?.(completed, total, name, `FAIL: ${msg}`); - failed++; - } - }); - - return { pushed, skipped, failed, errors }; -} - diff --git a/apps/cli/tsconfig.json b/apps/cli/tsconfig.json deleted file mode 100644 index 1a87488..0000000 --- a/apps/cli/tsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "compilerOptions": { - "target": "es2024", - "lib": ["es2024"], - "module": "nodenext", - "moduleResolution": "nodenext", - "outDir": "dist", - "rootDir": "src", - "strict": true, - "isolatedModules": true, - "forceConsistentCasingInFileNames": true, - "skipLibCheck": true, - "declaration": false, - "sourceMap": false - }, - "include": ["src/**/*.ts"] -} diff --git a/flake.nix b/flake.nix index 969a727..d12247a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,5 +1,5 @@ { - description = "Helios – Cloudflare-native Nix binary cache"; + description = "Helios: self-hosted Nix binary cache (Rust server and CLI, Zig core)"; inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; @@ -7,89 +7,63 @@ outputs = { self, nixpkgs }: let - systems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]; + # NAR serialisation is implemented for Linux; macOS builds are untested. + systems = [ "x86_64-linux" "aarch64-linux" ]; forAllSystems = fn: nixpkgs.lib.genAttrs systems (system: fn nixpkgs.legacyPackages.${system}); - pnpmDepsFor = pkgs: pkgs.fetchPnpmDeps { + helios' = pkgs: pkgs.rustPlatform.buildRustPackage { pname = "helios"; - version = "0.0.0"; - src = self; - hash = "sha256-FsWEGFPLmT7jU7QIv2FGqVHOdNMUqvqmnWmog2Wpfuw="; - fetcherVersion = 3; + version = "0.1.0"; + src = pkgs.lib.cleanSourceWith { + src = self; + filter = path: _type: + let base = baseNameOf path; in + !(builtins.elem base [ "target" ".zig-cache" "zig-out" "bench" ]); + }; + cargoLock.lockFile = ./Cargo.lock; + + nativeBuildInputs = [ pkgs.zig pkgs.pkg-config ]; + buildInputs = [ pkgs.zstd ]; + + # Nix builds must not depend on the build machine's CPU. `baseline` + # loses SHA-NI accelerated hashing; for a dedicated host, build with + # HELIOS_ZIG_CPU=native (or e.g. x86_64_v3+sha) instead. + HELIOS_ZIG_CPU = "baseline"; + # zig is a build tool of helios-core's build.rs, not the package builder. + dontUseZigBuild = true; + dontUseZigCheck = true; + dontUseZigInstall = true; + + meta = { + description = "Self-hosted Nix binary cache"; + mainProgram = "helios"; + }; }; in { - packages = forAllSystems (pkgs: { - default = pkgs.stdenv.mkDerivation { - pname = "helios"; - version = "0.0.0"; - src = self; - - nativeBuildInputs = [ - pkgs.nodejs_22 - pkgs.pnpm - pkgs.pnpmConfigHook - pkgs.makeWrapper - ]; - - pnpmDeps = pnpmDepsFor pkgs; - - buildPhase = '' - pnpm --filter @helios/cli exec tsc - ''; - - installPhase = '' - mkdir -p $out/lib/helios $out/bin - - # Copy the full workspace structure so pnpm symlinks resolve - cp -r apps $out/lib/helios/apps - cp -r packages $out/lib/helios/packages - cp -r workers $out/lib/helios/workers - cp -r node_modules $out/lib/helios/node_modules - cp package.json $out/lib/helios/package.json - - makeWrapper ${pkgs.nodejs_22}/bin/node $out/bin/helios \ - --add-flags "$out/lib/helios/apps/cli/dist/main.js" \ - --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.nix pkgs.zstd ]} - ''; - }; + packages = forAllSystems (pkgs: rec { + helios = helios' pkgs; + default = helios; }); devShells = forAllSystems (pkgs: { default = pkgs.mkShell { packages = [ - pkgs.nodejs_22 - pkgs.pnpm - pkgs.wrangler + pkgs.zig + pkgs.cargo + pkgs.rustc + pkgs.clippy + pkgs.rustfmt + pkgs.pkg-config + pkgs.zstd + pkgs.jq + pkgs.curl ]; }; }); checks = forAllSystems (pkgs: { - default = pkgs.stdenv.mkDerivation { - name = "helios-check"; - src = self; - - nativeBuildInputs = [ - pkgs.nodejs_22 - pkgs.pnpm - pkgs.pnpmConfigHook - ]; - - pnpmDeps = pnpmDepsFor pkgs; - - buildPhase = '' - # Domain package: full check (tsc + unit tests) - pnpm --filter @helios/cache-domain check - - # Worker: type-check only (integration tests need workerd runtime) - pnpm --filter @helios/cache-worker exec tsc --noEmit - ''; - - installPhase = '' - touch $out - ''; - }; + default = self.packages.${pkgs.stdenv.hostPlatform.system}.helios; }); }; } diff --git a/package.json b/package.json deleted file mode 100644 index 67a4975..0000000 --- a/package.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "packageManager": "pnpm@11.15.1", - "type": "module", - "name": "helios", - "private": true, - "license": "SEE LICENSE IN LICENSE", - "description": "Cloudflare-native Nix binary cache", - "scripts": { - "build": "echo 'wrangler bundles at deploy time'", - "deploy": "pnpm exec wrangler d1 migrations apply CACHE_DB --remote && pnpm exec wrangler deploy", - "dev": "pnpm --filter @helios/cache-worker dev", - "check": "pnpm -r check" - }, - "cloudflare": { - "bindings": { - "AUTH_TOKEN": { - "description": "Bearer token for write operations (push). Generate with `openssl rand -hex 32`." - }, - "SIGNING_PRIVATE_KEY": { - "description": "Ed25519 private key in PEM format for signing narinfo. Generate with `nix key generate-secret --key-name your-cache-1`." - } - } - }, - "devDependencies": { - "wrangler": "^4.112.0" - } -} diff --git a/packages/cache-domain/package.json b/packages/cache-domain/package.json deleted file mode 100644 index cce22ec..0000000 --- a/packages/cache-domain/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "@helios/cache-domain", - "version": "0.0.0", - "private": true, - "type": "module", - "exports": { - ".": "./src/index.ts" - }, - "scripts": { - "test": "vitest run", - "check": "tsc --noEmit && vitest run" - }, - "devDependencies": { - "typescript": "^7.0.2", - "vitest": "~4.1.10" - } -} diff --git a/packages/cache-domain/src/index.ts b/packages/cache-domain/src/index.ts deleted file mode 100644 index 1d90619..0000000 --- a/packages/cache-domain/src/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -export type { - Brand, - CacheName, - StorePathHash, - NarHash, - FileHash, - Compression, - R2ObjectKey, - NarInfoFields, - ParseError, -} from './types.js'; - -export { - parseCacheName, - parseStorePathHash, - parseNarHash, - parseFileHash, - parseCompression, - buildR2ObjectKey, -} from './parsers.js'; diff --git a/packages/cache-domain/src/parsers.ts b/packages/cache-domain/src/parsers.ts deleted file mode 100644 index 15405d8..0000000 --- a/packages/cache-domain/src/parsers.ts +++ /dev/null @@ -1,70 +0,0 @@ -import type { - CacheName, - Compression, - FileHash, - NarHash, - ParseError, - R2ObjectKey, - StorePathHash, -} from './types.js'; - -const NIX_BASE32_CHARS = "0123456789abcdfghijklmnpqrsvwxyz"; -const NIX_BASE32_RE = new RegExp(`^[${NIX_BASE32_CHARS}]+$`); -const HEX_RE = /^[0-9a-f]{64}$/; -const CACHE_NAME_RE = /^[a-z0-9]([a-z0-9-]{0,62}[a-z0-9])?$/; -const VALID_COMPRESSIONS = new Set(["zstd", "xz", "bzip2", "none"]); - -function err(message: string): ParseError { - return { kind: "ParseError", message }; -} - -export function parseCacheName(raw: string): CacheName | ParseError { - if (!CACHE_NAME_RE.test(raw)) { - return err( - "CacheName must be 1-64 lowercase alphanumeric chars or hyphens, no leading/trailing hyphen", - ); - } - return raw as CacheName; -} - -export function parseStorePathHash(raw: string): StorePathHash | ParseError { - if (raw.length !== 32) { - return err("StorePathHash must be exactly 32 characters"); - } - if (!NIX_BASE32_RE.test(raw)) { - return err("StorePathHash contains invalid nix base-32 characters"); - } - return raw as StorePathHash; -} - -export function parseNarHash(raw: string): NarHash | ParseError { - if (!raw.startsWith("sha256:")) { - return err("NarHash must start with 'sha256:'"); - } - const hash = raw.slice(7); - if (hash.length !== 52) { - return err("NarHash hash part must be exactly 52 characters"); - } - if (!NIX_BASE32_RE.test(hash)) { - return err("NarHash contains invalid nix base-32 characters"); - } - return raw as NarHash; -} - -export function parseFileHash(raw: string): FileHash | ParseError { - if (!HEX_RE.test(raw)) { - return err("FileHash must be exactly 64 lowercase hex characters"); - } - return raw as FileHash; -} - -export function parseCompression(raw: string): Compression | ParseError { - if (!VALID_COMPRESSIONS.has(raw as Compression)) { - return err(`Compression must be one of: ${[...VALID_COMPRESSIONS].join(", ")}`); - } - return raw as Compression; -} - -export function buildR2ObjectKey(fileHash: FileHash, compression: Compression): R2ObjectKey { - return `nars/sha256/${fileHash}/${compression}.nar` as R2ObjectKey; -} diff --git a/packages/cache-domain/src/types.ts b/packages/cache-domain/src/types.ts deleted file mode 100644 index 3ecb3ad..0000000 --- a/packages/cache-domain/src/types.ts +++ /dev/null @@ -1,27 +0,0 @@ -export type Brand = T & { readonly __brand: B }; - -export type CacheName = Brand; -export type StorePathHash = Brand; -export type NarHash = Brand; -export type FileHash = Brand; -export type Compression = "zstd" | "xz" | "bzip2" | "none"; -export type R2ObjectKey = Brand; - -export interface NarInfoFields { - readonly storePath: string; - readonly storePathHash: StorePathHash; - readonly narHash: NarHash; - readonly narSize: number; - readonly fileHash: FileHash; - readonly fileSize: number; - readonly compression: Compression; - readonly references: readonly StorePathHash[]; - readonly deriver?: string; - readonly system?: string; - readonly signatures: readonly string[]; -} - -export interface ParseError { - readonly kind: "ParseError"; - readonly message: string; -} diff --git a/packages/cache-domain/test/parsers.test.ts b/packages/cache-domain/test/parsers.test.ts deleted file mode 100644 index 0f022ef..0000000 --- a/packages/cache-domain/test/parsers.test.ts +++ /dev/null @@ -1,145 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { - parseCacheName, - parseStorePathHash, - parseNarHash, - parseFileHash, - parseCompression, - buildR2ObjectKey, -} from '../src/index.js'; -import type { FileHash, Compression } from '../src/index.js'; - -function isParseError(result: unknown): result is { kind: 'ParseError'; message: string } { - return ( - typeof result === 'object' && - result !== null && - (result as { kind?: string }).kind === 'ParseError' - ); -} - -describe('parseCacheName', () => { - it.each(['my-cache', 'a', 'a1b2', 'abc', 'a-b'])('accepts valid name %j', (input) => { - const result = parseCacheName(input); - expect(isParseError(result)).toBe(false); - expect(result).toBe(input); - }); - - it.each([ - { input: '', label: 'empty string' }, - { input: 'My-Cache', label: 'uppercase' }, - { input: '-leading', label: 'leading hyphen' }, - { input: 'trailing-', label: 'trailing hyphen' }, - { input: 'has spaces', label: 'spaces' }, - { input: 'a'.repeat(65), label: '65-char string' }, - { input: 'has_underscore', label: 'underscore' }, - ])('rejects $label', ({ input }) => { - const result = parseCacheName(input); - expect(isParseError(result)).toBe(true); - }); -}); - -describe('parseStorePathHash', () => { - it('accepts 32 valid nix-base32 chars', () => { - const input = '0v9z1234abcdfghijklmnpqrsvwxyz01'; - const result = parseStorePathHash(input); - expect(isParseError(result)).toBe(false); - expect(result).toBe(input); - }); - - it.each([ - { input: '0'.repeat(31), label: '31 chars (too short)' }, - { input: '0'.repeat(33), label: '33 chars (too long)' }, - { input: '0'.repeat(31) + 'e', label: "contains 'e'" }, - { input: '0'.repeat(31) + 'o', label: "contains 'o'" }, - { input: '0'.repeat(31) + 't', label: "contains 't'" }, - { input: '0'.repeat(31) + 'u', label: "contains 'u'" }, - { input: '0'.repeat(31) + 'A', label: 'uppercase letters' }, - ])('rejects $label', ({ input }) => { - const result = parseStorePathHash(input); - expect(isParseError(result)).toBe(true); - }); -}); - -describe('parseNarHash', () => { - it('accepts sha256: prefix + 52 valid nix-base32 chars (zeros)', () => { - const input = 'sha256:' + '0'.repeat(52); - const result = parseNarHash(input); - expect(isParseError(result)).toBe(false); - expect(result).toBe(input); - }); - - it('accepts sha256: prefix + 52 mixed valid nix-base32 chars', () => { - const input = 'sha256:' + 'abcdfghijklmnpqrsvwxyz0123456789abcdfghijklmnpqrsvwx'; - const result = parseNarHash(input); - expect(isParseError(result)).toBe(false); - expect(result).toBe(input); - }); - - it.each([ - { input: '0'.repeat(52), label: 'no sha256: prefix' }, - { input: 'md5:' + '0'.repeat(52), label: 'md5: prefix' }, - { input: 'sha256:' + '0'.repeat(51), label: '51 hash chars' }, - { input: 'sha256:' + '0'.repeat(53), label: '53 hash chars' }, - { input: 'sha256:' + '0'.repeat(51) + 'e', label: "hash contains 'e'" }, - ])('rejects $label', ({ input }) => { - const result = parseNarHash(input); - expect(isParseError(result)).toBe(true); - }); -}); - -describe('parseFileHash', () => { - it('accepts 64 lowercase hex chars (repeated a)', () => { - const input = 'a'.repeat(64); - const result = parseFileHash(input); - expect(isParseError(result)).toBe(false); - expect(result).toBe(input); - }); - - it('accepts 64 lowercase hex chars (mixed)', () => { - const input = '0123456789abcdef'.repeat(4); - const result = parseFileHash(input); - expect(isParseError(result)).toBe(false); - expect(result).toBe(input); - }); - - it.each([ - { input: 'a'.repeat(63), label: '63 chars' }, - { input: 'a'.repeat(65), label: '65 chars' }, - { input: 'a'.repeat(63) + 'g', label: "contains 'g'" }, - { input: 'A'.repeat(64), label: 'uppercase hex' }, - ])('rejects $label', ({ input }) => { - const result = parseFileHash(input); - expect(isParseError(result)).toBe(true); - }); -}); - -describe('parseCompression', () => { - it.each(['zstd', 'xz', 'bzip2', 'none'] as const)('accepts %j', (input) => { - const result = parseCompression(input); - expect(isParseError(result)).toBe(false); - expect(result).toBe(input); - }); - - it.each([ - { input: 'gzip', label: 'gzip' }, - { input: '', label: 'empty string' }, - { input: 'ZSTD', label: 'uppercase ZSTD' }, - ])('rejects $label', ({ input }) => { - const result = parseCompression(input); - expect(isParseError(result)).toBe(true); - }); -}); - -describe('buildR2ObjectKey', () => { - const fileHash = 'a'.repeat(64) as FileHash; - - it('builds key with zstd compression', () => { - const result = buildR2ObjectKey(fileHash, 'zstd' as Compression); - expect(result).toBe(`nars/sha256/${'a'.repeat(64)}/zstd.nar`); - }); - - it('builds key with none compression', () => { - const result = buildR2ObjectKey(fileHash, 'none' as Compression); - expect(result).toBe(`nars/sha256/${'a'.repeat(64)}/none.nar`); - }); -}); diff --git a/packages/cache-domain/tsconfig.json b/packages/cache-domain/tsconfig.json deleted file mode 100644 index 17187e0..0000000 --- a/packages/cache-domain/tsconfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "compilerOptions": { - "target": "es2024", - "lib": ["es2024"], - "module": "es2022", - "moduleResolution": "Bundler", - "resolveJsonModule": true, - "noEmit": true, - "isolatedModules": true, - "allowSyntheticDefaultImports": true, - "forceConsistentCasingInFileNames": true, - "strict": true, - "skipLibCheck": true - }, - "include": ["src/**/*.ts"], - "exclude": ["test"] -} diff --git a/packages/cache-domain/vitest.config.mts b/packages/cache-domain/vitest.config.mts deleted file mode 100644 index 43e56f4..0000000 --- a/packages/cache-domain/vitest.config.mts +++ /dev/null @@ -1,7 +0,0 @@ -import { defineConfig } from 'vitest/config'; - -export default defineConfig({ - test: { - include: ['test/**/*.test.ts'], - }, -}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml deleted file mode 100644 index ed212ed..0000000 --- a/pnpm-lock.yaml +++ /dev/null @@ -1,2723 +0,0 @@ -lockfileVersion: '9.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - -importers: - - .: - devDependencies: - wrangler: - specifier: ^4.112.0 - version: 4.112.0 - - apps/cli: - dependencies: - '@helios/cache-domain': - specifier: workspace:* - version: link:../../packages/cache-domain - devDependencies: - '@types/node': - specifier: ^26.1.1 - version: 26.1.1 - typescript: - specifier: ^7.0.2 - version: 7.0.2 - - packages/cache-domain: - devDependencies: - typescript: - specifier: ^7.0.2 - version: 7.0.2 - vitest: - specifier: ~4.1.10 - version: 4.1.10(@types/node@26.1.1)(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0)) - - workers/cache: - dependencies: - '@helios/cache-domain': - specifier: workspace:* - version: link:../../packages/cache-domain - aws4fetch: - specifier: ^1.0.20 - version: 1.0.20 - drizzle-orm: - specifier: ^0.45.2 - version: 0.45.2 - devDependencies: - '@cloudflare/vitest-pool-workers': - specifier: ^0.18.6 - version: 0.18.6(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10(@types/node@26.1.1)(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0))) - '@types/node': - specifier: ^26.1.1 - version: 26.1.1 - drizzle-kit: - specifier: ^0.31.10 - version: 0.31.10 - typescript: - specifier: ^7.0.2 - version: 7.0.2 - vitest: - specifier: ~4.1.10 - version: 4.1.10(@types/node@26.1.1)(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0)) - wrangler: - specifier: ^4.112.0 - version: 4.112.0 - -packages: - - '@cloudflare/kv-asset-handler@0.5.0': - resolution: {integrity: sha512-jxQYkj8dSIzc0cD6cMMNdOc1UVjqSqu8BZdor5s8cGjW2I8BjODt/kWPVdY+u9zj3ms75Q5qaZgnxUad83+eAg==} - engines: {node: '>=22.0.0'} - - '@cloudflare/unenv-preset@2.16.1': - resolution: {integrity: sha512-ECxObrMfyTl5bhQf/lZCXwo5G6xX9IAUo+nDMKK4SZ8m4Jvvxp52vilxyySSWh2YTZz8+HQ07qGH/2rEom1vDw==} - peerDependencies: - unenv: 2.0.0-rc.24 - workerd: '>1.20260305.0 <2.0.0-0' - peerDependenciesMeta: - workerd: - optional: true - - '@cloudflare/vitest-pool-workers@0.18.6': - resolution: {integrity: sha512-6JGqaQsQRZIVq/6jEC4ouJnShZriPIJ2X0yGndwMm+SiPP93pJi5Dp30dYAoztNrNJC7wWK7ec5slLfMBMZ8jA==} - peerDependencies: - '@vitest/runner': ^4.1.0 - '@vitest/snapshot': ^4.1.0 - vitest: ^4.1.0 - - '@cloudflare/workerd-darwin-64@1.20260714.1': - resolution: {integrity: sha512-ZWXqAN8G7Cx9hMRQuk+59ziJhR3j1F4iO+Qs8aHdfKZ3Dq5Yi/57xvkJTgCGBnW1YU/L78r8f6HEy51bwbTpNw==} - engines: {node: '>=16'} - cpu: [x64] - os: [darwin] - - '@cloudflare/workerd-darwin-arm64@1.20260714.1': - resolution: {integrity: sha512-tueWxWC3wyCbMG6zRAxsMXX0YLgrRWbiAPYFQ2uJ7dUH8G+5E7UTWaQS9B1HdJ0bpKFW1NWxhs1o2noKVFSUYg==} - engines: {node: '>=16'} - cpu: [arm64] - os: [darwin] - - '@cloudflare/workerd-linux-64@1.20260714.1': - resolution: {integrity: sha512-1VChTZRb0l0F7R4e1G5RtLKV4oFi6x+rQgxh2+yu887j3l/3TLgatuv1L8/5zhc9gKEhATTxOh0e52Rtd9dDWQ==} - engines: {node: '>=16'} - cpu: [x64] - os: [linux] - - '@cloudflare/workerd-linux-arm64@1.20260714.1': - resolution: {integrity: sha512-rMm3G+NirG2UdgHIRDdF1asNC6FqgIzZzkRG+VDhhDGcVxAQwvrMT1E38BivEvHr3G04MB4AfhcOczX0+GtRkQ==} - engines: {node: '>=16'} - cpu: [arm64] - os: [linux] - - '@cloudflare/workerd-windows-64@1.20260714.1': - resolution: {integrity: sha512-cGqnU3Hg2YZS/k3SAqrMp1DjpdsyFde72tWltdl6ZT9+SFz/Zrk/8gyTU1TcxC4YApXeNVH5TyU5cOGPgUJ0pg==} - engines: {node: '>=16'} - cpu: [x64] - os: [win32] - - '@cspotcode/source-map-support@0.8.1': - resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} - engines: {node: '>=12'} - - '@drizzle-team/brocli@0.10.2': - resolution: {integrity: sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w==} - - '@emnapi/runtime@1.10.0': - resolution: {integrity: sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==} - - '@esbuild-kit/core-utils@3.3.2': - resolution: {integrity: sha512-sPRAnw9CdSsRmEtnsl2WXWdyquogVpB3yZ3dgwJfe8zrOzTsV7cJvmwrKVa+0ma5BoiGJ+BoqkMvawbayKUsqQ==} - deprecated: 'Merged into tsx: https://tsx.hirok.io' - - '@esbuild-kit/esm-loader@2.6.5': - resolution: {integrity: sha512-FxEMIkJKnodyA1OaCUoEvbYRkoZlLZ4d/eXFu9Fh8CbBBgP5EmZxrfTRyN0qpXZ4vOvqnE5YdRdcrmUUXuU+dA==} - deprecated: 'Merged into tsx: https://tsx.hirok.io' - - '@esbuild/aix-ppc64@0.25.12': - resolution: {integrity: sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [aix] - - '@esbuild/aix-ppc64@0.27.7': - resolution: {integrity: sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [aix] - - '@esbuild/aix-ppc64@0.28.1': - resolution: {integrity: sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [aix] - - '@esbuild/android-arm64@0.18.20': - resolution: {integrity: sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ==} - engines: {node: '>=12'} - cpu: [arm64] - os: [android] - - '@esbuild/android-arm64@0.25.12': - resolution: {integrity: sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [android] - - '@esbuild/android-arm64@0.27.7': - resolution: {integrity: sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==} - engines: {node: '>=18'} - cpu: [arm64] - os: [android] - - '@esbuild/android-arm64@0.28.1': - resolution: {integrity: sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [android] - - '@esbuild/android-arm@0.18.20': - resolution: {integrity: sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw==} - engines: {node: '>=12'} - cpu: [arm] - os: [android] - - '@esbuild/android-arm@0.25.12': - resolution: {integrity: sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==} - engines: {node: '>=18'} - cpu: [arm] - os: [android] - - '@esbuild/android-arm@0.27.7': - resolution: {integrity: sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==} - engines: {node: '>=18'} - cpu: [arm] - os: [android] - - '@esbuild/android-arm@0.28.1': - resolution: {integrity: sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==} - engines: {node: '>=18'} - cpu: [arm] - os: [android] - - '@esbuild/android-x64@0.18.20': - resolution: {integrity: sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg==} - engines: {node: '>=12'} - cpu: [x64] - os: [android] - - '@esbuild/android-x64@0.25.12': - resolution: {integrity: sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==} - engines: {node: '>=18'} - cpu: [x64] - os: [android] - - '@esbuild/android-x64@0.27.7': - resolution: {integrity: sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==} - engines: {node: '>=18'} - cpu: [x64] - os: [android] - - '@esbuild/android-x64@0.28.1': - resolution: {integrity: sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==} - engines: {node: '>=18'} - cpu: [x64] - os: [android] - - '@esbuild/darwin-arm64@0.18.20': - resolution: {integrity: sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA==} - engines: {node: '>=12'} - cpu: [arm64] - os: [darwin] - - '@esbuild/darwin-arm64@0.25.12': - resolution: {integrity: sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [darwin] - - '@esbuild/darwin-arm64@0.27.7': - resolution: {integrity: sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [darwin] - - '@esbuild/darwin-arm64@0.28.1': - resolution: {integrity: sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==} - engines: {node: '>=18'} - cpu: [arm64] - os: [darwin] - - '@esbuild/darwin-x64@0.18.20': - resolution: {integrity: sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [darwin] - - '@esbuild/darwin-x64@0.25.12': - resolution: {integrity: sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==} - engines: {node: '>=18'} - cpu: [x64] - os: [darwin] - - '@esbuild/darwin-x64@0.27.7': - resolution: {integrity: sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [darwin] - - '@esbuild/darwin-x64@0.28.1': - resolution: {integrity: sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [darwin] - - '@esbuild/freebsd-arm64@0.18.20': - resolution: {integrity: sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw==} - engines: {node: '>=12'} - cpu: [arm64] - os: [freebsd] - - '@esbuild/freebsd-arm64@0.25.12': - resolution: {integrity: sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [freebsd] - - '@esbuild/freebsd-arm64@0.27.7': - resolution: {integrity: sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==} - engines: {node: '>=18'} - cpu: [arm64] - os: [freebsd] - - '@esbuild/freebsd-arm64@0.28.1': - resolution: {integrity: sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [freebsd] - - '@esbuild/freebsd-x64@0.18.20': - resolution: {integrity: sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [freebsd] - - '@esbuild/freebsd-x64@0.25.12': - resolution: {integrity: sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [freebsd] - - '@esbuild/freebsd-x64@0.27.7': - resolution: {integrity: sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [freebsd] - - '@esbuild/freebsd-x64@0.28.1': - resolution: {integrity: sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [freebsd] - - '@esbuild/linux-arm64@0.18.20': - resolution: {integrity: sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA==} - engines: {node: '>=12'} - cpu: [arm64] - os: [linux] - - '@esbuild/linux-arm64@0.25.12': - resolution: {integrity: sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==} - engines: {node: '>=18'} - cpu: [arm64] - os: [linux] - - '@esbuild/linux-arm64@0.27.7': - resolution: {integrity: sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==} - engines: {node: '>=18'} - cpu: [arm64] - os: [linux] - - '@esbuild/linux-arm64@0.28.1': - resolution: {integrity: sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==} - engines: {node: '>=18'} - cpu: [arm64] - os: [linux] - - '@esbuild/linux-arm@0.18.20': - resolution: {integrity: sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg==} - engines: {node: '>=12'} - cpu: [arm] - os: [linux] - - '@esbuild/linux-arm@0.25.12': - resolution: {integrity: sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==} - engines: {node: '>=18'} - cpu: [arm] - os: [linux] - - '@esbuild/linux-arm@0.27.7': - resolution: {integrity: sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==} - engines: {node: '>=18'} - cpu: [arm] - os: [linux] - - '@esbuild/linux-arm@0.28.1': - resolution: {integrity: sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==} - engines: {node: '>=18'} - cpu: [arm] - os: [linux] - - '@esbuild/linux-ia32@0.18.20': - resolution: {integrity: sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA==} - engines: {node: '>=12'} - cpu: [ia32] - os: [linux] - - '@esbuild/linux-ia32@0.25.12': - resolution: {integrity: sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==} - engines: {node: '>=18'} - cpu: [ia32] - os: [linux] - - '@esbuild/linux-ia32@0.27.7': - resolution: {integrity: sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==} - engines: {node: '>=18'} - cpu: [ia32] - os: [linux] - - '@esbuild/linux-ia32@0.28.1': - resolution: {integrity: sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==} - engines: {node: '>=18'} - cpu: [ia32] - os: [linux] - - '@esbuild/linux-loong64@0.18.20': - resolution: {integrity: sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg==} - engines: {node: '>=12'} - cpu: [loong64] - os: [linux] - - '@esbuild/linux-loong64@0.25.12': - resolution: {integrity: sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==} - engines: {node: '>=18'} - cpu: [loong64] - os: [linux] - - '@esbuild/linux-loong64@0.27.7': - resolution: {integrity: sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==} - engines: {node: '>=18'} - cpu: [loong64] - os: [linux] - - '@esbuild/linux-loong64@0.28.1': - resolution: {integrity: sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==} - engines: {node: '>=18'} - cpu: [loong64] - os: [linux] - - '@esbuild/linux-mips64el@0.18.20': - resolution: {integrity: sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ==} - engines: {node: '>=12'} - cpu: [mips64el] - os: [linux] - - '@esbuild/linux-mips64el@0.25.12': - resolution: {integrity: sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==} - engines: {node: '>=18'} - cpu: [mips64el] - os: [linux] - - '@esbuild/linux-mips64el@0.27.7': - resolution: {integrity: sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==} - engines: {node: '>=18'} - cpu: [mips64el] - os: [linux] - - '@esbuild/linux-mips64el@0.28.1': - resolution: {integrity: sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==} - engines: {node: '>=18'} - cpu: [mips64el] - os: [linux] - - '@esbuild/linux-ppc64@0.18.20': - resolution: {integrity: sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA==} - engines: {node: '>=12'} - cpu: [ppc64] - os: [linux] - - '@esbuild/linux-ppc64@0.25.12': - resolution: {integrity: sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [linux] - - '@esbuild/linux-ppc64@0.27.7': - resolution: {integrity: sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [linux] - - '@esbuild/linux-ppc64@0.28.1': - resolution: {integrity: sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [linux] - - '@esbuild/linux-riscv64@0.18.20': - resolution: {integrity: sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A==} - engines: {node: '>=12'} - cpu: [riscv64] - os: [linux] - - '@esbuild/linux-riscv64@0.25.12': - resolution: {integrity: sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==} - engines: {node: '>=18'} - cpu: [riscv64] - os: [linux] - - '@esbuild/linux-riscv64@0.27.7': - resolution: {integrity: sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==} - engines: {node: '>=18'} - cpu: [riscv64] - os: [linux] - - '@esbuild/linux-riscv64@0.28.1': - resolution: {integrity: sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==} - engines: {node: '>=18'} - cpu: [riscv64] - os: [linux] - - '@esbuild/linux-s390x@0.18.20': - resolution: {integrity: sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ==} - engines: {node: '>=12'} - cpu: [s390x] - os: [linux] - - '@esbuild/linux-s390x@0.25.12': - resolution: {integrity: sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==} - engines: {node: '>=18'} - cpu: [s390x] - os: [linux] - - '@esbuild/linux-s390x@0.27.7': - resolution: {integrity: sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==} - engines: {node: '>=18'} - cpu: [s390x] - os: [linux] - - '@esbuild/linux-s390x@0.28.1': - resolution: {integrity: sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==} - engines: {node: '>=18'} - cpu: [s390x] - os: [linux] - - '@esbuild/linux-x64@0.18.20': - resolution: {integrity: sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w==} - engines: {node: '>=12'} - cpu: [x64] - os: [linux] - - '@esbuild/linux-x64@0.25.12': - resolution: {integrity: sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==} - engines: {node: '>=18'} - cpu: [x64] - os: [linux] - - '@esbuild/linux-x64@0.27.7': - resolution: {integrity: sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==} - engines: {node: '>=18'} - cpu: [x64] - os: [linux] - - '@esbuild/linux-x64@0.28.1': - resolution: {integrity: sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==} - engines: {node: '>=18'} - cpu: [x64] - os: [linux] - - '@esbuild/netbsd-arm64@0.25.12': - resolution: {integrity: sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [netbsd] - - '@esbuild/netbsd-arm64@0.27.7': - resolution: {integrity: sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==} - engines: {node: '>=18'} - cpu: [arm64] - os: [netbsd] - - '@esbuild/netbsd-arm64@0.28.1': - resolution: {integrity: sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [netbsd] - - '@esbuild/netbsd-x64@0.18.20': - resolution: {integrity: sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A==} - engines: {node: '>=12'} - cpu: [x64] - os: [netbsd] - - '@esbuild/netbsd-x64@0.25.12': - resolution: {integrity: sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [netbsd] - - '@esbuild/netbsd-x64@0.27.7': - resolution: {integrity: sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==} - engines: {node: '>=18'} - cpu: [x64] - os: [netbsd] - - '@esbuild/netbsd-x64@0.28.1': - resolution: {integrity: sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==} - engines: {node: '>=18'} - cpu: [x64] - os: [netbsd] - - '@esbuild/openbsd-arm64@0.25.12': - resolution: {integrity: sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openbsd] - - '@esbuild/openbsd-arm64@0.27.7': - resolution: {integrity: sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openbsd] - - '@esbuild/openbsd-arm64@0.28.1': - resolution: {integrity: sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openbsd] - - '@esbuild/openbsd-x64@0.18.20': - resolution: {integrity: sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg==} - engines: {node: '>=12'} - cpu: [x64] - os: [openbsd] - - '@esbuild/openbsd-x64@0.25.12': - resolution: {integrity: sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==} - engines: {node: '>=18'} - cpu: [x64] - os: [openbsd] - - '@esbuild/openbsd-x64@0.27.7': - resolution: {integrity: sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==} - engines: {node: '>=18'} - cpu: [x64] - os: [openbsd] - - '@esbuild/openbsd-x64@0.28.1': - resolution: {integrity: sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==} - engines: {node: '>=18'} - cpu: [x64] - os: [openbsd] - - '@esbuild/openharmony-arm64@0.25.12': - resolution: {integrity: sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openharmony] - - '@esbuild/openharmony-arm64@0.27.7': - resolution: {integrity: sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openharmony] - - '@esbuild/openharmony-arm64@0.28.1': - resolution: {integrity: sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openharmony] - - '@esbuild/sunos-x64@0.18.20': - resolution: {integrity: sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [sunos] - - '@esbuild/sunos-x64@0.25.12': - resolution: {integrity: sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==} - engines: {node: '>=18'} - cpu: [x64] - os: [sunos] - - '@esbuild/sunos-x64@0.27.7': - resolution: {integrity: sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==} - engines: {node: '>=18'} - cpu: [x64] - os: [sunos] - - '@esbuild/sunos-x64@0.28.1': - resolution: {integrity: sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [sunos] - - '@esbuild/win32-arm64@0.18.20': - resolution: {integrity: sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg==} - engines: {node: '>=12'} - cpu: [arm64] - os: [win32] - - '@esbuild/win32-arm64@0.25.12': - resolution: {integrity: sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [win32] - - '@esbuild/win32-arm64@0.27.7': - resolution: {integrity: sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==} - engines: {node: '>=18'} - cpu: [arm64] - os: [win32] - - '@esbuild/win32-arm64@0.28.1': - resolution: {integrity: sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==} - engines: {node: '>=18'} - cpu: [arm64] - os: [win32] - - '@esbuild/win32-ia32@0.18.20': - resolution: {integrity: sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g==} - engines: {node: '>=12'} - cpu: [ia32] - os: [win32] - - '@esbuild/win32-ia32@0.25.12': - resolution: {integrity: sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==} - engines: {node: '>=18'} - cpu: [ia32] - os: [win32] - - '@esbuild/win32-ia32@0.27.7': - resolution: {integrity: sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==} - engines: {node: '>=18'} - cpu: [ia32] - os: [win32] - - '@esbuild/win32-ia32@0.28.1': - resolution: {integrity: sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==} - engines: {node: '>=18'} - cpu: [ia32] - os: [win32] - - '@esbuild/win32-x64@0.18.20': - resolution: {integrity: sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [win32] - - '@esbuild/win32-x64@0.25.12': - resolution: {integrity: sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==} - engines: {node: '>=18'} - cpu: [x64] - os: [win32] - - '@esbuild/win32-x64@0.27.7': - resolution: {integrity: sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==} - engines: {node: '>=18'} - cpu: [x64] - os: [win32] - - '@esbuild/win32-x64@0.28.1': - resolution: {integrity: sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==} - engines: {node: '>=18'} - cpu: [x64] - os: [win32] - - '@img/colour@1.1.0': - resolution: {integrity: sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==} - engines: {node: '>=18'} - - '@img/sharp-darwin-arm64@0.34.5': - resolution: {integrity: sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [darwin] - - '@img/sharp-darwin-x64@0.34.5': - resolution: {integrity: sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [darwin] - - '@img/sharp-libvips-darwin-arm64@1.2.4': - resolution: {integrity: sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g==} - cpu: [arm64] - os: [darwin] - - '@img/sharp-libvips-darwin-x64@1.2.4': - resolution: {integrity: sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg==} - cpu: [x64] - os: [darwin] - - '@img/sharp-libvips-linux-arm64@1.2.4': - resolution: {integrity: sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw==} - cpu: [arm64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-arm@1.2.4': - resolution: {integrity: sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A==} - cpu: [arm] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-ppc64@1.2.4': - resolution: {integrity: sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA==} - cpu: [ppc64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-riscv64@1.2.4': - resolution: {integrity: sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA==} - cpu: [riscv64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-s390x@1.2.4': - resolution: {integrity: sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ==} - cpu: [s390x] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linux-x64@1.2.4': - resolution: {integrity: sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw==} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@img/sharp-libvips-linuxmusl-arm64@1.2.4': - resolution: {integrity: sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw==} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@img/sharp-libvips-linuxmusl-x64@1.2.4': - resolution: {integrity: sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg==} - cpu: [x64] - os: [linux] - libc: [musl] - - '@img/sharp-linux-arm64@0.34.5': - resolution: {integrity: sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-arm@0.34.5': - resolution: {integrity: sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-ppc64@0.34.5': - resolution: {integrity: sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [ppc64] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-riscv64@0.34.5': - resolution: {integrity: sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [riscv64] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-s390x@0.34.5': - resolution: {integrity: sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [s390x] - os: [linux] - libc: [glibc] - - '@img/sharp-linux-x64@0.34.5': - resolution: {integrity: sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@img/sharp-linuxmusl-arm64@0.34.5': - resolution: {integrity: sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@img/sharp-linuxmusl-x64@0.34.5': - resolution: {integrity: sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [linux] - libc: [musl] - - '@img/sharp-wasm32@0.34.5': - resolution: {integrity: sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [wasm32] - - '@img/sharp-win32-arm64@0.34.5': - resolution: {integrity: sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [arm64] - os: [win32] - - '@img/sharp-win32-ia32@0.34.5': - resolution: {integrity: sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [ia32] - os: [win32] - - '@img/sharp-win32-x64@0.34.5': - resolution: {integrity: sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - cpu: [x64] - os: [win32] - - '@jridgewell/resolve-uri@3.1.2': - resolution: {integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==} - engines: {node: '>=6.0.0'} - - '@jridgewell/sourcemap-codec@1.5.5': - resolution: {integrity: sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==} - - '@jridgewell/trace-mapping@0.3.9': - resolution: {integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==} - - '@poppinss/colors@4.1.6': - resolution: {integrity: sha512-H9xkIdFswbS8n1d6vmRd8+c10t2Qe+rZITbbDHHkQixH5+2x1FDGmi/0K+WgWiqQFKPSlIYB7jlH6Kpfn6Fleg==} - - '@poppinss/dumper@0.6.5': - resolution: {integrity: sha512-NBdYIb90J7LfOI32dOewKI1r7wnkiH6m920puQ3qHUeZkxNkQiFnXVWoE6YtFSv6QOiPPf7ys6i+HWWecDz7sw==} - - '@poppinss/exception@1.2.3': - resolution: {integrity: sha512-dCED+QRChTVatE9ibtoaxc+WkdzOSjYTKi/+uacHWIsfodVfpsueo3+DKpgU5Px8qXjgmXkSvhXvSCz3fnP9lw==} - - '@rollup/rollup-android-arm-eabi@4.60.2': - resolution: {integrity: sha512-dnlp69efPPg6Uaw2dVqzWRfAWRnYVb1XJ8CyyhIbZeaq4CA5/mLeZ1IEt9QqQxmbdvagjLIm2ZL8BxXv5lH4Yw==} - cpu: [arm] - os: [android] - - '@rollup/rollup-android-arm64@4.60.2': - resolution: {integrity: sha512-OqZTwDRDchGRHHm/hwLOL7uVPB9aUvI0am/eQuWMNyFHf5PSEQmyEeYYheA0EPPKUO/l0uigCp+iaTjoLjVoHg==} - cpu: [arm64] - os: [android] - - '@rollup/rollup-darwin-arm64@4.60.2': - resolution: {integrity: sha512-UwRE7CGpvSVEQS8gUMBe1uADWjNnVgP3Iusyda1nSRwNDCsRjnGc7w6El6WLQsXmZTbLZx9cecegumcitNfpmA==} - cpu: [arm64] - os: [darwin] - - '@rollup/rollup-darwin-x64@4.60.2': - resolution: {integrity: sha512-gjEtURKLCC5VXm1I+2i1u9OhxFsKAQJKTVB8WvDAHF+oZlq0GTVFOlTlO1q3AlCTE/DF32c16ESvfgqR7343/g==} - cpu: [x64] - os: [darwin] - - '@rollup/rollup-freebsd-arm64@4.60.2': - resolution: {integrity: sha512-Bcl6CYDeAgE70cqZaMojOi/eK63h5Me97ZqAQoh77VPjMysA/4ORQBRGo3rRy45x4MzVlU9uZxs8Uwy7ZaKnBw==} - cpu: [arm64] - os: [freebsd] - - '@rollup/rollup-freebsd-x64@4.60.2': - resolution: {integrity: sha512-LU+TPda3mAE2QB0/Hp5VyeKJivpC6+tlOXd1VMoXV/YFMvk/MNk5iXeBfB4MQGRWyOYVJ01625vjkr0Az98OJQ==} - cpu: [x64] - os: [freebsd] - - '@rollup/rollup-linux-arm-gnueabihf@4.60.2': - resolution: {integrity: sha512-2QxQrM+KQ7DAW4o22j+XZ6RKdxjLD7BOWTP0Bv0tmjdyhXSsr2Ul1oJDQqh9Zf5qOwTuTc7Ek83mOFaKnodPjg==} - cpu: [arm] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-arm-musleabihf@4.60.2': - resolution: {integrity: sha512-TbziEu2DVsTEOPif2mKWkMeDMLoYjx95oESa9fkQQK7r/Orta0gnkcDpzwufEcAO2BLBsD7mZkXGFqEdMRRwfw==} - cpu: [arm] - os: [linux] - libc: [musl] - - '@rollup/rollup-linux-arm64-gnu@4.60.2': - resolution: {integrity: sha512-bO/rVDiDUuM2YfuCUwZ1t1cP+/yqjqz+Xf2VtkdppefuOFS2OSeAfgafaHNkFn0t02hEyXngZkxtGqXcXwO8Rg==} - cpu: [arm64] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-arm64-musl@4.60.2': - resolution: {integrity: sha512-hr26p7e93Rl0Za+JwW7EAnwAvKkehh12BU1Llm9Ykiibg4uIr2rbpxG9WCf56GuvidlTG9KiiQT/TXT1yAWxTA==} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@rollup/rollup-linux-loong64-gnu@4.60.2': - resolution: {integrity: sha512-pOjB/uSIyDt+ow3k/RcLvUAOGpysT2phDn7TTUB3n75SlIgZzM6NKAqlErPhoFU+npgY3/n+2HYIQVbF70P9/A==} - cpu: [loong64] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-loong64-musl@4.60.2': - resolution: {integrity: sha512-2/w+q8jszv9Ww1c+6uJT3OwqhdmGP2/4T17cu8WuwyUuuaCDDJ2ojdyYwZzCxx0GcsZBhzi3HmH+J5pZNXnd+Q==} - cpu: [loong64] - os: [linux] - libc: [musl] - - '@rollup/rollup-linux-ppc64-gnu@4.60.2': - resolution: {integrity: sha512-11+aL5vKheYgczxtPVVRhdptAM2H7fcDR5Gw4/bTcteuZBlH4oP9f5s9zYO9aGZvoGeBpqXI/9TZZihZ609wKw==} - cpu: [ppc64] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-ppc64-musl@4.60.2': - resolution: {integrity: sha512-i16fokAGK46IVZuV8LIIwMdtqhin9hfYkCh8pf8iC3QU3LpwL+1FSFGej+O7l3E/AoknL6Dclh2oTdnRMpTzFQ==} - cpu: [ppc64] - os: [linux] - libc: [musl] - - '@rollup/rollup-linux-riscv64-gnu@4.60.2': - resolution: {integrity: sha512-49FkKS6RGQoriDSK/6E2GkAsAuU5kETFCh7pG4yD/ylj9rKhTmO3elsnmBvRD4PgJPds5W2PkhC82aVwmUcJ7A==} - cpu: [riscv64] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-riscv64-musl@4.60.2': - resolution: {integrity: sha512-mjYNkHPfGpUR00DuM1ZZIgs64Hpf4bWcz9Z41+4Q+pgDx73UwWdAYyf6EG/lRFldmdHHzgrYyge5akFUW0D3mQ==} - cpu: [riscv64] - os: [linux] - libc: [musl] - - '@rollup/rollup-linux-s390x-gnu@4.60.2': - resolution: {integrity: sha512-ALyvJz965BQk8E9Al/JDKKDLH2kfKFLTGMlgkAbbYtZuJt9LU8DW3ZoDMCtQpXAltZxwBHevXz5u+gf0yA0YoA==} - cpu: [s390x] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-x64-gnu@4.60.2': - resolution: {integrity: sha512-UQjrkIdWrKI626Du8lCQ6MJp/6V1LAo2bOK9OTu4mSn8GGXIkPXk/Vsp4bLHCd9Z9Iz2OTEaokUE90VweJgIYQ==} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-x64-musl@4.60.2': - resolution: {integrity: sha512-bTsRGj6VlSdn/XD4CGyzMnzaBs9bsRxy79eTqTCBsA8TMIEky7qg48aPkvJvFe1HyzQ5oMZdg7AnVlWQSKLTnw==} - cpu: [x64] - os: [linux] - libc: [musl] - - '@rollup/rollup-openbsd-x64@4.60.2': - resolution: {integrity: sha512-6d4Z3534xitaA1FcMWP7mQPq5zGwBmGbhphh2DwaA1aNIXUu3KTOfwrWpbwI4/Gr0uANo7NTtaykFyO2hPuFLg==} - cpu: [x64] - os: [openbsd] - - '@rollup/rollup-openharmony-arm64@4.60.2': - resolution: {integrity: sha512-NetAg5iO2uN7eB8zE5qrZ3CSil+7IJt4WDFLcC75Ymywq1VZVD6qJ6EvNLjZ3rEm6gB7XW5JdT60c6MN35Z85Q==} - cpu: [arm64] - os: [openharmony] - - '@rollup/rollup-win32-arm64-msvc@4.60.2': - resolution: {integrity: sha512-NCYhOotpgWZ5kdxCZsv6Iudx0wX8980Q/oW4pNFNihpBKsDbEA1zpkfxJGC0yugsUuyDZ7gL37dbzwhR0VI7pQ==} - cpu: [arm64] - os: [win32] - - '@rollup/rollup-win32-ia32-msvc@4.60.2': - resolution: {integrity: sha512-RXsaOqXxfoUBQoOgvmmijVxJnW2IGB0eoMO7F8FAjaj0UTywUO/luSqimWBJn04WNgUkeNhh7fs7pESXajWmkg==} - cpu: [ia32] - os: [win32] - - '@rollup/rollup-win32-x64-gnu@4.60.2': - resolution: {integrity: sha512-qdAzEULD+/hzObedtmV6iBpdL5TIbKVztGiK7O3/KYSf+HIzU257+MX1EXJcyIiDbMAqmbwaufcYPvyRryeZtA==} - cpu: [x64] - os: [win32] - - '@rollup/rollup-win32-x64-msvc@4.60.2': - resolution: {integrity: sha512-Nd/SgG27WoA9e+/TdK74KnHz852TLa94ovOYySo/yMPuTmpckK/jIF2jSwS3g7ELSKXK13/cVdmg1Z/DaCWKxA==} - cpu: [x64] - os: [win32] - - '@sindresorhus/is@7.2.0': - resolution: {integrity: sha512-P1Cz1dWaFfR4IR+U13mqqiGsLFf1KbayybWwdd2vfctdV6hDpUkgCY0nKOLLTMSoRd/jJNjtbqzf13K8DCCXQw==} - engines: {node: '>=18'} - - '@speed-highlight/core@1.2.15': - resolution: {integrity: sha512-BMq1K3DsElxDWawkX6eLg9+CKJrTVGCBAWVuHXVUV2u0s2711qiChLSId6ikYPfxhdYocLNt3wWwSvDiTvFabw==} - - '@standard-schema/spec@1.1.0': - resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} - - '@types/chai@5.2.3': - resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} - - '@types/deep-eql@4.0.2': - resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} - - '@types/estree@1.0.8': - resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} - - '@types/node@26.1.1': - resolution: {integrity: sha512-nxAkRSVkN1Y0JC1W8ky/fTfkGsMmcrRsbx+3XoZE+rMOX71kLYTV7fLXpqud1GpbpP5TuffXFqfX7fH2GgZREw==} - - '@typescript/typescript-aix-ppc64@7.0.2': - resolution: {integrity: sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==} - engines: {node: '>=16.20.0'} - cpu: [ppc64] - os: [aix] - - '@typescript/typescript-darwin-arm64@7.0.2': - resolution: {integrity: sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==} - engines: {node: '>=16.20.0'} - cpu: [arm64] - os: [darwin] - - '@typescript/typescript-darwin-x64@7.0.2': - resolution: {integrity: sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==} - engines: {node: '>=16.20.0'} - cpu: [x64] - os: [darwin] - - '@typescript/typescript-freebsd-arm64@7.0.2': - resolution: {integrity: sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==} - engines: {node: '>=16.20.0'} - cpu: [arm64] - os: [freebsd] - - '@typescript/typescript-freebsd-x64@7.0.2': - resolution: {integrity: sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==} - engines: {node: '>=16.20.0'} - cpu: [x64] - os: [freebsd] - - '@typescript/typescript-linux-arm64@7.0.2': - resolution: {integrity: sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==} - engines: {node: '>=16.20.0'} - cpu: [arm64] - os: [linux] - - '@typescript/typescript-linux-arm@7.0.2': - resolution: {integrity: sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==} - engines: {node: '>=16.20.0'} - cpu: [arm] - os: [linux] - - '@typescript/typescript-linux-loong64@7.0.2': - resolution: {integrity: sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==} - engines: {node: '>=16.20.0'} - cpu: [loong64] - os: [linux] - - '@typescript/typescript-linux-mips64el@7.0.2': - resolution: {integrity: sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==} - engines: {node: '>=16.20.0'} - cpu: [mips64el] - os: [linux] - - '@typescript/typescript-linux-ppc64@7.0.2': - resolution: {integrity: sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==} - engines: {node: '>=16.20.0'} - cpu: [ppc64] - os: [linux] - - '@typescript/typescript-linux-riscv64@7.0.2': - resolution: {integrity: sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==} - engines: {node: '>=16.20.0'} - cpu: [riscv64] - os: [linux] - - '@typescript/typescript-linux-s390x@7.0.2': - resolution: {integrity: sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==} - engines: {node: '>=16.20.0'} - cpu: [s390x] - os: [linux] - - '@typescript/typescript-linux-x64@7.0.2': - resolution: {integrity: sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==} - engines: {node: '>=16.20.0'} - cpu: [x64] - os: [linux] - - '@typescript/typescript-netbsd-arm64@7.0.2': - resolution: {integrity: sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==} - engines: {node: '>=16.20.0'} - cpu: [arm64] - os: [netbsd] - - '@typescript/typescript-netbsd-x64@7.0.2': - resolution: {integrity: sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==} - engines: {node: '>=16.20.0'} - cpu: [x64] - os: [netbsd] - - '@typescript/typescript-openbsd-arm64@7.0.2': - resolution: {integrity: sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==} - engines: {node: '>=16.20.0'} - cpu: [arm64] - os: [openbsd] - - '@typescript/typescript-openbsd-x64@7.0.2': - resolution: {integrity: sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==} - engines: {node: '>=16.20.0'} - cpu: [x64] - os: [openbsd] - - '@typescript/typescript-sunos-x64@7.0.2': - resolution: {integrity: sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==} - engines: {node: '>=16.20.0'} - cpu: [x64] - os: [sunos] - - '@typescript/typescript-win32-arm64@7.0.2': - resolution: {integrity: sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==} - engines: {node: '>=16.20.0'} - cpu: [arm64] - os: [win32] - - '@typescript/typescript-win32-x64@7.0.2': - resolution: {integrity: sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==} - engines: {node: '>=16.20.0'} - cpu: [x64] - os: [win32] - - '@vitest/expect@4.1.10': - resolution: {integrity: sha512-YsCn+qAk1GWjQOWFEsEcL2gNQ0zmVmQu3T03qP6UyjhtmdtwtbuI+DASn/7iQB3HGTXkdBwGddzxPlmiql5vlA==} - - '@vitest/mocker@4.1.10': - resolution: {integrity: sha512-v0xaezt+DKEmKfaxg133ldzADrwLGd7Ze1MfQQTYfvs8OqZIwbxyxaYURivwV7sWy5fqn3rH5uOrSp07bp44Ow==} - peerDependencies: - msw: ^2.4.9 - vite: ^6.0.0 || ^7.0.0 || ^8.0.0 - peerDependenciesMeta: - msw: - optional: true - vite: - optional: true - - '@vitest/pretty-format@4.1.10': - resolution: {integrity: sha512-W1HsjSH4MXQ9YfmmhLAoIYf1HRfekQCGngeIgcei6MP5QQGWUe0gkopdZQaVCFO+JDJMrAJGwa5pRpNpvy4P8Q==} - - '@vitest/runner@4.1.10': - resolution: {integrity: sha512-IKI6kpIH+LmpROplyLwBBaCfMgOZOMsygVa6BARD6ahA04VRuJSa6OaVG7kRvSEMD870Vd91rSSw0eegtWyLGg==} - - '@vitest/snapshot@4.1.10': - resolution: {integrity: sha512-xRkfOT1qpTAi/Ti4Y1LtfRc3kEuqxGw59eN2jN9pRWMtS/XDevekhcFSqvQqjUNGksfjMJu3Y+oJ+4Ypn2OaJw==} - - '@vitest/spy@4.1.10': - resolution: {integrity: sha512-PLf/Ugvoq5wO/b4rwYCR1h2PSIdXz7wnkQFMiUpLdtM7l6pqVFcQIBEHyT1+l+cj7mNwAfZHzqXqDyjvOuwbDw==} - - '@vitest/utils@4.1.10': - resolution: {integrity: sha512-fy9am/HWxbaGt/Sawrp90vt6Y6jQwf1RX77cz3uwoJwJVMli/e1IEwRPnMNJ7vKfPTwo0diXifkpPvwH9v7nGA==} - - assertion-error@2.0.1: - resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} - engines: {node: '>=12'} - - aws4fetch@1.0.20: - resolution: {integrity: sha512-/djoAN709iY65ETD6LKCtyyEI04XIBP5xVvfmNxsEP0uJB5tyaGBztSryRr4HqMStr9R06PisQE7m9zDTXKu6g==} - - blake3-wasm@2.1.5: - resolution: {integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==} - - buffer-from@1.1.2: - resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==} - - chai@6.2.2: - resolution: {integrity: sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==} - engines: {node: '>=18'} - - cjs-module-lexer@1.2.3: - resolution: {integrity: sha512-0TNiGstbQmCFwt4akjjBg5pLRTSyj/PkWQ1ZoO2zntmg9yLqSRxwEa4iCfQLGjqhiqBfOJa7W/E8wfGrTDmlZQ==} - - convert-source-map@2.0.0: - resolution: {integrity: sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==} - - cookie@1.1.1: - resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} - engines: {node: '>=18'} - - detect-libc@2.1.2: - resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} - engines: {node: '>=8'} - - drizzle-kit@0.31.10: - resolution: {integrity: sha512-7OZcmQUrdGI+DUNNsKBn1aW8qSoKuTH7d0mYgSP8bAzdFzKoovxEFnoGQp2dVs82EOJeYycqRtciopszwUf8bw==} - hasBin: true - - drizzle-orm@0.45.2: - resolution: {integrity: sha512-kY0BSaTNYWnoDMVoyY8uxmyHjpJW1geOmBMdSSicKo9CIIWkSxMIj2rkeSR51b8KAPB7m+qysjuHme5nKP+E5Q==} - peerDependencies: - '@aws-sdk/client-rds-data': '>=3' - '@cloudflare/workers-types': '>=4' - '@electric-sql/pglite': '>=0.2.0' - '@libsql/client': '>=0.10.0' - '@libsql/client-wasm': '>=0.10.0' - '@neondatabase/serverless': '>=0.10.0' - '@op-engineering/op-sqlite': '>=2' - '@opentelemetry/api': ^1.4.1 - '@planetscale/database': '>=1.13' - '@prisma/client': '*' - '@tidbcloud/serverless': '*' - '@types/better-sqlite3': '*' - '@types/pg': '*' - '@types/sql.js': '*' - '@upstash/redis': '>=1.34.7' - '@vercel/postgres': '>=0.8.0' - '@xata.io/client': '*' - better-sqlite3: '>=7' - bun-types: '*' - expo-sqlite: '>=14.0.0' - gel: '>=2' - knex: '*' - kysely: '*' - mysql2: '>=2' - pg: '>=8' - postgres: '>=3' - prisma: '*' - sql.js: '>=1' - sqlite3: '>=5' - peerDependenciesMeta: - '@aws-sdk/client-rds-data': - optional: true - '@cloudflare/workers-types': - optional: true - '@electric-sql/pglite': - optional: true - '@libsql/client': - optional: true - '@libsql/client-wasm': - optional: true - '@neondatabase/serverless': - optional: true - '@op-engineering/op-sqlite': - optional: true - '@opentelemetry/api': - optional: true - '@planetscale/database': - optional: true - '@prisma/client': - optional: true - '@tidbcloud/serverless': - optional: true - '@types/better-sqlite3': - optional: true - '@types/pg': - optional: true - '@types/sql.js': - optional: true - '@upstash/redis': - optional: true - '@vercel/postgres': - optional: true - '@xata.io/client': - optional: true - better-sqlite3: - optional: true - bun-types: - optional: true - expo-sqlite: - optional: true - gel: - optional: true - knex: - optional: true - kysely: - optional: true - mysql2: - optional: true - pg: - optional: true - postgres: - optional: true - prisma: - optional: true - sql.js: - optional: true - sqlite3: - optional: true - - error-stack-parser-es@1.0.5: - resolution: {integrity: sha512-5qucVt2XcuGMcEGgWI7i+yZpmpByQ8J1lHhcL7PwqCwu9FPP3VUXzT4ltHe5i2z9dePwEHcDVOAfSnHsOlCXRA==} - - es-module-lexer@2.3.1: - resolution: {integrity: sha512-shc1dbU90Yl/xq1QrC7QRtfcwURZuVRfPhZbDoldJ1cn1gzDvBaBWlv0eFolj5+0znnPJz5TXLxsN77X/12KTA==} - - esbuild@0.18.20: - resolution: {integrity: sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA==} - engines: {node: '>=12'} - hasBin: true - - esbuild@0.25.12: - resolution: {integrity: sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==} - engines: {node: '>=18'} - hasBin: true - - esbuild@0.27.7: - resolution: {integrity: sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==} - engines: {node: '>=18'} - hasBin: true - - esbuild@0.28.1: - resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} - engines: {node: '>=18'} - hasBin: true - - estree-walker@3.0.3: - resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} - - expect-type@1.3.0: - resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} - engines: {node: '>=12.0.0'} - - fdir@6.5.0: - resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} - engines: {node: '>=12.0.0'} - peerDependencies: - picomatch: ^3 || ^4 - peerDependenciesMeta: - picomatch: - optional: true - - fsevents@2.3.3: - resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} - engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} - os: [darwin] - - get-tsconfig@4.14.0: - resolution: {integrity: sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==} - - kleur@4.1.5: - resolution: {integrity: sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==} - engines: {node: '>=6'} - - magic-string@0.30.21: - resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} - - miniflare@4.20260714.0: - resolution: {integrity: sha512-MYlTCLdWCPqvrYY2uLwOjXwmglXuiHE3TGGkbOW4BwjUPa1r07E0iuHwrNDIs/sxK21r+o90Jx58AV2KeNdJZw==} - engines: {node: '>=22.0.0'} - hasBin: true - - nanoid@3.3.11: - resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} - engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} - hasBin: true - - obug@2.1.4: - resolution: {integrity: sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==} - engines: {node: '>=12.20.0'} - - path-to-regexp@6.3.0: - resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} - - pathe@2.0.3: - resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} - - picocolors@1.1.1: - resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} - - picomatch@4.0.4: - resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} - engines: {node: '>=12'} - - postcss@8.5.10: - resolution: {integrity: sha512-pMMHxBOZKFU6HgAZ4eyGnwXF/EvPGGqUr0MnZ5+99485wwW41kW91A4LOGxSHhgugZmSChL5AlElNdwlNgcnLQ==} - engines: {node: ^10 || ^12 || >=14} - - resolve-pkg-maps@1.0.0: - resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} - - rollup@4.60.2: - resolution: {integrity: sha512-J9qZyW++QK/09NyN/zeO0dG/1GdGfyp9lV8ajHnRVLfo/uFsbji5mHnDgn/qYdUHyCkM2N+8VyspgZclfAh0eQ==} - engines: {node: '>=18.0.0', npm: '>=8.0.0'} - hasBin: true - - semver@7.7.4: - resolution: {integrity: sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==} - engines: {node: '>=10'} - hasBin: true - - sharp@0.34.5: - resolution: {integrity: sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==} - engines: {node: ^18.17.0 || ^20.3.0 || >=21.0.0} - - siginfo@2.0.0: - resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} - - source-map-js@1.2.1: - resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} - engines: {node: '>=0.10.0'} - - source-map-support@0.5.21: - resolution: {integrity: sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w==} - - source-map@0.6.1: - resolution: {integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==} - engines: {node: '>=0.10.0'} - - stackback@0.0.2: - resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} - - std-env@4.2.0: - resolution: {integrity: sha512-oCUKSupKTHX53EyjDtuZQ64pjLJ6yYCtpmEw0goYxtjG9KpbRe8KAsl2tBUGU9DyMcJ0RwJ8GqJAFzMXcXW1Rw==} - - supports-color@10.2.2: - resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} - engines: {node: '>=18'} - - tinybench@2.9.0: - resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} - - tinyexec@1.2.4: - resolution: {integrity: sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==} - engines: {node: '>=18'} - - tinyglobby@0.2.16: - resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==} - engines: {node: '>=12.0.0'} - - tinyrainbow@3.1.0: - resolution: {integrity: sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==} - engines: {node: '>=14.0.0'} - - tslib@2.8.1: - resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} - - tsx@4.21.0: - resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} - engines: {node: '>=18.0.0'} - hasBin: true - - typescript@7.0.2: - resolution: {integrity: sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==} - engines: {node: '>=16.20.0'} - hasBin: true - - undici-types@8.3.0: - resolution: {integrity: sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==} - - undici@7.28.0: - resolution: {integrity: sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==} - engines: {node: '>=20.18.1'} - - unenv@2.0.0-rc.24: - resolution: {integrity: sha512-i7qRCmY42zmCwnYlh9H2SvLEypEFGye5iRmEMKjcGi7zk9UquigRjFtTLz0TYqr0ZGLZhaMHl/foy1bZR+Cwlw==} - - vite@7.3.2: - resolution: {integrity: sha512-Bby3NOsna2jsjfLVOHKes8sGwgl4TT0E6vvpYgnAYDIF/tie7MRaFthmKuHx1NSXjiTueXH3do80FMQgvEktRg==} - engines: {node: ^20.19.0 || >=22.12.0} - hasBin: true - peerDependencies: - '@types/node': ^20.19.0 || >=22.12.0 - jiti: '>=1.21.0' - less: ^4.0.0 - lightningcss: ^1.21.0 - sass: ^1.70.0 - sass-embedded: ^1.70.0 - stylus: '>=0.54.8' - sugarss: ^5.0.0 - terser: ^5.16.0 - tsx: ^4.8.1 - yaml: ^2.4.2 - peerDependenciesMeta: - '@types/node': - optional: true - jiti: - optional: true - less: - optional: true - lightningcss: - optional: true - sass: - optional: true - sass-embedded: - optional: true - stylus: - optional: true - sugarss: - optional: true - terser: - optional: true - tsx: - optional: true - yaml: - optional: true - - vitest@4.1.10: - resolution: {integrity: sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==} - engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} - hasBin: true - peerDependencies: - '@edge-runtime/vm': '*' - '@opentelemetry/api': ^1.9.0 - '@types/node': ^20.0.0 || ^22.0.0 || >=24.0.0 - '@vitest/browser-playwright': 4.1.10 - '@vitest/browser-preview': 4.1.10 - '@vitest/browser-webdriverio': 4.1.10 - '@vitest/coverage-istanbul': 4.1.10 - '@vitest/coverage-v8': 4.1.10 - '@vitest/ui': 4.1.10 - happy-dom: '*' - jsdom: '*' - vite: ^6.0.0 || ^7.0.0 || ^8.0.0 - peerDependenciesMeta: - '@edge-runtime/vm': - optional: true - '@opentelemetry/api': - optional: true - '@types/node': - optional: true - '@vitest/browser-playwright': - optional: true - '@vitest/browser-preview': - optional: true - '@vitest/browser-webdriverio': - optional: true - '@vitest/coverage-istanbul': - optional: true - '@vitest/coverage-v8': - optional: true - '@vitest/ui': - optional: true - happy-dom: - optional: true - jsdom: - optional: true - - why-is-node-running@2.3.0: - resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} - engines: {node: '>=8'} - hasBin: true - - workerd@1.20260714.1: - resolution: {integrity: sha512-oIbQzfdyl9UQUnG6XLegcSq0Mgt/7WKDbFOoqGgOWCS+/fhyGB460uKEgdAQQ9RHCO/ttcNCX/KiMIQzdoeu3Q==} - engines: {node: '>=16'} - hasBin: true - - wrangler@4.112.0: - resolution: {integrity: sha512-5H+XUD0TySCv1LuktFHDIEOkboH2nTfQs+35L+USt3MtntjDTMVIJprLgQcL2WBjulOyjxpd1vyTiSTJVW5MjQ==} - engines: {node: '>=22.0.0'} - hasBin: true - peerDependencies: - '@cloudflare/workers-types': ^5.20260714.1 - peerDependenciesMeta: - '@cloudflare/workers-types': - optional: true - - ws@8.21.0: - resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} - engines: {node: '>=10.0.0'} - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: '>=5.0.2' - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - - youch-core@0.3.3: - resolution: {integrity: sha512-ho7XuGjLaJ2hWHoK8yFnsUGy2Y5uDpqSTq1FkHLK4/oqKtyUU1AFbOOxY4IpC9f0fTLjwYbslUz0Po5BpD1wrA==} - - youch@4.1.0-beta.10: - resolution: {integrity: sha512-rLfVLB4FgQneDr0dv1oddCVZmKjcJ6yX6mS4pU82Mq/Dt9a3cLZQ62pDBL4AUO+uVrCvtWz3ZFUL2HFAFJ/BXQ==} - - zod@3.25.76: - resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} - -snapshots: - - '@cloudflare/kv-asset-handler@0.5.0': {} - - '@cloudflare/unenv-preset@2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260714.1)': - dependencies: - unenv: 2.0.0-rc.24 - optionalDependencies: - workerd: 1.20260714.1 - - '@cloudflare/vitest-pool-workers@0.18.6(@vitest/runner@4.1.10)(@vitest/snapshot@4.1.10)(vitest@4.1.10(@types/node@26.1.1)(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0)))': - dependencies: - '@vitest/runner': 4.1.10 - '@vitest/snapshot': 4.1.10 - cjs-module-lexer: 1.2.3 - esbuild: 0.28.1 - miniflare: 4.20260714.0 - vitest: 4.1.10(@types/node@26.1.1)(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0)) - wrangler: 4.112.0 - zod: 3.25.76 - transitivePeerDependencies: - - '@cloudflare/workers-types' - - bufferutil - - utf-8-validate - - '@cloudflare/workerd-darwin-64@1.20260714.1': - optional: true - - '@cloudflare/workerd-darwin-arm64@1.20260714.1': - optional: true - - '@cloudflare/workerd-linux-64@1.20260714.1': - optional: true - - '@cloudflare/workerd-linux-arm64@1.20260714.1': - optional: true - - '@cloudflare/workerd-windows-64@1.20260714.1': - optional: true - - '@cspotcode/source-map-support@0.8.1': - dependencies: - '@jridgewell/trace-mapping': 0.3.9 - - '@drizzle-team/brocli@0.10.2': {} - - '@emnapi/runtime@1.10.0': - dependencies: - tslib: 2.8.1 - optional: true - - '@esbuild-kit/core-utils@3.3.2': - dependencies: - esbuild: 0.18.20 - source-map-support: 0.5.21 - - '@esbuild-kit/esm-loader@2.6.5': - dependencies: - '@esbuild-kit/core-utils': 3.3.2 - get-tsconfig: 4.14.0 - - '@esbuild/aix-ppc64@0.25.12': - optional: true - - '@esbuild/aix-ppc64@0.27.7': - optional: true - - '@esbuild/aix-ppc64@0.28.1': - optional: true - - '@esbuild/android-arm64@0.18.20': - optional: true - - '@esbuild/android-arm64@0.25.12': - optional: true - - '@esbuild/android-arm64@0.27.7': - optional: true - - '@esbuild/android-arm64@0.28.1': - optional: true - - '@esbuild/android-arm@0.18.20': - optional: true - - '@esbuild/android-arm@0.25.12': - optional: true - - '@esbuild/android-arm@0.27.7': - optional: true - - '@esbuild/android-arm@0.28.1': - optional: true - - '@esbuild/android-x64@0.18.20': - optional: true - - '@esbuild/android-x64@0.25.12': - optional: true - - '@esbuild/android-x64@0.27.7': - optional: true - - '@esbuild/android-x64@0.28.1': - optional: true - - '@esbuild/darwin-arm64@0.18.20': - optional: true - - '@esbuild/darwin-arm64@0.25.12': - optional: true - - '@esbuild/darwin-arm64@0.27.7': - optional: true - - '@esbuild/darwin-arm64@0.28.1': - optional: true - - '@esbuild/darwin-x64@0.18.20': - optional: true - - '@esbuild/darwin-x64@0.25.12': - optional: true - - '@esbuild/darwin-x64@0.27.7': - optional: true - - '@esbuild/darwin-x64@0.28.1': - optional: true - - '@esbuild/freebsd-arm64@0.18.20': - optional: true - - '@esbuild/freebsd-arm64@0.25.12': - optional: true - - '@esbuild/freebsd-arm64@0.27.7': - optional: true - - '@esbuild/freebsd-arm64@0.28.1': - optional: true - - '@esbuild/freebsd-x64@0.18.20': - optional: true - - '@esbuild/freebsd-x64@0.25.12': - optional: true - - '@esbuild/freebsd-x64@0.27.7': - optional: true - - '@esbuild/freebsd-x64@0.28.1': - optional: true - - '@esbuild/linux-arm64@0.18.20': - optional: true - - '@esbuild/linux-arm64@0.25.12': - optional: true - - '@esbuild/linux-arm64@0.27.7': - optional: true - - '@esbuild/linux-arm64@0.28.1': - optional: true - - '@esbuild/linux-arm@0.18.20': - optional: true - - '@esbuild/linux-arm@0.25.12': - optional: true - - '@esbuild/linux-arm@0.27.7': - optional: true - - '@esbuild/linux-arm@0.28.1': - optional: true - - '@esbuild/linux-ia32@0.18.20': - optional: true - - '@esbuild/linux-ia32@0.25.12': - optional: true - - '@esbuild/linux-ia32@0.27.7': - optional: true - - '@esbuild/linux-ia32@0.28.1': - optional: true - - '@esbuild/linux-loong64@0.18.20': - optional: true - - '@esbuild/linux-loong64@0.25.12': - optional: true - - '@esbuild/linux-loong64@0.27.7': - optional: true - - '@esbuild/linux-loong64@0.28.1': - optional: true - - '@esbuild/linux-mips64el@0.18.20': - optional: true - - '@esbuild/linux-mips64el@0.25.12': - optional: true - - '@esbuild/linux-mips64el@0.27.7': - optional: true - - '@esbuild/linux-mips64el@0.28.1': - optional: true - - '@esbuild/linux-ppc64@0.18.20': - optional: true - - '@esbuild/linux-ppc64@0.25.12': - optional: true - - '@esbuild/linux-ppc64@0.27.7': - optional: true - - '@esbuild/linux-ppc64@0.28.1': - optional: true - - '@esbuild/linux-riscv64@0.18.20': - optional: true - - '@esbuild/linux-riscv64@0.25.12': - optional: true - - '@esbuild/linux-riscv64@0.27.7': - optional: true - - '@esbuild/linux-riscv64@0.28.1': - optional: true - - '@esbuild/linux-s390x@0.18.20': - optional: true - - '@esbuild/linux-s390x@0.25.12': - optional: true - - '@esbuild/linux-s390x@0.27.7': - optional: true - - '@esbuild/linux-s390x@0.28.1': - optional: true - - '@esbuild/linux-x64@0.18.20': - optional: true - - '@esbuild/linux-x64@0.25.12': - optional: true - - '@esbuild/linux-x64@0.27.7': - optional: true - - '@esbuild/linux-x64@0.28.1': - optional: true - - '@esbuild/netbsd-arm64@0.25.12': - optional: true - - '@esbuild/netbsd-arm64@0.27.7': - optional: true - - '@esbuild/netbsd-arm64@0.28.1': - optional: true - - '@esbuild/netbsd-x64@0.18.20': - optional: true - - '@esbuild/netbsd-x64@0.25.12': - optional: true - - '@esbuild/netbsd-x64@0.27.7': - optional: true - - '@esbuild/netbsd-x64@0.28.1': - optional: true - - '@esbuild/openbsd-arm64@0.25.12': - optional: true - - '@esbuild/openbsd-arm64@0.27.7': - optional: true - - '@esbuild/openbsd-arm64@0.28.1': - optional: true - - '@esbuild/openbsd-x64@0.18.20': - optional: true - - '@esbuild/openbsd-x64@0.25.12': - optional: true - - '@esbuild/openbsd-x64@0.27.7': - optional: true - - '@esbuild/openbsd-x64@0.28.1': - optional: true - - '@esbuild/openharmony-arm64@0.25.12': - optional: true - - '@esbuild/openharmony-arm64@0.27.7': - optional: true - - '@esbuild/openharmony-arm64@0.28.1': - optional: true - - '@esbuild/sunos-x64@0.18.20': - optional: true - - '@esbuild/sunos-x64@0.25.12': - optional: true - - '@esbuild/sunos-x64@0.27.7': - optional: true - - '@esbuild/sunos-x64@0.28.1': - optional: true - - '@esbuild/win32-arm64@0.18.20': - optional: true - - '@esbuild/win32-arm64@0.25.12': - optional: true - - '@esbuild/win32-arm64@0.27.7': - optional: true - - '@esbuild/win32-arm64@0.28.1': - optional: true - - '@esbuild/win32-ia32@0.18.20': - optional: true - - '@esbuild/win32-ia32@0.25.12': - optional: true - - '@esbuild/win32-ia32@0.27.7': - optional: true - - '@esbuild/win32-ia32@0.28.1': - optional: true - - '@esbuild/win32-x64@0.18.20': - optional: true - - '@esbuild/win32-x64@0.25.12': - optional: true - - '@esbuild/win32-x64@0.27.7': - optional: true - - '@esbuild/win32-x64@0.28.1': - optional: true - - '@img/colour@1.1.0': {} - - '@img/sharp-darwin-arm64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-darwin-arm64': 1.2.4 - optional: true - - '@img/sharp-darwin-x64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-darwin-x64': 1.2.4 - optional: true - - '@img/sharp-libvips-darwin-arm64@1.2.4': - optional: true - - '@img/sharp-libvips-darwin-x64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-arm64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-arm@1.2.4': - optional: true - - '@img/sharp-libvips-linux-ppc64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-riscv64@1.2.4': - optional: true - - '@img/sharp-libvips-linux-s390x@1.2.4': - optional: true - - '@img/sharp-libvips-linux-x64@1.2.4': - optional: true - - '@img/sharp-libvips-linuxmusl-arm64@1.2.4': - optional: true - - '@img/sharp-libvips-linuxmusl-x64@1.2.4': - optional: true - - '@img/sharp-linux-arm64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-arm64': 1.2.4 - optional: true - - '@img/sharp-linux-arm@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-arm': 1.2.4 - optional: true - - '@img/sharp-linux-ppc64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-ppc64': 1.2.4 - optional: true - - '@img/sharp-linux-riscv64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-riscv64': 1.2.4 - optional: true - - '@img/sharp-linux-s390x@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-s390x': 1.2.4 - optional: true - - '@img/sharp-linux-x64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linux-x64': 1.2.4 - optional: true - - '@img/sharp-linuxmusl-arm64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 - optional: true - - '@img/sharp-linuxmusl-x64@0.34.5': - optionalDependencies: - '@img/sharp-libvips-linuxmusl-x64': 1.2.4 - optional: true - - '@img/sharp-wasm32@0.34.5': - dependencies: - '@emnapi/runtime': 1.10.0 - optional: true - - '@img/sharp-win32-arm64@0.34.5': - optional: true - - '@img/sharp-win32-ia32@0.34.5': - optional: true - - '@img/sharp-win32-x64@0.34.5': - optional: true - - '@jridgewell/resolve-uri@3.1.2': {} - - '@jridgewell/sourcemap-codec@1.5.5': {} - - '@jridgewell/trace-mapping@0.3.9': - dependencies: - '@jridgewell/resolve-uri': 3.1.2 - '@jridgewell/sourcemap-codec': 1.5.5 - - '@poppinss/colors@4.1.6': - dependencies: - kleur: 4.1.5 - - '@poppinss/dumper@0.6.5': - dependencies: - '@poppinss/colors': 4.1.6 - '@sindresorhus/is': 7.2.0 - supports-color: 10.2.2 - - '@poppinss/exception@1.2.3': {} - - '@rollup/rollup-android-arm-eabi@4.60.2': - optional: true - - '@rollup/rollup-android-arm64@4.60.2': - optional: true - - '@rollup/rollup-darwin-arm64@4.60.2': - optional: true - - '@rollup/rollup-darwin-x64@4.60.2': - optional: true - - '@rollup/rollup-freebsd-arm64@4.60.2': - optional: true - - '@rollup/rollup-freebsd-x64@4.60.2': - optional: true - - '@rollup/rollup-linux-arm-gnueabihf@4.60.2': - optional: true - - '@rollup/rollup-linux-arm-musleabihf@4.60.2': - optional: true - - '@rollup/rollup-linux-arm64-gnu@4.60.2': - optional: true - - '@rollup/rollup-linux-arm64-musl@4.60.2': - optional: true - - '@rollup/rollup-linux-loong64-gnu@4.60.2': - optional: true - - '@rollup/rollup-linux-loong64-musl@4.60.2': - optional: true - - '@rollup/rollup-linux-ppc64-gnu@4.60.2': - optional: true - - '@rollup/rollup-linux-ppc64-musl@4.60.2': - optional: true - - '@rollup/rollup-linux-riscv64-gnu@4.60.2': - optional: true - - '@rollup/rollup-linux-riscv64-musl@4.60.2': - optional: true - - '@rollup/rollup-linux-s390x-gnu@4.60.2': - optional: true - - '@rollup/rollup-linux-x64-gnu@4.60.2': - optional: true - - '@rollup/rollup-linux-x64-musl@4.60.2': - optional: true - - '@rollup/rollup-openbsd-x64@4.60.2': - optional: true - - '@rollup/rollup-openharmony-arm64@4.60.2': - optional: true - - '@rollup/rollup-win32-arm64-msvc@4.60.2': - optional: true - - '@rollup/rollup-win32-ia32-msvc@4.60.2': - optional: true - - '@rollup/rollup-win32-x64-gnu@4.60.2': - optional: true - - '@rollup/rollup-win32-x64-msvc@4.60.2': - optional: true - - '@sindresorhus/is@7.2.0': {} - - '@speed-highlight/core@1.2.15': {} - - '@standard-schema/spec@1.1.0': {} - - '@types/chai@5.2.3': - dependencies: - '@types/deep-eql': 4.0.2 - assertion-error: 2.0.1 - - '@types/deep-eql@4.0.2': {} - - '@types/estree@1.0.8': {} - - '@types/node@26.1.1': - dependencies: - undici-types: 8.3.0 - - '@typescript/typescript-aix-ppc64@7.0.2': - optional: true - - '@typescript/typescript-darwin-arm64@7.0.2': - optional: true - - '@typescript/typescript-darwin-x64@7.0.2': - optional: true - - '@typescript/typescript-freebsd-arm64@7.0.2': - optional: true - - '@typescript/typescript-freebsd-x64@7.0.2': - optional: true - - '@typescript/typescript-linux-arm64@7.0.2': - optional: true - - '@typescript/typescript-linux-arm@7.0.2': - optional: true - - '@typescript/typescript-linux-loong64@7.0.2': - optional: true - - '@typescript/typescript-linux-mips64el@7.0.2': - optional: true - - '@typescript/typescript-linux-ppc64@7.0.2': - optional: true - - '@typescript/typescript-linux-riscv64@7.0.2': - optional: true - - '@typescript/typescript-linux-s390x@7.0.2': - optional: true - - '@typescript/typescript-linux-x64@7.0.2': - optional: true - - '@typescript/typescript-netbsd-arm64@7.0.2': - optional: true - - '@typescript/typescript-netbsd-x64@7.0.2': - optional: true - - '@typescript/typescript-openbsd-arm64@7.0.2': - optional: true - - '@typescript/typescript-openbsd-x64@7.0.2': - optional: true - - '@typescript/typescript-sunos-x64@7.0.2': - optional: true - - '@typescript/typescript-win32-arm64@7.0.2': - optional: true - - '@typescript/typescript-win32-x64@7.0.2': - optional: true - - '@vitest/expect@4.1.10': - dependencies: - '@standard-schema/spec': 1.1.0 - '@types/chai': 5.2.3 - '@vitest/spy': 4.1.10 - '@vitest/utils': 4.1.10 - chai: 6.2.2 - tinyrainbow: 3.1.0 - - '@vitest/mocker@4.1.10(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0))': - dependencies: - '@vitest/spy': 4.1.10 - estree-walker: 3.0.3 - magic-string: 0.30.21 - optionalDependencies: - vite: 7.3.2(@types/node@26.1.1)(tsx@4.21.0) - - '@vitest/pretty-format@4.1.10': - dependencies: - tinyrainbow: 3.1.0 - - '@vitest/runner@4.1.10': - dependencies: - '@vitest/utils': 4.1.10 - pathe: 2.0.3 - - '@vitest/snapshot@4.1.10': - dependencies: - '@vitest/pretty-format': 4.1.10 - '@vitest/utils': 4.1.10 - magic-string: 0.30.21 - pathe: 2.0.3 - - '@vitest/spy@4.1.10': {} - - '@vitest/utils@4.1.10': - dependencies: - '@vitest/pretty-format': 4.1.10 - convert-source-map: 2.0.0 - tinyrainbow: 3.1.0 - - assertion-error@2.0.1: {} - - aws4fetch@1.0.20: {} - - blake3-wasm@2.1.5: {} - - buffer-from@1.1.2: {} - - chai@6.2.2: {} - - cjs-module-lexer@1.2.3: {} - - convert-source-map@2.0.0: {} - - cookie@1.1.1: {} - - detect-libc@2.1.2: {} - - drizzle-kit@0.31.10: - dependencies: - '@drizzle-team/brocli': 0.10.2 - '@esbuild-kit/esm-loader': 2.6.5 - esbuild: 0.25.12 - tsx: 4.21.0 - - drizzle-orm@0.45.2: {} - - error-stack-parser-es@1.0.5: {} - - es-module-lexer@2.3.1: {} - - esbuild@0.18.20: - optionalDependencies: - '@esbuild/android-arm': 0.18.20 - '@esbuild/android-arm64': 0.18.20 - '@esbuild/android-x64': 0.18.20 - '@esbuild/darwin-arm64': 0.18.20 - '@esbuild/darwin-x64': 0.18.20 - '@esbuild/freebsd-arm64': 0.18.20 - '@esbuild/freebsd-x64': 0.18.20 - '@esbuild/linux-arm': 0.18.20 - '@esbuild/linux-arm64': 0.18.20 - '@esbuild/linux-ia32': 0.18.20 - '@esbuild/linux-loong64': 0.18.20 - '@esbuild/linux-mips64el': 0.18.20 - '@esbuild/linux-ppc64': 0.18.20 - '@esbuild/linux-riscv64': 0.18.20 - '@esbuild/linux-s390x': 0.18.20 - '@esbuild/linux-x64': 0.18.20 - '@esbuild/netbsd-x64': 0.18.20 - '@esbuild/openbsd-x64': 0.18.20 - '@esbuild/sunos-x64': 0.18.20 - '@esbuild/win32-arm64': 0.18.20 - '@esbuild/win32-ia32': 0.18.20 - '@esbuild/win32-x64': 0.18.20 - - esbuild@0.25.12: - optionalDependencies: - '@esbuild/aix-ppc64': 0.25.12 - '@esbuild/android-arm': 0.25.12 - '@esbuild/android-arm64': 0.25.12 - '@esbuild/android-x64': 0.25.12 - '@esbuild/darwin-arm64': 0.25.12 - '@esbuild/darwin-x64': 0.25.12 - '@esbuild/freebsd-arm64': 0.25.12 - '@esbuild/freebsd-x64': 0.25.12 - '@esbuild/linux-arm': 0.25.12 - '@esbuild/linux-arm64': 0.25.12 - '@esbuild/linux-ia32': 0.25.12 - '@esbuild/linux-loong64': 0.25.12 - '@esbuild/linux-mips64el': 0.25.12 - '@esbuild/linux-ppc64': 0.25.12 - '@esbuild/linux-riscv64': 0.25.12 - '@esbuild/linux-s390x': 0.25.12 - '@esbuild/linux-x64': 0.25.12 - '@esbuild/netbsd-arm64': 0.25.12 - '@esbuild/netbsd-x64': 0.25.12 - '@esbuild/openbsd-arm64': 0.25.12 - '@esbuild/openbsd-x64': 0.25.12 - '@esbuild/openharmony-arm64': 0.25.12 - '@esbuild/sunos-x64': 0.25.12 - '@esbuild/win32-arm64': 0.25.12 - '@esbuild/win32-ia32': 0.25.12 - '@esbuild/win32-x64': 0.25.12 - - esbuild@0.27.7: - optionalDependencies: - '@esbuild/aix-ppc64': 0.27.7 - '@esbuild/android-arm': 0.27.7 - '@esbuild/android-arm64': 0.27.7 - '@esbuild/android-x64': 0.27.7 - '@esbuild/darwin-arm64': 0.27.7 - '@esbuild/darwin-x64': 0.27.7 - '@esbuild/freebsd-arm64': 0.27.7 - '@esbuild/freebsd-x64': 0.27.7 - '@esbuild/linux-arm': 0.27.7 - '@esbuild/linux-arm64': 0.27.7 - '@esbuild/linux-ia32': 0.27.7 - '@esbuild/linux-loong64': 0.27.7 - '@esbuild/linux-mips64el': 0.27.7 - '@esbuild/linux-ppc64': 0.27.7 - '@esbuild/linux-riscv64': 0.27.7 - '@esbuild/linux-s390x': 0.27.7 - '@esbuild/linux-x64': 0.27.7 - '@esbuild/netbsd-arm64': 0.27.7 - '@esbuild/netbsd-x64': 0.27.7 - '@esbuild/openbsd-arm64': 0.27.7 - '@esbuild/openbsd-x64': 0.27.7 - '@esbuild/openharmony-arm64': 0.27.7 - '@esbuild/sunos-x64': 0.27.7 - '@esbuild/win32-arm64': 0.27.7 - '@esbuild/win32-ia32': 0.27.7 - '@esbuild/win32-x64': 0.27.7 - - esbuild@0.28.1: - optionalDependencies: - '@esbuild/aix-ppc64': 0.28.1 - '@esbuild/android-arm': 0.28.1 - '@esbuild/android-arm64': 0.28.1 - '@esbuild/android-x64': 0.28.1 - '@esbuild/darwin-arm64': 0.28.1 - '@esbuild/darwin-x64': 0.28.1 - '@esbuild/freebsd-arm64': 0.28.1 - '@esbuild/freebsd-x64': 0.28.1 - '@esbuild/linux-arm': 0.28.1 - '@esbuild/linux-arm64': 0.28.1 - '@esbuild/linux-ia32': 0.28.1 - '@esbuild/linux-loong64': 0.28.1 - '@esbuild/linux-mips64el': 0.28.1 - '@esbuild/linux-ppc64': 0.28.1 - '@esbuild/linux-riscv64': 0.28.1 - '@esbuild/linux-s390x': 0.28.1 - '@esbuild/linux-x64': 0.28.1 - '@esbuild/netbsd-arm64': 0.28.1 - '@esbuild/netbsd-x64': 0.28.1 - '@esbuild/openbsd-arm64': 0.28.1 - '@esbuild/openbsd-x64': 0.28.1 - '@esbuild/openharmony-arm64': 0.28.1 - '@esbuild/sunos-x64': 0.28.1 - '@esbuild/win32-arm64': 0.28.1 - '@esbuild/win32-ia32': 0.28.1 - '@esbuild/win32-x64': 0.28.1 - - estree-walker@3.0.3: - dependencies: - '@types/estree': 1.0.8 - - expect-type@1.3.0: {} - - fdir@6.5.0(picomatch@4.0.4): - optionalDependencies: - picomatch: 4.0.4 - - fsevents@2.3.3: - optional: true - - get-tsconfig@4.14.0: - dependencies: - resolve-pkg-maps: 1.0.0 - - kleur@4.1.5: {} - - magic-string@0.30.21: - dependencies: - '@jridgewell/sourcemap-codec': 1.5.5 - - miniflare@4.20260714.0: - dependencies: - '@cspotcode/source-map-support': 0.8.1 - sharp: 0.34.5 - undici: 7.28.0 - workerd: 1.20260714.1 - ws: 8.21.0 - youch: 4.1.0-beta.10 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - nanoid@3.3.11: {} - - obug@2.1.4: {} - - path-to-regexp@6.3.0: {} - - pathe@2.0.3: {} - - picocolors@1.1.1: {} - - picomatch@4.0.4: {} - - postcss@8.5.10: - dependencies: - nanoid: 3.3.11 - picocolors: 1.1.1 - source-map-js: 1.2.1 - - resolve-pkg-maps@1.0.0: {} - - rollup@4.60.2: - dependencies: - '@types/estree': 1.0.8 - optionalDependencies: - '@rollup/rollup-android-arm-eabi': 4.60.2 - '@rollup/rollup-android-arm64': 4.60.2 - '@rollup/rollup-darwin-arm64': 4.60.2 - '@rollup/rollup-darwin-x64': 4.60.2 - '@rollup/rollup-freebsd-arm64': 4.60.2 - '@rollup/rollup-freebsd-x64': 4.60.2 - '@rollup/rollup-linux-arm-gnueabihf': 4.60.2 - '@rollup/rollup-linux-arm-musleabihf': 4.60.2 - '@rollup/rollup-linux-arm64-gnu': 4.60.2 - '@rollup/rollup-linux-arm64-musl': 4.60.2 - '@rollup/rollup-linux-loong64-gnu': 4.60.2 - '@rollup/rollup-linux-loong64-musl': 4.60.2 - '@rollup/rollup-linux-ppc64-gnu': 4.60.2 - '@rollup/rollup-linux-ppc64-musl': 4.60.2 - '@rollup/rollup-linux-riscv64-gnu': 4.60.2 - '@rollup/rollup-linux-riscv64-musl': 4.60.2 - '@rollup/rollup-linux-s390x-gnu': 4.60.2 - '@rollup/rollup-linux-x64-gnu': 4.60.2 - '@rollup/rollup-linux-x64-musl': 4.60.2 - '@rollup/rollup-openbsd-x64': 4.60.2 - '@rollup/rollup-openharmony-arm64': 4.60.2 - '@rollup/rollup-win32-arm64-msvc': 4.60.2 - '@rollup/rollup-win32-ia32-msvc': 4.60.2 - '@rollup/rollup-win32-x64-gnu': 4.60.2 - '@rollup/rollup-win32-x64-msvc': 4.60.2 - fsevents: 2.3.3 - - semver@7.7.4: {} - - sharp@0.34.5: - dependencies: - '@img/colour': 1.1.0 - detect-libc: 2.1.2 - semver: 7.7.4 - optionalDependencies: - '@img/sharp-darwin-arm64': 0.34.5 - '@img/sharp-darwin-x64': 0.34.5 - '@img/sharp-libvips-darwin-arm64': 1.2.4 - '@img/sharp-libvips-darwin-x64': 1.2.4 - '@img/sharp-libvips-linux-arm': 1.2.4 - '@img/sharp-libvips-linux-arm64': 1.2.4 - '@img/sharp-libvips-linux-ppc64': 1.2.4 - '@img/sharp-libvips-linux-riscv64': 1.2.4 - '@img/sharp-libvips-linux-s390x': 1.2.4 - '@img/sharp-libvips-linux-x64': 1.2.4 - '@img/sharp-libvips-linuxmusl-arm64': 1.2.4 - '@img/sharp-libvips-linuxmusl-x64': 1.2.4 - '@img/sharp-linux-arm': 0.34.5 - '@img/sharp-linux-arm64': 0.34.5 - '@img/sharp-linux-ppc64': 0.34.5 - '@img/sharp-linux-riscv64': 0.34.5 - '@img/sharp-linux-s390x': 0.34.5 - '@img/sharp-linux-x64': 0.34.5 - '@img/sharp-linuxmusl-arm64': 0.34.5 - '@img/sharp-linuxmusl-x64': 0.34.5 - '@img/sharp-wasm32': 0.34.5 - '@img/sharp-win32-arm64': 0.34.5 - '@img/sharp-win32-ia32': 0.34.5 - '@img/sharp-win32-x64': 0.34.5 - - siginfo@2.0.0: {} - - source-map-js@1.2.1: {} - - source-map-support@0.5.21: - dependencies: - buffer-from: 1.1.2 - source-map: 0.6.1 - - source-map@0.6.1: {} - - stackback@0.0.2: {} - - std-env@4.2.0: {} - - supports-color@10.2.2: {} - - tinybench@2.9.0: {} - - tinyexec@1.2.4: {} - - tinyglobby@0.2.16: - dependencies: - fdir: 6.5.0(picomatch@4.0.4) - picomatch: 4.0.4 - - tinyrainbow@3.1.0: {} - - tslib@2.8.1: - optional: true - - tsx@4.21.0: - dependencies: - esbuild: 0.27.7 - get-tsconfig: 4.14.0 - optionalDependencies: - fsevents: 2.3.3 - - typescript@7.0.2: - optionalDependencies: - '@typescript/typescript-aix-ppc64': 7.0.2 - '@typescript/typescript-darwin-arm64': 7.0.2 - '@typescript/typescript-darwin-x64': 7.0.2 - '@typescript/typescript-freebsd-arm64': 7.0.2 - '@typescript/typescript-freebsd-x64': 7.0.2 - '@typescript/typescript-linux-arm': 7.0.2 - '@typescript/typescript-linux-arm64': 7.0.2 - '@typescript/typescript-linux-loong64': 7.0.2 - '@typescript/typescript-linux-mips64el': 7.0.2 - '@typescript/typescript-linux-ppc64': 7.0.2 - '@typescript/typescript-linux-riscv64': 7.0.2 - '@typescript/typescript-linux-s390x': 7.0.2 - '@typescript/typescript-linux-x64': 7.0.2 - '@typescript/typescript-netbsd-arm64': 7.0.2 - '@typescript/typescript-netbsd-x64': 7.0.2 - '@typescript/typescript-openbsd-arm64': 7.0.2 - '@typescript/typescript-openbsd-x64': 7.0.2 - '@typescript/typescript-sunos-x64': 7.0.2 - '@typescript/typescript-win32-arm64': 7.0.2 - '@typescript/typescript-win32-x64': 7.0.2 - - undici-types@8.3.0: {} - - undici@7.28.0: {} - - unenv@2.0.0-rc.24: - dependencies: - pathe: 2.0.3 - - vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0): - dependencies: - esbuild: 0.27.7 - fdir: 6.5.0(picomatch@4.0.4) - picomatch: 4.0.4 - postcss: 8.5.10 - rollup: 4.60.2 - tinyglobby: 0.2.16 - optionalDependencies: - '@types/node': 26.1.1 - fsevents: 2.3.3 - tsx: 4.21.0 - - vitest@4.1.10(@types/node@26.1.1)(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0)): - dependencies: - '@vitest/expect': 4.1.10 - '@vitest/mocker': 4.1.10(vite@7.3.2(@types/node@26.1.1)(tsx@4.21.0)) - '@vitest/pretty-format': 4.1.10 - '@vitest/runner': 4.1.10 - '@vitest/snapshot': 4.1.10 - '@vitest/spy': 4.1.10 - '@vitest/utils': 4.1.10 - es-module-lexer: 2.3.1 - expect-type: 1.3.0 - magic-string: 0.30.21 - obug: 2.1.4 - pathe: 2.0.3 - picomatch: 4.0.4 - std-env: 4.2.0 - tinybench: 2.9.0 - tinyexec: 1.2.4 - tinyglobby: 0.2.16 - tinyrainbow: 3.1.0 - vite: 7.3.2(@types/node@26.1.1)(tsx@4.21.0) - why-is-node-running: 2.3.0 - optionalDependencies: - '@types/node': 26.1.1 - transitivePeerDependencies: - - msw - - why-is-node-running@2.3.0: - dependencies: - siginfo: 2.0.0 - stackback: 0.0.2 - - workerd@1.20260714.1: - optionalDependencies: - '@cloudflare/workerd-darwin-64': 1.20260714.1 - '@cloudflare/workerd-darwin-arm64': 1.20260714.1 - '@cloudflare/workerd-linux-64': 1.20260714.1 - '@cloudflare/workerd-linux-arm64': 1.20260714.1 - '@cloudflare/workerd-windows-64': 1.20260714.1 - - wrangler@4.112.0: - dependencies: - '@cloudflare/kv-asset-handler': 0.5.0 - '@cloudflare/unenv-preset': 2.16.1(unenv@2.0.0-rc.24)(workerd@1.20260714.1) - blake3-wasm: 2.1.5 - esbuild: 0.28.1 - miniflare: 4.20260714.0 - path-to-regexp: 6.3.0 - unenv: 2.0.0-rc.24 - workerd: 1.20260714.1 - optionalDependencies: - fsevents: 2.3.3 - transitivePeerDependencies: - - bufferutil - - utf-8-validate - - ws@8.21.0: {} - - youch-core@0.3.3: - dependencies: - '@poppinss/exception': 1.2.3 - error-stack-parser-es: 1.0.5 - - youch@4.1.0-beta.10: - dependencies: - '@poppinss/colors': 4.1.6 - '@poppinss/dumper': 0.6.5 - '@speed-highlight/core': 1.2.15 - cookie: 1.1.1 - youch-core: 0.3.3 - - zod@3.25.76: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml deleted file mode 100644 index 94830da..0000000 --- a/pnpm-workspace.yaml +++ /dev/null @@ -1,8 +0,0 @@ -packages: - - apps/* - - packages/* - - workers/* -allowBuilds: - esbuild: true - sharp: true - workerd: true diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh deleted file mode 100755 index dd2f7d5..0000000 --- a/scripts/smoke-test.sh +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Smoke test for the Helios Nix binary cache worker. -# -# Usage: -# ./scripts/smoke-test.sh -# -# Example: -# ./scripts/smoke-test.sh https://helios-cache.your-subdomain.workers.dev "your-token" -# ./scripts/smoke-test.sh https://cache.polmath.no "your-token" - -BASE="${1:?usage: smoke-test.sh }" -TOKEN="${2:?usage: smoke-test.sh }" - -# Strip trailing slash -BASE="${BASE%/}" - -# Resolve worker dir for wrangler d1 seeding -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -WORKER_DIR="$(cd "$SCRIPT_DIR/../workers/cache" && pwd)" - -PASS=0 -FAIL=0 - -check() { - local desc="$1" expected="$2" actual="$3" - if [ "$actual" = "$expected" ]; then - echo " ok: $desc" - PASS=$((PASS + 1)) - else - echo " FAIL: $desc (expected=$expected got=$actual)" - FAIL=$((FAIL + 1)) - fi -} - -echo "=== Smoke test against $BASE ===" -echo - -# ── 1. Health check ── -echo "1. Health check" -STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/healthz") -check "/healthz returns 200" "200" "$STATUS" - -BODY=$(curl -s "$BASE/healthz") -check "/healthz has ok=true" "true" "$(echo "$BODY" | grep -o '"ok" *: *true' | head -1 | grep -o 'true')" -echo - -# ── 2. Seed a cache via D1 ── -echo "2. Seeding test cache via D1" -CACHE_NAME="smoke-$(date +%s)" -wrangler d1 execute helios-cache --remote --yes \ - -c "$WORKER_DIR/wrangler.jsonc" \ - --command "INSERT OR IGNORE INTO caches (name, is_public) VALUES ('$CACHE_NAME', 1)" \ - > /dev/null 2>&1 -echo " created cache: $CACHE_NAME" -echo - -# ── 3. Cache info ── -echo "3. nix-cache-info" -STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/$CACHE_NAME/nix-cache-info") -check "returns 200" "200" "$STATUS" - -BODY=$(curl -s "$BASE/$CACHE_NAME/nix-cache-info") -check "contains StoreDir" "StoreDir: /nix/store" "$(echo "$BODY" | head -1)" -echo - -# ── 4. 404 for missing narinfo ── -echo "4. Missing narinfo returns 404" -STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/$CACHE_NAME/00000000000000000000000000000000.narinfo") -check "returns 404" "404" "$STATUS" -echo - -# ── 5. Full upload → publish → read flow ── -echo "5. Upload, complete, publish, read" - -FILE_HASH="$(openssl rand -hex 32)" -STORE_PATH_HASH="00000000000000000000000000000000" -NAR_HASH="sha256:1111111111111111111111111111111111111111111111111111" -NAR_CONTENT="nix-archive-smoke-test-payload1234" -FILE_SIZE=${#NAR_CONTENT} - -# 5a. Create upload session -echo " 5a. Creating upload session" -SESSION_STATUS=$(curl -s -o /tmp/smoke-session.json -w "%{http_code}" -X POST "$BASE/_api/v1/caches/$CACHE_NAME/upload-sessions" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d "{ - \"storePath\": \"/nix/store/${STORE_PATH_HASH}-smoke-test\", - \"storePathHash\": \"$STORE_PATH_HASH\", - \"narHash\": \"$NAR_HASH\", - \"narSize\": 2048, - \"fileHash\": \"$FILE_HASH\", - \"fileSize\": $FILE_SIZE, - \"compression\": \"none\" - }") - -SESSION_RESP=$(cat /tmp/smoke-session.json) -rm -f /tmp/smoke-session.json -SESSION_ID=$(echo "$SESSION_RESP" | grep -o '"sessionId":"[^"]*"' | cut -d'"' -f4) -R2_KEY=$(echo "$SESSION_RESP" | grep -o '"r2Key":"[^"]*"' | cut -d'"' -f4) - -if [ -z "$SESSION_ID" ]; then - echo " FAIL: could not create session (status=$SESSION_STATUS): $SESSION_RESP" - FAIL=$((FAIL + 1)) -else - check "session created (201)" "201" "$SESSION_STATUS" - echo " session=$SESSION_ID r2Key=$R2_KEY" - - # 5b. Upload blob via API - echo " 5b. Uploading blob via API" - UPLOAD_STATUS=$(curl -s -o /dev/null -w "%{http_code}" \ - -X PUT "$BASE/_api/v1/uploads/$SESSION_ID/blob" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/octet-stream" \ - --data-binary "$NAR_CONTENT") - check "blob upload returns 200" "200" "$UPLOAD_STATUS" - - # 5c. Complete - echo " 5c. Completing upload" - COMPLETE_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ - "$BASE/_api/v1/uploads/$SESSION_ID/complete" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{}') - check "complete returns 200" "200" "$COMPLETE_STATUS" - - # 5d. Publish - echo " 5d. Publishing path" - PUBLISH_RESP=$(curl -s -X POST "$BASE/_api/v1/uploads/$SESSION_ID/publish" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{}') - PUBLISH_STATUS=$(echo "$PUBLISH_RESP" | grep -o '"published":true' | head -1) - check "publish returns published=true" '"published":true' "$PUBLISH_STATUS" - - # 5e. Read narinfo - echo " 5e. Reading narinfo" - NARINFO_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/$CACHE_NAME/$STORE_PATH_HASH.narinfo") - check "narinfo returns 200" "200" "$NARINFO_STATUS" - - NARINFO=$(curl -s "$BASE/$CACHE_NAME/$STORE_PATH_HASH.narinfo") - check "narinfo contains StorePath" "/nix/store/${STORE_PATH_HASH}-smoke-test" \ - "$(echo "$NARINFO" | grep '^StorePath:' | cut -d' ' -f2)" - check "narinfo contains NarHash" "$NAR_HASH" \ - "$(echo "$NARINFO" | grep '^NarHash:' | cut -d' ' -f2)" - - # 5f. Read NAR blob - echo " 5f. Reading NAR blob" - NAR_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/$CACHE_NAME/nar/$FILE_HASH/none.nar") - check "NAR download returns 200" "200" "$NAR_STATUS" - - NAR_BODY=$(curl -s "$BASE/$CACHE_NAME/nar/$FILE_HASH/none.nar") - check "NAR body matches uploaded content" "$NAR_CONTENT" "$NAR_BODY" - - # 5g. Idempotent publish - echo " 5g. Duplicate publish is idempotent" - REPUB=$(curl -s -X POST "$BASE/_api/v1/uploads/$SESSION_ID/publish" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d '{}') - check "re-publish has alreadyExisted" '"alreadyExisted":true' \ - "$(echo "$REPUB" | grep -o '"alreadyExisted":true' | head -1)" -fi -echo - -# ── 6. Auth enforcement ── -echo "6. Auth enforcement" -UNAUTH=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ - "$BASE/_api/v1/caches/$CACHE_NAME/upload-sessions" \ - -H "Content-Type: application/json" \ - -d '{}') -check "write without auth returns 401" "401" "$UNAUTH" - -BAD_AUTH=$(curl -s -o /dev/null -w "%{http_code}" -X POST \ - "$BASE/_api/v1/caches/$CACHE_NAME/upload-sessions" \ - -H "Authorization: Bearer wrong-token" \ - -H "Content-Type: application/json" \ - -d '{}') -check "write with bad token returns 403" "403" "$BAD_AUTH" - -PUBLIC_READ=$(curl -s -o /dev/null -w "%{http_code}" "$BASE/$CACHE_NAME/nix-cache-info") -check "read without auth returns 200" "200" "$PUBLIC_READ" -echo - -# ── 7. get-missing-paths ── -echo "7. get-missing-paths" -MISSING_RESP=$(curl -s -X POST "$BASE/_api/v1/get-missing-paths" \ - -H "Authorization: Bearer $TOKEN" \ - -H "Content-Type: application/json" \ - -d "{\"cache\": \"$CACHE_NAME\", \"storePathHashes\": [\"$STORE_PATH_HASH\", \"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"]}") -check "published hash not in missing" "" \ - "$(echo "$MISSING_RESP" | grep -o "\"$STORE_PATH_HASH\"" | head -1)" -check "unknown hash is in missing" '"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"' \ - "$(echo "$MISSING_RESP" | grep -o '"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"' | head -1)" -echo - -# ── Summary ── -echo "=== Results: $PASS passed, $FAIL failed ===" -if [ "$FAIL" -gt 0 ]; then - exit 1 -fi diff --git a/workers/cache/.editorconfig b/workers/cache/.editorconfig deleted file mode 100644 index a727df3..0000000 --- a/workers/cache/.editorconfig +++ /dev/null @@ -1,12 +0,0 @@ -# http://editorconfig.org -root = true - -[*] -indent_style = tab -end_of_line = lf -charset = utf-8 -trim_trailing_whitespace = true -insert_final_newline = true - -[*.yml] -indent_style = space diff --git a/workers/cache/.gitignore b/workers/cache/.gitignore deleted file mode 100644 index 4138168..0000000 --- a/workers/cache/.gitignore +++ /dev/null @@ -1,167 +0,0 @@ -# Logs - -logs -_.log -npm-debug.log_ -yarn-debug.log* -yarn-error.log* -lerna-debug.log* -.pnpm-debug.log* - -# Diagnostic reports (https://nodejs.org/api/report.html) - -report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json - -# Runtime data - -pids -_.pid -_.seed -\*.pid.lock - -# Directory for instrumented libs generated by jscoverage/JSCover - -lib-cov - -# Coverage directory used by tools like istanbul - -coverage -\*.lcov - -# nyc test coverage - -.nyc_output - -# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) - -.grunt - -# Bower dependency directory (https://bower.io/) - -bower_components - -# node-waf configuration - -.lock-wscript - -# Compiled binary addons (https://nodejs.org/api/addons.html) - -build/Release - -# Dependency directories - -node_modules/ -jspm_packages/ - -# Snowpack dependency directory (https://snowpack.dev/) - -web_modules/ - -# TypeScript cache - -\*.tsbuildinfo - -# Optional npm cache directory - -.npm - -# Optional eslint cache - -.eslintcache - -# Optional stylelint cache - -.stylelintcache - -# Microbundle cache - -.rpt2_cache/ -.rts2_cache_cjs/ -.rts2_cache_es/ -.rts2_cache_umd/ - -# Optional REPL history - -.node_repl_history - -# Output of 'npm pack' - -\*.tgz - -# Yarn Integrity file - -.yarn-integrity - -# parcel-bundler cache (https://parceljs.org/) - -.cache -.parcel-cache - -# Next.js build output - -.next -out - -# Nuxt.js build / generate output - -.nuxt -dist - -# Gatsby files - -.cache/ - -# Comment in the public line in if your project uses Gatsby and not Next.js - -# https://nextjs.org/blog/next-9-1#public-directory-support - -# public - -# vuepress build output - -.vuepress/dist - -# vuepress v2.x temp and cache directory - -.temp -.cache - -# Docusaurus cache and generated files - -.docusaurus - -# Serverless directories - -.serverless/ - -# FuseBox cache - -.fusebox/ - -# DynamoDB Local files - -.dynamodb/ - -# TernJS port file - -.tern-port - -# Stores VSCode versions used for testing VSCode extensions - -.vscode-test - -# yarn v2 - -.yarn/cache -.yarn/unplugged -.yarn/build-state.yml -.yarn/install-state.gz -.pnp.\* - -# wrangler project - -.dev.vars* -!.dev.vars.example -.env* -!.env.example -.wrangler/ diff --git a/workers/cache/.prettierrc b/workers/cache/.prettierrc deleted file mode 100644 index 5c7b5d3..0000000 --- a/workers/cache/.prettierrc +++ /dev/null @@ -1,6 +0,0 @@ -{ - "printWidth": 140, - "singleQuote": true, - "semi": true, - "useTabs": true -} diff --git a/workers/cache/drizzle.config.ts b/workers/cache/drizzle.config.ts deleted file mode 100644 index d4cfa11..0000000 --- a/workers/cache/drizzle.config.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { defineConfig } from "drizzle-kit"; - -export default defineConfig({ - schema: "./src/db/schema.ts", - out: "./migrations", - dialect: "sqlite", -}); diff --git a/workers/cache/migrations/0000_cultured_night_nurse.sql b/workers/cache/migrations/0000_cultured_night_nurse.sql deleted file mode 100644 index 434ee57..0000000 --- a/workers/cache/migrations/0000_cultured_night_nurse.sql +++ /dev/null @@ -1,79 +0,0 @@ -CREATE TABLE `blob_objects` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `file_hash` text NOT NULL, - `file_size` integer NOT NULL, - `compression` text NOT NULL, - `r2_key` text NOT NULL, - `created_at` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL -); ---> statement-breakpoint -CREATE UNIQUE INDEX `blob_objects_r2_key_unique` ON `blob_objects` (`r2_key`);--> statement-breakpoint -CREATE UNIQUE INDEX `blob_objects_file_hash_compression_unique` ON `blob_objects` (`file_hash`,`compression`);--> statement-breakpoint -CREATE TABLE `caches` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `name` text NOT NULL, - `is_public` integer DEFAULT 1 NOT NULL, - `created_at` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL -); ---> statement-breakpoint -CREATE UNIQUE INDEX `caches_name_unique` ON `caches` (`name`);--> statement-breakpoint -CREATE TABLE `gc_marks` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `target_type` text NOT NULL, - `target_id` text NOT NULL, - `reason` text NOT NULL, - `marked_at` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL -); ---> statement-breakpoint -CREATE UNIQUE INDEX `gc_marks_target_unique` ON `gc_marks` (`target_type`,`target_id`);--> statement-breakpoint -CREATE TABLE `published_paths` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `cache_id` integer NOT NULL, - `store_path_hash` text NOT NULL, - `store_path` text NOT NULL, - `nar_hash` text NOT NULL, - `nar_size` integer NOT NULL, - `blob_object_id` integer NOT NULL, - `references_json` text DEFAULT '[]' NOT NULL, - `deriver` text, - `system` text, - `signatures_json` text DEFAULT '[]' NOT NULL, - `created_at` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL, - FOREIGN KEY (`cache_id`) REFERENCES `caches`(`id`) ON UPDATE no action ON DELETE no action, - FOREIGN KEY (`blob_object_id`) REFERENCES `blob_objects`(`id`) ON UPDATE no action ON DELETE no action -); ---> statement-breakpoint -CREATE UNIQUE INDEX `published_paths_cache_store_unique` ON `published_paths` (`cache_id`,`store_path_hash`);--> statement-breakpoint -CREATE INDEX `idx_published_paths_lookup` ON `published_paths` (`cache_id`,`store_path_hash`);--> statement-breakpoint -CREATE TABLE `upload_parts` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `session_id` text NOT NULL, - `part_number` integer NOT NULL, - `etag` text NOT NULL, - `size` integer NOT NULL, - FOREIGN KEY (`session_id`) REFERENCES `upload_sessions`(`id`) ON UPDATE no action ON DELETE no action -); ---> statement-breakpoint -CREATE UNIQUE INDEX `upload_parts_session_part_unique` ON `upload_parts` (`session_id`,`part_number`);--> statement-breakpoint -CREATE TABLE `upload_sessions` ( - `id` text PRIMARY KEY NOT NULL, - `cache_id` integer NOT NULL, - `store_path_hash` text NOT NULL, - `store_path` text NOT NULL, - `nar_hash` text NOT NULL, - `nar_size` integer NOT NULL, - `file_hash` text NOT NULL, - `file_size` integer NOT NULL, - `compression` text NOT NULL, - `references_json` text DEFAULT '[]' NOT NULL, - `deriver` text, - `system` text, - `status` text DEFAULT 'pending' NOT NULL, - `r2_upload_key` text, - `r2_upload_id` text, - `created_at` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL, - `expires_at` text NOT NULL, - FOREIGN KEY (`cache_id`) REFERENCES `caches`(`id`) ON UPDATE no action ON DELETE no action -); ---> statement-breakpoint -CREATE INDEX `idx_upload_sessions_status` ON `upload_sessions` (`status`,`expires_at`); \ No newline at end of file diff --git a/workers/cache/migrations/0001_overjoyed_exiles.sql b/workers/cache/migrations/0001_overjoyed_exiles.sql deleted file mode 100644 index a68632c..0000000 --- a/workers/cache/migrations/0001_overjoyed_exiles.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP INDEX `idx_published_paths_lookup`;--> statement-breakpoint -CREATE INDEX `idx_published_paths_blob_object` ON `published_paths` (`blob_object_id`); \ No newline at end of file diff --git a/workers/cache/migrations/0002_ordinary_wiccan.sql b/workers/cache/migrations/0002_ordinary_wiccan.sql deleted file mode 100644 index a8dae49..0000000 --- a/workers/cache/migrations/0002_ordinary_wiccan.sql +++ /dev/null @@ -1,27 +0,0 @@ -CREATE TABLE `api_tokens` ( - `jti` text PRIMARY KEY NOT NULL, - `subject` text NOT NULL, - `caches_json` text NOT NULL, - `perms_json` text NOT NULL, - `created_at` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL, - `expires_at` text NOT NULL, - `created_by` text NOT NULL, - `revoked_at` text, - `revoked_by` text, - `revocation_reason` text -); ---> statement-breakpoint -CREATE TABLE `audit_logs` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `timestamp` text DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) NOT NULL, - `actor` text NOT NULL, - `action` text NOT NULL, - `cache_name` text, - `detail` text DEFAULT '{}' NOT NULL, - `ip` text, - `status` integer NOT NULL -); ---> statement-breakpoint -CREATE INDEX `idx_audit_logs_actor` ON `audit_logs` (`actor`);--> statement-breakpoint -CREATE INDEX `idx_audit_logs_cache` ON `audit_logs` (`cache_name`);--> statement-breakpoint -CREATE INDEX `idx_audit_logs_time` ON `audit_logs` (`timestamp`); \ No newline at end of file diff --git a/workers/cache/migrations/meta/0000_snapshot.json b/workers/cache/migrations/meta/0000_snapshot.json deleted file mode 100644 index 2e46b24..0000000 --- a/workers/cache/migrations/meta/0000_snapshot.json +++ /dev/null @@ -1,554 +0,0 @@ -{ - "version": "6", - "dialect": "sqlite", - "id": "201f9766-f457-4259-b02e-f3aa0e4150c0", - "prevId": "00000000-0000-0000-0000-000000000000", - "tables": { - "blob_objects": { - "name": "blob_objects", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "file_hash": { - "name": "file_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_size": { - "name": "file_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "compression": { - "name": "compression", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "r2_key": { - "name": "r2_key", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "blob_objects_r2_key_unique": { - "name": "blob_objects_r2_key_unique", - "columns": [ - "r2_key" - ], - "isUnique": true - }, - "blob_objects_file_hash_compression_unique": { - "name": "blob_objects_file_hash_compression_unique", - "columns": [ - "file_hash", - "compression" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "caches": { - "name": "caches", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "is_public": { - "name": "is_public", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "caches_name_unique": { - "name": "caches_name_unique", - "columns": [ - "name" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "gc_marks": { - "name": "gc_marks", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "target_type": { - "name": "target_type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "target_id": { - "name": "target_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "reason": { - "name": "reason", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "marked_at": { - "name": "marked_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "gc_marks_target_unique": { - "name": "gc_marks_target_unique", - "columns": [ - "target_type", - "target_id" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "published_paths": { - "name": "published_paths", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "cache_id": { - "name": "cache_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path_hash": { - "name": "store_path_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path": { - "name": "store_path", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_hash": { - "name": "nar_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_size": { - "name": "nar_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "blob_object_id": { - "name": "blob_object_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "references_json": { - "name": "references_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "deriver": { - "name": "deriver", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "system": { - "name": "system", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "signatures_json": { - "name": "signatures_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "published_paths_cache_store_unique": { - "name": "published_paths_cache_store_unique", - "columns": [ - "cache_id", - "store_path_hash" - ], - "isUnique": true - }, - "idx_published_paths_lookup": { - "name": "idx_published_paths_lookup", - "columns": [ - "cache_id", - "store_path_hash" - ], - "isUnique": false - } - }, - "foreignKeys": { - "published_paths_cache_id_caches_id_fk": { - "name": "published_paths_cache_id_caches_id_fk", - "tableFrom": "published_paths", - "tableTo": "caches", - "columnsFrom": [ - "cache_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - }, - "published_paths_blob_object_id_blob_objects_id_fk": { - "name": "published_paths_blob_object_id_blob_objects_id_fk", - "tableFrom": "published_paths", - "tableTo": "blob_objects", - "columnsFrom": [ - "blob_object_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "upload_parts": { - "name": "upload_parts", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "session_id": { - "name": "session_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "part_number": { - "name": "part_number", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "etag": { - "name": "etag", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "size": { - "name": "size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "upload_parts_session_part_unique": { - "name": "upload_parts_session_part_unique", - "columns": [ - "session_id", - "part_number" - ], - "isUnique": true - } - }, - "foreignKeys": { - "upload_parts_session_id_upload_sessions_id_fk": { - "name": "upload_parts_session_id_upload_sessions_id_fk", - "tableFrom": "upload_parts", - "tableTo": "upload_sessions", - "columnsFrom": [ - "session_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "upload_sessions": { - "name": "upload_sessions", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "cache_id": { - "name": "cache_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path_hash": { - "name": "store_path_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path": { - "name": "store_path", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_hash": { - "name": "nar_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_size": { - "name": "nar_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_hash": { - "name": "file_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_size": { - "name": "file_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "compression": { - "name": "compression", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "references_json": { - "name": "references_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "deriver": { - "name": "deriver", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "system": { - "name": "system", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "r2_upload_key": { - "name": "r2_upload_key", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "r2_upload_id": { - "name": "r2_upload_id", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - }, - "expires_at": { - "name": "expires_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "idx_upload_sessions_status": { - "name": "idx_upload_sessions_status", - "columns": [ - "status", - "expires_at" - ], - "isUnique": false - } - }, - "foreignKeys": { - "upload_sessions_cache_id_caches_id_fk": { - "name": "upload_sessions_cache_id_caches_id_fk", - "tableFrom": "upload_sessions", - "tableTo": "caches", - "columnsFrom": [ - "cache_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - } - }, - "views": {}, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} - } -} \ No newline at end of file diff --git a/workers/cache/migrations/meta/0001_snapshot.json b/workers/cache/migrations/meta/0001_snapshot.json deleted file mode 100644 index a0812fe..0000000 --- a/workers/cache/migrations/meta/0001_snapshot.json +++ /dev/null @@ -1,553 +0,0 @@ -{ - "version": "6", - "dialect": "sqlite", - "id": "7e8a49f8-b25b-4ee7-8029-ba438ac76987", - "prevId": "201f9766-f457-4259-b02e-f3aa0e4150c0", - "tables": { - "blob_objects": { - "name": "blob_objects", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "file_hash": { - "name": "file_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_size": { - "name": "file_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "compression": { - "name": "compression", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "r2_key": { - "name": "r2_key", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "blob_objects_r2_key_unique": { - "name": "blob_objects_r2_key_unique", - "columns": [ - "r2_key" - ], - "isUnique": true - }, - "blob_objects_file_hash_compression_unique": { - "name": "blob_objects_file_hash_compression_unique", - "columns": [ - "file_hash", - "compression" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "caches": { - "name": "caches", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "is_public": { - "name": "is_public", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "caches_name_unique": { - "name": "caches_name_unique", - "columns": [ - "name" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "gc_marks": { - "name": "gc_marks", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "target_type": { - "name": "target_type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "target_id": { - "name": "target_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "reason": { - "name": "reason", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "marked_at": { - "name": "marked_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "gc_marks_target_unique": { - "name": "gc_marks_target_unique", - "columns": [ - "target_type", - "target_id" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "published_paths": { - "name": "published_paths", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "cache_id": { - "name": "cache_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path_hash": { - "name": "store_path_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path": { - "name": "store_path", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_hash": { - "name": "nar_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_size": { - "name": "nar_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "blob_object_id": { - "name": "blob_object_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "references_json": { - "name": "references_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "deriver": { - "name": "deriver", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "system": { - "name": "system", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "signatures_json": { - "name": "signatures_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "published_paths_cache_store_unique": { - "name": "published_paths_cache_store_unique", - "columns": [ - "cache_id", - "store_path_hash" - ], - "isUnique": true - }, - "idx_published_paths_blob_object": { - "name": "idx_published_paths_blob_object", - "columns": [ - "blob_object_id" - ], - "isUnique": false - } - }, - "foreignKeys": { - "published_paths_cache_id_caches_id_fk": { - "name": "published_paths_cache_id_caches_id_fk", - "tableFrom": "published_paths", - "tableTo": "caches", - "columnsFrom": [ - "cache_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - }, - "published_paths_blob_object_id_blob_objects_id_fk": { - "name": "published_paths_blob_object_id_blob_objects_id_fk", - "tableFrom": "published_paths", - "tableTo": "blob_objects", - "columnsFrom": [ - "blob_object_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "upload_parts": { - "name": "upload_parts", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "session_id": { - "name": "session_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "part_number": { - "name": "part_number", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "etag": { - "name": "etag", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "size": { - "name": "size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "upload_parts_session_part_unique": { - "name": "upload_parts_session_part_unique", - "columns": [ - "session_id", - "part_number" - ], - "isUnique": true - } - }, - "foreignKeys": { - "upload_parts_session_id_upload_sessions_id_fk": { - "name": "upload_parts_session_id_upload_sessions_id_fk", - "tableFrom": "upload_parts", - "tableTo": "upload_sessions", - "columnsFrom": [ - "session_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "upload_sessions": { - "name": "upload_sessions", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "cache_id": { - "name": "cache_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path_hash": { - "name": "store_path_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path": { - "name": "store_path", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_hash": { - "name": "nar_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_size": { - "name": "nar_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_hash": { - "name": "file_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_size": { - "name": "file_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "compression": { - "name": "compression", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "references_json": { - "name": "references_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "deriver": { - "name": "deriver", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "system": { - "name": "system", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "r2_upload_key": { - "name": "r2_upload_key", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "r2_upload_id": { - "name": "r2_upload_id", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - }, - "expires_at": { - "name": "expires_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "idx_upload_sessions_status": { - "name": "idx_upload_sessions_status", - "columns": [ - "status", - "expires_at" - ], - "isUnique": false - } - }, - "foreignKeys": { - "upload_sessions_cache_id_caches_id_fk": { - "name": "upload_sessions_cache_id_caches_id_fk", - "tableFrom": "upload_sessions", - "tableTo": "caches", - "columnsFrom": [ - "cache_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - } - }, - "views": {}, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} - } -} \ No newline at end of file diff --git a/workers/cache/migrations/meta/0002_snapshot.json b/workers/cache/migrations/meta/0002_snapshot.json deleted file mode 100644 index 694a7ac..0000000 --- a/workers/cache/migrations/meta/0002_snapshot.json +++ /dev/null @@ -1,724 +0,0 @@ -{ - "version": "6", - "dialect": "sqlite", - "id": "f6ad1415-55bd-4c46-8984-34fa48eaea35", - "prevId": "7e8a49f8-b25b-4ee7-8029-ba438ac76987", - "tables": { - "api_tokens": { - "name": "api_tokens", - "columns": { - "jti": { - "name": "jti", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "subject": { - "name": "subject", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "caches_json": { - "name": "caches_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "perms_json": { - "name": "perms_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - }, - "expires_at": { - "name": "expires_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_by": { - "name": "created_by", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "revoked_at": { - "name": "revoked_at", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "revoked_by": { - "name": "revoked_by", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "revocation_reason": { - "name": "revocation_reason", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "audit_logs": { - "name": "audit_logs", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "timestamp": { - "name": "timestamp", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - }, - "actor": { - "name": "actor", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "action": { - "name": "action", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "cache_name": { - "name": "cache_name", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "detail": { - "name": "detail", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'{}'" - }, - "ip": { - "name": "ip", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "idx_audit_logs_actor": { - "name": "idx_audit_logs_actor", - "columns": [ - "actor" - ], - "isUnique": false - }, - "idx_audit_logs_cache": { - "name": "idx_audit_logs_cache", - "columns": [ - "cache_name" - ], - "isUnique": false - }, - "idx_audit_logs_time": { - "name": "idx_audit_logs_time", - "columns": [ - "timestamp" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "blob_objects": { - "name": "blob_objects", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "file_hash": { - "name": "file_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_size": { - "name": "file_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "compression": { - "name": "compression", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "r2_key": { - "name": "r2_key", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "blob_objects_r2_key_unique": { - "name": "blob_objects_r2_key_unique", - "columns": [ - "r2_key" - ], - "isUnique": true - }, - "blob_objects_file_hash_compression_unique": { - "name": "blob_objects_file_hash_compression_unique", - "columns": [ - "file_hash", - "compression" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "caches": { - "name": "caches", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "is_public": { - "name": "is_public", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "caches_name_unique": { - "name": "caches_name_unique", - "columns": [ - "name" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "gc_marks": { - "name": "gc_marks", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "target_type": { - "name": "target_type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "target_id": { - "name": "target_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "reason": { - "name": "reason", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "marked_at": { - "name": "marked_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "gc_marks_target_unique": { - "name": "gc_marks_target_unique", - "columns": [ - "target_type", - "target_id" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "published_paths": { - "name": "published_paths", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "cache_id": { - "name": "cache_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path_hash": { - "name": "store_path_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path": { - "name": "store_path", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_hash": { - "name": "nar_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_size": { - "name": "nar_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "blob_object_id": { - "name": "blob_object_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "references_json": { - "name": "references_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "deriver": { - "name": "deriver", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "system": { - "name": "system", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "signatures_json": { - "name": "signatures_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - } - }, - "indexes": { - "published_paths_cache_store_unique": { - "name": "published_paths_cache_store_unique", - "columns": [ - "cache_id", - "store_path_hash" - ], - "isUnique": true - }, - "idx_published_paths_blob_object": { - "name": "idx_published_paths_blob_object", - "columns": [ - "blob_object_id" - ], - "isUnique": false - } - }, - "foreignKeys": { - "published_paths_cache_id_caches_id_fk": { - "name": "published_paths_cache_id_caches_id_fk", - "tableFrom": "published_paths", - "tableTo": "caches", - "columnsFrom": [ - "cache_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - }, - "published_paths_blob_object_id_blob_objects_id_fk": { - "name": "published_paths_blob_object_id_blob_objects_id_fk", - "tableFrom": "published_paths", - "tableTo": "blob_objects", - "columnsFrom": [ - "blob_object_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "upload_parts": { - "name": "upload_parts", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "session_id": { - "name": "session_id", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "part_number": { - "name": "part_number", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "etag": { - "name": "etag", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "size": { - "name": "size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "upload_parts_session_part_unique": { - "name": "upload_parts_session_part_unique", - "columns": [ - "session_id", - "part_number" - ], - "isUnique": true - } - }, - "foreignKeys": { - "upload_parts_session_id_upload_sessions_id_fk": { - "name": "upload_parts_session_id_upload_sessions_id_fk", - "tableFrom": "upload_parts", - "tableTo": "upload_sessions", - "columnsFrom": [ - "session_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "upload_sessions": { - "name": "upload_sessions", - "columns": { - "id": { - "name": "id", - "type": "text", - "primaryKey": true, - "notNull": true, - "autoincrement": false - }, - "cache_id": { - "name": "cache_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path_hash": { - "name": "store_path_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "store_path": { - "name": "store_path", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_hash": { - "name": "nar_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "nar_size": { - "name": "nar_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_hash": { - "name": "file_hash", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "file_size": { - "name": "file_size", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "compression": { - "name": "compression", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "references_json": { - "name": "references_json", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'[]'" - }, - "deriver": { - "name": "deriver", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "system": { - "name": "system", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "r2_upload_key": { - "name": "r2_upload_key", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "r2_upload_id": { - "name": "r2_upload_id", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))" - }, - "expires_at": { - "name": "expires_at", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "idx_upload_sessions_status": { - "name": "idx_upload_sessions_status", - "columns": [ - "status", - "expires_at" - ], - "isUnique": false - } - }, - "foreignKeys": { - "upload_sessions_cache_id_caches_id_fk": { - "name": "upload_sessions_cache_id_caches_id_fk", - "tableFrom": "upload_sessions", - "tableTo": "caches", - "columnsFrom": [ - "cache_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - } - }, - "views": {}, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} - } -} \ No newline at end of file diff --git a/workers/cache/migrations/meta/_journal.json b/workers/cache/migrations/meta/_journal.json deleted file mode 100644 index 2e084e2..0000000 --- a/workers/cache/migrations/meta/_journal.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "version": "7", - "dialect": "sqlite", - "entries": [ - { - "idx": 0, - "version": "6", - "when": 1776949256837, - "tag": "0000_cultured_night_nurse", - "breakpoints": true - }, - { - "idx": 1, - "version": "6", - "when": 1776966587890, - "tag": "0001_overjoyed_exiles", - "breakpoints": true - }, - { - "idx": 2, - "version": "6", - "when": 1777117423084, - "tag": "0002_ordinary_wiccan", - "breakpoints": true - } - ] -} \ No newline at end of file diff --git a/workers/cache/package.json b/workers/cache/package.json deleted file mode 100644 index a744c3e..0000000 --- a/workers/cache/package.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "@helios/cache-worker", - "version": "0.0.0", - "private": true, - "scripts": { - "deploy": "wrangler deploy", - "dev": "wrangler dev", - "test": "vitest run", - "cf-typegen": "wrangler types", - "check": "tsc --noEmit && vitest run" - }, - "devDependencies": { - "@cloudflare/vitest-pool-workers": "^0.18.6", - "@types/node": "^26.1.1", - "drizzle-kit": "^0.31.10", - "typescript": "^7.0.2", - "vitest": "~4.1.10", - "wrangler": "^4.112.0" - }, - "dependencies": { - "@helios/cache-domain": "workspace:*", - "aws4fetch": "^1.0.20", - "drizzle-orm": "^0.45.2" - } -} diff --git a/workers/cache/src/admin.ts b/workers/cache/src/admin.ts deleted file mode 100644 index 60c6609..0000000 --- a/workers/cache/src/admin.ts +++ /dev/null @@ -1,162 +0,0 @@ -import type { WorkerConfig } from "./config.js"; -import { signJwt, isTokenPermission, MAX_CACHES_PER_TOKEN, MAX_PERMS_PER_TOKEN, TOKEN_ISSUER, TOKEN_AUDIENCE } from "./jwt.js"; -import type { TokenClaims } from "./jwt.js"; -import type { TokenPermission } from "./db/types.js"; -import { - createApiToken, - listApiTokens, - revokeApiToken, -} from "./db/repository.js"; -import { jsonResponse, errorResponse, parseJsonBody } from "./responses.js"; - -const MAX_TOKEN_LIFETIME_DAYS = 365; -const DEFAULT_TOKEN_LIFETIME_DAYS = 90; -const MAX_SUBJECT_LENGTH = 256; -const MAX_REASON_LENGTH = 512; - -// ── Create Token ── - -export async function handleCreateToken( - request: Request, - config: WorkerConfig, -): Promise { - if (!config.jwtSecret) { - return errorResponse("Service unavailable", 503); - } - - const body = await parseJsonBody<{ - subject: string; - caches: string[]; - perms: string[]; - expiresInDays?: number; - }>(request); - if (body instanceof Response) return body; - - if (!body.subject || typeof body.subject !== "string") { - return errorResponse("subject must be a non-empty string", 400); - } - - if (body.subject.length > MAX_SUBJECT_LENGTH) { - return errorResponse("subject exceeds maximum length", 400); - } - - if (!Array.isArray(body.caches) || body.caches.length === 0) { - return errorResponse("caches must be a non-empty array of cache names", 400); - } - if (body.caches.length > MAX_CACHES_PER_TOKEN) { - return errorResponse(`caches exceeds maximum of ${MAX_CACHES_PER_TOKEN}`, 400); - } - for (const c of body.caches) { - if (typeof c !== "string" || c.length === 0) { - return errorResponse("each cache entry must be a non-empty string", 400); - } - } - - if (!Array.isArray(body.perms) || body.perms.length === 0) { - return errorResponse("perms must be a non-empty array", 400); - } - if (body.perms.length > MAX_PERMS_PER_TOKEN) { - return errorResponse(`perms exceeds maximum of ${MAX_PERMS_PER_TOKEN}`, 400); - } - const validatedPerms: TokenPermission[] = []; - for (const p of body.perms) { - if (typeof p !== "string" || !isTokenPermission(p)) { - return errorResponse(`invalid permission: ${String(p)}, expected "pull" or "push"`, 400); - } - validatedPerms.push(p); - } - - const expiresInDays = body.expiresInDays ?? DEFAULT_TOKEN_LIFETIME_DAYS; - if (!Number.isFinite(expiresInDays) || !Number.isInteger(expiresInDays) || expiresInDays < 1 || expiresInDays > MAX_TOKEN_LIFETIME_DAYS) { - return errorResponse(`expiresInDays must be between 1 and ${MAX_TOKEN_LIFETIME_DAYS}`, 400); - } - - const dedupedCaches = [...new Set(body.caches)]; - const dedupedPerms = [...new Set(validatedPerms)]; - - const now = Date.now() / 1000; - const jti = crypto.randomUUID(); - const exp = now + expiresInDays * 86400; - - const claims: TokenClaims = { - jti, - sub: body.subject, - iss: TOKEN_ISSUER, - aud: TOKEN_AUDIENCE, - caches: dedupedCaches, - perms: dedupedPerms, - iat: Math.floor(now), - exp: Math.floor(exp), - }; - - const token = await signJwt(claims, config.jwtSecret); - - const expiresAt = new Date(exp * 1000).toISOString(); - - await createApiToken(config.db, { - jti, - subject: body.subject, - cachesJson: JSON.stringify(dedupedCaches), - permsJson: JSON.stringify(dedupedPerms), - expiresAt, - createdBy: "admin", - }); - - return jsonResponse({ - token, - jti, - subject: body.subject, - caches: dedupedCaches, - perms: dedupedPerms, - expiresAt, - }, 201); -} - -// ── List Tokens ── - -export async function handleListTokens( - config: WorkerConfig, -): Promise { - const tokens = await listApiTokens(config.db); - const result = tokens.map((t) => ({ - jti: t.jti, - subject: t.subject, - caches: JSON.parse(t.cachesJson), - perms: JSON.parse(t.permsJson), - createdAt: t.createdAt, - expiresAt: t.expiresAt, - createdBy: t.createdBy, - revokedAt: t.revokedAt, - revokedBy: t.revokedBy, - revocationReason: t.revocationReason, - })); - return jsonResponse({ tokens: result }); -} - -// ── Revoke Token ── - -export async function handleRevokeToken( - request: Request, - config: WorkerConfig, - jti: string, -): Promise { - const body = await parseJsonBody<{ - reason: string; - }>(request); - if (body instanceof Response) return body; - - if (!body.reason || typeof body.reason !== "string") { - return errorResponse("reason must be a non-empty string", 400); - } - - if (body.reason.length > MAX_REASON_LENGTH) { - return errorResponse("reason exceeds maximum length", 400); - } - - const revoked = await revokeApiToken(config.db, jti, "admin", body.reason); - if (!revoked) { - return errorResponse("Token not found or already revoked", 404); - } - - return jsonResponse({ revoked: true, jti }); -} diff --git a/workers/cache/src/auth.ts b/workers/cache/src/auth.ts deleted file mode 100644 index d273eaf..0000000 --- a/workers/cache/src/auth.ts +++ /dev/null @@ -1,166 +0,0 @@ -import type { WorkerConfig } from "./config.js"; -import type { TokenClaims } from "./jwt.js"; -import { verifyJwt } from "./jwt.js"; -import { isTokenActive } from "./db/repository.js"; -import type { DrizzleD1Database } from "drizzle-orm/d1"; - -// Module-scope cache of recently confirmed-active token IDs (jti → cache -// entry deadline in ms). Workers isolates are reused across requests on the -// same edge node, so this avoids a D1 revocation-check round trip on every -// write request. Tradeoff: a freshly revoked token keeps working for up to -// TOKEN_ACTIVE_TTL_MS on warm isolates. Only positive results are cached; -// revocation is permanent, so negative results gain nothing from caching. -const TOKEN_ACTIVE_TTL_MS = 30_000; -const TOKEN_ACTIVE_CACHE_MAX_ENTRIES = 10_000; -const tokenActiveUntil = new Map(); - -async function isTokenActiveCached(db: DrizzleD1Database, jti: string): Promise { - const now = Date.now(); - const deadline = tokenActiveUntil.get(jti); - if (deadline !== undefined && deadline > now) { - return true; - } - - const active = await isTokenActive(db, jti); - if (active) { - // Bound memory: reset rather than evict; the cache refills on demand. - if (tokenActiveUntil.size >= TOKEN_ACTIVE_CACHE_MAX_ENTRIES) { - tokenActiveUntil.clear(); - } - tokenActiveUntil.set(jti, now + TOKEN_ACTIVE_TTL_MS); - } else { - tokenActiveUntil.delete(jti); - } - return active; -} - -/** Build a JSON error response. Always sets cache-control: no-store so auth failures don't get cached. */ -function authError(message: string, status: number): Response { - const headers: Record = { - "content-type": "application/json; charset=utf-8", - "cache-control": "no-store", - }; - if (status === 401) { - headers["www-authenticate"] = "Bearer"; - } - return new Response(JSON.stringify({ error: message }), { status, headers }); -} - -/** Extract bearer token from Authorization header, or return an error response. */ -function extractBearer(request: Request): string | Response { - const authHeader = request.headers.get("authorization"); - if (!authHeader) { - return authError("Unauthorized", 401); - } - const spaceIdx = authHeader.indexOf(" "); - if (spaceIdx === -1) { - return authError("Unauthorized", 401); - } - const scheme = authHeader.slice(0, spaceIdx); - const token = authHeader.slice(spaceIdx + 1); - if (scheme.toLowerCase() !== "bearer" || token.length === 0) { - return authError("Unauthorized", 401); - } - return token; -} - -const tsEncoder = new TextEncoder(); - -/** - * Timing-safe string comparison. Hashes both inputs with SHA-256 then - * compares digests byte-by-byte with constant-time XOR accumulation. - * Prevents timing side-channel attacks on secret comparison. - */ -async function timingSafeEqual(a: string, b: string): Promise { - const [da, db] = await Promise.all([ - crypto.subtle.digest("SHA-256", tsEncoder.encode(a)), - crypto.subtle.digest("SHA-256", tsEncoder.encode(b)), - ]); - const ba = new Uint8Array(da); - const bb = new Uint8Array(db); - let diff = a.length ^ b.length; - for (let i = 0; i < ba.length; i++) diff |= ba[i] ^ bb[i]; - return diff === 0; -} - -/** Represents the authenticated caller for write operations. */ -export type AuthIdentity = - | { readonly kind: "jwt"; readonly claims: TokenClaims } - | { readonly kind: "legacy" } - | { readonly kind: "admin" }; - -export type AuthResult = - | { readonly kind: "ok"; readonly identity: AuthIdentity } - | { readonly kind: "error"; readonly response: Response }; - -/** - * Authenticate a write request. Checks in order: - * 1. If JWT_SECRET is configured, try JWT verification + active check - * 2. If AUTH_TOKEN is configured, try static bearer token match - * 3. If neither is configured, fail closed (403) - */ -export async function authenticateWrite( - request: Request, - config: WorkerConfig, -): Promise { - // Require at least one auth method to be configured, otherwise reject. - if (!config.jwtSecret && !config.authToken) { - return { kind: "error", response: authError("Forbidden", 403) }; - } - - const bearer = extractBearer(request); - if (typeof bearer !== "string") { - return { kind: "error", response: bearer }; - } - - // Prefer JWT when a secret is set; fall back to the legacy token if JWT verification fails. - if (config.jwtSecret) { - const result = await verifyJwt(bearer, config.jwtSecret); - if (result.kind === "ok") { - const active = await isTokenActiveCached(config.db, result.claims.jti); - if (!active) { - return { kind: "error", response: authError("Unauthorized", 401) }; - } - return { kind: "ok", identity: { kind: "jwt", claims: result.claims } }; - } - if (!config.authToken) { - return { kind: "error", response: authError("Unauthorized", 401) }; - } - } - - if (config.authToken && await timingSafeEqual(bearer, config.authToken)) { - return { kind: "ok", identity: { kind: "legacy" } }; - } - - return { kind: "error", response: authError("Forbidden", 403) }; -} - -/** Authenticate an admin request using ADMIN_SECRET. */ -export async function authenticateAdmin( - request: Request, - config: WorkerConfig, -): Promise { - if (!config.adminSecret) { - return { kind: "error", response: authError("Forbidden", 403) }; - } - - const bearer = extractBearer(request); - if (typeof bearer !== "string") { - return { kind: "error", response: bearer }; - } - - if (!(await timingSafeEqual(bearer, config.adminSecret))) { - return { kind: "error", response: authError("Forbidden", 403) }; - } - - return { kind: "ok", identity: { kind: "admin" } }; -} - -/** Extract the actor identifier for audit logging. */ -export function actorId(identity: AuthIdentity): string { - switch (identity.kind) { - case "jwt": return identity.claims.jti; - case "legacy": return "legacy"; - case "admin": return "admin"; - } -} diff --git a/workers/cache/src/config.ts b/workers/cache/src/config.ts deleted file mode 100644 index 74458d6..0000000 --- a/workers/cache/src/config.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { drizzle } from "drizzle-orm/d1"; -import type { DrizzleD1Database } from "drizzle-orm/d1"; - -export interface WorkerConfig { - readonly bucket: R2Bucket; - readonly db: DrizzleD1Database; - readonly signingKeyName: string; - readonly signingPrivateKey: string; - readonly authToken: string; - readonly r2AccessKeyId: string; - readonly r2SecretAccessKey: string; - readonly r2Endpoint: string; - readonly r2BucketName: string; - readonly jwtSecret: string; - readonly adminSecret: string; - readonly ctx?: ExecutionContext; -} - -export function resolveConfig(env: Env): WorkerConfig { - if (!env.CACHE_BUCKET) { - throw new Error("Missing required binding: CACHE_BUCKET"); - } - if (!env.CACHE_DB) { - throw new Error("Missing required binding: CACHE_DB"); - } - return { - bucket: env.CACHE_BUCKET, - db: drizzle(env.CACHE_DB), - signingKeyName: env.SIGNING_KEY_NAME || "helios-cache-1", - signingPrivateKey: env.SIGNING_PRIVATE_KEY || "", - authToken: env.AUTH_TOKEN || "", - r2AccessKeyId: env.R2_ACCESS_KEY_ID || "", - r2SecretAccessKey: env.R2_SECRET_ACCESS_KEY || "", - r2Endpoint: env.R2_ENDPOINT || "", - r2BucketName: env.R2_BUCKET_NAME || "helios-cache", - jwtSecret: env.JWT_SECRET || "", - adminSecret: env.ADMIN_SECRET || "", - }; -} diff --git a/workers/cache/src/db/index.ts b/workers/cache/src/db/index.ts deleted file mode 100644 index 778281e..0000000 --- a/workers/cache/src/db/index.ts +++ /dev/null @@ -1,58 +0,0 @@ -export { - caches, - blobObjects, - publishedPaths, - uploadSessions, - uploadParts, - gcMarks, - apiTokens, - auditLogs, -} from './schema.js'; - -export type { - Cache, - BlobObject, - PublishedPath, - UploadSession, - UploadPart, - GcMark, - ApiToken, - AuditLog, - TokenPermission, - UploadSessionStatus, -} from './types.js'; - -export { - findCacheByName, - findCacheById, - createCache, - findBlobObject, - findBlobObjectById, - createBlobObject, - findPublishedPath, - findPublishedPathWithBlob, - findPublishedHashes, - createPublishedPath, - findUploadSession, - createUploadSession, - transitionSessionStatus, - transitionToMultipart, - findExpiredSessions, - upsertUploadPart, - upsertUploadParts, - findUploadParts, - createGcMark, - findGcMarks, - deleteGcMark, - findUnreferencedBlobObjects, - deleteBlobObject, - deleteUploadSession, - createApiToken, - findApiToken, - listApiTokens, - revokeApiToken, - isTokenActive, - deleteExpiredApiTokens, - createAuditLog, - deleteExpiredAuditLogs, -} from './repository.js'; diff --git a/workers/cache/src/db/repository.ts b/workers/cache/src/db/repository.ts deleted file mode 100644 index 346bb94..0000000 --- a/workers/cache/src/db/repository.ts +++ /dev/null @@ -1,507 +0,0 @@ -import { eq, and, notInArray, lt, inArray, isNull } from "drizzle-orm"; -import type { DrizzleD1Database } from "drizzle-orm/d1"; -import { - caches, - blobObjects, - publishedPaths, - uploadSessions, - uploadParts, - gcMarks, - apiTokens, - auditLogs, -} from "./schema.js"; -import type { - Cache, - BlobObject, - PublishedPath, - UploadSession, - UploadPart, - GcMark, - ApiToken, - UploadSessionStatus, -} from "./types.js"; - -// ── Caches ── - -export async function findCacheByName( - db: DrizzleD1Database, - name: string, -): Promise { - return db.select().from(caches).where(eq(caches.name, name)).get(); -} - -export async function findCacheById( - db: DrizzleD1Database, - id: number, -): Promise { - return db.select().from(caches).where(eq(caches.id, id)).get(); -} - -export async function createCache( - db: DrizzleD1Database, - name: string, -): Promise { - return db - .insert(caches) - .values({ name }) - .returning() - .get(); -} - -// ── Blob Objects ── - -export async function findBlobObject( - db: DrizzleD1Database, - fileHash: string, - compression: string, -): Promise { - return db - .select() - .from(blobObjects) - .where(and(eq(blobObjects.fileHash, fileHash), eq(blobObjects.compression, compression))) - .get(); -} - -export async function findBlobObjectById( - db: DrizzleD1Database, - id: number, -): Promise { - return db.select().from(blobObjects).where(eq(blobObjects.id, id)).get(); -} - -export async function createBlobObject( - db: DrizzleD1Database, - params: { - readonly fileHash: string; - readonly fileSize: number; - readonly compression: string; - readonly r2Key: string; - }, -): Promise { - return db - .insert(blobObjects) - .values(params) - .returning() - .get(); -} - -// ── Published Paths ── - -export async function findPublishedPathWithBlob( - db: DrizzleD1Database, - cacheId: number, - storePathHash: string, -): Promise<{ path: PublishedPath; blob: BlobObject } | undefined> { - const row = await db - .select({ - path: publishedPaths, - blob: blobObjects, - }) - .from(publishedPaths) - .innerJoin(blobObjects, eq(publishedPaths.blobObjectId, blobObjects.id)) - .where( - and( - eq(publishedPaths.cacheId, cacheId), - eq(publishedPaths.storePathHash, storePathHash), - ), - ) - .get(); - return row; -} - -export async function findPublishedPath( - db: DrizzleD1Database, - cacheId: number, - storePathHash: string, -): Promise { - return db - .select() - .from(publishedPaths) - .where( - and( - eq(publishedPaths.cacheId, cacheId), - eq(publishedPaths.storePathHash, storePathHash), - ), - ) - .get(); -} - -export async function createPublishedPath( - db: DrizzleD1Database, - params: { - readonly cacheId: number; - readonly storePathHash: string; - readonly storePath: string; - readonly narHash: string; - readonly narSize: number; - readonly blobObjectId: number; - readonly referencesJson: string; - readonly deriver: string | null; - readonly system: string | null; - readonly signaturesJson: string; - }, -): Promise { - return db - .insert(publishedPaths) - .values(params) - .returning() - .get(); -} - -export async function findPublishedHashes( - db: DrizzleD1Database, - cacheId: number, - storePathHashes: string[], -): Promise { - if (storePathHashes.length === 0) return []; - - // D1 limits bound parameters to 100 per query; reserve 1 for cacheId - const chunks: string[][] = []; - for (let i = 0; i < storePathHashes.length; i += 99) { - chunks.push(storePathHashes.slice(i, i + 99)); - } - - // Send all chunk queries in a single D1 batch round trip instead of one - // sequential round trip per chunk. - const queries = chunks.map((chunk) => - db - .select({ storePathHash: publishedPaths.storePathHash }) - .from(publishedPaths) - .where( - and( - eq(publishedPaths.cacheId, cacheId), - inArray(publishedPaths.storePathHash, chunk), - ), - ), - ); - - const [firstQuery, ...restQueries] = queries; - if (firstQuery === undefined) return []; - const chunkResults = await db.batch([firstQuery, ...restQueries]); - - const results: string[] = []; - for (const rows of chunkResults) { - for (const r of rows) { - results.push(r.storePathHash); - } - } - - return results; -} - -// ── Upload Sessions ── - -export async function findUploadSession( - db: DrizzleD1Database, - sessionId: string, -): Promise { - return db - .select() - .from(uploadSessions) - .where(eq(uploadSessions.id, sessionId)) - .get(); -} - -export async function createUploadSession( - db: DrizzleD1Database, - params: { - readonly id: string; - readonly cacheId: number; - readonly storePathHash: string; - readonly storePath: string; - readonly narHash: string; - readonly narSize: number; - readonly fileHash: string; - readonly fileSize: number; - readonly compression: string; - readonly referencesJson: string; - readonly deriver: string | null; - readonly system: string | null; - readonly r2UploadKey: string | null; - readonly r2UploadId: string | null; - readonly expiresAt: string; - }, -): Promise { - return db - .insert(uploadSessions) - .values(params) - .returning() - .get(); -} - -export async function transitionSessionStatus( - db: DrizzleD1Database, - sessionId: string, - fromStatus: UploadSessionStatus, - toStatus: UploadSessionStatus, -): Promise { - return db - .update(uploadSessions) - .set({ status: toStatus }) - .where(and(eq(uploadSessions.id, sessionId), eq(uploadSessions.status, fromStatus))) - .returning() - .get(); -} - -export async function transitionToMultipart( - db: DrizzleD1Database, - sessionId: string, - r2UploadId: string, -): Promise { - return db - .update(uploadSessions) - .set({ status: "uploading", r2UploadId }) - .where(and(eq(uploadSessions.id, sessionId), eq(uploadSessions.status, "pending"))) - .returning() - .get(); -} - -export async function findExpiredSessions( - db: DrizzleD1Database, - now: string, -): Promise { - return db - .select() - .from(uploadSessions) - .where( - and( - notInArray(uploadSessions.status, ["completed", "expired"]), - lt(uploadSessions.expiresAt, now), - ), - ) - .all(); -} - -// ── Upload Parts ── - -export async function upsertUploadPart( - db: DrizzleD1Database, - params: { - readonly sessionId: string; - readonly partNumber: number; - readonly etag: string; - readonly size: number; - }, -): Promise { - return db - .insert(uploadParts) - .values(params) - .onConflictDoUpdate({ - target: [uploadParts.sessionId, uploadParts.partNumber], - set: { etag: params.etag, size: params.size }, - }) - .returning() - .get(); -} - -export async function upsertUploadParts( - db: DrizzleD1Database, - sessionId: string, - parts: ReadonlyArray<{ - readonly partNumber: number; - readonly etag: string; - readonly size: number; - }>, -): Promise { - // One D1 batch round trip for all parts instead of one round trip each. - const statements = parts.map((part) => - db - .insert(uploadParts) - .values({ sessionId, partNumber: part.partNumber, etag: part.etag, size: part.size }) - .onConflictDoUpdate({ - target: [uploadParts.sessionId, uploadParts.partNumber], - set: { etag: part.etag, size: part.size }, - }), - ); - - const [firstStatement, ...restStatements] = statements; - if (firstStatement === undefined) return; - await db.batch([firstStatement, ...restStatements]); -} - -export async function findUploadParts( - db: DrizzleD1Database, - sessionId: string, -): Promise { - return db - .select() - .from(uploadParts) - .where(eq(uploadParts.sessionId, sessionId)) - .orderBy(uploadParts.partNumber) - .all(); -} - -// ── Blob Object Cleanup ── - -export async function findUnreferencedBlobObjects( - db: DrizzleD1Database, -): Promise { - const referenced = db - .select({ id: publishedPaths.blobObjectId }) - .from(publishedPaths); - - return db - .select() - .from(blobObjects) - .where(notInArray(blobObjects.id, referenced)) - .all(); -} - -export async function deleteBlobObject( - db: DrizzleD1Database, - id: number, -): Promise { - await db.delete(blobObjects).where(eq(blobObjects.id, id)).run(); -} - -// ── Upload Session Cleanup ── - -export async function deleteUploadSession( - db: DrizzleD1Database, - sessionId: string, -): Promise { - await db.delete(uploadParts).where(eq(uploadParts.sessionId, sessionId)).run(); - await db.delete(uploadSessions).where(eq(uploadSessions.id, sessionId)).run(); -} - -// ── GC Marks ── - -export async function createGcMark( - db: DrizzleD1Database, - params: { - readonly targetType: string; - readonly targetId: string; - readonly reason: string; - }, -): Promise { - return db - .insert(gcMarks) - .values(params) - .returning() - .get(); -} - -export async function findGcMarks( - db: DrizzleD1Database, - targetType: string, -): Promise { - return db - .select() - .from(gcMarks) - .where(eq(gcMarks.targetType, targetType)) - .all(); -} - -export async function deleteGcMark( - db: DrizzleD1Database, - targetType: string, - targetId: string, -): Promise { - await db - .delete(gcMarks) - .where(and(eq(gcMarks.targetType, targetType), eq(gcMarks.targetId, targetId))) - .run(); -} - -// ── API Tokens ── - -export async function createApiToken( - db: DrizzleD1Database, - params: { - readonly jti: string; - readonly subject: string; - readonly cachesJson: string; - readonly permsJson: string; - readonly expiresAt: string; - readonly createdBy: string; - }, -): Promise { - return db - .insert(apiTokens) - .values(params) - .returning() - .get(); -} - -export async function findApiToken( - db: DrizzleD1Database, - jti: string, -): Promise { - return db.select().from(apiTokens).where(eq(apiTokens.jti, jti)).get(); -} - -export async function listApiTokens( - db: DrizzleD1Database, -): Promise { - return db.select().from(apiTokens).all(); -} - -export async function revokeApiToken( - db: DrizzleD1Database, - jti: string, - revokedBy: string, - reason: string, -): Promise { - return db - .update(apiTokens) - .set({ - revokedAt: new Date().toISOString(), - revokedBy, - revocationReason: reason, - }) - .where(and(eq(apiTokens.jti, jti), isNull(apiTokens.revokedAt))) - .returning() - .get(); -} - -export async function isTokenActive( - db: DrizzleD1Database, - jti: string, -): Promise { - const row = await db - .select({ jti: apiTokens.jti }) - .from(apiTokens) - .where(and(eq(apiTokens.jti, jti), isNull(apiTokens.revokedAt))) - .get(); - return row !== undefined; -} - -export async function deleteExpiredApiTokens( - db: DrizzleD1Database, - now: string, -): Promise { - const result = await db - .delete(apiTokens) - .where(lt(apiTokens.expiresAt, now)) - .run(); - return result.meta.changes ?? 0; -} - -// ── Audit Logs ── - -export async function createAuditLog( - db: DrizzleD1Database, - params: { - readonly actor: string; - readonly action: string; - readonly cacheName: string | null; - readonly detail: string; - readonly ip: string | null; - readonly status: number; - }, -): Promise { - // No RETURNING: audit inserts are fire-and-forget on the hot write path, - // and skipping the row round trip keeps them as cheap as possible. - await db.insert(auditLogs).values(params).run(); -} - -export async function deleteExpiredAuditLogs( - db: DrizzleD1Database, - cutoff: string, -): Promise { - const result = await db - .delete(auditLogs) - .where(lt(auditLogs.timestamp, cutoff)) - .run(); - return result.meta.changes ?? 0; -} diff --git a/workers/cache/src/db/schema.ts b/workers/cache/src/db/schema.ts deleted file mode 100644 index 6f32dbd..0000000 --- a/workers/cache/src/db/schema.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { sqliteTable, text, integer, uniqueIndex, index } from "drizzle-orm/sqlite-core"; -import { sql } from "drizzle-orm"; - -const timestamp = (name: string) => - text(name).notNull().default(sql`(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))`); - -export const caches = sqliteTable("caches", { - id: integer("id").primaryKey({ autoIncrement: true }), - name: text("name").notNull().unique(), - isPublic: integer("is_public").notNull().default(1), - createdAt: timestamp("created_at"), -}); - -export const blobObjects = sqliteTable("blob_objects", { - id: integer("id").primaryKey({ autoIncrement: true }), - fileHash: text("file_hash").notNull(), - fileSize: integer("file_size").notNull(), - compression: text("compression").notNull(), - r2Key: text("r2_key").notNull().unique(), - createdAt: timestamp("created_at"), -}, (table) => [ - uniqueIndex("blob_objects_file_hash_compression_unique").on(table.fileHash, table.compression), -]); - -export const publishedPaths = sqliteTable("published_paths", { - id: integer("id").primaryKey({ autoIncrement: true }), - cacheId: integer("cache_id").notNull().references(() => caches.id), - storePathHash: text("store_path_hash").notNull(), - storePath: text("store_path").notNull(), - narHash: text("nar_hash").notNull(), - narSize: integer("nar_size").notNull(), - blobObjectId: integer("blob_object_id").notNull().references(() => blobObjects.id), - referencesJson: text("references_json").notNull().default("[]"), - deriver: text("deriver"), - system: text("system"), - signaturesJson: text("signatures_json").notNull().default("[]"), - createdAt: timestamp("created_at"), -}, (table) => [ - uniqueIndex("published_paths_cache_store_unique").on(table.cacheId, table.storePathHash), - index("idx_published_paths_blob_object").on(table.blobObjectId), -]); - -export const uploadSessions = sqliteTable("upload_sessions", { - id: text("id").primaryKey(), - cacheId: integer("cache_id").notNull().references(() => caches.id), - storePathHash: text("store_path_hash").notNull(), - storePath: text("store_path").notNull(), - narHash: text("nar_hash").notNull(), - narSize: integer("nar_size").notNull(), - fileHash: text("file_hash").notNull(), - fileSize: integer("file_size").notNull(), - compression: text("compression").notNull(), - referencesJson: text("references_json").notNull().default("[]"), - deriver: text("deriver"), - system: text("system"), - status: text("status").notNull().default("pending"), - r2UploadKey: text("r2_upload_key"), - r2UploadId: text("r2_upload_id"), - createdAt: timestamp("created_at"), - expiresAt: text("expires_at").notNull(), -}, (table) => [ - index("idx_upload_sessions_status").on(table.status, table.expiresAt), -]); - -export const uploadParts = sqliteTable("upload_parts", { - id: integer("id").primaryKey({ autoIncrement: true }), - sessionId: text("session_id").notNull().references(() => uploadSessions.id), - partNumber: integer("part_number").notNull(), - etag: text("etag").notNull(), - size: integer("size").notNull(), -}, (table) => [ - uniqueIndex("upload_parts_session_part_unique").on(table.sessionId, table.partNumber), -]); - -export const gcMarks = sqliteTable("gc_marks", { - id: integer("id").primaryKey({ autoIncrement: true }), - targetType: text("target_type").notNull(), - targetId: text("target_id").notNull(), - reason: text("reason").notNull(), - markedAt: text("marked_at").notNull().default(sql`(strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))`), -}, (table) => [ - uniqueIndex("gc_marks_target_unique").on(table.targetType, table.targetId), -]); - -export const apiTokens = sqliteTable("api_tokens", { - jti: text("jti").primaryKey(), - subject: text("subject").notNull(), - cachesJson: text("caches_json").notNull(), - permsJson: text("perms_json").notNull(), - createdAt: timestamp("created_at"), - expiresAt: text("expires_at").notNull(), - createdBy: text("created_by").notNull(), - revokedAt: text("revoked_at"), - revokedBy: text("revoked_by"), - revocationReason: text("revocation_reason"), -}); - -export const auditLogs = sqliteTable("audit_logs", { - id: integer("id").primaryKey({ autoIncrement: true }), - timestamp: timestamp("timestamp"), - actor: text("actor").notNull(), - action: text("action").notNull(), - cacheName: text("cache_name"), - detail: text("detail").notNull().default("{}"), - ip: text("ip"), - status: integer("status").notNull(), -}, (table) => [ - index("idx_audit_logs_actor").on(table.actor), - index("idx_audit_logs_cache").on(table.cacheName), - index("idx_audit_logs_time").on(table.timestamp), -]); diff --git a/workers/cache/src/db/types.ts b/workers/cache/src/db/types.ts deleted file mode 100644 index 241ba2c..0000000 --- a/workers/cache/src/db/types.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { InferSelectModel } from "drizzle-orm"; -import type { - caches, - blobObjects, - publishedPaths, - uploadSessions, - uploadParts, - gcMarks, - apiTokens, - auditLogs, -} from "./schema.js"; - -export type Cache = InferSelectModel; - -export type BlobObject = InferSelectModel; - -export type PublishedPath = InferSelectModel; - -export type UploadSession = InferSelectModel; - -export type UploadPart = InferSelectModel; - -export type GcMark = InferSelectModel; - -export type ApiToken = InferSelectModel; - -export type AuditLog = InferSelectModel; - -export type TokenPermission = "pull" | "push"; - -export type UploadSessionStatus = "pending" | "uploading" | "completed" | "expired"; diff --git a/workers/cache/src/env.d.ts b/workers/cache/src/env.d.ts deleted file mode 100644 index 3cb78b7..0000000 --- a/workers/cache/src/env.d.ts +++ /dev/null @@ -1,13 +0,0 @@ -// Secrets set via `wrangler secret put` — not in wrangler.jsonc vars. -declare namespace Cloudflare { - interface Env { - AUTH_TOKEN?: string; - SIGNING_PRIVATE_KEY?: string; - R2_ACCESS_KEY_ID?: string; - R2_SECRET_ACCESS_KEY?: string; - R2_ENDPOINT?: string; - R2_BUCKET_NAME?: string; - JWT_SECRET?: string; - ADMIN_SECRET?: string; - } -} diff --git a/workers/cache/src/gc.ts b/workers/cache/src/gc.ts deleted file mode 100644 index ded69ce..0000000 --- a/workers/cache/src/gc.ts +++ /dev/null @@ -1,112 +0,0 @@ -import type { WorkerConfig } from "./config.js"; -import { - findExpiredSessions, - transitionSessionStatus, - deleteUploadSession, - findUnreferencedBlobObjects, - deleteBlobObject, - deleteExpiredApiTokens, - deleteExpiredAuditLogs, -} from "./db/repository.js"; -import type { UploadSessionStatus } from "./db/types.js"; - -const VALID_TRANSITION_STATUSES = new Set(["pending", "uploading"]); - -function isTransitionableStatus(s: string): s is UploadSessionStatus { - return VALID_TRANSITION_STATUSES.has(s); -} - -const AUDIT_LOG_RETENTION_DAYS = 30; - -export interface GcResult { - readonly expiredSessions: number; - readonly deletedBlobs: number; - readonly expiredTokens: number; - readonly expiredAuditLogs: number; - readonly errors: string[]; -} - -export async function runGarbageCollection(config: WorkerConfig): Promise { - const errors: string[] = []; - let expiredSessions = 0; - let deletedBlobs = 0; - let expiredTokens = 0; - let expiredAuditLogs = 0; - - // Phase 1: Expire abandoned upload sessions - const now = new Date().toISOString(); - const expired = await findExpiredSessions(config.db, now); - - for (const session of expired) { - try { - // findExpiredSessions already excludes "completed" and "expired", - // but validate to satisfy the type system without an assertion. - if (!isTransitionableStatus(session.status)) continue; - - const transitioned = await transitionSessionStatus( - config.db, - session.id, - session.status, - "expired", - ); - if (!transitioned) continue; - - if (session.r2UploadId && session.r2UploadKey) { - try { - const multipart = config.bucket.resumeMultipartUpload(session.r2UploadKey, session.r2UploadId); - await multipart.abort(); - } catch { - // Multipart may already be completed or aborted - } - } - - if (session.r2UploadKey) { - try { - await config.bucket.delete(session.r2UploadKey); - } catch { - // R2 object may not exist - } - } - - await deleteUploadSession(config.db, session.id); - expiredSessions++; - } catch (err) { - errors.push(`Failed to expire session ${session.id}: ${err instanceof Error ? err.message : String(err)}`); - } - } - - // Phase 2: Delete unreferenced blob objects - const unreferenced = await findUnreferencedBlobObjects(config.db); - - for (const blob of unreferenced) { - try { - // Delete DB record first to prevent re-referencing during R2 delete - await deleteBlobObject(config.db, blob.id); - try { - await config.bucket.delete(blob.r2Key); - } catch { - // R2 object may already be gone - } - deletedBlobs++; - } catch (err) { - errors.push(`Failed to delete blob ${blob.id}: ${err instanceof Error ? err.message : String(err)}`); - } - } - - // Phase 3: Delete expired API tokens - try { - expiredTokens = await deleteExpiredApiTokens(config.db, now); - } catch (err) { - errors.push(`Failed to delete expired tokens: ${err instanceof Error ? err.message : String(err)}`); - } - - // Phase 4: Delete audit logs older than retention period - try { - const cutoff = new Date(Date.now() - AUDIT_LOG_RETENTION_DAYS * 86400_000).toISOString(); - expiredAuditLogs = await deleteExpiredAuditLogs(config.db, cutoff); - } catch (err) { - errors.push(`Failed to delete expired audit logs: ${err instanceof Error ? err.message : String(err)}`); - } - - return { expiredSessions, deletedBlobs, expiredTokens, expiredAuditLogs, errors }; -} diff --git a/workers/cache/src/index.ts b/workers/cache/src/index.ts deleted file mode 100644 index daed64d..0000000 --- a/workers/cache/src/index.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { resolveConfig } from "./config.js"; -import { handleRequest } from "./router.js"; -import { runGarbageCollection } from "./gc.js"; -import { logRequest } from "./logging.js"; - -export default { - async fetch(request, env, ctx): Promise { - const start = Date.now(); - try { - const config = { ...resolveConfig(env), ctx }; - const url = new URL(request.url); - const response = await handleRequest(request, config); - if (response.status >= 400 || url.pathname.startsWith("/_api/")) { - logRequest(request, response, start); - } - return response; - } catch (err) { - console.error("Unhandled error", { error: err instanceof Error ? err.message : String(err) }); - const response = new Response(JSON.stringify({ error: "Internal server error" }), { - status: 500, - headers: { "content-type": "application/json" }, - }); - logRequest(request, response, start); - return response; - } - }, - - async scheduled(event, env, ctx): Promise { - const config = resolveConfig(env); - const result = await runGarbageCollection(config); - - if (result.errors.length > 0) { - console.error("GC completed with errors", { - expiredSessions: result.expiredSessions, - deletedBlobs: result.deletedBlobs, - expiredTokens: result.expiredTokens, - expiredAuditLogs: result.expiredAuditLogs, - errors: result.errors, - }); - } else { - console.log("GC completed", { - expiredSessions: result.expiredSessions, - deletedBlobs: result.deletedBlobs, - expiredTokens: result.expiredTokens, - expiredAuditLogs: result.expiredAuditLogs, - }); - } - }, -} satisfies ExportedHandler; diff --git a/workers/cache/src/jwt.ts b/workers/cache/src/jwt.ts deleted file mode 100644 index 3569ba5..0000000 --- a/workers/cache/src/jwt.ts +++ /dev/null @@ -1,248 +0,0 @@ -import type { TokenPermission } from "./db/types.js"; - -export type TokenClaims = { - readonly jti: string; - readonly sub: string; - readonly iss: string; - readonly aud: string; - readonly caches: readonly string[]; - readonly perms: readonly TokenPermission[]; - readonly exp: number; - readonly iat: number; -}; - -export type JwtVerifyResult = - | { readonly kind: "ok"; readonly claims: TokenClaims } - | { readonly kind: "error"; readonly reason: string }; - -function base64UrlEncode(data: Uint8Array): string { - let binary = ""; - for (let i = 0; i < data.length; i++) { - binary += String.fromCharCode(data[i]); - } - return btoa(binary).replaceAll("+", "-").replaceAll("/", "_").replaceAll("=", ""); -} - -function base64UrlDecode(str: string): Uint8Array { - const base64 = str.replaceAll("-", "+").replaceAll("_", "/"); - const paddingNeeded = (4 - (base64.length % 4)) % 4; - const padded = paddingNeeded > 0 ? base64 + "====".slice(0, paddingNeeded) : base64; - const binary = atob(padded); - const bytes = new Uint8Array(binary.length); - for (let i = 0; i < binary.length; i++) { - bytes[i] = binary.charCodeAt(i); - } - return bytes; -} - -const HMAC_ALGO = { name: "HMAC", hash: "SHA-256" } as const; -const encoder = new TextEncoder(); -const decoder = new TextDecoder(); - -// Cache the imported CryptoKey at module scope. Workers isolates are reused -// across requests on the same edge node, so this avoids a costly importKey -// call on every verify. -let cachedSecret = ""; -let cachedKey: CryptoKey | undefined; - -async function getKey(secret: string): Promise { - if (secret === cachedSecret && cachedKey !== undefined) { - return cachedKey; - } - const key = await crypto.subtle.importKey( - "raw", - encoder.encode(secret), - HMAC_ALGO, - false, - ["sign", "verify"], - ); - cachedSecret = secret; - cachedKey = key; - return key; -} - -const EXPECTED_HEADER_JSON = '{"alg":"HS256","typ":"JWT"}'; -const HEADER_B64 = base64UrlEncode(encoder.encode(EXPECTED_HEADER_JSON)); -export const TOKEN_ISSUER = "helios-cache"; -export const TOKEN_AUDIENCE = "helios-cache"; -const MAX_TOKEN_BYTES = 4096; -const MAX_CLOCK_SKEW_SECONDS = 60; -export const MAX_CACHES_PER_TOKEN = 50; -export const MAX_PERMS_PER_TOKEN = 10; - -export const VALID_PERMISSIONS: ReadonlySet = new Set(["pull", "push"]); - -export function isTokenPermission(value: string): value is TokenPermission { - return VALID_PERMISSIONS.has(value); -} - -function validateClaims(obj: Record): JwtVerifyResult { - const jti = obj["jti"]; - if (typeof jti !== "string" || jti.length === 0) { - return { kind: "error", reason: "invalid token" }; - } - - const sub = obj["sub"]; - if (typeof sub !== "string" || sub.length === 0) { - return { kind: "error", reason: "invalid token" }; - } - - const exp = obj["exp"]; - if (typeof exp !== "number") { - return { kind: "error", reason: "invalid token" }; - } - - const iat = obj["iat"]; - if (typeof iat !== "number") { - return { kind: "error", reason: "invalid token" }; - } - - const iss = obj["iss"]; - if (typeof iss !== "string" || iss !== TOKEN_ISSUER) { - return { kind: "error", reason: "invalid token" }; - } - - const aud = obj["aud"]; - if (typeof aud !== "string" || aud !== TOKEN_AUDIENCE) { - return { kind: "error", reason: "invalid token" }; - } - - if (!Number.isSafeInteger(exp) || !Number.isSafeInteger(iat)) { - return { kind: "error", reason: "invalid token" }; - } - - // Reject tokens issued in the future, allowing a small clock skew between nodes. - const now = Date.now() / 1000; - if (iat > now + MAX_CLOCK_SKEW_SECONDS) { - return { kind: "error", reason: "invalid token" }; - } - - if (exp <= iat) { - return { kind: "error", reason: "invalid token" }; - } - - const rawCaches = obj["caches"]; - if (!Array.isArray(rawCaches) || rawCaches.length === 0) { - return { kind: "error", reason: "invalid token" }; - } - if (rawCaches.length > MAX_CACHES_PER_TOKEN) { - return { kind: "error", reason: "invalid token" }; - } - const validatedCaches: string[] = []; - for (let i = 0; i < rawCaches.length; i++) { - const entry: unknown = rawCaches[i]; - if (typeof entry !== "string" || entry.length === 0) { - return { kind: "error", reason: "invalid token" }; - } - validatedCaches.push(entry); - } - - const rawPerms = obj["perms"]; - if (!Array.isArray(rawPerms) || rawPerms.length === 0) { - return { kind: "error", reason: "invalid token" }; - } - if (rawPerms.length > MAX_PERMS_PER_TOKEN) { - return { kind: "error", reason: "invalid token" }; - } - const validatedPerms: TokenPermission[] = []; - for (let i = 0; i < rawPerms.length; i++) { - const perm: unknown = rawPerms[i]; - if (typeof perm !== "string" || !isTokenPermission(perm)) { - return { kind: "error", reason: "invalid token" }; - } - validatedPerms.push(perm); - } - - return { - kind: "ok", - claims: { jti, sub, iss, aud, caches: validatedCaches, perms: validatedPerms, exp, iat }, - }; -} - -export async function verifyJwt(token: string, secret: string): Promise { - if (token.length > MAX_TOKEN_BYTES) { - return { kind: "error", reason: "invalid token" }; - } - - const firstDot = token.indexOf("."); - if (firstDot === -1) { - return { kind: "error", reason: "invalid token" }; - } - const secondDot = token.indexOf(".", firstDot + 1); - if (secondDot === -1 || token.indexOf(".", secondDot + 1) !== -1) { - return { kind: "error", reason: "invalid token" }; - } - - const headerB64 = token.slice(0, firstDot); - const payloadB64 = token.slice(firstDot + 1, secondDot); - const signatureB64 = token.slice(secondDot + 1); - - // Compare the encoded header directly — saves a base64 decode and string parse. - if (headerB64 !== HEADER_B64) { - return { kind: "error", reason: "invalid token" }; - } - - // Parse the payload first so we can short-circuit on expiry before paying for HMAC verify. - let payloadJson: string; - try { - payloadJson = decoder.decode(base64UrlDecode(payloadB64)); - } catch { - return { kind: "error", reason: "invalid token" }; - } - - let rawParsed: unknown; - try { - rawParsed = JSON.parse(payloadJson); - } catch { - return { kind: "error", reason: "invalid token" }; - } - - if (typeof rawParsed !== "object" || rawParsed === null) { - return { kind: "error", reason: "invalid token" }; - } - - const claims: Record = Object.fromEntries(Object.entries(rawParsed)); - - const earlyExp = claims["exp"]; - if (typeof earlyExp === "number" && earlyExp <= Date.now() / 1000 - MAX_CLOCK_SKEW_SECONDS) { - return { kind: "error", reason: "invalid token" }; - } - - const key = await getKey(secret); - let signature: Uint8Array; - try { - signature = base64UrlDecode(signatureB64); - } catch { - return { kind: "error", reason: "invalid token" }; - } - - const data = encoder.encode(token.slice(0, secondDot)); - // Copy into a fresh ArrayBuffer because Uint8Array.buffer is ArrayBufferLike, - // which the CF runtime's BufferSource type does not accept. - const sigBuf = new ArrayBuffer(signature.byteLength); - new Uint8Array(sigBuf).set(signature); - const valid = await crypto.subtle.verify("HMAC", key, sigBuf, data); - if (!valid) { - return { kind: "error", reason: "invalid token" }; - } - - return validateClaims(claims); -} - -export async function signJwt(claims: TokenClaims, secret: string): Promise { - const payloadB64 = base64UrlEncode(encoder.encode(JSON.stringify(claims))); - - const key = await getKey(secret); - const signingInput = `${HEADER_B64}.${payloadB64}`; - const signature = new Uint8Array(await crypto.subtle.sign("HMAC", key, encoder.encode(signingInput))); - - return `${signingInput}.${base64UrlEncode(signature)}`; -} - -export function hasPermission(claims: TokenClaims, perm: TokenPermission): boolean { - return claims.perms.includes(perm); -} - -export function hasCacheAccess(claims: TokenClaims, cacheName: string): boolean { - return claims.caches.includes("*") || claims.caches.includes(cacheName); -} diff --git a/workers/cache/src/logging.ts b/workers/cache/src/logging.ts deleted file mode 100644 index b3e0abd..0000000 --- a/workers/cache/src/logging.ts +++ /dev/null @@ -1,15 +0,0 @@ -export function logRequest( - request: Request, - response: Response, - startTime: number, -): void { - const duration = Date.now() - startTime; - const url = new URL(request.url); - console.log(JSON.stringify({ - method: request.method, - path: url.pathname, - status: response.status, - duration_ms: duration, - cf_ray: request.headers.get("cf-ray") ?? undefined, - })); -} diff --git a/workers/cache/src/narinfo.ts b/workers/cache/src/narinfo.ts deleted file mode 100644 index 416bc72..0000000 --- a/workers/cache/src/narinfo.ts +++ /dev/null @@ -1,40 +0,0 @@ -import type { PublishedPath, BlobObject } from "./db/types.js"; - -export function renderNarinfo( - path: PublishedPath, - blob: BlobObject, - refs: readonly string[], - storedSigs: readonly string[], - signatures: readonly string[] = [], -): string { - const lines: string[] = [ - `StorePath: ${path.storePath}`, - `URL: nar/${blob.fileHash}/${blob.compression}.nar`, - `Compression: ${blob.compression}`, - `FileHash: sha256:${blob.fileHash}`, - `FileSize: ${blob.fileSize}`, - `NarHash: ${path.narHash}`, - `NarSize: ${path.narSize}`, - ]; - - if (refs.length > 0) { - lines.push(`References: ${refs.join(" ")}`); - } - - if (path.deriver) { - lines.push(`Deriver: ${path.deriver}`); - } - - if (path.system) { - lines.push(`System: ${path.system}`); - } - - for (const sig of storedSigs) { - lines.push(`Sig: ${sig}`); - } - for (const sig of signatures) { - lines.push(`Sig: ${sig}`); - } - - return lines.join("\n") + "\n"; -} diff --git a/workers/cache/src/presign.ts b/workers/cache/src/presign.ts deleted file mode 100644 index 0c46d75..0000000 --- a/workers/cache/src/presign.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { AwsClient } from "aws4fetch"; - -let cachedClient: AwsClient | undefined; -let cachedCredentials: string | undefined; - -function getClient(accessKeyId: string, secretAccessKey: string): AwsClient { - const key = `${accessKeyId}:${secretAccessKey}`; - if (cachedClient && cachedCredentials === key) { - return cachedClient; - } - cachedClient = new AwsClient({ - accessKeyId, - secretAccessKey, - service: "s3", - region: "auto", - }); - cachedCredentials = key; - return cachedClient; -} - -export async function createPresignedUrl( - endpoint: string, - accessKeyId: string, - secretAccessKey: string, - bucket: string, - key: string, - expiresIn: number, -): Promise { - const client = getClient(accessKeyId, secretAccessKey); - const url = new URL(`/${bucket}/${key}`, endpoint); - url.searchParams.set("X-Amz-Expires", String(expiresIn)); - - const signed = await client.sign(new Request(url, { method: "GET" }), { - aws: { signQuery: true }, - }); - - return signed.url; -} diff --git a/workers/cache/src/responses.ts b/workers/cache/src/responses.ts deleted file mode 100644 index 5f77e6e..0000000 --- a/workers/cache/src/responses.ts +++ /dev/null @@ -1,23 +0,0 @@ -const JSON_HEADERS: Record = { - "content-type": "application/json; charset=utf-8", - "cache-control": "no-store", -}; - -export function jsonResponse(body: unknown, status = 200): Response { - return new Response(JSON.stringify(body), { - status, - headers: JSON_HEADERS, - }); -} - -export function errorResponse(message: string, status: number): Response { - return jsonResponse({ error: message }, status); -} - -export async function parseJsonBody(request: Request): Promise { - try { - return await request.json(); - } catch { - return errorResponse("Invalid JSON in request body", 400); - } -} diff --git a/workers/cache/src/router.ts b/workers/cache/src/router.ts deleted file mode 100644 index bfda217..0000000 --- a/workers/cache/src/router.ts +++ /dev/null @@ -1,452 +0,0 @@ -import type { WorkerConfig } from "./config.js"; -import { findCacheByName, findPublishedPathWithBlob, createAuditLog } from "./db/repository.js"; -import { renderNarinfo } from "./narinfo.js"; -import { authenticateWrite, authenticateAdmin, actorId } from "./auth.js"; -import type { AuthIdentity } from "./auth.js"; -import { hasPermission, hasCacheAccess } from "./jwt.js"; -import { errorResponse } from "./responses.js"; -import { computeFingerprint, signNarinfo } from "./signing.js"; -import { parseCacheName, parseStorePathHash, parseFileHash, parseCompression } from "@helios/cache-domain"; -import type { CacheName } from "@helios/cache-domain"; -import { - handleCreateSession, - handleMultipart, - handleUploadPart, - handleUploadBlob, - handleComplete, - handlePublish, - handleGetMissingPaths, -} from "./uploads.js"; -import { createPresignedUrl } from "./presign.js"; -import { handleCreateToken, handleListTokens, handleRevokeToken } from "./admin.js"; - -// Pre-compiled route patterns. Compiled once per isolate, not per request. -const RE_SESSION_CREATE = /^\/_api\/v1\/caches\/([^/]+)\/upload-sessions$/; -const RE_MULTIPART = /^\/_api\/v1\/uploads\/([^/]+)\/multipart$/; -const RE_PART_UPLOAD = /^\/_api\/v1\/uploads\/([^/]+)\/part\/(\d+)$/; -const RE_BLOB = /^\/_api\/v1\/uploads\/([^/]+)\/blob$/; -const RE_COMPLETE = /^\/_api\/v1\/uploads\/([^/]+)\/complete$/; -const RE_PUBLISH = /^\/_api\/v1\/uploads\/([^/]+)\/publish$/; -const RE_REVOKE = /^\/_api\/v1\/admin\/tokens\/([^/]+)\/revoke$/; - -// Module-scope cache: cache name → D1 cache ID. -// Workers are short-lived isolates reused across requests on the same edge node, -// so this avoids a ~20ms D1 round trip on every narinfo warm hit. -const cacheNameToId = new Map(); - -async function resolveCacheId( - db: WorkerConfig["db"], - name: CacheName, -): Promise { - const cached = cacheNameToId.get(name); - if (cached !== undefined) return cached; - - const row = await findCacheByName(db, name); - if (!row) return undefined; - - cacheNameToId.set(name, row.id); - return row.id; -} - -export async function handleRequest( - request: Request, - config: WorkerConfig, -): Promise { - const url = new URL(request.url); - const method = request.method; - - if (url.pathname === "/") { - return json({ service: "helios-cache", status: "ok" }); - } - - if (url.pathname === "/healthz") { - return handleHealthz(config); - } - - // Admin API routes - if (url.pathname.startsWith("/_api/v1/admin/")) { - return handleAdminApi(request, config, url.pathname); - } - - // Write API routes (POST and PUT) - if (url.pathname.startsWith("/_api/v1/")) { - if (method !== "POST" && method !== "PUT") { - return new Response("Method Not Allowed", { - status: 405, - headers: { allow: "POST, PUT" }, - }); - } - return handleWriteApi(request, config, url.pathname); - } - - // Read paths (GET/HEAD only) - if (method !== "GET" && method !== "HEAD") { - return new Response("Method Not Allowed", { - status: 405, - headers: { allow: "GET, HEAD" }, - }); - } - - const segments = url.pathname.split("/").filter(Boolean); - if (segments.length < 2) { - return new Response("Not Found", { status: 404 }); - } - - // nix-cache-info is static — skip D1 lookup for the cache - if (segments[1] === "nix-cache-info") { - return NIX_CACHE_INFO_RESPONSE(); - } - - // NAR downloads are content-addressed — skip cache name D1 lookup - if (segments[1] === "nar" && segments.length === 4 && segments[3].endsWith(".nar")) { - return handleNarDownload(config, segments[2], segments[3].slice(0, -".nar".length), request, method); - } - - const cacheName = parseCacheName(segments[0]); - if (typeof cacheName === "object") { - return new Response("Not Found", { status: 404 }); - } - - if (segments[1].endsWith(".narinfo")) { - const hashStr = segments[1].slice(0, -".narinfo".length); - return handleNarinfo(config, cacheName, hashStr, method, request); - } - - return new Response("Not Found", { status: 404 }); -} - -function json(body: Record, status = 200): Response { - return new Response(JSON.stringify(body), { - status, - headers: { "content-type": "application/json; charset=utf-8" }, - }); -} - -function writeAuditLog( - config: WorkerConfig, - identity: AuthIdentity, - action: string, - cacheName: string | null, - status: number, - detail: string, - ip: string | null, -): void { - if (!config.ctx) return; - config.ctx.waitUntil( - createAuditLog(config.db, { - actor: actorId(identity), - action, - cacheName, - detail, - ip, - status, - }).catch((err) => { - console.error("Failed to write audit log", { error: err instanceof Error ? err.message : String(err) }); - }), - ); -} - -async function handleHealthz(config: WorkerConfig): Promise { - try { - await findCacheByName(config.db, "__healthz_probe__"); - return json({ ok: true, service: "helios-cache" }); - } catch { - return json({ ok: false, service: "helios-cache" }, 503); - } -} - -// Reused across requests — avoids per-request allocation -const textEncoder = new TextEncoder(); - -// Pre-computed static nix-cache-info body — avoids per-request string allocation -const NIX_CACHE_INFO_BODY = "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 40\n"; - -function NIX_CACHE_INFO_RESPONSE(): Response { - return new Response(NIX_CACHE_INFO_BODY, { - headers: { "content-type": "text/x-nix-cache-info" }, - }); -} - -// Nix mass-queries every substituter for every store path in a build, so most -// narinfo lookups are misses. Cache 404s briefly at the edge to keep that -// negative-lookup storm off D1. Short TTL so newly published paths appear quickly. -const NARINFO_NEGATIVE_TTL_SECONDS = 60; - -async function handleNarinfo( - config: WorkerConfig, - cacheName: CacheName, - hashStr: string, - method: string, - request: Request, -): Promise { - const storePathHash = parseStorePathHash(hashStr); - if (typeof storePathHash === "object") { - return new Response("Not Found", { status: 404 }); - } - - // Normalize to GET for shared cache (HEAD and GET share the same cache entry) - const cacheKey = new Request(request.url, { method: "GET" }); - const cache = await caches.open("helios-narinfo"); - const cached = await cache.match(cacheKey); - if (cached) { - if (method === "HEAD") { - return new Response(null, { status: cached.status, headers: cached.headers }); - } - return cached; - } - - // Unknown cache and unpublished path both produce a (negatively cached) 404. - const cacheId = await resolveCacheId(config.db, cacheName); - const result = cacheId === undefined - ? undefined - : await findPublishedPathWithBlob(config.db, cacheId, storePathHash); - if (!result) { - const notFoundHeaders = { "cache-control": `public, max-age=${NARINFO_NEGATIVE_TTL_SECONDS}` }; - if (config.ctx) { - // Fresh Response instead of clone(): a tee'd body whose sibling is - // never consumed can stall cache.put. - config.ctx.waitUntil( - cache.put(cacheKey, new Response("Not Found", { status: 404, headers: notFoundHeaders })), - ); - } - if (method === "HEAD") { - return new Response(null, { status: 404, headers: notFoundHeaders }); - } - return new Response("Not Found", { status: 404, headers: notFoundHeaders }); - } - - const refs: string[] = JSON.parse(result.path.referencesJson); - const storedSigs: string[] = JSON.parse(result.path.signaturesJson); - const fingerprint = computeFingerprint(result.path, refs); - const sig = await signNarinfo(fingerprint, config.signingKeyName, config.signingPrivateKey); - const signatures = sig ? [sig] : []; - const body = renderNarinfo(result.path, result.blob, refs, storedSigs, signatures); - - const response = new Response(body, { - headers: { - "content-type": "text/x-nix-narinfo", - "content-length": String(textEncoder.encode(body).byteLength), - "cache-control": "public, max-age=31536000, immutable", - }, - }); - - // Cache at edge (non-blocking) - if (config.ctx) { - config.ctx.waitUntil(cache.put(cacheKey, response.clone())); - } - - if (method === "HEAD") { - return new Response(null, { - status: 200, - headers: response.headers, - }); - } - - return response; -} - -async function handleNarDownload( - config: WorkerConfig, - rawFileHash: string, - rawCompression: string, - request: Request, - method: string, -): Promise { - const fileHash = parseFileHash(rawFileHash); - if (typeof fileHash === "object") { - return new Response("Not Found", { status: 404 }); - } - - const compression = parseCompression(rawCompression); - if (typeof compression === "object") { - return new Response("Not Found", { status: 404 }); - } - - // NAR keys are content-addressed, so we can skip the D1 lookup - // and go straight to R2 using the key derived from the URL - const r2Key = `nars/sha256/${fileHash}/${compression}.nar`; - - // Presigned R2 redirect (bypasses Worker entirely) - if (config.r2AccessKeyId && config.r2SecretAccessKey && config.r2Endpoint) { - const url = await createPresignedUrl( - config.r2Endpoint, - config.r2AccessKeyId, - config.r2SecretAccessKey, - config.r2BucketName, - r2Key, - 3600, - ); - return new Response(null, { - status: 302, - headers: { - location: url, - "cache-control": "public, max-age=3600", - }, - }); - } - - // Edge cache check (use a GET key for shared cache) - const cacheUrl = new URL(request.url); - const cacheRequest = new Request(cacheUrl, { method: "GET" }); - const cache = await caches.open("helios-nar"); - const cached = await cache.match(cacheRequest); - if (cached) { - if (method === "HEAD") { - return new Response(null, { status: 200, headers: cached.headers }); - } - return cached; - } - - if (method === "HEAD") { - const head = await config.bucket.head(r2Key); - if (!head) { - return new Response("Not Found", { status: 404 }); - } - return new Response(null, { - status: 200, - headers: { - "content-type": "application/x-nix-nar", - "content-length": String(head.size), - "cache-control": "public, max-age=31536000, immutable", - }, - }); - } - - const object = await config.bucket.get(r2Key); - if (!object) { - return new Response("Not Found", { status: 404 }); - } - - const response = new Response(object.body, { - headers: { - "content-type": "application/x-nix-nar", - "content-length": String(object.size), - "cache-control": "public, max-age=31536000, immutable", - }, - }); - - // Store in CF edge cache (non-blocking) - if (config.ctx) { - config.ctx.waitUntil(cache.put(cacheRequest, response.clone())); - } - - return response; -} - -async function handleAdminApi( - request: Request, - config: WorkerConfig, - pathname: string, -): Promise { - const authResult = await authenticateAdmin(request, config); - if (authResult.kind === "error") { - return authResult.response; - } - - const ip = request.headers.get("cf-connecting-ip"); - - if (pathname === "/_api/v1/admin/tokens" && request.method === "POST") { - const response = await handleCreateToken(request, config); - writeAuditLog(config, authResult.identity, "token.create", null, response.status, "{}", ip); - return response; - } - - if (pathname === "/_api/v1/admin/tokens" && request.method === "GET") { - const response = await handleListTokens(config); - writeAuditLog(config, authResult.identity, "token.list", null, response.status, "{}", ip); - return response; - } - - const revokeMatch = pathname.match(RE_REVOKE); - if (revokeMatch && request.method === "POST") { - const response = await handleRevokeToken(request, config, revokeMatch[1]); - writeAuditLog(config, authResult.identity, "token.revoke", null, response.status, JSON.stringify({ jti: revokeMatch[1] }), ip); - return response; - } - - return new Response("Not Found", { status: 404 }); -} - -async function handleWriteApi( - request: Request, - config: WorkerConfig, - pathname: string, -): Promise { - const authResult = await authenticateWrite(request, config); - if (authResult.kind === "error") { - return authResult.response; - } - - const identity = authResult.identity; - const ip = request.headers.get("cf-connecting-ip"); - - // Check push permission for JWT tokens - if (identity.kind === "jwt" && !hasPermission(identity.claims, "push")) { - const response = errorResponse("Forbidden", 403); - writeAuditLog(config, identity, "push", null, 403, '{"reason":"missing push permission"}', ip); - return response; - } - - if (pathname === "/_api/v1/get-missing-paths") { - const response = await handleGetMissingPaths(request, config, identity); - writeAuditLog(config, identity, "push", null, response.status, '{"endpoint":"get-missing-paths"}', ip); - return response; - } - - const sessionCreate = pathname.match(RE_SESSION_CREATE); - if (sessionCreate) { - const cacheName = sessionCreate[1]; - if (identity.kind === "jwt" && !hasCacheAccess(identity.claims, cacheName)) { - const response = errorResponse("Forbidden", 403); - writeAuditLog(config, identity, "push", cacheName, 403, '{"reason":"cache access denied"}', ip); - return response; - } - const response = await handleCreateSession(request, config, cacheName); - writeAuditLog(config, identity, "push", cacheName, response.status, '{"endpoint":"create-session"}', ip); - return response; - } - - // Session-scoped upload routes — all share the same audit pattern - const uploadRoutes: ReadonlyArray<{ - readonly pattern: RegExp; - readonly endpoint: string; - readonly handle: (match: RegExpMatchArray) => Promise; - }> = [ - { - pattern: RE_MULTIPART, - endpoint: "multipart", - handle: (m) => handleMultipart(config, m[1], identity), - }, - { - pattern: RE_PART_UPLOAD, - endpoint: "upload-part", - handle: (m) => handleUploadPart(request, config, m[1], parseInt(m[2], 10), identity), - }, - { - pattern: RE_BLOB, - endpoint: "upload-blob", - handle: (m) => handleUploadBlob(request, config, m[1], identity), - }, - { - pattern: RE_COMPLETE, - endpoint: "complete", - handle: (m) => handleComplete(request, config, m[1], identity), - }, - { - pattern: RE_PUBLISH, - endpoint: "publish", - handle: (m) => handlePublish(config, m[1], identity), - }, - ]; - - for (const route of uploadRoutes) { - const match = pathname.match(route.pattern); - if (match) { - const response = await route.handle(match); - writeAuditLog(config, identity, "push", null, response.status, `{"endpoint":"${route.endpoint}"}`, ip); - return response; - } - } - - return new Response("Not Found", { status: 404 }); -} diff --git a/workers/cache/src/signing.ts b/workers/cache/src/signing.ts deleted file mode 100644 index 81dd8f3..0000000 --- a/workers/cache/src/signing.ts +++ /dev/null @@ -1,59 +0,0 @@ -import type { PublishedPath } from "./db/types.js"; - -const textEncoder = new TextEncoder(); - -export function computeFingerprint(path: PublishedPath, refs: readonly string[]): string { - const refPaths = refs.map(r => `/nix/store/${r}`); - return `1;${path.storePath};${path.narHash};${path.narSize};${refPaths.join(",")}`; -} - -const PKCS8_ED25519_PREFIX = new Uint8Array([ - 0x30, 0x2e, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, - 0x03, 0x2b, 0x65, 0x70, 0x04, 0x22, 0x04, 0x20, -]); - -let cachedKeyMaterial: string | undefined; -let cachedKey: CryptoKey | undefined; - -async function getSigningKey(privateKeyBase64: string): Promise { - if (cachedKey && cachedKeyMaterial === privateKeyBase64) { - return cachedKey; - } - - const keyBytes = Uint8Array.from(atob(privateKeyBase64), c => c.charCodeAt(0)); - const seed = keyBytes.slice(0, 32); - const pkcs8 = new Uint8Array(PKCS8_ED25519_PREFIX.length + 32); - pkcs8.set(PKCS8_ED25519_PREFIX); - pkcs8.set(seed, PKCS8_ED25519_PREFIX.length); - const pkcs8Buffer = new ArrayBuffer(pkcs8.byteLength); - new Uint8Array(pkcs8Buffer).set(pkcs8); - - const key = await crypto.subtle.importKey( - "pkcs8", - pkcs8Buffer, - { name: "Ed25519" }, - false, - ["sign"], - ); - - cachedKeyMaterial = privateKeyBase64; - cachedKey = key; - return key; -} - -export async function signNarinfo( - fingerprint: string, - keyName: string, - privateKeyBase64: string, -): Promise { - if (!privateKeyBase64) { - return ""; - } - - const key = await getSigningKey(privateKeyBase64); - const data = textEncoder.encode(fingerprint); - const signature = await crypto.subtle.sign("Ed25519", key, data); - const sigBase64 = btoa(String.fromCharCode(...new Uint8Array(signature))); - - return `${keyName}:${sigBase64}`; -} diff --git a/workers/cache/src/uploads.ts b/workers/cache/src/uploads.ts deleted file mode 100644 index a1546b8..0000000 --- a/workers/cache/src/uploads.ts +++ /dev/null @@ -1,432 +0,0 @@ -import type { WorkerConfig } from "./config.js"; -import type { AuthIdentity } from "./auth.js"; -import { hasCacheAccess } from "./jwt.js"; -import { - findCacheByName, - findCacheById, - createUploadSession, - findUploadSession, - transitionSessionStatus, - transitionToMultipart, - upsertUploadPart, - upsertUploadParts, - findUploadParts, - createBlobObject, - findBlobObject, - createPublishedPath, - findPublishedPath, - findPublishedHashes, -} from "./db/repository.js"; -import type { UploadSession } from "./db/types.js"; -import { buildR2ObjectKey, parseFileHash, parseCompression, parseStorePathHash } from "@helios/cache-domain"; -import { jsonResponse, errorResponse, parseJsonBody } from "./responses.js"; -import type { DrizzleD1Database } from "drizzle-orm/d1"; - -// Module-scope cache: cache ID → cache name. Cache rows are immutable today -// (no rename or delete endpoints), so entries never go stale. Avoids a D1 -// round trip on every session-scoped upload request made with a JWT. -const cacheIdToName = new Map(); - -async function resolveCacheName( - db: DrizzleD1Database, - cacheId: number, -): Promise { - const cached = cacheIdToName.get(cacheId); - if (cached !== undefined) return cached; - - const row = await findCacheById(db, cacheId); - if (!row) return undefined; - - cacheIdToName.set(cacheId, row.name); - return row.name; -} - -// ── Session Auth Helper ── - -/** - * Load an upload session and verify the caller has access to its cache. - * Bearer-token callers are trusted implicitly; JWT callers must hold - * access to the cache the session belongs to. - */ -async function loadSessionWithAuthCheck( - config: WorkerConfig, - sessionId: string, - identity: AuthIdentity, -): Promise<{ session: UploadSession; error?: never } | { session?: never; error: Response }> { - const session = await findUploadSession(config.db, sessionId); - if (!session) { - return { error: errorResponse("Not found", 404) }; - } - - // Enforce session expiry at request time, not just via GC - if (session.expiresAt <= new Date().toISOString()) { - return { error: errorResponse("Not found", 404) }; - } - - if (identity.kind === "jwt") { - const cacheName = await resolveCacheName(config.db, session.cacheId); - if (cacheName === undefined || !hasCacheAccess(identity.claims, cacheName)) { - return { error: errorResponse("Forbidden", 403) }; - } - } - - return { session }; -} - -// ── Create Upload Session ── - -export async function handleCreateSession( - request: Request, - config: WorkerConfig, - cacheName: string, -): Promise { - const cache = await findCacheByName(config.db, cacheName); - if (!cache) { - return errorResponse("Cache not found", 404); - } - // Seed the ID → name cache so session-scoped requests skip the D1 lookup. - cacheIdToName.set(cache.id, cache.name); - - const body = await parseJsonBody<{ - storePath: string; - storePathHash: string; - narHash: string; - narSize: number; - fileHash: string; - fileSize: number; - compression: string; - references?: string[]; - deriver?: string; - system?: string; - }>(request); - if (body instanceof Response) return body; - - if (!body.storePath || !body.storePathHash || !body.narHash || !body.fileHash || !body.compression) { - return errorResponse("Missing required fields", 400); - } - - if (typeof body.narSize !== "number" || typeof body.fileSize !== "number") { - return errorResponse("narSize and fileSize must be numbers", 400); - } - - const parsedHash = parseStorePathHash(body.storePathHash); - if (typeof parsedHash === "object") { - return errorResponse(parsedHash.message, 400); - } - - if (!Number.isFinite(body.narSize) || body.narSize <= 0) { - return errorResponse("narSize must be a positive number", 400); - } - if (!Number.isFinite(body.fileSize) || body.fileSize <= 0) { - return errorResponse("fileSize must be a positive number", 400); - } - - const fileHash = parseFileHash(body.fileHash); - if (typeof fileHash === "object") { - return errorResponse(fileHash.message, 400); - } - - const compression = parseCompression(body.compression); - if (typeof compression === "object") { - return errorResponse(compression.message, 400); - } - - const r2Key = buildR2ObjectKey(fileHash, compression); - const sessionId = crypto.randomUUID(); - const expiresAt = new Date(Date.now() + 3600_000).toISOString(); - - const session = await createUploadSession(config.db, { - id: sessionId, - cacheId: cache.id, - storePathHash: body.storePathHash, - storePath: body.storePath, - narHash: body.narHash, - narSize: body.narSize, - fileHash: body.fileHash, - fileSize: body.fileSize, - compression: body.compression, - referencesJson: JSON.stringify(body.references ?? []), - deriver: body.deriver ?? null, - system: body.system ?? null, - r2UploadKey: r2Key, - r2UploadId: null, - expiresAt, - }); - - return jsonResponse({ - sessionId: session.id, - r2Key, - uploadMethod: "direct", - expiresAt, - }, 201); -} - -// ── Initiate Multipart Upload ── - -export async function handleMultipart( - config: WorkerConfig, - sessionId: string, - identity: AuthIdentity, -): Promise { - const result = await loadSessionWithAuthCheck(config, sessionId, identity); - if (result.error) return result.error; - const session = result.session; - - if (session.status !== "pending") { - return errorResponse("Conflict", 409); - } - - if (!session.r2UploadKey) { - return errorResponse("Internal error", 500); - } - - const multipart = await config.bucket.createMultipartUpload(session.r2UploadKey); - - const transitioned = await transitionToMultipart(config.db, sessionId, multipart.uploadId); - if (!transitioned) { - try { await multipart.abort(); } catch { /* best effort */ } - return errorResponse("Conflict", 409); - } - - return jsonResponse({ - uploadId: multipart.uploadId, - r2Key: session.r2UploadKey, - }); -} - -// ── Upload Part (Multipart) ── - -export async function handleUploadPart( - request: Request, - config: WorkerConfig, - sessionId: string, - partNumber: number, - identity: AuthIdentity, -): Promise { - const result = await loadSessionWithAuthCheck(config, sessionId, identity); - if (result.error) return result.error; - const session = result.session; - - if (session.status !== "uploading") { - return errorResponse("Conflict", 409); - } - - if (!session.r2UploadKey || !session.r2UploadId) { - return errorResponse("Bad request", 400); - } - - if (!request.body) { - return errorResponse("Request body is empty", 400); - } - - const multipart = config.bucket.resumeMultipartUpload(session.r2UploadKey, session.r2UploadId); - const uploadedPart = await multipart.uploadPart(partNumber, request.body); - - await upsertUploadPart(config.db, { - sessionId, - partNumber, - etag: uploadedPart.etag, - size: 0, - }); - - return jsonResponse({ partNumber, etag: uploadedPart.etag }); -} - -// ── Upload Blob ── - -export async function handleUploadBlob( - request: Request, - config: WorkerConfig, - sessionId: string, - identity: AuthIdentity, -): Promise { - const result = await loadSessionWithAuthCheck(config, sessionId, identity); - if (result.error) return result.error; - const session = result.session; - - if (session.status !== "pending" && session.status !== "uploading") { - return errorResponse("Conflict", 409); - } - - if (!session.r2UploadKey) { - return errorResponse("Internal error", 500); - } - - if (!request.body) { - return errorResponse("Request body is empty", 400); - } - - // Blob upload is idempotent (PUT overwrites), so allow both pending and uploading states. - // Attempt pending→uploading; if already uploading the transition fails but that is acceptable. - if (session.status === "pending") { - const transitioned = await transitionSessionStatus(config.db, sessionId, "pending", "uploading"); - if (!transitioned) { - // Re-read to check if someone else moved it to uploading (retry-safe) - const current = await findUploadSession(config.db, sessionId); - if (!current || current.status !== "uploading") { - return errorResponse("Conflict", 409); - } - } - } - - await config.bucket.put(session.r2UploadKey, request.body, { - httpMetadata: { contentType: "application/x-nix-nar" }, - }); - - return jsonResponse({ uploaded: true, r2Key: session.r2UploadKey }); -} - -// ── Complete Upload ── - -export async function handleComplete( - request: Request, - config: WorkerConfig, - sessionId: string, - identity: AuthIdentity, -): Promise { - const result = await loadSessionWithAuthCheck(config, sessionId, identity); - if (result.error) return result.error; - const session = result.session; - - if (session.status !== "uploading") { - return errorResponse("Conflict", 409); - } - - if (!session.r2UploadKey) { - return errorResponse("Internal error", 500); - } - - const body = await parseJsonBody<{ - parts?: Array<{ partNumber: number; etag: string; size: number }>; - }>(request); - if (body instanceof Response) return body; - - if (body.parts && body.parts.length > 0) { - await upsertUploadParts(config.db, sessionId, body.parts); - } - - if (session.r2UploadId) { - const parts = await findUploadParts(config.db, sessionId); - const multipart = config.bucket.resumeMultipartUpload(session.r2UploadKey, session.r2UploadId); - await multipart.complete(parts.map(p => ({ partNumber: p.partNumber, etag: p.etag }))); - } - - const head = await config.bucket.head(session.r2UploadKey); - if (!head) { - return errorResponse("Upload incomplete", 400); - } - - if (head.size !== session.fileSize) { - return errorResponse("File size mismatch", 400); - } - - const transitioned = await transitionSessionStatus(config.db, sessionId, "uploading", "completed"); - if (!transitioned) { - return errorResponse("Conflict", 409); - } - - return jsonResponse({ status: "completed" }); -} - -// ── Publish Path ── - -export async function handlePublish( - config: WorkerConfig, - sessionId: string, - identity: AuthIdentity, -): Promise { - const result = await loadSessionWithAuthCheck(config, sessionId, identity); - if (result.error) return result.error; - const session = result.session; - - if (session.status !== "completed") { - return errorResponse("Conflict", 409); - } - - const existing = await findPublishedPath(config.db, session.cacheId, session.storePathHash); - if (existing) { - return jsonResponse({ published: true, storePathHash: session.storePathHash, alreadyExisted: true }); - } - - let blob = await findBlobObject(config.db, session.fileHash, session.compression); - if (!blob) { - if (!session.r2UploadKey) { - return errorResponse("Internal error", 500); - } - try { - blob = await createBlobObject(config.db, { - fileHash: session.fileHash, - fileSize: session.fileSize, - compression: session.compression, - r2Key: session.r2UploadKey, - }); - } catch { - blob = await findBlobObject(config.db, session.fileHash, session.compression); - if (!blob) { - return errorResponse("Internal error", 500); - } - } - } - - try { - await createPublishedPath(config.db, { - cacheId: session.cacheId, - storePathHash: session.storePathHash, - storePath: session.storePath, - narHash: session.narHash, - narSize: session.narSize, - blobObjectId: blob.id, - referencesJson: session.referencesJson, - deriver: session.deriver, - system: session.system, - signaturesJson: "[]", - }); - } catch { - const raced = await findPublishedPath(config.db, session.cacheId, session.storePathHash); - if (raced) { - return jsonResponse({ published: true, storePathHash: session.storePathHash, alreadyExisted: true }); - } - return errorResponse("Internal error", 500); - } - - return jsonResponse({ published: true, storePathHash: session.storePathHash }); -} - -// ── Get Missing Paths ── - -const MAX_MISSING_PATHS_BATCH = 1000; - -export async function handleGetMissingPaths( - request: Request, - config: WorkerConfig, - identity: AuthIdentity, -): Promise { - const body = await parseJsonBody<{ - cache: string; - storePathHashes: string[]; - }>(request); - if (body instanceof Response) return body; - - if (!body.cache || !Array.isArray(body.storePathHashes)) { - return errorResponse("Missing required fields: cache, storePathHashes", 400); - } - - if (body.storePathHashes.length > MAX_MISSING_PATHS_BATCH) { - return errorResponse(`storePathHashes exceeds maximum of ${MAX_MISSING_PATHS_BATCH}`, 400); - } - - const cache = await findCacheByName(config.db, body.cache); - if (!cache) { - return errorResponse("Cache not found", 404); - } - - if (identity.kind === "jwt" && !hasCacheAccess(identity.claims, body.cache)) { - return errorResponse("Forbidden", 403); - } - - const existingHashes = await findPublishedHashes(config.db, cache.id, body.storePathHashes); - const existingSet = new Set(existingHashes); - const missing = body.storePathHashes.filter((h) => !existingSet.has(h)); - - return jsonResponse({ missing }); -} diff --git a/workers/cache/test/apply-migrations.ts b/workers/cache/test/apply-migrations.ts deleted file mode 100644 index 5311171..0000000 --- a/workers/cache/test/apply-migrations.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { applyD1Migrations } from "cloudflare:test"; -import { env } from "cloudflare:test"; - -await applyD1Migrations(env.CACHE_DB, env.TEST_MIGRATIONS); diff --git a/workers/cache/test/auth.spec.ts b/workers/cache/test/auth.spec.ts deleted file mode 100644 index ac2fcc0..0000000 --- a/workers/cache/test/auth.spec.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { env, createExecutionContext, waitOnExecutionContext, SELF } from "cloudflare:test"; -import { describe, it, expect, beforeAll } from "vitest"; -import { drizzle } from "drizzle-orm/d1"; -import { createCache } from "../src/db/repository.js"; -import worker from "../src"; - -const AUTH_TOKEN = "test-secret-token"; -const CACHE_NAME = "auth-test"; - -function envWithAuth(): typeof env { - return { ...env, AUTH_TOKEN }; -} - -function postWithWorker( - path: string, - body: unknown, - headers: Record = {}, -): Promise { - const request = new Request(`http://example.com${path}`, { - method: "POST", - headers: { "content-type": "application/json", ...headers }, - body: JSON.stringify(body), - }); - const ctx = createExecutionContext(); - const response = worker.fetch(request, envWithAuth(), ctx); - waitOnExecutionContext(ctx); - return response; -} - -beforeAll(async () => { - const db = drizzle(env.CACHE_DB); - await createCache(db, CACHE_NAME); -}); - -describe("write auth", () => { - it("rejects write requests without authorization header", async () => { - const res = await postWithWorker( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - { storePath: "/nix/store/test", storePathHash: "a".repeat(32), narHash: "sha256:" + "1".repeat(52), narSize: 100, fileHash: "b".repeat(64), fileSize: 100, compression: "zstd" }, - ); - - expect(res.status).toBe(401); - const body = await res.json<{ error: string }>(); - expect(body.error).toBe("Unauthorized"); - }); - - it("rejects write requests with wrong bearer token", async () => { - const res = await postWithWorker( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - { storePath: "/nix/store/test", storePathHash: "a".repeat(32), narHash: "sha256:" + "1".repeat(52), narSize: 100, fileHash: "b".repeat(64), fileSize: 100, compression: "zstd" }, - { authorization: "Bearer wrong-token" }, - ); - - expect(res.status).toBe(403); - const body = await res.json<{ error: string }>(); - expect(body.error).toBe("Forbidden"); - }); - - it("accepts write requests with valid bearer token", async () => { - const res = await postWithWorker( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - { storePath: "/nix/store/test", storePathHash: "c".repeat(32), narHash: "sha256:" + "1".repeat(52), narSize: 100, fileHash: "d".repeat(64), fileSize: 100, compression: "zstd" }, - { authorization: `Bearer ${AUTH_TOKEN}` }, - ); - - expect(res.status).toBe(201); - }); - - it("rejects non-bearer auth schemes", async () => { - const res = await postWithWorker( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - { storePath: "/nix/store/test", storePathHash: "a".repeat(32), narHash: "sha256:" + "1".repeat(52), narSize: 100, fileHash: "b".repeat(64), fileSize: 100, compression: "zstd" }, - { authorization: `Basic ${AUTH_TOKEN}` }, - ); - - expect(res.status).toBe(401); - }); -}); - -describe("read endpoints remain public", () => { - it("GET nix-cache-info works without auth even when AUTH_TOKEN is set", async () => { - const request = new Request(`http://example.com/${CACHE_NAME}/nix-cache-info`); - const ctx = createExecutionContext(); - const res = await worker.fetch(request, envWithAuth(), ctx); - await waitOnExecutionContext(ctx); - - expect(res.status).toBe(200); - const body = await res.text(); - expect(body).toContain("StoreDir: /nix/store"); - }); - - it("read endpoints via SELF work without auth (AUTH_TOKEN empty in test env)", async () => { - const res = await SELF.fetch(`http://example.com/${CACHE_NAME}/nix-cache-info`); - - expect(res.status).toBe(200); - }); -}); diff --git a/workers/cache/test/db.spec.ts b/workers/cache/test/db.spec.ts deleted file mode 100644 index a3cf6c5..0000000 --- a/workers/cache/test/db.spec.ts +++ /dev/null @@ -1,390 +0,0 @@ -import { env } from "cloudflare:test"; -import { describe, it, expect } from "vitest"; -import { drizzle } from "drizzle-orm/d1"; -import { - createCache, - findCacheByName, - createBlobObject, - findBlobObject, - createPublishedPath, - findPublishedPath, - createUploadSession, - findUploadSession, - transitionSessionStatus, - findExpiredSessions, - upsertUploadPart, - findUploadParts, - createGcMark, - findGcMarks, - deleteGcMark, -} from "../src/db/repository.js"; - -function getDb() { - return drizzle(env.CACHE_DB); -} - -function makeSessionParams(overrides: { - readonly id: string; - readonly cacheId: number; - readonly expiresAt?: string; - readonly storePathHash?: string; -}) { - return { - id: overrides.id, - cacheId: overrides.cacheId, - storePathHash: overrides.storePathHash ?? crypto.randomUUID(), - storePath: `/nix/store/${crypto.randomUUID()}`, - narHash: `sha256:${crypto.randomUUID()}`, - narSize: 1024, - fileHash: `sha256:${crypto.randomUUID()}`, - fileSize: 512, - compression: "zstd", - referencesJson: "[]", - deriver: null, - system: null, - r2UploadKey: null, - r2UploadId: null, - expiresAt: overrides.expiresAt ?? "2099-01-01T00:00:00.000Z", - }; -} - -describe("caches", () => { - it("createCache creates a public cache with expected fields", async () => { - const name = `cache-create-${crypto.randomUUID()}`; - const row = await createCache(getDb(), name); - - expect(row.name).toBe(name); - expect(typeof row.id).toBe("number"); - expect(typeof row.createdAt).toBe("string"); - }); - - it("findCacheByName returns a matching cache", async () => { - const name = `cache-find-${crypto.randomUUID()}`; - const created = await createCache(getDb(), name); - const found = await findCacheByName(getDb(), name); - - expect(found).toBeDefined(); - expect(found!.id).toBe(created.id); - expect(found!.name).toBe(name); - }); - - it("findCacheByName returns undefined for non-existent name", async () => { - const found = await findCacheByName(getDb(), `no-such-cache-${crypto.randomUUID()}`); - expect(found).toBeUndefined(); - }); - - it("createCache with duplicate name throws", async () => { - const name = `cache-dup-${crypto.randomUUID()}`; - await createCache(getDb(), name); - - await expect(createCache(getDb(), name)).rejects.toThrow(); - }); -}); - -describe("blob_objects", () => { - it("createBlobObject creates a row with expected fields", async () => { - const fileHash = `sha256:${crypto.randomUUID()}`; - const row = await createBlobObject(getDb(), { - fileHash, - fileSize: 2048, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - - expect(row.fileHash).toBe(fileHash); - expect(row.fileSize).toBe(2048); - expect(row.compression).toBe("zstd"); - expect(typeof row.id).toBe("number"); - expect(typeof row.createdAt).toBe("string"); - }); - - it("findBlobObject returns a matching blob", async () => { - const fileHash = `sha256:${crypto.randomUUID()}`; - const created = await createBlobObject(getDb(), { - fileHash, - fileSize: 100, - compression: "none", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - const found = await findBlobObject(getDb(), fileHash, "none"); - - expect(found).toBeDefined(); - expect(found!.id).toBe(created.id); - }); - - it("findBlobObject returns undefined for non-existent", async () => { - const found = await findBlobObject(getDb(), `sha256:${crypto.randomUUID()}`, "zstd"); - expect(found).toBeUndefined(); - }); - - it("createBlobObject with same fileHash+compression throws", async () => { - const fileHash = `sha256:${crypto.randomUUID()}`; - await createBlobObject(getDb(), { - fileHash, - fileSize: 100, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - - await expect( - createBlobObject(getDb(), { - fileHash, - fileSize: 200, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }), - ).rejects.toThrow(); - }); -}); - -describe("published_paths", () => { - it("createPublishedPath creates a row with expected fields", async () => { - const cache = await createCache(getDb(), `pp-create-${crypto.randomUUID()}`); - const blob = await createBlobObject(getDb(), { - fileHash: `sha256:${crypto.randomUUID()}`, - fileSize: 500, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - const storePathHash = crypto.randomUUID(); - - const row = await createPublishedPath(getDb(), { - cacheId: cache.id, - storePathHash, - storePath: `/nix/store/${storePathHash}`, - narHash: `sha256:${crypto.randomUUID()}`, - narSize: 4096, - blobObjectId: blob.id, - referencesJson: "[]", - deriver: null, - system: "x86_64-linux", - signaturesJson: "[]", - }); - - expect(row.cacheId).toBe(cache.id); - expect(row.storePathHash).toBe(storePathHash); - expect(row.blobObjectId).toBe(blob.id); - expect(row.system).toBe("x86_64-linux"); - expect(typeof row.id).toBe("number"); - expect(typeof row.createdAt).toBe("string"); - }); - - it("findPublishedPath returns a matching path", async () => { - const cache = await createCache(getDb(), `pp-find-${crypto.randomUUID()}`); - const blob = await createBlobObject(getDb(), { - fileHash: `sha256:${crypto.randomUUID()}`, - fileSize: 500, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - const storePathHash = crypto.randomUUID(); - - const created = await createPublishedPath(getDb(), { - cacheId: cache.id, - storePathHash, - storePath: `/nix/store/${storePathHash}`, - narHash: `sha256:${crypto.randomUUID()}`, - narSize: 4096, - blobObjectId: blob.id, - referencesJson: "[]", - deriver: null, - system: null, - signaturesJson: "[]", - }); - const found = await findPublishedPath(getDb(), cache.id, storePathHash); - - expect(found).toBeDefined(); - expect(found!.id).toBe(created.id); - }); - - it("findPublishedPath returns undefined for non-existent", async () => { - const found = await findPublishedPath(getDb(), 999999, crypto.randomUUID()); - expect(found).toBeUndefined(); - }); - - it("createPublishedPath with same cache+storePathHash throws", async () => { - const cache = await createCache(getDb(), `pp-dup-${crypto.randomUUID()}`); - const blob = await createBlobObject(getDb(), { - fileHash: `sha256:${crypto.randomUUID()}`, - fileSize: 500, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - const storePathHash = crypto.randomUUID(); - const common = { - cacheId: cache.id, - storePathHash, - storePath: `/nix/store/${storePathHash}`, - narHash: `sha256:${crypto.randomUUID()}`, - narSize: 4096, - blobObjectId: blob.id, - referencesJson: "[]", - deriver: null, - system: null, - signaturesJson: "[]", - }; - - await createPublishedPath(getDb(), common); - await expect(createPublishedPath(getDb(), common)).rejects.toThrow(); - }); -}); - -describe("upload_sessions", () => { - it("createUploadSession creates a session with status pending", async () => { - const cache = await createCache(getDb(), `us-create-${crypto.randomUUID()}`); - const sessionId = crypto.randomUUID(); - const row = await createUploadSession(getDb(), makeSessionParams({ id: sessionId, cacheId: cache.id })); - - expect(row.id).toBe(sessionId); - expect(row.cacheId).toBe(cache.id); - expect(row.status).toBe("pending"); - expect(typeof row.createdAt).toBe("string"); - }); - - it("findUploadSession returns a matching session", async () => { - const cache = await createCache(getDb(), `us-find-${crypto.randomUUID()}`); - const sessionId = crypto.randomUUID(); - await createUploadSession(getDb(), makeSessionParams({ id: sessionId, cacheId: cache.id })); - const found = await findUploadSession(getDb(), sessionId); - - expect(found).toBeDefined(); - expect(found!.id).toBe(sessionId); - }); - - it("transitionSessionStatus updates and returns the session", async () => { - const cache = await createCache(getDb(), `us-update-${crypto.randomUUID()}`); - const sessionId = crypto.randomUUID(); - await createUploadSession(getDb(), makeSessionParams({ id: sessionId, cacheId: cache.id })); - - const updated = await transitionSessionStatus(getDb(), sessionId, "pending", "uploading"); - - expect(updated).toBeDefined(); - expect(updated!.status).toBe("uploading"); - expect(updated!.id).toBe(sessionId); - }); - - it("findExpiredSessions returns only expired non-completed sessions", async () => { - const cache = await createCache(getDb(), `us-expired-${crypto.randomUUID()}`); - - const expiredId = crypto.randomUUID(); - await createUploadSession( - getDb(), - makeSessionParams({ id: expiredId, cacheId: cache.id, expiresAt: "2020-01-01T00:00:00.000Z" }), - ); - - const freshId = crypto.randomUUID(); - await createUploadSession( - getDb(), - makeSessionParams({ id: freshId, cacheId: cache.id, expiresAt: "2099-01-01T00:00:00.000Z" }), - ); - - const expired = await findExpiredSessions(getDb(), new Date().toISOString()); - const expiredIds = expired.map((s) => s.id); - - expect(expiredIds).toContain(expiredId); - expect(expiredIds).not.toContain(freshId); - }); - - it("findExpiredSessions excludes completed sessions even if expired", async () => { - const cache = await createCache(getDb(), `us-exp-compl-${crypto.randomUUID()}`); - - const sessionId = crypto.randomUUID(); - await createUploadSession( - getDb(), - makeSessionParams({ id: sessionId, cacheId: cache.id, expiresAt: "2020-01-01T00:00:00.000Z" }), - ); - await transitionSessionStatus(getDb(), sessionId, "pending", "completed"); - - const expired = await findExpiredSessions(getDb(), new Date().toISOString()); - const expiredIds = expired.map((s) => s.id); - - expect(expiredIds).not.toContain(sessionId); - }); -}); - -describe("upload_parts", () => { - it("upsertUploadPart creates a part with expected fields", async () => { - const cache = await createCache(getDb(), `up-create-${crypto.randomUUID()}`); - const sessionId = crypto.randomUUID(); - await createUploadSession(getDb(), makeSessionParams({ id: sessionId, cacheId: cache.id })); - - const row = await upsertUploadPart(getDb(), { - sessionId, - partNumber: 1, - etag: "etag-abc", - size: 1024, - }); - - expect(row.sessionId).toBe(sessionId); - expect(row.partNumber).toBe(1); - expect(row.etag).toBe("etag-abc"); - expect(row.size).toBe(1024); - expect(typeof row.id).toBe("number"); - }); - - it("findUploadParts returns parts ordered by partNumber", async () => { - const cache = await createCache(getDb(), `up-find-${crypto.randomUUID()}`); - const sessionId = crypto.randomUUID(); - await createUploadSession(getDb(), makeSessionParams({ id: sessionId, cacheId: cache.id })); - - await upsertUploadPart(getDb(), { sessionId, partNumber: 3, etag: "e3", size: 300 }); - await upsertUploadPart(getDb(), { sessionId, partNumber: 1, etag: "e1", size: 100 }); - await upsertUploadPart(getDb(), { sessionId, partNumber: 2, etag: "e2", size: 200 }); - - const parts = await findUploadParts(getDb(), sessionId); - - expect(parts).toHaveLength(3); - expect(parts[0].partNumber).toBe(1); - expect(parts[1].partNumber).toBe(2); - expect(parts[2].partNumber).toBe(3); - }); -}); - -describe("gc_marks", () => { - it("createGcMark creates a mark with expected fields", async () => { - const targetId = crypto.randomUUID(); - const row = await createGcMark(getDb(), { - targetType: "blob", - targetId, - reason: "orphaned", - }); - - expect(row.targetType).toBe("blob"); - expect(row.targetId).toBe(targetId); - expect(row.reason).toBe("orphaned"); - expect(typeof row.id).toBe("number"); - expect(typeof row.markedAt).toBe("string"); - }); - - it("findGcMarks returns only marks matching targetType", async () => { - const blobId = crypto.randomUUID(); - const sessionId = crypto.randomUUID(); - - await createGcMark(getDb(), { targetType: "blob_find", targetId: blobId, reason: "orphaned" }); - await createGcMark(getDb(), { targetType: "session_find", targetId: sessionId, reason: "expired" }); - - const blobMarks = await findGcMarks(getDb(), "blob_find"); - - expect(blobMarks).toHaveLength(1); - expect(blobMarks[0].targetId).toBe(blobId); - }); - - it("deleteGcMark removes the mark", async () => { - const targetId = crypto.randomUUID(); - await createGcMark(getDb(), { targetType: "blob_del", targetId, reason: "orphaned" }); - - await deleteGcMark(getDb(), "blob_del", targetId); - - const marks = await findGcMarks(getDb(), "blob_del"); - expect(marks).toHaveLength(0); - }); - - it("createGcMark with same target_type+target_id throws", async () => { - const targetId = crypto.randomUUID(); - await createGcMark(getDb(), { targetType: "blob_idem", targetId, reason: "orphaned" }); - - await expect( - createGcMark(getDb(), { targetType: "blob_idem", targetId, reason: "orphaned again" }), - ).rejects.toThrow(); - }); -}); \ No newline at end of file diff --git a/workers/cache/test/env.d.ts b/workers/cache/test/env.d.ts deleted file mode 100644 index 3972994..0000000 --- a/workers/cache/test/env.d.ts +++ /dev/null @@ -1,7 +0,0 @@ -import type { D1Migration } from "@cloudflare/vitest-pool-workers"; - -declare module "cloudflare:test" { - interface ProvidedEnv extends Env { - TEST_MIGRATIONS: D1Migration[]; - } -} diff --git a/workers/cache/test/gc.spec.ts b/workers/cache/test/gc.spec.ts deleted file mode 100644 index 9b2d564..0000000 --- a/workers/cache/test/gc.spec.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { env } from "cloudflare:test"; -import { describe, it, expect } from "vitest"; -import { drizzle } from "drizzle-orm/d1"; -import { - createCache, - createUploadSession, - createBlobObject, - createPublishedPath, - findUploadSession, - findBlobObjectById, -} from "../src/db/repository.js"; -import { runGarbageCollection } from "../src/gc.js"; -import type { WorkerConfig } from "../src/config.js"; - -function getConfig(): WorkerConfig { - return { - bucket: env.CACHE_BUCKET, - db: drizzle(env.CACHE_DB), - }; -} - -function makeSessionParams(overrides: { - readonly id: string; - readonly cacheId: number; - readonly expiresAt?: string; - readonly r2UploadKey?: string | null; -}) { - return { - id: overrides.id, - cacheId: overrides.cacheId, - storePathHash: crypto.randomUUID(), - storePath: `/nix/store/${crypto.randomUUID()}`, - narHash: `sha256:${crypto.randomUUID()}`, - narSize: 1024, - fileHash: `sha256:${crypto.randomUUID()}`, - fileSize: 512, - compression: "zstd", - referencesJson: "[]", - deriver: null, - system: null, - r2UploadKey: overrides.r2UploadKey ?? null, - r2UploadId: null, - expiresAt: overrides.expiresAt ?? "2099-01-01T00:00:00.000Z", - }; -} - -describe("gc: expire abandoned sessions", () => { - it("expires and deletes sessions past their expiresAt", async () => { - const config = getConfig(); - const cache = await createCache(config.db, `gc-expire-${crypto.randomUUID()}`); - const sessionId = crypto.randomUUID(); - await createUploadSession( - config.db, - makeSessionParams({ id: sessionId, cacheId: cache.id, expiresAt: "2020-01-01T00:00:00.000Z" }), - ); - - const result = await runGarbageCollection(config); - - expect(result.expiredSessions).toBeGreaterThanOrEqual(1); - expect(result.errors).toHaveLength(0); - - const found = await findUploadSession(config.db, sessionId); - expect(found).toBeUndefined(); - }); -}); - -describe("gc: delete unreferenced blobs", () => { - it("deletes blob objects not referenced by any published path", async () => { - const config = getConfig(); - const blob = await createBlobObject(config.db, { - fileHash: `sha256:${crypto.randomUUID()}`, - fileSize: 2048, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - - const result = await runGarbageCollection(config); - - expect(result.deletedBlobs).toBeGreaterThanOrEqual(1); - expect(result.errors).toHaveLength(0); - - const found = await findBlobObjectById(config.db, blob.id); - expect(found).toBeUndefined(); - }); -}); - -describe("gc: keep referenced blobs", () => { - it("preserves blob objects that are referenced by a published path", async () => { - const config = getConfig(); - const cache = await createCache(config.db, `gc-keep-${crypto.randomUUID()}`); - const blob = await createBlobObject(config.db, { - fileHash: `sha256:${crypto.randomUUID()}`, - fileSize: 2048, - compression: "zstd", - r2Key: `blobs/${crypto.randomUUID()}`, - }); - const storePathHash = crypto.randomUUID(); - await createPublishedPath(config.db, { - cacheId: cache.id, - storePathHash, - storePath: `/nix/store/${storePathHash}`, - narHash: `sha256:${crypto.randomUUID()}`, - narSize: 4096, - blobObjectId: blob.id, - referencesJson: "[]", - deriver: null, - system: null, - signaturesJson: "[]", - }); - - const result = await runGarbageCollection(config); - - const found = await findBlobObjectById(config.db, blob.id); - expect(found).toBeDefined(); - }); -}); - -describe("gc: keep active sessions", () => { - it("does not expire sessions with future expiresAt", async () => { - const config = getConfig(); - const cache = await createCache(config.db, `gc-active-${crypto.randomUUID()}`); - const sessionId = crypto.randomUUID(); - await createUploadSession( - config.db, - makeSessionParams({ id: sessionId, cacheId: cache.id, expiresAt: "2099-01-01T00:00:00.000Z" }), - ); - - const result = await runGarbageCollection(config); - - expect(result.expiredSessions).toBe(0); - - const found = await findUploadSession(config.db, sessionId); - expect(found).toBeDefined(); - }); -}); diff --git a/workers/cache/test/index.spec.ts b/workers/cache/test/index.spec.ts deleted file mode 100644 index eea5bd0..0000000 --- a/workers/cache/test/index.spec.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { - env, - createExecutionContext, - waitOnExecutionContext, - SELF, -} from "cloudflare:test"; -import { describe, it, expect } from "vitest"; -import worker from "../src"; - -describe("cache worker", () => { - it("serves service info on /", async () => { - const request = new Request( - "http://example.com/" - ); - const ctx = createExecutionContext(); - const response = await worker.fetch(request, env, ctx); - - await waitOnExecutionContext(ctx); - - expect(response.status).toBe(200); - expect(await response.json()).toEqual({ - service: "helios-cache", - status: "ok", - }); - }); - - it("serves a health check on /healthz", async () => { - const request = new Request("http://example.com/healthz"); - const response = await SELF.fetch(request); - - expect(response.status).toBe(200); - const body = await response.json<{ ok: boolean; service: string }>(); - expect(body.service).toBe("helios-cache"); - expect(body.ok).toBe(true); - }); - - it("rejects unsupported methods on cache paths", async () => { - const request = new Request("http://example.com/mycache/test.narinfo", { method: "POST" }); - const response = await SELF.fetch(request); - - expect(response.status).toBe(405); - expect(response.headers.get("allow")).toBe("GET, HEAD"); - }); - - it("returns 404 for unknown paths", async () => { - const request = new Request("http://example.com/nonexistent"); - const response = await SELF.fetch(request); - - expect(response.status).toBe(404); - }); -}); diff --git a/workers/cache/test/read-path.spec.ts b/workers/cache/test/read-path.spec.ts deleted file mode 100644 index 9b8994f..0000000 --- a/workers/cache/test/read-path.spec.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { env, SELF } from "cloudflare:test"; -import { describe, it, expect, beforeAll } from "vitest"; -import { drizzle } from "drizzle-orm/d1"; -import { createCache, createBlobObject, createPublishedPath } from "../src/db/repository.js"; -import { handleRequest } from "../src/router.js"; -import { resolveConfig } from "../src/config.js"; - -const STORE_PATH_HASH = "0".repeat(32); -const FILE_HASH = "a".repeat(64); -const NAR_HASH = `sha256:${"1".repeat(52)}`; -const R2_KEY = `nars/sha256/${FILE_HASH}/zstd.nar`; -const R2_BODY = new Uint8Array([1, 2, 3, 4]); - -function getDb() { - return drizzle(env.CACHE_DB); -} - -beforeAll(async () => { - const db = getDb(); - - const cache = await createCache(db, "test-cache"); - - const blob = await createBlobObject(db, { - fileHash: FILE_HASH, - fileSize: 1024, - compression: "zstd", - r2Key: R2_KEY, - }); - - await createPublishedPath(db, { - cacheId: cache.id, - storePathHash: STORE_PATH_HASH, - storePath: `/nix/store/${STORE_PATH_HASH}-hello`, - narHash: NAR_HASH, - narSize: 2048, - blobObjectId: blob.id, - referencesJson: "[]", - deriver: null, - system: null, - signaturesJson: "[]", - }); - - await env.CACHE_BUCKET.put(R2_KEY, R2_BODY); -}); - -describe("nix-cache-info", () => { - it("returns cache info for an existing cache", async () => { - const res = await SELF.fetch("http://example.com/test-cache/nix-cache-info"); - - expect(res.status).toBe(200); - expect(res.headers.get("content-type")).toBe("text/x-nix-cache-info"); - const body = await res.text(); - expect(body).toContain("StoreDir: /nix/store"); - }); - - it("returns 200 for any cache path", async () => { - const res = await SELF.fetch("http://example.com/nonexistent/nix-cache-info"); - - expect(res.status).toBe(200); - }); -}); - -describe("narinfo", () => { - it("returns narinfo for a published path", async () => { - const res = await SELF.fetch(`http://example.com/test-cache/${STORE_PATH_HASH}.narinfo`); - - expect(res.status).toBe(200); - expect(res.headers.get("content-type")).toBe("text/x-nix-narinfo"); - const body = await res.text(); - expect(body).toContain("StorePath:"); - expect(body).toContain("NarHash:"); - expect(body).toContain("FileHash:"); - expect(body).toContain("Compression: zstd"); - expect(body).toContain("URL: nar/"); - }); - - it("HEAD returns 200 with headers but no body", async () => { - const request = new Request(`http://example.com/test-cache/${STORE_PATH_HASH}.narinfo`, { - method: "HEAD", - }); - const config = resolveConfig(env); - const res = await handleRequest(request, config); - - expect(res.status).toBe(200); - expect(res.headers.get("content-type")).toBe("text/x-nix-narinfo"); - expect(res.headers.get("content-length")).toBeTruthy(); - expect(res.headers.get("cache-control")).toContain("immutable"); - const body = await res.text(); - expect(body).toBe(""); - }); - - it("returns 404 for a valid hash that is not published", async () => { - const res = await SELF.fetch(`http://example.com/test-cache/${"a".repeat(32)}.narinfo`); - - expect(res.status).toBe(404); - }); - - it("returns 404 for an invalid hash format", async () => { - const res = await SELF.fetch("http://example.com/test-cache/INVALID.narinfo"); - - expect(res.status).toBe(404); - }); - - it("returns 404 when the cache does not exist", async () => { - const res = await SELF.fetch(`http://example.com/noexist/${STORE_PATH_HASH}.narinfo`); - - expect(res.status).toBe(404); - }); -}); - -describe("nar download", () => { - it("returns the nar blob for a valid request", async () => { - const res = await SELF.fetch(`http://example.com/test-cache/nar/${FILE_HASH}/zstd.nar`); - - expect(res.status).toBe(200); - expect(res.headers.get("content-type")).toBe("application/x-nix-nar"); - expect(res.headers.get("cache-control")).toContain("immutable"); - const body = new Uint8Array(await res.arrayBuffer()); - expect(body).toEqual(R2_BODY); - }); - - it("returns 404 for a non-existent blob", async () => { - const res = await SELF.fetch(`http://example.com/test-cache/nar/${"b".repeat(64)}/zstd.nar`); - - expect(res.status).toBe(404); - }); - - it("returns 404 for an invalid file hash", async () => { - const res = await SELF.fetch("http://example.com/test-cache/nar/invalid/zstd.nar"); - - expect(res.status).toBe(404); - }); - - it("returns 404 for an invalid compression", async () => { - const res = await SELF.fetch(`http://example.com/test-cache/nar/${FILE_HASH}/gzip.nar`); - - expect(res.status).toBe(404); - }); -}); diff --git a/workers/cache/test/tsconfig.json b/workers/cache/test/tsconfig.json deleted file mode 100644 index 978ecd8..0000000 --- a/workers/cache/test/tsconfig.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "extends": "../tsconfig.json", - "compilerOptions": { - "types": ["@cloudflare/vitest-pool-workers"] - }, - "include": ["./**/*.ts", "../worker-configuration.d.ts"], - "exclude": [] -} diff --git a/workers/cache/test/upload-flow.spec.ts b/workers/cache/test/upload-flow.spec.ts deleted file mode 100644 index 775e6a8..0000000 --- a/workers/cache/test/upload-flow.spec.ts +++ /dev/null @@ -1,241 +0,0 @@ -import { env, SELF } from "cloudflare:test"; -import { describe, it, expect, beforeAll } from "vitest"; -import { drizzle } from "drizzle-orm/d1"; -import { createCache, findPublishedPath, findCacheByName } from "../src/db/repository.js"; - -const CACHE_NAME = "upload-test"; -const AUTH_TOKEN = "test-auth-token"; -const BLOB_DATA = new Uint8Array([1, 2, 3, 4]); - -function getDb() { - return drizzle(env.CACHE_DB); -} - -function post(path: string, body: unknown): Promise { - return SELF.fetch(`http://example.com${path}`, { - method: "POST", - headers: { - "content-type": "application/json", - authorization: `Bearer ${AUTH_TOKEN}`, - }, - body: JSON.stringify(body), - }); -} - -function sessionBody(overrides: { fileHash: string; storePathHash: string }) { - return { - storePath: `/nix/store/${overrides.storePathHash}-hello`, - storePathHash: overrides.storePathHash, - narHash: `sha256:${"1".repeat(52)}`, - narSize: 2048, - fileHash: overrides.fileHash, - fileSize: BLOB_DATA.byteLength, - compression: "zstd", - }; -} - -function uniqueFileHash() { - return crypto.randomUUID().replace(/-/g, "").slice(0, 64).padEnd(64, "0"); -} - -let storePathCounter = 1; -function uniqueStorePathHash() { - const c = String(storePathCounter++); - return c.repeat(32).slice(0, 32); -} - -async function createSessionAndUpload(fileHash: string, storePathHash: string) { - const createRes = await post( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - sessionBody({ fileHash, storePathHash }), - ); - expect(createRes.status).toBe(201); - const session = await createRes.json<{ sessionId: string; r2Key: string }>(); - - const blobRes = await SELF.fetch(`http://example.com/_api/v1/uploads/${session.sessionId}/blob`, { - method: "PUT", - headers: { authorization: `Bearer ${AUTH_TOKEN}` }, - body: BLOB_DATA, - }); - expect(blobRes.status).toBe(200); - - const completeRes = await post(`/_api/v1/uploads/${session.sessionId}/complete`, {}); - expect(completeRes.status).toBe(200); - - return session; -} - -beforeAll(async () => { - const db = getDb(); - await createCache(db, CACHE_NAME); -}); - -describe("create upload session", () => { - it("creates a session and returns 201 with expected fields", async () => { - const res = await post( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - sessionBody({ fileHash: uniqueFileHash(), storePathHash: uniqueStorePathHash() }), - ); - - expect(res.status).toBe(201); - const body = await res.json>(); - expect(body).toHaveProperty("sessionId"); - expect(body).toHaveProperty("r2Key"); - expect(body).toHaveProperty("uploadMethod", "direct"); - expect(body).toHaveProperty("expiresAt"); - }); - - it("returns 404 for non-existent cache", async () => { - const res = await post( - "/_api/v1/caches/no-such-cache/upload-sessions", - sessionBody({ fileHash: uniqueFileHash(), storePathHash: uniqueStorePathHash() }), - ); - - expect(res.status).toBe(404); - }); - - it("returns 400 for missing required fields", async () => { - const res = await post(`/_api/v1/caches/${CACHE_NAME}/upload-sessions`, {}); - - expect(res.status).toBe(400); - }); -}); - -describe("complete upload (direct)", () => { - it("completes a direct upload flow", async () => { - const fileHash = uniqueFileHash(); - const storePathHash = uniqueStorePathHash(); - - const createRes = await post( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - sessionBody({ fileHash, storePathHash }), - ); - expect(createRes.status).toBe(201); - const session = await createRes.json<{ sessionId: string; r2Key: string }>(); - - const blobRes = await SELF.fetch(`http://example.com/_api/v1/uploads/${session.sessionId}/blob`, { - method: "PUT", - headers: { authorization: `Bearer ${AUTH_TOKEN}` }, - body: BLOB_DATA, - }); - expect(blobRes.status).toBe(200); - - const completeRes = await post(`/_api/v1/uploads/${session.sessionId}/complete`, {}); - expect(completeRes.status).toBe(200); - const completeBody = await completeRes.json<{ status: string }>(); - expect(completeBody.status).toBe("completed"); - }); - - it("returns 404 for non-existent session", async () => { - const res = await post("/_api/v1/uploads/00000000-0000-0000-0000-000000000000/complete", {}); - - expect(res.status).toBe(404); - }); - - it("returns 409 for already-completed session", async () => { - const session = await createSessionAndUpload(uniqueFileHash(), uniqueStorePathHash()); - - const res = await post(`/_api/v1/uploads/${session.sessionId}/complete`, {}); - expect(res.status).toBe(409); - }); - - it("returns 409 when session has not uploaded a blob yet", async () => { - const createRes = await post( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - sessionBody({ fileHash: uniqueFileHash(), storePathHash: uniqueStorePathHash() }), - ); - expect(createRes.status).toBe(201); - const session = await createRes.json<{ sessionId: string }>(); - - const res = await post(`/_api/v1/uploads/${session.sessionId}/complete`, {}); - expect(res.status).toBe(409); - }); -}); - -describe("publish", () => { - it("publishes a completed upload and path is readable", async () => { - const fileHash = uniqueFileHash(); - const storePathHash = uniqueStorePathHash(); - const session = await createSessionAndUpload(fileHash, storePathHash); - - const publishRes = await post(`/_api/v1/uploads/${session.sessionId}/publish`, {}); - expect(publishRes.status).toBe(200); - const publishBody = await publishRes.json<{ published: boolean; storePathHash: string }>(); - expect(publishBody.published).toBe(true); - expect(publishBody.storePathHash).toBe(storePathHash); - - const db = getDb(); - const cache = await findCacheByName(db, CACHE_NAME); - expect(cache).toBeDefined(); - const published = await findPublishedPath(db, cache!.id, storePathHash); - expect(published).toBeDefined(); - }); - - it("returns 409 for uncompleted session", async () => { - const createRes = await post( - `/_api/v1/caches/${CACHE_NAME}/upload-sessions`, - sessionBody({ fileHash: uniqueFileHash(), storePathHash: uniqueStorePathHash() }), - ); - expect(createRes.status).toBe(201); - const session = await createRes.json<{ sessionId: string }>(); - - const res = await post(`/_api/v1/uploads/${session.sessionId}/publish`, {}); - expect(res.status).toBe(409); - }); - - it("concurrent duplicate publish does not create duplicate paths", async () => { - const session = await createSessionAndUpload(uniqueFileHash(), uniqueStorePathHash()); - - const [r1, r2] = await Promise.all([ - post(`/_api/v1/uploads/${session.sessionId}/publish`, {}), - post(`/_api/v1/uploads/${session.sessionId}/publish`, {}), - ]); - - expect(r1.status).toBe(200); - expect(r2.status).toBe(200); - const b1 = await r1.json<{ published: boolean; alreadyExisted?: boolean }>(); - const b2 = await r2.json<{ published: boolean; alreadyExisted?: boolean }>(); - expect(b1.published).toBe(true); - expect(b2.published).toBe(true); - }); - - it("duplicate publish is idempotent", async () => { - const session = await createSessionAndUpload(uniqueFileHash(), uniqueStorePathHash()); - - const first = await post(`/_api/v1/uploads/${session.sessionId}/publish`, {}); - expect(first.status).toBe(200); - - const second = await post(`/_api/v1/uploads/${session.sessionId}/publish`, {}); - expect(second.status).toBe(200); - const body = await second.json<{ alreadyExisted: boolean }>(); - expect(body.alreadyExisted).toBe(true); - }); -}); - -describe("get-missing-paths", () => { - it("returns only unpublished paths as missing", async () => { - const storePathHash = uniqueStorePathHash(); - const session = await createSessionAndUpload(uniqueFileHash(), storePathHash); - await post(`/_api/v1/uploads/${session.sessionId}/publish`, {}); - - const randomHash = "x".repeat(32).replace(/x/g, () => "abcdfg0123456789"[Math.floor(Math.random() * 16)]); - - const res = await post("/_api/v1/get-missing-paths", { - cache: CACHE_NAME, - storePathHashes: [storePathHash, randomHash], - }); - expect(res.status).toBe(200); - const body = await res.json<{ missing: string[] }>(); - expect(body.missing).not.toContain(storePathHash); - expect(body.missing).toContain(randomHash); - }); - - it("returns 404 for non-existent cache", async () => { - const res = await post("/_api/v1/get-missing-paths", { - cache: "nonexistent-cache", - storePathHashes: ["0".repeat(32)], - }); - - expect(res.status).toBe(404); - }); -}); diff --git a/workers/cache/tsconfig.json b/workers/cache/tsconfig.json deleted file mode 100644 index 9c0eaa5..0000000 --- a/workers/cache/tsconfig.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "compilerOptions": { - "target": "es2024", - "lib": ["es2024", "webworker"], - "module": "es2022", - "moduleResolution": "Bundler", - "resolveJsonModule": true, - "noEmit": true, - "isolatedModules": true, - "allowSyntheticDefaultImports": true, - "forceConsistentCasingInFileNames": true, - "strict": true, - "skipLibCheck": true, - "types": ["./worker-configuration.d.ts"] - }, - "exclude": ["test"], - "include": ["worker-configuration.d.ts", "src/**/*.ts"] -} diff --git a/workers/cache/vitest.config.mts b/workers/cache/vitest.config.mts deleted file mode 100644 index f3d3fe2..0000000 --- a/workers/cache/vitest.config.mts +++ /dev/null @@ -1,24 +0,0 @@ -import path from 'node:path'; -import { defineWorkersConfig, readD1Migrations } from '@cloudflare/vitest-pool-workers/config'; - -export default defineWorkersConfig(async () => { - const migrationsPath = path.join(import.meta.dirname, 'migrations'); - const migrations = await readD1Migrations(migrationsPath); - - return { - test: { - setupFiles: ['./test/apply-migrations.ts'], - poolOptions: { - workers: { - wrangler: { configPath: './wrangler.jsonc' }, - miniflare: { - bindings: { - TEST_MIGRATIONS: migrations, - AUTH_TOKEN: "test-auth-token", - }, - }, - }, - }, - }, - }; -}); diff --git a/workers/cache/worker-configuration.d.ts b/workers/cache/worker-configuration.d.ts deleted file mode 100644 index bd64b48..0000000 --- a/workers/cache/worker-configuration.d.ts +++ /dev/null @@ -1,13542 +0,0 @@ -/* eslint-disable */ -// Generated by Wrangler by running `wrangler types` (hash: b1675f5d714a0a73a72c63752a4938cd) -// Runtime types generated with workerd@1.20260421.1 2026-03-10 -declare namespace Cloudflare { - interface GlobalProps { - mainModule: typeof import("./src/index"); - } - interface Env { - CACHE_BUCKET: R2Bucket; - CACHE_DB: D1Database; - SIGNING_KEY_NAME: "helios-cache-1"; - } -} -interface Env extends Cloudflare.Env {} - -// Begin runtime types -/*! ***************************************************************************** -Copyright (c) Cloudflare. All rights reserved. -Copyright (c) Microsoft Corporation. All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use -this file except in compliance with the License. You may obtain a copy of the -License at http://www.apache.org/licenses/LICENSE-2.0 -THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED -WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -MERCHANTABLITY OR NON-INFRINGEMENT. -See the Apache Version 2.0 License for specific language governing permissions -and limitations under the License. -***************************************************************************** */ -/* eslint-disable */ -// noinspection JSUnusedGlobalSymbols -declare var onmessage: never; -/** - * The **`DOMException`** interface represents an abnormal event (called an **exception**) that occurs as a result of calling a method or accessing a property of a web API. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException) - */ -declare class DOMException extends Error { - constructor(message?: string, name?: string); - /** - * The **`message`** read-only property of the a message or description associated with the given error name. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException/message) - */ - readonly message: string; - /** - * The **`name`** read-only property of the one of the strings associated with an error name. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException/name) - */ - readonly name: string; - /** - * The **`code`** read-only property of the DOMException interface returns one of the legacy error code constants, or `0` if none match. - * @deprecated - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DOMException/code) - */ - readonly code: number; - static readonly INDEX_SIZE_ERR: number; - static readonly DOMSTRING_SIZE_ERR: number; - static readonly HIERARCHY_REQUEST_ERR: number; - static readonly WRONG_DOCUMENT_ERR: number; - static readonly INVALID_CHARACTER_ERR: number; - static readonly NO_DATA_ALLOWED_ERR: number; - static readonly NO_MODIFICATION_ALLOWED_ERR: number; - static readonly NOT_FOUND_ERR: number; - static readonly NOT_SUPPORTED_ERR: number; - static readonly INUSE_ATTRIBUTE_ERR: number; - static readonly INVALID_STATE_ERR: number; - static readonly SYNTAX_ERR: number; - static readonly INVALID_MODIFICATION_ERR: number; - static readonly NAMESPACE_ERR: number; - static readonly INVALID_ACCESS_ERR: number; - static readonly VALIDATION_ERR: number; - static readonly TYPE_MISMATCH_ERR: number; - static readonly SECURITY_ERR: number; - static readonly NETWORK_ERR: number; - static readonly ABORT_ERR: number; - static readonly URL_MISMATCH_ERR: number; - static readonly QUOTA_EXCEEDED_ERR: number; - static readonly TIMEOUT_ERR: number; - static readonly INVALID_NODE_TYPE_ERR: number; - static readonly DATA_CLONE_ERR: number; - get stack(): any; - set stack(value: any); -} -type WorkerGlobalScopeEventMap = { - fetch: FetchEvent; - scheduled: ScheduledEvent; - queue: QueueEvent; - unhandledrejection: PromiseRejectionEvent; - rejectionhandled: PromiseRejectionEvent; -}; -declare abstract class WorkerGlobalScope extends EventTarget { - EventTarget: typeof EventTarget; -} -/* The **`console`** object provides access to the debugging console (e.g., the Web console in Firefox). * - * The **`console`** object provides access to the debugging console (e.g., the Web console in Firefox). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console) - */ -interface Console { - "assert"(condition?: boolean, ...data: any[]): void; - /** - * The **`console.clear()`** static method clears the console if possible. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/clear_static) - */ - clear(): void; - /** - * The **`console.count()`** static method logs the number of times that this particular call to `count()` has been called. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/count_static) - */ - count(label?: string): void; - /** - * The **`console.countReset()`** static method resets counter used with console/count_static. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/countReset_static) - */ - countReset(label?: string): void; - /** - * The **`console.debug()`** static method outputs a message to the console at the 'debug' log level. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/debug_static) - */ - debug(...data: any[]): void; - /** - * The **`console.dir()`** static method displays a list of the properties of the specified JavaScript object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/dir_static) - */ - dir(item?: any, options?: any): void; - /** - * The **`console.dirxml()`** static method displays an interactive tree of the descendant elements of the specified XML/HTML element. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/dirxml_static) - */ - dirxml(...data: any[]): void; - /** - * The **`console.error()`** static method outputs a message to the console at the 'error' log level. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/error_static) - */ - error(...data: any[]): void; - /** - * The **`console.group()`** static method creates a new inline group in the Web console log, causing any subsequent console messages to be indented by an additional level, until console/groupEnd_static is called. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/group_static) - */ - group(...data: any[]): void; - /** - * The **`console.groupCollapsed()`** static method creates a new inline group in the console. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/groupCollapsed_static) - */ - groupCollapsed(...data: any[]): void; - /** - * The **`console.groupEnd()`** static method exits the current inline group in the console. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/groupEnd_static) - */ - groupEnd(): void; - /** - * The **`console.info()`** static method outputs a message to the console at the 'info' log level. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/info_static) - */ - info(...data: any[]): void; - /** - * The **`console.log()`** static method outputs a message to the console. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/log_static) - */ - log(...data: any[]): void; - /** - * The **`console.table()`** static method displays tabular data as a table. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/table_static) - */ - table(tabularData?: any, properties?: string[]): void; - /** - * The **`console.time()`** static method starts a timer you can use to track how long an operation takes. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/time_static) - */ - time(label?: string): void; - /** - * The **`console.timeEnd()`** static method stops a timer that was previously started by calling console/time_static. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/timeEnd_static) - */ - timeEnd(label?: string): void; - /** - * The **`console.timeLog()`** static method logs the current value of a timer that was previously started by calling console/time_static. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/timeLog_static) - */ - timeLog(label?: string, ...data: any[]): void; - timeStamp(label?: string): void; - /** - * The **`console.trace()`** static method outputs a stack trace to the console. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/trace_static) - */ - trace(...data: any[]): void; - /** - * The **`console.warn()`** static method outputs a warning message to the console at the 'warning' log level. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/console/warn_static) - */ - warn(...data: any[]): void; -} -declare const console: Console; -type BufferSource = ArrayBufferView | ArrayBuffer; -type TypedArray = Int8Array | Uint8Array | Uint8ClampedArray | Int16Array | Uint16Array | Int32Array | Uint32Array | Float32Array | Float64Array | BigInt64Array | BigUint64Array; -declare namespace WebAssembly { - class CompileError extends Error { - constructor(message?: string); - } - class RuntimeError extends Error { - constructor(message?: string); - } - type ValueType = "anyfunc" | "externref" | "f32" | "f64" | "i32" | "i64" | "v128"; - interface GlobalDescriptor { - value: ValueType; - mutable?: boolean; - } - class Global { - constructor(descriptor: GlobalDescriptor, value?: any); - value: any; - valueOf(): any; - } - type ImportValue = ExportValue | number; - type ModuleImports = Record; - type Imports = Record; - type ExportValue = Function | Global | Memory | Table; - type Exports = Record; - class Instance { - constructor(module: Module, imports?: Imports); - readonly exports: Exports; - } - interface MemoryDescriptor { - initial: number; - maximum?: number; - shared?: boolean; - } - class Memory { - constructor(descriptor: MemoryDescriptor); - readonly buffer: ArrayBuffer; - grow(delta: number): number; - } - type ImportExportKind = "function" | "global" | "memory" | "table"; - interface ModuleExportDescriptor { - kind: ImportExportKind; - name: string; - } - interface ModuleImportDescriptor { - kind: ImportExportKind; - module: string; - name: string; - } - abstract class Module { - static customSections(module: Module, sectionName: string): ArrayBuffer[]; - static exports(module: Module): ModuleExportDescriptor[]; - static imports(module: Module): ModuleImportDescriptor[]; - } - type TableKind = "anyfunc" | "externref"; - interface TableDescriptor { - element: TableKind; - initial: number; - maximum?: number; - } - class Table { - constructor(descriptor: TableDescriptor, value?: any); - readonly length: number; - get(index: number): any; - grow(delta: number, value?: any): number; - set(index: number, value?: any): void; - } - function instantiate(module: Module, imports?: Imports): Promise; - function validate(bytes: BufferSource): boolean; -} -/** - * The **`ServiceWorkerGlobalScope`** interface of the Service Worker API represents the global execution context of a service worker. - * Available only in secure contexts. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ServiceWorkerGlobalScope) - */ -interface ServiceWorkerGlobalScope extends WorkerGlobalScope { - DOMException: typeof DOMException; - WorkerGlobalScope: typeof WorkerGlobalScope; - btoa(data: string): string; - atob(data: string): string; - setTimeout(callback: (...args: any[]) => void, msDelay?: number): number; - setTimeout(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; - clearTimeout(timeoutId: number | null): void; - setInterval(callback: (...args: any[]) => void, msDelay?: number): number; - setInterval(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; - clearInterval(timeoutId: number | null): void; - queueMicrotask(task: Function): void; - structuredClone(value: T, options?: StructuredSerializeOptions): T; - reportError(error: any): void; - fetch(input: RequestInfo | URL, init?: RequestInit): Promise; - self: ServiceWorkerGlobalScope; - crypto: Crypto; - caches: CacheStorage; - scheduler: Scheduler; - performance: Performance; - Cloudflare: Cloudflare; - readonly origin: string; - Event: typeof Event; - ExtendableEvent: typeof ExtendableEvent; - CustomEvent: typeof CustomEvent; - PromiseRejectionEvent: typeof PromiseRejectionEvent; - FetchEvent: typeof FetchEvent; - TailEvent: typeof TailEvent; - TraceEvent: typeof TailEvent; - ScheduledEvent: typeof ScheduledEvent; - MessageEvent: typeof MessageEvent; - CloseEvent: typeof CloseEvent; - ReadableStreamDefaultReader: typeof ReadableStreamDefaultReader; - ReadableStreamBYOBReader: typeof ReadableStreamBYOBReader; - ReadableStream: typeof ReadableStream; - WritableStream: typeof WritableStream; - WritableStreamDefaultWriter: typeof WritableStreamDefaultWriter; - TransformStream: typeof TransformStream; - ByteLengthQueuingStrategy: typeof ByteLengthQueuingStrategy; - CountQueuingStrategy: typeof CountQueuingStrategy; - ErrorEvent: typeof ErrorEvent; - MessageChannel: typeof MessageChannel; - MessagePort: typeof MessagePort; - EventSource: typeof EventSource; - ReadableStreamBYOBRequest: typeof ReadableStreamBYOBRequest; - ReadableStreamDefaultController: typeof ReadableStreamDefaultController; - ReadableByteStreamController: typeof ReadableByteStreamController; - WritableStreamDefaultController: typeof WritableStreamDefaultController; - TransformStreamDefaultController: typeof TransformStreamDefaultController; - CompressionStream: typeof CompressionStream; - DecompressionStream: typeof DecompressionStream; - TextEncoderStream: typeof TextEncoderStream; - TextDecoderStream: typeof TextDecoderStream; - Headers: typeof Headers; - Body: typeof Body; - Request: typeof Request; - Response: typeof Response; - WebSocket: typeof WebSocket; - WebSocketPair: typeof WebSocketPair; - WebSocketRequestResponsePair: typeof WebSocketRequestResponsePair; - AbortController: typeof AbortController; - AbortSignal: typeof AbortSignal; - TextDecoder: typeof TextDecoder; - TextEncoder: typeof TextEncoder; - navigator: Navigator; - Navigator: typeof Navigator; - URL: typeof URL; - URLSearchParams: typeof URLSearchParams; - URLPattern: typeof URLPattern; - Blob: typeof Blob; - File: typeof File; - FormData: typeof FormData; - Crypto: typeof Crypto; - SubtleCrypto: typeof SubtleCrypto; - CryptoKey: typeof CryptoKey; - CacheStorage: typeof CacheStorage; - Cache: typeof Cache; - FixedLengthStream: typeof FixedLengthStream; - IdentityTransformStream: typeof IdentityTransformStream; - HTMLRewriter: typeof HTMLRewriter; -} -declare function addEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetAddEventListenerOptions | boolean): void; -declare function removeEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetEventListenerOptions | boolean): void; -/** - * The **`dispatchEvent()`** method of the EventTarget sends an Event to the object, (synchronously) invoking the affected event listeners in the appropriate order. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/dispatchEvent) - */ -declare function dispatchEvent(event: WorkerGlobalScopeEventMap[keyof WorkerGlobalScopeEventMap]): boolean; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/btoa) */ -declare function btoa(data: string): string; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/atob) */ -declare function atob(data: string): string; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setTimeout) */ -declare function setTimeout(callback: (...args: any[]) => void, msDelay?: number): number; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setTimeout) */ -declare function setTimeout(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/clearTimeout) */ -declare function clearTimeout(timeoutId: number | null): void; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setInterval) */ -declare function setInterval(callback: (...args: any[]) => void, msDelay?: number): number; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/setInterval) */ -declare function setInterval(callback: (...args: Args) => void, msDelay?: number, ...args: Args): number; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/clearInterval) */ -declare function clearInterval(timeoutId: number | null): void; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/queueMicrotask) */ -declare function queueMicrotask(task: Function): void; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/structuredClone) */ -declare function structuredClone(value: T, options?: StructuredSerializeOptions): T; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/reportError) */ -declare function reportError(error: any): void; -/* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Window/fetch) */ -declare function fetch(input: RequestInfo | URL, init?: RequestInit): Promise; -declare const self: ServiceWorkerGlobalScope; -/** -* The Web Crypto API provides a set of low-level functions for common cryptographic tasks. -* The Workers runtime implements the full surface of this API, but with some differences in -* the [supported algorithms](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/#supported-algorithms) -* compared to those implemented in most browsers. -* -* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/) -*/ -declare const crypto: Crypto; -/** -* The Cache API allows fine grained control of reading and writing from the Cloudflare global network cache. -* -* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/) -*/ -declare const caches: CacheStorage; -declare const scheduler: Scheduler; -/** -* The Workers runtime supports a subset of the Performance API, used to measure timing and performance, -* as well as timing of subrequests and other operations. -* -* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/) -*/ -declare const performance: Performance; -declare const Cloudflare: Cloudflare; -declare const origin: string; -declare const navigator: Navigator; -interface TestController { -} -interface ExecutionContext { - waitUntil(promise: Promise): void; - passThroughOnException(): void; - readonly exports: Cloudflare.Exports; - readonly props: Props; - cache?: CacheContext; -} -type ExportedHandlerFetchHandler = (request: Request>, env: Env, ctx: ExecutionContext) => Response | Promise; -type ExportedHandlerConnectHandler = (socket: Socket, env: Env, ctx: ExecutionContext) => void | Promise; -type ExportedHandlerTailHandler = (events: TraceItem[], env: Env, ctx: ExecutionContext) => void | Promise; -type ExportedHandlerTraceHandler = (traces: TraceItem[], env: Env, ctx: ExecutionContext) => void | Promise; -type ExportedHandlerTailStreamHandler = (event: TailStream.TailEvent, env: Env, ctx: ExecutionContext) => TailStream.TailEventHandlerType | Promise; -type ExportedHandlerScheduledHandler = (controller: ScheduledController, env: Env, ctx: ExecutionContext) => void | Promise; -type ExportedHandlerQueueHandler = (batch: MessageBatch, env: Env, ctx: ExecutionContext) => void | Promise; -type ExportedHandlerTestHandler = (controller: TestController, env: Env, ctx: ExecutionContext) => void | Promise; -interface ExportedHandler { - fetch?: ExportedHandlerFetchHandler; - connect?: ExportedHandlerConnectHandler; - tail?: ExportedHandlerTailHandler; - trace?: ExportedHandlerTraceHandler; - tailStream?: ExportedHandlerTailStreamHandler; - scheduled?: ExportedHandlerScheduledHandler; - test?: ExportedHandlerTestHandler; - email?: EmailExportedHandler; - queue?: ExportedHandlerQueueHandler; -} -interface StructuredSerializeOptions { - transfer?: any[]; -} -declare abstract class Navigator { - sendBeacon(url: string, body?: BodyInit): boolean; - readonly userAgent: string; - readonly hardwareConcurrency: number; - readonly platform: string; - readonly language: string; - readonly languages: string[]; -} -interface AlarmInvocationInfo { - readonly isRetry: boolean; - readonly retryCount: number; - readonly scheduledTime: number; -} -interface Cloudflare { - readonly compatibilityFlags: Record; -} -interface CachePurgeError { - code: number; - message: string; -} -interface CachePurgeResult { - success: boolean; - errors: CachePurgeError[]; -} -interface CachePurgeOptions { - tags?: string[]; - pathPrefixes?: string[]; - purgeEverything?: boolean; -} -interface CacheContext { - purge(options: CachePurgeOptions): Promise; -} -declare abstract class ColoLocalActorNamespace { - get(actorId: string): Fetcher; -} -interface DurableObject { - fetch(request: Request): Response | Promise; - connect?(socket: Socket): void | Promise; - alarm?(alarmInfo?: AlarmInvocationInfo): void | Promise; - webSocketMessage?(ws: WebSocket, message: string | ArrayBuffer): void | Promise; - webSocketClose?(ws: WebSocket, code: number, reason: string, wasClean: boolean): void | Promise; - webSocketError?(ws: WebSocket, error: unknown): void | Promise; -} -type DurableObjectStub = Fetcher & { - readonly id: DurableObjectId; - readonly name?: string; -}; -interface DurableObjectId { - toString(): string; - equals(other: DurableObjectId): boolean; - readonly name?: string; - readonly jurisdiction?: string; -} -declare abstract class DurableObjectNamespace { - newUniqueId(options?: DurableObjectNamespaceNewUniqueIdOptions): DurableObjectId; - idFromName(name: string): DurableObjectId; - idFromString(id: string): DurableObjectId; - get(id: DurableObjectId, options?: DurableObjectNamespaceGetDurableObjectOptions): DurableObjectStub; - getByName(name: string, options?: DurableObjectNamespaceGetDurableObjectOptions): DurableObjectStub; - jurisdiction(jurisdiction: DurableObjectJurisdiction): DurableObjectNamespace; -} -type DurableObjectJurisdiction = "eu" | "fedramp" | "fedramp-high"; -interface DurableObjectNamespaceNewUniqueIdOptions { - jurisdiction?: DurableObjectJurisdiction; -} -type DurableObjectLocationHint = "wnam" | "enam" | "sam" | "weur" | "eeur" | "apac" | "oc" | "afr" | "me"; -type DurableObjectRoutingMode = "primary-only"; -interface DurableObjectNamespaceGetDurableObjectOptions { - locationHint?: DurableObjectLocationHint; - routingMode?: DurableObjectRoutingMode; -} -interface DurableObjectClass<_T extends Rpc.DurableObjectBranded | undefined = undefined> { -} -interface DurableObjectState { - waitUntil(promise: Promise): void; - readonly exports: Cloudflare.Exports; - readonly props: Props; - readonly id: DurableObjectId; - readonly storage: DurableObjectStorage; - container?: Container; - facets: DurableObjectFacets; - blockConcurrencyWhile(callback: () => Promise): Promise; - acceptWebSocket(ws: WebSocket, tags?: string[]): void; - getWebSockets(tag?: string): WebSocket[]; - setWebSocketAutoResponse(maybeReqResp?: WebSocketRequestResponsePair): void; - getWebSocketAutoResponse(): WebSocketRequestResponsePair | null; - getWebSocketAutoResponseTimestamp(ws: WebSocket): Date | null; - setHibernatableWebSocketEventTimeout(timeoutMs?: number): void; - getHibernatableWebSocketEventTimeout(): number | null; - getTags(ws: WebSocket): string[]; - abort(reason?: string): void; -} -interface DurableObjectTransaction { - get(key: string, options?: DurableObjectGetOptions): Promise; - get(keys: string[], options?: DurableObjectGetOptions): Promise>; - list(options?: DurableObjectListOptions): Promise>; - put(key: string, value: T, options?: DurableObjectPutOptions): Promise; - put(entries: Record, options?: DurableObjectPutOptions): Promise; - delete(key: string, options?: DurableObjectPutOptions): Promise; - delete(keys: string[], options?: DurableObjectPutOptions): Promise; - rollback(): void; - getAlarm(options?: DurableObjectGetAlarmOptions): Promise; - setAlarm(scheduledTime: number | Date, options?: DurableObjectSetAlarmOptions): Promise; - deleteAlarm(options?: DurableObjectSetAlarmOptions): Promise; -} -interface DurableObjectStorage { - get(key: string, options?: DurableObjectGetOptions): Promise; - get(keys: string[], options?: DurableObjectGetOptions): Promise>; - list(options?: DurableObjectListOptions): Promise>; - put(key: string, value: T, options?: DurableObjectPutOptions): Promise; - put(entries: Record, options?: DurableObjectPutOptions): Promise; - delete(key: string, options?: DurableObjectPutOptions): Promise; - delete(keys: string[], options?: DurableObjectPutOptions): Promise; - deleteAll(options?: DurableObjectPutOptions): Promise; - transaction(closure: (txn: DurableObjectTransaction) => Promise): Promise; - getAlarm(options?: DurableObjectGetAlarmOptions): Promise; - setAlarm(scheduledTime: number | Date, options?: DurableObjectSetAlarmOptions): Promise; - deleteAlarm(options?: DurableObjectSetAlarmOptions): Promise; - sync(): Promise; - sql: SqlStorage; - kv: SyncKvStorage; - transactionSync(closure: () => T): T; - getCurrentBookmark(): Promise; - getBookmarkForTime(timestamp: number | Date): Promise; - onNextSessionRestoreBookmark(bookmark: string): Promise; -} -interface DurableObjectListOptions { - start?: string; - startAfter?: string; - end?: string; - prefix?: string; - reverse?: boolean; - limit?: number; - allowConcurrency?: boolean; - noCache?: boolean; -} -interface DurableObjectGetOptions { - allowConcurrency?: boolean; - noCache?: boolean; -} -interface DurableObjectGetAlarmOptions { - allowConcurrency?: boolean; -} -interface DurableObjectPutOptions { - allowConcurrency?: boolean; - allowUnconfirmed?: boolean; - noCache?: boolean; -} -interface DurableObjectSetAlarmOptions { - allowConcurrency?: boolean; - allowUnconfirmed?: boolean; -} -declare class WebSocketRequestResponsePair { - constructor(request: string, response: string); - get request(): string; - get response(): string; -} -interface DurableObjectFacets { - get(name: string, getStartupOptions: () => FacetStartupOptions | Promise>): Fetcher; - abort(name: string, reason: any): void; - delete(name: string): void; -} -interface FacetStartupOptions { - id?: DurableObjectId | string; - class: DurableObjectClass; -} -interface AnalyticsEngineDataset { - writeDataPoint(event?: AnalyticsEngineDataPoint): void; -} -interface AnalyticsEngineDataPoint { - indexes?: ((ArrayBuffer | string) | null)[]; - doubles?: number[]; - blobs?: ((ArrayBuffer | string) | null)[]; -} -/** - * The **`Event`** interface represents an event which takes place on an `EventTarget`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event) - */ -declare class Event { - constructor(type: string, init?: EventInit); - /** - * The **`type`** read-only property of the Event interface returns a string containing the event's type. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/type) - */ - get type(): string; - /** - * The **`eventPhase`** read-only property of the being evaluated. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/eventPhase) - */ - get eventPhase(): number; - /** - * The read-only **`composed`** property of the or not the event will propagate across the shadow DOM boundary into the standard DOM. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/composed) - */ - get composed(): boolean; - /** - * The **`bubbles`** read-only property of the Event interface indicates whether the event bubbles up through the DOM tree or not. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/bubbles) - */ - get bubbles(): boolean; - /** - * The **`cancelable`** read-only property of the Event interface indicates whether the event can be canceled, and therefore prevented as if the event never happened. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/cancelable) - */ - get cancelable(): boolean; - /** - * The **`defaultPrevented`** read-only property of the Event interface returns a boolean value indicating whether or not the call to Event.preventDefault() canceled the event. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/defaultPrevented) - */ - get defaultPrevented(): boolean; - /** - * The Event property **`returnValue`** indicates whether the default action for this event has been prevented or not. - * @deprecated - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/returnValue) - */ - get returnValue(): boolean; - /** - * The **`currentTarget`** read-only property of the Event interface identifies the element to which the event handler has been attached. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/currentTarget) - */ - get currentTarget(): EventTarget | undefined; - /** - * The read-only **`target`** property of the dispatched. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/target) - */ - get target(): EventTarget | undefined; - /** - * The deprecated **`Event.srcElement`** is an alias for the Event.target property. - * @deprecated - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/srcElement) - */ - get srcElement(): EventTarget | undefined; - /** - * The **`timeStamp`** read-only property of the Event interface returns the time (in milliseconds) at which the event was created. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/timeStamp) - */ - get timeStamp(): number; - /** - * The **`isTrusted`** read-only property of the when the event was generated by the user agent (including via user actions and programmatic methods such as HTMLElement.focus()), and `false` when the event was dispatched via The only exception is the `click` event, which initializes the `isTrusted` property to `false` in user agents. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/isTrusted) - */ - get isTrusted(): boolean; - /** - * The **`cancelBubble`** property of the Event interface is deprecated. - * @deprecated - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/cancelBubble) - */ - get cancelBubble(): boolean; - /** - * The **`cancelBubble`** property of the Event interface is deprecated. - * @deprecated - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/cancelBubble) - */ - set cancelBubble(value: boolean); - /** - * The **`stopImmediatePropagation()`** method of the If several listeners are attached to the same element for the same event type, they are called in the order in which they were added. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/stopImmediatePropagation) - */ - stopImmediatePropagation(): void; - /** - * The **`preventDefault()`** method of the Event interface tells the user agent that if the event does not get explicitly handled, its default action should not be taken as it normally would be. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/preventDefault) - */ - preventDefault(): void; - /** - * The **`stopPropagation()`** method of the Event interface prevents further propagation of the current event in the capturing and bubbling phases. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/stopPropagation) - */ - stopPropagation(): void; - /** - * The **`composedPath()`** method of the Event interface returns the event's path which is an array of the objects on which listeners will be invoked. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Event/composedPath) - */ - composedPath(): EventTarget[]; - static readonly NONE: number; - static readonly CAPTURING_PHASE: number; - static readonly AT_TARGET: number; - static readonly BUBBLING_PHASE: number; -} -interface EventInit { - bubbles?: boolean; - cancelable?: boolean; - composed?: boolean; -} -type EventListener = (event: EventType) => void; -interface EventListenerObject { - handleEvent(event: EventType): void; -} -type EventListenerOrEventListenerObject = EventListener | EventListenerObject; -/** - * The **`EventTarget`** interface is implemented by objects that can receive events and may have listeners for them. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget) - */ -declare class EventTarget = Record> { - constructor(); - /** - * The **`addEventListener()`** method of the EventTarget interface sets up a function that will be called whenever the specified event is delivered to the target. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/addEventListener) - */ - addEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetAddEventListenerOptions | boolean): void; - /** - * The **`removeEventListener()`** method of the EventTarget interface removes an event listener previously registered with EventTarget.addEventListener() from the target. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/removeEventListener) - */ - removeEventListener(type: Type, handler: EventListenerOrEventListenerObject, options?: EventTargetEventListenerOptions | boolean): void; - /** - * The **`dispatchEvent()`** method of the EventTarget sends an Event to the object, (synchronously) invoking the affected event listeners in the appropriate order. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventTarget/dispatchEvent) - */ - dispatchEvent(event: EventMap[keyof EventMap]): boolean; -} -interface EventTargetEventListenerOptions { - capture?: boolean; -} -interface EventTargetAddEventListenerOptions { - capture?: boolean; - passive?: boolean; - once?: boolean; - signal?: AbortSignal; -} -interface EventTargetHandlerObject { - handleEvent: (event: Event) => any | undefined; -} -/** - * The **`AbortController`** interface represents a controller object that allows you to abort one or more Web requests as and when desired. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortController) - */ -declare class AbortController { - constructor(); - /** - * The **`signal`** read-only property of the AbortController interface returns an AbortSignal object instance, which can be used to communicate with/abort an asynchronous operation as desired. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortController/signal) - */ - get signal(): AbortSignal; - /** - * The **`abort()`** method of the AbortController interface aborts an asynchronous operation before it has completed. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortController/abort) - */ - abort(reason?: any): void; -} -/** - * The **`AbortSignal`** interface represents a signal object that allows you to communicate with an asynchronous operation (such as a fetch request) and abort it if required via an AbortController object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal) - */ -declare abstract class AbortSignal extends EventTarget { - /** - * The **`AbortSignal.abort()`** static method returns an AbortSignal that is already set as aborted (and which does not trigger an AbortSignal/abort_event event). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/abort_static) - */ - static abort(reason?: any): AbortSignal; - /** - * The **`AbortSignal.timeout()`** static method returns an AbortSignal that will automatically abort after a specified time. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/timeout_static) - */ - static timeout(delay: number): AbortSignal; - /** - * The **`AbortSignal.any()`** static method takes an iterable of abort signals and returns an AbortSignal. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/any_static) - */ - static any(signals: AbortSignal[]): AbortSignal; - /** - * The **`aborted`** read-only property returns a value that indicates whether the asynchronous operations the signal is communicating with are aborted (`true`) or not (`false`). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/aborted) - */ - get aborted(): boolean; - /** - * The **`reason`** read-only property returns a JavaScript value that indicates the abort reason. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/reason) - */ - get reason(): any; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/abort_event) */ - get onabort(): any | null; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/abort_event) */ - set onabort(value: any | null); - /** - * The **`throwIfAborted()`** method throws the signal's abort AbortSignal.reason if the signal has been aborted; otherwise it does nothing. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/AbortSignal/throwIfAborted) - */ - throwIfAborted(): void; -} -interface Scheduler { - wait(delay: number, maybeOptions?: SchedulerWaitOptions): Promise; -} -interface SchedulerWaitOptions { - signal?: AbortSignal; -} -/** - * The **`ExtendableEvent`** interface extends the lifetime of the `install` and `activate` events dispatched on the global scope as part of the service worker lifecycle. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ExtendableEvent) - */ -declare abstract class ExtendableEvent extends Event { - /** - * The **`ExtendableEvent.waitUntil()`** method tells the event dispatcher that work is ongoing. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ExtendableEvent/waitUntil) - */ - waitUntil(promise: Promise): void; -} -/** - * The **`CustomEvent`** interface represents events initialized by an application for any purpose. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CustomEvent) - */ -declare class CustomEvent extends Event { - constructor(type: string, init?: CustomEventCustomEventInit); - /** - * The read-only **`detail`** property of the CustomEvent interface returns any data passed when initializing the event. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CustomEvent/detail) - */ - get detail(): T; -} -interface CustomEventCustomEventInit { - bubbles?: boolean; - cancelable?: boolean; - composed?: boolean; - detail?: any; -} -/** - * The **`Blob`** interface represents a blob, which is a file-like object of immutable, raw data; they can be read as text or binary data, or converted into a ReadableStream so its methods can be used for processing the data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob) - */ -declare class Blob { - constructor(type?: ((ArrayBuffer | ArrayBufferView) | string | Blob)[], options?: BlobOptions); - /** - * The **`size`** read-only property of the Blob interface returns the size of the Blob or File in bytes. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/size) - */ - get size(): number; - /** - * The **`type`** read-only property of the Blob interface returns the MIME type of the file. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/type) - */ - get type(): string; - /** - * The **`slice()`** method of the Blob interface creates and returns a new `Blob` object which contains data from a subset of the blob on which it's called. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/slice) - */ - slice(start?: number, end?: number, type?: string): Blob; - /** - * The **`arrayBuffer()`** method of the Blob interface returns a Promise that resolves with the contents of the blob as binary data contained in an ArrayBuffer. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/arrayBuffer) - */ - arrayBuffer(): Promise; - /** - * The **`bytes()`** method of the Blob interface returns a Promise that resolves with a Uint8Array containing the contents of the blob as an array of bytes. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/bytes) - */ - bytes(): Promise; - /** - * The **`text()`** method of the string containing the contents of the blob, interpreted as UTF-8. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/text) - */ - text(): Promise; - /** - * The **`stream()`** method of the Blob interface returns a ReadableStream which upon reading returns the data contained within the `Blob`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Blob/stream) - */ - stream(): ReadableStream; -} -interface BlobOptions { - type?: string; -} -/** - * The **`File`** interface provides information about files and allows JavaScript in a web page to access their content. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/File) - */ -declare class File extends Blob { - constructor(bits: ((ArrayBuffer | ArrayBufferView) | string | Blob)[] | undefined, name: string, options?: FileOptions); - /** - * The **`name`** read-only property of the File interface returns the name of the file represented by a File object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/name) - */ - get name(): string; - /** - * The **`lastModified`** read-only property of the File interface provides the last modified date of the file as the number of milliseconds since the Unix epoch (January 1, 1970 at midnight). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/File/lastModified) - */ - get lastModified(): number; -} -interface FileOptions { - type?: string; - lastModified?: number; -} -/** -* The Cache API allows fine grained control of reading and writing from the Cloudflare global network cache. -* -* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/) -*/ -declare abstract class CacheStorage { - /** - * The **`open()`** method of the the Cache object matching the `cacheName`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CacheStorage/open) - */ - open(cacheName: string): Promise; - readonly default: Cache; -} -/** -* The Cache API allows fine grained control of reading and writing from the Cloudflare global network cache. -* -* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/) -*/ -declare abstract class Cache { - /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/#delete) */ - delete(request: RequestInfo | URL, options?: CacheQueryOptions): Promise; - /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/#match) */ - match(request: RequestInfo | URL, options?: CacheQueryOptions): Promise; - /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/cache/#put) */ - put(request: RequestInfo | URL, response: Response): Promise; -} -interface CacheQueryOptions { - ignoreMethod?: boolean; -} -/** -* The Web Crypto API provides a set of low-level functions for common cryptographic tasks. -* The Workers runtime implements the full surface of this API, but with some differences in -* the [supported algorithms](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/#supported-algorithms) -* compared to those implemented in most browsers. -* -* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/web-crypto/) -*/ -declare abstract class Crypto { - /** - * The **`Crypto.subtle`** read-only property returns a cryptographic operations. - * Available only in secure contexts. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/subtle) - */ - get subtle(): SubtleCrypto; - /** - * The **`Crypto.getRandomValues()`** method lets you get cryptographically strong random values. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/getRandomValues) - */ - getRandomValues(buffer: T): T; - /** - * The **`randomUUID()`** method of the Crypto interface is used to generate a v4 UUID using a cryptographically secure random number generator. - * Available only in secure contexts. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Crypto/randomUUID) - */ - randomUUID(): string; - DigestStream: typeof DigestStream; -} -/** - * The **`SubtleCrypto`** interface of the Web Crypto API provides a number of low-level cryptographic functions. - * Available only in secure contexts. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto) - */ -declare abstract class SubtleCrypto { - /** - * The **`encrypt()`** method of the SubtleCrypto interface encrypts data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/encrypt) - */ - encrypt(algorithm: string | SubtleCryptoEncryptAlgorithm, key: CryptoKey, plainText: ArrayBuffer | ArrayBufferView): Promise; - /** - * The **`decrypt()`** method of the SubtleCrypto interface decrypts some encrypted data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/decrypt) - */ - decrypt(algorithm: string | SubtleCryptoEncryptAlgorithm, key: CryptoKey, cipherText: ArrayBuffer | ArrayBufferView): Promise; - /** - * The **`sign()`** method of the SubtleCrypto interface generates a digital signature. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/sign) - */ - sign(algorithm: string | SubtleCryptoSignAlgorithm, key: CryptoKey, data: ArrayBuffer | ArrayBufferView): Promise; - /** - * The **`verify()`** method of the SubtleCrypto interface verifies a digital signature. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/verify) - */ - verify(algorithm: string | SubtleCryptoSignAlgorithm, key: CryptoKey, signature: ArrayBuffer | ArrayBufferView, data: ArrayBuffer | ArrayBufferView): Promise; - /** - * The **`digest()`** method of the SubtleCrypto interface generates a _digest_ of the given data, using the specified hash function. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/digest) - */ - digest(algorithm: string | SubtleCryptoHashAlgorithm, data: ArrayBuffer | ArrayBufferView): Promise; - /** - * The **`generateKey()`** method of the SubtleCrypto interface is used to generate a new key (for symmetric algorithms) or key pair (for public-key algorithms). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/generateKey) - */ - generateKey(algorithm: string | SubtleCryptoGenerateKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; - /** - * The **`deriveKey()`** method of the SubtleCrypto interface can be used to derive a secret key from a master key. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/deriveKey) - */ - deriveKey(algorithm: string | SubtleCryptoDeriveKeyAlgorithm, baseKey: CryptoKey, derivedKeyAlgorithm: string | SubtleCryptoImportKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; - /** - * The **`deriveBits()`** method of the key. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/deriveBits) - */ - deriveBits(algorithm: string | SubtleCryptoDeriveKeyAlgorithm, baseKey: CryptoKey, length?: number | null): Promise; - /** - * The **`importKey()`** method of the SubtleCrypto interface imports a key: that is, it takes as input a key in an external, portable format and gives you a CryptoKey object that you can use in the Web Crypto API. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/importKey) - */ - importKey(format: string, keyData: (ArrayBuffer | ArrayBufferView) | JsonWebKey, algorithm: string | SubtleCryptoImportKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; - /** - * The **`exportKey()`** method of the SubtleCrypto interface exports a key: that is, it takes as input a CryptoKey object and gives you the key in an external, portable format. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/exportKey) - */ - exportKey(format: string, key: CryptoKey): Promise; - /** - * The **`wrapKey()`** method of the SubtleCrypto interface 'wraps' a key. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/wrapKey) - */ - wrapKey(format: string, key: CryptoKey, wrappingKey: CryptoKey, wrapAlgorithm: string | SubtleCryptoEncryptAlgorithm): Promise; - /** - * The **`unwrapKey()`** method of the SubtleCrypto interface 'unwraps' a key. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/SubtleCrypto/unwrapKey) - */ - unwrapKey(format: string, wrappedKey: ArrayBuffer | ArrayBufferView, unwrappingKey: CryptoKey, unwrapAlgorithm: string | SubtleCryptoEncryptAlgorithm, unwrappedKeyAlgorithm: string | SubtleCryptoImportKeyAlgorithm, extractable: boolean, keyUsages: string[]): Promise; - timingSafeEqual(a: ArrayBuffer | ArrayBufferView, b: ArrayBuffer | ArrayBufferView): boolean; -} -/** - * The **`CryptoKey`** interface of the Web Crypto API represents a cryptographic key obtained from one of the SubtleCrypto methods SubtleCrypto.generateKey, SubtleCrypto.deriveKey, SubtleCrypto.importKey, or SubtleCrypto.unwrapKey. - * Available only in secure contexts. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey) - */ -declare abstract class CryptoKey { - /** - * The read-only **`type`** property of the CryptoKey interface indicates which kind of key is represented by the object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/type) - */ - readonly type: string; - /** - * The read-only **`extractable`** property of the CryptoKey interface indicates whether or not the key may be extracted using `SubtleCrypto.exportKey()` or `SubtleCrypto.wrapKey()`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/extractable) - */ - readonly extractable: boolean; - /** - * The read-only **`algorithm`** property of the CryptoKey interface returns an object describing the algorithm for which this key can be used, and any associated extra parameters. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/algorithm) - */ - readonly algorithm: CryptoKeyKeyAlgorithm | CryptoKeyAesKeyAlgorithm | CryptoKeyHmacKeyAlgorithm | CryptoKeyRsaKeyAlgorithm | CryptoKeyEllipticKeyAlgorithm | CryptoKeyArbitraryKeyAlgorithm; - /** - * The read-only **`usages`** property of the CryptoKey interface indicates what can be done with the key. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CryptoKey/usages) - */ - readonly usages: string[]; -} -interface CryptoKeyPair { - publicKey: CryptoKey; - privateKey: CryptoKey; -} -interface JsonWebKey { - kty: string; - use?: string; - key_ops?: string[]; - alg?: string; - ext?: boolean; - crv?: string; - x?: string; - y?: string; - d?: string; - n?: string; - e?: string; - p?: string; - q?: string; - dp?: string; - dq?: string; - qi?: string; - oth?: RsaOtherPrimesInfo[]; - k?: string; -} -interface RsaOtherPrimesInfo { - r?: string; - d?: string; - t?: string; -} -interface SubtleCryptoDeriveKeyAlgorithm { - name: string; - salt?: (ArrayBuffer | ArrayBufferView); - iterations?: number; - hash?: (string | SubtleCryptoHashAlgorithm); - $public?: CryptoKey; - info?: (ArrayBuffer | ArrayBufferView); -} -interface SubtleCryptoEncryptAlgorithm { - name: string; - iv?: (ArrayBuffer | ArrayBufferView); - additionalData?: (ArrayBuffer | ArrayBufferView); - tagLength?: number; - counter?: (ArrayBuffer | ArrayBufferView); - length?: number; - label?: (ArrayBuffer | ArrayBufferView); -} -interface SubtleCryptoGenerateKeyAlgorithm { - name: string; - hash?: (string | SubtleCryptoHashAlgorithm); - modulusLength?: number; - publicExponent?: (ArrayBuffer | ArrayBufferView); - length?: number; - namedCurve?: string; -} -interface SubtleCryptoHashAlgorithm { - name: string; -} -interface SubtleCryptoImportKeyAlgorithm { - name: string; - hash?: (string | SubtleCryptoHashAlgorithm); - length?: number; - namedCurve?: string; - compressed?: boolean; -} -interface SubtleCryptoSignAlgorithm { - name: string; - hash?: (string | SubtleCryptoHashAlgorithm); - dataLength?: number; - saltLength?: number; -} -interface CryptoKeyKeyAlgorithm { - name: string; -} -interface CryptoKeyAesKeyAlgorithm { - name: string; - length: number; -} -interface CryptoKeyHmacKeyAlgorithm { - name: string; - hash: CryptoKeyKeyAlgorithm; - length: number; -} -interface CryptoKeyRsaKeyAlgorithm { - name: string; - modulusLength: number; - publicExponent: ArrayBuffer | ArrayBufferView; - hash?: CryptoKeyKeyAlgorithm; -} -interface CryptoKeyEllipticKeyAlgorithm { - name: string; - namedCurve: string; -} -interface CryptoKeyArbitraryKeyAlgorithm { - name: string; - hash?: CryptoKeyKeyAlgorithm; - namedCurve?: string; - length?: number; -} -declare class DigestStream extends WritableStream { - constructor(algorithm: string | SubtleCryptoHashAlgorithm); - readonly digest: Promise; - get bytesWritten(): number | bigint; -} -/** - * The **`TextDecoder`** interface represents a decoder for a specific text encoding, such as `UTF-8`, `ISO-8859-2`, `KOI8-R`, `GBK`, etc. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextDecoder) - */ -declare class TextDecoder { - constructor(label?: string, options?: TextDecoderConstructorOptions); - /** - * The **`TextDecoder.decode()`** method returns a string containing text decoded from the buffer passed as a parameter. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextDecoder/decode) - */ - decode(input?: (ArrayBuffer | ArrayBufferView), options?: TextDecoderDecodeOptions): string; - get encoding(): string; - get fatal(): boolean; - get ignoreBOM(): boolean; -} -/** - * The **`TextEncoder`** interface takes a stream of code points as input and emits a stream of UTF-8 bytes. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoder) - */ -declare class TextEncoder { - constructor(); - /** - * The **`TextEncoder.encode()`** method takes a string as input, and returns a Global_Objects/Uint8Array containing the text given in parameters encoded with the specific method for that TextEncoder object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoder/encode) - */ - encode(input?: string): Uint8Array; - /** - * The **`TextEncoder.encodeInto()`** method takes a string to encode and a destination Uint8Array to put resulting UTF-8 encoded text into, and returns a dictionary object indicating the progress of the encoding. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoder/encodeInto) - */ - encodeInto(input: string, buffer: Uint8Array): TextEncoderEncodeIntoResult; - get encoding(): string; -} -interface TextDecoderConstructorOptions { - fatal: boolean; - ignoreBOM: boolean; -} -interface TextDecoderDecodeOptions { - stream: boolean; -} -interface TextEncoderEncodeIntoResult { - read: number; - written: number; -} -/** - * The **`ErrorEvent`** interface represents events providing information related to errors in scripts or in files. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent) - */ -declare class ErrorEvent extends Event { - constructor(type: string, init?: ErrorEventErrorEventInit); - /** - * The **`filename`** read-only property of the ErrorEvent interface returns a string containing the name of the script file in which the error occurred. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/filename) - */ - get filename(): string; - /** - * The **`message`** read-only property of the ErrorEvent interface returns a string containing a human-readable error message describing the problem. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/message) - */ - get message(): string; - /** - * The **`lineno`** read-only property of the ErrorEvent interface returns an integer containing the line number of the script file on which the error occurred. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/lineno) - */ - get lineno(): number; - /** - * The **`colno`** read-only property of the ErrorEvent interface returns an integer containing the column number of the script file on which the error occurred. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/colno) - */ - get colno(): number; - /** - * The **`error`** read-only property of the ErrorEvent interface returns a JavaScript value, such as an Error or DOMException, representing the error associated with this event. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ErrorEvent/error) - */ - get error(): any; -} -interface ErrorEventErrorEventInit { - message?: string; - filename?: string; - lineno?: number; - colno?: number; - error?: any; -} -/** - * The **`MessageEvent`** interface represents a message received by a target object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent) - */ -declare class MessageEvent extends Event { - constructor(type: string, initializer: MessageEventInit); - /** - * The **`data`** read-only property of the The data sent by the message emitter; this can be any data type, depending on what originated this event. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/data) - */ - readonly data: any; - /** - * The **`origin`** read-only property of the origin of the message emitter. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/origin) - */ - readonly origin: string | null; - /** - * The **`lastEventId`** read-only property of the unique ID for the event. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/lastEventId) - */ - readonly lastEventId: string; - /** - * The **`source`** read-only property of the a WindowProxy, MessagePort, or a `MessageEventSource` (which can be a WindowProxy, message emitter. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/source) - */ - readonly source: MessagePort | null; - /** - * The **`ports`** read-only property of the containing all MessagePort objects sent with the message, in order. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageEvent/ports) - */ - readonly ports: MessagePort[]; -} -interface MessageEventInit { - data: ArrayBuffer | string; -} -/** - * The **`PromiseRejectionEvent`** interface represents events which are sent to the global script context when JavaScript Promises are rejected. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PromiseRejectionEvent) - */ -declare abstract class PromiseRejectionEvent extends Event { - /** - * The PromiseRejectionEvent interface's **`promise`** read-only property indicates the JavaScript rejected. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PromiseRejectionEvent/promise) - */ - readonly promise: Promise; - /** - * The PromiseRejectionEvent **`reason`** read-only property is any JavaScript value or Object which provides the reason passed into Promise.reject(). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/PromiseRejectionEvent/reason) - */ - readonly reason: any; -} -/** - * The **`FormData`** interface provides a way to construct a set of key/value pairs representing form fields and their values, which can be sent using the Window/fetch, XMLHttpRequest.send() or navigator.sendBeacon() methods. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData) - */ -declare class FormData { - constructor(); - /** - * The **`append()`** method of the FormData interface appends a new value onto an existing key inside a `FormData` object, or adds the key if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/append) - */ - append(name: string, value: string | Blob): void; - /** - * The **`append()`** method of the FormData interface appends a new value onto an existing key inside a `FormData` object, or adds the key if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/append) - */ - append(name: string, value: string): void; - /** - * The **`append()`** method of the FormData interface appends a new value onto an existing key inside a `FormData` object, or adds the key if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/append) - */ - append(name: string, value: Blob, filename?: string): void; - /** - * The **`delete()`** method of the FormData interface deletes a key and its value(s) from a `FormData` object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/delete) - */ - delete(name: string): void; - /** - * The **`get()`** method of the FormData interface returns the first value associated with a given key from within a `FormData` object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/get) - */ - get(name: string): (File | string) | null; - /** - * The **`getAll()`** method of the FormData interface returns all the values associated with a given key from within a `FormData` object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/getAll) - */ - getAll(name: string): (File | string)[]; - /** - * The **`has()`** method of the FormData interface returns whether a `FormData` object contains a certain key. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/has) - */ - has(name: string): boolean; - /** - * The **`set()`** method of the FormData interface sets a new value for an existing key inside a `FormData` object, or adds the key/value if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/set) - */ - set(name: string, value: string | Blob): void; - /** - * The **`set()`** method of the FormData interface sets a new value for an existing key inside a `FormData` object, or adds the key/value if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/set) - */ - set(name: string, value: string): void; - /** - * The **`set()`** method of the FormData interface sets a new value for an existing key inside a `FormData` object, or adds the key/value if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FormData/set) - */ - set(name: string, value: Blob, filename?: string): void; - /* Returns an array of key, value pairs for every entry in the list. */ - entries(): IterableIterator<[ - key: string, - value: File | string - ]>; - /* Returns a list of keys in the list. */ - keys(): IterableIterator; - /* Returns a list of values in the list. */ - values(): IterableIterator<(File | string)>; - forEach(callback: (this: This, value: File | string, key: string, parent: FormData) => void, thisArg?: This): void; - [Symbol.iterator](): IterableIterator<[ - key: string, - value: File | string - ]>; -} -interface ContentOptions { - html?: boolean; -} -declare class HTMLRewriter { - constructor(); - on(selector: string, handlers: HTMLRewriterElementContentHandlers): HTMLRewriter; - onDocument(handlers: HTMLRewriterDocumentContentHandlers): HTMLRewriter; - transform(response: Response): Response; -} -interface HTMLRewriterElementContentHandlers { - element?(element: Element): void | Promise; - comments?(comment: Comment): void | Promise; - text?(element: Text): void | Promise; -} -interface HTMLRewriterDocumentContentHandlers { - doctype?(doctype: Doctype): void | Promise; - comments?(comment: Comment): void | Promise; - text?(text: Text): void | Promise; - end?(end: DocumentEnd): void | Promise; -} -interface Doctype { - readonly name: string | null; - readonly publicId: string | null; - readonly systemId: string | null; -} -interface Element { - tagName: string; - readonly attributes: IterableIterator; - readonly removed: boolean; - readonly namespaceURI: string; - getAttribute(name: string): string | null; - hasAttribute(name: string): boolean; - setAttribute(name: string, value: string): Element; - removeAttribute(name: string): Element; - before(content: string | ReadableStream | Response, options?: ContentOptions): Element; - after(content: string | ReadableStream | Response, options?: ContentOptions): Element; - prepend(content: string | ReadableStream | Response, options?: ContentOptions): Element; - append(content: string | ReadableStream | Response, options?: ContentOptions): Element; - replace(content: string | ReadableStream | Response, options?: ContentOptions): Element; - remove(): Element; - removeAndKeepContent(): Element; - setInnerContent(content: string | ReadableStream | Response, options?: ContentOptions): Element; - onEndTag(handler: (tag: EndTag) => void | Promise): void; -} -interface EndTag { - name: string; - before(content: string | ReadableStream | Response, options?: ContentOptions): EndTag; - after(content: string | ReadableStream | Response, options?: ContentOptions): EndTag; - remove(): EndTag; -} -interface Comment { - text: string; - readonly removed: boolean; - before(content: string, options?: ContentOptions): Comment; - after(content: string, options?: ContentOptions): Comment; - replace(content: string, options?: ContentOptions): Comment; - remove(): Comment; -} -interface Text { - readonly text: string; - readonly lastInTextNode: boolean; - readonly removed: boolean; - before(content: string | ReadableStream | Response, options?: ContentOptions): Text; - after(content: string | ReadableStream | Response, options?: ContentOptions): Text; - replace(content: string | ReadableStream | Response, options?: ContentOptions): Text; - remove(): Text; -} -interface DocumentEnd { - append(content: string, options?: ContentOptions): DocumentEnd; -} -/** - * This is the event type for `fetch` events dispatched on the ServiceWorkerGlobalScope. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FetchEvent) - */ -declare abstract class FetchEvent extends ExtendableEvent { - /** - * The **`request`** read-only property of the the event handler. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FetchEvent/request) - */ - readonly request: Request; - /** - * The **`respondWith()`** method of allows you to provide a promise for a Response yourself. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/FetchEvent/respondWith) - */ - respondWith(promise: Response | Promise): void; - passThroughOnException(): void; -} -type HeadersInit = Headers | Iterable> | Record; -/** - * The **`Headers`** interface of the Fetch API allows you to perform various actions on HTTP request and response headers. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers) - */ -declare class Headers { - constructor(init?: HeadersInit); - /** - * The **`get()`** method of the Headers interface returns a byte string of all the values of a header within a `Headers` object with a given name. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/get) - */ - get(name: string): string | null; - getAll(name: string): string[]; - /** - * The **`getSetCookie()`** method of the Headers interface returns an array containing the values of all Set-Cookie headers associated with a response. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/getSetCookie) - */ - getSetCookie(): string[]; - /** - * The **`has()`** method of the Headers interface returns a boolean stating whether a `Headers` object contains a certain header. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/has) - */ - has(name: string): boolean; - /** - * The **`set()`** method of the Headers interface sets a new value for an existing header inside a `Headers` object, or adds the header if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/set) - */ - set(name: string, value: string): void; - /** - * The **`append()`** method of the Headers interface appends a new value onto an existing header inside a `Headers` object, or adds the header if it does not already exist. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/append) - */ - append(name: string, value: string): void; - /** - * The **`delete()`** method of the Headers interface deletes a header from the current `Headers` object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Headers/delete) - */ - delete(name: string): void; - forEach(callback: (this: This, value: string, key: string, parent: Headers) => void, thisArg?: This): void; - /* Returns an iterator allowing to go through all key/value pairs contained in this object. */ - entries(): IterableIterator<[ - key: string, - value: string - ]>; - /* Returns an iterator allowing to go through all keys of the key/value pairs contained in this object. */ - keys(): IterableIterator; - /* Returns an iterator allowing to go through all values of the key/value pairs contained in this object. */ - values(): IterableIterator; - [Symbol.iterator](): IterableIterator<[ - key: string, - value: string - ]>; -} -type BodyInit = ReadableStream | string | ArrayBuffer | ArrayBufferView | Blob | URLSearchParams | FormData | Iterable | AsyncIterable; -declare abstract class Body { - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/body) */ - get body(): ReadableStream | null; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/bodyUsed) */ - get bodyUsed(): boolean; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/arrayBuffer) */ - arrayBuffer(): Promise; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/bytes) */ - bytes(): Promise; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/text) */ - text(): Promise; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/json) */ - json(): Promise; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/formData) */ - formData(): Promise; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/blob) */ - blob(): Promise; -} -/** - * The **`Response`** interface of the Fetch API represents the response to a request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response) - */ -declare var Response: { - prototype: Response; - new (body?: BodyInit | null, init?: ResponseInit): Response; - error(): Response; - redirect(url: string, status?: number): Response; - json(any: any, maybeInit?: (ResponseInit | Response)): Response; -}; -/** - * The **`Response`** interface of the Fetch API represents the response to a request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response) - */ -interface Response extends Body { - /** - * The **`clone()`** method of the Response interface creates a clone of a response object, identical in every way, but stored in a different variable. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/clone) - */ - clone(): Response; - /** - * The **`status`** read-only property of the Response interface contains the HTTP status codes of the response. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/status) - */ - status: number; - /** - * The **`statusText`** read-only property of the Response interface contains the status message corresponding to the HTTP status code in Response.status. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/statusText) - */ - statusText: string; - /** - * The **`headers`** read-only property of the with the response. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/headers) - */ - headers: Headers; - /** - * The **`ok`** read-only property of the Response interface contains a Boolean stating whether the response was successful (status in the range 200-299) or not. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/ok) - */ - ok: boolean; - /** - * The **`redirected`** read-only property of the Response interface indicates whether or not the response is the result of a request you made which was redirected. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/redirected) - */ - redirected: boolean; - /** - * The **`url`** read-only property of the Response interface contains the URL of the response. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/url) - */ - url: string; - webSocket: WebSocket | null; - cf: any | undefined; - /** - * The **`type`** read-only property of the Response interface contains the type of the response. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Response/type) - */ - type: "default" | "error"; -} -interface ResponseInit { - status?: number; - statusText?: string; - headers?: HeadersInit; - cf?: any; - webSocket?: (WebSocket | null); - encodeBody?: "automatic" | "manual"; -} -type RequestInfo> = Request | string; -/** - * The **`Request`** interface of the Fetch API represents a resource request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request) - */ -declare var Request: { - prototype: Request; - new >(input: RequestInfo | URL, init?: RequestInit): Request; -}; -/** - * The **`Request`** interface of the Fetch API represents a resource request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request) - */ -interface Request> extends Body { - /** - * The **`clone()`** method of the Request interface creates a copy of the current `Request` object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/clone) - */ - clone(): Request; - /** - * The **`method`** read-only property of the `POST`, etc.) A String indicating the method of the request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/method) - */ - method: string; - /** - * The **`url`** read-only property of the Request interface contains the URL of the request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/url) - */ - url: string; - /** - * The **`headers`** read-only property of the with the request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/headers) - */ - headers: Headers; - /** - * The **`redirect`** read-only property of the Request interface contains the mode for how redirects are handled. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/redirect) - */ - redirect: string; - fetcher: Fetcher | null; - /** - * The read-only **`signal`** property of the Request interface returns the AbortSignal associated with the request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/signal) - */ - signal: AbortSignal; - cf?: Cf; - /** - * The **`integrity`** read-only property of the Request interface contains the subresource integrity value of the request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/integrity) - */ - integrity: string; - /** - * The **`keepalive`** read-only property of the Request interface contains the request's `keepalive` setting (`true` or `false`), which indicates whether the browser will keep the associated request alive if the page that initiated it is unloaded before the request is complete. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/keepalive) - */ - keepalive: boolean; - /** - * The **`cache`** read-only property of the Request interface contains the cache mode of the request. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Request/cache) - */ - cache?: "no-store" | "no-cache"; -} -interface RequestInit { - /* A string to set request's method. */ - method?: string; - /* A Headers object, an object literal, or an array of two-item arrays to set request's headers. */ - headers?: HeadersInit; - /* A BodyInit object or null to set request's body. */ - body?: BodyInit | null; - /* A string indicating whether request follows redirects, results in an error upon encountering a redirect, or returns the redirect (in an opaque fashion). Sets request's redirect. */ - redirect?: string; - fetcher?: (Fetcher | null); - cf?: Cf; - /* A string indicating how the request will interact with the browser's cache to set request's cache. */ - cache?: "no-store" | "no-cache"; - /* A cryptographic hash of the resource to be fetched by request. Sets request's integrity. */ - integrity?: string; - /* An AbortSignal to set request's signal. */ - signal?: (AbortSignal | null); - encodeResponseBody?: "automatic" | "manual"; -} -type Service Rpc.WorkerEntrypointBranded) | Rpc.WorkerEntrypointBranded | ExportedHandler | undefined = undefined> = T extends new (...args: any[]) => Rpc.WorkerEntrypointBranded ? Fetcher> : T extends Rpc.WorkerEntrypointBranded ? Fetcher : T extends Exclude ? never : Fetcher; -type Fetcher = (T extends Rpc.EntrypointBranded ? Rpc.Provider : unknown) & { - fetch(input: RequestInfo | URL, init?: RequestInit): Promise; - connect(address: SocketAddress | string, options?: SocketOptions): Socket; -}; -interface KVNamespaceListKey { - name: Key; - expiration?: number; - metadata?: Metadata; -} -type KVNamespaceListResult = { - list_complete: false; - keys: KVNamespaceListKey[]; - cursor: string; - cacheStatus: string | null; -} | { - list_complete: true; - keys: KVNamespaceListKey[]; - cacheStatus: string | null; -}; -interface KVNamespace { - get(key: Key, options?: Partial>): Promise; - get(key: Key, type: "text"): Promise; - get(key: Key, type: "json"): Promise; - get(key: Key, type: "arrayBuffer"): Promise; - get(key: Key, type: "stream"): Promise; - get(key: Key, options?: KVNamespaceGetOptions<"text">): Promise; - get(key: Key, options?: KVNamespaceGetOptions<"json">): Promise; - get(key: Key, options?: KVNamespaceGetOptions<"arrayBuffer">): Promise; - get(key: Key, options?: KVNamespaceGetOptions<"stream">): Promise; - get(key: Array, type: "text"): Promise>; - get(key: Array, type: "json"): Promise>; - get(key: Array, options?: Partial>): Promise>; - get(key: Array, options?: KVNamespaceGetOptions<"text">): Promise>; - get(key: Array, options?: KVNamespaceGetOptions<"json">): Promise>; - list(options?: KVNamespaceListOptions): Promise>; - put(key: Key, value: string | ArrayBuffer | ArrayBufferView | ReadableStream, options?: KVNamespacePutOptions): Promise; - getWithMetadata(key: Key, options?: Partial>): Promise>; - getWithMetadata(key: Key, type: "text"): Promise>; - getWithMetadata(key: Key, type: "json"): Promise>; - getWithMetadata(key: Key, type: "arrayBuffer"): Promise>; - getWithMetadata(key: Key, type: "stream"): Promise>; - getWithMetadata(key: Key, options: KVNamespaceGetOptions<"text">): Promise>; - getWithMetadata(key: Key, options: KVNamespaceGetOptions<"json">): Promise>; - getWithMetadata(key: Key, options: KVNamespaceGetOptions<"arrayBuffer">): Promise>; - getWithMetadata(key: Key, options: KVNamespaceGetOptions<"stream">): Promise>; - getWithMetadata(key: Array, type: "text"): Promise>>; - getWithMetadata(key: Array, type: "json"): Promise>>; - getWithMetadata(key: Array, options?: Partial>): Promise>>; - getWithMetadata(key: Array, options?: KVNamespaceGetOptions<"text">): Promise>>; - getWithMetadata(key: Array, options?: KVNamespaceGetOptions<"json">): Promise>>; - delete(key: Key): Promise; -} -interface KVNamespaceListOptions { - limit?: number; - prefix?: (string | null); - cursor?: (string | null); -} -interface KVNamespaceGetOptions { - type: Type; - cacheTtl?: number; -} -interface KVNamespacePutOptions { - expiration?: number; - expirationTtl?: number; - metadata?: (any | null); -} -interface KVNamespaceGetWithMetadataResult { - value: Value | null; - metadata: Metadata | null; - cacheStatus: string | null; -} -type QueueContentType = "text" | "bytes" | "json" | "v8"; -interface Queue { - metrics(): Promise; - send(message: Body, options?: QueueSendOptions): Promise; - sendBatch(messages: Iterable>, options?: QueueSendBatchOptions): Promise; -} -interface QueueSendMetrics { - backlogCount: number; - backlogBytes: number; - oldestMessageTimestamp?: Date; -} -interface QueueSendMetadata { - metrics: QueueSendMetrics; -} -interface QueueSendResponse { - metadata: QueueSendMetadata; -} -interface QueueSendBatchMetrics { - backlogCount: number; - backlogBytes: number; - oldestMessageTimestamp?: Date; -} -interface QueueSendBatchMetadata { - metrics: QueueSendBatchMetrics; -} -interface QueueSendBatchResponse { - metadata: QueueSendBatchMetadata; -} -interface QueueSendOptions { - contentType?: QueueContentType; - delaySeconds?: number; -} -interface QueueSendBatchOptions { - delaySeconds?: number; -} -interface MessageSendRequest { - body: Body; - contentType?: QueueContentType; - delaySeconds?: number; -} -interface QueueMetrics { - backlogCount: number; - backlogBytes: number; - oldestMessageTimestamp?: Date; -} -interface MessageBatchMetrics { - backlogCount: number; - backlogBytes: number; - oldestMessageTimestamp?: Date; -} -interface MessageBatchMetadata { - metrics: MessageBatchMetrics; -} -interface QueueRetryOptions { - delaySeconds?: number; -} -interface Message { - readonly id: string; - readonly timestamp: Date; - readonly body: Body; - readonly attempts: number; - retry(options?: QueueRetryOptions): void; - ack(): void; -} -interface QueueEvent extends ExtendableEvent { - readonly messages: readonly Message[]; - readonly queue: string; - readonly metadata: MessageBatchMetadata; - retryAll(options?: QueueRetryOptions): void; - ackAll(): void; -} -interface MessageBatch { - readonly messages: readonly Message[]; - readonly queue: string; - readonly metadata: MessageBatchMetadata; - retryAll(options?: QueueRetryOptions): void; - ackAll(): void; -} -interface R2Error extends Error { - readonly name: string; - readonly code: number; - readonly message: string; - readonly action: string; - readonly stack: any; -} -interface R2ListOptions { - limit?: number; - prefix?: string; - cursor?: string; - delimiter?: string; - startAfter?: string; - include?: ("httpMetadata" | "customMetadata")[]; -} -declare abstract class R2Bucket { - head(key: string): Promise; - get(key: string, options: R2GetOptions & { - onlyIf: R2Conditional | Headers; - }): Promise; - get(key: string, options?: R2GetOptions): Promise; - put(key: string, value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob, options?: R2PutOptions & { - onlyIf: R2Conditional | Headers; - }): Promise; - put(key: string, value: ReadableStream | ArrayBuffer | ArrayBufferView | string | null | Blob, options?: R2PutOptions): Promise; - createMultipartUpload(key: string, options?: R2MultipartOptions): Promise; - resumeMultipartUpload(key: string, uploadId: string): R2MultipartUpload; - delete(keys: string | string[]): Promise; - list(options?: R2ListOptions): Promise; -} -interface R2MultipartUpload { - readonly key: string; - readonly uploadId: string; - uploadPart(partNumber: number, value: ReadableStream | (ArrayBuffer | ArrayBufferView) | string | Blob, options?: R2UploadPartOptions): Promise; - abort(): Promise; - complete(uploadedParts: R2UploadedPart[]): Promise; -} -interface R2UploadedPart { - partNumber: number; - etag: string; -} -declare abstract class R2Object { - readonly key: string; - readonly version: string; - readonly size: number; - readonly etag: string; - readonly httpEtag: string; - readonly checksums: R2Checksums; - readonly uploaded: Date; - readonly httpMetadata?: R2HTTPMetadata; - readonly customMetadata?: Record; - readonly range?: R2Range; - readonly storageClass: string; - readonly ssecKeyMd5?: string; - writeHttpMetadata(headers: Headers): void; -} -interface R2ObjectBody extends R2Object { - get body(): ReadableStream; - get bodyUsed(): boolean; - arrayBuffer(): Promise; - bytes(): Promise; - text(): Promise; - json(): Promise; - blob(): Promise; -} -type R2Range = { - offset: number; - length?: number; -} | { - offset?: number; - length: number; -} | { - suffix: number; -}; -interface R2Conditional { - etagMatches?: string; - etagDoesNotMatch?: string; - uploadedBefore?: Date; - uploadedAfter?: Date; - secondsGranularity?: boolean; -} -interface R2GetOptions { - onlyIf?: (R2Conditional | Headers); - range?: (R2Range | Headers); - ssecKey?: (ArrayBuffer | string); -} -interface R2PutOptions { - onlyIf?: (R2Conditional | Headers); - httpMetadata?: (R2HTTPMetadata | Headers); - customMetadata?: Record; - md5?: ((ArrayBuffer | ArrayBufferView) | string); - sha1?: ((ArrayBuffer | ArrayBufferView) | string); - sha256?: ((ArrayBuffer | ArrayBufferView) | string); - sha384?: ((ArrayBuffer | ArrayBufferView) | string); - sha512?: ((ArrayBuffer | ArrayBufferView) | string); - storageClass?: string; - ssecKey?: (ArrayBuffer | string); -} -interface R2MultipartOptions { - httpMetadata?: (R2HTTPMetadata | Headers); - customMetadata?: Record; - storageClass?: string; - ssecKey?: (ArrayBuffer | string); -} -interface R2Checksums { - readonly md5?: ArrayBuffer; - readonly sha1?: ArrayBuffer; - readonly sha256?: ArrayBuffer; - readonly sha384?: ArrayBuffer; - readonly sha512?: ArrayBuffer; - toJSON(): R2StringChecksums; -} -interface R2StringChecksums { - md5?: string; - sha1?: string; - sha256?: string; - sha384?: string; - sha512?: string; -} -interface R2HTTPMetadata { - contentType?: string; - contentLanguage?: string; - contentDisposition?: string; - contentEncoding?: string; - cacheControl?: string; - cacheExpiry?: Date; -} -type R2Objects = { - objects: R2Object[]; - delimitedPrefixes: string[]; -} & ({ - truncated: true; - cursor: string; -} | { - truncated: false; -}); -interface R2UploadPartOptions { - ssecKey?: (ArrayBuffer | string); -} -declare abstract class ScheduledEvent extends ExtendableEvent { - readonly scheduledTime: number; - readonly cron: string; - noRetry(): void; -} -interface ScheduledController { - readonly scheduledTime: number; - readonly cron: string; - noRetry(): void; -} -interface QueuingStrategy { - highWaterMark?: (number | bigint); - size?: (chunk: T) => number | bigint; -} -interface UnderlyingSink { - type?: string; - start?: (controller: WritableStreamDefaultController) => void | Promise; - write?: (chunk: W, controller: WritableStreamDefaultController) => void | Promise; - abort?: (reason: any) => void | Promise; - close?: () => void | Promise; -} -interface UnderlyingByteSource { - type: "bytes"; - autoAllocateChunkSize?: number; - start?: (controller: ReadableByteStreamController) => void | Promise; - pull?: (controller: ReadableByteStreamController) => void | Promise; - cancel?: (reason: any) => void | Promise; -} -interface UnderlyingSource { - type?: "" | undefined; - start?: (controller: ReadableStreamDefaultController) => void | Promise; - pull?: (controller: ReadableStreamDefaultController) => void | Promise; - cancel?: (reason: any) => void | Promise; - expectedLength?: (number | bigint); -} -interface Transformer { - readableType?: string; - writableType?: string; - start?: (controller: TransformStreamDefaultController) => void | Promise; - transform?: (chunk: I, controller: TransformStreamDefaultController) => void | Promise; - flush?: (controller: TransformStreamDefaultController) => void | Promise; - cancel?: (reason: any) => void | Promise; - expectedLength?: number; -} -interface StreamPipeOptions { - preventAbort?: boolean; - preventCancel?: boolean; - /** - * Pipes this readable stream to a given writable stream destination. The way in which the piping process behaves under various error conditions can be customized with a number of passed options. It returns a promise that fulfills when the piping process completes successfully, or rejects if any errors were encountered. - * - * Piping a stream will lock it for the duration of the pipe, preventing any other consumer from acquiring a reader. - * - * Errors and closures of the source and destination streams propagate as follows: - * - * An error in this source readable stream will abort destination, unless preventAbort is truthy. The returned promise will be rejected with the source's error, or with any error that occurs during aborting the destination. - * - * An error in destination will cancel this source readable stream, unless preventCancel is truthy. The returned promise will be rejected with the destination's error, or with any error that occurs during canceling the source. - * - * When this source readable stream closes, destination will be closed, unless preventClose is truthy. The returned promise will be fulfilled once this process completes, unless an error is encountered while closing the destination, in which case it will be rejected with that error. - * - * If destination starts out closed or closing, this source readable stream will be canceled, unless preventCancel is true. The returned promise will be rejected with an error indicating piping to a closed stream failed, or with any error that occurs during canceling the source. - * - * The signal option can be set to an AbortSignal to allow aborting an ongoing pipe operation via the corresponding AbortController. In this case, this source readable stream will be canceled, and destination aborted, unless the respective options preventCancel or preventAbort are set. - */ - preventClose?: boolean; - signal?: AbortSignal; -} -type ReadableStreamReadResult = { - done: false; - value: R; -} | { - done: true; - value?: undefined; -}; -/** - * The `ReadableStream` interface of the Streams API represents a readable stream of byte data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream) - */ -interface ReadableStream { - /** - * The **`locked`** read-only property of the ReadableStream interface returns whether or not the readable stream is locked to a reader. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/locked) - */ - get locked(): boolean; - /** - * The **`cancel()`** method of the ReadableStream interface returns a Promise that resolves when the stream is canceled. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/cancel) - */ - cancel(reason?: any): Promise; - /** - * The **`getReader()`** method of the ReadableStream interface creates a reader and locks the stream to it. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/getReader) - */ - getReader(): ReadableStreamDefaultReader; - /** - * The **`getReader()`** method of the ReadableStream interface creates a reader and locks the stream to it. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/getReader) - */ - getReader(options: ReadableStreamGetReaderOptions): ReadableStreamBYOBReader; - /** - * The **`pipeThrough()`** method of the ReadableStream interface provides a chainable way of piping the current stream through a transform stream or any other writable/readable pair. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/pipeThrough) - */ - pipeThrough(transform: ReadableWritablePair, options?: StreamPipeOptions): ReadableStream; - /** - * The **`pipeTo()`** method of the ReadableStream interface pipes the current `ReadableStream` to a given WritableStream and returns a Promise that fulfills when the piping process completes successfully, or rejects if any errors were encountered. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/pipeTo) - */ - pipeTo(destination: WritableStream, options?: StreamPipeOptions): Promise; - /** - * The **`tee()`** method of the two-element array containing the two resulting branches as new ReadableStream instances. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream/tee) - */ - tee(): [ - ReadableStream, - ReadableStream - ]; - values(options?: ReadableStreamValuesOptions): AsyncIterableIterator; - [Symbol.asyncIterator](options?: ReadableStreamValuesOptions): AsyncIterableIterator; -} -/** - * The `ReadableStream` interface of the Streams API represents a readable stream of byte data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStream) - */ -declare const ReadableStream: { - prototype: ReadableStream; - new (underlyingSource: UnderlyingByteSource, strategy?: QueuingStrategy): ReadableStream; - new (underlyingSource?: UnderlyingSource, strategy?: QueuingStrategy): ReadableStream; -}; -/** - * The **`ReadableStreamDefaultReader`** interface of the Streams API represents a default reader that can be used to read stream data supplied from a network (such as a fetch request). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultReader) - */ -declare class ReadableStreamDefaultReader { - constructor(stream: ReadableStream); - get closed(): Promise; - cancel(reason?: any): Promise; - /** - * The **`read()`** method of the ReadableStreamDefaultReader interface returns a Promise providing access to the next chunk in the stream's internal queue. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultReader/read) - */ - read(): Promise>; - /** - * The **`releaseLock()`** method of the ReadableStreamDefaultReader interface releases the reader's lock on the stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultReader/releaseLock) - */ - releaseLock(): void; -} -/** - * The `ReadableStreamBYOBReader` interface of the Streams API defines a reader for a ReadableStream that supports zero-copy reading from an underlying byte source. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBReader) - */ -declare class ReadableStreamBYOBReader { - constructor(stream: ReadableStream); - get closed(): Promise; - cancel(reason?: any): Promise; - /** - * The **`read()`** method of the ReadableStreamBYOBReader interface is used to read data into a view on a user-supplied buffer from an associated readable byte stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBReader/read) - */ - read(view: T): Promise>; - /** - * The **`releaseLock()`** method of the ReadableStreamBYOBReader interface releases the reader's lock on the stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBReader/releaseLock) - */ - releaseLock(): void; - readAtLeast(minElements: number, view: T): Promise>; -} -interface ReadableStreamBYOBReaderReadableStreamBYOBReaderReadOptions { - min?: number; -} -interface ReadableStreamGetReaderOptions { - /** - * Creates a ReadableStreamBYOBReader and locks the stream to the new reader. - * - * This call behaves the same way as the no-argument variant, except that it only works on readable byte streams, i.e. streams which were constructed specifically with the ability to handle "bring your own buffer" reading. The returned BYOB reader provides the ability to directly read individual chunks from the stream via its read() method, into developer-supplied buffers, allowing more precise control over allocation. - */ - mode: "byob"; -} -/** - * The **`ReadableStreamBYOBRequest`** interface of the Streams API represents a 'pull request' for data from an underlying source that will made as a zero-copy transfer to a consumer (bypassing the stream's internal queues). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest) - */ -declare abstract class ReadableStreamBYOBRequest { - /** - * The **`view`** getter property of the ReadableStreamBYOBRequest interface returns the current view. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest/view) - */ - get view(): Uint8Array | null; - /** - * The **`respond()`** method of the ReadableStreamBYOBRequest interface is used to signal to the associated readable byte stream that the specified number of bytes were written into the ReadableStreamBYOBRequest.view. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest/respond) - */ - respond(bytesWritten: number): void; - /** - * The **`respondWithNewView()`** method of the ReadableStreamBYOBRequest interface specifies a new view that the consumer of the associated readable byte stream should write to instead of ReadableStreamBYOBRequest.view. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamBYOBRequest/respondWithNewView) - */ - respondWithNewView(view: ArrayBuffer | ArrayBufferView): void; - get atLeast(): number | null; -} -/** - * The **`ReadableStreamDefaultController`** interface of the Streams API represents a controller allowing control of a ReadableStream's state and internal queue. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController) - */ -declare abstract class ReadableStreamDefaultController { - /** - * The **`desiredSize`** read-only property of the required to fill the stream's internal queue. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/desiredSize) - */ - get desiredSize(): number | null; - /** - * The **`close()`** method of the ReadableStreamDefaultController interface closes the associated stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/close) - */ - close(): void; - /** - * The **`enqueue()`** method of the ```js-nolint enqueue(chunk) ``` - `chunk` - : The chunk to enqueue. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/enqueue) - */ - enqueue(chunk?: R): void; - /** - * The **`error()`** method of the with the associated stream to error. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableStreamDefaultController/error) - */ - error(reason: any): void; -} -/** - * The **`ReadableByteStreamController`** interface of the Streams API represents a controller for a readable byte stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController) - */ -declare abstract class ReadableByteStreamController { - /** - * The **`byobRequest`** read-only property of the ReadableByteStreamController interface returns the current BYOB request, or `null` if there are no pending requests. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/byobRequest) - */ - get byobRequest(): ReadableStreamBYOBRequest | null; - /** - * The **`desiredSize`** read-only property of the ReadableByteStreamController interface returns the number of bytes required to fill the stream's internal queue to its 'desired size'. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/desiredSize) - */ - get desiredSize(): number | null; - /** - * The **`close()`** method of the ReadableByteStreamController interface closes the associated stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/close) - */ - close(): void; - /** - * The **`enqueue()`** method of the ReadableByteStreamController interface enqueues a given chunk on the associated readable byte stream (the chunk is copied into the stream's internal queues). - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/enqueue) - */ - enqueue(chunk: ArrayBuffer | ArrayBufferView): void; - /** - * The **`error()`** method of the ReadableByteStreamController interface causes any future interactions with the associated stream to error with the specified reason. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ReadableByteStreamController/error) - */ - error(reason: any): void; -} -/** - * The **`WritableStreamDefaultController`** interface of the Streams API represents a controller allowing control of a WritableStream's state. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultController) - */ -declare abstract class WritableStreamDefaultController { - /** - * The read-only **`signal`** property of the WritableStreamDefaultController interface returns the AbortSignal associated with the controller. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultController/signal) - */ - get signal(): AbortSignal; - /** - * The **`error()`** method of the with the associated stream to error. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultController/error) - */ - error(reason?: any): void; -} -/** - * The **`TransformStreamDefaultController`** interface of the Streams API provides methods to manipulate the associated ReadableStream and WritableStream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController) - */ -declare abstract class TransformStreamDefaultController { - /** - * The **`desiredSize`** read-only property of the TransformStreamDefaultController interface returns the desired size to fill the queue of the associated ReadableStream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/desiredSize) - */ - get desiredSize(): number | null; - /** - * The **`enqueue()`** method of the TransformStreamDefaultController interface enqueues the given chunk in the readable side of the stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/enqueue) - */ - enqueue(chunk?: O): void; - /** - * The **`error()`** method of the TransformStreamDefaultController interface errors both sides of the stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/error) - */ - error(reason: any): void; - /** - * The **`terminate()`** method of the TransformStreamDefaultController interface closes the readable side and errors the writable side of the stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStreamDefaultController/terminate) - */ - terminate(): void; -} -interface ReadableWritablePair { - readable: ReadableStream; - /** - * Provides a convenient, chainable way of piping this readable stream through a transform stream (or any other { writable, readable } pair). It simply pipes the stream into the writable side of the supplied pair, and returns the readable side for further use. - * - * Piping a stream will lock it for the duration of the pipe, preventing any other consumer from acquiring a reader. - */ - writable: WritableStream; -} -/** - * The **`WritableStream`** interface of the Streams API provides a standard abstraction for writing streaming data to a destination, known as a sink. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream) - */ -declare class WritableStream { - constructor(underlyingSink?: UnderlyingSink, queuingStrategy?: QueuingStrategy); - /** - * The **`locked`** read-only property of the WritableStream interface returns a boolean indicating whether the `WritableStream` is locked to a writer. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/locked) - */ - get locked(): boolean; - /** - * The **`abort()`** method of the WritableStream interface aborts the stream, signaling that the producer can no longer successfully write to the stream and it is to be immediately moved to an error state, with any queued writes discarded. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/abort) - */ - abort(reason?: any): Promise; - /** - * The **`close()`** method of the WritableStream interface closes the associated stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/close) - */ - close(): Promise; - /** - * The **`getWriter()`** method of the WritableStream interface returns a new instance of WritableStreamDefaultWriter and locks the stream to that instance. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStream/getWriter) - */ - getWriter(): WritableStreamDefaultWriter; -} -/** - * The **`WritableStreamDefaultWriter`** interface of the Streams API is the object returned by WritableStream.getWriter() and once created locks the writer to the `WritableStream` ensuring that no other streams can write to the underlying sink. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter) - */ -declare class WritableStreamDefaultWriter { - constructor(stream: WritableStream); - /** - * The **`closed`** read-only property of the the stream errors or the writer's lock is released. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/closed) - */ - get closed(): Promise; - /** - * The **`ready`** read-only property of the that resolves when the desired size of the stream's internal queue transitions from non-positive to positive, signaling that it is no longer applying backpressure. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/ready) - */ - get ready(): Promise; - /** - * The **`desiredSize`** read-only property of the to fill the stream's internal queue. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/desiredSize) - */ - get desiredSize(): number | null; - /** - * The **`abort()`** method of the the producer can no longer successfully write to the stream and it is to be immediately moved to an error state, with any queued writes discarded. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/abort) - */ - abort(reason?: any): Promise; - /** - * The **`close()`** method of the stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/close) - */ - close(): Promise; - /** - * The **`write()`** method of the operation. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/write) - */ - write(chunk?: W): Promise; - /** - * The **`releaseLock()`** method of the corresponding stream. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WritableStreamDefaultWriter/releaseLock) - */ - releaseLock(): void; -} -/** - * The **`TransformStream`** interface of the Streams API represents a concrete implementation of the pipe chain _transform stream_ concept. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStream) - */ -declare class TransformStream { - constructor(transformer?: Transformer, writableStrategy?: QueuingStrategy, readableStrategy?: QueuingStrategy); - /** - * The **`readable`** read-only property of the TransformStream interface returns the ReadableStream instance controlled by this `TransformStream`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStream/readable) - */ - get readable(): ReadableStream; - /** - * The **`writable`** read-only property of the TransformStream interface returns the WritableStream instance controlled by this `TransformStream`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TransformStream/writable) - */ - get writable(): WritableStream; -} -declare class FixedLengthStream extends IdentityTransformStream { - constructor(expectedLength: number | bigint, queuingStrategy?: IdentityTransformStreamQueuingStrategy); -} -declare class IdentityTransformStream extends TransformStream { - constructor(queuingStrategy?: IdentityTransformStreamQueuingStrategy); -} -interface IdentityTransformStreamQueuingStrategy { - highWaterMark?: (number | bigint); -} -interface ReadableStreamValuesOptions { - preventCancel?: boolean; -} -/** - * The **`CompressionStream`** interface of the Compression Streams API is an API for compressing a stream of data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CompressionStream) - */ -declare class CompressionStream extends TransformStream { - constructor(format: "gzip" | "deflate" | "deflate-raw"); -} -/** - * The **`DecompressionStream`** interface of the Compression Streams API is an API for decompressing a stream of data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/DecompressionStream) - */ -declare class DecompressionStream extends TransformStream { - constructor(format: "gzip" | "deflate" | "deflate-raw"); -} -/** - * The **`TextEncoderStream`** interface of the Encoding API converts a stream of strings into bytes in the UTF-8 encoding. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextEncoderStream) - */ -declare class TextEncoderStream extends TransformStream { - constructor(); - get encoding(): string; -} -/** - * The **`TextDecoderStream`** interface of the Encoding API converts a stream of text in a binary encoding, such as UTF-8 etc., to a stream of strings. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/TextDecoderStream) - */ -declare class TextDecoderStream extends TransformStream { - constructor(label?: string, options?: TextDecoderStreamTextDecoderStreamInit); - get encoding(): string; - get fatal(): boolean; - get ignoreBOM(): boolean; -} -interface TextDecoderStreamTextDecoderStreamInit { - fatal?: boolean; - ignoreBOM?: boolean; -} -/** - * The **`ByteLengthQueuingStrategy`** interface of the Streams API provides a built-in byte length queuing strategy that can be used when constructing streams. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ByteLengthQueuingStrategy) - */ -declare class ByteLengthQueuingStrategy implements QueuingStrategy { - constructor(init: QueuingStrategyInit); - /** - * The read-only **`ByteLengthQueuingStrategy.highWaterMark`** property returns the total number of bytes that can be contained in the internal queue before backpressure is applied. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/ByteLengthQueuingStrategy/highWaterMark) - */ - get highWaterMark(): number; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/ByteLengthQueuingStrategy/size) */ - get size(): (chunk?: any) => number; -} -/** - * The **`CountQueuingStrategy`** interface of the Streams API provides a built-in chunk counting queuing strategy that can be used when constructing streams. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CountQueuingStrategy) - */ -declare class CountQueuingStrategy implements QueuingStrategy { - constructor(init: QueuingStrategyInit); - /** - * The read-only **`CountQueuingStrategy.highWaterMark`** property returns the total number of chunks that can be contained in the internal queue before backpressure is applied. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CountQueuingStrategy/highWaterMark) - */ - get highWaterMark(): number; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/CountQueuingStrategy/size) */ - get size(): (chunk?: any) => number; -} -interface QueuingStrategyInit { - /** - * Creates a new ByteLengthQueuingStrategy with the provided high water mark. - * - * Note that the provided high water mark will not be validated ahead of time. Instead, if it is negative, NaN, or not a number, the resulting ByteLengthQueuingStrategy will cause the corresponding stream constructor to throw. - */ - highWaterMark: number; -} -interface TracePreviewInfo { - id: string; - slug: string; - name: string; -} -interface ScriptVersion { - id?: string; - tag?: string; - message?: string; -} -declare abstract class TailEvent extends ExtendableEvent { - readonly events: TraceItem[]; - readonly traces: TraceItem[]; -} -interface TraceItem { - readonly event: (TraceItemFetchEventInfo | TraceItemJsRpcEventInfo | TraceItemConnectEventInfo | TraceItemScheduledEventInfo | TraceItemAlarmEventInfo | TraceItemQueueEventInfo | TraceItemEmailEventInfo | TraceItemTailEventInfo | TraceItemCustomEventInfo | TraceItemHibernatableWebSocketEventInfo) | null; - readonly eventTimestamp: number | null; - readonly logs: TraceLog[]; - readonly exceptions: TraceException[]; - readonly diagnosticsChannelEvents: TraceDiagnosticChannelEvent[]; - readonly scriptName: string | null; - readonly entrypoint?: string; - readonly scriptVersion?: ScriptVersion; - readonly dispatchNamespace?: string; - readonly scriptTags?: string[]; - readonly tailAttributes?: Record; - readonly preview?: TracePreviewInfo; - readonly durableObjectId?: string; - readonly outcome: string; - readonly executionModel: string; - readonly truncated: boolean; - readonly cpuTime: number; - readonly wallTime: number; -} -interface TraceItemAlarmEventInfo { - readonly scheduledTime: Date; -} -interface TraceItemConnectEventInfo { -} -interface TraceItemCustomEventInfo { -} -interface TraceItemScheduledEventInfo { - readonly scheduledTime: number; - readonly cron: string; -} -interface TraceItemQueueEventInfo { - readonly queue: string; - readonly batchSize: number; -} -interface TraceItemEmailEventInfo { - readonly mailFrom: string; - readonly rcptTo: string; - readonly rawSize: number; -} -interface TraceItemTailEventInfo { - readonly consumedEvents: TraceItemTailEventInfoTailItem[]; -} -interface TraceItemTailEventInfoTailItem { - readonly scriptName: string | null; -} -interface TraceItemFetchEventInfo { - readonly response?: TraceItemFetchEventInfoResponse; - readonly request: TraceItemFetchEventInfoRequest; -} -interface TraceItemFetchEventInfoRequest { - readonly cf?: any; - readonly headers: Record; - readonly method: string; - readonly url: string; - getUnredacted(): TraceItemFetchEventInfoRequest; -} -interface TraceItemFetchEventInfoResponse { - readonly status: number; -} -interface TraceItemJsRpcEventInfo { - readonly rpcMethod: string; -} -interface TraceItemHibernatableWebSocketEventInfo { - readonly getWebSocketEvent: TraceItemHibernatableWebSocketEventInfoMessage | TraceItemHibernatableWebSocketEventInfoClose | TraceItemHibernatableWebSocketEventInfoError; -} -interface TraceItemHibernatableWebSocketEventInfoMessage { - readonly webSocketEventType: string; -} -interface TraceItemHibernatableWebSocketEventInfoClose { - readonly webSocketEventType: string; - readonly code: number; - readonly wasClean: boolean; -} -interface TraceItemHibernatableWebSocketEventInfoError { - readonly webSocketEventType: string; -} -interface TraceLog { - readonly timestamp: number; - readonly level: string; - readonly message: any; -} -interface TraceException { - readonly timestamp: number; - readonly message: string; - readonly name: string; - readonly stack?: string; -} -interface TraceDiagnosticChannelEvent { - readonly timestamp: number; - readonly channel: string; - readonly message: any; -} -interface TraceMetrics { - readonly cpuTime: number; - readonly wallTime: number; -} -interface UnsafeTraceMetrics { - fromTrace(item: TraceItem): TraceMetrics; -} -/** - * The **`URL`** interface is used to parse, construct, normalize, and encode URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL) - */ -declare class URL { - constructor(url: string | URL, base?: string | URL); - /** - * The **`origin`** read-only property of the URL interface returns a string containing the Unicode serialization of the origin of the represented URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/origin) - */ - get origin(): string; - /** - * The **`href`** property of the URL interface is a string containing the whole URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/href) - */ - get href(): string; - /** - * The **`href`** property of the URL interface is a string containing the whole URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/href) - */ - set href(value: string); - /** - * The **`protocol`** property of the URL interface is a string containing the protocol or scheme of the URL, including the final `':'`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/protocol) - */ - get protocol(): string; - /** - * The **`protocol`** property of the URL interface is a string containing the protocol or scheme of the URL, including the final `':'`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/protocol) - */ - set protocol(value: string); - /** - * The **`username`** property of the URL interface is a string containing the username component of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/username) - */ - get username(): string; - /** - * The **`username`** property of the URL interface is a string containing the username component of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/username) - */ - set username(value: string); - /** - * The **`password`** property of the URL interface is a string containing the password component of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/password) - */ - get password(): string; - /** - * The **`password`** property of the URL interface is a string containing the password component of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/password) - */ - set password(value: string); - /** - * The **`host`** property of the URL interface is a string containing the host, which is the URL.hostname, and then, if the port of the URL is nonempty, a `':'`, followed by the URL.port of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/host) - */ - get host(): string; - /** - * The **`host`** property of the URL interface is a string containing the host, which is the URL.hostname, and then, if the port of the URL is nonempty, a `':'`, followed by the URL.port of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/host) - */ - set host(value: string); - /** - * The **`hostname`** property of the URL interface is a string containing either the domain name or IP address of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hostname) - */ - get hostname(): string; - /** - * The **`hostname`** property of the URL interface is a string containing either the domain name or IP address of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hostname) - */ - set hostname(value: string); - /** - * The **`port`** property of the URL interface is a string containing the port number of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/port) - */ - get port(): string; - /** - * The **`port`** property of the URL interface is a string containing the port number of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/port) - */ - set port(value: string); - /** - * The **`pathname`** property of the URL interface represents a location in a hierarchical structure. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/pathname) - */ - get pathname(): string; - /** - * The **`pathname`** property of the URL interface represents a location in a hierarchical structure. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/pathname) - */ - set pathname(value: string); - /** - * The **`search`** property of the URL interface is a search string, also called a _query string_, that is a string containing a `'?'` followed by the parameters of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/search) - */ - get search(): string; - /** - * The **`search`** property of the URL interface is a search string, also called a _query string_, that is a string containing a `'?'` followed by the parameters of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/search) - */ - set search(value: string); - /** - * The **`hash`** property of the URL interface is a string containing a `'#'` followed by the fragment identifier of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hash) - */ - get hash(): string; - /** - * The **`hash`** property of the URL interface is a string containing a `'#'` followed by the fragment identifier of the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/hash) - */ - set hash(value: string); - /** - * The **`searchParams`** read-only property of the access to the [MISSING: httpmethod('GET')] decoded query arguments contained in the URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/searchParams) - */ - get searchParams(): URLSearchParams; - /** - * The **`toJSON()`** method of the URL interface returns a string containing a serialized version of the URL, although in practice it seems to have the same effect as ```js-nolint toJSON() ``` None. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/toJSON) - */ - toJSON(): string; - /*function toString() { [native code] }*/ - toString(): string; - /** - * The **`URL.canParse()`** static method of the URL interface returns a boolean indicating whether or not an absolute URL, or a relative URL combined with a base URL, are parsable and valid. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/canParse_static) - */ - static canParse(url: string, base?: string): boolean; - /** - * The **`URL.parse()`** static method of the URL interface returns a newly created URL object representing the URL defined by the parameters. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/parse_static) - */ - static parse(url: string, base?: string): URL | null; - /** - * The **`createObjectURL()`** static method of the URL interface creates a string containing a URL representing the object given in the parameter. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/createObjectURL_static) - */ - static createObjectURL(object: File | Blob): string; - /** - * The **`revokeObjectURL()`** static method of the URL interface releases an existing object URL which was previously created by calling Call this method when you've finished using an object URL to let the browser know not to keep the reference to the file any longer. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URL/revokeObjectURL_static) - */ - static revokeObjectURL(object_url: string): void; -} -/** - * The **`URLSearchParams`** interface defines utility methods to work with the query string of a URL. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams) - */ -declare class URLSearchParams { - constructor(init?: (Iterable> | Record | string)); - /** - * The **`size`** read-only property of the URLSearchParams interface indicates the total number of search parameter entries. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/size) - */ - get size(): number; - /** - * The **`append()`** method of the URLSearchParams interface appends a specified key/value pair as a new search parameter. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/append) - */ - append(name: string, value: string): void; - /** - * The **`delete()`** method of the URLSearchParams interface deletes specified parameters and their associated value(s) from the list of all search parameters. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/delete) - */ - delete(name: string, value?: string): void; - /** - * The **`get()`** method of the URLSearchParams interface returns the first value associated to the given search parameter. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/get) - */ - get(name: string): string | null; - /** - * The **`getAll()`** method of the URLSearchParams interface returns all the values associated with a given search parameter as an array. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/getAll) - */ - getAll(name: string): string[]; - /** - * The **`has()`** method of the URLSearchParams interface returns a boolean value that indicates whether the specified parameter is in the search parameters. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/has) - */ - has(name: string, value?: string): boolean; - /** - * The **`set()`** method of the URLSearchParams interface sets the value associated with a given search parameter to the given value. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/set) - */ - set(name: string, value: string): void; - /** - * The **`URLSearchParams.sort()`** method sorts all key/value pairs contained in this object in place and returns `undefined`. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/URLSearchParams/sort) - */ - sort(): void; - /* Returns an array of key, value pairs for every entry in the search params. */ - entries(): IterableIterator<[ - key: string, - value: string - ]>; - /* Returns a list of keys in the search params. */ - keys(): IterableIterator; - /* Returns a list of values in the search params. */ - values(): IterableIterator; - forEach(callback: (this: This, value: string, key: string, parent: URLSearchParams) => void, thisArg?: This): void; - /*function toString() { [native code] }*/ - toString(): string; - [Symbol.iterator](): IterableIterator<[ - key: string, - value: string - ]>; -} -declare class URLPattern { - constructor(input?: (string | URLPatternInit), baseURL?: (string | URLPatternOptions), patternOptions?: URLPatternOptions); - get protocol(): string; - get username(): string; - get password(): string; - get hostname(): string; - get port(): string; - get pathname(): string; - get search(): string; - get hash(): string; - get hasRegExpGroups(): boolean; - test(input?: (string | URLPatternInit), baseURL?: string): boolean; - exec(input?: (string | URLPatternInit), baseURL?: string): URLPatternResult | null; -} -interface URLPatternInit { - protocol?: string; - username?: string; - password?: string; - hostname?: string; - port?: string; - pathname?: string; - search?: string; - hash?: string; - baseURL?: string; -} -interface URLPatternComponentResult { - input: string; - groups: Record; -} -interface URLPatternResult { - inputs: (string | URLPatternInit)[]; - protocol: URLPatternComponentResult; - username: URLPatternComponentResult; - password: URLPatternComponentResult; - hostname: URLPatternComponentResult; - port: URLPatternComponentResult; - pathname: URLPatternComponentResult; - search: URLPatternComponentResult; - hash: URLPatternComponentResult; -} -interface URLPatternOptions { - ignoreCase?: boolean; -} -/** - * A `CloseEvent` is sent to clients using WebSockets when the connection is closed. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent) - */ -declare class CloseEvent extends Event { - constructor(type: string, initializer?: CloseEventInit); - /** - * The **`code`** read-only property of the CloseEvent interface returns a WebSocket connection close code indicating the reason the connection was closed. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent/code) - */ - readonly code: number; - /** - * The **`reason`** read-only property of the CloseEvent interface returns the WebSocket connection close reason the server gave for closing the connection; that is, a concise human-readable prose explanation for the closure. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent/reason) - */ - readonly reason: string; - /** - * The **`wasClean`** read-only property of the CloseEvent interface returns `true` if the connection closed cleanly. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/CloseEvent/wasClean) - */ - readonly wasClean: boolean; -} -interface CloseEventInit { - code?: number; - reason?: string; - wasClean?: boolean; -} -type WebSocketEventMap = { - close: CloseEvent; - message: MessageEvent; - open: Event; - error: ErrorEvent; -}; -/** - * The `WebSocket` object provides the API for creating and managing a WebSocket connection to a server, as well as for sending and receiving data on the connection. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket) - */ -declare var WebSocket: { - prototype: WebSocket; - new (url: string, protocols?: (string[] | string)): WebSocket; - readonly READY_STATE_CONNECTING: number; - readonly CONNECTING: number; - readonly READY_STATE_OPEN: number; - readonly OPEN: number; - readonly READY_STATE_CLOSING: number; - readonly CLOSING: number; - readonly READY_STATE_CLOSED: number; - readonly CLOSED: number; -}; -/** - * The `WebSocket` object provides the API for creating and managing a WebSocket connection to a server, as well as for sending and receiving data on the connection. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket) - */ -interface WebSocket extends EventTarget { - accept(options?: WebSocketAcceptOptions): void; - /** - * The **`WebSocket.send()`** method enqueues the specified data to be transmitted to the server over the WebSocket connection, increasing the value of `bufferedAmount` by the number of bytes needed to contain the data. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/send) - */ - send(message: (ArrayBuffer | ArrayBufferView) | string): void; - /** - * The **`WebSocket.close()`** method closes the already `CLOSED`, this method does nothing. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/close) - */ - close(code?: number, reason?: string): void; - serializeAttachment(attachment: any): void; - deserializeAttachment(): any | null; - /** - * The **`WebSocket.readyState`** read-only property returns the current state of the WebSocket connection. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/readyState) - */ - readyState: number; - /** - * The **`WebSocket.url`** read-only property returns the absolute URL of the WebSocket as resolved by the constructor. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/url) - */ - url: string | null; - /** - * The **`WebSocket.protocol`** read-only property returns the name of the sub-protocol the server selected; this will be one of the strings specified in the `protocols` parameter when creating the WebSocket object, or the empty string if no connection is established. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/protocol) - */ - protocol: string | null; - /** - * The **`WebSocket.extensions`** read-only property returns the extensions selected by the server. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/extensions) - */ - extensions: string | null; - /** - * The **`WebSocket.binaryType`** property controls the type of binary data being received over the WebSocket connection. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/WebSocket/binaryType) - */ - binaryType: "blob" | "arraybuffer"; -} -interface WebSocketAcceptOptions { - /** - * When set to `true`, receiving a server-initiated WebSocket Close frame will not - * automatically send a reciprocal Close frame, leaving the connection in a half-open - * state. This is useful for proxying scenarios where you need to coordinate closing - * both sides independently. Defaults to `false` when the - * `no_web_socket_half_open_by_default` compatibility flag is enabled. - */ - allowHalfOpen?: boolean; -} -declare const WebSocketPair: { - new (): { - 0: WebSocket; - 1: WebSocket; - }; -}; -interface SqlStorage { - exec>(query: string, ...bindings: any[]): SqlStorageCursor; - get databaseSize(): number; - Cursor: typeof SqlStorageCursor; - Statement: typeof SqlStorageStatement; -} -declare abstract class SqlStorageStatement { -} -type SqlStorageValue = ArrayBuffer | string | number | null; -declare abstract class SqlStorageCursor> { - next(): { - done?: false; - value: T; - } | { - done: true; - value?: never; - }; - toArray(): T[]; - one(): T; - raw(): IterableIterator; - columnNames: string[]; - get rowsRead(): number; - get rowsWritten(): number; - [Symbol.iterator](): IterableIterator; -} -interface Socket { - get readable(): ReadableStream; - get writable(): WritableStream; - get closed(): Promise; - get opened(): Promise; - get upgraded(): boolean; - get secureTransport(): "on" | "off" | "starttls"; - close(): Promise; - startTls(options?: TlsOptions): Socket; -} -interface SocketOptions { - secureTransport?: string; - allowHalfOpen: boolean; - highWaterMark?: (number | bigint); -} -interface SocketAddress { - hostname: string; - port: number; -} -interface TlsOptions { - expectedServerHostname?: string; -} -interface SocketInfo { - remoteAddress?: string; - localAddress?: string; -} -/** - * The **`EventSource`** interface is web content's interface to server-sent events. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource) - */ -declare class EventSource extends EventTarget { - constructor(url: string, init?: EventSourceEventSourceInit); - /** - * The **`close()`** method of the EventSource interface closes the connection, if one is made, and sets the ```js-nolint close() ``` None. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/close) - */ - close(): void; - /** - * The **`url`** read-only property of the URL of the source. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/url) - */ - get url(): string; - /** - * The **`withCredentials`** read-only property of the the `EventSource` object was instantiated with CORS credentials set. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/withCredentials) - */ - get withCredentials(): boolean; - /** - * The **`readyState`** read-only property of the connection. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/readyState) - */ - get readyState(): number; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/open_event) */ - get onopen(): any | null; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/open_event) */ - set onopen(value: any | null); - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/message_event) */ - get onmessage(): any | null; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/message_event) */ - set onmessage(value: any | null); - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/error_event) */ - get onerror(): any | null; - /* [MDN Reference](https://developer.mozilla.org/docs/Web/API/EventSource/error_event) */ - set onerror(value: any | null); - static readonly CONNECTING: number; - static readonly OPEN: number; - static readonly CLOSED: number; - static from(stream: ReadableStream): EventSource; -} -interface EventSourceEventSourceInit { - withCredentials?: boolean; - fetcher?: Fetcher; -} -interface Container { - get running(): boolean; - start(options?: ContainerStartupOptions): void; - monitor(): Promise; - destroy(error?: any): Promise; - signal(signo: number): void; - getTcpPort(port: number): Fetcher; - setInactivityTimeout(durationMs: number | bigint): Promise; - interceptOutboundHttp(addr: string, binding: Fetcher): Promise; - interceptAllOutboundHttp(binding: Fetcher): Promise; - snapshotDirectory(options: ContainerDirectorySnapshotOptions): Promise; - snapshotContainer(options: ContainerSnapshotOptions): Promise; - interceptOutboundHttps(addr: string, binding: Fetcher): Promise; -} -interface ContainerDirectorySnapshot { - id: string; - size: number; - dir: string; - name?: string; -} -interface ContainerDirectorySnapshotOptions { - dir: string; - name?: string; -} -interface ContainerDirectorySnapshotRestoreParams { - snapshot: ContainerDirectorySnapshot; - mountPoint?: string; -} -interface ContainerSnapshot { - id: string; - size: number; - name?: string; -} -interface ContainerSnapshotOptions { - name?: string; -} -interface ContainerStartupOptions { - entrypoint?: string[]; - enableInternet: boolean; - env?: Record; - labels?: Record; - directorySnapshots?: ContainerDirectorySnapshotRestoreParams[]; - containerSnapshot?: ContainerSnapshot; -} -/** - * The **`MessagePort`** interface of the Channel Messaging API represents one of the two ports of a MessageChannel, allowing messages to be sent from one port and listening out for them arriving at the other. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort) - */ -declare abstract class MessagePort extends EventTarget { - /** - * The **`postMessage()`** method of the transfers ownership of objects to other browsing contexts. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort/postMessage) - */ - postMessage(data?: any, options?: (any[] | MessagePortPostMessageOptions)): void; - /** - * The **`close()`** method of the MessagePort interface disconnects the port, so it is no longer active. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort/close) - */ - close(): void; - /** - * The **`start()`** method of the MessagePort interface starts the sending of messages queued on the port. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessagePort/start) - */ - start(): void; - get onmessage(): any | null; - set onmessage(value: any | null); -} -/** - * The **`MessageChannel`** interface of the Channel Messaging API allows us to create a new message channel and send data through it via its two MessagePort properties. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageChannel) - */ -declare class MessageChannel { - constructor(); - /** - * The **`port1`** read-only property of the the port attached to the context that originated the channel. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageChannel/port1) - */ - readonly port1: MessagePort; - /** - * The **`port2`** read-only property of the the port attached to the context at the other end of the channel, which the message is initially sent to. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/MessageChannel/port2) - */ - readonly port2: MessagePort; -} -interface MessagePortPostMessageOptions { - transfer?: any[]; -} -type LoopbackForExport Rpc.EntrypointBranded) | ExportedHandler | undefined = undefined> = T extends new (...args: any[]) => Rpc.WorkerEntrypointBranded ? LoopbackServiceStub> : T extends new (...args: any[]) => Rpc.DurableObjectBranded ? LoopbackDurableObjectClass> : T extends ExportedHandler ? LoopbackServiceStub : undefined; -type LoopbackServiceStub = Fetcher & (T extends CloudflareWorkersModule.WorkerEntrypoint ? (opts: { - props?: Props; -}) => Fetcher : (opts: { - props?: any; -}) => Fetcher); -type LoopbackDurableObjectClass = DurableObjectClass & (T extends CloudflareWorkersModule.DurableObject ? (opts: { - props?: Props; -}) => DurableObjectClass : (opts: { - props?: any; -}) => DurableObjectClass); -interface LoopbackDurableObjectNamespace extends DurableObjectNamespace { -} -interface LoopbackColoLocalActorNamespace extends ColoLocalActorNamespace { -} -interface SyncKvStorage { - get(key: string): T | undefined; - list(options?: SyncKvListOptions): Iterable<[ - string, - T - ]>; - put(key: string, value: T): void; - delete(key: string): boolean; -} -interface SyncKvListOptions { - start?: string; - startAfter?: string; - end?: string; - prefix?: string; - reverse?: boolean; - limit?: number; -} -interface WorkerStub { - getEntrypoint(name?: string, options?: WorkerStubEntrypointOptions): Fetcher; - getDurableObjectClass(name?: string, options?: WorkerStubEntrypointOptions): DurableObjectClass; -} -interface WorkerStubEntrypointOptions { - props?: any; - limits?: workerdResourceLimits; -} -interface WorkerLoader { - get(name: string | null, getCode: () => WorkerLoaderWorkerCode | Promise): WorkerStub; - load(code: WorkerLoaderWorkerCode): WorkerStub; -} -interface WorkerLoaderModule { - js?: string; - cjs?: string; - text?: string; - data?: ArrayBuffer; - json?: any; - py?: string; - wasm?: ArrayBuffer; -} -interface WorkerLoaderWorkerCode { - compatibilityDate: string; - compatibilityFlags?: string[]; - allowExperimental?: boolean; - limits?: workerdResourceLimits; - mainModule: string; - modules: Record; - env?: any; - globalOutbound?: (Fetcher | null); - tails?: Fetcher[]; - streamingTails?: Fetcher[]; -} -interface workerdResourceLimits { - cpuMs?: number; - subRequests?: number; -} -/** -* The Workers runtime supports a subset of the Performance API, used to measure timing and performance, -* as well as timing of subrequests and other operations. -* -* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/) -*/ -declare abstract class Performance { - /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/#performancetimeorigin) */ - get timeOrigin(): number; - /* [Cloudflare Docs Reference](https://developers.cloudflare.com/workers/runtime-apis/performance/#performancenow) */ - now(): number; - /** - * The **`toJSON()`** method of the Performance interface is a Serialization; it returns a JSON representation of the Performance object. - * - * [MDN Reference](https://developer.mozilla.org/docs/Web/API/Performance/toJSON) - */ - toJSON(): object; -} -// ============ AI Search Error Interfaces ============ -interface AiSearchInternalError extends Error { -} -interface AiSearchNotFoundError extends Error { -} -// ============ AI Search Common Types ============ -/** A single message in a conversation-style search or chat request. */ -type AiSearchMessage = { - role: 'system' | 'developer' | 'user' | 'assistant' | 'tool'; - content: string | null; -}; -/** - * Common shape for `ai_search_options` used by both single-instance and multi-instance requests. - * Contains retrieval, query rewrite, reranking, and cache sub-options. - */ -type AiSearchOptions = { - retrieval?: { - /** Which retrieval backend to use. Defaults to the instance's configured index_method. */ - retrieval_type?: 'vector' | 'keyword' | 'hybrid'; - /** Fusion method for combining vector + keyword results. */ - fusion_method?: 'max' | 'rrf'; - /** How keyword terms are combined: "and" = all terms must match, "or" = any term matches. */ - keyword_match_mode?: 'and' | 'or'; - /** Minimum similarity score (0-1) for a result to be included. Default 0.4. */ - match_threshold?: number; - /** Maximum number of results to return (1-50). Default 10. */ - max_num_results?: number; - /** Vectorize metadata filters applied to the search. */ - filters?: VectorizeVectorMetadataFilter; - /** Number of surrounding chunks to include for context (0-3). Default 0. */ - context_expansion?: number; - /** If true, return only item metadata without chunk text. */ - metadata_only?: boolean; - /** If true (default), return empty results on retrieval failure instead of throwing. */ - return_on_failure?: boolean; - /** Boost results by metadata field values. Max 3 entries. */ - boost_by?: Array<{ - field: string; - direction?: 'asc' | 'desc' | 'exists' | 'not_exists'; - }>; - [key: string]: unknown; - }; - query_rewrite?: { - enabled?: boolean; - model?: string; - rewrite_prompt?: string; - [key: string]: unknown; - }; - reranking?: { - enabled?: boolean; - model?: string; - /** Match threshold (0-1, default 0.4) */ - match_threshold?: number; - [key: string]: unknown; - }; - cache?: { - enabled?: boolean; - cache_threshold?: 'super_strict_match' | 'close_enough' | 'flexible_friend' | 'anything_goes'; - }; - [key: string]: unknown; -}; -// ============ AI Search Request Types ============ -/** - * Request body for single-instance search. - * Exactly one of `query` or `messages` must be provided. - */ -type AiSearchSearchRequest = { - /** Simple query string. */ - query: string; - messages?: never; - ai_search_options?: AiSearchOptions; -} | { - query?: never; - /** Conversation-style input. At least one user message with non-empty content is required. */ - messages: AiSearchMessage[]; - ai_search_options?: AiSearchOptions; -}; -type AiSearchChatCompletionsRequest = { - messages: AiSearchMessage[]; - model?: string; - stream?: boolean; - ai_search_options?: AiSearchOptions; - [key: string]: unknown; -}; -// ============ AI Search Multi-Instance Types (Namespace-Scoped) ============ -/** `ai_search_options` shape for multi-instance requests — requires `instance_ids`. */ -type AiSearchMultiSearchOptions = AiSearchOptions & { - /** Instance IDs to search across (1-10). */ - instance_ids: string[]; -}; -/** - * Request for searching across multiple instances within a namespace. - * `ai_search_options` is required and must include `instance_ids`. - * Exactly one of `query` or `messages` must be provided. - */ -type AiSearchMultiSearchRequest = { - /** Simple query string. */ - query: string; - messages?: never; - ai_search_options: AiSearchMultiSearchOptions; -} | { - query?: never; - /** Conversation-style input. */ - messages: AiSearchMessage[]; - ai_search_options: AiSearchMultiSearchOptions; -}; -/** A search result chunk tagged with the instance it originated from. */ -type AiSearchMultiSearchChunk = AiSearchSearchResponse['chunks'][number] & { - instance_id: string; -}; -/** Describes a per-instance error during a multi-instance operation. */ -type AiSearchMultiSearchError = { - instance_id: string; - message: string; -}; -/** Response from a multi-instance search, with chunks tagged by instance and optional partial-failure errors. */ -type AiSearchMultiSearchResponse = { - search_query: string; - chunks: AiSearchMultiSearchChunk[]; - errors?: AiSearchMultiSearchError[]; -}; -/** Request for chat completions across multiple instances within a namespace. `ai_search_options` is required and must include `instance_ids`. */ -type AiSearchMultiChatCompletionsRequest = Omit & { - ai_search_options: AiSearchMultiSearchOptions; -}; -/** Response from multi-instance chat completions, with chunks tagged by instance and optional partial-failure errors. */ -type AiSearchMultiChatCompletionsResponse = Omit & { - chunks: AiSearchMultiSearchChunk[]; - errors?: AiSearchMultiSearchError[]; -}; -// ============ AI Search Response Types ============ -type AiSearchSearchResponse = { - search_query: string; - chunks: Array<{ - id: string; - type: string; - /** Match score (0-1) */ - score: number; - text: string; - item: { - timestamp?: number; - key: string; - metadata?: Record; - }; - scoring_details?: { - /** Keyword match score (0-1) */ - keyword_score?: number; - /** Vector similarity score (0-1) */ - vector_score?: number; - /** Keyword rank position */ - keyword_rank?: number; - /** Vector rank position */ - vector_rank?: number; - /** Reranking model score */ - reranking_score?: number; - /** Fusion method used to combine results */ - fusion_method?: 'rrf' | 'max'; - [key: string]: unknown; - }; - }>; -}; -type AiSearchChatCompletionsResponse = { - id?: string; - object?: string; - model?: string; - choices: Array<{ - index?: number; - message: { - role: 'system' | 'developer' | 'user' | 'assistant' | 'tool'; - content: string | null; - [key: string]: unknown; - }; - [key: string]: unknown; - }>; - chunks: AiSearchSearchResponse['chunks']; - [key: string]: unknown; -}; -type AiSearchStatsResponse = { - queued?: number; - running?: number; - completed?: number; - error?: number; - skipped?: number; - outdated?: number; - last_activity?: string; - /** Storage engine statistics. */ - engine?: { - vectorize?: { - vectorsCount: number; - dimensions: number; - }; - r2?: { - payloadSizeBytes: number; - metadataSizeBytes: number; - objectCount: number; - }; - }; -}; -// ============ AI Search Instance Info Types ============ -type AiSearchInstanceInfo = { - id: string; - type?: 'r2' | 'web-crawler' | string; - source?: string; - source_params?: unknown; - paused?: boolean; - status?: string; - namespace?: string; - created_at?: string; - modified_at?: string; - token_id?: string; - ai_gateway_id?: string; - rewrite_query?: boolean; - reranking?: boolean; - embedding_model?: string; - ai_search_model?: string; - rewrite_model?: string; - reranking_model?: string; - /** @deprecated Use index_method instead. */ - hybrid_search_enabled?: boolean; - /** Controls which storage backends are active. */ - index_method?: { - vector?: boolean; - keyword?: boolean; - }; - /** Fusion method for combining vector and keyword results. */ - fusion_method?: 'max' | 'rrf'; - indexing_options?: { - keyword_tokenizer?: 'porter' | 'trigram'; - } | null; - retrieval_options?: { - keyword_match_mode?: 'and' | 'or'; - boost_by?: Array<{ - field: string; - direction?: 'asc' | 'desc' | 'exists' | 'not_exists'; - }>; - } | null; - chunk?: boolean; - chunk_size?: number; - chunk_overlap?: number; - score_threshold?: number; - max_num_results?: number; - cache?: boolean; - cache_threshold?: 'super_strict_match' | 'close_enough' | 'flexible_friend' | 'anything_goes'; - custom_metadata?: Array<{ - field_name: string; - data_type: 'text' | 'number' | 'boolean' | 'datetime'; - }>; - /** Sync interval in seconds. */ - sync_interval?: 3600 | 7200 | 14400 | 21600 | 43200 | 86400; - metadata?: Record; - [key: string]: unknown; -}; -/** Pagination, search, and ordering parameters for listing instances within a namespace. */ -type AiSearchListInstancesParams = { - page?: number; - per_page?: number; - /** Search instances by ID. */ - search?: string; - /** Field to sort by. */ - order_by?: 'created_at'; - /** Sort direction. */ - order_by_direction?: 'asc' | 'desc'; -}; -type AiSearchListResponse = { - result: AiSearchInstanceInfo[]; - result_info?: { - count: number; - page: number; - per_page: number; - total_count: number; - }; -}; -// ============ AI Search Config Types ============ -type AiSearchConfig = { - /** Instance ID (1-32 chars, pattern: ^[a-z0-9_]+(?:-[a-z0-9_]+)*$) */ - id: string; - /** Instance type. Omit to create with built-in storage. */ - type?: 'r2' | 'web-crawler' | string; - /** Source URL (required for web-crawler type). */ - source?: string; - source_params?: unknown; - /** Token ID (UUID format) */ - token_id?: string; - ai_gateway_id?: string; - /** Enable query rewriting (default false) */ - rewrite_query?: boolean; - /** Enable reranking (default false) */ - reranking?: boolean; - embedding_model?: string; - ai_search_model?: string; - rewrite_model?: string; - reranking_model?: string; - /** @deprecated Use index_method instead. */ - hybrid_search_enabled?: boolean; - /** Controls which storage backends are used during indexing. Defaults to vector-only. */ - index_method?: { - vector?: boolean; - keyword?: boolean; - }; - /** Fusion method for combining vector and keyword results. "rrf" = reciprocal rank fusion (default), "max" = maximum score. */ - fusion_method?: 'max' | 'rrf'; - indexing_options?: { - keyword_tokenizer?: 'porter' | 'trigram'; - } | null; - retrieval_options?: { - keyword_match_mode?: 'and' | 'or'; - boost_by?: Array<{ - field: string; - direction?: 'asc' | 'desc' | 'exists' | 'not_exists'; - }>; - } | null; - chunk?: boolean; - chunk_size?: number; - chunk_overlap?: number; - /** Minimum similarity score (0-1) for a result to be included. */ - score_threshold?: number; - max_num_results?: number; - cache?: boolean; - /** Similarity threshold for cache hits. Stricter = fewer cache hits but higher relevance. */ - cache_threshold?: 'super_strict_match' | 'close_enough' | 'flexible_friend' | 'anything_goes'; - custom_metadata?: Array<{ - field_name: string; - data_type: 'text' | 'number' | 'boolean' | 'datetime'; - }>; - namespace?: string; - /** Sync interval in seconds. 3600=1h, 7200=2h, 14400=4h, 21600=6h, 43200=12h, 86400=24h. */ - sync_interval?: 3600 | 7200 | 14400 | 21600 | 43200 | 86400; - metadata?: Record; - [key: string]: unknown; -}; -// ============ AI Search Item Types ============ -type AiSearchItemInfo = { - id: string; - key: string; - status: 'completed' | 'error' | 'skipped' | 'queued' | 'running' | 'outdated'; - next_action?: 'INDEX' | 'DELETE' | null; - error?: string; - checksum?: string; - namespace?: string; - chunks_count?: number | null; - file_size?: number | null; - source_id?: string | null; - last_seen_at?: string; - created_at?: string; - metadata?: Record; - [key: string]: unknown; -}; -type AiSearchItemContentResult = { - body: ReadableStream; - contentType: string; - filename: string; - size: number; -}; -type AiSearchUploadItemOptions = { - metadata?: Record; -}; -type AiSearchListItemsParams = { - page?: number; - per_page?: number; - /** Search items by key name. */ - search?: string; - /** Sort order for results. */ - sort_by?: 'status' | 'modified_at'; - /** Filter items by processing status. */ - status?: 'queued' | 'running' | 'completed' | 'error' | 'skipped' | 'outdated'; - /** Filter items by source (e.g. "builtin" or "web-crawler:https://example.com"). */ - source?: string; - /** JSON-encoded Vectorize filter for metadata filtering. */ - metadata_filter?: string; -}; -type AiSearchListItemsResponse = { - result: AiSearchItemInfo[]; - result_info?: { - count: number; - page: number; - per_page: number; - total_count: number; - }; -}; -// ============ AI Search Item Logs Types ============ -type AiSearchItemLogsParams = { - /** Maximum number of log entries to return (1-100, default 50). */ - limit?: number; - /** Opaque cursor for pagination. Pass the `cursor` value from a previous response. */ - cursor?: string; -}; -type AiSearchItemLog = { - timestamp: string; - action: string; - message: string; - fileKey?: string; - chunkCount?: number; - processingTimeMs?: number; - errorType?: string; -}; -/** Paginated response for item processing logs (cursor-based). */ -type AiSearchItemLogsResponse = { - result: AiSearchItemLog[]; - result_info: { - count: number; - per_page: number; - cursor: string | null; - truncated: boolean; - }; -}; -// ============ AI Search Item Chunks Types ============ -type AiSearchItemChunksParams = { - /** Maximum number of chunks to return (1-100, default 20). */ - limit?: number; - /** Offset into the chunks list (default 0). */ - offset?: number; -}; -/** A single indexed chunk belonging to an item, including its text content and byte range. */ -type AiSearchItemChunk = { - id: string; - text: string; - start_byte: number; - end_byte: number; - item?: { - timestamp?: number; - key: string; - metadata?: Record; - }; -}; -/** Paginated response for item chunks (offset-based). */ -type AiSearchItemChunksResponse = { - result: AiSearchItemChunk[]; - result_info: { - count: number; - total: number; - limit: number; - offset: number; - }; -}; -// ============ AI Search Job Types ============ -type AiSearchJobInfo = { - id: string; - source: 'user' | 'schedule'; - description?: string; - last_seen_at?: string; - started_at?: string; - ended_at?: string; - end_reason?: string; -}; -type AiSearchJobLog = { - id: number; - message: string; - message_type: number; - created_at: number; -}; -type AiSearchCreateJobParams = { - description?: string; -}; -type AiSearchListJobsParams = { - page?: number; - per_page?: number; -}; -type AiSearchListJobsResponse = { - result: AiSearchJobInfo[]; - result_info?: { - count: number; - page: number; - per_page: number; - total_count: number; - }; -}; -type AiSearchJobLogsParams = { - page?: number; - per_page?: number; -}; -type AiSearchJobLogsResponse = { - result: AiSearchJobLog[]; - result_info?: { - count: number; - page: number; - per_page: number; - total_count: number; - }; -}; -// ============ AI Search Sub-Service Classes ============ -/** - * Single item service for an AI Search instance. - * Provides info, download, sync, logs, and chunks operations on a specific item. - */ -declare abstract class AiSearchItem { - /** Get metadata about this item. */ - info(): Promise; - /** - * Download the item's content. - * @returns Object with body stream, content type, filename, and size. - */ - download(): Promise; - /** - * Trigger re-indexing of this item. - * @returns The updated item info. - */ - sync(): Promise; - /** - * Retrieve processing logs for this item (cursor-based pagination). - * @param params Optional pagination parameters (limit, cursor). - * @returns Paginated log entries for this item. - */ - logs(params?: AiSearchItemLogsParams): Promise; - /** - * List indexed chunks for this item (offset-based pagination). - * @param params Optional pagination parameters (limit, offset). - * @returns Paginated chunk entries for this item. - */ - chunks(params?: AiSearchItemChunksParams): Promise; -} -/** - * Items collection service for an AI Search instance. - * Provides list, upload, and access to individual items. - */ -declare abstract class AiSearchItems { - /** List items in this instance. */ - list(params?: AiSearchListItemsParams): Promise; - /** - * Upload a file as an item. Behaves as an upsert: if an item with the same - * filename already exists, it is overwritten and re-indexed. - * @param name Filename for the uploaded item. - * @param content File content as a ReadableStream, Blob, or string. - * @param options Optional metadata to attach to the item. - * @returns The created item info. - */ - upload(name: string, content: ReadableStream | Blob | string, options?: AiSearchUploadItemOptions): Promise; - /** - * Upload a file and poll until processing completes. - * Behaves as an upsert: if an item with the same filename already exists, - * it is overwritten and re-indexed. - * @param name Filename for the uploaded item. - * @param content File content as a ReadableStream, Blob, or string. - * @param options Optional metadata and polling configuration. - * @returns The item info after processing completes (or timeout). - */ - uploadAndPoll(name: string, content: ReadableStream | Blob | string, options?: AiSearchUploadItemOptions & { - /** Polling interval in milliseconds (default 1000). */ - pollIntervalMs?: number; - /** Maximum time to wait in milliseconds (default 30000). */ - timeoutMs?: number; - }): Promise; - /** - * Get an item by ID. - * @param itemId The item identifier. - * @returns Item service for info, download, sync, logs, and chunks operations. - */ - get(itemId: string): AiSearchItem; - /** - * Delete an item from the instance. - * @param itemId The item identifier. - */ - delete(itemId: string): Promise; -} -/** - * Single job service for an AI Search instance. - * Provides info, logs, and cancel operations for a specific job. - */ -declare abstract class AiSearchJob { - /** Get metadata about this job. */ - info(): Promise; - /** Get logs for this job. */ - logs(params?: AiSearchJobLogsParams): Promise; - /** - * Cancel a running job. - * @returns The updated job info. - * @throws AiSearchNotFoundError if the job does not exist. - */ - cancel(): Promise; -} -/** - * Jobs collection service for an AI Search instance. - * Provides list, create, and access to individual jobs. - */ -declare abstract class AiSearchJobs { - /** List jobs for this instance. */ - list(params?: AiSearchListJobsParams): Promise; - /** - * Create a new indexing job. - * @param params Optional job parameters. - * @returns The created job info. - */ - create(params?: AiSearchCreateJobParams): Promise; - /** - * Get a job by ID. - * @param jobId The job identifier. - * @returns Job service for info, logs, and cancel operations. - */ - get(jobId: string): AiSearchJob; -} -// ============ AI Search Binding Classes ============ -/** - * Instance-level AI Search service. - * - * Used as: - * - The return type of `AiSearchNamespace.get(name)` (namespace binding) - * - The type of `env.BLOG_SEARCH` (single instance binding via `ai_search`) - * - * Provides search, chat, update, stats, items, and jobs operations. - * - * @example - * ```ts - * // Via namespace binding - * const instance = env.AI_SEARCH.get("blog"); - * const results = await instance.search({ - * query: "How does caching work?", - * }); - * - * // Via single instance binding - * const results = await env.BLOG_SEARCH.search({ - * messages: [{ role: "user", content: "How does caching work?" }], - * }); - * ``` - */ -declare abstract class AiSearchInstance { - /** - * Search the AI Search instance for relevant chunks. - * @param params Search request with query or messages and optional AI search options. - * @returns Search response with matching chunks and search query. - */ - search(params: AiSearchSearchRequest): Promise; - /** - * Generate chat completions with AI Search context (streaming). - * @param params Chat completions request with stream: true. - * @returns ReadableStream of server-sent events. - */ - chatCompletions(params: AiSearchChatCompletionsRequest & { - stream: true; - }): Promise; - /** - * Generate chat completions with AI Search context. - * @param params Chat completions request. - * @returns Chat completion response with choices and RAG chunks. - */ - chatCompletions(params: AiSearchChatCompletionsRequest): Promise; - /** - * Update the instance configuration. - * @param config Partial configuration to update. - * @returns Updated instance info. - */ - update(config: Partial): Promise; - /** Get metadata about this instance. */ - info(): Promise; - /** - * Get instance statistics (item count, indexing status, etc.). - * @returns Statistics with counts per status, last activity time, and engine details. - */ - stats(): Promise; - /** Items collection — list, upload, and manage items in this instance. */ - get items(): AiSearchItems; - /** Jobs collection — list, create, and inspect indexing jobs. */ - get jobs(): AiSearchJobs; -} -/** - * Namespace-level AI Search service. - * - * Used as the type of `env.AI_SEARCH` (namespace binding via `ai_search_namespaces`). - * Scoped to a single namespace. Provides dynamic instance access, creation, deletion, - * and multi-instance search/chat operations. - * - * @example - * ```ts - * // Access an instance within the namespace - * const blog = env.AI_SEARCH.get("blog"); - * const results = await blog.search({ query: "How does caching work?" }); - * - * // List all instances in the namespace - * const instances = await env.AI_SEARCH.list(); - * - * // Create a new instance with built-in storage - * const tenant = await env.AI_SEARCH.create({ id: "tenant-123" }); - * - * // Upload items into the instance - * await tenant.items.upload("doc.pdf", fileContent); - * - * // Search across multiple instances - * const multi = await env.AI_SEARCH.search({ - * query: "caching", - * ai_search_options: { instance_ids: ["blog", "docs"] }, - * }); - * - * // Delete an instance - * await env.AI_SEARCH.delete("tenant-123"); - * ``` - */ -declare abstract class AiSearchNamespace { - /** - * Get an instance by name within the bound namespace. - * @param name Instance name. - * @returns Instance service for search, chat, update, stats, items, and jobs. - */ - get(name: string): AiSearchInstance; - /** - * List instances in the bound namespace. - * @param params Optional pagination, search, and ordering parameters. - * @returns Array of instance metadata with pagination info. - */ - list(params?: AiSearchListInstancesParams): Promise; - /** - * Create a new instance within the bound namespace. - * @param config Instance configuration. Only `id` is required — omit `type` and `source` to create with built-in storage. - * @returns Instance service for the newly created instance. - * - * @example - * ```ts - * // Create with built-in storage (upload items manually) - * const instance = await env.AI_SEARCH.create({ id: "my-search" }); - * - * // Create with web crawler source - * const instance = await env.AI_SEARCH.create({ - * id: "docs-search", - * type: "web-crawler", - * source: "https://developers.cloudflare.com", - * }); - * ``` - */ - create(config: AiSearchConfig): Promise; - /** - * Delete an instance from the bound namespace. - * @param name Instance name to delete. - */ - delete(name: string): Promise; - /** - * Search across multiple instances within the bound namespace. - * Fans out to the specified instance_ids and merges results. - * @param params Search request with required `ai_search_options.instance_ids`. - * @returns Search response with chunks tagged by instance_id and optional partial-failure errors. - */ - search(params: AiSearchMultiSearchRequest): Promise; - /** - * Generate chat completions across multiple instances within the bound namespace (streaming). - * Fans out to the specified instance_ids, merges context, and generates a response. - * @param params Chat completions request with stream: true and required `ai_search_options.instance_ids`. - * @returns ReadableStream of server-sent events. - */ - chatCompletions(params: AiSearchMultiChatCompletionsRequest & { - stream: true; - }): Promise; - /** - * Generate chat completions across multiple instances within the bound namespace. - * Fans out to the specified instance_ids, merges context, and generates a response. - * @param params Chat completions request with required `ai_search_options.instance_ids`. - * @returns Chat completion response with choices, chunks tagged by instance_id, and optional partial-failure errors. - */ - chatCompletions(params: AiSearchMultiChatCompletionsRequest): Promise; -} -type AiImageClassificationInput = { - image: number[]; -}; -type AiImageClassificationOutput = { - score?: number; - label?: string; -}[]; -declare abstract class BaseAiImageClassification { - inputs: AiImageClassificationInput; - postProcessedOutputs: AiImageClassificationOutput; -} -type AiImageToTextInput = { - image: number[]; - prompt?: string; - max_tokens?: number; - temperature?: number; - top_p?: number; - top_k?: number; - seed?: number; - repetition_penalty?: number; - frequency_penalty?: number; - presence_penalty?: number; - raw?: boolean; - messages?: RoleScopedChatInput[]; -}; -type AiImageToTextOutput = { - description: string; -}; -declare abstract class BaseAiImageToText { - inputs: AiImageToTextInput; - postProcessedOutputs: AiImageToTextOutput; -} -type AiImageTextToTextInput = { - image: string; - prompt?: string; - max_tokens?: number; - temperature?: number; - ignore_eos?: boolean; - top_p?: number; - top_k?: number; - seed?: number; - repetition_penalty?: number; - frequency_penalty?: number; - presence_penalty?: number; - raw?: boolean; - messages?: RoleScopedChatInput[]; -}; -type AiImageTextToTextOutput = { - description: string; -}; -declare abstract class BaseAiImageTextToText { - inputs: AiImageTextToTextInput; - postProcessedOutputs: AiImageTextToTextOutput; -} -type AiMultimodalEmbeddingsInput = { - image: string; - text: string[]; -}; -type AiIMultimodalEmbeddingsOutput = { - data: number[][]; - shape: number[]; -}; -declare abstract class BaseAiMultimodalEmbeddings { - inputs: AiImageTextToTextInput; - postProcessedOutputs: AiImageTextToTextOutput; -} -type AiObjectDetectionInput = { - image: number[]; -}; -type AiObjectDetectionOutput = { - score?: number; - label?: string; -}[]; -declare abstract class BaseAiObjectDetection { - inputs: AiObjectDetectionInput; - postProcessedOutputs: AiObjectDetectionOutput; -} -type AiSentenceSimilarityInput = { - source: string; - sentences: string[]; -}; -type AiSentenceSimilarityOutput = number[]; -declare abstract class BaseAiSentenceSimilarity { - inputs: AiSentenceSimilarityInput; - postProcessedOutputs: AiSentenceSimilarityOutput; -} -type AiAutomaticSpeechRecognitionInput = { - audio: number[]; -}; -type AiAutomaticSpeechRecognitionOutput = { - text?: string; - words?: { - word: string; - start: number; - end: number; - }[]; - vtt?: string; -}; -declare abstract class BaseAiAutomaticSpeechRecognition { - inputs: AiAutomaticSpeechRecognitionInput; - postProcessedOutputs: AiAutomaticSpeechRecognitionOutput; -} -type AiSummarizationInput = { - input_text: string; - max_length?: number; -}; -type AiSummarizationOutput = { - summary: string; -}; -declare abstract class BaseAiSummarization { - inputs: AiSummarizationInput; - postProcessedOutputs: AiSummarizationOutput; -} -type AiTextClassificationInput = { - text: string; -}; -type AiTextClassificationOutput = { - score?: number; - label?: string; -}[]; -declare abstract class BaseAiTextClassification { - inputs: AiTextClassificationInput; - postProcessedOutputs: AiTextClassificationOutput; -} -type AiTextEmbeddingsInput = { - text: string | string[]; -}; -type AiTextEmbeddingsOutput = { - shape: number[]; - data: number[][]; -}; -declare abstract class BaseAiTextEmbeddings { - inputs: AiTextEmbeddingsInput; - postProcessedOutputs: AiTextEmbeddingsOutput; -} -type RoleScopedChatInput = { - role: "user" | "assistant" | "system" | "tool" | (string & NonNullable); - content: string; - name?: string; -}; -type AiTextGenerationToolLegacyInput = { - name: string; - description: string; - parameters?: { - type: "object" | (string & NonNullable); - properties: { - [key: string]: { - type: string; - description?: string; - }; - }; - required: string[]; - }; -}; -type AiTextGenerationToolInput = { - type: "function" | (string & NonNullable); - function: { - name: string; - description: string; - parameters?: { - type: "object" | (string & NonNullable); - properties: { - [key: string]: { - type: string; - description?: string; - }; - }; - required: string[]; - }; - }; -}; -type AiTextGenerationFunctionsInput = { - name: string; - code: string; -}; -type AiTextGenerationResponseFormat = { - type: string; - json_schema?: any; -}; -type AiTextGenerationInput = { - prompt?: string; - raw?: boolean; - stream?: boolean; - max_tokens?: number; - temperature?: number; - top_p?: number; - top_k?: number; - seed?: number; - repetition_penalty?: number; - frequency_penalty?: number; - presence_penalty?: number; - messages?: RoleScopedChatInput[]; - response_format?: AiTextGenerationResponseFormat; - tools?: AiTextGenerationToolInput[] | AiTextGenerationToolLegacyInput[] | (object & NonNullable); - functions?: AiTextGenerationFunctionsInput[]; -}; -type AiTextGenerationToolLegacyOutput = { - name: string; - arguments: unknown; -}; -type AiTextGenerationToolOutput = { - id: string; - type: "function"; - function: { - name: string; - arguments: string; - }; -}; -type UsageTags = { - prompt_tokens: number; - completion_tokens: number; - total_tokens: number; -}; -type AiTextGenerationOutput = { - response?: string; - tool_calls?: AiTextGenerationToolLegacyOutput[] & AiTextGenerationToolOutput[]; - usage?: UsageTags; -}; -declare abstract class BaseAiTextGeneration { - inputs: AiTextGenerationInput; - postProcessedOutputs: AiTextGenerationOutput; -} -type AiTextToSpeechInput = { - prompt: string; - lang?: string; -}; -type AiTextToSpeechOutput = Uint8Array | { - audio: string; -}; -declare abstract class BaseAiTextToSpeech { - inputs: AiTextToSpeechInput; - postProcessedOutputs: AiTextToSpeechOutput; -} -type AiTextToImageInput = { - prompt: string; - negative_prompt?: string; - height?: number; - width?: number; - image?: number[]; - image_b64?: string; - mask?: number[]; - num_steps?: number; - strength?: number; - guidance?: number; - seed?: number; -}; -type AiTextToImageOutput = ReadableStream; -declare abstract class BaseAiTextToImage { - inputs: AiTextToImageInput; - postProcessedOutputs: AiTextToImageOutput; -} -type AiTranslationInput = { - text: string; - target_lang: string; - source_lang?: string; -}; -type AiTranslationOutput = { - translated_text?: string; -}; -declare abstract class BaseAiTranslation { - inputs: AiTranslationInput; - postProcessedOutputs: AiTranslationOutput; -} -/** - * Workers AI support for OpenAI's Chat Completions API - */ -type ChatCompletionContentPartText = { - type: "text"; - text: string; -}; -type ChatCompletionContentPartImage = { - type: "image_url"; - image_url: { - url: string; - detail?: "auto" | "low" | "high"; - }; -}; -type ChatCompletionContentPartInputAudio = { - type: "input_audio"; - input_audio: { - /** Base64 encoded audio data. */ - data: string; - format: "wav" | "mp3"; - }; -}; -type ChatCompletionContentPartFile = { - type: "file"; - file: { - /** Base64 encoded file data. */ - file_data?: string; - /** The ID of an uploaded file. */ - file_id?: string; - filename?: string; - }; -}; -type ChatCompletionContentPartRefusal = { - type: "refusal"; - refusal: string; -}; -type ChatCompletionContentPart = ChatCompletionContentPartText | ChatCompletionContentPartImage | ChatCompletionContentPartInputAudio | ChatCompletionContentPartFile; -type FunctionDefinition = { - name: string; - description?: string; - parameters?: Record; - strict?: boolean | null; -}; -type ChatCompletionFunctionTool = { - type: "function"; - function: FunctionDefinition; -}; -type ChatCompletionCustomToolGrammarFormat = { - type: "grammar"; - grammar: { - definition: string; - syntax: "lark" | "regex"; - }; -}; -type ChatCompletionCustomToolTextFormat = { - type: "text"; -}; -type ChatCompletionCustomToolFormat = ChatCompletionCustomToolTextFormat | ChatCompletionCustomToolGrammarFormat; -type ChatCompletionCustomTool = { - type: "custom"; - custom: { - name: string; - description?: string; - format?: ChatCompletionCustomToolFormat; - }; -}; -type ChatCompletionTool = ChatCompletionFunctionTool | ChatCompletionCustomTool; -type ChatCompletionMessageFunctionToolCall = { - id: string; - type: "function"; - function: { - name: string; - /** JSON-encoded arguments string. */ - arguments: string; - }; -}; -type ChatCompletionMessageCustomToolCall = { - id: string; - type: "custom"; - custom: { - name: string; - input: string; - }; -}; -type ChatCompletionMessageToolCall = ChatCompletionMessageFunctionToolCall | ChatCompletionMessageCustomToolCall; -type ChatCompletionToolChoiceFunction = { - type: "function"; - function: { - name: string; - }; -}; -type ChatCompletionToolChoiceCustom = { - type: "custom"; - custom: { - name: string; - }; -}; -type ChatCompletionToolChoiceAllowedTools = { - type: "allowed_tools"; - allowed_tools: { - mode: "auto" | "required"; - tools: Array>; - }; -}; -type ChatCompletionToolChoiceOption = "none" | "auto" | "required" | ChatCompletionToolChoiceFunction | ChatCompletionToolChoiceCustom | ChatCompletionToolChoiceAllowedTools; -type DeveloperMessage = { - role: "developer"; - content: string | Array<{ - type: "text"; - text: string; - }>; - name?: string; -}; -type SystemMessage = { - role: "system"; - content: string | Array<{ - type: "text"; - text: string; - }>; - name?: string; -}; -/** - * Permissive merged content part used inside UserMessage arrays. - * - * Cabidela has a limitation where anyOf/oneOf with enum-based discrimination - * inside nested array items does not correctly match different branches for - * different array elements, so the schema uses a single merged object. - */ -type UserMessageContentPart = { - type: "text" | "image_url" | "input_audio" | "file"; - text?: string; - image_url?: { - url?: string; - detail?: "auto" | "low" | "high"; - }; - input_audio?: { - data?: string; - format?: "wav" | "mp3"; - }; - file?: { - file_data?: string; - file_id?: string; - filename?: string; - }; -}; -type UserMessage = { - role: "user"; - content: string | Array; - name?: string; -}; -type AssistantMessageContentPart = { - type: "text" | "refusal"; - text?: string; - refusal?: string; -}; -type AssistantMessage = { - role: "assistant"; - content?: string | null | Array; - refusal?: string | null; - name?: string; - audio?: { - id: string; - }; - tool_calls?: Array; - function_call?: { - name: string; - arguments: string; - }; -}; -type ToolMessage = { - role: "tool"; - content: string | Array<{ - type: "text"; - text: string; - }>; - tool_call_id: string; -}; -type FunctionMessage = { - role: "function"; - content: string; - name: string; -}; -type ChatCompletionMessageParam = DeveloperMessage | SystemMessage | UserMessage | AssistantMessage | ToolMessage | FunctionMessage; -type ChatCompletionsResponseFormatText = { - type: "text"; -}; -type ChatCompletionsResponseFormatJSONObject = { - type: "json_object"; -}; -type ResponseFormatJSONSchema = { - type: "json_schema"; - json_schema: { - name: string; - description?: string; - schema?: Record; - strict?: boolean | null; - }; -}; -type ResponseFormat = ChatCompletionsResponseFormatText | ChatCompletionsResponseFormatJSONObject | ResponseFormatJSONSchema; -type ChatCompletionsStreamOptions = { - include_usage?: boolean; - include_obfuscation?: boolean; -}; -type PredictionContent = { - type: "content"; - content: string | Array<{ - type: "text"; - text: string; - }>; -}; -type AudioParams = { - voice: string | { - id: string; - }; - format: "wav" | "aac" | "mp3" | "flac" | "opus" | "pcm16"; -}; -type WebSearchUserLocation = { - type: "approximate"; - approximate: { - city?: string; - country?: string; - region?: string; - timezone?: string; - }; -}; -type WebSearchOptions = { - search_context_size?: "low" | "medium" | "high"; - user_location?: WebSearchUserLocation; -}; -type ChatTemplateKwargs = { - /** Whether to enable reasoning, enabled by default. */ - enable_thinking?: boolean; - /** If false, preserves reasoning context between turns. */ - clear_thinking?: boolean; -}; -/** Shared optional properties used by both Prompt and Messages input branches. */ -type ChatCompletionsCommonOptions = { - model?: string; - audio?: AudioParams; - frequency_penalty?: number | null; - logit_bias?: Record | null; - logprobs?: boolean | null; - top_logprobs?: number | null; - max_tokens?: number | null; - max_completion_tokens?: number | null; - metadata?: Record | null; - modalities?: Array<"text" | "audio"> | null; - n?: number | null; - parallel_tool_calls?: boolean; - prediction?: PredictionContent; - presence_penalty?: number | null; - reasoning_effort?: "low" | "medium" | "high" | null; - chat_template_kwargs?: ChatTemplateKwargs; - response_format?: ResponseFormat; - seed?: number | null; - service_tier?: "auto" | "default" | "flex" | "scale" | "priority" | null; - stop?: string | Array | null; - store?: boolean | null; - stream?: boolean | null; - stream_options?: ChatCompletionsStreamOptions; - temperature?: number | null; - tool_choice?: ChatCompletionToolChoiceOption; - tools?: Array; - top_p?: number | null; - user?: string; - web_search_options?: WebSearchOptions; - function_call?: "none" | "auto" | { - name: string; - }; - functions?: Array; -}; -type PromptTokensDetails = { - cached_tokens?: number; - audio_tokens?: number; -}; -type CompletionTokensDetails = { - reasoning_tokens?: number; - audio_tokens?: number; - accepted_prediction_tokens?: number; - rejected_prediction_tokens?: number; -}; -type CompletionUsage = { - prompt_tokens: number; - completion_tokens: number; - total_tokens: number; - prompt_tokens_details?: PromptTokensDetails; - completion_tokens_details?: CompletionTokensDetails; -}; -type ChatCompletionTopLogprob = { - token: string; - logprob: number; - bytes: Array | null; -}; -type ChatCompletionTokenLogprob = { - token: string; - logprob: number; - bytes: Array | null; - top_logprobs: Array; -}; -type ChatCompletionAudio = { - id: string; - /** Base64 encoded audio bytes. */ - data: string; - expires_at: number; - transcript: string; -}; -type ChatCompletionUrlCitation = { - type: "url_citation"; - url_citation: { - url: string; - title: string; - start_index: number; - end_index: number; - }; -}; -type ChatCompletionResponseMessage = { - role: "assistant"; - content: string | null; - refusal: string | null; - annotations?: Array; - audio?: ChatCompletionAudio; - tool_calls?: Array; - function_call?: { - name: string; - arguments: string; - } | null; -}; -type ChatCompletionLogprobs = { - content: Array | null; - refusal?: Array | null; -}; -type ChatCompletionChoice = { - index: number; - message: ChatCompletionResponseMessage; - finish_reason: "stop" | "length" | "tool_calls" | "content_filter" | "function_call"; - logprobs: ChatCompletionLogprobs | null; -}; -type ChatCompletionsPromptInput = { - prompt: string; -} & ChatCompletionsCommonOptions; -type ChatCompletionsMessagesInput = { - messages: Array; -} & ChatCompletionsCommonOptions; -type ChatCompletionsOutput = { - id: string; - object: string; - created: number; - model: string; - choices: Array; - usage?: CompletionUsage; - system_fingerprint?: string | null; - service_tier?: "auto" | "default" | "flex" | "scale" | "priority" | null; -}; -/** - * Workers AI support for OpenAI's Responses API - * Reference: https://github.com/openai/openai-node/blob/master/src/resources/responses/responses.ts - * - * It's a stripped down version from its source. - * It currently supports basic function calling, json mode and accepts images as input. - * - * It does not include types for WebSearch, CodeInterpreter, FileInputs, MCP, CustomTools. - * We plan to add those incrementally as model + platform capabilities evolve. - */ -type ResponsesInput = { - background?: boolean | null; - conversation?: string | ResponseConversationParam | null; - include?: Array | null; - input?: string | ResponseInput; - instructions?: string | null; - max_output_tokens?: number | null; - parallel_tool_calls?: boolean | null; - previous_response_id?: string | null; - prompt_cache_key?: string; - reasoning?: Reasoning | null; - safety_identifier?: string; - service_tier?: "auto" | "default" | "flex" | "scale" | "priority" | null; - stream?: boolean | null; - stream_options?: StreamOptions | null; - temperature?: number | null; - text?: ResponseTextConfig; - tool_choice?: ToolChoiceOptions | ToolChoiceFunction; - tools?: Array; - top_p?: number | null; - truncation?: "auto" | "disabled" | null; -}; -type ResponsesOutput = { - id?: string; - created_at?: number; - output_text?: string; - error?: ResponseError | null; - incomplete_details?: ResponseIncompleteDetails | null; - instructions?: string | Array | null; - object?: "response"; - output?: Array; - parallel_tool_calls?: boolean; - temperature?: number | null; - tool_choice?: ToolChoiceOptions | ToolChoiceFunction; - tools?: Array; - top_p?: number | null; - max_output_tokens?: number | null; - previous_response_id?: string | null; - prompt?: ResponsePrompt | null; - reasoning?: Reasoning | null; - safety_identifier?: string; - service_tier?: "auto" | "default" | "flex" | "scale" | "priority" | null; - status?: ResponseStatus; - text?: ResponseTextConfig; - truncation?: "auto" | "disabled" | null; - usage?: ResponseUsage; -}; -type EasyInputMessage = { - content: string | ResponseInputMessageContentList; - role: "user" | "assistant" | "system" | "developer"; - type?: "message"; -}; -type ResponsesFunctionTool = { - name: string; - parameters: { - [key: string]: unknown; - } | null; - strict: boolean | null; - type: "function"; - description?: string | null; -}; -type ResponseIncompleteDetails = { - reason?: "max_output_tokens" | "content_filter"; -}; -type ResponsePrompt = { - id: string; - variables?: { - [key: string]: string | ResponseInputText | ResponseInputImage; - } | null; - version?: string | null; -}; -type Reasoning = { - effort?: ReasoningEffort | null; - generate_summary?: "auto" | "concise" | "detailed" | null; - summary?: "auto" | "concise" | "detailed" | null; -}; -type ResponseContent = ResponseInputText | ResponseInputImage | ResponseOutputText | ResponseOutputRefusal | ResponseContentReasoningText; -type ResponseContentReasoningText = { - text: string; - type: "reasoning_text"; -}; -type ResponseConversationParam = { - id: string; -}; -type ResponseCreatedEvent = { - response: Response; - sequence_number: number; - type: "response.created"; -}; -type ResponseCustomToolCallOutput = { - call_id: string; - output: string | Array; - type: "custom_tool_call_output"; - id?: string; -}; -type ResponseError = { - code: "server_error" | "rate_limit_exceeded" | "invalid_prompt" | "vector_store_timeout" | "invalid_image" | "invalid_image_format" | "invalid_base64_image" | "invalid_image_url" | "image_too_large" | "image_too_small" | "image_parse_error" | "image_content_policy_violation" | "invalid_image_mode" | "image_file_too_large" | "unsupported_image_media_type" | "empty_image_file" | "failed_to_download_image" | "image_file_not_found"; - message: string; -}; -type ResponseErrorEvent = { - code: string | null; - message: string; - param: string | null; - sequence_number: number; - type: "error"; -}; -type ResponseFailedEvent = { - response: Response; - sequence_number: number; - type: "response.failed"; -}; -type ResponseFormatText = { - type: "text"; -}; -type ResponseFormatJSONObject = { - type: "json_object"; -}; -type ResponseFormatTextConfig = ResponseFormatText | ResponseFormatTextJSONSchemaConfig | ResponseFormatJSONObject; -type ResponseFormatTextJSONSchemaConfig = { - name: string; - schema: { - [key: string]: unknown; - }; - type: "json_schema"; - description?: string; - strict?: boolean | null; -}; -type ResponseFunctionCallArgumentsDeltaEvent = { - delta: string; - item_id: string; - output_index: number; - sequence_number: number; - type: "response.function_call_arguments.delta"; -}; -type ResponseFunctionCallArgumentsDoneEvent = { - arguments: string; - item_id: string; - name: string; - output_index: number; - sequence_number: number; - type: "response.function_call_arguments.done"; -}; -type ResponseFunctionCallOutputItem = ResponseInputTextContent | ResponseInputImageContent; -type ResponseFunctionCallOutputItemList = Array; -type ResponseFunctionToolCall = { - arguments: string; - call_id: string; - name: string; - type: "function_call"; - id?: string; - status?: "in_progress" | "completed" | "incomplete"; -}; -interface ResponseFunctionToolCallItem extends ResponseFunctionToolCall { - id: string; -} -type ResponseFunctionToolCallOutputItem = { - id: string; - call_id: string; - output: string | Array; - type: "function_call_output"; - status?: "in_progress" | "completed" | "incomplete"; -}; -type ResponseIncludable = "message.input_image.image_url" | "message.output_text.logprobs"; -type ResponseIncompleteEvent = { - response: Response; - sequence_number: number; - type: "response.incomplete"; -}; -type ResponseInput = Array; -type ResponseInputContent = ResponseInputText | ResponseInputImage; -type ResponseInputImage = { - detail: "low" | "high" | "auto"; - type: "input_image"; - /** - * Base64 encoded image - */ - image_url?: string | null; -}; -type ResponseInputImageContent = { - type: "input_image"; - detail?: "low" | "high" | "auto" | null; - /** - * Base64 encoded image - */ - image_url?: string | null; -}; -type ResponseInputItem = EasyInputMessage | ResponseInputItemMessage | ResponseOutputMessage | ResponseFunctionToolCall | ResponseInputItemFunctionCallOutput | ResponseReasoningItem; -type ResponseInputItemFunctionCallOutput = { - call_id: string; - output: string | ResponseFunctionCallOutputItemList; - type: "function_call_output"; - id?: string | null; - status?: "in_progress" | "completed" | "incomplete" | null; -}; -type ResponseInputItemMessage = { - content: ResponseInputMessageContentList; - role: "user" | "system" | "developer"; - status?: "in_progress" | "completed" | "incomplete"; - type?: "message"; -}; -type ResponseInputMessageContentList = Array; -type ResponseInputMessageItem = { - id: string; - content: ResponseInputMessageContentList; - role: "user" | "system" | "developer"; - status?: "in_progress" | "completed" | "incomplete"; - type?: "message"; -}; -type ResponseInputText = { - text: string; - type: "input_text"; -}; -type ResponseInputTextContent = { - text: string; - type: "input_text"; -}; -type ResponseItem = ResponseInputMessageItem | ResponseOutputMessage | ResponseFunctionToolCallItem | ResponseFunctionToolCallOutputItem; -type ResponseOutputItem = ResponseOutputMessage | ResponseFunctionToolCall | ResponseReasoningItem; -type ResponseOutputItemAddedEvent = { - item: ResponseOutputItem; - output_index: number; - sequence_number: number; - type: "response.output_item.added"; -}; -type ResponseOutputItemDoneEvent = { - item: ResponseOutputItem; - output_index: number; - sequence_number: number; - type: "response.output_item.done"; -}; -type ResponseOutputMessage = { - id: string; - content: Array; - role: "assistant"; - status: "in_progress" | "completed" | "incomplete"; - type: "message"; -}; -type ResponseOutputRefusal = { - refusal: string; - type: "refusal"; -}; -type ResponseOutputText = { - text: string; - type: "output_text"; - logprobs?: Array; -}; -type ResponseReasoningItem = { - id: string; - summary: Array; - type: "reasoning"; - content?: Array; - encrypted_content?: string | null; - status?: "in_progress" | "completed" | "incomplete"; -}; -type ResponseReasoningSummaryItem = { - text: string; - type: "summary_text"; -}; -type ResponseReasoningContentItem = { - text: string; - type: "reasoning_text"; -}; -type ResponseReasoningTextDeltaEvent = { - content_index: number; - delta: string; - item_id: string; - output_index: number; - sequence_number: number; - type: "response.reasoning_text.delta"; -}; -type ResponseReasoningTextDoneEvent = { - content_index: number; - item_id: string; - output_index: number; - sequence_number: number; - text: string; - type: "response.reasoning_text.done"; -}; -type ResponseRefusalDeltaEvent = { - content_index: number; - delta: string; - item_id: string; - output_index: number; - sequence_number: number; - type: "response.refusal.delta"; -}; -type ResponseRefusalDoneEvent = { - content_index: number; - item_id: string; - output_index: number; - refusal: string; - sequence_number: number; - type: "response.refusal.done"; -}; -type ResponseStatus = "completed" | "failed" | "in_progress" | "cancelled" | "queued" | "incomplete"; -type ResponseStreamEvent = ResponseCompletedEvent | ResponseCreatedEvent | ResponseErrorEvent | ResponseFunctionCallArgumentsDeltaEvent | ResponseFunctionCallArgumentsDoneEvent | ResponseFailedEvent | ResponseIncompleteEvent | ResponseOutputItemAddedEvent | ResponseOutputItemDoneEvent | ResponseReasoningTextDeltaEvent | ResponseReasoningTextDoneEvent | ResponseRefusalDeltaEvent | ResponseRefusalDoneEvent | ResponseTextDeltaEvent | ResponseTextDoneEvent; -type ResponseCompletedEvent = { - response: Response; - sequence_number: number; - type: "response.completed"; -}; -type ResponseTextConfig = { - format?: ResponseFormatTextConfig; - verbosity?: "low" | "medium" | "high" | null; -}; -type ResponseTextDeltaEvent = { - content_index: number; - delta: string; - item_id: string; - logprobs: Array; - output_index: number; - sequence_number: number; - type: "response.output_text.delta"; -}; -type ResponseTextDoneEvent = { - content_index: number; - item_id: string; - logprobs: Array; - output_index: number; - sequence_number: number; - text: string; - type: "response.output_text.done"; -}; -type Logprob = { - token: string; - logprob: number; - top_logprobs?: Array; -}; -type TopLogprob = { - token?: string; - logprob?: number; -}; -type ResponseUsage = { - input_tokens: number; - output_tokens: number; - total_tokens: number; -}; -type Tool = ResponsesFunctionTool; -type ToolChoiceFunction = { - name: string; - type: "function"; -}; -type ToolChoiceOptions = "none"; -type ReasoningEffort = "minimal" | "low" | "medium" | "high" | null; -type StreamOptions = { - include_obfuscation?: boolean; -}; -/** Marks keys from T that aren't in U as optional never */ -type Without = { - [P in Exclude]?: never; -}; -/** Either T or U, but not both (mutually exclusive) */ -type XOR = (T & Without) | (U & Without); -type Ai_Cf_Baai_Bge_Base_En_V1_5_Input = { - text: string | string[]; - /** - * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. - */ - pooling?: "mean" | "cls"; -} | { - /** - * Batch of the embeddings requests to run using async-queue - */ - requests: { - text: string | string[]; - /** - * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. - */ - pooling?: "mean" | "cls"; - }[]; -}; -type Ai_Cf_Baai_Bge_Base_En_V1_5_Output = { - shape?: number[]; - /** - * Embeddings of the requested text values - */ - data?: number[][]; - /** - * The pooling method used in the embedding process. - */ - pooling?: "mean" | "cls"; -} | Ai_Cf_Baai_Bge_Base_En_V1_5_AsyncResponse; -interface Ai_Cf_Baai_Bge_Base_En_V1_5_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Baai_Bge_Base_En_V1_5 { - inputs: Ai_Cf_Baai_Bge_Base_En_V1_5_Input; - postProcessedOutputs: Ai_Cf_Baai_Bge_Base_En_V1_5_Output; -} -type Ai_Cf_Openai_Whisper_Input = string | { - /** - * An array of integers that represent the audio data constrained to 8-bit unsigned integer values - */ - audio: number[]; -}; -interface Ai_Cf_Openai_Whisper_Output { - /** - * The transcription - */ - text: string; - word_count?: number; - words?: { - word?: string; - /** - * The second this word begins in the recording - */ - start?: number; - /** - * The ending second when the word completes - */ - end?: number; - }[]; - vtt?: string; -} -declare abstract class Base_Ai_Cf_Openai_Whisper { - inputs: Ai_Cf_Openai_Whisper_Input; - postProcessedOutputs: Ai_Cf_Openai_Whisper_Output; -} -type Ai_Cf_Meta_M2M100_1_2B_Input = { - /** - * The text to be translated - */ - text: string; - /** - * The language code of the source text (e.g., 'en' for English). Defaults to 'en' if not specified - */ - source_lang?: string; - /** - * The language code to translate the text into (e.g., 'es' for Spanish) - */ - target_lang: string; -} | { - /** - * Batch of the embeddings requests to run using async-queue - */ - requests: { - /** - * The text to be translated - */ - text: string; - /** - * The language code of the source text (e.g., 'en' for English). Defaults to 'en' if not specified - */ - source_lang?: string; - /** - * The language code to translate the text into (e.g., 'es' for Spanish) - */ - target_lang: string; - }[]; -}; -type Ai_Cf_Meta_M2M100_1_2B_Output = { - /** - * The translated text in the target language - */ - translated_text?: string; -} | Ai_Cf_Meta_M2M100_1_2B_AsyncResponse; -interface Ai_Cf_Meta_M2M100_1_2B_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Meta_M2M100_1_2B { - inputs: Ai_Cf_Meta_M2M100_1_2B_Input; - postProcessedOutputs: Ai_Cf_Meta_M2M100_1_2B_Output; -} -type Ai_Cf_Baai_Bge_Small_En_V1_5_Input = { - text: string | string[]; - /** - * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. - */ - pooling?: "mean" | "cls"; -} | { - /** - * Batch of the embeddings requests to run using async-queue - */ - requests: { - text: string | string[]; - /** - * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. - */ - pooling?: "mean" | "cls"; - }[]; -}; -type Ai_Cf_Baai_Bge_Small_En_V1_5_Output = { - shape?: number[]; - /** - * Embeddings of the requested text values - */ - data?: number[][]; - /** - * The pooling method used in the embedding process. - */ - pooling?: "mean" | "cls"; -} | Ai_Cf_Baai_Bge_Small_En_V1_5_AsyncResponse; -interface Ai_Cf_Baai_Bge_Small_En_V1_5_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Baai_Bge_Small_En_V1_5 { - inputs: Ai_Cf_Baai_Bge_Small_En_V1_5_Input; - postProcessedOutputs: Ai_Cf_Baai_Bge_Small_En_V1_5_Output; -} -type Ai_Cf_Baai_Bge_Large_En_V1_5_Input = { - text: string | string[]; - /** - * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. - */ - pooling?: "mean" | "cls"; -} | { - /** - * Batch of the embeddings requests to run using async-queue - */ - requests: { - text: string | string[]; - /** - * The pooling method used in the embedding process. `cls` pooling will generate more accurate embeddings on larger inputs - however, embeddings created with cls pooling are not compatible with embeddings generated with mean pooling. The default pooling method is `mean` in order for this to not be a breaking change, but we highly suggest using the new `cls` pooling for better accuracy. - */ - pooling?: "mean" | "cls"; - }[]; -}; -type Ai_Cf_Baai_Bge_Large_En_V1_5_Output = { - shape?: number[]; - /** - * Embeddings of the requested text values - */ - data?: number[][]; - /** - * The pooling method used in the embedding process. - */ - pooling?: "mean" | "cls"; -} | Ai_Cf_Baai_Bge_Large_En_V1_5_AsyncResponse; -interface Ai_Cf_Baai_Bge_Large_En_V1_5_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Baai_Bge_Large_En_V1_5 { - inputs: Ai_Cf_Baai_Bge_Large_En_V1_5_Input; - postProcessedOutputs: Ai_Cf_Baai_Bge_Large_En_V1_5_Output; -} -type Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Input = string | { - /** - * The input text prompt for the model to generate a response. - */ - prompt?: string; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * Controls the creativity of the AI's responses by adjusting how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; - image: number[] | (string & NonNullable); - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; -}; -interface Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Output { - description?: string; -} -declare abstract class Base_Ai_Cf_Unum_Uform_Gen2_Qwen_500M { - inputs: Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Input; - postProcessedOutputs: Ai_Cf_Unum_Uform_Gen2_Qwen_500M_Output; -} -type Ai_Cf_Openai_Whisper_Tiny_En_Input = string | { - /** - * An array of integers that represent the audio data constrained to 8-bit unsigned integer values - */ - audio: number[]; -}; -interface Ai_Cf_Openai_Whisper_Tiny_En_Output { - /** - * The transcription - */ - text: string; - word_count?: number; - words?: { - word?: string; - /** - * The second this word begins in the recording - */ - start?: number; - /** - * The ending second when the word completes - */ - end?: number; - }[]; - vtt?: string; -} -declare abstract class Base_Ai_Cf_Openai_Whisper_Tiny_En { - inputs: Ai_Cf_Openai_Whisper_Tiny_En_Input; - postProcessedOutputs: Ai_Cf_Openai_Whisper_Tiny_En_Output; -} -interface Ai_Cf_Openai_Whisper_Large_V3_Turbo_Input { - audio: string | { - body?: object; - contentType?: string; - }; - /** - * Supported tasks are 'translate' or 'transcribe'. - */ - task?: string; - /** - * The language of the audio being transcribed or translated. - */ - language?: string; - /** - * Preprocess the audio with a voice activity detection model. - */ - vad_filter?: boolean; - /** - * A text prompt to help provide context to the model on the contents of the audio. - */ - initial_prompt?: string; - /** - * The prefix appended to the beginning of the output of the transcription and can guide the transcription result. - */ - prefix?: string; - /** - * The number of beams to use in beam search decoding. Higher values may improve accuracy at the cost of speed. - */ - beam_size?: number; - /** - * Whether to condition on previous text during transcription. Setting to false may help prevent hallucination loops. - */ - condition_on_previous_text?: boolean; - /** - * Threshold for detecting no-speech segments. Segments with no-speech probability above this value are skipped. - */ - no_speech_threshold?: number; - /** - * Threshold for filtering out segments with high compression ratio, which often indicate repetitive or hallucinated text. - */ - compression_ratio_threshold?: number; - /** - * Threshold for filtering out segments with low average log probability, indicating low confidence. - */ - log_prob_threshold?: number; - /** - * Optional threshold (in seconds) to skip silent periods that may cause hallucinations. - */ - hallucination_silence_threshold?: number; -} -interface Ai_Cf_Openai_Whisper_Large_V3_Turbo_Output { - transcription_info?: { - /** - * The language of the audio being transcribed or translated. - */ - language?: string; - /** - * The confidence level or probability of the detected language being accurate, represented as a decimal between 0 and 1. - */ - language_probability?: number; - /** - * The total duration of the original audio file, in seconds. - */ - duration?: number; - /** - * The duration of the audio after applying Voice Activity Detection (VAD) to remove silent or irrelevant sections, in seconds. - */ - duration_after_vad?: number; - }; - /** - * The complete transcription of the audio. - */ - text: string; - /** - * The total number of words in the transcription. - */ - word_count?: number; - segments?: { - /** - * The starting time of the segment within the audio, in seconds. - */ - start?: number; - /** - * The ending time of the segment within the audio, in seconds. - */ - end?: number; - /** - * The transcription of the segment. - */ - text?: string; - /** - * The temperature used in the decoding process, controlling randomness in predictions. Lower values result in more deterministic outputs. - */ - temperature?: number; - /** - * The average log probability of the predictions for the words in this segment, indicating overall confidence. - */ - avg_logprob?: number; - /** - * The compression ratio of the input to the output, measuring how much the text was compressed during the transcription process. - */ - compression_ratio?: number; - /** - * The probability that the segment contains no speech, represented as a decimal between 0 and 1. - */ - no_speech_prob?: number; - words?: { - /** - * The individual word transcribed from the audio. - */ - word?: string; - /** - * The starting time of the word within the audio, in seconds. - */ - start?: number; - /** - * The ending time of the word within the audio, in seconds. - */ - end?: number; - }[]; - }[]; - /** - * The transcription in WebVTT format, which includes timing and text information for use in subtitles. - */ - vtt?: string; -} -declare abstract class Base_Ai_Cf_Openai_Whisper_Large_V3_Turbo { - inputs: Ai_Cf_Openai_Whisper_Large_V3_Turbo_Input; - postProcessedOutputs: Ai_Cf_Openai_Whisper_Large_V3_Turbo_Output; -} -type Ai_Cf_Baai_Bge_M3_Input = Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts | Ai_Cf_Baai_Bge_M3_Input_Embedding | { - /** - * Batch of the embeddings requests to run using async-queue - */ - requests: (Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts_1 | Ai_Cf_Baai_Bge_M3_Input_Embedding_1)[]; -}; -interface Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts { - /** - * A query you wish to perform against the provided contexts. If no query is provided the model with respond with embeddings for contexts - */ - query?: string; - /** - * List of provided contexts. Note that the index in this array is important, as the response will refer to it. - */ - contexts: { - /** - * One of the provided context content - */ - text?: string; - }[]; - /** - * When provided with too long context should the model error out or truncate the context to fit? - */ - truncate_inputs?: boolean; -} -interface Ai_Cf_Baai_Bge_M3_Input_Embedding { - text: string | string[]; - /** - * When provided with too long context should the model error out or truncate the context to fit? - */ - truncate_inputs?: boolean; -} -interface Ai_Cf_Baai_Bge_M3_Input_QueryAnd_Contexts_1 { - /** - * A query you wish to perform against the provided contexts. If no query is provided the model with respond with embeddings for contexts - */ - query?: string; - /** - * List of provided contexts. Note that the index in this array is important, as the response will refer to it. - */ - contexts: { - /** - * One of the provided context content - */ - text?: string; - }[]; - /** - * When provided with too long context should the model error out or truncate the context to fit? - */ - truncate_inputs?: boolean; -} -interface Ai_Cf_Baai_Bge_M3_Input_Embedding_1 { - text: string | string[]; - /** - * When provided with too long context should the model error out or truncate the context to fit? - */ - truncate_inputs?: boolean; -} -type Ai_Cf_Baai_Bge_M3_Output = Ai_Cf_Baai_Bge_M3_Output_Query | Ai_Cf_Baai_Bge_M3_Output_EmbeddingFor_Contexts | Ai_Cf_Baai_Bge_M3_Output_Embedding | Ai_Cf_Baai_Bge_M3_AsyncResponse; -interface Ai_Cf_Baai_Bge_M3_Output_Query { - response?: { - /** - * Index of the context in the request - */ - id?: number; - /** - * Score of the context under the index. - */ - score?: number; - }[]; -} -interface Ai_Cf_Baai_Bge_M3_Output_EmbeddingFor_Contexts { - response?: number[][]; - shape?: number[]; - /** - * The pooling method used in the embedding process. - */ - pooling?: "mean" | "cls"; -} -interface Ai_Cf_Baai_Bge_M3_Output_Embedding { - shape?: number[]; - /** - * Embeddings of the requested text values - */ - data?: number[][]; - /** - * The pooling method used in the embedding process. - */ - pooling?: "mean" | "cls"; -} -interface Ai_Cf_Baai_Bge_M3_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Baai_Bge_M3 { - inputs: Ai_Cf_Baai_Bge_M3_Input; - postProcessedOutputs: Ai_Cf_Baai_Bge_M3_Output; -} -interface Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Input { - /** - * A text description of the image you want to generate. - */ - prompt: string; - /** - * The number of diffusion steps; higher values can improve quality but take longer. - */ - steps?: number; -} -interface Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Output { - /** - * The generated image in Base64 format. - */ - image?: string; -} -declare abstract class Base_Ai_Cf_Black_Forest_Labs_Flux_1_Schnell { - inputs: Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Input; - postProcessedOutputs: Ai_Cf_Black_Forest_Labs_Flux_1_Schnell_Output; -} -type Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Input = Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Prompt | Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Messages; -interface Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - image?: number[] | (string & NonNullable); - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; - /** - * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. - */ - lora?: string; -} -interface Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role?: string; - /** - * The tool call id. If you don't know what to put here you can fall back to 000000001 - */ - tool_call_id?: string; - content?: string | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }[] | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }; - }[]; - image?: number[] | (string & NonNullable); - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - /** - * If true, the response will be streamed back incrementally. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Controls the creativity of the AI's responses by adjusting how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -type Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Output = { - /** - * The generated text response from the model - */ - response?: string; - /** - * An array of tool calls requests made during the response generation - */ - tool_calls?: { - /** - * The arguments passed to be passed to the tool call request - */ - arguments?: object; - /** - * The name of the tool to be called - */ - name?: string; - }[]; -}; -declare abstract class Base_Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct { - inputs: Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Input; - postProcessedOutputs: Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct_Output; -} -type Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Input = Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Prompt | Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Messages | Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Async_Batch; -interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. - */ - lora?: string; - response_format?: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role: string; - content: string | { - /** - * Type of the content (text) - */ - type?: string; - /** - * Text content - */ - text?: string; - }[]; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_1; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_1 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Async_Batch { - requests?: { - /** - * User-supplied reference. This field will be present in the response as well it can be used to reference the request and response. It's NOT validated to be unique. - */ - external_reference?: string; - /** - * Prompt for the text generation model - */ - prompt?: string; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; - response_format?: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_2; - }[]; -} -interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_JSON_Mode_2 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -type Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Output = { - /** - * The generated text response from the model - */ - response: string; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * An array of tool calls requests made during the response generation - */ - tool_calls?: { - /** - * The arguments passed to be passed to the tool call request - */ - arguments?: object; - /** - * The name of the tool to be called - */ - name?: string; - }[]; -} | string | Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_AsyncResponse; -interface Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast { - inputs: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Input; - postProcessedOutputs: Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast_Output; -} -interface Ai_Cf_Meta_Llama_Guard_3_8B_Input { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender must alternate between 'user' and 'assistant'. - */ - role: "user" | "assistant"; - /** - * The content of the message as a string. - */ - content: string; - }[]; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Dictate the output format of the generated response. - */ - response_format?: { - /** - * Set to json_object to process and output generated text as JSON. - */ - type?: string; - }; -} -interface Ai_Cf_Meta_Llama_Guard_3_8B_Output { - response?: string | { - /** - * Whether the conversation is safe or not. - */ - safe?: boolean; - /** - * A list of what hazard categories predicted for the conversation, if the conversation is deemed unsafe. - */ - categories?: string[]; - }; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; -} -declare abstract class Base_Ai_Cf_Meta_Llama_Guard_3_8B { - inputs: Ai_Cf_Meta_Llama_Guard_3_8B_Input; - postProcessedOutputs: Ai_Cf_Meta_Llama_Guard_3_8B_Output; -} -interface Ai_Cf_Baai_Bge_Reranker_Base_Input { - /** - * A query you wish to perform against the provided contexts. - */ - /** - * Number of returned results starting with the best score. - */ - top_k?: number; - /** - * List of provided contexts. Note that the index in this array is important, as the response will refer to it. - */ - contexts: { - /** - * One of the provided context content - */ - text?: string; - }[]; -} -interface Ai_Cf_Baai_Bge_Reranker_Base_Output { - response?: { - /** - * Index of the context in the request - */ - id?: number; - /** - * Score of the context under the index. - */ - score?: number; - }[]; -} -declare abstract class Base_Ai_Cf_Baai_Bge_Reranker_Base { - inputs: Ai_Cf_Baai_Bge_Reranker_Base_Input; - postProcessedOutputs: Ai_Cf_Baai_Bge_Reranker_Base_Output; -} -type Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Input = Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Prompt | Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Messages; -interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. - */ - lora?: string; - response_format?: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role: string; - /** - * The content of the message as a string. - */ - content: string; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode_1; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_JSON_Mode_1 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -type Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Output = { - /** - * The generated text response from the model - */ - response: string; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * An array of tool calls requests made during the response generation - */ - tool_calls?: { - /** - * The arguments passed to be passed to the tool call request - */ - arguments?: object; - /** - * The name of the tool to be called - */ - name?: string; - }[]; -}; -declare abstract class Base_Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct { - inputs: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Input; - postProcessedOutputs: Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct_Output; -} -type Ai_Cf_Qwen_Qwq_32B_Input = Ai_Cf_Qwen_Qwq_32B_Prompt | Ai_Cf_Qwen_Qwq_32B_Messages; -interface Ai_Cf_Qwen_Qwq_32B_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * JSON schema that should be fulfilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Qwen_Qwq_32B_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role?: string; - /** - * The tool call id. If you don't know what to put here you can fall back to 000000001 - */ - tool_call_id?: string; - content?: string | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }[] | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - /** - * JSON schema that should be fufilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -type Ai_Cf_Qwen_Qwq_32B_Output = { - /** - * The generated text response from the model - */ - response: string; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * An array of tool calls requests made during the response generation - */ - tool_calls?: { - /** - * The arguments passed to be passed to the tool call request - */ - arguments?: object; - /** - * The name of the tool to be called - */ - name?: string; - }[]; -}; -declare abstract class Base_Ai_Cf_Qwen_Qwq_32B { - inputs: Ai_Cf_Qwen_Qwq_32B_Input; - postProcessedOutputs: Ai_Cf_Qwen_Qwq_32B_Output; -} -type Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Input = Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Prompt | Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Messages; -interface Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * JSON schema that should be fulfilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role?: string; - /** - * The tool call id. Must be supplied for tool calls for Mistral-3. If you don't know what to put here you can fall back to 000000001 - */ - tool_call_id?: string; - content?: string | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }[] | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - /** - * JSON schema that should be fufilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -type Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Output = { - /** - * The generated text response from the model - */ - response: string; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * An array of tool calls requests made during the response generation - */ - tool_calls?: { - /** - * The arguments passed to be passed to the tool call request - */ - arguments?: object; - /** - * The name of the tool to be called - */ - name?: string; - }[]; -}; -declare abstract class Base_Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct { - inputs: Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Input; - postProcessedOutputs: Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct_Output; -} -type Ai_Cf_Google_Gemma_3_12B_It_Input = Ai_Cf_Google_Gemma_3_12B_It_Prompt | Ai_Cf_Google_Gemma_3_12B_It_Messages; -interface Ai_Cf_Google_Gemma_3_12B_It_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * JSON schema that should be fufilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Google_Gemma_3_12B_It_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role?: string; - content?: string | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }[]; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - /** - * JSON schema that should be fufilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -type Ai_Cf_Google_Gemma_3_12B_It_Output = { - /** - * The generated text response from the model - */ - response: string; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * An array of tool calls requests made during the response generation - */ - tool_calls?: { - /** - * The arguments passed to be passed to the tool call request - */ - arguments?: object; - /** - * The name of the tool to be called - */ - name?: string; - }[]; -}; -declare abstract class Base_Ai_Cf_Google_Gemma_3_12B_It { - inputs: Ai_Cf_Google_Gemma_3_12B_It_Input; - postProcessedOutputs: Ai_Cf_Google_Gemma_3_12B_It_Output; -} -type Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Input = Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt | Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages | Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Async_Batch; -interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * JSON schema that should be fulfilled for the response. - */ - guided_json?: object; - response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role?: string; - /** - * The tool call id. If you don't know what to put here you can fall back to 000000001 - */ - tool_call_id?: string; - content?: string | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }[] | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; - /** - * JSON schema that should be fufilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Async_Batch { - requests: (Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt_Inner | Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages_Inner)[]; -} -interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Prompt_Inner { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * JSON schema that should be fulfilled for the response. - */ - guided_json?: object; - response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Messages_Inner { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role?: string; - /** - * The tool call id. If you don't know what to put here you can fall back to 000000001 - */ - tool_call_id?: string; - content?: string | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }[] | { - /** - * Type of the content provided - */ - type?: string; - text?: string; - image_url?: { - /** - * image uri with data (e.g. data:image/jpeg;base64,/9j/...). HTTP URL will not be accepted - */ - url?: string; - }; - }; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_JSON_Mode; - /** - * JSON schema that should be fufilled for the response. - */ - guided_json?: object; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -type Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Output = { - /** - * The generated text response from the model - */ - response: string; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * An array of tool calls requests made during the response generation - */ - tool_calls?: { - /** - * The tool call id. - */ - id?: string; - /** - * Specifies the type of tool (e.g., 'function'). - */ - type?: string; - /** - * Details of the function tool. - */ - function?: { - /** - * The name of the tool to be called - */ - name?: string; - /** - * The arguments passed to be passed to the tool call request - */ - arguments?: object; - }; - }[]; -}; -declare abstract class Base_Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct { - inputs: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Input; - postProcessedOutputs: Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct_Output; -} -type Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Input = Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Async_Batch; -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. - */ - lora?: string; - response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role: string; - content: string | { - /** - * Type of the content (text) - */ - type?: string; - /** - * Text content - */ - text?: string; - }[]; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_1; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_1 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Async_Batch { - requests: (Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt_1 | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages_1)[]; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Prompt_1 { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. - */ - lora?: string; - response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_2; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_2 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Messages_1 { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role: string; - content: string | { - /** - * Type of the content (text) - */ - type?: string; - /** - * Text content - */ - text?: string; - }[]; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_3; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_JSON_Mode_3 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -type Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Output = Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Chat_Completion_Response | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Text_Completion_Response | string | Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_AsyncResponse; -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Chat_Completion_Response { - /** - * Unique identifier for the completion - */ - id?: string; - /** - * Object type identifier - */ - object?: "chat.completion"; - /** - * Unix timestamp of when the completion was created - */ - created?: number; - /** - * Model used for the completion - */ - model?: string; - /** - * List of completion choices - */ - choices?: { - /** - * Index of the choice in the list - */ - index?: number; - /** - * The message generated by the model - */ - message?: { - /** - * Role of the message author - */ - role: string; - /** - * The content of the message - */ - content: string; - /** - * Internal reasoning content (if available) - */ - reasoning_content?: string; - /** - * Tool calls made by the assistant - */ - tool_calls?: { - /** - * Unique identifier for the tool call - */ - id: string; - /** - * Type of tool call - */ - type: "function"; - function: { - /** - * Name of the function to call - */ - name: string; - /** - * JSON string of arguments for the function - */ - arguments: string; - }; - }[]; - }; - /** - * Reason why the model stopped generating - */ - finish_reason?: string; - /** - * Stop reason (may be null) - */ - stop_reason?: string | null; - /** - * Log probabilities (if requested) - */ - logprobs?: {} | null; - }[]; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * Log probabilities for the prompt (if requested) - */ - prompt_logprobs?: {} | null; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Text_Completion_Response { - /** - * Unique identifier for the completion - */ - id?: string; - /** - * Object type identifier - */ - object?: "text_completion"; - /** - * Unix timestamp of when the completion was created - */ - created?: number; - /** - * Model used for the completion - */ - model?: string; - /** - * List of completion choices - */ - choices?: { - /** - * Index of the choice in the list - */ - index: number; - /** - * The generated text completion - */ - text: string; - /** - * Reason why the model stopped generating - */ - finish_reason: string; - /** - * Stop reason (may be null) - */ - stop_reason?: string | null; - /** - * Log probabilities (if requested) - */ - logprobs?: {} | null; - /** - * Log probabilities for the prompt (if requested) - */ - prompt_logprobs?: {} | null; - }[]; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; -} -interface Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8 { - inputs: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Input; - postProcessedOutputs: Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8_Output; -} -interface Ai_Cf_Deepgram_Nova_3_Input { - audio: { - body: object; - contentType: string; - }; - /** - * Sets how the model will interpret strings submitted to the custom_topic param. When strict, the model will only return topics submitted using the custom_topic param. When extended, the model will return its own detected topics in addition to those submitted using the custom_topic param. - */ - custom_topic_mode?: "extended" | "strict"; - /** - * Custom topics you want the model to detect within your input audio or text if present Submit up to 100 - */ - custom_topic?: string; - /** - * Sets how the model will interpret intents submitted to the custom_intent param. When strict, the model will only return intents submitted using the custom_intent param. When extended, the model will return its own detected intents in addition those submitted using the custom_intents param - */ - custom_intent_mode?: "extended" | "strict"; - /** - * Custom intents you want the model to detect within your input audio if present - */ - custom_intent?: string; - /** - * Identifies and extracts key entities from content in submitted audio - */ - detect_entities?: boolean; - /** - * Identifies the dominant language spoken in submitted audio - */ - detect_language?: boolean; - /** - * Recognize speaker changes. Each word in the transcript will be assigned a speaker number starting at 0 - */ - diarize?: boolean; - /** - * Identify and extract key entities from content in submitted audio - */ - dictation?: boolean; - /** - * Specify the expected encoding of your submitted audio - */ - encoding?: "linear16" | "flac" | "mulaw" | "amr-nb" | "amr-wb" | "opus" | "speex" | "g729"; - /** - * Arbitrary key-value pairs that are attached to the API response for usage in downstream processing - */ - extra?: string; - /** - * Filler Words can help transcribe interruptions in your audio, like 'uh' and 'um' - */ - filler_words?: boolean; - /** - * Key term prompting can boost or suppress specialized terminology and brands. - */ - keyterm?: string; - /** - * Keywords can boost or suppress specialized terminology and brands. - */ - keywords?: string; - /** - * The BCP-47 language tag that hints at the primary spoken language. Depending on the Model and API endpoint you choose only certain languages are available. - */ - language?: string; - /** - * Spoken measurements will be converted to their corresponding abbreviations. - */ - measurements?: boolean; - /** - * Opts out requests from the Deepgram Model Improvement Program. Refer to our Docs for pricing impacts before setting this to true. https://dpgr.am/deepgram-mip. - */ - mip_opt_out?: boolean; - /** - * Mode of operation for the model representing broad area of topic that will be talked about in the supplied audio - */ - mode?: "general" | "medical" | "finance"; - /** - * Transcribe each audio channel independently. - */ - multichannel?: boolean; - /** - * Numerals converts numbers from written format to numerical format. - */ - numerals?: boolean; - /** - * Splits audio into paragraphs to improve transcript readability. - */ - paragraphs?: boolean; - /** - * Profanity Filter looks for recognized profanity and converts it to the nearest recognized non-profane word or removes it from the transcript completely. - */ - profanity_filter?: boolean; - /** - * Add punctuation and capitalization to the transcript. - */ - punctuate?: boolean; - /** - * Redaction removes sensitive information from your transcripts. - */ - redact?: string; - /** - * Search for terms or phrases in submitted audio and replaces them. - */ - replace?: string; - /** - * Search for terms or phrases in submitted audio. - */ - search?: string; - /** - * Recognizes the sentiment throughout a transcript or text. - */ - sentiment?: boolean; - /** - * Apply formatting to transcript output. When set to true, additional formatting will be applied to transcripts to improve readability. - */ - smart_format?: boolean; - /** - * Detect topics throughout a transcript or text. - */ - topics?: boolean; - /** - * Segments speech into meaningful semantic units. - */ - utterances?: boolean; - /** - * Seconds to wait before detecting a pause between words in submitted audio. - */ - utt_split?: number; - /** - * The number of channels in the submitted audio - */ - channels?: number; - /** - * Specifies whether the streaming endpoint should provide ongoing transcription updates as more audio is received. When set to true, the endpoint sends continuous updates, meaning transcription results may evolve over time. Note: Supported only for webosockets. - */ - interim_results?: boolean; - /** - * Indicates how long model will wait to detect whether a speaker has finished speaking or pauses for a significant period of time. When set to a value, the streaming endpoint immediately finalizes the transcription for the processed time range and returns the transcript with a speech_final parameter set to true. Can also be set to false to disable endpointing - */ - endpointing?: string; - /** - * Indicates that speech has started. You'll begin receiving Speech Started messages upon speech starting. Note: Supported only for webosockets. - */ - vad_events?: boolean; - /** - * Indicates how long model will wait to send an UtteranceEnd message after a word has been transcribed. Use with interim_results. Note: Supported only for webosockets. - */ - utterance_end_ms?: boolean; -} -interface Ai_Cf_Deepgram_Nova_3_Output { - results?: { - channels?: { - alternatives?: { - confidence?: number; - transcript?: string; - words?: { - confidence?: number; - end?: number; - start?: number; - word?: string; - }[]; - }[]; - }[]; - summary?: { - result?: string; - short?: string; - }; - sentiments?: { - segments?: { - text?: string; - start_word?: number; - end_word?: number; - sentiment?: string; - sentiment_score?: number; - }[]; - average?: { - sentiment?: string; - sentiment_score?: number; - }; - }; - }; -} -declare abstract class Base_Ai_Cf_Deepgram_Nova_3 { - inputs: Ai_Cf_Deepgram_Nova_3_Input; - postProcessedOutputs: Ai_Cf_Deepgram_Nova_3_Output; -} -interface Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Input { - queries?: string | string[]; - /** - * Optional instruction for the task - */ - instruction?: string; - documents?: string | string[]; - text?: string | string[]; -} -interface Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Output { - data?: number[][]; - shape?: number[]; -} -declare abstract class Base_Ai_Cf_Qwen_Qwen3_Embedding_0_6B { - inputs: Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Input; - postProcessedOutputs: Ai_Cf_Qwen_Qwen3_Embedding_0_6B_Output; -} -type Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Input = { - /** - * readable stream with audio data and content-type specified for that data - */ - audio: { - body: object; - contentType: string; - }; - /** - * type of data PCM data that's sent to the inference server as raw array - */ - dtype?: "uint8" | "float32" | "float64"; -} | { - /** - * base64 encoded audio data - */ - audio: string; - /** - * type of data PCM data that's sent to the inference server as raw array - */ - dtype?: "uint8" | "float32" | "float64"; -}; -interface Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Output { - /** - * if true, end-of-turn was detected - */ - is_complete?: boolean; - /** - * probability of the end-of-turn detection - */ - probability?: number; -} -declare abstract class Base_Ai_Cf_Pipecat_Ai_Smart_Turn_V2 { - inputs: Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Input; - postProcessedOutputs: Ai_Cf_Pipecat_Ai_Smart_Turn_V2_Output; -} -declare abstract class Base_Ai_Cf_Openai_Gpt_Oss_120B { - inputs: XOR; - postProcessedOutputs: XOR; -} -declare abstract class Base_Ai_Cf_Openai_Gpt_Oss_20B { - inputs: XOR; - postProcessedOutputs: XOR; -} -interface Ai_Cf_Leonardo_Phoenix_1_0_Input { - /** - * A text description of the image you want to generate. - */ - prompt: string; - /** - * Controls how closely the generated image should adhere to the prompt; higher values make the image more aligned with the prompt - */ - guidance?: number; - /** - * Random seed for reproducibility of the image generation - */ - seed?: number; - /** - * The height of the generated image in pixels - */ - height?: number; - /** - * The width of the generated image in pixels - */ - width?: number; - /** - * The number of diffusion steps; higher values can improve quality but take longer - */ - num_steps?: number; - /** - * Specify what to exclude from the generated images - */ - negative_prompt?: string; -} -/** - * The generated image in JPEG format - */ -type Ai_Cf_Leonardo_Phoenix_1_0_Output = string; -declare abstract class Base_Ai_Cf_Leonardo_Phoenix_1_0 { - inputs: Ai_Cf_Leonardo_Phoenix_1_0_Input; - postProcessedOutputs: Ai_Cf_Leonardo_Phoenix_1_0_Output; -} -interface Ai_Cf_Leonardo_Lucid_Origin_Input { - /** - * A text description of the image you want to generate. - */ - prompt: string; - /** - * Controls how closely the generated image should adhere to the prompt; higher values make the image more aligned with the prompt - */ - guidance?: number; - /** - * Random seed for reproducibility of the image generation - */ - seed?: number; - /** - * The height of the generated image in pixels - */ - height?: number; - /** - * The width of the generated image in pixels - */ - width?: number; - /** - * The number of diffusion steps; higher values can improve quality but take longer - */ - num_steps?: number; - /** - * The number of diffusion steps; higher values can improve quality but take longer - */ - steps?: number; -} -interface Ai_Cf_Leonardo_Lucid_Origin_Output { - /** - * The generated image in Base64 format. - */ - image?: string; -} -declare abstract class Base_Ai_Cf_Leonardo_Lucid_Origin { - inputs: Ai_Cf_Leonardo_Lucid_Origin_Input; - postProcessedOutputs: Ai_Cf_Leonardo_Lucid_Origin_Output; -} -interface Ai_Cf_Deepgram_Aura_1_Input { - /** - * Speaker used to produce the audio. - */ - speaker?: "angus" | "asteria" | "arcas" | "orion" | "orpheus" | "athena" | "luna" | "zeus" | "perseus" | "helios" | "hera" | "stella"; - /** - * Encoding of the output audio. - */ - encoding?: "linear16" | "flac" | "mulaw" | "alaw" | "mp3" | "opus" | "aac"; - /** - * Container specifies the file format wrapper for the output audio. The available options depend on the encoding type.. - */ - container?: "none" | "wav" | "ogg"; - /** - * The text content to be converted to speech - */ - text: string; - /** - * Sample Rate specifies the sample rate for the output audio. Based on the encoding, different sample rates are supported. For some encodings, the sample rate is not configurable - */ - sample_rate?: number; - /** - * The bitrate of the audio in bits per second. Choose from predefined ranges or specific values based on the encoding type. - */ - bit_rate?: number; -} -/** - * The generated audio in MP3 format - */ -type Ai_Cf_Deepgram_Aura_1_Output = string; -declare abstract class Base_Ai_Cf_Deepgram_Aura_1 { - inputs: Ai_Cf_Deepgram_Aura_1_Input; - postProcessedOutputs: Ai_Cf_Deepgram_Aura_1_Output; -} -interface Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Input { - /** - * Input text to translate. Can be a single string or a list of strings. - */ - text: string | string[]; - /** - * Target langauge to translate to - */ - target_language: "asm_Beng" | "awa_Deva" | "ben_Beng" | "bho_Deva" | "brx_Deva" | "doi_Deva" | "eng_Latn" | "gom_Deva" | "gon_Deva" | "guj_Gujr" | "hin_Deva" | "hne_Deva" | "kan_Knda" | "kas_Arab" | "kas_Deva" | "kha_Latn" | "lus_Latn" | "mag_Deva" | "mai_Deva" | "mal_Mlym" | "mar_Deva" | "mni_Beng" | "mni_Mtei" | "npi_Deva" | "ory_Orya" | "pan_Guru" | "san_Deva" | "sat_Olck" | "snd_Arab" | "snd_Deva" | "tam_Taml" | "tel_Telu" | "urd_Arab" | "unr_Deva"; -} -interface Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Output { - /** - * Translated texts - */ - translations: string[]; -} -declare abstract class Base_Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B { - inputs: Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Input; - postProcessedOutputs: Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B_Output; -} -type Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Input = Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Async_Batch; -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. - */ - lora?: string; - response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role: string; - content: string | { - /** - * Type of the content (text) - */ - type?: string; - /** - * Text content - */ - text?: string; - }[]; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_1; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_1 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Async_Batch { - requests: (Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt_1 | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages_1)[]; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Prompt_1 { - /** - * The input text prompt for the model to generate a response. - */ - prompt: string; - /** - * Name of the LoRA (Low-Rank Adaptation) model to fine-tune the base model. - */ - lora?: string; - response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_2; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_2 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Messages_1 { - /** - * An array of message objects representing the conversation history. - */ - messages: { - /** - * The role of the message sender (e.g., 'user', 'assistant', 'system', 'tool'). - */ - role: string; - content: string | { - /** - * Type of the content (text) - */ - type?: string; - /** - * Text content - */ - text?: string; - }[]; - }[]; - functions?: { - name: string; - code: string; - }[]; - /** - * A list of tools available for the assistant to use. - */ - tools?: ({ - /** - * The name of the tool. More descriptive the better. - */ - name: string; - /** - * A brief description of what the tool does. - */ - description: string; - /** - * Schema defining the parameters accepted by the tool. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - } | { - /** - * Specifies the type of tool (e.g., 'function'). - */ - type: string; - /** - * Details of the function tool. - */ - function: { - /** - * The name of the function. - */ - name: string; - /** - * A brief description of what the function does. - */ - description: string; - /** - * Schema defining the parameters accepted by the function. - */ - parameters: { - /** - * The type of the parameters object (usually 'object'). - */ - type: string; - /** - * List of required parameter names. - */ - required?: string[]; - /** - * Definitions of each parameter. - */ - properties: { - [k: string]: { - /** - * The data type of the parameter. - */ - type: string; - /** - * A description of the expected parameter. - */ - description: string; - }; - }; - }; - }; - })[]; - response_format?: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_3; - /** - * If true, a chat template is not applied and you must adhere to the specific model's expected formatting. - */ - raw?: boolean; - /** - * If true, the response will be streamed back incrementally using SSE, Server Sent Events. - */ - stream?: boolean; - /** - * The maximum number of tokens to generate in the response. - */ - max_tokens?: number; - /** - * Controls the randomness of the output; higher values produce more random results. - */ - temperature?: number; - /** - * Adjusts the creativity of the AI's responses by controlling how many possible words it considers. Lower values make outputs more predictable; higher values allow for more varied and creative responses. - */ - top_p?: number; - /** - * Limits the AI to choose from the top 'k' most probable words. Lower values make responses more focused; higher values introduce more variety and potential surprises. - */ - top_k?: number; - /** - * Random seed for reproducibility of the generation. - */ - seed?: number; - /** - * Penalty for repeated tokens; higher values discourage repetition. - */ - repetition_penalty?: number; - /** - * Decreases the likelihood of the model repeating the same lines verbatim. - */ - frequency_penalty?: number; - /** - * Increases the likelihood of the model introducing new topics. - */ - presence_penalty?: number; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_JSON_Mode_3 { - type?: "json_object" | "json_schema"; - json_schema?: unknown; -} -type Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Output = Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Chat_Completion_Response | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Text_Completion_Response | string | Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_AsyncResponse; -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Chat_Completion_Response { - /** - * Unique identifier for the completion - */ - id?: string; - /** - * Object type identifier - */ - object?: "chat.completion"; - /** - * Unix timestamp of when the completion was created - */ - created?: number; - /** - * Model used for the completion - */ - model?: string; - /** - * List of completion choices - */ - choices?: { - /** - * Index of the choice in the list - */ - index?: number; - /** - * The message generated by the model - */ - message?: { - /** - * Role of the message author - */ - role: string; - /** - * The content of the message - */ - content: string; - /** - * Internal reasoning content (if available) - */ - reasoning_content?: string; - /** - * Tool calls made by the assistant - */ - tool_calls?: { - /** - * Unique identifier for the tool call - */ - id: string; - /** - * Type of tool call - */ - type: "function"; - function: { - /** - * Name of the function to call - */ - name: string; - /** - * JSON string of arguments for the function - */ - arguments: string; - }; - }[]; - }; - /** - * Reason why the model stopped generating - */ - finish_reason?: string; - /** - * Stop reason (may be null) - */ - stop_reason?: string | null; - /** - * Log probabilities (if requested) - */ - logprobs?: {} | null; - }[]; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; - /** - * Log probabilities for the prompt (if requested) - */ - prompt_logprobs?: {} | null; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Text_Completion_Response { - /** - * Unique identifier for the completion - */ - id?: string; - /** - * Object type identifier - */ - object?: "text_completion"; - /** - * Unix timestamp of when the completion was created - */ - created?: number; - /** - * Model used for the completion - */ - model?: string; - /** - * List of completion choices - */ - choices?: { - /** - * Index of the choice in the list - */ - index: number; - /** - * The generated text completion - */ - text: string; - /** - * Reason why the model stopped generating - */ - finish_reason: string; - /** - * Stop reason (may be null) - */ - stop_reason?: string | null; - /** - * Log probabilities (if requested) - */ - logprobs?: {} | null; - /** - * Log probabilities for the prompt (if requested) - */ - prompt_logprobs?: {} | null; - }[]; - /** - * Usage statistics for the inference request - */ - usage?: { - /** - * Total number of tokens in input - */ - prompt_tokens?: number; - /** - * Total number of tokens in output - */ - completion_tokens?: number; - /** - * Total number of input and output tokens - */ - total_tokens?: number; - }; -} -interface Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_AsyncResponse { - /** - * The async request id that can be used to obtain the results. - */ - request_id?: string; -} -declare abstract class Base_Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It { - inputs: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Input; - postProcessedOutputs: Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It_Output; -} -interface Ai_Cf_Pfnet_Plamo_Embedding_1B_Input { - /** - * Input text to embed. Can be a single string or a list of strings. - */ - text: string | string[]; -} -interface Ai_Cf_Pfnet_Plamo_Embedding_1B_Output { - /** - * Embedding vectors, where each vector is a list of floats. - */ - data: number[][]; - /** - * Shape of the embedding data as [number_of_embeddings, embedding_dimension]. - * - * @minItems 2 - * @maxItems 2 - */ - shape: [ - number, - number - ]; -} -declare abstract class Base_Ai_Cf_Pfnet_Plamo_Embedding_1B { - inputs: Ai_Cf_Pfnet_Plamo_Embedding_1B_Input; - postProcessedOutputs: Ai_Cf_Pfnet_Plamo_Embedding_1B_Output; -} -interface Ai_Cf_Deepgram_Flux_Input { - /** - * Encoding of the audio stream. Currently only supports raw signed little-endian 16-bit PCM. - */ - encoding: "linear16"; - /** - * Sample rate of the audio stream in Hz. - */ - sample_rate: string; - /** - * End-of-turn confidence required to fire an eager end-of-turn event. When set, enables EagerEndOfTurn and TurnResumed events. Valid Values 0.3 - 0.9. - */ - eager_eot_threshold?: string; - /** - * End-of-turn confidence required to finish a turn. Valid Values 0.5 - 0.9. - */ - eot_threshold?: string; - /** - * A turn will be finished when this much time has passed after speech, regardless of EOT confidence. - */ - eot_timeout_ms?: string; - /** - * Keyterm prompting can improve recognition of specialized terminology. Pass multiple keyterm query parameters to boost multiple keyterms. - */ - keyterm?: string; - /** - * Opts out requests from the Deepgram Model Improvement Program. Refer to Deepgram Docs for pricing impacts before setting this to true. https://dpgr.am/deepgram-mip - */ - mip_opt_out?: "true" | "false"; - /** - * Label your requests for the purpose of identification during usage reporting - */ - tag?: string; -} -/** - * Output will be returned as websocket messages. - */ -interface Ai_Cf_Deepgram_Flux_Output { - /** - * The unique identifier of the request (uuid) - */ - request_id?: string; - /** - * Starts at 0 and increments for each message the server sends to the client. - */ - sequence_id?: number; - /** - * The type of event being reported. - */ - event?: "Update" | "StartOfTurn" | "EagerEndOfTurn" | "TurnResumed" | "EndOfTurn"; - /** - * The index of the current turn - */ - turn_index?: number; - /** - * Start time in seconds of the audio range that was transcribed - */ - audio_window_start?: number; - /** - * End time in seconds of the audio range that was transcribed - */ - audio_window_end?: number; - /** - * Text that was said over the course of the current turn - */ - transcript?: string; - /** - * The words in the transcript - */ - words?: { - /** - * The individual punctuated, properly-cased word from the transcript - */ - word: string; - /** - * Confidence that this word was transcribed correctly - */ - confidence: number; - }[]; - /** - * Confidence that no more speech is coming in this turn - */ - end_of_turn_confidence?: number; -} -declare abstract class Base_Ai_Cf_Deepgram_Flux { - inputs: Ai_Cf_Deepgram_Flux_Input; - postProcessedOutputs: Ai_Cf_Deepgram_Flux_Output; -} -interface Ai_Cf_Deepgram_Aura_2_En_Input { - /** - * Speaker used to produce the audio. - */ - speaker?: "amalthea" | "andromeda" | "apollo" | "arcas" | "aries" | "asteria" | "athena" | "atlas" | "aurora" | "callista" | "cora" | "cordelia" | "delia" | "draco" | "electra" | "harmonia" | "helena" | "hera" | "hermes" | "hyperion" | "iris" | "janus" | "juno" | "jupiter" | "luna" | "mars" | "minerva" | "neptune" | "odysseus" | "ophelia" | "orion" | "orpheus" | "pandora" | "phoebe" | "pluto" | "saturn" | "thalia" | "theia" | "vesta" | "zeus"; - /** - * Encoding of the output audio. - */ - encoding?: "linear16" | "flac" | "mulaw" | "alaw" | "mp3" | "opus" | "aac"; - /** - * Container specifies the file format wrapper for the output audio. The available options depend on the encoding type.. - */ - container?: "none" | "wav" | "ogg"; - /** - * The text content to be converted to speech - */ - text: string; - /** - * Sample Rate specifies the sample rate for the output audio. Based on the encoding, different sample rates are supported. For some encodings, the sample rate is not configurable - */ - sample_rate?: number; - /** - * The bitrate of the audio in bits per second. Choose from predefined ranges or specific values based on the encoding type. - */ - bit_rate?: number; -} -/** - * The generated audio in MP3 format - */ -type Ai_Cf_Deepgram_Aura_2_En_Output = string; -declare abstract class Base_Ai_Cf_Deepgram_Aura_2_En { - inputs: Ai_Cf_Deepgram_Aura_2_En_Input; - postProcessedOutputs: Ai_Cf_Deepgram_Aura_2_En_Output; -} -interface Ai_Cf_Deepgram_Aura_2_Es_Input { - /** - * Speaker used to produce the audio. - */ - speaker?: "sirio" | "nestor" | "carina" | "celeste" | "alvaro" | "diana" | "aquila" | "selena" | "estrella" | "javier"; - /** - * Encoding of the output audio. - */ - encoding?: "linear16" | "flac" | "mulaw" | "alaw" | "mp3" | "opus" | "aac"; - /** - * Container specifies the file format wrapper for the output audio. The available options depend on the encoding type.. - */ - container?: "none" | "wav" | "ogg"; - /** - * The text content to be converted to speech - */ - text: string; - /** - * Sample Rate specifies the sample rate for the output audio. Based on the encoding, different sample rates are supported. For some encodings, the sample rate is not configurable - */ - sample_rate?: number; - /** - * The bitrate of the audio in bits per second. Choose from predefined ranges or specific values based on the encoding type. - */ - bit_rate?: number; -} -/** - * The generated audio in MP3 format - */ -type Ai_Cf_Deepgram_Aura_2_Es_Output = string; -declare abstract class Base_Ai_Cf_Deepgram_Aura_2_Es { - inputs: Ai_Cf_Deepgram_Aura_2_Es_Input; - postProcessedOutputs: Ai_Cf_Deepgram_Aura_2_Es_Output; -} -interface Ai_Cf_Black_Forest_Labs_Flux_2_Dev_Input { - multipart: { - body?: object; - contentType?: string; - }; -} -interface Ai_Cf_Black_Forest_Labs_Flux_2_Dev_Output { - /** - * Generated image as Base64 string. - */ - image?: string; -} -declare abstract class Base_Ai_Cf_Black_Forest_Labs_Flux_2_Dev { - inputs: Ai_Cf_Black_Forest_Labs_Flux_2_Dev_Input; - postProcessedOutputs: Ai_Cf_Black_Forest_Labs_Flux_2_Dev_Output; -} -interface Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B_Input { - multipart: { - body?: object; - contentType?: string; - }; -} -interface Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B_Output { - /** - * Generated image as Base64 string. - */ - image?: string; -} -declare abstract class Base_Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B { - inputs: Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B_Input; - postProcessedOutputs: Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B_Output; -} -interface Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B_Input { - multipart: { - body?: object; - contentType?: string; - }; -} -interface Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B_Output { - /** - * Generated image as Base64 string. - */ - image?: string; -} -declare abstract class Base_Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B { - inputs: Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B_Input; - postProcessedOutputs: Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B_Output; -} -declare abstract class Base_Ai_Cf_Zai_Org_Glm_4_7_Flash { - inputs: ChatCompletionsInput; - postProcessedOutputs: ChatCompletionsOutput; -} -declare abstract class Base_Ai_Cf_Moonshotai_Kimi_K2_5 { - inputs: ChatCompletionsInput; - postProcessedOutputs: ChatCompletionsOutput; -} -declare abstract class Base_Ai_Cf_Nvidia_Nemotron_3_120B_A12B { - inputs: ChatCompletionsInput; - postProcessedOutputs: ChatCompletionsOutput; -} -declare abstract class Base_Ai_Cf_Google_Gemma_4_26B_A4B_IT { - inputs: ChatCompletionsInput; - postProcessedOutputs: ChatCompletionsOutput; -} -interface AiModels { - "@cf/huggingface/distilbert-sst-2-int8": BaseAiTextClassification; - "@cf/stabilityai/stable-diffusion-xl-base-1.0": BaseAiTextToImage; - "@cf/runwayml/stable-diffusion-v1-5-inpainting": BaseAiTextToImage; - "@cf/runwayml/stable-diffusion-v1-5-img2img": BaseAiTextToImage; - "@cf/lykon/dreamshaper-8-lcm": BaseAiTextToImage; - "@cf/bytedance/stable-diffusion-xl-lightning": BaseAiTextToImage; - "@cf/myshell-ai/melotts": BaseAiTextToSpeech; - "@cf/google/embeddinggemma-300m": BaseAiTextEmbeddings; - "@cf/microsoft/resnet-50": BaseAiImageClassification; - "@cf/meta/llama-2-7b-chat-int8": BaseAiTextGeneration; - "@cf/mistral/mistral-7b-instruct-v0.1": BaseAiTextGeneration; - "@cf/meta/llama-2-7b-chat-fp16": BaseAiTextGeneration; - "@hf/thebloke/llama-2-13b-chat-awq": BaseAiTextGeneration; - "@hf/thebloke/mistral-7b-instruct-v0.1-awq": BaseAiTextGeneration; - "@hf/thebloke/zephyr-7b-beta-awq": BaseAiTextGeneration; - "@hf/thebloke/openhermes-2.5-mistral-7b-awq": BaseAiTextGeneration; - "@hf/thebloke/neural-chat-7b-v3-1-awq": BaseAiTextGeneration; - "@hf/thebloke/deepseek-coder-6.7b-base-awq": BaseAiTextGeneration; - "@hf/thebloke/deepseek-coder-6.7b-instruct-awq": BaseAiTextGeneration; - "@cf/deepseek-ai/deepseek-math-7b-instruct": BaseAiTextGeneration; - "@cf/defog/sqlcoder-7b-2": BaseAiTextGeneration; - "@cf/openchat/openchat-3.5-0106": BaseAiTextGeneration; - "@cf/tiiuae/falcon-7b-instruct": BaseAiTextGeneration; - "@cf/thebloke/discolm-german-7b-v1-awq": BaseAiTextGeneration; - "@cf/qwen/qwen1.5-0.5b-chat": BaseAiTextGeneration; - "@cf/qwen/qwen1.5-7b-chat-awq": BaseAiTextGeneration; - "@cf/qwen/qwen1.5-14b-chat-awq": BaseAiTextGeneration; - "@cf/tinyllama/tinyllama-1.1b-chat-v1.0": BaseAiTextGeneration; - "@cf/microsoft/phi-2": BaseAiTextGeneration; - "@cf/qwen/qwen1.5-1.8b-chat": BaseAiTextGeneration; - "@cf/mistral/mistral-7b-instruct-v0.2-lora": BaseAiTextGeneration; - "@hf/nousresearch/hermes-2-pro-mistral-7b": BaseAiTextGeneration; - "@hf/nexusflow/starling-lm-7b-beta": BaseAiTextGeneration; - "@hf/google/gemma-7b-it": BaseAiTextGeneration; - "@cf/meta-llama/llama-2-7b-chat-hf-lora": BaseAiTextGeneration; - "@cf/google/gemma-2b-it-lora": BaseAiTextGeneration; - "@cf/google/gemma-7b-it-lora": BaseAiTextGeneration; - "@hf/mistral/mistral-7b-instruct-v0.2": BaseAiTextGeneration; - "@cf/meta/llama-3-8b-instruct": BaseAiTextGeneration; - "@cf/fblgit/una-cybertron-7b-v2-bf16": BaseAiTextGeneration; - "@cf/meta/llama-3-8b-instruct-awq": BaseAiTextGeneration; - "@cf/meta/llama-3.1-8b-instruct-fp8": BaseAiTextGeneration; - "@cf/meta/llama-3.1-8b-instruct-awq": BaseAiTextGeneration; - "@cf/meta/llama-3.2-3b-instruct": BaseAiTextGeneration; - "@cf/meta/llama-3.2-1b-instruct": BaseAiTextGeneration; - "@cf/deepseek-ai/deepseek-r1-distill-qwen-32b": BaseAiTextGeneration; - "@cf/ibm-granite/granite-4.0-h-micro": BaseAiTextGeneration; - "@cf/facebook/bart-large-cnn": BaseAiSummarization; - "@cf/llava-hf/llava-1.5-7b-hf": BaseAiImageToText; - "@cf/baai/bge-base-en-v1.5": Base_Ai_Cf_Baai_Bge_Base_En_V1_5; - "@cf/openai/whisper": Base_Ai_Cf_Openai_Whisper; - "@cf/meta/m2m100-1.2b": Base_Ai_Cf_Meta_M2M100_1_2B; - "@cf/baai/bge-small-en-v1.5": Base_Ai_Cf_Baai_Bge_Small_En_V1_5; - "@cf/baai/bge-large-en-v1.5": Base_Ai_Cf_Baai_Bge_Large_En_V1_5; - "@cf/unum/uform-gen2-qwen-500m": Base_Ai_Cf_Unum_Uform_Gen2_Qwen_500M; - "@cf/openai/whisper-tiny-en": Base_Ai_Cf_Openai_Whisper_Tiny_En; - "@cf/openai/whisper-large-v3-turbo": Base_Ai_Cf_Openai_Whisper_Large_V3_Turbo; - "@cf/baai/bge-m3": Base_Ai_Cf_Baai_Bge_M3; - "@cf/black-forest-labs/flux-1-schnell": Base_Ai_Cf_Black_Forest_Labs_Flux_1_Schnell; - "@cf/meta/llama-3.2-11b-vision-instruct": Base_Ai_Cf_Meta_Llama_3_2_11B_Vision_Instruct; - "@cf/meta/llama-3.3-70b-instruct-fp8-fast": Base_Ai_Cf_Meta_Llama_3_3_70B_Instruct_Fp8_Fast; - "@cf/meta/llama-guard-3-8b": Base_Ai_Cf_Meta_Llama_Guard_3_8B; - "@cf/baai/bge-reranker-base": Base_Ai_Cf_Baai_Bge_Reranker_Base; - "@cf/qwen/qwen2.5-coder-32b-instruct": Base_Ai_Cf_Qwen_Qwen2_5_Coder_32B_Instruct; - "@cf/qwen/qwq-32b": Base_Ai_Cf_Qwen_Qwq_32B; - "@cf/mistralai/mistral-small-3.1-24b-instruct": Base_Ai_Cf_Mistralai_Mistral_Small_3_1_24B_Instruct; - "@cf/google/gemma-3-12b-it": Base_Ai_Cf_Google_Gemma_3_12B_It; - "@cf/meta/llama-4-scout-17b-16e-instruct": Base_Ai_Cf_Meta_Llama_4_Scout_17B_16E_Instruct; - "@cf/qwen/qwen3-30b-a3b-fp8": Base_Ai_Cf_Qwen_Qwen3_30B_A3B_Fp8; - "@cf/deepgram/nova-3": Base_Ai_Cf_Deepgram_Nova_3; - "@cf/qwen/qwen3-embedding-0.6b": Base_Ai_Cf_Qwen_Qwen3_Embedding_0_6B; - "@cf/pipecat-ai/smart-turn-v2": Base_Ai_Cf_Pipecat_Ai_Smart_Turn_V2; - "@cf/openai/gpt-oss-120b": Base_Ai_Cf_Openai_Gpt_Oss_120B; - "@cf/openai/gpt-oss-20b": Base_Ai_Cf_Openai_Gpt_Oss_20B; - "@cf/leonardo/phoenix-1.0": Base_Ai_Cf_Leonardo_Phoenix_1_0; - "@cf/leonardo/lucid-origin": Base_Ai_Cf_Leonardo_Lucid_Origin; - "@cf/deepgram/aura-1": Base_Ai_Cf_Deepgram_Aura_1; - "@cf/ai4bharat/indictrans2-en-indic-1B": Base_Ai_Cf_Ai4Bharat_Indictrans2_En_Indic_1B; - "@cf/aisingapore/gemma-sea-lion-v4-27b-it": Base_Ai_Cf_Aisingapore_Gemma_Sea_Lion_V4_27B_It; - "@cf/pfnet/plamo-embedding-1b": Base_Ai_Cf_Pfnet_Plamo_Embedding_1B; - "@cf/deepgram/flux": Base_Ai_Cf_Deepgram_Flux; - "@cf/deepgram/aura-2-en": Base_Ai_Cf_Deepgram_Aura_2_En; - "@cf/deepgram/aura-2-es": Base_Ai_Cf_Deepgram_Aura_2_Es; - "@cf/black-forest-labs/flux-2-dev": Base_Ai_Cf_Black_Forest_Labs_Flux_2_Dev; - "@cf/black-forest-labs/flux-2-klein-4b": Base_Ai_Cf_Black_Forest_Labs_Flux_2_Klein_4B; - "@cf/black-forest-labs/flux-2-klein-9b": Base_Ai_Cf_Black_Forest_Labs_Flux_2_Klein_9B; - "@cf/zai-org/glm-4.7-flash": Base_Ai_Cf_Zai_Org_Glm_4_7_Flash; - "@cf/moonshotai/kimi-k2.5": Base_Ai_Cf_Moonshotai_Kimi_K2_5; - "@cf/nvidia/nemotron-3-120b-a12b": Base_Ai_Cf_Nvidia_Nemotron_3_120B_A12B; -} -type AiOptions = { - /** - * Send requests as an asynchronous batch job, only works for supported models - * https://developers.cloudflare.com/workers-ai/features/batch-api - */ - queueRequest?: boolean; - /** - * Establish websocket connections, only works for supported models - */ - websocket?: boolean; - /** - * Tag your requests to group and view them in Cloudflare dashboard. - * - * Rules: - * Tags must only contain letters, numbers, and the symbols: : - . / @ - * Each tag can have maximum 50 characters. - * Maximum 5 tags are allowed each request. - * Duplicate tags will removed. - */ - tags?: string[]; - gateway?: GatewayOptions; - returnRawResponse?: boolean; - prefix?: string; - extraHeaders?: object; - signal?: AbortSignal; -}; -type AiModelsSearchParams = { - author?: string; - hide_experimental?: boolean; - page?: number; - per_page?: number; - search?: string; - source?: number; - task?: string; -}; -type AiModelsSearchObject = { - id: string; - source: number; - name: string; - description: string; - task: { - id: string; - name: string; - description: string; - }; - tags: string[]; - properties: { - property_id: string; - value: string; - }[]; -}; -type ChatCompletionsBase = XOR; -type ChatCompletionsInput = XOR; -interface InferenceUpstreamError extends Error { -} -interface AiInternalError extends Error { -} -type AiModelListType = Record; -type AiAsyncBatchResponse = { - request_id: string; -}; -declare abstract class Ai { - aiGatewayLogId: string | null; - gateway(gatewayId: string): AiGateway; - /** - * @deprecated Use the standalone `ai_search_namespaces` or `ai_search` Workers bindings instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ - aiSearch(): AiSearchNamespace; - /** - * @deprecated AutoRAG has been replaced by AI Search. - * Use the standalone `ai_search_namespaces` or `ai_search` Workers bindings instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - * - * @param autoragId Instance ID - */ - autorag(autoragId: string): AutoRAG; - // Batch request - run(model: Name, inputs: { - requests: AiModelList[Name]['inputs'][]; - }, options: AiOptions & { - queueRequest: true; - }): Promise; - // Raw response - run(model: Name, inputs: AiModelList[Name]['inputs'], options: AiOptions & { - returnRawResponse: true; - }): Promise; - // WebSocket - run(model: Name, inputs: AiModelList[Name]['inputs'], options: AiOptions & { - websocket: true; - }): Promise; - // Streaming - run(model: Name, inputs: AiModelList[Name]['inputs'] & { - stream: true; - }, options?: AiOptions): Promise; - // Normal (default) - known model - run(model: Name, inputs: AiModelList[Name]['inputs'], options?: AiOptions): Promise; - // Unknown model (gateway fallback) - run(model: string & {}, inputs: Record, options?: AiOptions): Promise>; - models(params?: AiModelsSearchParams): Promise; - toMarkdown(): ToMarkdownService; - toMarkdown(files: MarkdownDocument[], options?: ConversionRequestOptions): Promise; - toMarkdown(files: MarkdownDocument, options?: ConversionRequestOptions): Promise; -} -type GatewayRetries = { - maxAttempts?: 1 | 2 | 3 | 4 | 5; - retryDelayMs?: number; - backoff?: 'constant' | 'linear' | 'exponential'; -}; -type GatewayOptions = { - id: string; - cacheKey?: string; - cacheTtl?: number; - skipCache?: boolean; - metadata?: Record; - collectLog?: boolean; - eventId?: string; - requestTimeoutMs?: number; - retries?: GatewayRetries; -}; -type UniversalGatewayOptions = Exclude & { - /** - ** @deprecated - */ - id?: string; -}; -type AiGatewayPatchLog = { - score?: number | null; - feedback?: -1 | 1 | null; - metadata?: Record | null; -}; -type AiGatewayLog = { - id: string; - provider: string; - model: string; - model_type?: string; - path: string; - duration: number; - request_type?: string; - request_content_type?: string; - status_code: number; - response_content_type?: string; - success: boolean; - cached: boolean; - tokens_in?: number; - tokens_out?: number; - metadata?: Record; - step?: number; - cost?: number; - custom_cost?: boolean; - request_size: number; - request_head?: string; - request_head_complete: boolean; - response_size: number; - response_head?: string; - response_head_complete: boolean; - created_at: Date; -}; -type AIGatewayProviders = 'workers-ai' | 'anthropic' | 'aws-bedrock' | 'azure-openai' | 'google-vertex-ai' | 'huggingface' | 'openai' | 'perplexity-ai' | 'replicate' | 'groq' | 'cohere' | 'google-ai-studio' | 'mistral' | 'grok' | 'openrouter' | 'deepseek' | 'cerebras' | 'cartesia' | 'elevenlabs' | 'adobe-firefly'; -type AIGatewayHeaders = { - 'cf-aig-metadata': Record | string; - 'cf-aig-custom-cost': { - per_token_in?: number; - per_token_out?: number; - } | { - total_cost?: number; - } | string; - 'cf-aig-cache-ttl': number | string; - 'cf-aig-skip-cache': boolean | string; - 'cf-aig-cache-key': string; - 'cf-aig-event-id': string; - 'cf-aig-request-timeout': number | string; - 'cf-aig-max-attempts': number | string; - 'cf-aig-retry-delay': number | string; - 'cf-aig-backoff': string; - 'cf-aig-collect-log': boolean | string; - Authorization: string; - 'Content-Type': string; - [key: string]: string | number | boolean | object; -}; -type AIGatewayUniversalRequest = { - provider: AIGatewayProviders | string; // eslint-disable-line - endpoint: string; - headers: Partial; - query: unknown; -}; -interface AiGatewayInternalError extends Error { -} -interface AiGatewayLogNotFound extends Error { -} -declare abstract class AiGateway { - patchLog(logId: string, data: AiGatewayPatchLog): Promise; - getLog(logId: string): Promise; - run(data: AIGatewayUniversalRequest | AIGatewayUniversalRequest[], options?: { - gateway?: UniversalGatewayOptions; - extraHeaders?: object; - signal?: AbortSignal; - }): Promise; - getUrl(provider?: AIGatewayProviders | string): Promise; // eslint-disable-line -} -// Copyright (c) 2022-2025 Cloudflare, Inc. -// Licensed under the Apache 2.0 license found in the LICENSE file or at: -// https://opensource.org/licenses/Apache-2.0 -/** - * Artifacts — Git-compatible file storage on Cloudflare Workers. - * - * Provides programmatic access to create, manage, and fork repositories, - * and to issue and revoke scoped access tokens. - */ -/** Information about a repository. */ -interface ArtifactsRepoInfo { - /** Unique repository ID. */ - id: string; - /** Repository name. */ - name: string; - /** Repository description, or null if not set. */ - description: string | null; - /** Default branch name (e.g. "main"). */ - defaultBranch: string; - /** ISO 8601 creation timestamp. */ - createdAt: string; - /** ISO 8601 last-updated timestamp. */ - updatedAt: string; - /** ISO 8601 timestamp of the last push, or null if never pushed. */ - lastPushAt: string | null; - /** Fork source (e.g. "github:owner/repo", "artifacts:namespace/repo"), or null if not a fork. */ - source: string | null; - /** Whether the repository is read-only. */ - readOnly: boolean; - /** HTTPS git remote URL. */ - remote: string; -} -/** Result of creating a repository — includes the initial access token. */ -interface ArtifactsCreateRepoResult { - /** Unique repository ID. */ - id: string; - /** Repository name. */ - name: string; - /** Repository description, or null if not set. */ - description: string | null; - /** Default branch name. */ - defaultBranch: string; - /** HTTPS git remote URL. */ - remote: string; - /** Plaintext access token (only returned at creation time). */ - token: string; - /** ISO 8601 token expiry timestamp. */ - tokenExpiresAt: string; -} -/** Paginated list of repositories. */ -interface ArtifactsRepoListResult { - /** Repositories in this page (without the `remote` field). */ - repos: Omit[]; - /** Total number of repositories in the namespace. */ - total: number; - /** Cursor for the next page, if there are more results. */ - cursor?: string; -} -/** Result of creating an access token. */ -interface ArtifactsCreateTokenResult { - /** Unique token ID. */ - id: string; - /** Plaintext token (only returned at creation time). */ - plaintext: string; - /** Token scope: "read" or "write". */ - scope: 'read' | 'write'; - /** ISO 8601 token expiry timestamp. */ - expiresAt: string; -} -/** Token metadata (no plaintext). */ -interface ArtifactsTokenInfo { - /** Unique token ID. */ - id: string; - /** Token scope: "read" or "write". */ - scope: 'read' | 'write'; - /** Token state: "active", "expired", or "revoked". */ - state: 'active' | 'expired' | 'revoked'; - /** ISO 8601 creation timestamp. */ - createdAt: string; - /** ISO 8601 expiry timestamp. */ - expiresAt: string; -} -/** Paginated list of tokens for a repository. */ -interface ArtifactsTokenListResult { - /** Tokens in this page. */ - tokens: ArtifactsTokenInfo[]; - /** Total number of tokens for the repository. */ - total: number; -} -/** Handle for a single repository. Returned by Artifacts.get(). */ -interface ArtifactsRepo extends ArtifactsRepoInfo { - /** - * Create an access token for this repo. - * @param scope Token scope: "write" (default) or "read". - * @param ttl Time-to-live in seconds (default 86400, min 60, max 31536000). - */ - createToken(scope?: 'write' | 'read', ttl?: number): Promise; - /** List tokens for this repo (metadata only, no plaintext). */ - listTokens(): Promise; - /** - * Revoke a token by plaintext or ID. - * @param tokenOrId Plaintext token or token ID. - * @returns true if revoked, false if not found. - */ - revokeToken(tokenOrId: string): Promise; - // ── Fork ── - /** - * Fork this repo to a new repo. - * @param name Target repository name. - * @param opts Optional: description, readOnly flag, defaultBranchOnly (default true). - */ - fork(name: string, opts?: { - description?: string; - readOnly?: boolean; - defaultBranchOnly?: boolean; - }): Promise; -} -/** Artifacts binding — namespace-level operations. */ -interface Artifacts { - /** - * Create a new repository with an initial access token. - * @param name Repository name (alphanumeric, dots, hyphens, underscores). - * @param opts Optional: readOnly flag, description, default branch name. - * @returns Repo metadata with initial token. - */ - create(name: string, opts?: { - readOnly?: boolean; - description?: string; - setDefaultBranch?: string; - }): Promise; - /** - * Get a handle to an existing repository. - * @param name Repository name. - * @returns Repo handle. - */ - get(name: string): Promise; - /** - * Import a repository from an external git remote. - * @param params Source URL and optional branch/depth, plus target name and options. - * @returns Repo metadata with initial token. - */ - import(params: { - source: { - url: string; - branch?: string; - depth?: number; - }; - target: { - name: string; - opts?: { - description?: string; - readOnly?: boolean; - }; - }; - }): Promise; - /** - * List repositories with cursor-based pagination. - * @param opts Optional: limit (1–200, default 50), cursor for next page. - */ - list(opts?: { - limit?: number; - cursor?: string; - }): Promise; - /** - * Delete a repository and all associated tokens. - * @param name Repository name. - * @returns true if deleted, false if not found. - */ - delete(name: string): Promise; -} -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -interface AutoRAGInternalError extends Error { -} -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -interface AutoRAGNotFoundError extends Error { -} -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -interface AutoRAGUnauthorizedError extends Error { -} -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -interface AutoRAGNameNotSetError extends Error { -} -type ComparisonFilter = { - key: string; - type: 'eq' | 'ne' | 'gt' | 'gte' | 'lt' | 'lte'; - value: string | number | boolean; -}; -type CompoundFilter = { - type: 'and' | 'or'; - filters: ComparisonFilter[]; -}; -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -type AutoRagSearchRequest = { - query: string; - filters?: CompoundFilter | ComparisonFilter; - max_num_results?: number; - ranking_options?: { - ranker?: string; - score_threshold?: number; - }; - reranking?: { - enabled?: boolean; - model?: string; - }; - rewrite_query?: boolean; -}; -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -type AutoRagAiSearchRequest = AutoRagSearchRequest & { - stream?: boolean; - system_prompt?: string; -}; -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -type AutoRagAiSearchRequestStreaming = Omit & { - stream: true; -}; -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -type AutoRagSearchResponse = { - object: 'vector_store.search_results.page'; - search_query: string; - data: { - file_id: string; - filename: string; - score: number; - attributes: Record; - content: { - type: 'text'; - text: string; - }[]; - }[]; - has_more: boolean; - next_page: string | null; -}; -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -type AutoRagListResponse = { - id: string; - enable: boolean; - type: string; - source: string; - vectorize_name: string; - paused: boolean; - status: string; -}[]; -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -type AutoRagAiSearchResponse = AutoRagSearchResponse & { - response: string; -}; -/** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ -declare abstract class AutoRAG { - /** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ - list(): Promise; - /** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ - search(params: AutoRagSearchRequest): Promise; - /** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ - aiSearch(params: AutoRagAiSearchRequestStreaming): Promise; - /** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ - aiSearch(params: AutoRagAiSearchRequest): Promise; - /** - * @deprecated Use the standalone AI Search Workers binding instead. - * See https://developers.cloudflare.com/ai-search/usage/workers-binding/ - */ - aiSearch(params: AutoRagAiSearchRequest): Promise; -} -interface BasicImageTransformations { - /** - * Maximum width in image pixels. The value must be an integer. - */ - width?: number; - /** - * Maximum height in image pixels. The value must be an integer. - */ - height?: number; - /** - * Resizing mode as a string. It affects interpretation of width and height - * options: - * - scale-down: Similar to contain, but the image is never enlarged. If - * the image is larger than given width or height, it will be resized. - * Otherwise its original size will be kept. - * - contain: Resizes to maximum size that fits within the given width and - * height. If only a single dimension is given (e.g. only width), the - * image will be shrunk or enlarged to exactly match that dimension. - * Aspect ratio is always preserved. - * - cover: Resizes (shrinks or enlarges) to fill the entire area of width - * and height. If the image has an aspect ratio different from the ratio - * of width and height, it will be cropped to fit. - * - crop: The image will be shrunk and cropped to fit within the area - * specified by width and height. The image will not be enlarged. For images - * smaller than the given dimensions it's the same as scale-down. For - * images larger than the given dimensions, it's the same as cover. - * See also trim. - * - pad: Resizes to the maximum size that fits within the given width and - * height, and then fills the remaining area with a background color - * (white by default). Use of this mode is not recommended, as the same - * effect can be more efficiently achieved with the contain mode and the - * CSS object-fit: contain property. - * - squeeze: Stretches and deforms to the width and height given, even if it - * breaks aspect ratio - */ - fit?: "scale-down" | "contain" | "cover" | "crop" | "pad" | "squeeze"; - /** - * Image segmentation using artificial intelligence models. Sets pixels not - * within selected segment area to transparent e.g "foreground" sets every - * background pixel as transparent. - */ - segment?: "foreground"; - /** - * When cropping with fit: "cover", this defines the side or point that should - * be left uncropped. The value is either a string - * "left", "right", "top", "bottom", "auto", or "center" (the default), - * or an object {x, y} containing focal point coordinates in the original - * image expressed as fractions ranging from 0.0 (top or left) to 1.0 - * (bottom or right), 0.5 being the center. {fit: "cover", gravity: "top"} will - * crop bottom or left and right sides as necessary, but won’t crop anything - * from the top. {fit: "cover", gravity: {x:0.5, y:0.2}} will crop each side to - * preserve as much as possible around a point at 20% of the height of the - * source image. - */ - gravity?: 'face' | 'left' | 'right' | 'top' | 'bottom' | 'center' | 'auto' | 'entropy' | BasicImageTransformationsGravityCoordinates; - /** - * Background color to add underneath the image. Applies only to images with - * transparency (such as PNG). Accepts any CSS color (#RRGGBB, rgba(…), - * hsl(…), etc.) - */ - background?: string; - /** - * Number of degrees (90, 180, 270) to rotate the image by. width and height - * options refer to axes after rotation. - */ - rotate?: 0 | 90 | 180 | 270 | 360; -} -interface BasicImageTransformationsGravityCoordinates { - x?: number; - y?: number; - mode?: 'remainder' | 'box-center'; -} -/** - * In addition to the properties you can set in the RequestInit dict - * that you pass as an argument to the Request constructor, you can - * set certain properties of a `cf` object to control how Cloudflare - * features are applied to that new Request. - * - * Note: Currently, these properties cannot be tested in the - * playground. - */ -interface RequestInitCfProperties extends Record { - cacheEverything?: boolean; - /** - * A request's cache key is what determines if two requests are - * "the same" for caching purposes. If a request has the same cache key - * as some previous request, then we can serve the same cached response for - * both. (e.g. 'some-key') - * - * Only available for Enterprise customers. - */ - cacheKey?: string; - /** - * This allows you to append additional Cache-Tag response headers - * to the origin response without modifications to the origin server. - * This will allow for greater control over the Purge by Cache Tag feature - * utilizing changes only in the Workers process. - * - * Only available for Enterprise customers. - */ - cacheTags?: string[]; - /** - * Force response to be cached for a given number of seconds. (e.g. 300) - */ - cacheTtl?: number; - /** - * Force response to be cached for a given number of seconds based on the Origin status code. - * (e.g. { '200-299': 86400, '404': 1, '500-599': 0 }) - */ - cacheTtlByStatus?: Record; - /** - * Explicit Cache-Control header value to set on the response stored in cache. - * This gives full control over cache directives (e.g. 'public, max-age=3600, s-maxage=86400'). - * - * Cannot be used together with `cacheTtl` or the `cache` request option (`no-store`/`no-cache`), - * as these are mutually exclusive cache control mechanisms. Setting both will throw a TypeError. - * - * Can be used together with `cacheTtlByStatus`. - */ - cacheControl?: string; - /** - * Whether the response should be eligible for Cache Reserve storage. - */ - cacheReserveEligible?: boolean; - /** - * Whether to respect strong ETags (as opposed to weak ETags) from the origin. - */ - respectStrongEtag?: boolean; - /** - * Whether to strip ETag headers from the origin response before caching. - */ - stripEtags?: boolean; - /** - * Whether to strip Last-Modified headers from the origin response before caching. - */ - stripLastModified?: boolean; - /** - * Whether to enable Cache Deception Armor, which protects against web cache - * deception attacks by verifying the Content-Type matches the URL extension. - */ - cacheDeceptionArmor?: boolean; - /** - * Minimum file size in bytes for a response to be eligible for Cache Reserve storage. - */ - cacheReserveMinimumFileSize?: number; - scrapeShield?: boolean; - apps?: boolean; - image?: RequestInitCfPropertiesImage; - minify?: RequestInitCfPropertiesImageMinify; - mirage?: boolean; - polish?: "lossy" | "lossless" | "off"; - r2?: RequestInitCfPropertiesR2; - /** - * Redirects the request to an alternate origin server. You can use this, - * for example, to implement load balancing across several origins. - * (e.g.us-east.example.com) - * - * Note - For security reasons, the hostname set in resolveOverride must - * be proxied on the same Cloudflare zone of the incoming request. - * Otherwise, the setting is ignored. CNAME hosts are allowed, so to - * resolve to a host under a different domain or a DNS only domain first - * declare a CNAME record within your own zone’s DNS mapping to the - * external hostname, set proxy on Cloudflare, then set resolveOverride - * to point to that CNAME record. - */ - resolveOverride?: string; -} -interface RequestInitCfPropertiesImageDraw extends BasicImageTransformations { - /** - * Absolute URL of the image file to use for the drawing. It can be any of - * the supported file formats. For drawing of watermarks or non-rectangular - * overlays we recommend using PNG or WebP images. - */ - url: string; - /** - * Floating-point number between 0 (transparent) and 1 (opaque). - * For example, opacity: 0.5 makes overlay semitransparent. - */ - opacity?: number; - /** - * - If set to true, the overlay image will be tiled to cover the entire - * area. This is useful for stock-photo-like watermarks. - * - If set to "x", the overlay image will be tiled horizontally only - * (form a line). - * - If set to "y", the overlay image will be tiled vertically only - * (form a line). - */ - repeat?: true | "x" | "y"; - /** - * Position of the overlay image relative to a given edge. Each property is - * an offset in pixels. 0 aligns exactly to the edge. For example, left: 10 - * positions left side of the overlay 10 pixels from the left edge of the - * image it's drawn over. bottom: 0 aligns bottom of the overlay with bottom - * of the background image. - * - * Setting both left & right, or both top & bottom is an error. - * - * If no position is specified, the image will be centered. - */ - top?: number; - left?: number; - bottom?: number; - right?: number; -} -interface RequestInitCfPropertiesImage extends BasicImageTransformations { - /** - * Device Pixel Ratio. Default 1. Multiplier for width/height that makes it - * easier to specify higher-DPI sizes in . - */ - dpr?: number; - /** - * Allows you to trim your image. Takes dpr into account and is performed before - * resizing or rotation. - * - * It can be used as: - * - left, top, right, bottom - it will specify the number of pixels to cut - * off each side - * - width, height - the width/height you'd like to end up with - can be used - * in combination with the properties above - * - border - this will automatically trim the surroundings of an image based on - * it's color. It consists of three properties: - * - color: rgb or hex representation of the color you wish to trim (todo: verify the rgba bit) - * - tolerance: difference from color to treat as color - * - keep: the number of pixels of border to keep - */ - trim?: "border" | { - top?: number; - bottom?: number; - left?: number; - right?: number; - width?: number; - height?: number; - border?: boolean | { - color?: string; - tolerance?: number; - keep?: number; - }; - }; - /** - * Quality setting from 1-100 (useful values are in 60-90 range). Lower values - * make images look worse, but load faster. The default is 85. It applies only - * to JPEG and WebP images. It doesn’t have any effect on PNG. - */ - quality?: number | "low" | "medium-low" | "medium-high" | "high"; - /** - * Output format to generate. It can be: - * - avif: generate images in AVIF format. - * - webp: generate images in Google WebP format. Set quality to 100 to get - * the WebP-lossless format. - * - json: instead of generating an image, outputs information about the - * image, in JSON format. The JSON object will contain image size - * (before and after resizing), source image’s MIME type, file size, etc. - * - jpeg: generate images in JPEG format. - * - png: generate images in PNG format. - */ - format?: "avif" | "webp" | "json" | "jpeg" | "png" | "baseline-jpeg" | "png-force" | "svg"; - /** - * Whether to preserve animation frames from input files. Default is true. - * Setting it to false reduces animations to still images. This setting is - * recommended when enlarging images or processing arbitrary user content, - * because large GIF animations can weigh tens or even hundreds of megabytes. - * It is also useful to set anim:false when using format:"json" to get the - * response quicker without the number of frames. - */ - anim?: boolean; - /** - * What EXIF data should be preserved in the output image. Note that EXIF - * rotation and embedded color profiles are always applied ("baked in" into - * the image), and aren't affected by this option. Note that if the Polish - * feature is enabled, all metadata may have been removed already and this - * option may have no effect. - * - keep: Preserve most of EXIF metadata, including GPS location if there's - * any. - * - copyright: Only keep the copyright tag, and discard everything else. - * This is the default behavior for JPEG files. - * - none: Discard all invisible EXIF metadata. Currently WebP and PNG - * output formats always discard metadata. - */ - metadata?: "keep" | "copyright" | "none"; - /** - * Strength of sharpening filter to apply to the image. Floating-point - * number between 0 (no sharpening, default) and 10 (maximum). 1.0 is a - * recommended value for downscaled images. - */ - sharpen?: number; - /** - * Radius of a blur filter (approximate gaussian). Maximum supported radius - * is 250. - */ - blur?: number; - /** - * Overlays are drawn in the order they appear in the array (last array - * entry is the topmost layer). - */ - draw?: RequestInitCfPropertiesImageDraw[]; - /** - * Fetching image from authenticated origin. Setting this property will - * pass authentication headers (Authorization, Cookie, etc.) through to - * the origin. - */ - "origin-auth"?: "share-publicly"; - /** - * Adds a border around the image. The border is added after resizing. Border - * width takes dpr into account, and can be specified either using a single - * width property, or individually for each side. - */ - border?: { - color: string; - width: number; - } | { - color: string; - top: number; - right: number; - bottom: number; - left: number; - }; - /** - * Increase brightness by a factor. A value of 1.0 equals no change, a value - * of 0.5 equals half brightness, and a value of 2.0 equals twice as bright. - * 0 is ignored. - */ - brightness?: number; - /** - * Increase contrast by a factor. A value of 1.0 equals no change, a value of - * 0.5 equals low contrast, and a value of 2.0 equals high contrast. 0 is - * ignored. - */ - contrast?: number; - /** - * Increase exposure by a factor. A value of 1.0 equals no change, a value of - * 0.5 darkens the image, and a value of 2.0 lightens the image. 0 is ignored. - */ - gamma?: number; - /** - * Increase contrast by a factor. A value of 1.0 equals no change, a value of - * 0.5 equals low contrast, and a value of 2.0 equals high contrast. 0 is - * ignored. - */ - saturation?: number; - /** - * Flips the images horizontally, vertically, or both. Flipping is applied before - * rotation, so if you apply flip=h,rotate=90 then the image will be flipped - * horizontally, then rotated by 90 degrees. - */ - flip?: 'h' | 'v' | 'hv'; - /** - * Slightly reduces latency on a cache miss by selecting a - * quickest-to-compress file format, at a cost of increased file size and - * lower image quality. It will usually override the format option and choose - * JPEG over WebP or AVIF. We do not recommend using this option, except in - * unusual circumstances like resizing uncacheable dynamically-generated - * images. - */ - compression?: "fast"; -} -interface RequestInitCfPropertiesImageMinify { - javascript?: boolean; - css?: boolean; - html?: boolean; -} -interface RequestInitCfPropertiesR2 { - /** - * Colo id of bucket that an object is stored in - */ - bucketColoId?: number; -} -/** - * Request metadata provided by Cloudflare's edge. - */ -type IncomingRequestCfProperties = IncomingRequestCfPropertiesBase & IncomingRequestCfPropertiesBotManagementEnterprise & IncomingRequestCfPropertiesCloudflareForSaaSEnterprise & IncomingRequestCfPropertiesGeographicInformation & IncomingRequestCfPropertiesCloudflareAccessOrApiShield; -interface IncomingRequestCfPropertiesBase extends Record { - /** - * [ASN](https://www.iana.org/assignments/as-numbers/as-numbers.xhtml) of the incoming request. - * - * @example 395747 - */ - asn?: number; - /** - * The organization which owns the ASN of the incoming request. - * - * @example "Google Cloud" - */ - asOrganization?: string; - /** - * The original value of the `Accept-Encoding` header if Cloudflare modified it. - * - * @example "gzip, deflate, br" - */ - clientAcceptEncoding?: string; - /** - * The number of milliseconds it took for the request to reach your worker. - * - * @example 22 - */ - clientTcpRtt?: number; - /** - * The three-letter [IATA](https://en.wikipedia.org/wiki/IATA_airport_code) - * airport code of the data center that the request hit. - * - * @example "DFW" - */ - colo: string; - /** - * Represents the upstream's response to a - * [TCP `keepalive` message](https://tldp.org/HOWTO/TCP-Keepalive-HOWTO/overview.html) - * from cloudflare. - * - * For workers with no upstream, this will always be `1`. - * - * @example 3 - */ - edgeRequestKeepAliveStatus: IncomingRequestCfPropertiesEdgeRequestKeepAliveStatus; - /** - * The HTTP Protocol the request used. - * - * @example "HTTP/2" - */ - httpProtocol: string; - /** - * The browser-requested prioritization information in the request object. - * - * If no information was set, defaults to the empty string `""` - * - * @example "weight=192;exclusive=0;group=3;group-weight=127" - * @default "" - */ - requestPriority: string; - /** - * The TLS version of the connection to Cloudflare. - * In requests served over plaintext (without TLS), this property is the empty string `""`. - * - * @example "TLSv1.3" - */ - tlsVersion: string; - /** - * The cipher for the connection to Cloudflare. - * In requests served over plaintext (without TLS), this property is the empty string `""`. - * - * @example "AEAD-AES128-GCM-SHA256" - */ - tlsCipher: string; - /** - * Metadata containing the [`HELLO`](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.1.2) and [`FINISHED`](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.9) messages from this request's TLS handshake. - * - * If the incoming request was served over plaintext (without TLS) this field is undefined. - */ - tlsExportedAuthenticator?: IncomingRequestCfPropertiesExportedAuthenticatorMetadata; -} -interface IncomingRequestCfPropertiesBotManagementBase { - /** - * Cloudflare’s [level of certainty](https://developers.cloudflare.com/bots/concepts/bot-score/) that a request comes from a bot, - * represented as an integer percentage between `1` (almost certainly a bot) and `99` (almost certainly human). - * - * @example 54 - */ - score: number; - /** - * A boolean value that is true if the request comes from a good bot, like Google or Bing. - * Most customers choose to allow this traffic. For more details, see [Traffic from known bots](https://developers.cloudflare.com/firewall/known-issues-and-faq/#how-does-firewall-rules-handle-traffic-from-known-bots). - */ - verifiedBot: boolean; - /** - * A boolean value that is true if the request originates from a - * Cloudflare-verified proxy service. - */ - corporateProxy: boolean; - /** - * A boolean value that's true if the request matches [file extensions](https://developers.cloudflare.com/bots/reference/static-resources/) for many types of static resources. - */ - staticResource: boolean; - /** - * List of IDs that correlate to the Bot Management heuristic detections made on a request (you can have multiple heuristic detections on the same request). - */ - detectionIds: number[]; -} -interface IncomingRequestCfPropertiesBotManagement { - /** - * Results of Cloudflare's Bot Management analysis - */ - botManagement: IncomingRequestCfPropertiesBotManagementBase; - /** - * Duplicate of `botManagement.score`. - * - * @deprecated - */ - clientTrustScore: number; -} -interface IncomingRequestCfPropertiesBotManagementEnterprise extends IncomingRequestCfPropertiesBotManagement { - /** - * Results of Cloudflare's Bot Management analysis - */ - botManagement: IncomingRequestCfPropertiesBotManagementBase & { - /** - * A [JA3 Fingerprint](https://developers.cloudflare.com/bots/concepts/ja3-fingerprint/) to help profile specific SSL/TLS clients - * across different destination IPs, Ports, and X509 certificates. - */ - ja3Hash: string; - }; -} -interface IncomingRequestCfPropertiesCloudflareForSaaSEnterprise { - /** - * Custom metadata set per-host in [Cloudflare for SaaS](https://developers.cloudflare.com/cloudflare-for-platforms/cloudflare-for-saas/). - * - * This field is only present if you have Cloudflare for SaaS enabled on your account - * and you have followed the [required steps to enable it]((https://developers.cloudflare.com/cloudflare-for-platforms/cloudflare-for-saas/domain-support/custom-metadata/)). - */ - hostMetadata?: HostMetadata; -} -interface IncomingRequestCfPropertiesCloudflareAccessOrApiShield { - /** - * Information about the client certificate presented to Cloudflare. - * - * This is populated when the incoming request is served over TLS using - * either Cloudflare Access or API Shield (mTLS) - * and the presented SSL certificate has a valid - * [Certificate Serial Number](https://ldapwiki.com/wiki/Certificate%20Serial%20Number) - * (i.e., not `null` or `""`). - * - * Otherwise, a set of placeholder values are used. - * - * The property `certPresented` will be set to `"1"` when - * the object is populated (i.e. the above conditions were met). - */ - tlsClientAuth: IncomingRequestCfPropertiesTLSClientAuth | IncomingRequestCfPropertiesTLSClientAuthPlaceholder; -} -/** - * Metadata about the request's TLS handshake - */ -interface IncomingRequestCfPropertiesExportedAuthenticatorMetadata { - /** - * The client's [`HELLO` message](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.1.2), encoded in hexadecimal - * - * @example "44372ba35fa1270921d318f34c12f155dc87b682cf36a790cfaa3ba8737a1b5d" - */ - clientHandshake: string; - /** - * The server's [`HELLO` message](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.1.2), encoded in hexadecimal - * - * @example "44372ba35fa1270921d318f34c12f155dc87b682cf36a790cfaa3ba8737a1b5d" - */ - serverHandshake: string; - /** - * The client's [`FINISHED` message](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.9), encoded in hexadecimal - * - * @example "084ee802fe1348f688220e2a6040a05b2199a761f33cf753abb1b006792d3f8b" - */ - clientFinished: string; - /** - * The server's [`FINISHED` message](https://www.rfc-editor.org/rfc/rfc5246#section-7.4.9), encoded in hexadecimal - * - * @example "084ee802fe1348f688220e2a6040a05b2199a761f33cf753abb1b006792d3f8b" - */ - serverFinished: string; -} -/** - * Geographic data about the request's origin. - */ -interface IncomingRequestCfPropertiesGeographicInformation { - /** - * The [ISO 3166-1 Alpha 2](https://www.iso.org/iso-3166-country-codes.html) country code the request originated from. - * - * If your worker is [configured to accept TOR connections](https://support.cloudflare.com/hc/en-us/articles/203306930-Understanding-Cloudflare-Tor-support-and-Onion-Routing), this may also be `"T1"`, indicating a request that originated over TOR. - * - * If Cloudflare is unable to determine where the request originated this property is omitted. - * - * The country code `"T1"` is used for requests originating on TOR. - * - * @example "GB" - */ - country?: Iso3166Alpha2Code | "T1"; - /** - * If present, this property indicates that the request originated in the EU - * - * @example "1" - */ - isEUCountry?: "1"; - /** - * A two-letter code indicating the continent the request originated from. - * - * @example "AN" - */ - continent?: ContinentCode; - /** - * The city the request originated from - * - * @example "Austin" - */ - city?: string; - /** - * Postal code of the incoming request - * - * @example "78701" - */ - postalCode?: string; - /** - * Latitude of the incoming request - * - * @example "30.27130" - */ - latitude?: string; - /** - * Longitude of the incoming request - * - * @example "-97.74260" - */ - longitude?: string; - /** - * Timezone of the incoming request - * - * @example "America/Chicago" - */ - timezone?: string; - /** - * If known, the ISO 3166-2 name for the first level region associated with - * the IP address of the incoming request - * - * @example "Texas" - */ - region?: string; - /** - * If known, the ISO 3166-2 code for the first-level region associated with - * the IP address of the incoming request - * - * @example "TX" - */ - regionCode?: string; - /** - * Metro code (DMA) of the incoming request - * - * @example "635" - */ - metroCode?: string; -} -/** Data about the incoming request's TLS certificate */ -interface IncomingRequestCfPropertiesTLSClientAuth { - /** Always `"1"`, indicating that the certificate was presented */ - certPresented: "1"; - /** - * Result of certificate verification. - * - * @example "FAILED:self signed certificate" - */ - certVerified: Exclude; - /** The presented certificate's revokation status. - * - * - A value of `"1"` indicates the certificate has been revoked - * - A value of `"0"` indicates the certificate has not been revoked - */ - certRevoked: "1" | "0"; - /** - * The certificate issuer's [distinguished name](https://knowledge.digicert.com/generalinformation/INFO1745.html) - * - * @example "CN=cloudflareaccess.com, C=US, ST=Texas, L=Austin, O=Cloudflare" - */ - certIssuerDN: string; - /** - * The certificate subject's [distinguished name](https://knowledge.digicert.com/generalinformation/INFO1745.html) - * - * @example "CN=*.cloudflareaccess.com, C=US, ST=Texas, L=Austin, O=Cloudflare" - */ - certSubjectDN: string; - /** - * The certificate issuer's [distinguished name](https://knowledge.digicert.com/generalinformation/INFO1745.html) ([RFC 2253](https://www.rfc-editor.org/rfc/rfc2253.html) formatted) - * - * @example "CN=cloudflareaccess.com, C=US, ST=Texas, L=Austin, O=Cloudflare" - */ - certIssuerDNRFC2253: string; - /** - * The certificate subject's [distinguished name](https://knowledge.digicert.com/generalinformation/INFO1745.html) ([RFC 2253](https://www.rfc-editor.org/rfc/rfc2253.html) formatted) - * - * @example "CN=*.cloudflareaccess.com, C=US, ST=Texas, L=Austin, O=Cloudflare" - */ - certSubjectDNRFC2253: string; - /** The certificate issuer's distinguished name (legacy policies) */ - certIssuerDNLegacy: string; - /** The certificate subject's distinguished name (legacy policies) */ - certSubjectDNLegacy: string; - /** - * The certificate's serial number - * - * @example "00936EACBE07F201DF" - */ - certSerial: string; - /** - * The certificate issuer's serial number - * - * @example "2489002934BDFEA34" - */ - certIssuerSerial: string; - /** - * The certificate's Subject Key Identifier - * - * @example "BB:AF:7E:02:3D:FA:A6:F1:3C:84:8E:AD:EE:38:98:EC:D9:32:32:D4" - */ - certSKI: string; - /** - * The certificate issuer's Subject Key Identifier - * - * @example "BB:AF:7E:02:3D:FA:A6:F1:3C:84:8E:AD:EE:38:98:EC:D9:32:32:D4" - */ - certIssuerSKI: string; - /** - * The certificate's SHA-1 fingerprint - * - * @example "6b9109f323999e52259cda7373ff0b4d26bd232e" - */ - certFingerprintSHA1: string; - /** - * The certificate's SHA-256 fingerprint - * - * @example "acf77cf37b4156a2708e34c4eb755f9b5dbbe5ebb55adfec8f11493438d19e6ad3f157f81fa3b98278453d5652b0c1fd1d71e5695ae4d709803a4d3f39de9dea" - */ - certFingerprintSHA256: string; - /** - * The effective starting date of the certificate - * - * @example "Dec 22 19:39:00 2018 GMT" - */ - certNotBefore: string; - /** - * The effective expiration date of the certificate - * - * @example "Dec 22 19:39:00 2018 GMT" - */ - certNotAfter: string; -} -/** Placeholder values for TLS Client Authorization */ -interface IncomingRequestCfPropertiesTLSClientAuthPlaceholder { - certPresented: "0"; - certVerified: "NONE"; - certRevoked: "0"; - certIssuerDN: ""; - certSubjectDN: ""; - certIssuerDNRFC2253: ""; - certSubjectDNRFC2253: ""; - certIssuerDNLegacy: ""; - certSubjectDNLegacy: ""; - certSerial: ""; - certIssuerSerial: ""; - certSKI: ""; - certIssuerSKI: ""; - certFingerprintSHA1: ""; - certFingerprintSHA256: ""; - certNotBefore: ""; - certNotAfter: ""; -} -/** Possible outcomes of TLS verification */ -declare type CertVerificationStatus = -/** Authentication succeeded */ -"SUCCESS" -/** No certificate was presented */ - | "NONE" -/** Failed because the certificate was self-signed */ - | "FAILED:self signed certificate" -/** Failed because the certificate failed a trust chain check */ - | "FAILED:unable to verify the first certificate" -/** Failed because the certificate not yet valid */ - | "FAILED:certificate is not yet valid" -/** Failed because the certificate is expired */ - | "FAILED:certificate has expired" -/** Failed for another unspecified reason */ - | "FAILED"; -/** - * An upstream endpoint's response to a TCP `keepalive` message from Cloudflare. - */ -declare type IncomingRequestCfPropertiesEdgeRequestKeepAliveStatus = 0 /** Unknown */ | 1 /** no keepalives (not found) */ | 2 /** no connection re-use, opening keepalive connection failed */ | 3 /** no connection re-use, keepalive accepted and saved */ | 4 /** connection re-use, refused by the origin server (`TCP FIN`) */ | 5; /** connection re-use, accepted by the origin server */ -/** ISO 3166-1 Alpha-2 codes */ -declare type Iso3166Alpha2Code = "AD" | "AE" | "AF" | "AG" | "AI" | "AL" | "AM" | "AO" | "AQ" | "AR" | "AS" | "AT" | "AU" | "AW" | "AX" | "AZ" | "BA" | "BB" | "BD" | "BE" | "BF" | "BG" | "BH" | "BI" | "BJ" | "BL" | "BM" | "BN" | "BO" | "BQ" | "BR" | "BS" | "BT" | "BV" | "BW" | "BY" | "BZ" | "CA" | "CC" | "CD" | "CF" | "CG" | "CH" | "CI" | "CK" | "CL" | "CM" | "CN" | "CO" | "CR" | "CU" | "CV" | "CW" | "CX" | "CY" | "CZ" | "DE" | "DJ" | "DK" | "DM" | "DO" | "DZ" | "EC" | "EE" | "EG" | "EH" | "ER" | "ES" | "ET" | "FI" | "FJ" | "FK" | "FM" | "FO" | "FR" | "GA" | "GB" | "GD" | "GE" | "GF" | "GG" | "GH" | "GI" | "GL" | "GM" | "GN" | "GP" | "GQ" | "GR" | "GS" | "GT" | "GU" | "GW" | "GY" | "HK" | "HM" | "HN" | "HR" | "HT" | "HU" | "ID" | "IE" | "IL" | "IM" | "IN" | "IO" | "IQ" | "IR" | "IS" | "IT" | "JE" | "JM" | "JO" | "JP" | "KE" | "KG" | "KH" | "KI" | "KM" | "KN" | "KP" | "KR" | "KW" | "KY" | "KZ" | "LA" | "LB" | "LC" | "LI" | "LK" | "LR" | "LS" | "LT" | "LU" | "LV" | "LY" | "MA" | "MC" | "MD" | "ME" | "MF" | "MG" | "MH" | "MK" | "ML" | "MM" | "MN" | "MO" | "MP" | "MQ" | "MR" | "MS" | "MT" | "MU" | "MV" | "MW" | "MX" | "MY" | "MZ" | "NA" | "NC" | "NE" | "NF" | "NG" | "NI" | "NL" | "NO" | "NP" | "NR" | "NU" | "NZ" | "OM" | "PA" | "PE" | "PF" | "PG" | "PH" | "PK" | "PL" | "PM" | "PN" | "PR" | "PS" | "PT" | "PW" | "PY" | "QA" | "RE" | "RO" | "RS" | "RU" | "RW" | "SA" | "SB" | "SC" | "SD" | "SE" | "SG" | "SH" | "SI" | "SJ" | "SK" | "SL" | "SM" | "SN" | "SO" | "SR" | "SS" | "ST" | "SV" | "SX" | "SY" | "SZ" | "TC" | "TD" | "TF" | "TG" | "TH" | "TJ" | "TK" | "TL" | "TM" | "TN" | "TO" | "TR" | "TT" | "TV" | "TW" | "TZ" | "UA" | "UG" | "UM" | "US" | "UY" | "UZ" | "VA" | "VC" | "VE" | "VG" | "VI" | "VN" | "VU" | "WF" | "WS" | "YE" | "YT" | "ZA" | "ZM" | "ZW"; -/** The 2-letter continent codes Cloudflare uses */ -declare type ContinentCode = "AF" | "AN" | "AS" | "EU" | "NA" | "OC" | "SA"; -type CfProperties = IncomingRequestCfProperties | RequestInitCfProperties; -interface D1Meta { - duration: number; - size_after: number; - rows_read: number; - rows_written: number; - last_row_id: number; - changed_db: boolean; - changes: number; - /** - * The region of the database instance that executed the query. - */ - served_by_region?: string; - /** - * The three letters airport code of the colo that executed the query. - */ - served_by_colo?: string; - /** - * True if-and-only-if the database instance that executed the query was the primary. - */ - served_by_primary?: boolean; - timings?: { - /** - * The duration of the SQL query execution by the database instance. It doesn't include any network time. - */ - sql_duration_ms: number; - }; - /** - * Number of total attempts to execute the query, due to automatic retries. - * Note: All other fields in the response like `timings` only apply to the last attempt. - */ - total_attempts?: number; -} -interface D1Response { - success: true; - meta: D1Meta & Record; - error?: never; -} -type D1Result = D1Response & { - results: T[]; -}; -interface D1ExecResult { - count: number; - duration: number; -} -type D1SessionConstraint = -// Indicates that the first query should go to the primary, and the rest queries -// using the same D1DatabaseSession will go to any replica that is consistent with -// the bookmark maintained by the session (returned by the first query). -'first-primary' -// Indicates that the first query can go anywhere (primary or replica), and the rest queries -// using the same D1DatabaseSession will go to any replica that is consistent with -// the bookmark maintained by the session (returned by the first query). - | 'first-unconstrained'; -type D1SessionBookmark = string; -declare abstract class D1Database { - prepare(query: string): D1PreparedStatement; - batch(statements: D1PreparedStatement[]): Promise[]>; - exec(query: string): Promise; - /** - * Creates a new D1 Session anchored at the given constraint or the bookmark. - * All queries executed using the created session will have sequential consistency, - * meaning that all writes done through the session will be visible in subsequent reads. - * - * @param constraintOrBookmark Either the session constraint or the explicit bookmark to anchor the created session. - */ - withSession(constraintOrBookmark?: D1SessionBookmark | D1SessionConstraint): D1DatabaseSession; - /** - * @deprecated dump() will be removed soon, only applies to deprecated alpha v1 databases. - */ - dump(): Promise; -} -declare abstract class D1DatabaseSession { - prepare(query: string): D1PreparedStatement; - batch(statements: D1PreparedStatement[]): Promise[]>; - /** - * @returns The latest session bookmark across all executed queries on the session. - * If no query has been executed yet, `null` is returned. - */ - getBookmark(): D1SessionBookmark | null; -} -declare abstract class D1PreparedStatement { - bind(...values: unknown[]): D1PreparedStatement; - first(colName: string): Promise; - first>(): Promise; - run>(): Promise>; - all>(): Promise>; - raw(options: { - columnNames: true; - }): Promise<[ - string[], - ...T[] - ]>; - raw(options?: { - columnNames?: false; - }): Promise; -} -// `Disposable` was added to TypeScript's standard lib types in version 5.2. -// To support older TypeScript versions, define an empty `Disposable` interface. -// Users won't be able to use `using`/`Symbol.dispose` without upgrading to 5.2, -// but this will ensure type checking on older versions still passes. -// TypeScript's interface merging will ensure our empty interface is effectively -// ignored when `Disposable` is included in the standard lib. -interface Disposable { -} -/** - * The returned data after sending an email - */ -interface EmailSendResult { - /** - * The Email Message ID - */ - messageId: string; -} -/** - * An email message that can be sent from a Worker. - */ -interface EmailMessage { - /** - * Envelope From attribute of the email message. - */ - readonly from: string; - /** - * Envelope To attribute of the email message. - */ - readonly to: string; -} -/** - * An email message that is sent to a consumer Worker and can be rejected/forwarded. - */ -interface ForwardableEmailMessage extends EmailMessage { - /** - * Stream of the email message content. - */ - readonly raw: ReadableStream; - /** - * An [Headers object](https://developer.mozilla.org/en-US/docs/Web/API/Headers). - */ - readonly headers: Headers; - /** - * Size of the email message content. - */ - readonly rawSize: number; - /** - * Reject this email message by returning a permanent SMTP error back to the connecting client including the given reason. - * @param reason The reject reason. - * @returns void - */ - setReject(reason: string): void; - /** - * Forward this email message to a verified destination address of the account. - * @param rcptTo Verified destination address. - * @param headers A [Headers object](https://developer.mozilla.org/en-US/docs/Web/API/Headers). - * @returns A promise that resolves when the email message is forwarded. - */ - forward(rcptTo: string, headers?: Headers): Promise; - /** - * Reply to the sender of this email message with a new EmailMessage object. - * @param message The reply message. - * @returns A promise that resolves when the email message is replied. - */ - reply(message: EmailMessage): Promise; -} -/** A file attachment for an email message */ -type EmailAttachment = { - disposition: 'inline'; - contentId: string; - filename: string; - type: string; - content: string | ArrayBuffer | ArrayBufferView; -} | { - disposition: 'attachment'; - contentId?: undefined; - filename: string; - type: string; - content: string | ArrayBuffer | ArrayBufferView; -}; -/** An Email Address */ -interface EmailAddress { - name: string; - email: string; -} -/** - * A binding that allows a Worker to send email messages. - */ -interface SendEmail { - send(message: EmailMessage): Promise; - send(builder: { - from: string | EmailAddress; - to: string | string[]; - subject: string; - replyTo?: string | EmailAddress; - cc?: string | string[]; - bcc?: string | string[]; - headers?: Record; - text?: string; - html?: string; - attachments?: EmailAttachment[]; - }): Promise; -} -declare abstract class EmailEvent extends ExtendableEvent { - readonly message: ForwardableEmailMessage; -} -declare type EmailExportedHandler = (message: ForwardableEmailMessage, env: Env, ctx: ExecutionContext) => void | Promise; -declare module "cloudflare:email" { - let _EmailMessage: { - prototype: EmailMessage; - new (from: string, to: string, raw: ReadableStream | string): EmailMessage; - }; - export { _EmailMessage as EmailMessage }; -} -/** - * Evaluation context for targeting rules. - * Keys are attribute names (e.g. "userId", "country"), values are the attribute values. - */ -type FlagshipEvaluationContext = Record; -interface FlagshipEvaluationDetails { - flagKey: string; - value: T; - variant?: string | undefined; - reason?: string | undefined; - errorCode?: string | undefined; - errorMessage?: string | undefined; -} -interface FlagshipEvaluationError extends Error { -} -/** - * Feature flags binding for evaluating feature flags from a Cloudflare Workers script. - * - * @example - * ```typescript - * // Get a boolean flag value with a default - * const enabled = await env.FLAGS.getBooleanValue('my-feature', false); - * - * // Get a flag value with evaluation context for targeting - * const variant = await env.FLAGS.getStringValue('experiment', 'control', { - * userId: 'user-123', - * country: 'US', - * }); - * - * // Get full evaluation details including variant and reason - * const details = await env.FLAGS.getBooleanDetails('my-feature', false); - * console.log(details.variant, details.reason); - * ``` - */ -declare abstract class Flagship { - /** - * Get a flag value without type checking. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Optional default value returned when evaluation fails. - * @param context Optional evaluation context for targeting rules. - */ - get(flagKey: string, defaultValue?: unknown, context?: FlagshipEvaluationContext): Promise; - /** - * Get a boolean flag value. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getBooleanValue(flagKey: string, defaultValue: boolean, context?: FlagshipEvaluationContext): Promise; - /** - * Get a string flag value. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getStringValue(flagKey: string, defaultValue: string, context?: FlagshipEvaluationContext): Promise; - /** - * Get a number flag value. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getNumberValue(flagKey: string, defaultValue: number, context?: FlagshipEvaluationContext): Promise; - /** - * Get an object flag value. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getObjectValue(flagKey: string, defaultValue: T, context?: FlagshipEvaluationContext): Promise; - /** - * Get a boolean flag value with full evaluation details. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getBooleanDetails(flagKey: string, defaultValue: boolean, context?: FlagshipEvaluationContext): Promise>; - /** - * Get a string flag value with full evaluation details. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getStringDetails(flagKey: string, defaultValue: string, context?: FlagshipEvaluationContext): Promise>; - /** - * Get a number flag value with full evaluation details. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getNumberDetails(flagKey: string, defaultValue: number, context?: FlagshipEvaluationContext): Promise>; - /** - * Get an object flag value with full evaluation details. - * @param flagKey The key of the flag to evaluate. - * @param defaultValue Default value returned when evaluation fails or the flag type does not match. - * @param context Optional evaluation context for targeting rules. - */ - getObjectDetails(flagKey: string, defaultValue: T, context?: FlagshipEvaluationContext): Promise>; -} -/** - * Hello World binding to serve as an explanatory example. DO NOT USE - */ -interface HelloWorldBinding { - /** - * Retrieve the current stored value - */ - get(): Promise<{ - value: string; - ms?: number; - }>; - /** - * Set a new stored value - */ - set(value: string): Promise; -} -interface Hyperdrive { - /** - * Connect directly to Hyperdrive as if it's your database, returning a TCP socket. - * - * Calling this method returns an identical socket to if you call - * `connect("host:port")` using the `host` and `port` fields from this object. - * Pick whichever approach works better with your preferred DB client library. - * - * Note that this socket is not yet authenticated -- it's expected that your - * code (or preferably, the client library of your choice) will authenticate - * using the information in this class's readonly fields. - */ - connect(): Socket; - /** - * A valid DB connection string that can be passed straight into the typical - * client library/driver/ORM. This will typically be the easiest way to use - * Hyperdrive. - */ - readonly connectionString: string; - /* - * A randomly generated hostname that is only valid within the context of the - * currently running Worker which, when passed into `connect()` function from - * the "cloudflare:sockets" module, will connect to the Hyperdrive instance - * for your database. - */ - readonly host: string; - /* - * The port that must be paired the the host field when connecting. - */ - readonly port: number; - /* - * The username to use when authenticating to your database via Hyperdrive. - * Unlike the host and password, this will be the same every time - */ - readonly user: string; - /* - * The randomly generated password to use when authenticating to your - * database via Hyperdrive. Like the host field, this password is only valid - * within the context of the currently running Worker instance from which - * it's read. - */ - readonly password: string; - /* - * The name of the database to connect to. - */ - readonly database: string; -} -// Copyright (c) 2024 Cloudflare, Inc. -// Licensed under the Apache 2.0 license found in the LICENSE file or at: -// https://opensource.org/licenses/Apache-2.0 -type ImageInfoResponse = { - format: 'image/svg+xml'; -} | { - format: string; - fileSize: number; - width: number; - height: number; -}; -type ImageTransform = { - width?: number; - height?: number; - background?: string; - blur?: number; - border?: { - color?: string; - width?: number; - } | { - top?: number; - bottom?: number; - left?: number; - right?: number; - }; - brightness?: number; - contrast?: number; - fit?: 'scale-down' | 'contain' | 'pad' | 'squeeze' | 'cover' | 'crop'; - flip?: 'h' | 'v' | 'hv'; - gamma?: number; - segment?: 'foreground'; - gravity?: 'face' | 'left' | 'right' | 'top' | 'bottom' | 'center' | 'auto' | 'entropy' | { - x?: number; - y?: number; - mode: 'remainder' | 'box-center'; - }; - rotate?: 0 | 90 | 180 | 270; - saturation?: number; - sharpen?: number; - trim?: 'border' | { - top?: number; - bottom?: number; - left?: number; - right?: number; - width?: number; - height?: number; - border?: boolean | { - color?: string; - tolerance?: number; - keep?: number; - }; - }; -}; -type ImageDrawOptions = { - opacity?: number; - repeat?: boolean | string; - top?: number; - left?: number; - bottom?: number; - right?: number; -}; -type ImageInputOptions = { - encoding?: 'base64'; -}; -type ImageOutputOptions = { - format: 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp' | 'image/avif' | 'rgb' | 'rgba'; - quality?: number; - background?: string; - anim?: boolean; -}; -interface ImageMetadata { - id: string; - filename?: string; - uploaded?: string; - requireSignedURLs: boolean; - meta?: Record; - variants: string[]; - draft?: boolean; - creator?: string; -} -interface ImageUploadOptions { - id?: string; - filename?: string; - requireSignedURLs?: boolean; - metadata?: Record; - creator?: string; - encoding?: 'base64'; -} -interface ImageUpdateOptions { - requireSignedURLs?: boolean; - metadata?: Record; - creator?: string; -} -interface ImageListOptions { - limit?: number; - cursor?: string; - sortOrder?: 'asc' | 'desc'; - creator?: string; -} -interface ImageList { - images: ImageMetadata[]; - cursor?: string; - listComplete: boolean; -} -interface ImageHandle { - /** - * Get metadata for a hosted image - * @returns Image metadata, or null if not found - */ - details(): Promise; - /** - * Get the raw image data for a hosted image - * @returns ReadableStream of image bytes, or null if not found - */ - bytes(): Promise | null>; - /** - * Update hosted image metadata - * @param options Properties to update - * @returns Updated image metadata - * @throws {@link ImagesError} if update fails - */ - update(options: ImageUpdateOptions): Promise; - /** - * Delete a hosted image - * @returns True if deleted, false if not found - */ - delete(): Promise; -} -interface HostedImagesBinding { - /** - * Get a handle for a hosted image - * @param imageId The ID of the image (UUID or custom ID) - * @returns A handle for per-image operations - */ - image(imageId: string): ImageHandle; - /** - * Upload a new hosted image - * @param image The image file to upload - * @param options Upload configuration - * @returns Metadata for the uploaded image - * @throws {@link ImagesError} if upload fails - */ - upload(image: ReadableStream | ArrayBuffer, options?: ImageUploadOptions): Promise; - /** - * List hosted images with pagination - * @param options List configuration - * @returns List of images with pagination info - * @throws {@link ImagesError} if list fails - */ - list(options?: ImageListOptions): Promise; -} -interface ImagesBinding { - /** - * Get image metadata (type, width and height) - * @throws {@link ImagesError} with code 9412 if input is not an image - * @param stream The image bytes - */ - info(stream: ReadableStream, options?: ImageInputOptions): Promise; - /** - * Begin applying a series of transformations to an image - * @param stream The image bytes - * @returns A transform handle - */ - input(stream: ReadableStream, options?: ImageInputOptions): ImageTransformer; - /** - * Access hosted images CRUD operations - */ - readonly hosted: HostedImagesBinding; -} -interface ImageTransformer { - /** - * Apply transform next, returning a transform handle. - * You can then apply more transformations, draw, or retrieve the output. - * @param transform - */ - transform(transform: ImageTransform): ImageTransformer; - /** - * Draw an image on this transformer, returning a transform handle. - * You can then apply more transformations, draw, or retrieve the output. - * @param image The image (or transformer that will give the image) to draw - * @param options The options configuring how to draw the image - */ - draw(image: ReadableStream | ImageTransformer, options?: ImageDrawOptions): ImageTransformer; - /** - * Retrieve the image that results from applying the transforms to the - * provided input - * @param options Options that apply to the output e.g. output format - */ - output(options: ImageOutputOptions): Promise; -} -type ImageTransformationOutputOptions = { - encoding?: 'base64'; -}; -interface ImageTransformationResult { - /** - * The image as a response, ready to store in cache or return to users - */ - response(): Response; - /** - * The content type of the returned image - */ - contentType(): string; - /** - * The bytes of the response - */ - image(options?: ImageTransformationOutputOptions): ReadableStream; -} -interface ImagesError extends Error { - readonly code: number; - readonly message: string; - readonly stack?: string; -} -/** - * Media binding for transforming media streams. - * Provides the entry point for media transformation operations. - */ -interface MediaBinding { - /** - * Creates a media transformer from an input stream. - * @param media - The input media bytes - * @returns A MediaTransformer instance for applying transformations - */ - input(media: ReadableStream): MediaTransformer; -} -/** - * Media transformer for applying transformation operations to media content. - * Handles sizing, fitting, and other input transformation parameters. - */ -interface MediaTransformer { - /** - * Applies transformation options to the media content. - * @param transform - Configuration for how the media should be transformed - * @returns A generator for producing the transformed media output - */ - transform(transform?: MediaTransformationInputOptions): MediaTransformationGenerator; - /** - * Generates the final media output with specified options. - * @param output - Configuration for the output format and parameters - * @returns The final transformation result containing the transformed media - */ - output(output?: MediaTransformationOutputOptions): MediaTransformationResult; -} -/** - * Generator for producing media transformation results. - * Configures the output format and parameters for the transformed media. - */ -interface MediaTransformationGenerator { - /** - * Generates the final media output with specified options. - * @param output - Configuration for the output format and parameters - * @returns The final transformation result containing the transformed media - */ - output(output?: MediaTransformationOutputOptions): MediaTransformationResult; -} -/** - * Result of a media transformation operation. - * Provides multiple ways to access the transformed media content. - */ -interface MediaTransformationResult { - /** - * Returns the transformed media as a readable stream of bytes. - * @returns A promise containing a readable stream with the transformed media - */ - media(): Promise>; - /** - * Returns the transformed media as an HTTP response object. - * @returns The transformed media as a Promise, ready to store in cache or return to users - */ - response(): Promise; - /** - * Returns the MIME type of the transformed media. - * @returns A promise containing the content type string (e.g., 'image/jpeg', 'video/mp4') - */ - contentType(): Promise; -} -/** - * Configuration options for transforming media input. - * Controls how the media should be resized and fitted. - */ -type MediaTransformationInputOptions = { - /** How the media should be resized to fit the specified dimensions */ - fit?: 'contain' | 'cover' | 'scale-down'; - /** Target width in pixels */ - width?: number; - /** Target height in pixels */ - height?: number; -}; -/** - * Configuration options for Media Transformations output. - * Controls the format, timing, and type of the generated output. - */ -type MediaTransformationOutputOptions = { - /** - * Output mode determining the type of media to generate - */ - mode?: 'video' | 'spritesheet' | 'frame' | 'audio'; - /** Whether to include audio in the output */ - audio?: boolean; - /** - * Starting timestamp for frame extraction or start time for clips. (e.g. '2s'). - */ - time?: string; - /** - * Duration for video clips, audio extraction, and spritesheet generation (e.g. '5s'). - */ - duration?: string; - /** - * Number of frames in the spritesheet. - */ - imageCount?: number; - /** - * Output format for the generated media. - */ - format?: 'jpg' | 'png' | 'm4a'; -}; -/** - * Error object for media transformation operations. - * Extends the standard Error interface with additional media-specific information. - */ -interface MediaError extends Error { - readonly code: number; - readonly message: string; - readonly stack?: string; -} -declare module 'cloudflare:node' { - interface NodeStyleServer { - listen(...args: unknown[]): this; - address(): { - port?: number | null | undefined; - }; - } - export function httpServerHandler(port: number): ExportedHandler; - export function httpServerHandler(options: { - port: number; - }): ExportedHandler; - export function httpServerHandler(server: NodeStyleServer): ExportedHandler; -} -type Params

= Record; -type EventContext = { - request: Request>; - functionPath: string; - waitUntil: (promise: Promise) => void; - passThroughOnException: () => void; - next: (input?: Request | string, init?: RequestInit) => Promise; - env: Env & { - ASSETS: { - fetch: typeof fetch; - }; - }; - params: Params

; - data: Data; -}; -type PagesFunction = Record> = (context: EventContext) => Response | Promise; -type EventPluginContext = { - request: Request>; - functionPath: string; - waitUntil: (promise: Promise) => void; - passThroughOnException: () => void; - next: (input?: Request | string, init?: RequestInit) => Promise; - env: Env & { - ASSETS: { - fetch: typeof fetch; - }; - }; - params: Params

; - data: Data; - pluginArgs: PluginArgs; -}; -type PagesPluginFunction = Record, PluginArgs = unknown> = (context: EventPluginContext) => Response | Promise; -declare module "assets:*" { - export const onRequest: PagesFunction; -} -// Copyright (c) 2022-2023 Cloudflare, Inc. -// Licensed under the Apache 2.0 license found in the LICENSE file or at: -// https://opensource.org/licenses/Apache-2.0 -declare module "cloudflare:pipelines" { - export abstract class PipelineTransformationEntrypoint { - protected env: Env; - protected ctx: ExecutionContext; - constructor(ctx: ExecutionContext, env: Env); - /** - * run receives an array of PipelineRecord which can be - * transformed and returned to the pipeline - * @param records Incoming records from the pipeline to be transformed - * @param metadata Information about the specific pipeline calling the transformation entrypoint - * @returns A promise containing the transformed PipelineRecord array - */ - public run(records: I[], metadata: PipelineBatchMetadata): Promise; - } - export type PipelineRecord = Record; - export type PipelineBatchMetadata = { - pipelineId: string; - pipelineName: string; - }; - export interface Pipeline { - /** - * The Pipeline interface represents the type of a binding to a Pipeline - * - * @param records The records to send to the pipeline - */ - send(records: T[]): Promise; - } -} -// PubSubMessage represents an incoming PubSub message. -// The message includes metadata about the broker, the client, and the payload -// itself. -// https://developers.cloudflare.com/pub-sub/ -interface PubSubMessage { - // Message ID - readonly mid: number; - // MQTT broker FQDN in the form mqtts://BROKER.NAMESPACE.cloudflarepubsub.com:PORT - readonly broker: string; - // The MQTT topic the message was sent on. - readonly topic: string; - // The client ID of the client that published this message. - readonly clientId: string; - // The unique identifier (JWT ID) used by the client to authenticate, if token - // auth was used. - readonly jti?: string; - // A Unix timestamp (seconds from Jan 1, 1970), set when the Pub/Sub Broker - // received the message from the client. - readonly receivedAt: number; - // An (optional) string with the MIME type of the payload, if set by the - // client. - readonly contentType: string; - // Set to 1 when the payload is a UTF-8 string - // https://docs.oasis-open.org/mqtt/mqtt/v5.0/os/mqtt-v5.0-os.html#_Toc3901063 - readonly payloadFormatIndicator: number; - // Pub/Sub (MQTT) payloads can be UTF-8 strings, or byte arrays. - // You can use payloadFormatIndicator to inspect this before decoding. - payload: string | Uint8Array; -} -// JsonWebKey extended by kid parameter -interface JsonWebKeyWithKid extends JsonWebKey { - // Key Identifier of the JWK - readonly kid: string; -} -interface RateLimitOptions { - key: string; -} -interface RateLimitOutcome { - success: boolean; -} -interface RateLimit { - /** - * Rate limit a request based on the provided options. - * @see https://developers.cloudflare.com/workers/runtime-apis/bindings/rate-limit/ - * @returns A promise that resolves with the outcome of the rate limit. - */ - limit(options: RateLimitOptions): Promise; -} -// Namespace for RPC utility types. Unfortunately, we can't use a `module` here as these types need -// to referenced by `Fetcher`. This is included in the "importable" version of the types which -// strips all `module` blocks. -declare namespace Rpc { - // Branded types for identifying `WorkerEntrypoint`/`DurableObject`/`Target`s. - // TypeScript uses *structural* typing meaning anything with the same shape as type `T` is a `T`. - // For the classes exported by `cloudflare:workers` we want *nominal* typing (i.e. we only want to - // accept `WorkerEntrypoint` from `cloudflare:workers`, not any other class with the same shape) - export const __RPC_STUB_BRAND: '__RPC_STUB_BRAND'; - export const __RPC_TARGET_BRAND: '__RPC_TARGET_BRAND'; - export const __WORKER_ENTRYPOINT_BRAND: '__WORKER_ENTRYPOINT_BRAND'; - export const __DURABLE_OBJECT_BRAND: '__DURABLE_OBJECT_BRAND'; - export const __WORKFLOW_ENTRYPOINT_BRAND: '__WORKFLOW_ENTRYPOINT_BRAND'; - export interface RpcTargetBranded { - [__RPC_TARGET_BRAND]: never; - } - export interface WorkerEntrypointBranded { - [__WORKER_ENTRYPOINT_BRAND]: never; - } - export interface DurableObjectBranded { - [__DURABLE_OBJECT_BRAND]: never; - } - export interface WorkflowEntrypointBranded { - [__WORKFLOW_ENTRYPOINT_BRAND]: never; - } - export type EntrypointBranded = WorkerEntrypointBranded | DurableObjectBranded | WorkflowEntrypointBranded; - // Types that can be used through `Stub`s - export type Stubable = RpcTargetBranded | ((...args: any[]) => any); - // Types that can be passed over RPC - // The reason for using a generic type here is to build a serializable subset of structured - // cloneable composite types. This allows types defined with the "interface" keyword to pass the - // serializable check as well. Otherwise, only types defined with the "type" keyword would pass. - type Serializable = - // Structured cloneables - BaseType - // Structured cloneable composites - | Map ? Serializable : never, T extends Map ? Serializable : never> | Set ? Serializable : never> | ReadonlyArray ? Serializable : never> | { - [K in keyof T]: K extends number | string ? Serializable : never; - } - // Special types - | Stub - // Serialized as stubs, see `Stubify` - | Stubable; - // Base type for all RPC stubs, including common memory management methods. - // `T` is used as a marker type for unwrapping `Stub`s later. - interface StubBase extends Disposable { - [__RPC_STUB_BRAND]: T; - dup(): this; - } - export type Stub = Provider & StubBase; - // This represents all the types that can be sent as-is over an RPC boundary - type BaseType = void | undefined | null | boolean | number | bigint | string | TypedArray | ArrayBuffer | DataView | Date | Error | RegExp | ReadableStream | WritableStream | Request | Response | Headers; - // Recursively rewrite all `Stubable` types with `Stub`s - // prettier-ignore - type Stubify = T extends Stubable ? Stub : T extends Map ? Map, Stubify> : T extends Set ? Set> : T extends Array ? Array> : T extends ReadonlyArray ? ReadonlyArray> : T extends BaseType ? T : T extends { - [key: string | number]: any; - } ? { - [K in keyof T]: Stubify; - } : T; - // Recursively rewrite all `Stub`s with the corresponding `T`s. - // Note we use `StubBase` instead of `Stub` here to avoid circular dependencies: - // `Stub` depends on `Provider`, which depends on `Unstubify`, which would depend on `Stub`. - // prettier-ignore - type Unstubify = T extends StubBase ? V : T extends Map ? Map, Unstubify> : T extends Set ? Set> : T extends Array ? Array> : T extends ReadonlyArray ? ReadonlyArray> : T extends BaseType ? T : T extends { - [key: string | number]: unknown; - } ? { - [K in keyof T]: Unstubify; - } : T; - type UnstubifyAll = { - [I in keyof A]: Unstubify; - }; - // Utility type for adding `Provider`/`Disposable`s to `object` types only. - // Note `unknown & T` is equivalent to `T`. - type MaybeProvider = T extends object ? Provider : unknown; - type MaybeDisposable = T extends object ? Disposable : unknown; - // Type for method return or property on an RPC interface. - // - Stubable types are replaced by stubs. - // - Serializable types are passed by value, with stubable types replaced by stubs - // and a top-level `Disposer`. - // Everything else can't be passed over PRC. - // Technically, we use custom thenables here, but they quack like `Promise`s. - // Intersecting with `(Maybe)Provider` allows pipelining. - // prettier-ignore - type Result = R extends Stubable ? Promise> & Provider : R extends Serializable ? Promise & MaybeDisposable> & MaybeProvider : never; - // Type for method or property on an RPC interface. - // For methods, unwrap `Stub`s in parameters, and rewrite returns to be `Result`s. - // Unwrapping `Stub`s allows calling with `Stubable` arguments. - // For properties, rewrite types to be `Result`s. - // In each case, unwrap `Promise`s. - type MethodOrProperty = V extends (...args: infer P) => infer R ? (...args: UnstubifyAll

) => Result> : Result>; - // Type for the callable part of an `Provider` if `T` is callable. - // This is intersected with methods/properties. - type MaybeCallableProvider = T extends (...args: any[]) => any ? MethodOrProperty : unknown; - // Base type for all other types providing RPC-like interfaces. - // Rewrites all methods/properties to be `MethodOrProperty`s, while preserving callable types. - // `Reserved` names (e.g. stub method names like `dup()`) and symbols can't be accessed over RPC. - export type Provider = MaybeCallableProvider & Pick<{ - [K in keyof T]: MethodOrProperty; - }, Exclude>>; -} -declare namespace Cloudflare { - // Type of `env`. - // - // The specific project can extend `Env` by redeclaring it in project-specific files. Typescript - // will merge all declarations. - // - // You can use `wrangler types` to generate the `Env` type automatically. - interface Env { - } - // Project-specific parameters used to inform types. - // - // This interface is, again, intended to be declared in project-specific files, and then that - // declaration will be merged with this one. - // - // A project should have a declaration like this: - // - // interface GlobalProps { - // // Declares the main module's exports. Used to populate Cloudflare.Exports aka the type - // // of `ctx.exports`. - // mainModule: typeof import("my-main-module"); - // - // // Declares which of the main module's exports are configured with durable storage, and - // // thus should behave as Durable Object namsepace bindings. - // durableNamespaces: "MyDurableObject" | "AnotherDurableObject"; - // } - // - // You can use `wrangler types` to generate `GlobalProps` automatically. - interface GlobalProps { - } - // Evaluates to the type of a property in GlobalProps, defaulting to `Default` if it is not - // present. - type GlobalProp = K extends keyof GlobalProps ? GlobalProps[K] : Default; - // The type of the program's main module exports, if known. Requires `GlobalProps` to declare the - // `mainModule` property. - type MainModule = GlobalProp<"mainModule", {}>; - // The type of ctx.exports, which contains loopback bindings for all top-level exports. - type Exports = { - [K in keyof MainModule]: LoopbackForExport - // If the export is listed in `durableNamespaces`, then it is also a - // DurableObjectNamespace. - & (K extends GlobalProp<"durableNamespaces", never> ? MainModule[K] extends new (...args: any[]) => infer DoInstance ? DoInstance extends Rpc.DurableObjectBranded ? DurableObjectNamespace : DurableObjectNamespace : DurableObjectNamespace : {}); - }; -} -declare namespace CloudflareWorkersModule { - export type RpcStub = Rpc.Stub; - export const RpcStub: { - new (value: T): Rpc.Stub; - }; - export abstract class RpcTarget implements Rpc.RpcTargetBranded { - [Rpc.__RPC_TARGET_BRAND]: never; - } - // `protected` fields don't appear in `keyof`s, so can't be accessed over RPC - export abstract class WorkerEntrypoint implements Rpc.WorkerEntrypointBranded { - [Rpc.__WORKER_ENTRYPOINT_BRAND]: never; - protected ctx: ExecutionContext; - protected env: Env; - constructor(ctx: ExecutionContext, env: Env); - email?(message: ForwardableEmailMessage): void | Promise; - fetch?(request: Request): Response | Promise; - connect?(socket: Socket): void | Promise; - queue?(batch: MessageBatch): void | Promise; - scheduled?(controller: ScheduledController): void | Promise; - tail?(events: TraceItem[]): void | Promise; - tailStream?(event: TailStream.TailEvent): TailStream.TailEventHandlerType | Promise; - test?(controller: TestController): void | Promise; - trace?(traces: TraceItem[]): void | Promise; - } - export abstract class DurableObject implements Rpc.DurableObjectBranded { - [Rpc.__DURABLE_OBJECT_BRAND]: never; - protected ctx: DurableObjectState; - protected env: Env; - constructor(ctx: DurableObjectState, env: Env); - alarm?(alarmInfo?: AlarmInvocationInfo): void | Promise; - fetch?(request: Request): Response | Promise; - connect?(socket: Socket): void | Promise; - webSocketMessage?(ws: WebSocket, message: string | ArrayBuffer): void | Promise; - webSocketClose?(ws: WebSocket, code: number, reason: string, wasClean: boolean): void | Promise; - webSocketError?(ws: WebSocket, error: unknown): void | Promise; - } - export type WorkflowDurationLabel = 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'year'; - export type WorkflowSleepDuration = `${number} ${WorkflowDurationLabel}${'s' | ''}` | number; - export type WorkflowDelayDuration = WorkflowSleepDuration; - export type WorkflowTimeoutDuration = WorkflowSleepDuration; - export type WorkflowRetentionDuration = WorkflowSleepDuration; - export type WorkflowBackoff = 'constant' | 'linear' | 'exponential'; - export type WorkflowStepConfig = { - retries?: { - limit: number; - delay: WorkflowDelayDuration | number; - backoff?: WorkflowBackoff; - }; - timeout?: WorkflowTimeoutDuration | number; - }; - export type WorkflowEvent = { - payload: Readonly; - timestamp: Date; - instanceId: string; - }; - export type WorkflowStepEvent = { - payload: Readonly; - timestamp: Date; - type: string; - }; - export type WorkflowStepContext = { - step: { - name: string; - count: number; - }; - attempt: number; - config: WorkflowStepConfig; - }; - export abstract class WorkflowStep { - do>(name: string, callback: (ctx: WorkflowStepContext) => Promise): Promise; - do>(name: string, config: WorkflowStepConfig, callback: (ctx: WorkflowStepContext) => Promise): Promise; - sleep: (name: string, duration: WorkflowSleepDuration) => Promise; - sleepUntil: (name: string, timestamp: Date | number) => Promise; - waitForEvent>(name: string, options: { - type: string; - timeout?: WorkflowTimeoutDuration | number; - }): Promise>; - } - export type WorkflowInstanceStatus = 'queued' | 'running' | 'paused' | 'errored' | 'terminated' | 'complete' | 'waiting' | 'waitingForPause' | 'unknown'; - export abstract class WorkflowEntrypoint | unknown = unknown> implements Rpc.WorkflowEntrypointBranded { - [Rpc.__WORKFLOW_ENTRYPOINT_BRAND]: never; - protected ctx: ExecutionContext; - protected env: Env; - constructor(ctx: ExecutionContext, env: Env); - run(event: Readonly>, step: WorkflowStep): Promise; - } - export function waitUntil(promise: Promise): void; - export function withEnv(newEnv: unknown, fn: () => unknown): unknown; - export function withExports(newExports: unknown, fn: () => unknown): unknown; - export function withEnvAndExports(newEnv: unknown, newExports: unknown, fn: () => unknown): unknown; - export const env: Cloudflare.Env; - export const exports: Cloudflare.Exports; -} -declare module 'cloudflare:workers' { - export = CloudflareWorkersModule; -} -interface SecretsStoreSecret { - /** - * Get a secret from the Secrets Store, returning a string of the secret value - * if it exists, or throws an error if it does not exist - */ - get(): Promise; -} -declare module "cloudflare:sockets" { - function _connect(address: string | SocketAddress, options?: SocketOptions): Socket; - export { _connect as connect }; -} -/** - * Binding entrypoint for Cloudflare Stream. - * - * Usage: - * - Binding-level operations: - * `await env.STREAM.videos.upload` - * `await env.STREAM.videos.createDirectUpload` - * `await env.STREAM.videos.*` - * `await env.STREAM.watermarks.*` - * - Per-video operations: - * `await env.STREAM.video(id).downloads.*` - * `await env.STREAM.video(id).captions.*` - * - * Example usage: - * ```ts - * await env.STREAM.video(id).downloads.generate(); - * - * const video = env.STREAM.video(id) - * const captions = video.captions.list(); - * const videoDetails = video.details() - * ``` - */ -interface StreamBinding { - /** - * Returns a handle scoped to a single video for per-video operations. - * @param id The unique identifier for the video. - * @returns A handle for per-video operations. - */ - video(id: string): StreamVideoHandle; - /** - * Uploads a new video from a provided URL. - * @param url The URL to upload from. - * @param params Optional upload parameters. - * @returns The uploaded video details. - * @throws {BadRequestError} if the upload parameter is invalid or the URL is invalid - * @throws {QuotaReachedError} if the account storage capacity is exceeded - * @throws {MaxFileSizeError} if the file size is too large - * @throws {RateLimitedError} if the server received too many requests - * @throws {AlreadyUploadedError} if a video was already uploaded to this URL - * @throws {InternalError} if an unexpected error occurs - */ - upload(url: string, params?: StreamUrlUploadParams): Promise; - /** - * Creates a direct upload that allows video uploads without an API key. - * @param params Parameters for the direct upload - * @returns The direct upload details. - * @throws {BadRequestError} if the parameters are invalid - * @throws {RateLimitedError} if the server received too many requests - * @throws {InternalError} if an unexpected error occurs - */ - createDirectUpload(params: StreamDirectUploadCreateParams): Promise; - videos: StreamVideos; - watermarks: StreamWatermarks; -} -/** - * Handle for operations scoped to a single Stream video. - */ -interface StreamVideoHandle { - /** - * The unique identifier for the video. - */ - id: string; - /** - * Get a full videos details - * @returns The full video details. - * @throws {NotFoundError} if the video is not found - * @throws {InternalError} if an unexpected error occurs - */ - details(): Promise; - /** - * Update details for a single video. - * @param params The fields to update for the video. - * @returns The updated video details. - * @throws {NotFoundError} if the video is not found - * @throws {BadRequestError} if the parameters are invalid - * @throws {InternalError} if an unexpected error occurs - */ - update(params: StreamUpdateVideoParams): Promise; - /** - * Deletes a video and its copies from Cloudflare Stream. - * @returns A promise that resolves when deletion completes. - * @throws {NotFoundError} if the video is not found - * @throws {InternalError} if an unexpected error occurs - */ - delete(): Promise; - /** - * Creates a signed URL token for a video. - * @returns The signed token that was created. - * @throws {InternalError} if the signing key cannot be retrieved or the token cannot be signed - */ - generateToken(): Promise; - downloads: StreamScopedDownloads; - captions: StreamScopedCaptions; -} -interface StreamVideo { - /** - * The unique identifier for the video. - */ - id: string; - /** - * A user-defined identifier for the media creator. - */ - creator: string | null; - /** - * The thumbnail URL for the video. - */ - thumbnail: string; - /** - * The thumbnail timestamp percentage. - */ - thumbnailTimestampPct: number; - /** - * Indicates whether the video is ready to stream. - */ - readyToStream: boolean; - /** - * The date and time the video became ready to stream. - */ - readyToStreamAt: string | null; - /** - * Processing status information. - */ - status: StreamVideoStatus; - /** - * A user modifiable key-value store. - */ - meta: Record; - /** - * The date and time the video was created. - */ - created: string; - /** - * The date and time the video was last modified. - */ - modified: string; - /** - * The date and time at which the video will be deleted. - */ - scheduledDeletion: string | null; - /** - * The size of the video in bytes. - */ - size: number; - /** - * The preview URL for the video. - */ - preview?: string; - /** - * Origins allowed to display the video. - */ - allowedOrigins: Array; - /** - * Indicates whether signed URLs are required. - */ - requireSignedURLs: boolean | null; - /** - * The date and time the video was uploaded. - */ - uploaded: string | null; - /** - * The date and time when the upload URL expires. - */ - uploadExpiry: string | null; - /** - * The maximum size in bytes for direct uploads. - */ - maxSizeBytes: number | null; - /** - * The maximum duration in seconds for direct uploads. - */ - maxDurationSeconds: number | null; - /** - * The video duration in seconds. -1 indicates unknown. - */ - duration: number; - /** - * Input metadata for the original upload. - */ - input: StreamVideoInput; - /** - * Playback URLs for the video. - */ - hlsPlaybackUrl: string; - dashPlaybackUrl: string; - /** - * The watermark applied to the video, if any. - */ - watermark: StreamWatermark | null; - /** - * The live input id associated with the video, if any. - */ - liveInputId?: string | null; - /** - * The source video id if this is a clip. - */ - clippedFromId: string | null; - /** - * Public details associated with the video. - */ - publicDetails: StreamPublicDetails | null; -} -type StreamVideoStatus = { - /** - * The current processing state. - */ - state: string; - /** - * The current processing step. - */ - step?: string; - /** - * The percent complete as a string. - */ - pctComplete?: string; - /** - * An error reason code, if applicable. - */ - errorReasonCode: string; - /** - * An error reason text, if applicable. - */ - errorReasonText: string; -}; -type StreamVideoInput = { - /** - * The input width in pixels. - */ - width: number; - /** - * The input height in pixels. - */ - height: number; -}; -type StreamPublicDetails = { - /** - * The public title for the video. - */ - title: string | null; - /** - * The public share link. - */ - share_link: string | null; - /** - * The public channel link. - */ - channel_link: string | null; - /** - * The public logo URL. - */ - logo: string | null; -}; -type StreamDirectUpload = { - /** - * The URL an unauthenticated upload can use for a single multipart request. - */ - uploadURL: string; - /** - * A Cloudflare-generated unique identifier for a media item. - */ - id: string; - /** - * The watermark profile applied to the upload. - */ - watermark: StreamWatermark | null; - /** - * The scheduled deletion time, if any. - */ - scheduledDeletion: string | null; -}; -type StreamDirectUploadCreateParams = { - /** - * The maximum duration in seconds for a video upload. - */ - maxDurationSeconds: number; - /** - * The date and time after upload when videos will not be accepted. - */ - expiry?: string; - /** - * A user-defined identifier for the media creator. - */ - creator?: string; - /** - * A user modifiable key-value store used to reference other systems of record for - * managing videos. - */ - meta?: Record; - /** - * Lists the origins allowed to display the video. - */ - allowedOrigins?: Array; - /** - * Indicates whether the video can be accessed using the id. When set to `true`, - * a signed token must be generated with a signing key to view the video. - */ - requireSignedURLs?: boolean; - /** - * The thumbnail timestamp percentage. - */ - thumbnailTimestampPct?: number; - /** - * The date and time at which the video will be deleted. Include `null` to remove - * a scheduled deletion. - */ - scheduledDeletion?: string | null; - /** - * The watermark profile to apply. - */ - watermark?: StreamDirectUploadWatermark; -}; -type StreamDirectUploadWatermark = { - /** - * The unique identifier for the watermark profile. - */ - id: string; -}; -type StreamUrlUploadParams = { - /** - * Lists the origins allowed to display the video. Enter allowed origin - * domains in an array and use `*` for wildcard subdomains. Empty arrays allow the - * video to be viewed on any origin. - */ - allowedOrigins?: Array; - /** - * A user-defined identifier for the media creator. - */ - creator?: string; - /** - * A user modifiable key-value store used to reference other systems of - * record for managing videos. - */ - meta?: Record; - /** - * Indicates whether the video can be a accessed using the id. When - * set to `true`, a signed token must be generated with a signing key to view the - * video. - */ - requireSignedURLs?: boolean; - /** - * Indicates the date and time at which the video will be deleted. Omit - * the field to indicate no change, or include with a `null` value to remove an - * existing scheduled deletion. If specified, must be at least 30 days from upload - * time. - */ - scheduledDeletion?: string | null; - /** - * The timestamp for a thumbnail image calculated as a percentage value - * of the video's duration. To convert from a second-wise timestamp to a - * percentage, divide the desired timestamp by the total duration of the video. If - * this value is not set, the default thumbnail image is taken from 0s of the - * video. - */ - thumbnailTimestampPct?: number; - /** - * The identifier for the watermark profile - */ - watermarkId?: string; -}; -interface StreamScopedCaptions { - /** - * Uploads the caption or subtitle file to the endpoint for a specific BCP47 language. - * One caption or subtitle file per language is allowed. - * @param language The BCP 47 language tag for the caption or subtitle. - * @param input The caption or subtitle stream to upload. - * @returns The created caption entry. - * @throws {NotFoundError} if the video is not found - * @throws {BadRequestError} if the language or file is invalid - * @throws {InternalError} if an unexpected error occurs - */ - upload(language: string, input: ReadableStream): Promise; - /** - * Generate captions or subtitles for the provided language via AI. - * @param language The BCP 47 language tag to generate. - * @returns The generated caption entry. - * @throws {NotFoundError} if the video is not found - * @throws {BadRequestError} if the language is invalid - * @throws {StreamError} if a generated caption already exists - * @throws {StreamError} if the video duration is too long - * @throws {StreamError} if the video is missing audio - * @throws {StreamError} if the requested language is not supported - * @throws {InternalError} if an unexpected error occurs - */ - generate(language: string): Promise; - /** - * Lists the captions or subtitles. - * Use the language parameter to filter by a specific language. - * @param language The optional BCP 47 language tag to filter by. - * @returns The list of captions or subtitles. - * @throws {NotFoundError} if the video or caption is not found - * @throws {InternalError} if an unexpected error occurs - */ - list(language?: string): Promise; - /** - * Removes the captions or subtitles from a video. - * @param language The BCP 47 language tag to remove. - * @returns A promise that resolves when deletion completes. - * @throws {NotFoundError} if the video or caption is not found - * @throws {InternalError} if an unexpected error occurs - */ - delete(language: string): Promise; -} -interface StreamScopedDownloads { - /** - * Generates a download for a video when a video is ready to view. Available - * types are `default` and `audio`. Defaults to `default` when omitted. - * @param downloadType The download type to create. - * @returns The current downloads for the video. - * @throws {NotFoundError} if the video is not found - * @throws {BadRequestError} if the download type is invalid - * @throws {StreamError} if the video duration is too long to generate a download - * @throws {StreamError} if the video is not ready to stream - * @throws {InternalError} if an unexpected error occurs - */ - generate(downloadType?: StreamDownloadType): Promise; - /** - * Lists the downloads created for a video. - * @returns The current downloads for the video. - * @throws {NotFoundError} if the video or downloads are not found - * @throws {InternalError} if an unexpected error occurs - */ - get(): Promise; - /** - * Delete the downloads for a video. Available types are `default` and `audio`. - * Defaults to `default` when omitted. - * @param downloadType The download type to delete. - * @returns A promise that resolves when deletion completes. - * @throws {NotFoundError} if the video or downloads are not found - * @throws {InternalError} if an unexpected error occurs - */ - delete(downloadType?: StreamDownloadType): Promise; -} -interface StreamVideos { - /** - * Lists all videos in a users account. - * @returns The list of videos. - * @throws {BadRequestError} if the parameters are invalid - * @throws {InternalError} if an unexpected error occurs - */ - list(params?: StreamVideosListParams): Promise; -} -interface StreamWatermarks { - /** - * Generate a new watermark profile - * @param input The image stream to upload - * @param params The watermark creation parameters. - * @returns The created watermark profile. - * @throws {BadRequestError} if the parameters are invalid - * @throws {InvalidURLError} if the URL is invalid - * @throws {TooManyWatermarksError} if the number of allowed watermarks is reached - * @throws {InternalError} if an unexpected error occurs - */ - generate(input: ReadableStream, params: StreamWatermarkCreateParams): Promise; - /** - * Generate a new watermark profile - * @param url The image url to upload - * @param params The watermark creation parameters. - * @returns The created watermark profile. - * @throws {BadRequestError} if the parameters are invalid - * @throws {InvalidURLError} if the URL is invalid - * @throws {TooManyWatermarksError} if the number of allowed watermarks is reached - * @throws {InternalError} if an unexpected error occurs - */ - generate(url: string, params: StreamWatermarkCreateParams): Promise; - /** - * Lists all watermark profiles for an account. - * @returns The list of watermark profiles. - * @throws {InternalError} if an unexpected error occurs - */ - list(): Promise; - /** - * Retrieves details for a single watermark profile. - * @param watermarkId The watermark profile identifier. - * @returns The watermark profile details. - * @throws {NotFoundError} if the watermark is not found - * @throws {InternalError} if an unexpected error occurs - */ - get(watermarkId: string): Promise; - /** - * Deletes a watermark profile. - * @param watermarkId The watermark profile identifier. - * @returns A promise that resolves when deletion completes. - * @throws {NotFoundError} if the watermark is not found - * @throws {InternalError} if an unexpected error occurs - */ - delete(watermarkId: string): Promise; -} -type StreamUpdateVideoParams = { - /** - * Lists the origins allowed to display the video. Enter allowed origin - * domains in an array and use `*` for wildcard subdomains. Empty arrays allow the - * video to be viewed on any origin. - */ - allowedOrigins?: Array; - /** - * A user-defined identifier for the media creator. - */ - creator?: string; - /** - * The maximum duration in seconds for a video upload. Can be set for a - * video that is not yet uploaded to limit its duration. Uploads that exceed the - * specified duration will fail during processing. A value of `-1` means the value - * is unknown. - */ - maxDurationSeconds?: number; - /** - * A user modifiable key-value store used to reference other systems of - * record for managing videos. - */ - meta?: Record; - /** - * Indicates whether the video can be a accessed using the id. When - * set to `true`, a signed token must be generated with a signing key to view the - * video. - */ - requireSignedURLs?: boolean; - /** - * Indicates the date and time at which the video will be deleted. Omit - * the field to indicate no change, or include with a `null` value to remove an - * existing scheduled deletion. If specified, must be at least 30 days from upload - * time. - */ - scheduledDeletion?: string | null; - /** - * The timestamp for a thumbnail image calculated as a percentage value - * of the video's duration. To convert from a second-wise timestamp to a - * percentage, divide the desired timestamp by the total duration of the video. If - * this value is not set, the default thumbnail image is taken from 0s of the - * video. - */ - thumbnailTimestampPct?: number; -}; -type StreamCaption = { - /** - * Whether the caption was generated via AI. - */ - generated?: boolean; - /** - * The language label displayed in the native language to users. - */ - label: string; - /** - * The language tag in BCP 47 format. - */ - language: string; - /** - * The status of a generated caption. - */ - status?: 'ready' | 'inprogress' | 'error'; -}; -type StreamDownloadStatus = 'ready' | 'inprogress' | 'error'; -type StreamDownloadType = 'default' | 'audio'; -type StreamDownload = { - /** - * Indicates the progress as a percentage between 0 and 100. - */ - percentComplete: number; - /** - * The status of a generated download. - */ - status: StreamDownloadStatus; - /** - * The URL to access the generated download. - */ - url?: string; -}; -/** - * An object with download type keys. Each key is optional and only present if that - * download type has been created. - */ -type StreamDownloadGetResponse = { - /** - * The audio-only download. Only present if this download type has been created. - */ - audio?: StreamDownload; - /** - * The default video download. Only present if this download type has been created. - */ - default?: StreamDownload; -}; -type StreamWatermarkPosition = 'upperRight' | 'upperLeft' | 'lowerLeft' | 'lowerRight' | 'center'; -type StreamWatermark = { - /** - * The unique identifier for a watermark profile. - */ - id: string; - /** - * The size of the image in bytes. - */ - size: number; - /** - * The height of the image in pixels. - */ - height: number; - /** - * The width of the image in pixels. - */ - width: number; - /** - * The date and a time a watermark profile was created. - */ - created: string; - /** - * The source URL for a downloaded image. If the watermark profile was created via - * direct upload, this field is null. - */ - downloadedFrom: string | null; - /** - * A short description of the watermark profile. - */ - name: string; - /** - * The translucency of the image. A value of `0.0` makes the image completely - * transparent, and `1.0` makes the image completely opaque. Note that if the image - * is already semi-transparent, setting this to `1.0` will not make the image - * completely opaque. - */ - opacity: number; - /** - * The whitespace between the adjacent edges (determined by position) of the video - * and the image. `0.0` indicates no padding, and `1.0` indicates a fully padded - * video width or length, as determined by the algorithm. - */ - padding: number; - /** - * The size of the image relative to the overall size of the video. This parameter - * will adapt to horizontal and vertical videos automatically. `0.0` indicates no - * scaling (use the size of the image as-is), and `1.0 `fills the entire video. - */ - scale: number; - /** - * The location of the image. Valid positions are: `upperRight`, `upperLeft`, - * `lowerLeft`, `lowerRight`, and `center`. Note that `center` ignores the - * `padding` parameter. - */ - position: StreamWatermarkPosition; -}; -type StreamWatermarkCreateParams = { - /** - * A short description of the watermark profile. - */ - name?: string; - /** - * The translucency of the image. A value of `0.0` makes the image completely - * transparent, and `1.0` makes the image completely opaque. Note that if the - * image is already semi-transparent, setting this to `1.0` will not make the - * image completely opaque. - */ - opacity?: number; - /** - * The whitespace between the adjacent edges (determined by position) of the - * video and the image. `0.0` indicates no padding, and `1.0` indicates a fully - * padded video width or length, as determined by the algorithm. - */ - padding?: number; - /** - * The size of the image relative to the overall size of the video. This - * parameter will adapt to horizontal and vertical videos automatically. `0.0` - * indicates no scaling (use the size of the image as-is), and `1.0 `fills the - * entire video. - */ - scale?: number; - /** - * The location of the image. - */ - position?: StreamWatermarkPosition; -}; -type StreamVideosListParams = { - /** - * The maximum number of videos to return. - */ - limit?: number; - /** - * Return videos created before this timestamp. - * (RFC3339/RFC3339Nano) - */ - before?: string; - /** - * Comparison operator for the `before` field. - * @default 'lt' - */ - beforeComp?: StreamPaginationComparison; - /** - * Return videos created after this timestamp. - * (RFC3339/RFC3339Nano) - */ - after?: string; - /** - * Comparison operator for the `after` field. - * @default 'gte' - */ - afterComp?: StreamPaginationComparison; -}; -type StreamPaginationComparison = 'eq' | 'gt' | 'gte' | 'lt' | 'lte'; -/** - * Error object for Stream binding operations. - */ -interface StreamError extends Error { - readonly code: number; - readonly statusCode: number; - readonly message: string; - readonly stack?: string; -} -interface InternalError extends StreamError { - name: 'InternalError'; -} -interface BadRequestError extends StreamError { - name: 'BadRequestError'; -} -interface NotFoundError extends StreamError { - name: 'NotFoundError'; -} -interface ForbiddenError extends StreamError { - name: 'ForbiddenError'; -} -interface RateLimitedError extends StreamError { - name: 'RateLimitedError'; -} -interface QuotaReachedError extends StreamError { - name: 'QuotaReachedError'; -} -interface MaxFileSizeError extends StreamError { - name: 'MaxFileSizeError'; -} -interface InvalidURLError extends StreamError { - name: 'InvalidURLError'; -} -interface AlreadyUploadedError extends StreamError { - name: 'AlreadyUploadedError'; -} -interface TooManyWatermarksError extends StreamError { - name: 'TooManyWatermarksError'; -} -type MarkdownDocument = { - name: string; - blob: Blob; -}; -type ConversionResponse = { - id: string; - name: string; - mimeType: string; - format: 'markdown'; - tokens: number; - data: string; -} | { - id: string; - name: string; - mimeType: string; - format: 'error'; - error: string; -}; -type ImageConversionOptions = { - descriptionLanguage?: 'en' | 'es' | 'fr' | 'it' | 'pt' | 'de'; -}; -type EmbeddedImageConversionOptions = ImageConversionOptions & { - convert?: boolean; - maxConvertedImages?: number; -}; -type ConversionOptions = { - html?: { - images?: EmbeddedImageConversionOptions & { - convertOGImage?: boolean; - }; - hostname?: string; - cssSelector?: string; - }; - docx?: { - images?: EmbeddedImageConversionOptions; - }; - image?: ImageConversionOptions; - pdf?: { - images?: EmbeddedImageConversionOptions; - metadata?: boolean; - }; -}; -type ConversionRequestOptions = { - gateway?: GatewayOptions; - extraHeaders?: object; - conversionOptions?: ConversionOptions; -}; -type SupportedFileFormat = { - mimeType: string; - extension: string; -}; -declare abstract class ToMarkdownService { - transform(files: MarkdownDocument[], options?: ConversionRequestOptions): Promise; - transform(files: MarkdownDocument, options?: ConversionRequestOptions): Promise; - supported(): Promise; -} -declare namespace TailStream { - interface Header { - readonly name: string; - readonly value: string; - } - interface FetchEventInfo { - readonly type: "fetch"; - readonly method: string; - readonly url: string; - readonly cfJson?: object; - readonly headers: Header[]; - } - interface JsRpcEventInfo { - readonly type: "jsrpc"; - } - interface ScheduledEventInfo { - readonly type: "scheduled"; - readonly scheduledTime: Date; - readonly cron: string; - } - interface AlarmEventInfo { - readonly type: "alarm"; - readonly scheduledTime: Date; - } - interface QueueEventInfo { - readonly type: "queue"; - readonly queueName: string; - readonly batchSize: number; - } - interface EmailEventInfo { - readonly type: "email"; - readonly mailFrom: string; - readonly rcptTo: string; - readonly rawSize: number; - } - interface TraceEventInfo { - readonly type: "trace"; - readonly traces: (string | null)[]; - } - interface HibernatableWebSocketEventInfoMessage { - readonly type: "message"; - } - interface HibernatableWebSocketEventInfoError { - readonly type: "error"; - } - interface HibernatableWebSocketEventInfoClose { - readonly type: "close"; - readonly code: number; - readonly wasClean: boolean; - } - interface HibernatableWebSocketEventInfo { - readonly type: "hibernatableWebSocket"; - readonly info: HibernatableWebSocketEventInfoClose | HibernatableWebSocketEventInfoError | HibernatableWebSocketEventInfoMessage; - } - interface CustomEventInfo { - readonly type: "custom"; - } - interface FetchResponseInfo { - readonly type: "fetch"; - readonly statusCode: number; - } - interface ConnectEventInfo { - readonly type: "connect"; - } - type EventOutcome = "ok" | "canceled" | "exception" | "unknown" | "killSwitch" | "daemonDown" | "exceededCpu" | "exceededMemory" | "loadShed" | "responseStreamDisconnected" | "scriptNotFound"; - interface ScriptVersion { - readonly id: string; - readonly tag?: string; - readonly message?: string; - } - interface TracePreviewInfo { - readonly id: string; - readonly slug: string; - readonly name: string; - } - interface Onset { - readonly type: "onset"; - readonly attributes: Attribute[]; - // id for the span being opened by this Onset event. - readonly spanId: string; - readonly dispatchNamespace?: string; - readonly entrypoint?: string; - readonly executionModel: string; - readonly scriptName?: string; - readonly scriptTags?: string[]; - readonly scriptVersion?: ScriptVersion; - readonly preview?: TracePreviewInfo; - readonly info: FetchEventInfo | ConnectEventInfo | JsRpcEventInfo | ScheduledEventInfo | AlarmEventInfo | QueueEventInfo | EmailEventInfo | TraceEventInfo | HibernatableWebSocketEventInfo | CustomEventInfo; - } - interface Outcome { - readonly type: "outcome"; - readonly outcome: EventOutcome; - readonly cpuTime: number; - readonly wallTime: number; - } - interface SpanOpen { - readonly type: "spanOpen"; - readonly name: string; - // id for the span being opened by this SpanOpen event. - readonly spanId: string; - readonly info?: FetchEventInfo | JsRpcEventInfo | Attributes; - } - interface SpanClose { - readonly type: "spanClose"; - readonly outcome: EventOutcome; - } - interface DiagnosticChannelEvent { - readonly type: "diagnosticChannel"; - readonly channel: string; - readonly message: any; - } - interface Exception { - readonly type: "exception"; - readonly name: string; - readonly message: string; - readonly stack?: string; - } - interface Log { - readonly type: "log"; - readonly level: "debug" | "error" | "info" | "log" | "warn"; - readonly message: object; - } - interface DroppedEventsDiagnostic { - readonly diagnosticsType: "droppedEvents"; - readonly count: number; - } - interface StreamDiagnostic { - readonly type: 'streamDiagnostic'; - // To add new diagnostic types, define a new interface and add it to this union type. - readonly diagnostic: DroppedEventsDiagnostic; - } - // This marks the worker handler return information. - // This is separate from Outcome because the worker invocation can live for a long time after - // returning. For example - Websockets that return an http upgrade response but then continue - // streaming information or SSE http connections. - interface Return { - readonly type: "return"; - readonly info?: FetchResponseInfo; - } - interface Attribute { - readonly name: string; - readonly value: string | string[] | boolean | boolean[] | number | number[] | bigint | bigint[]; - } - interface Attributes { - readonly type: "attributes"; - readonly info: Attribute[]; - } - type EventType = Onset | Outcome | SpanOpen | SpanClose | DiagnosticChannelEvent | Exception | Log | StreamDiagnostic | Return | Attributes; - // Context in which this trace event lives. - interface SpanContext { - // Single id for the entire top-level invocation - // This should be a new traceId for the first worker stage invoked in the eyeball request and then - // same-account service-bindings should reuse the same traceId but cross-account service-bindings - // should use a new traceId. - readonly traceId: string; - // spanId in which this event is handled - // for Onset and SpanOpen events this would be the parent span id - // for Outcome and SpanClose these this would be the span id of the opening Onset and SpanOpen events - // For Hibernate and Mark this would be the span under which they were emitted. - // spanId is not set ONLY if: - // 1. This is an Onset event - // 2. We are not inheriting any SpanContext. (e.g. this is a cross-account service binding or a new top-level invocation) - readonly spanId?: string; - } - interface TailEvent { - // invocation id of the currently invoked worker stage. - // invocation id will always be unique to every Onset event and will be the same until the Outcome event. - readonly invocationId: string; - // Inherited spanContext for this event. - readonly spanContext: SpanContext; - readonly timestamp: Date; - readonly sequence: number; - readonly event: Event; - } - type TailEventHandler = (event: TailEvent) => void | Promise; - type TailEventHandlerObject = { - outcome?: TailEventHandler; - spanOpen?: TailEventHandler; - spanClose?: TailEventHandler; - diagnosticChannel?: TailEventHandler; - exception?: TailEventHandler; - log?: TailEventHandler; - return?: TailEventHandler; - attributes?: TailEventHandler; - }; - type TailEventHandlerType = TailEventHandler | TailEventHandlerObject; -} -// Copyright (c) 2022-2023 Cloudflare, Inc. -// Licensed under the Apache 2.0 license found in the LICENSE file or at: -// https://opensource.org/licenses/Apache-2.0 -/** - * Data types supported for holding vector metadata. - */ -type VectorizeVectorMetadataValue = string | number | boolean | string[]; -/** - * Additional information to associate with a vector. - */ -type VectorizeVectorMetadata = VectorizeVectorMetadataValue | Record; -type VectorFloatArray = Float32Array | Float64Array; -interface VectorizeError { - code?: number; - error: string; -} -/** - * Comparison logic/operation to use for metadata filtering. - * - * This list is expected to grow as support for more operations are released. - */ -type VectorizeVectorMetadataFilterOp = '$eq' | '$ne' | '$lt' | '$lte' | '$gt' | '$gte'; -type VectorizeVectorMetadataFilterCollectionOp = '$in' | '$nin'; -/** - * Filter criteria for vector metadata used to limit the retrieved query result set. - */ -type VectorizeVectorMetadataFilter = { - [field: string]: Exclude | null | { - [Op in VectorizeVectorMetadataFilterOp]?: Exclude | null; - } | { - [Op in VectorizeVectorMetadataFilterCollectionOp]?: Exclude[]; - }; -}; -/** - * Supported distance metrics for an index. - * Distance metrics determine how other "similar" vectors are determined. - */ -type VectorizeDistanceMetric = "euclidean" | "cosine" | "dot-product"; -/** - * Metadata return levels for a Vectorize query. - * - * Default to "none". - * - * @property all Full metadata for the vector return set, including all fields (including those un-indexed) without truncation. This is a more expensive retrieval, as it requires additional fetching & reading of un-indexed data. - * @property indexed Return all metadata fields configured for indexing in the vector return set. This level of retrieval is "free" in that no additional overhead is incurred returning this data. However, note that indexed metadata is subject to truncation (especially for larger strings). - * @property none No indexed metadata will be returned. - */ -type VectorizeMetadataRetrievalLevel = "all" | "indexed" | "none"; -interface VectorizeQueryOptions { - topK?: number; - namespace?: string; - returnValues?: boolean; - returnMetadata?: boolean | VectorizeMetadataRetrievalLevel; - filter?: VectorizeVectorMetadataFilter; -} -/** - * Information about the configuration of an index. - */ -type VectorizeIndexConfig = { - dimensions: number; - metric: VectorizeDistanceMetric; -} | { - preset: string; // keep this generic, as we'll be adding more presets in the future and this is only in a read capacity -}; -/** - * Metadata about an existing index. - * - * This type is exclusively for the Vectorize **beta** and will be deprecated once Vectorize RC is released. - * See {@link VectorizeIndexInfo} for its post-beta equivalent. - */ -interface VectorizeIndexDetails { - /** The unique ID of the index */ - readonly id: string; - /** The name of the index. */ - name: string; - /** (optional) A human readable description for the index. */ - description?: string; - /** The index configuration, including the dimension size and distance metric. */ - config: VectorizeIndexConfig; - /** The number of records containing vectors within the index. */ - vectorsCount: number; -} -/** - * Metadata about an existing index. - */ -interface VectorizeIndexInfo { - /** The number of records containing vectors within the index. */ - vectorCount: number; - /** Number of dimensions the index has been configured for. */ - dimensions: number; - /** ISO 8601 datetime of the last processed mutation on in the index. All changes before this mutation will be reflected in the index state. */ - processedUpToDatetime: number; - /** UUIDv4 of the last mutation processed by the index. All changes before this mutation will be reflected in the index state. */ - processedUpToMutation: number; -} -/** - * Represents a single vector value set along with its associated metadata. - */ -interface VectorizeVector { - /** The ID for the vector. This can be user-defined, and must be unique. It should uniquely identify the object, and is best set based on the ID of what the vector represents. */ - id: string; - /** The vector values */ - values: VectorFloatArray | number[]; - /** The namespace this vector belongs to. */ - namespace?: string; - /** Metadata associated with the vector. Includes the values of other fields and potentially additional details. */ - metadata?: Record; -} -/** - * Represents a matched vector for a query along with its score and (if specified) the matching vector information. - */ -type VectorizeMatch = Pick, "values"> & Omit & { - /** The score or rank for similarity, when returned as a result */ - score: number; -}; -/** - * A set of matching {@link VectorizeMatch} for a particular query. - */ -interface VectorizeMatches { - matches: VectorizeMatch[]; - count: number; -} -/** - * Results of an operation that performed a mutation on a set of vectors. - * Here, `ids` is a list of vectors that were successfully processed. - * - * This type is exclusively for the Vectorize **beta** and will be deprecated once Vectorize RC is released. - * See {@link VectorizeAsyncMutation} for its post-beta equivalent. - */ -interface VectorizeVectorMutation { - /* List of ids of vectors that were successfully processed. */ - ids: string[]; - /* Total count of the number of processed vectors. */ - count: number; -} -/** - * Result type indicating a mutation on the Vectorize Index. - * Actual mutations are processed async where the `mutationId` is the unique identifier for the operation. - */ -interface VectorizeAsyncMutation { - /** The unique identifier for the async mutation operation containing the changeset. */ - mutationId: string; -} -/** - * A Vectorize Vector Search Index for querying vectors/embeddings. - * - * This type is exclusively for the Vectorize **beta** and will be deprecated once Vectorize RC is released. - * See {@link Vectorize} for its new implementation. - */ -declare abstract class VectorizeIndex { - /** - * Get information about the currently bound index. - * @returns A promise that resolves with information about the current index. - */ - public describe(): Promise; - /** - * Use the provided vector to perform a similarity search across the index. - * @param vector Input vector that will be used to drive the similarity search. - * @param options Configuration options to massage the returned data. - * @returns A promise that resolves with matched and scored vectors. - */ - public query(vector: VectorFloatArray | number[], options?: VectorizeQueryOptions): Promise; - /** - * Insert a list of vectors into the index dataset. If a provided id exists, an error will be thrown. - * @param vectors List of vectors that will be inserted. - * @returns A promise that resolves with the ids & count of records that were successfully processed. - */ - public insert(vectors: VectorizeVector[]): Promise; - /** - * Upsert a list of vectors into the index dataset. If a provided id exists, it will be replaced with the new values. - * @param vectors List of vectors that will be upserted. - * @returns A promise that resolves with the ids & count of records that were successfully processed. - */ - public upsert(vectors: VectorizeVector[]): Promise; - /** - * Delete a list of vectors with a matching id. - * @param ids List of vector ids that should be deleted. - * @returns A promise that resolves with the ids & count of records that were successfully processed (and thus deleted). - */ - public deleteByIds(ids: string[]): Promise; - /** - * Get a list of vectors with a matching id. - * @param ids List of vector ids that should be returned. - * @returns A promise that resolves with the raw unscored vectors matching the id set. - */ - public getByIds(ids: string[]): Promise; -} -/** - * A Vectorize Vector Search Index for querying vectors/embeddings. - * - * Mutations in this version are async, returning a mutation id. - */ -declare abstract class Vectorize { - /** - * Get information about the currently bound index. - * @returns A promise that resolves with information about the current index. - */ - public describe(): Promise; - /** - * Use the provided vector to perform a similarity search across the index. - * @param vector Input vector that will be used to drive the similarity search. - * @param options Configuration options to massage the returned data. - * @returns A promise that resolves with matched and scored vectors. - */ - public query(vector: VectorFloatArray | number[], options?: VectorizeQueryOptions): Promise; - /** - * Use the provided vector-id to perform a similarity search across the index. - * @param vectorId Id for a vector in the index against which the index should be queried. - * @param options Configuration options to massage the returned data. - * @returns A promise that resolves with matched and scored vectors. - */ - public queryById(vectorId: string, options?: VectorizeQueryOptions): Promise; - /** - * Insert a list of vectors into the index dataset. If a provided id exists, an error will be thrown. - * @param vectors List of vectors that will be inserted. - * @returns A promise that resolves with a unique identifier of a mutation containing the insert changeset. - */ - public insert(vectors: VectorizeVector[]): Promise; - /** - * Upsert a list of vectors into the index dataset. If a provided id exists, it will be replaced with the new values. - * @param vectors List of vectors that will be upserted. - * @returns A promise that resolves with a unique identifier of a mutation containing the upsert changeset. - */ - public upsert(vectors: VectorizeVector[]): Promise; - /** - * Delete a list of vectors with a matching id. - * @param ids List of vector ids that should be deleted. - * @returns A promise that resolves with a unique identifier of a mutation containing the delete changeset. - */ - public deleteByIds(ids: string[]): Promise; - /** - * Get a list of vectors with a matching id. - * @param ids List of vector ids that should be returned. - * @returns A promise that resolves with the raw unscored vectors matching the id set. - */ - public getByIds(ids: string[]): Promise; -} -/** - * The interface for "version_metadata" binding - * providing metadata about the Worker Version using this binding. - */ -type WorkerVersionMetadata = { - /** The ID of the Worker Version using this binding */ - id: string; - /** The tag of the Worker Version using this binding */ - tag: string; - /** The timestamp of when the Worker Version was uploaded */ - timestamp: string; -}; -interface DynamicDispatchLimits { - /** - * Limit CPU time in milliseconds. - */ - cpuMs?: number; - /** - * Limit number of subrequests. - */ - subRequests?: number; -} -interface DynamicDispatchOptions { - /** - * Limit resources of invoked Worker script. - */ - limits?: DynamicDispatchLimits; - /** - * Arguments for outbound Worker script, if configured. - */ - outbound?: { - [key: string]: any; - }; -} -interface DispatchNamespace { - /** - * @param name Name of the Worker script. - * @param args Arguments to Worker script. - * @param options Options for Dynamic Dispatch invocation. - * @returns A Fetcher object that allows you to send requests to the Worker script. - * @throws If the Worker script does not exist in this dispatch namespace, an error will be thrown. - */ - get(name: string, args?: { - [key: string]: any; - }, options?: DynamicDispatchOptions): Fetcher; -} -declare module 'cloudflare:workflows' { - /** - * NonRetryableError allows for a user to throw a fatal error - * that makes a Workflow instance fail immediately without triggering a retry - */ - export class NonRetryableError extends Error { - public constructor(message: string, name?: string); - } -} -declare abstract class Workflow { - /** - * Get a handle to an existing instance of the Workflow. - * @param id Id for the instance of this Workflow - * @returns A promise that resolves with a handle for the Instance - */ - public get(id: string): Promise; - /** - * Create a new instance and return a handle to it. If a provided id exists, an error will be thrown. - * @param options Options when creating an instance including id and params - * @returns A promise that resolves with a handle for the Instance - */ - public create(options?: WorkflowInstanceCreateOptions): Promise; - /** - * Create a batch of instances and return handle for all of them. If a provided id exists, an error will be thrown. - * `createBatch` is limited at 100 instances at a time or when the RPC limit for the batch (1MiB) is reached. - * @param batch List of Options when creating an instance including name and params - * @returns A promise that resolves with a list of handles for the created instances. - */ - public createBatch(batch: WorkflowInstanceCreateOptions[]): Promise; -} -type WorkflowDurationLabel = 'second' | 'minute' | 'hour' | 'day' | 'week' | 'month' | 'year'; -type WorkflowSleepDuration = `${number} ${WorkflowDurationLabel}${'s' | ''}` | number; -type WorkflowRetentionDuration = WorkflowSleepDuration; -interface WorkflowInstanceCreateOptions { - /** - * An id for your Workflow instance. Must be unique within the Workflow. - */ - id?: string; - /** - * The event payload the Workflow instance is triggered with - */ - params?: PARAMS; - /** - * The retention policy for Workflow instance. - * Defaults to the maximum retention period available for the owner's account. - */ - retention?: { - successRetention?: WorkflowRetentionDuration; - errorRetention?: WorkflowRetentionDuration; - }; -} -type InstanceStatus = { - status: 'queued' // means that instance is waiting to be started (see concurrency limits) - | 'running' | 'paused' | 'errored' | 'terminated' // user terminated the instance while it was running - | 'complete' | 'waiting' // instance is hibernating and waiting for sleep or event to finish - | 'waitingForPause' // instance is finishing the current work to pause - | 'unknown'; - error?: { - name: string; - message: string; - }; - output?: unknown; -}; -interface WorkflowError { - code?: number; - message: string; -} -declare abstract class WorkflowInstance { - public id: string; - /** - * Pause the instance. - */ - public pause(): Promise; - /** - * Resume the instance. If it is already running, an error will be thrown. - */ - public resume(): Promise; - /** - * Terminate the instance. If it is errored, terminated or complete, an error will be thrown. - */ - public terminate(): Promise; - /** - * Restart the instance. - */ - public restart(): Promise; - /** - * Returns the current status of the instance. - */ - public status(): Promise; - /** - * Send an event to this instance. - */ - public sendEvent({ type, payload, }: { - type: string; - payload: unknown; - }): Promise; -} diff --git a/workers/cache/wrangler.jsonc b/workers/cache/wrangler.jsonc deleted file mode 100644 index 0fa8c8c..0000000 --- a/workers/cache/wrangler.jsonc +++ /dev/null @@ -1,31 +0,0 @@ -{ - "$schema": "node_modules/wrangler/config-schema.json", - "name": "helios-cache", - "main": "src/index.ts", - "compatibility_date": "2026-03-10", - "observability": { - "enabled": true - }, - "upload_source_maps": true, - "workers_dev": true, - "vars": { - "SIGNING_KEY_NAME": "helios-cache-1" - }, - "r2_buckets": [ - { - "binding": "CACHE_BUCKET", - "bucket_name": "helios-cache" - } - ], - "d1_databases": [ - { - "binding": "CACHE_DB", - "database_name": "helios-cache", - "database_id": "", - "migrations_dir": "migrations" - } - ], - "triggers": { - "crons": ["0 */6 * * *"] - } -} diff --git a/wrangler.jsonc b/wrangler.jsonc deleted file mode 100644 index 1bdd92f..0000000 --- a/wrangler.jsonc +++ /dev/null @@ -1,31 +0,0 @@ -{ - "$schema": "node_modules/wrangler/config-schema.json", - "name": "helios-cache", - "main": "workers/cache/src/index.ts", - "compatibility_date": "2026-03-10", - "observability": { - "enabled": true - }, - "upload_source_maps": true, - "workers_dev": true, - "vars": { - "SIGNING_KEY_NAME": "helios-cache-1" - }, - "r2_buckets": [ - { - "binding": "CACHE_BUCKET", - "bucket_name": "helios-cache" - } - ], - "d1_databases": [ - { - "binding": "CACHE_DB", - "database_name": "helios-cache", - "database_id": "", - "migrations_dir": "workers/cache/migrations" - } - ], - "triggers": { - "crons": ["0 */6 * * *"] - } -} From 439069e65f2ecb54ba1924009efed838d6c6f5a6 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 12:23:54 +0200 Subject: [PATCH 07/56] docs: describe the self-hosted deployment Rewrite the README, the reference documentation and the agent instructions for the Rust and Zig stack: the layout, server configuration, private caches, the push API, nginx with X-Accel-Redirect, and how to run the tests and benchmarks. --- AGENTS.md | 46 +++++++++------ README.md | 100 +++++++++++++------------------ docs.md | 174 ++++++++++++++++++++++++++++++++---------------------- 3 files changed, 175 insertions(+), 145 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c3995a3..8d265cc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,31 +1,43 @@ # Helios -This repository is a `pnpm` workspace for Cloudflare-native services. +A self-hosted Nix binary cache: Rust server and CLI over a Zig core. ## Workspace Layout -- `workers/*`: deployable Cloudflare Workers -- `apps/*`: operator or end-user applications that sit beside Workers -- `packages/*`: shared code used by workers or apps +- `pkg/*`: Zig packages for Nix formats and the NAR pipeline, one concern + each. Each builds and tests on its own (`zig build test`). +- `core/`: libhelios, the C ABI over `pkg/*`, compiled to a static library + by `crates/helios-core/build.rs`. `core/include/helios.h` is the contract. +- `crates/helios-core`: safe Rust bindings to libhelios. +- `crates/helios-server`, `crates/helios-cli`: the binaries. +- `bench/`: comparisons against Cachix's libraries and Nix C++. Not part of + the Cargo workspace. -Keep deployable runtime code inside `workers/`. Move shared protocol, parsing, and storage helpers into `packages/` only after they are reused. +Put Nix format logic in a `pkg/*` package and expose it through +`core/src/root.zig` only when Rust needs it. Keep calls across the C ABI +coarse, such as a whole NAR or narinfo per call. ## Tooling -- Use `pnpm`, not `npm`, for dependency management and scripts. -- Do not commit `package-lock.json` files. -- If `pnpm` or `wrangler` are missing locally, use `nix shell nixpkgs#nodejs nixpkgs#pnpm nixpkgs#wrangler`. -- Run workspace commands from the repository root unless a package-specific command is clearer. +- Use `nix develop` for zig 0.16, cargo, zstd and pkg-config. +- Zig 0.16 APIs changed a lot; check `zig env` for the std source rather + than relying on older examples. +- `HELIOS_ZIG_CPU` sets libhelios's `-Dcpu` (default `native`; the Nix + package uses `baseline`). -## Cloudflare Rules +## Performance Rules -- Before changing Workers, R2, D1, Durable Objects, Queues, or platform limits, fetch current Cloudflare docs. -- Prefer platform-native APIs such as `fetch`, Web Streams, and Web Crypto. -- Do not enable `nodejs_compat` unless a concrete dependency requires it. -- After changing a worker's bindings in `wrangler.jsonc`, run that package's `cf-typegen` script. +- Measure before and after; `bench/run.sh` and `bench/flake-lock.sh` + compare against the reference implementations on the same corpus. +- Keep the narinfo read path free of SQLite on misses and of per-request + signing. +- Keep a scalar reference for every SIMD routine, with a test asserting the + two agree. ## Verification -- Install dependencies with `pnpm install` from the repo root. -- Run `pnpm check` before finishing substantial changes. -- For worker-only iteration, use `pnpm --filter dev`. +- `cargo test --release` +- `./scripts/test-zig.sh` +- `./scripts/e2e.sh` before finishing changes to the server, the CLI or wire + formats; it substitutes a real closure through Nix with signatures + required. diff --git a/README.md b/README.md index 46d98b1..0d62a89 100644 --- a/README.md +++ b/README.md @@ -1,88 +1,70 @@ # Helios -A Cloudflare-native Nix binary cache. +A self-hosted Nix binary cache for a single server: a Rust HTTP server and +CLI on top of a Zig core library for NAR serialisation, hashing, +compression, narinfo and signing. -[![Deploy to Cloudflare](https://deploy.workers.cloudflare.com/button)](https://deploy.workers.cloudflare.com/?url=https://github.com/Polymath-AS/helios) -## Workspace +## Layout ``` -workers/cache/ Cloudflare Worker (main service) -packages/cache-domain/ Shared types and input parsers -apps/cli/ CLI for pushing store paths -scripts/ smoke-test.sh -``` - -## Setup - -```bash -nix develop # drops you into a shell with node, pnpm, wrangler -pnpm install +pkg/ Zig packages: nix-base32, nix-store-path, nix-archive, + nix-narinfo, nix-derivation, nix-flake-lock +core/ libhelios: C ABI over pkg/*, built as a static library +crates/ helios-core (Rust bindings), helios-server, helios-cli +bench/ benchmarks against Cachix's Nix libraries and Nix C++ +scripts/ e2e.sh (end-to-end test), test-zig.sh (Zig unit tests) ``` ## Develop ```bash -pnpm dev # runs the worker locally via wrangler -pnpm check # type-check + tests across the workspace -``` - -## Deploy - -See [docs.md](docs.md) for the full reference. Short version: - -```bash -wrangler r2 bucket create helios-cache -wrangler d1 create helios-cache -# update database_id in workers/cache/wrangler.jsonc -wrangler d1 migrations apply helios-cache --remote -wrangler secret put JWT_SECRET -wrangler secret put ADMIN_SECRET -wrangler secret put SIGNING_PRIVATE_KEY -wrangler secret put SIGNING_KEY_NAME -pnpm deploy +nix develop # zig, rust, zstd, pkg-config +cargo build --release # builds libhelios via zig automatically +cargo test --release +./scripts/test-zig.sh # unit tests for every pkg/* +./scripts/e2e.sh # server + CLI + real `nix copy` substitution ``` -## Push store paths +`nix build` produces `helios` (CLI) and `helios-server`. -Build the CLI (or `nix run .` for one-off use): +## Run the server ```bash -nix build # produces ./result/bin/helios +nix key generate-secret --key-name cache.example.com-1 > /var/lib/helios/signing.key +openssl rand -hex 32 > /var/lib/helios/jwt.secret +openssl rand -hex 32 > /var/lib/helios/admin.secret + +helios-server \ + --listen 127.0.0.1:8080 \ + --data-dir /var/lib/helios \ + --signing-key-file /var/lib/helios/signing.key \ + --jwt-secret-file /var/lib/helios/jwt.secret \ + --admin-secret-file /var/lib/helios/admin.secret + +helios-server --print-public-key --signing-key-file /var/lib/helios/signing.key ``` -Save credentials once: +## Push and substitute ```bash -helios login prod https://your-worker.workers.dev "$PUSH_TOKEN" -``` - -Push a single path: +helios login admin https://cache.example.com "$(cat admin.secret)" +helios cache create main +helios token create ci --caches main --perms push,pull # prints the token once -```bash -helios push main /nix/store/abc...-hello -``` - -Push a full closure: - -```bash +helios login ci https://cache.example.com "$TOKEN" helios push main --closure /run/current-system ``` -Push a flake output closure: - -```bash -helios push main --closure .#nixosConfigurations.myhost.config.system.build.toplevel -``` - -See [docs.md](docs.md) for token management and parallelism options. - -## Use as a substituter - ```nix -# configuration.nix or flake -nix.settings.substituters = [ "https://your-worker.workers.dev/main" ]; +nix.settings = { + substituters = [ "https://cache.example.com/main" ]; + trusted-public-keys = [ "cache.example.com-1:..." ]; # from --print-public-key +}; ``` +See [docs.md](docs.md) for configuration, private caches, the HTTP API +and deployment behind a reverse proxy. + ## License Source-available. See [LICENSE](LICENSE). diff --git a/docs.md b/docs.md index 0c1fbb5..82f2183 100644 --- a/docs.md +++ b/docs.md @@ -1,97 +1,133 @@ # Helios Documentation -## Deployment +## Storage + +Everything lives under `--data-dir`: + +| Path | Contents | +|------|----------| +| `helios.db` | SQLite (WAL): caches, blobs, published paths, tokens, audit log | +| `nar//.nar.zst` | Content-addressed compressed NARs | +| `tmp/` | In-flight uploads, renamed into `nar/` once verified | + +Uploads are decompressed and hashed by the server as they stream in, so a +path is only published against a NAR whose hash the server computed itself. +Narinfo is rendered and signed once, at publish time. + +## Server configuration + +Every flag can also be set through its environment variable. + +| Flag | Env | Default | +|------|-----|---------| +| `--listen` | `HELIOS_LISTEN` | `127.0.0.1:8080` | +| `--data-dir` | `HELIOS_DATA_DIR` | `/var/lib/helios` | +| `--signing-key-file` | `HELIOS_SIGNING_KEY_FILE` | unset: narinfo is unsigned | +| `--jwt-secret-file` | `HELIOS_JWT_SECRET_FILE` | unset: no API tokens | +| `--admin-secret-file` | `HELIOS_ADMIN_SECRET_FILE` | unset: no admin API | +| `--accel-redirect` | `HELIOS_ACCEL_REDIRECT` | unset: server streams NARs itself | +| `--trust-proxy` | `HELIOS_TRUST_PROXY` | `false` (audit IPs from `X-Forwarded-For`) | +| `--narinfo-cache-entries` | `HELIOS_NARINFO_CACHE_ENTRIES` | `262144` | +| `--max-upload-bytes` | `HELIOS_MAX_UPLOAD_BYTES` | 64 GiB | +| `--gc-interval-hours` | `HELIOS_GC_INTERVAL_HOURS` | `6` | +| `--audit-retention-days` | `HELIOS_AUDIT_RETENTION_DAYS` | `30` | + +Secret files must hold at least 16 bytes. Logging follows `RUST_LOG` +(default `helios_server=info`). + +## Reverse proxy and zero-copy downloads + +Terminate TLS in a reverse proxy. With `--accel-redirect /_nar`, the server +answers NAR requests with an `X-Accel-Redirect` header after the access +check, and nginx serves the file with `sendfile`: + +```nginx +location / { + proxy_pass http://127.0.0.1:8080; + proxy_request_buffering off; # stream uploads + client_max_body_size 0; +} +location /_nar/ { + internal; + alias /var/lib/helios/nar/; +} +``` + +## Caches and tokens ```bash -wrangler r2 bucket create helios-cache -wrangler d1 create helios-cache -# set database_id in workers/cache/wrangler.jsonc -wrangler d1 migrations apply helios-cache --remote -wrangler secret put JWT_SECRET -wrangler secret put ADMIN_SECRET -wrangler secret put SIGNING_PRIVATE_KEY -wrangler secret put SIGNING_KEY_NAME -pnpm deploy -``` +helios cache create main # public: anyone can read +helios cache create team --private # reads need a token with `pull` +helios cache list -| Secret | Purpose | -|--------|---------| -| `JWT_SECRET` | HMAC-SHA256 key for JWT token signing | -| `ADMIN_SECRET` | Bearer token for the admin API | -| `SIGNING_PRIVATE_KEY` | Ed25519 key for narinfo signing | -| `SIGNING_KEY_NAME` | Key name prefix for signatures | +helios token create ci --caches main --perms push --expires 90 +helios token create reader --caches team --perms pull +helios token list +helios token revoke "reason" # takes effect immediately +``` -## Setup +Admin commands use a server logged in with the admin secret; pass +`--server ` to pick one. Tokens are HS256 JWTs scoped to cache names +(or `*`) and to `push` and/or `pull`. -Configure the CLI with your server URL and admin secret: +For private caches, give Nix the token through netrc. Nix sends it as HTTP +Basic auth, and the server reads the password: -```bash -helios login prod https://your-worker.workers.dev $ADMIN_SECRET ``` - -## Using as a substituter +machine cache.example.com +password +``` ```nix -nix.settings.substituters = [ "https://your-worker.workers.dev/main" ]; +nix.settings.netrc-file = "/etc/nix/netrc"; ``` -## Managing tokens - -JWT tokens scope write access to specific caches. The signed JWT is -returned once on creation and never stored. Save it immediately. +## Pushing ```bash -# Create a token scoped to the "main" cache with push access -helios token create ci-runner --caches main --perms push --expires 90 - -# Create a token with access to all caches -helios token create admin-bot --caches "*" --perms push,pull - -# List all tokens -helios token list - -# Revoke a token -helios token revoke $JTI "employee offboarded" +helios push main /nix/store/...-hello # single paths +helios push main --closure .#nixosConfigurations.host.config.system.build.toplevel +helios push main --closure /run/current-system --jobs 8 --level 3 ``` -| Option | Default | Description | -|--------|---------|-------------| -| `--caches` | `*` | Comma-separated cache names or `*` for all | -| `--perms` | `push` | Comma-separated: `push` and/or `pull` | -| `--expires` | `90` | Token lifetime in days (1-365) | +The CLI serialises, zstd-compresses and hashes each NAR in one pass and +streams it straight into the upload, without temp files. It checks every +NAR hash against the Nix database. NARs the server already holds under any +cache are reused, and paths are published dependencies-first, in batches. -## Pushing store paths +## HTTP API -Log in with a push token (not the admin secret): - -```bash -helios login prod https://your-worker.workers.dev $PUSH_TOKEN -``` +Substituter endpoints (`GET`/`HEAD`): -Then push: +| Path | | +|------|--| +| `//nix-cache-info` | | +| `//.narinfo` | misses are answered from memory | +| `//nar/.nar.zst` | only served by caches that publish it | -```bash -# Single path -helios push main /nix/store/abc...-hello +Push endpoints (bearer token with `push` on the cache): -# Full closure -helios push main --closure /run/current-system +| Method and path | Body | | +|-----------------|------|--| +| `POST /_api/v2/caches//missing` | `{"hashes": [...]}` | store path hashes absent from the cache | +| `POST /_api/v2/caches//nars/known` | `{"narHashes": [...]}` | NAR hashes the server already has | +| `PUT /_api/v2/caches//nar?compression=zstd` | compressed NAR | returns the verified hashes and sizes | +| `POST /_api/v2/caches//paths` | `{"paths": [...]}` | publishes a batch in one transaction; `409 nar_required` lists paths without an uploaded NAR | -# Flake output closure -helios push main --closure .#nixosConfigurations.myhost.config.system.build.toplevel -``` +Admin endpoints (bearer admin secret): `GET/POST /_api/v2/admin/caches`, +`GET/POST /_api/v2/admin/tokens`, `POST /_api/v2/admin/tokens//revoke`. -Use `--jobs ` to control parallelism (default 8). +## Garbage collection -## Migrating from AUTH_TOKEN +Every `--gc-interval-hours`, the server removes stale temp uploads, blobs no +path references (after a one-hour grace period, so an upload is never +collected before it is published), expired tokens, and old audit log rows. -1. Set `JWT_SECRET` and `ADMIN_SECRET` via `wrangler secret put` -2. Log in with the admin secret: `helios login prod $URL $ADMIN_SECRET` -3. Create scoped tokens: `helios token create ci-runner --caches main` -4. Update publishers to log in with their JWT -5. Remove `AUTH_TOKEN` to disable legacy auth +## Benchmarks -## Audit logging +```bash +bench/run.sh # pkg/* vs Cachix's Haskell libraries and Nix C++ +bench/flake-lock.sh # pkg/nix-flake-lock vs cachix/nix-flake-lock +``` -All write operations and admin actions are logged to D1 with actor, -action, cache name, IP, and status code. Logs are retained for 30 days. +Both use the local store as their corpus and run single-threaded. From 3861854139e2ca0ff545d7ce51f2238b00e8e465 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 12:23:54 +0200 Subject: [PATCH 08/56] core: export sha256, hmac and randomness Export SHA-256 (the multi-block SHA-NI implementation), HMAC-SHA256 and the OS random number generator from libhelios, with Rust wrappers, a constant-time comparison and a version 4 UUID helper. This lets the server drop the hmac, sha2, subtle and uuid crates. Zig 0.16 moved randomness behind std.Io, so the export calls getrandom directly on Linux. The helios-core tests use a small scratch directory helper instead of the tempfile crate. --- Cargo.lock | 1 - core/include/helios.h | 7 +++ core/src/root.zig | 31 +++++++++++++ crates/helios-core/Cargo.toml | 2 - crates/helios-core/src/lib.rs | 84 ++++++++++++++++++++++++++++++++++- 5 files changed, 121 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0451ef0..170ee3e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -592,7 +592,6 @@ name = "helios-core" version = "0.1.0" dependencies = [ "pkg-config", - "tempfile", ] [[package]] diff --git a/core/include/helios.h b/core/include/helios.h index 7baa783..8dd83a8 100644 --- a/core/include/helios.h +++ b/core/include/helios.h @@ -106,6 +106,13 @@ typedef struct { int hl_narinfo_render(const hl_narinfo_input *in, const hl_signer *signer, char **out, size_t *out_len); void hl_free(char *ptr, size_t len); +/* ── Primitives ── */ + +void hl_sha256(const uint8_t *data, size_t len, uint8_t out[32]); +void hl_hmac_sha256(const uint8_t *key, size_t key_len, const uint8_t *msg, size_t msg_len, uint8_t out[32]); +/* Fills buf from the OS CSPRNG. */ +int hl_random(uint8_t *buf, size_t len); + #ifdef __cplusplus } #endif diff --git a/core/src/root.zig b/core/src/root.zig index db96907..99f70a3 100644 --- a/core/src/root.zig +++ b/core/src/root.zig @@ -219,3 +219,34 @@ export fn hl_free(ptr: ?[*]u8, len: usize) void { if (ptr) |p| gpa.free(p[0..len]); } + +// ── Small primitives for the Rust side (keeps hmac/sha2/uuid crates out) ── + +const builtin = @import("builtin"); + +export fn hl_sha256(data: [*]const u8, len: usize, out: *[32]u8) void { + archive.Sha256.hash(data[0..len], out, .{}); +} + +export fn hl_hmac_sha256(key: [*]const u8, key_len: usize, msg: [*]const u8, msg_len: usize, out: *[32]u8) void { + std.crypto.auth.hmac.sha2.HmacSha256.create(out, msg[0..msg_len], key[0..key_len]); +} + +/// Fills `buf` from the OS CSPRNG. +export fn hl_random(buf: [*]u8, len: usize) c_int { + if (builtin.os.tag == .linux) { + const linux = std.os.linux; + var off: usize = 0; + while (off < len) { + const rc = linux.getrandom(buf + off, len - off, 0); + switch (linux.errno(rc)) { + .SUCCESS => off += rc, + .INTR => {}, + else => return HL_E_IO, + } + } + } else { + std.c.arc4random_buf(buf, len); + } + return HL_OK; +} diff --git a/crates/helios-core/Cargo.toml b/crates/helios-core/Cargo.toml index de80468..54e76b3 100644 --- a/crates/helios-core/Cargo.toml +++ b/crates/helios-core/Cargo.toml @@ -12,5 +12,3 @@ links = "helios" [build-dependencies] pkg-config = "0.3.34" -[dev-dependencies] -tempfile = "3.27.0" diff --git a/crates/helios-core/src/lib.rs b/crates/helios-core/src/lib.rs index 2bb1fa1..708c115 100644 --- a/crates/helios-core/src/lib.rs +++ b/crates/helios-core/src/lib.rs @@ -70,6 +70,9 @@ mod sys { pub fn hl_signer_public_key(s: *const Signer, out: *mut u8, cap: usize, out_len: *mut usize) -> c_int; pub fn hl_narinfo_render(input: *const NarinfoInput, signer: *const Signer, out: *mut *mut u8, out_len: *mut usize) -> c_int; pub fn hl_free(ptr: *mut u8, len: usize); + pub fn hl_sha256(data: *const u8, len: usize, out: *mut [u8; 32]); + pub fn hl_hmac_sha256(key: *const u8, key_len: usize, msg: *const u8, msg_len: usize, out: *mut [u8; 32]); + pub fn hl_random(buf: *mut u8, len: usize) -> c_int; } } @@ -368,12 +371,91 @@ pub fn render_narinfo(input: &NarinfoInput<'_>, signer: Option<&Signer>) -> Resu Ok(text) } +// ── Primitives ── + +pub fn sha256(data: &[u8]) -> [u8; 32] { + let mut out = [0u8; 32]; + unsafe { sys::hl_sha256(data.as_ptr(), data.len(), &mut out) }; + out +} + +pub fn hmac_sha256(key: &[u8], msg: &[u8]) -> [u8; 32] { + let mut out = [0u8; 32]; + unsafe { sys::hl_hmac_sha256(key.as_ptr(), key.len(), msg.as_ptr(), msg.len(), &mut out) }; + out +} + +/// Constant-time equality for equal-length byte strings. +pub fn ct_eq(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + let diff = a.iter().zip(b).fold(0u8, |acc, (x, y)| acc | (x ^ y)); + std::hint::black_box(diff) == 0 +} + +/// Fills `buf` from the OS CSPRNG. +pub fn random_bytes(buf: &mut [u8]) { + let rc = unsafe { sys::hl_random(buf.as_mut_ptr(), buf.len()) }; + assert_eq!(rc, 0, "the OS random number generator failed"); +} + +/// A random (version 4) UUID in canonical hyphenated form. +pub fn uuid_v4() -> String { + let mut b = [0u8; 16]; + random_bytes(&mut b); + b[6] = (b[6] & 0x0f) | 0x40; + b[8] = (b[8] & 0x3f) | 0x80; + let hex: String = b.iter().map(|x| format!("{x:02x}")).collect(); + format!("{}-{}-{}-{}-{}", &hex[0..8], &hex[8..12], &hex[12..16], &hex[16..20], &hex[20..32]) +} + #[cfg(test)] mod tests { use super::*; const KEY: &str = "test-1:suJsWimvBNFUIc0JJE18OfOMygH/f2GuTC2/XwD6rPKkV+1VKilMIkXl6Ax9hqeKpUF/BSOH+Fqng4tqZlirhw=="; + /// A scratch directory removed on drop (keeps tempfile out of the tree). + struct TempDir(std::path::PathBuf); + + impl TempDir { + fn new() -> Self { + let dir = std::env::temp_dir().join(format!("helios-core-test-{}", uuid_v4())); + std::fs::create_dir(&dir).unwrap(); + Self(dir) + } + fn path(&self) -> &Path { + &self.0 + } + } + + impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + #[test] + fn primitives() { + // RFC 4231 test case 2. + let mac = hmac_sha256(b"Jefe", b"what do ya want for nothing?"); + assert_eq!(nix32_encode(&mac).len(), 52); + assert_eq!( + mac.iter().map(|b| format!("{b:02x}")).collect::(), + "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843" + ); + assert_eq!( + sha256(b"abc").iter().map(|b| format!("{b:02x}")).collect::(), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + assert!(ct_eq(b"same", b"same") && !ct_eq(b"same", b"diff") && !ct_eq(b"a", b"ab")); + let (a, b) = (uuid_v4(), uuid_v4()); + assert_ne!(a, b); + assert_eq!(a.len(), 36); + assert_eq!(&a[14..15], "4"); + } + #[test] fn nix32_round_trip() { let bytes = [7u8; 20]; @@ -392,7 +474,7 @@ mod tests { #[test] fn dump_matches_compressor_and_verifier() { - let dir = tempfile::tempdir().unwrap(); + let dir = TempDir::new(); let root = dir.path().join("pkg"); std::fs::create_dir_all(root.join("bin")).unwrap(); std::fs::write(root.join("bin/hello"), b"#!/bin/sh\necho hi\n").unwrap(); From 1a72024388240aaf3fa5de53006e3553a0e36b36 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 12:23:54 +0200 Subject: [PATCH 09/56] build: add sqlite to the dev shell and package The server is switching from a compiled-in SQLite to the system library, so the dev shell and the package now provide it. --- flake.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index d12247a..8114e8e 100644 --- a/flake.nix +++ b/flake.nix @@ -23,7 +23,7 @@ cargoLock.lockFile = ./Cargo.lock; nativeBuildInputs = [ pkgs.zig pkgs.pkg-config ]; - buildInputs = [ pkgs.zstd ]; + buildInputs = [ pkgs.zstd pkgs.sqlite ]; # Nix builds must not depend on the build machine's CPU. `baseline` # loses SHA-NI accelerated hashing; for a dedicated host, build with @@ -56,6 +56,7 @@ pkgs.rustfmt pkgs.pkg-config pkgs.zstd + pkgs.sqlite pkgs.jq pkgs.curl ]; From 210aa56f61160d5c1c7a9997974517431ddebf8f Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 12:23:54 +0200 Subject: [PATCH 10/56] server: cut the dependency tree from 116 to 64 crates Serve HTTP/1.1 only, since TLS and HTTP/2 end at the reverse proxy, and parse the upload query by hand, dropping axum's http2 and query features. Link the system SQLite by default, with a bundled-sqlite feature to compile it in, and turn off default features on rusqlite, quick_cache, clap and futures-util. tracing is replaced by a small logger that emits journald priorities under systemd and reads its level from HELIOS_LOG, parking_lot by std locks, tokio-util's ReaderStream by an unfold stream, and hmac, sha2, subtle and uuid by the libhelios exports. mimalloc is gone too; without it narinfo misses still run at 1.18M requests per second. The server now also shuts down gracefully on SIGTERM, which is how systemd stops services. --- Cargo.lock | 369 +----------------------------- crates/helios-server/Cargo.toml | 31 +-- crates/helios-server/src/admin.rs | 9 +- crates/helios-server/src/audit.rs | 4 +- crates/helios-server/src/auth.rs | 21 +- crates/helios-server/src/db.rs | 4 +- crates/helios-server/src/error.rs | 2 +- crates/helios-server/src/gc.rs | 11 +- crates/helios-server/src/log.rs | 56 +++++ crates/helios-server/src/main.rs | 26 ++- crates/helios-server/src/push.rs | 23 +- crates/helios-server/src/read.rs | 15 +- crates/helios-server/src/state.rs | 24 +- 13 files changed, 152 insertions(+), 443 deletions(-) create mode 100644 crates/helios-server/src/log.rs diff --git a/Cargo.lock b/Cargo.lock index 170ee3e..dcc63fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "aho-corasick" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" -dependencies = [ - "memchr", -] - [[package]] name = "anstream" version = "1.0.0" @@ -104,7 +95,6 @@ checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ "axum-core", "bytes", - "form_urlencoded", "futures-util", "http", "http-body", @@ -120,7 +110,6 @@ dependencies = [ "serde_core", "serde_json", "serde_path_to_error", - "serde_urlencoded", "sync_wrapper", "tokio", "tower", @@ -158,15 +147,6 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" -[[package]] -name = "block-buffer" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" -dependencies = [ - "hybrid-array", -] - [[package]] name = "bumpalo" version = "3.20.3" @@ -263,12 +243,6 @@ dependencies = [ "cc", ] -[[package]] -name = "cmov" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" - [[package]] name = "colorchoice" version = "1.0.5" @@ -285,12 +259,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "const-oid" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" - [[package]] name = "core-foundation" version = "0.10.1" @@ -316,36 +284,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crypto-common" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" -dependencies = [ - "hybrid-array", -] - -[[package]] -name = "ctutils" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" -dependencies = [ - "cmov", -] - -[[package]] -name = "digest" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" -dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", - "ctutils", -] - [[package]] name = "displaydoc" version = "0.2.7" @@ -391,12 +329,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" -[[package]] -name = "fastrand" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" - [[package]] name = "find-msvc-tools" version = "0.1.14" @@ -536,15 +468,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbrown" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" -dependencies = [ - "foldhash", -] - [[package]] name = "hashbrown" version = "0.17.1" @@ -560,7 +483,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a596f1b20ed2cc5ecac41a164aaebc7258057060f06c0cf7a2ba3991ee7990fb" dependencies = [ - "hashbrown 0.17.1", + "hashbrown", ] [[package]] @@ -605,31 +528,11 @@ dependencies = [ "clap", "futures-util", "helios-core", - "hmac", - "mimalloc", - "parking_lot", "quick_cache", - "reqwest", "rusqlite", "serde", "serde_json", - "sha2", - "subtle", - "tempfile", "tokio", - "tokio-util", - "tracing", - "tracing-subscriber", - "uuid", -] - -[[package]] -name = "hmac" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" -dependencies = [ - "digest", ] [[package]] @@ -677,15 +580,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "hybrid-array" -version = "0.4.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" -dependencies = [ - "typenum", -] - [[package]] name = "hyper" version = "1.11.1" @@ -858,7 +752,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", - "hashbrown 0.17.1", + "hashbrown", ] [[package]] @@ -949,12 +843,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - [[package]] name = "libc" version = "0.2.189" @@ -981,27 +869,12 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "linux-raw-sys" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" - [[package]] name = "litemap" version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - [[package]] name = "log" version = "0.4.34" @@ -1014,15 +887,6 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - [[package]] name = "matchit" version = "0.8.4" @@ -1061,15 +925,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "nu-ansi-term" -version = "0.50.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "once_cell" version = "1.21.4" @@ -1088,29 +943,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] - [[package]] name = "percent-encoding" version = "2.3.2" @@ -1154,9 +986,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "403c1a912fec895cafb223201e368234842acb9220aaf08ab042ae89ba5f135c" dependencies = [ "equivalent", - "foldhash", - "hashbrown 0.17.1", - "parking_lot", + "hashbrown", ] [[package]] @@ -1257,32 +1087,6 @@ dependencies = [ "rand_core", ] -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags", -] - -[[package]] -name = "regex-automata" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" - [[package]] name = "reqwest" version = "0.13.5" @@ -1338,16 +1142,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rsqlite-vfs" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c51c9ae4df8a7fba42103df5c621fa3c37eccf3a3c650879e90fc48b11cc192c" -dependencies = [ - "hashbrown 0.16.1", - "thiserror", -] - [[package]] name = "rusqlite" version = "0.40.2" @@ -1360,7 +1154,6 @@ dependencies = [ "hashlink", "libsqlite3-sys", "smallvec", - "sqlite-wasm-rs", ] [[package]] @@ -1378,19 +1171,6 @@ dependencies = [ "semver", ] -[[package]] -name = "rustix" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" -dependencies = [ - "bitflags", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.61.2", -] - [[package]] name = "rustls" version = "0.23.45" @@ -1472,12 +1252,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" -[[package]] -name = "ryu" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" - [[package]] name = "same-file" version = "1.0.6" @@ -1496,12 +1270,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "security-framework" version = "3.7.0" @@ -1585,38 +1353,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "sha2" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - [[package]] name = "shlex" version = "2.0.1" @@ -1671,18 +1407,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "sqlite-wasm-rs" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc3efc0da82635d7e1ced0053bbbfa8c7ab9645d0bf36ceb4f7127bb85315d75" -dependencies = [ - "cc", - "js-sys", - "rsqlite-vfs", - "wasm-bindgen", -] - [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1743,19 +1467,6 @@ dependencies = [ "syn 3.0.6", ] -[[package]] -name = "tempfile" -version = "3.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" -dependencies = [ - "fastrand", - "getrandom 0.4.3", - "once_cell", - "rustix", - "windows-sys 0.61.2", -] - [[package]] name = "thiserror" version = "2.0.21" @@ -1776,15 +1487,6 @@ dependencies = [ "syn 3.0.6", ] -[[package]] -name = "thread_local" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" -dependencies = [ - "cfg-if", -] - [[package]] name = "tinystr" version = "0.8.4" @@ -1915,21 +1617,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "pin-project-lite", - "tracing-attributes", "tracing-core", ] -[[package]] -name = "tracing-attributes" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "tracing-core" version = "0.1.36" @@ -1937,36 +1627,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex-automata", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", ] [[package]] @@ -1975,12 +1635,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" - [[package]] name = "unicode-ident" version = "1.0.26" @@ -2017,23 +1671,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "1.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" -dependencies = [ - "getrandom 0.4.3", - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - [[package]] name = "vcpkg" version = "0.2.15" diff --git a/crates/helios-server/Cargo.toml b/crates/helios-server/Cargo.toml index 206d20a..5f54ea0 100644 --- a/crates/helios-server/Cargo.toml +++ b/crates/helios-server/Cargo.toml @@ -5,29 +5,22 @@ edition.workspace = true license-file.workspace = true publish.workspace = true +[features] +default = [] +# Compile SQLite into the binary instead of linking the system library. +bundled-sqlite = ["rusqlite/bundled"] + [dependencies] anyhow = "1.0.104" -axum = { version = "0.8.9", default-features = false, features = ["http1", "http2", "tokio", "query", "json"] } +# HTTP/1.1 only: TLS and HTTP/2 end at the reverse proxy. +axum = { version = "0.8.9", default-features = false, features = ["http1", "tokio", "json"] } base64 = "0.23.1" bytes = "1.12.1" -clap = { version = "4.6.7", features = ["derive", "env"] } -futures-util = "0.3.34" +clap = { version = "4.6.7", default-features = false, features = ["std", "derive", "env", "help", "usage", "error-context"] } +futures-util = { version = "0.3.34", default-features = false, features = ["std"] } helios-core = { path = "../helios-core" } -hmac = "0.13.0" -mimalloc = "0.1.52" -parking_lot = "0.12.5" -quick_cache = "0.7.0" -rusqlite = { version = "0.40.2", features = ["bundled", "array", "fallible_uint"] } +quick_cache = { version = "0.7.0", default-features = false } +rusqlite = { version = "0.40.2", default-features = false, features = ["cache", "array", "fallible_uint"] } serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" -sha2 = "0.11.0" -subtle = "2.6.1" -tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "net", "fs", "signal", "sync", "time", "io-util"] } -tokio-util = { version = "0.7.19", features = ["io"] } -tracing = "0.1.44" -tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } -uuid = { version = "1.26.1", features = ["v4"] } - -[dev-dependencies] -reqwest = { version = "0.13.5", default-features = false, features = ["json", "stream", "rustls"] } -tempfile = "3.27.0" +tokio = { version = "1.53.1", features = ["rt-multi-thread", "net", "fs", "signal", "sync", "time", "io-util"] } diff --git a/crates/helios-server/src/admin.rs b/crates/helios-server/src/admin.rs index 5445cc8..9dfff71 100644 --- a/crates/helios-server/src/admin.rs +++ b/crates/helios-server/src/admin.rs @@ -7,6 +7,7 @@ use rusqlite::params; use serde::Deserialize; use serde_json::json; +use crate::state::Locked; use crate::audit::Audit; use crate::auth::{self, AUDIENCE, Claims, ISSUER, MAX_CACHES_PER_TOKEN, MAX_PERMS_PER_TOKEN, Perm}; use crate::error::{ApiError, ApiResult}; @@ -59,7 +60,7 @@ pub async fn create_cache( pub async fn list_caches(State(st): State, headers: HeaderMap) -> ApiResult> { st.authorize_admin(&headers)?; - let mut caches: Vec<_> = st.caches.read().iter().map(|(name, c)| json!({ "name": name, "public": c.public })).collect(); + let mut caches: Vec<_> = st.caches.rd().iter().map(|(name, c)| json!({ "name": name, "public": c.public })).collect(); caches.sort_by(|a, b| a["name"].as_str().cmp(&b["name"].as_str())); Ok(Json(json!({ "caches": caches }))) } @@ -103,7 +104,7 @@ pub async fn create_token( let now = crate::db::now(); let claims = Claims { - jti: uuid::Uuid::new_v4().to_string(), + jti: helios_core::uuid_v4(), sub: req.subject, iss: ISSUER.into(), aud: AUDIENCE.into(), @@ -131,7 +132,7 @@ pub async fn create_token( }) }) .await??; - st.tokens.write().insert(claims.jti.as_str().into(), TokenState { expires_at: claims.exp, revoked: false }); + st.tokens.wr().insert(claims.jti.as_str().into(), TokenState { expires_at: claims.exp, revoked: false }); audit.log(&st, &who, "token.create", None, StatusCode::CREATED, json!({ "jti": claims.jti, "subject": claims.sub })); Ok(( StatusCode::CREATED, @@ -207,7 +208,7 @@ pub async fn revoke_token( if changed == 0 { return Err(ApiError::new(StatusCode::NOT_FOUND, "token not found or already revoked")); } - if let Some(t) = st.tokens.write().get_mut(jti.as_str()) { + if let Some(t) = st.tokens.wr().get_mut(jti.as_str()) { t.revoked = true; } audit.log(&st, &who, "token.revoke", None, StatusCode::OK, json!({ "jti": jti })); diff --git a/crates/helios-server/src/audit.rs b/crates/helios-server/src/audit.rs index 14cbf41..24ce329 100644 --- a/crates/helios-server/src/audit.rs +++ b/crates/helios-server/src/audit.rs @@ -87,8 +87,8 @@ pub async fn run(db: Db, mut rx: mpsc::UnboundedReceiver) { .await; match result { Ok(Ok(())) => {} - Ok(Err(e)) => tracing::error!(error = %e, "writing audit log"), - Err(e) => tracing::error!(error = %e, "audit writer panicked"), + Ok(Err(e)) => crate::log::error!("writing audit log: {e}"), + Err(e) => crate::log::error!("audit writer panicked: {e}"), } } } diff --git a/crates/helios-server/src/auth.rs b/crates/helios-server/src/auth.rs index 8853d3f..2f36034 100644 --- a/crates/helios-server/src/auth.rs +++ b/crates/helios-server/src/auth.rs @@ -4,15 +4,12 @@ use axum::http::{HeaderMap, header}; use base64::Engine; use base64::engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD}; -use hmac::{KeyInit, Mac}; use serde::{Deserialize, Serialize}; -use subtle::ConstantTimeEq; +use crate::state::Locked; use crate::error::ApiError; use crate::state::{AppState, CacheInfo}; -type HmacSha256 = hmac::Hmac; - pub const ISSUER: &str = "helios-cache"; pub const AUDIENCE: &str = "helios-cache"; const HEADER_B64: &str = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"; // {"alg":"HS256","typ":"JWT"} @@ -63,9 +60,7 @@ impl Identity { pub fn sign(claims: &Claims, secret: &[u8]) -> String { let payload = URL_SAFE_NO_PAD.encode(serde_json::to_vec(claims).expect("claims serialise")); let input = format!("{HEADER_B64}.{payload}"); - let mut mac = HmacSha256::new_from_slice(secret).expect("HMAC accepts any key length"); - mac.update(input.as_bytes()); - let sig = URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes()); + let sig = URL_SAFE_NO_PAD.encode(helios_core::hmac_sha256(secret, input.as_bytes())); format!("{input}.{sig}") } @@ -79,9 +74,9 @@ fn verify(token: &str, secret: &[u8], now: i64) -> Option { return None; } let sig = URL_SAFE_NO_PAD.decode(sig_b64).ok()?; - let mut mac = HmacSha256::new_from_slice(secret).ok()?; - mac.update(signed.as_bytes()); - mac.verify_slice(&sig).ok()?; + if !helios_core::ct_eq(&helios_core::hmac_sha256(secret, signed.as_bytes()), &sig) { + return None; + } let claims: Claims = serde_json::from_slice(&URL_SAFE_NO_PAD.decode(payload_b64).ok()?).ok()?; let valid = !claims.jti.is_empty() @@ -114,10 +109,8 @@ fn credential(headers: &HeaderMap) -> Option { } fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { - use sha2::Digest; // Hash first so the comparison does not leak the secret's length. - let (da, db) = (sha2::Sha256::digest(a), sha2::Sha256::digest(b)); - da.as_slice().ct_eq(db.as_slice()).into() + helios_core::ct_eq(&helios_core::sha256(a), &helios_core::sha256(b)) } impl AppState { @@ -126,7 +119,7 @@ impl AppState { let token = credential(headers).ok_or_else(ApiError::unauthorized)?; let now = crate::db::now(); let claims = verify(&token, secret, now).ok_or_else(ApiError::unauthorized)?; - match self.tokens.read().get(claims.jti.as_str()) { + match self.tokens.rd().get(claims.jti.as_str()) { Some(t) if !t.revoked && t.expires_at > now => Ok(claims), _ => Err(ApiError::unauthorized()), } diff --git a/crates/helios-server/src/db.rs b/crates/helios-server/src/db.rs index 3b8d405..ae5d2e8 100644 --- a/crates/helios-server/src/db.rs +++ b/crates/helios-server/src/db.rs @@ -5,7 +5,7 @@ use std::cell::RefCell; use std::path::{Path, PathBuf}; use std::sync::Arc; -use parking_lot::Mutex; +use std::sync::{Mutex, PoisonError}; use rusqlite::{Connection, OpenFlags}; const MIGRATIONS: &[&str] = &[r#" @@ -89,7 +89,7 @@ impl Db { /// Runs `f` on the writer connection. Blocking: call from spawn_blocking. pub fn write(&self, f: impl FnOnce(&mut Connection) -> T) -> T { - f(&mut self.writer.lock()) + f(&mut self.writer.lock().unwrap_or_else(PoisonError::into_inner)) } /// Runs `f` on this thread's read-only connection. Blocking. diff --git a/crates/helios-server/src/error.rs b/crates/helios-server/src/error.rs index 2774a4f..d19afad 100644 --- a/crates/helios-server/src/error.rs +++ b/crates/helios-server/src/error.rs @@ -31,7 +31,7 @@ impl ApiError { } pub fn internal(err: impl std::fmt::Display) -> Self { - tracing::error!(error = %err, "internal error"); + crate::log::error!("internal error: {err}"); Self::new(StatusCode::INTERNAL_SERVER_ERROR, "internal error") } diff --git a/crates/helios-server/src/gc.rs b/crates/helios-server/src/gc.rs index f26a8ae..fec1cf1 100644 --- a/crates/helios-server/src/gc.rs +++ b/crates/helios-server/src/gc.rs @@ -6,6 +6,7 @@ use std::time::{Duration, SystemTime}; use helios_core::Compression; use rusqlite::params; +use crate::state::Locked; use crate::state::Shared; /// Blobs uploaded but not yet published are kept this long, so a push that @@ -30,9 +31,9 @@ pub async fn run_forever(st: Shared) { interval.tick().await; let st2 = st.clone(); match tokio::task::spawn_blocking(move || collect(&st2)).await { - Ok(Ok(stats)) => tracing::info!(?stats, "gc finished"), - Ok(Err(e)) => tracing::error!(error = %e, "gc failed"), - Err(e) => tracing::error!(error = %e, "gc panicked"), + Ok(Ok(stats)) => crate::log::info!("gc finished: {stats:?}"), + Ok(Err(e)) => crate::log::error!("gc failed: {e}"), + Err(e) => crate::log::error!("gc panicked: {e}"), } } } @@ -82,7 +83,7 @@ pub fn collect(st: &Shared) -> anyhow::Result { match std::fs::remove_file(st.nar_path(&hash, compression)) { Ok(()) => stats.bytes += size, Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => tracing::warn!(error = %e, "removing blob file"), + Err(e) => crate::log::warning!("removing blob file: {e}"), } } Ok(victims.len()) @@ -94,7 +95,7 @@ pub fn collect(st: &Shared) -> anyhow::Result { } stats.tokens = st.db.write(|conn| conn.execute("DELETE FROM tokens WHERE expires_at < ?1", [now]))?; - st.tokens.write().retain(|_, t| t.expires_at >= now); + st.tokens.wr().retain(|_, t| t.expires_at >= now); let audit_cutoff = now - st.cfg.audit_retention.as_secs() as i64; stats.audit_rows = st.db.write(|conn| conn.execute("DELETE FROM audit_log WHERE ts < ?1", [audit_cutoff]))?; diff --git a/crates/helios-server/src/log.rs b/crates/helios-server/src/log.rs new file mode 100644 index 0000000..a0f8fe5 --- /dev/null +++ b/crates/helios-server/src/log.rs @@ -0,0 +1,56 @@ +//! Minimal leveled logging to stderr. Under systemd (JOURNAL_STREAM is set) +//! lines carry a `` syslog priority prefix so journald records levels; +//! journald adds timestamps. The level comes from HELIOS_LOG +//! (error, warn, info, debug; default info). + +use std::sync::atomic::{AtomicU8, Ordering}; + +pub const ERROR: u8 = 3; +pub const WARN: u8 = 4; +pub const INFO: u8 = 6; +pub const DEBUG: u8 = 7; + +static LEVEL: AtomicU8 = AtomicU8::new(INFO); +static JOURNAL: AtomicU8 = AtomicU8::new(0); + +pub fn init() { + let level = match std::env::var("HELIOS_LOG").as_deref() { + Ok("error") => ERROR, + Ok("warn") => WARN, + Ok("debug") => DEBUG, + _ => INFO, + }; + LEVEL.store(level, Ordering::Relaxed); + JOURNAL.store(u8::from(std::env::var_os("JOURNAL_STREAM").is_some()), Ordering::Relaxed); +} + +pub fn enabled(level: u8) -> bool { + level <= LEVEL.load(Ordering::Relaxed) +} + +pub fn emit(level: u8, args: std::fmt::Arguments<'_>) { + let label = match level { + ERROR => "error", + WARN => "warn", + INFO => "info", + _ => "debug", + }; + if JOURNAL.load(Ordering::Relaxed) == 1 { + eprintln!("<{level}>{args}"); + } else { + eprintln!("{label}: {args}"); + } +} + +macro_rules! log { + ($level:expr, $($arg:tt)*) => { + if $crate::log::enabled($level) { + $crate::log::emit($level, format_args!($($arg)*)); + } + }; +} +macro_rules! error { ($($arg:tt)*) => { $crate::log::log!($crate::log::ERROR, $($arg)*) }; } +macro_rules! warning { ($($arg:tt)*) => { $crate::log::log!($crate::log::WARN, $($arg)*) }; } +macro_rules! info { ($($arg:tt)*) => { $crate::log::log!($crate::log::INFO, $($arg)*) }; } + +pub(crate) use {error, info, log, warning}; diff --git a/crates/helios-server/src/main.rs b/crates/helios-server/src/main.rs index 85ed52d..677d2d0 100644 --- a/crates/helios-server/src/main.rs +++ b/crates/helios-server/src/main.rs @@ -5,6 +5,7 @@ mod config; mod db; mod error; mod gc; +mod log; mod push; mod read; mod state; @@ -22,9 +23,6 @@ use serde_json::json; use crate::config::{Args, Config}; use crate::state::{AppState, Shared}; -#[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; - fn load_signer(args: &Args) -> anyhow::Result> { let Some(path) = &args.signing_key_file else { return Ok(None) }; let text = std::fs::read_to_string(path).map_err(|e| anyhow::anyhow!("reading {}: {e}", path.display()))?; @@ -67,7 +65,7 @@ pub async fn build_state(args: &Args) -> anyhow::Result { let db = db::Db::open(&cfg.data_dir.join("helios.db"))?; let signer = load_signer(args)?; if signer.is_none() { - tracing::warn!("no signing key configured; narinfo will be unsigned"); + crate::log::warning!("no signing key configured; narinfo will be unsigned"); } let (audit_tx, audit_rx) = tokio::sync::mpsc::unbounded_channel(); tokio::spawn(audit::run(db.clone(), audit_rx)); @@ -77,9 +75,7 @@ pub async fn build_state(args: &Args) -> anyhow::Result { } fn main() -> anyhow::Result<()> { - tracing_subscriber::fmt() - .with_env_filter(tracing_subscriber::EnvFilter::try_from_default_env().unwrap_or_else(|_| "helios_server=info".into())) - .init(); + log::init(); let args = Args::parse(); if args.print_public_key { @@ -91,12 +87,20 @@ fn main() -> anyhow::Result<()> { tokio::runtime::Builder::new_multi_thread().enable_all().build()?.block_on(async move { let st = build_state(&args).await?; let listener = tokio::net::TcpListener::bind(args.listen).await?; - tracing::info!(listen = %args.listen, "helios listening"); + crate::log::info!("listening on {}", args.listen); axum::serve(listener, router(st).into_make_service_with_connect_info::()) - .with_graceful_shutdown(async { - let _ = tokio::signal::ctrl_c().await; - }) + .with_graceful_shutdown(shutdown_signal()) .await?; Ok(()) }) } + +/// SIGTERM (systemd stop) or SIGINT: finish in-flight requests, then exit. +async fn shutdown_signal() { + use tokio::signal::unix::{SignalKind, signal}; + let mut term = signal(SignalKind::terminate()).expect("installing SIGTERM handler"); + let term = std::pin::pin!(term.recv()); + let int = std::pin::pin!(tokio::signal::ctrl_c()); + futures_util::future::select(term, int).await; + crate::log::info!("shutting down"); +} diff --git a/crates/helios-server/src/push.rs b/crates/helios-server/src/push.rs index 18f72eb..22df909 100644 --- a/crates/helios-server/src/push.rs +++ b/crates/helios-server/src/push.rs @@ -12,8 +12,8 @@ use std::rc::Rc; use axum::Json; use axum::body::Body; -use axum::extract::{Path, Query, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode, Uri}; use base64::Engine; use bytes::Bytes; use futures_util::StreamExt; @@ -23,6 +23,7 @@ use rusqlite::{OptionalExtension, params}; use serde::{Deserialize, Serialize}; use serde_json::json; +use crate::state::Locked; use crate::audit::Audit; use crate::auth::Perm; use crate::error::{ApiError, ApiResult}; @@ -54,7 +55,7 @@ pub async fn missing( if req.hashes.len() > MAX_MISSING_BATCH { return Err(ApiError::bad_request(format!("at most {MAX_MISSING_BATCH} hashes per request"))); } - let index = st.index.read(); + let index = st.index.rd(); let missing: Vec<&String> = req .hashes .iter() @@ -108,11 +109,6 @@ pub async fn known( // ── Upload ── -#[derive(Deserialize)] -pub struct UploadQuery { - compression: Option, -} - #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct UploadResp { @@ -125,17 +121,18 @@ pub struct UploadResp { pub async fn upload( State(st): State, Path(name): Path, - Query(q): Query, + uri: Uri, audit: Audit, headers: HeaderMap, body: Body, ) -> ApiResult<(StatusCode, Json)> { cache_for(&st, &name)?; let identity = st.authorize(&headers, &name, Perm::Push)?; - let compression = Compression::parse(q.compression.as_deref().unwrap_or("zstd")) + let requested = uri.query().unwrap_or("").split('&').find_map(|kv| kv.strip_prefix("compression=")).unwrap_or("zstd"); + let compression = Compression::parse(requested) .ok_or_else(|| ApiError::bad_request("compression must be zstd or none"))?; - let tmp = st.tmp_dir().join(uuid::Uuid::new_v4().to_string()); + let tmp = st.tmp_dir().join(helios_core::uuid_v4()); let (tx, mut rx) = tokio::sync::mpsc::channel::(32); let worker_tmp = tmp.clone(); // Disk writes, hashing and decompression happen off the async runtime. @@ -344,7 +341,7 @@ pub async fn publish( let mut prepared = Vec::with_capacity(req.paths.len()); let mut already = 0usize; { - let index = st.index.read(); + let index = st.index.rd(); for spec in req.paths { let p = prepare(cache, spec).map_err(ApiError::bad_request)?; if index.contains(&p.key) { @@ -369,7 +366,7 @@ pub async fn publish( let published = rendered_keys.len(); { - let mut index = st.index.write(); + let mut index = st.index.wr(); for key in rendered_keys { index.insert(key); } diff --git a/crates/helios-server/src/read.rs b/crates/helios-server/src/read.rs index 9c0e6e1..92a2ac0 100644 --- a/crates/helios-server/src/read.rs +++ b/crates/helios-server/src/read.rs @@ -10,6 +10,7 @@ use bytes::Bytes; use helios_core::Compression; use rusqlite::{OptionalExtension, params}; +use crate::state::Locked; use crate::error::{ApiError, ApiResult}; use crate::state::{PathKey, Shared}; @@ -75,7 +76,7 @@ async fn serve(st: Shared, method: Method, uri: Uri, headers: HeaderMap) -> ApiR return Ok(not_found(NEGATIVE)); }; let key = PathKey { cache: cache.id, hash }; - if !st.index.read().contains(&key) { + if !st.index.rd().contains(&key) { return Ok(not_found(if cache.public { NEGATIVE } else { PRIVATE })); } let body = match st.narinfo.get(&key) { @@ -155,6 +156,16 @@ async fn nar_response( Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(not_found(NEGATIVE)), Err(e) => return Err(ApiError::internal(e)), }; - let stream = tokio_util::io::ReaderStream::with_capacity(file, 512 * 1024); + // 512 KiB reads; the stream ends after EOF or the first error. + let stream = futures_util::stream::unfold(Some(file), |file| async move { + use tokio::io::AsyncReadExt; + let mut file = file?; + let mut buf = bytes::BytesMut::with_capacity(512 * 1024); + match file.read_buf(&mut buf).await { + Ok(0) => None, + Ok(_) => Some((Ok(buf.freeze()), Some(file))), + Err(e) => Some((Err(e), None)), + } + }); builder.body(Body::from_stream(stream)).map_err(ApiError::internal) } diff --git a/crates/helios-server/src/state.rs b/crates/helios-server/src/state.rs index 4efb8cd..432ac56 100644 --- a/crates/helios-server/src/state.rs +++ b/crates/helios-server/src/state.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use bytes::Bytes; use helios_core::{Compression, Signer}; -use parking_lot::RwLock; +use std::sync::{PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard}; use rusqlite::params; use tokio::sync::mpsc; @@ -109,7 +109,7 @@ impl AppState { Ok((caches, index, tokens)) })?; - tracing::info!(caches = caches.len(), paths = index.len(), tokens = tokens.len(), "loaded state"); + crate::log::info!("loaded state: {} caches, {} paths, {} tokens", caches.len(), index.len(), tokens.len()); let narinfo = quick_cache::sync::Cache::with( cfg.narinfo_cache_entries, cfg.narinfo_cache_entries as u64, @@ -130,7 +130,7 @@ impl AppState { } pub fn cache(&self, name: &str) -> Option { - self.caches.read().get(name).copied() + self.caches.rd().get(name).copied() } pub fn nar_path(&self, file_hash: &[u8; 32], compression: Compression) -> PathBuf { @@ -155,8 +155,24 @@ impl AppState { })?; Ok(id.map(|id| { let info = CacheInfo { id, public }; - self.caches.write().insert(name.into(), info); + self.caches.wr().insert(name.into(), info); info })) } } + +/// Lock access that ignores poisoning: every critical section here leaves +/// the data consistent, so a panic elsewhere must not wedge the server. +pub trait Locked { + fn rd(&self) -> RwLockReadGuard<'_, T>; + fn wr(&self) -> RwLockWriteGuard<'_, T>; +} + +impl Locked for RwLock { + fn rd(&self) -> RwLockReadGuard<'_, T> { + self.read().unwrap_or_else(PoisonError::into_inner) + } + fn wr(&self) -> RwLockWriteGuard<'_, T> { + self.write().unwrap_or_else(PoisonError::into_inner) + } +} From 3b905f069dee07cbe8b3abf645dd6d70063a30c8 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 12:23:54 +0200 Subject: [PATCH 11/56] cli: replace reqwest with a small hyper client reqwest brought in aws-lc-rs, URL and IDNA handling, tower and proxy support that a CLI talking to one configured server does not need. The CLI now uses a small HTTP/1.1 client over hyper, with rustls on the ring provider and the system CA roots, honouring NIX_SSL_CERT_FILE. This takes the CLI from 115 crates to 56. tokio-stream and mimalloc are dropped as well; a full closure push takes the same 3.3 seconds without mimalloc. --- Cargo.lock | 1235 ++-------------------------- crates/helios-cli/Cargo.toml | 24 +- crates/helios-cli/src/api.rs | 76 +- crates/helios-cli/src/main.rs | 18 +- crates/helios-cli/src/nix.rs | 4 +- crates/helios-cli/src/push.rs | 4 +- crates/helios-cli/src/transport.rs | 219 +++++ 7 files changed, 363 insertions(+), 1217 deletions(-) create mode 100644 crates/helios-cli/src/transport.rs diff --git a/Cargo.lock b/Cargo.lock index dcc63fb..c60361b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,56 +2,12 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "anstream" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - [[package]] name = "anstyle" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" -[[package]] -name = "anstyle-parse" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys 0.61.2", -] - [[package]] name = "anyhow" version = "1.0.104" @@ -64,29 +20,6 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" -[[package]] -name = "aws-lc-rs" -version = "1.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", - "pkg-config", -] - [[package]] name = "axum" version = "0.8.9" @@ -147,12 +80,6 @@ version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" -[[package]] -name = "bumpalo" -version = "3.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" - [[package]] name = "bytes" version = "1.12.1" @@ -166,8 +93,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex", ] @@ -177,23 +102,6 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - -[[package]] -name = "chacha20" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" -dependencies = [ - "cfg-if", - "cpufeatures", - "rand_core", -] - [[package]] name = "clap" version = "4.6.7" @@ -210,10 +118,8 @@ version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ - "anstream", "anstyle", "clap_lex", - "strsim", ] [[package]] @@ -225,7 +131,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.6", + "syn", ] [[package]] @@ -234,31 +140,6 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" -[[package]] -name = "cmake" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" -dependencies = [ - "cc", -] - -[[package]] -name = "colorchoice" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" - -[[package]] -name = "combine" -version = "4.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" -dependencies = [ - "bytes", - "memchr", -] - [[package]] name = "core-foundation" version = "0.10.1" @@ -275,32 +156,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "cpufeatures" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" -dependencies = [ - "libc", -] - -[[package]] -name = "displaydoc" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", -] - -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - [[package]] name = "equivalent" version = "1.0.2" @@ -335,33 +190,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - [[package]] name = "foldhash" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "futures-channel" version = "0.3.34" @@ -377,29 +211,6 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" -[[package]] -name = "futures-io" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" - -[[package]] -name = "futures-macro" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", -] - -[[package]] -name = "futures-sink" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" - [[package]] name = "futures-task" version = "0.3.34" @@ -413,11 +224,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", - "futures-io", - "futures-macro", - "futures-sink", "futures-task", - "memchr", "pin-project-lite", "slab", ] @@ -429,43 +236,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", - "js-sys", "libc", "wasi", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi", - "rand_core", - "wasm-bindgen", -] - -[[package]] -name = "h2" -version = "0.4.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", ] [[package]] @@ -502,12 +274,15 @@ dependencies = [ "clap", "futures-util", "helios-core", - "mimalloc", - "reqwest", + "http", + "http-body-util", + "hyper", + "rustls", + "rustls-native-certs", "serde", "serde_json", "tokio", - "tokio-stream", + "tokio-rustls", ] [[package]] @@ -590,7 +365,6 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", "http", "http-body", "httparse", @@ -602,573 +376,157 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.27.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfa8e654703247911e29c23fbeaa261834bd9bb74efba2f9acddc37bfb127f53" -dependencies = [ - "http", - "hyper", - "hyper-util", - "rustls", - "tokio", - "tokio-rustls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64", "bytes", - "futures-channel", - "futures-util", "http", "http-body", - "httparse", "hyper", - "ipnet", - "libc", - "percent-encoding", "pin-project-lite", - "socket2", "tokio", "tower-service", - "tracing", ] [[package]] -name = "icu_collections" -version = "2.3.0" +name = "itoa" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" -dependencies = [ - "displaydoc", - "potential_utf", - "utf8_iter", - "yoke", - "zerofrom", - "zerovec", -] +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] -name = "icu_locale_core" -version = "2.3.0" +name = "libc" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] -name = "icu_normalizer" -version = "2.3.0" +name = "libsqlite3-sys" +version = "0.38.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", + "cc", + "pkg-config", + "vcpkg", ] [[package]] -name = "icu_normalizer_data" -version = "2.3.0" +name = "matchit" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" [[package]] -name = "icu_properties" -version = "2.3.0" +name = "memchr" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] -name = "icu_properties_data" -version = "2.3.0" +name = "mime" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] -name = "icu_provider" -version = "2.3.1" +name = "mio" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", + "libc", + "wasi", + "windows-sys 0.61.2", ] [[package]] -name = "idna" -version = "1.1.0" +name = "once_cell" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] -name = "idna_adapter" -version = "1.2.2" +name = "openssl-probe" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" -dependencies = [ - "icu_normalizer", - "icu_properties", -] +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" [[package]] -name = "indexmap" -version = "2.14.2" +name = "percent-encoding" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" -dependencies = [ - "equivalent", - "hashbrown", -] +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] -name = "ipnet" -version = "2.12.2" +name = "pin-project-lite" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] -name = "is_terminal_polyfill" -version = "1.70.2" +name = "pkg-config" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] -name = "itoa" -version = "1.0.18" +name = "proc-macro2" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] [[package]] -name = "jni" -version = "0.22.4" +name = "quick_cache" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +checksum = "403c1a912fec895cafb223201e368234842acb9220aaf08ab042ae89ba5f135c" dependencies = [ - "cfg-if", - "combine", - "jni-macros", - "jni-sys", - "log", - "simd_cesu8", - "thiserror", - "walkdir", - "windows-link", + "equivalent", + "hashbrown", ] [[package]] -name = "jni-macros" -version = "0.22.4" +name = "quote" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", - "quote", - "rustc_version", - "simd_cesu8", - "syn 2.0.119", ] [[package]] -name = "jni-sys" -version = "0.4.1" +name = "ring" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ - "jni-sys-macros", + "cc", + "cfg-if", + "getrandom", + "libc", + "untrusted", + "windows-sys 0.52.0", ] [[package]] -name = "jni-sys-macros" -version = "0.4.1" +name = "rusqlite" +version = "0.40.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" dependencies = [ - "quote", - "syn 2.0.119", -] - -[[package]] -name = "jobserver" -version = "0.1.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" -dependencies = [ - "getrandom 0.4.3", - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" -dependencies = [ - "cfg-if", - "futures-util", - "wasm-bindgen", -] - -[[package]] -name = "libc" -version = "0.2.189" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" - -[[package]] -name = "libmimalloc-sys" -version = "0.1.49" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" -dependencies = [ - "cc", -] - -[[package]] -name = "libsqlite3-sys" -version = "0.38.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1d20bef17f513b9b3004532233187769cd072d790971f4e4da0e346eb6401e8" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "litemap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" - -[[package]] -name = "log" -version = "0.4.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" - -[[package]] -name = "lru-slab" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" - -[[package]] -name = "matchit" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "mimalloc" -version = "0.1.52" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" -dependencies = [ - "libmimalloc-sys", -] - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "mio" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.61.2", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "once_cell_polyfill" -version = "1.70.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" - -[[package]] -name = "openssl-probe" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "pkg-config" -version = "0.3.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" - -[[package]] -name = "potential_utf" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" -dependencies = [ - "zerovec", -] - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quick_cache" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403c1a912fec895cafb223201e368234842acb9220aaf08ab042ae89ba5f135c" -dependencies = [ - "equivalent", - "hashbrown", -] - -[[package]] -name = "quinn" -version = "0.11.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" -dependencies = [ - "aws-lc-rs", - "bytes", - "getrandom 0.4.3", - "lru-slab", - "rand", - "rand_pcg", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.61.2", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "rand" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" -dependencies = [ - "chacha20", - "getrandom 0.4.3", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - -[[package]] -name = "rand_pcg" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" -dependencies = [ - "rand_core", -] - -[[package]] -name = "reqwest" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" -dependencies = [ - "base64", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-pki-types", - "rustls-platform-verifier", - "serde", - "serde_json", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tokio-util", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", -] - -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.17", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - -[[package]] -name = "rusqlite" -version = "0.40.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f2a97da3e3873c73cb2a2e71b35c40ff95e0b1eefa8d72d8499a6928c3b5b3" -dependencies = [ - "bitflags", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", -] - -[[package]] -name = "rustc-hash" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" - -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", ] [[package]] @@ -1177,8 +535,8 @@ version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ - "aws-lc-rs", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -1203,64 +561,20 @@ version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ - "web-time", "zeroize", ] -[[package]] -name = "rustls-platform-verifier" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" -dependencies = [ - "core-foundation", - "core-foundation-sys", - "jni", - "log", - "once_cell", - "rustls", - "rustls-native-certs", - "rustls-platform-verifier-android", - "rustls-webpki", - "security-framework", - "security-framework-sys", - "webpki-root-certs", - "windows-sys 0.61.2", -] - -[[package]] -name = "rustls-platform-verifier-android" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" - [[package]] name = "rustls-webpki" version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ - "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", ] -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "schannel" version = "0.1.29" @@ -1293,12 +607,6 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - [[package]] name = "serde" version = "1.0.229" @@ -1326,7 +634,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn", ] [[package]] @@ -1369,22 +677,6 @@ dependencies = [ "libc", ] -[[package]] -name = "simd_cesu8" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" -dependencies = [ - "rustc_version", - "simdutf8", -] - -[[package]] -name = "simdutf8" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" - [[package]] name = "slab" version = "0.4.12" @@ -1407,35 +699,12 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - [[package]] name = "subtle" version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "syn" -version = "2.0.119" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "3.0.6" @@ -1452,56 +721,6 @@ name = "sync_wrapper" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - -[[package]] -name = "synstructure" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", -] - -[[package]] -name = "thiserror" -version = "2.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", -] - -[[package]] -name = "tinystr" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "tokio" @@ -1527,7 +746,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.6", + "syn", ] [[package]] @@ -1540,31 +759,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-stream" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "libc", - "pin-project-lite", - "tokio", -] - [[package]] name = "tower" version = "0.5.3" @@ -1580,24 +774,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "tower-http" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" -dependencies = [ - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", - "url", -] - [[package]] name = "tower-layer" version = "0.3.3" @@ -1610,25 +786,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" -[[package]] -name = "tracing" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" -dependencies = [ - "pin-project-lite", - "tracing-core", -] - -[[package]] -name = "tracing-core" -version = "0.1.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" -dependencies = [ - "once_cell", -] - [[package]] name = "try-lock" version = "0.2.5" @@ -1647,46 +804,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "url" -version = "2.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - [[package]] name = "vcpkg" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - [[package]] name = "want" version = "0.3.1" @@ -1702,113 +825,6 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasm-bindgen" -version = "0.2.129" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" -dependencies = [ - "js-sys", - "tokio", - "wasm-bindgen", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.129" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.129" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn 3.0.6", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.129" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "wasm-streams" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "web-sys" -version = "0.3.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki-root-certs" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "windows-link" version = "0.2.1" @@ -1897,95 +913,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "writeable" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" - -[[package]] -name = "yoke" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", - "synstructure", -] - -[[package]] -name = "zerofrom" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", - "synstructure", -] - [[package]] name = "zeroize" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" -[[package]] -name = "zerotrie" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.6", -] - [[package]] name = "zmij" version = "1.0.23" diff --git a/crates/helios-cli/Cargo.toml b/crates/helios-cli/Cargo.toml index e4742a0..2352efc 100644 --- a/crates/helios-cli/Cargo.toml +++ b/crates/helios-cli/Cargo.toml @@ -5,20 +5,24 @@ edition.workspace = true license-file.workspace = true publish.workspace = true +[[bin]] +name = "helios" +path = "src/main.rs" + [dependencies] anyhow = "1.0.104" base64 = "0.23.1" bytes = "1.12.1" -clap = { version = "4.6.7", features = ["derive", "env"] } -futures-util = "0.3.34" +clap = { version = "4.6.7", default-features = false, features = ["std", "derive", "env", "help", "usage", "error-context"] } +futures-util = { version = "0.3.34", default-features = false, features = ["std"] } helios-core = { path = "../helios-core" } -mimalloc = "0.1.52" -reqwest = { version = "0.13.5", default-features = false, features = ["json", "stream", "rustls", "http2"] } +http = "1.4" +http-body-util = "0.1" +# HTTP/1.1 client only; TLS via rustls with the ring provider (no aws-lc-rs). +hyper = { version = "1.8", default-features = false, features = ["client", "http1"] } +rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } +rustls-native-certs = "0.8" serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" -tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "sync", "process", "io-util"] } -tokio-stream = "0.1.19" - -[[bin]] -name = "helios" -path = "src/main.rs" +tokio = { version = "1.53.1", features = ["rt-multi-thread", "net", "sync", "process", "io-util"] } +tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } diff --git a/crates/helios-cli/src/api.rs b/crates/helios-cli/src/api.rs index d4af4d9..8894406 100644 --- a/crates/helios-cli/src/api.rs +++ b/crates/helios-cli/src/api.rs @@ -1,15 +1,14 @@ use anyhow::{Context, bail}; -use reqwest::{Body, Response, StatusCode}; +use http::{Method, StatusCode}; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use crate::config::Server; +use crate::transport::{self, Body}; -#[derive(Clone)] pub struct Client { - http: reqwest::Client, - base: String, + http: transport::Client, token: String, } @@ -38,36 +37,29 @@ pub enum Publish { MissingNars(Vec), } -async fn error_text(resp: Response) -> String { - let status = resp.status(); - let body = resp.text().await.unwrap_or_default(); - let msg = serde_json::from_str::(&body) +fn error_text(status: StatusCode, body: &[u8]) -> String { + let msg = serde_json::from_slice::(body) .ok() .and_then(|v| v.get("error").and_then(Value::as_str).map(str::to_owned)) - .unwrap_or(body); + .unwrap_or_else(|| String::from_utf8_lossy(body).into_owned()); format!("{status}: {msg}") } impl Client { pub fn new(server: &Server) -> anyhow::Result { - let http = reqwest::Client::builder() - .user_agent(concat!("helios-cli/", env!("CARGO_PKG_VERSION"))) - .tcp_nodelay(true) - .pool_max_idle_per_host(64) - .build()?; - Ok(Self { http, base: server.server.trim_end_matches('/').to_owned(), token: server.token.clone() }) + Ok(Self { http: transport::Client::new(server.server.trim_end_matches('/'))?, token: server.token.clone() }) } - fn url(&self, path: &str) -> String { - format!("{}/_api/v2{path}", self.base) + async fn call(&self, method: Method, path: &str, content_type: Option<&str>, body: Body) -> anyhow::Result { + let (status, bytes) = self.http.send(method, &format!("/_api/v2{path}"), &self.token, content_type, body).await?; + if !status.is_success() { + bail!("{}", error_text(status, &bytes)); + } + serde_json::from_slice(&bytes).with_context(|| format!("decoding response from {path}")) } - async fn json(&self, req: reqwest::RequestBuilder) -> anyhow::Result { - let resp = req.bearer_auth(&self.token).send().await?; - if !resp.status().is_success() { - bail!("{}", error_text(resp).await); - } - Ok(resp.json().await?) + async fn post(&self, path: &str, body: &Value) -> anyhow::Result { + self.call(Method::POST, path, Some("application/json"), transport::full(serde_json::to_vec(body)?)).await } pub async fn missing(&self, cache: &str, hashes: &[&str]) -> anyhow::Result> { @@ -75,7 +67,7 @@ impl Client { struct R { missing: Vec, } - let r: R = self.json(self.http.post(self.url(&format!("/caches/{cache}/missing"))).json(&json!({ "hashes": hashes }))).await?; + let r: R = self.post(&format!("/caches/{cache}/missing"), &json!({ "hashes": hashes })).await?; Ok(r.missing) } @@ -84,50 +76,42 @@ impl Client { struct R { known: Vec, } - let r: R = self - .json(self.http.post(self.url(&format!("/caches/{cache}/nars/known"))).json(&json!({ "narHashes": nar_hashes }))) - .await?; + let r: R = self.post(&format!("/caches/{cache}/nars/known"), &json!({ "narHashes": nar_hashes })).await?; Ok(r.known) } pub async fn upload(&self, cache: &str, body: Body) -> anyhow::Result { - self.json( - self.http - .put(self.url(&format!("/caches/{cache}/nar?compression=zstd"))) - .header("content-type", "application/x-nix-nar") - .body(body), - ) - .await + self.call(Method::PUT, &format!("/caches/{cache}/nar?compression=zstd"), Some("application/x-nix-nar"), body).await } pub async fn publish(&self, cache: &str, paths: &[PathSpec]) -> anyhow::Result { - let resp = self + let body = transport::full(serde_json::to_vec(&json!({ "paths": paths }))?); + let (status, bytes) = self .http - .post(self.url(&format!("/caches/{cache}/paths"))) - .bearer_auth(&self.token) - .json(&json!({ "paths": paths })) - .send() + .send(Method::POST, &format!("/_api/v2/caches/{cache}/paths"), &self.token, Some("application/json"), body) .await?; - if resp.status() == StatusCode::CONFLICT { - let v: Value = resp.json().await?; + if status == StatusCode::CONFLICT { + let v: Value = serde_json::from_slice(&bytes)?; if v["error"] == "nar_required" { let missing = v["missing"].as_array().context("missing list")?.iter().filter_map(|m| m.as_str().map(str::to_owned)).collect(); return Ok(Publish::MissingNars(missing)); } bail!("publish conflict: {}", v["error"]); } - if !resp.status().is_success() { - bail!("{}", error_text(resp).await); + if !status.is_success() { + bail!("{}", error_text(status, &bytes)); } - let v: Value = resp.json().await?; + let v: Value = serde_json::from_slice(&bytes)?; Ok(Publish::Done { published: v["published"].as_u64().unwrap_or(0) }) } pub async fn admin_get(&self, path: &str) -> anyhow::Result { - self.json(self.http.get(self.url(path))).await + self.call(Method::GET, path, None, transport::full(Bytes::new())).await } pub async fn admin_post(&self, path: &str, body: Value) -> anyhow::Result { - self.json(self.http.post(self.url(path)).json(&body)).await + self.post(path, &body).await } } + +use bytes::Bytes; diff --git a/crates/helios-cli/src/main.rs b/crates/helios-cli/src/main.rs index 4c7aa9e..ba63289 100644 --- a/crates/helios-cli/src/main.rs +++ b/crates/helios-cli/src/main.rs @@ -1,14 +1,12 @@ mod api; mod config; +mod transport; mod nix; mod push; use clap::{Parser, Subcommand}; use serde_json::{Value, json}; -#[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; - /// Helios Nix binary cache CLI. #[derive(Parser)] #[command(version)] @@ -116,9 +114,17 @@ async fn run(cli: Cli) -> anyhow::Result<()> { Ok(()) } -#[tokio::main] -async fn main() { - if let Err(e) = run(Cli::parse()).await { +fn main() { + // Nix environments point at their CA bundle with NIX_SSL_CERT_FILE. + if std::env::var_os("SSL_CERT_FILE").is_none() + && let Some(file) = std::env::var_os("NIX_SSL_CERT_FILE") + { + // SAFETY: single-threaded; the runtime has not started yet. + unsafe { std::env::set_var("SSL_CERT_FILE", file) }; + } + let cli = Cli::parse(); + let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build().expect("starting the tokio runtime"); + if let Err(e) = runtime.block_on(run(cli)) { eprintln!("error: {e:#}"); std::process::exit(1); } diff --git a/crates/helios-cli/src/nix.rs b/crates/helios-cli/src/nix.rs index 553ec4a..9e04ab7 100644 --- a/crates/helios-cli/src/nix.rs +++ b/crates/helios-cli/src/nix.rs @@ -37,7 +37,7 @@ fn full(p: &str) -> String { } async fn run(args: &[&str], installables: &[String]) -> anyhow::Result { - Ok(Command::new("nix") + Command::new("nix") .args(["--extra-experimental-features", "nix-command flakes", "path-info"]) .args(args) .arg("--") @@ -45,7 +45,7 @@ async fn run(args: &[&str], installables: &[String]) -> anyhow::Result anyhow::Result> { diff --git a/crates/helios-cli/src/push.rs b/crates/helios-cli/src/push.rs index 1a6e008..b6a0a9a 100644 --- a/crates/helios-cli/src/push.rs +++ b/crates/helios-cli/src/push.rs @@ -12,7 +12,6 @@ use bytes::Bytes; use futures_util::StreamExt; use helios_core::{DumpOptions, HL_E_UNSUPPORTED_OS}; use tokio::sync::mpsc; -use tokio_stream::wrappers::ReceiverStream; use crate::api::{Client, PathSpec, Publish}; use crate::nix::{self, PathInfo}; @@ -82,7 +81,8 @@ async fn upload_one(client: &Client, cache: &str, info: &PathInfo, opts: DumpOpt let (tx, rx) = mpsc::channel(8); let path = info.path.clone(); let producer = tokio::task::spawn_blocking(move || produce(&path, opts, tx)); - let body = reqwest::Body::wrap_stream(ReceiverStream::new(rx)); + let chunks = futures_util::stream::unfold(rx, |mut rx| async move { rx.recv().await.map(|c| (c, rx)) }); + let body = crate::transport::stream(chunks); let uploaded = client.upload(cache, body).await; let digest = producer.await?; // An upload failure aborts the producer; report the upload's error. diff --git a/crates/helios-cli/src/transport.rs b/crates/helios-cli/src/transport.rs new file mode 100644 index 0000000..b110d46 --- /dev/null +++ b/crates/helios-cli/src/transport.rs @@ -0,0 +1,219 @@ +//! A small HTTP/1.1 client for one Helios server: hyper over TCP, rustls +//! (ring) for https, system CA roots, and a keep-alive connection pool. +//! It replaces reqwest, which pulled in URL/IDNA handling, proxy support +//! and aws-lc-rs that a CLI talking to one configured server never needs. + +use std::io; +use std::net::IpAddr; +use std::pin::Pin; +use std::sync::{Arc, Mutex, PoisonError}; +use std::task::{Context, Poll}; + +use anyhow::{Context as _, bail}; +use bytes::Bytes; +use http::{HeaderValue, Method, Request, Response, header}; +use http_body_util::combinators::UnsyncBoxBody; +use http_body_util::{BodyExt, Full, StreamBody}; +use hyper::body::{Frame, Incoming}; +use hyper::client::conn::http1::SendRequest; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; +use tokio::net::TcpStream; + +pub type Body = UnsyncBoxBody; + +pub fn full(bytes: impl Into) -> Body { + Full::new(bytes.into()).map_err(|never| match never {}).boxed_unsync() +} + +pub fn stream(s: S) -> Body +where + S: futures_util::Stream> + Send + 'static, +{ + use futures_util::StreamExt; + StreamBody::new(s.map(|r| r.map(Frame::data))).boxed_unsync() +} + +struct Target { + tls: bool, + host: String, + port: u16, + /// Path prefix without a trailing slash, e.g. "" or "/helios". + prefix: String, +} + +fn parse_base(url: &str) -> anyhow::Result { + let (tls, rest) = if let Some(r) = url.strip_prefix("https://") { + (true, r) + } else if let Some(r) = url.strip_prefix("http://") { + (false, r) + } else { + bail!("server URL must start with http:// or https://: {url}"); + }; + let (authority, path) = rest.split_once('/').map_or((rest, ""), |(a, p)| (a, p)); + let (host, port) = if let Some(v6) = authority.strip_prefix('[') { + let (h, after) = v6.split_once(']').context("unterminated IPv6 address")?; + (h.to_owned(), after.strip_prefix(':')) + } else { + match authority.rsplit_once(':') { + Some((h, p)) => (h.to_owned(), Some(p)), + None => (authority.to_owned(), None), + } + }; + if host.is_empty() { + bail!("server URL has no host: {url}"); + } + let port = match port { + Some(p) => p.parse().with_context(|| format!("invalid port in {url}"))?, + None if tls => 443, + None => 80, + }; + let prefix = format!("/{}", path.trim_matches('/')); + Ok(Target { tls, host, port, prefix: if prefix == "/" { String::new() } else { prefix } }) +} + +fn tls_config() -> anyhow::Result> { + let mut roots = rustls::RootCertStore::empty(); + let native = rustls_native_certs::load_native_certs(); + for cert in native.certs { + let _ = roots.add(cert); + } + if roots.is_empty() { + bail!("no CA certificates found (set SSL_CERT_FILE or NIX_SSL_CERT_FILE)"); + } + let config = rustls::ClientConfig::builder_with_provider(Arc::new(rustls::crypto::ring::default_provider())) + .with_safe_default_protocol_versions()? + .with_root_certificates(roots) + .with_no_client_auth(); + Ok(Arc::new(config)) +} + +/// hyper's I/O traits over a tokio stream. +struct Io(T); + +impl hyper::rt::Read for Io { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, mut buf: hyper::rt::ReadBufCursor<'_>) -> Poll> { + // SAFETY: tokio's ReadBuf only writes initialised bytes into the + // uninitialised tail, and we advance by exactly what was filled. + let filled = unsafe { + let mut rb = ReadBuf::uninit(buf.as_mut()); + match Pin::new(&mut self.0).poll_read(cx, &mut rb) { + Poll::Ready(Ok(())) => rb.filled().len(), + other => return other, + } + }; + unsafe { buf.advance(filled) }; + Poll::Ready(Ok(())) + } +} + +impl hyper::rt::Write for Io { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + Pin::new(&mut self.0).poll_write(cx, buf) + } + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.0).poll_flush(cx) + } + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.0).poll_shutdown(cx) + } +} + +pub struct Client { + target: Target, + host_header: HeaderValue, + tls: Option>, + idle: Mutex>>, +} + +impl Client { + pub fn new(base: &str) -> anyhow::Result { + let target = parse_base(base)?; + let default_port = if target.tls { 443 } else { 80 }; + let host = if target.host.contains(':') { format!("[{}]", target.host) } else { target.host.clone() }; + let host_header = if target.port == default_port { host } else { format!("{host}:{}", target.port) }; + let tls = if target.tls { Some(tls_config()?) } else { None }; + Ok(Self { host_header: HeaderValue::from_str(&host_header)?, target, tls, idle: Mutex::new(Vec::new()) }) + } + + pub fn path(&self, p: &str) -> String { + format!("{}{p}", self.target.prefix) + } + + async fn connect(&self) -> anyhow::Result> { + let tcp = TcpStream::connect((self.target.host.as_str(), self.target.port)) + .await + .with_context(|| format!("connecting to {}:{}", self.target.host, self.target.port))?; + tcp.set_nodelay(true)?; + let sender = match &self.tls { + None => { + let (sender, conn) = hyper::client::conn::http1::handshake(Io(tcp)).await?; + tokio::spawn(conn); + sender + } + Some(config) => { + let name = match self.target.host.parse::() { + Ok(ip) => rustls::pki_types::ServerName::IpAddress(ip.into()), + Err(_) => rustls::pki_types::ServerName::try_from(self.target.host.clone())?, + }; + let tls = tokio_rustls::TlsConnector::from(config.clone()).connect(name, tcp).await?; + let (sender, conn) = hyper::client::conn::http1::handshake(Io(tls)).await?; + tokio::spawn(conn); + sender + } + }; + Ok(sender) + } + + fn checkout(&self) -> Option> { + let mut idle = self.idle.lock().unwrap_or_else(PoisonError::into_inner); + while let Some(s) = idle.pop() { + if !s.is_closed() { + return Some(s); + } + } + None + } + + /// Sends a request and reads the whole response body. + pub async fn send(&self, method: Method, path: &str, token: &str, content_type: Option<&str>, body: Body) -> anyhow::Result<(http::StatusCode, Bytes)> { + let mut req = Request::builder() + .method(method) + .uri(self.path(path)) + .header(header::HOST, self.host_header.clone()) + .header(header::USER_AGENT, concat!("helios-cli/", env!("CARGO_PKG_VERSION"))) + .header(header::AUTHORIZATION, format!("Bearer {token}")); + if let Some(ct) = content_type { + req = req.header(header::CONTENT_TYPE, ct); + } + let req = req.body(body)?; + let mut sender = match self.checkout() { + Some(s) => s, + None => self.connect().await?, + }; + sender.ready().await?; + let resp: Response = sender.send_request(req).await?; + let status = resp.status(); + let bytes = resp.into_body().collect().await?.to_bytes(); + if !sender.is_closed() { + self.idle.lock().unwrap_or_else(PoisonError::into_inner).push(sender); + } + Ok((status, bytes)) + } +} + +#[cfg(test)] +mod tests { + use super::parse_base; + + #[test] + fn parses_base_urls() { + let t = parse_base("https://cache.example.com").unwrap(); + assert!(t.tls && t.host == "cache.example.com" && t.port == 443 && t.prefix.is_empty()); + let t = parse_base("http://127.0.0.1:8080/helios/").unwrap(); + assert!(!t.tls && t.host == "127.0.0.1" && t.port == 8080 && t.prefix == "/helios"); + let t = parse_base("https://[::1]:9443").unwrap(); + assert!(t.host == "::1" && t.port == 9443); + assert!(parse_base("ftp://x").is_err()); + assert!(parse_base("https://:80").is_err()); + } +} From 3cab8b1ad892d106e6c630b28aab54821914f58e Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 12:23:54 +0200 Subject: [PATCH 12/56] docs: document logging and the dependency policy Replace RUST_LOG with HELIOS_LOG in the reference, note the bundled-sqlite feature, and record in AGENTS.md which dependencies are deliberately absent and why. --- AGENTS.md | 10 ++++++++++ docs.md | 8 ++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8d265cc..2ad3eae 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,9 +22,19 @@ coarse, such as a whole NAR or narinfo per call. - Use `nix develop` for zig 0.16, cargo, zstd and pkg-config. - Zig 0.16 APIs changed a lot; check `zig env` for the std source rather than relying on older examples. +- System libraries: zstd and sqlite (pkg-config). - `HELIOS_ZIG_CPU` sets libhelios's `-Dcpu` (default `native`; the Nix package uses `baseline`). +## Dependencies + +Keep the dependency tree small; every crate is build time, audit surface and +supply-chain risk. Before adding one, check whether libhelios, std or a few +dozen lines cover it, and disable default features. Deliberately absent: +reqwest (and `url`/`idna`), aws-lc-rs, tracing, parking_lot, uuid, +hmac/sha2 (libhelios provides SHA-256, HMAC and randomness) and mimalloc +(measured: no gain here). + ## Performance Rules - Measure before and after; `bench/run.sh` and `bench/flake-lock.sh` diff --git a/docs.md b/docs.md index 82f2183..93989b7 100644 --- a/docs.md +++ b/docs.md @@ -32,8 +32,12 @@ Every flag can also be set through its environment variable. | `--gc-interval-hours` | `HELIOS_GC_INTERVAL_HOURS` | `6` | | `--audit-retention-days` | `HELIOS_AUDIT_RETENTION_DAYS` | `30` | -Secret files must hold at least 16 bytes. Logging follows `RUST_LOG` -(default `helios_server=info`). +Secret files must hold at least 16 bytes. `HELIOS_LOG` sets the log level +(`error`, `warn`, `info` or `debug`; default `info`). Under systemd, log lines +carry journald priorities. + +The server links the system SQLite; build with `--features bundled-sqlite` +to compile SQLite in instead. ## Reverse proxy and zero-copy downloads From 3bc251e2c4c09e76bafe31a61fcfcb598c57ad31 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 12:31:08 +0200 Subject: [PATCH 13/56] nix: add a nixos module for the server The flake now exports nixosModules.default, which runs helios-server on any NixOS machine as the services.helios systemd unit, and overlays.default for pkgs.helios. Secrets are file paths passed to the unit as systemd credentials, so they can be root-only and come from any secret manager. The unit runs as its own user under a strict sandbox: no capabilities, a read-only system, and write access only to its data directory, which defaults to the StateDirectory and can point anywhere else. When a domain is set, nginx fronts the server with ACME TLS and HTTP/2. NAR downloads are served by nginx itself: the server checks access and answers with X-Accel-Redirect to an internal location aliased to the data directory. A VM test in the flake checks deploys the module behind nginx, pushes a closure from a client, substitutes it back through Nix with signatures required, and checks the sandbox, NARs on disk under a custom data directory, restarts, and clean shutdown on SIGTERM. The package source is now limited to what the build reads, so module and documentation edits no longer rebuild it. --- flake.nix | 30 +++++- nix/module.nix | 244 +++++++++++++++++++++++++++++++++++++++++++++++++ nix/test.nix | 99 ++++++++++++++++++++ 3 files changed, 369 insertions(+), 4 deletions(-) create mode 100644 nix/module.nix create mode 100644 nix/test.nix diff --git a/flake.nix b/flake.nix index 8114e8e..374807a 100644 --- a/flake.nix +++ b/flake.nix @@ -7,18 +7,24 @@ outputs = { self, nixpkgs }: let - # NAR serialisation is implemented for Linux; macOS builds are untested. + # NAR serialisation is implemented for Linux only. systems = [ "x86_64-linux" "aarch64-linux" ]; forAllSystems = fn: nixpkgs.lib.genAttrs systems (system: fn nixpkgs.legacyPackages.${system}); helios' = pkgs: pkgs.rustPlatform.buildRustPackage { pname = "helios"; version = "0.1.0"; + # Only what the build reads, so docs and module edits do not rebuild it. src = pkgs.lib.cleanSourceWith { src = self; filter = path: _type: - let base = baseNameOf path; in - !(builtins.elem base [ "target" ".zig-cache" "zig-out" "bench" ]); + let + rel = pkgs.lib.removePrefix "${toString self}/" (toString path); + top = builtins.head (pkgs.lib.splitString "/" rel); + base = baseNameOf path; + in + builtins.elem top [ "Cargo.toml" "Cargo.lock" "core" "crates" "pkg" ] + && !(builtins.elem base [ "target" ".zig-cache" "zig-out" ]); }; cargoLock.lockFile = ./Cargo.lock; @@ -46,6 +52,17 @@ default = helios; }); + overlays.default = final: _prev: { helios = helios' final; }; + + # services.helios; the package defaults to this flake's build. + nixosModules.default = + { lib, pkgs, ... }: + { + imports = [ ./nix/module.nix ]; + services.helios.package = lib.mkDefault (helios' pkgs); + }; + nixosModules.helios = self.nixosModules.default; + devShells = forAllSystems (pkgs: { default = pkgs.mkShell { packages = [ @@ -64,7 +81,12 @@ }); checks = forAllSystems (pkgs: { - default = self.packages.${pkgs.stdenv.hostPlatform.system}.helios; + package = self.packages.${pkgs.stdenv.hostPlatform.system}.helios; + nixos = import ./nix/test.nix { + inherit pkgs; + module = ./nix/module.nix; + helios = self.packages.${pkgs.stdenv.hostPlatform.system}.helios; + }; }); }; } diff --git a/nix/module.nix b/nix/module.nix new file mode 100644 index 0000000..e7dbf24 --- /dev/null +++ b/nix/module.nix @@ -0,0 +1,244 @@ +# services.helios: the Helios binary cache server, optionally behind nginx +# with ACME TLS. NAR downloads go through nginx's sendfile via +# X-Accel-Redirect after the server has checked access. +{ + config, + lib, + pkgs, + ... +}: + +let + cfg = config.services.helios; + inherit (lib) + mkEnableOption + mkIf + mkOption + optional + optionalString + types + ; + + dataDir = cfg.dataDir; + defaultDataDir = "/var/lib/helios"; + secret = name: file: optional (file != null) "${name}:${file}"; + credential = flag: name: file: optionalString (file != null) " ${flag} %d/${name}"; +in +{ + options.services.helios = { + enable = mkEnableOption "the Helios Nix binary cache"; + + package = mkOption { + type = types.package; + description = "Package providing `helios-server`."; + }; + + dataDir = mkOption { + type = types.str; + default = defaultDataDir; + description = '' + Where the database and NARs live. The default is managed by systemd + (StateDirectory); another path, such as a mounted data disk, is + created with the right ownership and made writable in the sandbox. + ''; + }; + + listen = mkOption { + type = types.str; + default = "127.0.0.1:8080"; + description = "Address the server listens on. Keep it on loopback when nginx is enabled."; + }; + + signingKeyFile = mkOption { + type = types.nullOr types.path; + default = null; + description = '' + Nix secret key (`nix key generate-secret`) used to sign narinfo. Read by + systemd as a credential, so it may be root-only (for example an agenix + secret). Without it narinfo is unsigned. + ''; + }; + + jwtSecretFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "HMAC secret for API tokens (at least 16 bytes). Without it no tokens can be issued or used."; + }; + + adminSecretFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "Bearer secret for the admin API (at least 16 bytes). Without it the admin API is disabled."; + }; + + logLevel = mkOption { + type = types.enum [ "error" "warn" "info" "debug" ]; + default = "info"; + }; + + settings = { + narinfoCacheEntries = mkOption { + type = types.ints.positive; + default = 262144; + description = "Rendered narinfo bodies kept in memory."; + }; + maxUploadBytes = mkOption { + type = types.ints.positive; + default = 64 * 1024 * 1024 * 1024; + description = "Largest accepted compressed NAR."; + }; + gcIntervalHours = mkOption { + type = types.ints.positive; + default = 6; + }; + auditRetentionDays = mkOption { + type = types.ints.unsigned; + default = 30; + }; + }; + + domain = mkOption { + type = types.nullOr types.str; + default = null; + example = "cache.example.com"; + description = "Public host name. When set, nginx serves the cache on it."; + }; + + nginx = { + enable = mkOption { + type = types.bool; + default = cfg.domain != null; + defaultText = lib.literalExpression "config.services.helios.domain != null"; + description = "Put nginx in front of the server (TLS, HTTP/2, zero-copy NAR downloads)."; + }; + acme = mkOption { + type = types.bool; + default = true; + description = "Obtain a certificate for `domain` from Let's Encrypt. Requires `security.acme` terms and email."; + }; + }; + + openFirewall = mkOption { + type = types.bool; + default = cfg.nginx.enable; + defaultText = lib.literalExpression "config.services.helios.nginx.enable"; + description = "Open TCP 80 and 443."; + }; + }; + + config = mkIf cfg.enable { + assertions = [ + { + assertion = cfg.nginx.enable -> cfg.domain != null; + message = "services.helios.nginx.enable requires services.helios.domain"; + } + ]; + + users.users.helios = { + isSystemUser = true; + group = "helios"; + home = dataDir; + }; + users.groups.helios = { }; + + systemd.services.helios = { + description = "Helios Nix binary cache"; + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + environment = { + HELIOS_LOG = cfg.logLevel; + HELIOS_LISTEN = cfg.listen; + HELIOS_DATA_DIR = dataDir; + HELIOS_NARINFO_CACHE_ENTRIES = toString cfg.settings.narinfoCacheEntries; + HELIOS_MAX_UPLOAD_BYTES = toString cfg.settings.maxUploadBytes; + HELIOS_GC_INTERVAL_HOURS = toString cfg.settings.gcIntervalHours; + HELIOS_AUDIT_RETENTION_DAYS = toString cfg.settings.auditRetentionDays; + } + // lib.optionalAttrs cfg.nginx.enable { + HELIOS_ACCEL_REDIRECT = "/_nar"; + HELIOS_TRUST_PROXY = "true"; + }; + + serviceConfig = { + ExecStart = + "${cfg.package}/bin/helios-server" + + credential "--signing-key-file" "signing-key" cfg.signingKeyFile + + credential "--jwt-secret-file" "jwt-secret" cfg.jwtSecretFile + + credential "--admin-secret-file" "admin-secret" cfg.adminSecretFile; + LoadCredential = + secret "signing-key" cfg.signingKeyFile + ++ secret "jwt-secret" cfg.jwtSecretFile + ++ secret "admin-secret" cfg.adminSecretFile; + + User = "helios"; + Group = "helios"; + # nginx (in group helios) reads NARs for X-Accel-Redirect. + StateDirectory = mkIf (dataDir == defaultDataDir) "helios"; + StateDirectoryMode = "0750"; + ReadWritePaths = mkIf (dataDir != defaultDataDir) [ dataDir ]; + UMask = "0027"; + Restart = "on-failure"; + RestartSec = 2; + LimitNOFILE = 65536; + + # Sandboxing: the server only needs its state directory and the network. + NoNewPrivileges = true; + CapabilityBoundingSet = ""; + AmbientCapabilities = ""; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectKernelLogs = true; + ProtectControlGroups = true; + ProtectClock = true; + ProtectHostname = true; + ProtectProc = "invisible"; + ProcSubset = "pid"; + RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + LockPersonality = true; + MemoryDenyWriteExecute = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ "@system-service" "~@privileged" ]; + }; + }; + + systemd.tmpfiles.rules = mkIf (dataDir != defaultDataDir) [ "d '${dataDir}' 0750 helios helios -" ]; + + users.users.nginx.extraGroups = mkIf cfg.nginx.enable [ "helios" ]; + + services.nginx = mkIf cfg.nginx.enable { + enable = true; + recommendedProxySettings = true; + recommendedTlsSettings = true; + recommendedOptimisation = true; + virtualHosts.${cfg.domain} = { + enableACME = cfg.nginx.acme; + forceSSL = cfg.nginx.acme; + locations."/" = { + proxyPass = "http://${cfg.listen}"; + extraConfig = '' + # Stream uploads straight to the server instead of spooling them. + proxy_request_buffering off; + client_max_body_size 0; + proxy_read_timeout 1h; + proxy_send_timeout 1h; + ''; + }; + locations."/_nar/" = { + alias = "${dataDir}/nar/"; + extraConfig = '' + internal; + ''; + }; + }; + }; + + networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ 80 443 ]; + }; +} diff --git a/nix/test.nix b/nix/test.nix new file mode 100644 index 0000000..b9c83d4 --- /dev/null +++ b/nix/test.nix @@ -0,0 +1,99 @@ +# NixOS VM test for services.helios: a server behind nginx and a client +# that pushes a closure with the CLI, then substitutes it back through Nix +# with signatures required. +{ pkgs, module, helios }: + +pkgs.testers.runNixOSTest { + name = "helios"; + + nodes.server = + { lib, ... }: + { + imports = [ module ]; + services.helios = { + enable = true; + package = helios; + domain = "server"; + # A custom location exercises the tmpfiles and sandbox path handling. + dataDir = "/srv/helios"; + nginx.acme = false; + signingKeyFile = "/etc/helios/signing-key"; + jwtSecretFile = "/etc/helios/jwt-secret"; + adminSecretFile = "/etc/helios/admin-secret"; + }; + # Started by the test once the secrets exist. + systemd.services.helios.wantedBy = lib.mkForce [ ]; + nix.settings.experimental-features = [ "nix-command" ]; + }; + + nodes.client = { + environment.systemPackages = [ + helios + pkgs.hello + pkgs.jq + ]; + nix.settings.experimental-features = [ "nix-command" ]; + virtualisation.memorySize = 2048; + }; + + testScript = '' + import json + + start_all() + + with subtest("secrets and startup"): + server.succeed( + "install -d -m 0700 /etc/helios", + "nix key generate-secret --key-name test-1 > /etc/helios/signing-key", + "head -c 32 /dev/urandom | base64 > /etc/helios/jwt-secret", + "head -c 32 /dev/urandom | base64 > /etc/helios/admin-secret", + "chmod 0400 /etc/helios/*", + "systemctl start helios", + ) + server.wait_for_unit("helios.service") + server.wait_for_unit("nginx.service") + server.wait_for_open_port(8080) + server.wait_for_open_port(80) + client.wait_for_unit("multi-user.target") + pubkey = server.succeed("nix key convert-secret-to-public < /etc/helios/signing-key").strip() + admin = server.succeed("cat /etc/helios/admin-secret").strip() + + with subtest("sandboxing"): + # The unit reads its secrets as credentials, not from /etc. + server.fail("sudo -u helios cat /etc/helios/signing-key") + props = server.succeed("systemctl show helios -p NoNewPrivileges -p ProtectSystem -p User") + assert "NoNewPrivileges=yes" in props and "ProtectSystem=strict" in props and "User=helios" in props, props + + with subtest("admin and push"): + client.succeed(f"helios login admin http://server {admin}") + client.succeed("helios cache create main") + token = json.loads(client.succeed("helios token create ci --caches main --perms push,pull 2>/dev/null"))["token"] + client.succeed(f"helios login ci http://server {token}") + client.succeed("helios push main --closure ${pkgs.hello}") + out = client.succeed("helios push main --closure ${pkgs.hello} 2>&1") + assert "already in 'main'" in out, out + + with subtest("substitute with signatures required"): + client.succeed( + f"NIX_CONFIG='trusted-public-keys = {pubkey}\nrequire-sigs = true' " + "nix copy --from http://server/main --to /tmp/store ${pkgs.hello}" + ) + client.succeed(f"nix store verify --store /tmp/store --trusted-public-keys '{pubkey}' -r ${pkgs.hello}") + + with subtest("nginx serves NARs from disk"): + url = client.succeed("curl -sf http://server/main/$(basename ${pkgs.hello} | cut -c1-32).narinfo | sed -n 's/^URL: //p'").strip() + client.succeed(f"curl -sf -o /dev/null http://server/main/{url}") + # The internal location is not reachable directly. + client.fail("curl -sf -o /dev/null http://server/_nar/") + server.succeed("find /srv/helios/nar -name '*.nar.zst' | grep -q .") + server.succeed("stat -c %U:%G:%a /srv/helios | grep -qx helios:helios:750") + + with subtest("state survives restarts; SIGTERM shuts down cleanly"): + server.succeed("systemctl restart helios") + server.wait_for_open_port(8080) + client.succeed("curl -sf http://server/main/$(basename ${pkgs.hello} | cut -c1-32).narinfo") + server.succeed("systemctl stop helios") + server.succeed("journalctl -u helios | grep -q 'shutting down'") + server.fail("systemctl is-failed helios") + ''; +} From a5a09849ee7f0bd733086172f9a047d54f844051 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 12:37:29 +0200 Subject: [PATCH 14/56] docs: document the nixos module Add a NixOS section to the reference with a flake and configuration example and the module options, point the README at it, and list the VM test among the checks in AGENTS.md. --- AGENTS.md | 3 +++ README.md | 6 ++++-- docs.md | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2ad3eae..b9eb18e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,6 +10,7 @@ A self-hosted Nix binary cache: Rust server and CLI over a Zig core. by `crates/helios-core/build.rs`. `core/include/helios.h` is the contract. - `crates/helios-core`: safe Rust bindings to libhelios. - `crates/helios-server`, `crates/helios-cli`: the binaries. +- `nix/`: the NixOS module (`services.helios`) and its VM test. - `bench/`: comparisons against Cachix's libraries and Nix C++. Not part of the Cargo workspace. @@ -48,6 +49,8 @@ hmac/sha2 (libhelios provides SHA-256, HMAC and randomness) and mimalloc - `cargo test --release` - `./scripts/test-zig.sh` +- `nix build .#checks.x86_64-linux.nixos` after changing `nix/`: a VM test of + the NixOS module. - `./scripts/e2e.sh` before finishing changes to the server, the CLI or wire formats; it substitutes a real closure through Nix with signatures required. diff --git a/README.md b/README.md index 0d62a89..3113f4a 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,10 @@ nix.settings = { }; ``` -See [docs.md](docs.md) for configuration, private caches, the HTTP API -and deployment behind a reverse proxy. +On NixOS, use the flake's `nixosModules.default` (`services.helios`). + +See [docs.md](docs.md) for the NixOS module, configuration, private caches, +the HTTP API and deployment behind a reverse proxy. ## License diff --git a/docs.md b/docs.md index 93989b7..31d01f8 100644 --- a/docs.md +++ b/docs.md @@ -39,9 +39,63 @@ carry journald priorities. The server links the system SQLite; build with `--features bundled-sqlite` to compile SQLite in instead. +## NixOS + +The flake exports `nixosModules.default`. It runs the server as a hardened +systemd service and, when `domain` is set, puts nginx in front of it with +TLS from Let's Encrypt and zero-copy NAR downloads: + +```nix +# flake.nix +{ + inputs.helios.url = "github:Polymath-AS/helios"; + outputs = { nixpkgs, helios, ... }: { + nixosConfigurations.cache = nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + modules = [ helios.nixosModules.default ./configuration.nix ]; + }; + }; +} +``` + +```nix +# configuration.nix +{ + services.helios = { + enable = true; + domain = "cache.example.com"; + signingKeyFile = "/run/secrets/helios-signing-key"; + jwtSecretFile = "/run/secrets/helios-jwt-secret"; + adminSecretFile = "/run/secrets/helios-admin-secret"; + }; + security.acme = { + acceptTerms = true; + defaults.email = "ops@example.com"; + }; +} +``` + +Secret files are handed to the service as systemd credentials, so they can +be root-only and come from any secret manager (agenix, sops-nix, or files +placed by hand). Without `domain`, the module runs only the server on +`listen`, for use behind another proxy or on a private network. + +| Option | Default | | +|--------|---------|--| +| `listen` | `127.0.0.1:8080` | keep on loopback behind nginx | +| `dataDir` | `/var/lib/helios` | any other path, such as a mounted disk, is created and allowed in the sandbox | +| `domain` | `null` | enables nginx for this host name | +| `nginx.acme` | `true` | set `false` for plain HTTP or your own certificates | +| `openFirewall` | on with nginx | opens 80 and 443 | +| `logLevel` | `info` | | +| `settings.*` | | `narinfoCacheEntries`, `maxUploadBytes`, `gcIntervalHours`, `auditRetentionDays` | + +`overlays.default` adds `pkgs.helios`. `nix flake check` runs a VM test that +pushes a closure through the module and substitutes it back. + ## Reverse proxy and zero-copy downloads -Terminate TLS in a reverse proxy. With `--accel-redirect /_nar`, the server +Outside NixOS, terminate TLS in a reverse proxy. With `--accel-redirect /_nar`, the server answers NAR requests with an `X-Accel-Redirect` header after the access check, and nginx serves the file with `sendfile`: From 5ea7885daf48e3b81e4407c1ebba6fc50ce82ebd Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 13:03:23 +0200 Subject: [PATCH 15/56] server: add a maintenance api on a unix socket helios-daemon needs to evict paths, scrub blobs and maintain the database, but the server's in-memory path index and narinfo cache must stay in step with SQLite, so the server remains the only process that changes it. With --admin-socket it serves a small internal API on a Unix socket, created 0660 so that directory and group permissions decide who may use it: stats, least-recently-used paths, eviction, blob listing, quarantine of a bad blob, GC, WAL checkpoints, optimize, and rotated online backups via VACUUM INTO. Eviction deletes the rows under the writer lock, drops the paths from the index and narinfo cache, and only then collects the blobs no path references any more, so nothing is served for a NAR whose file is gone. Quarantine unpublishes every path a blob backs and moves its file to quarantine/ for inspection. Least-recently-used order comes from a new paths.accessed_at column. Narinfo hits are recorded in sharded in-memory sets and flushed once a minute, before every LRU query and at shutdown; request counters are striped across cache lines. Neither shows up in narinfo throughput, which stays at about 1.2M requests per second. The upload grace period that protects in-flight pushes from GC is now configurable with --upload-grace-seconds. --- crates/helios-server/src/config.rs | 12 ++ crates/helios-server/src/db.rs | 9 +- crates/helios-server/src/gc.rs | 69 ++++-- crates/helios-server/src/internal.rs | 310 +++++++++++++++++++++++++++ crates/helios-server/src/main.rs | 34 ++- crates/helios-server/src/push.rs | 3 + crates/helios-server/src/read.rs | 4 + crates/helios-server/src/state.rs | 4 + crates/helios-server/src/stats.rs | 116 ++++++++++ 9 files changed, 544 insertions(+), 17 deletions(-) create mode 100644 crates/helios-server/src/internal.rs create mode 100644 crates/helios-server/src/stats.rs diff --git a/crates/helios-server/src/config.rs b/crates/helios-server/src/config.rs index e8d8905..b5adb0b 100644 --- a/crates/helios-server/src/config.rs +++ b/crates/helios-server/src/config.rs @@ -46,6 +46,11 @@ pub struct Args { #[arg(long, env = "HELIOS_MAX_UPLOAD_BYTES", default_value_t = 64 << 30)] pub max_upload_bytes: u64, + /// Seconds an uploaded NAR is kept before it must be published. Protects + /// pushes in flight from GC and eviction. + #[arg(long, env = "HELIOS_UPLOAD_GRACE_SECONDS", default_value_t = 3600)] + pub upload_grace_seconds: u64, + /// Hours between garbage collection runs. #[arg(long, env = "HELIOS_GC_INTERVAL_HOURS", default_value_t = 6)] pub gc_interval_hours: u64, @@ -54,6 +59,11 @@ pub struct Args { #[arg(long, env = "HELIOS_AUDIT_RETENTION_DAYS", default_value_t = 30)] pub audit_retention_days: u64, + /// Unix socket for the maintenance API used by helios-daemon. Access is + /// controlled by the socket's permissions (created 0660). + #[arg(long, env = "HELIOS_ADMIN_SOCKET")] + pub admin_socket: Option, + /// Print the public key for `trusted-public-keys` and exit. #[arg(long)] pub print_public_key: bool, @@ -68,6 +78,7 @@ pub struct Config { pub trust_proxy: bool, pub narinfo_cache_entries: usize, pub max_upload_bytes: u64, + pub upload_grace: Duration, pub gc_interval: Duration, pub audit_retention: Duration, } @@ -90,6 +101,7 @@ impl Config { trust_proxy: args.trust_proxy, narinfo_cache_entries: args.narinfo_cache_entries.max(16), max_upload_bytes: args.max_upload_bytes, + upload_grace: Duration::from_secs(args.upload_grace_seconds), gc_interval: Duration::from_secs(args.gc_interval_hours.max(1) * 3600), audit_retention: Duration::from_secs(args.audit_retention_days * 86400), }) diff --git a/crates/helios-server/src/db.rs b/crates/helios-server/src/db.rs index ae5d2e8..bdb9006 100644 --- a/crates/helios-server/src/db.rs +++ b/crates/helios-server/src/db.rs @@ -58,7 +58,14 @@ CREATE TABLE audit_log ( status INTEGER NOT NULL ); CREATE INDEX audit_log_ts ON audit_log (ts); -"#]; +"#, + // Last narinfo hit per path, for least-recently-used eviction. + r#" +ALTER TABLE paths ADD COLUMN accessed_at INTEGER NOT NULL DEFAULT 0; +UPDATE paths SET accessed_at = created_at; +CREATE INDEX paths_accessed ON paths (accessed_at); +"#, +]; const PRAGMAS: &str = " PRAGMA busy_timeout = 5000; diff --git a/crates/helios-server/src/gc.rs b/crates/helios-server/src/gc.rs index fec1cf1..06fa694 100644 --- a/crates/helios-server/src/gc.rs +++ b/crates/helios-server/src/gc.rs @@ -1,5 +1,6 @@ //! Periodic garbage collection: stale temp uploads, unreferenced blobs past -//! a grace period, expired tokens, and old audit log entries. +//! a grace period, expired tokens, and old audit log entries. Also flushes +//! recorded narinfo hits to `paths.accessed_at` for LRU eviction. use std::time::{Duration, SystemTime}; @@ -9,13 +10,12 @@ use rusqlite::params; use crate::state::Locked; use crate::state::Shared; -/// Blobs uploaded but not yet published are kept this long, so a push that -/// uploads first and publishes afterwards is never raced by GC. -const BLOB_GRACE: Duration = Duration::from_secs(3600); const TMP_MAX_AGE: Duration = Duration::from_secs(3600); const BLOB_BATCH: usize = 1000; +const ACCESS_FLUSH: Duration = Duration::from_secs(60); -#[derive(Debug, Default)] +#[derive(Debug, Default, serde::Serialize)] +#[serde(rename_all = "camelCase")] pub struct GcStats { pub tmp_files: usize, pub blobs: usize, @@ -55,7 +55,22 @@ pub fn collect(st: &Shared) -> anyhow::Result { } } - let cutoff = now - BLOB_GRACE.as_secs() as i64; + (stats.blobs, stats.bytes) = collect_blobs(st)?; + + stats.tokens = st.db.write(|conn| conn.execute("DELETE FROM tokens WHERE expires_at < ?1", [now]))?; + st.tokens.wr().retain(|_, t| t.expires_at >= now); + + let audit_cutoff = now - st.cfg.audit_retention.as_secs() as i64; + stats.audit_rows = st.db.write(|conn| conn.execute("DELETE FROM audit_log WHERE ts < ?1", [audit_cutoff]))?; + Ok(stats) +} + +/// Deletes blobs no path references, once past the upload grace period, so +/// a push that uploads first and publishes afterwards is never raced. +/// Returns (blobs, bytes) removed. +pub fn collect_blobs(st: &Shared) -> anyhow::Result<(usize, u64)> { + let cutoff = crate::db::now() - st.cfg.upload_grace.as_secs() as i64; + let (mut total, mut freed) = (0usize, 0u64); loop { // Row delete and unlink share the writer lock with upload's // rename+upsert, so a re-uploaded blob can never lose its file. @@ -81,23 +96,49 @@ pub fn collect(st: &Shared) -> anyhow::Result { continue; }; match std::fs::remove_file(st.nar_path(&hash, compression)) { - Ok(()) => stats.bytes += size, + Ok(()) => freed += size, Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} Err(e) => crate::log::warning!("removing blob file: {e}"), } } Ok(victims.len()) })?; - stats.blobs += n; + total += n; if n < BLOB_BATCH { - break; + return Ok((total, freed)); } } +} - stats.tokens = st.db.write(|conn| conn.execute("DELETE FROM tokens WHERE expires_at < ?1", [now]))?; - st.tokens.wr().retain(|_, t| t.expires_at >= now); +/// Writes recorded narinfo hits to `paths.accessed_at` in one transaction. +pub fn flush_access(st: &Shared) -> rusqlite::Result { + let keys = st.access.drain(); + if keys.is_empty() { + return Ok(0); + } + let now = crate::db::now(); + st.db.write(|conn| { + let tx = conn.transaction()?; + { + let mut stmt = tx.prepare_cached("UPDATE paths SET accessed_at = ?1 WHERE cache_id = ?2 AND hash = ?3")?; + for k in &keys { + stmt.execute(params![now, k.cache, &k.hash[..]])?; + } + } + tx.commit()?; + Ok(keys.len()) + }) +} - let audit_cutoff = now - st.cfg.audit_retention.as_secs() as i64; - stats.audit_rows = st.db.write(|conn| conn.execute("DELETE FROM audit_log WHERE ts < ?1", [audit_cutoff]))?; - Ok(stats) +pub async fn run_access_flush(st: Shared) { + let mut interval = tokio::time::interval(ACCESS_FLUSH); + loop { + interval.tick().await; + let st2 = st.clone(); + match tokio::task::spawn_blocking(move || flush_access(&st2)).await { + Ok(Ok(_)) => {} + Ok(Err(e)) => crate::log::error!("flushing access times: {e}"), + Err(e) => crate::log::error!("access flush panicked: {e}"), + } + } } diff --git a/crates/helios-server/src/internal.rs b/crates/helios-server/src/internal.rs new file mode 100644 index 0000000..113efe6 --- /dev/null +++ b/crates/helios-server/src/internal.rs @@ -0,0 +1,310 @@ +//! Maintenance API for helios-daemon, served only on a Unix socket +//! (`--admin-socket`) whose file permissions are the access control. +//! +//! The server stays the only process that changes the database, so its +//! in-memory index and narinfo cache always agree with it; the daemon +//! decides what to do and asks. + +use std::path::PathBuf; + +use axum::extract::State; +use axum::http::{StatusCode, Uri}; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use helios_core::Compression; +use rusqlite::{OptionalExtension, params}; +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::error::{ApiError, ApiResult}; +use crate::state::{Locked, PathKey, Shared}; + +pub fn router(st: Shared) -> Router { + Router::new() + .route("/v1/stats", get(stats)) + .route("/v1/lru", get(lru)) + .route("/v1/evict", post(evict)) + .route("/v1/blobs", get(blobs)) + .route("/v1/quarantine", post(quarantine)) + .route("/v1/gc", post(gc)) + .route("/v1/db/checkpoint", post(checkpoint)) + .route("/v1/db/optimize", post(optimize)) + .route("/v1/db/backup", post(backup)) + .with_state(st) +} + +async fn blocking(f: impl FnOnce() -> ApiResult + Send + 'static) -> ApiResult { + tokio::task::spawn_blocking(f).await? +} + +fn internal(e: impl std::fmt::Display) -> ApiError { + ApiError::internal(e) +} + +/// A numeric query parameter (the public API has no query strings to speak of). +fn param(uri: &Uri, name: &str) -> Option { + uri.query()?.split('&').find_map(|kv| kv.strip_prefix(name)?.strip_prefix('=')?.parse().ok()) +} + +async fn stats(State(st): State) -> ApiResult> { + let db_st = st.clone(); + let (caches, blob_count, blob_bytes) = blocking(move || { + db_st.db.read(|conn| { + let mut stmt = conn.prepare_cached( + "SELECT c.name, c.is_public, count(p.id) FROM caches c LEFT JOIN paths p ON p.cache_id = c.id GROUP BY c.id ORDER BY c.name", + )?; + let caches: Vec = stmt + .query_map([], |r| Ok(json!({ "name": r.get::<_, String>(0)?, "public": r.get::<_, bool>(1)?, "paths": r.get::<_, u64>(2)? })))? + .collect::>()?; + let (count, bytes): (u64, u64) = conn.query_row("SELECT count(*), coalesce(sum(file_size), 0) FROM blobs", [], |r| Ok((r.get(0)?, r.get(1)?)))?; + Ok((caches, count, bytes)) + }) + .map_err(internal) + }) + .await?; + let c = &st.counters; + Ok(Json(json!({ + "caches": caches, + "blobs": { "count": blob_count, "bytes": blob_bytes }, + "indexedPaths": st.index.rd().len(), + "narinfoCacheEntries": st.narinfo.len(), + "requests": { + "narinfoHits": c.narinfo_hits.get(), + "narinfoMisses": c.narinfo_misses.get(), + "nars": c.nar_requests.get(), + "uploads": c.uploads.get(), + "uploadBytes": c.upload_bytes.get(), + "publishedPaths": c.published_paths.get(), + }, + }))) +} + +/// Least recently accessed paths first, with the size of their NAR. +async fn lru(State(st): State, uri: Uri) -> ApiResult> { + let limit = param(&uri, "limit").unwrap_or(500).clamp(1, 10_000); + let paths = blocking(move || { + // Pending hits must count before anything is chosen for eviction. + crate::gc::flush_access(&st).map_err(internal)?; + st.db + .read(|conn| { + let mut stmt = conn.prepare_cached( + "SELECT p.cache_id, p.hash, p.store_path, p.accessed_at, b.file_size + FROM paths p JOIN blobs b ON b.id = p.blob_id ORDER BY p.accessed_at, p.id LIMIT ?1", + )?; + stmt.query_map([limit], |r| { + let hash: Vec = r.get(1)?; + Ok(json!({ + "cache": r.get::<_, u32>(0)?, + "hash": helios_core::nix32_encode(&hash), + "storePath": r.get::<_, String>(2)?, + "accessedAt": r.get::<_, i64>(3)?, + "narBytes": r.get::<_, u64>(4)?, + })) + })? + .collect::>>() + }) + .map_err(internal) + }) + .await?; + Ok(Json(json!({ "paths": paths }))) +} + +#[derive(Deserialize)] +struct EvictPath { + cache: u32, + hash: String, +} + +#[derive(Deserialize)] +struct EvictReq { + paths: Vec, +} + +fn forget(st: &Shared, keys: &[PathKey]) { + let mut index = st.index.wr(); + for k in keys { + index.remove(k); + st.narinfo.remove(k); + } +} + +/// Deletes paths, then any blobs that no longer back a path. +async fn evict(State(st): State, Json(req): Json) -> ApiResult> { + let keys: Vec = req + .paths + .iter() + .map(|p| helios_core::nix32_decode::<20>(&p.hash).map(|hash| PathKey { cache: p.cache, hash })) + .collect::>() + .ok_or_else(|| ApiError::bad_request("invalid path hash"))?; + blocking(move || { + let evicted = st.db.write(|conn| -> rusqlite::Result { + let tx = conn.transaction()?; + let mut n = 0; + { + let mut del = tx.prepare_cached("DELETE FROM paths WHERE cache_id = ?1 AND hash = ?2")?; + for k in &keys { + n += del.execute(params![k.cache, &k.hash[..]])?; + } + } + tx.commit()?; + Ok(n) + }); + let evicted = evicted.map_err(internal)?; + // Stop serving before the files go. + forget(&st, &keys); + let (blobs, bytes) = crate::gc::collect_blobs(&st).map_err(internal)?; + crate::log::info!("evicted {evicted} paths, freeing {blobs} blobs ({bytes} bytes)"); + Ok(Json(json!({ "evicted": evicted, "freedBlobs": blobs, "freedBytes": bytes }))) + }) + .await +} + +/// Blobs in id order, for the integrity scrub. +async fn blobs(State(st): State, uri: Uri) -> ApiResult> { + let after: i64 = param(&uri, "after").unwrap_or(0); + let limit = param(&uri, "limit").unwrap_or(1000).clamp(1, 10_000); + let data_dir = st.cfg.data_dir.clone(); + let blobs = blocking(move || { + st.db + .read(|conn| { + let mut stmt = conn.prepare_cached( + "SELECT id, file_hash, compression, file_size, nar_hash, nar_size FROM blobs WHERE id > ?1 ORDER BY id LIMIT ?2", + )?; + stmt.query_map(params![after, limit], |r| { + let file_hash: Vec = r.get(1)?; + let nar_hash: Vec = r.get(4)?; + let compression: String = r.get(2)?; + let path = <[u8; 32]>::try_from(file_hash.as_slice()) + .ok() + .zip(Compression::parse(&compression)) + .map(|(h, c)| st.nar_path(&h, c).strip_prefix(&data_dir).map(PathBuf::from).unwrap_or_default()); + Ok(json!({ + "id": r.get::<_, i64>(0)?, + "fileHash": helios_core::nix32_encode(&file_hash), + "compression": compression, + "fileSize": r.get::<_, u64>(3)?, + "narHash": helios_core::nix32_encode(&nar_hash), + "narSize": r.get::<_, u64>(5)?, + "path": path, + })) + })? + .collect::>>() + }) + .map_err(internal) + }) + .await?; + Ok(Json(json!({ "blobs": blobs }))) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct QuarantineReq { + file_hash: String, + reason: String, +} + +/// Takes a corrupt or missing blob out of service: every path it backs is +/// unpublished, and its file is moved aside for inspection. +async fn quarantine(State(st): State, Json(req): Json) -> ApiResult> { + let file_hash = helios_core::nix32_decode::<32>(&req.file_hash).ok_or_else(|| ApiError::bad_request("invalid file hash"))?; + blocking(move || { + let found = st.db.write(|conn| -> rusqlite::Result, String)>> { + let tx = conn.transaction()?; + let Some((id, compression)) = tx + .query_row("SELECT id, compression FROM blobs WHERE file_hash = ?1", [&file_hash[..]], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?))) + .optional()? + else { + return Ok(None); + }; + let keys: Vec = { + let mut stmt = tx.prepare_cached("SELECT cache_id, hash FROM paths WHERE blob_id = ?1")?; + stmt.query_map([id], |r| Ok((r.get::<_, u32>(0)?, r.get::<_, Vec>(1)?)))? + .filter_map(|row| row.ok().and_then(|(cache, h)| <[u8; 20]>::try_from(h.as_slice()).ok().map(|hash| PathKey { cache, hash }))) + .collect() + }; + tx.execute("DELETE FROM paths WHERE blob_id = ?1", [id])?; + tx.execute("DELETE FROM blobs WHERE id = ?1", [id])?; + tx.commit()?; + Ok(Some((keys, compression))) + }); + let Some((keys, compression)) = found.map_err(internal)? else { + return Err(ApiError::new(StatusCode::NOT_FOUND, "blob not found")); + }; + forget(&st, &keys); + if let Some(compression) = Compression::parse(&compression) { + let src = st.nar_path(&file_hash, compression); + let dir = st.cfg.data_dir.join("quarantine"); + let moved = std::fs::create_dir_all(&dir).and_then(|()| std::fs::rename(&src, dir.join(src.file_name().unwrap_or_default()))); + if let Err(e) = moved + && e.kind() != std::io::ErrorKind::NotFound + { + crate::log::warning!("moving quarantined blob: {e}"); + } + } + crate::log::warning!("quarantined blob {}: {} ({} paths unpublished)", req.file_hash, req.reason, keys.len()); + Ok(Json(json!({ "pathsRemoved": keys.len() }))) + }) + .await +} + +async fn gc(State(st): State) -> ApiResult> { + blocking(move || { + let stats = crate::gc::collect(&st).map_err(internal)?; + Ok(Json(serde_json::to_value(stats).map_err(internal)?)) + }) + .await +} + +async fn checkpoint(State(st): State) -> ApiResult> { + blocking(move || { + let (busy, log, done) = st + .db + .write(|conn| conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?, r.get::<_, i64>(2)?)))) + .map_err(internal)?; + Ok(Json(json!({ "busy": busy != 0, "walPages": log, "checkpointedPages": done }))) + }) + .await +} + +async fn optimize(State(st): State) -> ApiResult> { + blocking(move || { + st.db.write(|conn| conn.execute_batch("PRAGMA optimize;")).map_err(internal)?; + Ok(Json(json!({ "optimized": true }))) + }) + .await +} + +#[derive(Deserialize)] +struct BackupReq { + /// Backups to retain, newest first. + keep: usize, +} + +/// A consistent online copy of the database under `/backups`, via +/// VACUUM INTO on a read connection so writers are not blocked. +async fn backup(State(st): State, Json(req): Json) -> ApiResult> { + blocking(move || { + let dir = st.cfg.data_dir.join("backups"); + std::fs::create_dir_all(&dir).map_err(internal)?; + let dest = dir.join(format!("helios-{}.db", crate::db::now())); + let dest_str = dest.to_str().ok_or_else(|| internal("backup path is not UTF-8"))?.to_owned(); + st.db.read(|conn| conn.execute("VACUUM INTO ?1", [&dest_str])).map_err(internal)?; + let bytes = std::fs::metadata(&dest).map(|m| m.len()).unwrap_or(0); + + let mut existing: Vec = std::fs::read_dir(&dir) + .map_err(internal)? + .flatten() + .map(|e| e.path()) + .filter(|p| p.file_name().and_then(|n| n.to_str()).is_some_and(|n| n.starts_with("helios-") && n.ends_with(".db"))) + .collect(); + existing.sort(); + let mut removed = 0; + while existing.len() > req.keep.max(1) { + if std::fs::remove_file(existing.remove(0)).is_ok() { + removed += 1; + } + } + Ok(Json(json!({ "path": dest, "bytes": bytes, "removed": removed }))) + }) + .await +} diff --git a/crates/helios-server/src/main.rs b/crates/helios-server/src/main.rs index 677d2d0..ef6cf20 100644 --- a/crates/helios-server/src/main.rs +++ b/crates/helios-server/src/main.rs @@ -5,10 +5,12 @@ mod config; mod db; mod error; mod gc; +mod internal; mod log; mod push; mod read; mod state; +mod stats; use std::net::SocketAddr; use std::sync::Arc; @@ -71,6 +73,7 @@ pub async fn build_state(args: &Args) -> anyhow::Result { tokio::spawn(audit::run(db.clone(), audit_rx)); let st = Arc::new(AppState::load(cfg, db, signer, audit_tx)?); tokio::spawn(gc::run_forever(st.clone())); + tokio::spawn(gc::run_access_flush(st.clone())); Ok(st) } @@ -86,11 +89,38 @@ fn main() -> anyhow::Result<()> { tokio::runtime::Builder::new_multi_thread().enable_all().build()?.block_on(async move { let st = build_state(&args).await?; + let (stop_tx, stop_rx) = tokio::sync::watch::channel(false); + tokio::spawn(async move { + shutdown_signal().await; + let _ = stop_tx.send(true); + }); + let stopped = |mut rx: tokio::sync::watch::Receiver| async move { + let _ = rx.wait_for(|stop| *stop).await; + }; + + let admin = match &args.admin_socket { + Some(path) => { + let _ = std::fs::remove_file(path); + let listener = tokio::net::UnixListener::bind(path)?; + // Group access only: the socket directory decides who that is. + std::fs::set_permissions(path, std::os::unix::fs::PermissionsExt::from_mode(0o660))?; + crate::log::info!("maintenance API on {}", path.display()); + let serve = axum::serve(listener, internal::router(st.clone())).with_graceful_shutdown(stopped(stop_rx.clone())); + Some(tokio::spawn(async move { serve.await })) + } + None => None, + }; + let listener = tokio::net::TcpListener::bind(args.listen).await?; crate::log::info!("listening on {}", args.listen); - axum::serve(listener, router(st).into_make_service_with_connect_info::()) - .with_graceful_shutdown(shutdown_signal()) + axum::serve(listener, router(st.clone()).into_make_service_with_connect_info::()) + .with_graceful_shutdown(stopped(stop_rx)) .await?; + if let Some(admin) = admin { + admin.await??; + } + // Keep the hits recorded since the last flush. + tokio::task::spawn_blocking(move || gc::flush_access(&st)).await??; Ok(()) }) } diff --git a/crates/helios-server/src/push.rs b/crates/helios-server/src/push.rs index 22df909..2aa4157 100644 --- a/crates/helios-server/src/push.rs +++ b/crates/helios-server/src/push.rs @@ -223,6 +223,8 @@ pub async fn upload( nar_hash: format!("sha256:{}", helios_core::nix32_encode(&digest.nar_hash)), nar_size: digest.nar_size, }; + st.counters.uploads.inc(); + st.counters.upload_bytes.add(resp.file_size); audit.log(&st, &identity, "nar.upload", Some(&name), StatusCode::CREATED, json!({ "fileHash": resp.file_hash, "size": resp.file_size })); Ok((StatusCode::CREATED, Json(resp))) } @@ -365,6 +367,7 @@ pub async fn publish( } let published = rendered_keys.len(); + st.counters.published_paths.add(published as u64); { let mut index = st.index.wr(); for key in rendered_keys { diff --git a/crates/helios-server/src/read.rs b/crates/helios-server/src/read.rs index 92a2ac0..cc49cfd 100644 --- a/crates/helios-server/src/read.rs +++ b/crates/helios-server/src/read.rs @@ -77,8 +77,11 @@ async fn serve(st: Shared, method: Method, uri: Uri, headers: HeaderMap) -> ApiR }; let key = PathKey { cache: cache.id, hash }; if !st.index.rd().contains(&key) { + st.counters.narinfo_misses.inc(); return Ok(not_found(if cache.public { NEGATIVE } else { PRIVATE })); } + st.counters.narinfo_hits.inc(); + st.access.touch(key); let body = match st.narinfo.get(&key) { Some(body) => body, None => { @@ -103,6 +106,7 @@ async fn serve(st: Shared, method: Method, uri: Uri, headers: HeaderMap) -> ApiR .into_response()) } Route::Nar(hash, compression) => { + st.counters.nar_requests.inc(); let Some(file_hash) = helios_core::nix32_decode::<32>(hash) else { return Ok(not_found(NEGATIVE)); }; diff --git a/crates/helios-server/src/state.rs b/crates/helios-server/src/state.rs index 432ac56..4bee131 100644 --- a/crates/helios-server/src/state.rs +++ b/crates/helios-server/src/state.rs @@ -73,6 +73,8 @@ pub struct AppState { pub narinfo: quick_cache::sync::Cache>, pub tokens: RwLock, TokenState>>, pub audit: mpsc::UnboundedSender, + pub counters: crate::stats::Counters, + pub access: crate::stats::Access, } pub type Shared = Arc; @@ -126,6 +128,8 @@ impl AppState { narinfo, tokens: RwLock::new(tokens), audit, + counters: Default::default(), + access: Default::default(), }) } diff --git a/crates/helios-server/src/stats.rs b/crates/helios-server/src/stats.rs new file mode 100644 index 0000000..cdc42aa --- /dev/null +++ b/crates/helios-server/src/stats.rs @@ -0,0 +1,116 @@ +//! Request counters and path access tracking, kept off shared cache lines: +//! the narinfo path runs at over a million requests per second, where a +//! single atomic or lock touched by every core would become the bottleneck. + +use std::collections::HashSet; +use std::hash::BuildHasherDefault; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::{Mutex, PoisonError}; + +use crate::state::{PassThroughHasher, PathKey}; + +const STRIPES: usize = 32; + +#[repr(align(128))] +#[derive(Default)] +struct Padded(AtomicU64); + +static NEXT_SLOT: AtomicUsize = AtomicUsize::new(0); + +thread_local! { + /// Each thread increments its own stripe. + static SLOT: usize = NEXT_SLOT.fetch_add(1, Ordering::Relaxed) % STRIPES; +} + +/// A monotonically increasing counter striped across cache lines. +#[derive(Default)] +pub struct Counter { + stripes: [Padded; STRIPES], +} + +impl Counter { + pub fn add(&self, n: u64) { + let slot = SLOT.with(|s| *s); + self.stripes[slot].0.fetch_add(n, Ordering::Relaxed); + } + + pub fn inc(&self) { + self.add(1); + } + + pub fn get(&self) -> u64 { + self.stripes.iter().map(|s| s.0.load(Ordering::Relaxed)).sum() + } +} + +#[derive(Default)] +pub struct Counters { + pub narinfo_hits: Counter, + pub narinfo_misses: Counter, + pub nar_requests: Counter, + pub uploads: Counter, + pub upload_bytes: Counter, + pub published_paths: Counter, +} + +type KeySet = HashSet>; + +const SHARDS: usize = 64; + +/// Paths served since the last flush. Sharded by hash so concurrent +/// requests rarely share a lock. +pub struct Access { + shards: Vec>, +} + +impl Default for Access { + fn default() -> Self { + Self { shards: (0..SHARDS).map(|_| Mutex::new(KeySet::default())).collect() } + } +} + +impl Access { + pub fn touch(&self, key: PathKey) { + let shard = &self.shards[usize::from(key.hash[19]) % SHARDS]; + shard.lock().unwrap_or_else(PoisonError::into_inner).insert(key); + } + + /// Takes every recorded key, leaving the tracker empty. + pub fn drain(&self) -> Vec { + let mut out = Vec::new(); + for shard in &self.shards { + let taken = std::mem::take(&mut *shard.lock().unwrap_or_else(PoisonError::into_inner)); + out.extend(taken); + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn counters_sum_across_threads() { + let c = std::sync::Arc::new(Counter::default()); + let threads: Vec<_> = (0..8) + .map(|_| { + let c = c.clone(); + std::thread::spawn(move || (0..1000).for_each(|_| c.inc())) + }) + .collect(); + threads.into_iter().for_each(|t| t.join().unwrap()); + assert_eq!(c.get(), 8000); + } + + #[test] + fn access_deduplicates_and_drains() { + let a = Access::default(); + let k = PathKey { cache: 1, hash: [7; 20] }; + a.touch(k); + a.touch(k); + a.touch(PathKey { cache: 2, hash: [7; 20] }); + assert_eq!(a.drain().len(), 2); + assert!(a.drain().is_empty()); + } +} From a4a89b7dad6749b1f72c0f1de091d274f5a77d52 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 13:11:30 +0200 Subject: [PATCH 16/56] server: report nar bytes awaiting gc in stats Evicting a path only unpublishes it; its NAR is deleted by GC once no path uses it and its upload grace period has passed. A quota policy that watched deleted bytes alone would see nothing freed for recent NARs and keep evicting. The stats endpoint now reports unreferencedBytes, the NARs no path uses, so callers can budget against what is actually in use. Blob GC also compared creation time strictly against the cutoff, so with a zero grace period a NAR created in the current second was never collected; the comparison is now inclusive. --- crates/helios-server/src/gc.rs | 2 +- crates/helios-server/src/internal.rs | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/crates/helios-server/src/gc.rs b/crates/helios-server/src/gc.rs index 06fa694..ec81e25 100644 --- a/crates/helios-server/src/gc.rs +++ b/crates/helios-server/src/gc.rs @@ -78,7 +78,7 @@ pub fn collect_blobs(st: &Shared) -> anyhow::Result<(usize, u64)> { let tx = conn.transaction()?; let victims: Vec<(i64, Vec, String, u64)> = { let mut stmt = tx.prepare_cached( - "SELECT id, file_hash, compression, file_size FROM blobs b WHERE created_at < ?1 + "SELECT id, file_hash, compression, file_size FROM blobs b WHERE created_at <= ?1 AND NOT EXISTS (SELECT 1 FROM paths p WHERE p.blob_id = b.id) LIMIT ?2", )?; stmt.query_map(params![cutoff, BLOB_BATCH], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))? diff --git a/crates/helios-server/src/internal.rs b/crates/helios-server/src/internal.rs index 113efe6..712aebd 100644 --- a/crates/helios-server/src/internal.rs +++ b/crates/helios-server/src/internal.rs @@ -48,7 +48,7 @@ fn param(uri: &Uri, name: &str) -> Option { async fn stats(State(st): State) -> ApiResult> { let db_st = st.clone(); - let (caches, blob_count, blob_bytes) = blocking(move || { + let (caches, blob_count, blob_bytes, unreferenced) = blocking(move || { db_st.db.read(|conn| { let mut stmt = conn.prepare_cached( "SELECT c.name, c.is_public, count(p.id) FROM caches c LEFT JOIN paths p ON p.cache_id = c.id GROUP BY c.id ORDER BY c.name", @@ -57,7 +57,13 @@ async fn stats(State(st): State) -> ApiResult> { .query_map([], |r| Ok(json!({ "name": r.get::<_, String>(0)?, "public": r.get::<_, bool>(1)?, "paths": r.get::<_, u64>(2)? })))? .collect::>()?; let (count, bytes): (u64, u64) = conn.query_row("SELECT count(*), coalesce(sum(file_size), 0) FROM blobs", [], |r| Ok((r.get(0)?, r.get(1)?)))?; - Ok((caches, count, bytes)) + // Blobs no path uses: GC deletes them once their upload grace ends. + let unreferenced: u64 = conn.query_row( + "SELECT coalesce(sum(file_size), 0) FROM blobs b WHERE NOT EXISTS (SELECT 1 FROM paths p WHERE p.blob_id = b.id)", + [], + |r| r.get(0), + )?; + Ok((caches, count, bytes, unreferenced)) }) .map_err(internal) }) @@ -65,7 +71,7 @@ async fn stats(State(st): State) -> ApiResult> { let c = &st.counters; Ok(Json(json!({ "caches": caches, - "blobs": { "count": blob_count, "bytes": blob_bytes }, + "blobs": { "count": blob_count, "bytes": blob_bytes, "unreferencedBytes": unreferenced }, "indexedPaths": st.index.rd().len(), "narinfoCacheEntries": st.narinfo.len(), "requests": { From b2152ed9b143777ada9e29fbc3cb3864603bd18c Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 13:16:56 +0200 Subject: [PATCH 17/56] daemon: add helios-daemon for cache maintenance helios-daemon runs beside helios-server and drives it over the maintenance socket; it never writes the database itself. Auto-GC keeps the NARs in use under a byte quota, and optionally the data filesystem above a free-space floor, by evicting least-recently-used paths between a high and a low watermark. Each round re-reads the server's stats, so NARs still shared with other paths or inside the upload grace period cannot drive it into evicting everything. An integrity scrub re-reads every NAR at a rate limit, decompresses and re-hashes it through libhelios, and compares file hash, NAR hash and NAR size with the database. A missing or corrupt NAR is quarantined, which unpublishes the paths it backs. The daemon also schedules WAL checkpoints, PRAGMA optimize and rotated database backups, and serves Prometheus metrics that combine the server's request counters, its own task counters and disk space. Failed runs are logged and counted and retried on the next tick, since the server may be restarting. The crate adds no dependencies the workspace does not already build. --- Cargo.lock | 18 +++ crates/helios-daemon/Cargo.toml | 21 +++ crates/helios-daemon/src/autogc.rs | 148 +++++++++++++++++ crates/helios-daemon/src/client.rs | 56 +++++++ crates/helios-daemon/src/io.rs | 38 +++++ crates/helios-daemon/src/log.rs | 54 +++++++ crates/helios-daemon/src/main.rs | 239 ++++++++++++++++++++++++++++ crates/helios-daemon/src/maint.rs | 30 ++++ crates/helios-daemon/src/metrics.rs | 113 +++++++++++++ crates/helios-daemon/src/scrub.rs | 124 +++++++++++++++ crates/helios-daemon/src/state.rs | 56 +++++++ 11 files changed, 897 insertions(+) create mode 100644 crates/helios-daemon/Cargo.toml create mode 100644 crates/helios-daemon/src/autogc.rs create mode 100644 crates/helios-daemon/src/client.rs create mode 100644 crates/helios-daemon/src/io.rs create mode 100644 crates/helios-daemon/src/log.rs create mode 100644 crates/helios-daemon/src/main.rs create mode 100644 crates/helios-daemon/src/maint.rs create mode 100644 crates/helios-daemon/src/metrics.rs create mode 100644 crates/helios-daemon/src/scrub.rs create mode 100644 crates/helios-daemon/src/state.rs diff --git a/Cargo.lock b/Cargo.lock index c60361b..fc625da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -292,6 +292,24 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "helios-daemon" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "clap", + "futures-util", + "helios-core", + "http", + "http-body-util", + "hyper", + "libc", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "helios-server" version = "0.1.0" diff --git a/crates/helios-daemon/Cargo.toml b/crates/helios-daemon/Cargo.toml new file mode 100644 index 0000000..444a850 --- /dev/null +++ b/crates/helios-daemon/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "helios-daemon" +description = "Maintenance daemon for helios-server: auto-GC, integrity scrub, SQLite maintenance, metrics" +version.workspace = true +edition.workspace = true +license-file.workspace = true +publish.workspace = true + +[dependencies] +anyhow = "1.0.104" +bytes = "1.12.1" +clap = { version = "4.6.7", default-features = false, features = ["std", "derive", "env", "help", "usage", "error-context"] } +futures-util = { version = "0.3.34", default-features = false, features = ["std"] } +helios-core = { path = "../helios-core" } +http = "1.4" +http-body-util = "0.1" +hyper = { version = "1.8", default-features = false, features = ["client", "server", "http1"] } +libc = "0.2" +serde = { version = "1.0.229", features = ["derive"] } +serde_json = "1.0.151" +tokio = { version = "1.53.1", features = ["rt-multi-thread", "net", "time", "signal", "sync", "io-util"] } diff --git a/crates/helios-daemon/src/autogc.rs b/crates/helios-daemon/src/autogc.rs new file mode 100644 index 0000000..e565c4b --- /dev/null +++ b/crates/helios-daemon/src/autogc.rs @@ -0,0 +1,148 @@ +//! Keeps the cache under a byte quota (and optionally the disk above a +//! free-space floor) by evicting least-recently-used paths. +//! +//! Eviction starts above the high watermark and stops at the low one, so +//! it runs in occasional batches rather than on every upload. + +use serde::Deserialize; +use serde_json::json; + +use crate::log; +use crate::state::{Daemon, add, now, set}; + +#[derive(Clone, Copy, Debug)] +pub struct Policy { + /// Target NAR bytes; 0 disables the quota. + pub quota: u64, + pub high: f64, + pub low: f64, + /// Keep at least this much free on the data filesystem; 0 disables. + pub min_free: u64, +} + +#[derive(Deserialize)] +struct Stats { + blobs: Blobs, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Blobs { + bytes: u64, + /// NARs no path uses any more; GC deletes them once their upload + /// grace period ends, so they no longer count against the quota. + unreferenced_bytes: u64, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Candidate { + cache: u32, + hash: String, + nar_bytes: u64, +} + +#[derive(Deserialize)] +struct Lru { + paths: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Evicted { + evicted: u64, + freed_bytes: u64, +} + +/// Bytes to free now, or 0 when within policy. +pub fn to_free(policy: &Policy, used: u64, free_disk: u64) -> u64 { + let over_quota = if policy.quota > 0 && used as f64 > policy.quota as f64 * policy.high { + used - (policy.quota as f64 * policy.low) as u64 + } else { + 0 + }; + let short_disk = if policy.min_free > 0 && free_disk < policy.min_free { + // Overshoot a little so the next upload does not trigger it again. + (policy.min_free as f64 * 1.1) as u64 - free_disk + } else { + 0 + }; + over_quota.max(short_disk) +} + +/// Upper bound on eviction rounds per run, as a backstop. +const MAX_ROUNDS: usize = 10_000; + +/// Referenced NAR bytes and bytes to free, from fresh server stats. +async fn measure(d: &Daemon, policy: &Policy) -> anyhow::Result<(u64, u64)> { + let stats: Stats = d.client.get("/v1/stats").await?; + let referenced = stats.blobs.bytes.saturating_sub(stats.blobs.unreferenced_bytes); + let free_disk = crate::state::disk_space(&d.data_dir).map(|(free, _)| free).unwrap_or(u64::MAX); + // Unreferenced NARs will be deleted soon; count them as free already. + let free_soon = free_disk.saturating_add(stats.blobs.unreferenced_bytes); + Ok((referenced, to_free(policy, referenced, free_soon))) +} + +pub async fn run(d: &Daemon, policy: &Policy) -> anyhow::Result<()> { + add(&d.metrics.gc_runs, 1); + set(&d.metrics.gc_last_run, now()); + let (referenced, mut need) = measure(d, policy).await?; + if need == 0 { + return Ok(()); + } + log::info!("auto-gc: {referenced} bytes of NARs in use, freeing {need}"); + let (mut evicted, mut freed) = (0u64, 0u64); + for _ in 0..MAX_ROUNDS { + let lru: Lru = d.client.get("/v1/lru?limit=500").await?; + // Enough of the oldest paths to cover what is left, by NAR size. + let mut batch = Vec::new(); + let mut planned = 0u64; + for c in lru.paths { + planned += c.nar_bytes; + batch.push(json!({ "cache": c.cache, "hash": c.hash })); + if planned >= need { + break; + } + } + if batch.is_empty() { + break; + } + let r: Evicted = d.client.post("/v1/evict", json!({ "paths": batch })).await?; + if r.evicted == 0 { + break; + } + evicted += r.evicted; + freed += r.freed_bytes; + // NARs shared with paths still in use stay; re-measure rather than + // trusting the plan. + need = measure(d, policy).await?.1; + if need == 0 { + break; + } + } + add(&d.metrics.gc_evicted_paths, evicted); + add(&d.metrics.gc_freed_bytes, freed); + log::info!("auto-gc: evicted {evicted} paths, {freed} bytes deleted now (the rest after the upload grace period)"); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const GIB: u64 = 1 << 30; + + #[test] + fn watermarks() { + let p = Policy { quota: 100 * GIB, high: 0.9, low: 0.8, min_free: 0 }; + assert_eq!(to_free(&p, 89 * GIB, u64::MAX), 0); + assert_eq!(to_free(&p, 95 * GIB, u64::MAX), 15 * GIB); + } + + #[test] + fn free_space_floor() { + let p = Policy { quota: 0, high: 0.9, low: 0.8, min_free: 10 * GIB }; + assert_eq!(to_free(&p, 1, 20 * GIB), 0); + assert_eq!(to_free(&p, 1, 5 * GIB), 6 * GIB); + } +} diff --git a/crates/helios-daemon/src/client.rs b/crates/helios-daemon/src/client.rs new file mode 100644 index 0000000..784a8d8 --- /dev/null +++ b/crates/helios-daemon/src/client.rs @@ -0,0 +1,56 @@ +//! Client for helios-server's maintenance API on its Unix socket. Requests +//! are rare (minutes apart), so each one gets a fresh connection. + +use std::path::PathBuf; + +use anyhow::{Context, bail}; +use bytes::Bytes; +use http::{Method, Request, header}; +use http_body_util::{BodyExt, Full}; +use serde::de::DeserializeOwned; +use serde_json::Value; +use tokio::net::UnixStream; + +use crate::io::Io; + +#[derive(Clone)] +pub struct Client { + socket: PathBuf, +} + +impl Client { + pub fn new(socket: PathBuf) -> Self { + Self { socket } + } + + async fn call(&self, method: Method, path: &str, body: Option) -> anyhow::Result { + let stream = UnixStream::connect(&self.socket) + .await + .with_context(|| format!("connecting to {}", self.socket.display()))?; + let (mut sender, conn) = hyper::client::conn::http1::handshake(Io(stream)).await?; + tokio::spawn(conn); + let mut req = Request::builder().method(method).uri(path).header(header::HOST, "helios"); + let payload = match body { + Some(v) => { + req = req.header(header::CONTENT_TYPE, "application/json"); + Bytes::from(serde_json::to_vec(&v)?) + } + None => Bytes::new(), + }; + let resp = sender.send_request(req.body(Full::new(payload))?).await?; + let status = resp.status(); + let bytes = resp.into_body().collect().await?.to_bytes(); + if !status.is_success() { + bail!("{path}: {status}: {}", String::from_utf8_lossy(&bytes)); + } + serde_json::from_slice(&bytes).with_context(|| format!("decoding {path}")) + } + + pub async fn get(&self, path: &str) -> anyhow::Result { + self.call(Method::GET, path, None).await + } + + pub async fn post(&self, path: &str, body: Value) -> anyhow::Result { + self.call(Method::POST, path, Some(body)).await + } +} diff --git a/crates/helios-daemon/src/io.rs b/crates/helios-daemon/src/io.rs new file mode 100644 index 0000000..6433b7a --- /dev/null +++ b/crates/helios-daemon/src/io.rs @@ -0,0 +1,38 @@ +//! hyper's I/O traits over tokio streams (hyper-util would bring a crate +//! for these forty lines). + +use std::io; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +pub struct Io(pub T); + +impl hyper::rt::Read for Io { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, mut buf: hyper::rt::ReadBufCursor<'_>) -> Poll> { + // SAFETY: tokio's ReadBuf only writes initialised bytes into the + // uninitialised tail, and we advance by exactly what was filled. + let filled = unsafe { + let mut rb = ReadBuf::uninit(buf.as_mut()); + match Pin::new(&mut self.0).poll_read(cx, &mut rb) { + Poll::Ready(Ok(())) => rb.filled().len(), + other => return other, + } + }; + unsafe { buf.advance(filled) }; + Poll::Ready(Ok(())) + } +} + +impl hyper::rt::Write for Io { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + Pin::new(&mut self.0).poll_write(cx, buf) + } + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.0).poll_flush(cx) + } + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.0).poll_shutdown(cx) + } +} diff --git a/crates/helios-daemon/src/log.rs b/crates/helios-daemon/src/log.rs new file mode 100644 index 0000000..b3be188 --- /dev/null +++ b/crates/helios-daemon/src/log.rs @@ -0,0 +1,54 @@ +//! Leveled logging to stderr, with journald priorities under systemd. +//! Same format as helios-server; HELIOS_LOG sets the level. + +use std::sync::atomic::{AtomicU8, Ordering}; + +pub const ERROR: u8 = 3; +pub const WARN: u8 = 4; +pub const INFO: u8 = 6; +pub const DEBUG: u8 = 7; + +static LEVEL: AtomicU8 = AtomicU8::new(INFO); +static JOURNAL: AtomicU8 = AtomicU8::new(0); + +pub fn init() { + let level = match std::env::var("HELIOS_LOG").as_deref() { + Ok("error") => ERROR, + Ok("warn") => WARN, + Ok("debug") => DEBUG, + _ => INFO, + }; + LEVEL.store(level, Ordering::Relaxed); + JOURNAL.store(u8::from(std::env::var_os("JOURNAL_STREAM").is_some()), Ordering::Relaxed); +} + +pub fn enabled(level: u8) -> bool { + level <= LEVEL.load(Ordering::Relaxed) +} + +pub fn emit(level: u8, args: std::fmt::Arguments<'_>) { + let label = match level { + ERROR => "error", + WARN => "warn", + INFO => "info", + _ => "debug", + }; + if JOURNAL.load(Ordering::Relaxed) == 1 { + eprintln!("<{level}>{args}"); + } else { + eprintln!("{label}: {args}"); + } +} + +macro_rules! log { + ($level:expr, $($arg:tt)*) => { + if $crate::log::enabled($level) { + $crate::log::emit($level, format_args!($($arg)*)); + } + }; +} +macro_rules! error { ($($arg:tt)*) => { $crate::log::log!($crate::log::ERROR, $($arg)*) }; } +macro_rules! warning { ($($arg:tt)*) => { $crate::log::log!($crate::log::WARN, $($arg)*) }; } +macro_rules! info { ($($arg:tt)*) => { $crate::log::log!($crate::log::INFO, $($arg)*) }; } + +pub(crate) use {error, info, log, warning}; diff --git a/crates/helios-daemon/src/main.rs b/crates/helios-daemon/src/main.rs new file mode 100644 index 0000000..c41a71f --- /dev/null +++ b/crates/helios-daemon/src/main.rs @@ -0,0 +1,239 @@ +//! helios-daemon: maintenance for a helios-server, driven over its admin +//! socket. It never writes the database itself; it decides what to do and +//! the server does it, so the server's in-memory state stays consistent. + +mod autogc; +mod client; +mod io; +mod log; +mod maint; +mod metrics; +mod scrub; +mod state; + +use std::future::Future; +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::atomic::AtomicU64; +use std::time::Duration; + +use clap::Parser; + +use crate::state::{Daemon, add}; + +/// Maintenance daemon for helios-server. +#[derive(Parser, Debug)] +#[command(version)] +struct Args { + /// helios-server's maintenance socket (its --admin-socket). + #[arg(long, env = "HELIOS_ADMIN_SOCKET")] + socket: PathBuf, + + /// helios-server's data directory, read for scrubbing and disk space. + #[arg(long, env = "HELIOS_DATA_DIR", default_value = "/var/lib/helios")] + data_dir: PathBuf, + + /// Keep stored NARs under this size (e.g. 500G); 0 disables the quota. + #[arg(long, env = "HELIOS_QUOTA", default_value = "0", value_parser = parse_size)] + quota: u64, + + /// Start evicting above this fraction of the quota. + #[arg(long, env = "HELIOS_QUOTA_HIGH", default_value_t = 0.9)] + quota_high: f64, + + /// Evict down to this fraction of the quota. + #[arg(long, env = "HELIOS_QUOTA_LOW", default_value_t = 0.8)] + quota_low: f64, + + /// Also evict while the data filesystem has less free space than this (e.g. 20G); 0 disables. + #[arg(long, env = "HELIOS_MIN_FREE", default_value = "0", value_parser = parse_size)] + min_free: u64, + + /// Time between auto-GC checks. + #[arg(long, env = "HELIOS_GC_INTERVAL", default_value = "5m", value_parser = parse_duration)] + gc_interval: Duration, + + /// Time between full integrity scrubs; 0 disables. + #[arg(long, env = "HELIOS_SCRUB_INTERVAL", default_value = "7d", value_parser = parse_duration)] + scrub_interval: Duration, + + /// Scrub read rate limit (e.g. 64M per second); 0 is unlimited. + #[arg(long, env = "HELIOS_SCRUB_RATE", default_value = "64M", value_parser = parse_size)] + scrub_rate: u64, + + /// Time between WAL checkpoints. + #[arg(long, env = "HELIOS_CHECKPOINT_INTERVAL", default_value = "15m", value_parser = parse_duration)] + checkpoint_interval: Duration, + + /// Time between query planner statistics updates (PRAGMA optimize). + #[arg(long, env = "HELIOS_OPTIMIZE_INTERVAL", default_value = "1d", value_parser = parse_duration)] + optimize_interval: Duration, + + /// Time between database backups; 0 disables. + #[arg(long, env = "HELIOS_BACKUP_INTERVAL", default_value = "1d", value_parser = parse_duration)] + backup_interval: Duration, + + /// Database backups to keep. + #[arg(long, env = "HELIOS_BACKUP_KEEP", default_value_t = 7)] + backup_keep: usize, + + /// Serve Prometheus metrics on this address. + #[arg(long, env = "HELIOS_METRICS_LISTEN")] + metrics_listen: Option, +} + +/// Bytes with an optional binary suffix: 1024, 64M, 500G, 2T. +fn parse_size(s: &str) -> Result { + let s = s.trim(); + let (num, mult) = match s.char_indices().last() { + Some((i, c)) if c.is_ascii_alphabetic() => { + let mult = match c.to_ascii_uppercase() { + 'K' => 1u64 << 10, + 'M' => 1 << 20, + 'G' => 1 << 30, + 'T' => 1 << 40, + _ => return Err(format!("unknown size suffix in {s}")), + }; + (&s[..i], mult) + } + _ => (s, 1), + }; + let n: f64 = num.parse().map_err(|_| format!("invalid size {s}"))?; + if n < 0.0 { + return Err(format!("invalid size {s}")); + } + Ok((n * mult as f64) as u64) +} + +/// A duration with a unit: 30s, 5m, 24h, 7d. A bare 0 disables. +fn parse_duration(s: &str) -> Result { + let s = s.trim(); + if s == "0" { + return Ok(Duration::ZERO); + } + let split = s.find(|c: char| c.is_ascii_alphabetic()).ok_or_else(|| format!("{s}: missing unit (s, m, h or d)"))?; + let n: u64 = s[..split].parse().map_err(|_| format!("invalid duration {s}"))?; + let unit = match &s[split..] { + "s" => 1, + "m" => 60, + "h" => 3600, + "d" => 86400, + other => return Err(format!("unknown duration unit {other}")), + }; + Ok(Duration::from_secs(n * unit)) +} + +/// Runs `task` every `every`, logging and counting failures instead of +/// exiting: the server may be restarting. +async fn periodic(name: &'static str, every: Duration, delay_first: bool, errors: &AtomicU64, mut task: F) +where + F: FnMut() -> Fut, + Fut: Future>, +{ + if delay_first { + tokio::time::sleep(every).await; + } + let mut interval = tokio::time::interval(every); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + interval.tick().await; + if let Err(e) = task().await { + add(errors, 1); + log::error!("{name}: {e:#}"); + } + } +} + +async fn shutdown() { + use tokio::signal::unix::{SignalKind, signal}; + let mut term = signal(SignalKind::terminate()).expect("installing SIGTERM handler"); + let term = std::pin::pin!(term.recv()); + let int = std::pin::pin!(tokio::signal::ctrl_c()); + futures_util::future::select(term, int).await; +} + +fn main() -> anyhow::Result<()> { + log::init(); + let args = Args::parse(); + anyhow::ensure!( + (0.0..=1.0).contains(&args.quota_low) && args.quota_low < args.quota_high && args.quota_high <= 1.0, + "watermarks must satisfy 0 <= low < high <= 1" + ); + let policy = autogc::Policy { quota: args.quota, high: args.quota_high, low: args.quota_low, min_free: args.min_free }; + let d = Arc::new(Daemon { client: client::Client::new(args.socket.clone()), data_dir: args.data_dir.clone(), metrics: Default::default() }); + + tokio::runtime::Builder::new_current_thread().enable_all().build()?.block_on(async move { + let mut tasks = Vec::new(); + if policy.quota > 0 || policy.min_free > 0 { + let d = d.clone(); + tasks.push(tokio::spawn(async move { + periodic("auto-gc", args.gc_interval.max(Duration::from_secs(1)), false, &d.metrics.errors_gc, || autogc::run(&d, &policy)).await + })); + } + if !args.scrub_interval.is_zero() { + let d = d.clone(); + let rate = args.scrub_rate; + // Wait a full interval first so restarts do not re-read everything. + tasks.push(tokio::spawn(async move { + periodic("scrub", args.scrub_interval, true, &d.metrics.errors_scrub, || scrub::run(&d, rate)).await + })); + } + { + let d = d.clone(); + tasks.push(tokio::spawn(async move { + periodic("checkpoint", args.checkpoint_interval.max(Duration::from_secs(1)), true, &d.metrics.errors_db, || maint::checkpoint(&d)).await + })); + } + { + let d = d.clone(); + tasks.push(tokio::spawn(async move { + periodic("optimize", args.optimize_interval.max(Duration::from_secs(1)), true, &d.metrics.errors_db, || maint::optimize(&d)).await + })); + } + if !args.backup_interval.is_zero() { + let d = d.clone(); + let keep = args.backup_keep; + tasks.push(tokio::spawn(async move { + periodic("backup", args.backup_interval, false, &d.metrics.errors_db, || maint::backup(&d, keep)).await + })); + } + if let Some(addr) = args.metrics_listen { + let d = d.clone(); + tasks.push(tokio::spawn(async move { + if let Err(e) = metrics::serve(d, addr).await { + log::error!("metrics: {e:#}"); + } + })); + } + log::info!("helios-daemon started ({} tasks)", tasks.len()); + shutdown().await; + log::info!("shutting down"); + Ok(()) + }) +} + +#[cfg(test)] +mod tests { + use super::{parse_duration, parse_size}; + use std::time::Duration; + + #[test] + fn durations() { + assert_eq!(parse_duration("30s"), Ok(Duration::from_secs(30))); + assert_eq!(parse_duration("7d"), Ok(Duration::from_secs(7 * 86400))); + assert_eq!(parse_duration("0"), Ok(Duration::ZERO)); + assert!(parse_duration("5").is_err()); + assert!(parse_duration("5w").is_err()); + } + + #[test] + fn sizes() { + assert_eq!(parse_size("0"), Ok(0)); + assert_eq!(parse_size("1024"), Ok(1024)); + assert_eq!(parse_size("64M"), Ok(64 << 20)); + assert_eq!(parse_size("1.5g"), Ok(3 << 29)); + assert!(parse_size("5X").is_err()); + assert!(parse_size("-1").is_err()); + } +} diff --git a/crates/helios-daemon/src/maint.rs b/crates/helios-daemon/src/maint.rs new file mode 100644 index 0000000..6151daa --- /dev/null +++ b/crates/helios-daemon/src/maint.rs @@ -0,0 +1,30 @@ +//! SQLite upkeep, performed by the server on request: WAL checkpoints (so +//! the WAL does not grow without bound under constant reads), query +//! planner statistics, and rotated online backups. + +use serde_json::{Value, json}; + +use crate::log; +use crate::state::{Daemon, add, now, set}; + +pub async fn checkpoint(d: &Daemon) -> anyhow::Result<()> { + let r: Value = d.client.post("/v1/db/checkpoint", json!({})).await?; + add(&d.metrics.db_checkpoints, 1); + if r["busy"] == true { + log::warning!("db: checkpoint could not complete while readers were active"); + } + Ok(()) +} + +pub async fn optimize(d: &Daemon) -> anyhow::Result<()> { + let _: Value = d.client.post("/v1/db/optimize", json!({})).await?; + Ok(()) +} + +pub async fn backup(d: &Daemon, keep: usize) -> anyhow::Result<()> { + let r: Value = d.client.post("/v1/db/backup", json!({ "keep": keep })).await?; + add(&d.metrics.db_backups, 1); + set(&d.metrics.db_last_backup, now()); + log::info!("db: backed up to {} ({} bytes)", r["path"].as_str().unwrap_or("?"), r["bytes"]); + Ok(()) +} diff --git a/crates/helios-daemon/src/metrics.rs b/crates/helios-daemon/src/metrics.rs new file mode 100644 index 0000000..c8337aa --- /dev/null +++ b/crates/helios-daemon/src/metrics.rs @@ -0,0 +1,113 @@ +//! Prometheus text exposition on `--metrics-listen`: the server's counters +//! (fetched per scrape over the admin socket), the daemon's own, and disk +//! space of the data filesystem. + +use std::fmt::Write as _; +use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; + +use bytes::Bytes; +use http::{Response, StatusCode, header}; +use http_body_util::Full; +use hyper::service::service_fn; +use serde_json::Value; + +use crate::io::Io; +use crate::log; +use crate::state::Daemon; + +fn line(out: &mut String, name: &str, kind: &str, help: &str, samples: &[(&str, u64)]) { + let _ = writeln!(out, "# HELP {name} {help}\n# TYPE {name} {kind}"); + for (labels, v) in samples { + let _ = writeln!(out, "{name}{labels} {v}"); + } +} + +fn get(c: &AtomicU64) -> u64 { + c.load(Ordering::Relaxed) +} + +pub async fn render(d: &Daemon) -> String { + let mut out = String::new(); + let server: Option = d.client.get("/v1/stats").await.ok(); + line(&mut out, "helios_up", "gauge", "Whether helios-server answered on the admin socket.", &[("", u64::from(server.is_some()))]); + if let Some(s) = &server { + let n = |v: &Value| v.as_u64().unwrap_or(0); + let caches: Vec<(String, u64)> = s["caches"] + .as_array() + .into_iter() + .flatten() + .map(|c| (format!("{{cache=\"{}\"}}", c["name"].as_str().unwrap_or("")), n(&c["paths"]))) + .collect(); + let caches: Vec<(&str, u64)> = caches.iter().map(|(l, v)| (l.as_str(), *v)).collect(); + line(&mut out, "helios_cache_paths", "gauge", "Published paths per cache.", &caches); + line(&mut out, "helios_blobs", "gauge", "Stored NAR files.", &[("", n(&s["blobs"]["count"]))]); + line(&mut out, "helios_blob_bytes", "gauge", "Bytes of stored NAR files.", &[("", n(&s["blobs"]["bytes"]))]); + let r = &s["requests"]; + line( + &mut out, + "helios_narinfo_requests_total", + "counter", + "Narinfo lookups since the server started.", + &[("{result=\"hit\"}", n(&r["narinfoHits"])), ("{result=\"miss\"}", n(&r["narinfoMisses"]))], + ); + line(&mut out, "helios_nar_requests_total", "counter", "NAR download requests.", &[("", n(&r["nars"]))]); + line(&mut out, "helios_uploads_total", "counter", "Verified NAR uploads.", &[("", n(&r["uploads"]))]); + line(&mut out, "helios_upload_bytes_total", "counter", "Bytes of verified NAR uploads.", &[("", n(&r["uploadBytes"]))]); + line(&mut out, "helios_published_paths_total", "counter", "Paths published.", &[("", n(&r["publishedPaths"]))]); + } + if let Ok((free, total)) = crate::state::disk_space(&d.data_dir) { + line(&mut out, "helios_disk_free_bytes", "gauge", "Free bytes on the data filesystem.", &[("", free)]); + line(&mut out, "helios_disk_size_bytes", "gauge", "Size of the data filesystem.", &[("", total)]); + } + let m = &d.metrics; + line(&mut out, "helios_gc_runs_total", "counter", "Auto-GC checks.", &[("", get(&m.gc_runs))]); + line(&mut out, "helios_gc_evicted_paths_total", "counter", "Paths evicted by auto-GC.", &[("", get(&m.gc_evicted_paths))]); + line(&mut out, "helios_gc_freed_bytes_total", "counter", "NAR bytes freed by auto-GC.", &[("", get(&m.gc_freed_bytes))]); + line(&mut out, "helios_gc_last_run_timestamp_seconds", "gauge", "Last auto-GC check.", &[("", get(&m.gc_last_run))]); + line( + &mut out, + "helios_scrub_blobs_total", + "counter", + "Blobs checked by the integrity scrub.", + &[("{result=\"ok\"}", get(&m.scrub_ok)), ("{result=\"corrupt\"}", get(&m.scrub_corrupt)), ("{result=\"missing\"}", get(&m.scrub_missing))], + ); + line(&mut out, "helios_scrub_bytes_total", "counter", "Bytes read by the integrity scrub.", &[("", get(&m.scrub_bytes))]); + line(&mut out, "helios_scrub_last_complete_timestamp_seconds", "gauge", "End of the last full scrub pass.", &[("", get(&m.scrub_last_complete))]); + line(&mut out, "helios_db_checkpoints_total", "counter", "WAL checkpoints.", &[("", get(&m.db_checkpoints))]); + line(&mut out, "helios_db_backups_total", "counter", "Database backups.", &[("", get(&m.db_backups))]); + line(&mut out, "helios_db_last_backup_timestamp_seconds", "gauge", "Last database backup.", &[("", get(&m.db_last_backup))]); + line( + &mut out, + "helios_maintenance_errors_total", + "counter", + "Failed maintenance runs.", + &[("{task=\"gc\"}", get(&m.errors_gc)), ("{task=\"scrub\"}", get(&m.errors_scrub)), ("{task=\"db\"}", get(&m.errors_db))], + ); + out +} + +pub async fn serve(d: Arc, addr: SocketAddr) -> anyhow::Result<()> { + let listener = tokio::net::TcpListener::bind(addr).await?; + log::info!("metrics on http://{addr}/metrics"); + loop { + let (stream, _) = listener.accept().await?; + let d = d.clone(); + tokio::spawn(async move { + let svc = service_fn(move |req: http::Request| { + let d = d.clone(); + async move { + if req.uri().path() == "/metrics" { + Response::builder() + .header(header::CONTENT_TYPE, "text/plain; version=0.0.4") + .body(Full::new(Bytes::from(render(&d).await))) + } else { + Response::builder().status(StatusCode::NOT_FOUND).body(Full::new(Bytes::from_static(b"not found\n"))) + } + } + }); + let _ = hyper::server::conn::http1::Builder::new().serve_connection(Io(stream), svc).await; + }); + } +} diff --git a/crates/helios-daemon/src/scrub.rs b/crates/helios-daemon/src/scrub.rs new file mode 100644 index 0000000..f7d80f4 --- /dev/null +++ b/crates/helios-daemon/src/scrub.rs @@ -0,0 +1,124 @@ +//! Integrity scrub: re-reads every blob, decompresses and re-hashes it, and +//! compares the file hash, NAR hash and NAR size with the database. A blob +//! that is missing or corrupt is quarantined, which unpublishes its paths +//! so clients never download bad data. Reads are rate limited. + +use std::io::Read; +use std::time::{Duration, Instant}; + +use helios_core::{Compression, Verifier}; +use serde::Deserialize; +use serde_json::json; + +use crate::log; +use crate::state::{Daemon, add, now, set}; + +#[derive(Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +struct Blob { + id: i64, + file_hash: String, + compression: String, + file_size: u64, + nar_hash: String, + nar_size: u64, + path: Option, +} + +#[derive(Deserialize)] +struct Page { + blobs: Vec, +} + +enum Verdict { + Ok, + Missing, + Corrupt(String), +} + +fn check(data_dir: &std::path::Path, b: &Blob) -> anyhow::Result { + let (Some(rel), Some(compression)) = (&b.path, Compression::parse(&b.compression)) else { + return Ok(Verdict::Corrupt(format!("unsupported compression {}", b.compression))); + }; + let mut file = match std::fs::File::open(data_dir.join(rel)) { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Verdict::Missing), + Err(e) => return Err(e.into()), + }; + let mut v = Verifier::new(compression)?; + let mut buf = vec![0u8; 1 << 20]; + loop { + let n = file.read(&mut buf)?; + if n == 0 { + break; + } + if let Err(e) = v.update(&buf[..n]) { + return Ok(Verdict::Corrupt(format!("undecodable: {e}"))); + } + } + let d = match v.finish() { + Ok(d) => d, + Err(e) => return Ok(Verdict::Corrupt(format!("undecodable: {e}"))), + }; + let problems: Vec<&str> = [ + (d.file_size != b.file_size, "file size"), + (helios_core::nix32_encode(&d.file_hash) != b.file_hash, "file hash"), + (helios_core::nix32_encode(&d.nar_hash) != b.nar_hash, "NAR hash"), + (d.nar_size != b.nar_size, "NAR size"), + ] + .into_iter() + .filter_map(|(bad, what)| bad.then_some(what)) + .collect(); + Ok(if problems.is_empty() { Verdict::Ok } else { Verdict::Corrupt(format!("{} mismatch", problems.join(", "))) }) +} + +/// One full pass over every blob. `rate` is bytes per second (0 = unlimited). +pub async fn run(d: &Daemon, rate: u64) -> anyhow::Result<()> { + let started = Instant::now(); + let (mut after, mut bytes, mut blobs) = (0i64, 0u64, 0u64); + loop { + let page: Page = d.client.get(&format!("/v1/blobs?after={after}&limit=500")).await?; + let Some(last) = page.blobs.last() else { break }; + after = last.id; + for b in page.blobs { + let dir = d.data_dir.clone(); + let blob = b.clone(); + let verdict = tokio::task::spawn_blocking(move || check(&dir, &blob)).await??; + blobs += 1; + bytes += b.file_size; + add(&d.metrics.scrub_bytes, b.file_size); + let reason = match verdict { + Verdict::Ok => { + add(&d.metrics.scrub_ok, 1); + None + } + Verdict::Missing => { + add(&d.metrics.scrub_missing, 1); + Some("file missing".to_owned()) + } + Verdict::Corrupt(why) => { + add(&d.metrics.scrub_corrupt, 1); + Some(why) + } + }; + if let Some(reason) = reason { + log::error!("scrub: blob {} is bad ({reason}); quarantining", b.file_hash); + // A blob GC removed since it was listed is already gone. + if let Err(e) = d.client.post::("/v1/quarantine", json!({ "fileHash": b.file_hash, "reason": reason })).await + && !e.to_string().contains("404") + { + return Err(e); + } + } + if rate > 0 { + let due = Duration::from_secs_f64(bytes as f64 / rate as f64); + if let Some(wait) = due.checked_sub(started.elapsed()) { + tokio::time::sleep(wait).await; + } + } + } + } + set(&d.metrics.scrub_last_complete, now()); + log::info!("scrub: checked {blobs} blobs ({bytes} bytes) in {:.0?}", started.elapsed()); + Ok(()) +} diff --git a/crates/helios-daemon/src/state.rs b/crates/helios-daemon/src/state.rs new file mode 100644 index 0000000..896c1df --- /dev/null +++ b/crates/helios-daemon/src/state.rs @@ -0,0 +1,56 @@ +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::client::Client; + +/// Daemon-side counters, exported on /metrics. +#[derive(Default)] +pub struct Metrics { + pub gc_runs: AtomicU64, + pub gc_evicted_paths: AtomicU64, + pub gc_freed_bytes: AtomicU64, + pub gc_last_run: AtomicU64, + pub scrub_ok: AtomicU64, + pub scrub_corrupt: AtomicU64, + pub scrub_missing: AtomicU64, + pub scrub_bytes: AtomicU64, + pub scrub_last_complete: AtomicU64, + pub db_checkpoints: AtomicU64, + pub db_backups: AtomicU64, + pub db_last_backup: AtomicU64, + pub errors_gc: AtomicU64, + pub errors_scrub: AtomicU64, + pub errors_db: AtomicU64, +} + +pub fn add(counter: &AtomicU64, n: u64) { + counter.fetch_add(n, Ordering::Relaxed); +} + +pub fn set(counter: &AtomicU64, v: u64) { + counter.store(v, Ordering::Relaxed); +} + +pub fn now() -> u64 { + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs()) +} + +pub struct Daemon { + pub client: Client, + pub data_dir: PathBuf, + pub metrics: Metrics, +} + +/// Free and total bytes of the filesystem holding `path`. +// statvfs fields are c_ulong: the casts matter on 32-bit targets. +#[allow(clippy::unnecessary_cast)] +pub fn disk_space(path: &std::path::Path) -> std::io::Result<(u64, u64)> { + use std::os::unix::ffi::OsStrExt; + let c = std::ffi::CString::new(path.as_os_str().as_bytes())?; + let mut s: libc::statvfs = unsafe { std::mem::zeroed() }; + if unsafe { libc::statvfs(c.as_ptr(), &mut s) } != 0 { + return Err(std::io::Error::last_os_error()); + } + let frag = s.f_frsize as u64; + Ok((s.f_bavail as u64 * frag, s.f_blocks as u64 * frag)) +} From 0b0a7de1b96f725a6067f6fddd996068bcc538e2 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 13:18:20 +0200 Subject: [PATCH 18/56] cli: add watch-store to push paths as they are built helios queue-paths is meant for Nix's post-build-hook. It writes the paths in $OUT_PATHS to a spool directory under a temporary name and renames them into place, then returns, so builds never wait on the network or on the cache being up. helios watch-store drains the spool and pushes each batch's closure, deleting entries only after a successful push and backing off up to a minute while the server is unreachable. Services and CI can now point the CLI at a server with --url and --token-file (or HELIOS_URL, HELIOS_TOKEN_FILE and HELIOS_TOKEN) instead of a saved login. --- crates/helios-cli/Cargo.toml | 2 +- crates/helios-cli/src/main.rs | 54 +++++++++++++++- crates/helios-cli/src/push.rs | 1 + crates/helios-cli/src/watch.rs | 109 +++++++++++++++++++++++++++++++++ 4 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 crates/helios-cli/src/watch.rs diff --git a/crates/helios-cli/Cargo.toml b/crates/helios-cli/Cargo.toml index 2352efc..772e920 100644 --- a/crates/helios-cli/Cargo.toml +++ b/crates/helios-cli/Cargo.toml @@ -24,5 +24,5 @@ rustls = { version = "0.23", default-features = false, features = ["ring", "std" rustls-native-certs = "0.8" serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" -tokio = { version = "1.53.1", features = ["rt-multi-thread", "net", "sync", "process", "io-util"] } +tokio = { version = "1.53.1", features = ["rt-multi-thread", "net", "sync", "process", "io-util", "time"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } diff --git a/crates/helios-cli/src/main.rs b/crates/helios-cli/src/main.rs index ba63289..fe5dd4c 100644 --- a/crates/helios-cli/src/main.rs +++ b/crates/helios-cli/src/main.rs @@ -3,7 +3,9 @@ mod config; mod transport; mod nix; mod push; +mod watch; +use anyhow::Context; use clap::{Parser, Subcommand}; use serde_json::{Value, json}; @@ -15,6 +17,14 @@ struct Cli { #[arg(long, global = true, env = "HELIOS_SERVER")] server: Option, + /// Server URL, instead of a saved login (for services and CI). + #[arg(long, global = true, env = "HELIOS_URL")] + url: Option, + + /// File holding the token for --url (or set HELIOS_TOKEN). + #[arg(long, global = true, env = "HELIOS_TOKEN_FILE")] + token_file: Option, + #[command(subcommand)] command: Command, } @@ -38,6 +48,26 @@ enum Command { #[arg(long, default_value_t = 3, value_parser = clap::value_parser!(i32).range(1..=19))] level: i32, }, + /// Push paths as they are built: drain a spool filled by `queue-paths`. + WatchStore { + cache: String, + /// Spool directory shared with the post-build hook. + #[arg(long, default_value = "/var/lib/helios-watch-store/spool")] + spool: std::path::PathBuf, + /// Parallel uploads. + #[arg(long, short = 'j', default_value_t = 8)] + jobs: usize, + /// zstd compression level (1-19). + #[arg(long, default_value_t = 3, value_parser = clap::value_parser!(i32).range(1..=19))] + level: i32, + }, + /// Spool store paths for `watch-store` (Nix post-build-hook; reads $OUT_PATHS). + QueuePaths { + #[arg(long, default_value = "/var/lib/helios-watch-store/spool")] + spool: std::path::PathBuf, + /// Paths to queue; defaults to $OUT_PATHS. + paths: Vec, + }, /// Manage caches (admin). #[command(subcommand)] Cache(CacheCmd), @@ -89,12 +119,32 @@ async fn run(cli: Cli) -> anyhow::Result<()> { println!("logged in to '{name}' at {url}"); return Ok(()); } - let client = api::Client::new(&config::server(cli.server.as_deref())?)?; + // Runs inside the nix-daemon's post-build hook: no server needed, and it + // must return quickly. + if let Command::QueuePaths { spool, paths } = &cli.command { + let joined = if paths.is_empty() { std::env::var("OUT_PATHS").unwrap_or_default() } else { paths.join(" ") }; + watch::queue(spool, &joined)?; + return Ok(()); + } + let server = match &cli.url { + Some(url) => { + let token = match &cli.token_file { + Some(file) => std::fs::read_to_string(file).with_context(|| format!("reading {}", file.display()))?, + None => std::env::var("HELIOS_TOKEN").context("--url needs --token-file or HELIOS_TOKEN")?, + }; + config::Server { server: url.clone(), token: token.trim().to_owned() } + } + None => config::server(cli.server.as_deref())?, + }; + let client = api::Client::new(&server)?; match cli.command { - Command::Login { .. } => unreachable!(), + Command::Login { .. } | Command::QueuePaths { .. } => unreachable!(), Command::Push { cache, installables, closure, jobs, level } => { push::push(&client, &cache, &installables, push::Options { jobs, level, closure }).await?; } + Command::WatchStore { cache, spool, jobs, level } => { + watch::watch(&client, &cache, &spool, push::Options { jobs, level, closure: true }).await?; + } Command::Cache(CacheCmd::Create { name, private }) => { print(&client.admin_post("/admin/caches", json!({ "name": name, "public": !private })).await?); } diff --git a/crates/helios-cli/src/push.rs b/crates/helios-cli/src/push.rs index b6a0a9a..2b986ff 100644 --- a/crates/helios-cli/src/push.rs +++ b/crates/helios-cli/src/push.rs @@ -19,6 +19,7 @@ use crate::nix::{self, PathInfo}; const QUERY_BATCH: usize = 50_000; const PUBLISH_BATCH: usize = 1_000; +#[derive(Clone, Copy)] pub struct Options { pub jobs: usize, pub level: i32, diff --git a/crates/helios-cli/src/watch.rs b/crates/helios-cli/src/watch.rs new file mode 100644 index 0000000..a52951e --- /dev/null +++ b/crates/helios-cli/src/watch.rs @@ -0,0 +1,109 @@ +//! Push-on-build. Nix's post-build-hook runs `helios queue-paths`, which +//! drops the built paths into a spool directory and returns at once, so +//! builds never wait on the network. `helios watch-store` drains the spool +//! and pushes each batch's closure, retrying with backoff while the +//! server is unreachable. +//! +//! Spool entries are files of newline-separated store paths, written to a +//! temporary name and renamed into place so readers never see half a file. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::Context; + +use crate::api::Client; +use crate::push; + +const POLL: Duration = Duration::from_secs(1); +const MAX_BACKOFF: Duration = Duration::from_secs(60); + +fn valid(path: &str) -> bool { + path.strip_prefix("/nix/store/").is_some_and(helios_core::store_basename_valid) +} + +/// Writes `paths` (whitespace-separated, as in $OUT_PATHS) as one spool entry. +pub fn queue(spool: &Path, paths: &str) -> anyhow::Result { + let paths: Vec<&str> = paths.split_whitespace().filter(|p| valid(p)).collect(); + if paths.is_empty() { + return Ok(0); + } + std::fs::create_dir_all(spool).with_context(|| format!("creating {}", spool.display()))?; + let stamp = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_nanos()); + let name = format!("{stamp:020}-{}", std::process::id()); + let tmp = spool.join(format!(".{name}")); + std::fs::write(&tmp, paths.join("\n") + "\n")?; + std::fs::rename(&tmp, spool.join(name))?; + Ok(paths.len()) +} + +/// Spool entries in arrival order, skipping ones still being written. +fn pending(spool: &Path) -> anyhow::Result> { + let mut entries: Vec = match std::fs::read_dir(spool) { + Ok(rd) => rd + .flatten() + .map(|e| e.path()) + .filter(|p| p.file_name().and_then(|n| n.to_str()).is_some_and(|n| !n.starts_with('.'))) + .collect(), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(e) => return Err(e.into()), + }; + entries.sort(); + Ok(entries) +} + +pub async fn watch(client: &Client, cache: &str, spool: &Path, opts: push::Options) -> anyhow::Result<()> { + eprintln!("watching {} for paths to push to '{cache}'", spool.display()); + let mut backoff = POLL; + loop { + let entries = pending(spool)?; + if entries.is_empty() { + tokio::time::sleep(POLL).await; + continue; + } + let mut paths: Vec = Vec::new(); + for e in &entries { + let text = std::fs::read_to_string(e).unwrap_or_default(); + paths.extend(text.lines().filter(|p| valid(p)).map(str::to_owned)); + } + paths.sort(); + paths.dedup(); + // Paths garbage-collected since they were built cannot be pushed. + paths.retain(|p| Path::new(p).exists()); + let result = if paths.is_empty() { + Ok(()) + } else { + push::push(client, cache, &paths, push::Options { closure: true, ..opts }).await + }; + match result { + Ok(()) => { + for e in &entries { + let _ = std::fs::remove_file(e); + } + backoff = POLL; + } + Err(e) => { + eprintln!("push failed, retrying in {}s: {e:#}", backoff.as_secs()); + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(MAX_BACKOFF); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn queue_writes_valid_paths_atomically() { + let dir = std::env::temp_dir().join(format!("helios-spool-{}", helios_core::uuid_v4())); + let n = queue(&dir, "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello /etc/passwd\n/nix/store/bad").unwrap(); + assert_eq!(n, 1); + let entries = pending(&dir).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!(std::fs::read_to_string(&entries[0]).unwrap(), "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello\n"); + assert_eq!(queue(&dir, "").unwrap(), 0); + std::fs::remove_dir_all(&dir).unwrap(); + } +} From 8e8f99e0df17e024b2a123c06fbb36b71f5147eb Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 13:28:01 +0200 Subject: [PATCH 19/56] nix: run the maintenance daemon and watch-store from the module services.helios.daemon runs helios-daemon beside the server by default, with WAL checkpoints, daily backups and a weekly scrub; auto-GC starts once a quota or a free-space floor is set. It runs as its own user at idle I/O priority, without network access unless metrics are enabled. The server's maintenance socket lives in a setgid directory owned by a dedicated helios-admin group, so the daemon can reach it and nginx, which shares the helios group to read NARs, cannot. services.helios.watchStore is for build machines and works without the server. It installs a Nix post-build hook that spools each build's outputs and a unit that pushes them with helios watch-store. The hook never fails, because a failing post-build hook fails the build. The VM test now also checks socket permissions, metrics, rotated backups, a scrub that quarantines a corrupted NAR, a build pushed by watch-store without anyone running push, eviction under a quota, and a clean SIGTERM shutdown of the daemon. --- nix/module.nix | 265 ++++++++++++++++++++++++++++++++++++++++++++----- nix/test.nix | 118 ++++++++++++++++++---- 2 files changed, 339 insertions(+), 44 deletions(-) diff --git a/nix/module.nix b/nix/module.nix index e7dbf24..67fa72c 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -22,6 +22,64 @@ let dataDir = cfg.dataDir; defaultDataDir = "/var/lib/helios"; secret = name: file: optional (file != null) "${name}:${file}"; + + # The maintenance socket lives in a setgid directory, so it belongs to + # group helios-admin: the daemon can reach it, nginx (group helios) cannot. + adminDir = "/run/helios-admin"; + adminSocket = "${adminDir}/admin.sock"; + + daemonArgs = lib.cli.toCommandLineShellGNU { } ( + { + socket = adminSocket; + data-dir = dataDir; + gc-interval = cfg.daemon.gcInterval; + scrub-interval = if cfg.daemon.scrub.interval == null then "0" else cfg.daemon.scrub.interval; + scrub-rate = cfg.daemon.scrub.rate; + checkpoint-interval = cfg.daemon.checkpointInterval; + optimize-interval = cfg.daemon.optimizeInterval; + backup-interval = if cfg.daemon.backup.interval == null then "0" else cfg.daemon.backup.interval; + backup-keep = cfg.daemon.backup.keep; + quota-high = cfg.daemon.quotaHigh; + quota-low = cfg.daemon.quotaLow; + } + // lib.optionalAttrs (cfg.daemon.quota != null) { quota = cfg.daemon.quota; } + // lib.optionalAttrs (cfg.daemon.minFree != null) { min-free = cfg.daemon.minFree; } + // lib.optionalAttrs (cfg.daemon.metrics.listen != null) { metrics-listen = cfg.daemon.metrics.listen; } + ); + + watchSpool = "/var/lib/helios-watch-store/spool"; + # Runs as root inside the nix-daemon after every build; a failing + # post-build hook fails the build, so queueing problems are only logged. + queueHook = pkgs.writeShellScript "helios-queue-paths" '' + ${cfg.package}/bin/helios queue-paths --spool ${watchSpool} \ + || echo "helios: could not queue $OUT_PATHS for pushing" >&2 + exit 0 + ''; + + hardening = { + NoNewPrivileges = true; + CapabilityBoundingSet = ""; + AmbientCapabilities = ""; + ProtectSystem = "strict"; + ProtectHome = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectKernelTunables = true; + ProtectKernelModules = true; + ProtectKernelLogs = true; + ProtectControlGroups = true; + ProtectClock = true; + ProtectHostname = true; + ProtectProc = "invisible"; + ProcSubset = "pid"; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + LockPersonality = true; + MemoryDenyWriteExecute = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ "@system-service" "~@privileged" ]; + }; credential = flag: name: file: optionalString (file != null) " ${flag} %d/${name}"; in { @@ -124,9 +182,106 @@ in defaultText = lib.literalExpression "config.services.helios.nginx.enable"; description = "Open TCP 80 and 443."; }; + + daemon = { + enable = mkOption { + type = types.bool; + default = true; + description = '' + Run helios-daemon beside the server: WAL checkpoints, database + backups and an integrity scrub, plus auto-GC once a quota is set. + ''; + }; + quota = mkOption { + type = types.nullOr types.str; + default = null; + example = "500G"; + description = "Keep stored NARs under this size by evicting least-recently-used paths. Null disables auto-GC by size."; + }; + quotaHigh = mkOption { + type = types.float; + default = 0.9; + description = "Start evicting above this fraction of the quota."; + }; + quotaLow = mkOption { + type = types.float; + default = 0.8; + description = "Evict down to this fraction of the quota."; + }; + minFree = mkOption { + type = types.nullOr types.str; + default = null; + example = "20G"; + description = "Also evict while the data filesystem has less free space than this."; + }; + gcInterval = mkOption { + type = types.str; + default = "5m"; + }; + scrub = { + interval = mkOption { + type = types.nullOr types.str; + default = "7d"; + description = "Time between full integrity scrubs; null disables."; + }; + rate = mkOption { + type = types.str; + default = "64M"; + description = "Scrub read rate per second; \"0\" is unlimited."; + }; + }; + checkpointInterval = mkOption { + type = types.str; + default = "15m"; + }; + optimizeInterval = mkOption { + type = types.str; + default = "1d"; + }; + backup = { + interval = mkOption { + type = types.nullOr types.str; + default = "1d"; + description = "Time between database backups (to /backups); null disables."; + }; + keep = mkOption { + type = types.ints.positive; + default = 7; + }; + }; + metrics.listen = mkOption { + type = types.nullOr types.str; + default = null; + example = "127.0.0.1:9120"; + description = "Serve Prometheus metrics on this address."; + }; + }; + + watchStore = { + enable = mkEnableOption "pushing every locally built path to a Helios cache (post-build hook)"; + url = mkOption { + type = types.str; + example = "https://cache.example.com"; + description = "Helios server to push to."; + }; + cache = mkOption { + type = types.str; + example = "main"; + }; + tokenFile = mkOption { + type = types.path; + description = "File holding a push token for `cache`; read as a systemd credential."; + }; + jobs = mkOption { + type = types.ints.positive; + default = 8; + description = "Parallel uploads."; + }; + }; }; - config = mkIf cfg.enable { + config = lib.mkMerge [ + (mkIf cfg.enable { assertions = [ { assertion = cfg.nginx.enable -> cfg.domain != null; @@ -154,6 +309,9 @@ in HELIOS_GC_INTERVAL_HOURS = toString cfg.settings.gcIntervalHours; HELIOS_AUDIT_RETENTION_DAYS = toString cfg.settings.auditRetentionDays; } + // lib.optionalAttrs cfg.daemon.enable { + HELIOS_ADMIN_SOCKET = adminSocket; + } // lib.optionalAttrs cfg.nginx.enable { HELIOS_ACCEL_REDIRECT = "/_nar"; HELIOS_TRUST_PROXY = "true"; @@ -175,37 +333,16 @@ in # nginx (in group helios) reads NARs for X-Accel-Redirect. StateDirectory = mkIf (dataDir == defaultDataDir) "helios"; StateDirectoryMode = "0750"; - ReadWritePaths = mkIf (dataDir != defaultDataDir) [ dataDir ]; + ReadWritePaths = optional (dataDir != defaultDataDir) dataDir ++ optional cfg.daemon.enable adminDir; UMask = "0027"; Restart = "on-failure"; RestartSec = 2; LimitNOFILE = 65536; - # Sandboxing: the server only needs its state directory and the network. - NoNewPrivileges = true; - CapabilityBoundingSet = ""; - AmbientCapabilities = ""; - ProtectSystem = "strict"; - ProtectHome = true; - PrivateTmp = true; - PrivateDevices = true; - ProtectKernelTunables = true; - ProtectKernelModules = true; - ProtectKernelLogs = true; - ProtectControlGroups = true; - ProtectClock = true; - ProtectHostname = true; - ProtectProc = "invisible"; - ProcSubset = "pid"; + # The server only needs its state directory and the network. RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ]; - RestrictNamespaces = true; - RestrictRealtime = true; - RestrictSUIDSGID = true; - LockPersonality = true; - MemoryDenyWriteExecute = true; - SystemCallArchitectures = "native"; - SystemCallFilter = [ "@system-service" "~@privileged" ]; - }; + } + // hardening; }; systemd.tmpfiles.rules = mkIf (dataDir != defaultDataDir) [ "d '${dataDir}' 0750 helios helios -" ]; @@ -240,5 +377,79 @@ in }; networking.firewall.allowedTCPPorts = mkIf cfg.openFirewall [ 80 443 ]; - }; + }) + + (mkIf (cfg.enable && cfg.daemon.enable) { + users.users.helios-daemon = { + isSystemUser = true; + group = "helios-admin"; + # Reads NARs for the scrub. + extraGroups = [ "helios" ]; + }; + users.groups.helios-admin = { }; + + systemd.tmpfiles.rules = [ "d ${adminDir} 2750 helios helios-admin -" ]; + + systemd.services.helios-daemon = { + description = "Helios cache maintenance"; + wantedBy = [ "multi-user.target" ]; + after = [ "helios.service" ]; + wants = [ "helios.service" ]; + environment.HELIOS_LOG = cfg.logLevel; + serviceConfig = { + ExecStart = "${cfg.package}/bin/helios-daemon ${daemonArgs}"; + User = "helios-daemon"; + Group = "helios-admin"; + SupplementaryGroups = [ "helios" ]; + Restart = "on-failure"; + RestartSec = 5; + # Maintenance must not compete with serving. + Nice = 10; + CPUSchedulingPolicy = "batch"; + IOSchedulingClass = "idle"; + # Reaches the server over its Unix socket; the network only for metrics. + PrivateNetwork = cfg.daemon.metrics.listen == null; + RestrictAddressFamilies = [ "AF_UNIX" ] ++ lib.optionals (cfg.daemon.metrics.listen != null) [ "AF_INET" "AF_INET6" ]; + } + // hardening; + }; + }) + + (mkIf cfg.watchStore.enable { + users.users.helios-watch-store = { + isSystemUser = true; + group = "helios-watch-store"; + home = "/var/lib/helios-watch-store"; + }; + users.groups.helios-watch-store = { }; + + nix.settings.post-build-hook = queueHook; + systemd.tmpfiles.rules = [ "d ${watchSpool} 0750 helios-watch-store helios-watch-store -" ]; + + systemd.services.helios-watch-store = { + description = "Push locally built paths to ${cfg.watchStore.url}"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" "nix-daemon.socket" ]; + wants = [ "network-online.target" ]; + path = [ config.nix.package ]; + environment = { + HELIOS_URL = cfg.watchStore.url; + HOME = "/var/lib/helios-watch-store"; + }; + serviceConfig = { + ExecStart = "${cfg.package}/bin/helios --token-file %d/token watch-store ${lib.escapeShellArg cfg.watchStore.cache} --spool ${watchSpool} --jobs ${toString cfg.watchStore.jobs}"; + LoadCredential = [ "token:${cfg.watchStore.tokenFile}" ]; + User = "helios-watch-store"; + Group = "helios-watch-store"; + StateDirectory = "helios-watch-store"; + StateDirectoryMode = "0750"; + Restart = "always"; + RestartSec = 5; + Nice = 10; + RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ]; + } + // hardening; + }; + }) + ]; } diff --git a/nix/test.nix b/nix/test.nix index b9c83d4..da6d6a1 100644 --- a/nix/test.nix +++ b/nix/test.nix @@ -1,8 +1,12 @@ -# NixOS VM test for services.helios: a server behind nginx and a client -# that pushes a closure with the CLI, then substitutes it back through Nix -# with signatures required. +# NixOS VM test for services.helios: a server behind nginx with the +# maintenance daemon, and a client that pushes a closure with the CLI, +# substitutes it back through Nix with signatures required, and pushes +# new builds automatically with watchStore. { pkgs, module, helios }: +let + system = pkgs.stdenv.hostPlatform.system; +in pkgs.testers.runNixOSTest { name = "helios"; @@ -20,27 +24,72 @@ pkgs.testers.runNixOSTest { signingKeyFile = "/etc/helios/signing-key"; jwtSecretFile = "/etc/helios/jwt-secret"; adminSecretFile = "/etc/helios/admin-secret"; + daemon = { + metrics.listen = "127.0.0.1:9120"; + checkpointInterval = "5s"; + scrub = { + interval = "5s"; + rate = "0"; + }; + backup = { + interval = "5s"; + keep = 2; + }; + }; }; # Started by the test once the secrets exist. systemd.services.helios.wantedBy = lib.mkForce [ ]; + systemd.services.helios-daemon.wantedBy = lib.mkForce [ ]; nix.settings.experimental-features = [ "nix-command" ]; }; - nodes.client = { - environment.systemPackages = [ - helios - pkgs.hello - pkgs.jq - ]; - nix.settings.experimental-features = [ "nix-command" ]; - virtualisation.memorySize = 2048; - }; + nodes.client = + { lib, ... }: + { + imports = [ module ]; + services.helios = { + package = helios; + watchStore = { + enable = true; + url = "http://server"; + cache = "main"; + tokenFile = "/etc/helios-push-token"; + }; + }; + systemd.services.helios-watch-store.wantedBy = lib.mkForce [ ]; + environment.systemPackages = [ + helios + pkgs.hello + ]; + # A derivation that builds offline in the VM. storePath gives the + # builder string context, so the build sandbox includes busybox. + environment.etc."watched.nix".text = '' + derivation { + name = "watched"; + system = "${system}"; + builder = "''${builtins.storePath "${pkgs.busybox}"}/bin/sh"; + args = [ "-c" "echo built > $out" ]; + } + ''; + nix.settings.experimental-features = [ "nix-command" ]; + virtualisation.memorySize = 2048; + }; testScript = '' import json start_all() + def narinfo_url(path): + return f"http://server/main/{path.split('/')[-1][:32]}.narinfo" + + def metric(name): + out = server.succeed("curl -sf http://127.0.0.1:9120/metrics") + for line in out.splitlines(): + if line.startswith(name + " ") or line.startswith(name + "{"): + return line + raise Exception(f"{name} not in metrics:\n{out}") + with subtest("secrets and startup"): server.succeed( "install -d -m 0700 /etc/helios", @@ -48,9 +97,10 @@ pkgs.testers.runNixOSTest { "head -c 32 /dev/urandom | base64 > /etc/helios/jwt-secret", "head -c 32 /dev/urandom | base64 > /etc/helios/admin-secret", "chmod 0400 /etc/helios/*", - "systemctl start helios", + "systemctl start helios helios-daemon", ) server.wait_for_unit("helios.service") + server.wait_for_unit("helios-daemon.service") server.wait_for_unit("nginx.service") server.wait_for_open_port(8080) server.wait_for_open_port(80) @@ -63,6 +113,10 @@ pkgs.testers.runNixOSTest { server.fail("sudo -u helios cat /etc/helios/signing-key") props = server.succeed("systemctl show helios -p NoNewPrivileges -p ProtectSystem -p User") assert "NoNewPrivileges=yes" in props and "ProtectSystem=strict" in props and "User=helios" in props, props + # nginx may read NARs but not reach the maintenance socket. + server.succeed("stat -c %G /run/helios-admin/admin.sock | grep -qx helios-admin") + server.fail("sudo -u nginx curl -sf --unix-socket /run/helios-admin/admin.sock http://x/v1/stats") + server.succeed("sudo -u helios-daemon curl -sf --unix-socket /run/helios-admin/admin.sock http://x/v1/stats") with subtest("admin and push"): client.succeed(f"helios login admin http://server {admin}") @@ -81,19 +135,49 @@ pkgs.testers.runNixOSTest { client.succeed(f"nix store verify --store /tmp/store --trusted-public-keys '{pubkey}' -r ${pkgs.hello}") with subtest("nginx serves NARs from disk"): - url = client.succeed("curl -sf http://server/main/$(basename ${pkgs.hello} | cut -c1-32).narinfo | sed -n 's/^URL: //p'").strip() + url = client.succeed(f"curl -sf {narinfo_url('${pkgs.hello}')} | sed -n 's/^URL: //p'").strip() client.succeed(f"curl -sf -o /dev/null http://server/main/{url}") # The internal location is not reachable directly. client.fail("curl -sf -o /dev/null http://server/_nar/") server.succeed("find /srv/helios/nar -name '*.nar.zst' | grep -q .") server.succeed("stat -c %U:%G:%a /srv/helios | grep -qx helios:helios:750") - with subtest("state survives restarts; SIGTERM shuts down cleanly"): + with subtest("state survives restarts"): server.succeed("systemctl restart helios") server.wait_for_open_port(8080) - client.succeed("curl -sf http://server/main/$(basename ${pkgs.hello} | cut -c1-32).narinfo") - server.succeed("systemctl stop helios") + client.succeed(f"curl -sf {narinfo_url('${pkgs.hello}')}") + + with subtest("daemon: metrics, backups, scrub"): + server.wait_until_succeeds("curl -sf http://127.0.0.1:9120/metrics | grep -qx 'helios_up 1'") + assert int(metric('helios_cache_paths').split()[-1]) > 0 + server.wait_until_succeeds("test $(ls /srv/helios/backups | wc -l) -eq 2", timeout=60) + # Corrupt a dependency's NAR; the scrub must take it out of service. + url = client.succeed(f"curl -sf {narinfo_url('${pkgs.glibc}')} | sed -n 's/^URL: nar\\///p'").strip() + server.succeed(f"printf garbage | dd of=/srv/helios/nar/{url[:2]}/{url} bs=1 seek=100 conv=notrunc") + server.wait_until_succeeds("ls /srv/helios/quarantine | grep -q .", timeout=60) + client.fail(f"curl -sf {narinfo_url('${pkgs.glibc}')}") + assert metric('helios_scrub_blobs_total{result="corrupt"}').endswith(" 1") + + with subtest("watch-store pushes new builds"): + client.succeed(f"echo {token} > /etc/helios-push-token", "systemctl start helios-watch-store") + built = client.succeed("nix-build /etc/watched.nix --no-out-link").strip() + client.wait_until_succeeds(f"curl -sf {narinfo_url(built)}", timeout=60) + + with subtest("auto-gc evicts least recently used paths"): + before = int(metric('helios_cache_paths').split()[-1]) + server.succeed( + "sudo -u helios-daemon timeout 5 ${helios}/bin/helios-daemon " + "--socket /run/helios-admin/admin.sock --data-dir /srv/helios " + "--quota 1K --gc-interval 1s --scrub-interval 0 --backup-interval 0 || test $? -eq 124" + ) + after = int(metric('helios_cache_paths').split()[-1]) + assert after < before, (before, after) + + with subtest("SIGTERM shuts down cleanly"): + server.succeed("systemctl stop helios-daemon helios") server.succeed("journalctl -u helios | grep -q 'shutting down'") + server.succeed("journalctl -u helios-daemon | grep -q 'shutting down'") server.fail("systemctl is-failed helios") + server.fail("systemctl is-failed helios-daemon") ''; } From cd9981cd9c34973e8fdc1717a11246c50e3028f1 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 13:29:02 +0200 Subject: [PATCH 20/56] docs: document the daemon and watch-store Describe helios-daemon's auto-GC, integrity scrub, SQLite upkeep and metrics, the maintenance socket they run over, the watch-store flow for build machines, and the NixOS options for both. The server reference gains --admin-socket and --upload-grace-seconds, and the storage table the backups and quarantine directories. --- AGENTS.md | 4 ++- README.md | 7 +++-- docs.md | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 84 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b9eb18e..bb834b3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,7 +9,9 @@ A self-hosted Nix binary cache: Rust server and CLI over a Zig core. - `core/`: libhelios, the C ABI over `pkg/*`, compiled to a static library by `crates/helios-core/build.rs`. `core/include/helios.h` is the contract. - `crates/helios-core`: safe Rust bindings to libhelios. -- `crates/helios-server`, `crates/helios-cli`: the binaries. +- `crates/helios-server`, `crates/helios-cli`, `crates/helios-daemon`: the + binaries. The daemon drives the server over its maintenance socket and + never writes the database itself. - `nix/`: the NixOS module (`services.helios`) and its VM test. - `bench/`: comparisons against Cachix's libraries and Nix C++. Not part of the Cargo workspace. diff --git a/README.md b/README.md index 3113f4a..569ac78 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,9 @@ compression, narinfo and signing. pkg/ Zig packages: nix-base32, nix-store-path, nix-archive, nix-narinfo, nix-derivation, nix-flake-lock core/ libhelios: C ABI over pkg/*, built as a static library -crates/ helios-core (Rust bindings), helios-server, helios-cli +crates/ helios-core (Rust bindings), helios-server, helios-cli, + helios-daemon (auto-GC, scrub, backups, metrics) +nix/ NixOS module (services.helios) and its VM test bench/ benchmarks against Cachix's Nix libraries and Nix C++ scripts/ e2e.sh (end-to-end test), test-zig.sh (Zig unit tests) ``` @@ -62,7 +64,8 @@ nix.settings = { }; ``` -On NixOS, use the flake's `nixosModules.default` (`services.helios`). +On NixOS, use the flake's `nixosModules.default` (`services.helios`): the +server, the maintenance daemon, and push-on-build for build machines. See [docs.md](docs.md) for the NixOS module, configuration, private caches, the HTTP API and deployment behind a reverse proxy. diff --git a/docs.md b/docs.md index 31d01f8..3c2445a 100644 --- a/docs.md +++ b/docs.md @@ -9,6 +9,8 @@ Everything lives under `--data-dir`: | `helios.db` | SQLite (WAL): caches, blobs, published paths, tokens, audit log | | `nar//.nar.zst` | Content-addressed compressed NARs | | `tmp/` | In-flight uploads, renamed into `nar/` once verified | +| `backups/` | Database backups made on request of helios-daemon | +| `quarantine/` | NARs the integrity scrub found missing or corrupt | Uploads are decompressed and hashed by the server as they stream in, so a path is only published against a NAR whose hash the server computed itself. @@ -31,6 +33,8 @@ Every flag can also be set through its environment variable. | `--max-upload-bytes` | `HELIOS_MAX_UPLOAD_BYTES` | 64 GiB | | `--gc-interval-hours` | `HELIOS_GC_INTERVAL_HOURS` | `6` | | `--audit-retention-days` | `HELIOS_AUDIT_RETENTION_DAYS` | `30` | +| `--upload-grace-seconds` | `HELIOS_UPLOAD_GRACE_SECONDS` | `3600`: an unpublished upload is kept this long | +| `--admin-socket` | `HELIOS_ADMIN_SOCKET` | unset: no maintenance API (see helios-daemon) | Secret files must hold at least 16 bytes. `HELIOS_LOG` sets the log level (`error`, `warn`, `info` or `debug`; default `info`). Under systemd, log lines @@ -90,8 +94,33 @@ placed by hand). Without `domain`, the module runs only the server on | `logLevel` | `info` | | | `settings.*` | | `narinfoCacheEntries`, `maxUploadBytes`, `gcIntervalHours`, `auditRetentionDays` | -`overlays.default` adds `pkgs.helios`. `nix flake check` runs a VM test that -pushes a closure through the module and substitutes it back. +`services.helios.daemon` (on by default) runs helios-daemon beside the +server; see [Maintenance](#maintenance). To keep the cache under a size: + +```nix +services.helios.daemon = { + quota = "500G"; + minFree = "20G"; + metrics.listen = "127.0.0.1:9120"; +}; +``` + +`services.helios.watchStore` goes on build machines, and needs no server +there. It installs a Nix post-build hook, so every path the machine builds is +pushed to a cache: + +```nix +services.helios.watchStore = { + enable = true; + url = "https://cache.example.com"; + cache = "main"; + tokenFile = "/run/secrets/helios-push-token"; +}; +``` + +`overlays.default` adds `pkgs.helios`. `nix flake check` runs a VM test of +all of the above: push, signed substitution, sandboxing, metrics, backups, +a scrub that catches a corrupted NAR, watch-store, eviction and shutdown. ## Reverse proxy and zero-copy downloads @@ -178,9 +207,53 @@ Admin endpoints (bearer admin secret): `GET/POST /_api/v2/admin/caches`, ## Garbage collection Every `--gc-interval-hours`, the server removes stale temp uploads, blobs no -path references (after a one-hour grace period, so an upload is never +path references (once past `--upload-grace-seconds`, so an upload is never collected before it is published), expired tokens, and old audit log rows. +## Maintenance + +helios-daemon keeps a cache healthy without touching the database itself: +it decides what to do and asks the server, over the Unix socket given to +the server as `--admin-socket`, so the server's in-memory index always +agrees with SQLite. The socket's permissions are its access control. + +- **Auto-GC.** With `--quota`, stored NARs are kept under that size. + Eviction starts above `--quota-high` (0.9) and removes the least recently + used paths down to `--quota-low` (0.8). Recency comes from narinfo hits, + which the server records in memory and flushes once a minute. + `--min-free` also evicts while the disk is short on space. NARs another + path still uses stay, and an evicted NAR is deleted once its upload grace + period has passed. +- **Integrity scrub.** Every `--scrub-interval` (7d), every NAR is read at up + to `--scrub-rate` (64M per second), then decompressed and re-hashed. A + missing or corrupt NAR is quarantined, which unpublishes its paths, so no + client downloads bad data. +- **SQLite.** WAL checkpoints (`--checkpoint-interval`, 15m), `PRAGMA + optimize` (`--optimize-interval`, 1d) and online backups to + `/backups` (`--backup-interval`, 1d; `--backup-keep`, 7). +- **Metrics.** `--metrics-listen` serves Prometheus metrics: + - cache paths, NAR bytes, and narinfo hits and misses; + - uploads; + - auto-GC, scrub and backup results; + - disk space. + +Intervals take a unit (`30s`, `5m`, `24h`, `7d`) and sizes an optional +binary suffix (`64M`, `500G`). A failed run is logged, counted in +`helios_maintenance_errors_total`, and retried on the next tick. + +The maintenance API (`/v1/stats`, `/v1/lru`, `/v1/evict`, `/v1/blobs`, +`/v1/quarantine`, `/v1/gc`, `/v1/db/checkpoint`, `/v1/db/optimize`, +`/v1/db/backup`) is internal and may change between versions. + +## Watch-store + +On a build machine, `helios queue-paths` works as Nix's `post-build-hook`. +It spools each build's outputs and returns at once, so builds never wait on +the network. `helios watch-store ` pushes the spooled closures and +retries with backoff while the server is unreachable. Both take `--spool`, +and the CLI accepts `--url` with `--token-file` (or `HELIOS_URL` and +`HELIOS_TOKEN`) instead of a saved login. + ## Benchmarks ```bash From c0d98008108effb1e11cba57a7a2822066b196de Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 21/56] core: generate nix signing keys Signer.generate derives an Ed25519 key pair from a seed and writes it in the name:base64 form that nix key generate-secret produces, so a fresh deployment can create its own signing key without Nix installed. libhelios exposes it as hl_signer_generate, seeded from hl_random, and helios-core as Signer::generate. --- core/include/helios.h | 2 ++ core/src/root.zig | 15 +++++++++++++++ crates/helios-core/src/lib.rs | 19 +++++++++++++++++++ pkg/nix-narinfo/src/render.zig | 26 ++++++++++++++++++++++++++ 4 files changed, 62 insertions(+) diff --git a/core/include/helios.h b/core/include/helios.h index 8dd83a8..f4fcc3b 100644 --- a/core/include/helios.h +++ b/core/include/helios.h @@ -83,6 +83,8 @@ hl_signer *hl_signer_new(const char *key, size_t len); void hl_signer_free(hl_signer *s); /* Writes ":". Sets *out_len even on HL_E_BUFFER. */ int hl_signer_public_key(const hl_signer *s, char *out, size_t cap, size_t *out_len); +/* Generates a new secret key in the same format from the OS CSPRNG. */ +int hl_signer_generate(const char *name, size_t name_len, char *out, size_t cap, size_t *out_len); typedef struct { const char *ptr; diff --git a/core/src/root.zig b/core/src/root.zig index 99f70a3..94993ed 100644 --- a/core/src/root.zig +++ b/core/src/root.zig @@ -162,6 +162,21 @@ export fn hl_signer_free(s: ?*narinfo.Signer) void { if (s) |p| p.destroy(gpa); } +/// Generates a Nix secret key named `name` from the OS CSPRNG. +export fn hl_signer_generate(name: [*]const u8, name_len: usize, out: ?[*]u8, cap: usize, out_len: *usize) c_int { + var seed: [32]u8 = undefined; + const rc = hl_random(&seed, seed.len); + if (rc != HL_OK) return rc; + defer std.crypto.secureZero(u8, &seed); + var list: std.ArrayList(u8) = .empty; + defer { + std.crypto.secureZero(u8, list.items); + list.deinit(gpa); + } + narinfo.Signer.generate(&list, gpa, name[0..name_len], seed) catch |e| return code(e); + return copyOut(list.items, out, cap, out_len); +} + fn copyOut(bytes: []const u8, out: ?[*]u8, cap: usize, out_len: *usize) c_int { out_len.* = bytes.len; if (bytes.len > cap or out == null) return HL_E_BUFFER; diff --git a/crates/helios-core/src/lib.rs b/crates/helios-core/src/lib.rs index 708c115..a014e9f 100644 --- a/crates/helios-core/src/lib.rs +++ b/crates/helios-core/src/lib.rs @@ -68,6 +68,7 @@ mod sys { pub fn hl_signer_new(key: *const u8, len: usize) -> *mut Signer; pub fn hl_signer_free(s: *mut Signer); pub fn hl_signer_public_key(s: *const Signer, out: *mut u8, cap: usize, out_len: *mut usize) -> c_int; + pub fn hl_signer_generate(name: *const u8, name_len: usize, out: *mut u8, cap: usize, out_len: *mut usize) -> c_int; pub fn hl_narinfo_render(input: *const NarinfoInput, signer: *const Signer, out: *mut *mut u8, out_len: *mut usize) -> c_int; pub fn hl_free(ptr: *mut u8, len: usize); pub fn hl_sha256(data: *const u8, len: usize, out: *mut [u8; 32]); @@ -315,6 +316,15 @@ impl Signer { (!raw.is_null()).then_some(Self { raw }) } + /// A new Nix secret key named `name`, in `nix key generate-secret` format. + pub fn generate(name: &str) -> Result { + let mut buf = vec![0u8; 256]; + let mut len = 0usize; + check(unsafe { sys::hl_signer_generate(name.as_ptr(), name.len(), buf.as_mut_ptr(), buf.len(), &mut len) })?; + buf.truncate(len); + Ok(String::from_utf8(buf).expect("key name was valid UTF-8")) + } + /// `:`, the value for `trusted-public-keys`. pub fn public_key(&self) -> String { let mut buf = vec![0u8; 256]; @@ -465,6 +475,15 @@ mod tests { assert_eq!(nix32_decode::<20>("not-valid"), None); } + #[test] + fn generated_keys_round_trip() { + let key = Signer::generate("gen-1").unwrap(); + assert!(key.starts_with("gen-1:")); + assert_ne!(key, Signer::generate("gen-1").unwrap()); + assert!(Signer::new(&key).unwrap().public_key().starts_with("gen-1:")); + assert!(Signer::generate("bad:name").is_err()); + } + #[test] fn signer_public_key() { let signer = Signer::new(KEY).unwrap(); diff --git a/pkg/nix-narinfo/src/render.zig b/pkg/nix-narinfo/src/render.zig index 22f521b..a56fdc7 100644 --- a/pkg/nix-narinfo/src/render.zig +++ b/pkg/nix-narinfo/src/render.zig @@ -33,6 +33,20 @@ pub const Signer = struct { /// Comb table for the base point: signing does 64 additions, not 252 doublings. table: basemul.Table, + /// Writes a new Nix secret key, `:` (the + /// `nix key generate-secret` format), derived from a random `seed`. + pub fn generate(out: *std.ArrayList(u8), allocator: std.mem.Allocator, name: []const u8, seed: [32]u8) Error!void { + if (name.len == 0) return error.InvalidKey; + for (name) |c| if (c == ':' or c == '\n' or c == ' ') return error.InvalidKey; + const key_pair = Ed25519.KeyPair.generateDeterministic(seed) catch return error.InvalidKey; + const bytes = key_pair.secret_key.toBytes(); + var b64: [std.base64.standard.Encoder.calcSize(bytes.len)]u8 = undefined; + _ = std.base64.standard.Encoder.encode(&b64, &bytes); + try out.appendSlice(allocator, name); + try out.append(allocator, ':'); + try out.appendSlice(allocator, &b64); + } + /// Parses a Nix secret key: `:`. pub fn parse(allocator: std.mem.Allocator, text: []const u8) Error!*Signer { const trimmed = std.mem.trim(u8, text, " \t\r\n"); @@ -281,3 +295,15 @@ test "fast signing is byte-identical to std Ed25519" { try std.testing.expectEqualSlices(u8, &theirs, &ours); } } + +test "generated keys parse and sign" { + const a = std.testing.allocator; + var text: std.ArrayList(u8) = .empty; + defer text.deinit(a); + try Signer.generate(&text, a, "gen-1", [_]u8{42} ** 32); + const signer = try Signer.parse(a, text.items); + defer signer.destroy(a); + const theirs = (try signer.key_pair.sign("msg", null)).toBytes(); + try std.testing.expectEqualSlices(u8, &theirs, &(try signer.signRaw("msg"))); + try std.testing.expectError(error.InvalidKey, Signer.generate(&text, a, "bad:name", [_]u8{1} ** 32)); +} From b79bf7c29b405440c9f3ccbad5f99e3c4ef77444 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 22/56] server: add generate-secrets for first-run setup helios-server generate-secrets --dir creates a signing key, a token secret and an admin secret, each 0600 and only when missing, and writes and prints the public key. Rerunning it is safe, which lets a container entrypoint call it on every start. --- crates/helios-server/src/config.rs | 18 +++++++++++- crates/helios-server/src/main.rs | 44 ++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/crates/helios-server/src/config.rs b/crates/helios-server/src/config.rs index b5adb0b..63da1b2 100644 --- a/crates/helios-server/src/config.rs +++ b/crates/helios-server/src/config.rs @@ -2,12 +2,15 @@ use std::net::SocketAddr; use std::path::PathBuf; use std::time::Duration; -use clap::Parser; +use clap::{Parser, Subcommand}; /// Helios Nix binary cache server. #[derive(Parser, Debug, Clone)] #[command(version)] pub struct Args { + #[command(subcommand)] + pub command: Option, + /// Address to listen on. #[arg(long, env = "HELIOS_LISTEN", default_value = "127.0.0.1:8080")] pub listen: SocketAddr, @@ -69,6 +72,19 @@ pub struct Args { pub print_public_key: bool, } +#[derive(Subcommand, Debug, Clone)] +pub enum Command { + /// Create whichever of signing-key, jwt-secret and admin-secret are + /// missing in DIR (mode 0600), and write public-key. Safe to rerun. + GenerateSecrets { + #[arg(long, env = "HELIOS_SECRETS_DIR")] + dir: PathBuf, + /// Name for a new signing key, as it appears in `trusted-public-keys`. + #[arg(long, env = "HELIOS_KEY_NAME", default_value = "helios-1")] + key_name: String, + }, +} + #[derive(Debug, Clone)] pub struct Config { pub data_dir: PathBuf, diff --git a/crates/helios-server/src/main.rs b/crates/helios-server/src/main.rs index ef6cf20..b0ec707 100644 --- a/crates/helios-server/src/main.rs +++ b/crates/helios-server/src/main.rs @@ -81,6 +81,9 @@ fn main() -> anyhow::Result<()> { log::init(); let args = Args::parse(); + if let Some(config::Command::GenerateSecrets { dir, key_name }) = &args.command { + return generate_secrets(dir, key_name); + } if args.print_public_key { let signer = load_signer(&args)?.ok_or_else(|| anyhow::anyhow!("--signing-key-file is required"))?; println!("{}", signer.public_key()); @@ -134,3 +137,44 @@ async fn shutdown_signal() { futures_util::future::select(term, int).await; crate::log::info!("shutting down"); } + +/// Writes `contents` to `path` only if it does not exist, readable by the +/// owner alone. Returns whether it was created. +fn create_secret(path: &std::path::Path, contents: impl FnOnce() -> anyhow::Result) -> anyhow::Result { + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; + if path.exists() { + return Ok(false); + } + let mut file = std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(path)?; + file.write_all(contents()?.as_bytes())?; + file.write_all(b"\n")?; + Ok(true) +} + +fn random_hex() -> anyhow::Result { + let mut bytes = [0u8; 32]; + helios_core::random_bytes(&mut bytes); + Ok(bytes.iter().map(|b| format!("{b:02x}")).collect()) +} + +fn generate_secrets(dir: &std::path::Path, key_name: &str) -> anyhow::Result<()> { + use std::os::unix::fs::PermissionsExt; + std::fs::create_dir_all(dir)?; + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))?; + let signing = dir.join("signing-key"); + for (name, created) in [ + ("signing-key", create_secret(&signing, || Ok(helios_core::Signer::generate(key_name)?))?), + ("jwt-secret", create_secret(&dir.join("jwt-secret"), random_hex)?), + ("admin-secret", create_secret(&dir.join("admin-secret"), random_hex)?), + ] { + if created { + log::info!("created {}", dir.join(name).display()); + } + } + let key = std::fs::read_to_string(&signing)?; + let public = helios_core::Signer::new(&key).ok_or_else(|| anyhow::anyhow!("{} is not a Nix secret key", signing.display()))?.public_key(); + std::fs::write(dir.join("public-key"), format!("{public}\n"))?; + println!("{public}"); + Ok(()) +} From d027fc031b96a74784a788ed287ca7bdd10cb7d1 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 23/56] nix: add an oci image with the server and daemon packages.docker is a layered image that runs helios-server and helios-daemon as uid 10000 with all state in /var/lib/helios. The entrypoint generates secrets on first start unless HELIOS_*_FILE provides them, forwards SIGTERM to both processes, and exits non-zero when either dies so the restart policy applies. helios-admin and helios-public-key are wrappers for docker exec, since the image has no shell utilities. /etc/passwd and /etc/group are copied in rather than linked into the store: docker exec rejects absolute symlinks when resolving the user. checks.docker runs the image under Docker in a VM, pushes to it, and substitutes back with signatures required. --- flake.nix | 7 +++ nix/docker-test.nix | 57 +++++++++++++++++++ nix/docker.nix | 134 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 198 insertions(+) create mode 100644 nix/docker-test.nix create mode 100644 nix/docker.nix diff --git a/flake.nix b/flake.nix index 374807a..edd78d2 100644 --- a/flake.nix +++ b/flake.nix @@ -50,6 +50,8 @@ packages = forAllSystems (pkgs: rec { helios = helios' pkgs; default = helios; + # OCI image with the server and maintenance daemon; see nix/docker.nix. + docker = import ./nix/docker.nix { inherit pkgs helios; }; }); overlays.default = final: _prev: { helios = helios' final; }; @@ -87,6 +89,11 @@ module = ./nix/module.nix; helios = self.packages.${pkgs.stdenv.hostPlatform.system}.helios; }; + docker = import ./nix/docker-test.nix { + inherit pkgs; + image = self.packages.${pkgs.stdenv.hostPlatform.system}.docker; + helios = self.packages.${pkgs.stdenv.hostPlatform.system}.helios; + }; }); }; } diff --git a/nix/docker-test.nix b/nix/docker-test.nix new file mode 100644 index 0000000..50d8f66 --- /dev/null +++ b/nix/docker-test.nix @@ -0,0 +1,57 @@ +# VM test for the OCI image: load it into Docker, run it with generated +# secrets, administer it with docker exec, push to it from the host, and +# substitute back through Nix with signatures required. +{ pkgs, image, helios }: + +pkgs.testers.runNixOSTest { + name = "helios-docker"; + + nodes.machine = { + virtualisation.docker.enable = true; + virtualisation.diskSize = 4096; + virtualisation.memorySize = 2048; + environment.systemPackages = [ + helios + pkgs.hello + ]; + nix.settings.experimental-features = [ "nix-command" ]; + }; + + testScript = '' + import json + + machine.wait_for_unit("docker.service") + tag = machine.succeed("docker load < ${image} | sed -n 's/^Loaded image: //p'").strip() + machine.succeed(f"docker run -d --name helios -p 8080:8080 -v helios:/var/lib/helios {tag}") + machine.wait_until_succeeds("curl -sf http://127.0.0.1:8080/healthz") + + with subtest("runs unprivileged with generated secrets"): + assert machine.succeed("docker inspect -f '{{.Config.User}}' helios").strip() == "10000:10000" + # The container's init (the entrypoint) runs as the image user. + pid = machine.succeed("docker inspect -f '{{.State.Pid}}' helios").strip() + machine.succeed(f"grep -Eq '^Uid:\\s+10000\\s' /proc/{pid}/status") + machine.wait_until_succeeds("docker top helios | grep -q helios-daemon") + pubkey = machine.succeed("docker exec helios helios-public-key").strip() + assert pubkey.startswith("helios-1:"), pubkey + + with subtest("administer with docker exec, push from the host"): + machine.succeed("docker exec helios helios-admin cache create main") + token = json.loads(machine.succeed("docker exec helios helios-admin token create ci --caches main --perms push,pull 2>/dev/null"))["token"] + machine.succeed(f"helios login ci http://127.0.0.1:8080 {token}") + machine.succeed("helios push main --closure ${pkgs.hello}") + + with subtest("substitute with signatures required"): + machine.succeed( + f"NIX_CONFIG='trusted-public-keys = {pubkey}\nrequire-sigs = true' " + "nix copy --from http://127.0.0.1:8080/main --to /tmp/store ${pkgs.hello}" + ) + + with subtest("docker stop is clean; data and keys persist"): + machine.succeed("docker stop helios") + assert machine.succeed("docker inspect -f '{{.State.ExitCode}}' helios").strip() == "0" + machine.succeed("docker start helios") + machine.wait_until_succeeds("curl -sf http://127.0.0.1:8080/healthz") + machine.succeed("curl -sf http://127.0.0.1:8080/main/$(basename ${pkgs.hello} | cut -c1-32).narinfo") + assert machine.succeed("docker exec helios helios-public-key").strip() == pubkey + ''; +} diff --git a/nix/docker.nix b/nix/docker.nix new file mode 100644 index 0000000..bfc31d5 --- /dev/null +++ b/nix/docker.nix @@ -0,0 +1,134 @@ +# OCI image: helios-server and helios-daemon in one container, running as +# an unprivileged user, with everything persistent under /var/lib/helios. +# +# docker load < $(nix build .#docker --print-out-paths) +# docker run -d -p 8080:8080 -v helios:/var/lib/helios helios: +# +# On first start, without HELIOS_*_FILE secrets configured, the entrypoint +# generates a signing key, token secret and admin secret into +# /var/lib/helios/secrets. +{ pkgs, helios }: + +let + uid = "10000"; + + entrypoint = pkgs.writeShellApplication { + name = "helios-entrypoint"; + runtimeInputs = [ helios ]; + text = '' + data="''${HELIOS_DATA_DIR:-/var/lib/helios}" + secrets="$data/secrets" + + # Generate secrets on first start unless they are provided (for + # example as Docker secrets under /run/secrets). + if [[ -z "''${HELIOS_SIGNING_KEY_FILE:-}''${HELIOS_JWT_SECRET_FILE:-}''${HELIOS_ADMIN_SECRET_FILE:-}" ]]; then + pubkey=$(helios-server generate-secrets --dir "$secrets" --key-name "''${HELIOS_KEY_NAME:-helios-1}") + echo "helios: signing key $pubkey (secrets in $secrets)" >&2 + export HELIOS_SIGNING_KEY_FILE="$secrets/signing-key" + export HELIOS_JWT_SECRET_FILE="$secrets/jwt-secret" + export HELIOS_ADMIN_SECRET_FILE="$secrets/admin-secret" + fi + + if [[ "''${1:-serve}" != serve ]]; then + exec "$@" + fi + + export HELIOS_ADMIN_SOCKET="''${HELIOS_ADMIN_SOCKET:-/run/helios/admin.sock}" + helios-server & + server=$! + daemon= + if [[ "''${HELIOS_DAEMON:-1}" != 0 ]]; then + for _ in {1..100}; do + [[ -S "$HELIOS_ADMIN_SOCKET" ]] && break + read -rt 0.1 <> <(:) || true + done + helios-daemon --socket "$HELIOS_ADMIN_SOCKET" --data-dir "$data" & + daemon=$! + fi + + # docker stop: forward SIGTERM so both shut down cleanly. + stopping=0 + # shellcheck disable=SC2329 # invoked by the trap below + stop() { + stopping=1 + kill -TERM "$server" ''${daemon:+"$daemon"} 2>/dev/null || true + } + trap stop TERM INT + + # If either process exits on its own, take the other down too. + status=0 + wait -n || status=$? + kill -TERM "$server" ''${daemon:+"$daemon"} 2>/dev/null || true + wait || true + if (( stopping )); then + exit 0 + fi + exit $(( status == 0 ? 1 : status )) + ''; + }; + + # docker exec helpers; the image has no shell utilities of its own. + admin = pkgs.writeShellApplication { + name = "helios-admin"; + runtimeInputs = [ helios ]; + text = '' + # The CLI, pointed at this container's server with the admin secret: + # docker exec helios helios-admin cache create main + listen="''${HELIOS_LISTEN:-0.0.0.0:8080}" + exec helios --url "http://127.0.0.1:''${listen##*:}" \ + --token-file "''${HELIOS_ADMIN_SECRET_FILE:-''${HELIOS_DATA_DIR:-/var/lib/helios}/secrets/admin-secret}" "$@" + ''; + }; + publicKey = pkgs.writeShellApplication { + name = "helios-public-key"; + runtimeInputs = [ helios ]; + text = '' + exec helios-server --print-public-key \ + --signing-key-file "''${HELIOS_SIGNING_KEY_FILE:-''${HELIOS_DATA_DIR:-/var/lib/helios}/secrets/signing-key}" + ''; + }; + + nss = pkgs.dockerTools.fakeNss.override { + extraPasswdLines = [ "helios:x:${uid}:${uid}:helios:/var/lib/helios:/bin/false" ]; + extraGroupLines = [ "helios:x:${uid}:" ]; + }; +in +pkgs.dockerTools.buildLayeredImage { + name = "helios"; + tag = helios.version; + contents = [ + helios + entrypoint + admin + publicKey + pkgs.cacert + ]; + # Copies, not store symlinks: docker exec resolves the user through + # /etc/passwd and /etc/group and refuses absolute symlinks there. + fakeRootCommands = '' + mkdir -p etc var/lib/helios run/helios tmp + cp -L ${nss}/etc/passwd ${nss}/etc/group ${nss}/etc/nsswitch.conf etc/ + chown ${uid}:${uid} var/lib/helios run/helios + chmod 0750 var/lib/helios run/helios + chmod 1777 tmp + ''; + config = { + Entrypoint = [ "${entrypoint}/bin/helios-entrypoint" ]; + Cmd = [ "serve" ]; + User = "${uid}:${uid}"; + WorkingDir = "/var/lib/helios"; + Env = [ + "HELIOS_LISTEN=0.0.0.0:8080" + "HELIOS_DATA_DIR=/var/lib/helios" + "HOME=/var/lib/helios" + "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + ]; + ExposedPorts."8080/tcp" = { }; + Volumes."/var/lib/helios" = { }; + Labels = { + "org.opencontainers.image.title" = "helios"; + "org.opencontainers.image.description" = "Self-hosted Nix binary cache"; + "org.opencontainers.image.version" = helios.version; + }; + }; +} From a758933c3c31ad8f3b28ab950ccfcfd6a1d8af95 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 24/56] docs: document the docker image --- AGENTS.md | 6 ++++-- README.md | 2 ++ docs.md | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index bb834b3..b4722be 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,8 @@ A self-hosted Nix binary cache: Rust server and CLI over a Zig core. - `crates/helios-server`, `crates/helios-cli`, `crates/helios-daemon`: the binaries. The daemon drives the server over its maintenance socket and never writes the database itself. -- `nix/`: the NixOS module (`services.helios`) and its VM test. +- `nix/`: the NixOS module (`services.helios`), the OCI image, and their VM + tests. - `bench/`: comparisons against Cachix's libraries and Nix C++. Not part of the Cargo workspace. @@ -52,7 +53,8 @@ hmac/sha2 (libhelios provides SHA-256, HMAC and randomness) and mimalloc - `cargo test --release` - `./scripts/test-zig.sh` - `nix build .#checks.x86_64-linux.nixos` after changing `nix/`: a VM test of - the NixOS module. + the NixOS module. `.#checks.x86_64-linux.docker` likewise for the OCI + image. - `./scripts/e2e.sh` before finishing changes to the server, the CLI or wire formats; it substitutes a real closure through Nix with signatures required. diff --git a/README.md b/README.md index 569ac78..b7613cf 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,8 @@ nix.settings = { On NixOS, use the flake's `nixosModules.default` (`services.helios`): the server, the maintenance daemon, and push-on-build for build machines. +Elsewhere, `nix build .#docker` builds an OCI image of the server and daemon +that generates its keys on first start. See [docs.md](docs.md) for the NixOS module, configuration, private caches, the HTTP API and deployment behind a reverse proxy. diff --git a/docs.md b/docs.md index 3c2445a..a0deeb0 100644 --- a/docs.md +++ b/docs.md @@ -122,6 +122,38 @@ services.helios.watchStore = { all of the above: push, signed substitution, sandboxing, metrics, backups, a scrub that catches a corrupted NAR, watch-store, eviction and shutdown. +## Docker + +`packages..docker` is an OCI image with the server and the +maintenance daemon, running as uid 10000 with all state in +`/var/lib/helios`: + +```sh +docker load < "$(nix build .#docker --print-out-paths)" +docker run -d --name helios -p 8080:8080 -v helios:/var/lib/helios \ + -e HELIOS_QUOTA=500G helios:0.1.0 +docker exec helios helios-public-key # for trusted-public-keys +docker exec helios helios-admin cache create main +docker exec helios helios-admin token create ci --caches main --perms push,pull +``` + +On first start the entrypoint generates a signing key (named by +`HELIOS_KEY_NAME`, default `helios-1`), a token secret and an admin secret +into `/var/lib/helios/secrets`, and logs the public key. It never overwrites +them, so keep the volume and the keys survive upgrades. To provide your own +instead, for example as Docker secrets, set all three of +`HELIOS_SIGNING_KEY_FILE`, `HELIOS_JWT_SECRET_FILE` and +`HELIOS_ADMIN_SECRET_FILE`. + +Both binaries take their flags as `HELIOS_*` environment variables (see +`helios-server --help` and `helios-daemon --help`), such as `HELIOS_QUOTA`, +`HELIOS_MIN_FREE`, `HELIOS_METRICS_LISTEN` and `HELIOS_LOG`. Set +`HELIOS_DAEMON=0` to run the server alone. `docker stop` shuts both down +cleanly; if either exits on its own, the container exits non-zero so the +restart policy takes over. The image has no reverse proxy: put one in front +for TLS. `nix build .#checks.x86_64-linux.docker` runs the image in a VM +under real Docker. + ## Reverse proxy and zero-copy downloads Outside NixOS, terminate TLS in a reverse proxy. With `--accel-redirect /_nar`, the server From f116901c746b312b81a15b8d0ce0da4230b40aa9 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 25/56] build: format rust with a checked-in rustfmt config The code was written to long lines by hand and never run through rustfmt. max_width 150 with the Max heuristics matches that style, so applying it changes little, and CI can now enforce it. --- crates/helios-cli/src/api.rs | 6 ++-- crates/helios-cli/src/main.rs | 12 ++++++-- crates/helios-cli/src/push.rs | 8 +---- crates/helios-cli/src/transport.rs | 9 +++++- crates/helios-cli/src/watch.rs | 12 ++------ crates/helios-core/build.rs | 10 ++----- crates/helios-core/src/lib.rs | 13 +++----- crates/helios-daemon/src/autogc.rs | 7 ++--- crates/helios-daemon/src/client.rs | 4 +-- crates/helios-daemon/src/main.rs | 9 +++--- crates/helios-daemon/src/metrics.rs | 4 +-- crates/helios-server/src/admin.rs | 2 +- crates/helios-server/src/audit.rs | 12 +++----- crates/helios-server/src/auth.rs | 2 +- crates/helios-server/src/db.rs | 9 +++--- crates/helios-server/src/gc.rs | 3 +- crates/helios-server/src/internal.rs | 45 ++++++++++++++++------------ crates/helios-server/src/main.rs | 4 +-- crates/helios-server/src/push.rs | 24 +++++---------- crates/helios-server/src/read.rs | 6 ++-- crates/helios-server/src/state.rs | 2 +- rustfmt.toml | 2 ++ 22 files changed, 87 insertions(+), 118 deletions(-) create mode 100644 rustfmt.toml diff --git a/crates/helios-cli/src/api.rs b/crates/helios-cli/src/api.rs index 8894406..f2d6221 100644 --- a/crates/helios-cli/src/api.rs +++ b/crates/helios-cli/src/api.rs @@ -86,10 +86,8 @@ impl Client { pub async fn publish(&self, cache: &str, paths: &[PathSpec]) -> anyhow::Result { let body = transport::full(serde_json::to_vec(&json!({ "paths": paths }))?); - let (status, bytes) = self - .http - .send(Method::POST, &format!("/_api/v2/caches/{cache}/paths"), &self.token, Some("application/json"), body) - .await?; + let (status, bytes) = + self.http.send(Method::POST, &format!("/_api/v2/caches/{cache}/paths"), &self.token, Some("application/json"), body).await?; if status == StatusCode::CONFLICT { let v: Value = serde_json::from_slice(&bytes)?; if v["error"] == "nar_required" { diff --git a/crates/helios-cli/src/main.rs b/crates/helios-cli/src/main.rs index fe5dd4c..677fc37 100644 --- a/crates/helios-cli/src/main.rs +++ b/crates/helios-cli/src/main.rs @@ -1,8 +1,8 @@ mod api; mod config; -mod transport; mod nix; mod push; +mod transport; mod watch; use anyhow::Context; @@ -102,7 +102,10 @@ enum TokenCmd { expires: i64, }, List, - Revoke { jti: String, reason: String }, + Revoke { + jti: String, + reason: String, + }, } fn split(s: &str) -> Vec { @@ -151,7 +154,10 @@ async fn run(cli: Cli) -> anyhow::Result<()> { Command::Cache(CacheCmd::List) => print(&client.admin_get("/admin/caches").await?), Command::Token(TokenCmd::Create { subject, caches, perms, expires }) => { let v = client - .admin_post("/admin/tokens", json!({ "subject": subject, "caches": split(&caches), "perms": split(&perms), "expiresInDays": expires })) + .admin_post( + "/admin/tokens", + json!({ "subject": subject, "caches": split(&caches), "perms": split(&perms), "expiresInDays": expires }), + ) .await?; eprintln!("store this token now; it is not shown again"); print(&v); diff --git a/crates/helios-cli/src/push.rs b/crates/helios-cli/src/push.rs index 2b986ff..d1e79fe 100644 --- a/crates/helios-cli/src/push.rs +++ b/crates/helios-cli/src/push.rs @@ -154,12 +154,7 @@ pub async fn push(client: &Client, cache: &str, installables: &[String], opts: O known.extend(client.known(cache, chunk).await?); } let uploads: Vec<&PathInfo> = todo.iter().filter(|i| !known.contains(&i.nar_hash)).collect(); - eprintln!( - "{total} paths, {} missing from '{cache}': uploading {}, reusing {} NARs", - todo.len(), - uploads.len(), - todo.len() - uploads.len() - ); + eprintln!("{total} paths, {} missing from '{cache}': uploading {}, reusing {} NARs", todo.len(), uploads.len(), todo.len() - uploads.len()); let mut uploaded_bytes = upload_all(client, cache, &uploads, &opts).await?; let ordered = nix::topo_order(todo); @@ -202,4 +197,3 @@ pub async fn push(client: &Client, cache: &str, installables: &[String], opts: O ); Ok(()) } - diff --git a/crates/helios-cli/src/transport.rs b/crates/helios-cli/src/transport.rs index b110d46..6701944 100644 --- a/crates/helios-cli/src/transport.rs +++ b/crates/helios-cli/src/transport.rs @@ -175,7 +175,14 @@ impl Client { } /// Sends a request and reads the whole response body. - pub async fn send(&self, method: Method, path: &str, token: &str, content_type: Option<&str>, body: Body) -> anyhow::Result<(http::StatusCode, Bytes)> { + pub async fn send( + &self, + method: Method, + path: &str, + token: &str, + content_type: Option<&str>, + body: Body, + ) -> anyhow::Result<(http::StatusCode, Bytes)> { let mut req = Request::builder() .method(method) .uri(self.path(path)) diff --git a/crates/helios-cli/src/watch.rs b/crates/helios-cli/src/watch.rs index a52951e..11c3253 100644 --- a/crates/helios-cli/src/watch.rs +++ b/crates/helios-cli/src/watch.rs @@ -40,11 +40,7 @@ pub fn queue(spool: &Path, paths: &str) -> anyhow::Result { /// Spool entries in arrival order, skipping ones still being written. fn pending(spool: &Path) -> anyhow::Result> { let mut entries: Vec = match std::fs::read_dir(spool) { - Ok(rd) => rd - .flatten() - .map(|e| e.path()) - .filter(|p| p.file_name().and_then(|n| n.to_str()).is_some_and(|n| !n.starts_with('.'))) - .collect(), + Ok(rd) => rd.flatten().map(|e| e.path()).filter(|p| p.file_name().and_then(|n| n.to_str()).is_some_and(|n| !n.starts_with('.'))).collect(), Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), Err(e) => return Err(e.into()), }; @@ -70,11 +66,7 @@ pub async fn watch(client: &Client, cache: &str, spool: &Path, opts: push::Optio paths.dedup(); // Paths garbage-collected since they were built cannot be pushed. paths.retain(|p| Path::new(p).exists()); - let result = if paths.is_empty() { - Ok(()) - } else { - push::push(client, cache, &paths, push::Options { closure: true, ..opts }).await - }; + let result = if paths.is_empty() { Ok(()) } else { push::push(client, cache, &paths, push::Options { closure: true, ..opts }).await }; match result { Ok(()) => { for e in &entries { diff --git a/crates/helios-core/build.rs b/crates/helios-core/build.rs index 6859ff9..3b36cd9 100644 --- a/crates/helios-core/build.rs +++ b/crates/helios-core/build.rs @@ -43,20 +43,14 @@ fn main() { if env::var_os("ZIG_GLOBAL_CACHE_DIR").is_none() { cmd.env("ZIG_GLOBAL_CACHE_DIR", out.join("zig-global-cache")); } - let status = cmd - .status() - .unwrap_or_else(|e| panic!("failed to run `{zig} build` (is zig 0.16 on PATH?): {e}")); + let status = cmd.status().unwrap_or_else(|e| panic!("failed to run `{zig} build` (is zig 0.16 on PATH?): {e}")); assert!(status.success(), "zig build failed for libhelios"); println!("cargo:rustc-link-search=native={}", prefix.join("lib").display()); println!("cargo:rustc-link-lib=static=helios"); println!("cargo:include={}", prefix.join("include").display()); - if pkg_config::Config::new() - .atleast_version("1.5") - .probe("libzstd") - .is_err() - { + if pkg_config::Config::new().atleast_version("1.5").probe("libzstd").is_err() { println!("cargo:rustc-link-lib=zstd"); } } diff --git a/crates/helios-core/src/lib.rs b/crates/helios-core/src/lib.rs index a014e9f..df29dd4 100644 --- a/crates/helios-core/src/lib.rs +++ b/crates/helios-core/src/lib.rs @@ -187,9 +187,7 @@ unsafe extern "C" fn trampoline bool>(ctx: *mut c_void, buf: pub fn dump_nar bool>(path: &Path, opts: &DumpOptions, mut write: F) -> Result { let cpath = CString::new(path.as_os_str().as_bytes()).map_err(|_| Error { code: HL_E_INVALID, errno: 0 })?; let mut out = sys::Digest::default(); - let rc = unsafe { - sys::hl_nar_dump(cpath.as_ptr(), &opts.raw(), trampoline::, &mut write as *mut F as *mut c_void, &mut out) - }; + let rc = unsafe { sys::hl_nar_dump(cpath.as_ptr(), &opts.raw(), trampoline::, &mut write as *mut F as *mut c_void, &mut out) }; check(rc)?; Ok(out.into()) } @@ -451,10 +449,7 @@ mod tests { // RFC 4231 test case 2. let mac = hmac_sha256(b"Jefe", b"what do ya want for nothing?"); assert_eq!(nix32_encode(&mac).len(), 52); - assert_eq!( - mac.iter().map(|b| format!("{b:02x}")).collect::(), - "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843" - ); + assert_eq!(mac.iter().map(|b| format!("{b:02x}")).collect::(), "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843"); assert_eq!( sha256(b"abc").iter().map(|b| format!("{b:02x}")).collect::(), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" @@ -538,8 +533,8 @@ mod tests { let hashes = Command::new("nix-store").args(["-q", "--hash"]).args(&paths).output().unwrap(); let hashes = String::from_utf8(hashes.stdout).unwrap(); for (path, expected) in paths.iter().zip(hashes.lines()) { - let digest = dump_nar(Path::new(path), &DumpOptions { level: 1, threads: 0, size_hint: 0 }, |_| true) - .unwrap_or_else(|e| panic!("{path}: {e}")); + let digest = + dump_nar(Path::new(path), &DumpOptions { level: 1, threads: 0, size_hint: 0 }, |_| true).unwrap_or_else(|e| panic!("{path}: {e}")); assert_eq!(format!("sha256:{}", nix32_encode(&digest.nar_hash)), expected, "{path}"); } } diff --git a/crates/helios-daemon/src/autogc.rs b/crates/helios-daemon/src/autogc.rs index e565c4b..3e679f0 100644 --- a/crates/helios-daemon/src/autogc.rs +++ b/crates/helios-daemon/src/autogc.rs @@ -56,11 +56,8 @@ struct Evicted { /// Bytes to free now, or 0 when within policy. pub fn to_free(policy: &Policy, used: u64, free_disk: u64) -> u64 { - let over_quota = if policy.quota > 0 && used as f64 > policy.quota as f64 * policy.high { - used - (policy.quota as f64 * policy.low) as u64 - } else { - 0 - }; + let over_quota = + if policy.quota > 0 && used as f64 > policy.quota as f64 * policy.high { used - (policy.quota as f64 * policy.low) as u64 } else { 0 }; let short_disk = if policy.min_free > 0 && free_disk < policy.min_free { // Overshoot a little so the next upload does not trigger it again. (policy.min_free as f64 * 1.1) as u64 - free_disk diff --git a/crates/helios-daemon/src/client.rs b/crates/helios-daemon/src/client.rs index 784a8d8..07a5c6d 100644 --- a/crates/helios-daemon/src/client.rs +++ b/crates/helios-daemon/src/client.rs @@ -24,9 +24,7 @@ impl Client { } async fn call(&self, method: Method, path: &str, body: Option) -> anyhow::Result { - let stream = UnixStream::connect(&self.socket) - .await - .with_context(|| format!("connecting to {}", self.socket.display()))?; + let stream = UnixStream::connect(&self.socket).await.with_context(|| format!("connecting to {}", self.socket.display()))?; let (mut sender, conn) = hyper::client::conn::http1::handshake(Io(stream)).await?; tokio::spawn(conn); let mut req = Request::builder().method(method).uri(path).header(header::HOST, "helios"); diff --git a/crates/helios-daemon/src/main.rs b/crates/helios-daemon/src/main.rs index c41a71f..897e4f8 100644 --- a/crates/helios-daemon/src/main.rs +++ b/crates/helios-daemon/src/main.rs @@ -175,14 +175,15 @@ fn main() -> anyhow::Result<()> { let d = d.clone(); let rate = args.scrub_rate; // Wait a full interval first so restarts do not re-read everything. - tasks.push(tokio::spawn(async move { - periodic("scrub", args.scrub_interval, true, &d.metrics.errors_scrub, || scrub::run(&d, rate)).await - })); + tasks.push(tokio::spawn( + async move { periodic("scrub", args.scrub_interval, true, &d.metrics.errors_scrub, || scrub::run(&d, rate)).await }, + )); } { let d = d.clone(); tasks.push(tokio::spawn(async move { - periodic("checkpoint", args.checkpoint_interval.max(Duration::from_secs(1)), true, &d.metrics.errors_db, || maint::checkpoint(&d)).await + periodic("checkpoint", args.checkpoint_interval.max(Duration::from_secs(1)), true, &d.metrics.errors_db, || maint::checkpoint(&d)) + .await })); } { diff --git a/crates/helios-daemon/src/metrics.rs b/crates/helios-daemon/src/metrics.rs index c8337aa..d93a4e7 100644 --- a/crates/helios-daemon/src/metrics.rs +++ b/crates/helios-daemon/src/metrics.rs @@ -99,9 +99,7 @@ pub async fn serve(d: Arc, addr: SocketAddr) -> anyhow::Result<()> { let d = d.clone(); async move { if req.uri().path() == "/metrics" { - Response::builder() - .header(header::CONTENT_TYPE, "text/plain; version=0.0.4") - .body(Full::new(Bytes::from(render(&d).await))) + Response::builder().header(header::CONTENT_TYPE, "text/plain; version=0.0.4").body(Full::new(Bytes::from(render(&d).await))) } else { Response::builder().status(StatusCode::NOT_FOUND).body(Full::new(Bytes::from_static(b"not found\n"))) } diff --git a/crates/helios-server/src/admin.rs b/crates/helios-server/src/admin.rs index 9dfff71..c180fb4 100644 --- a/crates/helios-server/src/admin.rs +++ b/crates/helios-server/src/admin.rs @@ -7,10 +7,10 @@ use rusqlite::params; use serde::Deserialize; use serde_json::json; -use crate::state::Locked; use crate::audit::Audit; use crate::auth::{self, AUDIENCE, Claims, ISSUER, MAX_CACHES_PER_TOKEN, MAX_PERMS_PER_TOKEN, Perm}; use crate::error::{ApiError, ApiResult}; +use crate::state::Locked; use crate::state::{Shared, TokenState}; const MAX_TOKEN_LIFETIME_DAYS: i64 = 365; diff --git a/crates/helios-server/src/audit.rs b/crates/helios-server/src/audit.rs index 24ce329..ca6bf7d 100644 --- a/crates/helios-server/src/audit.rs +++ b/crates/helios-server/src/audit.rs @@ -33,11 +33,8 @@ impl FromRequestParts for Audit { type Rejection = std::convert::Infallible; async fn from_request_parts(parts: &mut Parts, st: &Shared) -> Result { - let forwarded = st - .cfg - .trust_proxy - .then(|| parts.headers.get("x-forwarded-for")?.to_str().ok()?.split(',').next()?.trim().parse().ok()) - .flatten(); + let forwarded = + st.cfg.trust_proxy.then(|| parts.headers.get("x-forwarded-for")?.to_str().ok()?.split(',').next()?.trim().parse().ok()).flatten(); let peer = parts.extensions.get::>().map(|c| c.0.ip()); Ok(Self { ip: forwarded.or(peer) }) } @@ -74,9 +71,8 @@ pub async fn run(db: Db, mut rx: mpsc::UnboundedReceiver) { db.write(|conn| -> rusqlite::Result<()> { let tx = conn.transaction()?; { - let mut stmt = tx.prepare_cached( - "INSERT INTO audit_log (ts, actor, action, cache, detail, ip, status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", - )?; + let mut stmt = tx + .prepare_cached("INSERT INTO audit_log (ts, actor, action, cache, detail, ip, status) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)")?; for e in &events { stmt.execute(params![e.ts, e.actor, e.action, e.cache, e.detail, e.ip, e.status])?; } diff --git a/crates/helios-server/src/auth.rs b/crates/helios-server/src/auth.rs index 2f36034..f283b9d 100644 --- a/crates/helios-server/src/auth.rs +++ b/crates/helios-server/src/auth.rs @@ -6,8 +6,8 @@ use base64::Engine; use base64::engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD}; use serde::{Deserialize, Serialize}; -use crate::state::Locked; use crate::error::ApiError; +use crate::state::Locked; use crate::state::{AppState, CacheInfo}; pub const ISSUER: &str = "helios-cache"; diff --git a/crates/helios-server/src/db.rs b/crates/helios-server/src/db.rs index bdb9006..4f9475c 100644 --- a/crates/helios-server/src/db.rs +++ b/crates/helios-server/src/db.rs @@ -5,10 +5,11 @@ use std::cell::RefCell; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::sync::{Mutex, PoisonError}; use rusqlite::{Connection, OpenFlags}; +use std::sync::{Mutex, PoisonError}; -const MIGRATIONS: &[&str] = &[r#" +const MIGRATIONS: &[&str] = &[ + r#" CREATE TABLE caches ( id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, @@ -130,7 +131,5 @@ fn migrate(conn: &mut Connection) -> rusqlite::Result<()> { } pub fn now() -> i64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |d| d.as_secs() as i64) + std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_secs() as i64) } diff --git a/crates/helios-server/src/gc.rs b/crates/helios-server/src/gc.rs index ec81e25..c945fe1 100644 --- a/crates/helios-server/src/gc.rs +++ b/crates/helios-server/src/gc.rs @@ -81,8 +81,7 @@ pub fn collect_blobs(st: &Shared) -> anyhow::Result<(usize, u64)> { "SELECT id, file_hash, compression, file_size FROM blobs b WHERE created_at <= ?1 AND NOT EXISTS (SELECT 1 FROM paths p WHERE p.blob_id = b.id) LIMIT ?2", )?; - stmt.query_map(params![cutoff, BLOB_BATCH], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))? - .collect::>()? + stmt.query_map(params![cutoff, BLOB_BATCH], |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?)))?.collect::>()? }; { let mut del = tx.prepare_cached("DELETE FROM blobs WHERE id = ?1")?; diff --git a/crates/helios-server/src/internal.rs b/crates/helios-server/src/internal.rs index 712aebd..d5dbb5f 100644 --- a/crates/helios-server/src/internal.rs +++ b/crates/helios-server/src/internal.rs @@ -49,23 +49,26 @@ fn param(uri: &Uri, name: &str) -> Option { async fn stats(State(st): State) -> ApiResult> { let db_st = st.clone(); let (caches, blob_count, blob_bytes, unreferenced) = blocking(move || { - db_st.db.read(|conn| { - let mut stmt = conn.prepare_cached( - "SELECT c.name, c.is_public, count(p.id) FROM caches c LEFT JOIN paths p ON p.cache_id = c.id GROUP BY c.id ORDER BY c.name", - )?; - let caches: Vec = stmt - .query_map([], |r| Ok(json!({ "name": r.get::<_, String>(0)?, "public": r.get::<_, bool>(1)?, "paths": r.get::<_, u64>(2)? })))? - .collect::>()?; - let (count, bytes): (u64, u64) = conn.query_row("SELECT count(*), coalesce(sum(file_size), 0) FROM blobs", [], |r| Ok((r.get(0)?, r.get(1)?)))?; - // Blobs no path uses: GC deletes them once their upload grace ends. - let unreferenced: u64 = conn.query_row( - "SELECT coalesce(sum(file_size), 0) FROM blobs b WHERE NOT EXISTS (SELECT 1 FROM paths p WHERE p.blob_id = b.id)", - [], - |r| r.get(0), - )?; - Ok((caches, count, bytes, unreferenced)) - }) - .map_err(internal) + db_st + .db + .read(|conn| { + let mut stmt = conn.prepare_cached( + "SELECT c.name, c.is_public, count(p.id) FROM caches c LEFT JOIN paths p ON p.cache_id = c.id GROUP BY c.id ORDER BY c.name", + )?; + let caches: Vec = stmt + .query_map([], |r| Ok(json!({ "name": r.get::<_, String>(0)?, "public": r.get::<_, bool>(1)?, "paths": r.get::<_, u64>(2)? })))? + .collect::>()?; + let (count, bytes): (u64, u64) = + conn.query_row("SELECT count(*), coalesce(sum(file_size), 0) FROM blobs", [], |r| Ok((r.get(0)?, r.get(1)?)))?; + // Blobs no path uses: GC deletes them once their upload grace ends. + let unreferenced: u64 = conn.query_row( + "SELECT coalesce(sum(file_size), 0) FROM blobs b WHERE NOT EXISTS (SELECT 1 FROM paths p WHERE p.blob_id = b.id)", + [], + |r| r.get(0), + )?; + Ok((caches, count, bytes, unreferenced)) + }) + .map_err(internal) }) .await?; let c = &st.counters; @@ -217,7 +220,9 @@ async fn quarantine(State(st): State, Json(req): Json) -> let found = st.db.write(|conn| -> rusqlite::Result, String)>> { let tx = conn.transaction()?; let Some((id, compression)) = tx - .query_row("SELECT id, compression FROM blobs WHERE file_hash = ?1", [&file_hash[..]], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?))) + .query_row("SELECT id, compression FROM blobs WHERE file_hash = ?1", [&file_hash[..]], |r| { + Ok((r.get::<_, i64>(0)?, r.get::<_, String>(1)?)) + }) .optional()? else { return Ok(None); @@ -265,7 +270,9 @@ async fn checkpoint(State(st): State) -> ApiResult> { blocking(move || { let (busy, log, done) = st .db - .write(|conn| conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?, r.get::<_, i64>(2)?)))) + .write(|conn| { + conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| Ok((r.get::<_, i64>(0)?, r.get::<_, i64>(1)?, r.get::<_, i64>(2)?))) + }) .map_err(internal)?; Ok(Json(json!({ "busy": busy != 0, "walPages": log, "checkpointedPages": done }))) }) diff --git a/crates/helios-server/src/main.rs b/crates/helios-server/src/main.rs index b0ec707..850d792 100644 --- a/crates/helios-server/src/main.rs +++ b/crates/helios-server/src/main.rs @@ -53,9 +53,7 @@ pub fn router(st: Shared) -> Router { async fn healthz(State(st): State) -> (StatusCode, axum::Json) { let db = st.db.clone(); - let ok = tokio::task::spawn_blocking(move || db.read(|c| c.query_row("SELECT 1", [], |_| Ok(()))).is_ok()) - .await - .unwrap_or(false); + let ok = tokio::task::spawn_blocking(move || db.read(|c| c.query_row("SELECT 1", [], |_| Ok(()))).is_ok()).await.unwrap_or(false); let status = if ok { StatusCode::OK } else { StatusCode::SERVICE_UNAVAILABLE }; (status, axum::Json(json!({ "ok": ok, "service": "helios" }))) } diff --git a/crates/helios-server/src/push.rs b/crates/helios-server/src/push.rs index 2aa4157..204e5a8 100644 --- a/crates/helios-server/src/push.rs +++ b/crates/helios-server/src/push.rs @@ -23,10 +23,10 @@ use rusqlite::{OptionalExtension, params}; use serde::{Deserialize, Serialize}; use serde_json::json; -use crate::state::Locked; use crate::audit::Audit; use crate::auth::Perm; use crate::error::{ApiError, ApiResult}; +use crate::state::Locked; use crate::state::{CacheInfo, PathKey, Shared}; const MAX_MISSING_BATCH: usize = 100_000; @@ -97,13 +97,7 @@ pub async fn known( }) }) .await??; - let known: Vec<&String> = req - .nar_hashes - .iter() - .zip(&parsed) - .filter(|(_, h)| h.is_some_and(|h| found.contains(&h[..]))) - .map(|(s, _)| s) - .collect(); + let known: Vec<&String> = req.nar_hashes.iter().zip(&parsed).filter(|(_, h)| h.is_some_and(|h| found.contains(&h[..]))).map(|(s, _)| s).collect(); Ok(Json(json!({ "known": known }))) } @@ -129,8 +123,7 @@ pub async fn upload( cache_for(&st, &name)?; let identity = st.authorize(&headers, &name, Perm::Push)?; let requested = uri.query().unwrap_or("").split('&').find_map(|kv| kv.strip_prefix("compression=")).unwrap_or("zstd"); - let compression = Compression::parse(requested) - .ok_or_else(|| ApiError::bad_request("compression must be zstd or none"))?; + let compression = Compression::parse(requested).ok_or_else(|| ApiError::bad_request("compression must be zstd or none"))?; let tmp = st.tmp_dir().join(helios_core::uuid_v4()); let (tx, mut rx) = tokio::sync::mpsc::channel::(32); @@ -398,9 +391,8 @@ fn render_and_insert(st: &Shared, prepared: Vec) -> ApiResult "SELECT id, file_hash, file_size, compression, nar_size FROM blobs WHERE nar_hash = ?1 ORDER BY compression = 'zstd' DESC, id LIMIT 1", )?; - let mut by_file = conn.prepare_cached( - "SELECT id, file_hash, file_size, compression, nar_size FROM blobs WHERE file_hash = ?1 AND nar_hash = ?2", - )?; + let mut by_file = + conn.prepare_cached("SELECT id, file_hash, file_size, compression, nar_size FROM blobs WHERE file_hash = ?1 AND nar_hash = ?2")?; let row = |r: &rusqlite::Row<'_>| { Ok(Blob { id: r.get(0)?, @@ -429,10 +421,8 @@ fn render_and_insert(st: &Shared, prepared: Vec) -> ApiResult let threads = std::thread::available_parallelism().map_or(1, |n| n.get()).min(work.len().div_ceil(64).max(1)); let chunk = work.len().div_ceil(threads).max(1); let rendered: Vec> = std::thread::scope(|scope| { - let handles: Vec<_> = work - .chunks(chunk) - .map(|part| scope.spawn(move || part.iter().map(|(p, b)| render(st, p, b)).collect::>())) - .collect(); + let handles: Vec<_> = + work.chunks(chunk).map(|part| scope.spawn(move || part.iter().map(|(p, b)| render(st, p, b)).collect::>())).collect(); handles.into_iter().flat_map(|h| h.join().expect("render thread panicked")).collect() }); let rendered: Vec = rendered.into_iter().collect::>()?; diff --git a/crates/helios-server/src/read.rs b/crates/helios-server/src/read.rs index cc49cfd..0240c70 100644 --- a/crates/helios-server/src/read.rs +++ b/crates/helios-server/src/read.rs @@ -10,8 +10,8 @@ use bytes::Bytes; use helios_core::Compression; use rusqlite::{OptionalExtension, params}; -use crate::state::Locked; use crate::error::{ApiError, ApiResult}; +use crate::state::Locked; use crate::state::{PathKey, Shared}; const NIX_CACHE_INFO: &str = "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 40\n"; @@ -140,9 +140,7 @@ async fn nar_response( method: &Method, cache_control: HeaderValue, ) -> ApiResult { - let mut builder = Response::builder() - .header(header::CONTENT_TYPE, "application/x-nix-nar") - .header(header::CACHE_CONTROL, cache_control); + let mut builder = Response::builder().header(header::CONTENT_TYPE, "application/x-nix-nar").header(header::CACHE_CONTROL, cache_control); if let Some(prefix) = &st.cfg.accel_redirect { let name = helios_core::nix32_encode(file_hash); diff --git a/crates/helios-server/src/state.rs b/crates/helios-server/src/state.rs index 4bee131..55fc5fb 100644 --- a/crates/helios-server/src/state.rs +++ b/crates/helios-server/src/state.rs @@ -9,8 +9,8 @@ use std::sync::Arc; use bytes::Bytes; use helios_core::{Compression, Signer}; -use std::sync::{PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard}; use rusqlite::params; +use std::sync::{PoisonError, RwLock, RwLockReadGuard, RwLockWriteGuard}; use tokio::sync::mpsc; use crate::audit::AuditEvent; diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..3121924 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,2 @@ +max_width = 150 +use_small_heuristics = "Max" From c167522390166561886fa9d36d5e1e0e925b6f1e Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 26/56] nix: wait for the new admin socket on container restart A restarted container keeps /run/helios, so the entrypoint saw the last run's socket, started the daemon before the server was listening, and its first backup failed with connection refused. Remove the socket before starting the server. The entrypoint now has coreutils, which also replaces the read -t workaround for sleep. --- nix/docker-test.nix | 3 +++ nix/docker.nix | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/nix/docker-test.nix b/nix/docker-test.nix index 50d8f66..99f878c 100644 --- a/nix/docker-test.nix +++ b/nix/docker-test.nix @@ -53,5 +53,8 @@ pkgs.testers.runNixOSTest { machine.wait_until_succeeds("curl -sf http://127.0.0.1:8080/healthz") machine.succeed("curl -sf http://127.0.0.1:8080/main/$(basename ${pkgs.hello} | cut -c1-32).narinfo") assert machine.succeed("docker exec helios helios-public-key").strip() == pubkey + # The daemon waited for the new server, not the last run's socket. + machine.wait_until_succeeds("docker logs helios 2>&1 | grep -c 'helios-daemon started' | grep -qx 2") + machine.fail("docker logs helios 2>&1 | grep -q 'Connection refused'") ''; } diff --git a/nix/docker.nix b/nix/docker.nix index bfc31d5..e6bd5cc 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -14,7 +14,7 @@ let entrypoint = pkgs.writeShellApplication { name = "helios-entrypoint"; - runtimeInputs = [ helios ]; + runtimeInputs = [ helios pkgs.coreutils ]; text = '' data="''${HELIOS_DATA_DIR:-/var/lib/helios}" secrets="$data/secrets" @@ -34,13 +34,16 @@ let fi export HELIOS_ADMIN_SOCKET="''${HELIOS_ADMIN_SOCKET:-/run/helios/admin.sock}" + # A restarted container keeps the last run's socket; without this the + # wait below would pass before the server is listening. + rm -f "$HELIOS_ADMIN_SOCKET" helios-server & server=$! daemon= if [[ "''${HELIOS_DAEMON:-1}" != 0 ]]; then for _ in {1..100}; do [[ -S "$HELIOS_ADMIN_SOCKET" ]] && break - read -rt 0.1 <> <(:) || true + sleep 0.1 done helios-daemon --socket "$HELIOS_ADMIN_SOCKET" --data-dir "$data" & daemon=$! From ba5199f21cec9b8a5df179ff88e3592360388112 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 27/56] ci: test with nix and publish the image to ghcr Each push and pull request runs rustfmt, clippy, the Rust and Zig tests and the end-to-end test on x86_64 and aarch64 runners, the NixOS and Docker VM tests on x86_64 (the only runners with KVM), and builds the package and image on aarch64. Pushes to master and v* tags then push each architecture's image with skopeo and join them into one multi-arch tag on ghcr.io/polymath-as/helios: master, or the version and latest for a tag, which must match the package version. Actions are pinned by commit. --- .github/workflows/ci.yml | 106 +++++++++++++++++++++++++++++++++++++++ AGENTS.md | 4 ++ README.md | 4 +- docs.md | 11 ++-- nix/docker.nix | 2 + 5 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ec5aa77 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,106 @@ +# Tests on x86_64 and aarch64, then publishes the OCI image to +# ghcr.io//helios on pushes to master and on v* tags. +name: ci + +on: + push: + branches: [master] + tags: ["v*"] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + test: + strategy: + fail-fast: false + matrix: + include: + - runner: ubuntu-24.04 + system: x86_64-linux + arch: amd64 + - runner: ubuntu-24.04-arm + system: aarch64-linux + arch: arm64 + runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1 + with: + extra_nix_config: | + system-features = nixos-test benchmark big-parallel kvm + + - name: Format + run: nix develop -c cargo fmt --check + - name: Clippy + run: nix develop -c cargo clippy --release --all-targets -- -D warnings + - name: Rust tests + run: nix develop -c cargo test --release + - name: Zig tests + run: nix develop -c ./scripts/test-zig.sh + - name: End to end + run: nix develop -c sh -c 'cargo build --release && ./scripts/e2e.sh' + + # The NixOS and Docker VM tests need KVM, which only the x86_64 + # runners have. + - name: Enable KVM + if: matrix.system == 'x86_64-linux' + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Nix checks + if: matrix.system == 'x86_64-linux' + run: nix flake check -L + - name: Nix build + if: matrix.system != 'x86_64-linux' + run: nix build -L .#checks.${{ matrix.system }}.package .#docker + + - name: Push image + if: github.event_name == 'push' + env: + ARCH: ${{ matrix.arch }} + TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + version=$(nix eval --raw .#helios.version) + if [[ "$GITHUB_REF_TYPE" == tag && "$GITHUB_REF_NAME" != "v$version" ]]; then + echo "tag $GITHUB_REF_NAME does not match package version $version" >&2 + exit 1 + fi + image="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/helios" + nix run --inputs-from . nixpkgs#skopeo -- copy \ + --dest-creds "$GITHUB_ACTOR:$TOKEN" \ + "docker-archive:$(nix build --print-out-paths .#docker)" \ + "docker://$image:sha-${GITHUB_SHA::12}-$ARCH" + + # Joins the per-architecture images into one multi-arch tag. + image: + if: github.event_name == 'push' + needs: test + runs-on: ubuntu-24.04 + permissions: + packages: write + steps: + - name: Publish manifest + env: + TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "$TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + image="ghcr.io/${GITHUB_REPOSITORY_OWNER,,}/helios" + sha="sha-${GITHUB_SHA::12}" + tags=(-t "$image:$sha") + if [[ "$GITHUB_REF_TYPE" == tag ]]; then + tags+=(-t "$image:${GITHUB_REF_NAME#v}" -t "$image:latest") + else + tags+=(-t "$image:$GITHUB_REF_NAME") + fi + docker buildx imagetools create "${tags[@]}" "$image:$sha-amd64" "$image:$sha-arm64" diff --git a/AGENTS.md b/AGENTS.md index b4722be..67bde23 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,6 +50,10 @@ hmac/sha2 (libhelios provides SHA-256, HMAC and randomness) and mimalloc ## Verification +CI (`.github/workflows/ci.yml`) runs all of this on x86_64 and aarch64 +(the VM tests on x86_64 only) and publishes the image on pushes. + +- `cargo fmt --check` and `cargo clippy --release --all-targets -- -D warnings` - `cargo test --release` - `./scripts/test-zig.sh` - `nix build .#checks.x86_64-linux.nixos` after changing `nix/`: a VM test of diff --git a/README.md b/README.md index b7613cf..0a00689 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,8 @@ nix.settings = { On NixOS, use the flake's `nixosModules.default` (`services.helios`): the server, the maintenance daemon, and push-on-build for build machines. -Elsewhere, `nix build .#docker` builds an OCI image of the server and daemon -that generates its keys on first start. +Elsewhere, run the `ghcr.io/polymath-as/helios` image: the server and daemon +in one container that generates its keys on first start. See [docs.md](docs.md) for the NixOS module, configuration, private caches, the HTTP API and deployment behind a reverse proxy. diff --git a/docs.md b/docs.md index a0deeb0..66a48bc 100644 --- a/docs.md +++ b/docs.md @@ -124,14 +124,15 @@ a scrub that catches a corrupted NAR, watch-store, eviction and shutdown. ## Docker -`packages..docker` is an OCI image with the server and the -maintenance daemon, running as uid 10000 with all state in -`/var/lib/helios`: +`ghcr.io/polymath-as/helios` is an OCI image with the server and the +maintenance daemon for x86_64 and aarch64, running as uid 10000 with all +state in `/var/lib/helios`. CI publishes `latest` and the version for each +`v*` tag, `master` for the branch, and `sha-` for every build. To +build it yourself, `nix build .#docker` gives a tarball for `docker load`. ```sh -docker load < "$(nix build .#docker --print-out-paths)" docker run -d --name helios -p 8080:8080 -v helios:/var/lib/helios \ - -e HELIOS_QUOTA=500G helios:0.1.0 + -e HELIOS_QUOTA=500G ghcr.io/polymath-as/helios:latest docker exec helios helios-public-key # for trusted-public-keys docker exec helios helios-admin cache create main docker exec helios helios-admin token create ci --caches main --perms push,pull diff --git a/nix/docker.nix b/nix/docker.nix index e6bd5cc..9f9441e 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -132,6 +132,8 @@ pkgs.dockerTools.buildLayeredImage { "org.opencontainers.image.title" = "helios"; "org.opencontainers.image.description" = "Self-hosted Nix binary cache"; "org.opencontainers.image.version" = helios.version; + # Links the GHCR package to the repository. + "org.opencontainers.image.source" = "https://github.com/Polymath-AS/helios"; }; }; } From 82bb81f058740fa442de65ca63090ceb7e23700a Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 28/56] server: say why a token is refused Every auth failure answered a bare "unauthorized" or "forbidden", which leaves the token holder guessing whether it is expired, revoked, scoped to another cache or missing a permission. Name the reason instead: the holder already knows the claims, so this leaks nothing to them. --- crates/helios-server/src/auth.rs | 37 ++++++++++++++++++++++--------- crates/helios-server/src/error.rs | 8 ------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/crates/helios-server/src/auth.rs b/crates/helios-server/src/auth.rs index f283b9d..066c7fe 100644 --- a/crates/helios-server/src/auth.rs +++ b/crates/helios-server/src/auth.rs @@ -1,7 +1,7 @@ //! HS256 API tokens and the admin bearer secret. Revocation is checked //! against the in-memory token table, so it takes effect immediately. -use axum::http::{HeaderMap, header}; +use axum::http::{HeaderMap, StatusCode, header}; use base64::Engine; use base64::engine::general_purpose::{STANDARD, URL_SAFE_NO_PAD}; use serde::{Deserialize, Serialize}; @@ -37,6 +37,15 @@ pub struct Claims { pub exp: i64, } +impl std::fmt::Display for Perm { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Perm::Pull => "pull", + Perm::Push => "push", + }) + } +} + impl Claims { pub fn allows(&self, cache: &str, perm: Perm) -> bool { self.perms.contains(&perm) && self.caches.iter().any(|c| c == "*" || c == cache) @@ -108,6 +117,10 @@ fn credential(headers: &HeaderMap) -> Option { None } +fn unauthorized(message: &str) -> ApiError { + ApiError::new(StatusCode::UNAUTHORIZED, message) +} + fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { // Hash first so the comparison does not leak the secret's length. helios_core::ct_eq(&helios_core::sha256(a), &helios_core::sha256(b)) @@ -115,13 +128,17 @@ fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { impl AppState { fn token_claims(&self, headers: &HeaderMap) -> Result { - let secret = self.cfg.jwt_secret.as_deref().ok_or_else(ApiError::forbidden)?; - let token = credential(headers).ok_or_else(ApiError::unauthorized)?; + let secret = self.cfg.jwt_secret.as_deref().ok_or_else(|| ApiError::new(StatusCode::FORBIDDEN, "token auth is not configured"))?; + let token = credential(headers).ok_or_else(|| unauthorized("a token is required"))?; let now = crate::db::now(); - let claims = verify(&token, secret, now).ok_or_else(ApiError::unauthorized)?; + // Say why a token is refused: its holder already knows its claims, + // and a bare "unauthorized" leaves them guessing. + let claims = verify(&token, secret, now).ok_or_else(|| unauthorized("invalid or expired token"))?; match self.tokens.rd().get(claims.jti.as_str()) { - Some(t) if !t.revoked && t.expires_at > now => Ok(claims), - _ => Err(ApiError::unauthorized()), + Some(t) if t.revoked => Err(unauthorized("token has been revoked")), + Some(t) if t.expires_at > now => Ok(claims), + Some(_) => Err(unauthorized("token has expired")), + None => Err(unauthorized("token was not issued by this server")), } } @@ -129,7 +146,7 @@ impl AppState { pub fn authorize(&self, headers: &HeaderMap, cache: &str, perm: Perm) -> Result { let claims = self.token_claims(headers)?; if !claims.allows(cache, perm) { - return Err(ApiError::forbidden()); + return Err(ApiError::new(StatusCode::FORBIDDEN, format!("token does not grant {perm} on cache '{cache}'"))); } Ok(Identity::Token(claims)) } @@ -143,10 +160,10 @@ impl AppState { } pub fn authorize_admin(&self, headers: &HeaderMap) -> Result { - let secret = self.cfg.admin_secret.as_deref().ok_or_else(ApiError::forbidden)?; - let given = credential(headers).ok_or_else(ApiError::unauthorized)?; + let secret = self.cfg.admin_secret.as_deref().ok_or_else(|| ApiError::new(StatusCode::FORBIDDEN, "the admin API is not configured"))?; + let given = credential(headers).ok_or_else(|| unauthorized("the admin secret is required"))?; if !constant_time_eq(given.as_bytes(), secret) { - return Err(ApiError::forbidden()); + return Err(ApiError::new(StatusCode::FORBIDDEN, "the admin API takes the admin secret, not a cache token")); } Ok(Identity::Admin) } diff --git a/crates/helios-server/src/error.rs b/crates/helios-server/src/error.rs index d19afad..a0b608a 100644 --- a/crates/helios-server/src/error.rs +++ b/crates/helios-server/src/error.rs @@ -22,14 +22,6 @@ impl ApiError { Self::new(StatusCode::NOT_FOUND, "not found") } - pub fn forbidden() -> Self { - Self::new(StatusCode::FORBIDDEN, "forbidden") - } - - pub fn unauthorized() -> Self { - Self::new(StatusCode::UNAUTHORIZED, "unauthorized") - } - pub fn internal(err: impl std::fmt::Display) -> Self { crate::log::error!("internal error: {err}"); Self::new(StatusCode::INTERNAL_SERVER_ERROR, "internal error") From 587ad0a3fe9dc47a5bd0ea89bf61f7e22e26d34e Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 29/56] cli: create read-only tokens by default helios token create defaulted to push on every cache. Default to pull and require --caches, so a token only gets what it is given. The end-to-end test covers the default, that push does not imply pull, and the refusal messages. --- crates/helios-cli/src/main.rs | 10 ++++++---- docs.md | 8 ++++++-- scripts/e2e.sh | 7 ++++++- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/crates/helios-cli/src/main.rs b/crates/helios-cli/src/main.rs index 677fc37..36188ee 100644 --- a/crates/helios-cli/src/main.rs +++ b/crates/helios-cli/src/main.rs @@ -91,11 +91,13 @@ enum CacheCmd { enum TokenCmd { Create { subject: String, - /// Comma-separated cache names, or "*". - #[arg(long, default_value = "*")] + /// Comma-separated cache names, or "*" for all of them. + #[arg(long)] caches: String, - /// Comma-separated: push, pull. - #[arg(long, default_value = "push")] + /// Comma-separated: pull, push. Read-only unless push is asked for; + /// push does not imply pull, so a builder can upload without reading + /// private caches. + #[arg(long, default_value = "pull")] perms: String, /// Lifetime in days (1-365). #[arg(long, default_value_t = 90)] diff --git a/docs.md b/docs.md index 66a48bc..63ee2ff 100644 --- a/docs.md +++ b/docs.md @@ -180,15 +180,19 @@ helios cache create main # public: anyone can read helios cache create team --private # reads need a token with `pull` helios cache list +helios token create reader --caches team # read-only (the default) helios token create ci --caches main --perms push --expires 90 -helios token create reader --caches team --perms pull helios token list helios token revoke "reason" # takes effect immediately ``` Admin commands use a server logged in with the admin secret; pass `--server ` to pick one. Tokens are HS256 JWTs scoped to cache names -(or `*`) and to `push` and/or `pull`. +(or `*`) and to `pull` and/or `push`. Tokens are read-only unless created +with `push`, and `push` does not imply `pull`: a builder can upload to a +private cache without being able to read it. A refused request says why, +such as `token does not grant push on cache 'main'` or `token has been +revoked`. For private caches, give Nix the token through netrc. Nix sends it as HTTP Basic auth, and the server reads the password: diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 0d78223..a5df380 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -57,7 +57,9 @@ helios login admin "$URL" "$(cat "$WORK/admin")" >/dev/null helios cache create main >/dev/null helios cache create secret --private >/dev/null PUSH_TOKEN="$(helios token create ci --caches main,secret --perms push,pull 2>/dev/null | jq -r .token)" -PULL_ONLY="$(helios token create reader --caches secret --perms pull 2>/dev/null | jq -r .token)" +PULL_ONLY="$(helios token create reader --caches secret 2>/dev/null | jq -r .token)" +PUSH_ONLY="$(helios token create builder --caches secret --perms push 2>/dev/null | jq -r .token)" +check "tokens are read-only by default" '["pull"]' "$(helios token list | jq -c '.tokens[] | select(.subject=="reader") | .perms')" check "token issued" true "$([ -n "$PUSH_TOKEN" ] && echo true)" check "admin rejects push token" 403 "$(status -H "authorization: Bearer $PUSH_TOKEN" "$URL/_api/v2/admin/tokens")" @@ -90,6 +92,8 @@ helios push secret "$TARGET" >/dev/null 2>&1 check "private narinfo without auth" 401 "$(status "$URL/secret/$HASH.narinfo")" check "private narinfo with netrc-style basic auth" 200 "$(status -u "reader:$PULL_ONLY" "$URL/secret/$HASH.narinfo")" check "pull-only token cannot push" 403 "$(status -X POST -H "authorization: Bearer $PULL_ONLY" -H 'content-type: application/json' -d '{"hashes":[]}' "$URL/_api/v2/caches/secret/missing")" +check "push-only token cannot read" 403 "$(status -u "builder:$PUSH_ONLY" "$URL/secret/$HASH.narinfo")" +check "refusals say why" "token does not grant push on cache 'secret'" "$(curl -s -X POST -H "authorization: Bearer $PULL_ONLY" -H 'content-type: application/json' -d '{"hashes":[]}' "$URL/_api/v2/caches/secret/missing" | jq -r .error)" echo "=== abuse" check "garbage upload rejected" 400 "$(status -X PUT -H "authorization: Bearer $PUSH_TOKEN" --data-binary 'not a zstd stream' "$URL/_api/v2/caches/main/nar")" @@ -101,6 +105,7 @@ check "publish without uploaded NAR" 409 "$(status -X POST -H "authorization: Be JTI="$(helios --server admin token list | jq -r '.tokens[] | select(.subject=="reader") | .jti')" helios --server admin token revoke "$JTI" "e2e" >/dev/null check "revocation is immediate" 401 "$(status -u "reader:$PULL_ONLY" "$URL/secret/$HASH.narinfo")" +check "revoked token says so" "token has been revoked" "$(curl -s -u "reader:$PULL_ONLY" "$URL/secret/$HASH.narinfo" | jq -r .error)" echo echo "passed=$PASS failed=$FAIL" From a9de8fdacda7189f25bb08f29c2cdfdee9597045 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 30/56] log: move the binaries to tracing The server and daemon each had a hand-rolled logger that formatted a line, and the CLI wrote to stderr directly. Replace all three with tracing, set up once in the new helios-log crate, and log events with fields (blob, cache, bytes) rather than prose. HELIOS_LOG takes a level or per-target directives, parsed with Targets instead of env-filter and its regex engine. HELIOS_LOG_FORMAT picks journald (structured entries, the default when stderr is the journal), json for container log collectors, or text. JOURNAL_STREAM only counts when it matches stderr: a shell started from a service inherits it, and the CLI would otherwise log into the journal instead of the terminal. The CLI keeps its plain output, with warning: and error: prefixes. Requests log at debug; a 5xx also logs at warn with its method and path. Fatal errors at startup now go through the logger too. This adds 12 crates to the server (64 to 76) and the daemon. Narinfo hits measure about 1% slower (726k to 717k req/s), with or without the request middleware, which points at code layout rather than per-request work. --- AGENTS.md | 11 ++- Cargo.lock | 110 +++++++++++++++++++++++++ crates/helios-cli/Cargo.toml | 2 + crates/helios-cli/src/main.rs | 7 +- crates/helios-cli/src/push.rs | 10 +-- crates/helios-cli/src/watch.rs | 4 +- crates/helios-daemon/Cargo.toml | 2 + crates/helios-daemon/src/autogc.rs | 5 +- crates/helios-daemon/src/log.rs | 54 ------------- crates/helios-daemon/src/main.rs | 27 +++++-- crates/helios-daemon/src/maint.rs | 5 +- crates/helios-daemon/src/metrics.rs | 3 +- crates/helios-daemon/src/scrub.rs | 5 +- crates/helios-log/Cargo.toml | 12 +++ crates/helios-log/src/lib.rs | 115 +++++++++++++++++++++++++++ crates/helios-server/Cargo.toml | 2 + crates/helios-server/src/audit.rs | 4 +- crates/helios-server/src/error.rs | 2 +- crates/helios-server/src/gc.rs | 12 +-- crates/helios-server/src/internal.rs | 6 +- crates/helios-server/src/log.rs | 56 ------------- crates/helios-server/src/main.rs | 50 +++++++++--- crates/helios-server/src/state.rs | 2 +- docs.md | 22 ++++- nix/module.nix | 4 +- 25 files changed, 362 insertions(+), 170 deletions(-) delete mode 100644 crates/helios-daemon/src/log.rs create mode 100644 crates/helios-log/Cargo.toml create mode 100644 crates/helios-log/src/lib.rs delete mode 100644 crates/helios-server/src/log.rs diff --git a/AGENTS.md b/AGENTS.md index 67bde23..e4de078 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,8 +10,9 @@ A self-hosted Nix binary cache: Rust server and CLI over a Zig core. by `crates/helios-core/build.rs`. `core/include/helios.h` is the contract. - `crates/helios-core`: safe Rust bindings to libhelios. - `crates/helios-server`, `crates/helios-cli`, `crates/helios-daemon`: the - binaries. The daemon drives the server over its maintenance socket and - never writes the database itself. + binaries. `crates/helios-log` sets up their logging. The daemon drives + the server over its maintenance socket and never writes the database + itself. - `nix/`: the NixOS module (`services.helios`), the OCI image, and their VM tests. - `bench/`: comparisons against Cachix's libraries and Nix C++. Not part of @@ -35,10 +36,14 @@ coarse, such as a whole NAR or narinfo per call. Keep the dependency tree small; every crate is build time, audit surface and supply-chain risk. Before adding one, check whether libhelios, std or a few dozen lines cover it, and disable default features. Deliberately absent: -reqwest (and `url`/`idna`), aws-lc-rs, tracing, parking_lot, uuid, +reqwest (and `url`/`idna`), aws-lc-rs, parking_lot, uuid, hmac/sha2 (libhelios provides SHA-256, HMAC and randomness) and mimalloc (measured: no gain here). +Logging is `tracing` with `tracing-subscriber` (fmt, ansi and json only; no +`env-filter`, whose regex engine `Targets` makes unnecessary) and +`tracing-journald`, set up once in `crates/helios-log`. + ## Performance Rules - Measure before and after; `bench/run.sh` and `bench/flake-lock.sh` diff --git a/Cargo.lock b/Cargo.lock index fc625da..c95edfa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -274,6 +274,7 @@ dependencies = [ "clap", "futures-util", "helios-core", + "helios-log", "http", "http-body-util", "hyper", @@ -283,6 +284,7 @@ dependencies = [ "serde_json", "tokio", "tokio-rustls", + "tracing", ] [[package]] @@ -301,6 +303,7 @@ dependencies = [ "clap", "futures-util", "helios-core", + "helios-log", "http", "http-body-util", "hyper", @@ -308,6 +311,16 @@ dependencies = [ "serde", "serde_json", "tokio", + "tracing", +] + +[[package]] +name = "helios-log" +version = "0.1.0" +dependencies = [ + "tracing", + "tracing-journald", + "tracing-subscriber", ] [[package]] @@ -321,11 +334,13 @@ dependencies = [ "clap", "futures-util", "helios-core", + "helios-log", "quick_cache", "rusqlite", "serde", "serde_json", "tokio", + "tracing", ] [[package]] @@ -415,6 +430,12 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "libc" version = "0.2.189" @@ -461,6 +482,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -679,6 +709,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "2.0.1" @@ -740,6 +779,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + [[package]] name = "tokio" version = "1.53.1" @@ -804,6 +852,62 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-journald" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d3a81ed245bfb62592b1e2bc153e77656d94ee6a0497683a65a12ccaf2438d0" +dependencies = [ + "libc", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "nu-ansi-term", + "serde", + "serde_json", + "sharded-slab", + "thread_local", + "tracing-core", + "tracing-serde", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -822,6 +926,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/crates/helios-cli/Cargo.toml b/crates/helios-cli/Cargo.toml index 772e920..5c12eef 100644 --- a/crates/helios-cli/Cargo.toml +++ b/crates/helios-cli/Cargo.toml @@ -16,6 +16,7 @@ bytes = "1.12.1" clap = { version = "4.6.7", default-features = false, features = ["std", "derive", "env", "help", "usage", "error-context"] } futures-util = { version = "0.3.34", default-features = false, features = ["std"] } helios-core = { path = "../helios-core" } +helios-log = { path = "../helios-log" } http = "1.4" http-body-util = "0.1" # HTTP/1.1 client only; TLS via rustls with the ring provider (no aws-lc-rs). @@ -25,4 +26,5 @@ rustls-native-certs = "0.8" serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" tokio = { version = "1.53.1", features = ["rt-multi-thread", "net", "sync", "process", "io-util", "time"] } +tracing = { version = "0.1.44", default-features = false, features = ["std"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } diff --git a/crates/helios-cli/src/main.rs b/crates/helios-cli/src/main.rs index 36188ee..e406a79 100644 --- a/crates/helios-cli/src/main.rs +++ b/crates/helios-cli/src/main.rs @@ -121,7 +121,7 @@ fn print(v: &Value) { async fn run(cli: Cli) -> anyhow::Result<()> { if let Command::Login { name, url, token } = &cli.command { config::login(name, url, token)?; - println!("logged in to '{name}' at {url}"); + tracing::info!("logged in to '{name}' at {url}"); return Ok(()); } // Runs inside the nix-daemon's post-build hook: no server needed, and it @@ -161,7 +161,7 @@ async fn run(cli: Cli) -> anyhow::Result<()> { json!({ "subject": subject, "caches": split(&caches), "perms": split(&perms), "expiresInDays": expires }), ) .await?; - eprintln!("store this token now; it is not shown again"); + tracing::warn!("store this token now; it is not shown again"); print(&v); } Command::Token(TokenCmd::List) => print(&client.admin_get("/admin/tokens").await?), @@ -180,10 +180,11 @@ fn main() { // SAFETY: single-threaded; the runtime has not started yet. unsafe { std::env::set_var("SSL_CERT_FILE", file) }; } + helios_log::init(helios_log::Style::Cli); let cli = Cli::parse(); let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build().expect("starting the tokio runtime"); if let Err(e) = runtime.block_on(run(cli)) { - eprintln!("error: {e:#}"); + tracing::error!("{e:#}"); std::process::exit(1); } } diff --git a/crates/helios-cli/src/push.rs b/crates/helios-cli/src/push.rs index d1e79fe..b0a6486 100644 --- a/crates/helios-cli/src/push.rs +++ b/crates/helios-cli/src/push.rs @@ -117,10 +117,10 @@ async fn upload_all(client: &Client, cache: &str, paths: &[&PathInfo], opts: &Op match result { Ok(size) => { bytes += size; - eprintln!("[{done}/{total}] {} {} → {} ({:.2}s)", name(&info.path), human(info.nar_size), human(size), took.as_secs_f64()); + tracing::info!("[{done}/{total}] {} {} → {} ({:.2}s)", name(&info.path), human(info.nar_size), human(size), took.as_secs_f64()); } Err(e) => { - eprintln!("[{done}/{total}] {} FAILED: {e:#}", name(&info.path)); + tracing::error!("[{done}/{total}] {} failed: {e:#}", name(&info.path)); failures.push(info.path.clone()); } } @@ -143,7 +143,7 @@ pub async fn push(client: &Client, cache: &str, installables: &[String], opts: O } let todo: Vec = infos.into_iter().filter(|i| missing.contains(&i.hash)).collect(); if todo.is_empty() { - eprintln!("all {total} paths already in '{cache}'"); + tracing::info!("all {total} paths already in '{cache}'"); return Ok(()); } @@ -154,7 +154,7 @@ pub async fn push(client: &Client, cache: &str, installables: &[String], opts: O known.extend(client.known(cache, chunk).await?); } let uploads: Vec<&PathInfo> = todo.iter().filter(|i| !known.contains(&i.nar_hash)).collect(); - eprintln!("{total} paths, {} missing from '{cache}': uploading {}, reusing {} NARs", todo.len(), uploads.len(), todo.len() - uploads.len()); + tracing::info!("{total} paths, {} missing from '{cache}': uploading {}, reusing {} NARs", todo.len(), uploads.len(), todo.len() - uploads.len()); let mut uploaded_bytes = upload_all(client, cache, &uploads, &opts).await?; let ordered = nix::topo_order(todo); @@ -190,7 +190,7 @@ pub async fn push(client: &Client, cache: &str, installables: &[String], opts: O } let secs = started.elapsed().as_secs_f64(); - eprintln!( + tracing::info!( "published {published} paths to '{cache}', uploaded {} in {secs:.2}s ({}/s)", human(uploaded_bytes), human((uploaded_bytes as f64 / secs.max(1e-3)) as u64) diff --git a/crates/helios-cli/src/watch.rs b/crates/helios-cli/src/watch.rs index 11c3253..17fb121 100644 --- a/crates/helios-cli/src/watch.rs +++ b/crates/helios-cli/src/watch.rs @@ -49,7 +49,7 @@ fn pending(spool: &Path) -> anyhow::Result> { } pub async fn watch(client: &Client, cache: &str, spool: &Path, opts: push::Options) -> anyhow::Result<()> { - eprintln!("watching {} for paths to push to '{cache}'", spool.display()); + tracing::info!(spool = %spool.display(), cache, "watching for paths to push"); let mut backoff = POLL; loop { let entries = pending(spool)?; @@ -75,7 +75,7 @@ pub async fn watch(client: &Client, cache: &str, spool: &Path, opts: push::Optio backoff = POLL; } Err(e) => { - eprintln!("push failed, retrying in {}s: {e:#}", backoff.as_secs()); + tracing::warn!(retry_in = ?backoff, error = format!("{e:#}"), "push failed"); tokio::time::sleep(backoff).await; backoff = (backoff * 2).min(MAX_BACKOFF); } diff --git a/crates/helios-daemon/Cargo.toml b/crates/helios-daemon/Cargo.toml index 444a850..3292049 100644 --- a/crates/helios-daemon/Cargo.toml +++ b/crates/helios-daemon/Cargo.toml @@ -12,6 +12,7 @@ bytes = "1.12.1" clap = { version = "4.6.7", default-features = false, features = ["std", "derive", "env", "help", "usage", "error-context"] } futures-util = { version = "0.3.34", default-features = false, features = ["std"] } helios-core = { path = "../helios-core" } +helios-log = { path = "../helios-log" } http = "1.4" http-body-util = "0.1" hyper = { version = "1.8", default-features = false, features = ["client", "server", "http1"] } @@ -19,3 +20,4 @@ libc = "0.2" serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" tokio = { version = "1.53.1", features = ["rt-multi-thread", "net", "time", "signal", "sync", "io-util"] } +tracing = { version = "0.1.44", default-features = false, features = ["std"] } diff --git a/crates/helios-daemon/src/autogc.rs b/crates/helios-daemon/src/autogc.rs index 3e679f0..c9fdf51 100644 --- a/crates/helios-daemon/src/autogc.rs +++ b/crates/helios-daemon/src/autogc.rs @@ -7,7 +7,6 @@ use serde::Deserialize; use serde_json::json; -use crate::log; use crate::state::{Daemon, add, now, set}; #[derive(Clone, Copy, Debug)] @@ -87,7 +86,7 @@ pub async fn run(d: &Daemon, policy: &Policy) -> anyhow::Result<()> { if need == 0 { return Ok(()); } - log::info!("auto-gc: {referenced} bytes of NARs in use, freeing {need}"); + tracing::info!(referenced, need, "over quota, evicting"); let (mut evicted, mut freed) = (0u64, 0u64); for _ in 0..MAX_ROUNDS { let lru: Lru = d.client.get("/v1/lru?limit=500").await?; @@ -119,7 +118,7 @@ pub async fn run(d: &Daemon, policy: &Policy) -> anyhow::Result<()> { } add(&d.metrics.gc_evicted_paths, evicted); add(&d.metrics.gc_freed_bytes, freed); - log::info!("auto-gc: evicted {evicted} paths, {freed} bytes deleted now (the rest after the upload grace period)"); + tracing::info!(paths = evicted, freed_now = freed, "evicted (the rest is freed after the upload grace period)"); Ok(()) } diff --git a/crates/helios-daemon/src/log.rs b/crates/helios-daemon/src/log.rs deleted file mode 100644 index b3be188..0000000 --- a/crates/helios-daemon/src/log.rs +++ /dev/null @@ -1,54 +0,0 @@ -//! Leveled logging to stderr, with journald priorities under systemd. -//! Same format as helios-server; HELIOS_LOG sets the level. - -use std::sync::atomic::{AtomicU8, Ordering}; - -pub const ERROR: u8 = 3; -pub const WARN: u8 = 4; -pub const INFO: u8 = 6; -pub const DEBUG: u8 = 7; - -static LEVEL: AtomicU8 = AtomicU8::new(INFO); -static JOURNAL: AtomicU8 = AtomicU8::new(0); - -pub fn init() { - let level = match std::env::var("HELIOS_LOG").as_deref() { - Ok("error") => ERROR, - Ok("warn") => WARN, - Ok("debug") => DEBUG, - _ => INFO, - }; - LEVEL.store(level, Ordering::Relaxed); - JOURNAL.store(u8::from(std::env::var_os("JOURNAL_STREAM").is_some()), Ordering::Relaxed); -} - -pub fn enabled(level: u8) -> bool { - level <= LEVEL.load(Ordering::Relaxed) -} - -pub fn emit(level: u8, args: std::fmt::Arguments<'_>) { - let label = match level { - ERROR => "error", - WARN => "warn", - INFO => "info", - _ => "debug", - }; - if JOURNAL.load(Ordering::Relaxed) == 1 { - eprintln!("<{level}>{args}"); - } else { - eprintln!("{label}: {args}"); - } -} - -macro_rules! log { - ($level:expr, $($arg:tt)*) => { - if $crate::log::enabled($level) { - $crate::log::emit($level, format_args!($($arg)*)); - } - }; -} -macro_rules! error { ($($arg:tt)*) => { $crate::log::log!($crate::log::ERROR, $($arg)*) }; } -macro_rules! warning { ($($arg:tt)*) => { $crate::log::log!($crate::log::WARN, $($arg)*) }; } -macro_rules! info { ($($arg:tt)*) => { $crate::log::log!($crate::log::INFO, $($arg)*) }; } - -pub(crate) use {error, info, log, warning}; diff --git a/crates/helios-daemon/src/main.rs b/crates/helios-daemon/src/main.rs index 897e4f8..c79b60b 100644 --- a/crates/helios-daemon/src/main.rs +++ b/crates/helios-daemon/src/main.rs @@ -5,7 +5,6 @@ mod autogc; mod client; mod io; -mod log; mod maint; mod metrics; mod scrub; @@ -19,6 +18,7 @@ use std::sync::atomic::AtomicU64; use std::time::Duration; use clap::Parser; +use tracing::Instrument; use crate::state::{Daemon, add}; @@ -138,9 +138,9 @@ where interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); loop { interval.tick().await; - if let Err(e) = task().await { + if let Err(e) = task().instrument(tracing::info_span!("task", name)).await { add(errors, 1); - log::error!("{name}: {e:#}"); + tracing::error!(task = name, error = format!("{e:#}"), "task failed"); } } } @@ -153,8 +153,19 @@ async fn shutdown() { futures_util::future::select(term, int).await; } -fn main() -> anyhow::Result<()> { - log::init(); +fn main() -> std::process::ExitCode { + helios_log::init(helios_log::Style::Service); + // Fatal errors go through the logger too, so JSON and journald get them. + match run() { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(e) => { + tracing::error!("{e:#}"); + std::process::ExitCode::FAILURE + } + } +} + +fn run() -> anyhow::Result<()> { let args = Args::parse(); anyhow::ensure!( (0.0..=1.0).contains(&args.quota_low) && args.quota_low < args.quota_high && args.quota_high <= 1.0, @@ -203,13 +214,13 @@ fn main() -> anyhow::Result<()> { let d = d.clone(); tasks.push(tokio::spawn(async move { if let Err(e) = metrics::serve(d, addr).await { - log::error!("metrics: {e:#}"); + tracing::error!(error = format!("{e:#}"), "metrics server failed"); } })); } - log::info!("helios-daemon started ({} tasks)", tasks.len()); + tracing::info!(tasks = tasks.len(), "helios-daemon started"); shutdown().await; - log::info!("shutting down"); + tracing::info!("shutting down"); Ok(()) }) } diff --git a/crates/helios-daemon/src/maint.rs b/crates/helios-daemon/src/maint.rs index 6151daa..fac804a 100644 --- a/crates/helios-daemon/src/maint.rs +++ b/crates/helios-daemon/src/maint.rs @@ -4,14 +4,13 @@ use serde_json::{Value, json}; -use crate::log; use crate::state::{Daemon, add, now, set}; pub async fn checkpoint(d: &Daemon) -> anyhow::Result<()> { let r: Value = d.client.post("/v1/db/checkpoint", json!({})).await?; add(&d.metrics.db_checkpoints, 1); if r["busy"] == true { - log::warning!("db: checkpoint could not complete while readers were active"); + tracing::warn!("checkpoint could not complete while readers were active"); } Ok(()) } @@ -25,6 +24,6 @@ pub async fn backup(d: &Daemon, keep: usize) -> anyhow::Result<()> { let r: Value = d.client.post("/v1/db/backup", json!({ "keep": keep })).await?; add(&d.metrics.db_backups, 1); set(&d.metrics.db_last_backup, now()); - log::info!("db: backed up to {} ({} bytes)", r["path"].as_str().unwrap_or("?"), r["bytes"]); + tracing::info!(path = r["path"].as_str().unwrap_or("?"), bytes = r["bytes"].as_u64(), "backed up the database"); Ok(()) } diff --git a/crates/helios-daemon/src/metrics.rs b/crates/helios-daemon/src/metrics.rs index d93a4e7..d21aad0 100644 --- a/crates/helios-daemon/src/metrics.rs +++ b/crates/helios-daemon/src/metrics.rs @@ -14,7 +14,6 @@ use hyper::service::service_fn; use serde_json::Value; use crate::io::Io; -use crate::log; use crate::state::Daemon; fn line(out: &mut String, name: &str, kind: &str, help: &str, samples: &[(&str, u64)]) { @@ -90,7 +89,7 @@ pub async fn render(d: &Daemon) -> String { pub async fn serve(d: Arc, addr: SocketAddr) -> anyhow::Result<()> { let listener = tokio::net::TcpListener::bind(addr).await?; - log::info!("metrics on http://{addr}/metrics"); + tracing::info!(url = %format_args!("http://{addr}/metrics"), "metrics listening"); loop { let (stream, _) = listener.accept().await?; let d = d.clone(); diff --git a/crates/helios-daemon/src/scrub.rs b/crates/helios-daemon/src/scrub.rs index f7d80f4..eb8f008 100644 --- a/crates/helios-daemon/src/scrub.rs +++ b/crates/helios-daemon/src/scrub.rs @@ -10,7 +10,6 @@ use helios_core::{Compression, Verifier}; use serde::Deserialize; use serde_json::json; -use crate::log; use crate::state::{Daemon, add, now, set}; #[derive(Deserialize, Clone)] @@ -102,7 +101,7 @@ pub async fn run(d: &Daemon, rate: u64) -> anyhow::Result<()> { } }; if let Some(reason) = reason { - log::error!("scrub: blob {} is bad ({reason}); quarantining", b.file_hash); + tracing::error!(blob = %b.file_hash, %reason, "bad blob, quarantining"); // A blob GC removed since it was listed is already gone. if let Err(e) = d.client.post::("/v1/quarantine", json!({ "fileHash": b.file_hash, "reason": reason })).await && !e.to_string().contains("404") @@ -119,6 +118,6 @@ pub async fn run(d: &Daemon, rate: u64) -> anyhow::Result<()> { } } set(&d.metrics.scrub_last_complete, now()); - log::info!("scrub: checked {blobs} blobs ({bytes} bytes) in {:.0?}", started.elapsed()); + tracing::info!(blobs, bytes, elapsed = ?started.elapsed(), "scrub finished"); Ok(()) } diff --git a/crates/helios-log/Cargo.toml b/crates/helios-log/Cargo.toml new file mode 100644 index 0000000..3ec9470 --- /dev/null +++ b/crates/helios-log/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "helios-log" +description = "Logging setup shared by the helios binaries" +version.workspace = true +edition.workspace = true +license-file.workspace = true +publish.workspace = true + +[dependencies] +tracing = { version = "0.1.44", default-features = false, features = ["std"] } +tracing-journald = "0.3.2" +tracing-subscriber = { version = "0.3.23", default-features = false, features = ["std", "fmt", "ansi", "json"] } diff --git a/crates/helios-log/src/lib.rs b/crates/helios-log/src/lib.rs new file mode 100644 index 0000000..7acea2a --- /dev/null +++ b/crates/helios-log/src/lib.rs @@ -0,0 +1,115 @@ +//! Logging setup shared by the helios binaries, over `tracing`. +//! +//! `HELIOS_LOG` filters events: a level (`error`, `warn`, `info`, `debug`, +//! `trace`; default `info`), optionally with per-target directives such as +//! `info,helios_server=debug`. `HELIOS_LOG_FORMAT` picks the output: +//! +//! - `journald`: structured entries with priorities, event fields and span +//! fields as journal fields. The default when stderr is the journal. +//! - `json`: one object per line on stderr, for container log collectors. +//! - `text`: human-readable lines on stderr. The default otherwise. + +use std::io::IsTerminal; + +use tracing::{Event, Level, Subscriber}; +use tracing_subscriber::Layer; +use tracing_subscriber::filter::{LevelFilter, Targets}; +use tracing_subscriber::fmt::format::Writer; +use tracing_subscriber::fmt::{self, FmtContext, FormatEvent, FormatFields}; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::registry::{LookupSpan, Registry}; +use tracing_subscriber::util::SubscriberInitExt; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Style { + /// A long-running service: text lines carry a timestamp, level, spans + /// and target. + Service, + /// A command run by hand: text lines are the message alone, prefixed + /// with `warning:` or `error:` the way command-line tools report. + Cli, +} + +type Output = Box + Send + Sync>; + +/// Installs the global subscriber. Call once, first thing in `main`. +pub fn init(style: Style) { + let (filter, bad_filter) = match std::env::var("HELIOS_LOG") { + Ok(spec) if !spec.trim().is_empty() => match spec.parse::() { + Ok(filter) => (filter, None), + Err(e) => (default_filter(), Some(format!("ignoring HELIOS_LOG={spec:?}: {e}"))), + }, + _ => (default_filter(), None), + }; + + let format = std::env::var("HELIOS_LOG_FORMAT").unwrap_or_default(); + let (output, fallback) = match format.as_str() { + "json" => (json(), None), + "text" => (text(style), None), + "journald" => journald().map_or_else(|e| (text(style), Some(format!("journald unavailable, logging to stderr: {e}"))), |j| (j, None)), + "" | "auto" if stderr_is_journal() => journald().map_or_else(|_| (text(style), None), |j| (j, None)), + "" | "auto" => (text(style), None), + other => (text(style), Some(format!("unknown HELIOS_LOG_FORMAT {other:?}; expected text, json or journald"))), + }; + + let _ = tracing_subscriber::registry().with(output.with_filter(filter)).try_init(); + for problem in [bad_filter, fallback].into_iter().flatten() { + tracing::warn!("{problem}"); + } +} + +/// Whether stderr is the journal: systemd sets JOURNAL_STREAM to its +/// `device:inode`, but children inherit the variable, such as a shell +/// started from a service, so only a match counts. +fn stderr_is_journal() -> bool { + use std::os::fd::AsFd; + use std::os::unix::fs::MetadataExt; + let Some((dev, ino)) = std::env::var("JOURNAL_STREAM").ok().and_then(|s| s.split_once(':').map(|(d, i)| (d.parse::(), i.parse::()))) + else { + return false; + }; + let Ok(stderr) = std::io::stderr().as_fd().try_clone_to_owned() else { return false }; + let Ok(meta) = std::fs::File::from(stderr).metadata() else { return false }; + dev == Ok(meta.dev()) && ino == Ok(meta.ino()) +} + +fn default_filter() -> Targets { + Targets::new().with_default(LevelFilter::INFO) +} + +fn journald() -> std::io::Result { + Ok(Box::new(tracing_journald::layer()?)) +} + +fn json() -> Output { + Box::new(fmt::layer().json().flatten_event(true).with_current_span(true).with_span_list(false).with_writer(std::io::stderr)) +} + +fn text(style: Style) -> Output { + let ansi = std::io::stderr().is_terminal(); + match style { + Style::Service => Box::new(fmt::layer().with_ansi(ansi).with_writer(std::io::stderr)), + Style::Cli => Box::new(fmt::layer().event_format(CliFormat).with_ansi(ansi).with_writer(std::io::stderr)), + } +} + +/// `message key=value`, with a level prefix except on `info`. +struct CliFormat; + +impl FormatEvent for CliFormat +where + S: Subscriber + for<'a> LookupSpan<'a>, + N: for<'a> FormatFields<'a> + 'static, +{ + fn format_event(&self, ctx: &FmtContext<'_, S, N>, mut writer: Writer<'_>, event: &Event<'_>) -> std::fmt::Result { + match *event.metadata().level() { + Level::ERROR => write!(writer, "error: ")?, + Level::WARN => write!(writer, "warning: ")?, + Level::INFO => {} + Level::DEBUG => write!(writer, "debug: ")?, + Level::TRACE => write!(writer, "trace: ")?, + } + ctx.format_fields(writer.by_ref(), event)?; + writeln!(writer) + } +} diff --git a/crates/helios-server/Cargo.toml b/crates/helios-server/Cargo.toml index 5f54ea0..701b220 100644 --- a/crates/helios-server/Cargo.toml +++ b/crates/helios-server/Cargo.toml @@ -19,8 +19,10 @@ bytes = "1.12.1" clap = { version = "4.6.7", default-features = false, features = ["std", "derive", "env", "help", "usage", "error-context"] } futures-util = { version = "0.3.34", default-features = false, features = ["std"] } helios-core = { path = "../helios-core" } +helios-log = { path = "../helios-log" } quick_cache = { version = "0.7.0", default-features = false } rusqlite = { version = "0.40.2", default-features = false, features = ["cache", "array", "fallible_uint"] } serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" tokio = { version = "1.53.1", features = ["rt-multi-thread", "net", "fs", "signal", "sync", "time", "io-util"] } +tracing = { version = "0.1.44", default-features = false, features = ["std"] } diff --git a/crates/helios-server/src/audit.rs b/crates/helios-server/src/audit.rs index ca6bf7d..0ee70ab 100644 --- a/crates/helios-server/src/audit.rs +++ b/crates/helios-server/src/audit.rs @@ -83,8 +83,8 @@ pub async fn run(db: Db, mut rx: mpsc::UnboundedReceiver) { .await; match result { Ok(Ok(())) => {} - Ok(Err(e)) => crate::log::error!("writing audit log: {e}"), - Err(e) => crate::log::error!("audit writer panicked: {e}"), + Ok(Err(e)) => tracing::error!(error = %e, "writing audit log"), + Err(e) => tracing::error!(error = %e, "audit writer panicked"), } } } diff --git a/crates/helios-server/src/error.rs b/crates/helios-server/src/error.rs index a0b608a..2f14228 100644 --- a/crates/helios-server/src/error.rs +++ b/crates/helios-server/src/error.rs @@ -23,7 +23,7 @@ impl ApiError { } pub fn internal(err: impl std::fmt::Display) -> Self { - crate::log::error!("internal error: {err}"); + tracing::error!(error = %err, "internal error"); Self::new(StatusCode::INTERNAL_SERVER_ERROR, "internal error") } diff --git a/crates/helios-server/src/gc.rs b/crates/helios-server/src/gc.rs index c945fe1..3d5aef5 100644 --- a/crates/helios-server/src/gc.rs +++ b/crates/helios-server/src/gc.rs @@ -31,9 +31,9 @@ pub async fn run_forever(st: Shared) { interval.tick().await; let st2 = st.clone(); match tokio::task::spawn_blocking(move || collect(&st2)).await { - Ok(Ok(stats)) => crate::log::info!("gc finished: {stats:?}"), - Ok(Err(e)) => crate::log::error!("gc failed: {e}"), - Err(e) => crate::log::error!("gc panicked: {e}"), + Ok(Ok(stats)) => tracing::info!(?stats, "gc finished"), + Ok(Err(e)) => tracing::error!(error = %e, "gc failed"), + Err(e) => tracing::error!(error = %e, "gc panicked"), } } } @@ -97,7 +97,7 @@ pub fn collect_blobs(st: &Shared) -> anyhow::Result<(usize, u64)> { match std::fs::remove_file(st.nar_path(&hash, compression)) { Ok(()) => freed += size, Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} - Err(e) => crate::log::warning!("removing blob file: {e}"), + Err(e) => tracing::warn!(error = %e, "removing blob file"), } } Ok(victims.len()) @@ -136,8 +136,8 @@ pub async fn run_access_flush(st: Shared) { let st2 = st.clone(); match tokio::task::spawn_blocking(move || flush_access(&st2)).await { Ok(Ok(_)) => {} - Ok(Err(e)) => crate::log::error!("flushing access times: {e}"), - Err(e) => crate::log::error!("access flush panicked: {e}"), + Ok(Err(e)) => tracing::error!(error = %e, "flushing access times"), + Err(e) => tracing::error!(error = %e, "access flush panicked"), } } } diff --git a/crates/helios-server/src/internal.rs b/crates/helios-server/src/internal.rs index d5dbb5f..790a6fe 100644 --- a/crates/helios-server/src/internal.rs +++ b/crates/helios-server/src/internal.rs @@ -162,7 +162,7 @@ async fn evict(State(st): State, Json(req): Json) -> ApiResult // Stop serving before the files go. forget(&st, &keys); let (blobs, bytes) = crate::gc::collect_blobs(&st).map_err(internal)?; - crate::log::info!("evicted {evicted} paths, freeing {blobs} blobs ({bytes} bytes)"); + tracing::info!(paths = evicted, blobs, bytes, "evicted"); Ok(Json(json!({ "evicted": evicted, "freedBlobs": blobs, "freedBytes": bytes }))) }) .await @@ -249,10 +249,10 @@ async fn quarantine(State(st): State, Json(req): Json) -> if let Err(e) = moved && e.kind() != std::io::ErrorKind::NotFound { - crate::log::warning!("moving quarantined blob: {e}"); + tracing::warn!(error = %e, "moving quarantined blob"); } } - crate::log::warning!("quarantined blob {}: {} ({} paths unpublished)", req.file_hash, req.reason, keys.len()); + tracing::warn!(blob = %req.file_hash, reason = %req.reason, unpublished = keys.len(), "quarantined blob"); Ok(Json(json!({ "pathsRemoved": keys.len() }))) }) .await diff --git a/crates/helios-server/src/log.rs b/crates/helios-server/src/log.rs deleted file mode 100644 index a0f8fe5..0000000 --- a/crates/helios-server/src/log.rs +++ /dev/null @@ -1,56 +0,0 @@ -//! Minimal leveled logging to stderr. Under systemd (JOURNAL_STREAM is set) -//! lines carry a `` syslog priority prefix so journald records levels; -//! journald adds timestamps. The level comes from HELIOS_LOG -//! (error, warn, info, debug; default info). - -use std::sync::atomic::{AtomicU8, Ordering}; - -pub const ERROR: u8 = 3; -pub const WARN: u8 = 4; -pub const INFO: u8 = 6; -pub const DEBUG: u8 = 7; - -static LEVEL: AtomicU8 = AtomicU8::new(INFO); -static JOURNAL: AtomicU8 = AtomicU8::new(0); - -pub fn init() { - let level = match std::env::var("HELIOS_LOG").as_deref() { - Ok("error") => ERROR, - Ok("warn") => WARN, - Ok("debug") => DEBUG, - _ => INFO, - }; - LEVEL.store(level, Ordering::Relaxed); - JOURNAL.store(u8::from(std::env::var_os("JOURNAL_STREAM").is_some()), Ordering::Relaxed); -} - -pub fn enabled(level: u8) -> bool { - level <= LEVEL.load(Ordering::Relaxed) -} - -pub fn emit(level: u8, args: std::fmt::Arguments<'_>) { - let label = match level { - ERROR => "error", - WARN => "warn", - INFO => "info", - _ => "debug", - }; - if JOURNAL.load(Ordering::Relaxed) == 1 { - eprintln!("<{level}>{args}"); - } else { - eprintln!("{label}: {args}"); - } -} - -macro_rules! log { - ($level:expr, $($arg:tt)*) => { - if $crate::log::enabled($level) { - $crate::log::emit($level, format_args!($($arg)*)); - } - }; -} -macro_rules! error { ($($arg:tt)*) => { $crate::log::log!($crate::log::ERROR, $($arg)*) }; } -macro_rules! warning { ($($arg:tt)*) => { $crate::log::log!($crate::log::WARN, $($arg)*) }; } -macro_rules! info { ($($arg:tt)*) => { $crate::log::log!($crate::log::INFO, $($arg)*) }; } - -pub(crate) use {error, info, log, warning}; diff --git a/crates/helios-server/src/main.rs b/crates/helios-server/src/main.rs index 850d792..33cd8ea 100644 --- a/crates/helios-server/src/main.rs +++ b/crates/helios-server/src/main.rs @@ -6,7 +6,6 @@ mod db; mod error; mod gc; mod internal; -mod log; mod push; mod read; mod state; @@ -21,6 +20,7 @@ use axum::http::StatusCode; use axum::routing::{get, post, put}; use clap::Parser; use serde_json::json; +use tracing::Instrument; use crate::config::{Args, Config}; use crate::state::{AppState, Shared}; @@ -48,9 +48,26 @@ pub fn router(st: Shared) -> Router { .route("/healthz", get(healthz)) .nest("/_api/v2", api) .fallback(read::handle) + .layer(axum::middleware::from_fn(trace_request)) .with_state(st) } +/// Requests log at debug. Server errors log at warn with the request, which +/// the handler's own error event does not name. +async fn trace_request(req: axum::extract::Request, next: axum::middleware::Next) -> axum::response::Response { + let method = req.method().clone(); + let uri = req.uri().clone(); + let started = std::time::Instant::now(); + let resp = next.run(req).instrument(tracing::debug_span!("request", %method, path = uri.path())).await; + let status = resp.status(); + if status.is_server_error() { + tracing::warn!(%method, path = uri.path(), status = status.as_u16(), elapsed = ?started.elapsed(), "request failed"); + } else { + tracing::debug!(%method, path = uri.path(), status = status.as_u16(), elapsed = ?started.elapsed(), "request"); + } + resp +} + async fn healthz(State(st): State) -> (StatusCode, axum::Json) { let db = st.db.clone(); let ok = tokio::task::spawn_blocking(move || db.read(|c| c.query_row("SELECT 1", [], |_| Ok(()))).is_ok()).await.unwrap_or(false); @@ -60,12 +77,14 @@ async fn healthz(State(st): State) -> (StatusCode, axum::Json anyhow::Result { let cfg = Config::from_args(args)?; - std::fs::create_dir_all(cfg.data_dir.join("nar"))?; - std::fs::create_dir_all(cfg.data_dir.join("tmp"))?; + for dir in ["nar", "tmp"] { + let dir = cfg.data_dir.join(dir); + std::fs::create_dir_all(&dir).map_err(|e| anyhow::anyhow!("creating {}: {e}", dir.display()))?; + } let db = db::Db::open(&cfg.data_dir.join("helios.db"))?; let signer = load_signer(args)?; if signer.is_none() { - crate::log::warning!("no signing key configured; narinfo will be unsigned"); + tracing::warn!("no signing key configured; narinfo will be unsigned"); } let (audit_tx, audit_rx) = tokio::sync::mpsc::unbounded_channel(); tokio::spawn(audit::run(db.clone(), audit_rx)); @@ -75,8 +94,19 @@ pub async fn build_state(args: &Args) -> anyhow::Result { Ok(st) } -fn main() -> anyhow::Result<()> { - log::init(); +fn main() -> std::process::ExitCode { + helios_log::init(helios_log::Style::Service); + // Fatal errors go through the logger too, so JSON and journald get them. + match run() { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(e) => { + tracing::error!("{e:#}"); + std::process::ExitCode::FAILURE + } + } +} + +fn run() -> anyhow::Result<()> { let args = Args::parse(); if let Some(config::Command::GenerateSecrets { dir, key_name }) = &args.command { @@ -105,7 +135,7 @@ fn main() -> anyhow::Result<()> { let listener = tokio::net::UnixListener::bind(path)?; // Group access only: the socket directory decides who that is. std::fs::set_permissions(path, std::os::unix::fs::PermissionsExt::from_mode(0o660))?; - crate::log::info!("maintenance API on {}", path.display()); + tracing::info!(socket = %path.display(), "maintenance API listening"); let serve = axum::serve(listener, internal::router(st.clone())).with_graceful_shutdown(stopped(stop_rx.clone())); Some(tokio::spawn(async move { serve.await })) } @@ -113,7 +143,7 @@ fn main() -> anyhow::Result<()> { }; let listener = tokio::net::TcpListener::bind(args.listen).await?; - crate::log::info!("listening on {}", args.listen); + tracing::info!(addr = %args.listen, "listening"); axum::serve(listener, router(st.clone()).into_make_service_with_connect_info::()) .with_graceful_shutdown(stopped(stop_rx)) .await?; @@ -133,7 +163,7 @@ async fn shutdown_signal() { let term = std::pin::pin!(term.recv()); let int = std::pin::pin!(tokio::signal::ctrl_c()); futures_util::future::select(term, int).await; - crate::log::info!("shutting down"); + tracing::info!("shutting down"); } /// Writes `contents` to `path` only if it does not exist, readable by the @@ -167,7 +197,7 @@ fn generate_secrets(dir: &std::path::Path, key_name: &str) -> anyhow::Result<()> ("admin-secret", create_secret(&dir.join("admin-secret"), random_hex)?), ] { if created { - log::info!("created {}", dir.join(name).display()); + tracing::info!(path = %dir.join(name).display(), "created secret"); } } let key = std::fs::read_to_string(&signing)?; diff --git a/crates/helios-server/src/state.rs b/crates/helios-server/src/state.rs index 55fc5fb..6c01874 100644 --- a/crates/helios-server/src/state.rs +++ b/crates/helios-server/src/state.rs @@ -111,7 +111,7 @@ impl AppState { Ok((caches, index, tokens)) })?; - crate::log::info!("loaded state: {} caches, {} paths, {} tokens", caches.len(), index.len(), tokens.len()); + tracing::info!(caches = caches.len(), paths = index.len(), tokens = tokens.len(), "loaded state"); let narinfo = quick_cache::sync::Cache::with( cfg.narinfo_cache_entries, cfg.narinfo_cache_entries as u64, diff --git a/docs.md b/docs.md index 63ee2ff..959b26b 100644 --- a/docs.md +++ b/docs.md @@ -36,9 +36,22 @@ Every flag can also be set through its environment variable. | `--upload-grace-seconds` | `HELIOS_UPLOAD_GRACE_SECONDS` | `3600`: an unpublished upload is kept this long | | `--admin-socket` | `HELIOS_ADMIN_SOCKET` | unset: no maintenance API (see helios-daemon) | -Secret files must hold at least 16 bytes. `HELIOS_LOG` sets the log level -(`error`, `warn`, `info` or `debug`; default `info`). Under systemd, log lines -carry journald priorities. +Secret files must hold at least 16 bytes. + +## Logging + +The server, daemon and CLI log through `tracing`, configured the same way: + +| Variable | | +|----------|--| +| `HELIOS_LOG` | a level (`error`, `warn`, `info`, `debug`, `trace`; default `info`), optionally with per-target directives, such as `info,helios_server=debug` | +| `HELIOS_LOG_FORMAT` | `journald`, `json` or `text`. The default is `journald` under systemd and `text` otherwise | + +Under journald, events are structured entries: their fields (`blob`, +`cache`, `bytes` and so on) are journal fields, so `journalctl -u helios +PRIORITY=4` or `journalctl BLOB=` filters on them. `json` writes one +object per line for container log collectors. Requests log at `debug`; a +request that fails with a 5xx logs at `warn` with its method and path. The server links the system SQLite; build with `--features bundled-sqlite` to compile SQLite in instead. @@ -148,7 +161,8 @@ instead, for example as Docker secrets, set all three of Both binaries take their flags as `HELIOS_*` environment variables (see `helios-server --help` and `helios-daemon --help`), such as `HELIOS_QUOTA`, -`HELIOS_MIN_FREE`, `HELIOS_METRICS_LISTEN` and `HELIOS_LOG`. Set +`HELIOS_MIN_FREE`, `HELIOS_METRICS_LISTEN`, `HELIOS_LOG` and +`HELIOS_LOG_FORMAT=json` (see [Logging](#logging)). Set `HELIOS_DAEMON=0` to run the server alone. `docker stop` shuts both down cleanly; if either exits on its own, the container exits non-zero so the restart policy takes over. The image has no reverse proxy: put one in front diff --git a/nix/module.nix b/nix/module.nix index 67fa72c..707c36a 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -130,8 +130,10 @@ in }; logLevel = mkOption { - type = types.enum [ "error" "warn" "info" "debug" ]; + type = types.str; default = "info"; + example = "info,helios_server=debug"; + description = "HELIOS_LOG for the server and daemon: a level, optionally with per-target directives."; }; settings = { From eeb8b26effd159918800c5996ef82c7ab1184fc4 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 31/56] push: upload large NARs in resumable chunks A NAR was always one PUT, so pushes failed behind proxies that cap request bodies: 100 MB on Cloudflare's free plan, 32 MiB on Cloud Run. Attic users hit this repeatedly (zhaofengli/attic#141, #220, #287). The server gains a chunked upload: open a session, append chunks at a stated offset, complete. A chunk is verified as it arrives and appended whole or not at all, so after a failed request the client reads the offset and resends only if it did not land; a stale offset gets 409 with the current one. Sessions belong to the token that opened them, expire after an hour idle and do not survive a restart. helios push and watch-store send NARs larger than --chunk-size (32 MiB) that way and retry each chunk with backoff; smaller NARs still go in one request, and --chunk-size 0 streams every NAR whole. --- crates/helios-cli/src/api.rs | 63 ++++++++ crates/helios-cli/src/main.rs | 18 ++- crates/helios-cli/src/push.rs | 67 ++++++++- crates/helios-server/src/chunked.rs | 225 ++++++++++++++++++++++++++++ crates/helios-server/src/main.rs | 4 + crates/helios-server/src/push.rs | 16 +- crates/helios-server/src/state.rs | 2 + docs.md | 17 +++ scripts/e2e.sh | 38 ++++- 9 files changed, 431 insertions(+), 19 deletions(-) create mode 100644 crates/helios-server/src/chunked.rs diff --git a/crates/helios-cli/src/api.rs b/crates/helios-cli/src/api.rs index f2d6221..e3e866f 100644 --- a/crates/helios-cli/src/api.rs +++ b/crates/helios-cli/src/api.rs @@ -84,6 +84,69 @@ impl Client { self.call(Method::PUT, &format!("/caches/{cache}/nar?compression=zstd"), Some("application/x-nix-nar"), body).await } + /// Starts a chunked upload; returns its id. + pub async fn upload_create(&self, cache: &str) -> anyhow::Result { + #[derive(Deserialize)] + struct R { + id: String, + } + let r: R = self.call(Method::POST, &format!("/caches/{cache}/uploads?compression=zstd"), None, transport::full(Bytes::new())).await?; + Ok(r.id) + } + + /// Appends `chunk` at `offset`, retrying transient failures. After a + /// failed request the chunk may or may not have landed; the server's + /// offset says which, and a chunk is never appended twice. + pub async fn upload_append(&self, cache: &str, id: &str, offset: u64, chunk: Bytes) -> anyhow::Result { + #[derive(Deserialize)] + struct R { + offset: u64, + } + let end = offset + chunk.len() as u64; + let path = format!("/_api/v2/caches/{cache}/uploads/{id}"); + let mut delay = std::time::Duration::from_millis(250); + for attempt in 1.. { + let failure = + match self.http.send(Method::PATCH, &format!("{path}?offset={offset}"), &self.token, None, transport::full(chunk.clone())).await { + Ok((status, bytes)) if status.is_success() => return Ok(serde_json::from_slice::(&bytes)?.offset), + Ok((status, bytes)) if status == StatusCode::CONFLICT => { + let at = serde_json::from_slice::(&bytes).ok().and_then(|v| v["offset"].as_u64()); + match at { + Some(at) if at == end => return Ok(end), + _ => bail!("{}", error_text(status, &bytes)), + } + } + Ok((status, bytes)) if status.is_server_error() || status == StatusCode::TOO_MANY_REQUESTS => error_text(status, &bytes), + Ok((status, bytes)) => bail!("{}", error_text(status, &bytes)), + Err(e) => format!("{e:#}"), + }; + if attempt == 5 { + bail!("uploading a chunk at offset {offset}: {failure}"); + } + tracing::warn!("uploading a chunk at offset {offset} failed, retrying in {delay:?}: {failure}"); + tokio::time::sleep(delay).await; + delay *= 2; + // The request may have landed before the failure; if so, move on. + if let Ok(r) = self.call::(Method::GET, &format!("/caches/{cache}/uploads/{id}"), None, transport::full(Bytes::new())).await { + match r.offset { + at if at == end => return Ok(end), + at if at == offset => {} + at => bail!("server is at offset {at}, expected {offset} or {end}"), + } + } + } + unreachable!() + } + + pub async fn upload_complete(&self, cache: &str, id: &str) -> anyhow::Result { + self.call(Method::POST, &format!("/caches/{cache}/uploads/{id}/complete"), None, transport::full(Bytes::new())).await + } + + pub async fn upload_abort(&self, cache: &str, id: &str) { + let _ = + self.http.send(Method::DELETE, &format!("/_api/v2/caches/{cache}/uploads/{id}"), &self.token, None, transport::full(Bytes::new())).await; + } + pub async fn publish(&self, cache: &str, paths: &[PathSpec]) -> anyhow::Result { let body = transport::full(serde_json::to_vec(&json!({ "paths": paths }))?); let (status, bytes) = diff --git a/crates/helios-cli/src/main.rs b/crates/helios-cli/src/main.rs index e406a79..fe571a2 100644 --- a/crates/helios-cli/src/main.rs +++ b/crates/helios-cli/src/main.rs @@ -47,6 +47,11 @@ enum Command { /// zstd compression level (1-19). #[arg(long, default_value_t = 3, value_parser = clap::value_parser!(i32).range(1..=19))] level: i32, + /// Upload NARs in chunks of this many MiB, for proxies that cap + /// request bodies (Cloudflare, Cloud Run). 0 sends each NAR in one + /// request; so does a NAR that fits in one chunk. + #[arg(long, default_value_t = 32, value_parser = clap::value_parser!(u64).range(0..=64))] + chunk_size: u64, }, /// Push paths as they are built: drain a spool filled by `queue-paths`. WatchStore { @@ -60,6 +65,11 @@ enum Command { /// zstd compression level (1-19). #[arg(long, default_value_t = 3, value_parser = clap::value_parser!(i32).range(1..=19))] level: i32, + /// Upload NARs in chunks of this many MiB, for proxies that cap + /// request bodies (Cloudflare, Cloud Run). 0 sends each NAR in one + /// request; so does a NAR that fits in one chunk. + #[arg(long, default_value_t = 32, value_parser = clap::value_parser!(u64).range(0..=64))] + chunk_size: u64, }, /// Spool store paths for `watch-store` (Nix post-build-hook; reads $OUT_PATHS). QueuePaths { @@ -144,11 +154,11 @@ async fn run(cli: Cli) -> anyhow::Result<()> { let client = api::Client::new(&server)?; match cli.command { Command::Login { .. } | Command::QueuePaths { .. } => unreachable!(), - Command::Push { cache, installables, closure, jobs, level } => { - push::push(&client, &cache, &installables, push::Options { jobs, level, closure }).await?; + Command::Push { cache, installables, closure, jobs, level, chunk_size } => { + push::push(&client, &cache, &installables, push::Options { jobs, level, closure, chunk_size: (chunk_size << 20) as usize }).await?; } - Command::WatchStore { cache, spool, jobs, level } => { - watch::watch(&client, &cache, &spool, push::Options { jobs, level, closure: true }).await?; + Command::WatchStore { cache, spool, jobs, level, chunk_size } => { + watch::watch(&client, &cache, &spool, push::Options { jobs, level, closure: true, chunk_size: (chunk_size << 20) as usize }).await?; } Command::Cache(CacheCmd::Create { name, private }) => { print(&client.admin_post("/admin/caches", json!({ "name": name, "public": !private })).await?); diff --git a/crates/helios-cli/src/push.rs b/crates/helios-cli/src/push.rs index b0a6486..675a581 100644 --- a/crates/helios-cli/src/push.rs +++ b/crates/helios-cli/src/push.rs @@ -8,12 +8,12 @@ use std::time::Instant; use anyhow::{Context, bail}; use base64::Engine; -use bytes::Bytes; +use bytes::{Bytes, BytesMut}; use futures_util::StreamExt; use helios_core::{DumpOptions, HL_E_UNSUPPORTED_OS}; use tokio::sync::mpsc; -use crate::api::{Client, PathSpec, Publish}; +use crate::api::{Client, PathSpec, Publish, Uploaded}; use crate::nix::{self, PathInfo}; const QUERY_BATCH: usize = 50_000; @@ -24,6 +24,8 @@ pub struct Options { pub jobs: usize, pub level: i32, pub closure: bool, + /// Bytes per chunked-upload request; 0 streams each NAR in one request. + pub chunk_size: usize, } fn parse_sha256(s: &str) -> Option<[u8; 32]> { @@ -78,13 +80,62 @@ fn produce(path: &str, opts: DumpOptions, tx: mpsc::Sender anyhow::Result { - let (tx, rx) = mpsc::channel(8); +/// Sends the compressed NAR arriving on `rx` in `chunk_size` pieces, or in +/// one request if it fits in one chunk. +async fn send_chunked( + client: &Client, + cache: &str, + rx: &mut mpsc::Receiver>, + chunk_size: usize, +) -> anyhow::Result { + let mut buf = BytesMut::new(); + let mut session: Option = None; + let mut offset = 0u64; + let result = async { + loop { + let done = match rx.recv().await { + Some(chunk) => { + buf.extend_from_slice(&chunk?); + false + } + None => true, + }; + if done && session.is_none() && buf.len() <= chunk_size { + return client.upload(cache, crate::transport::full(buf.split().freeze())).await; + } + while buf.len() >= chunk_size || (done && !buf.is_empty()) { + let chunk = buf.split_to(buf.len().min(chunk_size)).freeze(); + let id = match &session { + Some(id) => id.clone(), + None => session.insert(client.upload_create(cache).await?).clone(), + }; + offset = client.upload_append(cache, &id, offset, chunk).await?; + } + if done { + let id = session.as_deref().context("chunked upload without a session")?; + return client.upload_complete(cache, id).await; + } + } + } + .await; + if let (Err(_), Some(id)) = (&result, &session) { + client.upload_abort(cache, id).await; + } + result +} + +async fn upload_one(client: &Client, cache: &str, info: &PathInfo, opts: DumpOptions, chunk_size: usize) -> anyhow::Result { + let (tx, mut rx) = mpsc::channel(8); let path = info.path.clone(); let producer = tokio::task::spawn_blocking(move || produce(&path, opts, tx)); - let chunks = futures_util::stream::unfold(rx, |mut rx| async move { rx.recv().await.map(|c| (c, rx)) }); - let body = crate::transport::stream(chunks); - let uploaded = client.upload(cache, body).await; + let uploaded = if chunk_size == 0 { + let chunks = futures_util::stream::unfold(rx, |mut rx| async move { rx.recv().await.map(|c| (c, rx)) }); + client.upload(cache, crate::transport::stream(chunks)).await + } else { + let uploaded = send_chunked(client, cache, &mut rx, chunk_size).await; + drop(rx); // Stops the producer if the upload gave up early. + uploaded + }; let digest = producer.await?; // An upload failure aborts the producer; report the upload's error. let uploaded = uploaded?; @@ -109,7 +160,7 @@ async fn upload_all(client: &Client, cache: &str, paths: &[&PathInfo], opts: &Op let mut results = futures_util::stream::iter(paths.iter().map(|info| async move { let dump = DumpOptions { level: opts.level, threads, size_hint: info.nar_size }; let started = Instant::now(); - (info, upload_one(client, cache, info, dump).await, started.elapsed()) + (info, upload_one(client, cache, info, dump, opts.chunk_size).await, started.elapsed()) })) .buffer_unordered(opts.jobs.max(1)); while let Some((info, result, took)) = results.next().await { diff --git a/crates/helios-server/src/chunked.rs b/crates/helios-server/src/chunked.rs new file mode 100644 index 0000000..a58c316 --- /dev/null +++ b/crates/helios-server/src/chunked.rs @@ -0,0 +1,225 @@ +//! Chunked uploads, for NARs larger than a proxy in front of the server +//! accepts in one request (Cloudflare's free plan stops at 100 MB, Cloud +//! Run at 32 MiB). +//! +//! 1. `POST /_api/v2/caches/{cache}/uploads?compression=zstd` → `{id, offset: 0}` +//! 2. `PATCH /_api/v2/caches/{cache}/uploads/{id}?offset=N` with the next +//! chunk as the body → `{offset}`. A chunk is appended whole or not at +//! all, so after a failed request the client asks for the offset and +//! resends from there; a stale offset gets 409 with the current one. +//! 3. `GET /_api/v2/caches/{cache}/uploads/{id}` → `{offset}` +//! 4. `POST /_api/v2/caches/{cache}/uploads/{id}/complete` → as `PUT /nar` +//! 5. `DELETE /_api/v2/caches/{cache}/uploads/{id}` abandons the upload. +//! +//! Chunks are verified as they arrive, as in a single-request upload. +//! Sessions live in memory: after a server restart the client starts the +//! NAR again. An idle session is dropped after an hour, and GC removes its +//! file. + +use std::io::Write; +use std::path::PathBuf; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::{Duration, Instant}; + +use axum::Json; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode, Uri}; +use bytes::Bytes; +use helios_core::{Compression, Verifier}; +use serde_json::json; + +use crate::audit::Audit; +use crate::auth::{Identity, Perm}; +use crate::error::{ApiError, ApiResult}; +use crate::push::{UploadResp, cache_for, store_blob}; +use crate::state::Shared; + +const MAX_SESSIONS: usize = 1024; +const IDLE_TIMEOUT: Duration = Duration::from_secs(3600); + +pub struct Session { + cache: Box, + actor: String, + compression: Compression, + tmp: PathBuf, + file: std::fs::File, + /// `None` once the stream is found invalid or the upload is finished. + verifier: Option, + offset: u64, + touched: Instant, +} + +pub type Sessions = Mutex, Arc>>>; + +fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { + m.lock().unwrap_or_else(PoisonError::into_inner) +} + +fn query<'a>(uri: &'a Uri, key: &str) -> Option<&'a str> { + uri.query()?.split('&').find_map(|kv| kv.strip_prefix(key)?.strip_prefix('=')) +} + +fn not_found() -> ApiError { + ApiError::new(StatusCode::NOT_FOUND, "upload not found; it expired or the server restarted, so start the NAR again") +} + +/// The session `id`, if it belongs to `cache` and to whoever is asking. +fn session(st: &Shared, cache: &str, id: &str, who: &Identity) -> ApiResult>> { + let s = lock(&st.uploads).get(id).cloned().ok_or_else(not_found)?; + { + let g = lock(&s); + if &*g.cache != cache || g.actor != who.actor() { + return Err(not_found()); + } + } + Ok(s) +} + +fn drop_session(st: &Shared, id: &str) { + if let Some(s) = lock(&st.uploads).remove(id) { + let _ = std::fs::remove_file(&lock(&s).tmp); + } +} + +pub async fn create( + State(st): State, + Path(name): Path, + uri: Uri, + headers: HeaderMap, +) -> ApiResult<(StatusCode, Json)> { + cache_for(&st, &name)?; + let who = st.authorize(&headers, &name, Perm::Push)?; + let compression = + Compression::parse(query(&uri, "compression").unwrap_or("zstd")).ok_or_else(|| ApiError::bad_request("compression must be zstd or none"))?; + + let id = helios_core::uuid_v4(); + let tmp = st.tmp_dir().join(format!("upload-{id}")); + let file = std::fs::File::create_new(&tmp).map_err(ApiError::internal)?; + let verifier = Verifier::new(compression).map_err(ApiError::internal)?; + let new = Session { + cache: name.into(), + actor: who.actor().to_owned(), + compression, + tmp, + file, + verifier: Some(verifier), + offset: 0, + touched: Instant::now(), + }; + + let mut sessions = lock(&st.uploads); + let expired: Vec> = + sessions.iter().filter(|(_, s)| s.try_lock().is_ok_and(|g| g.touched.elapsed() > IDLE_TIMEOUT)).map(|(id, _)| id.clone()).collect(); + for old in expired { + if let Some(s) = sessions.remove(&old) { + let _ = std::fs::remove_file(&lock(&s).tmp); + } + } + if sessions.len() >= MAX_SESSIONS { + let _ = std::fs::remove_file(&new.tmp); + return Err(ApiError::new(StatusCode::SERVICE_UNAVAILABLE, "too many uploads in progress")); + } + sessions.insert(id.as_str().into(), Arc::new(Mutex::new(new))); + Ok((StatusCode::CREATED, Json(json!({ "id": id, "offset": 0 })))) +} + +pub async fn append( + State(st): State, + Path((name, id)): Path<(String, String)>, + uri: Uri, + headers: HeaderMap, + body: Bytes, +) -> ApiResult> { + let at: u64 = query(&uri, "offset").and_then(|o| o.parse().ok()).ok_or_else(|| ApiError::bad_request("offset is required"))?; + let who = st.authorize(&headers, &name, Perm::Push)?; + let s = session(&st, &name, &id, &who)?; + let max = st.cfg.max_upload_bytes; + let result = tokio::task::spawn_blocking(move || -> ApiResult { + let mut g = lock(&s); + let g = &mut *g; + g.touched = Instant::now(); + if at != g.offset { + return Err(ApiError::new(StatusCode::CONFLICT, "offset mismatch").with(json!({ "offset": g.offset }))); + } + if g.offset + body.len() as u64 > max { + return Err(ApiError::new(StatusCode::PAYLOAD_TOO_LARGE, "upload too large")); + } + let verifier = g.verifier.as_mut().ok_or_else(|| ApiError::bad_request("upload is no longer accepting data"))?; + if let Err(e) = verifier.update(&body) { + g.verifier = None; + return Err(ApiError::bad_request(format!("invalid NAR stream: {e}"))); + } + // A short write leaves the file ahead of the offset; later chunks + // would then land in the wrong place, so give up on the upload. + if let Err(e) = g.file.write_all(&body) { + g.verifier = None; + return Err(ApiError::internal(e)); + } + g.offset += body.len() as u64; + Ok(g.offset) + }) + .await?; + match result { + Ok(offset) => Ok(Json(json!({ "offset": offset }))), + Err(e) => { + if e.status != StatusCode::CONFLICT { + drop_session(&st, &id); + } + Err(e) + } + } +} + +pub async fn status(State(st): State, Path((name, id)): Path<(String, String)>, headers: HeaderMap) -> ApiResult> { + let who = st.authorize(&headers, &name, Perm::Push)?; + let s = session(&st, &name, &id, &who)?; + let offset = lock(&s).offset; + Ok(Json(json!({ "offset": offset }))) +} + +pub async fn complete( + State(st): State, + Path((name, id)): Path<(String, String)>, + audit: Audit, + headers: HeaderMap, +) -> ApiResult<(StatusCode, Json)> { + let who = st.authorize(&headers, &name, Perm::Push)?; + let s = session(&st, &name, &id, &who)?; + // Taken out of the map first, so a concurrent chunk cannot extend it. + lock(&st.uploads).remove(id.as_str()); + let (tmp, compression, digest) = tokio::task::spawn_blocking(move || { + let mut g = lock(&s); + let digest = match g.verifier.take() { + Some(v) => v.finish().map_err(|e| ApiError::bad_request(format!("invalid NAR stream: {e}"))), + None => Err(ApiError::bad_request("upload is no longer accepting data")), + } + .and_then(|d| g.file.sync_data().map(|()| d).map_err(ApiError::internal)); + (g.tmp.clone(), g.compression, digest) + }) + .await?; + let digest = match digest { + Ok(d) => d, + Err(err) => { + let _ = tokio::fs::remove_file(&tmp).await; + audit.log(&st, &who, "nar.upload", Some(&name), err.status, json!({ "error": err.message, "chunked": true })); + return Err(err); + } + }; + let resp = store_blob(&st, tmp, digest, compression).await?; + audit.log( + &st, + &who, + "nar.upload", + Some(&name), + StatusCode::CREATED, + json!({ "fileHash": resp.file_hash, "size": resp.file_size, "chunked": true }), + ); + Ok((StatusCode::CREATED, Json(resp))) +} + +pub async fn abort(State(st): State, Path((name, id)): Path<(String, String)>, headers: HeaderMap) -> ApiResult { + let who = st.authorize(&headers, &name, Perm::Push)?; + session(&st, &name, &id, &who)?; + drop_session(&st, &id); + Ok(StatusCode::NO_CONTENT) +} diff --git a/crates/helios-server/src/main.rs b/crates/helios-server/src/main.rs index 33cd8ea..284ec44 100644 --- a/crates/helios-server/src/main.rs +++ b/crates/helios-server/src/main.rs @@ -1,6 +1,7 @@ mod admin; mod audit; mod auth; +mod chunked; mod config; mod db; mod error; @@ -37,6 +38,9 @@ pub fn router(st: Shared) -> Router { .route("/caches/{cache}/missing", post(push::missing)) .route("/caches/{cache}/nars/known", post(push::known)) .route("/caches/{cache}/nar", put(push::upload).layer(DefaultBodyLimit::disable())) + .route("/caches/{cache}/uploads", post(chunked::create)) + .route("/caches/{cache}/uploads/{id}", get(chunked::status).patch(chunked::append).delete(chunked::abort)) + .route("/caches/{cache}/uploads/{id}/complete", post(chunked::complete)) .route("/caches/{cache}/paths", post(push::publish)) .route("/admin/caches", post(admin::create_cache).get(admin::list_caches)) .route("/admin/tokens", post(admin::create_token).get(admin::list_tokens)) diff --git a/crates/helios-server/src/push.rs b/crates/helios-server/src/push.rs index 204e5a8..530291a 100644 --- a/crates/helios-server/src/push.rs +++ b/crates/helios-server/src/push.rs @@ -33,7 +33,7 @@ const MAX_MISSING_BATCH: usize = 100_000; const MAX_PUBLISH_BATCH: usize = 5_000; const STORE_DIR: &str = "/nix/store/"; -fn cache_for(st: &Shared, name: &str) -> ApiResult { +pub(crate) fn cache_for(st: &Shared, name: &str) -> ApiResult { st.cache(name).ok_or_else(|| ApiError::new(StatusCode::NOT_FOUND, "cache not found")) } @@ -106,8 +106,8 @@ pub async fn known( #[derive(Serialize)] #[serde(rename_all = "camelCase")] pub struct UploadResp { - file_hash: String, - file_size: u64, + pub(crate) file_hash: String, + pub(crate) file_size: u64, nar_hash: String, nar_size: u64, } @@ -180,6 +180,13 @@ pub async fn upload( (None, Ok(d)) => d, }; + let resp = store_blob(&st, tmp, digest, compression).await?; + audit.log(&st, &identity, "nar.upload", Some(&name), StatusCode::CREATED, json!({ "fileHash": resp.file_hash, "size": resp.file_size })); + Ok((StatusCode::CREATED, Json(resp))) +} + +/// Moves a verified upload at `tmp` into the blob store and records it. +pub(crate) async fn store_blob(st: &Shared, tmp: std::path::PathBuf, digest: helios_core::Digest, compression: Compression) -> ApiResult { let dest = st.nar_path(&digest.file_hash, compression); let st2 = st.clone(); tokio::task::spawn_blocking(move || -> ApiResult<()> { @@ -218,8 +225,7 @@ pub async fn upload( }; st.counters.uploads.inc(); st.counters.upload_bytes.add(resp.file_size); - audit.log(&st, &identity, "nar.upload", Some(&name), StatusCode::CREATED, json!({ "fileHash": resp.file_hash, "size": resp.file_size })); - Ok((StatusCode::CREATED, Json(resp))) + Ok(resp) } // ── Publish ── diff --git a/crates/helios-server/src/state.rs b/crates/helios-server/src/state.rs index 6c01874..4d8ef20 100644 --- a/crates/helios-server/src/state.rs +++ b/crates/helios-server/src/state.rs @@ -75,6 +75,7 @@ pub struct AppState { pub audit: mpsc::UnboundedSender, pub counters: crate::stats::Counters, pub access: crate::stats::Access, + pub uploads: crate::chunked::Sessions, } pub type Shared = Arc; @@ -130,6 +131,7 @@ impl AppState { audit, counters: Default::default(), access: Default::default(), + uploads: Default::default(), }) } diff --git a/docs.md b/docs.md index 959b26b..77b1521 100644 --- a/docs.md +++ b/docs.md @@ -233,6 +233,13 @@ streams it straight into the upload, without temp files. It checks every NAR hash against the Nix database. NARs the server already holds under any cache are reused, and paths are published dependencies-first, in batches. +A NAR larger than `--chunk-size` (MiB, default 32) is uploaded in chunks of +that size, so pushes work through proxies that cap request bodies, such as +Cloudflare (100 MB on the free plan) and Cloud Run (32 MiB). Each chunk is +retried on its own after a network or server error, and the server's offset +decides whether it has to be sent again. `--chunk-size 0` streams every NAR +in one request. + ## HTTP API Substituter endpoints (`GET`/`HEAD`): @@ -250,8 +257,18 @@ Push endpoints (bearer token with `push` on the cache): | `POST /_api/v2/caches//missing` | `{"hashes": [...]}` | store path hashes absent from the cache | | `POST /_api/v2/caches//nars/known` | `{"narHashes": [...]}` | NAR hashes the server already has | | `PUT /_api/v2/caches//nar?compression=zstd` | compressed NAR | returns the verified hashes and sizes | +| `POST /_api/v2/caches//uploads?compression=zstd` | | starts a chunked upload: `{"id", "offset": 0}` | +| `PATCH /_api/v2/caches//uploads/?offset=` | the next chunk, up to 64 MiB | appends it whole or not at all: `{"offset"}`; a stale offset gets `409` with the current one | +| `GET /_api/v2/caches//uploads/` | | `{"offset"}`, to resume after a failed request | +| `POST /_api/v2/caches//uploads//complete` | | as `PUT /nar` | +| `DELETE /_api/v2/caches//uploads/` | | abandons it | | `POST /_api/v2/caches//paths` | `{"paths": [...]}` | publishes a batch in one transaction; `409 nar_required` lists paths without an uploaded NAR | +Chunks are verified as they arrive, as in a single `PUT`. An upload belongs +to the token that started it, is dropped after an hour without a chunk or +after an invalid one, and does not survive a server restart (the client +then starts that NAR again). + Admin endpoints (bearer admin secret): `GET/POST /_api/v2/admin/caches`, `GET/POST /_api/v2/admin/tokens`, `POST /_api/v2/admin/tokens//revoke`. diff --git a/scripts/e2e.sh b/scripts/e2e.sh index a5df380..0c469f2 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -38,7 +38,8 @@ head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' >"$WORK/jwt" head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' >"$WORK/admin" PUBKEY="$("$BIN/helios-server" --print-public-key --signing-key-file "$WORK/key")" -RUST_LOG=helios_server=warn "$BIN/helios-server" \ +# Debug logs each request, which the chunked-upload checks count. +HELIOS_LOG=warn,helios_server=debug "$BIN/helios-server" 2>"$WORK/server.log" \ --listen "127.0.0.1:$PORT" \ --data-dir "$WORK/data" \ --signing-key-file "$WORK/key" \ @@ -56,13 +57,46 @@ echo "=== admin" helios login admin "$URL" "$(cat "$WORK/admin")" >/dev/null helios cache create main >/dev/null helios cache create secret --private >/dev/null -PUSH_TOKEN="$(helios token create ci --caches main,secret --perms push,pull 2>/dev/null | jq -r .token)" +helios cache create chunked >/dev/null +PUSH_TOKEN="$(helios token create ci --caches main,secret,chunked --perms push,pull 2>/dev/null | jq -r .token)" PULL_ONLY="$(helios token create reader --caches secret 2>/dev/null | jq -r .token)" PUSH_ONLY="$(helios token create builder --caches secret --perms push 2>/dev/null | jq -r .token)" check "tokens are read-only by default" '["pull"]' "$(helios token list | jq -c '.tokens[] | select(.subject=="reader") | .perms')" check "token issued" true "$([ -n "$PUSH_TOKEN" ] && echo true)" check "admin rejects push token" 403 "$(status -H "authorization: Bearer $PUSH_TOKEN" "$URL/_api/v2/admin/tokens")" +echo "=== chunked upload" + +helios login ci "$URL" "$PUSH_TOKEN" >/dev/null +# 1 MiB chunks: the closure's larger NARs go in pieces, the rest in one request. +helios push chunked --closure --chunk-size 1 "$TARGET" 2>&1 | tail -1 +check "large NARs were sent in chunks" true "$([ "$(grep -c 'method=PATCH' "$WORK/server.log")" -gt 1 ] && echo true)" +check "chunks fit the chunk size" 0 "$(grep -c 'status=413' "$WORK/server.log")" +NIX_CONFIG="trusted-public-keys = $PUBKEY +require-sigs = true" nix copy --from "$URL/chunked" --to "$WORK/chunked-store" "$TARGET" 2>&1 | tail -3 +check "chunked closure substitutes" "$(nix-store -qR "$TARGET" | wc -l)" "$(nix-store --store "$WORK/chunked-store" -qR "$TARGET" | wc -l)" + +AUTH=(-H "authorization: Bearer $PUSH_TOKEN") +new_upload() { curl -s -X POST "${AUTH[@]}" "$URL/_api/v2/caches/chunked/uploads" | jq -r .id; } +NAR="$WORK/one.nar.zst" +nix-store --dump "$TARGET" | zstd -q >"$NAR" +NAR_BYTES="$(stat -c %s "$NAR")" +HALF=$((NAR_BYTES / 2)) +ID="$(new_upload)" +head -c "$HALF" "$NAR" >"$WORK/part1" +tail -c +$((HALF + 1)) "$NAR" >"$WORK/part2" +check "chunk at the wrong offset" 409 "$(status -X PATCH "${AUTH[@]}" --data-binary @"$WORK/part2" "$URL/_api/v2/caches/chunked/uploads/$ID?offset=$HALF")" +curl -s -o /dev/null -X PATCH "${AUTH[@]}" --data-binary @"$WORK/part1" "$URL/_api/v2/caches/chunked/uploads/$ID?offset=0" +check "resume reads the offset" "$HALF" "$(curl -s "${AUTH[@]}" "$URL/_api/v2/caches/chunked/uploads/$ID" | jq .offset)" +check "another token cannot see the upload" 404 "$(status -H "authorization: Bearer $PUSH_ONLY" "$URL/_api/v2/caches/secret/uploads/$ID")" +curl -s -o /dev/null -X PATCH "${AUTH[@]}" --data-binary @"$WORK/part2" "$URL/_api/v2/caches/chunked/uploads/$ID?offset=$HALF" +EXPECT="sha256:$(nix-store -q --hash "$TARGET" | cut -d: -f2)" +check "completed upload has the NAR hash" "$EXPECT" "$(curl -s -X POST "${AUTH[@]}" "$URL/_api/v2/caches/chunked/uploads/$ID/complete" | jq -r .narHash)" +check "completed upload is gone" 404 "$(status "${AUTH[@]}" "$URL/_api/v2/caches/chunked/uploads/$ID")" +ID="$(new_upload)" +check "garbage chunk rejected" 400 "$(status -X PATCH "${AUTH[@]}" --data-binary 'not zstd' "$URL/_api/v2/caches/chunked/uploads/$ID?offset=0")" +check "rejected upload is dropped" 404 "$(status "${AUTH[@]}" "$URL/_api/v2/caches/chunked/uploads/$ID")" + echo "=== push $TARGET" helios login ci "$URL" "$PUSH_TOKEN" >/dev/null helios push main --closure "$TARGET" From 1f3859f71c673e2817b069c0b3c20d89c1d23173 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 32/56] cli: add helios use to configure nix for a cache Setting up a client meant writing nix.conf and, for a private cache, a netrc entry by hand, and getting either wrong only shows up as "does not appear to be a binary cache" (zhaofengli/attic#140). helios use adds the substituter, the signing key and, for a private cache, a netrc entry with the login's token to the user's Nix config; rerunning it changes nothing, and a new token replaces the old entry. The URL is the one given to helios login rather than one the server reports, which behind a proxy is wrong (attic#325). --print gives system-wide settings, and it warns when the Nix daemon would ignore a user's substituters. The server gains GET /_api/v2/caches/ for the signing key. --- README.md | 8 +- crates/helios-cli/src/api.rs | 4 + crates/helios-cli/src/main.rs | 14 ++ crates/helios-cli/src/substituter.rs | 192 +++++++++++++++++++++++++++ crates/helios-server/src/main.rs | 1 + crates/helios-server/src/read.rs | 13 ++ docs.md | 26 ++-- scripts/e2e.sh | 15 +++ 8 files changed, 260 insertions(+), 13 deletions(-) create mode 100644 crates/helios-cli/src/substituter.rs diff --git a/README.md b/README.md index 0a00689..236335a 100644 --- a/README.md +++ b/README.md @@ -57,11 +57,9 @@ helios login ci https://cache.example.com "$TOKEN" helios push main --closure /run/current-system ``` -```nix -nix.settings = { - substituters = [ "https://cache.example.com/main" ]; - trusted-public-keys = [ "cache.example.com-1:..." ]; # from --print-public-key -}; +```bash +helios use main # this user's Nix pulls from the cache +helios use main --print # settings for NixOS or /etc/nix/nix.conf ``` On NixOS, use the flake's `nixosModules.default` (`services.helios`): the diff --git a/crates/helios-cli/src/api.rs b/crates/helios-cli/src/api.rs index e3e866f..7f2815d 100644 --- a/crates/helios-cli/src/api.rs +++ b/crates/helios-cli/src/api.rs @@ -84,6 +84,10 @@ impl Client { self.call(Method::PUT, &format!("/caches/{cache}/nar?compression=zstd"), Some("application/x-nix-nar"), body).await } + pub async fn cache_info(&self, cache: &str) -> anyhow::Result { + self.call(Method::GET, &format!("/caches/{cache}"), None, transport::full(Bytes::new())).await + } + /// Starts a chunked upload; returns its id. pub async fn upload_create(&self, cache: &str) -> anyhow::Result { #[derive(Deserialize)] diff --git a/crates/helios-cli/src/main.rs b/crates/helios-cli/src/main.rs index fe571a2..92a1f58 100644 --- a/crates/helios-cli/src/main.rs +++ b/crates/helios-cli/src/main.rs @@ -2,6 +2,7 @@ mod api; mod config; mod nix; mod push; +mod substituter; mod transport; mod watch; @@ -71,6 +72,15 @@ enum Command { #[arg(long, default_value_t = 32, value_parser = clap::value_parser!(u64).range(0..=64))] chunk_size: u64, }, + /// Configure Nix to substitute from a cache: its URL, signing key and, + /// for a private cache, this login's token in netrc. + Use { + cache: String, + /// Print system-wide settings (nix.conf, NixOS, netrc) instead of + /// writing the user's nix.conf. + #[arg(long)] + print: bool, + }, /// Spool store paths for `watch-store` (Nix post-build-hook; reads $OUT_PATHS). QueuePaths { #[arg(long, default_value = "/var/lib/helios-watch-store/spool")] @@ -157,6 +167,10 @@ async fn run(cli: Cli) -> anyhow::Result<()> { Command::Push { cache, installables, closure, jobs, level, chunk_size } => { push::push(&client, &cache, &installables, push::Options { jobs, level, closure, chunk_size: (chunk_size << 20) as usize }).await?; } + Command::Use { cache, print } => { + let plan = substituter::plan(&client, &server, &cache).await?; + if print { plan.print() } else { plan.apply()? } + } Command::WatchStore { cache, spool, jobs, level, chunk_size } => { watch::watch(&client, &cache, &spool, push::Options { jobs, level, closure: true, chunk_size: (chunk_size << 20) as usize }).await?; } diff --git a/crates/helios-cli/src/substituter.rs b/crates/helios-cli/src/substituter.rs new file mode 100644 index 0000000..08fec38 --- /dev/null +++ b/crates/helios-cli/src/substituter.rs @@ -0,0 +1,192 @@ +//! `helios use `: configure Nix to substitute from a cache. +//! +//! The substituter URL is the one given to `helios login`, never an address +//! the server reports about itself: behind a proxy the server does not know +//! the address clients reach it at (compare zhaofengli/attic#325). + +use std::io::Write; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, bail}; +use serde::Deserialize; + +use crate::api::Client; +use crate::config::Server; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CacheInfo { + pub public: bool, + pub public_key: Option, +} + +pub struct Plan { + url: String, + public_key: Option, + /// `(host, token)` for private caches, which Nix reads from netrc. + netrc: Option<(String, String)>, +} + +pub async fn plan(client: &Client, server: &Server, cache: &str) -> anyhow::Result { + let info = client.cache_info(cache).await?; + let netrc = (!info.public).then(|| Ok::<_, anyhow::Error>((host(&server.server)?.to_owned(), server.token.clone()))).transpose()?; + Ok(Plan { url: format!("{}/{cache}", server.server), public_key: info.public_key, netrc }) +} + +/// The netrc `machine` for a URL: its host, without scheme, port or path. +fn host(url: &str) -> anyhow::Result<&str> { + let rest = url.split_once("://").map_or(url, |(_, r)| r); + let authority = rest.split('/').next().unwrap_or(rest); + let host = authority.rsplit_once('@').map_or(authority, |(_, h)| h); + let host = match host.strip_prefix('[') { + Some(v6) => v6.split(']').next().unwrap_or(v6), + None => host.split(':').next().unwrap_or(host), + }; + if host.is_empty() { + bail!("no host in server URL {url}"); + } + Ok(host) +} + +fn nix_config_dir() -> anyhow::Result { + let base = match std::env::var_os("XDG_CONFIG_HOME") { + Some(dir) if !dir.is_empty() => PathBuf::from(dir), + _ => PathBuf::from(std::env::var_os("HOME").context("HOME is not set")?).join(".config"), + }; + Ok(base.join("nix")) +} + +impl Plan { + fn conf_lines(&self, netrc_file: Option<&Path>) -> Vec { + let mut lines = vec![format!("extra-substituters = {}", self.url)]; + if let Some(key) = &self.public_key { + lines.push(format!("extra-trusted-public-keys = {key}")); + } + if let (Some(_), Some(file)) = (&self.netrc, netrc_file) { + lines.push(format!("netrc-file = {}", file.display())); + } + lines + } + + /// Settings for a NixOS configuration or `/etc/nix/nix.conf`, which + /// apply to every user. + pub fn print(&self) { + println!("# nix.conf"); + for line in self.conf_lines(self.netrc.as_ref().map(|_| Path::new("/etc/nix/netrc"))) { + println!("{line}"); + } + println!("\n# NixOS"); + println!("nix.settings = {{"); + println!(" extra-substituters = [ \"{}\" ];", self.url); + if let Some(key) = &self.public_key { + println!(" extra-trusted-public-keys = [ \"{key}\" ];"); + } + if self.netrc.is_some() { + println!(" netrc-file = \"/etc/nix/netrc\"; # holding the netrc entry below"); + } + println!("}};"); + if let Some((host, token)) = &self.netrc { + println!("\n# netrc (keep it readable by root only)\nmachine {host}\nlogin helios\npassword {token}"); + } + } + + /// Adds the settings to the user's nix.conf (and netrc), leaving lines + /// already there alone. + pub fn apply(&self) -> anyhow::Result<()> { + let dir = nix_config_dir()?; + std::fs::create_dir_all(&dir)?; + let conf = dir.join("nix.conf"); + let existing = + std::fs::read_to_string(&conf).or_else(|e| if e.kind() == std::io::ErrorKind::NotFound { Ok(String::new()) } else { Err(e) })?; + + let mut netrc_file = None; + if let Some((host, token)) = &self.netrc { + // An existing netrc-file setting wins; Nix reads only one file. + let file = existing + .lines() + .find_map(|l| l.trim().strip_prefix("netrc-file")?.trim_start().strip_prefix('=').map(|v| PathBuf::from(v.trim()))) + .unwrap_or_else(|| dir.join("netrc")); + add_netrc(&file, host, token)?; + netrc_file = Some(file); + } + + let missing: Vec = self + .conf_lines(netrc_file.as_deref()) + .into_iter() + .filter(|line| { + let key = line.split('=').next().unwrap_or("").trim(); + // netrc-file is single-valued: keep the one that is there. + !existing.lines().any(|l| l.trim() == line || (key == "netrc-file" && l.trim().starts_with("netrc-file"))) + }) + .collect(); + if !missing.is_empty() { + let mut out = std::fs::OpenOptions::new().append(true).create(true).open(&conf)?; + if !existing.is_empty() && !existing.ends_with('\n') { + writeln!(out)?; + } + for line in &missing { + writeln!(out, "{line}")?; + } + } + tracing::info!("configured {} in {}", self.url, conf.display()); + if self.public_key.is_none() { + tracing::warn!("the server has no signing key; Nix refuses its paths unless require-sigs is off"); + } + if !nix_trusts_user() { + tracing::warn!( + "the Nix daemon does not trust this user, so it ignores substituters set here; \ + add them system-wide instead (helios use --print shows how) or list this user in trusted-users" + ); + } + Ok(()) + } +} + +/// Appends `machine host` to a netrc, replacing an earlier entry for it. +fn add_netrc(file: &Path, host: &str, token: &str) -> anyhow::Result<()> { + let existing = std::fs::read_to_string(file).unwrap_or_default(); + // netrc entries are whitespace-separated tokens; drop the one for this + // host so a new login replaces its token. + let mut kept = String::new(); + let mut tokens = existing.split_whitespace().peekable(); + while let Some(t) = tokens.next() { + if t == "machine" && tokens.peek() == Some(&host) { + tokens.next(); + while let Some(&next) = tokens.peek() { + if next == "machine" || next == "default" { + break; + } + tokens.next(); + } + continue; + } + kept.push_str(if t == "machine" || t == "default" { "\n" } else { " " }); + kept.push_str(t); + } + let body = format!("{}\nmachine {host} login helios password {token}\n", kept.trim()); + let mut out = std::fs::OpenOptions::new().write(true).create(true).truncate(true).mode(0o600).open(file)?; + out.write_all(body.trim_start().as_bytes())?; + Ok(()) +} + +fn nix_trusts_user() -> bool { + let Ok(out) = std::process::Command::new("nix").args(["--extra-experimental-features", "nix-command", "store", "info", "--json"]).output() else { + return true; // No way to tell; do not warn. + }; + serde_json::from_slice::(&out.stdout).map_or(true, |v| v["trusted"] != false && v["trusted"] != 0) +} + +#[cfg(test)] +mod tests { + use super::host; + + #[test] + fn hosts() { + assert_eq!(host("https://cache.example.com").unwrap(), "cache.example.com"); + assert_eq!(host("http://127.0.0.1:8080/").unwrap(), "127.0.0.1"); + assert_eq!(host("http://[::1]:8080").unwrap(), "::1"); + assert_eq!(host("https://u@cache.example.com:443/x").unwrap(), "cache.example.com"); + assert!(host("http://").is_err()); + } +} diff --git a/crates/helios-server/src/main.rs b/crates/helios-server/src/main.rs index 284ec44..d632579 100644 --- a/crates/helios-server/src/main.rs +++ b/crates/helios-server/src/main.rs @@ -35,6 +35,7 @@ fn load_signer(args: &Args) -> anyhow::Result> { pub fn router(st: Shared) -> Router { let api = Router::new() + .route("/caches/{cache}", get(read::cache_info)) .route("/caches/{cache}/missing", post(push::missing)) .route("/caches/{cache}/nars/known", post(push::known)) .route("/caches/{cache}/nar", put(push::upload).layer(DefaultBodyLimit::disable())) diff --git a/crates/helios-server/src/read.rs b/crates/helios-server/src/read.rs index 0240c70..2d468c8 100644 --- a/crates/helios-server/src/read.rs +++ b/crates/helios-server/src/read.rs @@ -171,3 +171,16 @@ async fn nar_response( }); builder.body(Body::from_stream(stream)).map_err(ApiError::internal) } + +/// `GET /_api/v2/caches/{cache}`: what a client needs to substitute from the +/// cache. Readable by whoever can read the cache itself. +pub async fn cache_info( + State(st): State, + axum::extract::Path(name): axum::extract::Path, + headers: HeaderMap, +) -> ApiResult> { + let cache = st.cache(&name).ok_or_else(ApiError::not_found)?; + st.authorize_read(&headers, &name, cache)?; + let public_key = st.signer.as_ref().map(helios_core::Signer::public_key); + Ok(axum::Json(serde_json::json!({ "name": name, "public": cache.public, "publicKey": public_key }))) +} diff --git a/docs.md b/docs.md index 77b1521..55f4eaf 100644 --- a/docs.md +++ b/docs.md @@ -208,18 +208,25 @@ private cache without being able to read it. A refused request says why, such as `token does not grant push on cache 'main'` or `token has been revoked`. -For private caches, give Nix the token through netrc. Nix sends it as HTTP -Basic auth, and the server reads the password: +## Substituting -``` -machine cache.example.com -password -``` +`helios use` configures Nix to pull from a cache, using the URL given to +`helios login` (not an address the server reports, which behind a proxy is +the wrong one): -```nix -nix.settings.netrc-file = "/etc/nix/netrc"; +```bash +helios login reader https://cache.example.com "$READ_TOKEN" +helios use team # adds to ~/.config/nix/nix.conf and netrc +helios use team --print # system-wide settings, for NixOS or /etc/nix ``` +It adds the substituter and the cache's signing key and, for a private +cache, a netrc entry with the login's token (Nix sends it as HTTP Basic +auth; the server reads the password). Running it again changes nothing, and +logging in with a new token replaces the old entry. The Nix daemon ignores +substituters from users it does not trust; `helios use` warns when that is +the case, and `--print` gives the `nix.settings` to use instead. + ## Pushing ```bash @@ -250,6 +257,9 @@ Substituter endpoints (`GET`/`HEAD`): | `//.narinfo` | misses are answered from memory | | `//nar/.nar.zst` | only served by caches that publish it | +`GET /_api/v2/caches/` returns `{"name", "public", "publicKey"}` to +anyone who can read the cache. + Push endpoints (bearer token with `push` on the cache): | Method and path | Body | | diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 0c469f2..bc0e7cc 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -129,6 +129,21 @@ check "pull-only token cannot push" 403 "$(status -X POST -H "authorization: Bea check "push-only token cannot read" 403 "$(status -u "builder:$PUSH_ONLY" "$URL/secret/$HASH.narinfo")" check "refusals say why" "token does not grant push on cache 'secret'" "$(curl -s -X POST -H "authorization: Bearer $PULL_ONLY" -H 'content-type: application/json' -d '{"hashes":[]}' "$URL/_api/v2/caches/secret/missing" | jq -r .error)" +echo "=== helios use" +helios login reader "$URL" "$PULL_ONLY" >/dev/null 2>&1 +helios use secret 2>/dev/null +helios use secret 2>/dev/null +NIXCONF="$XDG_CONFIG_HOME/nix/nix.conf" +check "nix.conf gets the login URL, once" 1 "$(grep -cx "extra-substituters = $URL/secret" "$NIXCONF")" +check "nix.conf gets the signing key" 1 "$(grep -cx "extra-trusted-public-keys = $PUBKEY" "$NIXCONF")" +check "netrc is private" 600 "$(stat -c %a "$XDG_CONFIG_HOME/nix/netrc")" +check "netrc holds the token once" 1 "$(grep -c "password $PULL_ONLY" "$XDG_CONFIG_HOME/nix/netrc")" +# Signatures checked against the configured key, fetched with netrc auth. +nix store verify --no-contents --store "$URL/secret" "$TARGET" && check "nix reads the private cache with this config" 0 0 +check "and not without it" fails "$(XDG_CONFIG_HOME="$WORK/empty" nix store verify --no-contents --store "$URL/secret" "$TARGET" >/dev/null 2>&1 || echo fails)" +check "--print shows system-wide settings" 1 "$(helios use secret --print | grep -c "^machine 127.0.0.1 *\$")" +helios login ci "$URL" "$PUSH_TOKEN" >/dev/null 2>&1 + echo "=== abuse" check "garbage upload rejected" 400 "$(status -X PUT -H "authorization: Bearer $PUSH_TOKEN" --data-binary 'not a zstd stream' "$URL/_api/v2/caches/main/nar")" check "raw non-NAR upload rejected" 400 "$(status -X PUT -H "authorization: Bearer $PUSH_TOKEN" --data-binary 'hello' "$URL/_api/v2/caches/main/nar?compression=none")" From 493ef6fd540198dcd6a1efe5fbe8f68b2c3af18d Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 33/56] nix: declare caches in the module Caches could only be created by hand with helios cache create, so a NixOS host was not fully described by its configuration (zhaofengli/attic#169). helios-server --caches (HELIOS_CACHES) takes names such as main,team:private, creates missing ones at startup and gives existing ones the listed visibility. Unlisted caches are left alone: removing a cache from the list must not delete its paths. The module exposes it as services.helios.caches..public. --- crates/helios-server/src/admin.rs | 2 +- crates/helios-server/src/config.rs | 41 +++++++++++++++++++++++ crates/helios-server/src/main.rs | 5 +++ crates/helios-server/src/state.rs | 53 ++++++++++++++++++++++++++++++ docs.md | 17 +++++++++- nix/module.nix | 20 +++++++++++ nix/test.nix | 7 +++- 7 files changed, 142 insertions(+), 3 deletions(-) diff --git a/crates/helios-server/src/admin.rs b/crates/helios-server/src/admin.rs index c180fb4..ddb16e8 100644 --- a/crates/helios-server/src/admin.rs +++ b/crates/helios-server/src/admin.rs @@ -18,7 +18,7 @@ const DEFAULT_TOKEN_LIFETIME_DAYS: i64 = 90; const MAX_SUBJECT_LEN: usize = 256; const MAX_REASON_LEN: usize = 512; -fn valid_cache_name(name: &str) -> bool { +pub(crate) fn valid_cache_name(name: &str) -> bool { let b = name.as_bytes(); (1..=64).contains(&b.len()) && b.iter().all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || *c == b'-') diff --git a/crates/helios-server/src/config.rs b/crates/helios-server/src/config.rs index 63da1b2..aad1f60 100644 --- a/crates/helios-server/src/config.rs +++ b/crates/helios-server/src/config.rs @@ -67,11 +67,36 @@ pub struct Args { #[arg(long, env = "HELIOS_ADMIN_SOCKET")] pub admin_socket: Option, + /// Caches to create at startup, comma-separated: `name` for a public + /// cache, `name:private` for a private one. A listed cache that exists + /// takes the listed visibility; unlisted caches are left alone. + #[arg(long, env = "HELIOS_CACHES", value_delimiter = ',', value_parser = parse_declared_cache)] + pub caches: Vec, + /// Print the public key for `trusted-public-keys` and exit. #[arg(long)] pub print_public_key: bool, } +#[derive(Debug, Clone)] +pub struct DeclaredCache { + pub name: String, + pub public: bool, +} + +fn parse_declared_cache(s: &str) -> Result { + let (name, public) = match s.trim().split_once(':') { + None => (s.trim(), true), + Some((name, "public")) => (name, true), + Some((name, "private")) => (name, false), + Some((_, other)) => return Err(format!("unknown visibility {other:?}; expected public or private")), + }; + if !crate::admin::valid_cache_name(name) { + return Err(format!("invalid cache name {name:?}: 1-64 of a-z, 0-9 and -, not starting or ending with -")); + } + Ok(DeclaredCache { name: name.to_owned(), public }) +} + #[derive(Subcommand, Debug, Clone)] pub enum Command { /// Create whichever of signing-key, jwt-secret and admin-secret are @@ -123,3 +148,19 @@ impl Config { }) } } + +#[cfg(test)] +mod tests { + use super::parse_declared_cache; + + #[test] + fn declared_caches() { + let c = parse_declared_cache("main").unwrap(); + assert_eq!((c.name.as_str(), c.public), ("main", true)); + assert!(!parse_declared_cache("team:private").unwrap().public); + assert!(parse_declared_cache("team:public").unwrap().public); + assert!(parse_declared_cache("team:secret").is_err()); + assert!(parse_declared_cache("Bad_Name").is_err()); + assert!(parse_declared_cache("").is_err()); + } +} diff --git a/crates/helios-server/src/main.rs b/crates/helios-server/src/main.rs index d632579..778e6fd 100644 --- a/crates/helios-server/src/main.rs +++ b/crates/helios-server/src/main.rs @@ -94,6 +94,11 @@ pub async fn build_state(args: &Args) -> anyhow::Result { let (audit_tx, audit_rx) = tokio::sync::mpsc::unbounded_channel(); tokio::spawn(audit::run(db.clone(), audit_rx)); let st = Arc::new(AppState::load(cfg, db, signer, audit_tx)?); + for c in &args.caches { + if let Some(change) = st.declare_cache(&c.name, c.public)? { + tracing::info!(cache = c.name, public = c.public, "declared cache {change}"); + } + } tokio::spawn(gc::run_forever(st.clone())); tokio::spawn(gc::run_access_flush(st.clone())); Ok(st) diff --git a/crates/helios-server/src/state.rs b/crates/helios-server/src/state.rs index 4d8ef20..1a37f03 100644 --- a/crates/helios-server/src/state.rs +++ b/crates/helios-server/src/state.rs @@ -148,6 +148,22 @@ impl AppState { self.cfg.data_dir.join("tmp") } + /// Creates `name`, or gives an existing cache the visibility `public`. + /// Returns what changed, if anything. + pub fn declare_cache(&self, name: &str, public: bool) -> rusqlite::Result> { + if self.insert_cache(name, public)?.is_some() { + return Ok(Some("created")); + } + if self.cache(name).is_some_and(|c| c.public == public) { + return Ok(None); + } + self.db.write(|conn| conn.execute("UPDATE caches SET is_public = ?2 WHERE name = ?1", params![name, public]))?; + if let Some(c) = self.caches.wr().get_mut(name) { + c.public = public; + } + Ok(Some(if public { "made public" } else { "made private" })) + } + pub fn insert_cache(&self, name: &str, public: bool) -> rusqlite::Result> { let id = self.db.write(|conn| -> rusqlite::Result> { conn.execute( @@ -182,3 +198,40 @@ impl Locked for RwLock { self.write().unwrap_or_else(PoisonError::into_inner) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn declared_caches_are_created_and_take_the_declared_visibility() { + let dir = std::env::temp_dir().join(format!("helios-state-test-{}", helios_core::uuid_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let cfg = Config { + data_dir: dir.clone(), + jwt_secret: None, + admin_secret: None, + accel_redirect: None, + trust_proxy: false, + narinfo_cache_entries: 16, + max_upload_bytes: 1 << 20, + upload_grace: std::time::Duration::from_secs(1), + gc_interval: std::time::Duration::from_secs(3600), + audit_retention: std::time::Duration::from_secs(3600), + }; + let db = Db::open(&dir.join("helios.db")).unwrap(); + let (tx, _rx) = mpsc::unbounded_channel(); + let st = AppState::load(cfg.clone(), db, None, tx).unwrap(); + + assert_eq!(st.declare_cache("team", false).unwrap(), Some("created")); + assert_eq!(st.declare_cache("team", false).unwrap(), None); + assert_eq!(st.declare_cache("team", true).unwrap(), Some("made public")); + assert!(st.cache("team").unwrap().public); + + // The change is in the database, not only in memory. + let (tx, _rx) = mpsc::unbounded_channel(); + let reloaded = AppState::load(cfg, Db::open(&dir.join("helios.db")).unwrap(), None, tx).unwrap(); + assert!(reloaded.cache("team").unwrap().public); + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/docs.md b/docs.md index 55f4eaf..d02785e 100644 --- a/docs.md +++ b/docs.md @@ -35,6 +35,7 @@ Every flag can also be set through its environment variable. | `--audit-retention-days` | `HELIOS_AUDIT_RETENTION_DAYS` | `30` | | `--upload-grace-seconds` | `HELIOS_UPLOAD_GRACE_SECONDS` | `3600`: an unpublished upload is kept this long | | `--admin-socket` | `HELIOS_ADMIN_SOCKET` | unset: no maintenance API (see helios-daemon) | +| `--caches` | `HELIOS_CACHES` | unset: caches to create at startup, such as `main,team:private`; a listed cache that exists takes the listed visibility, and unlisted caches are left alone | Secret files must hold at least 16 bytes. @@ -92,6 +93,18 @@ TLS from Let's Encrypt and zero-copy NAR downloads: } ``` +Caches can be declared instead of created by hand: + +```nix +services.helios.caches = { + main = { }; + team.public = false; # reads need a token with pull +}; +``` + +A declared cache that exists takes the declared visibility. Removing one from +the list leaves it and its paths in place; there is no declarative delete. + Secret files are handed to the service as systemd credentials, so they can be root-only and come from any secret manager (agenix, sops-nix, or files placed by hand). Without `domain`, the module runs only the server on @@ -106,6 +119,7 @@ placed by hand). Without `domain`, the module runs only the server on | `openFirewall` | on with nginx | opens 80 and 443 | | `logLevel` | `info` | | | `settings.*` | | `narinfoCacheEntries`, `maxUploadBytes`, `gcIntervalHours`, `auditRetentionDays` | +| `caches..public` | `true` | caches created at startup; see below | `services.helios.daemon` (on by default) runs helios-daemon beside the server; see [Maintenance](#maintenance). To keep the cache under a size: @@ -161,7 +175,8 @@ instead, for example as Docker secrets, set all three of Both binaries take their flags as `HELIOS_*` environment variables (see `helios-server --help` and `helios-daemon --help`), such as `HELIOS_QUOTA`, -`HELIOS_MIN_FREE`, `HELIOS_METRICS_LISTEN`, `HELIOS_LOG` and +`HELIOS_CACHES=main,team:private`, `HELIOS_MIN_FREE`, `HELIOS_METRICS_LISTEN`, +`HELIOS_LOG` and `HELIOS_LOG_FORMAT=json` (see [Logging](#logging)). Set `HELIOS_DAEMON=0` to run the server alone. `docker stop` shuts both down cleanly; if either exits on its own, the container exits non-zero so the diff --git a/nix/module.nix b/nix/module.nix index 707c36a..43b7ebb 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -129,6 +129,23 @@ in description = "Bearer secret for the admin API (at least 16 bytes). Without it the admin API is disabled."; }; + caches = mkOption { + type = types.attrsOf (types.submodule { + options.public = mkOption { + type = types.bool; + default = true; + description = "Whether anyone may read the cache; a private cache needs a token with pull."; + }; + }); + default = { }; + example = { main = { }; team.public = false; }; + description = '' + Caches to create at startup. A declared cache that already exists takes + the declared visibility. Removing a cache from here does not delete it + or its paths. + ''; + }; + logLevel = mkOption { type = types.str; default = "info"; @@ -311,6 +328,9 @@ in HELIOS_GC_INTERVAL_HOURS = toString cfg.settings.gcIntervalHours; HELIOS_AUDIT_RETENTION_DAYS = toString cfg.settings.auditRetentionDays; } + // lib.optionalAttrs (cfg.caches != { }) { + HELIOS_CACHES = lib.concatStringsSep "," (lib.mapAttrsToList (name: c: if c.public then name else "${name}:private") cfg.caches); + } // lib.optionalAttrs cfg.daemon.enable { HELIOS_ADMIN_SOCKET = adminSocket; } diff --git a/nix/test.nix b/nix/test.nix index da6d6a1..776f895 100644 --- a/nix/test.nix +++ b/nix/test.nix @@ -20,6 +20,10 @@ pkgs.testers.runNixOSTest { domain = "server"; # A custom location exercises the tmpfiles and sandbox path handling. dataDir = "/srv/helios"; + caches = { + main = { }; + team.public = false; + }; nginx.acme = false; signingKeyFile = "/etc/helios/signing-key"; jwtSecretFile = "/etc/helios/jwt-secret"; @@ -120,7 +124,8 @@ pkgs.testers.runNixOSTest { with subtest("admin and push"): client.succeed(f"helios login admin http://server {admin}") - client.succeed("helios cache create main") + caches = {c["name"]: c["public"] for c in json.loads(client.succeed("helios cache list"))["caches"]} + assert caches == {"main": True, "team": False}, caches token = json.loads(client.succeed("helios token create ci --caches main --perms push,pull 2>/dev/null"))["token"] client.succeed(f"helios login ci http://server {token}") client.succeed("helios push main --closure ${pkgs.hello}") From cc8a95266a2d0ec0f511338b67c5874f82421565 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 34/56] gc: let pins keep paths and their closures Auto-GC evicts the least recently used paths, so a release or system fetched rarely could vanish from the cache with nothing to hold it (zhaofengli/attic#83). A pin names a store path in a cache. It protects the path and its closure, followed through the published narinfo References when auto-GC runs, so dependencies pushed after the pin count too. The candidate list skips protected paths, and evict re-checks under the writer lock since a pin may be newer than the list. Pins do not stop the scrub from unpublishing a corrupt NAR. The API is GET, POST and DELETE on /caches//pins (pull to list, push to change), with helios pin, unpin, pins and push --pin. The daemon logs when what is left over quota is pinned. --- crates/helios-cli/src/api.rs | 13 +++ crates/helios-cli/src/main.rs | 47 ++++++++- crates/helios-daemon/src/autogc.rs | 3 + crates/helios-server/src/db.rs | 11 ++ crates/helios-server/src/internal.rs | 33 ++++-- crates/helios-server/src/main.rs | 3 + crates/helios-server/src/pins.rs | 151 +++++++++++++++++++++++++++ docs.md | 27 +++++ nix/test.nix | 16 ++- scripts/e2e.sh | 16 ++- 10 files changed, 305 insertions(+), 15 deletions(-) create mode 100644 crates/helios-server/src/pins.rs diff --git a/crates/helios-cli/src/api.rs b/crates/helios-cli/src/api.rs index 7f2815d..3ca1183 100644 --- a/crates/helios-cli/src/api.rs +++ b/crates/helios-cli/src/api.rs @@ -84,6 +84,19 @@ impl Client { self.call(Method::PUT, &format!("/caches/{cache}/nar?compression=zstd"), Some("application/x-nix-nar"), body).await } + pub async fn pin(&self, cache: &str, store_paths: &[String]) -> anyhow::Result { + self.post(&format!("/caches/{cache}/pins"), &json!({ "storePaths": store_paths })).await + } + + pub async fn unpin(&self, cache: &str, store_path: &str) -> anyhow::Result { + let base = store_path.strip_prefix("/nix/store/").unwrap_or(store_path); + self.call(Method::DELETE, &format!("/caches/{cache}/pins/{base}"), None, transport::full(Bytes::new())).await + } + + pub async fn pins(&self, cache: &str) -> anyhow::Result { + self.call(Method::GET, &format!("/caches/{cache}/pins"), None, transport::full(Bytes::new())).await + } + pub async fn cache_info(&self, cache: &str) -> anyhow::Result { self.call(Method::GET, &format!("/caches/{cache}"), None, transport::full(Bytes::new())).await } diff --git a/crates/helios-cli/src/main.rs b/crates/helios-cli/src/main.rs index 92a1f58..b3fc236 100644 --- a/crates/helios-cli/src/main.rs +++ b/crates/helios-cli/src/main.rs @@ -48,6 +48,10 @@ enum Command { /// zstd compression level (1-19). #[arg(long, default_value_t = 3, value_parser = clap::value_parser!(i32).range(1..=19))] level: i32, + /// Pin the given paths afterwards, so auto-GC keeps them and their + /// closures. + #[arg(long)] + pin: bool, /// Upload NARs in chunks of this many MiB, for proxies that cap /// request bodies (Cloudflare, Cloud Run). 0 sends each NAR in one /// request; so does a NAR that fits in one chunk. @@ -72,6 +76,21 @@ enum Command { #[arg(long, default_value_t = 32, value_parser = clap::value_parser!(u64).range(0..=64))] chunk_size: u64, }, + /// Protect store paths, with their closures in the cache, from auto-GC. + Pin { + cache: String, + /// Store paths, or installables to resolve with Nix. + #[arg(required = true)] + paths: Vec, + }, + /// Remove pins. + Unpin { + cache: String, + #[arg(required = true)] + paths: Vec, + }, + /// List a cache's pins. + Pins { cache: String }, /// Configure Nix to substitute from a cache: its URL, signing key and, /// for a private cache, this login's token in netrc. Use { @@ -164,9 +183,20 @@ async fn run(cli: Cli) -> anyhow::Result<()> { let client = api::Client::new(&server)?; match cli.command { Command::Login { .. } | Command::QueuePaths { .. } => unreachable!(), - Command::Push { cache, installables, closure, jobs, level, chunk_size } => { + Command::Push { cache, installables, closure, jobs, level, chunk_size, pin } => { push::push(&client, &cache, &installables, push::Options { jobs, level, closure, chunk_size: (chunk_size << 20) as usize }).await?; + if pin { + pin_paths(&client, &cache, &installables).await?; + } + } + Command::Pin { cache, paths } => pin_paths(&client, &cache, &paths).await?, + Command::Unpin { cache, paths } => { + for path in store_paths(&paths).await? { + client.unpin(&cache, &path).await?; + tracing::info!("unpinned {path}"); + } } + Command::Pins { cache } => print(&client.pins(&cache).await?), Command::Use { cache, print } => { let plan = substituter::plan(&client, &server, &cache).await?; if print { plan.print() } else { plan.apply()? } @@ -212,3 +242,18 @@ fn main() { std::process::exit(1); } } + +/// Store paths as given, or resolved by Nix when any is an installable. +async fn store_paths(args: &[String]) -> anyhow::Result> { + if args.iter().all(|a| a.starts_with("/nix/store/")) { + return Ok(args.to_vec()); + } + Ok(nix::path_infos(args, false).await?.into_iter().map(|i| i.path).collect()) +} + +async fn pin_paths(client: &api::Client, cache: &str, args: &[String]) -> anyhow::Result<()> { + let paths = store_paths(args).await?; + let r = client.pin(cache, &paths).await?; + tracing::info!("pinned {} paths in '{cache}' ({} already were)", r["pinned"], r["alreadyPinned"]); + Ok(()) +} diff --git a/crates/helios-daemon/src/autogc.rs b/crates/helios-daemon/src/autogc.rs index c9fdf51..299b3dd 100644 --- a/crates/helios-daemon/src/autogc.rs +++ b/crates/helios-daemon/src/autogc.rs @@ -116,6 +116,9 @@ pub async fn run(d: &Daemon, policy: &Policy) -> anyhow::Result<()> { break; } } + if need > 0 { + tracing::warn!(need, "cannot free enough: what is left is pinned, or shared with paths still in use"); + } add(&d.metrics.gc_evicted_paths, evicted); add(&d.metrics.gc_freed_bytes, freed); tracing::info!(paths = evicted, freed_now = freed, "evicted (the rest is freed after the upload grace period)"); diff --git a/crates/helios-server/src/db.rs b/crates/helios-server/src/db.rs index 4f9475c..ca5e4d8 100644 --- a/crates/helios-server/src/db.rs +++ b/crates/helios-server/src/db.rs @@ -65,6 +65,17 @@ CREATE INDEX audit_log_ts ON audit_log (ts); ALTER TABLE paths ADD COLUMN accessed_at INTEGER NOT NULL DEFAULT 0; UPDATE paths SET accessed_at = created_at; CREATE INDEX paths_accessed ON paths (accessed_at); +"#, + // Paths auto-GC must keep, with their closures. + r#" +CREATE TABLE pins ( + cache_id INTEGER NOT NULL REFERENCES caches (id), + hash BLOB NOT NULL, + store_path TEXT NOT NULL, + created_at INTEGER NOT NULL, + created_by TEXT NOT NULL, + PRIMARY KEY (cache_id, hash) +); "#, ]; diff --git a/crates/helios-server/src/internal.rs b/crates/helios-server/src/internal.rs index 790a6fe..9650d07 100644 --- a/crates/helios-server/src/internal.rs +++ b/crates/helios-server/src/internal.rs @@ -89,6 +89,7 @@ async fn stats(State(st): State) -> ApiResult> { } /// Least recently accessed paths first, with the size of their NAR. +/// Pinned paths and their closures are never candidates. async fn lru(State(st): State, uri: Uri) -> ApiResult> { let limit = param(&uri, "limit").unwrap_or(500).clamp(1, 10_000); let paths = blocking(move || { @@ -96,21 +97,32 @@ async fn lru(State(st): State, uri: Uri) -> ApiResult> { crate::gc::flush_access(&st).map_err(internal)?; st.db .read(|conn| { + let protected = crate::pins::protected(conn)?; + // No LIMIT: rows stream, and pinned ones are skipped until + // enough candidates are found. let mut stmt = conn.prepare_cached( "SELECT p.cache_id, p.hash, p.store_path, p.accessed_at, b.file_size - FROM paths p JOIN blobs b ON b.id = p.blob_id ORDER BY p.accessed_at, p.id LIMIT ?1", + FROM paths p JOIN blobs b ON b.id = p.blob_id ORDER BY p.accessed_at, p.id", )?; - stmt.query_map([limit], |r| { + let mut rows = stmt.query([])?; + let mut out = Vec::new(); + while out.len() < limit + && let Some(r) = rows.next()? + { + let cache: u32 = r.get(0)?; let hash: Vec = r.get(1)?; - Ok(json!({ - "cache": r.get::<_, u32>(0)?, + if <[u8; 20]>::try_from(hash.as_slice()).is_ok_and(|hash| protected.contains(&PathKey { cache, hash })) { + continue; + } + out.push(json!({ + "cache": cache, "hash": helios_core::nix32_encode(&hash), "storePath": r.get::<_, String>(2)?, "accessedAt": r.get::<_, i64>(3)?, "narBytes": r.get::<_, u64>(4)?, - })) - })? - .collect::>>() + })); + } + Ok(out) }) .map_err(internal) }) @@ -137,9 +149,10 @@ fn forget(st: &Shared, keys: &[PathKey]) { } } -/// Deletes paths, then any blobs that no longer back a path. +/// Deletes paths, then any blobs that no longer back a path. Pinned paths +/// are kept, even when asked for: a pin may be newer than the candidate list. async fn evict(State(st): State, Json(req): Json) -> ApiResult> { - let keys: Vec = req + let mut keys: Vec = req .paths .iter() .map(|p| helios_core::nix32_decode::<20>(&p.hash).map(|hash| PathKey { cache: p.cache, hash })) @@ -148,6 +161,8 @@ async fn evict(State(st): State, Json(req): Json) -> ApiResult blocking(move || { let evicted = st.db.write(|conn| -> rusqlite::Result { let tx = conn.transaction()?; + let protected = crate::pins::protected(&tx)?; + keys.retain(|k| !protected.contains(k)); let mut n = 0; { let mut del = tx.prepare_cached("DELETE FROM paths WHERE cache_id = ?1 AND hash = ?2")?; diff --git a/crates/helios-server/src/main.rs b/crates/helios-server/src/main.rs index 778e6fd..9df88ce 100644 --- a/crates/helios-server/src/main.rs +++ b/crates/helios-server/src/main.rs @@ -7,6 +7,7 @@ mod db; mod error; mod gc; mod internal; +mod pins; mod push; mod read; mod state; @@ -39,6 +40,8 @@ pub fn router(st: Shared) -> Router { .route("/caches/{cache}/missing", post(push::missing)) .route("/caches/{cache}/nars/known", post(push::known)) .route("/caches/{cache}/nar", put(push::upload).layer(DefaultBodyLimit::disable())) + .route("/caches/{cache}/pins", get(pins::list).post(pins::add)) + .route("/caches/{cache}/pins/{path}", axum::routing::delete(pins::remove)) .route("/caches/{cache}/uploads", post(chunked::create)) .route("/caches/{cache}/uploads/{id}", get(chunked::status).patch(chunked::append).delete(chunked::abort)) .route("/caches/{cache}/uploads/{id}/complete", post(chunked::complete)) diff --git a/crates/helios-server/src/pins.rs b/crates/helios-server/src/pins.rs new file mode 100644 index 0000000..4a2ff42 --- /dev/null +++ b/crates/helios-server/src/pins.rs @@ -0,0 +1,151 @@ +//! Pins: store paths auto-GC must not evict, with their closures in the +//! cache. A pinned system is no use without its dependencies, so a pin +//! protects everything it references, directly or not, including +//! dependencies pushed after the pin. +//! +//! - `GET /_api/v2/caches/{cache}/pins` (pull) +//! - `POST /_api/v2/caches/{cache}/pins` `{"storePaths": [...]}` (push) +//! - `DELETE /_api/v2/caches/{cache}/pins/{store path basename}` (push) +//! +//! Pins only hold back eviction. A path the integrity scrub finds corrupt +//! is still unpublished. + +use std::collections::HashSet; + +use axum::Json; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use rusqlite::{Connection, params}; +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::audit::Audit; +use crate::auth::Perm; +use crate::error::{ApiError, ApiResult}; +use crate::push::cache_for; +use crate::state::{PathKey, Shared}; + +const MAX_PINS_PER_REQUEST: usize = 1000; + +/// `/nix/store/-` or its basename, as (hash, basename). +fn parse_path(s: &str) -> Option<([u8; 20], &str)> { + let base = s.strip_prefix("/nix/store/").unwrap_or(s); + if !helios_core::store_basename_valid(base) { + return None; + } + Some((helios_core::nix32_decode::<20>(&base[..32])?, base)) +} + +/// Every path a pin protects: pinned paths and their closures, following +/// the References of each narinfo within its cache. +pub fn protected(conn: &Connection) -> rusqlite::Result> { + let mut seen = HashSet::new(); + let mut todo: Vec = conn + .prepare_cached("SELECT cache_id, hash FROM pins")? + .query_map([], |r| Ok((r.get::<_, u32>(0)?, r.get::<_, Vec>(1)?)))? + .filter_map(|row| row.ok().and_then(|(cache, hash)| Some(PathKey { cache, hash: hash.try_into().ok()? }))) + .collect(); + let mut narinfo = conn.prepare_cached("SELECT narinfo FROM paths WHERE cache_id = ?1 AND hash = ?2")?; + while let Some(key) = todo.pop() { + if !seen.insert(key) { + continue; + } + let text: Option> = match narinfo.query_row(params![key.cache, &key.hash[..]], |r| r.get(0)) { + Ok(t) => Some(t), + Err(rusqlite::Error::QueryReturnedNoRows) => None, // Pinned before it was pushed. + Err(e) => return Err(e), + }; + let Some(text) = text else { continue }; + let refs = std::str::from_utf8(&text).unwrap_or("").lines().find_map(|l| l.strip_prefix("References:")).unwrap_or(""); + for r in refs.split_ascii_whitespace() { + if let Some((hash, _)) = parse_path(r) { + todo.push(PathKey { cache: key.cache, hash }); + } + } + } + Ok(seen) +} + +pub async fn list(State(st): State, Path(name): Path, headers: HeaderMap) -> ApiResult> { + let cache = cache_for(&st, &name)?; + st.authorize(&headers, &name, Perm::Pull)?; + let pins = tokio::task::spawn_blocking(move || { + st.db.read(|conn| { + conn.prepare_cached("SELECT store_path, created_at, created_by FROM pins WHERE cache_id = ?1 ORDER BY store_path")? + .query_map([cache.id], |r| { + Ok(json!({ "storePath": format!("/nix/store/{}", r.get::<_, String>(0)?), "createdAt": r.get::<_, i64>(1)?, "createdBy": r.get::<_, String>(2)? })) + })? + .collect::>>() + }) + }) + .await??; + Ok(Json(json!({ "pins": pins }))) +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AddReq { + store_paths: Vec, +} + +pub async fn add( + State(st): State, + Path(name): Path, + audit: Audit, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult<(StatusCode, Json)> { + let cache = cache_for(&st, &name)?; + let who = st.authorize(&headers, &name, Perm::Push)?; + if req.store_paths.is_empty() || req.store_paths.len() > MAX_PINS_PER_REQUEST { + return Err(ApiError::bad_request(format!("storePaths must hold 1-{MAX_PINS_PER_REQUEST} store paths"))); + } + let parsed: Vec<([u8; 20], String)> = req + .store_paths + .iter() + .map(|p| parse_path(p).map(|(h, base)| (h, base.to_owned())).ok_or_else(|| ApiError::bad_request(format!("invalid store path: {p}")))) + .collect::>()?; + let actor = who.actor().to_owned(); + let st2 = st.clone(); + let added = tokio::task::spawn_blocking(move || { + st2.db.write(|conn| -> rusqlite::Result { + let tx = conn.transaction()?; + let mut n = 0; + { + let mut insert = tx.prepare_cached( + "INSERT INTO pins (cache_id, hash, store_path, created_at, created_by) VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT (cache_id, hash) DO NOTHING", + )?; + for (hash, base) in &parsed { + n += insert.execute(params![cache.id, &hash[..], base, crate::db::now(), actor])?; + } + } + tx.commit()?; + Ok(n) + }) + }) + .await??; + audit.log(&st, &who, "pin.add", Some(&name), StatusCode::CREATED, json!({ "pinned": added, "requested": req.store_paths.len() })); + Ok((StatusCode::CREATED, Json(json!({ "pinned": added, "alreadyPinned": req.store_paths.len() - added })))) +} + +pub async fn remove( + State(st): State, + Path((name, path)): Path<(String, String)>, + audit: Audit, + headers: HeaderMap, +) -> ApiResult> { + let cache = cache_for(&st, &name)?; + let who = st.authorize(&headers, &name, Perm::Push)?; + let (hash, _) = parse_path(&path).ok_or_else(|| ApiError::bad_request(format!("invalid store path: {path}")))?; + let st2 = st.clone(); + let removed = tokio::task::spawn_blocking(move || { + st2.db.write(|conn| conn.execute("DELETE FROM pins WHERE cache_id = ?1 AND hash = ?2", params![cache.id, &hash[..]])) + }) + .await??; + if removed == 0 { + return Err(ApiError::new(StatusCode::NOT_FOUND, "not pinned")); + } + audit.log(&st, &who, "pin.remove", Some(&name), StatusCode::OK, json!({ "storePath": path })); + Ok(Json(json!({ "unpinned": removed }))) +} diff --git a/docs.md b/docs.md index d02785e..bbc59de 100644 --- a/docs.md +++ b/docs.md @@ -294,6 +294,10 @@ to the token that started it, is dropped after an hour without a chunk or after an invalid one, and does not survive a server restart (the client then starts that NAR again). +Pin endpoints: `GET /_api/v2/caches//pins` (`pull`), `POST +/_api/v2/caches//pins` with `{"storePaths": [...]}` and `DELETE +/_api/v2/caches//pins/` (`push`). + Admin endpoints (bearer admin secret): `GET/POST /_api/v2/admin/caches`, `GET/POST /_api/v2/admin/tokens`, `POST /_api/v2/admin/tokens//revoke`. @@ -317,6 +321,7 @@ agrees with SQLite. The socket's permissions are its access control. `--min-free` also evicts while the disk is short on space. NARs another path still uses stay, and an evicted NAR is deleted once its upload grace period has passed. + Pinned paths stay too; see [Pins](#pins). - **Integrity scrub.** Every `--scrub-interval` (7d), every NAR is read at up to `--scrub-rate` (64M per second), then decompressed and re-hashed. A missing or corrupt NAR is quarantined, which unpublishes its paths, so no @@ -338,6 +343,28 @@ The maintenance API (`/v1/stats`, `/v1/lru`, `/v1/evict`, `/v1/blobs`, `/v1/quarantine`, `/v1/gc`, `/v1/db/checkpoint`, `/v1/db/optimize`, `/v1/db/backup`) is internal and may change between versions. +## Pins + +A pin keeps a store path and its closure in a cache through auto-GC, for +releases or systems that must stay substitutable however rarely they are +fetched: + +```bash +helios push main --closure --pin .#nixosConfigurations.host.config.system.build.toplevel +helios pin main /nix/store/...-release # or an installable +helios pins main +helios unpin main /nix/store/...-release +``` + +The closure is followed through the References of the published narinfo +when auto-GC runs, so dependencies pushed after the pin are protected too. +It stops at a dependency the cache does not have: pin after pushing the +closure (as `--pin` does). Pinning and unpinning need `push` on the cache, +listing needs `pull`. Pins only hold back eviction: the integrity scrub +still unpublishes a corrupt NAR, and pushing the closure again restores it. +When what is left over quota is pinned, the daemon logs that it cannot +free enough. + ## Watch-store On a build machine, `helios queue-paths` works as Nix's `post-build-hook`. diff --git a/nix/test.nix b/nix/test.nix index 776f895..46eca01 100644 --- a/nix/test.nix +++ b/nix/test.nix @@ -154,7 +154,7 @@ pkgs.testers.runNixOSTest { with subtest("daemon: metrics, backups, scrub"): server.wait_until_succeeds("curl -sf http://127.0.0.1:9120/metrics | grep -qx 'helios_up 1'") - assert int(metric('helios_cache_paths').split()[-1]) > 0 + assert int(metric('helios_cache_paths{cache="main"}').split()[-1]) > 0 server.wait_until_succeeds("test $(ls /srv/helios/backups | wc -l) -eq 2", timeout=60) # Corrupt a dependency's NAR; the scrub must take it out of service. url = client.succeed(f"curl -sf {narinfo_url('${pkgs.glibc}')} | sed -n 's/^URL: nar\\///p'").strip() @@ -168,15 +168,23 @@ pkgs.testers.runNixOSTest { built = client.succeed("nix-build /etc/watched.nix --no-out-link").strip() client.wait_until_succeeds(f"curl -sf {narinfo_url(built)}", timeout=60) - with subtest("auto-gc evicts least recently used paths"): - before = int(metric('helios_cache_paths').split()[-1]) + with subtest("auto-gc evicts least recently used paths, except pinned closures"): + # The scrub unpublished glibc; a re-push restores the whole closure, + # which --pin then protects. + client.succeed("helios push main --closure --pin ${pkgs.hello}") + client.succeed(f"curl -sf {narinfo_url('${pkgs.glibc}')}") + pinned = client.succeed("nix-store -qR ${pkgs.hello}").split() + before = int(metric('helios_cache_paths{cache="main"}').split()[-1]) server.succeed( "sudo -u helios-daemon timeout 5 ${helios}/bin/helios-daemon " "--socket /run/helios-admin/admin.sock --data-dir /srv/helios " "--quota 1K --gc-interval 1s --scrub-interval 0 --backup-interval 0 || test $? -eq 124" ) - after = int(metric('helios_cache_paths').split()[-1]) + after = int(metric('helios_cache_paths{cache="main"}').split()[-1]) assert after < before, (before, after) + for p in pinned: + client.succeed(f"curl -sf http://server/main/{p[11:43]}.narinfo") + assert after == len(pinned), (after, pinned) with subtest("SIGTERM shuts down cleanly"): server.succeed("systemctl stop helios-daemon helios") diff --git a/scripts/e2e.sh b/scripts/e2e.sh index bc0e7cc..2a3df12 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -44,7 +44,8 @@ HELIOS_LOG=warn,helios_server=debug "$BIN/helios-server" 2>"$WORK/server.log" \ --data-dir "$WORK/data" \ --signing-key-file "$WORK/key" \ --jwt-secret-file "$WORK/jwt" \ - --admin-secret-file "$WORK/admin" & + --admin-secret-file "$WORK/admin" \ + --admin-socket "$WORK/admin.sock" & SERVER_PID=$! for _ in $(seq 50); do curl -sf "$URL/healthz" >/dev/null && break; sleep 0.1; done @@ -144,6 +145,19 @@ check "and not without it" fails "$(XDG_CONFIG_HOME="$WORK/empty" nix store veri check "--print shows system-wide settings" 1 "$(helios use secret --print | grep -c "^machine 127.0.0.1 *\$")" helios login ci "$URL" "$PUSH_TOKEN" >/dev/null 2>&1 +echo "=== pins" +# Auto-GC takes its candidates from /v1/lru; main is cache 1, chunked cache 3. +lru_count() { curl -s --unix-socket "$WORK/admin.sock" "http://x/v1/lru?limit=10000" | jq --argjson c "$1" --argjson closure "$(nix-store -qR "$TARGET" | jq -R . | jq -s .)" '[.paths[] | select(.cache == $c and ((.storePath | if startswith("/nix/store/") then . else "/nix/store/" + . end) as $p | $closure | index($p)))] | length'; } +check "the closure starts as a GC candidate" "$CLOSURE_SIZE" "$(lru_count 1)" +helios pin main "$TARGET" 2>/dev/null +check "pins are listed" "$TARGET" "$(helios pins main | jq -r '.pins[0].storePath')" +check "a pinned closure is not a GC candidate" 0 "$(lru_count 1)" +check "the same paths in another cache still are" "$CLOSURE_SIZE" "$(lru_count 3)" +check "pull-only token cannot pin" 403 "$(status -X POST -H "authorization: Bearer $PULL_ONLY" -H 'content-type: application/json' -d "{\"storePaths\":[\"$TARGET\"]}" "$URL/_api/v2/caches/secret/pins")" +check "invalid pin rejected" 400 "$(status -X POST "${AUTH[@]}" -H 'content-type: application/json' -d '{"storePaths":["/etc/passwd"]}' "$URL/_api/v2/caches/main/pins")" +helios unpin main "$TARGET" 2>/dev/null +check "unpinned closure is a candidate again" "$CLOSURE_SIZE" "$(lru_count 1)" + echo "=== abuse" check "garbage upload rejected" 400 "$(status -X PUT -H "authorization: Bearer $PUSH_TOKEN" --data-binary 'not a zstd stream' "$URL/_api/v2/caches/main/nar")" check "raw non-NAR upload rejected" 400 "$(status -X PUT -H "authorization: Bearer $PUSH_TOKEN" --data-binary 'hello' "$URL/_api/v2/caches/main/nar?compression=none")" From 6ca2690446e4200148c7175f5056d2d10a46ee8c Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 35/56] core: export detached signing hl_signer_sign signs an arbitrary message with the key, as :, for signatures that are not a narinfo's. A test checks it gives the narinfo's Sig for the same fingerprint. --- core/include/helios.h | 2 ++ core/src/root.zig | 9 ++++++++ crates/helios-core/src/lib.rs | 39 +++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+) diff --git a/core/include/helios.h b/core/include/helios.h index f4fcc3b..6ae2ee3 100644 --- a/core/include/helios.h +++ b/core/include/helios.h @@ -83,6 +83,8 @@ hl_signer *hl_signer_new(const char *key, size_t len); void hl_signer_free(hl_signer *s); /* Writes ":". Sets *out_len even on HL_E_BUFFER. */ int hl_signer_public_key(const hl_signer *s, char *out, size_t cap, size_t *out_len); +/* A detached Ed25519 signature over msg, as `:`. */ +int hl_signer_sign(const hl_signer *s, const uint8_t *msg, size_t msg_len, char *out, size_t cap, size_t *out_len); /* Generates a new secret key in the same format from the OS CSPRNG. */ int hl_signer_generate(const char *name, size_t name_len, char *out, size_t cap, size_t *out_len); diff --git a/core/src/root.zig b/core/src/root.zig index 94993ed..7f89eba 100644 --- a/core/src/root.zig +++ b/core/src/root.zig @@ -191,6 +191,15 @@ export fn hl_signer_public_key(s: *const narinfo.Signer, out: ?[*]u8, cap: usize return copyOut(list.items, out, cap, out_len); } +/// A detached signature over `msg`, as `:`. +export fn hl_signer_sign(s: *const narinfo.Signer, msg: ?[*]const u8, msg_len: usize, out: ?[*]u8, cap: usize, out_len: *usize) c_int { + var list: std.ArrayList(u8) = .empty; + defer list.deinit(gpa); + const m: []const u8 = if (msg) |p| p[0..msg_len] else ""; + s.sign(&list, gpa, m) catch |e| return code(e); + return copyOut(list.items, out, cap, out_len); +} + pub const Str = extern struct { ptr: ?[*]const u8, len: usize, diff --git a/crates/helios-core/src/lib.rs b/crates/helios-core/src/lib.rs index df29dd4..33a3afd 100644 --- a/crates/helios-core/src/lib.rs +++ b/crates/helios-core/src/lib.rs @@ -68,6 +68,7 @@ mod sys { pub fn hl_signer_new(key: *const u8, len: usize) -> *mut Signer; pub fn hl_signer_free(s: *mut Signer); pub fn hl_signer_public_key(s: *const Signer, out: *mut u8, cap: usize, out_len: *mut usize) -> c_int; + pub fn hl_signer_sign(s: *const Signer, msg: *const u8, msg_len: usize, out: *mut u8, cap: usize, out_len: *mut usize) -> c_int; pub fn hl_signer_generate(name: *const u8, name_len: usize, out: *mut u8, cap: usize, out_len: *mut usize) -> c_int; pub fn hl_narinfo_render(input: *const NarinfoInput, signer: *const Signer, out: *mut *mut u8, out_len: *mut usize) -> c_int; pub fn hl_free(ptr: *mut u8, len: usize); @@ -332,6 +333,17 @@ impl Signer { buf.truncate(len); String::from_utf8(buf).expect("key name was valid UTF-8") } + + /// A detached signature over `msg`, as `:`. + pub fn sign(&self, msg: &[u8]) -> String { + // Room for the key name (as in public_key) plus 88 base64 characters. + let mut buf = vec![0u8; 512]; + let mut len = 0usize; + let rc = unsafe { sys::hl_signer_sign(self.raw, msg.as_ptr(), msg.len(), buf.as_mut_ptr(), buf.len(), &mut len) }; + assert_eq!(rc, 0, "signature fits in 512 bytes"); + buf.truncate(len); + String::from_utf8(buf).expect("key name was valid UTF-8") + } } impl Drop for Signer { @@ -479,6 +491,33 @@ mod tests { assert!(Signer::generate("bad:name").is_err()); } + #[test] + fn detached_signature_matches_the_narinfo_one() { + // The same key over the same fingerprint gives the narinfo's Sig. + let signer = Signer::new(KEY).unwrap(); + let path = "/nix/store/7rjj86a15146cq1d3qy068lml7n8ykzm-hello-2.12"; + let nar_hash = [7u8; 32]; + let narinfo = render_narinfo( + &NarinfoInput { + store_path: path, + nar_hash: &nar_hash, + nar_size: 1234, + file_hash: &[9u8; 32], + file_size: 100, + compression: "zstd", + references: "", + deriver: "", + system: "", + }, + Some(&signer), + ) + .unwrap(); + let text = String::from_utf8(narinfo).unwrap(); + let sig = text.lines().find_map(|l| l.strip_prefix("Sig: ")).unwrap(); + let fingerprint = format!("1;{path};sha256:{};1234;", nix32_encode(&nar_hash)); + assert_eq!(signer.sign(fingerprint.as_bytes()), sig); + } + #[test] fn signer_public_key() { let signer = Signer::new(KEY).unwrap(); From 367fa1e16967b11d631c240d22dc26edf71492e8 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 36/56] push: publish build traces for content-addressed derivations A CA derivation's output path is only known once built, so Nix looks it up in the cache's build traces (realisations) before it can substitute. The server served none, so CA derivations were always rebuilt (zhaofengli/attic#188). helios push now gathers the build traces of the CA outputs it pushes from nix store build-trace info (nix realisation info on older Nix) and publishes them. The server accepts an entry only when its output path is published in the cache, signs it with the cache key over the fingerprint Nix defines, and serves it from memory in the format the pusher produced: build-trace-v2//.doi from Nix 2.35 on, realisations/sha256:!.doi before. The VM test builds a two-level CA derivation with Nix 2.34, pushes it, and substitutes it into a fresh store with --max-jobs 0. Nix 2.34 and Determinate Nix 2.35 accepted entries with a deliberately wrong signature too, so the signatures follow the upstream definition but no available Nix checks them. --- crates/helios-cli/src/api.rs | 4 + crates/helios-cli/src/nix.rs | 35 ++++ crates/helios-cli/src/push.rs | 44 ++++- crates/helios-server/src/db.rs | 11 ++ crates/helios-server/src/main.rs | 2 + crates/helios-server/src/read.rs | 14 +- crates/helios-server/src/state.rs | 6 +- crates/helios-server/src/traces.rs | 288 +++++++++++++++++++++++++++++ docs.md | 22 +++ nix/test.nix | 35 +++- scripts/e2e.sh | 16 ++ 11 files changed, 472 insertions(+), 5 deletions(-) create mode 100644 crates/helios-server/src/traces.rs diff --git a/crates/helios-cli/src/api.rs b/crates/helios-cli/src/api.rs index 3ca1183..1863b8b 100644 --- a/crates/helios-cli/src/api.rs +++ b/crates/helios-cli/src/api.rs @@ -84,6 +84,10 @@ impl Client { self.call(Method::PUT, &format!("/caches/{cache}/nar?compression=zstd"), Some("application/x-nix-nar"), body).await } + pub async fn publish_build_traces(&self, cache: &str, entries: &[Value]) -> anyhow::Result { + self.post(&format!("/caches/{cache}/build-traces"), &json!({ "entries": entries })).await + } + pub async fn pin(&self, cache: &str, store_paths: &[String]) -> anyhow::Result { self.post(&format!("/caches/{cache}/pins"), &json!({ "storePaths": store_paths })).await } diff --git a/crates/helios-cli/src/nix.rs b/crates/helios-cli/src/nix.rs index 9e04ab7..6f26e12 100644 --- a/crates/helios-cli/src/nix.rs +++ b/crates/helios-cli/src/nix.rs @@ -15,6 +15,8 @@ pub struct PathInfo { /// Full store paths. pub references: Vec, pub deriver: Option, + /// Set for content-addressed paths, such as CA derivation outputs. + pub ca: Option, } #[derive(Deserialize)] @@ -28,6 +30,8 @@ struct RawInfo { references: Vec, #[serde(default)] deriver: Option, + #[serde(default)] + ca: Option, } const STORE_DIR: &str = "/nix/store/"; @@ -96,6 +100,7 @@ fn parse(stdout: &[u8]) -> anyhow::Result> { nar_size: info.nar_size, references: info.references.iter().map(|r| full(r)).collect(), deriver: info.deriver.map(|d| full(&d)), + ca: info.ca, path, }) }) @@ -137,6 +142,35 @@ pub fn topo_order(infos: Vec) -> Vec { order.into_iter().map(|i| slots[i].take().expect("each index placed once")).collect() } +/// Build trace entries (realisations) for these derivations' outputs, as +/// this Nix prints them: `{"key", "value"}` from 2.35 on, `{"id", "outPath"}` +/// before. Empty when this Nix has CA derivations disabled. +pub async fn build_traces(derivers: &[String]) -> anyhow::Result> { + let outputs: Vec = derivers.iter().map(|d| format!("{d}^*")).collect(); + let mut out = None; + // `store build-trace` is the current name; `realisation` the old one. + for sub in [&["store", "build-trace", "info"][..], &["realisation", "info"][..]] { + let o = Command::new("nix") + .args(["--extra-experimental-features", "nix-command ca-derivations"]) + .args(sub) + .args(["--json", "--"]) + .args(&outputs) + .stderr(std::process::Stdio::piped()) + .output() + .await + .context("running nix store build-trace info")?; + if o.status.success() { + out = Some(o); + break; + } + tracing::debug!("nix {}: {}", sub.join(" "), String::from_utf8_lossy(&o.stderr).trim()); + } + let Some(out) = out else { return Ok(Vec::new()) }; + let entries: Vec = serde_json::from_slice(&out.stdout).context("parsing build trace entries")?; + // Store paths given as installables come back as `{"opaquePath"}`. + Ok(entries.into_iter().filter(|e| e.get("key").is_some() || e.get("id").is_some()).collect()) +} + #[cfg(test)] mod tests { use super::*; @@ -149,6 +183,7 @@ mod tests { nar_size: 0, references: refs.iter().map(|r| format!("{STORE_DIR}{r}")).collect(), deriver: None, + ca: None, } } diff --git a/crates/helios-cli/src/push.rs b/crates/helios-cli/src/push.rs index 675a581..fa9025f 100644 --- a/crates/helios-cli/src/push.rs +++ b/crates/helios-cli/src/push.rs @@ -183,8 +183,50 @@ async fn upload_all(client: &Client, cache: &str, paths: &[&PathInfo], opts: &Op } pub async fn push(client: &Client, cache: &str, installables: &[String], opts: Options) -> anyhow::Result<()> { - let started = Instant::now(); let infos = nix::path_infos(installables, opts.closure).await?; + // Outputs of content-addressed derivations need build traces as well, + // or Nix cannot tell which paths a CA derivation produced. + let derivers: Vec = + infos.iter().filter(|i| i.ca.is_some()).filter_map(|i| i.deriver.clone()).collect::>().into_iter().collect(); + let paths: HashSet = infos.iter().map(|i| i.path.clone()).collect(); + push_infos(client, cache, infos, opts).await?; + if !derivers.is_empty() { + push_build_traces(client, cache, &derivers, &paths).await?; + } + Ok(()) +} + +async fn push_build_traces(client: &Client, cache: &str, derivers: &[String], pushed: &HashSet) -> anyhow::Result<()> { + let entries = match nix::build_traces(derivers).await { + Ok(entries) => entries, + Err(e) => { + tracing::warn!("not pushing build traces: {e:#}"); + return Ok(()); + } + }; + // Only outputs this push published; the server refuses the rest. + let entries: Vec = entries + .into_iter() + .filter(|e| { + // `value.outPath` from Nix 2.35 on, `outPath` before. + e["value"]["outPath"] + .as_str() + .or(e["outPath"].as_str()) + .is_some_and(|p| pushed.contains(&if p.starts_with('/') { p.to_owned() } else { format!("/nix/store/{p}") })) + }) + .collect(); + let mut published = 0; + for chunk in entries.chunks(PUBLISH_BATCH) { + published += client.publish_build_traces(cache, chunk).await?["published"].as_u64().unwrap_or(0); + } + if !entries.is_empty() { + tracing::info!("published {published} build traces to '{cache}' ({} already there)", entries.len() as u64 - published); + } + Ok(()) +} + +async fn push_infos(client: &Client, cache: &str, infos: Vec, opts: Options) -> anyhow::Result<()> { + let started = Instant::now(); let total = infos.len(); let mut missing: HashSet = HashSet::new(); diff --git a/crates/helios-server/src/db.rs b/crates/helios-server/src/db.rs index ca5e4d8..1585d3e 100644 --- a/crates/helios-server/src/db.rs +++ b/crates/helios-server/src/db.rs @@ -76,6 +76,17 @@ CREATE TABLE pins ( created_by TEXT NOT NULL, PRIMARY KEY (cache_id, hash) ); +"#, + // Build traces of content-addressed derivations, rendered and signed. + r#" +CREATE TABLE build_traces ( + cache_id INTEGER NOT NULL REFERENCES caches (id), + id TEXT NOT NULL, + out_path TEXT NOT NULL, + body BLOB NOT NULL, + created_at INTEGER NOT NULL, + PRIMARY KEY (cache_id, id) +); "#, ]; diff --git a/crates/helios-server/src/main.rs b/crates/helios-server/src/main.rs index 9df88ce..9fa39cb 100644 --- a/crates/helios-server/src/main.rs +++ b/crates/helios-server/src/main.rs @@ -12,6 +12,7 @@ mod push; mod read; mod state; mod stats; +mod traces; use std::net::SocketAddr; use std::sync::Arc; @@ -40,6 +41,7 @@ pub fn router(st: Shared) -> Router { .route("/caches/{cache}/missing", post(push::missing)) .route("/caches/{cache}/nars/known", post(push::known)) .route("/caches/{cache}/nar", put(push::upload).layer(DefaultBodyLimit::disable())) + .route("/caches/{cache}/build-traces", post(traces::publish)) .route("/caches/{cache}/pins", get(pins::list).post(pins::add)) .route("/caches/{cache}/pins/{path}", axum::routing::delete(pins::remove)) .route("/caches/{cache}/uploads", post(chunked::create)) diff --git a/crates/helios-server/src/read.rs b/crates/helios-server/src/read.rs index 2d468c8..bee2418 100644 --- a/crates/helios-server/src/read.rs +++ b/crates/helios-server/src/read.rs @@ -1,5 +1,6 @@ //! The substituter read path: `//nix-cache-info`, -//! `//.narinfo` and `//nar/.nar[.zst]`. +//! `//.narinfo`, `//nar/.nar[.zst]` and +//! `//build-trace-v2//.doi` (or `realisations/.doi`). //! URLs are parsed by hand, without allocating. use axum::body::Body; @@ -25,12 +26,17 @@ enum Route<'a> { CacheInfo, Narinfo(&'a str), Nar(&'a str, Compression), + /// `build-trace-v2/...` or `realisations/...` + BuildTrace(&'a str), } fn route(rest: &str) -> Option> { if rest == "nix-cache-info" { return Some(Route::CacheInfo); } + if rest.starts_with("build-trace-v2/") || rest.starts_with("realisations/") { + return Some(Route::BuildTrace(rest)); + } if let Some(hash) = rest.strip_suffix(".narinfo") { return (hash.len() == 32).then_some(Route::Narinfo(hash)); } @@ -71,6 +77,12 @@ async fn serve(st: Shared, method: Method, uri: Uri, headers: HeaderMap) -> ApiR match route { Route::CacheInfo => Ok(([(header::CONTENT_TYPE, "text/x-nix-cache-info")], NIX_CACHE_INFO).into_response()), + Route::BuildTrace(id) => Ok(match crate::traces::lookup(&st, cache.id, id) { + Some(body) => { + ([(header::CONTENT_TYPE, HeaderValue::from_static("application/json")), (header::CACHE_CONTROL, cache_control)], body).into_response() + } + None => not_found(if cache.public { NEGATIVE } else { PRIVATE }), + }), Route::Narinfo(hash) => { let Some(hash) = helios_core::nix32_decode::<20>(hash) else { return Ok(not_found(NEGATIVE)); diff --git a/crates/helios-server/src/state.rs b/crates/helios-server/src/state.rs index 1a37f03..5232654 100644 --- a/crates/helios-server/src/state.rs +++ b/crates/helios-server/src/state.rs @@ -76,13 +76,14 @@ pub struct AppState { pub counters: crate::stats::Counters, pub access: crate::stats::Access, pub uploads: crate::chunked::Sessions, + pub traces: RwLock, } pub type Shared = Arc; impl AppState { pub fn load(cfg: Config, db: Db, signer: Option, audit: mpsc::UnboundedSender) -> anyhow::Result { - let (caches, index, tokens) = db.read(|conn| { + let (caches, index, tokens, traces) = db.read(|conn| { let mut caches = HashMap::new(); let mut stmt = conn.prepare("SELECT id, name, is_public FROM caches")?; let mut rows = stmt.query([])?; @@ -109,7 +110,7 @@ impl AppState { let jti: String = r.get(0)?; tokens.insert(jti.into_boxed_str(), TokenState { expires_at: r.get(1)?, revoked: r.get(2)? }); } - Ok((caches, index, tokens)) + Ok((caches, index, tokens, crate::traces::load(conn)?)) })?; tracing::info!(caches = caches.len(), paths = index.len(), tokens = tokens.len(), "loaded state"); @@ -132,6 +133,7 @@ impl AppState { counters: Default::default(), access: Default::default(), uploads: Default::default(), + traces: RwLock::new(traces), }) } diff --git a/crates/helios-server/src/traces.rs b/crates/helios-server/src/traces.rs new file mode 100644 index 0000000..cc0e24a --- /dev/null +++ b/crates/helios-server/src/traces.rs @@ -0,0 +1,288 @@ +//! Build traces ("realisations"): which output path a content-addressed +//! derivation produced, so Nix can substitute CA derivations instead of +//! building them (compare zhaofengli/attic#188). +//! +//! Nix has two formats, and a cache serves whichever its pushers produce: +//! +//! - Nix 2.35 and later: `//build-trace-v2//.doi`, +//! keyed by derivation path, with structured signatures. +//! - Before 2.35: `//realisations/sha256:!.doi`, keyed by +//! the derivation's hash modulo, which only Nix computes. +//! +//! CA derivations are experimental in Nix, and so are both formats. +//! +//! `POST /_api/v2/caches/{cache}/build-traces` takes entries as `nix store +//! build-trace info --json` (or the older `nix realisation info --json`) +//! prints them. Each output path must already be published in the cache. +//! Client signatures are dropped and the entry is signed with the cache's +//! key, as narinfo is; entries are rendered and signed once, at publish +//! time, and served from memory. + +use std::collections::HashMap; + +use axum::Json; +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use bytes::Bytes; +use rusqlite::params; +use serde::Deserialize; +use serde_json::{Value, json}; + +use crate::audit::Audit; +use crate::auth::Perm; +use crate::error::{ApiError, ApiResult}; +use crate::push::cache_for; +use crate::state::{Locked, PathKey, Shared}; + +const MAX_ENTRIES: usize = 5_000; + +/// An entry's file name without `.doi`: `/` for the current +/// format, `sha256:!` for the old one. The two cannot collide. +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct TraceKey { + pub cache: u32, + pub id: Box, +} + +pub struct Trace { + /// The output path, which must still be published for the entry to be + /// served. + pub out: PathKey, + pub body: Bytes, +} + +pub type Traces = HashMap; + +fn out_key(cache: u32, out: &str) -> Option { + let hash = helios_core::store_basename_valid(out).then(|| helios_core::nix32_decode::<20>(&out[..32])).flatten()?; + Some(PathKey { cache, hash }) +} + +pub fn load(conn: &rusqlite::Connection) -> rusqlite::Result { + let mut traces = Traces::new(); + let mut stmt = conn.prepare("SELECT cache_id, id, out_path, body FROM build_traces")?; + let mut rows = stmt.query([])?; + while let Some(r) = rows.next()? { + let cache: u32 = r.get(0)?; + let (id, out): (String, String) = (r.get(1)?, r.get(2)?); + if let Some(key) = out_key(cache, &out) { + traces.insert(TraceKey { cache, id: id.into() }, Trace { out: key, body: Bytes::from(r.get::<_, Vec>(3)?) }); + } + } + Ok(traces) +} + +/// Nix output names: the characters of store path names. +fn output_name_valid(s: &str) -> bool { + (1..=255).contains(&s.len()) && s.bytes().all(|c| c.is_ascii_alphanumeric() || b"+-._?=".contains(&c)) +} + +fn basename(s: &str) -> &str { + s.strip_prefix("/nix/store/").unwrap_or(s) +} + +/// An entry in either format, validated. +enum Parsed<'a> { + /// ``, ``. + Current(&'a str, &'a str), + /// `sha256:!`. + Legacy(&'a str), +} + +impl Parsed<'_> { + fn id(&self) -> String { + match self { + Parsed::Current(drv, output) => format!("{drv}/{output}"), + Parsed::Legacy(id) => (*id).to_owned(), + } + } +} + +fn legacy_id_valid(id: &str) -> bool { + let Some((hash, output)) = id.split_once('!') else { return false }; + hash.strip_prefix("sha256:").is_some_and(|h| h.len() == 64 && h.bytes().all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c))) + && output_name_valid(output) +} + +/// The signed file body. The signature covers what Nix's +/// `UnkeyedRealisation::fingerprint` produces for the format: the entry's +/// JSON with keys sorted and the signatures removed. Every value is limited +/// to store path and hash characters, so no JSON escaping is involved. +fn render(st: &Shared, entry: &Parsed<'_>, out: &str) -> Vec { + let sign = |fingerprint: String| st.signer.as_ref().map(|s| s.sign(fingerprint.as_bytes())); + let body = match entry { + Parsed::Current(drv, output) => { + let sig = sign(format!(r#"{{"key":{{"drvPath":"{drv}","outputName":"{output}"}},"value":{{"outPath":"{out}"}}}}"#)); + let signatures: Vec = sig + .iter() + .map(|s| { + let (name, sig) = s.split_once(':').expect("signatures are name:base64"); + json!({ "keyName": name, "sig": sig }) + }) + .collect(); + json!({ "outPath": out, "signatures": signatures }) + } + Parsed::Legacy(id) => { + let sig = sign(format!(r#"{{"dependentRealisations":{{}},"id":"{id}","outPath":"{out}"}}"#)); + json!({ "dependentRealisations": {}, "id": id, "outPath": out, "signatures": sig.into_iter().collect::>() }) + } + }; + serde_json::to_vec(&body).expect("serialisable") +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct EntryKey { + drv_path: String, + output_name: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct EntryValue { + out_path: String, +} + +/// `{"key": {...}, "value": {...}}` (Nix 2.35+) or `{"id", "outPath"}`. +#[derive(Deserialize)] +#[serde(untagged)] +enum Entry { + #[serde(rename_all = "camelCase")] + Current { key: EntryKey, value: EntryValue }, + #[serde(rename_all = "camelCase")] + Legacy { id: String, out_path: String }, +} + +#[derive(Deserialize)] +pub struct PublishReq { + entries: Vec, +} + +pub async fn publish( + State(st): State, + Path(name): Path, + audit: Audit, + headers: HeaderMap, + Json(req): Json, +) -> ApiResult<(StatusCode, Json)> { + let cache = cache_for(&st, &name)?; + let who = st.authorize(&headers, &name, Perm::Push)?; + if req.entries.len() > MAX_ENTRIES { + return Err(ApiError::bad_request(format!("at most {MAX_ENTRIES} entries per request"))); + } + + let mut rows = Vec::with_capacity(req.entries.len()); + let mut missing = Vec::new(); + { + let index = st.index.rd(); + for e in &req.entries { + let (parsed, out) = match e { + Entry::Current { key, value } => { + let drv = basename(&key.drv_path); + if !(helios_core::store_basename_valid(drv) && drv.ends_with(".drv")) { + return Err(ApiError::bad_request(format!("invalid derivation path: {}", key.drv_path))); + } + if !output_name_valid(&key.output_name) { + return Err(ApiError::bad_request(format!("invalid output name: {}", key.output_name))); + } + (Parsed::Current(drv, &key.output_name), basename(&value.out_path)) + } + Entry::Legacy { id, out_path } => { + if !legacy_id_valid(id) { + return Err(ApiError::bad_request(format!("invalid build trace id: {id}"))); + } + (Parsed::Legacy(id), basename(out_path)) + } + }; + let key = out_key(cache.id, out).ok_or_else(|| ApiError::bad_request(format!("invalid output path: {out}")))?; + if !index.contains(&key) { + missing.push(format!("/nix/store/{out}")); + continue; + } + rows.push((parsed.id(), out.to_owned(), key, render(&st, &parsed, out))); + } + } + if !missing.is_empty() { + return Err(ApiError::new(StatusCode::CONFLICT, "output paths are not published in this cache").with(json!({ "missing": missing }))); + } + + let st2 = st.clone(); + let inserted = tokio::task::spawn_blocking(move || -> ApiResult> { + let now = crate::db::now(); + st2.db + .write(|conn| -> rusqlite::Result> { + let tx = conn.transaction()?; + let mut inserted = Vec::new(); + { + // The first entry for an output wins, as for paths: a + // non-deterministic build must not flip what is served. + let mut insert = tx.prepare_cached( + "INSERT INTO build_traces (cache_id, id, out_path, body, created_at) + VALUES (?1, ?2, ?3, ?4, ?5) ON CONFLICT DO NOTHING", + )?; + for (id, out, key, body) in rows { + if insert.execute(params![cache.id, id, out, body, now])? > 0 { + inserted.push((TraceKey { cache: cache.id, id: id.into() }, Trace { out: key, body: body.into() })); + } + } + } + tx.commit()?; + Ok(inserted) + }) + .map_err(ApiError::from) + }) + .await??; + + let published = inserted.len(); + { + let mut traces = st.traces.wr(); + for (k, v) in inserted { + traces.insert(k, v); + } + } + let already = req.entries.len() - published; + audit.log(&st, &who, "build_trace.publish", Some(&name), StatusCode::CREATED, json!({ "published": published, "alreadyExisted": already })); + Ok((StatusCode::CREATED, Json(json!({ "published": published, "alreadyExisted": already })))) +} + +/// The body for a request path below the cache, if it names a build trace +/// whose output is still published. +pub fn lookup(st: &Shared, cache: u32, rest: &str) -> Option { + let (current, id) = match rest.strip_prefix("build-trace-v2/") { + Some(id) => (true, id), + None => (false, rest.strip_prefix("realisations/")?), + }; + let id = id.strip_suffix(".doi")?; + // Old clients may percent-encode the `:` and `!` of a hash-keyed id. + let id = if current { id.to_owned() } else { id.replace("%3A", ":").replace("%3a", ":").replace("%21", "!") }; + if current == legacy_id_valid(&id) { + return None; // Each format only under its own prefix. + } + let traces = st.traces.rd(); + let trace = traces.get(&TraceKey { cache, id: id.into() })?; + st.index.rd().contains(&trace.out).then(|| trace.body.clone()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn legacy_ids() { + let hex = "a".repeat(64); + assert!(legacy_id_valid(&format!("sha256:{hex}!out"))); + assert!(!legacy_id_valid(&format!("sha256:{hex}"))); + assert!(!legacy_id_valid(&format!("sha1:{hex}!out"))); + assert!(!legacy_id_valid(&format!("sha256:{}!out", "A".repeat(64)))); + assert!(!legacy_id_valid(&format!("sha256:{hex}!bad/name"))); + } + + #[test] + fn both_entry_shapes_parse() { + let current: Entry = + serde_json::from_str(r#"{"key":{"drvPath":"x.drv","outputName":"out"},"value":{"outPath":"y","signatures":[]}}"#).unwrap(); + assert!(matches!(current, Entry::Current { .. })); + let legacy: Entry = serde_json::from_str(r#"{"id":"sha256:ab!out","outPath":"y","signatures":[],"dependentRealisations":{}}"#).unwrap(); + assert!(matches!(legacy, Entry::Legacy { .. })); + } +} diff --git a/docs.md b/docs.md index bbc59de..03eeacc 100644 --- a/docs.md +++ b/docs.md @@ -271,6 +271,7 @@ Substituter endpoints (`GET`/`HEAD`): | `//nix-cache-info` | | | `//.narinfo` | misses are answered from memory | | `//nar/.nar.zst` | only served by caches that publish it | +| `//build-trace-v2//.doi`, `//realisations/.doi` | build traces of CA derivations, from memory | `GET /_api/v2/caches/` returns `{"name", "public", "publicKey"}` to anyone who can read the cache. @@ -294,6 +295,10 @@ to the token that started it, is dropped after an hour without a chunk or after an invalid one, and does not survive a server restart (the client then starts that NAR again). +`POST /_api/v2/caches//build-traces` (`push`) takes `{"entries": [...]}` +as `nix store build-trace info --json` (or `nix realisation info --json`) +prints them. + Pin endpoints: `GET /_api/v2/caches//pins` (`pull`), `POST /_api/v2/caches//pins` with `{"storePaths": [...]}` and `DELETE /_api/v2/caches//pins/` (`push`). @@ -365,6 +370,23 @@ still unpublishes a corrupt NAR, and pushing the closure again restores it. When what is left over quota is pinned, the daemon logs that it cannot free enough. +## Content-addressed derivations + +A content-addressed (CA) derivation's output path is only known once it is +built, so Nix looks it up in the cache's build traces ("realisations") +before it can substitute. `helios push` publishes them for the CA outputs it +pushes, when Nix has the experimental `ca-derivations` feature enabled. + +The cache serves whichever format the pushing Nix produces: Nix 2.35 and +later use `//build-trace-v2//.doi`, earlier versions +`//realisations/sha256:!.doi`. Push and substitute +with the same side of that change. An entry is accepted only when its output +path is published in the cache, and is signed with the cache's key over the +fingerprint Nix defines. Current Nix does not check those signatures when +substituting (a wrong one is accepted too); what it checks is the output +path's signed narinfo. Both formats are experimental upstream and may +change. + ## Watch-store On a build machine, `helios queue-paths` works as Nix's `post-build-hook`. diff --git a/nix/test.nix b/nix/test.nix index 46eca01..c86f19b 100644 --- a/nix/test.nix +++ b/nix/test.nix @@ -75,12 +75,32 @@ pkgs.testers.runNixOSTest { args = [ "-c" "echo built > $out" ]; } ''; - nix.settings.experimental-features = [ "nix-command" ]; + # Content-addressed derivations, whose outputs Nix finds through build + # traces. `top` depends on `dep`, so both traces are needed. + environment.etc."ca.nix".text = '' + let + sh = "''${builtins.storePath "${pkgs.busybox}"}/bin/sh"; + ca = name: script: derivation { + inherit name; + system = "${system}"; + builder = sh; + args = [ "-c" script ]; + __contentAddressed = true; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + }; + in rec { + dep = ca "ca-dep" "echo dep > $out"; + top = ca "ca-top" "echo top ''${dep} > $out"; + } + ''; + nix.settings.experimental-features = [ "nix-command" "ca-derivations" ]; virtualisation.memorySize = 2048; }; testScript = '' import json + import re start_all() @@ -168,6 +188,19 @@ pkgs.testers.runNixOSTest { built = client.succeed("nix-build /etc/watched.nix --no-out-link").strip() client.wait_until_succeeds(f"curl -sf {narinfo_url(built)}", timeout=60) + with subtest("content-addressed derivations substitute through build traces"): + top = client.succeed("nix-build /etc/ca.nix -A top --no-out-link").strip() + out = client.succeed(f"helios push main --closure {top} 2>&1") + assert re.search(r"published [1-9][0-9]* build traces", out), out + # A fresh store may build nothing, so both outputs have to be found + # through the signed traces. + client.succeed("nix copy --no-check-sigs --to /tmp/ca-store ${pkgs.busybox}") + substituted = client.succeed( + "nix-build --store /tmp/ca-store /etc/ca.nix -A top --no-out-link --max-jobs 0 " + f"--option substituters http://server/main --option trusted-public-keys '{pubkey}' --option require-sigs true" + ).strip() + assert substituted == top, (substituted, top) + with subtest("auto-gc evicts least recently used paths, except pinned closures"): # The scrub unpublished glibc; a re-push restores the whole closure, # which --pin then protects. diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 2a3df12..f1c0893 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -158,6 +158,22 @@ check "invalid pin rejected" 400 "$(status -X POST "${AUTH[@]}" -H 'content-type helios unpin main "$TARGET" 2>/dev/null check "unpinned closure is a candidate again" "$CLOSURE_SIZE" "$(lru_count 1)" +echo "=== build traces" +OUT_BASE="$(basename "$TARGET")" +HEX="$(printf '%064d' 0 | tr 0 a)" +traces() { curl -s -o /dev/null -w '%{http_code}' -X POST "${AUTH[@]}" -H 'content-type: application/json' -d "{\"entries\":[$1]}" "$URL/_api/v2/caches/main/build-traces"; } +check "a current-format trace publishes" 201 "$(traces "{\"key\":{\"drvPath\":\"$HASH-x.drv\",\"outputName\":\"out\"},\"value\":{\"outPath\":\"$OUT_BASE\",\"signatures\":[]}}")" +check "a legacy trace publishes" 201 "$(traces "{\"id\":\"sha256:$HEX!out\",\"outPath\":\"$OUT_BASE\",\"signatures\":[],\"dependentRealisations\":{}}")" +check "a trace needs its output published" 409 "$(traces "{\"id\":\"sha256:$HEX!dev\",\"outPath\":\"00000000000000000000000000000000-x\"}")" +check "an invalid trace id is refused" 400 "$(traces "{\"id\":\"sha1:abc!out\",\"outPath\":\"$OUT_BASE\"}")" +CURRENT="$(curl -sf "$URL/main/build-trace-v2/$HASH-x.drv/out.doi")" +check "current trace served, signed by the cache" "$OUT_BASE ${PUBKEY%%:*}" "$(echo "$CURRENT" | jq -r '.outPath + " " + .signatures[0].keyName')" +LEGACY="$(curl -sf "$URL/main/realisations/sha256:$HEX!out.doi")" +check "legacy trace served, signed by the cache" "sha256:$HEX!out ${PUBKEY%%:*}" "$(echo "$LEGACY" | jq -r '.id + " " + (.signatures[0] | split(":")[0])')" +check "legacy trace served percent-encoded" 200 "$(status "$URL/main/realisations/sha256%3A$HEX%21out.doi")" +check "each format only under its prefix" 404 "$(status "$URL/main/build-trace-v2/sha256:$HEX!out.doi")" +check "unknown trace" 404 "$(status "$URL/main/build-trace-v2/$HASH-y.drv/out.doi")" + echo "=== abuse" check "garbage upload rejected" 400 "$(status -X PUT -H "authorization: Bearer $PUSH_TOKEN" --data-binary 'not a zstd stream' "$URL/_api/v2/caches/main/nar")" check "raw non-NAR upload rejected" 400 "$(status -X PUT -H "authorization: Bearer $PUSH_TOKEN" --data-binary 'hello' "$URL/_api/v2/caches/main/nar?compression=none")" From 4b953ffc2ea9f267caccd0e48cd69449bc417889 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 37/56] bench: name reference implementations by their packages Describe the comparisons by the libraries they run (the Haskell Nix libraries, the nix-flake-lock crate, Nix C++), and take the nix-flake-lock crate from crates.io instead of a git checkout. CI installs upstream Nix with the nix-installer-action. --- .github/workflows/ci.yml | 6 ++++-- AGENTS.md | 4 ++-- README.md | 2 +- bench/flake-lock-rs/Cargo.lock | 3 ++- bench/flake-lock-rs/Cargo.toml | 4 ++-- bench/flake-lock.sh | 4 ++-- bench/haskell/Bench.hs | 2 +- bench/haskell/ghc.nix | 2 +- bench/run.sh | 5 +++-- docs.md | 4 ++-- pkg/nix-flake-lock/src/root.zig | 2 +- 11 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec5aa77..76acf00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,9 +34,11 @@ jobs: packages: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: cachix/install-nix-action@13d8dd58da0234aa297dedd986986ccb8e7f3e24 # v31.11.1 + - uses: DeterminateSystems/nix-installer-action@3138316df39ed29be04236d7ffc686fa525866aa # v23 with: - extra_nix_config: | + # Upstream Nix, as users of the module and image run. + determinate: false + extra-conf: | system-features = nixos-test benchmark big-parallel kvm - name: Format diff --git a/AGENTS.md b/AGENTS.md index e4de078..4acf072 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,8 +15,8 @@ A self-hosted Nix binary cache: Rust server and CLI over a Zig core. itself. - `nix/`: the NixOS module (`services.helios`), the OCI image, and their VM tests. -- `bench/`: comparisons against Cachix's libraries and Nix C++. Not part of - the Cargo workspace. +- `bench/`: comparisons against the Haskell Nix libraries, the Rust + nix-flake-lock crate and Nix C++. Not part of the Cargo workspace. Put Nix format logic in a `pkg/*` package and expose it through `core/src/root.zig` only when Rust needs it. Keep calls across the C ABI diff --git a/README.md b/README.md index 236335a..effcd3e 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ core/ libhelios: C ABI over pkg/*, built as a static library crates/ helios-core (Rust bindings), helios-server, helios-cli, helios-daemon (auto-GC, scrub, backups, metrics) nix/ NixOS module (services.helios) and its VM test -bench/ benchmarks against Cachix's Nix libraries and Nix C++ +bench/ benchmarks against the Haskell and Rust Nix libraries and Nix C++ scripts/ e2e.sh (end-to-end test), test-zig.sh (Zig unit tests) ``` diff --git a/bench/flake-lock-rs/Cargo.lock b/bench/flake-lock-rs/Cargo.lock index 3e6dc2a..8a97319 100644 --- a/bench/flake-lock-rs/Cargo.lock +++ b/bench/flake-lock-rs/Cargo.lock @@ -24,7 +24,8 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "nix-flake-lock" version = "0.1.0" -source = "git+https://github.com/cachix/nix-flake-lock?rev=61c13ac#61c13ac3b0c13dd394dcad2c93f76c89c6b541bd" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7815d99678fd80c5d646d68ab0360132754e4c0ba0c5e203bc7b4f353de6d9" dependencies = [ "itoa", "memchr", diff --git a/bench/flake-lock-rs/Cargo.toml b/bench/flake-lock-rs/Cargo.toml index b9bd2de..2d4d2e5 100644 --- a/bench/flake-lock-rs/Cargo.toml +++ b/bench/flake-lock-rs/Cargo.toml @@ -1,4 +1,4 @@ -# Reference harness: cachix/nix-flake-lock, pinned. Not part of the workspace. +# Reference harness: the nix-flake-lock crate (Rust), pinned. Not part of the workspace. [package] name = "flake-lock-rs" version = "0.0.0" @@ -6,7 +6,7 @@ edition = "2024" publish = false [dependencies] -nix-flake-lock = { git = "https://github.com/cachix/nix-flake-lock", rev = "61c13ac" } +nix-flake-lock = "=0.1.0" [profile.release] lto = "fat" diff --git a/bench/flake-lock.sh b/bench/flake-lock.sh index 949b7db..46e0b30 100755 --- a/bench/flake-lock.sh +++ b/bench/flake-lock.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# pkg/nix-flake-lock (Zig) vs cachix/nix-flake-lock (Rust, pinned in +# pkg/nix-flake-lock (Zig) vs the nix-flake-lock crate (Rust, pinned in # bench/flake-lock-rs): a differential check that both produce identical # canonical output and validation results, then a single-threaded benchmark. # @@ -19,7 +19,7 @@ RS="$ROOT/bench/flake-lock-rs/target/release/flake-lock-rs" ZIG="$ROOT/bench/zig/zig-out/bin/flake-lock" mkdir -p "$W/fixtures" "$W/real" "$W/fuzz" -UPSTREAM="$(find "${CARGO_HOME:-$HOME/.cargo}/git/checkouts" -path '*nix-flake-lock*/tests/fixtures' -type d | head -1)" +UPSTREAM="$(find "${CARGO_HOME:-$HOME/.cargo}/registry/src" -path '*/nix-flake-lock-0.1.0/tests/fixtures' -type d | head -1)"| head -1)" cp "$UPSTREAM"/*.lock "$W/fixtures/" "$RS" gen "$W/fixtures" i=0 diff --git a/bench/haskell/Bench.hs b/bench/haskell/Bench.hs index 98690ce..69e5249 100644 --- a/bench/haskell/Bench.hs +++ b/bench/haskell/Bench.hs @@ -2,7 +2,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE PackageImports #-} --- Benchmarks for the Haskell Nix libraries in Cachix's dependency tree +-- Benchmarks for the Haskell Nix libraries -- (nix-narinfo, nix-derivation, hnix-store-core, hnix-store-nar, ed25519). -- Mirrors bench/zig/src/main.zig: same corpus, same output format -- (` `), best of 7 rounds after a warm-up. diff --git a/bench/haskell/ghc.nix b/bench/haskell/ghc.nix index 91552b4..0289f20 100644 --- a/bench/haskell/ghc.nix +++ b/bench/haskell/ghc.nix @@ -1,4 +1,4 @@ -# GHC with the Haskell Nix libraries Cachix builds on. +# GHC with the Haskell Nix libraries (nix-narinfo, nix-derivation, hnix-store). { pkgs ? import { } }: pkgs.haskellPackages.ghcWithPackages (p: [ p.nix-narinfo diff --git a/bench/run.sh b/bench/run.sh index 4748a7f..b9b9dad 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# Benchmarks pkg/* (Zig) against the Haskell libraries Cachix uses, plus -# Nix's own C++ NAR serialiser. Single-threaded, same corpus for both. +# Benchmarks pkg/* (Zig) against the Haskell Nix libraries (nix-narinfo, +# nix-derivation, hnix-store) and Nix's own C++ NAR serialiser. +# Single-threaded, same corpus for both. # # Usage: bench/run.sh [store-path] (inside `nix develop`) set -euo pipefail diff --git a/docs.md b/docs.md index 03eeacc..4d31dd0 100644 --- a/docs.md +++ b/docs.md @@ -399,8 +399,8 @@ and the CLI accepts `--url` with `--token-file` (or `HELIOS_URL` and ## Benchmarks ```bash -bench/run.sh # pkg/* vs Cachix's Haskell libraries and Nix C++ -bench/flake-lock.sh # pkg/nix-flake-lock vs cachix/nix-flake-lock +bench/run.sh # pkg/* vs the Haskell Nix libraries and Nix C++ +bench/flake-lock.sh # pkg/nix-flake-lock vs the nix-flake-lock crate ``` Both use the local store as their corpus and run single-threaded. diff --git a/pkg/nix-flake-lock/src/root.zig b/pkg/nix-flake-lock/src/root.zig index 08ad8ff..10d1cec 100644 --- a/pkg/nix-flake-lock/src/root.zig +++ b/pkg/nix-flake-lock/src/root.zig @@ -1,7 +1,7 @@ //! Nix `flake.lock` parsing (versions 5-7), `follows` validation and //! resolution, and canonical version 7 serialisation, without Nix. //! -//! Semantics follow cachix/nix-flake-lock (Rust): last duplicate key wins, +//! Semantics follow the nix-flake-lock crate (Rust): last duplicate key wins, //! version 5 `info` is overlaid onto `locked`, unreachable nodes are //! dropped, and output is byte-identical to Nix's canonical writer. //! From e0da841b0bf9939afeb3ec25fd47aeffa4b01c04 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 38/56] pkg: reject mismatched signing keys and duplicate references A secret key embeds its public half. Ed25519.KeyPair.fromSecretKey only checks it against the seed under runtime safety, so the ReleaseFast build libhelios ships accepted a corrupt key, advertised a public key no signature verified under, and signed under two public halves, which leaks the secret scalar. Derive the key pair from the seed and refuse a mismatch. render sorted references but kept duplicates, so References and the signed fingerprint listed one twice while Nix, which reads them into a set, fingerprints it once and rejects the signature. Drop adjacent duplicates. test-zig.sh now runs every package in ReleaseFast as well as Debug, so behaviour that depends on runtime safety is covered. --- pkg/nix-narinfo/src/render.zig | 52 +++++++++++++++++++++++++++++++++- scripts/test-zig.sh | 10 +++++-- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/pkg/nix-narinfo/src/render.zig b/pkg/nix-narinfo/src/render.zig index a56fdc7..6512b5d 100644 --- a/pkg/nix-narinfo/src/render.zig +++ b/pkg/nix-narinfo/src/render.zig @@ -61,7 +61,10 @@ pub const Signer = struct { if (n != raw.len) return error.InvalidKey; decoder.decode(&raw, b64) catch return error.InvalidKey; const secret = Ed25519.SecretKey.fromBytes(raw) catch return error.InvalidKey; - const key_pair = Ed25519.KeyPair.fromSecretKey(secret) catch return error.InvalidKey; + // std's fromSecretKey checks the embedded public half only under + // runtime safety; derive it from the seed in every build mode. + const key_pair = Ed25519.KeyPair.generateDeterministic(secret.seed()) catch return error.InvalidKey; + if (!std.mem.eql(u8, &key_pair.public_key.toBytes(), &secret.publicKeyBytes())) return error.InvalidKey; var expanded: [64]u8 = undefined; std.crypto.hash.sha2.Sha512.hash(raw[0..32], &expanded, .{}); @@ -174,6 +177,14 @@ pub fn render(allocator: std.mem.Allocator, in: Input, signer: ?*const Signer) E try refs.append(allocator, r); } std.mem.sort([]const u8, refs.items, {}, lessThanStr); + // Nix reads references into a set, so a duplicate would break the Sig. + var unique: usize = 0; + for (refs.items) |r| { + if (unique > 0 and std.mem.eql(u8, refs.items[unique - 1], r)) continue; + refs.items[unique] = r; + unique += 1; + } + refs.shrinkRetainingCapacity(unique); var nar_hash: [52]u8 = undefined; nix32.encode(&nar_hash, in.nar_hash); @@ -307,3 +318,42 @@ test "generated keys parse and sign" { try std.testing.expectEqualSlices(u8, &theirs, &(try signer.signRaw("msg"))); try std.testing.expectError(error.InvalidKey, Signer.generate(&text, a, "bad:name", [_]u8{1} ** 32)); } + +test "rejects a secret key whose public half does not match its seed" { + const a = std.testing.allocator; + const prefix = "test-1:"; + const good = prefix ++ "suJsWimvBNFUIc0JJE18OfOMygH/f2GuTC2/XwD6rPKkV+1VKilMIkXl6Ax9hqeKpUF/BSOH+Fqng4tqZlirhw=="; + var raw: [64]u8 = undefined; + try std.base64.standard.Decoder.decode(&raw, good[prefix.len..]); + // A valid point, but the public key of another seed. + raw[32..64].* = (try Ed25519.KeyPair.generateDeterministic([_]u8{7} ** 32)).public_key.toBytes(); + var text: [good.len]u8 = undefined; + @memcpy(text[0..prefix.len], prefix); + _ = std.base64.standard.Encoder.encode(text[prefix.len..], &raw); + try std.testing.expectError(error.InvalidKey, Signer.parse(a, &text)); +} + +test "duplicate references are rendered and signed once" { + const a = std.testing.allocator; + const signer = try Signer.parse(a, "test-1:suJsWimvBNFUIc0JJE18OfOMygH/f2GuTC2/XwD6rPKkV+1VKilMIkXl6Ax9hqeKpUF/BSOH+Fqng4tqZlirhw=="); + defer signer.destroy(a); + const nar = [_]u8{1} ** 32; + const file = [_]u8{2} ** 32; + var in: Input = .{ + .store_path = "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello", + .nar_hash = &nar, + .nar_size = 1234, + .file_hash = &file, + .file_size = 99, + .compression = "zstd", + .references = "1mdqa9w1p6cmli6976v4wi0sw9r4p5pr-b 0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello 1mdqa9w1p6cmli6976v4wi0sw9r4p5pr-b", + .deriver = "", + .system = "", + }; + const dup = try render(a, in, signer); + defer a.free(dup); + in.references = "0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello 1mdqa9w1p6cmli6976v4wi0sw9r4p5pr-b"; + const once = try render(a, in, signer); + defer a.free(once); + try std.testing.expectEqualStrings(once, dup); +} diff --git a/scripts/test-zig.sh b/scripts/test-zig.sh index 4ebf9a3..65a0205 100755 --- a/scripts/test-zig.sh +++ b/scripts/test-zig.sh @@ -1,10 +1,14 @@ #!/usr/bin/env bash -# Runs the unit tests of every Zig package, then builds libhelios. +# Runs the unit tests of every Zig package, then builds libhelios. Tests run +# in Debug and in ReleaseFast, which libhelios ships with: std skips some +# checks without runtime safety. set -euo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" for pkg in "$ROOT"/pkg/*/; do - echo "== $(basename "$pkg")" - (cd "$pkg" && zig build test -Dcpu=native --summary failures) + for mode in Debug ReleaseFast; do + echo "== $(basename "$pkg") ($mode)" + (cd "$pkg" && zig build test -Doptimize=$mode -Dcpu=native --summary failures) + done done echo "== libhelios" (cd "$ROOT/core" && zig build -Doptimize=ReleaseFast -Dcpu=native) From 463b298f710615bffcc457e2e074fdca9a38cc02 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 39/56] core: compress with a long window and the exact NAR size The compressor takes window_log, which turns on zstd long-distance matching with a window of up to 2^27 (128 MiB, the largest a stock Nix decoder accepts), and pledges the exact NAR size instead of hinting it. With the size pledged zstd fits the window to the NAR and records the size in the frame, so a decoder allocates no more than each NAR needs. hl_dump_options gains window_log and size_hint becomes nar_size. Also: the CCtx leaked when allocating the output buffer failed, and Signer::public_key and sign panicked on key names longer than their fixed buffers; they now grow the buffer to the size libhelios reports. --- core/include/helios.h | 3 +- core/src/root.zig | 5 +-- crates/helios-core/src/lib.rs | 63 +++++++++++++++++++++++------------ pkg/nix-archive/src/sink.zig | 25 ++++++++++---- pkg/nix-archive/src/tests.zig | 31 ++++++++++++++++- pkg/nix-archive/src/zstd.zig | 5 +-- 6 files changed, 98 insertions(+), 34 deletions(-) diff --git a/core/include/helios.h b/core/include/helios.h index 6ae2ee3..15e71b8 100644 --- a/core/include/helios.h +++ b/core/include/helios.h @@ -46,7 +46,8 @@ typedef struct { typedef struct { int level; /* zstd level; 0 disables compression */ int threads; /* zstd worker threads; 0 = calling thread */ - uint64_t size_hint; /* expected NAR size, 0 if unknown */ + uint64_t nar_size; /* exact NAR size, pledged to zstd; 0 if unknown */ + int window_log; /* 0: the level's window; 10-27: long-distance matching */ } hl_dump_options; /* Return non-zero to abort the pipeline with HL_E_ABORTED. */ diff --git a/core/src/root.zig b/core/src/root.zig index 7f89eba..af3443d 100644 --- a/core/src/root.zig +++ b/core/src/root.zig @@ -83,11 +83,12 @@ export fn hl_store_basename_valid(base: [*]const u8, len: usize) bool { pub const DumpOptions = extern struct { level: c_int, threads: c_int, - size_hint: u64, + nar_size: u64, + window_log: c_int, }; fn sinkOptions(opts: *const DumpOptions) archive.Options { - return .{ .level = opts.level, .threads = opts.threads, .size_hint = opts.size_hint }; + return .{ .level = opts.level, .threads = opts.threads, .nar_size = opts.nar_size, .window_log = opts.window_log }; } export fn hl_nar_dump(path: [*:0]const u8, opts: *const DumpOptions, write: WriteFn, ctx: ?*anyopaque, out: *Digest) c_int { diff --git a/crates/helios-core/src/lib.rs b/crates/helios-core/src/lib.rs index 33a3afd..7333197 100644 --- a/crates/helios-core/src/lib.rs +++ b/crates/helios-core/src/lib.rs @@ -22,7 +22,8 @@ mod sys { pub struct DumpOptions { pub level: c_int, pub threads: c_int, - pub size_hint: u64, + pub nar_size: u64, + pub window_log: c_int, } #[repr(C)] @@ -157,19 +158,23 @@ pub struct DumpOptions { pub level: i32, /// zstd worker threads; 0 compresses on the calling thread. pub threads: i32, - /// Expected NAR size, 0 if unknown. - pub size_hint: u64, + /// The exact NAR size, 0 if unknown. zstd fits its window to it and + /// records it in the frame; a stream of another length fails. + pub nar_size: u64, + /// 0 keeps the level's window; 10-27 enables long-distance matching + /// with a 2^window_log window. + pub window_log: i32, } impl Default for DumpOptions { fn default() -> Self { - Self { level: 3, threads: 0, size_hint: 0 } + Self { level: 3, threads: 0, nar_size: 0, window_log: 0 } } } impl DumpOptions { fn raw(&self) -> sys::DumpOptions { - sys::DumpOptions { level: self.level, threads: self.threads, size_hint: self.size_hint } + sys::DumpOptions { level: self.level, threads: self.threads, nar_size: self.nar_size, window_log: self.window_log } } } @@ -317,35 +322,41 @@ impl Signer { /// A new Nix secret key named `name`, in `nix key generate-secret` format. pub fn generate(name: &str) -> Result { - let mut buf = vec![0u8; 256]; - let mut len = 0usize; - check(unsafe { sys::hl_signer_generate(name.as_ptr(), name.len(), buf.as_mut_ptr(), buf.len(), &mut len) })?; - buf.truncate(len); + // The name, ':' and 88 base64 characters: one call, so one key. + let buf = out_string(name.len() + 89, |out, cap, len| unsafe { sys::hl_signer_generate(name.as_ptr(), name.len(), out, cap, len) })?; Ok(String::from_utf8(buf).expect("key name was valid UTF-8")) } /// `:`, the value for `trusted-public-keys`. pub fn public_key(&self) -> String { - let mut buf = vec![0u8; 256]; - let mut len = 0usize; - let rc = unsafe { sys::hl_signer_public_key(self.raw, buf.as_mut_ptr(), buf.len(), &mut len) }; - assert_eq!(rc, 0, "public key fits in 256 bytes"); - buf.truncate(len); + let buf = out_string(128, |out, cap, len| unsafe { sys::hl_signer_public_key(self.raw, out, cap, len) }).expect("public key renders"); String::from_utf8(buf).expect("key name was valid UTF-8") } /// A detached signature over `msg`, as `:`. pub fn sign(&self, msg: &[u8]) -> String { - // Room for the key name (as in public_key) plus 88 base64 characters. - let mut buf = vec![0u8; 512]; - let mut len = 0usize; - let rc = unsafe { sys::hl_signer_sign(self.raw, msg.as_ptr(), msg.len(), buf.as_mut_ptr(), buf.len(), &mut len) }; - assert_eq!(rc, 0, "signature fits in 512 bytes"); - buf.truncate(len); + let buf = out_string(160, |out, cap, len| unsafe { sys::hl_signer_sign(self.raw, msg.as_ptr(), msg.len(), out, cap, len) }) + .expect("signing succeeds"); String::from_utf8(buf).expect("key name was valid UTF-8") } } +/// Calls `f` with a buffer of `cap` bytes, and once more with the length it +/// asked for if that was too small: the key name has no fixed bound. +fn out_string(cap: usize, f: impl Fn(*mut u8, usize, &mut usize) -> c_int) -> Result, Error> { + const HL_E_BUFFER: c_int = -11; + let mut buf = vec![0u8; cap]; + let mut len = 0usize; + let mut rc = f(buf.as_mut_ptr(), buf.len(), &mut len); + if rc == HL_E_BUFFER { + buf.resize(len, 0); + rc = f(buf.as_mut_ptr(), buf.len(), &mut len); + } + check(rc)?; + buf.truncate(len); + Ok(buf) +} + impl Drop for Signer { fn drop(&mut self) { unsafe { sys::hl_signer_free(self.raw) } @@ -525,6 +536,14 @@ mod tests { assert!(Signer::new("garbage").is_none()); } + #[test] + fn long_key_names_fit() { + let name = "k".repeat(600); + let signer = Signer::new(&Signer::generate(&name).unwrap()).unwrap(); + assert_eq!(signer.public_key().len(), name.len() + 1 + 44); + assert_eq!(signer.sign(b"msg").len(), name.len() + 1 + 88); + } + #[test] fn dump_matches_compressor_and_verifier() { let dir = TempDir::new(); @@ -536,7 +555,7 @@ mod tests { std::os::unix::fs::symlink("bin/hello", root.join("link")).unwrap(); let mut compressed = Vec::new(); - let digest = dump_nar(&root, &DumpOptions { level: 3, threads: 2, size_hint: 0 }, |c| { + let digest = dump_nar(&root, &DumpOptions { level: 3, threads: 2, ..Default::default() }, |c| { compressed.extend_from_slice(c); true }) @@ -573,7 +592,7 @@ mod tests { let hashes = String::from_utf8(hashes.stdout).unwrap(); for (path, expected) in paths.iter().zip(hashes.lines()) { let digest = - dump_nar(Path::new(path), &DumpOptions { level: 1, threads: 0, size_hint: 0 }, |_| true).unwrap_or_else(|e| panic!("{path}: {e}")); + dump_nar(Path::new(path), &DumpOptions { level: 1, ..Default::default() }, |_| true).unwrap_or_else(|e| panic!("{path}: {e}")); assert_eq!(format!("sha256:{}", nix32_encode(&digest.nar_hash)), expected, "{path}"); } } diff --git a/pkg/nix-archive/src/sink.zig b/pkg/nix-archive/src/sink.zig index b753868..9c7d6b4 100644 --- a/pkg/nix-archive/src/sink.zig +++ b/pkg/nix-archive/src/sink.zig @@ -23,8 +23,15 @@ pub const Options = struct { level: c_int = 3, /// zstd worker threads; 0 compresses on the calling thread. threads: c_int = 0, - /// Expected NAR size, 0 when unknown. Lets zstd size its tables. - size_hint: u64 = 0, + /// The exact NAR size, 0 when unknown. It is pledged to zstd, which then + /// sizes its window to the NAR and records the size in the frame, so a + /// decoder allocates no more than the NAR needs. A stream of another + /// length fails. + nar_size: u64 = 0, + /// 0 keeps the level's window. Otherwise long-distance matching with a + /// window of 2^window_log bytes (at most 27, the largest a stock Nix + /// decoder accepts), which finds repeats across a large NAR. + window_log: c_int = 0, }; const in_capacity = 1 << 20; @@ -53,17 +60,23 @@ pub const Sink = struct { errdefer allocator.free(in_buf); var cctx: ?*zstd.CCtx = null; + // At function scope so a failed out_buf allocation frees it too. + errdefer _ = zstd.ZSTD_freeCCtx(cctx); var out_len: usize = 0; if (options.level != 0) { cctx = zstd.ZSTD_createCCtx() orelse return error.OutOfMemory; - errdefer _ = zstd.ZSTD_freeCCtx(cctx); if (zstd.isError(zstd.ZSTD_CCtx_setParameter(cctx.?, zstd.c_compressionLevel, options.level))) return error.Zstd; - if (options.threads > 0 and (options.size_hint == 0 or options.size_hint >= multithread_threshold)) { + if (options.threads > 0 and (options.nar_size == 0 or options.nar_size >= multithread_threshold)) { // Fails harmlessly on a single-threaded libzstd build. _ = zstd.ZSTD_CCtx_setParameter(cctx.?, zstd.c_nbWorkers, options.threads); } - if (options.size_hint > 0 and options.size_hint <= std.math.maxInt(c_int)) { - _ = zstd.ZSTD_CCtx_setParameter(cctx.?, zstd.c_srcSizeHint, @intCast(options.size_hint)); + if (options.window_log != 0) { + if (options.window_log < 10 or options.window_log > 27) return error.Zstd; + if (zstd.isError(zstd.ZSTD_CCtx_setParameter(cctx.?, zstd.c_enableLongDistanceMatching, 1))) return error.Zstd; + if (zstd.isError(zstd.ZSTD_CCtx_setParameter(cctx.?, zstd.c_windowLog, options.window_log))) return error.Zstd; + } + if (options.nar_size > 0) { + if (zstd.isError(zstd.ZSTD_CCtx_setPledgedSrcSize(cctx.?, options.nar_size))) return error.Zstd; } out_len = zstd.ZSTD_CStreamOutSize(); } diff --git a/pkg/nix-archive/src/tests.zig b/pkg/nix-archive/src/tests.zig index 0f6040a..ec35a46 100644 --- a/pkg/nix-archive/src/tests.zig +++ b/pkg/nix-archive/src/tests.zig @@ -1,5 +1,6 @@ const std = @import("std"); const Sink = @import("sink.zig").Sink; +const Options = @import("sink.zig").Options; const Verifier = @import("verify.zig").Verifier; const Collect = struct { @@ -21,13 +22,20 @@ fn fakeNar(a: std.mem.Allocator, body_len: usize) ![]u8 { } fn roundTrip(level: c_int, threads: c_int, body_len: usize) !void { + try roundTripWith(.{ .level = level, .threads = threads }, body_len, false); +} + +fn roundTripWith(options: Options, body_len: usize, pledge: bool) !void { + const level = options.level; const a = std.testing.allocator; const nar = try fakeNar(a, body_len); defer a.free(nar); + var opts = options; + if (pledge) opts.nar_size = nar.len; var out: Collect = .{}; defer out.list.deinit(a); - var sink = try Sink.init(a, .{ .level = level, .threads = threads }, Collect.write, &out); + var sink = try Sink.init(a, opts, Collect.write, &out); defer sink.deinit(a); // Uneven writes exercise the buffering paths. var off: usize = 0; @@ -65,6 +73,27 @@ test "compress then verify round-trips" { try roundTrip(0, 0, 5000); } +test "long-distance matching with an exact pledged size round-trips" { + try roundTripWith(.{ .level = 3, .window_log = 27 }, 3 << 20, true); + try roundTripWith(.{ .level = 9, .threads = 4, .window_log = 27 }, 9 << 20, true); + try roundTripWith(.{ .level = 3 }, 1000, true); +} + +test "a pledged size the stream does not match fails" { + const a = std.testing.allocator; + var out: Collect = .{}; + defer out.list.deinit(a); + var sink = try Sink.init(a, .{ .nar_size = 10 }, Collect.write, &out); + defer sink.deinit(a); + try sink.write("only five"); + try std.testing.expectError(error.Zstd, sink.finish()); +} + +test "an out-of-range window is refused" { + var out: Collect = .{}; + try std.testing.expectError(error.Zstd, Sink.init(std.testing.allocator, .{ .window_log = 28 }, Collect.write, &out)); +} + test "verifier rejects truncated and non-NAR input" { const a = std.testing.allocator; const nar = try fakeNar(a, 100_000); diff --git a/pkg/nix-archive/src/zstd.zig b/pkg/nix-archive/src/zstd.zig index cdf9e73..f9de889 100644 --- a/pkg/nix-archive/src/zstd.zig +++ b/pkg/nix-archive/src/zstd.zig @@ -18,8 +18,8 @@ pub const OutBuffer = extern struct { pub const c_compressionLevel: c_int = 100; pub const c_checksumFlag: c_int = 201; pub const c_nbWorkers: c_int = 400; -/// ZSTD_c_srcSizeHint (experimental, stable since 1.4.x). -pub const c_srcSizeHint: c_int = 1004; +pub const c_windowLog: c_int = 101; +pub const c_enableLongDistanceMatching: c_int = 160; pub const e_continue: c_int = 0; pub const e_end: c_int = 2; @@ -28,6 +28,7 @@ pub extern "c" fn ZSTD_createCCtx() ?*CCtx; pub extern "c" fn ZSTD_freeCCtx(cctx: ?*CCtx) usize; pub extern "c" fn ZSTD_CCtx_setParameter(cctx: *CCtx, param: c_int, value: c_int) usize; pub extern "c" fn ZSTD_compressStream2(cctx: *CCtx, output: *OutBuffer, input: *InBuffer, end_op: c_int) usize; +pub extern "c" fn ZSTD_CCtx_setPledgedSrcSize(cctx: *CCtx, pledged_src_size: c_ulonglong) usize; pub extern "c" fn ZSTD_CStreamOutSize() usize; pub extern "c" fn ZSTD_createDCtx() ?*DCtx; From c2c6f80a027e00c06b53289fbbaacf80762d81b3 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 40/56] server: stop leaking private NARs and fix review findings The serious one: a push token for any cache could publish a path in its own public cache over a NAR that only another, private cache held, given just the NAR hash, then fetch it without auth. nars/known even confirmed the hash existed. A new blob_caches table records which caches a NAR was uploaded to or published in, and a cache may only reuse a NAR it holds or that a public cache serves; anything else must be uploaded again. The narSize mismatch error no longer reveals the real size. Also fixed: - New paths had accessed_at 0, so auto-GC evicted them first. - Publish and evict updated the in-memory index after releasing the writer lock, so the index could disagree with the database; both now update it under the lock. A narinfo read before an eviction or re-push could be cached after it; an epoch check prevents that. - Publish re-checked a blob by rowid, which SQLite reuses; it now checks the file hash too, and a blob collected meanwhile becomes a nar_required the client recovers from instead of a dead-end 409. - GC swept the file of a chunked upload idle past an hour while its session lived on; idle sessions now expire with their files, GC skips live ones, sessions are capped per token, and offsets are read without blocking on a chunk in progress. - Upload I/O failures were reported as 400; they are 500 now. A blob file whose row failed to insert is removed, and the directory is synced after the rename. - Quarantine moved the file outside the writer lock, racing a re-upload of the same NAR. - Private caches sent cacheable 404s; every private response is now private, no-cache. narinfo and build traces lost immutable, since eviction removes them. - A build trace whose output was evicted could never be replaced; trace signing ran under the index lock. - Pins may only name published paths, at most 10000 per cache. - Two backups in one second failed, and a failed one could count towards --backup-keep. - PathKey hashing used client-chosen bytes directly; it is now keyed per process, so pushes cannot flood the index with collisions. - The audit log trusted the leftmost X-Forwarded-For entry. The server advertises compression defaults (--compression-level 3, --compression-window-log 27) in GET /_api/v2/caches/, which push tokens may now read. scripts/e2e.sh covers the leak, uncompressed uploads and the defaults. --- crates/helios-server/src/audit.rs | 4 +- crates/helios-server/src/chunked.rs | 155 ++++++++++++++--------- crates/helios-server/src/config.rs | 23 ++++ crates/helios-server/src/db.rs | 10 ++ crates/helios-server/src/gc.rs | 8 ++ crates/helios-server/src/internal.rs | 48 ++++--- crates/helios-server/src/pins.rs | 21 +++- crates/helios-server/src/push.rs | 181 +++++++++++++++++---------- crates/helios-server/src/read.rs | 84 +++++++++---- crates/helios-server/src/state.rs | 32 ++++- crates/helios-server/src/traces.rs | 41 +++--- scripts/e2e.sh | 26 +++- 12 files changed, 448 insertions(+), 185 deletions(-) diff --git a/crates/helios-server/src/audit.rs b/crates/helios-server/src/audit.rs index 0ee70ab..2c1fef8 100644 --- a/crates/helios-server/src/audit.rs +++ b/crates/helios-server/src/audit.rs @@ -33,8 +33,10 @@ impl FromRequestParts for Audit { type Rejection = std::convert::Infallible; async fn from_request_parts(parts: &mut Parts, st: &Shared) -> Result { + // The proxy appends the peer it saw; everything left of that is + // whatever the client claimed, so only the last entry is trusted. let forwarded = - st.cfg.trust_proxy.then(|| parts.headers.get("x-forwarded-for")?.to_str().ok()?.split(',').next()?.trim().parse().ok()).flatten(); + st.cfg.trust_proxy.then(|| parts.headers.get("x-forwarded-for")?.to_str().ok()?.rsplit(',').next()?.trim().parse().ok()).flatten(); let peer = parts.extensions.get::>().map(|c| c.0.ip()); Ok(Self { ip: forwarded.or(peer) }) } diff --git a/crates/helios-server/src/chunked.rs b/crates/helios-server/src/chunked.rs index a58c316..e4dd7ad 100644 --- a/crates/helios-server/src/chunked.rs +++ b/crates/helios-server/src/chunked.rs @@ -13,13 +13,13 @@ //! //! Chunks are verified as they arrive, as in a single-request upload. //! Sessions live in memory: after a server restart the client starts the -//! NAR again. An idle session is dropped after an hour, and GC removes its -//! file. +//! NAR again. A session idle for an hour is dropped with its file; until +//! then, GC leaves the file alone. use std::io::Write; use std::path::PathBuf; +use std::sync::atomic::{AtomicI64, AtomicU64, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; -use std::time::{Duration, Instant}; use axum::Json; use axum::extract::{Path, State}; @@ -35,21 +35,36 @@ use crate::push::{UploadResp, cache_for, store_blob}; use crate::state::Shared; const MAX_SESSIONS: usize = 1024; -const IDLE_TIMEOUT: Duration = Duration::from_secs(3600); +/// Per token, so one client cannot take every slot. +const MAX_SESSIONS_PER_ACTOR: usize = 32; +const IDLE_TIMEOUT_SECS: i64 = 3600; +const FILE_PREFIX: &str = "upload-"; pub struct Session { cache: Box, actor: String, compression: Compression, tmp: PathBuf, + /// Bytes appended so far; read without waiting for a chunk in progress. + offset: AtomicU64, + /// Unix time of the last request. + touched: AtomicI64, + io: Mutex, +} + +struct Io { file: std::fs::File, /// `None` once the stream is found invalid or the upload is finished. verifier: Option, - offset: u64, - touched: Instant, } -pub type Sessions = Mutex, Arc>>>; +impl Session { + fn idle(&self, now: i64) -> bool { + now - self.touched.load(Ordering::Relaxed) > IDLE_TIMEOUT_SECS + } +} + +pub type Sessions = Mutex, Arc>>; fn lock(m: &Mutex) -> std::sync::MutexGuard<'_, T> { m.lock().unwrap_or_else(PoisonError::into_inner) @@ -63,21 +78,45 @@ fn not_found() -> ApiError { ApiError::new(StatusCode::NOT_FOUND, "upload not found; it expired or the server restarted, so start the NAR again") } -/// The session `id`, if it belongs to `cache` and to whoever is asking. -fn session(st: &Shared, cache: &str, id: &str, who: &Identity) -> ApiResult>> { +/// Drops idle sessions and their files. Returns how many. +pub fn expire(st: &Shared) -> usize { + let now = crate::db::now(); + let expired: Vec> = { + let mut sessions = lock(&st.uploads); + let ids: Vec> = sessions.iter().filter(|(_, s)| s.idle(now)).map(|(id, _)| id.clone()).collect(); + ids.iter().filter_map(|id| sessions.remove(id)).collect() + }; + for s in &expired { + let _ = std::fs::remove_file(&s.tmp); + } + expired.len() +} + +/// Whether a file in the tmp directory belongs to a live upload, which GC +/// must leave alone however old its mtime. +pub fn is_live(st: &Shared, file_name: &str) -> bool { + file_name.strip_prefix(FILE_PREFIX).is_some_and(|id| lock(&st.uploads).contains_key(id)) +} + +/// The session `id`, if it belongs to `cache` and to whoever is asking and +/// has not gone idle. +fn session(st: &Shared, cache: &str, id: &str, who: &Identity) -> ApiResult> { let s = lock(&st.uploads).get(id).cloned().ok_or_else(not_found)?; - { - let g = lock(&s); - if &*g.cache != cache || g.actor != who.actor() { - return Err(not_found()); - } + if &*s.cache != cache || s.actor != who.actor() { + return Err(not_found()); + } + let now = crate::db::now(); + if s.idle(now) { + drop_session(st, id); + return Err(not_found()); } + s.touched.store(now, Ordering::Relaxed); Ok(s) } fn drop_session(st: &Shared, id: &str) { if let Some(s) = lock(&st.uploads).remove(id) { - let _ = std::fs::remove_file(&lock(&s).tmp); + let _ = std::fs::remove_file(&s.tmp); } } @@ -91,35 +130,39 @@ pub async fn create( let who = st.authorize(&headers, &name, Perm::Push)?; let compression = Compression::parse(query(&uri, "compression").unwrap_or("zstd")).ok_or_else(|| ApiError::bad_request("compression must be zstd or none"))?; + expire(&st); + { + let sessions = lock(&st.uploads); + if sessions.len() >= MAX_SESSIONS { + return Err(ApiError::new(StatusCode::SERVICE_UNAVAILABLE, "too many uploads in progress")); + } + if sessions.values().filter(|s| s.actor == who.actor()).count() >= MAX_SESSIONS_PER_ACTOR { + return Err(ApiError::new( + StatusCode::TOO_MANY_REQUESTS, + format!("at most {MAX_SESSIONS_PER_ACTOR} uploads in progress per token; complete or abandon one"), + )); + } + } let id = helios_core::uuid_v4(); - let tmp = st.tmp_dir().join(format!("upload-{id}")); - let file = std::fs::File::create_new(&tmp).map_err(ApiError::internal)?; + let tmp = st.tmp_dir().join(format!("{FILE_PREFIX}{id}")); let verifier = Verifier::new(compression).map_err(ApiError::internal)?; + let file = tokio::task::spawn_blocking({ + let tmp = tmp.clone(); + move || std::fs::File::create_new(tmp) + }) + .await? + .map_err(ApiError::internal)?; let new = Session { cache: name.into(), actor: who.actor().to_owned(), compression, tmp, - file, - verifier: Some(verifier), - offset: 0, - touched: Instant::now(), + offset: AtomicU64::new(0), + touched: AtomicI64::new(crate::db::now()), + io: Mutex::new(Io { file, verifier: Some(verifier) }), }; - - let mut sessions = lock(&st.uploads); - let expired: Vec> = - sessions.iter().filter(|(_, s)| s.try_lock().is_ok_and(|g| g.touched.elapsed() > IDLE_TIMEOUT)).map(|(id, _)| id.clone()).collect(); - for old in expired { - if let Some(s) = sessions.remove(&old) { - let _ = std::fs::remove_file(&lock(&s).tmp); - } - } - if sessions.len() >= MAX_SESSIONS { - let _ = std::fs::remove_file(&new.tmp); - return Err(ApiError::new(StatusCode::SERVICE_UNAVAILABLE, "too many uploads in progress")); - } - sessions.insert(id.as_str().into(), Arc::new(Mutex::new(new))); + lock(&st.uploads).insert(id.as_str().into(), Arc::new(new)); Ok((StatusCode::CREATED, Json(json!({ "id": id, "offset": 0 })))) } @@ -135,28 +178,29 @@ pub async fn append( let s = session(&st, &name, &id, &who)?; let max = st.cfg.max_upload_bytes; let result = tokio::task::spawn_blocking(move || -> ApiResult { - let mut g = lock(&s); - let g = &mut *g; - g.touched = Instant::now(); - if at != g.offset { - return Err(ApiError::new(StatusCode::CONFLICT, "offset mismatch").with(json!({ "offset": g.offset }))); + let mut io = lock(&s.io); + let io = &mut *io; + let offset = s.offset.load(Ordering::Acquire); + if at != offset { + return Err(ApiError::new(StatusCode::CONFLICT, "offset mismatch").with(json!({ "offset": offset }))); } - if g.offset + body.len() as u64 > max { + if offset + body.len() as u64 > max { return Err(ApiError::new(StatusCode::PAYLOAD_TOO_LARGE, "upload too large")); } - let verifier = g.verifier.as_mut().ok_or_else(|| ApiError::bad_request("upload is no longer accepting data"))?; + let verifier = io.verifier.as_mut().ok_or_else(|| ApiError::new(StatusCode::CONFLICT, "upload is completing or was abandoned"))?; if let Err(e) = verifier.update(&body) { - g.verifier = None; + io.verifier = None; return Err(ApiError::bad_request(format!("invalid NAR stream: {e}"))); } // A short write leaves the file ahead of the offset; later chunks // would then land in the wrong place, so give up on the upload. - if let Err(e) = g.file.write_all(&body) { - g.verifier = None; + if let Err(e) = io.file.write_all(&body) { + io.verifier = None; return Err(ApiError::internal(e)); } - g.offset += body.len() as u64; - Ok(g.offset) + let offset = offset + body.len() as u64; + s.offset.store(offset, Ordering::Release); + Ok(offset) }) .await?; match result { @@ -173,8 +217,7 @@ pub async fn append( pub async fn status(State(st): State, Path((name, id)): Path<(String, String)>, headers: HeaderMap) -> ApiResult> { let who = st.authorize(&headers, &name, Perm::Push)?; let s = session(&st, &name, &id, &who)?; - let offset = lock(&s).offset; - Ok(Json(json!({ "offset": offset }))) + Ok(Json(json!({ "offset": s.offset.load(Ordering::Acquire) }))) } pub async fn complete( @@ -183,18 +226,19 @@ pub async fn complete( audit: Audit, headers: HeaderMap, ) -> ApiResult<(StatusCode, Json)> { + let cache_id = cache_for(&st, &name)?.id; let who = st.authorize(&headers, &name, Perm::Push)?; let s = session(&st, &name, &id, &who)?; // Taken out of the map first, so a concurrent chunk cannot extend it. lock(&st.uploads).remove(id.as_str()); - let (tmp, compression, digest) = tokio::task::spawn_blocking(move || { - let mut g = lock(&s); - let digest = match g.verifier.take() { + let (tmp, compression) = (s.tmp.clone(), s.compression); + let digest = tokio::task::spawn_blocking(move || { + let mut io = lock(&s.io); + match io.verifier.take() { Some(v) => v.finish().map_err(|e| ApiError::bad_request(format!("invalid NAR stream: {e}"))), None => Err(ApiError::bad_request("upload is no longer accepting data")), } - .and_then(|d| g.file.sync_data().map(|()| d).map_err(ApiError::internal)); - (g.tmp.clone(), g.compression, digest) + .and_then(|d| io.file.sync_data().map(|()| d).map_err(ApiError::internal)) }) .await?; let digest = match digest { @@ -205,7 +249,7 @@ pub async fn complete( return Err(err); } }; - let resp = store_blob(&st, tmp, digest, compression).await?; + let resp = store_blob(&st, cache_id, tmp, digest, compression).await?; audit.log( &st, &who, @@ -216,7 +260,6 @@ pub async fn complete( ); Ok((StatusCode::CREATED, Json(resp))) } - pub async fn abort(State(st): State, Path((name, id)): Path<(String, String)>, headers: HeaderMap) -> ApiResult { let who = st.authorize(&headers, &name, Perm::Push)?; session(&st, &name, &id, &who)?; diff --git a/crates/helios-server/src/config.rs b/crates/helios-server/src/config.rs index aad1f60..d095f8c 100644 --- a/crates/helios-server/src/config.rs +++ b/crates/helios-server/src/config.rs @@ -73,6 +73,18 @@ pub struct Args { #[arg(long, env = "HELIOS_CACHES", value_delimiter = ',', value_parser = parse_declared_cache)] pub caches: Vec, + /// zstd level clients compress with unless told otherwise (0 uploads + /// uncompressed). Level 3 with a 2^27 window gives NARs about 9% smaller + /// than plain level 2 while compressing faster than most uplinks. + #[arg(long, env = "HELIOS_COMPRESSION_LEVEL", default_value_t = 3, value_parser = clap::value_parser!(i32).range(0..=19))] + pub compression_level: i32, + + /// zstd window for long-distance matching clients use unless told + /// otherwise, as a power of two (10-27), or 0 for the level's own. + /// 27 (128 MiB) is the largest a stock Nix decoder accepts. + #[arg(long, env = "HELIOS_COMPRESSION_WINDOW_LOG", default_value_t = 27, value_parser = parse_window_log)] + pub compression_window_log: i32, + /// Print the public key for `trusted-public-keys` and exit. #[arg(long)] pub print_public_key: bool, @@ -84,6 +96,13 @@ pub struct DeclaredCache { pub public: bool, } +fn parse_window_log(s: &str) -> Result { + match s.parse::() { + Ok(n) if n == 0 || (10..=27).contains(&n) => Ok(n), + _ => Err("expected 0, or 10-27".into()), + } +} + fn parse_declared_cache(s: &str) -> Result { let (name, public) = match s.trim().split_once(':') { None => (s.trim(), true), @@ -122,6 +141,8 @@ pub struct Config { pub upload_grace: Duration, pub gc_interval: Duration, pub audit_retention: Duration, + pub compression_level: i32, + pub compression_window_log: i32, } pub fn read_secret(path: &Option) -> anyhow::Result>> { @@ -145,6 +166,8 @@ impl Config { upload_grace: Duration::from_secs(args.upload_grace_seconds), gc_interval: Duration::from_secs(args.gc_interval_hours.max(1) * 3600), audit_retention: Duration::from_secs(args.audit_retention_days * 86400), + compression_level: args.compression_level, + compression_window_log: args.compression_window_log, }) } } diff --git a/crates/helios-server/src/db.rs b/crates/helios-server/src/db.rs index 1585d3e..a9b9514 100644 --- a/crates/helios-server/src/db.rs +++ b/crates/helios-server/src/db.rs @@ -87,6 +87,16 @@ CREATE TABLE build_traces ( created_at INTEGER NOT NULL, PRIMARY KEY (cache_id, id) ); +"#, + // Caches each blob was uploaded to or published in: a cache may only + // reuse a blob it holds, or one any public cache serves. + r#" +CREATE TABLE blob_caches ( + blob_id INTEGER NOT NULL REFERENCES blobs (id), + cache_id INTEGER NOT NULL REFERENCES caches (id), + PRIMARY KEY (blob_id, cache_id) +) WITHOUT ROWID; +INSERT OR IGNORE INTO blob_caches (blob_id, cache_id) SELECT DISTINCT blob_id, cache_id FROM paths; "#, ]; diff --git a/crates/helios-server/src/gc.rs b/crates/helios-server/src/gc.rs index 3d5aef5..72a5abf 100644 --- a/crates/helios-server/src/gc.rs +++ b/crates/helios-server/src/gc.rs @@ -42,8 +42,14 @@ pub fn collect(st: &Shared) -> anyhow::Result { let mut stats = GcStats::default(); let now = crate::db::now(); + // Idle chunked uploads first: their files are then ordinary stale ones, + // and a live upload's file is skipped however old its mtime. + crate::chunked::expire(st); if let Ok(entries) = std::fs::read_dir(st.tmp_dir()) { for entry in entries.flatten() { + if crate::chunked::is_live(st, &entry.file_name().to_string_lossy()) { + continue; + } let stale = entry .metadata() .and_then(|m| m.modified()) @@ -85,7 +91,9 @@ pub fn collect_blobs(st: &Shared) -> anyhow::Result<(usize, u64)> { }; { let mut del = tx.prepare_cached("DELETE FROM blobs WHERE id = ?1")?; + let mut unlink = tx.prepare_cached("DELETE FROM blob_caches WHERE blob_id = ?1")?; for (id, ..) in &victims { + unlink.execute([id])?; del.execute([id])?; } } diff --git a/crates/helios-server/src/internal.rs b/crates/helios-server/src/internal.rs index 9650d07..9df63a1 100644 --- a/crates/helios-server/src/internal.rs +++ b/crates/helios-server/src/internal.rs @@ -143,6 +143,7 @@ struct EvictReq { fn forget(st: &Shared, keys: &[PathKey]) { let mut index = st.index.wr(); + st.narinfo_epoch.fetch_add(1, std::sync::atomic::Ordering::Release); for k in keys { index.remove(k); st.narinfo.remove(k); @@ -171,11 +172,12 @@ async fn evict(State(st): State, Json(req): Json) -> ApiResult } } tx.commit()?; + // Under the writer lock, so a publish cannot interleave, and + // before the files go. + forget(&st, &keys); Ok(n) }); let evicted = evicted.map_err(internal)?; - // Stop serving before the files go. - forget(&st, &keys); let (blobs, bytes) = crate::gc::collect_blobs(&st).map_err(internal)?; tracing::info!(paths = evicted, blobs, bytes, "evicted"); Ok(Json(json!({ "evicted": evicted, "freedBlobs": blobs, "freedBytes": bytes }))) @@ -232,7 +234,7 @@ struct QuarantineReq { async fn quarantine(State(st): State, Json(req): Json) -> ApiResult> { let file_hash = helios_core::nix32_decode::<32>(&req.file_hash).ok_or_else(|| ApiError::bad_request("invalid file hash"))?; blocking(move || { - let found = st.db.write(|conn| -> rusqlite::Result, String)>> { + let found = st.db.write(|conn| -> rusqlite::Result>> { let tx = conn.transaction()?; let Some((id, compression)) = tx .query_row("SELECT id, compression FROM blobs WHERE file_hash = ?1", [&file_hash[..]], |r| { @@ -249,24 +251,27 @@ async fn quarantine(State(st): State, Json(req): Json) -> .collect() }; tx.execute("DELETE FROM paths WHERE blob_id = ?1", [id])?; + tx.execute("DELETE FROM blob_caches WHERE blob_id = ?1", [id])?; tx.execute("DELETE FROM blobs WHERE id = ?1", [id])?; tx.commit()?; - Ok(Some((keys, compression))) + forget(&st, &keys); + // Still under the writer lock: a re-upload of the same NAR must + // not have its good file moved aside by this. + if let Some(compression) = Compression::parse(&compression) { + let src = st.nar_path(&file_hash, compression); + let dir = st.cfg.data_dir.join("quarantine"); + let moved = std::fs::create_dir_all(&dir).and_then(|()| std::fs::rename(&src, dir.join(src.file_name().unwrap_or_default()))); + if let Err(e) = moved + && e.kind() != std::io::ErrorKind::NotFound + { + tracing::warn!(error = %e, "moving quarantined blob"); + } + } + Ok(Some(keys)) }); - let Some((keys, compression)) = found.map_err(internal)? else { + let Some(keys) = found.map_err(internal)? else { return Err(ApiError::new(StatusCode::NOT_FOUND, "blob not found")); }; - forget(&st, &keys); - if let Some(compression) = Compression::parse(&compression) { - let src = st.nar_path(&file_hash, compression); - let dir = st.cfg.data_dir.join("quarantine"); - let moved = std::fs::create_dir_all(&dir).and_then(|()| std::fs::rename(&src, dir.join(src.file_name().unwrap_or_default()))); - if let Err(e) = moved - && e.kind() != std::io::ErrorKind::NotFound - { - tracing::warn!(error = %e, "moving quarantined blob"); - } - } tracing::warn!(blob = %req.file_hash, reason = %req.reason, unpublished = keys.len(), "quarantined blob"); Ok(Json(json!({ "pathsRemoved": keys.len() }))) }) @@ -314,9 +319,16 @@ async fn backup(State(st): State, Json(req): Json) -> ApiResu blocking(move || { let dir = st.cfg.data_dir.join("backups"); std::fs::create_dir_all(&dir).map_err(internal)?; - let dest = dir.join(format!("helios-{}.db", crate::db::now())); + // Milliseconds: VACUUM INTO refuses an existing file. Names still sort + // by age, after older backups named in seconds. + let millis = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.as_millis()); + let dest = dir.join(format!("helios-{millis}.db")); let dest_str = dest.to_str().ok_or_else(|| internal("backup path is not UTF-8"))?.to_owned(); - st.db.read(|conn| conn.execute("VACUUM INTO ?1", [&dest_str])).map_err(internal)?; + if let Err(e) = st.db.read(|conn| conn.execute("VACUUM INTO ?1", [&dest_str])) { + // A partial file must not count towards --backup-keep. + let _ = std::fs::remove_file(&dest); + return Err(internal(e)); + } let bytes = std::fs::metadata(&dest).map(|m| m.len()).unwrap_or(0); let mut existing: Vec = std::fs::read_dir(&dir) diff --git a/crates/helios-server/src/pins.rs b/crates/helios-server/src/pins.rs index 4a2ff42..76eb76a 100644 --- a/crates/helios-server/src/pins.rs +++ b/crates/helios-server/src/pins.rs @@ -23,9 +23,11 @@ use crate::audit::Audit; use crate::auth::Perm; use crate::error::{ApiError, ApiResult}; use crate::push::cache_for; -use crate::state::{PathKey, Shared}; +use crate::state::{Locked, PathKey, Shared}; const MAX_PINS_PER_REQUEST: usize = 1000; +/// Each eviction walks every pinned closure under the writer lock. +const MAX_PINS_PER_CACHE: i64 = 10_000; /// `/nix/store/-` or its basename, as (hash, basename). fn parse_path(s: &str) -> Option<([u8; 20], &str)> { @@ -105,11 +107,26 @@ pub async fn add( .iter() .map(|p| parse_path(p).map(|(h, base)| (h, base.to_owned())).ok_or_else(|| ApiError::bad_request(format!("invalid store path: {p}")))) .collect::>()?; + // Only published paths: a pin stops at what the cache lacks, so pinning + // ahead of the push would protect less than it seems to. + let missing: Vec = { + let index = st.index.rd(); + parsed.iter().filter(|(hash, _)| !index.contains(&PathKey { cache: cache.id, hash: *hash })).map(|(_, b)| format!("/nix/store/{b}")).collect() + }; + if !missing.is_empty() { + return Err( + ApiError::new(StatusCode::CONFLICT, "pin paths after pushing them; these are not in the cache").with(json!({ "missing": missing })) + ); + } let actor = who.actor().to_owned(); let st2 = st.clone(); let added = tokio::task::spawn_blocking(move || { - st2.db.write(|conn| -> rusqlite::Result { + st2.db.write(|conn| -> ApiResult { let tx = conn.transaction()?; + let pinned: i64 = tx.query_row("SELECT count(*) FROM pins WHERE cache_id = ?1", [cache.id], |r| r.get(0))?; + if pinned + parsed.len() as i64 > MAX_PINS_PER_CACHE { + return Err(ApiError::bad_request(format!("a cache holds at most {MAX_PINS_PER_CACHE} pins; it has {pinned}"))); + } let mut n = 0; { let mut insert = tx.prepare_cached( diff --git a/crates/helios-server/src/push.rs b/crates/helios-server/src/push.rs index 530291a..7f20930 100644 --- a/crates/helios-server/src/push.rs +++ b/crates/helios-server/src/push.rs @@ -33,6 +33,18 @@ const MAX_MISSING_BATCH: usize = 100_000; const MAX_PUBLISH_BATCH: usize = 5_000; const STORE_DIR: &str = "/nix/store/"; +/// SQL condition on blob `b`: the cache bound to parameter `?n` may reuse +/// it without uploading. That holds for a blob uploaded to or published in +/// that cache, and for one a public cache serves, which anyone can fetch. +/// Anything else would let a push token for one cache publish, and so +/// read, another cache's private NARs by their hash. +fn reusable(n: u8) -> String { + format!( + "(EXISTS (SELECT 1 FROM blob_caches bc WHERE bc.blob_id = b.id AND bc.cache_id = ?{n}) + OR EXISTS (SELECT 1 FROM paths p JOIN caches c ON c.id = p.cache_id WHERE p.blob_id = b.id AND c.is_public = 1))" + ) +} + pub(crate) fn cache_for(st: &Shared, name: &str) -> ApiResult { st.cache(name).ok_or_else(|| ApiError::new(StatusCode::NOT_FOUND, "cache not found")) } @@ -81,7 +93,7 @@ pub async fn known( headers: HeaderMap, Json(req): Json, ) -> ApiResult> { - cache_for(&st, &name)?; + let cache = cache_for(&st, &name)?; st.authorize(&headers, &name, Perm::Push)?; if req.nar_hashes.len() > MAX_MISSING_BATCH { return Err(ApiError::bad_request(format!("at most {MAX_MISSING_BATCH} hashes per request"))); @@ -91,8 +103,8 @@ pub async fn known( let values: Vec = parsed.iter().flatten().map(|h| Value::Blob(h.to_vec())).collect(); let found = tokio::task::spawn_blocking(move || { db.read(|conn| { - let mut stmt = conn.prepare_cached("SELECT DISTINCT nar_hash FROM blobs WHERE nar_hash IN rarray(?1)")?; - let rows = stmt.query_map([Rc::new(values)], |r| r.get::<_, Vec>(0))?; + let mut stmt = conn.prepare_cached(&format!("SELECT DISTINCT nar_hash FROM blobs b WHERE nar_hash IN rarray(?1) AND {}", reusable(2)))?; + let rows = stmt.query_map(params![Rc::new(values), cache.id], |r| r.get::<_, Vec>(0))?; rows.collect::>>>() }) }) @@ -120,7 +132,7 @@ pub async fn upload( headers: HeaderMap, body: Body, ) -> ApiResult<(StatusCode, Json)> { - cache_for(&st, &name)?; + let cache = cache_for(&st, &name)?; let identity = st.authorize(&headers, &name, Perm::Push)?; let requested = uri.query().unwrap_or("").split('&').find_map(|kv| kv.strip_prefix("compression=")).unwrap_or("zstd"); let compression = Compression::parse(requested).ok_or_else(|| ApiError::bad_request("compression must be zstd or none"))?; @@ -129,17 +141,19 @@ pub async fn upload( let (tx, mut rx) = tokio::sync::mpsc::channel::(32); let worker_tmp = tmp.clone(); // Disk writes, hashing and decompression happen off the async runtime. - let worker = tokio::task::spawn_blocking(move || -> Result { - let file = std::fs::File::create_new(&worker_tmp).map_err(|e| e.to_string())?; + // A bad stream is the client's error (400); a failed write is ours (500). + let worker = tokio::task::spawn_blocking(move || -> ApiResult { + let invalid = |e: helios_core::Error| ApiError::bad_request(format!("invalid NAR stream: {e}")); + let file = std::fs::File::create_new(&worker_tmp).map_err(ApiError::internal)?; let mut out = std::io::BufWriter::with_capacity(1 << 20, file); - let mut verifier = Verifier::new(compression).map_err(|e| e.to_string())?; + let mut verifier = Verifier::new(compression).map_err(ApiError::internal)?; while let Some(chunk) = rx.blocking_recv() { - verifier.update(&chunk).map_err(|e| format!("invalid NAR stream: {e}"))?; - out.write_all(&chunk).map_err(|e| e.to_string())?; + verifier.update(&chunk).map_err(invalid)?; + out.write_all(&chunk).map_err(ApiError::internal)?; } - let file = out.into_inner().map_err(|e| e.to_string())?; - file.sync_data().map_err(|e| e.to_string())?; - verifier.finish().map_err(|e| format!("invalid NAR stream: {e}")) + let file = out.into_inner().map_err(|e| ApiError::internal(e.into_error()))?; + file.sync_data().map_err(ApiError::internal)?; + verifier.finish().map_err(invalid) }); let mut stream = body.into_data_stream(); @@ -171,49 +185,76 @@ pub async fn upload( audit.log(&st, &identity, "nar.upload", Some(&name), err.status, json!({ "error": err.message })); return Err(err); } - (None, Err(msg)) => { + (None, Err(err)) => { let _ = tokio::fs::remove_file(&tmp).await; - let err = ApiError::bad_request(msg); audit.log(&st, &identity, "nar.upload", Some(&name), err.status, json!({ "error": err.message })); return Err(err); } (None, Ok(d)) => d, }; - let resp = store_blob(&st, tmp, digest, compression).await?; + let resp = store_blob(&st, cache.id, tmp, digest, compression).await?; audit.log(&st, &identity, "nar.upload", Some(&name), StatusCode::CREATED, json!({ "fileHash": resp.file_hash, "size": resp.file_size })); Ok((StatusCode::CREATED, Json(resp))) } -/// Moves a verified upload at `tmp` into the blob store and records it. -pub(crate) async fn store_blob(st: &Shared, tmp: std::path::PathBuf, digest: helios_core::Digest, compression: Compression) -> ApiResult { +/// Moves a verified upload at `tmp` into the blob store and records it as +/// held by `cache_id`: uploading a NAR entitles that cache to reuse it. +pub(crate) async fn store_blob( + st: &Shared, + cache_id: u32, + tmp: std::path::PathBuf, + digest: helios_core::Digest, + compression: Compression, +) -> ApiResult { let dest = st.nar_path(&digest.file_hash, compression); let st2 = st.clone(); tokio::task::spawn_blocking(move || -> ApiResult<()> { - if let Some(parent) = dest.parent() { - std::fs::create_dir_all(parent).map_err(ApiError::internal)?; - } + let parent = dest.parent().expect("blob paths have a parent"); + std::fs::create_dir_all(parent).map_err(ApiError::internal)?; // Rename and row upsert happen under the writer lock so GC can never // delete the file between them. Renaming over an existing blob is // harmless: the content is identical by hash. st2.db.write(|conn| -> ApiResult<()> { - std::fs::rename(&tmp, &dest).map_err(ApiError::internal)?; - conn.prepare_cached( - "INSERT INTO blobs (file_hash, file_size, compression, nar_hash, nar_size, created_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6) - ON CONFLICT (file_hash) DO UPDATE SET created_at = excluded.created_at", - )? - .execute(params![ - &digest.file_hash[..], - digest.file_size, - compression.as_str(), - &digest.nar_hash[..], - digest.nar_size, - crate::db::now() - ])?; + match std::fs::rename(&tmp, &dest) { + Ok(()) => {} + // GC removed an upload left idle past the grace period. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Err(ApiError::new(StatusCode::NOT_FOUND, "the upload expired before it completed; upload the NAR again")); + } + Err(e) => return Err(ApiError::internal(e)), + } + // The rename must survive a crash before a row refers to it. + std::fs::File::open(parent).and_then(|d| d.sync_all()).map_err(ApiError::internal)?; + let recorded = (|| -> rusqlite::Result<()> { + let tx = conn.transaction()?; + tx.prepare_cached( + "INSERT INTO blobs (file_hash, file_size, compression, nar_hash, nar_size, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT (file_hash) DO UPDATE SET created_at = excluded.created_at", + )? + .execute(params![ + &digest.file_hash[..], + digest.file_size, + compression.as_str(), + &digest.nar_hash[..], + digest.nar_size, + crate::db::now() + ])?; + tx.prepare_cached("INSERT OR IGNORE INTO blob_caches (blob_id, cache_id) SELECT id, ?2 FROM blobs WHERE file_hash = ?1")? + .execute(params![&digest.file_hash[..], cache_id])?; + tx.commit() + })(); + if let Err(e) = recorded { + // A file without a row is never collected; remove it unless + // an earlier upload of the same NAR owns it. + if conn.query_row("SELECT 1 FROM blobs WHERE file_hash = ?1", [&digest.file_hash[..]], |_| Ok(())).is_err() { + let _ = std::fs::remove_file(&dest); + } + return Err(e.into()); + } Ok(()) - })?; - Ok(()) + }) }) .await??; @@ -288,6 +329,7 @@ struct Rendered { key: PathKey, store_path: String, blob_id: i64, + file_hash: [u8; 32], narinfo: Vec, } @@ -367,12 +409,6 @@ pub async fn publish( let published = rendered_keys.len(); st.counters.published_paths.add(published as u64); - { - let mut index = st.index.wr(); - for key in rendered_keys { - index.insert(key); - } - } audit.log(&st, &identity, "path.publish", Some(&name), StatusCode::CREATED, json!({ "published": published, "alreadyExisted": already })); Ok((StatusCode::CREATED, Json(json!({ "published": published, "alreadyExisted": already })))) } @@ -393,12 +429,15 @@ struct Blob { fn render_and_insert(st: &Shared, prepared: Vec) -> ApiResult { // Resolve blobs on a read connection, outside the writer lock. let blobs: Vec> = st.db.read(|conn| { - let mut by_nar = conn.prepare_cached( - "SELECT id, file_hash, file_size, compression, nar_size FROM blobs - WHERE nar_hash = ?1 ORDER BY compression = 'zstd' DESC, id LIMIT 1", - )?; - let mut by_file = - conn.prepare_cached("SELECT id, file_hash, file_size, compression, nar_size FROM blobs WHERE file_hash = ?1 AND nar_hash = ?2")?; + let mut by_nar = conn.prepare_cached(&format!( + "SELECT id, file_hash, file_size, compression, nar_size FROM blobs b + WHERE nar_hash = ?1 AND {} ORDER BY compression = 'zstd' DESC, id LIMIT 1", + reusable(2) + ))?; + let mut by_file = conn.prepare_cached(&format!( + "SELECT id, file_hash, file_size, compression, nar_size FROM blobs b WHERE file_hash = ?1 AND nar_hash = ?3 AND {}", + reusable(2) + ))?; let row = |r: &rusqlite::Row<'_>| { Ok(Blob { id: r.get(0)?, @@ -411,8 +450,8 @@ fn render_and_insert(st: &Shared, prepared: Vec) -> ApiResult prepared .iter() .map(|p| match &p.file_hash { - Some(fh) => by_file.query_row(params![&fh[..], &p.nar_hash[..]], row).optional(), - None => by_nar.query_row(params![&p.nar_hash[..]], row).optional(), + Some(fh) => by_file.query_row(params![&fh[..], p.key.cache, &p.nar_hash[..]], row).optional(), + None => by_nar.query_row(params![&p.nar_hash[..], p.key.cache], row).optional(), }) .collect() })?; @@ -434,38 +473,50 @@ fn render_and_insert(st: &Shared, prepared: Vec) -> ApiResult let rendered: Vec = rendered.into_iter().collect::>()?; let now = crate::db::now(); - let keys = st.db.write(|conn| { + st.db.write(|conn| -> ApiResult { let tx = conn.transaction()?; let mut keys = Vec::with_capacity(rendered.len()); + let mut gone = Vec::new(); { - let mut blob_exists = tx.prepare_cached("SELECT 1 FROM blobs WHERE id = ?1")?; + // By file hash too: SQLite reuses the rowid of a deleted blob. + let mut blob_exists = tx.prepare_cached("SELECT 1 FROM blobs WHERE id = ?1 AND file_hash = ?2")?; let mut insert = tx.prepare_cached( - "INSERT INTO paths (cache_id, hash, blob_id, store_path, narinfo, created_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6) ON CONFLICT (cache_id, hash) DO NOTHING", + "INSERT INTO paths (cache_id, hash, blob_id, store_path, narinfo, created_at, accessed_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?6) ON CONFLICT (cache_id, hash) DO NOTHING", )?; + let mut link = tx.prepare_cached("INSERT OR IGNORE INTO blob_caches (blob_id, cache_id) VALUES (?1, ?2)")?; for r in &rendered { // Re-checked under the writer lock: GC may have collected the blob. - if !blob_exists.exists([r.blob_id])? { - return Err(rusqlite::Error::QueryReturnedNoRows); + if !blob_exists.exists(params![r.blob_id, &r.file_hash[..]])? { + gone.push(r.store_path.clone()); + continue; } if insert.execute(params![r.key.cache, &r.key.hash[..], r.blob_id, r.store_path, r.narinfo, now])? > 0 { keys.push(r.key); } + link.execute(params![r.blob_id, r.key.cache])?; } } + if !gone.is_empty() { + // Nothing is committed; the client uploads those NARs again. + return Ok(Outcome::MissingNars(gone)); + } tx.commit()?; - Ok(keys) - }); - match keys { - Ok(keys) => Ok(Outcome::Published(keys)), - Err(rusqlite::Error::QueryReturnedNoRows) => Err(ApiError::new(StatusCode::CONFLICT, "blob was garbage collected; retry")), - Err(e) => Err(e.into()), - } + // Under the writer lock, so an eviction cannot run between the + // commit and the index update. + let mut index = st.index.wr(); + st.narinfo_epoch.fetch_add(1, std::sync::atomic::Ordering::Release); + for key in &keys { + index.insert(*key); + st.narinfo.remove(key); + } + Ok(Outcome::Published(keys)) + }) } fn render(st: &Shared, p: &Prepared, b: &Blob) -> ApiResult { if b.nar_size != p.nar_size { - return Err(ApiError::bad_request(format!("{}: narSize {} does not match uploaded NAR ({})", p.store_path, p.nar_size, b.nar_size))); + return Err(ApiError::bad_request(format!("{}: narSize {} does not match the uploaded NAR", p.store_path, p.nar_size))); } let narinfo = helios_core::render_narinfo( &NarinfoInput { @@ -482,5 +533,5 @@ fn render(st: &Shared, p: &Prepared, b: &Blob) -> ApiResult { st.signer.as_ref(), ) .map_err(|e| ApiError::bad_request(format!("{}: {e}", p.store_path)))?; - Ok(Rendered { key: p.key, store_path: p.store_path.clone(), blob_id: b.id, narinfo }) + Ok(Rendered { key: p.key, store_path: p.store_path.clone(), blob_id: b.id, file_hash: b.file_hash, narinfo }) } diff --git a/crates/helios-server/src/read.rs b/crates/helios-server/src/read.rs index bee2418..0140813 100644 --- a/crates/helios-server/src/read.rs +++ b/crates/helios-server/src/read.rs @@ -16,10 +16,32 @@ use crate::state::Locked; use crate::state::{PathKey, Shared}; const NIX_CACHE_INFO: &str = "StoreDir: /nix/store\nWantMassQuery: 1\nPriority: 40\n"; -const IMMUTABLE: HeaderValue = HeaderValue::from_static("public, max-age=31536000, immutable"); -const PRIVATE: HeaderValue = HeaderValue::from_static("private, max-age=3600"); -/// Most narinfo requests are misses (Nix asks every substituter about every -/// path). Let proxies cache them briefly; new paths still appear quickly. +/// Cache-Control for each kind of response. +struct Caching { + /// NARs are named by their content, so they never change. + nar: HeaderValue, + /// narinfo and build traces can be evicted, quarantined or re-pushed. + metadata: HeaderValue, + /// Most narinfo requests are misses (Nix asks every substituter about + /// every path); new paths still appear quickly. + miss: HeaderValue, +} + +const PUBLIC: Caching = Caching { + nar: HeaderValue::from_static("public, max-age=31536000, immutable"), + metadata: HeaderValue::from_static("public, max-age=3600"), + miss: HeaderValue::from_static("public, max-age=60"), +}; + +/// Nothing from a private cache may be kept by a shared proxy, and the +/// client's own copy is revalidated. +const PRIVATE: Caching = Caching { + nar: HeaderValue::from_static("private, no-cache"), + metadata: HeaderValue::from_static("private, no-cache"), + miss: HeaderValue::from_static("private, no-cache"), +}; + +/// Before the cache is known (unknown cache or URL). const NEGATIVE: HeaderValue = HeaderValue::from_static("public, max-age=60"); enum Route<'a> { @@ -73,30 +95,36 @@ async fn serve(st: Shared, method: Method, uri: Uri, headers: HeaderMap) -> ApiR return Ok(not_found(NEGATIVE)); }; st.authorize_read(&headers, name, cache)?; - let cache_control = if cache.public { IMMUTABLE } else { PRIVATE }; + let caching = if cache.public { &PUBLIC } else { &PRIVATE }; match route { - Route::CacheInfo => Ok(([(header::CONTENT_TYPE, "text/x-nix-cache-info")], NIX_CACHE_INFO).into_response()), + Route::CacheInfo => Ok(( + [(header::CONTENT_TYPE, HeaderValue::from_static("text/x-nix-cache-info")), (header::CACHE_CONTROL, caching.metadata.clone())], + NIX_CACHE_INFO, + ) + .into_response()), Route::BuildTrace(id) => Ok(match crate::traces::lookup(&st, cache.id, id) { Some(body) => { - ([(header::CONTENT_TYPE, HeaderValue::from_static("application/json")), (header::CACHE_CONTROL, cache_control)], body).into_response() + ([(header::CONTENT_TYPE, HeaderValue::from_static("application/json")), (header::CACHE_CONTROL, caching.metadata.clone())], body) + .into_response() } - None => not_found(if cache.public { NEGATIVE } else { PRIVATE }), + None => not_found(caching.miss.clone()), }), Route::Narinfo(hash) => { let Some(hash) = helios_core::nix32_decode::<20>(hash) else { - return Ok(not_found(NEGATIVE)); + return Ok(not_found(caching.miss.clone())); }; let key = PathKey { cache: cache.id, hash }; if !st.index.rd().contains(&key) { st.counters.narinfo_misses.inc(); - return Ok(not_found(if cache.public { NEGATIVE } else { PRIVATE })); + return Ok(not_found(caching.miss.clone())); } st.counters.narinfo_hits.inc(); st.access.touch(key); let body = match st.narinfo.get(&key) { Some(body) => body, None => { + let epoch = st.narinfo_epoch.load(std::sync::atomic::Ordering::Acquire); let db = st.db.clone(); let loaded = tokio::task::spawn_blocking(move || { db.read(|conn| { @@ -107,20 +135,25 @@ async fn serve(st: Shared, method: Method, uri: Uri, headers: HeaderMap) -> ApiR }) .await??; let Some(text) = loaded else { - return Ok(not_found(NEGATIVE)); + return Ok(not_found(caching.miss.clone())); }; let body = Bytes::from(text); - st.narinfo.insert(key, body.clone()); + // Checked under the index lock that publish and eviction + // hold while they bump the epoch. + let index = st.index.rd(); + if index.contains(&key) && st.narinfo_epoch.load(std::sync::atomic::Ordering::Acquire) == epoch { + st.narinfo.insert(key, body.clone()); + } body } }; - Ok(([(header::CONTENT_TYPE, HeaderValue::from_static("text/x-nix-narinfo")), (header::CACHE_CONTROL, cache_control)], body) + Ok(([(header::CONTENT_TYPE, HeaderValue::from_static("text/x-nix-narinfo")), (header::CACHE_CONTROL, caching.metadata.clone())], body) .into_response()) } Route::Nar(hash, compression) => { st.counters.nar_requests.inc(); let Some(file_hash) = helios_core::nix32_decode::<32>(hash) else { - return Ok(not_found(NEGATIVE)); + return Ok(not_found(caching.miss.clone())); }; let db = st.db.clone(); let cache_id = cache.id; @@ -137,9 +170,9 @@ async fn serve(st: Shared, method: Method, uri: Uri, headers: HeaderMap) -> ApiR }) .await??; let Some(size) = size else { - return Ok(not_found(NEGATIVE)); + return Ok(not_found(caching.miss.clone())); }; - nar_response(&st, &file_hash, compression, size, &method, cache_control).await + nar_response(&st, &file_hash, compression, size, &method, caching).await } } } @@ -150,9 +183,9 @@ async fn nar_response( compression: Compression, size: u64, method: &Method, - cache_control: HeaderValue, + caching: &Caching, ) -> ApiResult { - let mut builder = Response::builder().header(header::CONTENT_TYPE, "application/x-nix-nar").header(header::CACHE_CONTROL, cache_control); + let mut builder = Response::builder().header(header::CONTENT_TYPE, "application/x-nix-nar").header(header::CACHE_CONTROL, caching.nar.clone()); if let Some(prefix) = &st.cfg.accel_redirect { let name = helios_core::nix32_encode(file_hash); @@ -167,7 +200,7 @@ async fn nar_response( } let file = match tokio::fs::File::open(st.nar_path(file_hash, compression)).await { Ok(f) => f, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(not_found(NEGATIVE)), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(not_found(caching.miss.clone())), Err(e) => return Err(ApiError::internal(e)), }; // 512 KiB reads; the stream ends after EOF or the first error. @@ -192,7 +225,16 @@ pub async fn cache_info( headers: HeaderMap, ) -> ApiResult> { let cache = st.cache(&name).ok_or_else(ApiError::not_found)?; - st.authorize_read(&headers, &name, cache)?; + // Pushers read it too, for the compression defaults; a push-only token + // learns nothing here it could not from pushing. + if st.authorize_read(&headers, &name, cache).is_err() { + st.authorize(&headers, &name, crate::auth::Perm::Push)?; + } let public_key = st.signer.as_ref().map(helios_core::Signer::public_key); - Ok(axum::Json(serde_json::json!({ "name": name, "public": cache.public, "publicKey": public_key }))) + Ok(axum::Json(serde_json::json!({ + "name": name, + "public": cache.public, + "publicKey": public_key, + "compression": { "level": st.cfg.compression_level, "windowLog": st.cfg.compression_window_log }, + }))) } diff --git a/crates/helios-server/src/state.rs b/crates/helios-server/src/state.rs index 5232654..c9f710d 100644 --- a/crates/helios-server/src/state.rs +++ b/crates/helios-server/src/state.rs @@ -24,12 +24,28 @@ pub struct PathKey { pub hash: [u8; 20], } +/// Per-process key for `PathKey` hashes. Store path hashes come from +/// clients, so a fixed hash would let a pusher publish paths that all +/// collide and slow every lookup to a crawl. +static HASH_KEY: std::sync::LazyLock<[u64; 3]> = std::sync::LazyLock::new(|| { + let mut bytes = [0u8; 24]; + helios_core::random_bytes(&mut bytes); + let word = |i: usize| u64::from_le_bytes(bytes[i * 8..i * 8 + 8].try_into().expect("8 bytes")); + [word(0), word(1), word(2)] +}); + +/// The 128-bit product folded to 64 bits, as in foldhash and wyhash. +fn fold_mul(a: u64, b: u64) -> u64 { + let full = u128::from(a) * u128::from(b); + (full as u64) ^ ((full >> 64) as u64) +} + impl Hash for PathKey { fn hash(&self, state: &mut H) { - // Store path hashes are uniformly distributed already. - let mut word = [0u8; 8]; - word.copy_from_slice(&self.hash[..8]); - state.write_u64(u64::from_ne_bytes(word) ^ u64::from(self.cache)); + let k = &*HASH_KEY; + let word = |i: usize| u64::from_le_bytes(self.hash[i..i + 8].try_into().expect("8 bytes")); + let tail = u64::from(u32::from_le_bytes(self.hash[16..20].try_into().expect("4 bytes"))) | (u64::from(self.cache) << 32); + state.write_u64(fold_mul(fold_mul(word(0) ^ k[0], word(8) ^ k[1]) ^ tail, k[2])); } } @@ -77,6 +93,11 @@ pub struct AppState { pub access: crate::stats::Access, pub uploads: crate::chunked::Sessions, pub traces: RwLock, + /// Bumped, under the index write lock, whenever paths are published or + /// forgotten. A narinfo read from the database is cached only if this + /// did not move meanwhile, so a body from before an eviction or re-push + /// is never cached after it. + pub narinfo_epoch: std::sync::atomic::AtomicU64, } pub type Shared = Arc; @@ -134,6 +155,7 @@ impl AppState { access: Default::default(), uploads: Default::default(), traces: RwLock::new(traces), + narinfo_epoch: Default::default(), }) } @@ -220,6 +242,8 @@ mod tests { upload_grace: std::time::Duration::from_secs(1), gc_interval: std::time::Duration::from_secs(3600), audit_retention: std::time::Duration::from_secs(3600), + compression_level: 3, + compression_window_log: 27, }; let db = Db::open(&dir.join("helios.db")).unwrap(); let (tx, _rx) = mpsc::unbounded_channel(); diff --git a/crates/helios-server/src/traces.rs b/crates/helios-server/src/traces.rs index cc0e24a..91f9d39 100644 --- a/crates/helios-server/src/traces.rs +++ b/crates/helios-server/src/traces.rs @@ -82,18 +82,18 @@ fn basename(s: &str) -> &str { } /// An entry in either format, validated. -enum Parsed<'a> { +enum Parsed { /// ``, ``. - Current(&'a str, &'a str), + Current(String, String), /// `sha256:!`. - Legacy(&'a str), + Legacy(String), } -impl Parsed<'_> { +impl Parsed { fn id(&self) -> String { match self { Parsed::Current(drv, output) => format!("{drv}/{output}"), - Parsed::Legacy(id) => (*id).to_owned(), + Parsed::Legacy(id) => id.clone(), } } } @@ -108,7 +108,7 @@ fn legacy_id_valid(id: &str) -> bool { /// `UnkeyedRealisation::fingerprint` produces for the format: the entry's /// JSON with keys sorted and the signatures removed. Every value is limited /// to store path and hash characters, so no JSON escaping is involved. -fn render(st: &Shared, entry: &Parsed<'_>, out: &str) -> Vec { +fn render(st: &Shared, entry: &Parsed, out: &str) -> Vec { let sign = |fingerprint: String| st.signer.as_ref().map(|s| s.sign(fingerprint.as_bytes())); let body = match entry { Parsed::Current(drv, output) => { @@ -185,13 +185,13 @@ pub async fn publish( if !output_name_valid(&key.output_name) { return Err(ApiError::bad_request(format!("invalid output name: {}", key.output_name))); } - (Parsed::Current(drv, &key.output_name), basename(&value.out_path)) + (Parsed::Current(drv.to_owned(), key.output_name.clone()), basename(&value.out_path)) } Entry::Legacy { id, out_path } => { if !legacy_id_valid(id) { return Err(ApiError::bad_request(format!("invalid build trace id: {id}"))); } - (Parsed::Legacy(id), basename(out_path)) + (Parsed::Legacy(id.clone()), basename(out_path)) } }; let key = out_key(cache.id, out).ok_or_else(|| ApiError::bad_request(format!("invalid output path: {out}")))?; @@ -199,7 +199,8 @@ pub async fn publish( missing.push(format!("/nix/store/{out}")); continue; } - rows.push((parsed.id(), out.to_owned(), key, render(&st, &parsed, out))); + // Signed later, off the index lock. + rows.push((parsed, out.to_owned(), key)); } } if !missing.is_empty() { @@ -208,22 +209,28 @@ pub async fn publish( let st2 = st.clone(); let inserted = tokio::task::spawn_blocking(move || -> ApiResult> { + let rendered: Vec<_> = rows.into_iter().map(|(parsed, out, key)| (parsed.id(), render(&st2, &parsed, &out), out, key)).collect(); let now = crate::db::now(); st2.db .write(|conn| -> rusqlite::Result> { let tx = conn.transaction()?; let mut inserted = Vec::new(); { - // The first entry for an output wins, as for paths: a - // non-deterministic build must not flip what is served. - let mut insert = tx.prepare_cached( - "INSERT INTO build_traces (cache_id, id, out_path, body, created_at) - VALUES (?1, ?2, ?3, ?4, ?5) ON CONFLICT DO NOTHING", + let mut upsert = tx.prepare_cached( + "INSERT INTO build_traces (cache_id, id, out_path, body, created_at) VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT (cache_id, id) DO UPDATE SET out_path = excluded.out_path, body = excluded.body, created_at = excluded.created_at", )?; - for (id, out, key, body) in rows { - if insert.execute(params![cache.id, id, out, body, now])? > 0 { - inserted.push((TraceKey { cache: cache.id, id: id.into() }, Trace { out: key, body: body.into() })); + let (traces, index) = (st2.traces.rd(), st2.index.rd()); + for (id, body, out, key) in rendered { + let trace_key = TraceKey { cache: cache.id, id: id.as_str().into() }; + // The first entry for an output wins while its path is + // served, so a non-deterministic build cannot flip it; + // once that path is gone, a newer build may take over. + if traces.get(&trace_key).is_some_and(|t| index.contains(&t.out)) { + continue; } + upsert.execute(params![cache.id, id, out, body, now])?; + inserted.push((trace_key, Trace { out: key, body: body.into() })); } } tx.commit()?; diff --git a/scripts/e2e.sh b/scripts/e2e.sh index f1c0893..7b3d307 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -59,7 +59,8 @@ helios login admin "$URL" "$(cat "$WORK/admin")" >/dev/null helios cache create main >/dev/null helios cache create secret --private >/dev/null helios cache create chunked >/dev/null -PUSH_TOKEN="$(helios token create ci --caches main,secret,chunked --perms push,pull 2>/dev/null | jq -r .token)" +helios cache create plain >/dev/null +PUSH_TOKEN="$(helios token create ci --caches main,secret,chunked,plain --perms push,pull 2>/dev/null | jq -r .token)" PULL_ONLY="$(helios token create reader --caches secret 2>/dev/null | jq -r .token)" PUSH_ONLY="$(helios token create builder --caches secret --perms push 2>/dev/null | jq -r .token)" check "tokens are read-only by default" '["pull"]' "$(helios token list | jq -c '.tokens[] | select(.subject=="reader") | .perms')" @@ -77,6 +78,16 @@ NIX_CONFIG="trusted-public-keys = $PUBKEY require-sigs = true" nix copy --from "$URL/chunked" --to "$WORK/chunked-store" "$TARGET" 2>&1 | tail -3 check "chunked closure substitutes" "$(nix-store -qR "$TARGET" | wc -l)" "$(nix-store --store "$WORK/chunked-store" -qR "$TARGET" | wc -l)" +# Level 0: raw NARs, served as such. A fresh path, so no NAR is reused. +head -c 100000 /dev/urandom >"$WORK/plain-file" +PLAIN="$(nix-store --add "$WORK/plain-file")" +helios push plain --level 0 "$PLAIN" 2>&1 | tail -1 +check "level 0 publishes uncompressed NARs" "none" "$(curl -s "$URL/plain/$(basename "$PLAIN" | cut -c1-32).narinfo" | sed -n 's/^Compression: //p')" +NIX_CONFIG="trusted-public-keys = $PUBKEY +require-sigs = true" nix copy --from "$URL/plain" --to "$WORK/plain-store" "$PLAIN" 2>&1 | tail -3 +check "an uncompressed NAR substitutes" "$(nix-store -q --hash "$PLAIN")" "$(nix-store --store "$WORK/plain-store" -q --hash "$PLAIN")" +check "the server advertises compression defaults to pushers" '3 27' "$(curl -s -H "authorization: Bearer $PUSH_ONLY" "$URL/_api/v2/caches/secret" | jq -r '"\(.compression.level) \(.compression.windowLog)"')" + AUTH=(-H "authorization: Bearer $PUSH_TOKEN") new_upload() { curl -s -X POST "${AUTH[@]}" "$URL/_api/v2/caches/chunked/uploads" | jq -r .id; } NAR="$WORK/one.nar.zst" @@ -158,6 +169,19 @@ check "invalid pin rejected" 400 "$(status -X POST "${AUTH[@]}" -H 'content-type helios unpin main "$TARGET" 2>/dev/null check "unpinned closure is a candidate again" "$CLOSURE_SIZE" "$(lru_count 1)" +echo "=== private NARs stay private" +OTHER="$(helios --server admin token create other --caches main --perms push 2>/dev/null | jq -r .token)" +head -c 4096 /dev/urandom >"$WORK/secret-file" +nix-store --dump "$WORK/secret-file" | zstd -q >"$WORK/secret.nar.zst" +SECRET_NAR="$(curl -s -X PUT -H "authorization: Bearer $PUSH_TOKEN" --data-binary @"$WORK/secret.nar.zst" "$URL/_api/v2/caches/secret/nar" | jq -r .narHash)" +SECRET_SIZE="$(nix-store --dump "$WORK/secret-file" | wc -c)" +FAKE_PATH="/nix/store/$(printf 'a%.0s' $(seq 32))-leak" +LEAK="{\"paths\":[{\"storePath\":\"$FAKE_PATH\",\"narHash\":\"$SECRET_NAR\",\"narSize\":$SECRET_SIZE}]}" +check "another cache does not see a private NAR as known" "[]" "$(curl -s -X POST -H "authorization: Bearer $OTHER" -H 'content-type: application/json' -d "{\"narHashes\":[\"$SECRET_NAR\"]}" "$URL/_api/v2/caches/main/nars/known" | jq -c .known)" +check "nor can it publish a path over it" 409 "$(status -X POST -H "authorization: Bearer $OTHER" -H 'content-type: application/json' -d "$LEAK" "$URL/_api/v2/caches/main/paths")" +curl -s -o /dev/null -X PUT -H "authorization: Bearer $OTHER" --data-binary @"$WORK/secret.nar.zst" "$URL/_api/v2/caches/main/nar" +check "uploading the NAR itself entitles it" 201 "$(status -X POST -H "authorization: Bearer $OTHER" -H 'content-type: application/json' -d "$LEAK" "$URL/_api/v2/caches/main/paths")" + echo "=== build traces" OUT_BASE="$(basename "$TARGET")" HEX="$(printf '%064d' 0 | tr 0 a)" From 4a347da4c6296d6b163b4c65339242dfcffbcaba Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 41/56] cli: compress with the server defaults and harden push and use helios push and watch-store take --level (0-19, 0 uploads raw NARs) and --window-log (10-27, or 0), and use the cache's advertised defaults for whichever is not given: zstd level 3 with a 2^27 window, about 9% smaller than plain level 2 on a NixOS closure. Fixes from review: - One NAR that kept failing stopped push from publishing anything, and stopped watch-store for good. Push now publishes everything that uploaded, leaving out failed paths and their dependents. watch-store pushes in batches, falls back to one spool entry at a time, and after five failures, counted only while the server is reachable, moves an entry to failed/. - A large spool overflowed nix path-info's argument list; long path lists now go through --stdin. - The transport had no timeouts or keepalive, so a half-open connection hung push and watch-store forever. Connect, TLS and the handshake are bounded, keepalive is on, and a connection that moves no bytes for five minutes fails. - A local serialisation failure was reported as the server's "invalid NAR stream". - helios use appended its netrc entry after a default entry, where curl never reaches it, and rewrote the file in ways that broke macdefs and quoted values. It now edits only its own entry. It also shadowed /etc/nix/netrc by pointing netrc-file at a new file; it writes to the effective netrc where it can. - Token files kept loose permissions and were rewritten in place; they are now written to a 0600 temp file and renamed. - A trailing slash in --url gave substituter URLs with //. - Unreadable spool entries were deleted; they move to failed/. --- crates/helios-cli/src/api.rs | 10 +- crates/helios-cli/src/config.rs | 63 ++++++-- crates/helios-cli/src/main.rs | 63 ++++++-- crates/helios-cli/src/nix.rs | 71 ++++++--- crates/helios-cli/src/push.rs | 148 ++++++++++++++----- crates/helios-cli/src/substituter.rs | 207 +++++++++++++++++++++++---- crates/helios-cli/src/transport.rs | 125 +++++++++++++++- crates/helios-cli/src/watch.rs | 153 +++++++++++++++++--- 8 files changed, 706 insertions(+), 134 deletions(-) diff --git a/crates/helios-cli/src/api.rs b/crates/helios-cli/src/api.rs index 1863b8b..1daee74 100644 --- a/crates/helios-cli/src/api.rs +++ b/crates/helios-cli/src/api.rs @@ -80,8 +80,9 @@ impl Client { Ok(r.known) } - pub async fn upload(&self, cache: &str, body: Body) -> anyhow::Result { - self.call(Method::PUT, &format!("/caches/{cache}/nar?compression=zstd"), Some("application/x-nix-nar"), body).await + /// `compression` is the narinfo name: `zstd`, or `none` for a raw NAR. + pub async fn upload(&self, cache: &str, compression: &str, body: Body) -> anyhow::Result { + self.call(Method::PUT, &format!("/caches/{cache}/nar?compression={compression}"), Some("application/x-nix-nar"), body).await } pub async fn publish_build_traces(&self, cache: &str, entries: &[Value]) -> anyhow::Result { @@ -106,12 +107,13 @@ impl Client { } /// Starts a chunked upload; returns its id. - pub async fn upload_create(&self, cache: &str) -> anyhow::Result { + pub async fn upload_create(&self, cache: &str, compression: &str) -> anyhow::Result { #[derive(Deserialize)] struct R { id: String, } - let r: R = self.call(Method::POST, &format!("/caches/{cache}/uploads?compression=zstd"), None, transport::full(Bytes::new())).await?; + let r: R = + self.call(Method::POST, &format!("/caches/{cache}/uploads?compression={compression}"), None, transport::full(Bytes::new())).await?; Ok(r.id) } diff --git a/crates/helios-cli/src/config.rs b/crates/helios-cli/src/config.rs index 41e9b4b..93e11cc 100644 --- a/crates/helios-cli/src/config.rs +++ b/crates/helios-cli/src/config.rs @@ -1,7 +1,7 @@ //! `~/.config/helios/config.json`. use std::collections::BTreeMap; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use anyhow::{Context, bail}; use serde::{Deserialize, Serialize}; @@ -38,18 +38,42 @@ pub fn load() -> anyhow::Result { } pub fn login(name: &str, server: &str, token: &str) -> anyhow::Result<()> { - use std::io::Write; - use std::os::unix::fs::OpenOptionsExt; - let mut cfg = load()?; cfg.servers.insert(name.into(), Server { server: server.trim_end_matches('/').into(), token: token.into() }); cfg.default_server = Some(name.into()); let path = path()?; std::fs::create_dir_all(path.parent().expect("config path has a parent"))?; - let mut file = std::fs::OpenOptions::new().write(true).create(true).truncate(true).mode(0o600).open(&path)?; - file.write_all(&serde_json::to_vec_pretty(&cfg)?)?; - file.write_all(b"\n")?; - Ok(()) + let mut body = serde_json::to_vec_pretty(&cfg)?; + body.push(b'\n'); + write_secret(&path, &body) +} + +/// Replaces `path` with `contents`, readable by the owner only: a fresh +/// 0600 file in the same directory, synced and renamed over the old one, so +/// a crash leaves the old or the new file and never a torn or +/// world-readable one. A symlink is followed, so the file it names is +/// replaced. +pub fn write_secret(path: &Path, contents: &[u8]) -> anyhow::Result<()> { + use std::io::Write; + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + + let path = std::fs::canonicalize(path).unwrap_or_else(|_| path.to_owned()); + let dir = path.parent().with_context(|| format!("{} has no directory", path.display()))?; + let name = path.file_name().with_context(|| format!("{} has no file name", path.display()))?.to_string_lossy(); + let tmp = dir.join(format!(".{name}.{}.tmp", helios_core::uuid_v4())); + let written = (|| { + let mut file = std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(&tmp)?; + // The mode above is subject to the umask; this is not. + file.set_permissions(std::fs::Permissions::from_mode(0o600))?; + file.write_all(contents)?; + file.sync_all()?; + std::fs::rename(&tmp, &path)?; + std::fs::File::open(dir)?.sync_all() + })(); + if written.is_err() { + let _ = std::fs::remove_file(&tmp); + } + written.with_context(|| format!("writing {}", path.display())) } pub fn server(name: Option<&str>) -> anyhow::Result { @@ -59,3 +83,26 @@ pub fn server(name: Option<&str>) -> anyhow::Result { }; cfg.servers.get(&key).cloned().with_context(|| format!("server '{key}' not found; run: helios login {key} ")) } + +#[cfg(test)] +mod tests { + use super::write_secret; + use std::os::unix::fs::PermissionsExt; + + #[test] + fn secrets_replace_files_privately() { + let dir = std::env::temp_dir().join(format!("helios-config-{}", helios_core::uuid_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("netrc"); + std::fs::write(&file, "old").unwrap(); + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o644)).unwrap(); + let link = dir.join("link"); + std::os::unix::fs::symlink(&file, &link).unwrap(); + write_secret(&link, b"new").unwrap(); + assert_eq!(std::fs::read_to_string(&file).unwrap(), "new"); + assert_eq!(std::fs::metadata(&file).unwrap().permissions().mode() & 0o777, 0o600); + assert!(std::fs::symlink_metadata(&link).unwrap().file_type().is_symlink()); + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 2, "no temporary files left"); + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/crates/helios-cli/src/main.rs b/crates/helios-cli/src/main.rs index b3fc236..08db0e0 100644 --- a/crates/helios-cli/src/main.rs +++ b/crates/helios-cli/src/main.rs @@ -45,9 +45,14 @@ enum Command { /// Parallel uploads. #[arg(long, short = 'j', default_value_t = 8)] jobs: usize, - /// zstd compression level (1-19). - #[arg(long, default_value_t = 3, value_parser = clap::value_parser!(i32).range(1..=19))] - level: i32, + /// zstd level, 1-19, or 0 to upload uncompressed. Default: the + /// server's (3 unless configured). + #[arg(long, value_parser = clap::value_parser!(i32).range(0..=19))] + level: Option, + /// zstd window for long-distance matching, as a power of two + /// (10-27), or 0 for the level's own. Default: the server's (27). + #[arg(long, value_parser = parse_window_log)] + window_log: Option, /// Pin the given paths afterwards, so auto-GC keeps them and their /// closures. #[arg(long)] @@ -67,9 +72,14 @@ enum Command { /// Parallel uploads. #[arg(long, short = 'j', default_value_t = 8)] jobs: usize, - /// zstd compression level (1-19). - #[arg(long, default_value_t = 3, value_parser = clap::value_parser!(i32).range(1..=19))] - level: i32, + /// zstd level, 1-19, or 0 to upload uncompressed. Default: the + /// server's (3 unless configured). + #[arg(long, value_parser = clap::value_parser!(i32).range(0..=19))] + level: Option, + /// zstd window for long-distance matching, as a power of two + /// (10-27), or 0 for the level's own. Default: the server's (27). + #[arg(long, value_parser = parse_window_log)] + window_log: Option, /// Upload NARs in chunks of this many MiB, for proxies that cap /// request bodies (Cloudflare, Cloud Run). 0 sends each NAR in one /// request; so does a NAR that fits in one chunk. @@ -176,15 +186,17 @@ async fn run(cli: Cli) -> anyhow::Result<()> { Some(file) => std::fs::read_to_string(file).with_context(|| format!("reading {}", file.display()))?, None => std::env::var("HELIOS_TOKEN").context("--url needs --token-file or HELIOS_TOKEN")?, }; - config::Server { server: url.clone(), token: token.trim().to_owned() } + config::Server { server: url.trim_end_matches('/').to_owned(), token: token.trim().to_owned() } } None => config::server(cli.server.as_deref())?, }; let client = api::Client::new(&server)?; match cli.command { Command::Login { .. } | Command::QueuePaths { .. } => unreachable!(), - Command::Push { cache, installables, closure, jobs, level, chunk_size, pin } => { - push::push(&client, &cache, &installables, push::Options { jobs, level, closure, chunk_size: (chunk_size << 20) as usize }).await?; + Command::Push { cache, installables, closure, jobs, level, window_log, chunk_size, pin } => { + let (level, window_log) = compression(&client, &cache, level, window_log).await; + let opts = push::Options { jobs, level, window_log, closure, chunk_size: (chunk_size << 20) as usize }; + push::push(&client, &cache, &installables, opts).await?; if pin { pin_paths(&client, &cache, &installables).await?; } @@ -201,8 +213,10 @@ async fn run(cli: Cli) -> anyhow::Result<()> { let plan = substituter::plan(&client, &server, &cache).await?; if print { plan.print() } else { plan.apply()? } } - Command::WatchStore { cache, spool, jobs, level, chunk_size } => { - watch::watch(&client, &cache, &spool, push::Options { jobs, level, closure: true, chunk_size: (chunk_size << 20) as usize }).await?; + Command::WatchStore { cache, spool, jobs, level, window_log, chunk_size } => { + let (level, window_log) = compression(&client, &cache, level, window_log).await; + let opts = push::Options { jobs, level, window_log, closure: true, chunk_size: (chunk_size << 20) as usize }; + watch::watch(&client, &cache, &spool, opts).await?; } Command::Cache(CacheCmd::Create { name, private }) => { print(&client.admin_post("/admin/caches", json!({ "name": name, "public": !private })).await?); @@ -257,3 +271,30 @@ async fn pin_paths(client: &api::Client, cache: &str, args: &[String]) -> anyhow tracing::info!("pinned {} paths in '{cache}' ({} already were)", r["pinned"], r["alreadyPinned"]); Ok(()) } + +fn parse_window_log(s: &str) -> Result { + match s.parse::() { + Ok(n) if n == 0 || (10..=27).contains(&n) => Ok(n), + _ => Err("expected 0, or 10-27 (27, a 128 MiB window, is the largest a stock Nix decoder accepts)".into()), + } +} + +/// The level and window to compress with: what was asked for, else the +/// server's defaults for the cache, else level 3 with a 2^27 window. +async fn compression(client: &api::Client, cache: &str, level: Option, window_log: Option) -> (i32, i32) { + let server = if level.is_none() || window_log.is_none() { + match client.cache_info(cache).await { + Ok(info) => info.compression, + Err(e) => { + tracing::debug!("no compression defaults from the server: {e:#}"); + None + } + } + } else { + None + }; + let level = level.or(server.map(|c| c.level)).unwrap_or(3); + // An uncompressed upload has no window. + let window_log = if level == 0 { 0 } else { window_log.or(server.map(|c| c.window_log)).unwrap_or(27) }; + (level, window_log) +} diff --git a/crates/helios-cli/src/nix.rs b/crates/helios-cli/src/nix.rs index 6f26e12..4fc7ce4 100644 --- a/crates/helios-cli/src/nix.rs +++ b/crates/helios-cli/src/nix.rs @@ -40,16 +40,43 @@ fn full(p: &str) -> String { if p.starts_with('/') { p.to_owned() } else { format!("{STORE_DIR}{p}") } } +/// Installables beyond this many go to Nix on stdin rather than argv, +/// which the kernel caps (E2BIG). +const MAX_ARGS: usize = 1_000; + +/// Runs `cmd` with `installables` as arguments, or on stdin (`--stdin`) +/// when there are too many for one command line. +async fn output(mut cmd: Command, installables: &[String]) -> std::io::Result { + use tokio::io::AsyncWriteExt; + cmd.stdout(std::process::Stdio::piped()).stderr(std::process::Stdio::piped()); + if installables.len() <= MAX_ARGS { + return cmd.arg("--").args(installables).output().await; + } + let mut child = cmd.arg("--stdin").stdin(std::process::Stdio::piped()).spawn()?; + let mut stdin = child.stdin.take().expect("stdin is piped"); + let input = installables.join("\n") + "\n"; + // Feed stdin while collecting the output, or a full pipe deadlocks both. + let (written, out) = futures_util::future::join( + async move { + let r = stdin.write_all(input.as_bytes()).await; + drop(stdin); + r + }, + child.wait_with_output(), + ) + .await; + let out = out?; + // Nix closes the pipe early when it fails; its stderr says why. + if out.status.success() { + written?; + } + Ok(out) +} + async fn run(args: &[&str], installables: &[String]) -> anyhow::Result { - Command::new("nix") - .args(["--extra-experimental-features", "nix-command flakes", "path-info"]) - .args(args) - .arg("--") - .args(installables) - .stderr(std::process::Stdio::piped()) - .output() - .await - .context("running nix path-info (is nix on PATH?)") + let mut cmd = Command::new("nix"); + cmd.args(["--extra-experimental-features", "nix-command flakes", "path-info"]).args(args); + output(cmd, installables).await.context("running nix path-info (is nix on PATH?)") } pub async fn path_infos(installables: &[String], closure: bool) -> anyhow::Result> { @@ -150,15 +177,9 @@ pub async fn build_traces(derivers: &[String]) -> anyhow::Result = (0..n).map(|i| format!("p{i}")).collect(); + let out = rt.block_on(output(cmd, &args)).unwrap(); + String::from_utf8(out.stdout).unwrap() + }; + assert_eq!(run(2).split_whitespace().collect::>(), ["--", "p0", "p1", "0"]); + assert_eq!(run(MAX_ARGS + 1).split_whitespace().collect::>(), ["--stdin", &(MAX_ARGS + 1).to_string()]); + } } diff --git a/crates/helios-cli/src/push.rs b/crates/helios-cli/src/push.rs index fa9025f..153b046 100644 --- a/crates/helios-cli/src/push.rs +++ b/crates/helios-cli/src/push.rs @@ -23,6 +23,8 @@ const PUBLISH_BATCH: usize = 1_000; pub struct Options { pub jobs: usize, pub level: i32, + /// zstd window for long-distance matching (10-27), or 0 for the level's. + pub window_log: i32, pub closure: bool, /// Bytes per chunked-upload request; 0 streams each NAR in one request. pub chunk_size: usize, @@ -85,6 +87,7 @@ fn produce(path: &str, opts: DumpOptions, tx: mpsc::Sender>, chunk_size: usize, ) -> anyhow::Result { @@ -101,13 +104,13 @@ async fn send_chunked( None => true, }; if done && session.is_none() && buf.len() <= chunk_size { - return client.upload(cache, crate::transport::full(buf.split().freeze())).await; + return client.upload(cache, compression, crate::transport::full(buf.split().freeze())).await; } while buf.len() >= chunk_size || (done && !buf.is_empty()) { let chunk = buf.split_to(buf.len().min(chunk_size)).freeze(); let id = match &session { Some(id) => id.clone(), - None => session.insert(client.upload_create(cache).await?).clone(), + None => session.insert(client.upload_create(cache, compression).await?).clone(), }; offset = client.upload_append(cache, &id, offset, chunk).await?; } @@ -125,21 +128,32 @@ async fn send_chunked( } async fn upload_one(client: &Client, cache: &str, info: &PathInfo, opts: DumpOptions, chunk_size: usize) -> anyhow::Result { + let compression = if opts.level == 0 { "none" } else { "zstd" }; let (tx, mut rx) = mpsc::channel(8); let path = info.path.clone(); - let producer = tokio::task::spawn_blocking(move || produce(&path, opts, tx)); + // Also reports whether the upload had already stopped reading, which + // tells a cause (local serialisation failure) from an effect. + let producer = tokio::task::spawn_blocking(move || { + let digest = produce(&path, opts, tx.clone()); + (digest, tx.is_closed()) + }); let uploaded = if chunk_size == 0 { let chunks = futures_util::stream::unfold(rx, |mut rx| async move { rx.recv().await.map(|c| (c, rx)) }); - client.upload(cache, crate::transport::stream(chunks)).await + client.upload(cache, compression, crate::transport::stream(chunks)).await } else { - let uploaded = send_chunked(client, cache, &mut rx, chunk_size).await; + let uploaded = send_chunked(client, cache, compression, &mut rx, chunk_size).await; drop(rx); // Stops the producer if the upload gave up early. uploaded }; - let digest = producer.await?; - // An upload failure aborts the producer; report the upload's error. - let uploaded = uploaded?; - let digest = digest?; + let (digest, upload_stopped) = producer.await?; + // A failed upload aborts the producer, so then the upload's error is + // the cause; a failed producer cuts the stream short, and the server's + // complaint about it is only the effect. + let (uploaded, digest) = match (uploaded, digest) { + (Err(e), Err(_)) if upload_stopped => return Err(e), + (_, Err(e)) => return Err(e), + (uploaded, Ok(digest)) => (uploaded?, digest), + }; let ours = format!("sha256:{}", helios_core::nix32_encode(&digest.nar_hash)); if parse_sha256(&info.nar_hash) != Some(digest.nar_hash) { @@ -151,14 +165,15 @@ async fn upload_one(client: &Client, cache: &str, info: &PathInfo, opts: DumpOpt Ok(uploaded.file_size) } -async fn upload_all(client: &Client, cache: &str, paths: &[&PathInfo], opts: &Options) -> anyhow::Result { +/// Uploads `paths`, returning the bytes sent and the paths that failed. +async fn upload_all(client: &Client, cache: &str, paths: &[&PathInfo], opts: &Options) -> (u64, HashSet) { let total = paths.len(); let threads = (std::thread::available_parallelism().map_or(4, |n| n.get()) / opts.jobs.max(1)).max(1) as i32; let mut done = 0usize; let mut bytes = 0u64; - let mut failures = Vec::new(); + let mut failures = HashSet::new(); let mut results = futures_util::stream::iter(paths.iter().map(|info| async move { - let dump = DumpOptions { level: opts.level, threads, size_hint: info.nar_size }; + let dump = DumpOptions { level: opts.level, threads, nar_size: info.nar_size, window_log: opts.window_log }; let started = Instant::now(); (info, upload_one(client, cache, info, dump, opts.chunk_size).await, started.elapsed()) })) @@ -172,14 +187,26 @@ async fn upload_all(client: &Client, cache: &str, paths: &[&PathInfo], opts: &Op } Err(e) => { tracing::error!("[{done}/{total}] {} failed: {e:#}", name(&info.path)); - failures.push(info.path.clone()); + failures.insert(info.path.clone()); } } } - if !failures.is_empty() { - bail!("{} of {total} uploads failed", failures.len()); + (bytes, failures) +} + +/// Drops from `batch` the paths in `bad` and those referring to one, +/// adding the latter to `bad`. `batch` comes dependencies first, so one +/// pass reaches every dependent, here and in later batches. +fn without_bad<'a>(batch: &'a [PathInfo], bad: &mut HashSet) -> Vec<&'a PathInfo> { + let mut kept = Vec::with_capacity(batch.len()); + for info in batch { + if bad.contains(&info.path) || info.references.iter().any(|r| *r != info.path && bad.contains(r)) { + bad.insert(info.path.clone()); + } else { + kept.push(info); + } } - Ok(bytes) + kept } pub async fn push(client: &Client, cache: &str, installables: &[String], opts: Options) -> anyhow::Result<()> { @@ -188,11 +215,16 @@ pub async fn push(client: &Client, cache: &str, installables: &[String], opts: O // or Nix cannot tell which paths a CA derivation produced. let derivers: Vec = infos.iter().filter(|i| i.ca.is_some()).filter_map(|i| i.deriver.clone()).collect::>().into_iter().collect(); - let paths: HashSet = infos.iter().map(|i| i.path.clone()).collect(); - push_infos(client, cache, infos, opts).await?; + let mut paths: HashSet = infos.iter().map(|i| i.path.clone()).collect(); + let total = paths.len(); + let unpublished = push_infos(client, cache, infos, opts).await?; + paths.retain(|p| !unpublished.contains(p)); if !derivers.is_empty() { push_build_traces(client, cache, &derivers, &paths).await?; } + if !unpublished.is_empty() { + bail!("{} of {total} paths not pushed: they, or paths they refer to, failed to upload", unpublished.len()); + } Ok(()) } @@ -225,7 +257,10 @@ async fn push_build_traces(client: &Client, cache: &str, derivers: &[String], pu Ok(()) } -async fn push_infos(client: &Client, cache: &str, infos: Vec, opts: Options) -> anyhow::Result<()> { +/// Uploads and publishes what is missing. A path that fails to upload is +/// left out with everything referring to it, and the rest is published; +/// returns the paths left out. +async fn push_infos(client: &Client, cache: &str, infos: Vec, opts: Options) -> anyhow::Result> { let started = Instant::now(); let total = infos.len(); @@ -237,7 +272,7 @@ async fn push_infos(client: &Client, cache: &str, infos: Vec, opts: Op let todo: Vec = infos.into_iter().filter(|i| missing.contains(&i.hash)).collect(); if todo.is_empty() { tracing::info!("all {total} paths already in '{cache}'"); - return Ok(()); + return Ok(HashSet::new()); } // Paths whose NAR the server already holds (from any cache) skip the upload. @@ -248,23 +283,27 @@ async fn push_infos(client: &Client, cache: &str, infos: Vec, opts: Op } let uploads: Vec<&PathInfo> = todo.iter().filter(|i| !known.contains(&i.nar_hash)).collect(); tracing::info!("{total} paths, {} missing from '{cache}': uploading {}, reusing {} NARs", todo.len(), uploads.len(), todo.len() - uploads.len()); - let mut uploaded_bytes = upload_all(client, cache, &uploads, &opts).await?; + let (mut uploaded_bytes, mut bad) = upload_all(client, cache, &uploads, &opts).await; let ordered = nix::topo_order(todo); let mut published = 0u64; for batch in ordered.chunks(PUBLISH_BATCH) { - let specs: Vec = batch - .iter() - .map(|i| PathSpec { - store_path: i.path.clone(), - nar_hash: i.nar_hash.clone(), - nar_size: i.nar_size, - references: i.references.clone(), - deriver: i.deriver.clone(), - }) - .collect(); let mut retried = false; loop { + let kept = without_bad(batch, &mut bad); + if kept.is_empty() { + break; + } + let specs: Vec = kept + .iter() + .map(|i| PathSpec { + store_path: i.path.clone(), + nar_hash: i.nar_hash.clone(), + nar_size: i.nar_size, + references: i.references.clone(), + deriver: i.deriver.clone(), + }) + .collect(); match client.publish(cache, &specs).await? { Publish::Done { published: n } => { published += n; @@ -274,10 +313,21 @@ async fn push_infos(client: &Client, cache: &str, infos: Vec, opts: Op Publish::MissingNars(paths) if !retried => { retried = true; let wanted: HashSet<&str> = paths.iter().map(String::as_str).collect(); - let redo: Vec<&PathInfo> = batch.iter().filter(|i| wanted.contains(i.path.as_str())).collect(); - uploaded_bytes += upload_all(client, cache, &redo, &opts).await?; + let redo: Vec<&PathInfo> = kept.into_iter().filter(|i| wanted.contains(i.path.as_str())).collect(); + let (bytes, failed) = upload_all(client, cache, &redo, &opts).await; + uploaded_bytes += bytes; + bad.extend(failed); + } + Publish::MissingNars(paths) => { + let wanted: HashSet = paths.into_iter().collect(); + let lacking: Vec = kept.iter().filter(|i| wanted.contains(&i.path)).map(|i| i.path.clone()).collect(); + // Each round drops at least one path, so this ends. + if lacking.is_empty() { + bail!("server lacks NARs for {} paths not in this batch", wanted.len()); + } + tracing::error!("server still lacks NARs for {} paths; leaving them out", lacking.len()); + bad.extend(lacking); } - Publish::MissingNars(paths) => bail!("server still lacks NARs for {} paths", paths.len()), } } } @@ -288,5 +338,33 @@ async fn push_infos(client: &Client, cache: &str, infos: Vec, opts: Op human(uploaded_bytes), human((uploaded_bytes as f64 / secs.max(1e-3)) as u64) ); - Ok(()) + Ok(bad) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn info(name: &str, refs: &[&str]) -> PathInfo { + PathInfo { + path: name.to_owned(), + hash: String::new(), + nar_hash: String::new(), + nar_size: 0, + references: refs.iter().map(|r| (*r).to_owned()).collect(), + deriver: None, + ca: None, + } + } + + #[test] + fn failed_paths_hold_back_their_dependents() { + let first = [info("libc", &["libc"]), info("lib", &["libc"]), info("other", &[])]; + let second = [info("app", &["lib", "other"]), info("tool", &["other"])]; + let mut bad: HashSet = ["lib".to_owned()].into(); + let names = |v: Vec<&PathInfo>| v.into_iter().map(|i| i.path.clone()).collect::>(); + assert_eq!(names(without_bad(&first, &mut bad)), ["libc", "other"]); + assert_eq!(names(without_bad(&second, &mut bad)), ["tool"]); + assert!(bad.contains("app")); + } } diff --git a/crates/helios-cli/src/substituter.rs b/crates/helios-cli/src/substituter.rs index 08fec38..9ff51e0 100644 --- a/crates/helios-cli/src/substituter.rs +++ b/crates/helios-cli/src/substituter.rs @@ -5,7 +5,6 @@ //! the address clients reach it at (compare zhaofengli/attic#325). use std::io::Write; -use std::os::unix::fs::OpenOptionsExt; use std::path::{Path, PathBuf}; use anyhow::{Context, bail}; @@ -19,6 +18,16 @@ use crate::config::Server; pub struct CacheInfo { pub public: bool, pub public_key: Option, + /// The server's compression defaults; absent from older servers. + #[serde(default)] + pub compression: Option, +} + +#[derive(Deserialize, Clone, Copy)] +#[serde(rename_all = "camelCase")] +pub struct Compression { + pub level: i32, + pub window_log: i32, } pub struct Plan { @@ -102,13 +111,7 @@ impl Plan { let mut netrc_file = None; if let Some((host, token)) = &self.netrc { - // An existing netrc-file setting wins; Nix reads only one file. - let file = existing - .lines() - .find_map(|l| l.trim().strip_prefix("netrc-file")?.trim_start().strip_prefix('=').map(|v| PathBuf::from(v.trim()))) - .unwrap_or_else(|| dir.join("netrc")); - add_netrc(&file, host, token)?; - netrc_file = Some(file); + netrc_file = self.write_netrc(&dir, &existing, host, token)?; } let missing: Vec = self @@ -141,33 +144,147 @@ impl Plan { } Ok(()) } -} -/// Appends `machine host` to a netrc, replacing an earlier entry for it. -fn add_netrc(file: &Path, host: &str, token: &str) -> anyhow::Result<()> { - let existing = std::fs::read_to_string(file).unwrap_or_default(); - // netrc entries are whitespace-separated tokens; drop the one for this - // host so a new login replaces its token. - let mut kept = String::new(); - let mut tokens = existing.split_whitespace().peekable(); - while let Some(t) = tokens.next() { - if t == "machine" && tokens.peek() == Some(&host) { - tokens.next(); - while let Some(&next) = tokens.peek() { - if next == "machine" || next == "default" { - break; + /// Adds the token to the netrc file Nix reads. Nix reads only one, so + /// pointing it at a new one would hide the entries of the one it reads + /// now (by default `/etc/nix/netrc`). Returns the file to name in the + /// user's nix.conf, if any. + fn write_netrc(&self, dir: &Path, conf: &str, host: &str, token: &str) -> anyhow::Result> { + // One set in the user's nix.conf already is the one. + if let Some(file) = conf.lines().find_map(conf_netrc_file) { + add_netrc(&file, None, host, token)?; + tracing::info!("added the token for {host} to {}", file.display()); + return Ok(Some(file)); + } + let effective = effective_netrc_file(); + if let Some(file) = &effective + && std::fs::OpenOptions::new().append(true).open(file).is_ok() + { + add_netrc(file, None, host, token)?; + tracing::info!("added the token for {host} to {}", file.display()); + return Ok(None); + } + let own = dir.join("netrc"); + let mut base = None; + if let Some(file) = effective.filter(|f| *f != own) { + match std::fs::read_to_string(&file) { + Ok(text) if !own.exists() => { + tracing::info!("copying the entries of {} to {}, which Nix reads instead from now on", file.display(), own.display()); + base = Some(text); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Ok(_) => { + tracing::warn!("Nix reads {} instead of {} from now on; copy over any entries it still needs", own.display(), file.display()) } - tokens.next(); + Err(e) => tracing::warn!( + "Nix reads {} instead of {} from now on, which could not be read to copy its entries over ({e}); \ + copy any it still needs by hand", + own.display(), + file.display() + ), + } + } + add_netrc(&own, base, host, token)?; + tracing::info!("added the token for {host} to {}", own.display()); + Ok(Some(own)) + } +} + +/// Sets the token for `host` in a netrc file. `base` is the content to +/// start from instead of the file's own, when the file is new. +fn add_netrc(file: &Path, base: Option, host: &str, token: &str) -> anyhow::Result<()> { + let existing = match base { + Some(base) => base, + None => match std::fs::read_to_string(file) { + Ok(text) => text, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(e) => return Err(e).with_context(|| format!("reading {}", file.display())), + }, + }; + crate::config::write_secret(file, edit_netrc(&existing, host, token).as_bytes()) +} + +/// `netrc` with the entry for `machine host` set to the token. The old +/// entry's lines go (its usual forms: on one line, or `login`, `password` +/// and `account` on lines of their own), the new entry takes their place, +/// or comes before any `default` entry, which ends the entries curl reads. +/// Every other line, with comments, macros and quoting, stays as it was. +fn edit_netrc(netrc: &str, host: &str, token: &str) -> String { + let mut kept: Vec<&str> = Vec::new(); + let mut at: Option = None; + let (mut removing, mut in_macro) = (false, false); + for line in netrc.split_inclusive('\n') { + let words: Vec<&str> = line.split_whitespace().collect(); + if in_macro { + // A macro body runs to the next empty line. + if words.is_empty() { + (in_macro, removing) = (false, false); + kept.push(line); + } else if !removing { + kept.push(line); } continue; } - kept.push_str(if t == "machine" || t == "default" { "\n" } else { " " }); - kept.push_str(t); + match words.first().copied() { + None => {} + Some(w) if w.starts_with('#') => {} + Some(first @ ("machine" | "default")) => { + removing = false; + if first == "default" { + at.get_or_insert(kept.len()); + } + // Only an entry alone on its line; one sharing a line with + // others is left alone. + let ours = first == "machine" && words.get(1) == Some(&host) && !words[2..].iter().any(|w| matches!(*w, "machine" | "default")); + if ours { + at.get_or_insert(kept.len()); + removing = true; + in_macro = words.contains(&"macdef"); + continue; + } + } + Some("login" | "password" | "account" | "macdef") if removing => { + in_macro = words.contains(&"macdef"); + continue; + } + Some(_) => removing = false, + } + in_macro = words.contains(&"macdef") && !words[0].starts_with('#'); + kept.push(line); + } + let at = at.unwrap_or(kept.len()); + let mut out: String = kept[..at].concat(); + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); } - let body = format!("{}\nmachine {host} login helios password {token}\n", kept.trim()); - let mut out = std::fs::OpenOptions::new().write(true).create(true).truncate(true).mode(0o600).open(file)?; - out.write_all(body.trim_start().as_bytes())?; - Ok(()) + out.push_str(&format!("machine {host} login helios password {token}\n")); + out.push_str(&kept[at..].concat()); + out +} + +/// The netrc file Nix reads, as `nix config show` reports it with every +/// configuration file applied. +fn effective_netrc_file() -> Option { + let nix = |args: &[&str]| { + let out = std::process::Command::new("nix") + .args(["--extra-experimental-features", "nix-command"]) + .args(args) + .stderr(std::process::Stdio::null()) + .output() + .ok()?; + out.status.success().then(|| String::from_utf8_lossy(&out.stdout).into_owned()) + }; + let value = match nix(&["config", "show", "netrc-file"]) { + Some(v) => v.trim().to_owned(), + // Before `nix config show`. + None => nix(&["show-config"])?.lines().find_map(conf_netrc_file)?.to_string_lossy().into_owned(), + }; + (!value.is_empty()).then(|| PathBuf::from(value)) +} + +/// The value of a `netrc-file = ...` line. +fn conf_netrc_file(line: &str) -> Option { + line.trim().strip_prefix("netrc-file")?.trim_start().strip_prefix('=').map(|v| PathBuf::from(v.trim())) } fn nix_trusts_user() -> bool { @@ -179,7 +296,7 @@ fn nix_trusts_user() -> bool { #[cfg(test)] mod tests { - use super::host; + use super::{edit_netrc, host}; #[test] fn hosts() { @@ -189,4 +306,34 @@ mod tests { assert_eq!(host("https://u@cache.example.com:443/x").unwrap(), "cache.example.com"); assert!(host("http://").is_err()); } + + #[test] + fn netrc_entries_go_before_default() { + let netrc = "machine a login x password y\ndefault login anonymous password me@example.com\n"; + assert_eq!( + edit_netrc(netrc, "h", "T"), + "machine a login x password y\nmachine h login helios password T\ndefault login anonymous password me@example.com\n" + ); + // An entry stranded after `default` by an older helios moves up. + let stale = "default login x password y\nmachine h login helios password old\n"; + assert_eq!(edit_netrc(stale, "h", "T"), "machine h login helios password T\ndefault login x password y\n"); + assert_eq!(edit_netrc("", "h", "T"), "machine h login helios password T\n"); + assert_eq!(edit_netrc("machine a login x password y", "h", "T"), "machine a login x password y\nmachine h login helios password T\n"); + } + + #[test] + fn netrc_edits_leave_other_lines_alone() { + let netrc = "# tokens\nmachine h\n login old\n # rotated yearly\n password old\n\nmachine o login l password \"a b\"\n"; + assert_eq!( + edit_netrc(netrc, "h", "T"), + "# tokens\nmachine h login helios password T\n # rotated yearly\n\nmachine o login l password \"a b\"\n" + ); + let macros = "machine o login l password p macdef init\nmachine h is not an entry here\n\nmachine h login a password b\n"; + assert_eq!( + edit_netrc(macros, "h", "T"), + "machine o login l password p macdef init\nmachine h is not an entry here\n\nmachine h login helios password T\n" + ); + let own_macro = "machine h login a password b\nmacdef init\ncd /x\n\nmachine o login l password p\n"; + assert_eq!(edit_netrc(own_macro, "h", "T"), "machine h login helios password T\n\nmachine o login l password p\n"); + } } diff --git a/crates/helios-cli/src/transport.rs b/crates/helios-cli/src/transport.rs index 6701944..4d2cc5d 100644 --- a/crates/helios-cli/src/transport.rs +++ b/crates/helios-cli/src/transport.rs @@ -8,6 +8,7 @@ use std::net::IpAddr; use std::pin::Pin; use std::sync::{Arc, Mutex, PoisonError}; use std::task::{Context, Poll}; +use std::time::Duration; use anyhow::{Context as _, bail}; use bytes::Bytes; @@ -17,7 +18,8 @@ use http_body_util::{BodyExt, Full, StreamBody}; use hyper::body::{Frame, Incoming}; use hyper::client::conn::http1::SendRequest; use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; -use tokio::net::TcpStream; +use tokio::net::{TcpSocket, TcpStream}; +use tokio::time::{Instant, Sleep}; pub type Body = UnsyncBoxBody; @@ -118,6 +120,70 @@ impl hyper::rt::Write for Io { } } +/// Connecting, including the TLS and HTTP handshakes. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(30); +/// Connecting to one of the host's addresses, before trying the next. +const ADDRESS_TIMEOUT: Duration = Duration::from_secs(10); +/// A connection that moves no bytes either way for this long is dead (a +/// half-open connection otherwise hangs forever). Large uploads are fine: +/// they keep making progress. +const IDLE_TIMEOUT: Duration = Duration::from_secs(300); + +/// A stream that fails reads and writes once it has made no progress in +/// either direction for `limit`. +struct Idle { + inner: T, + limit: Duration, + timer: Pin>, + last: Instant, +} + +impl Idle { + fn new(inner: T, limit: Duration) -> Self { + Self { inner, limit, timer: Box::pin(tokio::time::sleep(limit)), last: Instant::now() } + } + + /// Records progress, or checks the deadline when there was none. + fn track(&mut self, cx: &mut Context<'_>, poll: Poll>) -> Poll> { + if poll.is_ready() { + self.last = Instant::now(); + return poll; + } + loop { + if self.timer.as_mut().poll(cx).is_pending() { + return Poll::Pending; + } + let deadline = self.last + self.limit; + if Instant::now() >= deadline { + return Poll::Ready(Err(io::Error::new(io::ErrorKind::TimedOut, format!("connection idle for {:?}", self.limit)))); + } + self.timer.as_mut().reset(deadline); + } + } +} + +impl AsyncRead for Idle { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let poll = Pin::new(&mut self.inner).poll_read(cx, buf); + self.track(cx, poll) + } +} + +impl AsyncWrite for Idle { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + let poll = Pin::new(&mut self.inner).poll_write(cx, buf); + self.track(cx, poll) + } + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let poll = Pin::new(&mut self.inner).poll_flush(cx); + self.track(cx, poll) + } + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let poll = Pin::new(&mut self.inner).poll_shutdown(cx); + self.track(cx, poll) + } +} + pub struct Client { target: Target, host_header: HeaderValue, @@ -139,11 +205,32 @@ impl Client { format!("{}{p}", self.target.prefix) } + /// TCP to the first of the host's addresses that answers, with + /// keepalive on so the kernel notices a peer that went away. + async fn tcp(&self) -> io::Result { + let mut last = io::Error::new(io::ErrorKind::NotFound, "host has no addresses"); + for addr in tokio::net::lookup_host((self.target.host.as_str(), self.target.port)).await? { + let socket = if addr.is_ipv4() { TcpSocket::new_v4() } else { TcpSocket::new_v6() }?; + socket.set_keepalive(true)?; + socket.set_nodelay(true)?; + match tokio::time::timeout(ADDRESS_TIMEOUT, socket.connect(addr)).await { + Ok(Ok(tcp)) => return Ok(tcp), + Ok(Err(e)) => last = e, + Err(_) => last = io::Error::new(io::ErrorKind::TimedOut, format!("{addr} did not answer in {ADDRESS_TIMEOUT:?}")), + } + } + Err(last) + } + async fn connect(&self) -> anyhow::Result> { - let tcp = TcpStream::connect((self.target.host.as_str(), self.target.port)) - .await - .with_context(|| format!("connecting to {}:{}", self.target.host, self.target.port))?; - tcp.set_nodelay(true)?; + let connected = tokio::time::timeout(CONNECT_TIMEOUT, self.handshake()).await; + connected + .unwrap_or_else(|_| Err(anyhow::anyhow!("timed out after {CONNECT_TIMEOUT:?}"))) + .with_context(|| format!("connecting to {}:{}", self.target.host, self.target.port)) + } + + async fn handshake(&self) -> anyhow::Result> { + let tcp = Idle::new(self.tcp().await?, IDLE_TIMEOUT); let sender = match &self.tls { None => { let (sender, conn) = hyper::client::conn::http1::handshake(Io(tcp)).await?; @@ -210,7 +297,33 @@ impl Client { #[cfg(test)] mod tests { - use super::parse_base; + use super::{Idle, parse_base}; + use std::time::Duration; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + #[test] + fn idle_connections_time_out() { + let rt = tokio::runtime::Builder::new_current_thread().enable_time().build().unwrap(); + rt.block_on(async { + let (a, mut b) = tokio::io::duplex(64); + let mut a = Idle::new(a, Duration::from_millis(100)); + let writer = tokio::spawn(async move { + for _ in 0..3 { + tokio::time::sleep(Duration::from_millis(60)).await; + b.write_all(b"x").await.unwrap(); + } + b + }); + // Progress keeps it alive past the limit. + let mut buf = [0u8; 4]; + for _ in 0..3 { + assert_eq!(a.read(&mut buf).await.unwrap(), 1); + } + let _open = writer.await.unwrap(); + let err = a.read(&mut buf).await.unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::TimedOut); + }); + } #[test] fn parses_base_urls() { diff --git a/crates/helios-cli/src/watch.rs b/crates/helios-cli/src/watch.rs index 17fb121..c91c605 100644 --- a/crates/helios-cli/src/watch.rs +++ b/crates/helios-cli/src/watch.rs @@ -6,7 +6,11 @@ //! //! Spool entries are files of newline-separated store paths, written to a //! temporary name and renamed into place so readers never see half a file. +//! A batch that fails while the server is reachable is retried one entry +//! at a time, so one bad entry cannot hold up the rest; an entry that keeps +//! failing on its own moves to `failed/` in the spool. +use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::time::Duration; @@ -17,6 +21,13 @@ use crate::push; const POLL: Duration = Duration::from_secs(1); const MAX_BACKOFF: Duration = Duration::from_secs(60); +/// Paths named by one batch (each with its closure); an entry naming more +/// goes alone. +const MAX_BATCH_PATHS: usize = 1_000; +/// Failed pushes of an entry on its own, with the server reachable, before +/// it is set aside. +const MAX_STRIKES: u32 = 5; +const FAILED_DIR: &str = "failed"; fn valid(path: &str) -> bool { path.strip_prefix("/nix/store/").is_some_and(helios_core::store_basename_valid) @@ -40,7 +51,12 @@ pub fn queue(spool: &Path, paths: &str) -> anyhow::Result { /// Spool entries in arrival order, skipping ones still being written. fn pending(spool: &Path) -> anyhow::Result> { let mut entries: Vec = match std::fs::read_dir(spool) { - Ok(rd) => rd.flatten().map(|e| e.path()).filter(|p| p.file_name().and_then(|n| n.to_str()).is_some_and(|n| !n.starts_with('.'))).collect(), + Ok(rd) => rd + .flatten() + .filter(|e| e.file_type().is_ok_and(|t| t.is_file())) + .map(|e| e.path()) + .filter(|p| p.file_name().and_then(|n| n.to_str()).is_some_and(|n| !n.starts_with('.'))) + .collect(), Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), Err(e) => return Err(e.into()), }; @@ -48,38 +64,107 @@ fn pending(spool: &Path) -> anyhow::Result> { Ok(entries) } +/// Moves an entry out of the queue into `failed/`, for a person to look at. +fn set_aside(spool: &Path, entry: &Path, why: &str) { + let dir = spool.join(FAILED_DIR); + let moved = std::fs::create_dir_all(&dir).and_then(|()| std::fs::rename(entry, dir.join(entry.file_name().unwrap_or_default()))); + match moved { + Ok(()) => tracing::warn!(entry = %entry.display(), to = %dir.display(), "{why}; set the entry aside"), + Err(e) => tracing::error!(entry = %entry.display(), error = %e, "{why}; could not set the entry aside"), + } +} + +/// The oldest entries and their paths, up to `MAX_BATCH_PATHS` paths (at +/// least one entry). Unreadable entries are set aside. +fn next_batch(spool: &Path, entries: &[PathBuf]) -> Vec<(PathBuf, Vec)> { + let mut batch = Vec::new(); + let mut count = 0; + for entry in entries { + let text = match std::fs::read_to_string(entry) { + Ok(text) => text, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => continue, + Err(e) => { + set_aside(spool, entry, &format!("unreadable spool entry: {e}")); + continue; + } + }; + let paths: Vec = text.lines().filter(|p| valid(p)).map(str::to_owned).collect(); + if !batch.is_empty() && count + paths.len() > MAX_BATCH_PATHS { + break; + } + count += paths.len(); + batch.push((entry.clone(), paths)); + } + batch +} + +async fn push_paths(client: &Client, cache: &str, paths: impl Iterator, opts: push::Options) -> anyhow::Result<()> { + let mut paths: Vec = paths.cloned().collect(); + paths.sort(); + paths.dedup(); + // Paths garbage-collected since they were built cannot be pushed. + paths.retain(|p| Path::new(p).exists()); + if paths.is_empty() { + return Ok(()); + } + push::push(client, cache, &paths, push::Options { closure: true, ..opts }).await +} + +/// Whether the server answers this client, so a failure is the entry's +/// fault rather than the network's or the server's. +async fn reachable(client: &Client, cache: &str) -> bool { + client.missing(cache, &[]).await.is_ok() +} + pub async fn watch(client: &Client, cache: &str, spool: &Path, opts: push::Options) -> anyhow::Result<()> { tracing::info!(spool = %spool.display(), cache, "watching for paths to push"); let mut backoff = POLL; + let mut strikes: HashMap = HashMap::new(); loop { let entries = pending(spool)?; - if entries.is_empty() { + let batch = next_batch(spool, &entries); + if batch.is_empty() { tokio::time::sleep(POLL).await; continue; } - let mut paths: Vec = Vec::new(); - for e in &entries { - let text = std::fs::read_to_string(e).unwrap_or_default(); - paths.extend(text.lines().filter(|p| valid(p)).map(str::to_owned)); - } - paths.sort(); - paths.dedup(); - // Paths garbage-collected since they were built cannot be pushed. - paths.retain(|p| Path::new(p).exists()); - let result = if paths.is_empty() { Ok(()) } else { push::push(client, cache, &paths, push::Options { closure: true, ..opts }).await }; - match result { - Ok(()) => { - for e in &entries { - let _ = std::fs::remove_file(e); - } - backoff = POLL; + let Err(e) = push_paths(client, cache, batch.iter().flat_map(|(_, p)| p), opts).await else { + for (entry, _) in &batch { + let _ = std::fs::remove_file(entry); + strikes.remove(entry); } - Err(e) => { - tracing::warn!(retry_in = ?backoff, error = format!("{e:#}"), "push failed"); - tokio::time::sleep(backoff).await; - backoff = (backoff * 2).min(MAX_BACKOFF); + backoff = POLL; + continue; + }; + tracing::warn!(entries = batch.len(), error = format!("{e:#}"), "push failed"); + // With the server reachable, find the entries at fault and push the others. + if reachable(client, cache).await { + for (entry, paths) in &batch { + if batch.len() > 1 { + match push_paths(client, cache, paths.iter(), opts).await { + Ok(()) => { + let _ = std::fs::remove_file(entry); + strikes.remove(entry); + continue; + } + Err(e) => { + tracing::warn!(entry = %entry.display(), error = format!("{e:#}"), "push of one entry failed"); + if !reachable(client, cache).await { + break; + } + } + } + } + let n = strikes.entry(entry.clone()).or_default(); + *n += 1; + if *n >= MAX_STRIKES { + strikes.remove(entry); + set_aside(spool, entry, &format!("push failed {MAX_STRIKES} times")); + } } } + tracing::info!(retry_in = ?backoff, "retrying failed pushes"); + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(MAX_BACKOFF); } } @@ -98,4 +183,28 @@ mod tests { assert_eq!(queue(&dir, "").unwrap(), 0); std::fs::remove_dir_all(&dir).unwrap(); } + + #[test] + fn batches_are_bounded_and_skip_unreadable_entries() { + let dir = std::env::temp_dir().join(format!("helios-spool-{}", helios_core::uuid_v4())); + let path = "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello"; + let many = vec![path; MAX_BATCH_PATHS].join(" "); + queue(&dir, path).unwrap(); + // A directory where an entry should be: reading it fails. + std::fs::create_dir_all(dir.join("00000000000000000000-0")).unwrap(); + queue(&dir, &many).unwrap(); + queue(&dir, path).unwrap(); + let entries = pending(&dir).unwrap(); + assert_eq!(entries.len(), 3, "directories are not entries"); + let mut unreadable = entries.clone(); + unreadable.insert(0, dir.join("00000000000000000000-0")); + let batch = next_batch(&dir, &unreadable); + assert_eq!(batch.len(), 1, "the large entry waits for the next batch"); + assert!(dir.join(FAILED_DIR).join("00000000000000000000-0").exists(), "unreadable entry set aside"); + std::fs::remove_file(&batch[0].0).unwrap(); + let batch = next_batch(&dir, &pending(&dir).unwrap()); + assert_eq!(batch.len(), 1); + assert_eq!(batch[0].1.len(), MAX_BATCH_PATHS); + std::fs::remove_dir_all(&dir).unwrap(); + } } From d72f7e3627f400c7c9576eed636ec3682dd72384 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 42/56] daemon: resume scrubs and keep auto-gc to what helios holds A read error ended the whole scrub, so the blob at a bad sector, the case scrub exists for, stopped every later pass at the same place, and any restart began a multi-day pass again a full interval later. Read errors are now counted and skipped, the position is saved in the state directory (--state-dir, or systemd's StateDirectory) so a restart resumes, and admin-socket calls are retried. The rate limit applies per buffer rather than between blobs, and shutdown stops a scrub in progress. With --min-free, a disk filled by something else made auto-GC evict the whole cache. It now evicts for free space only when that can help, and at most a quarter of the cache per run. Also: one accept() error ended the metrics endpoint, and requests over the admin socket had no timeout. --- crates/helios-daemon/src/autogc.rs | 82 +++++++--- crates/helios-daemon/src/client.rs | 15 ++ crates/helios-daemon/src/main.rs | 39 ++++- crates/helios-daemon/src/metrics.rs | 18 ++- crates/helios-daemon/src/scrub.rs | 226 ++++++++++++++++++++++++---- crates/helios-daemon/src/state.rs | 8 +- 6 files changed, 333 insertions(+), 55 deletions(-) diff --git a/crates/helios-daemon/src/autogc.rs b/crates/helios-daemon/src/autogc.rs index 299b3dd..3ae423d 100644 --- a/crates/helios-daemon/src/autogc.rs +++ b/crates/helios-daemon/src/autogc.rs @@ -53,24 +53,43 @@ struct Evicted { freed_bytes: u64, } -/// Bytes to free now, or 0 when within policy. -pub fn to_free(policy: &Policy, used: u64, free_disk: u64) -> u64 { - let over_quota = +/// Bytes to free now, for each reason; 0 when within policy. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Need { + pub quota: u64, + pub disk: u64, +} + +pub fn to_free(policy: &Policy, used: u64, free_disk: u64) -> Need { + let quota = if policy.quota > 0 && used as f64 > policy.quota as f64 * policy.high { used - (policy.quota as f64 * policy.low) as u64 } else { 0 }; - let short_disk = if policy.min_free > 0 && free_disk < policy.min_free { + let disk = if policy.min_free > 0 && free_disk < policy.min_free { // Overshoot a little so the next upload does not trigger it again. (policy.min_free as f64 * 1.1) as u64 - free_disk } else { 0 }; - over_quota.max(short_disk) + Need { quota, disk } +} + +/// One run evicts at most this share of what helios holds for the +/// free-space floor, so a disk filled by something else drains the cache +/// slowly, with warnings, rather than at once. +const MAX_DISK_SHARE: f64 = 0.25; + +/// Bytes this run should evict. The quota is helios's own, and is met in +/// full; the free-space floor only as far as evicting helios data can +/// meet it, and within the per-run cap. +pub fn budget(need: Need, held: u64) -> u64 { + let disk = if need.disk > held { 0 } else { need.disk.min((held as f64 * MAX_DISK_SHARE) as u64) }; + need.quota.max(disk) } /// Upper bound on eviction rounds per run, as a backstop. const MAX_ROUNDS: usize = 10_000; /// Referenced NAR bytes and bytes to free, from fresh server stats. -async fn measure(d: &Daemon, policy: &Policy) -> anyhow::Result<(u64, u64)> { +async fn measure(d: &Daemon, policy: &Policy) -> anyhow::Result<(u64, Need)> { let stats: Stats = d.client.get("/v1/stats").await?; let referenced = stats.blobs.bytes.saturating_sub(stats.blobs.unreferenced_bytes); let free_disk = crate::state::disk_space(&d.data_dir).map(|(free, _)| free).unwrap_or(u64::MAX); @@ -82,11 +101,22 @@ async fn measure(d: &Daemon, policy: &Policy) -> anyhow::Result<(u64, u64)> { pub async fn run(d: &Daemon, policy: &Policy) -> anyhow::Result<()> { add(&d.metrics.gc_runs, 1); set(&d.metrics.gc_last_run, now()); - let (referenced, mut need) = measure(d, policy).await?; - if need == 0 { + let (referenced, need) = measure(d, policy).await?; + if need.disk > referenced { + tracing::warn!( + short = need.disk, + held = referenced, + "the data filesystem lacks more free space than helios holds; not evicting for it, as something else fills the disk" + ); + } + let mut left = budget(need, referenced); + if left == 0 { return Ok(()); } - tracing::info!(referenced, need, "over quota, evicting"); + if left < need.quota.max(need.disk) && need.disk <= referenced { + tracing::warn!(short = need.disk, evicting = left, "low on disk space; evicting only part of what is needed in this run"); + } + tracing::info!(referenced, need = left, "evicting"); let (mut evicted, mut freed) = (0u64, 0u64); for _ in 0..MAX_ROUNDS { let lru: Lru = d.client.get("/v1/lru?limit=500").await?; @@ -96,7 +126,7 @@ pub async fn run(d: &Daemon, policy: &Policy) -> anyhow::Result<()> { for c in lru.paths { planned += c.nar_bytes; batch.push(json!({ "cache": c.cache, "hash": c.hash })); - if planned >= need { + if planned >= left { break; } } @@ -110,14 +140,15 @@ pub async fn run(d: &Daemon, policy: &Policy) -> anyhow::Result<()> { evicted += r.evicted; freed += r.freed_bytes; // NARs shared with paths still in use stay; re-measure rather than - // trusting the plan. - need = measure(d, policy).await?.1; - if need == 0 { + // trusting the plan, and never go past this run's budget. + let (referenced, need) = measure(d, policy).await?; + left = left.saturating_sub(planned).min(budget(need, referenced)); + if left == 0 { break; } } - if need > 0 { - tracing::warn!(need, "cannot free enough: what is left is pinned, or shared with paths still in use"); + if left > 0 { + tracing::warn!(need = left, "cannot free enough: what is left is pinned, or shared with paths still in use"); } add(&d.metrics.gc_evicted_paths, evicted); add(&d.metrics.gc_freed_bytes, freed); @@ -134,14 +165,27 @@ mod tests { #[test] fn watermarks() { let p = Policy { quota: 100 * GIB, high: 0.9, low: 0.8, min_free: 0 }; - assert_eq!(to_free(&p, 89 * GIB, u64::MAX), 0); - assert_eq!(to_free(&p, 95 * GIB, u64::MAX), 15 * GIB); + assert_eq!(to_free(&p, 89 * GIB, u64::MAX), Need { quota: 0, disk: 0 }); + assert_eq!(to_free(&p, 95 * GIB, u64::MAX), Need { quota: 15 * GIB, disk: 0 }); + assert_eq!(budget(to_free(&p, 95 * GIB, u64::MAX), 95 * GIB), 15 * GIB); } #[test] fn free_space_floor() { let p = Policy { quota: 0, high: 0.9, low: 0.8, min_free: 10 * GIB }; - assert_eq!(to_free(&p, 1, 20 * GIB), 0); - assert_eq!(to_free(&p, 1, 5 * GIB), 6 * GIB); + assert_eq!(to_free(&p, 1, 20 * GIB), Need { quota: 0, disk: 0 }); + assert_eq!(to_free(&p, 1, 5 * GIB), Need { quota: 0, disk: 6 * GIB }); + } + + #[test] + fn free_space_floor_spares_the_cache() { + let need = Need { quota: 0, disk: 6 * GIB }; + // Evicting all of it would not be enough: something else fills the disk. + assert_eq!(budget(need, 5 * GIB), 0); + // A run evicts a share at most. + assert_eq!(budget(need, 8 * GIB), 2 * GIB); + assert_eq!(budget(need, 100 * GIB), 6 * GIB); + // The quota is met in full regardless. + assert_eq!(budget(Need { quota: 7 * GIB, disk: 6 * GIB }, 8 * GIB), 7 * GIB); } } diff --git a/crates/helios-daemon/src/client.rs b/crates/helios-daemon/src/client.rs index 07a5c6d..9bc147a 100644 --- a/crates/helios-daemon/src/client.rs +++ b/crates/helios-daemon/src/client.rs @@ -2,6 +2,7 @@ //! are rare (minutes apart), so each one gets a fresh connection. use std::path::PathBuf; +use std::time::Duration; use anyhow::{Context, bail}; use bytes::Bytes; @@ -13,6 +14,12 @@ use tokio::net::UnixStream; use crate::io::Io; +/// Queries: a server that takes longer is stuck. +const GET_TIMEOUT: Duration = Duration::from_secs(60); +/// Operations, which may be slow (a backup of a large database, a large +/// eviction) but must not hang a task forever. +const POST_TIMEOUT: Duration = Duration::from_secs(30 * 60); + #[derive(Clone)] pub struct Client { socket: PathBuf, @@ -24,6 +31,14 @@ impl Client { } async fn call(&self, method: Method, path: &str, body: Option) -> anyhow::Result { + let limit = if method == Method::GET { GET_TIMEOUT } else { POST_TIMEOUT }; + match tokio::time::timeout(limit, self.request(method, path, body)).await { + Ok(result) => result, + Err(_) => bail!("{path}: no response in {limit:?}"), + } + } + + async fn request(&self, method: Method, path: &str, body: Option) -> anyhow::Result { let stream = UnixStream::connect(&self.socket).await.with_context(|| format!("connecting to {}", self.socket.display()))?; let (mut sender, conn) = hyper::client::conn::http1::handshake(Io(stream)).await?; tokio::spawn(conn); diff --git a/crates/helios-daemon/src/main.rs b/crates/helios-daemon/src/main.rs index c79b60b..de201aa 100644 --- a/crates/helios-daemon/src/main.rs +++ b/crates/helios-daemon/src/main.rs @@ -34,6 +34,12 @@ struct Args { #[arg(long, env = "HELIOS_DATA_DIR", default_value = "/var/lib/helios")] data_dir: PathBuf, + /// Directory for the daemon's own state, such as where an unfinished + /// scrub resumes (default: systemd's $STATE_DIRECTORY; without either, + /// a restart begins the scrub again). + #[arg(long, env = "HELIOS_DAEMON_STATE_DIR")] + state_dir: Option, + /// Keep stored NARs under this size (e.g. 500G); 0 disables the quota. #[arg(long, env = "HELIOS_QUOTA", default_value = "0", value_parser = parse_size)] quota: u64, @@ -172,9 +178,25 @@ fn run() -> anyhow::Result<()> { "watermarks must satisfy 0 <= low < high <= 1" ); let policy = autogc::Policy { quota: args.quota, high: args.quota_high, low: args.quota_low, min_free: args.min_free }; - let d = Arc::new(Daemon { client: client::Client::new(args.socket.clone()), data_dir: args.data_dir.clone(), metrics: Default::default() }); + // systemd's StateDirectory= may name several directories; the first is ours. + let state_dir = args.state_dir.clone().or_else(|| { + let dirs = std::env::var_os("STATE_DIRECTORY")?; + std::env::split_paths(&dirs).next().filter(|p| !p.as_os_str().is_empty()) + }); + if let Some(dir) = &state_dir { + std::fs::create_dir_all(dir).map_err(|e| anyhow::anyhow!("creating {}: {e}", dir.display()))?; + } + let d = Arc::new(Daemon { + client: client::Client::new(args.socket.clone()), + data_dir: args.data_dir.clone(), + state_dir, + metrics: Default::default(), + stopping: Default::default(), + }); - tokio::runtime::Builder::new_current_thread().enable_all().build()?.block_on(async move { + let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?; + let stopping = d.stopping.clone(); + let result = runtime.block_on(async move { let mut tasks = Vec::new(); if policy.quota > 0 || policy.min_free > 0 { let d = d.clone(); @@ -185,9 +207,11 @@ fn run() -> anyhow::Result<()> { if !args.scrub_interval.is_zero() { let d = d.clone(); let rate = args.scrub_rate; - // Wait a full interval first so restarts do not re-read everything. + // Wait a full interval first so restarts do not re-read + // everything, unless a pass was cut short. + let delay = !scrub::resuming(&d); tasks.push(tokio::spawn( - async move { periodic("scrub", args.scrub_interval, true, &d.metrics.errors_scrub, || scrub::run(&d, rate)).await }, + async move { periodic("scrub", args.scrub_interval, delay, &d.metrics.errors_scrub, || scrub::run(&d, rate)).await }, )); } { @@ -222,7 +246,12 @@ fn run() -> anyhow::Result<()> { shutdown().await; tracing::info!("shutting down"); Ok(()) - }) + }); + // Blocking work (a scrub read) sees this and stops; do not wait long + // for anything that does not. + stopping.store(true, std::sync::atomic::Ordering::Relaxed); + runtime.shutdown_timeout(Duration::from_secs(5)); + result } #[cfg(test)] diff --git a/crates/helios-daemon/src/metrics.rs b/crates/helios-daemon/src/metrics.rs index d21aad0..bfa37d9 100644 --- a/crates/helios-daemon/src/metrics.rs +++ b/crates/helios-daemon/src/metrics.rs @@ -70,7 +70,12 @@ pub async fn render(d: &Daemon) -> String { "helios_scrub_blobs_total", "counter", "Blobs checked by the integrity scrub.", - &[("{result=\"ok\"}", get(&m.scrub_ok)), ("{result=\"corrupt\"}", get(&m.scrub_corrupt)), ("{result=\"missing\"}", get(&m.scrub_missing))], + &[ + ("{result=\"ok\"}", get(&m.scrub_ok)), + ("{result=\"corrupt\"}", get(&m.scrub_corrupt)), + ("{result=\"missing\"}", get(&m.scrub_missing)), + ("{result=\"unreadable\"}", get(&m.scrub_unreadable)), + ], ); line(&mut out, "helios_scrub_bytes_total", "counter", "Bytes read by the integrity scrub.", &[("", get(&m.scrub_bytes))]); line(&mut out, "helios_scrub_last_complete_timestamp_seconds", "gauge", "End of the last full scrub pass.", &[("", get(&m.scrub_last_complete))]); @@ -91,7 +96,16 @@ pub async fn serve(d: Arc, addr: SocketAddr) -> anyhow::Result<()> { let listener = tokio::net::TcpListener::bind(addr).await?; tracing::info!(url = %format_args!("http://{addr}/metrics"), "metrics listening"); loop { - let (stream, _) = listener.accept().await?; + // Accepting fails transiently (EMFILE, a connection reset before + // it was accepted); the endpoint must outlive that. + let stream = match listener.accept().await { + Ok((stream, _)) => stream, + Err(e) => { + tracing::warn!(error = %e, "accepting a metrics connection failed"); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + continue; + } + }; let d = d.clone(); tokio::spawn(async move { let svc = service_fn(move |req: http::Request| { diff --git a/crates/helios-daemon/src/scrub.rs b/crates/helios-daemon/src/scrub.rs index eb8f008..b494841 100644 --- a/crates/helios-daemon/src/scrub.rs +++ b/crates/helios-daemon/src/scrub.rs @@ -1,9 +1,13 @@ //! Integrity scrub: re-reads every blob, decompresses and re-hashes it, and //! compares the file hash, NAR hash and NAR size with the database. A blob //! that is missing or corrupt is quarantined, which unpublishes its paths -//! so clients never download bad data. Reads are rate limited. +//! so clients never download bad data. Reads are rate limited, and a pass +//! cut short resumes where it stopped. use std::io::Read; +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU64, Ordering}; use std::time::{Duration, Instant}; use helios_core::{Compression, Verifier}; @@ -33,27 +37,65 @@ enum Verdict { Ok, Missing, Corrupt(String), + /// The file is there but reading it failed (EIO, EACCES): not proof of + /// corruption, so it is counted and reported rather than quarantined. + Unreadable(String), + /// The daemon is shutting down. + Stopped, } -fn check(data_dir: &std::path::Path, b: &Blob) -> anyhow::Result { +/// Holds reads to `rate` bytes per second (0 = unlimited) over a pass. +struct Throttle { + rate: u64, + started: Instant, + bytes: AtomicU64, +} + +impl Throttle { + /// Accounts for `n` bytes read, sleeping until they are due; false + /// once the daemon is stopping. + fn take(&self, n: u64, stopping: &AtomicBool) -> bool { + let total = self.bytes.fetch_add(n, Ordering::Relaxed) + n; + loop { + if stopping.load(Ordering::Relaxed) { + return false; + } + if self.rate == 0 { + return true; + } + let due = Duration::from_secs_f64(total as f64 / self.rate as f64); + match due.checked_sub(self.started.elapsed()) { + Some(wait) if !wait.is_zero() => std::thread::sleep(wait.min(Duration::from_millis(100))), + _ => return true, + } + } + } +} + +fn check(data_dir: &Path, b: &Blob, throttle: &Throttle, stopping: &AtomicBool) -> anyhow::Result { let (Some(rel), Some(compression)) = (&b.path, Compression::parse(&b.compression)) else { return Ok(Verdict::Corrupt(format!("unsupported compression {}", b.compression))); }; let mut file = match std::fs::File::open(data_dir.join(rel)) { Ok(f) => f, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Verdict::Missing), - Err(e) => return Err(e.into()), + Err(e) => return Ok(Verdict::Unreadable(format!("opening: {e}"))), }; let mut v = Verifier::new(compression)?; let mut buf = vec![0u8; 1 << 20]; loop { - let n = file.read(&mut buf)?; - if n == 0 { - break; - } + let n = match file.read(&mut buf) { + Ok(0) => break, + Ok(n) => n, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => return Ok(Verdict::Unreadable(format!("reading: {e}"))), + }; if let Err(e) = v.update(&buf[..n]) { return Ok(Verdict::Corrupt(format!("undecodable: {e}"))); } + if !throttle.take(n as u64, stopping) { + return Ok(Verdict::Stopped); + } } let d = match v.finish() { Ok(d) => d, @@ -71,22 +113,84 @@ fn check(data_dir: &std::path::Path, b: &Blob) -> anyhow::Result { Ok(if problems.is_empty() { Verdict::Ok } else { Verdict::Corrupt(format!("{} mismatch", problems.join(", "))) }) } -/// One full pass over every blob. `rate` is bytes per second (0 = unlimited). +/// The id of the last blob checked in the current pass, so a pass cut +/// short (by a restart, or a server that stayed away) resumes where it +/// stopped. Kept in the state directory when there is one. +static CURSOR: AtomicI64 = AtomicI64::new(0); +const CURSOR_FILE: &str = "scrub-cursor"; + +fn load_cursor(d: &Daemon) -> i64 { + let Some(dir) = &d.state_dir else { return CURSOR.load(Ordering::Relaxed) }; + match std::fs::read_to_string(dir.join(CURSOR_FILE)) { + Ok(text) => text.trim().parse().unwrap_or(0), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => 0, + Err(e) => { + tracing::warn!(error = %e, "reading the scrub position failed; starting over"); + 0 + } + } +} + +fn save_cursor(d: &Daemon, id: i64) { + CURSOR.store(id, Ordering::Relaxed); + let Some(dir) = &d.state_dir else { return }; + let tmp = dir.join(format!(".{CURSOR_FILE}")); + if let Err(e) = std::fs::write(&tmp, format!("{id}\n")).and_then(|()| std::fs::rename(&tmp, dir.join(CURSOR_FILE))) { + tracing::warn!(error = %e, dir = %dir.display(), "saving the scrub position failed"); + } +} + +/// Whether a pass was cut short and should resume now rather than after a +/// full interval. +pub fn resuming(d: &Daemon) -> bool { + load_cursor(d) > 0 +} + +/// Retries `f` through a short server outage (a restart) before giving up. +async fn retrying(what: &str, mut f: F) -> anyhow::Result +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let mut delay = Duration::from_secs(2); + for attempt in 1.. { + match f().await { + Ok(v) => return Ok(v), + Err(e) if attempt < ATTEMPTS => { + tracing::warn!(error = format!("{e:#}"), retry_in = ?delay, "{what} failed"); + tokio::time::sleep(delay).await; + delay *= 2; + } + Err(e) => return Err(e), + } + } + unreachable!() +} + +const ATTEMPTS: u32 = 5; +const SAVE_EVERY: Duration = Duration::from_secs(10); + +/// One full pass over every blob, resuming one cut short. `rate` is bytes +/// per second (0 = unlimited). pub async fn run(d: &Daemon, rate: u64) -> anyhow::Result<()> { let started = Instant::now(); - let (mut after, mut bytes, mut blobs) = (0i64, 0u64, 0u64); + let throttle = Arc::new(Throttle { rate, started, bytes: AtomicU64::new(0) }); + let mut after = load_cursor(d); + if after > 0 { + tracing::info!(after, "resuming the scrub"); + } + let (mut bytes, mut blobs, mut saved) = (0u64, 0u64, Instant::now()); loop { - let page: Page = d.client.get(&format!("/v1/blobs?after={after}&limit=500")).await?; - let Some(last) = page.blobs.last() else { break }; - after = last.id; + let path = format!("/v1/blobs?after={after}&limit=500"); + let page: Page = retrying("listing blobs", || d.client.get(&path)).await?; + if page.blobs.is_empty() { + break; + } for b in page.blobs { - let dir = d.data_dir.clone(); - let blob = b.clone(); - let verdict = tokio::task::spawn_blocking(move || check(&dir, &blob)).await??; - blobs += 1; - bytes += b.file_size; - add(&d.metrics.scrub_bytes, b.file_size); + let (dir, blob, throttle, stopping) = (d.data_dir.clone(), b.clone(), throttle.clone(), d.stopping.clone()); + let verdict = tokio::task::spawn_blocking(move || check(&dir, &blob, &throttle, &stopping)).await??; let reason = match verdict { + Verdict::Stopped => return Ok(()), Verdict::Ok => { add(&d.metrics.scrub_ok, 1); None @@ -99,25 +203,91 @@ pub async fn run(d: &Daemon, rate: u64) -> anyhow::Result<()> { add(&d.metrics.scrub_corrupt, 1); Some(why) } + Verdict::Unreadable(why) => { + add(&d.metrics.scrub_unreadable, 1); + tracing::warn!(blob = %b.file_hash, reason = %why, "unreadable blob, skipping"); + None + } }; + blobs += 1; + bytes += b.file_size; + add(&d.metrics.scrub_bytes, b.file_size); if let Some(reason) = reason { tracing::error!(blob = %b.file_hash, %reason, "bad blob, quarantining"); + let body = json!({ "fileHash": b.file_hash, "reason": reason }); // A blob GC removed since it was listed is already gone. - if let Err(e) = d.client.post::("/v1/quarantine", json!({ "fileHash": b.file_hash, "reason": reason })).await - && !e.to_string().contains("404") - { - return Err(e); - } + retrying("quarantining", || async { + match d.client.post::("/v1/quarantine", body.clone()).await { + Err(e) if e.to_string().contains("404") => Ok(()), + r => r.map(drop), + } + }) + .await?; } - if rate > 0 { - let due = Duration::from_secs_f64(bytes as f64 / rate as f64); - if let Some(wait) = due.checked_sub(started.elapsed()) { - tokio::time::sleep(wait).await; - } + after = b.id; + if saved.elapsed() >= SAVE_EVERY { + save_cursor(d, after); + saved = Instant::now(); } } + save_cursor(d, after); } + save_cursor(d, 0); set(&d.metrics.scrub_last_complete, now()); tracing::info!(blobs, bytes, elapsed = ?started.elapsed(), "scrub finished"); Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + fn blob(path: &str) -> Blob { + Blob { + id: 1, + file_hash: String::new(), + compression: "zstd".into(), + file_size: 0, + nar_hash: String::new(), + nar_size: 0, + path: Some(path.into()), + } + } + + #[test] + fn read_errors_are_not_corruption() { + let dir = std::env::temp_dir().join(format!("helios-scrub-{}", helios_core::uuid_v4())); + std::fs::create_dir_all(dir.join("nar")).unwrap(); + let throttle = Throttle { rate: 0, started: Instant::now(), bytes: AtomicU64::new(0) }; + let stopping = AtomicBool::new(false); + // Reading a directory fails with EISDIR. + assert!(matches!(check(&dir, &blob("nar"), &throttle, &stopping).unwrap(), Verdict::Unreadable(_))); + assert!(matches!(check(&dir, &blob("gone"), &throttle, &stopping).unwrap(), Verdict::Missing)); + std::fs::write(dir.join("junk"), b"not zstd").unwrap(); + assert!(matches!(check(&dir, &blob("junk"), &throttle, &stopping).unwrap(), Verdict::Corrupt(_))); + stopping.store(true, Ordering::Relaxed); + assert!(!throttle.take(1, &stopping)); + std::fs::remove_dir_all(&dir).unwrap(); + } + + #[test] + fn the_scrub_position_survives_a_restart() { + let dir = std::env::temp_dir().join(format!("helios-scrub-{}", helios_core::uuid_v4())); + std::fs::create_dir_all(&dir).unwrap(); + let d = Daemon { + client: crate::client::Client::new(dir.join("sock")), + data_dir: dir.clone(), + state_dir: Some(dir.clone()), + metrics: Default::default(), + stopping: Default::default(), + }; + assert!(!resuming(&d)); + save_cursor(&d, 42); + CURSOR.store(0, Ordering::Relaxed); + assert_eq!(load_cursor(&d), 42); + assert!(resuming(&d)); + save_cursor(&d, 0); + assert!(!resuming(&d)); + std::fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/crates/helios-daemon/src/state.rs b/crates/helios-daemon/src/state.rs index 896c1df..7e859ff 100644 --- a/crates/helios-daemon/src/state.rs +++ b/crates/helios-daemon/src/state.rs @@ -1,5 +1,6 @@ use std::path::PathBuf; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use crate::client::Client; @@ -13,6 +14,7 @@ pub struct Metrics { pub scrub_ok: AtomicU64, pub scrub_corrupt: AtomicU64, pub scrub_missing: AtomicU64, + pub scrub_unreadable: AtomicU64, pub scrub_bytes: AtomicU64, pub scrub_last_complete: AtomicU64, pub db_checkpoints: AtomicU64, @@ -38,7 +40,11 @@ pub fn now() -> u64 { pub struct Daemon { pub client: Client, pub data_dir: PathBuf, + /// Where the daemon keeps its own state, such as the scrub position. + pub state_dir: Option, pub metrics: Metrics, + /// Set on shutdown, for long blocking work to stop early. + pub stopping: Arc, } /// Free and total bytes of the filesystem holding `path`. From 70e53462ce97ecc1938e971f87968942fbb13a49 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 43/56] nix: harden the module, image and CI - The secret options were types.path, so a path literal copied a signing key into the world-readable store. They are absolute path strings now, and a store path is refused. - Without nginx the module created an empty nginx user and failed evaluation. - The image ignored docker stop while starting: bash runs as PID 1 and set its trap only after starting the server. The VM test now stops the container during startup. - CI ran pull request code with permission to publish images. The test job is read-only with credentials not persisted, and a separate job builds and pushes images on push events only. - The daemon gets a state directory for its scrub position, and the module exposes the compression defaults as settings.compressionLevel and settings.compressionWindowLog. --- .github/workflows/ci.yml | 30 +++++++++++++++++++++++++++--- nix/docker-test.nix | 5 +++++ nix/docker.nix | 39 ++++++++++++++++++++++++++------------- nix/module.nix | 36 +++++++++++++++++++++++++++++++----- 4 files changed, 89 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76acf00..533028a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,9 +31,10 @@ jobs: runs-on: ${{ matrix.runner }} permissions: contents: read - packages: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - uses: DeterminateSystems/nix-installer-action@3138316df39ed29be04236d7ffc686fa525866aa # v23 with: # Upstream Nix, as users of the module and image run. @@ -67,8 +68,31 @@ jobs: if: matrix.system != 'x86_64-linux' run: nix build -L .#checks.${{ matrix.system }}.package .#docker + # Builds and pushes each architecture's image. Kept apart from the tests, + # which run pull request code, so only pushes ever hold a write token. + push: + if: github.event_name == 'push' + needs: test + strategy: + matrix: + include: + - runner: ubuntu-24.04 + arch: amd64 + - runner: ubuntu-24.04-arm + arch: arm64 + runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: DeterminateSystems/nix-installer-action@3138316df39ed29be04236d7ffc686fa525866aa # v23 + with: + determinate: false + - name: Push image - if: github.event_name == 'push' env: ARCH: ${{ matrix.arch }} TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -87,7 +111,7 @@ jobs: # Joins the per-architecture images into one multi-arch tag. image: if: github.event_name == 'push' - needs: test + needs: push runs-on: ubuntu-24.04 permissions: packages: write diff --git a/nix/docker-test.nix b/nix/docker-test.nix index 99f878c..01b5b61 100644 --- a/nix/docker-test.nix +++ b/nix/docker-test.nix @@ -56,5 +56,10 @@ pkgs.testers.runNixOSTest { # The daemon waited for the new server, not the last run's socket. machine.wait_until_succeeds("docker logs helios 2>&1 | grep -c 'helios-daemon started' | grep -qx 2") machine.fail("docker logs helios 2>&1 | grep -q 'Connection refused'") + + with subtest("docker stop during startup is clean"): + # Without a handler yet, PID 1 would ignore the TERM until the kill. + machine.succeed("docker stop helios && docker start helios && timeout 20 docker stop -t 60 helios") + assert machine.succeed("docker inspect -f '{{.State.ExitCode}}' helios").strip() == "0" ''; } diff --git a/nix/docker.nix b/nix/docker.nix index 9f9441e..f346c20 100644 --- a/nix/docker.nix +++ b/nix/docker.nix @@ -19,6 +19,21 @@ let data="''${HELIOS_DATA_DIR:-/var/lib/helios}" secrets="$data/secrets" + # docker stop: forward SIGTERM so both shut down cleanly. Installed + # first because bash is PID 1, which ignores signals without a + # handler, and a stop can come before either process has started. + stopping=0 + server= + daemon= + # shellcheck disable=SC2329 # invoked by the trap below + stop() { + stopping=1 + if [[ -n "$server$daemon" ]]; then + kill -TERM ''${server:+"$server"} ''${daemon:+"$daemon"} 2>/dev/null || true + fi + } + trap stop TERM INT + # Generate secrets on first start unless they are provided (for # example as Docker secrets under /run/secrets). if [[ -z "''${HELIOS_SIGNING_KEY_FILE:-}''${HELIOS_JWT_SECRET_FILE:-}''${HELIOS_ADMIN_SECRET_FILE:-}" ]]; then @@ -32,6 +47,9 @@ let if [[ "''${1:-serve}" != serve ]]; then exec "$@" fi + if (( stopping )); then + exit 0 + fi export HELIOS_ADMIN_SOCKET="''${HELIOS_ADMIN_SOCKET:-/run/helios/admin.sock}" # A restarted container keeps the last run's socket; without this the @@ -39,25 +57,20 @@ let rm -f "$HELIOS_ADMIN_SOCKET" helios-server & server=$! - daemon= + # The trap may have run between the fork and the assignment. + if (( stopping )); then stop; fi if [[ "''${HELIOS_DAEMON:-1}" != 0 ]]; then for _ in {1..100}; do - [[ -S "$HELIOS_ADMIN_SOCKET" ]] && break + if [[ -S "$HELIOS_ADMIN_SOCKET" ]] || (( stopping )); then break; fi sleep 0.1 done - helios-daemon --socket "$HELIOS_ADMIN_SOCKET" --data-dir "$data" & - daemon=$! + if (( ! stopping )); then + helios-daemon --socket "$HELIOS_ADMIN_SOCKET" --data-dir "$data" --state-dir "$data/daemon" & + daemon=$! + if (( stopping )); then stop; fi + fi fi - # docker stop: forward SIGTERM so both shut down cleanly. - stopping=0 - # shellcheck disable=SC2329 # invoked by the trap below - stop() { - stopping=1 - kill -TERM "$server" ''${daemon:+"$daemon"} 2>/dev/null || true - } - trap stop TERM INT - # If either process exits on its own, take the other down too. status=0 wait -n || status=$? diff --git a/nix/module.nix b/nix/module.nix index 43b7ebb..a78f858 100644 --- a/nix/module.nix +++ b/nix/module.nix @@ -23,6 +23,13 @@ let defaultDataDir = "/var/lib/helios"; secret = name: file: optional (file != null) "${name}:${file}"; + # Secrets are runtime paths given as strings: a path literal, or a string + # built from one, would put the secret in the world-readable store. + secretFileAssertion = option: file: { + assertion = file == null || (lib.hasPrefix "/" file && !(lib.hasPrefix "${builtins.storeDir}/" file)); + message = "services.helios.${option} must be an absolute path outside the Nix store, such as \"/run/secrets/helios\", given as a string: store paths are world-readable."; + }; + # The maintenance socket lives in a setgid directory, so it belongs to # group helios-admin: the daemon can reach it, nginx (group helios) cannot. adminDir = "/run/helios-admin"; @@ -108,7 +115,7 @@ in }; signingKeyFile = mkOption { - type = types.nullOr types.path; + type = types.nullOr types.str; default = null; description = '' Nix secret key (`nix key generate-secret`) used to sign narinfo. Read by @@ -118,13 +125,13 @@ in }; jwtSecretFile = mkOption { - type = types.nullOr types.path; + type = types.nullOr types.str; default = null; description = "HMAC secret for API tokens (at least 16 bytes). Without it no tokens can be issued or used."; }; adminSecretFile = mkOption { - type = types.nullOr types.path; + type = types.nullOr types.str; default = null; description = "Bearer secret for the admin API (at least 16 bytes). Without it the admin API is disabled."; }; @@ -172,6 +179,16 @@ in type = types.ints.unsigned; default = 30; }; + compressionLevel = mkOption { + type = types.ints.between 0 19; + default = 3; + description = "zstd level clients compress with unless told otherwise; 0 uploads uncompressed."; + }; + compressionWindowLog = mkOption { + type = types.addCheck types.int (n: n == 0 || (n >= 10 && n <= 27)); + default = 27; + description = "zstd long-distance matching window clients use, as a power of two (10-27), or 0 for the level's own."; + }; }; domain = mkOption { @@ -288,7 +305,7 @@ in example = "main"; }; tokenFile = mkOption { - type = types.path; + type = types.str; description = "File holding a push token for `cache`; read as a systemd credential."; }; jobs = mkOption { @@ -306,6 +323,9 @@ in assertion = cfg.nginx.enable -> cfg.domain != null; message = "services.helios.nginx.enable requires services.helios.domain"; } + (secretFileAssertion "signingKeyFile" cfg.signingKeyFile) + (secretFileAssertion "jwtSecretFile" cfg.jwtSecretFile) + (secretFileAssertion "adminSecretFile" cfg.adminSecretFile) ]; users.users.helios = { @@ -327,6 +347,8 @@ in HELIOS_MAX_UPLOAD_BYTES = toString cfg.settings.maxUploadBytes; HELIOS_GC_INTERVAL_HOURS = toString cfg.settings.gcIntervalHours; HELIOS_AUDIT_RETENTION_DAYS = toString cfg.settings.auditRetentionDays; + HELIOS_COMPRESSION_LEVEL = toString cfg.settings.compressionLevel; + HELIOS_COMPRESSION_WINDOW_LOG = toString cfg.settings.compressionWindowLog; } // lib.optionalAttrs (cfg.caches != { }) { HELIOS_CACHES = lib.concatStringsSep "," (lib.mapAttrsToList (name: c: if c.public then name else "${name}:private") cfg.caches); @@ -369,7 +391,7 @@ in systemd.tmpfiles.rules = mkIf (dataDir != defaultDataDir) [ "d '${dataDir}' 0750 helios helios -" ]; - users.users.nginx.extraGroups = mkIf cfg.nginx.enable [ "helios" ]; + users.users.nginx = mkIf cfg.nginx.enable { extraGroups = [ "helios" ]; }; services.nginx = mkIf cfg.nginx.enable { enable = true; @@ -421,6 +443,8 @@ in serviceConfig = { ExecStart = "${cfg.package}/bin/helios-daemon ${daemonArgs}"; User = "helios-daemon"; + # Where an unfinished scrub resumes after a restart. + StateDirectory = "helios-daemon"; Group = "helios-admin"; SupplementaryGroups = [ "helios" ]; Restart = "on-failure"; @@ -438,6 +462,8 @@ in }) (mkIf cfg.watchStore.enable { + assertions = [ (secretFileAssertion "watchStore.tokenFile" cfg.watchStore.tokenFile) ]; + users.users.helios-watch-store = { isSystemUser = true; group = "helios-watch-store"; From bdbe87ba389bb41ced58ab0e1776f878cf743a41 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Mon, 28 Sep 2026 19:59:56 +0200 Subject: [PATCH 44/56] docs: document compression and the codec benchmark bench/compression.sh compresses every NAR of a closure separately, as a cache serves them, with each codec a stock Nix can decode, and reports size, speed and round-trip correctness. The numbers behind the default come from it, and docs.md gains a compression section. --- bench/compression.sh | 108 +++++++++++++++++++++++++++++++++++++++++++ docs.md | 37 +++++++++++++-- 2 files changed, 141 insertions(+), 4 deletions(-) create mode 100755 bench/compression.sh diff --git a/bench/compression.sh b/bench/compression.sh new file mode 100755 index 0000000..6b5bf86 --- /dev/null +++ b/bench/compression.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Compares NAR compression codecs that Nix clients can decode (the narinfo +# `Compression` field), plus how much a closure dedups across NARs. +# Every NAR is compressed on its own, the way a cache serves it. Times are +# single-threaded CPU seconds (user+sys), summed over the closure; jobs run +# in parallel across NARs. +# +# Usage: bench/compression.sh [store-path] +# CODECS="zstd-3 xz-6" only these codecs (default: all below) +# JOBS=N parallel jobs (default: nproc) +# OUT=file.tsv keep the per-NAR results (default: discarded) +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +# Pin every codec to the flake's nixpkgs so runs are comparable. +if [ -z "${HELIOS_BENCH_SHELL:-}" ]; then + export HELIOS_BENCH_SHELL=1 + exec nix shell --inputs-from "$ROOT" nixpkgs#{zstd,xz,brotli,lz4,lzop,lzip,gzip,bzip2,coreutils,findutils,gawk} -c "$0" "$@" +fi + +TARGET="$(readlink -f "${1:-/run/current-system}")" +JOBS="${JOBS:-$(nproc)}" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +# name|narinfo Compression|compress|decompress. zstd windows stay at 2^27 +# and brotli at its default, the largest a stock Nix decoder accepts. +ALL=( + "none|none|cat|cat" + "lz4-1|lz4|lz4 -q -1 -c|lz4 -q -d -c" + "lz4-9|lz4|lz4 -q -9 -c|lz4 -q -d -c" + "lzop-1|lzop|lzop -q -1 -c|lzop -q -d -c" + "gzip-6|gzip|gzip -6 -c|gzip -d -c" + "gzip-9|gzip|gzip -9 -c|gzip -d -c" + "bzip2-9|bzip2|bzip2 -9 -c|bzip2 -d -c" + "zstd-1|zstd|zstd -q -T1 -1 -c|zstd -q -d -c" + "zstd-2|zstd|zstd -q -T1 -2 -c|zstd -q -d -c" + "zstd-3|zstd|zstd -q -T1 -3 -c|zstd -q -d -c" + "zstd-9|zstd|zstd -q -T1 -9 -c|zstd -q -d -c" + "zstd-19|zstd|zstd -q -T1 -19 -c|zstd -q -d -c" + "zstd-3-long|zstd|zstd -q -T1 -3 --long=27 -c|zstd -q -d -c" + "zstd-6-long|zstd|zstd -q -T1 -6 --long=27 -c|zstd -q -d -c" + "zstd-9-long|zstd|zstd -q -T1 -9 --long=27 -c|zstd -q -d -c" + "zstd-12-long|zstd|zstd -q -T1 -12 --long=27 -c|zstd -q -d -c" + "zstd-19-long|zstd|zstd -q -T1 -19 --long=27 -c|zstd -q -d -c" + "zstd-22-long|zstd|zstd -q -T1 --ultra -22 --long=27 -c|zstd -q -d -c" + "br-6|br|brotli -q 6 -c|brotli -d -c" + "br-11|br|brotli -q 11 -c|brotli -d -c" + "xz-6|xz|xz -T1 -6 -c|xz -d -c" + "xz-9e|xz|xz -T1 -9e -c|xz -d -c" + "lzip-6|lzip|lzip -q -6 -c|lzip -q -d -c" +) +: >"$WORK/codecs" +for c in "${ALL[@]}"; do + if [ -z "${CODECS:-}" ] || [[ " $CODECS " == *" ${c%%|*} "* ]]; then echo "$c" >>"$WORK/codecs"; fi +done + +echo "dumping closure of $TARGET..." >&2 +mkdir -p "$WORK/nar" +nix-store -qR "$TARGET" >"$WORK/paths" +xargs -P "$JOBS" -I{} sh -c 'nix-store --dump "$1" >"$2/nar/$(basename "$1").nar"' _ {} "$WORK" <"$WORK/paths" +RAW=$(du -cb "$WORK"/nar/*.nar | tail -1 | cut -f1) +echo "$(wc -l <"$WORK/paths") paths, $((RAW / 1048576)) MiB of NAR" >&2 + +# One job per (codec, NAR), largest NARs first so the pool drains evenly. +job() { + IFS='|' read -r name _ comp decomp <<<"$1" + local nar=$2 tmp; tmp="$(mktemp -p "$WORK")" + TIMEFORMAT='%3U %3S' + local ct dt + ct=$( { time $comp <"$nar" >"$tmp"; } 2>&1 ) + dt=$( { time $decomp <"$tmp" >/dev/null; } 2>&1 ) + $decomp <"$tmp" | cmp -s - "$nar" || { echo "roundtrip failed: $name $nar" >&2; exit 1; } + printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$name" "$(basename "$nar")" "$(stat -c%s "$nar")" "$(stat -c%s "$tmp")" \ + "$(awk '{print $1+$2}' <<<"$ct")" "$(awk '{print $1+$2}' <<<"$dt")" + rm -f "$tmp" +} +export -f job; export WORK +ls -S "$WORK"/nar/*.nar >"$WORK/nars" +while read -r c; do while read -r n; do printf '%s\0%s\0' "$c" "$n"; done <"$WORK/nars"; done <"$WORK/codecs" \ + | xargs -0 -n2 -P "$JOBS" bash -c 'job "$1" "$2"' _ >"$WORK/results.tsv" +[ -n "${OUT:-}" ] && cp "$WORK/results.tsv" "$OUT" + +echo +printf '%-14s %-6s %11s %7s %9s %12s %12s\n' codec narinfo "size MiB" ratio "vs zstd-3" "comp MB/s" "decomp MB/s" +awk -F'\t' -v order="$(cut -d'|' -f1,2 "$WORK/codecs" | tr '\n' ' ')" ' + { raw[$1]+=$3; out[$1]+=$4; ct[$1]+=$5; dt[$1]+=$6 } + END { + n = split(order, o, " "); base = ("zstd-3" in out) ? out["zstd-3"] : 0 + for (i = 1; i <= n; i++) { + split(o[i], f, "|"); c = f[1] + printf "%-14s %-6s %11.1f %7.2f %9s %12.0f %12.0f\n", c, f[2], out[c]/1048576, raw[c]/out[c], + base ? sprintf("%+.1f%%", (out[c]/base-1)*100) : "-", + ct[c] ? raw[c]/ct[c]/1e6 : 0, dt[c] ? raw[c]/dt[c]/1e6 : 0 + } + }' "$WORK/results.tsv" + +# Headroom beyond per-NAR compression, which only the storage side can use. +echo +echo "storage-side redundancy (not servable as-is):" +find "$WORK/nar" -name '*.nar' -print0 | xargs -0 cat | zstd -q -T"$JOBS" -9 --long=31 -c | wc -c \ + | awk -v raw="$RAW" '{printf " whole closure as one zstd-9 --long=31 stream: %.1f MiB (%.2fx)\n", $1/1048576, raw/$1}' +xargs -I{} find {} -type f -printf '%s\t%p\n' <"$WORK/paths" | LC_ALL=C sort -t $'\t' -k2,2 >"$WORK/sizes" +xargs -I{} find {} -type f -print0 <"$WORK/paths" | xargs -0 -n 2000 -P "$JOBS" sha256sum \ + | awk '{h=$1; $1=""; sub(/^ \*?/, ""); print $0 "\t" h}' | LC_ALL=C sort -t $'\t' -k1,1 >"$WORK/hashes" +LC_ALL=C join -t $'\t' -1 2 -2 1 "$WORK/sizes" "$WORK/hashes" \ + | awk -F'\t' '{ tot+=$2; if (!seen[$3]++) uniq+=$2 } END { + printf " regular files: %.1f MiB, %.1f MiB unique by content (%.1f%% duplicate)\n", tot/1048576, uniq/1048576, (1-uniq/tot)*100 }' diff --git a/docs.md b/docs.md index 4d31dd0..aee54c7 100644 --- a/docs.md +++ b/docs.md @@ -36,6 +36,8 @@ Every flag can also be set through its environment variable. | `--upload-grace-seconds` | `HELIOS_UPLOAD_GRACE_SECONDS` | `3600`: an unpublished upload is kept this long | | `--admin-socket` | `HELIOS_ADMIN_SOCKET` | unset: no maintenance API (see helios-daemon) | | `--caches` | `HELIOS_CACHES` | unset: caches to create at startup, such as `main,team:private`; a listed cache that exists takes the listed visibility, and unlisted caches are left alone | +| `--compression-level` | `HELIOS_COMPRESSION_LEVEL` | `3`: zstd level clients use unless told otherwise (0: uncompressed) | +| `--compression-window-log` | `HELIOS_COMPRESSION_WINDOW_LOG` | `27`: long-distance matching window clients use (0: the level's own) | Secret files must hold at least 16 bytes. @@ -118,7 +120,7 @@ placed by hand). Without `domain`, the module runs only the server on | `nginx.acme` | `true` | set `false` for plain HTTP or your own certificates | | `openFirewall` | on with nginx | opens 80 and 443 | | `logLevel` | `info` | | -| `settings.*` | | `narinfoCacheEntries`, `maxUploadBytes`, `gcIntervalHours`, `auditRetentionDays` | +| `settings.*` | | `narinfoCacheEntries`, `maxUploadBytes`, `gcIntervalHours`, `auditRetentionDays`, `compressionLevel`, `compressionWindowLog` | | `caches..public` | `true` | caches created at startup; see below | `services.helios.daemon` (on by default) runs helios-daemon beside the @@ -247,13 +249,15 @@ the case, and `--print` gives the `nix.settings` to use instead. ```bash helios push main /nix/store/...-hello # single paths helios push main --closure .#nixosConfigurations.host.config.system.build.toplevel -helios push main --closure /run/current-system --jobs 8 --level 3 +helios push main --closure /run/current-system --jobs 8 --level 9 ``` The CLI serialises, zstd-compresses and hashes each NAR in one pass and streams it straight into the upload, without temp files. It checks every -NAR hash against the Nix database. NARs the server already holds under any -cache are reused, and paths are published dependencies-first, in batches. +NAR hash against the Nix database. NARs the cache already holds, or that a +public cache serves, are reused instead of uploaded; a NAR only another +private cache holds is uploaded again, since reusing it would expose it. +Paths are published dependencies-first, in batches. A NAR larger than `--chunk-size` (MiB, default 32) is uploaded in chunks of that size, so pushes work through proxies that cap request bodies, such as @@ -262,6 +266,31 @@ retried on its own after a network or server error, and the server's offset decides whether it has to be sent again. `--chunk-size 0` streams every NAR in one request. +## Compression + +NARs are compressed with zstd at level 3 with long-distance matching over a +128 MiB window (`--window-log 27`). On an 8.2 GiB NixOS system closure that +stores 3357 MiB, against 3697 MiB for plain zstd level 2 and 3568 MiB for +level 3: 9% and 6% smaller. It costs about half the single-core speed of +level 2 (129 against 260 MB/s), but push compresses on every core, so even +a two-core machine stays ahead of most uplinks, and fewer bytes means a +faster push. + +| Level | Size vs level 2 | MB/s per core | +|-------|-----------------|---------------| +| 3, long window (default) | -9.2% | 129 | +| 6, long window | -12.2% | 63 | +| 9, long window | -14.3% | 33 | +| 19, long window | -21.2% | 3 | + +`--level` (0-19; 0 uploads uncompressed) and `--window-log` (10-27, or 0 +for the level's own window) override the server's defaults, which are set +with `HELIOS_COMPRESSION_LEVEL` and `HELIOS_COMPRESSION_WINDOW_LOG`; clients +without either flag use them. Every NAR is compressed with its exact size +pledged to zstd, so the window never exceeds the NAR and a decoder allocates +no more than the NAR needs. 27 is the largest window a stock Nix decoder +accepts. `bench/compression.sh` reproduces the measurements on any closure. + ## HTTP API Substituter endpoints (`GET`/`HEAD`): From 400c4cb992cf63ae2ff51ea80ff22667e3ee9d3f Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Sat, 3 Oct 2026 19:32:44 +0200 Subject: [PATCH 45/56] nix: update nixpkgs nixpkgs moves from 2026-04-18 to 2026-10-01: rustc 1.98.1, Nix 2.34.8, zstd 1.5.7, sqlite 3.53.3, GCC 16 and binutils 2.46; zig stays 0.16.0, still the newest. binutils 2.46 refuses to link libhelios into the Rust binaries (".eh_frame_hdr refers to overlapping FDEs"): the compiler-rt Zig bundled into the archive duplicates functions Rust's compiler_builtins already provides. Stop bundling it; those symbols come from compiler_builtins and libc as for the rest of the binary. --- core/build.zig | 2 +- flake.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/build.zig b/core/build.zig index 104b5ea..7e03ce0 100644 --- a/core/build.zig +++ b/core/build.zig @@ -23,7 +23,7 @@ pub fn build(b: *std.Build) void { } const lib = b.addLibrary(.{ .name = "helios", .linkage = .static, .root_module = mod }); - lib.bundle_compiler_rt = true; + lib.bundle_compiler_rt = false; lib.installHeader(b.path("include/helios.h"), "helios.h"); b.installArtifact(lib); } diff --git a/flake.lock b/flake.lock index dfdfdf9..12b201d 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1776548001, - "narHash": "sha256-ZSK0NL4a1BwVbbTBoSnWgbJy9HeZFXLYQizjb2DPF24=", + "lastModified": 1790822859, + "narHash": "sha256-69xHQhAeMAD2wDXO7T2pcOZIF9Sga2W+JkmY2a11Ops=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b12141ef619e0a9c1c84dc8c684040326f27cdcc", + "rev": "c59305bab2065cfecc4944690d9eedbb56f3a9fa", "type": "github" }, "original": { From fbdb6dca7362457ff4e3da8b886bc77d4df92621 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Sat, 3 Oct 2026 19:35:09 +0200 Subject: [PATCH 46/56] build: update rust dependencies Raise every requirement to the newest stable release and refresh the lock file: hyper 1.11, http 1.5, http-body-util 0.1.5, libc 0.2.190, rustls 0.23.45, rustls-native-certs 0.8.4, tokio 1.53.2 and tokio-rustls 0.26.6; cc, lazy_static and mio move with them. The rest were already current. libc 1.0 and rustls 0.24 are pre-releases and stay out. --- Cargo.lock | 24 ++++++++++++------------ crates/helios-cli/Cargo.toml | 14 +++++++------- crates/helios-daemon/Cargo.toml | 10 +++++----- crates/helios-server/Cargo.toml | 2 +- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c95edfa..e28ccef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -88,9 +88,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cc" -version = "1.5.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" dependencies = [ "find-msvc-tools", "shlex", @@ -432,15 +432,15 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "lazy_static" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" [[package]] name = "libc" -version = "0.2.189" +version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "libsqlite3-sys" @@ -473,9 +473,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mio" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi", @@ -790,9 +790,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.53.1" +version = "1.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" dependencies = [ "bytes", "libc", @@ -817,9 +817,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c85f2c3ef0b1cd58b36682f4b17aaa995f0e5db534d85692b4903abce21f67" +checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ "rustls", "tokio", diff --git a/crates/helios-cli/Cargo.toml b/crates/helios-cli/Cargo.toml index 5c12eef..67463c2 100644 --- a/crates/helios-cli/Cargo.toml +++ b/crates/helios-cli/Cargo.toml @@ -17,14 +17,14 @@ clap = { version = "4.6.7", default-features = false, features = ["std", "derive futures-util = { version = "0.3.34", default-features = false, features = ["std"] } helios-core = { path = "../helios-core" } helios-log = { path = "../helios-log" } -http = "1.4" -http-body-util = "0.1" +http = "1.5" +http-body-util = "0.1.5" # HTTP/1.1 client only; TLS via rustls with the ring provider (no aws-lc-rs). -hyper = { version = "1.8", default-features = false, features = ["client", "http1"] } -rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } -rustls-native-certs = "0.8" +hyper = { version = "1.11", default-features = false, features = ["client", "http1"] } +rustls = { version = "0.23.45", default-features = false, features = ["ring", "std", "tls12"] } +rustls-native-certs = "0.8.4" serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" -tokio = { version = "1.53.1", features = ["rt-multi-thread", "net", "sync", "process", "io-util", "time"] } +tokio = { version = "1.53.2", features = ["rt-multi-thread", "net", "sync", "process", "io-util", "time"] } tracing = { version = "0.1.44", default-features = false, features = ["std"] } -tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } +tokio-rustls = { version = "0.26.6", default-features = false, features = ["ring", "tls12"] } diff --git a/crates/helios-daemon/Cargo.toml b/crates/helios-daemon/Cargo.toml index 3292049..009e87a 100644 --- a/crates/helios-daemon/Cargo.toml +++ b/crates/helios-daemon/Cargo.toml @@ -13,11 +13,11 @@ clap = { version = "4.6.7", default-features = false, features = ["std", "derive futures-util = { version = "0.3.34", default-features = false, features = ["std"] } helios-core = { path = "../helios-core" } helios-log = { path = "../helios-log" } -http = "1.4" -http-body-util = "0.1" -hyper = { version = "1.8", default-features = false, features = ["client", "server", "http1"] } -libc = "0.2" +http = "1.5" +http-body-util = "0.1.5" +hyper = { version = "1.11", default-features = false, features = ["client", "server", "http1"] } +libc = "0.2.190" serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" -tokio = { version = "1.53.1", features = ["rt-multi-thread", "net", "time", "signal", "sync", "io-util"] } +tokio = { version = "1.53.2", features = ["rt-multi-thread", "net", "time", "signal", "sync", "io-util"] } tracing = { version = "0.1.44", default-features = false, features = ["std"] } diff --git a/crates/helios-server/Cargo.toml b/crates/helios-server/Cargo.toml index 701b220..8a6fd38 100644 --- a/crates/helios-server/Cargo.toml +++ b/crates/helios-server/Cargo.toml @@ -24,5 +24,5 @@ quick_cache = { version = "0.7.0", default-features = false } rusqlite = { version = "0.40.2", default-features = false, features = ["cache", "array", "fallible_uint"] } serde = { version = "1.0.229", features = ["derive"] } serde_json = "1.0.151" -tokio = { version = "1.53.1", features = ["rt-multi-thread", "net", "fs", "signal", "sync", "time", "io-util"] } +tokio = { version = "1.53.2", features = ["rt-multi-thread", "net", "fs", "signal", "sync", "time", "io-util"] } tracing = { version = "0.1.44", default-features = false, features = ["std"] } From 80b52615e4bb312d3d9c45c7d0e521352b235e40 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Sun, 4 Oct 2026 16:04:37 +0200 Subject: [PATCH 47/56] bench: fix the fixture lookup and the push token bench/flake-lock.sh had a stray "| head -1)" from an earlier edit and failed to find the nix-flake-lock fixtures. bench/run.sh created its push token with the default permissions, which became read-only. --- bench/flake-lock.sh | 2 +- bench/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/flake-lock.sh b/bench/flake-lock.sh index 46e0b30..95ff79b 100755 --- a/bench/flake-lock.sh +++ b/bench/flake-lock.sh @@ -19,7 +19,7 @@ RS="$ROOT/bench/flake-lock-rs/target/release/flake-lock-rs" ZIG="$ROOT/bench/zig/zig-out/bin/flake-lock" mkdir -p "$W/fixtures" "$W/real" "$W/fuzz" -UPSTREAM="$(find "${CARGO_HOME:-$HOME/.cargo}/registry/src" -path '*/nix-flake-lock-0.1.0/tests/fixtures' -type d | head -1)"| head -1)" +UPSTREAM="$(find "${CARGO_HOME:-$HOME/.cargo}/registry/src" -path '*/nix-flake-lock-0.1.0/tests/fixtures' -type d | head -1)" cp "$UPSTREAM"/*.lock "$W/fixtures/" "$RS" gen "$W/fixtures" i=0 diff --git a/bench/run.sh b/bench/run.sh index b9b9dad..c46f4d2 100755 --- a/bench/run.sh +++ b/bench/run.sh @@ -28,7 +28,7 @@ export XDG_CONFIG_HOME="$WORK/config" H="$ROOT/target/release/helios" "$H" login a http://127.0.0.1:18099 "$(cat "$WORK/secret")" >/dev/null "$H" cache create main >/dev/null -"$H" login ci http://127.0.0.1:18099 "$("$H" --server a token create bench --caches main 2>/dev/null | jq -r .token)" >/dev/null +"$H" login ci http://127.0.0.1:18099 "$("$H" --server a token create bench --caches main --perms push 2>/dev/null | jq -r .token)" >/dev/null "$H" push main -r "$TARGET" >/dev/null 2>&1 nix-store -qR "$TARGET" >"$WORK/paths" while read -r p; do printf '%s\0' "$p"; done <"$WORK/paths" >"$C/paths.bin" From a414405a0ade5312277b76823551045e709e1267 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Sun, 4 Oct 2026 16:38:27 +0200 Subject: [PATCH 48/56] build: move to zig 0.17 nixpkgs now packages zig 0.17.0 (zig_0_17; plain zig is still 0.16), so the flake builds and develops with it, and nixpkgs moves to 2026-10-03 for it. The port is mechanical: @intFromEnum to @backingInt, array repetition to @splat, dupeZ to dupeSentinel, std.meta.Int to @Int, stackFallback to BufferFirstAllocator, std.fmt.allocPrint to Allocator.print, and minimum_zig_version 0.17.0 in every package. Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 omitted. In the hot paths here that cost up to 10% (drv-parse, narinfo-render), so every module omits them again in ReleaseFast builds. --- AGENTS.md | 4 ++-- bench/zig/build.zig | 6 ++++++ bench/zig/build.zig.zon | 2 +- bench/zig/src/flake_lock.zig | 2 +- bench/zig/src/main.zig | 10 +++++----- core/build.zig | 3 +++ core/build.zig.zon | 2 +- crates/helios-core/build.rs | 2 +- flake.lock | 6 +++--- flake.nix | 4 ++-- pkg/nix-archive/build.zig | 3 +++ pkg/nix-archive/build.zig.zon | 2 +- pkg/nix-archive/src/walk.zig | 8 ++++---- pkg/nix-base32/build.zig | 3 +++ pkg/nix-base32/build.zig.zon | 2 +- pkg/nix-base32/src/root.zig | 10 +++++----- pkg/nix-derivation/build.zig | 3 +++ pkg/nix-derivation/build.zig.zon | 2 +- pkg/nix-derivation/src/root.zig | 4 ++-- pkg/nix-flake-lock/build.zig | 3 +++ pkg/nix-flake-lock/build.zig.zon | 2 +- pkg/nix-flake-lock/src/bump.zig | 2 +- pkg/nix-flake-lock/src/root.zig | 20 +++++++++++--------- pkg/nix-narinfo/build.zig | 3 +++ pkg/nix-narinfo/build.zig.zon | 2 +- pkg/nix-narinfo/src/parse.zig | 4 ++-- pkg/nix-narinfo/src/render.zig | 14 +++++++------- pkg/nix-store-path/build.zig | 3 +++ pkg/nix-store-path/build.zig.zon | 2 +- pkg/nix-store-path/src/root.zig | 4 ++-- 30 files changed, 83 insertions(+), 54 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4acf072..6a2b09c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -24,8 +24,8 @@ coarse, such as a whole NAR or narinfo per call. ## Tooling -- Use `nix develop` for zig 0.16, cargo, zstd and pkg-config. -- Zig 0.16 APIs changed a lot; check `zig env` for the std source rather +- Use `nix develop` for zig 0.17, cargo, zstd and pkg-config. +- Zig 0.17 APIs changed a lot; check `zig env` for the std source rather than relying on older examples. - System libraries: zstd and sqlite (pkg-config). - `HELIOS_ZIG_CPU` sets libhelios's `-Dcpu` (default `native`; the Nix diff --git a/bench/zig/build.zig b/bench/zig/build.zig index 19a9224..c72ba88 100644 --- a/bench/zig/build.zig +++ b/bench/zig/build.zig @@ -15,6 +15,9 @@ pub fn build(b: *std.Build) void { .root_source_file = b.path("src/main.zig"), .target = target, .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, .link_libc = true, }); inline for (packages) |p| mod.addImport(p[0], b.dependency(p[1], .{ .target = target, .optimize = optimize }).module(p[0])); @@ -25,6 +28,9 @@ pub fn build(b: *std.Build) void { .root_source_file = b.path("src/flake_lock.zig"), .target = target, .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, .link_libc = true, }); fl_mod.addImport("nix-flake-lock", b.dependency("nix_flake_lock", .{ .target = target, .optimize = optimize }).module("nix-flake-lock")); diff --git a/bench/zig/build.zig.zon b/bench/zig/build.zig.zon index 5cdca22..5e0b4b6 100644 --- a/bench/zig/build.zig.zon +++ b/bench/zig/build.zig.zon @@ -2,7 +2,7 @@ .name = .helios_bench, .version = "0.1.0", .fingerprint = 0xc67ae359dd92ff1e, - .minimum_zig_version = "0.16.0", + .minimum_zig_version = "0.17.0", .dependencies = .{ .nix_base32 = .{ .path = "../../pkg/nix-base32" }, .nix_store_path = .{ .path = "../../pkg/nix-store-path" }, diff --git a/bench/zig/src/flake_lock.zig b/bench/zig/src/flake_lock.zig index 89f9cf5..3026995 100644 --- a/bench/zig/src/flake_lock.zig +++ b/bench/zig/src/flake_lock.zig @@ -14,7 +14,7 @@ fn now() u64 { } fn readFile(path: []const u8) ![]u8 { - const z = try gpa.dupeZ(u8, path); + const z = try gpa.dupeSentinel(u8, path, 0); defer gpa.free(z); const fd: i32 = @intCast(@as(isize, @bitCast(linux.openat(linux.AT.FDCWD, z, .{ .ACCMODE = .RDONLY }, 0)))); if (fd < 0) return error.Open; diff --git a/bench/zig/src/main.zig b/bench/zig/src/main.zig index 92268d2..864a6d7 100644 --- a/bench/zig/src/main.zig +++ b/bench/zig/src/main.zig @@ -35,7 +35,7 @@ fn readFile(path: [:0]const u8) ![]u8 { } fn records(dir: []const u8, name: []const u8) ![]const []const u8 { - const path = try std.fmt.allocPrintSentinel(gpa, "{s}/{s}", .{ dir, name }, 0); + const path = try gpa.printSentinel("{s}/{s}", .{ dir, name }, 0); const data = try readFile(path); var out: std.ArrayList([]const u8) = .empty; var it = std.mem.splitScalar(u8, data, 0); @@ -184,7 +184,7 @@ fn narHash(c: *const Ctx) !void { for (c.paths) |p| { var sink = try archive.Sink.init(gpa, .{ .level = 0 }, discard, null); defer sink.deinit(gpa); - const z = try gpa.dupeZ(u8, p); + const z = try gpa.dupeSentinel(u8, p, 0); defer gpa.free(z); try archive.dump(&sink, z); std.mem.doNotOptimizeAway(try sink.finish()); @@ -208,9 +208,9 @@ pub fn main() !void { hashes32[i] = p.store_path[store_path.store_dir.len..][0..32]; hashes52[i] = p.nar_hash["sha256:".len..]; try base32.decode(&raw20[i], hashes32[i]); - fingerprints[i] = try std.fmt.allocPrint(gpa, "1;{s};{s};{d};{s}", .{ p.store_path, p.nar_hash, p.nar_size, p.references }); + fingerprints[i] = try gpa.print("1;{s};{s};{d};{s}", .{ p.store_path, p.nar_hash, p.nar_size, p.references }); } - const key = try readFile(try std.fmt.allocPrintSentinel(gpa, "{s}/key", .{dir}, 0)); + const key = try readFile(try gpa.printSentinel("{s}/key", .{dir}, 0)); const ctx: Ctx = .{ .narinfos = narinfos, .parsed = parsed, @@ -228,7 +228,7 @@ pub fn main() !void { for (ctx.paths) |p| { var sink = try archive.Sink.init(gpa, .{ .level = 0 }, discard, null); defer sink.deinit(gpa); - try archive.dump(&sink, try gpa.dupeZ(u8, p)); + try archive.dump(&sink, try gpa.dupeSentinel(u8, p, 0)); bytes += (try sink.finish()).nar_size; } const t0 = now(); diff --git a/core/build.zig b/core/build.zig index 7e03ce0..ce6d6ec 100644 --- a/core/build.zig +++ b/core/build.zig @@ -13,6 +13,9 @@ pub fn build(b: *std.Build) void { .root_source_file = b.path("src/root.zig"), .target = target, .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, .link_libc = true, // Linked into Rust binaries, which are PIE. .pic = true, diff --git a/core/build.zig.zon b/core/build.zig.zon index 07254df..72b2285 100644 --- a/core/build.zig.zon +++ b/core/build.zig.zon @@ -2,7 +2,7 @@ .name = .libhelios, .version = "0.1.0", .fingerprint = 0x27391ab28c1ecfe2, - .minimum_zig_version = "0.16.0", + .minimum_zig_version = "0.17.0", .dependencies = .{ .nix_base32 = .{ .path = "../pkg/nix-base32" }, .nix_store_path = .{ .path = "../pkg/nix-store-path" }, diff --git a/crates/helios-core/build.rs b/crates/helios-core/build.rs index 3b36cd9..4f4d013 100644 --- a/crates/helios-core/build.rs +++ b/crates/helios-core/build.rs @@ -43,7 +43,7 @@ fn main() { if env::var_os("ZIG_GLOBAL_CACHE_DIR").is_none() { cmd.env("ZIG_GLOBAL_CACHE_DIR", out.join("zig-global-cache")); } - let status = cmd.status().unwrap_or_else(|e| panic!("failed to run `{zig} build` (is zig 0.16 on PATH?): {e}")); + let status = cmd.status().unwrap_or_else(|e| panic!("failed to run `{zig} build` (is zig 0.17 on PATH?): {e}")); assert!(status.success(), "zig build failed for libhelios"); println!("cargo:rustc-link-search=native={}", prefix.join("lib").display()); diff --git a/flake.lock b/flake.lock index 12b201d..4b67419 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1790822859, - "narHash": "sha256-69xHQhAeMAD2wDXO7T2pcOZIF9Sga2W+JkmY2a11Ops=", + "lastModified": 1791048980, + "narHash": "sha256-KgItSKML8Xvte0B7/uGnBDsYzSnnKHcOaiUbgWBXLXw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c59305bab2065cfecc4944690d9eedbb56f3a9fa", + "rev": "a7868a727837f3c09cee2ce0ca671c76b1589fed", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index edd78d2..f7987eb 100644 --- a/flake.nix +++ b/flake.nix @@ -28,7 +28,7 @@ }; cargoLock.lockFile = ./Cargo.lock; - nativeBuildInputs = [ pkgs.zig pkgs.pkg-config ]; + nativeBuildInputs = [ pkgs.zig_0_17 pkgs.pkg-config ]; buildInputs = [ pkgs.zstd pkgs.sqlite ]; # Nix builds must not depend on the build machine's CPU. `baseline` @@ -68,7 +68,7 @@ devShells = forAllSystems (pkgs: { default = pkgs.mkShell { packages = [ - pkgs.zig + pkgs.zig_0_17 pkgs.cargo pkgs.rustc pkgs.clippy diff --git a/pkg/nix-archive/build.zig b/pkg/nix-archive/build.zig index 2da35ef..ad848ee 100644 --- a/pkg/nix-archive/build.zig +++ b/pkg/nix-archive/build.zig @@ -8,6 +8,9 @@ pub fn build(b: *std.Build) void { .root_source_file = b.path("src/root.zig"), .target = target, .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, }); mod.link_libc = true; diff --git a/pkg/nix-archive/build.zig.zon b/pkg/nix-archive/build.zig.zon index 440b90e..b3505b5 100644 --- a/pkg/nix-archive/build.zig.zon +++ b/pkg/nix-archive/build.zig.zon @@ -2,7 +2,7 @@ .name = .nix_archive, .version = "0.1.0", .fingerprint = 0xcb3a212dbdaff4ec, - .minimum_zig_version = "0.16.0", + .minimum_zig_version = "0.17.0", .dependencies = .{ }, .paths = .{ "build.zig", "build.zig.zon", "src" }, diff --git a/pkg/nix-archive/src/walk.zig b/pkg/nix-archive/src/walk.zig index d47e73f..4306ddf 100644 --- a/pkg/nix-archive/src/walk.zig +++ b/pkg/nix-archive/src/walk.zig @@ -24,7 +24,7 @@ const linux = std.os.linux; fn check(rc: usize) Error!usize { const e = linux.errno(rc); if (e == .SUCCESS) return rc; - last_errno = @intFromEnum(e); + last_errno = @backingInt(e); return error.Io; } @@ -73,7 +73,7 @@ const Writer = struct { } fn pad(self: *Writer, n: u64) Error!void { - const zeros = [_]u8{0} ** 8; + const zeros: [8]u8 = @splat(0); const rem: usize = @intCast(n % 8); if (rem != 0) try self.sink.write(zeros[0 .. 8 - rem]); } @@ -152,7 +152,7 @@ const Writer = struct { var entries: std.ArrayList(Entry) = .empty; const buf = self.dents; while (true) { - const n = try check(linux.getdents64(fd, buf.ptr, buf.len)); + const n = try check(linux.getdents64(fd, buf.ptr, @intCast(buf.len))); if (n == 0) break; var off: usize = 0; while (off < n) { @@ -161,7 +161,7 @@ const Writer = struct { const entry_name = std.mem.sliceTo(@as([*:0]const u8, @ptrCast(&buf[off + 19])), 0); off += reclen; if (std.mem.eql(u8, entry_name, ".") or std.mem.eql(u8, entry_name, "..")) continue; - try entries.append(self.arena, .{ .name = try self.arena.dupeZ(u8, entry_name), .dtype = dtype }); + try entries.append(self.arena, .{ .name = try self.arena.dupeSentinel(u8, entry_name, 0), .dtype = dtype }); } } diff --git a/pkg/nix-base32/build.zig b/pkg/nix-base32/build.zig index 1fe3eb0..dee0106 100644 --- a/pkg/nix-base32/build.zig +++ b/pkg/nix-base32/build.zig @@ -8,6 +8,9 @@ pub fn build(b: *std.Build) void { .root_source_file = b.path("src/root.zig"), .target = target, .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, }); const tests = b.addTest(.{ .root_module = mod }); diff --git a/pkg/nix-base32/build.zig.zon b/pkg/nix-base32/build.zig.zon index 938d1e5..21afb7c 100644 --- a/pkg/nix-base32/build.zig.zon +++ b/pkg/nix-base32/build.zig.zon @@ -2,7 +2,7 @@ .name = .nix_base32, .version = "0.1.0", .fingerprint = 0x9c08ba94cedadc5b, - .minimum_zig_version = "0.16.0", + .minimum_zig_version = "0.17.0", .dependencies = .{ }, .paths = .{ "build.zig", "build.zig.zon", "src" }, diff --git a/pkg/nix-base32/src/root.zig b/pkg/nix-base32/src/root.zig index fad33c3..6ee1049 100644 --- a/pkg/nix-base32/src/root.zig +++ b/pkg/nix-base32/src/root.zig @@ -12,7 +12,7 @@ pub const alphabet = "0123456789abcdfghijklmnpqrsvwxyz"; const invalid: u8 = 0xff; const reverse: [256]u8 = blk: { - var table = [_]u8{invalid} ** 256; + var table: [256]u8 = @splat(invalid); for (alphabet, 0..) |c, i| table[c] = @intCast(i); break :blk table; }; @@ -21,7 +21,7 @@ const reverse: [256]u8 = blk: { /// one AVX2 register (two SSE registers, one half of an AVX-512 one). pub const lanes = 32; const V = @Vector(lanes, u8); -const Mask = std.meta.Int(.unsigned, lanes); +const Mask = @Int(.unsigned, lanes); fn splat(c: u8) V { return @splat(c); @@ -190,16 +190,16 @@ test "store path hash length" { test "rejects invalid characters and overflow" { var out: [20]u8 = undefined; - try std.testing.expectError(error.InvalidNix32, decode(&out, "e" ** 32)); + try std.testing.expectError(error.InvalidNix32, decode(&out, &@as([32]u8, @splat('e')))); // 52 chars carry 260 bits; the top 4 must be zero for a 32-byte digest. var digest: [32]u8 = undefined; - try std.testing.expectError(error.InvalidNix32, decode(&digest, "z" ** 52)); + try std.testing.expectError(error.InvalidNix32, decode(&digest, &@as([52]u8, @splat('z')))); } test "vector classifier agrees with the table for every byte in every lane" { for (0..256) |c| { for (0..lanes) |lane| { - var in = [_]u8{'0'} ** lanes; + var in: [lanes]u8 = @splat('0'); in[lane] = @intCast(c); try std.testing.expectEqual(isValidScalar(&in), isValid(&in)); var d: V = undefined; diff --git a/pkg/nix-derivation/build.zig b/pkg/nix-derivation/build.zig index 570f659..95a2656 100644 --- a/pkg/nix-derivation/build.zig +++ b/pkg/nix-derivation/build.zig @@ -8,6 +8,9 @@ pub fn build(b: *std.Build) void { .root_source_file = b.path("src/root.zig"), .target = target, .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, }); const tests = b.addTest(.{ .root_module = mod }); diff --git a/pkg/nix-derivation/build.zig.zon b/pkg/nix-derivation/build.zig.zon index 12cfd89..a25f4d5 100644 --- a/pkg/nix-derivation/build.zig.zon +++ b/pkg/nix-derivation/build.zig.zon @@ -2,7 +2,7 @@ .name = .nix_derivation, .version = "0.1.0", .fingerprint = 0x7a251e78739a22ca, - .minimum_zig_version = "0.16.0", + .minimum_zig_version = "0.17.0", .dependencies = .{ }, .paths = .{ "build.zig", "build.zig.zon", "src" }, diff --git a/pkg/nix-derivation/src/root.zig b/pkg/nix-derivation/src/root.zig index 57e8016..4b22806 100644 --- a/pkg/nix-derivation/src/root.zig +++ b/pkg/nix-derivation/src/root.zig @@ -39,7 +39,7 @@ pub const Derivation = struct { const lanes = 32; const V = @Vector(lanes, u8); -const Mask = std.meta.Int(.unsigned, lanes); +const Mask = @Int(.unsigned, lanes); /// Index of the first '"' or '\\' at or after `start`. pub fn indexOfQuoteOrEscape(s: []const u8, start: usize) ?usize { @@ -197,7 +197,7 @@ test "parses a derivation with escapes" { } test "vector quote search matches scalar" { - var buf = [_]u8{'x'} ** 100; + var buf: [100]u8 = @splat('x'); for (0..buf.len) |i| { for ([_]u8{ '"', '\\' }) |c| { buf[i] = c; diff --git a/pkg/nix-flake-lock/build.zig b/pkg/nix-flake-lock/build.zig index 7987c9c..4d6b4eb 100644 --- a/pkg/nix-flake-lock/build.zig +++ b/pkg/nix-flake-lock/build.zig @@ -8,6 +8,9 @@ pub fn build(b: *std.Build) void { .root_source_file = b.path("src/root.zig"), .target = target, .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, }); const tests = b.addTest(.{ .root_module = mod }); diff --git a/pkg/nix-flake-lock/build.zig.zon b/pkg/nix-flake-lock/build.zig.zon index 1a4fa42..0693adc 100644 --- a/pkg/nix-flake-lock/build.zig.zon +++ b/pkg/nix-flake-lock/build.zig.zon @@ -2,7 +2,7 @@ .name = .nix_flake_lock, .version = "0.1.0", .fingerprint = 0x6174916c0100c5fd, - .minimum_zig_version = "0.16.0", + .minimum_zig_version = "0.17.0", .dependencies = .{ }, .paths = .{ "build.zig", "build.zig.zon", "src" }, diff --git a/pkg/nix-flake-lock/src/bump.zig b/pkg/nix-flake-lock/src/bump.zig index d09f75a..73ecf56 100644 --- a/pkg/nix-flake-lock/src/bump.zig +++ b/pkg/nix-flake-lock/src/bump.zig @@ -1,5 +1,5 @@ //! Single-threaded bump allocator. std.heap.ArenaAllocator is lock-free and -//! thread-safe in Zig 0.16, which puts atomics on every allocation; parsing +//! thread-safe since Zig 0.16, which puts atomics on every allocation; parsing //! allocates from one thread only, so plain pointer bumping is enough. //! Chunks are chained through a header at their start, so a document that //! fits in its first chunk costs exactly one malloc. diff --git a/pkg/nix-flake-lock/src/root.zig b/pkg/nix-flake-lock/src/root.zig index 10d1cec..19e89d6 100644 --- a/pkg/nix-flake-lock/src/root.zig +++ b/pkg/nix-flake-lock/src/root.zig @@ -118,8 +118,9 @@ pub const LockFile = struct { // Only `follows` edges can fail, and every parsed node is reachable, // so a graph without non-empty follows edges is valid as is. if (!self.hasFollows()) return; - var fallback = std.heap.stackFallback(4096, gpa); - var bump = try Bump.init(fallback.get(), 2 * 1024); + var first: [4096]u8 = undefined; + var fallback: std.heap.BufferFirstAllocator = .init(&first, gpa); + var bump = try Bump.init(fallback.allocator(), 2 * 1024); defer bump.deinit(); var r: Resolver = .init(bump.allocator(), self); const visited = try bump.alloc(bool, self.nodes.len); @@ -173,7 +174,7 @@ fn findSorted(comptime T: type, items: []const T, name: []const u8) ?usize { const lanes = 32; const V = @Vector(lanes, u8); -const Mask = std.meta.Int(.unsigned, lanes); +const Mask = @Int(.unsigned, lanes); inline fn splat(c: u8) V { return @splat(c); @@ -1136,7 +1137,7 @@ const Writer = struct { const gop = try suffix.getOrPut(tmp, in.name); if (!gop.found_existing) gop.value_ptr.* = 2; while (true) { - const candidate = try std.fmt.allocPrint(tmp, "{s}_{d}", .{ in.name, gop.value_ptr.* }); + const candidate = try tmp.print("{s}_{d}", .{ in.name, gop.value_ptr.* }); gop.value_ptr.* += 1; if (!taken(small, assigned.items, &used, candidate)) { name = candidate; @@ -1153,8 +1154,9 @@ const Writer = struct { } fn write(w: *Writer) Allocator.Error!void { - var fallback = std.heap.stackFallback(8192, w.gpa); - var bump = try Bump.init(fallback.get(), 4096); + var first: [8192]u8 = undefined; + var fallback: std.heap.BufferFirstAllocator = .init(&first, w.gpa); + var bump = try Bump.init(fallback.allocator(), 4096); defer bump.deinit(); const tmp = bump.allocator(); const nm = try w.names(tmp); @@ -1306,7 +1308,7 @@ test "rejects malformed and adversarial inputs" { .{ "{\"version\":7,\"root\":\"r\",\"nodes\":{\"r\":{\"inputs\":{\"a\":\"b\"}},\"b\":{\"locked\":{\"x\":null},\"original\":{}}}}", error.UnsupportedInputAttributeType }, }; inline for (cases) |c| try testing.expectError(c[1], parse(a, c[0], .{}, null)); - try testing.expectError(error.NestingTooDeep, parse(a, "{\"x\":" ++ "[" ** 300 ++ "]" ** 300 ++ "}", .{}, null)); + try testing.expectError(error.NestingTooDeep, parse(a, "{\"x\":" ++ &@as([300]u8, @splat('[')) ++ &@as([300]u8, @splat(']')) ++ "}", .{}, null)); } test "follows validation detects cycles and missing targets" { @@ -1336,7 +1338,7 @@ test "unicode escapes and string escaping on output" { } test "vector scanners agree with scalar definitions" { - var buf = [_]u8{'a'} ** 80; + var buf: [80]u8 = @splat('a'); for (0..buf.len) |i| { for ([_]u8{ '"', '\\', 0x01, 0x80, 0xff }) |c| { buf[i] = c; @@ -1344,7 +1346,7 @@ test "vector scanners agree with scalar definitions" { if (c < 0x80) try testing.expectEqual(i, scanEscape(&buf, 0)); buf[i] = 'a'; } - var sp = [_]u8{' '} ** 80; + var sp: [80]u8 = @splat(' '); sp[i] = 'x'; try testing.expectEqual(i, skipSpace(&sp, 0)); } diff --git a/pkg/nix-narinfo/build.zig b/pkg/nix-narinfo/build.zig index 8fe6d66..a591f02 100644 --- a/pkg/nix-narinfo/build.zig +++ b/pkg/nix-narinfo/build.zig @@ -8,6 +8,9 @@ pub fn build(b: *std.Build) void { .root_source_file = b.path("src/root.zig"), .target = target, .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, }); mod.addImport("nix-base32", b.dependency("nix_base32", .{ .target = target, .optimize = optimize }).module("nix-base32")); mod.addImport("nix-store-path", b.dependency("nix_store_path", .{ .target = target, .optimize = optimize }).module("nix-store-path")); diff --git a/pkg/nix-narinfo/build.zig.zon b/pkg/nix-narinfo/build.zig.zon index cf7be98..0752b8d 100644 --- a/pkg/nix-narinfo/build.zig.zon +++ b/pkg/nix-narinfo/build.zig.zon @@ -2,7 +2,7 @@ .name = .nix_narinfo, .version = "0.1.0", .fingerprint = 0xf254156c7cc4c9ba, - .minimum_zig_version = "0.16.0", + .minimum_zig_version = "0.17.0", .dependencies = .{ .nix_base32 = .{ .path = "../nix-base32" }, .nix_store_path = .{ .path = "../nix-store-path" }, diff --git a/pkg/nix-narinfo/src/parse.zig b/pkg/nix-narinfo/src/parse.zig index c32e4e0..be760ca 100644 --- a/pkg/nix-narinfo/src/parse.zig +++ b/pkg/nix-narinfo/src/parse.zig @@ -38,7 +38,7 @@ pub const NarInfo = struct { const lanes = 32; const V = @Vector(lanes, u8); -const Mask = std.meta.Int(.unsigned, lanes); +const Mask = @Int(.unsigned, lanes); pub fn indexOfNewline(s: []const u8, start: usize) ?usize { var i = start; @@ -147,7 +147,7 @@ test "parses a narinfo" { } test "vector newline search matches scalar" { - var buf = [_]u8{'x'} ** 100; + var buf: [100]u8 = @splat('x'); for (0..buf.len) |i| { buf[i] = '\n'; for (0..i + 1) |start| try std.testing.expectEqual(@as(?usize, i), indexOfNewline(&buf, start)); diff --git a/pkg/nix-narinfo/src/render.zig b/pkg/nix-narinfo/src/render.zig index 6512b5d..563706c 100644 --- a/pkg/nix-narinfo/src/render.zig +++ b/pkg/nix-narinfo/src/render.zig @@ -251,8 +251,8 @@ test "renders and signs a narinfo" { // Secret key from `nix key generate-secret --key-name test-1`. const signer = try Signer.parse(a, "test-1:suJsWimvBNFUIc0JJE18OfOMygH/f2GuTC2/XwD6rPKkV+1VKilMIkXl6Ax9hqeKpUF/BSOH+Fqng4tqZlirhw=="); defer signer.destroy(a); - const nar = [_]u8{1} ** 32; - const file = [_]u8{2} ** 32; + const nar: [32]u8 = @splat(1); + const file: [32]u8 = @splat(2); const text = try render(a, .{ .store_path = "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello", .nar_hash = &nar, @@ -311,12 +311,12 @@ test "generated keys parse and sign" { const a = std.testing.allocator; var text: std.ArrayList(u8) = .empty; defer text.deinit(a); - try Signer.generate(&text, a, "gen-1", [_]u8{42} ** 32); + try Signer.generate(&text, a, "gen-1", @as([32]u8, @splat(42))); const signer = try Signer.parse(a, text.items); defer signer.destroy(a); const theirs = (try signer.key_pair.sign("msg", null)).toBytes(); try std.testing.expectEqualSlices(u8, &theirs, &(try signer.signRaw("msg"))); - try std.testing.expectError(error.InvalidKey, Signer.generate(&text, a, "bad:name", [_]u8{1} ** 32)); + try std.testing.expectError(error.InvalidKey, Signer.generate(&text, a, "bad:name", @as([32]u8, @splat(1)))); } test "rejects a secret key whose public half does not match its seed" { @@ -326,7 +326,7 @@ test "rejects a secret key whose public half does not match its seed" { var raw: [64]u8 = undefined; try std.base64.standard.Decoder.decode(&raw, good[prefix.len..]); // A valid point, but the public key of another seed. - raw[32..64].* = (try Ed25519.KeyPair.generateDeterministic([_]u8{7} ** 32)).public_key.toBytes(); + raw[32..64].* = (try Ed25519.KeyPair.generateDeterministic(@as([32]u8, @splat(7)))).public_key.toBytes(); var text: [good.len]u8 = undefined; @memcpy(text[0..prefix.len], prefix); _ = std.base64.standard.Encoder.encode(text[prefix.len..], &raw); @@ -337,8 +337,8 @@ test "duplicate references are rendered and signed once" { const a = std.testing.allocator; const signer = try Signer.parse(a, "test-1:suJsWimvBNFUIc0JJE18OfOMygH/f2GuTC2/XwD6rPKkV+1VKilMIkXl6Ax9hqeKpUF/BSOH+Fqng4tqZlirhw=="); defer signer.destroy(a); - const nar = [_]u8{1} ** 32; - const file = [_]u8{2} ** 32; + const nar: [32]u8 = @splat(1); + const file: [32]u8 = @splat(2); var in: Input = .{ .store_path = "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello", .nar_hash = &nar, diff --git a/pkg/nix-store-path/build.zig b/pkg/nix-store-path/build.zig index 44b7bcf..eaa2a57 100644 --- a/pkg/nix-store-path/build.zig +++ b/pkg/nix-store-path/build.zig @@ -8,6 +8,9 @@ pub fn build(b: *std.Build) void { .root_source_file = b.path("src/root.zig"), .target = target, .optimize = optimize, + // Zig 0.17 keeps frame pointers in ReleaseFast, which 0.16 dropped; + // they cost up to 10% in the hot paths here. + .omit_frame_pointer = if (optimize == .fast) true else null, }); mod.addImport("nix-base32", b.dependency("nix_base32", .{ .target = target, .optimize = optimize }).module("nix-base32")); diff --git a/pkg/nix-store-path/build.zig.zon b/pkg/nix-store-path/build.zig.zon index 55b9cd1..3508b82 100644 --- a/pkg/nix-store-path/build.zig.zon +++ b/pkg/nix-store-path/build.zig.zon @@ -2,7 +2,7 @@ .name = .nix_store_path, .version = "0.1.0", .fingerprint = 0xdcfce087c7a4494d, - .minimum_zig_version = "0.16.0", + .minimum_zig_version = "0.17.0", .dependencies = .{ .nix_base32 = .{ .path = "../nix-base32" }, }, diff --git a/pkg/nix-store-path/src/root.zig b/pkg/nix-store-path/src/root.zig index 9e571d1..5843d30 100644 --- a/pkg/nix-store-path/src/root.zig +++ b/pkg/nix-store-path/src/root.zig @@ -13,7 +13,7 @@ pub const max_name_len = 211; const lanes = 32; const V = @Vector(lanes, u8); -const Mask = std.meta.Int(.unsigned, lanes); +const Mask = @Int(.unsigned, lanes); fn splat(c: u8) V { return @splat(c); @@ -88,7 +88,7 @@ test "validates base names" { test "vector name check agrees with scalar for every byte in every lane" { for (0..256) |c| { for (0..lanes + 3) |pos| { - var name = [_]u8{'a'} ** (lanes + 3); + var name: [lanes + 3]u8 = @splat('a'); name[pos] = @intCast(c); try std.testing.expectEqual(isValidNameScalar(&name), isValidName(&name)); } From cafe1d932ce48f2d864281872c3c39d3aeed9578 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Sun, 4 Oct 2026 16:38:27 +0200 Subject: [PATCH 49/56] bench: escape benchmark outputs through one barrier Results were passed to doNotOptimizeAway by value. For an array of 64 bytes or less that is one register barrier per byte, 20 or 32 asm statements around a 5 ns routine, so the harness timed its own barriers: under Zig 0.17 base32-encode-20 seemed twice as slow while its machine code was identical. A pointer barrier makes the output escape once. --- bench/zig/src/main.zig | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/bench/zig/src/main.zig b/bench/zig/src/main.zig index 864a6d7..15e70b3 100644 --- a/bench/zig/src/main.zig +++ b/bench/zig/src/main.zig @@ -120,7 +120,7 @@ fn decode32(c: *const Ctx) !void { for (c.hashes32) |h| { var out: [20]u8 = undefined; try base32.decode(&out, h); - std.mem.doNotOptimizeAway(out); + std.mem.doNotOptimizeAway(&out); } } @@ -128,7 +128,7 @@ fn decode32Scalar(c: *const Ctx) !void { for (c.hashes32) |h| { var out: [20]u8 = undefined; try base32.decodeScalar(&out, h); - std.mem.doNotOptimizeAway(out); + std.mem.doNotOptimizeAway(&out); } } @@ -136,7 +136,7 @@ fn decode52(c: *const Ctx) !void { for (c.hashes52) |h| { var out: [32]u8 = undefined; try base32.decode(&out, h); - std.mem.doNotOptimizeAway(out); + std.mem.doNotOptimizeAway(&out); } } @@ -144,14 +144,17 @@ fn encode20(c: *const Ctx) !void { for (c.raw20) |*r| { var out: [32]u8 = undefined; base32.encode(&out, r); - std.mem.doNotOptimizeAway(out); + std.mem.doNotOptimizeAway(&out); } } fn storePathParse(c: *const Ctx) !void { for (c.paths) |p| { const base = store_path.baseName(p) orelse return error.Invalid; - std.mem.doNotOptimizeAway(store_path.hashOf(base)); + { + const h = store_path.hashOf(base); + std.mem.doNotOptimizeAway(&h); + } } } @@ -165,7 +168,7 @@ fn encode20Scalar(c: *const Ctx) !void { for (c.raw20) |*r| { var out: [32]u8 = undefined; base32.encodeScalar(&out, r); - std.mem.doNotOptimizeAway(out); + std.mem.doNotOptimizeAway(&out); } } From a2a0d64fe69e8f6cc71c421567bc9376a07c6aba Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Sun, 4 Oct 2026 16:38:35 +0200 Subject: [PATCH 50/56] pkg: recover zig 0.17 regressions in nix32 and flake.lock Measured against zig 0.16 (median of five runs, same corpus): - nix32 decode packed each 8-digit group by extracting digits from a vector register one byte at a time, which LLVM 22 schedules worse: base32-decode-20 went from 7.2 to 9.1 ns. One big-endian u64 load and three mask-and-shift steps now pack a group without extracts: 6.9 ns, and base32-decode-32 16.3 to 13.1 ns. - Zig 0.17 inlines the vector name check into baseName, which made store path parsing 7% slower than calling it. It stays out of line. - flake.lock validation on 8000-input documents was 13-17% slower in the follows resolver's hash maps. Edges are now numbered (node, input) through a prefix sum, so cycle detection and the target cache are plain arrays: validation is 0-8% faster than under 0.16, and output is still identical to the nix-flake-lock crate on 20302 documents. --- pkg/nix-base32/src/root.zig | 11 ++++- pkg/nix-flake-lock/src/root.zig | 84 +++++++++++++++++++++------------ pkg/nix-store-path/src/root.zig | 4 +- 3 files changed, 65 insertions(+), 34 deletions(-) diff --git a/pkg/nix-base32/src/root.zig b/pkg/nix-base32/src/root.zig index 6ee1049..10ad36c 100644 --- a/pkg/nix-base32/src/root.zig +++ b/pkg/nix-base32/src/root.zig @@ -110,8 +110,15 @@ fn pack(out: []u8, d: []const u8) DecodeError!void { const len = d.len; var k: usize = 0; while (8 * k + 8 <= len and 5 * k + 5 <= out.len) : (k += 1) { - var v: u40 = 0; - inline for (0..8) |i| v |= @as(u40, d[len - 1 - (8 * k + i)]) << (5 * i); + // One load puts digit 8k+i in byte i (big-endian, as the digits run + // most significant first). Three mask-and-shift steps then close + // the 3-bit gaps between them: pairs into 10 bits, pairs of those + // into 20, then the two halves into 40. Unlike shifting each digit + // in, this does not depend on how the backend extracts bytes. + const word = std.mem.readInt(u64, d[len - 8 - 8 * k ..][0..8], .big); + const pairs = (word & 0x001F001F001F001F) | ((word & 0x1F001F001F001F00) >> 3); + const quads = (pairs & 0x000003FF000003FF) | ((pairs & 0x03FF000003FF0000) >> 6); + const v: u40 = @truncate((quads & 0xFFFFF) | ((quads >> 32) << 20)); std.mem.writeInt(u40, out[5 * k ..][0..5], v, .little); } @memset(out[5 * k ..], 0); diff --git a/pkg/nix-flake-lock/src/root.zig b/pkg/nix-flake-lock/src/root.zig index 19e89d6..70dc36a 100644 --- a/pkg/nix-flake-lock/src/root.zig +++ b/pkg/nix-flake-lock/src/root.zig @@ -100,7 +100,7 @@ pub const LockFile = struct { nodes: []const Node, pub fn resolve(self: *const LockFile, gpa: Allocator, path: Path) ValidationError!?u32 { - var r: Resolver = .init(gpa, self); + var r: Resolver = try .init(gpa, self); defer r.deinit(); return r.resolve(path, false); } @@ -122,7 +122,7 @@ pub const LockFile = struct { var fallback: std.heap.BufferFirstAllocator = .init(&first, gpa); var bump = try Bump.init(fallback.allocator(), 2 * 1024); defer bump.deinit(); - var r: Resolver = .init(bump.allocator(), self); + var r: Resolver = try .init(bump.allocator(), self); const visited = try bump.alloc(bool, self.nodes.len); @memset(visited, false); visited[root_id] = true; @@ -835,64 +835,86 @@ fn build(p: *Parser, version: u8, root_key: []const u8, wire: []WireNode) ParseE // ── follows resolution ── -const EdgeKey = struct { node: u32, input: u32 }; - const Resolver = struct { gpa: Allocator, lock: *const LockFile, - /// Resolved follows targets, keyed by the path slice's address. - cache: std.AutoHashMapUnmanaged(usize, ?u32) = .empty, - active: std.AutoHashMapUnmanaged(EdgeKey, void) = .empty, - - const Step = union(enum) { component: []const u8, finish: struct { edge: EdgeKey, target: usize } }; - - fn init(gpa: Allocator, lock: *const LockFile) Resolver { - return .{ .gpa = gpa, .lock = lock }; + /// Input `i` of node `n` is edge `first_input[n] + i`, so the per-edge + /// state below is plain arrays: no hashing on the resolve path. + first_input: []u32, + /// Edges on the chain being resolved, for cycle detection. + active: []bool, + /// Resolved follows target per edge, or `unknown`. + cache: []u32, + steps: std.ArrayList(Step) = .empty, + + const unknown = std.math.maxInt(u32); + + const Step = union(enum) { component: []const u8, finish: u32 }; + + fn init(gpa: Allocator, lock: *const LockFile) Allocator.Error!Resolver { + const first_input = try gpa.alloc(u32, lock.nodes.len); + errdefer gpa.free(first_input); + var edges: u32 = 0; + for (lock.nodes, first_input) |n, *first| { + first.* = edges; + edges += @intCast(n.inputs.len); + } + const active = try gpa.alloc(bool, edges); + errdefer gpa.free(active); + @memset(active, false); + const cache = try gpa.alloc(u32, edges); + @memset(cache, unknown); + return .{ .gpa = gpa, .lock = lock, .first_input = first_input, .active = active, .cache = cache }; } fn deinit(r: *Resolver) void { - r.cache.deinit(r.gpa); - r.active.deinit(r.gpa); + r.gpa.free(r.first_input); + r.gpa.free(r.active); + r.gpa.free(r.cache); + r.steps.deinit(r.gpa); } fn resolve(r: *Resolver, path: Path, use_cache: bool) ValidationError!?u32 { - var steps: std.ArrayList(Step) = .empty; - defer steps.deinit(r.gpa); + r.steps.clearRetainingCapacity(); + // Every active edge has its finish step still on the stack; an early + // return clears them so the next call starts clean. + defer for (r.steps.items) |step| switch (step) { + .finish => |e| r.active[e] = false, + .component => {}, + }; var i = path.len; while (i > 0) { i -= 1; - try steps.append(r.gpa, .{ .component = path[i] }); + try r.steps.append(r.gpa, .{ .component = path[i] }); } var current: u32 = root_id; - r.active.clearRetainingCapacity(); - while (steps.pop()) |step| switch (step) { + while (r.steps.pop()) |step| switch (step) { .component => |name| { const node = &r.lock.nodes[current]; const idx = findSorted(Input, node.inputs, name) orelse return null; switch (node.inputs[idx].edge) { .node => |child| current = child, .follows => |target| { - const addr = @intFromPtr(target.ptr); - if (use_cache) if (r.cache.get(addr)) |cached| { - current = cached orelse return null; + const edge = r.first_input[current] + @as(u32, @intCast(idx)); + if (use_cache and r.cache[edge] != unknown) { + current = r.cache[edge]; continue; - }; - const edge: EdgeKey = .{ .node = current, .input = @intCast(idx) }; - const gop = try r.active.getOrPut(r.gpa, edge); - if (gop.found_existing) return error.FollowsCycle; - try steps.append(r.gpa, .{ .finish = .{ .edge = edge, .target = addr } }); + } + if (r.active[edge]) return error.FollowsCycle; + r.active[edge] = true; + try r.steps.append(r.gpa, .{ .finish = edge }); var j = target.len; while (j > 0) { j -= 1; - try steps.append(r.gpa, .{ .component = target[j] }); + try r.steps.append(r.gpa, .{ .component = target[j] }); } current = root_id; }, } }, - .finish => |f| { - _ = r.active.remove(f.edge); - if (use_cache) try r.cache.put(r.gpa, f.target, current); + .finish => |edge| { + r.active[edge] = false; + if (use_cache) r.cache[edge] = current; }, }; return current; diff --git a/pkg/nix-store-path/src/root.zig b/pkg/nix-store-path/src/root.zig index 5843d30..e73c8a4 100644 --- a/pkg/nix-store-path/src/root.zig +++ b/pkg/nix-store-path/src/root.zig @@ -39,7 +39,9 @@ fn isNameChar(c: u8) bool { }; } -pub fn isValidName(name: []const u8) bool { +// Out of line: Zig 0.17 inlines this vector loop into callers such as +// baseName, which made store path parsing about 7% slower than a call. +pub noinline fn isValidName(name: []const u8) bool { if (name.len == 0 or name.len > max_name_len or name[0] == '.') return false; var i: usize = 0; while (i + lanes <= name.len) : (i += lanes) { From ad9bfa447dd60e89224372c4c5e6405899302b2a Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Sun, 4 Oct 2026 19:39:04 +0200 Subject: [PATCH 51/56] pkg: replace api zig 0.17 deprecates The 0.17 port compiled, but still used what 0.17 deprecates. Checked against every declaration the 0.17 std documents as deprecated and the release notes: - builtin.cpu and builtin.os become builtin.target.cpu and .os - std.mem.indexOf and indexOfScalar become find and findScalar, and our own indexOfNewline and indexOfQuoteOrEscape follow as findNewline and findQuoteOrEscape - std.fmt.bufPrint becomes std.mem.print - std.fs.path becomes std.Io.Dir.path - std.unicode.utf8Decode, deprecated as easy to misuse, becomes the length-specific utf8Decode2/3/4; flake.lock output is still identical to the nix-flake-lock crate on 20310 documents Benchmarks are unchanged. --- bench/zig/src/flake_lock.zig | 2 +- core/src/root.zig | 2 +- pkg/nix-archive/src/sha256.zig | 4 ++-- pkg/nix-archive/src/walk.zig | 4 ++-- pkg/nix-derivation/src/root.zig | 8 ++++---- pkg/nix-flake-lock/src/root.zig | 22 ++++++++++++++-------- pkg/nix-narinfo/src/parse.zig | 10 +++++----- pkg/nix-narinfo/src/render.zig | 12 ++++++------ 8 files changed, 35 insertions(+), 29 deletions(-) diff --git a/bench/zig/src/flake_lock.zig b/bench/zig/src/flake_lock.zig index 3026995..b8c04e3 100644 --- a/bench/zig/src/flake_lock.zig +++ b/bench/zig/src/flake_lock.zig @@ -100,7 +100,7 @@ pub fn main() !void { if (out.items.len > 1 << 20) flush(&out); continue; } - const name = std.fs.path.basename(path); + const name = std.Io.Dir.path.basename(path); const iters = iterations(bytes.len); var doc = try fl.parse(gpa, bytes, .{}, null); defer doc.deinit(); diff --git a/core/src/root.zig b/core/src/root.zig index af3443d..6868026 100644 --- a/core/src/root.zig +++ b/core/src/root.zig @@ -259,7 +259,7 @@ export fn hl_hmac_sha256(key: [*]const u8, key_len: usize, msg: [*]const u8, msg /// Fills `buf` from the OS CSPRNG. export fn hl_random(buf: [*]u8, len: usize) c_int { - if (builtin.os.tag == .linux) { + if (builtin.target.os.tag == .linux) { const linux = std.os.linux; var off: usize = 0; while (off < len) { diff --git a/pkg/nix-archive/src/sha256.zig b/pkg/nix-archive/src/sha256.zig index d0732db..fa37c27 100644 --- a/pkg/nix-archive/src/sha256.zig +++ b/pkg/nix-archive/src/sha256.zig @@ -10,9 +10,9 @@ const std = @import("std"); const builtin = @import("builtin"); -pub const accelerated = builtin.cpu.arch == .x86_64 and +pub const accelerated = builtin.target.cpu.arch == .x86_64 and builtin.zig_backend != .stage2_c and - builtin.cpu.hasAll(.x86, &.{ .sha, .avx2 }); + builtin.target.cpu.hasAll(.x86, &.{ .sha, .avx2 }); pub const Sha256 = if (accelerated) Fast else std.crypto.hash.sha2.Sha256; diff --git a/pkg/nix-archive/src/walk.zig b/pkg/nix-archive/src/walk.zig index 4306ddf..21e1107 100644 --- a/pkg/nix-archive/src/walk.zig +++ b/pkg/nix-archive/src/walk.zig @@ -7,7 +7,7 @@ const std = @import("std"); const builtin = @import("builtin"); const Sink = @import("sink.zig").Sink; -pub const supported = builtin.os.tag == .linux; +pub const supported = builtin.target.os.tag == .linux; pub const Error = error{ Io, @@ -156,7 +156,7 @@ const Writer = struct { if (n == 0) break; var off: usize = 0; while (off < n) { - const reclen = std.mem.readInt(u16, buf[off + 16 ..][0..2], builtin.cpu.arch.endian()); + const reclen = std.mem.readInt(u16, buf[off + 16 ..][0..2], builtin.target.cpu.arch.endian()); const dtype = buf[off + 18]; const entry_name = std.mem.sliceTo(@as([*:0]const u8, @ptrCast(&buf[off + 19])), 0); off += reclen; diff --git a/pkg/nix-derivation/src/root.zig b/pkg/nix-derivation/src/root.zig index 4b22806..457b903 100644 --- a/pkg/nix-derivation/src/root.zig +++ b/pkg/nix-derivation/src/root.zig @@ -42,7 +42,7 @@ const V = @Vector(lanes, u8); const Mask = @Int(.unsigned, lanes); /// Index of the first '"' or '\\' at or after `start`. -pub fn indexOfQuoteOrEscape(s: []const u8, start: usize) ?usize { +pub fn findQuoteOrEscape(s: []const u8, start: usize) ?usize { var i = start; const quote: V = @splat('"'); const backslash: V = @splat('\\'); @@ -73,7 +73,7 @@ const Parser = struct { fn string(p: *Parser) Error![]const u8 { try p.expect("\""); const start = p.pos; - var i = indexOfQuoteOrEscape(p.s, start) orelse return error.Syntax; + var i = findQuoteOrEscape(p.s, start) orelse return error.Syntax; if (p.s[i] == '"') { p.pos = i + 1; return p.s[start..i]; @@ -94,7 +94,7 @@ const Parser = struct { 't' => '\t', else => |c| c, }); - const next = indexOfQuoteOrEscape(p.s, i + 2) orelse return error.Syntax; + const next = findQuoteOrEscape(p.s, i + 2) orelse return error.Syntax; try out.appendSlice(p.arena, p.s[i + 2 .. next]); i = next; } @@ -201,7 +201,7 @@ test "vector quote search matches scalar" { for (0..buf.len) |i| { for ([_]u8{ '"', '\\' }) |c| { buf[i] = c; - for (0..i + 1) |start| try std.testing.expectEqual(@as(?usize, i), indexOfQuoteOrEscape(&buf, start)); + for (0..i + 1) |start| try std.testing.expectEqual(@as(?usize, i), findQuoteOrEscape(&buf, start)); buf[i] = 'x'; } } diff --git a/pkg/nix-flake-lock/src/root.zig b/pkg/nix-flake-lock/src/root.zig index 70dc36a..88690b3 100644 --- a/pkg/nix-flake-lock/src/root.zig +++ b/pkg/nix-flake-lock/src/root.zig @@ -393,7 +393,13 @@ const Parser = struct { else => { const len = std.unicode.utf8ByteSequenceLength(c) catch return p.fail(error.InvalidUtf8, i); if (i + len > p.s.len) return p.fail(error.InvalidUtf8, i); - _ = std.unicode.utf8Decode(p.s[i..][0..len]) catch return p.fail(error.InvalidUtf8, i); + const seq = p.s[i..]; + _ = switch (len) { + 2 => std.unicode.utf8Decode2(seq[0..2].*), + 3 => std.unicode.utf8Decode3(seq[0..3].*), + 4 => std.unicode.utf8Decode4(seq[0..4].*), + else => return p.fail(error.InvalidUtf8, i), + } catch return p.fail(error.InvalidUtf8, i); p.pos = i + len; }, } @@ -1285,10 +1291,10 @@ test "version 5 info overlay, duplicate keys and renaming" { var out: std.ArrayList(u8) = .empty; defer out.deinit(testing.allocator); try doc.lock.writeNix(testing.allocator, &out); - try testing.expect(std.mem.indexOf(u8, out.items, "\"lastModified\": 7,") != null); - try testing.expect(std.mem.indexOf(u8, out.items, "\"a_2\": {") != null); - try testing.expect(std.mem.indexOf(u8, out.items, "\"k\": \"new\"") != null); - try testing.expect(std.mem.indexOf(u8, out.items, "\"flake\": false,") != null); + try testing.expect(std.mem.find(u8, out.items, "\"lastModified\": 7,") != null); + try testing.expect(std.mem.find(u8, out.items, "\"a_2\": {") != null); + try testing.expect(std.mem.find(u8, out.items, "\"k\": \"new\"") != null); + try testing.expect(std.mem.find(u8, out.items, "\"flake\": false,") != null); // Re-parsing canonical output is a fixed point. var doc2 = try parse(testing.allocator, out.items, .{}, null); defer doc2.deinit(); @@ -1311,8 +1317,8 @@ test "large graphs use the hashed collision path" { var out: std.ArrayList(u8) = .empty; defer out.deinit(testing.allocator); try doc.lock.writeNix(testing.allocator, &out); - try testing.expect(std.mem.indexOf(u8, out.items, "\"s_100\": {") != null); - try testing.expect(std.mem.indexOf(u8, out.items, "\"s_101\"") == null); + try testing.expect(std.mem.find(u8, out.items, "\"s_100\": {") != null); + try testing.expect(std.mem.find(u8, out.items, "\"s_101\"") == null); } test "rejects malformed and adversarial inputs" { @@ -1356,7 +1362,7 @@ test "unicode escapes and string escaping on output" { var out: std.ArrayList(u8) = .empty; defer out.deinit(testing.allocator); try doc.lock.writeNix(testing.allocator, &out); - try testing.expect(std.mem.indexOf(u8, out.items, "\"é😀 \\\"q\\\" \\t é\"") != null); + try testing.expect(std.mem.find(u8, out.items, "\"é😀 \\\"q\\\" \\t é\"") != null); } test "vector scanners agree with scalar definitions" { diff --git a/pkg/nix-narinfo/src/parse.zig b/pkg/nix-narinfo/src/parse.zig index be760ca..0fac67b 100644 --- a/pkg/nix-narinfo/src/parse.zig +++ b/pkg/nix-narinfo/src/parse.zig @@ -40,7 +40,7 @@ const lanes = 32; const V = @Vector(lanes, u8); const Mask = @Int(.unsigned, lanes); -pub fn indexOfNewline(s: []const u8, start: usize) ?usize { +pub fn findNewline(s: []const u8, start: usize) ?usize { var i = start; const nl: V = @splat('\n'); while (i + lanes <= s.len) : (i += lanes) { @@ -65,11 +65,11 @@ pub fn parse(text: []const u8) Error!NarInfo { var have_size = false; var pos: usize = 0; while (pos < text.len) { - const end = indexOfNewline(text, pos) orelse text.len; + const end = findNewline(text, pos) orelse text.len; const line = text[pos..end]; pos = end + 1; if (line.len == 0) continue; - const colon = std.mem.indexOfScalar(u8, line, ':') orelse return error.InvalidLine; + const colon = std.mem.findScalar(u8, line, ':') orelse return error.InvalidLine; const key = line[0..colon]; // Nix skips exactly ": "; tolerate a bare ":" before end of line. const value = if (colon + 2 <= line.len) line[colon + 2 ..] else ""; @@ -150,8 +150,8 @@ test "vector newline search matches scalar" { var buf: [100]u8 = @splat('x'); for (0..buf.len) |i| { buf[i] = '\n'; - for (0..i + 1) |start| try std.testing.expectEqual(@as(?usize, i), indexOfNewline(&buf, start)); + for (0..i + 1) |start| try std.testing.expectEqual(@as(?usize, i), findNewline(&buf, start)); buf[i] = 'x'; } - try std.testing.expectEqual(@as(?usize, null), indexOfNewline(&buf, 0)); + try std.testing.expectEqual(@as(?usize, null), findNewline(&buf, 0)); } diff --git a/pkg/nix-narinfo/src/render.zig b/pkg/nix-narinfo/src/render.zig index 563706c..d4cf831 100644 --- a/pkg/nix-narinfo/src/render.zig +++ b/pkg/nix-narinfo/src/render.zig @@ -50,7 +50,7 @@ pub const Signer = struct { /// Parses a Nix secret key: `:`. pub fn parse(allocator: std.mem.Allocator, text: []const u8) Error!*Signer { const trimmed = std.mem.trim(u8, text, " \t\r\n"); - const colon = std.mem.indexOfScalar(u8, trimmed, ':') orelse return error.InvalidKey; + const colon = std.mem.findScalar(u8, trimmed, ':') orelse return error.InvalidKey; const name = trimmed[0..colon]; if (name.len == 0) return error.InvalidKey; for (name) |c| if (c == '\n' or c == ' ') return error.InvalidKey; @@ -200,8 +200,8 @@ pub fn render(allocator: std.mem.Allocator, in: Input, signer: ?*const Signer) E } }.line; var num_buf: [2][20]u8 = undefined; - const nar_size = std.fmt.bufPrint(&num_buf[0], "{d}", .{in.nar_size}) catch unreachable; - const file_size = std.fmt.bufPrint(&num_buf[1], "{d}", .{in.file_size}) catch unreachable; + const nar_size = std.mem.print(&num_buf[0], "{d}", .{in.nar_size}) catch unreachable; + const file_size = std.mem.print(&num_buf[1], "{d}", .{in.file_size}) catch unreachable; try w(&out, allocator, &.{ "StorePath: ", in.store_path }); try w(&out, allocator, &.{ "URL: nar/", &file_hash, ext }); @@ -265,8 +265,8 @@ test "renders and signs a narinfo" { .system = "x86_64-linux", }, signer); defer a.free(text); - try std.testing.expect(std.mem.indexOf(u8, text, "References: 0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello 1mdqa9w1p6cmli6976v4wi0sw9r4p5pr-b\n") != null); - try std.testing.expect(std.mem.indexOf(u8, text, "\nSig: test-1:") != null); + try std.testing.expect(std.mem.find(u8, text, "References: 0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello 1mdqa9w1p6cmli6976v4wi0sw9r4p5pr-b\n") != null); + try std.testing.expect(std.mem.find(u8, text, "\nSig: test-1:") != null); const empty = try render(a, .{ .store_path = "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello", .nar_hash = &nar, @@ -279,7 +279,7 @@ test "renders and signs a narinfo" { .system = "", }, null); defer a.free(empty); - try std.testing.expect(std.mem.indexOf(u8, empty, "\nReferences: \nDeriver: ") != null); + try std.testing.expect(std.mem.find(u8, empty, "\nReferences: \nDeriver: ") != null); try std.testing.expectError(error.InvalidSystem, render(a, .{ .store_path = "/nix/store/0mdqa9w1p6cmli6976v4wi0sw9r4p5pr-hello", .nar_hash = &nar, From 90d9cdf189e973f8a807fc8079fc4de5fc1a6c68 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Sun, 4 Oct 2026 21:00:23 +0200 Subject: [PATCH 52/56] bench: compare whole processes with poop bench/poop.sh runs three comparisons through poop, which counts wall time, peak RSS, cycles, instructions and cache and branch misses across child processes: NAR serialise and SHA-256 against nix hash path, flake.lock canonicalisation against the nix-flake-lock crate, and helios push against nix copy to a zstd file cache, both at level 3 and signing. Nix reads the store read-only itself there, so none of its work hides in the nix-daemon. The bench binary gains a single-pass nar-once mode for it. Results print to the terminal; .gitignore now keeps benchmark output (txt, tsv, perf.data) out of the tree. --- .gitignore | 5 +++ AGENTS.md | 3 ++ bench/poop.sh | 90 ++++++++++++++++++++++++++++++++++++++++++ bench/zig/src/main.zig | 2 + 4 files changed, 100 insertions(+) create mode 100755 bench/poop.sh diff --git a/.gitignore b/.gitignore index ef21e25..c8709fc 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,8 @@ result-* # Zig .zig-cache/ zig-out/ + +# Benchmark output (bench/*.sh, perf, poop): keep results out of the tree +/bench/**/*.txt +*.tsv +perf.data* diff --git a/AGENTS.md b/AGENTS.md index 6a2b09c..7727c5f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,6 +48,9 @@ Logging is `tracing` with `tracing-subscriber` (fmt, ansi and json only; no - Measure before and after; `bench/run.sh` and `bench/flake-lock.sh` compare against the reference implementations on the same corpus. + `bench/poop.sh` compares whole processes with poop (wall time, peak RSS, + cycles, cache and branch misses), including `helios push` against + `nix copy`. Benchmark output stays out of the tree. - Keep the narinfo read path free of SQLite on misses and of per-request signing. - Keep a scalar reference for every SIMD routine, with a test asserting the diff --git a/bench/poop.sh b/bench/poop.sh new file mode 100755 index 0000000..61033a7 --- /dev/null +++ b/bench/poop.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Whole-process comparisons with poop (https://github.com/andrewrk/poop): +# wall time, peak RSS, cycles, instructions, cache and branch misses, on the +# same work, counting child processes: +# +# 1. NAR serialise + SHA-256 of a closure: pkg/nix-archive (one pass of +# the bench binary) vs `nix hash path` +# 2. flake.lock canonicalisation and validation: pkg/nix-flake-lock vs the +# nix-flake-lock crate, on every flake.lock under $FLAKE_LOCK_SEARCH +# 3. Pushing a closure: `helios push` to a fresh local server (started, +# populated and stopped each run) vs `nix copy` to a fresh zstd file +# cache, both compressing at level 3 and signing. Nix reads the store +# itself (read-only), so none of its work hides in the nix-daemon. +# +# poop counts user-space events only; I/O shows in wall time. Results go to +# the terminal, never to files in the tree. +# +# Usage (inside `nix develop`): bench/poop.sh [store-path] +# default store path: python3's closure (about 200 MiB of NAR) +# DURATION=ms per command (default 5000) +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SEARCH="${FLAKE_LOCK_SEARCH:-$HOME}" +DURATION="${DURATION:-5000}" +W="$(mktemp -d)" +trap 'rm -rf "$W"' EXIT + +echo "building..." >&2 +(cd "$ROOT" && cargo build --release -q) +(cd "$ROOT/bench/zig" && zig build -Doptimize=ReleaseFast -Dcpu=native) +cargo build -q --release --manifest-path "$ROOT/bench/flake-lock-rs/Cargo.toml" +POOP="$(nix build --no-link --print-out-paths --inputs-from "$ROOT" nixpkgs#poop)/bin/poop" +TARGET="$(readlink -f "${1:-$(nix build --no-link --print-out-paths --inputs-from "$ROOT" nixpkgs#python3)}")" +BIN="$ROOT/target/release" + +nix-store -qR "$TARGET" >"$W/paths" +while read -r p; do printf '%s\0' "$p"; done <"$W/paths" >"$W/paths.bin" +: >"$W/narinfo.bin" +: >"$W/drv.bin" +nix key generate-secret --key-name poop-1 >"$W/key" +find "$SEARCH" -name flake.lock -not -path '*/.cache/*' -not -path '*/node_modules/*' -size -16M 2>/dev/null >"$W/locks" || true +echo "closure of $TARGET: $(wc -l <"$W/paths") paths, $(nix path-info -rs "$TARGET" | awk '{s+=$2} END {printf "%.0f", s/1048576}') MiB of NAR" >&2 +echo "flake.lock files: $(wc -l <"$W/locks")" >&2 + +# poop splits each command on spaces without a shell, so every side runs +# from a small script; both sides pay the same shell start. +script() { + local name=$1 + shift + printf '#!/bin/sh\nset -e\n%s\n' "$*" >"$W/$name" + chmod +x "$W/$name" +} + +script helios-nar "BENCH_DIR=$W BENCH_ONLY=nar-once exec $ROOT/bench/zig/zig-out/bin/bench" +script nix-nar "exec xargs nix hash path --type sha256 <$W/paths >/dev/null 2>&1" + +script helios-flake-lock "FL_MODE=canon FL_LIST=$W/locks exec $ROOT/bench/zig/zig-out/bin/flake-lock >/dev/null" +script rust-flake-lock "exec xargs $ROOT/bench/flake-lock-rs/target/release/flake-lock-rs canon <$W/locks >/dev/null" + +script helios-push " +d=\$(mktemp -d -p $W) +$BIN/helios-server generate-secrets --dir \$d/s >/dev/null 2>&1 +HELIOS_LOG=error $BIN/helios-server --listen 127.0.0.1:18299 --data-dir \$d/data \\ + --signing-key-file \$d/s/signing-key --jwt-secret-file \$d/s/jwt-secret --admin-secret-file \$d/s/admin-secret & +pid=\$! +until curl -sf http://127.0.0.1:18299/healthz >/dev/null; do sleep 0.01; done +export XDG_CONFIG_HOME=\$d/cfg HELIOS_LOG=error +$BIN/helios login admin http://127.0.0.1:18299 \$(cat \$d/s/admin-secret) >/dev/null 2>&1 +$BIN/helios cache create main >/dev/null +$BIN/helios login ci http://127.0.0.1:18299 \$($BIN/helios token create ci --caches main --perms push 2>/dev/null | sed -n 's/.*\"token\": *\"\\([^\"]*\\)\".*/\\1/p') >/dev/null 2>&1 +$BIN/helios push main --closure \"\$@\" $TARGET >/dev/null 2>&1 +kill \$pid; wait \$pid || true +rm -rf \$d" +# The same, without the long window: zstd level 3 exactly as Nix runs it. +sed 's|--closure "\$@"|--closure --window-log 0|' "$W/helios-push" >"$W/helios-push-plain" +chmod +x "$W/helios-push-plain" +script nix-copy " +d=\$(mktemp -d -p $W) +nix copy --extra-experimental-features read-only-local-store --from 'local?read-only=true' --to \"file://\$d?compression=zstd&compression-level=3¶llel-compression=true&secret-key=$W/key\" $TARGET +rm -rf \$d" + +echo +echo "== NAR serialise + SHA-256" +"$POOP" -d "$DURATION" "$W/nix-nar" "$W/helios-nar" +echo +echo "== flake.lock canonicalise + validate" +"$POOP" -d "$DURATION" "$W/rust-flake-lock" "$W/helios-flake-lock" +echo +echo "== push a closure (compress at zstd 3, sign, store)" +"$POOP" -d "$DURATION" "$W/nix-copy" "$W/helios-push-plain" "$W/helios-push" diff --git a/bench/zig/src/main.zig b/bench/zig/src/main.zig index 15e70b3..e92d074 100644 --- a/bench/zig/src/main.zig +++ b/bench/zig/src/main.zig @@ -226,6 +226,8 @@ pub fn main() !void { .signer = try narinfo.Signer.parse(gpa, key), }; + // One untimed pass, for whole-process tools (bench/poop.sh). + if (std.mem.eql(u8, only, "nar-once")) return narHash(&ctx); if (std.mem.eql(u8, only, "nar")) { var bytes: u64 = 0; for (ctx.paths) |p| { From 0e3180dc4259a4ce7e9ef79817373c4e4de231b0 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Sun, 4 Oct 2026 22:01:29 +0200 Subject: [PATCH 53/56] nix-archive: map large files a window at a time Files of 1 MiB or more were mapped whole with MAP_POPULATE, so peak RSS grew with the largest file in a closure: hashing python3's closure peaked at 70.7 MB for its 64 MiB libpython3.14.a, against 47.8 MB for nix hash path. They are now mapped 16 MiB at a time, falling back to read() from where the mappings stopped if one fails: 20.4 MB peak, at the same speed (poop, bench/poop.sh). A test serialises a 40 MiB file that ends mid-page against an independent NAR encoding. --- crates/helios-core/src/lib.rs | 29 +++++++++++++++++++++++++++++ pkg/nix-archive/src/walk.zig | 24 ++++++++++++++++-------- 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/crates/helios-core/src/lib.rs b/crates/helios-core/src/lib.rs index 7333197..16e0578 100644 --- a/crates/helios-core/src/lib.rs +++ b/crates/helios-core/src/lib.rs @@ -573,6 +573,35 @@ mod tests { assert_eq!(aborted.unwrap_err().code, HL_E_ABORTED); } + /// Files are mapped a 16 MiB window at a time; one spanning several + /// windows and ending mid-page must still serialise exactly. + #[test] + fn large_files_serialise_across_mapping_windows() { + let dir = TempDir::new(); + let file = dir.path().join("big"); + let data: Vec = (0..(40usize << 20) + 12345).map(|i| (i * 31 % 251) as u8).collect(); + std::fs::write(&file, &data).unwrap(); + + let mut nar = Vec::new(); + let digest = dump_nar(&file, &DumpOptions { level: 0, ..Default::default() }, |c| { + nar.extend_from_slice(c); + true + }) + .unwrap(); + + let mut expected = Vec::new(); + let mut str = |s: &[u8]| { + expected.extend_from_slice(&(s.len() as u64).to_le_bytes()); + expected.extend_from_slice(s); + expected.resize(expected.len().next_multiple_of(8), 0); + }; + for s in [&b"nix-archive-1"[..], b"(", b"type", b"regular", b"contents", &data, b")"] { + str(s); + } + assert!(nar == expected, "NAR differs from the reference encoding"); + assert_eq!(digest.nar_hash, sha256(&expected)); + } + /// Our NAR serialiser must agree byte-for-byte with Nix: compare NAR /// hashes against the Nix database for a real closure. #[test] diff --git a/pkg/nix-archive/src/walk.zig b/pkg/nix-archive/src/walk.zig index 21e1107..d5807ee 100644 --- a/pkg/nix-archive/src/walk.zig +++ b/pkg/nix-archive/src/walk.zig @@ -34,6 +34,8 @@ fn closeFd(fd: i32) void { /// Files at least this large are mmapped; sink.write feeds them unbuffered. const mmap_threshold = @import("sink.zig").direct_threshold; +/// Bytes mapped at once; a page multiple, so every window offset is aligned. +const mmap_window = 16 << 20; const Kind = enum { regular, executable, symlink, directory }; @@ -114,18 +116,24 @@ const Writer = struct { // Large files are hashed straight out of the page cache: no // kernel-to-user copy, and the sink feeds the mapping without // buffering it. Store paths are immutable, so the mapping is stable. + // A window at a time, so peak RSS does not grow with the file. + var done: u64 = 0; if (size >= mmap_threshold) { - const map_len: usize = @intCast(size); - const rc = linux.mmap(null, map_len, .{ .READ = true }, .{ .TYPE = .PRIVATE, .POPULATE = true }, fd, 0); - if (linux.errno(rc) == .SUCCESS) { + while (done < size) { + const window: usize = @intCast(@min(size - done, mmap_window)); + const rc = linux.mmap(null, window, .{ .READ = true }, .{ .TYPE = .PRIVATE, .POPULATE = true }, fd, @intCast(done)); + if (linux.errno(rc) != .SUCCESS) { + // Read the rest instead, from where the mappings stopped. + _ = try check(linux.lseek(fd, @intCast(done), linux.SEEK.SET)); + break; + } const ptr: [*]u8 = @ptrFromInt(rc); - defer _ = linux.munmap(ptr, map_len); - _ = linux.madvise(ptr, map_len, linux.MADV.SEQUENTIAL); - try self.sink.write(ptr[0..map_len]); - return self.pad(size); + defer _ = linux.munmap(ptr, window); + try self.sink.write(ptr[0..window]); + done += window; } } - var remaining = size; + var remaining = size - done; while (remaining > 0) { var dst = self.sink.spare(); if (dst.len == 0) { From 6e72d1d6faea3fe4f11a9640bb156c744ba542bf Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Sun, 4 Oct 2026 22:01:29 +0200 Subject: [PATCH 54/56] push: let the server's verification stand for the client's hashes helios push hashed every NAR twice, raw and compressed, and the server hashed both again while verifying the upload: four SHA-256 passes where nix copy makes two, 29% of a push's cycles. The server's hashes are the authoritative ones, so the compressor gains a hash option (on by default) that the CLI turns off, and the client checks the NAR hash the server verified against the Nix database instead. A corrupt local store is still caught, after the upload rather than before, and its NAR is never published. Pushing python3's closure at zstd 3 (poop, bench/poop.sh): 416 to 360 ms and 4.40 to 3.74 G cycles, against 336 ms and 3.42 G for nix copy to a file cache, which verifies nothing on the other side. The default long window: 503 to 473 ms. --- core/include/helios.h | 1 + core/src/root.zig | 3 ++- crates/helios-cli/src/api.rs | 1 - crates/helios-cli/src/push.rs | 22 +++++++++++++--------- crates/helios-core/src/lib.rs | 8 ++++++-- pkg/nix-archive/src/sink.zig | 20 +++++++++++++++----- 6 files changed, 37 insertions(+), 18 deletions(-) diff --git a/core/include/helios.h b/core/include/helios.h index 15e71b8..d04d1c8 100644 --- a/core/include/helios.h +++ b/core/include/helios.h @@ -48,6 +48,7 @@ typedef struct { int threads; /* zstd worker threads; 0 = calling thread */ uint64_t nar_size; /* exact NAR size, pledged to zstd; 0 if unknown */ int window_log; /* 0: the level's window; 10-27: long-distance matching */ + bool hash; /* false skips SHA-256; the digest then has sizes only */ } hl_dump_options; /* Return non-zero to abort the pipeline with HL_E_ABORTED. */ diff --git a/core/src/root.zig b/core/src/root.zig index 6868026..7dfccd7 100644 --- a/core/src/root.zig +++ b/core/src/root.zig @@ -85,10 +85,11 @@ pub const DumpOptions = extern struct { threads: c_int, nar_size: u64, window_log: c_int, + hash: bool, }; fn sinkOptions(opts: *const DumpOptions) archive.Options { - return .{ .level = opts.level, .threads = opts.threads, .nar_size = opts.nar_size, .window_log = opts.window_log }; + return .{ .level = opts.level, .threads = opts.threads, .nar_size = opts.nar_size, .window_log = opts.window_log, .hash = opts.hash }; } export fn hl_nar_dump(path: [*:0]const u8, opts: *const DumpOptions, write: WriteFn, ctx: ?*anyopaque, out: *Digest) c_int { diff --git a/crates/helios-cli/src/api.rs b/crates/helios-cli/src/api.rs index 1daee74..ff485df 100644 --- a/crates/helios-cli/src/api.rs +++ b/crates/helios-cli/src/api.rs @@ -26,7 +26,6 @@ pub struct PathSpec { #[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase")] pub struct Uploaded { - pub file_hash: String, pub file_size: u64, pub nar_hash: String, pub nar_size: u64, diff --git a/crates/helios-cli/src/push.rs b/crates/helios-cli/src/push.rs index 153b046..fc75100 100644 --- a/crates/helios-cli/src/push.rs +++ b/crates/helios-cli/src/push.rs @@ -1,6 +1,7 @@ -//! `helios push`: NARs are serialised, compressed and hashed by libhelios -//! and streamed straight into the upload request, with no temp files and no -//! `nix store dump-path | zstd` subprocesses. +//! `helios push`: NARs are serialised and compressed by libhelios and +//! streamed straight into the upload request, with no temp files and no +//! `nix store dump-path | zstd` subprocesses. The server hashes what it +//! receives, so the client does not hash again. use std::collections::HashSet; use std::path::Path; @@ -59,7 +60,7 @@ fn produce(path: &str, opts: DumpOptions, tx: mpsc::Sender {} other => return other.with_context(|| format!("serialising {path}")), } - // Non-Linux: let Nix serialise, libhelios compresses and hashes. + // Non-Linux: let Nix serialise, libhelios compresses. use std::io::Read; let mut child = std::process::Command::new("nix-store") .args(["--dump", path]) @@ -155,11 +156,14 @@ async fn upload_one(client: &Client, cache: &str, info: &PathInfo, opts: DumpOpt (uploaded, Ok(digest)) => (uploaded?, digest), }; - let ours = format!("sha256:{}", helios_core::nix32_encode(&digest.nar_hash)); - if parse_sha256(&info.nar_hash) != Some(digest.nar_hash) { - bail!("{}: NAR hash {ours} differs from the Nix database ({})", info.path, info.nar_hash); + // The NAR is not hashed here: the server hashes what it received, and + // that is checked against the Nix database, which still catches a + // corrupt local store (the NAR is then never published). + let verified = parse_sha256(&uploaded.nar_hash); + if verified.is_none() || verified != parse_sha256(&info.nar_hash) { + bail!("{}: NAR hash {} differs from the Nix database ({})", info.path, uploaded.nar_hash, info.nar_hash); } - if uploaded.nar_hash != ours || uploaded.file_hash != helios_core::nix32_encode(&digest.file_hash) || uploaded.nar_size != digest.nar_size { + if uploaded.nar_size != digest.nar_size || uploaded.file_size != digest.file_size { bail!("{}: server verified different content than was sent", info.path); } Ok(uploaded.file_size) @@ -173,7 +177,7 @@ async fn upload_all(client: &Client, cache: &str, paths: &[&PathInfo], opts: &Op let mut bytes = 0u64; let mut failures = HashSet::new(); let mut results = futures_util::stream::iter(paths.iter().map(|info| async move { - let dump = DumpOptions { level: opts.level, threads, nar_size: info.nar_size, window_log: opts.window_log }; + let dump = DumpOptions { level: opts.level, threads, nar_size: info.nar_size, window_log: opts.window_log, hash: false }; let started = Instant::now(); (info, upload_one(client, cache, info, dump, opts.chunk_size).await, started.elapsed()) })) diff --git a/crates/helios-core/src/lib.rs b/crates/helios-core/src/lib.rs index 16e0578..1feed37 100644 --- a/crates/helios-core/src/lib.rs +++ b/crates/helios-core/src/lib.rs @@ -24,6 +24,7 @@ mod sys { pub threads: c_int, pub nar_size: u64, pub window_log: c_int, + pub hash: bool, } #[repr(C)] @@ -164,17 +165,20 @@ pub struct DumpOptions { /// 0 keeps the level's window; 10-27 enables long-distance matching /// with a 2^window_log window. pub window_log: i32, + /// false skips both SHA-256 passes; the digest then carries sizes only, + /// for an uploader whose server verifies the content anyway. + pub hash: bool, } impl Default for DumpOptions { fn default() -> Self { - Self { level: 3, threads: 0, nar_size: 0, window_log: 0 } + Self { level: 3, threads: 0, nar_size: 0, window_log: 0, hash: true } } } impl DumpOptions { fn raw(&self) -> sys::DumpOptions { - sys::DumpOptions { level: self.level, threads: self.threads, nar_size: self.nar_size, window_log: self.window_log } + sys::DumpOptions { level: self.level, threads: self.threads, nar_size: self.nar_size, window_log: self.window_log, hash: self.hash } } } diff --git a/pkg/nix-archive/src/sink.zig b/pkg/nix-archive/src/sink.zig index 9c7d6b4..0e15fa5 100644 --- a/pkg/nix-archive/src/sink.zig +++ b/pkg/nix-archive/src/sink.zig @@ -32,6 +32,9 @@ pub const Options = struct { /// window of 2^window_log bytes (at most 27, the largest a stock Nix /// decoder accepts), which finds repeats across a large NAR. window_log: c_int = 0, + /// false skips both SHA-256 passes; the digest then carries sizes only. + /// For an uploader whose server verifies the content anyway. + hash: bool = true, }; const in_capacity = 1 << 20; @@ -44,6 +47,7 @@ pub const direct_threshold = in_capacity; const multithread_threshold = 4 << 20; pub const Sink = struct { + hash: bool = true, nar_hasher: Sha256 = .init(.{}), nar_size: u64 = 0, file_hasher: Sha256 = .init(.{}), @@ -83,6 +87,7 @@ pub const Sink = struct { const out_buf = try allocator.alloc(u8, out_len); return .{ + .hash = options.hash, .cctx = cctx, .in_buf = in_buf, .out_buf = out_buf, @@ -132,7 +137,7 @@ pub const Sink = struct { } fn feed(self: *Sink, data: []const u8) Error!void { - self.nar_hasher.update(data); + if (self.hash) self.nar_hasher.update(data); self.nar_size += data.len; const cctx = self.cctx orelse return self.emit(data); var input: zstd.InBuffer = .{ .src = data.ptr, .size = data.len, .pos = 0 }; @@ -145,7 +150,7 @@ pub const Sink = struct { fn emit(self: *Sink, data: []const u8) Error!void { // Uncompressed output is the NAR itself: hash it once, not twice. - if (self.cctx != null) self.file_hasher.update(data); + if (self.hash and self.cctx != null) self.file_hasher.update(data); self.file_size += data.len; if (self.write_fn(self.write_ctx, data.ptr, data.len) != 0) return error.Aborted; } @@ -163,9 +168,14 @@ pub const Sink = struct { } } var digest: Digest = undefined; - self.nar_hasher.final(&digest.nar_hash); - if (self.cctx != null) self.file_hasher.final(&digest.file_hash) else { - digest.file_hash = digest.nar_hash; + if (!self.hash) { + @memset(&digest.nar_hash, 0); + @memset(&digest.file_hash, 0); + } else { + self.nar_hasher.final(&digest.nar_hash); + if (self.cctx != null) self.file_hasher.final(&digest.file_hash) else { + digest.file_hash = digest.nar_hash; + } } digest.nar_size = self.nar_size; digest.file_size = self.file_size; From f2296fd82ebad63a2d9abb07cf9210e6d9e8aba0 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Sun, 4 Oct 2026 22:14:04 +0200 Subject: [PATCH 55/56] cli: give nix subprocesses no stdin nix path-info, nix store info, nix config show and the nix-store --dump fallback inherited helios's stdin. Nix does not read it, but a test that stood in for nix with `cat` hung for half an hour whenever the test runner left stdin open as a pipe, and a subprocess has no business waiting on whatever the caller left open. They now get /dev/null; only the --stdin path list still gets a pipe. --- crates/helios-cli/src/nix.rs | 3 ++- crates/helios-cli/src/push.rs | 1 + crates/helios-cli/src/substituter.rs | 7 ++++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/helios-cli/src/nix.rs b/crates/helios-cli/src/nix.rs index 4fc7ce4..0628354 100644 --- a/crates/helios-cli/src/nix.rs +++ b/crates/helios-cli/src/nix.rs @@ -50,7 +50,8 @@ async fn output(mut cmd: Command, installables: &[String]) -> std::io::Result Option { .args(["--extra-experimental-features", "nix-command"]) .args(args) .stderr(std::process::Stdio::null()) + .stdin(std::process::Stdio::null()) .output() .ok()?; out.status.success().then(|| String::from_utf8_lossy(&out.stdout).into_owned()) @@ -288,7 +289,11 @@ fn conf_netrc_file(line: &str) -> Option { } fn nix_trusts_user() -> bool { - let Ok(out) = std::process::Command::new("nix").args(["--extra-experimental-features", "nix-command", "store", "info", "--json"]).output() else { + let Ok(out) = std::process::Command::new("nix") + .args(["--extra-experimental-features", "nix-command", "store", "info", "--json"]) + .stdin(std::process::Stdio::null()) + .output() + else { return true; // No way to tell; do not warn. }; serde_json::from_slice::(&out.stdout).map_or(true, |v| v["trusted"] != false && v["trusted"] != 0) From e8d1436f6b52739d9bf3885f061f767ab9e2ed26 Mon Sep 17 00:00:00 2001 From: Uzair Aftab Date: Sun, 4 Oct 2026 22:50:13 +0200 Subject: [PATCH 56/56] tests: drop redundant tests and merge the fragmented ones Rust: - helios-core: the three Signer binding tests become one, and the nix32 smoke test joins the primitives test; signing logic is tested in pkg/nix-narinfo. - auth: the header constant and scope checks join the token test. - traces: drop a test that serde picks an enum variant; e2e publishes and serves both formats. - daemon: the duration and size parsers share a test, and so do the two free-space floor tests. Zig: - nix-base32: the encodedLen check folds into the known-vector test. - nix-archive: the two "refused" tests merge; a test now covers the hash = false option, which had none. e2e drops seven checks a later check already proves: the token being non-empty, nix-cache-info, a narinfo hit and a NAR download (all needed by the signed substitution that follows), the chunk-size log scan (the chunked closure substitutes), the signing key in nix.conf (Nix verifies signatures through it) and an unknown trace 404. --- crates/helios-core/src/lib.rs | 35 ++++++++++-------------------- crates/helios-daemon/src/autogc.rs | 4 ---- crates/helios-daemon/src/main.rs | 6 +---- crates/helios-server/src/auth.rs | 12 +++------- crates/helios-server/src/traces.rs | 9 -------- pkg/nix-archive/src/tests.zig | 30 ++++++++++++++++++++----- pkg/nix-base32/src/root.zig | 6 +---- scripts/e2e.sh | 7 ------ 8 files changed, 41 insertions(+), 68 deletions(-) diff --git a/crates/helios-core/src/lib.rs b/crates/helios-core/src/lib.rs index 1feed37..95b41cf 100644 --- a/crates/helios-core/src/lib.rs +++ b/crates/helios-core/src/lib.rs @@ -486,24 +486,28 @@ mod tests { assert_ne!(a, b); assert_eq!(a.len(), 36); assert_eq!(&a[14..15], "4"); - } - - #[test] - fn nix32_round_trip() { + // nix32 lengths and the error path across the C ABI. let bytes = [7u8; 20]; - let text = nix32_encode(&bytes); - assert_eq!(text.len(), 32); - assert_eq!(nix32_decode::<20>(&text), Some(bytes)); + assert_eq!(nix32_decode::<20>(&nix32_encode(&bytes)), Some(bytes)); assert_eq!(nix32_decode::<20>("not-valid"), None); } + /// The Signer binding: generation, parsing, and buffers that grow for + /// long key names. Signing itself is tested in pkg/nix-narinfo. #[test] - fn generated_keys_round_trip() { + fn signer() { let key = Signer::generate("gen-1").unwrap(); assert!(key.starts_with("gen-1:")); assert_ne!(key, Signer::generate("gen-1").unwrap()); assert!(Signer::new(&key).unwrap().public_key().starts_with("gen-1:")); assert!(Signer::generate("bad:name").is_err()); + assert!(Signer::new(KEY).unwrap().public_key().starts_with("test-1:")); + assert!(Signer::new("garbage").is_none()); + + let name = "k".repeat(600); + let long = Signer::new(&Signer::generate(&name).unwrap()).unwrap(); + assert_eq!(long.public_key().len(), name.len() + 1 + 44); + assert_eq!(long.sign(b"msg").len(), name.len() + 1 + 88); } #[test] @@ -533,21 +537,6 @@ mod tests { assert_eq!(signer.sign(fingerprint.as_bytes()), sig); } - #[test] - fn signer_public_key() { - let signer = Signer::new(KEY).unwrap(); - assert!(signer.public_key().starts_with("test-1:")); - assert!(Signer::new("garbage").is_none()); - } - - #[test] - fn long_key_names_fit() { - let name = "k".repeat(600); - let signer = Signer::new(&Signer::generate(&name).unwrap()).unwrap(); - assert_eq!(signer.public_key().len(), name.len() + 1 + 44); - assert_eq!(signer.sign(b"msg").len(), name.len() + 1 + 88); - } - #[test] fn dump_matches_compressor_and_verifier() { let dir = TempDir::new(); diff --git a/crates/helios-daemon/src/autogc.rs b/crates/helios-daemon/src/autogc.rs index 3ae423d..b31061a 100644 --- a/crates/helios-daemon/src/autogc.rs +++ b/crates/helios-daemon/src/autogc.rs @@ -175,10 +175,6 @@ mod tests { let p = Policy { quota: 0, high: 0.9, low: 0.8, min_free: 10 * GIB }; assert_eq!(to_free(&p, 1, 20 * GIB), Need { quota: 0, disk: 0 }); assert_eq!(to_free(&p, 1, 5 * GIB), Need { quota: 0, disk: 6 * GIB }); - } - - #[test] - fn free_space_floor_spares_the_cache() { let need = Need { quota: 0, disk: 6 * GIB }; // Evicting all of it would not be enough: something else fills the disk. assert_eq!(budget(need, 5 * GIB), 0); diff --git a/crates/helios-daemon/src/main.rs b/crates/helios-daemon/src/main.rs index de201aa..c1eb115 100644 --- a/crates/helios-daemon/src/main.rs +++ b/crates/helios-daemon/src/main.rs @@ -260,16 +260,12 @@ mod tests { use std::time::Duration; #[test] - fn durations() { + fn durations_and_sizes() { assert_eq!(parse_duration("30s"), Ok(Duration::from_secs(30))); assert_eq!(parse_duration("7d"), Ok(Duration::from_secs(7 * 86400))); assert_eq!(parse_duration("0"), Ok(Duration::ZERO)); assert!(parse_duration("5").is_err()); assert!(parse_duration("5w").is_err()); - } - - #[test] - fn sizes() { assert_eq!(parse_size("0"), Ok(0)); assert_eq!(parse_size("1024"), Ok(1024)); assert_eq!(parse_size("64M"), Ok(64 << 20)); diff --git a/crates/helios-server/src/auth.rs b/crates/helios-server/src/auth.rs index 066c7fe..2e84d12 100644 --- a/crates/helios-server/src/auth.rs +++ b/crates/helios-server/src/auth.rs @@ -187,7 +187,9 @@ mod tests { } #[test] - fn round_trip_and_tamper() { + fn tokens() { + // The pre-encoded header must stay what it claims to be. + assert_eq!(URL_SAFE_NO_PAD.encode(br#"{"alg":"HS256","typ":"JWT"}"#), HEADER_B64); let now = 1_700_000_000; let token = sign(&claims(now), b"0123456789abcdef"); assert!(verify(&token, b"0123456789abcdef", now).is_some()); @@ -196,15 +198,7 @@ mod tests { let mut tampered = token.clone(); tampered.insert(token.find('.').unwrap() + 3, 'x'); assert!(verify(&tampered, b"0123456789abcdef", now).is_none()); - } - #[test] - fn header_constant_matches() { - assert_eq!(URL_SAFE_NO_PAD.encode(br#"{"alg":"HS256","typ":"JWT"}"#), HEADER_B64); - } - - #[test] - fn scope_checks() { let c = claims(0); assert!(c.allows("main", Perm::Push)); assert!(!c.allows("main", Perm::Pull)); diff --git a/crates/helios-server/src/traces.rs b/crates/helios-server/src/traces.rs index 91f9d39..275adb4 100644 --- a/crates/helios-server/src/traces.rs +++ b/crates/helios-server/src/traces.rs @@ -283,13 +283,4 @@ mod tests { assert!(!legacy_id_valid(&format!("sha256:{}!out", "A".repeat(64)))); assert!(!legacy_id_valid(&format!("sha256:{hex}!bad/name"))); } - - #[test] - fn both_entry_shapes_parse() { - let current: Entry = - serde_json::from_str(r#"{"key":{"drvPath":"x.drv","outputName":"out"},"value":{"outPath":"y","signatures":[]}}"#).unwrap(); - assert!(matches!(current, Entry::Current { .. })); - let legacy: Entry = serde_json::from_str(r#"{"id":"sha256:ab!out","outPath":"y","signatures":[],"dependentRealisations":{}}"#).unwrap(); - assert!(matches!(legacy, Entry::Legacy { .. })); - } } diff --git a/pkg/nix-archive/src/tests.zig b/pkg/nix-archive/src/tests.zig index ec35a46..8726b27 100644 --- a/pkg/nix-archive/src/tests.zig +++ b/pkg/nix-archive/src/tests.zig @@ -79,8 +79,11 @@ test "long-distance matching with an exact pledged size round-trips" { try roundTripWith(.{ .level = 3 }, 1000, true); } -test "a pledged size the stream does not match fails" { +test "invalid options and mismatched pledges are refused" { const a = std.testing.allocator; + var none: Collect = .{}; + try std.testing.expectError(error.Zstd, Sink.init(a, .{ .window_log = 28 }, Collect.write, &none)); + var out: Collect = .{}; defer out.list.deinit(a); var sink = try Sink.init(a, .{ .nar_size = 10 }, Collect.write, &out); @@ -89,11 +92,6 @@ test "a pledged size the stream does not match fails" { try std.testing.expectError(error.Zstd, sink.finish()); } -test "an out-of-range window is refused" { - var out: Collect = .{}; - try std.testing.expectError(error.Zstd, Sink.init(std.testing.allocator, .{ .window_log = 28 }, Collect.write, &out)); -} - test "verifier rejects truncated and non-NAR input" { const a = std.testing.allocator; const nar = try fakeNar(a, 100_000); @@ -115,3 +113,23 @@ test "verifier rejects truncated and non-NAR input" { try raw.update("definitely not a nar archive...."); try std.testing.expectError(error.NotNar, raw.finish()); } + +test "hash = false writes the same stream and sizes without hashing" { + const a = std.testing.allocator; + const nar = try fakeNar(a, 100_000); + defer a.free(nar); + var outs: [2]Collect = .{ .{}, .{} }; + defer for (&outs) |*o| o.list.deinit(a); + var digests: [2]@import("sink.zig").Digest = undefined; + for ([_]bool{ true, false }, &outs, &digests) |hash, *out, *digest| { + var sink = try Sink.init(a, .{ .level = 3, .hash = hash }, Collect.write, out); + defer sink.deinit(a); + try sink.write(nar); + digest.* = try sink.finish(); + } + try std.testing.expectEqualSlices(u8, outs[0].list.items, outs[1].list.items); + try std.testing.expectEqual(digests[0].nar_size, digests[1].nar_size); + try std.testing.expectEqual(digests[0].file_size, digests[1].file_size); + try std.testing.expectEqualSlices(u8, &@as([32]u8, @splat(0)), &digests[1].nar_hash); + try std.testing.expectEqualSlices(u8, &@as([32]u8, @splat(0)), &digests[1].file_hash); +} diff --git a/pkg/nix-base32/src/root.zig b/pkg/nix-base32/src/root.zig index 10ad36c..de246e9 100644 --- a/pkg/nix-base32/src/root.zig +++ b/pkg/nix-base32/src/root.zig @@ -180,6 +180,7 @@ pub fn isValidScalar(in: []const u8) bool { } test "sha256 of empty string round-trips" { + try std.testing.expectEqual(@as(usize, 32), encodedLen(20)); // a store path hash var digest: [32]u8 = undefined; std.crypto.hash.sha2.Sha256.hash("", &digest, .{}); var text: [52]u8 = undefined; @@ -190,11 +191,6 @@ test "sha256 of empty string round-trips" { try std.testing.expectEqualSlices(u8, &digest, &back); } -test "store path hash length" { - try std.testing.expectEqual(@as(usize, 32), encodedLen(20)); - try std.testing.expectEqual(@as(usize, 52), encodedLen(32)); -} - test "rejects invalid characters and overflow" { var out: [20]u8 = undefined; try std.testing.expectError(error.InvalidNix32, decode(&out, &@as([32]u8, @splat('e')))); diff --git a/scripts/e2e.sh b/scripts/e2e.sh index 7b3d307..5cac065 100755 --- a/scripts/e2e.sh +++ b/scripts/e2e.sh @@ -64,7 +64,6 @@ PUSH_TOKEN="$(helios token create ci --caches main,secret,chunked,plain --perms PULL_ONLY="$(helios token create reader --caches secret 2>/dev/null | jq -r .token)" PUSH_ONLY="$(helios token create builder --caches secret --perms push 2>/dev/null | jq -r .token)" check "tokens are read-only by default" '["pull"]' "$(helios token list | jq -c '.tokens[] | select(.subject=="reader") | .perms')" -check "token issued" true "$([ -n "$PUSH_TOKEN" ] && echo true)" check "admin rejects push token" 403 "$(status -H "authorization: Bearer $PUSH_TOKEN" "$URL/_api/v2/admin/tokens")" echo "=== chunked upload" @@ -73,7 +72,6 @@ helios login ci "$URL" "$PUSH_TOKEN" >/dev/null # 1 MiB chunks: the closure's larger NARs go in pieces, the rest in one request. helios push chunked --closure --chunk-size 1 "$TARGET" 2>&1 | tail -1 check "large NARs were sent in chunks" true "$([ "$(grep -c 'method=PATCH' "$WORK/server.log")" -gt 1 ] && echo true)" -check "chunks fit the chunk size" 0 "$(grep -c 'status=413' "$WORK/server.log")" NIX_CONFIG="trusted-public-keys = $PUBKEY require-sigs = true" nix copy --from "$URL/chunked" --to "$WORK/chunked-store" "$TARGET" 2>&1 | tail -3 check "chunked closure substitutes" "$(nix-store -qR "$TARGET" | wc -l)" "$(nix-store --store "$WORK/chunked-store" -qR "$TARGET" | wc -l)" @@ -118,13 +116,10 @@ check "second push is a no-op" "all $CLOSURE_SIZE paths already in 'main'" "$AGA echo "=== read path" HASH="$(basename "$TARGET" | cut -c1-32)" -check "nix-cache-info" 200 "$(status "$URL/main/nix-cache-info")" -check "narinfo hit" 200 "$(status "$URL/main/$HASH.narinfo")" check "narinfo HEAD" 200 "$(status -I "$URL/main/$HASH.narinfo")" check "narinfo miss" 404 "$(status "$URL/main/00000000000000000000000000000000.narinfo")" check "unknown cache" 404 "$(status "$URL/nope/$HASH.narinfo")" NAR_URL="$(curl -s "$URL/main/$HASH.narinfo" | sed -n 's/^URL: //p')" -check "nar download" 200 "$(status "$URL/main/$NAR_URL")" check "nar not served from a cache that lacks it" 404 "$(status -H "authorization: Bearer $PUSH_TOKEN" "$URL/secret/$NAR_URL")" echo "=== substitute into a fresh store (signatures required)" @@ -147,7 +142,6 @@ helios use secret 2>/dev/null helios use secret 2>/dev/null NIXCONF="$XDG_CONFIG_HOME/nix/nix.conf" check "nix.conf gets the login URL, once" 1 "$(grep -cx "extra-substituters = $URL/secret" "$NIXCONF")" -check "nix.conf gets the signing key" 1 "$(grep -cx "extra-trusted-public-keys = $PUBKEY" "$NIXCONF")" check "netrc is private" 600 "$(stat -c %a "$XDG_CONFIG_HOME/nix/netrc")" check "netrc holds the token once" 1 "$(grep -c "password $PULL_ONLY" "$XDG_CONFIG_HOME/nix/netrc")" # Signatures checked against the configured key, fetched with netrc auth. @@ -196,7 +190,6 @@ LEGACY="$(curl -sf "$URL/main/realisations/sha256:$HEX!out.doi")" check "legacy trace served, signed by the cache" "sha256:$HEX!out ${PUBKEY%%:*}" "$(echo "$LEGACY" | jq -r '.id + " " + (.signatures[0] | split(":")[0])')" check "legacy trace served percent-encoded" 200 "$(status "$URL/main/realisations/sha256%3A$HEX%21out.doi")" check "each format only under its prefix" 404 "$(status "$URL/main/build-trace-v2/sha256:$HEX!out.doi")" -check "unknown trace" 404 "$(status "$URL/main/build-trace-v2/$HASH-y.drv/out.doi")" echo "=== abuse" check "garbage upload rejected" 400 "$(status -X PUT -H "authorization: Bearer $PUSH_TOKEN" --data-binary 'not a zstd stream' "$URL/_api/v2/caches/main/nar")"