From 7f9bd5993d34d2ce7be050a5f4980ed8315b81a1 Mon Sep 17 00:00:00 2001 From: Schmitt Christian Date: Sat, 31 Oct 2026 16:53:53 +0000 Subject: [PATCH 1/2] apt repositories: hosted and proxy, signed indexes (#176) Debian packages for apt, like Nexus' apt-hosted and apt-proxy, served at /repository//. - crates/formats/apt: the apt format plugin. Packages are :, versions are ordered as dpkg orders them, a .deb's control file (gzip, xz, zstd or plain control member) is read within bounds, the Section names the component, and the pool layout, Packages and Release are Debian's. Uploads: Nexus' raw-body POST, a form, or a PUT; a DELETE of a pool file deletes the version. Proxies pass the upstream's files on unchanged and check packages against the upstream's Packages index. - The plugin contract grows: format options (apt's distribution) in the descriptor, checked by the plugin and set through the admin API; host- kept signed indexes (Query::Index, FormatPlugin::index), made when a client asks and the packages, options or keys changed, replaced in one transaction, by-hash copies kept a day; files of package formats (file, fetch-file, serve-file, file_package for selectors); file-at locations (immutable files, a size and digest checked before caching); and Prereleases::None, so apt's 1.0-1 is a release in browse and retention. - Signing keys per repository (repository_signing_key): Ed25519 keys made with rPGP, generated at creation or imported, sealed with the secret encryption, every key signing InRelease and Release.gpg so a key is rotated without warnings; /admin/repositories/{name}/signing-keys. - Web UI: the format's options in the repository form, a signing keys panel, the sources.list, key and auth.conf.d snippets, and the distribution browsed by component and architecture. - Tests: the plugin's units (version order, traversal, indexes), server integration tests, and .github/scripts/apt-e2e.sh with the real apt-get in debian:bookworm and gpgv (CI job apt-e2e). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012e6DJbxCkG53eCuDWqGGFy --- .cargo/audit.toml | 11 +- .github/scripts/apt-e2e.sh | 325 ++++++ .github/scripts/ci-changes.sh | 4 +- .github/workflows/ci.yml | 67 +- .github/workflows/release.yml | 2 +- ...8a1378ead641fd8af96ac1d4b1ba08dc14fb6.json | 30 + ...2b1a8dc0c2d19bfe31f08a98d5375f772a07f.json | 24 + ...a767e380d05c155eae7f876cb89c0e0d94750.json | 38 + ...0fe5ba7059f3e08f6e4fec7c21fd610d95787.json | 112 ++ ...1603025aa4611d362dd99f3c5f6f298acc1f.json} | 5 +- ...0b50a45ee9dae856e075d23a8984abbb60d43.json | 28 + ...b9a2a43c2f39ae867f52e6393f4f187a5723.json} | 5 +- ...7a04a1e81a99d0e8218c8d047d5c17be4095.json} | 18 +- ...b031318564ce98d8c1d90d07463a3c08465ba.json | 18 + ...19dba6794896e11ee86029c56e2c39efed5bb.json | 15 + ...deb0e3a4897bc8f6f4752cd2a6016a9b2d133.json | 14 + ...2c9a1c697ebf1b3b815ccc181a21716d2b9b6.json | 15 + ...07e080df16a30cc40389632467e12f29693a8.json | 113 ++ ...9faca8efb5fd75ff5d7b67ad8987900dc2674.json | 28 + ...57a4d8b826dd348b737c3c11942efff3d4682.json | 32 + ...4a27478e154a8596d1f40e212bf904917db5a.json | 15 + ...fa5066bfdad1491e3c61bd0a4dfdc1b40b61a.json | 28 + ...9a3f8ca3ab658acab7e1727470c9816efed7c.json | 50 + ...5e8e7574e053537d15aec3ce04939c920acd.json} | 5 +- Cargo.lock | 1032 ++++++++++++++++- Cargo.toml | 1 + DEVELOPMENT.md | 23 +- README.md | 14 +- crates/core/src/format.rs | 14 + crates/core/src/repo/proxy.rs | 8 +- crates/core/src/retention/evaluate.rs | 49 +- crates/db/src/index_dao.rs | 165 +++ crates/db/src/lib.rs | 2 + crates/db/src/model.rs | 20 + crates/db/src/raw_dao.rs | 20 +- crates/db/src/secret_dao.rs | 17 +- crates/db/src/signing_key_dao.rs | 207 ++++ crates/db/tests/migrations.rs | 3 + crates/db/tests/raw.rs | 1 + crates/formats/README.md | 7 +- crates/formats/apt/Cargo.toml | 31 + crates/formats/apt/src/control.rs | 360 ++++++ crates/formats/apt/src/deb.rs | 312 +++++ crates/formats/apt/src/index.rs | 468 ++++++++ crates/formats/apt/src/layout.rs | 334 ++++++ crates/formats/apt/src/lib.rs | 340 ++++++ crates/formats/apt/src/manifest.rs | 249 ++++ crates/formats/apt/src/proxy.rs | 284 +++++ crates/formats/apt/src/version.rs | 338 ++++++ crates/formats/docker/src/lib.rs | 3 + crates/formats/docker/src/upstream.rs | 1 + .../formats/npm/examples/packument_memory.rs | 1 + crates/formats/npm/src/lib.rs | 3 + crates/formats/npm/src/proxy.rs | 1 + crates/formats/npm/tests/protocol.rs | 1 + crates/formats/nuget/src/lib.rs | 3 + crates/formats/nuget/tests/protocol.rs | 1 + crates/formats/nuget/tests/upstream.rs | 2 + crates/formats/raw/src/lib.rs | 3 + crates/formats/raw/src/listing.rs | 1 + crates/formats/raw/src/sidecar.rs | 1 + crates/plugin-api/src/lib.rs | 45 + crates/plugin-api/src/testing.rs | 1 + crates/plugin-api/src/types.rs | 217 +++- crates/plugin/src/descriptor.rs | 87 ++ crates/plugin/src/format.rs | 14 + crates/server/Cargo.toml | 7 + crates/server/src/admin/crypto.rs | 4 + crates/server/src/admin/formats.rs | 35 +- crates/server/src/admin/mod.rs | 9 + crates/server/src/admin/proxy_cache.rs | 12 +- crates/server/src/admin/repositories.rs | 174 ++- crates/server/src/admin/signing_keys.rs | 186 +++ crates/server/src/browse.rs | 7 +- crates/server/src/crypto.rs | 1 + crates/server/src/files/mod.rs | 30 + crates/server/src/files/proxy.rs | 70 +- crates/server/src/files/serve.rs | 1 + crates/server/src/files/store.rs | 1 + crates/server/src/images/dispatch.rs | 1 + crates/server/src/images/proxy.rs | 2 + crates/server/src/import/feed.rs | 1 + crates/server/src/indexes/mod.rs | 490 ++++++++ crates/server/src/indexes/signing.rs | 328 ++++++ crates/server/src/lib.rs | 1 + crates/server/src/plugins.rs | 103 +- crates/server/src/state.rs | 6 +- crates/server/tests/apt.rs | 860 ++++++++++++++ crates/server/tests/plugin_contract.rs | 3 + docs/README.md | 3 + docs/SUMMARY.md | 1 + docs/apt.md | 257 ++++ docs/changes/2.2.0/176.feature.md | 11 + docs/changes/2.2.0/176.limitation.md | 2 + docs/format-plugins.md | 101 +- migrations/0009_apt.sql | 57 + web/src/api/packages.ts | 2 + web/src/api/repositories.ts | 77 ++ web/src/components/AptIndex.test.ts | 28 + web/src/components/AptIndex.tsx | 225 ++++ web/src/components/PackageConnect.test.ts | 46 +- web/src/components/PackageConnect.tsx | 106 +- web/src/components/RepositoryForm.test.tsx | 81 ++ web/src/components/RepositoryForm.tsx | 65 +- web/src/components/SigningKeysPanel.tsx | 154 +++ web/src/queries/useRepositories.ts | 35 + web/src/routes/RepoBrowse.tsx | 24 +- web/src/routes/admin/repositories/Edit.tsx | 5 + 108 files changed, 9144 insertions(+), 154 deletions(-) create mode 100755 .github/scripts/apt-e2e.sh create mode 100644 .sqlx/query-1d6cf687e785d880b8bff0305e68a1378ead641fd8af96ac1d4b1ba08dc14fb6.json create mode 100644 .sqlx/query-273a55548f16aa07d44458ec0882b1a8dc0c2d19bfe31f08a98d5375f772a07f.json create mode 100644 .sqlx/query-3b1a02033a827f289b9abbe10dca767e380d05c155eae7f876cb89c0e0d94750.json create mode 100644 .sqlx/query-4ee4d8c2ff7670dd0011282b8b10fe5ba7059f3e08f6e4fec7c21fd610d95787.json rename .sqlx/{query-57d5cf84d10de51c9f83dc2983e57a82292802112810b3d84da175738969d81b.json => query-614a470f4cd20e218fd1d0419bc71603025aa4611d362dd99f3c5f6f298acc1f.json} (62%) create mode 100644 .sqlx/query-639f6b8e885bbe8625935b8d5300b50a45ee9dae856e075d23a8984abbb60d43.json rename .sqlx/{query-83f41652bbb35c3c080c900e787eb2612023fdef91698d9e2bf116a118d43c55.json => query-7e22a7e6b91779a6e5a529fcc1b3b9a2a43c2f39ae867f52e6393f4f187a5723.json} (66%) rename .sqlx/{query-22abc387b0e4e8f6a15b06634ca8c4361fe031cda1a2247530cdf863d234a5c3.json => query-9434ba17b934138df78f5e0f94da7a04a1e81a99d0e8218c8d047d5c17be4095.json} (89%) create mode 100644 .sqlx/query-970a94e7badf03a92f54e1f054cb031318564ce98d8c1d90d07463a3c08465ba.json create mode 100644 .sqlx/query-980e592028b0baa37f9d2e43e4819dba6794896e11ee86029c56e2c39efed5bb.json create mode 100644 .sqlx/query-9ad3d42a5f4e433417b0d70e17edeb0e3a4897bc8f6f4752cd2a6016a9b2d133.json create mode 100644 .sqlx/query-9aff1f3427275d97cd3b03d0c2c2c9a1c697ebf1b3b815ccc181a21716d2b9b6.json create mode 100644 .sqlx/query-b5ac992cdbf1cb4c70996b43c1d07e080df16a30cc40389632467e12f29693a8.json create mode 100644 .sqlx/query-c6ded420e770d039677246cb5949faca8efb5fd75ff5d7b67ad8987900dc2674.json create mode 100644 .sqlx/query-da9d18c9d644ad6e08b43a5461257a4d8b826dd348b737c3c11942efff3d4682.json create mode 100644 .sqlx/query-db170fd0494ce43b0069120bb594a27478e154a8596d1f40e212bf904917db5a.json create mode 100644 .sqlx/query-e9029250004eef3621d5633a906fa5066bfdad1491e3c61bd0a4dfdc1b40b61a.json create mode 100644 .sqlx/query-f89af9953b3876e659a23f181dd9a3f8ca3ab658acab7e1727470c9816efed7c.json rename .sqlx/{query-6f2fe45f241984b02fa6a38441d64e9639c08f0146065d0a1589ac36596f500f.json => query-fc7378cc859a92d60264d8343c125e8e7574e053537d15aec3ce04939c920acd.json} (67%) create mode 100644 crates/db/src/index_dao.rs create mode 100644 crates/db/src/signing_key_dao.rs create mode 100644 crates/formats/apt/Cargo.toml create mode 100644 crates/formats/apt/src/control.rs create mode 100644 crates/formats/apt/src/deb.rs create mode 100644 crates/formats/apt/src/index.rs create mode 100644 crates/formats/apt/src/layout.rs create mode 100644 crates/formats/apt/src/lib.rs create mode 100644 crates/formats/apt/src/manifest.rs create mode 100644 crates/formats/apt/src/proxy.rs create mode 100644 crates/formats/apt/src/version.rs create mode 100644 crates/server/src/admin/signing_keys.rs create mode 100644 crates/server/src/indexes/mod.rs create mode 100644 crates/server/src/indexes/signing.rs create mode 100644 crates/server/tests/apt.rs create mode 100644 docs/apt.md create mode 100644 docs/changes/2.2.0/176.feature.md create mode 100644 docs/changes/2.2.0/176.limitation.md create mode 100644 migrations/0009_apt.sql create mode 100644 web/src/components/AptIndex.test.ts create mode 100644 web/src/components/AptIndex.tsx create mode 100644 web/src/components/SigningKeysPanel.tsx diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 9e353b7e..ba6b8868 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -4,7 +4,14 @@ [advisories] ignore = [ # rsa 0.9 Marvin timing side channel (no fixed release): a dev-dependency - # only (forge-core, forge-server and forge-e2e tests sign test JWTs with - # it); not in the shipped forge-server binary. + # (forge-core, forge-server and forge-e2e tests sign test JWTs with it), + # and linked into forge-server by rPGP (pgp, apt's signing keys), which + # never signs or decrypts with RSA there: Forge's signing keys are Ed25519, + # generated or imported (other algorithms are refused). "RUSTSEC-2023-0071", + # crypto-bigint 0.5 timing variability (no fixed 0.5 release): linked by + # rPGP (pgp) for its NIST-curve and X448 support (elliptic-curve 0.13, + # cx448), which forge-server never uses with a secret key: apt's signing + # keys are Ed25519 (curve25519-dalek), other algorithms are refused. + "RUSTSEC-2026-0387", ] diff --git a/.github/scripts/apt-e2e.sh b/.github/scripts/apt-e2e.sh new file mode 100755 index 00000000..62a3a8a7 --- /dev/null +++ b/.github/scripts/apt-e2e.sh @@ -0,0 +1,325 @@ +#!/usr/bin/env bash +# End-to-end check of apt repositories with the real apt-get (used by +# ci.yml): boots forge-server, then +# +# 1. hosted: uploads packages built with dpkg-deb (Nexus' raw-body POST +# and a PUT), refuses a redeploy and path traversal, verifies InRelease +# and Release.gpg with gpgv against the repository's key.gpg, and in a +# debian:bookworm container adds the source with signed-by and +# credentials in /etc/apt/auth.conf.d, runs apt-get update (no warning), +# installs a package, and upgrades it after a newer version is uploaded; +# 2. key rotation: adds a second signing key; InRelease and Release.gpg +# verify with either key alone, and apt-get update trusting only the +# old key still has no warning; the old key is then deleted; +# 3. proxy: a mock Debian mirror (dpkg-scanpackages, signed with its own +# key, by-hash copies) behind a proxy repository; apt-get installs +# through it, then installs again in a fresh container while the mirror +# is down. +# +# Usage: apt-e2e.sh +# Needs on PATH: curl, jq, python3, docker, dpkg-deb, dpkg-scanpackages, +# gpg, gpgv, sha256sum, gzip, openssl. Postgres and S3 come from the +# environment (defaults match ci.yml): +# FORGE_E2E_DATABASE_URL (postgres://forge:forge@localhost:5432/forge) +# FORGE_E2E_S3_ENDPOINT (http://localhost:7070), access/secret forge-e2e-key +# FORGE_E2E_PORT (8441: forge-server's) +# FORGE_E2E_MIRROR_PORT (5031: the mock mirror's) +set -euo pipefail + +server_bin="$(realpath "$1")" +results="$(mkdir -p "$2" && realpath "$2")" +db_url="${FORGE_E2E_DATABASE_URL:-postgres://forge:forge@localhost:5432/forge}" +s3_endpoint="${FORGE_E2E_S3_ENDPOINT:-http://localhost:7070}" +port="${FORGE_E2E_PORT:-8441}" +mirror_port="${FORGE_E2E_MIRROR_PORT:-5031}" +base="http://127.0.0.1:$port" +admin_password="e2e-$(openssl rand -hex 12)" +# Docker Hub images come through mirror.gcr.io (AGENTS.md), pinned. +debian_image="mirror.gcr.io/library/debian:bookworm@sha256:d74df7598108a173a621eda4d58a2adbf850b2d9a9d81bc2cca1a6c21174a880" + +work=$(mktemp -d) +server_pid="" +mirror_pid="" +cleanup() { + if [ -n "$server_pid" ]; then kill "$server_pid" 2>/dev/null || true; fi + if [ -n "$mirror_pid" ]; then kill "$mirror_pid" 2>/dev/null || true; fi + cp "$work/server.log" "$work/mirror.log" "$work"/apt-*.log "$results/" 2>/dev/null || true + rm -rf "$work" +} +trap cleanup EXIT + +step() { echo; echo "── $* ──"; } +fail() { echo "FAIL: $*" >&2; exit 1; } +expect_status() { # + local want=$1 + shift + local got + got=$(curl -sS -o "$work/body" -w '%{http_code}' "$@") + [ "$got" = "$want" ] || fail "curl $* answered $got, not $want: $(head -c 300 "$work/body")" +} + +# A binary package [section] in $work/debs. +deb() { + local name=$1 version=$2 arch=$3 section=${4:-utils} + local dir="$work/build/${name}_${version}_${arch}" + mkdir -p "$dir/DEBIAN" "$dir/usr/bin" "$work/debs" + printf '#!/bin/sh\necho "%s %s"\n' "$name" "$version" > "$dir/usr/bin/$name" + chmod 755 "$dir/usr/bin/$name" + cat > "$dir/DEBIAN/control" < +Section: $section +Priority: optional +Description: $name for the apt e2e + A package built by apt-e2e.sh. +EOF + dpkg-deb -Zxz --build --root-owner-group "$dir" "$work/debs/${name}_${version}_${arch}.deb" >/dev/null +} + +# Run