From 54cfbfe620e6001b4e33bbf584224eb477ce9f43 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 13:32:46 +0000 Subject: [PATCH] Retention: cron schedules, more triggers and filters, format retention hooks Policies of every shape (package, raw file, image) take a 5-field cron `schedule` in an IANA `schedule_timezone` as an alternative to `min_run_interval_seconds`, and `max_run_seconds`. The worker computes the next run from the schedule (croner, Vixie-cron rules across DST; chrono-tz compiled in), a saved schedule waits for its next time, Run now still runs at once, and a run that reaches its time limit stops before its next batch and leaves the rest to the next run (`deferred` in the run summary). Package rules gain the triggers `prerelease_superseded_for_days`, `unlisted_for_days` (with `component.unlisted_at`, kept by a trigger on every unlist path; versions unlisted before take `updated_at`) and `max_versions_per_package` (hosted and proxy), and the filters `version_pattern`, `exclude_version_pattern`, `selector` (a content selector, which cannot be deleted while a rule names it) and `release_type: snapshots`. Image rules gain `tag_pattern`, `exclude_tag_pattern` and globs in `protected_tags`. Protections win over every trigger. The format contract gets `version_class` and `retention_partition`, which the evaluator asks for classes and keep-N partitions. The admin API validates every new field, the preview lists a schedule's next runs and each decision's reason, the Retention screen edits all of it, and docs/retention.md lists every rule with examples. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012e6DJc3No8xu6rxoMc3pFJ --- ...d0edf17b4b43fe4020fd90489a2a88fb7c2c.json} | 18 +- ...563f11eb852912862ee377291217d5d4438f8.json | 17 - ...81769c74787c94e8d3fa8f23fd050ef9b2d9.json} | 18 +- ...9287b465825ce10502cd3299f7ecaf41d4084.json | 28 + ...d6b0361712949ca498a1ea7a7a8cb4012f59e.json | 18 + ...aad637e039ae6ef341bc7ac788715bcf88fb5.json | 18 + ...110ca930bb5ce71555fb59af293af05986043.json | 15 + ...baa119b01b48789559f7c20145d2c368f7cbb.json | 17 - Cargo.lock | 145 +- Cargo.toml | 9 + README.md | 6 +- crates/core/Cargo.toml | 4 + crates/core/src/format.rs | 21 + crates/core/src/retention/apply.rs | 81 +- crates/core/src/retention/evaluate.rs | 1226 ++++++++++++++--- crates/core/src/retention/files.rs | 62 +- crates/core/src/retention/images.rs | 255 +++- crates/core/src/retention/mod.rs | 59 +- crates/core/src/retention/model.rs | 407 +++++- crates/core/src/retention/schedule.rs | 345 +++++ crates/core/src/retention/worker.rs | 95 +- crates/core/tests/retention_apply.rs | 98 +- crates/core/tests/retention_worker.rs | 208 ++- crates/db/src/retention_dao.rs | 86 +- crates/db/tests/migrations.rs | 74 + crates/db/tests/retention.rs | 239 +++- crates/plugin-api/src/lib.rs | 28 + crates/plugin-api/src/types.rs | 17 + crates/plugin/src/format.rs | 66 + crates/server/Cargo.toml | 2 + crates/server/src/admin/content_selectors.rs | 28 +- crates/server/src/admin/retention.rs | 109 +- crates/server/tests/admin_retention.rs | 384 +++++- crates/server/tests/raw.rs | 1 + crates/server/tests/registry.rs | 1 + docs/README.md | 6 +- docs/SUMMARY.md | 2 +- docs/changes/2.2.0/185.feature.md | 19 + docs/changes/2.2.0/185.upgrade.md | 8 + docs/docker.md | 17 +- docs/format-plugins.md | 39 +- docs/npm.md | 3 +- docs/raw.md | 7 +- docs/retention.md | 370 ++++- migrations/0011_retention_unlisted_at.sql | 45 + web/src/api/retention.ts | 39 +- .../components/RetentionPolicyForm.test.ts | 89 ++ web/src/components/RetentionPolicyForm.tsx | 296 +++- .../routes/admin/repositories/Retention.tsx | 30 +- .../admin/repositories/retention.test.tsx | 98 ++ 50 files changed, 4881 insertions(+), 392 deletions(-) rename .sqlx/{query-f87de28a0a8dbc58f3e231d8a84b037970cee3202a4d0d5a680f733ffa01c18d.json => query-039cc5d684741f9b87c7a56d99b3d0edf17b4b43fe4020fd90489a2a88fb7c2c.json} (70%) delete mode 100644 .sqlx/query-1aeb41f9748ce3776c8bd76538f563f11eb852912862ee377291217d5d4438f8.json rename .sqlx/{query-73abec0f274ad4948b5b907c6bc9060725361eb9e97453bc631711c8f5718dc9.json => query-1c1ec5df18ded5289ec21d5d96f781769c74787c94e8d3fa8f23fd050ef9b2d9.json} (71%) create mode 100644 .sqlx/query-26e9e6f52611dc47418158b10fe9287b465825ce10502cd3299f7ecaf41d4084.json create mode 100644 .sqlx/query-5ba94dea0a96eb1fe29086f8e03d6b0361712949ca498a1ea7a7a8cb4012f59e.json create mode 100644 .sqlx/query-73ba20cf0d00b6eac1d7762e06faad637e039ae6ef341bc7ac788715bcf88fb5.json create mode 100644 .sqlx/query-a5efd4737ae2ef552f36d832cda110ca930bb5ce71555fb59af293af05986043.json delete mode 100644 .sqlx/query-c84063b83a57f6b0a57e1921dd7baa119b01b48789559f7c20145d2c368f7cbb.json create mode 100644 crates/core/src/retention/schedule.rs create mode 100644 docs/changes/2.2.0/185.feature.md create mode 100644 docs/changes/2.2.0/185.upgrade.md create mode 100644 migrations/0011_retention_unlisted_at.sql diff --git a/.sqlx/query-f87de28a0a8dbc58f3e231d8a84b037970cee3202a4d0d5a680f733ffa01c18d.json b/.sqlx/query-039cc5d684741f9b87c7a56d99b3d0edf17b4b43fe4020fd90489a2a88fb7c2c.json similarity index 70% rename from .sqlx/query-f87de28a0a8dbc58f3e231d8a84b037970cee3202a4d0d5a680f733ffa01c18d.json rename to .sqlx/query-039cc5d684741f9b87c7a56d99b3d0edf17b4b43fe4020fd90489a2a88fb7c2c.json index 518762ee..3da243bf 100644 --- a/.sqlx/query-f87de28a0a8dbc58f3e231d8a84b037970cee3202a4d0d5a680f733ffa01c18d.json +++ b/.sqlx/query-039cc5d684741f9b87c7a56d99b3d0edf17b4b43fe4020fd90489a2a88fb7c2c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n c.id AS component_id,\n c.name AS name,\n c.version AS version,\n c.listed AS listed,\n c.created_at AS created_at,\n c.last_downloaded_at AS last_downloaded_at,\n COALESCE((SELECT SUM(a.size) FROM asset a WHERE a.component_id = c.id), 0)::bigint\n AS \"size_bytes!\",\n (\n SELECT a.blob_sha256\n FROM asset a\n WHERE a.component_id = c.id AND (a.path = $2 OR a.path LIKE '%.nupkg')\n ORDER BY a.path\n LIMIT 1\n ) AS \"blob_sha256?: String\"\n FROM component c\n WHERE c.repository_id = $1\n ORDER BY c.name, c.id\n ", + "query": "\n SELECT\n c.id AS component_id,\n c.name AS name,\n c.version AS version,\n c.listed AS listed,\n c.created_at AS created_at,\n c.last_downloaded_at AS last_downloaded_at,\n c.unlisted_at AS unlisted_at,\n COALESCE((SELECT SUM(a.size) FROM asset a WHERE a.component_id = c.id), 0)::bigint\n AS \"size_bytes!\",\n (\n SELECT a.blob_sha256\n FROM asset a\n WHERE a.component_id = c.id AND (a.path = $2 OR a.path LIKE '%.nupkg')\n ORDER BY a.path\n LIMIT 1\n ) AS \"blob_sha256?: String\"\n FROM component c\n WHERE c.repository_id = $1\n ORDER BY c.name, c.id\n ", "describe": { "columns": [ { @@ -71,12 +71,23 @@ }, { "ordinal": 6, + "name": "unlisted_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "component", + "name": "unlisted_at" + } + } + }, + { + "ordinal": 7, "name": "size_bytes!", "type_info": "Int8", "origin": "Expression" }, { - "ordinal": 7, + "ordinal": 8, "name": "blob_sha256?: String", "type_info": "Text", "origin": "Expression" @@ -95,9 +106,10 @@ false, false, true, + true, null, null ] }, - "hash": "f87de28a0a8dbc58f3e231d8a84b037970cee3202a4d0d5a680f733ffa01c18d" + "hash": "039cc5d684741f9b87c7a56d99b3d0edf17b4b43fe4020fd90489a2a88fb7c2c" } diff --git a/.sqlx/query-1aeb41f9748ce3776c8bd76538f563f11eb852912862ee377291217d5d4438f8.json b/.sqlx/query-1aeb41f9748ce3776c8bd76538f563f11eb852912862ee377291217d5d4438f8.json deleted file mode 100644 index 7d806acd..00000000 --- a/.sqlx/query-1aeb41f9748ce3776c8bd76538f563f11eb852912862ee377291217d5d4438f8.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE retention_run\n SET state = 'idle',\n finished_at = now(),\n locked_until = NULL,\n current_run_id = NULL,\n next_run_at = $2,\n last_run_summary = $3\n WHERE repository_id = $1\n AND ($4::uuid IS NULL OR current_run_id = $4)\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Timestamptz", - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "1aeb41f9748ce3776c8bd76538f563f11eb852912862ee377291217d5d4438f8" -} diff --git a/.sqlx/query-73abec0f274ad4948b5b907c6bc9060725361eb9e97453bc631711c8f5718dc9.json b/.sqlx/query-1c1ec5df18ded5289ec21d5d96f781769c74787c94e8d3fa8f23fd050ef9b2d9.json similarity index 71% rename from .sqlx/query-73abec0f274ad4948b5b907c6bc9060725361eb9e97453bc631711c8f5718dc9.json rename to .sqlx/query-1c1ec5df18ded5289ec21d5d96f781769c74787c94e8d3fa8f23fd050ef9b2d9.json index c31bd5d4..4103e5b9 100644 --- a/.sqlx/query-73abec0f274ad4948b5b907c6bc9060725361eb9e97453bc631711c8f5718dc9.json +++ b/.sqlx/query-1c1ec5df18ded5289ec21d5d96f781769c74787c94e8d3fa8f23fd050ef9b2d9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n c.id AS component_id,\n c.name AS name,\n c.version AS version,\n c.listed AS listed,\n c.created_at AS created_at,\n c.last_downloaded_at AS last_downloaded_at,\n EXISTS (\n SELECT 1 FROM asset a\n WHERE a.component_id = c.id AND (a.path = $3 OR a.path LIKE '%.nupkg')\n ) AS \"has_nupkg!\",\n COALESCE((SELECT SUM(a.size) FROM asset a WHERE a.component_id = c.id), 0)::bigint\n AS \"size_bytes!\"\n FROM component c\n WHERE c.repository_id = $1 AND c.name = $2\n ORDER BY c.id\n FOR UPDATE OF c\n ", + "query": "\n SELECT\n c.id AS component_id,\n c.name AS name,\n c.version AS version,\n c.listed AS listed,\n c.created_at AS created_at,\n c.last_downloaded_at AS last_downloaded_at,\n c.unlisted_at AS unlisted_at,\n EXISTS (\n SELECT 1 FROM asset a\n WHERE a.component_id = c.id AND (a.path = $3 OR a.path LIKE '%.nupkg')\n ) AS \"has_nupkg!\",\n COALESCE((SELECT SUM(a.size) FROM asset a WHERE a.component_id = c.id), 0)::bigint\n AS \"size_bytes!\"\n FROM component c\n WHERE c.repository_id = $1 AND c.name = $2\n ORDER BY c.id\n FOR UPDATE OF c\n ", "describe": { "columns": [ { @@ -71,12 +71,23 @@ }, { "ordinal": 6, + "name": "unlisted_at", + "type_info": "Timestamptz", + "origin": { + "Table": { + "table": "component", + "name": "unlisted_at" + } + } + }, + { + "ordinal": 7, "name": "has_nupkg!", "type_info": "Bool", "origin": "Expression" }, { - "ordinal": 7, + "ordinal": 8, "name": "size_bytes!", "type_info": "Int8", "origin": "Expression" @@ -96,9 +107,10 @@ false, false, true, + true, null, null ] }, - "hash": "73abec0f274ad4948b5b907c6bc9060725361eb9e97453bc631711c8f5718dc9" + "hash": "1c1ec5df18ded5289ec21d5d96f781769c74787c94e8d3fa8f23fd050ef9b2d9" } diff --git a/.sqlx/query-26e9e6f52611dc47418158b10fe9287b465825ce10502cd3299f7ecaf41d4084.json b/.sqlx/query-26e9e6f52611dc47418158b10fe9287b465825ce10502cd3299f7ecaf41d4084.json new file mode 100644 index 00000000..bb19db7f --- /dev/null +++ b/.sqlx/query-26e9e6f52611dc47418158b10fe9287b465825ce10502cd3299f7ecaf41d4084.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT r.name\n FROM repository r\n WHERE r.retention_policy IS NOT NULL\n AND jsonb_path_exists(\n r.retention_policy,\n '$.rules[*] ? (@.selector == $name)',\n jsonb_build_object('name', $1::text)\n )\n ORDER BY r.name\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "name", + "type_info": "Text", + "origin": { + "Table": { + "table": "repository", + "name": "name" + } + } + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "26e9e6f52611dc47418158b10fe9287b465825ce10502cd3299f7ecaf41d4084" +} diff --git a/.sqlx/query-5ba94dea0a96eb1fe29086f8e03d6b0361712949ca498a1ea7a7a8cb4012f59e.json b/.sqlx/query-5ba94dea0a96eb1fe29086f8e03d6b0361712949ca498a1ea7a7a8cb4012f59e.json new file mode 100644 index 00000000..c1740a47 --- /dev/null +++ b/.sqlx/query-5ba94dea0a96eb1fe29086f8e03d6b0361712949ca498a1ea7a7a8cb4012f59e.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE retention_run\n SET state = 'idle',\n finished_at = now(),\n locked_until = NULL,\n current_run_id = NULL,\n next_run_at = CASE\n WHEN $5::jsonb IS NULL\n OR (SELECT r.retention_policy FROM repository r WHERE r.id = $1) = $5::jsonb\n THEN $2\n ELSE retention_run.next_run_at\n END,\n last_error = $3\n WHERE repository_id = $1\n AND ($4::uuid IS NULL OR current_run_id = $4)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Timestamptz", + "Text", + "Uuid", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "5ba94dea0a96eb1fe29086f8e03d6b0361712949ca498a1ea7a7a8cb4012f59e" +} diff --git a/.sqlx/query-73ba20cf0d00b6eac1d7762e06faad637e039ae6ef341bc7ac788715bcf88fb5.json b/.sqlx/query-73ba20cf0d00b6eac1d7762e06faad637e039ae6ef341bc7ac788715bcf88fb5.json new file mode 100644 index 00000000..1817097e --- /dev/null +++ b/.sqlx/query-73ba20cf0d00b6eac1d7762e06faad637e039ae6ef341bc7ac788715bcf88fb5.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE retention_run\n SET state = 'idle',\n finished_at = now(),\n locked_until = NULL,\n current_run_id = NULL,\n next_run_at = CASE\n WHEN $5::jsonb IS NULL\n OR (SELECT r.retention_policy FROM repository r WHERE r.id = $1) = $5::jsonb\n THEN $2\n ELSE retention_run.next_run_at\n END,\n last_run_summary = $3\n WHERE repository_id = $1\n AND ($4::uuid IS NULL OR current_run_id = $4)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Timestamptz", + "Jsonb", + "Uuid", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "73ba20cf0d00b6eac1d7762e06faad637e039ae6ef341bc7ac788715bcf88fb5" +} diff --git a/.sqlx/query-a5efd4737ae2ef552f36d832cda110ca930bb5ce71555fb59af293af05986043.json b/.sqlx/query-a5efd4737ae2ef552f36d832cda110ca930bb5ce71555fb59af293af05986043.json new file mode 100644 index 00000000..5a795efc --- /dev/null +++ b/.sqlx/query-a5efd4737ae2ef552f36d832cda110ca930bb5ce71555fb59af293af05986043.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE retention_run SET next_run_at = $2 WHERE repository_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "a5efd4737ae2ef552f36d832cda110ca930bb5ce71555fb59af293af05986043" +} diff --git a/.sqlx/query-c84063b83a57f6b0a57e1921dd7baa119b01b48789559f7c20145d2c368f7cbb.json b/.sqlx/query-c84063b83a57f6b0a57e1921dd7baa119b01b48789559f7c20145d2c368f7cbb.json deleted file mode 100644 index e7e970ba..00000000 --- a/.sqlx/query-c84063b83a57f6b0a57e1921dd7baa119b01b48789559f7c20145d2c368f7cbb.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE retention_run\n SET state = 'idle',\n finished_at = now(),\n locked_until = NULL,\n current_run_id = NULL,\n next_run_at = $2,\n last_error = $3\n WHERE repository_id = $1\n AND ($4::uuid IS NULL OR current_run_id = $4)\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Timestamptz", - "Text", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "c84063b83a57f6b0a57e1921dd7baa119b01b48789559f7c20145d2c368f7cbb" -} diff --git a/Cargo.lock b/Cargo.lock index 71d5529b..358a622a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1045,6 +1045,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf", +] + [[package]] name = "cipher" version = "0.5.2" @@ -1207,6 +1217,17 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "croner" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b5b023fc2978536bbb2a3b71237964fc4630bec1eb31077b4977127f4cdb9c0" +dependencies = [ + "chrono", + "derive_builder", + "strum", +] + [[package]] name = "crossbeam-channel" version = "0.5.17" @@ -1285,14 +1306,38 @@ dependencies = [ "cmov", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.24.1", + "darling_macro 0.24.1", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.119", ] [[package]] @@ -1308,13 +1353,24 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn 2.0.119", +] + [[package]] name = "darling_macro" version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ - "darling_core", + "darling_core 0.24.1", "quote", "syn 3.0.6", ] @@ -1408,6 +1464,37 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.10.7" @@ -1667,6 +1754,8 @@ dependencies = [ "axum", "base64 0.23.1", "chrono", + "chrono-tz", + "croner", "flate2", "forge-cache", "forge-crypto", @@ -1873,6 +1962,7 @@ dependencies = [ "base64 0.23.1", "bytes", "chrono", + "chrono-tz", "figment", "flate2", "forge-cache", @@ -3494,6 +3584,24 @@ dependencies = [ "getrandom 0.4.3", ] +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.13" @@ -4357,7 +4465,7 @@ version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" dependencies = [ - "darling", + "darling 0.24.1", "proc-macro2", "quote", "syn 3.0.6", @@ -4482,6 +4590,12 @@ dependencies = [ "time", ] +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + [[package]] name = "slab" version = "0.4.12" @@ -4763,6 +4877,27 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "subtle" version = "2.6.1" @@ -4866,7 +5001,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", diff --git a/Cargo.toml b/Cargo.toml index f9123884..98797614 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -207,6 +207,15 @@ tempfile = { version = "3" } # Date/time (shared with forge-db's model types) chrono = { version = "0.4", default-features = false, features = ["clock"] } +# Retention schedules (`schedule` + `schedule_timezone` in a retention +# policy): croner parses 5-field cron patterns and finds their next time in a +# chrono time zone, with Vixie-cron/OCPS rules across daylight-saving +# changes (a time in a skipped hour runs right after it, a time in a repeated +# hour once); chrono-tz is the IANA time-zone database compiled in, so a zone +# resolves the same in a container image without /usr/share/zoneinfo. +croner = { version = "4.0.1", default-features = false, features = ["chrono"] } +chrono-tz = { version = "0.10" } + # Tests build unoptimized, yet the mock IdPs generate RSA keys and every test # server hashes its bootstrap admin password: ~30x (RSA-2048 keygen, ~3 s) and # ~15x (Argon2 default, ~0.3 s) slower than optimized. Optimize just those diff --git a/README.md b/README.md index 8a8241ee..e502e4c9 100644 --- a/README.md +++ b/README.md @@ -220,9 +220,9 @@ The guides below are also a site, with navigation and search: (hosted, proxy, group): uploads with curl, ranges and checksum sidecars, listings, the deployment policy, proxies of release downloads (the pdfium example) and retention. -- **[Retention size budgets](./docs/retention.md)** — keep a repository - under a logical size: the budget in the retention policy, how size is - counted, and what goes first. +- **[Retention](./docs/retention.md)** — the retention rules of each kind + of repository with examples, cron schedules in a time zone with a time + limit per run, and size budgets. - **[Trusted publishing](./docs/trusted-publishing.md)** — configure short-lived workload identity for GitHub Actions, GitLab CI, and compatible OIDC issuers, and exchange a job's OIDC token for a short-lived API key (`POST /api/v2/token`) diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 313385c8..ac9d1d93 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -31,6 +31,10 @@ uuid = { workspace = true } # into a GlobSet held behind Arc in the cached effective set. globset = { workspace = true } +# Retention schedules: cron patterns and IANA time zones. +croner = { workspace = true } +chrono-tz = { workspace = true } + # Auth argon2 = { workspace = true } sha2 = { workspace = true } diff --git a/crates/core/src/format.rs b/crates/core/src/format.rs index e4422a31..692aff96 100644 --- a/crates/core/src/format.rs +++ b/crates/core/src/format.rs @@ -7,6 +7,7 @@ use std::cmp::Ordering; +pub use forge_plugin_api::VersionClass; use forge_plugin_api::{FetchResult, Remote, UpstreamOp, UpstreamStep}; use crate::domain::{FormatError, ParsedUpload}; @@ -82,6 +83,26 @@ pub trait Format: Send + Sync { /// Lowercase / canonicalize a package id for case-insensitive lookups. fn lower_id(&self, id: &str) -> String; + /// Retention: the class of a **normalized** version (release, prerelease + /// or snapshot). The default is SemVer's rule: a version with a `-` is a + /// prerelease (build metadata is gone once normalized), any other a + /// release. See [`forge_plugin_api::FormatPlugin::version_class`]. + fn version_class(&self, version: &str) -> VersionClass { + if version.contains('-') { + VersionClass::Prerelease + } else { + VersionClass::Release + } + } + + /// Retention: the partition of package `id`'s versions within which the + /// keep-N rules count; `None` (the default) keeps the whole package one + /// partition. See [`forge_plugin_api::FormatPlugin::retention_partition`]. + fn retention_partition(&self, id: &str, version: &str) -> Option { + let _ = (id, version); + None + } + /// The asset path a version's package archive is stored under (the /// `asset.path` of its component). Internal to the repository: clients /// never see it. NuGet keeps `package.nupkg`. diff --git a/crates/core/src/retention/apply.rs b/crates/core/src/retention/apply.rs index 79ec6a08..431c8d71 100644 --- a/crates/core/src/retention/apply.rs +++ b/crates/core/src/retention/apply.rs @@ -36,6 +36,12 @@ //! When `dry_run` is set (preview or a `dry_run_only` policy) no mutation runs; //! we still record `dry_run = true` audit rows so an operator can inspect what //! *would* happen. +//! +//! With a `deadline` (the policy's `max_run_seconds`), the run stops before +//! the first batch or eviction that would start after it: the decisions left +//! are counted as `deferred`, and the next run evaluates the repository again +//! and goes on with them. The first step always goes, so a run whose +//! evaluation alone outlasts its window still makes progress. use std::collections::{HashMap, HashSet}; @@ -49,9 +55,10 @@ use forge_db::{asset_dao, blob_dao, component_dao}; use crate::format::Format; use crate::retention::evaluate::{ evaluate, evaluate_proxy, size_budget_protected_ids, CandidateComponent, Decision, - ProxyCandidate, ProxyDecision, + ProxyCandidate, ProxyDecision, Selectors, }; use crate::retention::model::{Action, RetentionPolicy}; +use crate::retention::schedule::RunClock; /// What a run's decisions were made under, re-checked before each one is /// applied (see the module docs). @@ -63,6 +70,9 @@ pub struct RunFence<'a> { pub policy: &'a RetentionPolicy, /// The repository's format: orders versions for the re-evaluation. pub format: &'a dyn Format, + /// The content selectors the policy's rules name, as the run resolved + /// them. + pub selectors: &'a Selectors, } /// Whether the run goes on after a step. @@ -98,6 +108,9 @@ pub struct ApplyContext<'a> { /// Fence and re-evaluate every mutating step (the worker always sets /// it). `None` applies the decisions as given. pub fence: Option>, + /// When the run must stop applying (`max_run_seconds` after it + /// started); `None` applies everything. + pub deadline: Option, } /// Whether a download-driven decision became stale: the target was fetched @@ -143,15 +156,22 @@ pub async fn apply( ) -> Result { let total = hosted.len() + proxy.len(); let mut summary = RunSummary::default(); + let mut clock = RunClock::new(ctx.deadline); let mut done = 0; if ctx.dry_run { for d in hosted { + if clock.out_of_time() { + return Ok(deferred(ctx, summary, total - done)); + } record_hosted_dry_run(ctx, d, &mut summary).await?; + done += 1; } - done += hosted.len(); } else { for batch in package_batches(hosted) { + if clock.out_of_time() { + return Ok(deferred(ctx, summary, total - done)); + } if apply_hosted_batch(ctx, &batch, &mut summary).await? == Step::Stop { return Ok(stopped(ctx, summary, total - done)); } @@ -160,6 +180,9 @@ pub async fn apply( } if ctx.dry_run { for d in proxy { + if clock.out_of_time() { + return Ok(deferred(ctx, summary, total - done)); + } if apply_proxy(ctx, d, &mut summary).await? == Step::Stop { return Ok(stopped(ctx, summary, total - done)); } @@ -167,6 +190,9 @@ pub async fn apply( } } else { for d in proxy.iter().filter(|d| d.reason.size_budget.is_none()) { + if clock.out_of_time() { + return Ok(deferred(ctx, summary, total - done)); + } if apply_proxy(ctx, d, &mut summary).await? == Step::Stop { return Ok(stopped(ctx, summary, total - done)); } @@ -182,6 +208,9 @@ pub async fn apply( }); if revalidate_protections { for batch in proxy_budget_batches(proxy) { + if clock.out_of_time() { + return Ok(deferred(ctx, summary, total - done)); + } if apply_proxy_budget_batch(ctx, &batch, &mut summary).await? == Step::Stop { return Ok(stopped(ctx, summary, total - done)); } @@ -189,6 +218,9 @@ pub async fn apply( } } else { for d in proxy.iter().filter(|d| d.reason.size_budget.is_some()) { + if clock.out_of_time() { + return Ok(deferred(ctx, summary, total - done)); + } if apply_proxy(ctx, d, &mut summary).await? == Step::Stop { return Ok(stopped(ctx, summary, total - done)); } @@ -249,6 +281,19 @@ fn stopped(ctx: &ApplyContext<'_>, mut summary: RunSummary, remaining: usize) -> summary } +/// The summary of a run that reached its deadline with `remaining` +/// decisions unapplied (deferred: the next run evaluates them again). +fn deferred(ctx: &ApplyContext<'_>, mut summary: RunSummary, remaining: usize) -> RunSummary { + tracing::info!( + repository = ctx.repository_name, + run_id = %ctx.run_id, + remaining, + "retention run stopped at its max_run_seconds; the next run goes on with the rest" + ); + summary.deferred += remaining as u64; + summary +} + /// What evaluating `package_name`'s current versions (locked by this call) /// under the run's policy decides now, by component id. async fn decided_now( @@ -273,11 +318,18 @@ async fn decided_now( listed: r.listed, created_at: r.created_at, last_downloaded_at: r.last_downloaded_at, + unlisted_at: r.unlisted_at, size_bytes: u64::try_from(r.size_bytes).unwrap_or_default(), }) .collect(); let now = ctx.evaluated_at.unwrap_or_else(chrono::Utc::now); - let mut ordinary = evaluate(fence.format, now, fence.policy, &candidates); + let mut ordinary = evaluate( + fence.format, + now, + fence.policy, + fence.selectors, + &candidates, + ); if !budget_ids.is_empty() { let tagged: HashSet<(String, String)> = forge_db::tag_dao::tagged_versions(&mut *tx, ctx.repository_id) @@ -293,8 +345,15 @@ async fn decided_now( .map(|again| (again.component_id, again.action)) .collect(); if !budget_ids.is_empty() && fence.policy.size_budget.is_some() { - let (protected, _) = - size_budget_protected_ids(fence.format, fence.policy, &candidates, &[], &ordinary, &[]); + let (protected, _) = size_budget_protected_ids( + fence.format, + fence.policy, + fence.selectors, + &candidates, + &[], + &ordinary, + &[], + ); for candidate in &candidates { if budget_ids.contains(&candidate.component_id) && !protected.contains(&candidate.component_id) @@ -718,10 +777,18 @@ async fn apply_proxy_budget_batch( fence.format, ctx.evaluated_at.unwrap_or_else(chrono::Utc::now), fence.policy, + fence.selectors, + &candidates, + ); + let (_, protected) = size_budget_protected_ids( + fence.format, + fence.policy, + fence.selectors, + &[], &candidates, + &[], + &ordinary, ); - let (_, protected) = - size_budget_protected_ids(fence.format, fence.policy, &[], &candidates, &[], &ordinary); let candidates_by_id: HashMap = candidates .iter() .map(|candidate| (candidate.cached_version_id, candidate)) diff --git a/crates/core/src/retention/evaluate.rs b/crates/core/src/retention/evaluate.rs index ea068e90..0f5ca676 100644 --- a/crates/core/src/retention/evaluate.rs +++ b/crates/core/src/retention/evaluate.rs @@ -3,29 +3,38 @@ //! Given a policy and a snapshot of a repository's components (or a proxy repo's //! cached versions), [`evaluate`] / [`evaluate_proxy`] compute the set of //! `Decision`s the worker should apply. There is **no I/O** here: every input is -//! plain data plus a `&dyn Format` used only to order versions, so the whole -//! algorithm is unit-testable without a database or clock. +//! plain data, the content selectors the rules name ([`Selectors`], resolved +//! by the caller), and a `&dyn Format` used to order and classify versions, +//! so the whole algorithm is unit-testable without a database or clock. //! //! ## Algorithm (hosted, [`evaluate`]) //! -//! 1. Filter out components whose version fails to apply to any rule's -//! `release_type` / `package_name_prefix` — handled per rule. -//! 2. **Group by package id.** Within a group, order versions newest-first using +//! 1. **Group by package id.** Within a group, order versions newest-first using //! [`Format::compare_versions`] (format-agnostic — we never import a concrete //! format here). -//! 3. **Compute protections per group, per rule.** Each rule's protections -//! are taken over that rule's own *pool* — the group's versions its -//! `release_type` / `package_name_prefix` filters admit: `keep_last_n` is -//! the newest N **listed** versions of the pool (unlisted versions neither -//! count nor are protected), `keep_n_most_downloaded` the N most-recently -//! downloaded versions of the pool, listed or not (nulls last). The -//! protected set is the **union** over all rules; a protected version is -//! **never** deleted or unlisted, by any rule. -//! 4. For each version, a rule *selects* it iff the rule's filters match **and** -//! all of the rule's present triggers are satisfied (ANDed): -//! `older_than_days` (age since `created_at`) and `not_downloaded_for_days` +//! 2. **A rule's pool** is the group's versions its filters admit: +//! `release_type` (the version's [`Format::version_class`]), +//! `package_name_prefix`, `selector` (a content selector's globs on the +//! package name) and `version_pattern` / `exclude_version_pattern` (globs +//! on the version). Keep-N counts run per **partition** of the pool +//! ([`Format::retention_partition`]; the whole package by default). +//! 3. **Compute protections per group, per rule**, over each partition of +//! the rule's pool: `keep_last_n` is the newest N **listed** versions +//! (unlisted versions neither count nor are protected), +//! `keep_n_most_downloaded` the N most-recently downloaded versions, +//! listed or not (nulls last). The protected set is the **union** over all +//! rules; a protected version is **never** deleted or unlisted, by any +//! rule, whatever trigger selects it. +//! 4. For each version, a rule *selects* it iff the rule's pool holds it +//! **and** all of the rule's present triggers are satisfied (ANDed): +//! `older_than_days` (age since `created_at`), `not_downloaded_for_days` //! (days since the last download or, for a version never downloaded, since -//! it was published — so a fresh push is not cold on its first run). +//! it was published — so a fresh push is not cold on its first run), +//! `prerelease_superseded_for_days` (a prerelease or snapshot, once the +//! first listed release of its package that orders after it has been +//! published that long), `unlisted_for_days` (unlisted that long, by +//! `unlisted_at`) and `max_versions_per_package` (not among the newest N +//! of its partition of the pool, listed or not). //! 5. Protection overrides selection. Otherwise the highest-severity action //! wins (`hard_delete` > `unlist`). //! @@ -33,21 +42,25 @@ //! //! `scope: proxy_cached` rules evict a cached version when `not_pulled_for_days` //! is satisfied, comparing against `last_downloaded_at` or, when never pulled, -//! `first_seen_at`. Only versions whose bytes are cached locally are -//! candidates. Their protections work as on hosted rules, over each package's -//! cached versions: `keep_last_n` keeps the newest N the upstream lists, -//! `keep_n_most_downloaded` the N pulled last, and a version any proxy rule -//! keeps is never evicted. Proxy entries are never unlisted — eviction always -//! deletes the cached bytes (the version stays listed and re-caches on the -//! next pull). +//! `first_seen_at`, and/or when `max_versions_per_package` counts it out of +//! its package's newest cached versions. Only versions whose bytes are cached +//! locally are candidates. Filters and protections work as on hosted rules, +//! over each package's cached versions: `keep_last_n` keeps the newest N the +//! upstream lists, `keep_n_most_downloaded` the N pulled last, and a version +//! any proxy rule keeps is never evicted. Proxy entries are never unlisted — +//! eviction always deletes the cached bytes (the version stays listed and +//! re-caches on the next pull). use std::collections::{HashMap, HashSet}; use chrono::{DateTime, Utc}; +use globset::GlobMatcher; -use crate::format::Format; +use crate::authz::CompiledSelector; +use crate::format::{Format, VersionClass}; use crate::retention::model::{ - Action, Criteria, ReleaseType, RetentionPolicy, RetentionRule, Scope, SizeBudgetSummary, + version_glob, Action, Criteria, ReleaseType, RetentionPolicy, RetentionRule, Scope, + SizeBudgetSummary, }; /// One hosted component in the evaluation snapshot. All fields come straight @@ -66,6 +79,9 @@ pub struct CandidateComponent { pub created_at: DateTime, /// Last download timestamp, or `None` if never downloaded. pub last_downloaded_at: Option>, + /// When the version was unlisted (`component.unlisted_at`, for + /// `unlisted_for_days`); `None` while it is listed. + pub unlisted_at: Option>, /// Sum of this version's asset sizes, in bytes. pub size_bytes: u64, } @@ -166,32 +182,61 @@ pub struct PackagePlan { pub budget: Option, } +/// The content selectors a policy's rules name (`selector`), compiled and +/// looked up by name. The caller resolves them (the worker and the preview +/// read them from `content_selector`); a rule naming a selector missing +/// here admits nothing. +#[derive(Debug, Clone, Default)] +pub struct Selectors { + by_name: HashMap, +} + +impl Selectors { + /// No selectors: rules that name one admit nothing. + pub fn none() -> Selectors { + Selectors::default() + } + + /// Add `selector` under `name`. + pub fn insert(&mut self, name: impl Into, selector: CompiledSelector) { + self.by_name.insert(name.into(), selector); + } + + /// The selector named `name`. + pub fn get(&self, name: &str) -> Option<&CompiledSelector> { + self.by_name.get(name) + } +} + /// What one run of `policy` decides for a package repository as of `now`: /// the rules' decisions, less those on versions a package tag names /// (`tagged_hosted_ids`, which retention never touches), then the size /// budget's on top of them when the policy has one (`repository_size_bytes` /// is the repository's size then). A version the budget deletes is not also /// unlisted. The worker and the preview both plan with this. +#[allow(clippy::too_many_arguments)] pub fn plan_package_run( format: &dyn Format, now: DateTime, policy: &RetentionPolicy, + selectors: &Selectors, hosted: &[CandidateComponent], proxy: &[ProxyCandidate], tagged_hosted_ids: &HashSet, repository_size_bytes: Option, ) -> PackagePlan { - let mut hosted_decisions = evaluate(format, now, policy, hosted); + let mut hosted_decisions = evaluate(format, now, policy, selectors, hosted); let before = hosted_decisions.len(); hosted_decisions.retain(|d| !tagged_hosted_ids.contains(&d.component_id)); let spared = before - hosted_decisions.len(); - let mut proxy_decisions = evaluate_proxy(format, now, policy, proxy); + let mut proxy_decisions = evaluate_proxy(format, now, policy, selectors, proxy); let mut budget = None; if let (Some(_), Some(size)) = (policy.size_budget, repository_size_bytes) { let mut plan = evaluate_size_budget( format, policy, + selectors, hosted, proxy, size, @@ -218,7 +263,7 @@ pub fn plan_package_run( /// Structured explanation of why a version was selected, recorded on the audit /// row's `reason` jsonb. -#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct DecisionReason { /// Whether `older_than_days` was a matched trigger (and its threshold). #[serde(skip_serializing_if = "Option::is_none")] @@ -229,6 +274,23 @@ pub struct DecisionReason { /// Whether `not_pulled_for_days` was a matched trigger (proxy). #[serde(skip_serializing_if = "Option::is_none")] pub not_pulled_for_days: Option, + /// Whether `prerelease_superseded_for_days` was a matched trigger. + #[serde(skip_serializing_if = "Option::is_none")] + pub prerelease_superseded_for_days: Option, + /// The release that superseded the prerelease: the first one published + /// of the listed releases that order after it. + #[serde(skip_serializing_if = "Option::is_none")] + pub superseded_by: Option, + /// Whether `unlisted_for_days` was a matched trigger. + #[serde(skip_serializing_if = "Option::is_none")] + pub unlisted_for_days: Option, + /// Whether `max_versions_per_package` was a matched trigger. + #[serde(skip_serializing_if = "Option::is_none")] + pub max_versions_per_package: Option, + /// With `max_versions_per_package`: the version's place among its + /// package's (partition's) versions in the rule's pool, newest first. + #[serde(skip_serializing_if = "Option::is_none")] + pub rank: Option, #[serde(skip_serializing_if = "Option::is_none")] pub size_budget: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -239,33 +301,90 @@ pub struct DecisionReason { pub last_used_at: Option>, } -/// Whether a **normalized** version string denotes a prerelease. -/// -/// Format-agnostic heuristic: a normalized version carrying a prerelease label -/// renders it after a `-` (NuGet's `to_normalized_string`, SemVer 2.0.0). We do -/// not import a concrete format here, honoring the "order/classify via the seam" -/// rule. Build metadata (`+...`) is already stripped during normalization. -fn is_prerelease(normalized_version: &str) -> bool { - normalized_version.contains('-') +/// A rule's filters, compiled once per evaluation. +struct RuleFilter<'a> { + rule: &'a RetentionRule, + /// `package_name_prefix`, lowercased. + prefix: Option, + include: Option, + exclude: Option, + selector: Option<&'a CompiledSelector>, + /// A glob that does not compile, or a selector the caller did not + /// resolve: the rule admits nothing (validation refuses both, so this + /// only fails closed). + admits_nothing: bool, } -/// Whether a rule's static filters (`release_type`, `package_name_prefix`) -/// admit a given component, independent of its triggers. -fn rule_filters_match(rule: &RetentionRule, package_name: &str, version: &str) -> bool { - match rule.release_type { - ReleaseType::Any => {} - ReleaseType::Releases if is_prerelease(version) => return false, - ReleaseType::Prereleases if !is_prerelease(version) => return false, - _ => {} - } - if let Some(prefix) = &rule.package_name_prefix { - // Case-insensitive prefix match. `package_name` is already lowercased by - // the loader; lowercase the configured prefix to be safe. - if !package_name.starts_with(&crate::format::lower_invariant(prefix)) { +impl<'a> RuleFilter<'a> { + fn new(rule: &'a RetentionRule, selectors: &'a Selectors) -> Self { + let mut admits_nothing = false; + let mut glob = |pattern: &Option| match pattern.as_deref().map(version_glob) { + None => None, + Some(Ok(matcher)) => Some(matcher), + Some(Err(_)) => { + admits_nothing = true; + None + } + }; + let include = glob(&rule.version_pattern); + let exclude = glob(&rule.exclude_version_pattern); + let selector = match &rule.selector { + None => None, + Some(name) => { + let found = selectors.get(name); + admits_nothing |= found.is_none(); + found + } + }; + RuleFilter { + rule, + prefix: rule + .package_name_prefix + .as_deref() + .map(crate::format::lower_invariant), + include, + exclude, + selector, + admits_nothing, + } + } + + /// Whether the rule's filters admit `version` of `package_name`, + /// independent of its triggers. + fn admits(&self, format: &dyn Format, package_name: &str, version: &str) -> bool { + if self.admits_nothing { return false; } + let class_ok = match self.rule.release_type { + ReleaseType::Any => true, + ReleaseType::Releases => format.version_class(version) == VersionClass::Release, + ReleaseType::Prereleases => format.version_class(version) == VersionClass::Prerelease, + ReleaseType::Snapshots => format.version_class(version) == VersionClass::Snapshot, + }; + // `package_name` is already lowercased by the loader. + class_ok + && self + .prefix + .as_deref() + .is_none_or(|prefix| package_name.starts_with(prefix)) + && self.include.as_ref().is_none_or(|g| g.is_match(version)) + && !self.exclude.as_ref().is_some_and(|g| g.is_match(version)) + && self.selector.is_none_or(|s| s.matches(package_name)) } - true +} + +/// The compiled filters of `policy`'s rules of `scope`, in policy order. +fn rule_filters<'a>( + policy: &'a RetentionPolicy, + scope: Scope, + selectors: &'a Selectors, +) -> Vec> { + policy + .rules + .iter() + .filter(|r| r.scope == scope) + .map(|r| RuleFilter::new(r, selectors)) + .collect() } /// Whole-days elapsed from `then` to `now` (floored). Negative spans (a @@ -279,6 +398,15 @@ fn days_since(now: DateTime, then: DateTime) -> i64 { } } +/// What one candidate's group says about it for one rule's triggers. +struct TriggerInputs<'a> { + /// The release that superseded it ([`superseding_releases`]). + superseded_by: Option<&'a CandidateComponent>, + /// Its place (1-based) in its partition of the rule's pool, when that is + /// past the rule's `max_versions_per_package`. + over_cap: Option, +} + /// Evaluate the hosted `Criteria` triggers (ANDed) for one component. Returns /// `Some(reason)` when **all present triggers** match, else `None`. A criteria /// block with no triggers never selects (the rule was rejected at validation @@ -287,17 +415,10 @@ fn triggers_match( c: &Criteria, now: DateTime, cand: &CandidateComponent, + inputs: &TriggerInputs<'_>, ) -> Option { let mut any_trigger = false; - let mut reason = DecisionReason { - older_than_days: None, - not_downloaded_for_days: None, - not_pulled_for_days: None, - size_budget: None, - target_bytes: None, - size_bytes: None, - last_used_at: None, - }; + let mut reason = DecisionReason::default(); if let Some(days) = c.older_than_days { any_trigger = true; @@ -320,6 +441,37 @@ fn triggers_match( reason.not_downloaded_for_days = Some(days); } + if let Some(days) = c.prerelease_superseded_for_days { + any_trigger = true; + // A release is never superseded; a prerelease only by a release. + let release = inputs.superseded_by?; + if days_since(now, release.created_at) < days as i64 { + return None; + } + reason.prerelease_superseded_for_days = Some(days); + reason.superseded_by = Some(release.version.clone()); + } + + if let Some(days) = c.unlisted_for_days { + any_trigger = true; + // A listed version is not unlisted at all; an unlisted one without + // its unlisting time (never after the migration) is not judged. + if cand.listed { + return None; + } + let unlisted_at = cand.unlisted_at?; + if days_since(now, unlisted_at) < days as i64 { + return None; + } + reason.unlisted_for_days = Some(days); + } + + if let Some(cap) = c.max_versions_per_package { + any_trigger = true; + reason.rank = Some(inputs.over_cap?); + reason.max_versions_per_package = Some(cap); + } + if any_trigger { Some(reason) } else { @@ -381,65 +533,134 @@ fn group_by_package(candidates: &[C]) -> Vec> { groups } +/// What the rules of one scope see in one package's candidates, ordered +/// newest-first by version. +struct GroupView { + /// Per rule (in filter order), whether it admits each candidate. + admitted: Vec>, + /// Each candidate's partition ([`Format::retention_partition`]). + partition: Vec>, +} + +impl GroupView { + fn new( + format: &dyn Format, + filters: &[RuleFilter<'_>], + ordered: &[&C], + ) -> Self { + GroupView { + admitted: filters + .iter() + .map(|f| { + ordered + .iter() + .map(|c| f.admits(format, c.package_name(), c.version())) + .collect() + }) + .collect(), + partition: ordered + .iter() + .map(|c| format.retention_partition(c.package_name(), c.version())) + .collect(), + } + } + + /// Rule `rule`'s pool split by partition: each part newest-first, the + /// parts in the order they first appear. + fn pools(&self, rule: usize) -> Vec> { + let mut parts: Vec> = Vec::new(); + let mut index: HashMap<&Option, usize> = HashMap::new(); + for (i, admitted) in self.admitted[rule].iter().enumerate() { + if !admitted { + continue; + } + let at = *index.entry(&self.partition[i]).or_insert_with(|| { + parts.push(Vec::new()); + parts.len() - 1 + }); + parts[at].push(i); + } + parts + } + + /// Per rule, each candidate's place (1-based) in its partition of the + /// rule's pool when that is past the rule's `max_versions_per_package`. + fn over_cap(&self, filters: &[RuleFilter<'_>]) -> Vec>> { + filters + .iter() + .enumerate() + .map(|(r, f)| { + let mut ranks = vec![None; self.partition.len()]; + if let Some(cap) = f.rule.criteria.max_versions_per_package { + for pool in self.pools(r) { + for (rank, &idx) in pool.iter().enumerate().skip(cap as usize) { + ranks[idx] = Some(u32::try_from(rank + 1).unwrap_or(u32::MAX)); + } + } + } + ranks + }) + .collect() + } +} + /// Compute which of a group's versions (indices into its newest-first -/// ordering) are **protected**: the union, over every rule of `scope`, of +/// ordering) are **protected**: the union, over every rule of `filters`, of /// that rule's `keep_last_n` / `keep_n_most_downloaded` protections. /// /// Each rule's protections are computed over the versions the rule itself -/// governs — those its `release_type` / `package_name_prefix` filters admit — -/// never over the whole package. Otherwise a `releases`-only rule with -/// `keep_last_n: 3` would "protect" the three newest prereleases and leave -/// every stable release unprotected. `keep_last_n` further counts only -/// **listed** versions, so unlisted versions cannot use up its slots; -/// `keep_n_most_downloaded` ranks listed and unlisted versions alike by -/// download recency (see [`Criteria`]). +/// governs — those its filters admit — never over the whole package. +/// Otherwise a `releases`-only rule with `keep_last_n: 3` would "protect" +/// the three newest prereleases and leave every stable release unprotected. +/// They count per partition of that pool ([`Format::retention_partition`]). +/// `keep_last_n` further counts only **listed** versions, so unlisted +/// versions cannot use up its slots; `keep_n_most_downloaded` ranks listed +/// and unlisted versions alike by download recency (see [`Criteria`]). /// /// `ordered` is the group's candidates sorted newest-first (by version): a /// hosted package's components, or a proxy package's cached versions. fn protected_indices( - policy: &RetentionPolicy, - scope: Scope, + filters: &[RuleFilter<'_>], + view: &GroupView, ordered: &[&C], ) -> Vec { let mut protected = vec![false; ordered.len()]; - for rule in policy.rules.iter().filter(|r| r.scope == scope) { - let c = &rule.criteria; + for (r, filter) in filters.iter().enumerate() { + let c = &filter.rule.criteria; if !c.has_protection() { continue; } - // The rule's pool, still newest-first. - let pool: Vec = (0..ordered.len()) - .filter(|&i| rule_filters_match(rule, ordered[i].package_name(), ordered[i].version())) - .collect(); - - if let Some(n) = c.keep_last_n { - for &idx in pool - .iter() - .filter(|&&i| ordered[i].listed()) - .take(n as usize) - { - protected[idx] = true; + for pool in view.pools(r) { + if let Some(n) = c.keep_last_n { + for &idx in pool + .iter() + .filter(|&&i| ordered[i].listed()) + .take(n as usize) + { + protected[idx] = true; + } } - } - if let Some(n) = c.keep_n_most_downloaded { - // Order by download recency: most-recent first, nulls last. The - // sort is stable, so ties keep the pool's version-desc order. - let mut by_download = pool.clone(); - by_download.sort_by(|&a, &b| { - match ( - ordered[a].last_downloaded_at(), - ordered[b].last_downloaded_at(), - ) { - (Some(x), Some(y)) => y.cmp(&x), // more recent first - (Some(_), None) => std::cmp::Ordering::Less, - (None, Some(_)) => std::cmp::Ordering::Greater, - (None, None) => std::cmp::Ordering::Equal, + if let Some(n) = c.keep_n_most_downloaded { + // Order by download recency: most-recent first, nulls last. + // The sort is stable, so ties keep the pool's version-desc + // order. + let mut by_download = pool.clone(); + by_download.sort_by(|&a, &b| { + match ( + ordered[a].last_downloaded_at(), + ordered[b].last_downloaded_at(), + ) { + (Some(x), Some(y)) => y.cmp(&x), // more recent first + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => std::cmp::Ordering::Equal, + } + }); + for &idx in by_download.iter().take(n as usize) { + protected[idx] = true; } - }); - for &idx in by_download.iter().take(n as usize) { - protected[idx] = true; } } } @@ -447,22 +668,85 @@ fn protected_indices( protected } +/// [`protected_indices`] of one group, for the size budget. +fn protected_in( + format: &dyn Format, + filters: &[RuleFilter<'_>], + ordered: &[&C], +) -> Vec { + let view = GroupView::new(format, filters, ordered); + protected_indices(filters, &view, ordered) +} + +/// For each of a hosted group's versions (newest first), the release that +/// supersedes it when it is not a release itself: of the package's +/// **listed** releases that order strictly after it, the one published +/// first (from then on the prerelease has had a release). +fn superseding_releases<'g>( + format: &dyn Format, + ordered: &[&'g CandidateComponent], +) -> Vec> { + let classes: Vec = ordered + .iter() + .map(|c| format.version_class(&c.version)) + .collect(); + let mut out = vec![None; ordered.len()]; + // The first-published listed release among the strictly newer versions. + let mut first: Option<&CandidateComponent> = None; + let mut i = 0; + while i < ordered.len() { + // A run of versions the format orders equal: none supersedes another. + let mut j = i + 1; + while j < ordered.len() + && format.compare_versions(&ordered[j].version, &ordered[i].version) + == std::cmp::Ordering::Equal + { + j += 1; + } + for k in i..j { + if classes[k] != VersionClass::Release { + out[k] = first; + } + } + for k in i..j { + let c = ordered[k]; + if c.listed + && classes[k] == VersionClass::Release + && first.is_none_or(|f| c.created_at < f.created_at) + { + first = Some(c); + } + } + i = j; + } + out +} + /// Evaluate a hosted retention policy over a component snapshot, producing the /// ordered set of decisions to apply. **Pure**: no I/O, `now` is injected. /// -/// `format` is used only to order versions within a package group. The cost is -/// linear in the snapshot plus sorting each package's versions, but it is -/// still CPU-bound on a large repository, so async callers run it on the -/// blocking pool. +/// `format` orders and classifies versions within a package group, and +/// partitions them for keep-N; `selectors` are the content selectors the +/// rules name. The cost is linear in the snapshot plus sorting each +/// package's versions, but it is still CPU-bound on a large repository, so +/// async callers run it on the blocking pool. pub fn evaluate( format: &dyn Format, now: DateTime, policy: &RetentionPolicy, + selectors: &Selectors, candidates: &[CandidateComponent], ) -> Vec { if !policy.enabled { return Vec::new(); } + let filters = rule_filters(policy, Scope::Hosted, selectors); + if filters.is_empty() { + return Vec::new(); + } + let supersession = filters + .iter() + .any(|f| f.rule.criteria.prerelease_superseded_for_days.is_some()); let mut decisions = Vec::new(); @@ -470,7 +754,14 @@ pub fn evaluate( // This group's components, newest-first by version. group.sort_by(|a, b| format.compare_versions(&b.version, &a.version)); // desc - let protected = protected_indices(policy, Scope::Hosted, &group); + let view = GroupView::new(format, &filters, &group); + let protected = protected_indices(&filters, &view, &group); + let over_cap = view.over_cap(&filters); + let superseded = if supersession { + superseding_releases(format, &group) + } else { + vec![None; group.len()] + }; for (idx, cand) in group.iter().enumerate() { if protected[idx] { @@ -479,19 +770,23 @@ pub fn evaluate( // Find the highest-severity action among rules that select it. let mut winning: Option<(&RetentionRule, DecisionReason)> = None; - for rule in policy.rules.iter().filter(|r| r.scope == Scope::Hosted) { - if !rule_filters_match(rule, &cand.package_name, &cand.version) { + for (r, filter) in filters.iter().enumerate() { + if !view.admitted[r][idx] { continue; } - let Some(reason) = triggers_match(&rule.criteria, now, cand) else { + let inputs = TriggerInputs { + superseded_by: superseded[idx], + over_cap: over_cap[r][idx], + }; + let Some(reason) = triggers_match(&filter.rule.criteria, now, cand, &inputs) else { continue; }; let better = match &winning { None => true, - Some((cur, _)) => rule.action.severity() > cur.action.severity(), + Some((cur, _)) => filter.rule.action.severity() > cur.action.severity(), }; if better { - winning = Some((rule, reason)); + winning = Some((filter.rule, reason)); } } @@ -521,18 +816,23 @@ pub fn evaluate( /// snapshot. **Pure**: no I/O. Only `scope: proxy_cached` rules participate. /// /// `format` orders each package's cached versions newest-first, for the -/// rules' `keep_last_n` protections; a version any proxy rule protects is -/// never evicted. Like [`evaluate`], async callers run it on the blocking -/// pool. +/// rules' keep-N protections and `max_versions_per_package`; a version any +/// proxy rule protects is never evicted. Like [`evaluate`], async callers +/// run it on the blocking pool. pub fn evaluate_proxy( format: &dyn Format, now: DateTime, policy: &RetentionPolicy, + selectors: &Selectors, candidates: &[ProxyCandidate], ) -> Vec { if !policy.enabled { return Vec::new(); } + let filters = rule_filters(policy, Scope::ProxyCached, selectors); + if filters.is_empty() { + return Vec::new(); + } let mut decisions = Vec::new(); @@ -540,57 +840,61 @@ pub fn evaluate_proxy( // This package's cached versions, newest-first by version. group.sort_by(|a, b| format.compare_versions(&b.version, &a.version)); // desc - let protected = protected_indices(policy, Scope::ProxyCached, &group); + let view = GroupView::new(format, &filters, &group); + let protected = protected_indices(&filters, &view, &group); + let over_cap = view.over_cap(&filters); for (idx, cand) in group.iter().enumerate() { if protected[idx] { continue; // protection always wins — never evict. } - let mut winning: Option<(&RetentionRule, u32)> = None; - for rule in policy - .rules - .iter() - .filter(|r| r.scope == Scope::ProxyCached) - { - // release_type / prefix filters apply to proxy entries too. - if !rule_filters_match(rule, &cand.package_name, &cand.version) { + // All proxy rules delete; of the rules that select the version, + // keep the one with the smallest `not_pulled_for_days` for the + // most specific reason (the first such rule on a tie). + let mut winning: Option<(&RetentionRule, DecisionReason)> = None; + for (r, filter) in filters.iter().enumerate() { + if !view.admitted[r][idx] { continue; } - let Some(days) = rule.criteria.not_pulled_for_days else { - continue; - }; - // Use last pull, else first-seen (never pulled since caching). - let reference = cand.last_downloaded_at.unwrap_or(cand.first_seen_at); - if days_since(now, reference) < days as i64 { - continue; + let c = &filter.rule.criteria; + let mut reason = DecisionReason::default(); + if let Some(days) = c.not_pulled_for_days { + // Use last pull, else first-seen (never pulled since caching). + let reference = cand.last_downloaded_at.unwrap_or(cand.first_seen_at); + if days_since(now, reference) < days as i64 { + continue; + } + reason.not_pulled_for_days = Some(days); + } + if let Some(cap) = c.max_versions_per_package { + let Some(rank) = over_cap[r][idx] else { + continue; + }; + reason.max_versions_per_package = Some(cap); + reason.rank = Some(rank); + } + if reason == DecisionReason::default() { + continue; // no proxy trigger (refused at validation). } - // All proxy rules delete; first matching rule is enough, but keep - // the one with the smallest threshold for the most specific reason. + let threshold = + |reason: &DecisionReason| reason.not_pulled_for_days.unwrap_or(u32::MAX); let take = match &winning { None => true, - Some((_, cur_days)) => days < *cur_days, + Some((_, cur)) => threshold(&reason) < threshold(cur), }; if take { - winning = Some((rule, days)); + winning = Some((filter.rule, reason)); } } - if let Some((rule, days)) = winning { + if let Some((rule, reason)) = winning { decisions.push(ProxyDecision { cached_version_id: cand.cached_version_id, package_name: cand.package_name.clone(), version: cand.version.clone(), rule_id: rule.id.clone(), - reason: DecisionReason { - older_than_days: None, - not_downloaded_for_days: None, - not_pulled_for_days: Some(days), - size_budget: None, - target_bytes: None, - size_bytes: None, - last_used_at: None, - }, + reason, }); } } @@ -604,6 +908,7 @@ pub fn evaluate_proxy( pub fn evaluate_size_budget( format: &dyn Format, policy: &RetentionPolicy, + selectors: &Selectors, hosted: &[CandidateComponent], proxy: &[ProxyCandidate], repository_size_bytes: u64, @@ -620,6 +925,8 @@ pub fn evaluate_size_budget( target_reached: true, }; }; + let hosted_filters = rule_filters(policy, Scope::Hosted, selectors); + let proxy_filters = rule_filters(policy, Scope::ProxyCached, selectors); let hosted_decisions = inputs.hosted_decisions; let proxy_decisions = inputs.proxy_decisions; let tagged_hosted_ids = inputs.tagged_hosted_ids; @@ -679,7 +986,7 @@ pub fn evaluate_size_budget( } } let ordered: Vec<&CandidateComponent> = survivors.iter().collect(); - let protected = protected_indices(policy, Scope::Hosted, &ordered); + let protected = protected_in(format, &hosted_filters, &ordered); let newest_listed = survivors.iter().find(|c| c.listed).map(|c| c.component_id); for (idx, candidate) in survivors.into_iter().enumerate() { if !protected[idx] @@ -709,15 +1016,13 @@ pub fn evaluate_size_budget( action: Action::HardDelete, rule_id: "size_budget".into(), reason: DecisionReason { - older_than_days: None, - not_downloaded_for_days: None, - not_pulled_for_days: None, size_budget: Some(budget.max_bytes), target_bytes: Some(target_bytes), size_bytes: Some(candidate.size_bytes), last_used_at: Some( candidate.last_downloaded_at.unwrap_or(candidate.created_at), ), + ..DecisionReason::default() }, }); size_after = size_after.saturating_sub(candidate.size_bytes); @@ -731,7 +1036,7 @@ pub fn evaluate_size_budget( .cloned() .collect(); let ordered: Vec<&ProxyCandidate> = survivors.iter().collect(); - let protected = protected_indices(policy, Scope::ProxyCached, &ordered); + let protected = protected_in(format, &proxy_filters, &ordered); proxy_protected.extend(survivors.iter().zip(protected).filter_map( |(candidate, protected)| protected.then_some(candidate.cached_version_id), )); @@ -759,9 +1064,6 @@ pub fn evaluate_size_budget( version: candidate.version.clone(), rule_id: "size_budget".into(), reason: DecisionReason { - older_than_days: None, - not_downloaded_for_days: None, - not_pulled_for_days: None, size_budget: Some(budget.max_bytes), target_bytes: Some(target_bytes), size_bytes: Some(candidate.size_bytes), @@ -770,6 +1072,7 @@ pub fn evaluate_size_budget( .last_downloaded_at .unwrap_or(candidate.first_seen_at), ), + ..DecisionReason::default() }, }); size_after = size_after.saturating_sub(candidate.size_bytes); @@ -791,11 +1094,14 @@ pub fn evaluate_size_budget( pub fn size_budget_protected_ids( format: &dyn Format, policy: &RetentionPolicy, + selectors: &Selectors, hosted: &[CandidateComponent], proxy: &[ProxyCandidate], hosted_decisions: &[Decision], proxy_decisions: &[ProxyDecision], ) -> (HashSet, HashSet) { + let hosted_filters = rule_filters(policy, Scope::Hosted, selectors); + let proxy_filters = rule_filters(policy, Scope::ProxyCached, selectors); let hosted_hard_deleted: HashSet = hosted_decisions .iter() .filter(|decision| decision.action == Action::HardDelete) @@ -824,7 +1130,7 @@ pub fn size_budget_protected_ids( } } let ordered: Vec<&CandidateComponent> = survivors.iter().collect(); - let protected = protected_indices(policy, Scope::Hosted, &ordered); + let protected = protected_in(format, &hosted_filters, &ordered); let newest_listed = survivors.iter().find(|c| c.listed).map(|c| c.component_id); hosted_ids.extend( survivors @@ -843,7 +1149,7 @@ pub fn size_budget_protected_ids( .cloned() .collect(); let ordered: Vec<&ProxyCandidate> = survivors.iter().collect(); - let protected = protected_indices(policy, Scope::ProxyCached, &ordered); + let protected = protected_in(format, &proxy_filters, &ordered); proxy_ids.extend( survivors .iter() @@ -932,6 +1238,7 @@ mod tests { listed: true, created_at: days_ago(created_days_ago), last_downloaded_at: last_dl_days_ago.map(days_ago), + unlisted_at: None, size_bytes: 1, } } @@ -942,6 +1249,9 @@ mod tests { enabled: true, dry_run_only: false, min_run_interval_seconds: 3600, + schedule: None, + schedule_timezone: None, + max_run_seconds: None, size_budget: None, rules, } @@ -954,6 +1264,9 @@ mod tests { scope: Scope::Hosted, release_type: ReleaseType::Any, package_name_prefix: None, + selector: None, + version_pattern: None, + exclude_version_pattern: None, criteria, } } @@ -964,11 +1277,16 @@ mod tests { v } + fn is_prerelease(version: &str) -> bool { + fmt().version_class(version) == VersionClass::Prerelease + } + #[test] - fn is_prerelease_detects_label() { - assert!(!is_prerelease("1.0.0")); + fn the_default_version_class_is_semver() { + assert_eq!(fmt().version_class("1.0.0"), VersionClass::Release); assert!(is_prerelease("1.0.0-beta")); assert!(is_prerelease("2.1.0-rc.1")); + assert_eq!(fmt().retention_partition("lib", "1.0.0"), None); } #[test] @@ -987,7 +1305,7 @@ mod tests { ..Default::default() }, )]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(deleted_versions(&d), vec!["1.0.1", "1.0.2"]); } @@ -1008,7 +1326,7 @@ mod tests { ..Default::default() }, )]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); // newest two (2.0.0, 1.5.0) protected; 1.0.0 deleted. assert_eq!(deleted_versions(&d), vec!["1.0.0"]); } @@ -1027,7 +1345,7 @@ mod tests { ..Default::default() }, )]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(deleted_versions(&d), vec!["2.0.0"]); } @@ -1047,7 +1365,7 @@ mod tests { ..Default::default() }, )]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(deleted_versions(&d), vec!["1.0.0", "3.0.0"]); } @@ -1067,7 +1385,7 @@ mod tests { ..Default::default() }, )]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(deleted_versions(&d), vec!["1.0.0"]); } @@ -1088,7 +1406,7 @@ mod tests { ..Default::default() }, )]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(deleted_versions(&d), vec!["2.0.0", "3.0.0"]); } @@ -1109,7 +1427,7 @@ mod tests { }, ) }]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(deleted_versions(&d), vec!["1.0.0"]); } @@ -1130,7 +1448,7 @@ mod tests { }, ) }]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(deleted_versions(&d), vec!["2.0.0-beta"]); } @@ -1151,7 +1469,7 @@ mod tests { }, ) }]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(d.len(), 1); assert_eq!(d[0].package_name, "acme.core"); } @@ -1173,7 +1491,7 @@ mod tests { ..Default::default() }, )]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(deleted_versions(&d), vec!["1.0.0"]); } @@ -1205,7 +1523,7 @@ mod tests { }, ), ]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(deleted_versions(&d), vec!["1.0.0"]); } @@ -1230,7 +1548,7 @@ mod tests { }, ), ]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(d.len(), 1); assert_eq!(d[0].action, Action::HardDelete); assert_eq!(d[0].rule_id, "d"); @@ -1247,7 +1565,7 @@ mod tests { ..Default::default() }, )]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(d.len(), 1); assert_eq!(d[0].action, Action::Unlist); } @@ -1265,7 +1583,7 @@ mod tests { ..Default::default() }, )]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(deleted_versions(&d), vec!["1.1.0"]); // A hard-delete rule still reaches an unlisted version. @@ -1279,7 +1597,7 @@ mod tests { ..Default::default() }, )]); - let d = evaluate(&fmt(), now(), &p, &[hidden]); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &[hidden]); assert_eq!(d.len(), 1); assert_eq!(d[0].action, Action::HardDelete); } @@ -1296,7 +1614,7 @@ mod tests { }, )]); p.enabled = false; - assert!(evaluate(&fmt(), now(), &p, &cands).is_empty()); + assert!(evaluate(&fmt(), now(), &p, &Selectors::none(), &cands).is_empty()); } #[test] @@ -1309,7 +1627,7 @@ mod tests { ..Default::default() }, )]); - assert!(evaluate(&fmt(), now(), &p, &[]).is_empty()); + assert!(evaluate(&fmt(), now(), &p, &Selectors::none(), &[]).is_empty()); } #[test] @@ -1330,7 +1648,7 @@ mod tests { ..Default::default() }, )]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); let mut pairs: Vec<(String, String)> = d .iter() .map(|x| (x.package_name.clone(), x.version.clone())) @@ -1346,6 +1664,7 @@ mod tests { fn unlisted(mut c: CandidateComponent) -> CandidateComponent { c.listed = false; + c.unlisted_at = Some(c.created_at); c } @@ -1376,7 +1695,7 @@ mod tests { }, ) }]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(deleted_versions(&d), vec!["1.0.0"]); } @@ -1402,7 +1721,7 @@ mod tests { ..Default::default() }, )]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(deleted_versions(&d), vec!["1.0.0", "1.3.0", "1.4.0"]); } @@ -1438,7 +1757,7 @@ mod tests { }, ), ]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); let mut pairs: Vec<(String, String)> = d .iter() .map(|x| (x.package_name.clone(), x.version.clone())) @@ -1484,7 +1803,7 @@ mod tests { }, ), ]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(deleted_versions(&d), vec!["1.0.0", "2.0.0-beta.1"]); } @@ -1510,7 +1829,7 @@ mod tests { }, ) }]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(deleted_versions(&d), vec!["1.0.0"]); } @@ -1533,7 +1852,7 @@ mod tests { ..Default::default() }, )]); - let d = evaluate(&fmt(), now(), &p, &cands); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); assert_eq!(deleted_versions(&d), vec!["1.0.0", "1.2.0"]); } @@ -1606,10 +1925,10 @@ mod tests { }, ]); - let decisions = evaluate(&fmt(), now(), &p, &all); + let decisions = evaluate(&fmt(), now(), &p, &Selectors::none(), &all); let expected: Vec = by_package .iter() - .flat_map(|group| evaluate(&fmt(), now(), &p, group)) + .flat_map(|group| evaluate(&fmt(), now(), &p, &Selectors::none(), group)) .collect(); assert_eq!(decisions.len(), expected.len()); assert!( @@ -1662,6 +1981,9 @@ mod tests { scope: Scope::ProxyCached, release_type: ReleaseType::Any, package_name_prefix: None, + selector: None, + version_pattern: None, + exclude_version_pattern: None, criteria: Criteria { not_pulled_for_days: Some(days), ..Default::default() @@ -1678,7 +2000,7 @@ mod tests { proxy(4, "lib", "4.0.0", 10, None), // never pulled, seen 10d ago → warm ]; let p = policy_with(vec![proxy_rule("evict", 30)]); - let d = evaluate_proxy(&fmt(), now(), &p, &cands); + let d = evaluate_proxy(&fmt(), now(), &p, &Selectors::none(), &cands); let mut ids: Vec = d.iter().map(|x| x.cached_version_id).collect(); ids.sort_unstable(); assert_eq!(ids, vec![1, 3]); @@ -1701,11 +2023,11 @@ mod tests { proxy_rule("p", 30), ]); // hosted evaluator only sees the hosted rule - let hd = evaluate(&fmt(), now(), &p, &hosted_cands); + let hd = evaluate(&fmt(), now(), &p, &Selectors::none(), &hosted_cands); assert_eq!(hd.len(), 1); assert_eq!(hd[0].rule_id, "h"); // proxy evaluator only sees the proxy rule - let pd = evaluate_proxy(&fmt(), now(), &p, &proxy_cands); + let pd = evaluate_proxy(&fmt(), now(), &p, &Selectors::none(), &proxy_cands); assert_eq!(pd.len(), 1); assert_eq!(pd[0].rule_id, "p"); } @@ -1715,7 +2037,7 @@ mod tests { let cands = vec![proxy(1, "lib", "1.0.0", 100, None)]; let mut p = policy_with(vec![proxy_rule("p", 1)]); p.enabled = false; - assert!(evaluate_proxy(&fmt(), now(), &p, &cands).is_empty()); + assert!(evaluate_proxy(&fmt(), now(), &p, &Selectors::none(), &cands).is_empty()); } /// A proxy rule evicting what nobody pulled for 30 days, keeping the @@ -1740,7 +2062,7 @@ mod tests { } fn evicted(p: &RetentionPolicy, cands: &[ProxyCandidate]) -> Vec { - let mut ids: Vec = evaluate_proxy(&fmt(), now(), p, cands) + let mut ids: Vec = evaluate_proxy(&fmt(), now(), p, &Selectors::none(), cands) .iter() .map(|d| d.cached_version_id) .collect(); @@ -1837,6 +2159,7 @@ mod tests { let plan = evaluate_size_budget( &fmt(), &policy, + &Selectors::none(), &candidates, &[], 120, @@ -1883,6 +2206,7 @@ mod tests { let plan = evaluate_size_budget( &fmt(), &policy, + &Selectors::none(), &candidates, &[], 150, @@ -1915,12 +2239,7 @@ mod tests { rule_id: "prerelease".into(), reason: DecisionReason { older_than_days: Some(1), - not_downloaded_for_days: None, - not_pulled_for_days: None, - size_budget: None, - target_bytes: None, - size_bytes: None, - last_used_at: None, + ..DecisionReason::default() }, }]; let mut policy = policy_with(Vec::new()); @@ -1931,6 +2250,7 @@ mod tests { let plan = evaluate_size_budget( &fmt(), &policy, + &Selectors::none(), &candidates, &[], 200, @@ -1960,6 +2280,7 @@ mod tests { let plan = evaluate_size_budget( &fmt(), &policy, + &Selectors::none(), &[], &[], 95, @@ -1991,6 +2312,7 @@ mod tests { let plan = evaluate_size_budget( &fmt(), &policy, + &Selectors::none(), &candidates, &[], 150, @@ -2024,6 +2346,7 @@ mod tests { let plan = evaluate_size_budget( &fmt(), &policy, + &Selectors::none(), &[], &candidates, 150, @@ -2044,4 +2367,521 @@ mod tests { assert!(plan.target_reached); assert!(plan.proxy.iter().all(|d| d.rule_id == "size_budget")); } + + // ---- #185: more triggers and filters ---- + + fn with_criteria(id: &str, action: Action, criteria: Criteria) -> RetentionRule { + rule(id, action, criteria) + } + + fn hard_delete(criteria: Criteria) -> RetentionPolicy { + policy_with(vec![with_criteria("r", Action::HardDelete, criteria)]) + } + + fn superseded(days: u32) -> Criteria { + Criteria { + prerelease_superseded_for_days: Some(days), + ..Default::default() + } + } + + #[test] + fn prerelease_superseded_once_a_release_has_been_out_long_enough() { + let cands = vec![ + comp(1, "lib", "2.2.0-beta.1", 120, None), + comp(2, "lib", "2.2.0", 90, None), + comp(3, "lib", "2.3.0-ci.417", 60, None), + comp(4, "lib", "2.3.0", 20, None), + // No release orders after it: never superseded. + comp(5, "lib", "2.4.0-ci.5", 10, None), + ]; + let d = evaluate( + &fmt(), + now(), + &hard_delete(superseded(14)), + &Selectors::none(), + &cands, + ); + assert_eq!(deleted_versions(&d), vec!["2.2.0-beta.1", "2.3.0-ci.417"]); + let ci = d.iter().find(|d| d.version == "2.3.0-ci.417").unwrap(); + assert_eq!(ci.reason.prerelease_superseded_for_days, Some(14)); + assert_eq!(ci.reason.superseded_by.as_deref(), Some("2.3.0")); + // The beta has had a release since 2.2.0 came out, 90 days ago. + let beta = d.iter().find(|d| d.version == "2.2.0-beta.1").unwrap(); + assert_eq!(beta.reason.superseded_by.as_deref(), Some("2.2.0")); + + // 2.3.0 has been out for 20 days only. + let d = evaluate( + &fmt(), + now(), + &hard_delete(superseded(30)), + &Selectors::none(), + &cands, + ); + assert_eq!(deleted_versions(&d), vec!["2.2.0-beta.1"]); + } + + #[test] + fn supersession_counts_from_the_first_published_newer_release_that_is_listed() { + // 3.0.0 was published before 2.0.0 (a hotfix line): the prerelease + // has had a newer release for 50 days. An unlisted release supersedes + // nothing. + let mut cands = vec![ + comp(1, "lib", "2.0.0-rc.1", 100, None), + comp(2, "lib", "2.0.0", 5, None), + comp(3, "lib", "3.0.0", 50, None), + unlisted(comp(4, "lib", "4.0.0", 80, None)), + ]; + let d = evaluate( + &fmt(), + now(), + &hard_delete(superseded(40)), + &Selectors::none(), + &cands, + ); + assert_eq!(deleted_versions(&d), vec!["2.0.0-rc.1"]); + assert_eq!(d[0].reason.superseded_by.as_deref(), Some("3.0.0")); + + // Without 3.0.0 only the 5-day-old 2.0.0 is left. + cands.remove(2); + let d = evaluate( + &fmt(), + now(), + &hard_delete(superseded(40)), + &Selectors::none(), + &cands, + ); + assert!(d.is_empty(), "{d:?}"); + } + + #[test] + fn unlisted_for_days_deletes_versions_unlisted_long_enough() { + let mut long_ago = unlisted(comp(1, "lib", "1.0.0", 400, None)); + long_ago.unlisted_at = Some(days_ago(100)); + let mut lately = unlisted(comp(2, "lib", "1.1.0", 400, None)); + lately.unlisted_at = Some(days_ago(10)); + // Unlisted, but when is not known: not judged. + let mut unknown = unlisted(comp(3, "lib", "1.2.0", 400, None)); + unknown.unlisted_at = None; + let listed = comp(4, "lib", "1.3.0", 400, None); + let cands = vec![long_ago, lately, unknown, listed]; + let p = hard_delete(Criteria { + unlisted_for_days: Some(90), + ..Default::default() + }); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); + assert_eq!(deleted_versions(&d), vec!["1.0.0"]); + assert_eq!(d[0].action, Action::HardDelete); + assert_eq!(d[0].reason.unlisted_for_days, Some(90)); + } + + #[test] + fn two_step_policy_unlists_then_deletes_later() { + // Rule 1 unlists versions older than 30 days, rule 2 deletes what has + // been unlisted for 60 days. + let mut old_unlisted = unlisted(comp(1, "lib", "1.0.0", 200, None)); + old_unlisted.unlisted_at = Some(days_ago(70)); + let cands = vec![ + old_unlisted, + comp(2, "lib", "1.1.0", 40, None), + comp(3, "lib", "1.2.0", 5, None), + ]; + let p = policy_with(vec![ + with_criteria( + "unlist-old", + Action::Unlist, + Criteria { + older_than_days: Some(30), + ..Default::default() + }, + ), + with_criteria( + "delete-unlisted", + Action::HardDelete, + Criteria { + unlisted_for_days: Some(60), + ..Default::default() + }, + ), + ]); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); + let by_version: HashMap<&str, (&str, Action)> = d + .iter() + .map(|d| (d.version.as_str(), (d.rule_id.as_str(), d.action))) + .collect(); + assert_eq!(by_version["1.0.0"], ("delete-unlisted", Action::HardDelete)); + assert_eq!(by_version["1.1.0"], ("unlist-old", Action::Unlist)); + assert_eq!(by_version.len(), 2); + } + + #[test] + fn max_versions_per_package_is_a_pure_count() { + // All published today: no age needed. Unlisted versions count too. + let cands = vec![ + comp(1, "lib", "1.0.0", 0, None), + unlisted(comp(2, "lib", "1.1.0", 0, None)), + comp(3, "lib", "1.2.0", 0, None), + comp(4, "lib", "1.3.0", 0, None), + comp(5, "lib", "1.4.0", 0, None), + comp(6, "other", "1.0.0", 0, None), + ]; + let p = hard_delete(Criteria { + max_versions_per_package: Some(3), + ..Default::default() + }); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); + assert_eq!(deleted_versions(&d), vec!["1.0.0", "1.1.0"]); + let ranks: HashMap<&str, Option> = d + .iter() + .map(|d| (d.version.as_str(), d.reason.rank)) + .collect(); + assert_eq!(ranks["1.1.0"], Some(4)); + assert_eq!(ranks["1.0.0"], Some(5)); + assert!(d + .iter() + .all(|d| d.reason.max_versions_per_package == Some(3))); + } + + #[test] + fn max_versions_per_package_counts_the_rules_own_pool_and_ands_with_age() { + let cands = vec![ + comp(1, "lib", "1.0.0", 100, None), + comp(2, "lib", "1.1.0-ci.1", 100, None), + comp(3, "lib", "1.1.0-ci.2", 100, None), + comp(4, "lib", "1.1.0-ci.3", 1, None), + comp(5, "lib", "1.1.0-ci.4", 100, None), + comp(6, "lib", "1.1.0", 100, None), + ]; + // At most 1 prerelease (the newest, ci.4), and only those older + // than 30 days go: ci.3 is too young. + let p = policy_with(vec![RetentionRule { + release_type: ReleaseType::Prereleases, + ..with_criteria( + "ci", + Action::HardDelete, + Criteria { + max_versions_per_package: Some(1), + older_than_days: Some(30), + ..Default::default() + }, + ) + }]); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); + assert_eq!(deleted_versions(&d), vec!["1.1.0-ci.1", "1.1.0-ci.2"]); + } + + #[test] + fn version_patterns_include_and_exclude_ignoring_case() { + let cands = vec![ + comp(1, "lib", "1.0.0-ci.1", 100, None), + comp(2, "lib", "1.0.0-pr.7", 100, None), + comp(3, "lib", "1.0.0-beta.1", 100, None), + comp(4, "lib", "1.0.0-ci.keep", 100, None), + comp(5, "lib", "1.0.0", 100, None), + ]; + let p = policy_with(vec![RetentionRule { + version_pattern: Some("{*-CI.*,*-pr.*}".into()), + exclude_version_pattern: Some("*.keep".into()), + ..with_criteria( + "ci-and-pr", + Action::HardDelete, + Criteria { + older_than_days: Some(30), + ..Default::default() + }, + ) + }]); + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); + assert_eq!(deleted_versions(&d), vec!["1.0.0-ci.1", "1.0.0-pr.7"]); + } + + fn selector(includes: &[&str], excludes: &[&str]) -> CompiledSelector { + crate::authz::SelectorPatterns { + includes: includes.iter().map(|s| s.to_string()).collect(), + excludes: excludes.iter().map(|s| s.to_string()).collect(), + } + .compile() + .unwrap() + } + + #[test] + fn a_selector_filters_packages_and_an_unresolved_one_admits_nothing() { + let cands = vec![ + comp(1, "acme.core", "1.0.0", 100, None), + comp(2, "acme.keep.this", "1.0.0", 100, None), + comp(3, "other.lib", "1.0.0", 100, None), + ]; + let p = policy_with(vec![RetentionRule { + selector: Some("internal".into()), + ..with_criteria( + "internal", + Action::HardDelete, + Criteria { + older_than_days: Some(30), + ..Default::default() + }, + ) + }]); + let mut selectors = Selectors::none(); + selectors.insert("internal", selector(&["Acme.*"], &["acme.keep.*"])); + let d = evaluate(&fmt(), now(), &p, &selectors, &cands); + assert_eq!(d.len(), 1); + assert_eq!(d[0].package_name, "acme.core"); + // Fail closed: a selector the caller could not resolve selects + // nothing. + assert!(evaluate(&fmt(), now(), &p, &Selectors::none(), &cands).is_empty()); + } + + #[test] + fn protections_win_over_every_new_trigger() { + // Every version would be selected by each trigger below... + let mut gone = unlisted(comp(1, "lib", "1.0.0-ci.1", 100, Some(50))); + gone.unlisted_at = Some(days_ago(100)); + let mut gone_too = unlisted(comp(2, "lib", "1.0.0-ci.2", 100, Some(60))); + gone_too.unlisted_at = Some(days_ago(100)); + let cands = vec![ + gone, + gone_too, + comp(3, "lib", "1.0.0", 100, None), + comp(4, "lib", "2.0.0", 100, None), + ]; + let triggers = [ + Criteria { + prerelease_superseded_for_days: Some(1), + ..Default::default() + }, + Criteria { + unlisted_for_days: Some(1), + ..Default::default() + }, + Criteria { + max_versions_per_package: Some(1), + ..Default::default() + }, + ]; + for criteria in triggers { + let alone = hard_delete(criteria.clone()); + let selected = evaluate(&fmt(), now(), &alone, &Selectors::none(), &cands); + assert!(!selected.is_empty(), "{criteria:?} selects nothing alone"); + // ...but a rule that keeps them (in this rule or another one) + // wins: the newest listed and the most recently downloaded. + for keep_in_same_rule in [true, false] { + let keep = Criteria { + keep_last_n: Some(10), + keep_n_most_downloaded: Some(10), + older_than_days: Some(1000), + ..Default::default() + }; + let p = if keep_in_same_rule { + hard_delete(Criteria { + keep_last_n: Some(10), + keep_n_most_downloaded: Some(10), + ..criteria.clone() + }) + } else { + policy_with(vec![ + with_criteria("keep", Action::HardDelete, keep), + with_criteria("trigger", Action::HardDelete, criteria.clone()), + ]) + }; + let d = evaluate(&fmt(), now(), &p, &Selectors::none(), &cands); + assert!(d.is_empty(), "{criteria:?}: {d:?}"); + } + } + } + + /// A Maven-like format: `-SNAPSHOT` builds are written + /// `--`, are snapshots, sort before their release + /// and are counted per base version; releases are counted per package. + struct SnapshotFormat; + + impl SnapshotFormat { + fn split(v: &str) -> (Vec, Option<&str>) { + let (base, build) = v.split_once('-').map_or((v, None), |(b, s)| (b, Some(s))); + ( + base.split('.').map(|c| c.parse().unwrap_or(0)).collect(), + build, + ) + } + } + + impl Format for SnapshotFormat { + fn id(&self) -> &'static str { + "snapshots" + } + fn parse_upload(&self, _f: &mut dyn ReadSeek) -> Result { + unreachable!() + } + fn normalize_version(&self, raw: &str) -> Result { + Ok(raw.to_string()) + } + fn compare_versions(&self, a: &str, b: &str) -> Ordering { + let (ab, abuild) = Self::split(a); + let (bb, bbuild) = Self::split(b); + ab.cmp(&bb).then(match (abuild, bbuild) { + (None, None) => Ordering::Equal, + (None, Some(_)) => Ordering::Greater, + (Some(_), None) => Ordering::Less, + (Some(x), Some(y)) => x.cmp(y), + }) + } + fn lower_id(&self, id: &str) -> String { + id.to_lowercase() + } + fn version_class(&self, version: &str) -> VersionClass { + if version.contains('-') { + VersionClass::Snapshot + } else { + VersionClass::Release + } + } + fn retention_partition(&self, _id: &str, version: &str) -> Option { + version + .split_once('-') + .map(|(base, _)| format!("{base}-SNAPSHOT")) + } + } + + #[test] + fn format_hooks_partition_keep_n_and_class_snapshots() { + let cands = vec![ + comp(1, "g:a", "1.0-20260101.000000-1", 100, None), + comp(2, "g:a", "1.0-20260102.000000-2", 100, None), + comp(3, "g:a", "1.0-20260103.000000-3", 100, None), + comp(4, "g:a", "1.1-20260104.000000-1", 100, None), + comp(5, "g:a", "1.1-20260105.000000-2", 100, None), + comp(6, "g:a", "0.9", 100, None), + comp(7, "g:a", "0.8", 100, None), + ]; + // Keep the newest build of each -SNAPSHOT version; delete the other + // builds older than 30 days. Releases are not snapshots. + let p = policy_with(vec![RetentionRule { + release_type: ReleaseType::Snapshots, + ..with_criteria( + "snapshots", + Action::HardDelete, + Criteria { + keep_last_n: Some(1), + older_than_days: Some(30), + ..Default::default() + }, + ) + }]); + let d = evaluate(&SnapshotFormat, now(), &p, &Selectors::none(), &cands); + assert_eq!( + deleted_versions(&d), + vec![ + "1.0-20260101.000000-1", + "1.0-20260102.000000-2", + "1.1-20260104.000000-1" + ] + ); + + // The newest 2 builds of each snapshot version, as a count. + let p = policy_with(vec![RetentionRule { + release_type: ReleaseType::Snapshots, + ..with_criteria( + "builds", + Action::HardDelete, + Criteria { + max_versions_per_package: Some(2), + ..Default::default() + }, + ) + }]); + let d = evaluate(&SnapshotFormat, now(), &p, &Selectors::none(), &cands); + assert_eq!(deleted_versions(&d), vec!["1.0-20260101.000000-1"]); + + // A version's snapshots go once its release exists. + let mut released = cands.clone(); + released.push(comp(8, "g:a", "1.0", 20, None)); + let d = evaluate( + &SnapshotFormat, + now(), + &hard_delete(superseded(7)), + &Selectors::none(), + &released, + ); + assert_eq!( + deleted_versions(&d), + vec![ + "1.0-20260101.000000-1", + "1.0-20260102.000000-2", + "1.0-20260103.000000-3" + ] + ); + assert!(d + .iter() + .all(|d| d.reason.superseded_by.as_deref() == Some("1.0"))); + + // Release rules count the releases of the package (one partition): + // `prereleases` admits no snapshot of this format. + let p = policy_with(vec![RetentionRule { + release_type: ReleaseType::Releases, + ..with_criteria( + "releases", + Action::HardDelete, + Criteria { + keep_last_n: Some(1), + older_than_days: Some(30), + ..Default::default() + }, + ) + }]); + let d = evaluate(&SnapshotFormat, now(), &p, &Selectors::none(), &cands); + assert_eq!(deleted_versions(&d), vec!["0.8"]); + let p = policy_with(vec![RetentionRule { + release_type: ReleaseType::Prereleases, + ..with_criteria( + "none", + Action::HardDelete, + Criteria { + older_than_days: Some(30), + ..Default::default() + }, + ) + }]); + assert!(evaluate(&SnapshotFormat, now(), &p, &Selectors::none(), &cands).is_empty()); + } + + #[test] + fn proxy_rules_take_filters_and_max_versions_per_package() { + let cands = vec![ + proxy(1, "lib", "1.0.0", 100, Some(1)), + proxy(2, "lib", "1.1.0", 100, Some(1)), + proxy(3, "lib", "1.2.0-ci.1", 100, Some(1)), + proxy(4, "lib", "1.2.0", 100, Some(1)), + ]; + // Pulled yesterday, but more than 2 cached versions: a count needs + // no cold trigger. + let mut count = proxy_rule("count", 30); + count.criteria.not_pulled_for_days = None; + count.criteria.max_versions_per_package = Some(2); + let d = evaluate_proxy( + &fmt(), + now(), + &policy_with(vec![count.clone()]), + &Selectors::none(), + &cands, + ); + let ids: Vec = d.iter().map(|d| d.cached_version_id).collect(); + assert_eq!(ids, vec![2, 1]); + assert!(d + .iter() + .all(|d| d.reason.max_versions_per_package == Some(2))); + // Only the CI builds count and go. + count.version_pattern = Some("*-ci.*".into()); + count.criteria.max_versions_per_package = Some(1); + let mut second = cands.clone(); + second.push(proxy(5, "lib", "1.3.0-ci.1", 100, Some(1))); + let d = evaluate_proxy( + &fmt(), + now(), + &policy_with(vec![count]), + &Selectors::none(), + &second, + ); + let ids: Vec = d.iter().map(|d| d.cached_version_id).collect(); + assert_eq!(ids, vec![3]); + } } diff --git a/crates/core/src/retention/files.rs b/crates/core/src/retention/files.rs index 1cc7c082..1054cd73 100644 --- a/crates/core/src/retention/files.rs +++ b/crates/core/src/retention/files.rs @@ -1,8 +1,9 @@ //! Retention of raw files. //! //! A raw repository's `retention_policy` has the envelope of a package -//! policy (`version`, `enabled`, `dry_run_only`, `min_run_interval_seconds`, -//! `rules`) with file rules. Each rule looks at the files under its +//! policy (`version`, `enabled`, `dry_run_only`, `min_run_interval_seconds` +//! or a `schedule` in a `schedule_timezone`, `max_run_seconds`, `rules`) with +//! file rules. Each rule looks at the files under its //! `path_prefix` (all files without one), directory by directory: //! //! * `keep_newest_per_directory` keeps the N most recently deployed files of @@ -45,6 +46,13 @@ pub struct FileRetentionPolicy { pub dry_run_only: bool, #[serde(default = "default_interval")] pub min_run_interval_seconds: u64, + /// A cron schedule, as for package policies. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schedule: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schedule_timezone: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_run_seconds: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub size_budget: Option, pub rules: Vec, @@ -96,6 +104,12 @@ pub fn validate_file_policy(policy: &FileRetentionPolicy, kind: RepoKind) -> Res "min_run_interval_seconds must be at most {MAX_MIN_RUN_INTERVAL_SECONDS}" )); } + super::schedule::validate( + policy.schedule.as_deref(), + policy.schedule_timezone.as_deref(), + policy.max_run_seconds, + Utc::now(), + )?; if policy.rules.is_empty() && policy.size_budget.is_none() { return Err("a policy needs at least one rule".into()); } @@ -360,13 +374,16 @@ const FILE_BATCH: usize = 100; /// Apply decisions (or, `dry_run`, only audit them). /// /// Decisions are applied per directory, in batches of up to -/// [`FILE_BATCH`], one transaction each. With a `fence`, each batch first +/// [`FILE_BATCH`], one transaction each. With a `deadline`, the run stops +/// before the first batch that would start after it (the first batch always +/// goes); the decisions left are `deferred` to the next run. With a `fence`, each batch first /// checks that the run still owns its lease and that the stored policy is /// still the one it evaluated, or the run stops (the next run evaluates the /// rest). It then locks the directory's files and evaluates the policy on /// them again: a decision their current state no longer produces (a file /// downloaded or redeployed since, a newer one deleted meanwhile) is /// skipped. +#[allow(clippy::too_many_arguments)] pub async fn apply_files( pool: &PgPool, repository_id: i64, @@ -375,8 +392,10 @@ pub async fn apply_files( grace: Duration, dry_run: bool, fence: Option<&FileRunFence<'_>>, + deadline: Option, ) -> sqlx::Result { let mut summary = RunSummary::default(); + let mut clock = super::schedule::RunClock::new(deadline); let mut directories: Vec<(&str, Vec<&FileDecision>)> = Vec::new(); let mut index: HashMap<&str, usize> = HashMap::new(); for d in decisions { @@ -390,6 +409,17 @@ pub async fn apply_files( let mut remaining = decisions.len(); for (dir, all) in &directories { for batch in all.chunks(FILE_BATCH) { + if clock.out_of_time() { + tracing::info!( + repository_id, + %run_id, + remaining, + "file retention run stopped at its max_run_seconds; \ + the next run goes on with the rest" + ); + summary.deferred += remaining as u64; + return Ok(summary); + } let mut tx = forge_db::begin(pool).await?; let mut decided_now: Option> = None; if let Some(fence) = fence.filter(|_| !dry_run) { @@ -510,6 +540,9 @@ mod tests { enabled: true, dry_run_only: false, min_run_interval_seconds: 3600, + schedule: None, + schedule_timezone: None, + max_run_seconds: None, size_budget: None, rules, } @@ -604,6 +637,29 @@ mod tests { assert!(parsed.is_err(), "unknown fields are refused"); } + #[test] + fn schedule_fields_are_validated() { + let mut keep = rule("keep"); + keep.keep_newest_per_directory = Some(3); + let mut p = policy(vec![keep]); + p.schedule = Some("0 3 * * *".into()); + p.schedule_timezone = Some("Europe/Berlin".into()); + p.max_run_seconds = Some(3600); + assert!(validate_file_policy(&p, RepoKind::Hosted).is_ok()); + let json = serde_json::to_value(&p).unwrap(); + assert_eq!(json["schedule"], "0 3 * * *"); + p.schedule = Some("0 3 * *".into()); + assert!(validate_file_policy(&p, RepoKind::Hosted).is_err()); + p.schedule = None; + assert!( + validate_file_policy(&p, RepoKind::Hosted).is_err(), + "a time zone needs a schedule" + ); + p.schedule_timezone = None; + p.max_run_seconds = Some(1); + assert!(validate_file_policy(&p, RepoKind::Hosted).is_err()); + } + #[test] fn budget_only_file_policy_is_valid_and_budget_is_validated() { let mut p = policy(Vec::new()); diff --git a/crates/core/src/retention/images.rs b/crates/core/src/retention/images.rs index 8f07d07a..db30529c 100644 --- a/crates/core/src/retention/images.rs +++ b/crates/core/src/retention/images.rs @@ -1,14 +1,18 @@ //! Retention of Docker / OCI images. //! //! A docker repository's `retention_policy` has the envelope of a package -//! policy (`version`, `enabled`, `dry_run_only`, `min_run_interval_seconds`, -//! `rules`) with image rules: +//! policy (`version`, `enabled`, `dry_run_only`, `min_run_interval_seconds` +//! or a `schedule` in a `schedule_timezone`, `max_run_seconds`, `rules`) with +//! image rules: //! //! * **Tag rules** delete tags: per image (optionally only those under an -//! `image_prefix`), the `keep_last_n_tags` most recently pushed tags and the -//! `protected_tags` stay; any other tag goes when every set trigger holds -//! (`tag_older_than_days` since it was pushed, `tag_not_pulled_for_days`). -//! A rule with only `keep_last_n_tags` keeps N tags per image. +//! `image_prefix`), among the tags the rule looks at (those `tag_pattern` +//! matches and `exclude_tag_pattern` does not, all without either), the +//! `keep_last_n_tags` most recently pushed tags and the `protected_tags` +//! (names or globs: `latest`, `release-*`) stay; any other tag goes when +//! every set trigger holds (`tag_older_than_days` since it was pushed, +//! `tag_not_pulled_for_days`). A rule with only `keep_last_n_tags` keeps N +//! tags per image. //! * **Untagged rules** (`delete_untagged_after_days`) delete manifests that //! have had no tag for that long — unless something kept refers to them: //! the children of a kept index, and the referrers (signatures, SBOMs, @@ -24,6 +28,7 @@ use chrono::{DateTime, Duration, Utc}; use forge_db::model::RepoKind; use forge_db::oci_dao::{self, RetentionManifest, RetentionTag}; use forge_db::retention_dao::{self, AuditRow, RunSummary}; +use globset::{GlobMatcher, GlobSet, GlobSetBuilder}; use serde::{Deserialize, Serialize}; use sqlx::PgPool; @@ -47,6 +52,13 @@ pub struct ImageRetentionPolicy { pub dry_run_only: bool, #[serde(default = "default_interval")] pub min_run_interval_seconds: u64, + /// A cron schedule, as for package policies. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schedule: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schedule_timezone: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_run_seconds: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub size_budget: Option, pub rules: Vec, @@ -60,11 +72,19 @@ pub struct ImageRule { /// Only images whose name starts with this (e.g. `team/`). #[serde(default, skip_serializing_if = "Option::is_none")] pub image_prefix: Option, + /// Only tags this glob matches (`pr-*`); all tags without one. Tags are + /// matched as written (case matters). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tag_pattern: Option, + /// Never tags this glob matches (`v*`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exclude_tag_pattern: Option, /// Tags kept per image: the most recently pushed ones. #[serde(default, skip_serializing_if = "Option::is_none")] pub keep_last_n_tags: Option, - /// Tags never deleted by this rule (e.g. `latest`, `stable`), kept in - /// addition to the `keep_last_n_tags` newest others. + /// Tags never deleted by this rule (e.g. `latest`, `stable`, or a glob + /// such as `release-*`), kept in addition to the `keep_last_n_tags` + /// newest others. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub protected_tags: Vec, /// Trigger: the tag was pushed at least this many days ago. @@ -93,6 +113,73 @@ impl ImageRule { } } +/// Compile a tag glob: `*`, `?`, `[…]` and `{a,b}` over the whole tag, case +/// sensitive (`pr-*`, `release-*`). A plain tag matches itself. +fn tag_glob(pattern: &str) -> Result { + globset::GlobBuilder::new(pattern) + .literal_separator(false) + .build() +} + +/// A tag rule's tag globs, compiled once per evaluation. +struct TagFilter { + include: Option, + exclude: Option, + protected: GlobSet, + /// A glob that does not compile (validation refuses one): the rule then + /// deletes no tag. + broken: bool, +} + +impl TagFilter { + fn new(rule: &ImageRule) -> Self { + let mut broken = false; + let mut matcher = |pattern: &Option| match pattern.as_deref().map(tag_glob) { + None => None, + Some(Ok(glob)) => Some(glob.compile_matcher()), + Some(Err(_)) => { + broken = true; + None + } + }; + let include = matcher(&rule.tag_pattern); + let exclude = matcher(&rule.exclude_tag_pattern); + let mut protected = GlobSetBuilder::new(); + for tag in &rule.protected_tags { + match tag_glob(tag) { + Ok(glob) => { + protected.add(glob); + } + Err(_) => broken = true, + } + } + let protected = protected.build().unwrap_or_else(|_| { + broken = true; + GlobSet::empty() + }); + TagFilter { + include, + exclude, + protected, + broken, + } + } + + /// Whether `protected_tags` names `tag`. + fn protects(&self, tag: &str) -> bool { + self.protected.is_match(tag) + } + + /// Whether the rule looks at `tag`: its patterns admit it and it is not + /// protected. + fn in_pool(&self, tag: &str) -> bool { + !self.broken + && self.include.as_ref().is_none_or(|g| g.is_match(tag)) + && !self.exclude.as_ref().is_some_and(|g| g.is_match(tag)) + && !self.protects(tag) + } +} + /// Validate an image policy for a repository of `kind`. pub fn validate_image_policy(policy: &ImageRetentionPolicy, kind: RepoKind) -> Result<(), String> { if kind == RepoKind::Group { @@ -111,6 +198,12 @@ pub fn validate_image_policy(policy: &ImageRetentionPolicy, kind: RepoKind) -> R "min_run_interval_seconds must be at most {MAX_MIN_RUN_INTERVAL_SECONDS}" )); } + super::schedule::validate( + policy.schedule.as_deref(), + policy.schedule_timezone.as_deref(), + policy.max_run_seconds, + Utc::now(), + )?; if policy.rules.is_empty() && policy.size_budget.is_none() { return Err("a policy needs at least one rule".into()); } @@ -158,6 +251,37 @@ pub fn validate_image_policy(policy: &ImageRetentionPolicy, kind: RepoKind) -> R rule.id )); } + for (field, pattern) in [ + ("tag_pattern", &rule.tag_pattern), + ("exclude_tag_pattern", &rule.exclude_tag_pattern), + ] { + let Some(pattern) = pattern else { + continue; + }; + if !rule.has_tag_criteria() { + return Err(format!("rule {:?}: {field} needs a tag criterion", rule.id)); + } + if pattern.trim().is_empty() { + return Err(format!("rule {:?}: {field} is empty", rule.id)); + } + if let Err(e) = tag_glob(pattern) { + return Err(format!( + "rule {:?}: {field} is not a valid glob: {e}", + rule.id + )); + } + } + for tag in &rule.protected_tags { + if tag.trim().is_empty() { + return Err(format!("rule {:?}: a protected tag is empty", rule.id)); + } + if let Err(e) = tag_glob(tag) { + return Err(format!( + "rule {:?}: protected tag {tag:?} is not a valid glob: {e}", + rule.id + )); + } + } } Ok(()) } @@ -208,15 +332,17 @@ pub fn evaluate_images( } let mut deleted_tags: HashSet<(&str, &str)> = HashSet::new(); for rule in policy.rules.iter().filter(|r| r.has_tag_criteria()) { + let filter = TagFilter::new(rule); for (image, image_tags) in &by_image { if !rule.matches(image) { continue; } - // Protected tags stay on top of the N kept ones. + // The tags the rule looks at; protected tags stay on top of the + // N kept ones. let mut ordered: Vec<&RetentionTag> = image_tags .iter() .copied() - .filter(|t| !rule.protected_tags.iter().any(|p| p == &t.tag)) + .filter(|t| filter.in_pool(&t.tag)) .collect(); ordered.sort_by(|a, b| b.updated_at.cmp(&a.updated_at).then(a.tag.cmp(&b.tag))); let keep = rule.keep_last_n_tags.unwrap_or(0) as usize; @@ -890,6 +1016,7 @@ fn budget_protected_tags( by_image.entry(&tag.image).or_default().push(tag); } for rule in &policy.rules { + let filter = TagFilter::new(rule); for (image, image_tags) in &by_image { if !rule.matches(image) { continue; @@ -897,13 +1024,13 @@ fn budget_protected_tags( protected.extend( image_tags .iter() - .filter(|t| rule.protected_tags.iter().any(|p| p == &t.tag)) + .filter(|t| filter.protects(&t.tag)) .map(|t| (t.image.clone(), t.tag.clone())), ); if let Some(keep) = rule.keep_last_n_tags { let mut ordered: Vec<_> = image_tags .iter() - .filter(|t| !rule.protected_tags.iter().any(|p| p == &t.tag)) + .filter(|t| filter.in_pool(&t.tag)) .copied() .collect(); ordered.sort_by(|a, b| b.updated_at.cmp(&a.updated_at).then(a.tag.cmp(&b.tag))); @@ -1047,6 +1174,10 @@ impl DecidedNow { /// Apply decisions (or, `dry_run`, only audit them). /// +/// With a `deadline`, the run stops before the first batch that would start +/// after it (the first batch always goes); the decisions left are +/// `deferred` to the next run. +/// /// Decisions are applied per image, in batches of up to [`IMAGE_BATCH`] /// (its tags first, then its manifests, indexes before their children), /// one transaction each. With a `fence`, each batch first checks that the @@ -1057,6 +1188,7 @@ impl DecidedNow { /// state no longer produces (a tag pulled or pushed since, newer tags /// deleted meanwhile, a referrer whose subject was tagged again) is /// skipped. +#[allow(clippy::too_many_arguments)] pub async fn apply_images( pool: &PgPool, repository_id: i64, @@ -1065,11 +1197,24 @@ pub async fn apply_images( grace: Duration, dry_run: bool, fence: Option<&ImageRunFence<'_>>, + deadline: Option, ) -> sqlx::Result { let mut summary = RunSummary::default(); + let mut clock = super::schedule::RunClock::new(deadline); let batches = image_batches(decisions); let mut remaining = decisions.tags.len() + decisions.manifests.len(); for (image, batch) in &batches { + if clock.out_of_time() { + tracing::info!( + repository_id, + %run_id, + remaining, + "image retention run stopped at its max_run_seconds; \ + the next run goes on with the rest" + ); + summary.deferred += remaining as u64; + return Ok(summary); + } let mut tx = forge_db::begin(pool).await?; let mut decided = None; if let Some(fence) = fence.filter(|_| !dry_run) { @@ -1424,6 +1569,9 @@ mod tests { enabled: true, dry_run_only: false, min_run_interval_seconds: 3600, + schedule: None, + schedule_timezone: None, + max_run_seconds: None, size_budget: None, rules, } @@ -1432,6 +1580,8 @@ mod tests { ImageRule { id: id.into(), image_prefix: None, + tag_pattern: None, + exclude_tag_pattern: None, keep_last_n_tags: None, protected_tags: Vec::new(), tag_older_than_days: None, @@ -1459,6 +1609,59 @@ mod tests { assert_eq!(deleted, vec!["1"]); } + #[test] + fn tag_patterns_choose_the_tags_a_rule_looks_at() { + let tags = vec![ + tag("app", "pr-12", 1, 30), + tag("app", "pr-13", 2, 20), + tag("app", "pr-14", 3, 1), + tag("app", "pr-keep", 4, 30), + tag("app", "v1.0", 5, 30), + tag("app", "main", 6, 30), + ]; + // Delete `pr-*` tags after 7 days, except `pr-keep`. + let mut r = rule("prs"); + r.tag_pattern = Some("pr-*".into()); + r.exclude_tag_pattern = Some("*-keep".into()); + r.tag_older_than_days = Some(7); + let d = evaluate_images(now(), &policy(vec![r.clone()]), &tags, &[]); + let mut deleted: Vec<&str> = d.tags.iter().map(|t| t.tag.as_str()).collect(); + deleted.sort_unstable(); + assert_eq!(deleted, vec!["pr-12", "pr-13"]); + + // keep_last_n_tags counts only the tags the rule looks at: the newest + // `pr-*` tag is kept, whatever `main` and `v1.0` are. + r.tag_older_than_days = None; + r.exclude_tag_pattern = None; + r.keep_last_n_tags = Some(1); + let d = evaluate_images(now(), &policy(vec![r]), &tags, &[]); + let mut deleted: Vec<&str> = d.tags.iter().map(|t| t.tag.as_str()).collect(); + deleted.sort_unstable(); + assert_eq!(deleted, vec!["pr-12", "pr-13", "pr-keep"]); + } + + #[test] + fn protected_tags_take_globs_in_rules_and_the_budget() { + let tags = vec![ + tag("app", "release-1", 1, 300), + tag("app", "release-2", 2, 200), + tag("app", "latest", 2, 200), + tag("app", "build-7", 3, 100), + tag("app", "build-8", 4, 90), + ]; + let mut r = rule("old"); + r.tag_older_than_days = Some(30); + r.protected_tags = vec!["release-*".into(), "latest".into()]; + let d = evaluate_images(now(), &policy(vec![r.clone()]), &tags, &[]); + let mut deleted: Vec<&str> = d.tags.iter().map(|t| t.tag.as_str()).collect(); + deleted.sort_unstable(); + assert_eq!(deleted, vec!["build-7", "build-8"]); + let kept = budget_protected_tags(&policy(vec![r]), &tags); + let mut kept: Vec<&str> = kept.iter().map(|(_, t)| t.as_str()).collect(); + kept.sort_unstable(); + assert_eq!(kept, vec!["latest", "release-1", "release-2"]); + } + #[test] fn untagged_manifests_go_unless_something_kept_refers_to_them() { // 10: tagged index of 11 and 12. 11, 12 untagged children (kept). @@ -1595,6 +1798,34 @@ mod tests { serde_json::json!({"version": 1, "rules": [{"id": "x", "keep": 1}]}), ); assert!(parsed.is_err(), "unknown fields are refused"); + + // Tag globs compile, are not empty, and need a tag criterion. + let mut prs = rule("prs"); + prs.tag_pattern = Some("pr-*".into()); + prs.tag_older_than_days = Some(7); + prs.protected_tags = vec!["release-*".into()]; + assert!(validate_image_policy(&policy(vec![prs.clone()]), RepoKind::Hosted).is_ok()); + for broken in ["pr-[", ""] { + let mut bad = prs.clone(); + bad.tag_pattern = Some(broken.into()); + assert!(validate_image_policy(&policy(vec![bad]), RepoKind::Hosted).is_err()); + let mut bad = prs.clone(); + bad.protected_tags = vec![broken.into()]; + assert!(validate_image_policy(&policy(vec![bad]), RepoKind::Hosted).is_err()); + } + let mut untagged = rule("untagged"); + untagged.delete_untagged_after_days = Some(7); + untagged.exclude_tag_pattern = Some("v*".into()); + assert!(validate_image_policy(&policy(vec![untagged]), RepoKind::Hosted).is_err()); + + // The schedule fields of the envelope. + let mut scheduled = ok.clone(); + scheduled.schedule = Some("30 2 * * 0".into()); + scheduled.schedule_timezone = Some("America/New_York".into()); + scheduled.max_run_seconds = Some(1800); + assert!(validate_image_policy(&scheduled, RepoKind::Hosted).is_ok()); + scheduled.schedule_timezone = Some("Nowhere/Land".into()); + assert!(validate_image_policy(&scheduled, RepoKind::Hosted).is_err()); } #[test] diff --git a/crates/core/src/retention/mod.rs b/crates/core/src/retention/mod.rs index b3a8dc67..6deae522 100644 --- a/crates/core/src/retention/mod.rs +++ b/crates/core/src/retention/mod.rs @@ -17,12 +17,13 @@ pub mod evaluate; pub mod files; pub mod images; pub mod model; +pub mod schedule; pub mod worker; pub use apply::{apply, ApplyContext, RunFence}; pub use evaluate::{ evaluate, evaluate_proxy, CandidateComponent, Decision, DecisionReason, ProxyCandidate, - ProxyDecision, + ProxyDecision, Selectors, }; pub use model::{ validate_policy, Action, Criteria, PolicyError, ReleaseType, RetentionPolicy, RetentionRule, @@ -54,3 +55,59 @@ pub async fn tagged_candidate_ids( .map(|candidate| candidate.component_id) .collect()) } + +/// Why the content selectors a policy names could not be resolved. +#[derive(Debug, thiserror::Error)] +pub enum SelectorError { + /// A rule names a selector that does not exist (any more). + #[error("content selector {0:?} does not exist")] + Missing(String), + /// A selector's stored patterns do not read or compile. + #[error("content selector {name:?} cannot be used: {reason}")] + Invalid { + /// The selector's name. + name: String, + /// What is wrong with it. + reason: String, + }, + /// Reading the selectors failed. + #[error(transparent)] + Db(#[from] sqlx::Error), +} + +/// The content selectors `policy`'s rules name (`selector`), read from +/// `content_selector` and compiled, for the evaluator. A name that matches +/// no selector is an error: a run or a preview then refuses the policy +/// instead of evaluating a rule that would admit nothing. +pub async fn resolve_selectors( + pool: &sqlx::PgPool, + policy: &model::RetentionPolicy, +) -> Result { + let names = model::selector_names(policy); + let mut selectors = Selectors::none(); + if names.is_empty() { + return Ok(selectors); + } + let stored = forge_db::authz_dao::list_content_selectors(pool).await?; + for name in names { + let Some(row) = stored.iter().find(|s| s.name == name) else { + return Err(SelectorError::Missing(name)); + }; + let invalid = |reason: String| SelectorError::Invalid { + name: name.clone(), + reason, + }; + let (includes, excludes) = forge_db::authz_dao::SelectorRow { + id: row.id, + includes: row.includes.clone(), + excludes: row.excludes.clone(), + } + .patterns() + .map_err(|e| invalid(e.to_string()))?; + let compiled = crate::authz::SelectorPatterns { includes, excludes } + .compile() + .map_err(|e| invalid(e.to_string()))?; + selectors.insert(name.clone(), compiled); + } + Ok(selectors) +} diff --git a/crates/core/src/retention/model.rs b/crates/core/src/retention/model.rs index 8fb47e56..8677c62a 100644 --- a/crates/core/src/retention/model.rs +++ b/crates/core/src/retention/model.rs @@ -9,10 +9,18 @@ //! //! Parity subset of Nexus cleanup policies + Artifactory retention: //! * protections: `keep_last_n`, `keep_n_most_downloaded` -//! * triggers (ANDed within a rule): `older_than_days`, `not_downloaded_for_days` -//! * filters: `release_type` (release vs prerelease), `package_name_prefix` -//! * proxy rules: `scope: proxy_cached` with `not_pulled_for_days`, and the -//! same protections over each package's cached versions +//! * triggers (ANDed within a rule): `older_than_days`, `not_downloaded_for_days`, +//! `prerelease_superseded_for_days`, `unlisted_for_days`, +//! `max_versions_per_package` +//! * filters: `release_type` (release, prerelease or snapshot, as the format +//! classes a version), `package_name_prefix`, `selector` (a content +//! selector), `version_pattern` / `exclude_version_pattern` (globs) +//! * proxy rules: `scope: proxy_cached` with `not_pulled_for_days` (or +//! `max_versions_per_package`), and the same protections over each +//! package's cached versions +//! * scheduling: `min_run_interval_seconds`, or a cron `schedule` in a +//! `schedule_timezone`, and `max_run_seconds` (see +//! [`crate::retention::schedule`]) //! //! These types are **pure data** + a `validate_policy` function; no I/O. They are //! (de)serialized straight to/from the jsonb column with serde. @@ -45,6 +53,19 @@ pub struct RetentionPolicy { /// never schedules runs less than 60 s apart, whatever this says. #[serde(default = "default_min_run_interval")] pub min_run_interval_seconds: u64, + /// A 5-field cron pattern (`0 3 * * *`): the worker runs the policy at + /// its times instead of every `min_run_interval_seconds`. A manual run + /// still runs at once. See [`crate::retention::schedule`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schedule: Option, + /// The IANA time zone of `schedule`'s times (`Europe/Berlin`); UTC when + /// absent. Only with a `schedule`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub schedule_timezone: Option, + /// How long a run may apply decisions before it stops; the next run + /// goes on with what is left. `None`: as long as it needs. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_run_seconds: Option, /// Optional repository-wide logical-size budget. #[serde(default, skip_serializing_if = "Option::is_none")] pub size_budget: Option, @@ -118,6 +139,20 @@ pub struct RetentionRule { /// Optional case-insensitive package-id prefix filter (`None` ⇒ all ids). #[serde(default)] pub package_name_prefix: Option, + /// Optional content selector, by name: only packages its include and + /// exclude globs admit (as a grant scoped by it would). Resolved when + /// the policy is stored and on every run; a selector that is gone fails + /// the run. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selector: Option, + /// Optional glob on the normalized version, ignoring case (`*-ci.*`, + /// `{*-pr.*,*-ci.*}`): only versions it matches. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version_pattern: Option, + /// Optional glob on the normalized version, ignoring case: never the + /// versions it matches. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub exclude_version_pattern: Option, /// The protections + triggers for this rule. #[serde(default)] pub criteria: Criteria, @@ -163,17 +198,20 @@ pub enum Scope { ProxyCached, } -/// Release-type filter for a rule. +/// Release-type filter for a rule: the version classes it admits, as the +/// repository's format classes a version ([`crate::format::Format::version_class`]). #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ReleaseType { /// Match every version. #[default] Any, - /// Match only stable releases (no prerelease label). + /// Match only stable releases (for SemVer formats: no prerelease label). Releases, /// Match only prerelease versions. Prereleases, + /// Match only snapshot builds (formats that have them, such as Maven). + Snapshots, } /// Protections + triggers for a rule. All fields optional; an empty `Criteria` @@ -215,6 +253,24 @@ pub struct Criteria { /// `scope: proxy_cached` rules. #[serde(default, skip_serializing_if = "Option::is_none")] pub not_pulled_for_days: Option, + /// Trigger: a prerelease (or snapshot) whose package has had, for at + /// least this many days, a **listed release that orders after it** (for + /// SemVer: a release of the same or a higher base version, so + /// `2.3.0-ci.417` once `2.3.0` or `2.4.0` is out). Counted from the + /// publication of the first such release. Hosted rules only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prerelease_superseded_for_days: Option, + /// Trigger: unlisted for at least this many days (`component.unlisted_at`); + /// a listed version never matches. With `action: hard_delete` it is the + /// second step of "unlist, then delete later". Hosted rules only. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub unlisted_for_days: Option, + /// Trigger: not among the newest N versions (by format version order, + /// listed or not) of its package in this rule's pool, per partition + /// ([`crate::format::Format::retention_partition`]). A pure count: no age + /// needed. On proxy rules it counts the package's cached versions. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub max_versions_per_package: Option, } impl Criteria { @@ -225,6 +281,9 @@ impl Criteria { self.older_than_days.is_some() || self.not_downloaded_for_days.is_some() || self.not_pulled_for_days.is_some() + || self.prerelease_superseded_for_days.is_some() + || self.unlisted_for_days.is_some() + || self.max_versions_per_package.is_some() } /// Whether this rule defines any protection. @@ -260,7 +319,11 @@ pub enum PolicyError { #[error("rule id must not be empty")] EmptyRuleId, /// A rule had no deletion trigger (only protections / nothing). - #[error("rule {0:?} has no trigger criterion (older_than_days / not_downloaded_for_days / not_pulled_for_days)")] + #[error( + "rule {0:?} has no trigger criterion (older_than_days, not_downloaded_for_days, \ + prerelease_superseded_for_days, unlisted_for_days, max_versions_per_package or \ + not_pulled_for_days)" + )] RuleHasNoTrigger(String), /// A `keep_*` count was zero (meaningless — protects nothing; use absence /// instead). @@ -313,6 +376,35 @@ pub enum PolicyError { /// The upper bound. max: u64, }, + /// `schedule`, `schedule_timezone` or `max_run_seconds` is invalid. + #[error("{0}")] + InvalidSchedule(String), + /// A version glob does not compile. + #[error("rule {rule:?} field {field} is not a valid glob: {reason}")] + InvalidPattern { + /// The offending rule id. + rule: String, + /// The offending field name. + field: &'static str, + /// What is wrong with it. + reason: String, + }, + /// A rule's `selector` names no selector (empty). + #[error("rule {0:?} field selector must name a content selector")] + EmptySelector(String), + /// `unlisted_for_days` selects versions already unlisted: unlisting them + /// again does nothing. + #[error( + "rule {0:?} uses unlisted_for_days, which selects versions already unlisted; \ + its action must be hard_delete" + )] + UnlistedTriggerUnlists(String), + /// `prerelease_superseded_for_days` on a rule that admits only releases. + #[error( + "rule {0:?} uses prerelease_superseded_for_days, but release_type \"releases\" \ + admits no prerelease" + )] + SupersededOnReleases(String), } /// Upper bound on a policy's `min_run_interval_seconds` (one year). Larger @@ -353,6 +445,13 @@ pub fn validate_policy(policy: &RetentionPolicy, repo_kind: RepoKind) -> Result< max: MAX_MIN_RUN_INTERVAL_SECONDS, }); } + crate::retention::schedule::validate( + policy.schedule.as_deref(), + policy.schedule_timezone.as_deref(), + policy.max_run_seconds, + chrono::Utc::now(), + ) + .map_err(PolicyError::InvalidSchedule)?; let mut seen_ids: Vec<&str> = Vec::with_capacity(policy.rules.len()); for rule in &policy.rules { @@ -396,6 +495,20 @@ fn validate_rule(rule: &RetentionRule, repo_kind: RepoKind) -> Result<(), Policy field: "not_downloaded_for_days", }); } + // A proxy's cached versions are listed or not as the upstream + // says, and are never unlisted here. + if c.prerelease_superseded_for_days.is_some() { + return Err(PolicyError::HostedFieldOnProxyScope { + rule: rule.id.clone(), + field: "prerelease_superseded_for_days", + }); + } + if c.unlisted_for_days.is_some() { + return Err(PolicyError::HostedFieldOnProxyScope { + rule: rule.id.clone(), + field: "unlisted_for_days", + }); + } } Scope::Hosted if repo_kind == RepoKind::Proxy => { return Err(PolicyError::HostedScopeOnProxy(rule.id.clone())); @@ -420,6 +533,12 @@ fn validate_rule(rule: &RetentionRule, repo_kind: RepoKind) -> Result<(), Policy field: "keep_n_most_downloaded", }); } + if c.max_versions_per_package == Some(0) { + return Err(PolicyError::ZeroCount { + rule: rule.id.clone(), + field: "max_versions_per_package", + }); + } // Days must be positive when present. if c.older_than_days == Some(0) { return Err(PolicyError::ZeroDays { @@ -439,6 +558,52 @@ fn validate_rule(rule: &RetentionRule, repo_kind: RepoKind) -> Result<(), Policy field: "not_pulled_for_days", }); } + if c.prerelease_superseded_for_days == Some(0) { + return Err(PolicyError::ZeroDays { + rule: rule.id.clone(), + field: "prerelease_superseded_for_days", + }); + } + if c.unlisted_for_days == Some(0) { + return Err(PolicyError::ZeroDays { + rule: rule.id.clone(), + field: "unlisted_for_days", + }); + } + if c.unlisted_for_days.is_some() && rule.action == Action::Unlist { + return Err(PolicyError::UnlistedTriggerUnlists(rule.id.clone())); + } + if c.prerelease_superseded_for_days.is_some() && rule.release_type == ReleaseType::Releases { + return Err(PolicyError::SupersededOnReleases(rule.id.clone())); + } + + // Filters must compile (a broken one would admit nothing at run time). + for (field, pattern) in [ + ("version_pattern", &rule.version_pattern), + ("exclude_version_pattern", &rule.exclude_version_pattern), + ] { + if let Some(pattern) = pattern { + let refused = if pattern.trim().is_empty() { + Some("it is empty".to_string()) + } else { + version_glob(pattern).err().map(|e| e.to_string()) + }; + if let Some(reason) = refused { + return Err(PolicyError::InvalidPattern { + rule: rule.id.clone(), + field, + reason, + }); + } + } + } + if rule + .selector + .as_deref() + .is_some_and(|s| s.trim().is_empty()) + { + return Err(PolicyError::EmptySelector(rule.id.clone())); + } // Every rule must express at least one trigger; pure-protection rules // delete nothing and are almost certainly a config error. @@ -449,6 +614,28 @@ fn validate_rule(rule: &RetentionRule, repo_kind: RepoKind) -> Result<(), Policy Ok(()) } +/// Compile a rule's version glob: matched against the whole normalized +/// version, ignoring case (`*-ci.*`, `{*-pr.*,*-ci.*}`, `1.?.*`). +pub fn version_glob(pattern: &str) -> Result { + Ok(globset::GlobBuilder::new(pattern) + .case_insensitive(true) + .literal_separator(false) + .build()? + .compile_matcher()) +} + +/// The content selectors a policy's rules name, each once. +pub fn selector_names(policy: &RetentionPolicy) -> Vec { + let mut names: Vec = policy + .rules + .iter() + .filter_map(|r| r.selector.clone()) + .collect(); + names.sort(); + names.dedup(); + names +} + #[cfg(test)] mod tests { use super::*; @@ -462,12 +649,14 @@ mod tests { scope: Scope::Hosted, release_type: ReleaseType::Any, package_name_prefix: None, + selector: None, + version_pattern: None, + exclude_version_pattern: None, criteria: Criteria { keep_last_n: Some(10), - keep_n_most_downloaded: None, older_than_days: Some(90), not_downloaded_for_days: Some(30), - not_pulled_for_days: None, + ..Default::default() }, } } @@ -479,6 +668,9 @@ mod tests { scope: Scope::ProxyCached, release_type: ReleaseType::Any, package_name_prefix: None, + selector: None, + version_pattern: None, + exclude_version_pattern: None, criteria: Criteria { not_pulled_for_days: Some(30), ..Default::default() @@ -492,6 +684,9 @@ mod tests { enabled: true, dry_run_only: false, min_run_interval_seconds: 3600, + schedule: None, + schedule_timezone: None, + max_run_seconds: None, size_budget: None, rules: vec![example_hosted_rule()], } @@ -504,6 +699,9 @@ mod tests { enabled: true, dry_run_only: false, min_run_interval_seconds: 3600, + schedule: None, + schedule_timezone: None, + max_run_seconds: None, size_budget: None, rules: vec![example_hosted_rule(), example_proxy_rule()], }; @@ -813,6 +1011,9 @@ mod tests { scope: Scope::Hosted, release_type: ReleaseType::Any, package_name_prefix: None, + selector: None, + version_pattern: None, + exclude_version_pattern: None, criteria: Criteria { keep_last_n: Some(5), ..Default::default() @@ -844,4 +1045,192 @@ mod tests { // Even disabled, a broken rule is rejected so a later enable is safe. assert!(validate_policy(&p, RepoKind::Hosted).is_err()); } + + #[test] + fn new_triggers_and_filters_round_trip_and_validate() { + let json = r#"{ + "version": 1, + "schedule": "0 3 * * *", + "schedule_timezone": "Europe/Berlin", + "max_run_seconds": 3600, + "rules": [ + { + "id": "ci", + "action": "hard_delete", + "release_type": "prereleases", + "selector": "internal", + "version_pattern": "*-ci.*", + "exclude_version_pattern": "*-ci.0", + "criteria": { "prerelease_superseded_for_days": 14 } + }, + { + "id": "unlisted", + "action": "hard_delete", + "criteria": { "unlisted_for_days": 90 } + }, + { + "id": "cap", + "action": "unlist", + "release_type": "snapshots", + "criteria": { "max_versions_per_package": 50 } + } + ] + }"#; + let policy: RetentionPolicy = serde_json::from_str(json).unwrap(); + validate_policy(&policy, RepoKind::Hosted).unwrap(); + assert_eq!(policy.schedule.as_deref(), Some("0 3 * * *")); + assert_eq!(policy.rules[0].selector.as_deref(), Some("internal")); + assert_eq!(policy.rules[2].release_type, ReleaseType::Snapshots); + assert_eq!(selector_names(&policy), vec!["internal".to_string()]); + let back: RetentionPolicy = + serde_json::from_value(serde_json::to_value(&policy).unwrap()).unwrap(); + assert_eq!(back, policy); + // Absent new fields are not written, so policies stored before keep + // their shape. + let plain = serde_json::to_value(minimal_valid()).unwrap(); + for absent in ["schedule", "schedule_timezone", "max_run_seconds"] { + assert!(plain.get(absent).is_none(), "{absent}"); + } + assert!(plain["rules"][0].get("version_pattern").is_none()); + // Unknown fields are still refused. + let typo = json.replace("version_pattern", "version_patern"); + assert!(serde_json::from_str::(&typo).is_err()); + } + + fn with_rule(edit: impl FnOnce(&mut RetentionRule)) -> RetentionPolicy { + let mut p = minimal_valid(); + edit(&mut p.rules[0]); + p + } + + #[test] + fn rejects_invalid_new_triggers_and_filters() { + let zero_days = with_rule(|r| r.criteria.prerelease_superseded_for_days = Some(0)); + assert!(matches!( + validate_policy(&zero_days, RepoKind::Hosted), + Err(PolicyError::ZeroDays { + field: "prerelease_superseded_for_days", + .. + }) + )); + let zero_days = with_rule(|r| r.criteria.unlisted_for_days = Some(0)); + assert!(matches!( + validate_policy(&zero_days, RepoKind::Hosted), + Err(PolicyError::ZeroDays { + field: "unlisted_for_days", + .. + }) + )); + let zero_count = with_rule(|r| r.criteria.max_versions_per_package = Some(0)); + assert!(matches!( + validate_policy(&zero_count, RepoKind::Hosted), + Err(PolicyError::ZeroCount { + field: "max_versions_per_package", + .. + }) + )); + let unlists_unlisted = with_rule(|r| { + r.action = Action::Unlist; + r.criteria.unlisted_for_days = Some(30); + }); + assert_eq!( + validate_policy(&unlists_unlisted, RepoKind::Hosted), + Err(PolicyError::UnlistedTriggerUnlists( + "keep-recent-releases".into() + )) + ); + let releases_only = with_rule(|r| { + r.release_type = ReleaseType::Releases; + r.criteria.prerelease_superseded_for_days = Some(7); + }); + assert_eq!( + validate_policy(&releases_only, RepoKind::Hosted), + Err(PolicyError::SupersededOnReleases( + "keep-recent-releases".into() + )) + ); + for (field, pattern) in [ + ("version_pattern", "1.[0"), + ("exclude_version_pattern", " "), + ] { + let p = with_rule(|r| { + if field == "version_pattern" { + r.version_pattern = Some(pattern.into()); + } else { + r.exclude_version_pattern = Some(pattern.into()); + } + }); + assert!( + matches!( + validate_policy(&p, RepoKind::Hosted), + Err(PolicyError::InvalidPattern { field: f, .. }) if f == field + ), + "{field}" + ); + } + let empty_selector = with_rule(|r| r.selector = Some(" ".into())); + assert_eq!( + validate_policy(&empty_selector, RepoKind::Hosted), + Err(PolicyError::EmptySelector("keep-recent-releases".into())) + ); + } + + #[test] + fn hosted_only_triggers_are_refused_on_proxy_rules() { + for field in ["prerelease_superseded_for_days", "unlisted_for_days"] { + let mut rule = example_proxy_rule(); + if field == "unlisted_for_days" { + rule.criteria.unlisted_for_days = Some(30); + } else { + rule.criteria.prerelease_superseded_for_days = Some(30); + } + let p = RetentionPolicy { + rules: vec![rule], + ..minimal_valid() + }; + assert_eq!( + validate_policy(&p, RepoKind::Proxy), + Err(PolicyError::HostedFieldOnProxyScope { + rule: "proxy-evict-cold".into(), + field, + }) + ); + } + // A count is a trigger of its own on a proxy too, with filters. + let mut rule = example_proxy_rule(); + rule.criteria.not_pulled_for_days = None; + rule.criteria.max_versions_per_package = Some(5); + rule.version_pattern = Some("*-ci.*".into()); + let p = RetentionPolicy { + rules: vec![rule], + ..minimal_valid() + }; + validate_policy(&p, RepoKind::Proxy).unwrap(); + } + + #[test] + fn schedule_fields_are_validated() { + let mut p = minimal_valid(); + p.schedule = Some("0 3 * * 1-5".into()); + p.schedule_timezone = Some("Europe/Berlin".into()); + p.max_run_seconds = Some(7200); + validate_policy(&p, RepoKind::Hosted).unwrap(); + for (schedule, zone, max) in [ + (Some("0 3 * *"), None, None), + (Some("0 3 * * *"), Some("Europe/Atlantis"), None), + (None, Some("UTC"), None), + (None, None, Some(10)), + ] { + p.schedule = schedule.map(Into::into); + p.schedule_timezone = zone.map(Into::into); + p.max_run_seconds = max; + assert!( + matches!( + validate_policy(&p, RepoKind::Hosted), + Err(PolicyError::InvalidSchedule(_)) + ), + "{schedule:?} {zone:?} {max:?}" + ); + } + } } diff --git a/crates/core/src/retention/schedule.rs b/crates/core/src/retention/schedule.rs new file mode 100644 index 00000000..95119d27 --- /dev/null +++ b/crates/core/src/retention/schedule.rs @@ -0,0 +1,345 @@ +//! When a repository's retention policy runs. +//! +//! Every policy shape (package, raw file and image policies) has the same +//! scheduling fields in its envelope: +//! +//! * `min_run_interval_seconds` (default 3600): a run at most that often; +//! * `schedule`: a 5-field cron pattern (`minute hour day-of-month month +//! day-of-week`, e.g. `0 3 * * *`), as an alternative to the interval: the +//! worker runs the policy at the pattern's times instead; +//! * `schedule_timezone`: the IANA time zone the pattern's times are in +//! (`Europe/Berlin`); UTC without one; +//! * `max_run_seconds`: how long a run may apply decisions before it stops +//! (the end of a maintenance window). The next run evaluates the +//! repository again and goes on with what is left. +//! +//! A pattern's times follow the zone's daylight-saving changes as Vixie cron +//! does ([croner]'s rules, the Open Cron Pattern Specification): a time that +//! falls into the hour the clocks skip runs right after the change, on that +//! day; a time in the hour they repeat runs once, at its first occurrence. +//! `0 3 * * *` in `Europe/Berlin` therefore runs at 03:00 Berlin time every +//! day, whether that is 01:00 or 02:00 UTC. +//! +//! The functions here are pure: the worker passes the database's clock. + +use std::str::FromStr; + +use chrono::{DateTime, Utc}; +use chrono_tz::Tz; +use croner::parser::{CronParser, Seconds, Year}; +use croner::Cron; +use serde_json::Value as JsonValue; + +/// The shortest `max_run_seconds` a policy may set. +pub const MIN_MAX_RUN_SECONDS: u64 = 60; + +/// The longest `max_run_seconds` a policy may set (a week). +pub const MAX_MAX_RUN_SECONDS: u64 = 7 * 24 * 60 * 60; + +/// A policy's `schedule` in its `schedule_timezone`, parsed. +#[derive(Debug, Clone)] +pub struct Schedule { + cron: Cron, + zone: Tz, +} + +impl Schedule { + /// Parse a 5-field cron `pattern` (`@daily`, `@weekly` and the other + /// nicknames are accepted too) for the IANA time zone `zone` (UTC when + /// `None`). + pub fn parse(pattern: &str, zone: Option<&str>) -> Result { + let cron = CronParser::builder() + .seconds(Seconds::Disallowed) + .year(Year::Disallowed) + .build() + .parse(pattern) + .map_err(|e| { + format!( + "schedule {pattern:?} is not a cron pattern of 5 fields \ + (minute hour day-of-month month day-of-week, e.g. \"0 3 * * *\"): {e}" + ) + })?; + let zone = match zone { + None => Tz::UTC, + Some(name) => Tz::from_str(name).map_err(|_| { + format!( + "schedule_timezone {name:?} is not an IANA time zone \ + (e.g. \"Europe/Berlin\" or \"UTC\")" + ) + })?, + }; + Ok(Schedule { cron, zone }) + } + + /// The time zone the pattern's times are in. + pub fn zone(&self) -> Tz { + self.zone + } + + /// The first time of the pattern strictly after `after`, or `None` when + /// the pattern names no time within croner's search range (e.g. the 30th + /// of February). + pub fn next_after(&self, after: DateTime) -> Option> { + self.cron + .find_next_occurrence(&after.with_timezone(&self.zone), false) + .ok() + .map(|t| t.with_timezone(&Utc)) + } + + /// The pattern's next `n` times after `after`. + pub fn next_runs(&self, after: DateTime, n: usize) -> Vec> { + let mut runs = Vec::with_capacity(n); + let mut at = after; + while runs.len() < n { + let Some(next) = self.next_after(at) else { + break; + }; + runs.push(next); + at = next; + } + runs + } +} + +/// Check an envelope's scheduling fields: `schedule` parses and names a time +/// after `now`, `schedule_timezone` is a zone and comes with a schedule, and +/// `max_run_seconds` is within [`MIN_MAX_RUN_SECONDS`] and +/// [`MAX_MAX_RUN_SECONDS`]. +pub fn validate( + schedule: Option<&str>, + zone: Option<&str>, + max_run_seconds: Option, + now: DateTime, +) -> Result<(), String> { + match schedule { + Some(pattern) => { + let parsed = Schedule::parse(pattern, zone)?; + if parsed.next_after(now).is_none() { + return Err(format!("schedule {pattern:?} names no time that comes")); + } + } + None if zone.is_some() => { + return Err("schedule_timezone needs a schedule".into()); + } + None => {} + } + if let Some(secs) = max_run_seconds { + if !(MIN_MAX_RUN_SECONDS..=MAX_MAX_RUN_SECONDS).contains(&secs) { + return Err(format!( + "max_run_seconds must be between {MIN_MAX_RUN_SECONDS} and \ + {MAX_MAX_RUN_SECONDS} (a week)" + )); + } + } + Ok(()) +} + +/// The schedule a stored policy of any shape names, when it names one that +/// parses (validation refuses any other, so `None` means an interval policy). +pub fn of_policy(policy: &JsonValue) -> Option { + let pattern = policy.get("schedule")?.as_str()?; + let zone = policy.get("schedule_timezone").and_then(JsonValue::as_str); + Schedule::parse(pattern, zone).ok() +} + +/// How long a run of a stored policy of any shape may apply decisions +/// (`max_run_seconds`), or `None` when it may take as long as it needs. +pub fn max_run_duration(policy: &JsonValue) -> Option { + policy + .get("max_run_seconds") + .and_then(JsonValue::as_u64) + .map(|secs| std::time::Duration::from_secs(secs.min(MAX_MAX_RUN_SECONDS))) +} + +/// A run's time limit as its apply loops see it: [`Self::out_of_time`] is +/// asked before each step (a batch, an eviction), and says stop once the +/// deadline passed. The first step always goes, so a run whose evaluation +/// alone outlasts its `max_run_seconds` still makes progress. +#[derive(Debug, Clone, Copy)] +pub(crate) struct RunClock { + deadline: Option, + stepped: bool, +} + +impl RunClock { + pub(crate) fn new(deadline: Option) -> Self { + RunClock { + deadline, + stepped: false, + } + } + + /// Whether the run must stop before the step it is about to take. + pub(crate) fn out_of_time(&mut self) -> bool { + let out = self.stepped + && self + .deadline + .is_some_and(|deadline| std::time::Instant::now() >= deadline); + self.stepped = true; + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + + fn utc(y: i32, mo: u32, d: u32, h: u32, mi: u32) -> DateTime { + Utc.with_ymd_and_hms(y, mo, d, h, mi, 0).unwrap() + } + + fn berlin(y: i32, mo: u32, d: u32, h: u32, mi: u32) -> DateTime { + chrono_tz::Europe::Berlin + .with_ymd_and_hms(y, mo, d, h, mi, 0) + .earliest() + .unwrap() + .with_timezone(&Utc) + } + + #[test] + fn a_daily_run_stays_at_its_local_time_across_the_spring_change() { + // Berlin moves from CET (UTC+1) to CEST (UTC+2) on 2026-03-29 at + // 02:00 local time. + let s = Schedule::parse("0 3 * * *", Some("Europe/Berlin")).unwrap(); + let runs = s.next_runs(utc(2026, 3, 27, 12, 0), 4); + assert_eq!( + runs, + vec![ + utc(2026, 3, 28, 2, 0), // 03:00 CET + utc(2026, 3, 29, 1, 0), // 03:00 CEST, the day of the change + utc(2026, 3, 30, 1, 0), + utc(2026, 3, 31, 1, 0), + ] + ); + for run in &runs { + let local = run.with_timezone(&chrono_tz::Europe::Berlin); + assert_eq!(local.format("%H:%M").to_string(), "03:00", "{local}"); + } + } + + #[test] + fn a_daily_run_stays_at_its_local_time_across_the_autumn_change() { + // Back from CEST to CET on 2026-10-25 at 03:00 local time. + let s = Schedule::parse("0 3 * * *", Some("Europe/Berlin")).unwrap(); + let runs = s.next_runs(utc(2026, 10, 23, 12, 0), 4); + assert_eq!( + runs, + vec![ + utc(2026, 10, 24, 1, 0), // 03:00 CEST + utc(2026, 10, 25, 2, 0), // 03:00 CET, the day of the change + utc(2026, 10, 26, 2, 0), + utc(2026, 10, 27, 2, 0), + ] + ); + for run in &runs { + let local = run.with_timezone(&chrono_tz::Europe::Berlin); + assert_eq!(local.format("%H:%M").to_string(), "03:00", "{local}"); + } + } + + #[test] + fn a_time_in_the_skipped_hour_runs_once_right_after_it() { + // 02:30 does not exist in Berlin on 2026-03-29: the run is at 03:00 + // CEST, once, and the next day at 02:30 again. + let s = Schedule::parse("30 2 * * *", Some("Europe/Berlin")).unwrap(); + assert_eq!( + s.next_runs(utc(2026, 3, 28, 12, 0), 2), + vec![berlin(2026, 3, 29, 3, 0), berlin(2026, 3, 30, 2, 30)] + ); + } + + #[test] + fn a_time_in_the_repeated_hour_runs_once() { + // 02:30 happens twice in Berlin on 2026-10-25 (CEST, then CET): the + // run is at the first, and the next one a day later. + let s = Schedule::parse("30 2 * * *", Some("Europe/Berlin")).unwrap(); + let runs = s.next_runs(utc(2026, 10, 24, 12, 0), 2); + assert_eq!( + runs, + vec![utc(2026, 10, 25, 0, 30), utc(2026, 10, 26, 1, 30)] + ); + } + + #[test] + fn utc_is_the_default_zone_and_the_next_time_is_strictly_later() { + let s = Schedule::parse("0 3 * * 1-5", None).unwrap(); + assert_eq!(s.zone(), Tz::UTC); + // Friday 2026-10-30 03:00 → the next weekday is Monday. + assert_eq!( + s.next_after(utc(2026, 10, 30, 3, 0)), + Some(utc(2026, 11, 2, 3, 0)) + ); + assert_eq!( + s.next_after(utc(2026, 10, 30, 2, 59)), + Some(utc(2026, 10, 30, 3, 0)) + ); + let daily = Schedule::parse("@daily", None).unwrap(); + assert_eq!( + daily.next_after(utc(2026, 10, 30, 3, 0)), + Some(utc(2026, 10, 31, 0, 0)) + ); + } + + #[test] + fn validation() { + let now = utc(2026, 10, 31, 12, 0); + assert!(validate(None, None, None, now).is_ok()); + assert!(validate(Some("0 3 * * *"), Some("Europe/Berlin"), Some(3600), now).is_ok()); + assert!(validate(Some("*/15 * * * *"), None, None, now).is_ok()); + for bad in [ + "", + "0 3 * *", + "0 0 3 * * *", + "0 3 * * * 2026", + "61 * * * *", + "nonsense", + ] { + let err = validate(Some(bad), None, None, now).unwrap_err(); + assert!(err.contains("5 fields"), "{bad:?}: {err}"); + } + let err = validate(Some("0 3 * * *"), Some("Mars/Olympus"), None, now).unwrap_err(); + assert!(err.contains("IANA"), "{err}"); + let err = validate(None, Some("Europe/Berlin"), None, now).unwrap_err(); + assert!(err.contains("needs a schedule"), "{err}"); + for secs in [0, 59, MAX_MAX_RUN_SECONDS + 1] { + assert!(validate(None, None, Some(secs), now).is_err(), "{secs}"); + } + assert!(validate(None, None, Some(MIN_MAX_RUN_SECONDS), now).is_ok()); + } + + #[test] + fn a_clock_past_its_deadline_lets_the_first_step_go() { + let mut past = RunClock::new(Some(std::time::Instant::now())); + assert!(!past.out_of_time()); + assert!(past.out_of_time()); + let mut unlimited = RunClock::new(None); + assert!(!unlimited.out_of_time()); + assert!(!unlimited.out_of_time()); + let mut later = RunClock::new(Some( + std::time::Instant::now() + std::time::Duration::from_secs(3600), + )); + assert!(!later.out_of_time()); + assert!(!later.out_of_time()); + } + + #[test] + fn reads_a_stored_policy_of_any_shape() { + let policy = serde_json::json!({ + "version": 1, + "schedule": "0 3 * * *", + "schedule_timezone": "Europe/Berlin", + "max_run_seconds": 7200, + "rules": [], + }); + let s = of_policy(&policy).unwrap(); + assert_eq!(s.zone(), chrono_tz::Europe::Berlin); + assert_eq!( + max_run_duration(&policy), + Some(std::time::Duration::from_secs(7200)) + ); + let interval = serde_json::json!({"version": 1, "rules": []}); + assert!(of_policy(&interval).is_none()); + assert!(max_run_duration(&interval).is_none()); + } +} diff --git a/crates/core/src/retention/worker.rs b/crates/core/src/retention/worker.rs index 5f6ae2c4..e8e6fa8e 100644 --- a/crates/core/src/retention/worker.rs +++ b/crates/core/src/retention/worker.rs @@ -14,9 +14,15 @@ //! 3. runs the **pure** [`evaluate`] / [`evaluate_proxy`]; //! 4. [`apply`]s the decisions — each in its own tx, decref + `blob_outbox` //! only (no direct S3 delete); -//! 5. records the run summary + schedules the next run on the ledger; +//! 5. records the run summary + schedules the next run on the ledger: at the +//! policy's cron `schedule` (in its `schedule_timezone`), or +//! `min_run_interval_seconds` later ([`crate::retention::schedule`]); //! 6. invalidates the read cache for every affected package. //! +//! A policy's `max_run_seconds` bounds how long a run applies decisions: it +//! stops before the first batch that would start later, and the next run +//! evaluates the repository again and goes on with what is left. +//! //! All S3 deletion flows through the transactional blob outbox; this worker //! never touches S3. A `dry_run_only` policy (or a disabled one) audits/no-ops //! but never mutates. @@ -26,7 +32,7 @@ use std::panic::AssertUnwindSafe; use std::sync::Arc; use std::time::Duration as StdDuration; -use chrono::{Duration, Utc}; +use chrono::{DateTime, Duration, Utc}; use futures_util::FutureExt; use sqlx::PgPool; use tracing::Instrument; @@ -41,6 +47,7 @@ use crate::retention::evaluate::{ plan_package_run, CandidateComponent, PackagePlan, ProxyCandidate, }; use crate::retention::model::{validate_policy, RetentionPolicy, SizeBudgetSummary}; +use crate::retention::{schedule, SelectorError}; /// The shortest interval the worker ever schedules between two automatic runs /// of a repository, whatever the config and the policy say (both accept 0). @@ -189,12 +196,15 @@ impl RetentionWorker { // The size before the run, for the bytes it frees (metrics only: a // failed read records none). let size_before = self.repository_size(&repo).await.ok(); + // The policy's `max_run_seconds`, counted from the claim. + let deadline = + schedule::max_run_duration(&repo.policy).map(|limit| std::time::Instant::now() + limit); // A panic in `process` (a bug, or a panicking `Format`) is recorded // like any other failed run instead of unwinding past `fail_run` and // leaving the ledger row `running` until its lease expires. Nothing is // shared mutably across the unwind: every applied decision committed // in its batch's transaction, and the in-flight one rolled back on drop. - let result = AssertUnwindSafe(self.process(&repo, run_id)) + let result = AssertUnwindSafe(self.process(&repo, run_id, deadline)) .catch_unwind() .await .unwrap_or_else(|panic| Err(WorkerError::Panicked(panic_message(&*panic)))); @@ -212,13 +222,14 @@ impl RetentionWorker { let now = forge_db::db_now(&self.pool) .await .unwrap_or_else(|_| Utc::now()); - let next = now + self.retry_interval(&repo.policy); + let next = self.next_run_at(&repo.policy, now); if let Err(e2) = retention_dao::fail_run( &self.pool, repo.repository_id, Some(run_id), &e.to_string(), next, + Some(&repo.policy), ) .await { @@ -277,6 +288,7 @@ impl RetentionWorker { &self, repo: &retention_dao::ClaimedRepo, run_id: uuid::Uuid, + deadline: Option, ) -> Result { let kind = RepoKind::from(repo.kind.as_str()); // An images format's repositories (Docker) hold images. @@ -285,7 +297,7 @@ impl RetentionWorker { .resolve(&repo.format) .is_some_and(|format| format.stores_images()) { - return self.process_images(repo, kind, run_id).await; + return self.process_images(repo, kind, run_id, deadline).await; } // A files format's repositories (raw) hold files under paths. if self @@ -293,7 +305,7 @@ impl RetentionWorker { .resolve(&repo.format) .is_some_and(|format| format.stores_files()) { - return self.process_files(repo, kind, run_id).await; + return self.process_files(repo, kind, run_id, deadline).await; } // Parse + validate the policy. A malformed or repo-kind-incoherent policy @@ -316,7 +328,7 @@ impl RetentionWorker { // The database's clock: `next_run_at` is compared with it, and the // ages evaluated below are of timestamps it wrote. let now = forge_db::db_now(&self.pool).await?; - let next_run_at = now + self.next_interval(&policy); + let next_run_at = self.next_run_at(&repo.policy, now); // Disabled policy ⇒ no-op run (still advance the schedule). if !policy.enabled { @@ -326,11 +338,22 @@ impl RetentionWorker { Some(run_id), &RunSummary::default(), next_run_at, + Some(&repo.policy), ) .await?; return Ok(RunSummary::default()); } + // The content selectors the rules name. One that is gone (deleted + // since the policy was stored) fails the run without mutating + // anything: a rule must never run with a filter it cannot apply. + let selectors = crate::retention::resolve_selectors(&self.pool, &policy) + .await + .map_err(|e| match e { + SelectorError::Db(e) => WorkerError::Db(e), + other => WorkerError::Policy(other.to_string()), + })?; + // ── Snapshot candidates (no tx held across apply; apply re-locks rows). ── // A proxy's components are cache entries owned by its // `proxy_cached_version` rows, never hosted content, so they are only @@ -352,6 +375,7 @@ impl RetentionWorker { listed: r.listed, created_at: r.created_at, last_downloaded_at: r.last_downloaded_at, + unlisted_at: r.unlisted_at, size_bytes: u64::try_from(r.size_bytes).unwrap_or_default(), }) .collect() @@ -395,11 +419,13 @@ impl RetentionWorker { let evaluation = { let format = Arc::clone(&format); let policy = policy.clone(); + let selectors = selectors.clone(); tokio::task::spawn_blocking(move || { plan_package_run( format.as_ref(), now, &policy, + &selectors, &hosted, &proxy, &tagged_hosted_ids, @@ -446,7 +472,9 @@ impl RetentionWorker { stored_policy: &repo.policy, policy: &policy, format: format.as_ref(), + selectors: &selectors, }), + deadline, }; let mut summary = apply(&ctx, &hosted_decisions, &proxy_decisions).await?; summary.evaluated = evaluated; @@ -466,6 +494,7 @@ impl RetentionWorker { Some(run_id), &summary, next_run_at, + Some(&repo.policy), ) .await? { @@ -494,6 +523,7 @@ impl RetentionWorker { repo: &retention_dao::ClaimedRepo, kind: RepoKind, run_id: uuid::Uuid, + deadline: Option, ) -> Result { use crate::retention::images; let policy: images::ImageRetentionPolicy = serde_json::from_value(repo.policy.clone()) @@ -501,12 +531,7 @@ impl RetentionWorker { images::validate_image_policy(&policy, kind) .map_err(|e| WorkerError::Policy(format!("invalid retention_policy: {e}")))?; let now = forge_db::db_now(&self.pool).await?; - let secs = policy - .min_run_interval_seconds - .min(crate::retention::model::MAX_MIN_RUN_INTERVAL_SECONDS); - let next_run_at = now - + Duration::seconds(i64::try_from(secs).unwrap_or(i64::MAX)) - .max(self.config.default_run_interval); + let next_run_at = self.next_run_at(&repo.policy, now); let mut summary = RunSummary::default(); if policy.enabled { let (decisions, evaluated, budget) = @@ -524,6 +549,7 @@ impl RetentionWorker { self.config.delete_grace, policy.dry_run_only, Some(&fence), + deadline, ) .await?; summary.evaluated = evaluated; @@ -542,6 +568,7 @@ impl RetentionWorker { Some(run_id), &summary, next_run_at, + Some(&repo.policy), ) .await? { @@ -559,6 +586,7 @@ impl RetentionWorker { repo: &retention_dao::ClaimedRepo, kind: RepoKind, run_id: uuid::Uuid, + deadline: Option, ) -> Result { use crate::retention::files; let policy: files::FileRetentionPolicy = serde_json::from_value(repo.policy.clone()) @@ -566,12 +594,7 @@ impl RetentionWorker { files::validate_file_policy(&policy, kind) .map_err(|e| WorkerError::Policy(format!("invalid retention_policy: {e}")))?; let now = forge_db::db_now(&self.pool).await?; - let secs = policy - .min_run_interval_seconds - .min(crate::retention::model::MAX_MIN_RUN_INTERVAL_SECONDS); - let next_run_at = now - + Duration::seconds(i64::try_from(secs).unwrap_or(i64::MAX)) - .max(self.config.default_run_interval); + let next_run_at = self.next_run_at(&repo.policy, now); let mut summary = RunSummary::default(); if policy.enabled { let (decisions, evaluated, budget) = @@ -589,6 +612,7 @@ impl RetentionWorker { self.config.delete_grace, policy.dry_run_only, Some(&fence), + deadline, ) .await?; summary.evaluated = evaluated; @@ -607,6 +631,7 @@ impl RetentionWorker { Some(run_id), &summary, next_run_at, + Some(&repo.policy), ) .await? { @@ -619,24 +644,24 @@ impl RetentionWorker { Ok(summary) } - /// The next-run interval for a policy: the larger of the repo's - /// `min_run_interval_seconds` and the global default (which [`Self::new`] - /// raised to at least [`MIN_RUN_INTERVAL_SECONDS`]). - fn next_interval(&self, policy: &RetentionPolicy) -> Duration { - // Clamped defensively: validation already bounds the policy value, and - // an out-of-range interval must never panic the worker loop. - let secs = policy - .min_run_interval_seconds - .min(crate::retention::model::MAX_MIN_RUN_INTERVAL_SECONDS); - let per_repo = Duration::seconds(i64::try_from(secs).unwrap_or(i64::MAX)); - per_repo.max(self.config.default_run_interval) + /// When a repository whose run started at `now` runs next, finished or + /// failed: at the first time of its policy's `schedule` that is at least + /// the floor (the global default, which [`Self::new`] raised to at least + /// [`MIN_RUN_INTERVAL_SECONDS`]) after `now`; without a schedule, after + /// [`Self::interval`]. Read from the raw document of any policy shape + /// (which may be what failed to parse), so a broken policy is not + /// retried every minute either. + fn next_run_at(&self, policy: &serde_json::Value, now: DateTime) -> DateTime { + let floor = self.config.default_run_interval; + schedule::of_policy(policy) + .and_then(|schedule| schedule.next_after(now + floor - Duration::seconds(1))) + .unwrap_or_else(|| now + self.interval(policy)) } - /// When a failed run is tried again: after its policy's - /// `min_run_interval_seconds`, read from the raw document (which may be - /// what failed to parse) or the default when it names none, and no sooner - /// than the floor, so a broken policy is not retried every minute. - fn retry_interval(&self, policy: &serde_json::Value) -> Duration { + /// The next-run interval for a policy: the larger of its + /// `min_run_interval_seconds` (the default when it names none) and the + /// floor. + fn interval(&self, policy: &serde_json::Value) -> Duration { let secs = policy .get("min_run_interval_seconds") .and_then(serde_json::Value::as_u64) diff --git a/crates/core/tests/retention_apply.rs b/crates/core/tests/retention_apply.rs index cfca5966..98120578 100644 --- a/crates/core/tests/retention_apply.rs +++ b/crates/core/tests/retention_apply.rs @@ -26,12 +26,7 @@ const GRACE: Duration = Duration::seconds(60); fn reason() -> DecisionReason { DecisionReason { older_than_days: Some(30), - not_downloaded_for_days: None, - not_pulled_for_days: None, - size_budget: None, - target_bytes: None, - size_bytes: None, - last_used_at: None, + ..DecisionReason::default() } } @@ -120,6 +115,7 @@ fn ctx<'a>(pool: &'a PgPool, repo_id: i64, name: &'a str, run_id: uuid::Uuid) -> dry_run: false, evaluated_at: None, fence: None, + deadline: None, } } @@ -424,13 +420,8 @@ fn evict(cached_version_id: i64) -> ProxyDecision { version: "1.0.0".into(), rule_id: "evict".into(), reason: DecisionReason { - older_than_days: None, - not_downloaded_for_days: None, not_pulled_for_days: Some(30), - size_budget: None, - target_bytes: None, - size_bytes: None, - last_used_at: None, + ..DecisionReason::default() }, } } @@ -567,3 +558,86 @@ async fn proxy_eviction_of_a_row_without_cached_bytes_is_skipped() { assert_eq!(rows.len(), 1); assert_eq!(rows[0].0, "skipped"); } + +/// `max_run_seconds`: a run past its deadline stops before its next batch +/// and leaves the rest `deferred`; the next run (a re-evaluation, here the +/// decisions not applied yet) goes on where it stopped. The first batch of +/// a run always goes, so each run makes progress however short its window. +#[tokio::test] +async fn a_run_past_its_deadline_stops_and_the_next_one_resumes() { + let pool = common::pool().await; + let n = common::unique(); + let repo = repository_dao::insert_repository(&pool, NewRepository::hosted(format!("d-{n}"))) + .await + .unwrap(); + // Three packages: one batch each. + let mut decisions = Vec::new(); + for name in ["alpha", "beta", "gamma"] { + let sha = seed_blob(&pool, common::unique(), 1).await; + let id = seed_component(&pool, repo.id, name, "1.0.0", &sha).await; + decisions.push(Decision { + component_id: id, + package_name: name.into(), + version: "1.0.0".into(), + action: Action::HardDelete, + rule_id: "r".into(), + reason: reason(), + }); + } + let remaining = |pool: &PgPool| { + let pool = pool.clone(); + let repo_id = repo.id; + async move { + sqlx::query_scalar::<_, String>( + "SELECT name FROM component WHERE repository_id = $1 ORDER BY name", + ) + .bind(repo_id) + .fetch_all(&pool) + .await + .unwrap() + } + }; + + // The deadline has passed already: the first batch goes, the rest waits. + let mut left = decisions.clone(); + let mut runs = 0; + while !left.is_empty() { + runs += 1; + let ctx = ApplyContext { + deadline: Some(std::time::Instant::now()), + ..ctx(&pool, repo.id, &repo.name, uuid::Uuid::new_v4()) + }; + let summary = apply(&ctx, &left, &[]).await.expect("apply"); + assert_eq!(summary.deleted, 1, "run {runs}"); + assert_eq!(summary.deferred, left.len() as u64 - 1, "run {runs}"); + assert_eq!(summary.skipped, 0); + left.remove(0); + let expected: Vec = left.iter().map(|d| d.package_name.clone()).collect(); + assert_eq!(remaining(&pool).await, expected, "run {runs}"); + } + assert_eq!(runs, 3); + + // Without a deadline in reach, one run applies everything. + let sha = seed_blob(&pool, common::unique(), 1).await; + let id = seed_component(&pool, repo.id, "delta", "1.0.0", &sha).await; + let sha = seed_blob(&pool, common::unique(), 1).await; + let id2 = seed_component(&pool, repo.id, "epsilon", "1.0.0", &sha).await; + let all: Vec = [(id, "delta"), (id2, "epsilon")] + .into_iter() + .map(|(component_id, name)| Decision { + component_id, + package_name: name.into(), + version: "1.0.0".into(), + action: Action::HardDelete, + rule_id: "r".into(), + reason: reason(), + }) + .collect(); + let ctx = ApplyContext { + deadline: Some(std::time::Instant::now() + std::time::Duration::from_secs(3600)), + ..ctx(&pool, repo.id, &repo.name, uuid::Uuid::new_v4()) + }; + let summary = apply(&ctx, &all, &[]).await.expect("apply"); + assert_eq!((summary.deleted, summary.deferred), (2, 0)); + assert!(remaining(&pool).await.is_empty()); +} diff --git a/crates/core/tests/retention_worker.rs b/crates/core/tests/retention_worker.rs index 59ea8ba4..4820df97 100644 --- a/crates/core/tests/retention_worker.rs +++ b/crates/core/tests/retention_worker.rs @@ -20,6 +20,7 @@ use std::time::Duration as StdDuration; use chrono::{DateTime, Duration, Utc}; use forge_core::repo::NoopCacheInvalidator; +use forge_core::retention::evaluate::Selectors; use forge_core::retention::worker::{ RetentionConfig, RetentionWorker, WorkerError, MIN_RUN_INTERVAL_SECONDS, }; @@ -881,12 +882,18 @@ async fn claim_and_evaluate( listed: r.listed, created_at: r.created_at, last_downloaded_at: r.last_downloaded_at, + unlisted_at: r.unlisted_at, size_bytes: u64::try_from(r.size_bytes).unwrap_or_default(), }, ) .collect(); - let decisions = - forge_core::retention::evaluate::evaluate(&nuget(), Utc::now(), &policy, &candidates); + let decisions = forge_core::retention::evaluate::evaluate( + &nuget(), + Utc::now(), + &policy, + &Selectors::none(), + &candidates, + ); (claimed, run_id, policy, decisions) } @@ -910,7 +917,9 @@ async fn apply_fenced( stored_policy: &claimed.policy, policy, format: &format, + selectors: &Selectors::none(), }), + deadline: None, }; forge_core::retention::apply::apply(&ctx, decisions, &[]) .await @@ -1291,14 +1300,20 @@ async fn proxy_size_budget_rechecks_keep_last_n_after_a_newer_version_is_unliste size_bytes: u64::try_from(r.size_bytes).unwrap_or_default(), }) .collect(); - let ordinary = - forge_core::retention::evaluate::evaluate_proxy(&format, now, &policy, &candidates); + let ordinary = forge_core::retention::evaluate::evaluate_proxy( + &format, + now, + &policy, + &Selectors::none(), + &candidates, + ); let size_before = retention_dao::package_repository_size(&pool, repo.id) .await .unwrap(); let mut plan = forge_core::retention::evaluate::evaluate_size_budget( &format, &policy, + &Selectors::none(), &[], &candidates, size_before, @@ -1339,7 +1354,9 @@ async fn proxy_size_budget_rechecks_keep_last_n_after_a_newer_version_is_unliste stored_policy: &claimed.policy, policy: &policy, format: &format, + selectors: &Selectors::none(), }), + deadline: None, }; let summary = forge_core::retention::apply::apply(&ctx, &[], &decisions) .await @@ -1367,3 +1384,186 @@ async fn proxy_size_budget_rechecks_keep_last_n_after_a_newer_version_is_unliste assert!(component_exists(&pool, repo.id, "lib", "2.0.0").await); assert!(component_exists(&pool, repo.id, "lib", "3.0.0").await); } + +/// #185: a policy with a cron schedule runs next at the schedule's next +/// time in its time zone, not `min_run_interval_seconds` later. +#[tokio::test] +async fn a_scheduled_policy_runs_next_at_its_schedule() { + let pool = pool().await; + let n = unique(); + let repo = + repository_dao::insert_repository(&pool, NewRepository::hosted(format!("rw-cron-{n}"))) + .await + .unwrap(); + arm( + &pool, + repo.id, + &json!({ + "version": 1, + "schedule": "0 3 * * *", + "schedule_timezone": "Europe/Berlin", + "max_run_seconds": 3600, + "rules": [{ "id": "sweep", "action": "hard_delete", "criteria": { "older_than_days": 30 } }] + }), + ) + .await; + + let before = Utc::now(); + let summary = worker(&pool, config()) + .tick_repo(repo.id) + .await + .unwrap() + .unwrap(); + assert_eq!(summary.deferred, 0, "the run had all the time it needed"); + let run = retention_dao::get_run(&pool, repo.id) + .await + .unwrap() + .unwrap(); + let local = run.next_run_at.with_timezone(&chrono_tz::Europe::Berlin); + assert_eq!(local.format("%H:%M:%S").to_string(), "03:00:00", "{local}"); + assert!(run.next_run_at > before); + assert!( + run.next_run_at <= before + Duration::hours(25), + "{}", + run.next_run_at + ); + let expected = + forge_core::retention::schedule::Schedule::parse("0 3 * * *", Some("Europe/Berlin")) + .unwrap() + .next_after(before) + .unwrap(); + assert_eq!(run.next_run_at, expected); +} + +/// #185: `unlisted_for_days` deletes versions unlisted long enough, by the +/// time the unlisting recorded; `prerelease_superseded_for_days` deletes a +/// prerelease once its release has been out long enough. +#[tokio::test] +async fn unlisted_and_superseded_versions_go_after_their_days() { + let pool = pool().await; + let n = unique(); + let repo = + repository_dao::insert_repository(&pool, NewRepository::hosted(format!("rw-two-{n}"))) + .await + .unwrap(); + for (i, (name, version, days)) in [ + ("old", "1.0.0", 400), + ("old", "1.1.0", 400), + ("old", "1.2.0", 400), + ("app", "2.0.0-ci.1", 60), + ("app", "2.0.0", 30), + ("app", "2.1.0-ci.1", 5), + ] + .into_iter() + .enumerate() + { + let sha = seed_blob(&pool, n * 100 + i as u64, 1).await; + seed_component(&pool, repo.id, name, version, &sha, days).await; + } + // 1.0.0 was unlisted 100 days ago, 1.1.0 yesterday; 1.2.0 is listed. + for version in ["1.0.0", "1.1.0"] { + component_dao::set_listed(&pool, repo.id, "old", version, false) + .await + .unwrap(); + } + sqlx::query( + "UPDATE component SET unlisted_at = now() - interval '100 days' \ + WHERE repository_id = $1 AND name = 'old' AND version = '1.0.0'", + ) + .bind(repo.id) + .execute(&pool) + .await + .unwrap(); + arm( + &pool, + repo.id, + &json!({ + "version": 1, + "rules": [ + { "id": "unlisted", "action": "hard_delete", "criteria": { "unlisted_for_days": 90 } }, + { "id": "ci", "action": "hard_delete", "release_type": "prereleases", + "version_pattern": "*-ci.*", + "criteria": { "prerelease_superseded_for_days": 14 } } + ] + }), + ) + .await; + + let summary = worker(&pool, config()) + .tick_repo(repo.id) + .await + .unwrap() + .unwrap(); + assert_eq!(summary.deleted, 2); + assert!(!component_exists(&pool, repo.id, "old", "1.0.0").await); + assert!(component_exists(&pool, repo.id, "old", "1.1.0").await); + assert!(component_exists(&pool, repo.id, "old", "1.2.0").await); + assert!(!component_exists(&pool, repo.id, "app", "2.0.0-ci.1").await); + assert!(component_exists(&pool, repo.id, "app", "2.0.0").await); + assert!(component_exists(&pool, repo.id, "app", "2.1.0-ci.1").await); + let reasons: Vec<(String, Value)> = sqlx::query_as( + "SELECT rule_id, reason FROM retention_audit WHERE repository_id = $1 ORDER BY rule_id", + ) + .bind(repo.id) + .fetch_all(&pool) + .await + .unwrap(); + assert_eq!(reasons.len(), 2); + assert_eq!(reasons[0].0, "ci"); + assert_eq!(reasons[0].1["superseded_by"], "2.0.0"); + assert_eq!(reasons[1].0, "unlisted"); + assert_eq!(reasons[1].1["unlisted_for_days"], 90); +} + +/// #185: a rule's content selector is read on every run. While it is +/// missing the run fails without touching anything; once it exists, the +/// rule covers the packages it admits only. +#[tokio::test] +async fn a_rules_selector_filters_packages_and_a_missing_one_fails_the_run() { + let pool = pool().await; + let n = unique(); + let repo = + repository_dao::insert_repository(&pool, NewRepository::hosted(format!("rw-sel-{n}"))) + .await + .unwrap(); + for (i, name) in ["acme.core", "acme.keep", "other"].into_iter().enumerate() { + let sha = seed_blob(&pool, n * 100 + i as u64, 1).await; + seed_component(&pool, repo.id, name, "1.0.0", &sha, 100).await; + } + let selector = format!("rw-acme-{n}"); + arm( + &pool, + repo.id, + &json!({ + "version": 1, + "rules": [{ "id": "acme", "action": "hard_delete", "selector": selector, + "criteria": { "older_than_days": 30 } }] + }), + ) + .await; + + let w = worker(&pool, config()); + let err = w.tick_repo(repo.id).await.unwrap_err(); + assert!( + matches!(&err, WorkerError::Policy(m) if m.contains("does not exist")), + "{err}" + ); + for name in ["acme.core", "acme.keep", "other"] { + assert!(component_exists(&pool, repo.id, name, "1.0.0").await); + } + + forge_db::authz_dao::insert_content_selector( + &pool, + &selector, + &["acme.*".to_string()], + &["acme.keep".to_string()], + ) + .await + .unwrap(); + retention_dao::schedule_now(&pool, repo.id).await.unwrap(); + let summary = w.tick_repo(repo.id).await.unwrap().unwrap(); + assert_eq!(summary.deleted, 1); + assert!(!component_exists(&pool, repo.id, "acme.core", "1.0.0").await); + assert!(component_exists(&pool, repo.id, "acme.keep", "1.0.0").await); + assert!(component_exists(&pool, repo.id, "other", "1.0.0").await); +} diff --git a/crates/db/src/retention_dao.rs b/crates/db/src/retention_dao.rs index cea1d1d4..7bcba59b 100644 --- a/crates/db/src/retention_dao.rs +++ b/crates/db/src/retention_dao.rs @@ -51,6 +51,9 @@ pub struct CandidateRow { pub created_at: DateTime, /// Denormalized last-download timestamp (`component.last_downloaded_at`). pub last_downloaded_at: Option>, + /// When the version was unlisted (`component.unlisted_at`), `None` while + /// it is listed. + pub unlisted_at: Option>, /// Sum of the component's asset sizes, in bytes. pub size_bytes: i64, } @@ -183,6 +186,10 @@ pub struct RunSummary { pub blobs_enqueued: u64, /// Decisions skipped at apply time (row changed under us). pub skipped: u64, + /// Decisions left for the next run: the run stopped at its policy's + /// `max_run_seconds`, and the next run evaluates them again. + #[serde(default, skip_serializing_if = "is_zero")] + pub deferred: u64, /// Logical size before a size-budget run. #[serde(default, skip_serializing_if = "Option::is_none")] pub size_before: Option, @@ -197,6 +204,10 @@ pub struct RunSummary { pub target_reached: Option, } +fn is_zero(n: &u64) -> bool { + *n == 0 +} + /// Ensure a `retention_run` ledger row exists for `repository_id` (idempotent). /// /// Created lazily the first time a repo with a policy is seen; `next_run_at` @@ -344,12 +355,19 @@ where /// With `run_id`, the update is **fenced** to the run that still owns the /// lease: a worker whose lease expired and was re-claimed by another instance /// cannot clear the newer owner's lease. Returns whether the row was updated. +/// +/// `policy` is the stored policy the run evaluated. When the repository's +/// policy is another one by now (it was changed while the run worked), +/// `next_run_at` is left as the write of the new policy set it +/// ([`write_policy_at`]), so a schedule saved mid-run is not overridden by +/// the old policy's next time. `None` always sets it. pub async fn finish_run<'e, E>( executor: E, repository_id: i64, run_id: Option, summary: &RunSummary, next_run_at: DateTime, + policy: Option<&JsonValue>, ) -> sqlx::Result where E: PgExecutor<'e>, @@ -362,7 +380,12 @@ where finished_at = now(), locked_until = NULL, current_run_id = NULL, - next_run_at = $2, + next_run_at = CASE + WHEN $5::jsonb IS NULL + OR (SELECT r.retention_policy FROM repository r WHERE r.id = $1) = $5::jsonb + THEN $2 + ELSE retention_run.next_run_at + END, last_run_summary = $3 WHERE repository_id = $1 AND ($4::uuid IS NULL OR current_run_id = $4) @@ -371,6 +394,7 @@ where next_run_at, summary_json, run_id, + policy, ) .execute(executor) .await?; @@ -378,13 +402,15 @@ where } /// Record a run-level error and release the lease so the repo is retried. -/// Fenced to `run_id` like [`finish_run`]. +/// Fenced to `run_id`, and keeping the next run time of a policy changed +/// meanwhile, like [`finish_run`]. pub async fn fail_run<'e, E>( executor: E, repository_id: i64, run_id: Option, error: &str, next_run_at: DateTime, + policy: Option<&JsonValue>, ) -> sqlx::Result where E: PgExecutor<'e>, @@ -396,7 +422,12 @@ where finished_at = now(), locked_until = NULL, current_run_id = NULL, - next_run_at = $2, + next_run_at = CASE + WHEN $5::jsonb IS NULL + OR (SELECT r.retention_policy FROM repository r WHERE r.id = $1) = $5::jsonb + THEN $2 + ELSE retention_run.next_run_at + END, last_error = $3 WHERE repository_id = $1 AND ($4::uuid IS NULL OR current_run_id = $4) @@ -405,6 +436,7 @@ where next_run_at, error, run_id, + policy, ) .execute(executor) .await?; @@ -481,6 +513,7 @@ where c.listed AS listed, c.created_at AS created_at, c.last_downloaded_at AS last_downloaded_at, + c.unlisted_at AS unlisted_at, COALESCE((SELECT SUM(a.size) FROM asset a WHERE a.component_id = c.id), 0)::bigint AS "size_bytes!", ( @@ -512,6 +545,7 @@ where listed: r.listed, created_at: r.created_at, last_downloaded_at: r.last_downloaded_at, + unlisted_at: r.unlisted_at, size_bytes: r.size_bytes, }, sha, @@ -687,6 +721,19 @@ pub async fn write_policy( pool: &PgPool, repository_id: i64, policy: Option<&JsonValue>, +) -> sqlx::Result { + write_policy_at(pool, repository_id, policy, None).await +} + +/// [`write_policy`], and with `next_run_at`, set the ledger's next run to it +/// (a policy with a schedule runs at the schedule's next time, not as soon +/// as it is saved). `None` leaves an existing ledger row's next run as it +/// is, and a new row due at once. +pub async fn write_policy_at( + pool: &PgPool, + repository_id: i64, + policy: Option<&JsonValue>, + next_run_at: Option>, ) -> sqlx::Result { let mut tx = crate::begin(pool).await?; let res = sqlx::query!( @@ -708,11 +755,42 @@ pub async fn write_policy( if policy.is_some() { upsert_retention_run(&mut *tx, repository_id).await?; + if let Some(next_run_at) = next_run_at { + sqlx::query!( + r#"UPDATE retention_run SET next_run_at = $2 WHERE repository_id = $1"#, + repository_id, + next_run_at, + ) + .execute(&mut *tx) + .await?; + } } tx.commit().await?; Ok(true) } +/// The repositories whose retention policy names the content selector +/// `name` in a rule (`"selector": name`), by name: the selector cannot be +/// deleted while they do. +pub async fn repositories_using_selector(pool: &PgPool, name: &str) -> sqlx::Result> { + sqlx::query_scalar!( + r#" + SELECT r.name + FROM repository r + WHERE r.retention_policy IS NOT NULL + AND jsonb_path_exists( + r.retention_policy, + '$.rules[*] ? (@.selector == $name)', + jsonb_build_object('name', $1::text) + ) + ORDER BY r.name + "#, + name, + ) + .fetch_all(pool) + .await +} + /// Fetch the current `retention_run` state for a repository (for the admin GET /// surface). Returned as a typed tuple of the columns the API needs. pub async fn get_run(pool: &PgPool, repository_id: i64) -> sqlx::Result> { @@ -858,6 +936,7 @@ pub async fn lock_package_candidates( c.listed AS listed, c.created_at AS created_at, c.last_downloaded_at AS last_downloaded_at, + c.unlisted_at AS unlisted_at, EXISTS ( SELECT 1 FROM asset a WHERE a.component_id = c.id AND (a.path = $3 OR a.path LIKE '%.nupkg') @@ -885,6 +964,7 @@ pub async fn lock_package_candidates( listed: r.listed, created_at: r.created_at, last_downloaded_at: r.last_downloaded_at, + unlisted_at: r.unlisted_at, size_bytes: r.size_bytes, }) .collect()) diff --git a/crates/db/tests/migrations.rs b/crates/db/tests/migrations.rs index 30a0bd96..1257048a 100644 --- a/crates/db/tests/migrations.rs +++ b/crates/db/tests/migrations.rs @@ -954,6 +954,80 @@ async fn an_import_of_a_nexus_repository_survives_the_rename() { ); } +/// Versions unlisted before `component.unlisted_at` existed (0011) take +/// their `updated_at` (every unlist path set it to that moment), listed ones +/// none; from then on the trigger records each unlisting. +#[tokio::test] +async fn versions_unlisted_before_the_upgrade_count_from_their_last_change() { + let (server, _container) = server().await; + let url = fresh_database(&server, "unlisted_at").await; + let pool = forge_db::connect(&url).await.expect("connect"); + sqlx::migrate!("../../migrations") + .run_to(7, &pool) + .await + .expect("migrate to 0007"); + + let repo = repository_dao::insert_repository(&pool, NewRepository::hosted("nuget")) + .await + .unwrap(); + let insert = |version: &'static str, listed: bool, changed_days_ago: i32| { + let pool = pool.clone(); + async move { + sqlx::query_scalar::<_, i64>( + "INSERT INTO component (repository_id, name, original_id, version, \ + version_original, listed, created_at, updated_at) \ + VALUES ($1, 'lib', 'Lib', $2, $2, $3, now() - interval '400 days', \ + now() - make_interval(days => $4)) RETURNING id", + ) + .bind(repo.id) + .bind(version) + .bind(listed) + .bind(changed_days_ago) + .fetch_one(&pool) + .await + .expect("insert a version") + } + }; + let unlisted = insert("1.0.0", false, 120).await; + let listed = insert("2.0.0", true, 30).await; + + forge_db::migrate(&pool).await.expect("migrate the rest"); + + let row = |id: i64| { + let pool = pool.clone(); + async move { + sqlx::query_as::< + _, + ( + Option>, + chrono::DateTime, + ), + >("SELECT unlisted_at, updated_at FROM component WHERE id = $1") + .bind(id) + .fetch_one(&pool) + .await + .unwrap() + } + }; + let (unlisted_at, updated_at) = row(unlisted).await; + assert_eq!( + unlisted_at, + Some(updated_at), + "the last change is the default" + ); + assert_eq!(row(listed).await.0, None); + + // From now on, the trigger. + sqlx::query("UPDATE component SET listed = false WHERE id = $1") + .bind(listed) + .execute(&pool) + .await + .unwrap(); + let (now_unlisted, _) = row(listed).await; + let age = chrono::Utc::now() - now_unlisted.expect("unlisting records the time"); + assert!(age < chrono::Duration::minutes(5), "{age}"); +} + /// A Postgres server for one test: a throwaway Postgres 18 container matching /// docker-compose (db/user/pw = forge), or, with FORGE_TEST_DATABASE_URL (no /// Docker), that server. Returns the URL of a database on it, and the diff --git a/crates/db/tests/retention.rs b/crates/db/tests/retention.rs index 6e909dae..d383386d 100644 --- a/crates/db/tests/retention.rs +++ b/crates/db/tests/retention.rs @@ -193,7 +193,7 @@ async fn finished_run_advances_next_run_and_goes_idle() { blobs_enqueued: 2, ..Default::default() }; - retention_dao::finish_run(&pool, repo_id, None, &summary, next) + retention_dao::finish_run(&pool, repo_id, None, &summary, next, None) .await .expect("finish"); @@ -227,6 +227,7 @@ async fn running_repo_is_not_claimed_until_due_and_idle() { None, &RunSummary::default(), now + Duration::seconds(3600), + None, ) .await .unwrap(); @@ -516,6 +517,7 @@ async fn schedule_now_makes_repo_immediately_due() { None, &RunSummary::default(), now + Duration::seconds(99999), + None, ) .await .unwrap(); @@ -571,11 +573,12 @@ async fn run_completion_and_renewal_are_fenced_to_the_lease_owner() { Some(stranger), &RunSummary::default(), now + Duration::seconds(3600), + None, ) .await .unwrap()); assert!( - !retention_dao::fail_run(&pool, repo_id, Some(stranger), "x", now) + !retention_dao::fail_run(&pool, repo_id, Some(stranger), "x", now, None) .await .unwrap() ); @@ -591,6 +594,7 @@ async fn run_completion_and_renewal_are_fenced_to_the_lease_owner() { Some(run_id), &RunSummary::default(), now + Duration::seconds(3600), + None, ) .await .unwrap()); @@ -600,3 +604,234 @@ async fn run_completion_and_renewal_are_fenced_to_the_lease_owner() { .unwrap(); assert_eq!(run.state, "idle"); } + +/// `component.unlisted_at` (for `unlisted_for_days`), kept by a trigger on +/// every path: set when a version is unlisted or arrives unlisted, kept while +/// it stays unlisted, cleared when it is listed again. +#[tokio::test] +async fn unlisting_records_when_on_every_path() { + let pool = common::pool().await; + let n = common::unique(); + let repo = repository_dao::insert_repository(&pool, NewRepository::hosted(format!("unl-{n}"))) + .await + .unwrap(); + let insert = |version: &'static str, listed: bool| { + let pool = pool.clone(); + async move { + component_dao::insert_component( + &pool, + NewComponent { + repository_id: repo.id, + name: "lib".into(), + original_id: "Lib".into(), + version: version.into(), + version_original: version.into(), + listed, + }, + ) + .await + .unwrap() + .id + } + }; + let unlisted_at = |id: i64| { + let pool = pool.clone(); + async move { + sqlx::query_scalar::<_, Option>>( + "SELECT unlisted_at FROM component WHERE id = $1", + ) + .bind(id) + .fetch_one(&pool) + .await + .unwrap() + } + }; + + let listed = insert("1.0.0", true).await; + assert_eq!(unlisted_at(listed).await, None); + // A version pushed unlisted. + let pushed_unlisted = insert("2.0.0", false).await; + assert!(unlisted_at(pushed_unlisted).await.is_some()); + + // A client's unlist (NuGet's DELETE, npm) and its relist. + assert!( + component_dao::set_listed(&pool, repo.id, "lib", "1.0.0", false) + .await + .unwrap() + ); + let first = unlisted_at(listed) + .await + .expect("unlisting records the time"); + // Backdated, as a test or an operator might; unlisting again keeps it. + sqlx::query("UPDATE component SET unlisted_at = now() - interval '40 days' WHERE id = $1") + .bind(listed) + .execute(&pool) + .await + .unwrap(); + let backdated = unlisted_at(listed).await.unwrap(); + assert!(backdated < first - Duration::days(39)); + component_dao::set_listed(&pool, repo.id, "lib", "1.0.0", false) + .await + .unwrap(); + assert_eq!(unlisted_at(listed).await, Some(backdated)); + component_dao::set_listed(&pool, repo.id, "lib", "1.0.0", true) + .await + .unwrap(); + assert_eq!(unlisted_at(listed).await, None); + + // Retention's own unlist goes through the same column; the candidates + // carry it. + sqlx::query("UPDATE component SET listed = false, updated_at = now() WHERE id = $1") + .bind(listed) + .execute(&pool) + .await + .unwrap(); + let sha = format!("{:064x}", n); + blob_dao::upsert_pending(&pool, &sha, 10, &format!("k/{sha}")) + .await + .unwrap(); + asset_dao::insert_asset( + &pool, + NewAsset { + component_id: listed, + path: "lib.1.0.0.nupkg".into(), + content_type: "application/octet-stream".into(), + blob_sha256: sha, + size: 10, + sha512_base64: None, + }, + ) + .await + .unwrap(); + let cands = retention_dao::load_candidate_components(&pool, repo.id, "package.nupkg") + .await + .unwrap(); + assert_eq!(cands.len(), 1); + assert!(!cands[0].0.listed); + assert_eq!(cands[0].0.unlisted_at, unlisted_at(listed).await); + assert!(cands[0].0.unlisted_at.is_some()); +} + +/// A policy with a schedule sets the ledger's next run when it is saved; +/// a run that finishes after its policy was changed keeps the next run the +/// change set, instead of the old policy's. +#[tokio::test] +async fn a_saved_schedule_sets_the_next_run_and_a_stale_run_keeps_it() { + let pool = common::pool().await; + let repo_id = armed_hosted_repo(&pool, common::unique()).await; + let old_policy = hosted_policy_json(); + + let window = Utc::now() + Duration::days(1); + let mut scheduled = hosted_policy_json(); + scheduled["schedule"] = json!("0 3 * * *"); + assert!( + retention_dao::write_policy_at(&pool, repo_id, Some(&scheduled), Some(window)) + .await + .unwrap() + ); + let run = retention_dao::get_run(&pool, repo_id) + .await + .unwrap() + .unwrap(); + assert!((run.next_run_at - window).num_milliseconds().abs() < 1); + + // A run of the old policy ends: its next time is not the one to keep. + let stale_next = Utc::now() + Duration::hours(1); + assert!(retention_dao::finish_run( + &pool, + repo_id, + None, + &RunSummary::default(), + stale_next, + Some(&old_policy), + ) + .await + .unwrap()); + let run = retention_dao::get_run(&pool, repo_id) + .await + .unwrap() + .unwrap(); + assert!((run.next_run_at - window).num_milliseconds().abs() < 1); + assert_eq!(run.state, "idle"); + assert!( + retention_dao::fail_run(&pool, repo_id, None, "x", stale_next, Some(&old_policy)) + .await + .unwrap() + ); + let run = retention_dao::get_run(&pool, repo_id) + .await + .unwrap() + .unwrap(); + assert!((run.next_run_at - window).num_milliseconds().abs() < 1); + + // A run of the stored policy sets its next run. + let stored = retention_dao::read_policy(&pool, repo_id) + .await + .unwrap() + .unwrap(); + retention_dao::finish_run( + &pool, + repo_id, + None, + &RunSummary::default(), + stale_next, + Some(&stored), + ) + .await + .unwrap(); + let run = retention_dao::get_run(&pool, repo_id) + .await + .unwrap() + .unwrap(); + assert!((run.next_run_at - stale_next).num_milliseconds().abs() < 1); +} + +/// A run that stopped at its time limit records how many decisions it left. +#[tokio::test] +async fn the_summary_records_deferred_decisions_only_when_there_are_some() { + let summary = RunSummary { + deleted: 3, + deferred: 7, + ..RunSummary::default() + }; + let json = serde_json::to_value(&summary).unwrap(); + assert_eq!(json["deferred"], 7); + let none = serde_json::to_value(RunSummary::default()).unwrap(); + assert!(none.get("deferred").is_none()); + // A summary recorded before the field existed reads as none deferred. + let back: RunSummary = serde_json::from_value(json!({ + "evaluated": 1, "unlisted": 0, "deleted": 0, "blobs_enqueued": 0, "skipped": 0 + })) + .unwrap(); + assert_eq!(back.deferred, 0); +} + +/// The repositories whose retention rules name a content selector. +#[tokio::test] +async fn repositories_using_a_selector_are_found_by_their_rules() { + let pool = common::pool().await; + let n = common::unique(); + let selector = format!("sel-{n}"); + let using = armed_hosted_repo(&pool, n).await; + let mut policy = hosted_policy_json(); + policy["rules"][0]["selector"] = json!(selector); + retention_dao::write_policy(&pool, using, Some(&policy)) + .await + .unwrap(); + let other = armed_hosted_repo(&pool, common::unique()).await; + let mut unrelated = hosted_policy_json(); + unrelated["rules"][0]["selector"] = json!(format!("{selector}-other")); + retention_dao::write_policy(&pool, other, Some(&unrelated)) + .await + .unwrap(); + let names = retention_dao::repositories_using_selector(&pool, &selector) + .await + .unwrap(); + assert_eq!(names, vec![format!("ret-{n}")]); + assert!( + retention_dao::repositories_using_selector(&pool, "nobody-uses-this") + .await + .unwrap() + .is_empty() + ); +} diff --git a/crates/plugin-api/src/lib.rs b/crates/plugin-api/src/lib.rs index 7b6a188d..705ab7fc 100644 --- a/crates/plugin-api/src/lib.rs +++ b/crates/plugin-api/src/lib.rs @@ -88,6 +88,34 @@ pub trait FormatPlugin: Send + Sync + 'static { a.cmp(b) } + /// Retention: the class of a normalized version (release, prerelease + /// or snapshot), which a rule's `release_type` filters on and + /// `prerelease_superseded_for_days` reads. The default is SemVer's rule: + /// a version with a `-` is a prerelease, any other a release. A format + /// whose versions say it otherwise (Maven: `33.0.0-jre` is a release, + /// `1.0-SNAPSHOT` a snapshot) overrides it. + fn version_class(&self, version: &str) -> VersionClass { + if version.contains('-') { + VersionClass::Prerelease + } else { + VersionClass::Release + } + } + + /// Retention: the partition of a package's versions within which the + /// keep-N rules count (`keep_last_n`, `keep_n_most_downloaded`, + /// `max_versions_per_package`), so that a format defines what "the + /// newest N" means. `None` (the default): the package is one partition, + /// so its newest N versions are kept. A format returns a key of its own + /// for versions it counts apart: Maven the base version of a timestamped + /// snapshot build (the newest N builds of each `-SNAPSHOT` version), apt + /// the architecture a build is for. `id` is canonical, `version` + /// normalized; called once per version and rule, so keep it quick. + fn retention_partition(&self, id: &str, version: &str) -> Option { + let _ = (id, version); + None + } + /// Read a package's identity, manifest and dependencies. `package` is /// positioned at its start. The default reads none. fn parse_package(&self, package: &mut dyn PackageSource) -> Result { diff --git a/crates/plugin-api/src/types.rs b/crates/plugin-api/src/types.rs index e22248fa..a5b1a1f1 100644 --- a/crates/plugin-api/src/types.rs +++ b/crates/plugin-api/src/types.rs @@ -75,6 +75,23 @@ pub struct ParsedPackage { pub dependencies: Vec, } +/// What kind of version a normalized version is, as retention rules see it +/// ([`FormatPlugin::version_class`]): a rule's `release_type` picks one of +/// them, and `prerelease_superseded_for_days` deletes what is not a release +/// once a release orders after it. +/// +/// [`FormatPlugin::version_class`]: crate::FormatPlugin::version_class +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum VersionClass { + /// A stable release (`1.2.0`; Maven's `33.0.0-jre`). + Release, + /// A prerelease (`1.2.0-beta.1`, `2.3.0-ci.417`). + Prerelease, + /// A build of a version still in development (Maven's `-SNAPSHOT` + /// builds), which its release supersedes. + Snapshot, +} + /// How a repository stores its packages. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RepositoryKind { diff --git a/crates/plugin/src/format.rs b/crates/plugin/src/format.rs index f24f2720..fca741d7 100644 --- a/crates/plugin/src/format.rs +++ b/crates/plugin/src/format.rs @@ -302,6 +302,14 @@ impl Format for PluginFormat { self.plugin.canonical_id(id) } + fn version_class(&self, version: &str) -> forge_core::format::VersionClass { + self.plugin.version_class(version) + } + + fn retention_partition(&self, id: &str, version: &str) -> Option { + self.plugin.retention_partition(id, version) + } + /// Whether the plugin takes `id`, which is never longer than /// [`MAX_ID_LENGTH`]. fn validate_id(&self, id: &str) -> Result<(), FormatError> { @@ -507,4 +515,62 @@ mod tests { assert!(matches!(parse(parsed), Err(FormatError::InvalidPackage(_)))); } } + + /// A plugin with retention hooks of its own: Maven-like snapshots, + /// counted per base version. + struct Snapshots; + + impl FormatPlugin for Snapshots { + fn descriptor(&self) -> &'static api::Descriptor { + &UNLIMITED + } + + fn version_class(&self, version: &str) -> api::VersionClass { + if version.ends_with("-SNAPSHOT") { + api::VersionClass::Snapshot + } else { + api::VersionClass::Release + } + } + + fn retention_partition(&self, _id: &str, version: &str) -> Option { + version.strip_suffix("-SNAPSHOT").map(str::to_string) + } + + fn handle(&self, _repo: &api::Repository, _req: &api::Request) -> api::Action { + api::Action::Respond(api::Response::text(404, "not found")) + } + + fn render( + &self, + _repo: &api::Repository, + _req: &api::Request, + _data: &[api::Loaded], + ) -> api::Action { + api::Action::Respond(api::Response::text(404, "not found")) + } + } + + #[test] + fn retention_hooks_reach_the_host() { + use forge_core::format::VersionClass; + // The defaults: SemVer's prereleases, one partition per package. + let plain = unlimited(None, package("demo", "1.0.0")); + assert_eq!(plain.version_class("1.0.0"), VersionClass::Release); + assert_eq!(plain.version_class("1.0.0-ci.1"), VersionClass::Prerelease); + assert_eq!(plain.retention_partition("demo", "1.0.0-ci.1"), None); + // A plugin's own. + let snapshots = PluginFormat::new(&Plugin::builtin(Snapshots)); + assert_eq!(snapshots.version_class("33.0.0-jre"), VersionClass::Release); + assert_eq!( + snapshots.version_class("1.0-SNAPSHOT"), + VersionClass::Snapshot + ); + assert_eq!( + snapshots + .retention_partition("g:a", "1.0-SNAPSHOT") + .as_deref(), + Some("1.0") + ); + } } diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 608e876b..1e4bdea0 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -130,6 +130,8 @@ zip = { version = "9.0", default-features = false, features = ["deflate"] } serde_json = { workspace = true } tempfile = { workspace = true } chrono = { workspace = true, features = ["serde"] } +# Retention schedules' local times in the admin API tests. +chrono-tz = { workspace = true } # Mock OIDC IdP for the /auth route integration tests: serve # discovery/JWKS/token, sign a self-issued ID token with a throwaway RSA key. # Mirrors the forge-core OIDC test harness. diff --git a/crates/server/src/admin/content_selectors.rs b/crates/server/src/admin/content_selectors.rs index dee5b877..5874dcaa 100644 --- a/crates/server/src/admin/content_selectors.rs +++ b/crates/server/src/admin/content_selectors.rs @@ -8,7 +8,8 @@ //! * `GET /admin/content-selectors/{id}` — fetch one. //! * `PATCH /admin/content-selectors/{id}` — update description/includes/excludes //! (true PATCH: an omitted field keeps its stored value). -//! * `DELETE /admin/content-selectors/{id}` — delete (cascades scoped grants). +//! * `DELETE /admin/content-selectors/{id}` — delete (cascades scoped grants); +//! refused (409) while a repository's retention policy names it in a rule. //! * `GET /admin/content-selectors/{id}/grants` — the grants the selector //! scopes, in every role: what a delete removes. //! * `POST /admin/content-selectors/match` — test a package name against @@ -242,8 +243,31 @@ pub async fn test_match(Json(body): Json) -> Response { } } -/// `DELETE /admin/content-selectors/{id}` +/// `DELETE /admin/content-selectors/{id}`. A selector a retention rule +/// names (`"selector": ""`) is not deleted (409, naming the +/// repositories): the rule would no longer know which packages it covers. pub async fn delete(State(state): State, Path(id): Path) -> Response { + let selector = match authz_dao::find_content_selector_by_id(&state.pool, id).await { + Ok(Some(selector)) => selector, + Ok(None) => return (StatusCode::NOT_FOUND, "content selector not found").into_response(), + Err(e) => return db_error(e), + }; + match forge_db::retention_dao::repositories_using_selector(&state.pool, &selector.name).await { + Ok(repositories) if !repositories.is_empty() => { + return ( + StatusCode::CONFLICT, + format!( + "content selector {:?} is used by the retention policy of {}; \ + remove it from their rules first", + selector.name, + repositories.join(", ") + ), + ) + .into_response(); + } + Ok(_) => {} + Err(e) => return db_error(e), + } match authz_dao::delete_content_selector(&state.pool, id).await { Ok(true) => { invalidate_all(&state.cache).await; diff --git a/crates/server/src/admin/retention.rs b/crates/server/src/admin/retention.rs index 9f211c89..c7fb2684 100644 --- a/crates/server/src/admin/retention.rs +++ b/crates/server/src/admin/retention.rs @@ -6,12 +6,14 @@ //! [`forge_db::retention_dao`]. //! //! * `GET …/retention` — the stored policy + the ledger run state. -//! * `PUT …/retention` — validate + store a policy (arms the worker). +//! * `PUT …/retention` — validate + store a policy (arms the worker; +//! a policy with a cron `schedule` is first due at the schedule's next time). //! * `DELETE …/retention` — clear the policy (disarm; ledger row left). //! * `POST …/retention/preview` — **dry run**: snapshot + the pure evaluator, //! returning what *would* be deleted/unlisted with **zero** mutation and //! **zero** outbox rows. Evaluates the stored policy, or a draft `policy` -//! from the body (checked like `PUT`, never stored). +//! from the body (checked like `PUT`, never stored). A policy with a +//! `schedule` also lists its next run times (`next_runs`). //! * `POST …/retention/run` — schedule a run promptly (`next_run_at=now()`); //! the background worker claims it on its next poll (does **not** run inline). //! Refused (409) while retention runs are switched off for every repository @@ -28,11 +30,12 @@ use axum::http::StatusCode; use axum::response::{IntoResponse, Json, Response}; use chrono::Utc; use forge_core::retention::evaluate::{ - plan_package_run, CandidateComponent, PackagePlan, ProxyCandidate, + plan_package_run, CandidateComponent, PackagePlan, ProxyCandidate, Selectors, }; use forge_core::retention::files::{validate_file_policy, FileDecision, FileRetentionPolicy}; use forge_core::retention::images::{validate_image_policy, ImageDecisions, ImageRetentionPolicy}; use forge_core::retention::model::{validate_policy, RetentionPolicy, SizeBudgetSummary}; +use forge_core::retention::{resolve_selectors, schedule, SelectorError}; use forge_db::model::RepoKind; use forge_db::{repository_dao, retention_dao}; use serde::{Deserialize, Serialize}; @@ -119,6 +122,57 @@ async fn preview_now( } } +/// How many of a scheduled policy's next run times a preview lists. +const PREVIEW_NEXT_RUNS: usize = 5; + +/// The content selectors `policy`'s rules name, or the response refusing it +/// (`invalid` words the 400) when one is missing or unusable. +async fn selectors_for( + state: &AppState, + policy: &RetentionPolicy, + invalid: impl FnOnce(SelectorError) -> Response, +) -> Result { + match resolve_selectors(&state.pool, policy).await { + Ok(selectors) => Ok(selectors), + Err(SelectorError::Db(e)) => Err(db_error(e)), + Err(other) => Err(invalid(other)), + } +} + +/// The next run to set on the ledger when `policy` (any shape, as stored) +/// is saved: a schedule's next time after `now`; for an interval policy, +/// `min_run_interval_seconds` after the last run started (and not before +/// now), when the ledger has the next run later than that (a schedule +/// replaced by an interval, a shorter interval). `None` leaves the ledger's +/// next run as it is. +async fn next_run_on_save( + state: &AppState, + repository_id: i64, + policy: &JsonValue, + now: chrono::DateTime, +) -> Result>, Response> { + if let Some(schedule) = schedule::of_policy(policy) { + return Ok(schedule.next_after(now)); + } + let Some(run) = retention_dao::get_run(&state.pool, repository_id) + .await + .map_err(db_error)? + else { + return Ok(None); + }; + let secs = policy + .get("min_run_interval_seconds") + .and_then(JsonValue::as_u64) + .unwrap_or(3600) + .clamp( + forge_core::retention::worker::MIN_RUN_INTERVAL_SECONDS as u64, + forge_core::retention::model::MAX_MIN_RUN_INTERVAL_SECONDS, + ); + let interval = chrono::Duration::seconds(i64::try_from(secs).unwrap_or(i64::MAX)); + let due = (run.started_at.unwrap_or(now) + interval).max(now); + Ok((run.next_run_at > due).then_some(due)) +} + /// `GET /admin/repositories/{name}/retention` pub async fn get(State(state): State, Path(name): Path) -> Response { let repo = match resolve_repo(&state, &name).await { @@ -218,14 +272,32 @@ pub async fn put( if let Err(e) = validate_policy(&policy, repo.kind()) { return bad_request(format!("invalid retention policy: {e}")); } + // Every content selector a rule names must exist. + if let Err(resp) = selectors_for(&state, &policy, |e| { + bad_request(format!("invalid retention policy: {e}")) + }) + .await + { + return resp; + } serde_json::to_value(&policy) }; let policy_json = match normalized { Ok(v) => v, Err(e) => return bad_request(format!("could not serialize policy: {e}")), }; + let now = match forge_db::db_now(&state.pool).await { + Ok(now) => now, + Err(e) => return db_error(e), + }; + let next_run_at = match next_run_on_save(&state, repo.id, &policy_json, now).await { + Ok(next) => next, + Err(resp) => return resp, + }; - match retention_dao::write_policy(&state.pool, repo.id, Some(&policy_json)).await { + match retention_dao::write_policy_at(&state.pool, repo.id, Some(&policy_json), next_run_at) + .await + { Ok(true) => { let current_size_bytes = match current_size_bytes(&state, &repo).await { Ok(size) => size, @@ -350,6 +422,9 @@ pub struct PreviewView { /// (`size_before`, `size_after`, `target_bytes`, `target_reached`). #[serde(flatten)] pub budget: Option, + /// With a `schedule`: the next times it would run, in UTC. + #[serde(skip_serializing_if = "Option::is_none")] + pub next_runs: Option>>, } /// A preview's size budget: the size before, the size projected once the @@ -408,17 +483,26 @@ pub async fn preview( file_decisions: state.formats.stores_files(&repo.format).then(Vec::new), counts: PreviewCounts::default(), budget: None, + next_runs: None, }) .into_response() } Err(e) => return db_error(e), }, }; + // A schedule's next times, as of the instant the preview evaluates at. + let next_runs = match schedule::of_policy(&policy_json) { + Some(schedule) => match preview_now(&state, at).await { + Ok(now) => Some(schedule.next_runs(now, PREVIEW_NEXT_RUNS)), + Err(resp) => return resp, + }, + None => None, + }; if state.formats.stores_images(&repo.format) { - return preview_images(&state, &repo, policy_json, source, at).await; + return preview_images(&state, &repo, policy_json, source, at, next_runs).await; } if state.formats.stores_files(&repo.format) { - return preview_files(&state, &repo, policy_json, source, at).await; + return preview_files(&state, &repo, policy_json, source, at, next_runs).await; } let mut policy: RetentionPolicy = match source.parse(policy_json) { Ok(p) => p, @@ -427,6 +511,10 @@ pub async fn preview( if let Err(e) = validate_policy(&policy, repo.kind()) { return source.invalid(e); } + let selectors = match selectors_for(&state, &policy, |e| source.invalid(e)).await { + Ok(selectors) => selectors, + Err(resp) => return resp, + }; if !source.enable(&mut policy.enabled) { return Json(PreviewView { decisions: Vec::new(), @@ -435,6 +523,7 @@ pub async fn preview( file_decisions: None, counts: PreviewCounts::default(), budget: None, + next_runs, }) .into_response(); } @@ -470,6 +559,7 @@ pub async fn preview( listed: r.listed, created_at: r.created_at, last_downloaded_at: r.last_downloaded_at, + unlisted_at: r.unlisted_at, size_bytes: u64::try_from(r.size_bytes).unwrap_or_default(), }) .collect(), @@ -519,6 +609,7 @@ pub async fn preview( format.as_ref(), now, &policy, + &selectors, &hosted, &proxy, &tagged_hosted_ids, @@ -561,6 +652,7 @@ pub async fn preview( deleted, }, budget: budget.map(PreviewBudget::from), + next_runs, }) .into_response() } @@ -572,6 +664,7 @@ async fn preview_images( policy_json: JsonValue, source: PolicySource, at: Option>, + next_runs: Option>>, ) -> Response { let mut policy: ImageRetentionPolicy = match source.parse(policy_json) { Ok(p) => p, @@ -590,6 +683,7 @@ async fn preview_images( ..PreviewCounts::default() }, budget: None, + next_runs: next_runs.clone(), }; if !source.enable(&mut policy.enabled) { return Json(empty(0)).into_response(); @@ -614,6 +708,7 @@ async fn preview_images( deleted, }, budget: budget.map(PreviewBudget::from), + next_runs, }) .into_response() } @@ -627,6 +722,7 @@ async fn preview_files( policy_json: JsonValue, source: PolicySource, at: Option>, + next_runs: Option>>, ) -> Response { let mut policy: FileRetentionPolicy = match source.parse(policy_json) { Ok(p) => p, @@ -646,6 +742,7 @@ async fn preview_files( }, file_decisions: Some(decisions), budget: None, + next_runs: next_runs.clone(), }; if !source.enable(&mut policy.enabled) { return Json(view(Vec::new(), 0)).into_response(); diff --git a/crates/server/tests/admin_retention.rs b/crates/server/tests/admin_retention.rs index 6f1a8d6c..f940bc61 100644 --- a/crates/server/tests/admin_retention.rs +++ b/crates/server/tests/admin_retention.rs @@ -1089,7 +1089,7 @@ async fn retention_settings_switch_runs_off_and_on() { assert_eq!(view["runs_enabled"], json!(false), "{view}"); // Nothing is claimed while off, even a repository that is due. - retention_dao::fail_run(&srv.pool, repo_id, Some(run_id), "test", now) + retention_dao::fail_run(&srv.pool, repo_id, Some(run_id), "test", now, None) .await .unwrap(); let mut tx = srv.pool.begin().await.unwrap(); @@ -1213,3 +1213,385 @@ async fn proxy_rules_keep_the_newest_and_recently_pulled_cached_versions() { .unwrap(); assert_eq!(resp.status(), 400); } + +/// #185: the new fields are validated like the others (unknown fields 422, an +/// invalid value 400 naming it), a rule's content selector must exist, and a +/// policy with a schedule is first due at the schedule's next time, not at +/// once. A selector a policy names cannot be deleted. +#[tokio::test] +async fn schedules_triggers_and_selectors_are_validated_and_stored() { + let srv = common::start().await; + let (u, p) = srv.admin_basic(); + let client = srv.client(); + let s = common::unique(); + let name = format!("rets{s}"); + let repo_id = seed_hosted_with_versions(&srv, &name, "Sample.Lib", &["1.0.0"]).await; + let url = format!("{}/admin/repositories/{name}/retention", srv.base_url); + let put = |policy: Value| { + client + .put(&url) + .basic_auth(&u, Some(&p)) + .json(&json!({ "policy": policy })) + .send() + }; + let selector = format!("ret-ci-{s}"); + let policy = json!({ + "version": 1, + "schedule": "0 3 * * *", + "schedule_timezone": "Europe/Berlin", + "max_run_seconds": 3600, + "rules": [ + { "id": "ci", "action": "hard_delete", "release_type": "prereleases", + "selector": selector, "version_pattern": "*-ci.*", + "exclude_version_pattern": "*-ci.0", + "criteria": { "prerelease_superseded_for_days": 14, "keep_last_n": 2 } }, + { "id": "unlisted", "action": "hard_delete", + "criteria": { "unlisted_for_days": 90 } }, + { "id": "cap", "action": "unlist", + "criteria": { "max_versions_per_package": 50 } } + ] + }); + + // The selector does not exist yet. + let resp = put(policy.clone()).await.unwrap(); + assert_eq!(resp.status(), 400); + let msg = resp.text().await.unwrap(); + assert!(msg.contains("does not exist"), "{msg}"); + + let resp = client + .post(format!("{}/admin/content-selectors", srv.base_url)) + .basic_auth(&u, Some(&p)) + .json(&json!({ "name": selector, "includes": ["sample.*"], "excludes": [] })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201); + let selector_id = resp.json::().await.unwrap()["id"].as_i64().unwrap(); + + let before = chrono::Utc::now(); + let resp = put(policy.clone()).await.unwrap(); + assert_eq!(resp.status(), 200, "{}", resp.text().await.unwrap()); + let view: Value = client + .get(&url) + .basic_auth(&u, Some(&p)) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + // Stored in its canonical form, with every new field. + let stored = &view["policy"]; + assert_eq!(stored["schedule"], "0 3 * * *", "{view}"); + assert_eq!(stored["schedule_timezone"], "Europe/Berlin"); + assert_eq!(stored["max_run_seconds"], 3600); + assert_eq!(stored["rules"][0]["selector"], selector.as_str()); + assert_eq!(stored["rules"][0]["version_pattern"], "*-ci.*"); + assert_eq!(stored["rules"][0]["exclude_version_pattern"], "*-ci.0"); + assert_eq!( + stored["rules"][0]["criteria"]["prerelease_superseded_for_days"], + 14 + ); + assert_eq!(stored["rules"][1]["criteria"]["unlisted_for_days"], 90); + assert_eq!( + stored["rules"][2]["criteria"]["max_versions_per_package"], + 50 + ); + // Due at the next 03:00 in Berlin, not now. + let next: chrono::DateTime = + serde_json::from_value(view["last_run"]["next_run_at"].clone()).unwrap(); + let expected = + forge_core::retention::schedule::Schedule::parse("0 3 * * *", Some("Europe/Berlin")) + .unwrap() + .next_after(before) + .unwrap(); + assert_eq!(next, expected, "{view}"); + + // Run now still runs at once. + let resp = client + .post(format!("{url}/run")) + .basic_auth(&u, Some(&p)) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202); + let due: bool = sqlx::query_scalar( + "SELECT next_run_at <= now() + interval '1 minute' FROM retention_run WHERE repository_id = $1", + ) + .bind(repo_id) + .fetch_one(&srv.pool) + .await + .unwrap(); + assert!(due); + + // A schedule replaced by an interval runs by the interval again, not at + // the old schedule's next time. + sqlx::query("UPDATE retention_run SET next_run_at = now() + interval '20 hours' WHERE repository_id = $1") + .bind(repo_id) + .execute(&srv.pool) + .await + .unwrap(); + let mut hourly = policy.clone(); + let envelope = hourly.as_object_mut().unwrap(); + for key in ["schedule", "schedule_timezone"] { + envelope.remove(key); + } + let resp = put(hourly).await.unwrap(); + assert_eq!(resp.status(), 200, "{}", resp.text().await.unwrap()); + let soon: bool = sqlx::query_scalar( + "SELECT next_run_at <= now() + interval '61 minutes' FROM retention_run WHERE repository_id = $1", + ) + .bind(repo_id) + .fetch_one(&srv.pool) + .await + .unwrap(); + assert!(soon); + + // The selector is in use. + let resp = client + .delete(format!( + "{}/admin/content-selectors/{selector_id}", + srv.base_url + )) + .basic_auth(&u, Some(&p)) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 409); + let msg = resp.text().await.unwrap(); + assert!(msg.contains(&name), "{msg}"); + + // Invalid values: 400 naming the field. + for (edit, needle) in [ + (json!({"schedule": "0 3 * *"}), "5 fields"), + ( + json!({"schedule": "0 3 * * *", "schedule_timezone": "Europe/Atlantis"}), + "IANA", + ), + ( + json!({"schedule": null, "schedule_timezone": "UTC"}), + "needs a schedule", + ), + (json!({"max_run_seconds": 5}), "max_run_seconds"), + ] { + let mut bad = policy.clone(); + for (k, v) in edit.as_object().unwrap() { + if v.is_null() { + bad.as_object_mut().unwrap().remove(k); + } else { + bad[k] = v.clone(); + } + } + let resp = put(bad).await.unwrap(); + assert_eq!(resp.status(), 400, "{edit}"); + let msg = resp.text().await.unwrap(); + assert!(msg.contains(needle), "{edit}: {msg}"); + } + for (rule, field, value, needle) in [ + (0, "version_pattern", json!("1.[0"), "version_pattern"), + (1, "action", json!("unlist"), "unlisted_for_days"), + (2, "release_type", json!("nightlies"), "unknown variant"), + ] { + let mut bad = policy.clone(); + bad["rules"][rule][field] = value; + let resp = put(bad).await.unwrap(); + assert!( + resp.status() == 400 || resp.status() == 422, + "{field}: {}", + resp.status() + ); + let msg = resp.text().await.unwrap(); + assert!(msg.contains(needle), "{field}: {msg}"); + } + let mut zero = policy.clone(); + zero["rules"][2]["criteria"]["max_versions_per_package"] = json!(0); + assert_eq!(put(zero).await.unwrap().status(), 400); + let mut typo = policy.clone(); + typo["rules"][0]["criteria"]["prerelease_superseded_days"] = json!(14); + assert_eq!(put(typo).await.unwrap().status(), 422); + let mut typo = policy.clone(); + typo["schedule_tz"] = json!("UTC"); + assert_eq!(put(typo).await.unwrap().status(), 422); +} + +/// #185: the preview shows each new trigger's decisions with the reason a +/// run records (the release that superseded a prerelease, how long a +/// version has been unlisted, its place past the cap), filters narrow them, +/// protections win, and a scheduled draft lists its next runs. +#[tokio::test] +async fn preview_shows_the_new_triggers_decisions_and_the_next_runs() { + let srv = common::start().await; + let (u, p) = srv.admin_basic(); + let client = srv.client(); + let s = common::unique(); + let name = format!("retp{s}"); + let repo_id = seed_hosted_with_versions( + &srv, + &name, + "Sample.Lib", + &[ + "1.0.0-ci.1", + "1.0.0-ci.2", + "1.0.0", + "1.1.0", + "1.2.0", + "2.0.0-beta.1", + ], + ) + .await; + // 1.0.0 was published 20 days ago, the CI builds before it; 1.1.0 has + // been unlisted for 100 days. + sqlx::query( + "UPDATE component SET created_at = now() - interval '20 days' \ + WHERE repository_id = $1 AND version = '1.0.0'", + ) + .bind(repo_id) + .execute(&srv.pool) + .await + .unwrap(); + sqlx::query( + "UPDATE component SET listed = false WHERE repository_id = $1 AND version = '1.1.0'", + ) + .bind(repo_id) + .execute(&srv.pool) + .await + .unwrap(); + sqlx::query( + "UPDATE component SET unlisted_at = now() - interval '100 days' \ + WHERE repository_id = $1 AND version = '1.1.0'", + ) + .bind(repo_id) + .execute(&srv.pool) + .await + .unwrap(); + let url = format!("{}/admin/repositories/{name}/retention", srv.base_url); + let preview = |policy: Value| { + let client = client.clone(); + let (u, p, url) = (u.clone(), p.clone(), url.clone()); + async move { + let resp = client + .post(format!("{url}/preview")) + .basic_auth(&u, Some(&p)) + .json(&json!({ "policy": policy })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200); + resp.json::().await.unwrap() + } + }; + let by_version = |body: &Value| -> std::collections::BTreeMap { + body["decisions"] + .as_array() + .unwrap() + .iter() + .map(|d| (d["version"].as_str().unwrap().to_string(), d.clone())) + .collect() + }; + + let body = preview(json!({ + "version": 1, + "schedule": "30 2 * * *", + "schedule_timezone": "Europe/Berlin", + "rules": [ + { "id": "ci", "action": "hard_delete", "version_pattern": "*-ci.*", + "criteria": { "prerelease_superseded_for_days": 14 } }, + { "id": "unlisted", "action": "hard_delete", + "criteria": { "unlisted_for_days": 90 } } + ] + })) + .await; + let decided = by_version(&body); + assert_eq!( + decided.keys().collect::>(), + vec!["1.0.0-ci.1", "1.0.0-ci.2", "1.1.0"], + "{body}" + ); + assert_eq!(decided["1.0.0-ci.1"]["reason"]["superseded_by"], "1.0.0"); + assert_eq!( + decided["1.0.0-ci.1"]["reason"]["prerelease_superseded_for_days"], + 14 + ); + assert_eq!(decided["1.1.0"]["reason"]["unlisted_for_days"], 90); + assert_eq!(decided["1.1.0"]["rule_id"], "unlisted"); + let runs = body["next_runs"].as_array().unwrap(); + assert_eq!(runs.len(), 5); + for run in runs { + let at: chrono::DateTime = serde_json::from_value(run.clone()).unwrap(); + let local = at.with_timezone(&chrono_tz::Europe::Berlin); + // 02:30, or 03:00 on the day the clocks skip it. + assert!( + ["02:30", "03:00"].contains(&local.format("%H:%M").to_string().as_str()), + "{local}" + ); + } + + // A count: two newest of the package stay, listed or not; keep_last_n + // (newest listed) in another rule wins over it. + let body = preview(json!({ + "version": 1, + "rules": [ + { "id": "cap", "action": "hard_delete", + "criteria": { "max_versions_per_package": 2 } }, + { "id": "keep", "action": "hard_delete", "release_type": "releases", + "criteria": { "keep_last_n": 2, "older_than_days": 1000 } } + ] + })) + .await; + let decided = by_version(&body); + assert_eq!( + decided.keys().collect::>(), + vec!["1.0.0-ci.1", "1.0.0-ci.2", "1.1.0"], + "2.0.0-beta.1 and 1.2.0 are the newest two; 1.2.0 and 1.0.0 the newest listed \ + releases, which the other rule keeps: {body}" + ); + assert_eq!(decided["1.1.0"]["reason"]["rank"], 3); + assert_eq!(decided["1.0.0-ci.1"]["reason"]["rank"], 6); + assert!(body.get("next_runs").is_none(), "no schedule: {body}"); +} + +/// #185: image rules take tag patterns and globs in `protected_tags`. +#[tokio::test] +async fn image_rules_take_tag_patterns_and_glob_protected_tags() { + let srv = common::start().await; + let (u, p) = srv.admin_basic(); + let client = srv.client(); + let s = common::unique(); + let name = format!("reti{s}"); + let resp = client + .post(format!("{}/admin/repositories", srv.base_url)) + .basic_auth(&u, Some(&p)) + .json(&json!({ "name": name, "kind": "hosted", "format": "docker" })) + .send() + .await + .unwrap(); + assert!(resp.status().is_success(), "{}", resp.text().await.unwrap()); + let url = format!("{}/admin/repositories/{name}/retention", srv.base_url); + let policy = json!({ + "version": 1, + "schedule": "0 4 * * 0", + "rules": [ + { "id": "prs", "tag_pattern": "pr-*", "exclude_tag_pattern": "pr-keep*", + "tag_older_than_days": 7, "protected_tags": ["release-*", "latest"] } + ] + }); + let resp = client + .put(&url) + .basic_auth(&u, Some(&p)) + .json(&json!({ "policy": policy })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "{}", resp.text().await.unwrap()); + let mut bad = policy.clone(); + bad["rules"][0]["tag_pattern"] = json!("pr-["); + let resp = client + .put(&url) + .basic_auth(&u, Some(&p)) + .json(&json!({ "policy": bad })) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); + assert!(resp.text().await.unwrap().contains("tag_pattern")); +} diff --git a/crates/server/tests/raw.rs b/crates/server/tests/raw.rs index 8f58a6de..937c6857 100644 --- a/crates/server/tests/raw.rs +++ b/crates/server/tests/raw.rs @@ -1412,6 +1412,7 @@ async fn retention_deletes_old_files_and_keeps_the_newest() { chrono::Duration::zero(), false, None, + None, ) .await .unwrap(); diff --git a/crates/server/tests/registry.rs b/crates/server/tests/registry.rs index 7b436d3d..a4a428e7 100644 --- a/crates/server/tests/registry.rs +++ b/crates/server/tests/registry.rs @@ -4392,6 +4392,7 @@ async fn apply_image_run( chrono::Duration::seconds(60), false, Some(&fence), + None, ) .await .unwrap() diff --git a/docs/README.md b/docs/README.md index 3b067bf3..fdd36c33 100644 --- a/docs/README.md +++ b/docs/README.md @@ -19,8 +19,10 @@ its [quickstart](../README.md#quickstart); base, components and overlays, for production and for an evaluation. - **[Monitoring](../deploy/monitoring/README.md)**: an example OpenTelemetry Collector configuration, Grafana dashboard and Prometheus alerting rules. -- **[Retention size budgets](retention.md)**: keep a repository under a - logical size: the budget, how size is counted, and what goes first. +- **[Retention](retention.md)**: every retention rule with examples (age, + downloads, superseded prereleases, unlisted versions, version and tag + patterns, keep-N), cron schedules in a time zone with a time limit per + run, and size budgets. - **[Importing repositories](import.md)**: copy a hosted repository of another server (a NuGet feed, a registry: Nexus, Artifactory, another Forge) into Forge, with its publish dates. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 57d5006a..1c599617 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -8,7 +8,7 @@ - [Helm chart](../deploy/helm/forge/README.md) - [Kubernetes manifests](../deploy/kubernetes/README.md) - [Monitoring](../deploy/monitoring/README.md) -- [Retention size budgets](retention.md) +- [Retention](retention.md) - [Importing repositories](import.md) - [Importing from Nexus](nexus-import.md) diff --git a/docs/changes/2.2.0/185.feature.md b/docs/changes/2.2.0/185.feature.md new file mode 100644 index 00000000..ffbd6f9b --- /dev/null +++ b/docs/changes/2.2.0/185.feature.md @@ -0,0 +1,19 @@ +Retention policies run on a cron schedule in a time zone (`schedule`, +`schedule_timezone`: `0 3 * * *` in `Europe/Berlin` runs at 03:00 Berlin +time, summer and winter), as an alternative to `min_run_interval_seconds`, +and `max_run_seconds` stops a run at the end of its window, the next run +going on with the rest; Run now still runs at once, and the Retention +screen shows the schedule, the next run and a draft's next run times. This +holds for package, raw file and image policies. Package rules gain the +triggers `prerelease_superseded_for_days` (a prerelease once a release that +comes after it has been out N days), `unlisted_for_days` (delete what was +unlisted N days ago) and `max_versions_per_package` (a pure count, on proxy +rules too), and the filters `version_pattern` / `exclude_version_pattern` +(globs on the version), `selector` (a content selector, which cannot be +deleted while a rule names it) and `release_type: snapshots`; image rules +gain `tag_pattern` / `exclude_tag_pattern` and globs in `protected_tags`. +Protections win over every trigger, the preview shows each decision's +reason, and the Retention screen edits every field. The format contract +gets retention hooks (`version_class`, `retention_partition`) so a format +says what a prerelease, a snapshot and "the newest N" are; `docs/retention.md` +lists every rule with examples. diff --git a/docs/changes/2.2.0/185.upgrade.md b/docs/changes/2.2.0/185.upgrade.md new file mode 100644 index 00000000..a86e5b80 --- /dev/null +++ b/docs/changes/2.2.0/185.upgrade.md @@ -0,0 +1,8 @@ +A version now records when it was unlisted (`component.unlisted_at`, kept +by a database trigger on every path that unlists), for retention's +`unlisted_for_days`. The upgrade gives versions that are unlisted already +the time of their last change (`updated_at`): every way of unlisting set it +then, so it is never earlier than the unlisting, and a version changed +after it was unlisted counts from that later change. A content selector a +retention rule names can no longer be deleted (409) until no rule names it. +Policies stored before keep running as they did. diff --git a/docs/docker.md b/docs/docker.md index a0d38cd1..35a18557 100644 --- a/docs/docker.md +++ b/docs/docker.md @@ -334,7 +334,8 @@ A docker repository's retention policy (`PUT "version": 1, "rules": [ {"id": "keep-10", "image_prefix": "team/", "keep_last_n_tags": 10, - "protected_tags": ["latest", "stable"]}, + "protected_tags": ["latest", "stable", "release-*"]}, + {"id": "pr-tags", "tag_pattern": "pr-*", "tag_older_than_days": 7}, {"id": "stale-tags", "tag_older_than_days": 90, "tag_not_pulled_for_days": 30}, {"id": "untagged", "delete_untagged_after_days": 7} ] @@ -342,8 +343,11 @@ A docker repository's retention policy (`PUT ``` - **Tag rules** delete tags. Per image (only those under `image_prefix`, when - set), the `keep_last_n_tags` most recently pushed tags and the - `protected_tags` stay; any other tag goes when every trigger the rule sets + set), a rule looks at the tags its `tag_pattern` glob matches and its + `exclude_tag_pattern` does not (every tag without either; `pr-*`, `v*`, + case as written). Of those, the `keep_last_n_tags` most recently pushed + tags and the `protected_tags` (names, or globs such as `release-*`) stay; + any other tag goes when every trigger the rule sets holds (`tag_older_than_days` since its push, `tag_not_pulled_for_days`). A `GET` or a `HEAD` of a tag's manifest counts as a pull, so a tag that containerd (Kubernetes, nerdctl, BuildKit) only resolves stays pulled. A @@ -361,8 +365,9 @@ A docker repository's retention policy (`PUT tag for that long, except what something kept refers to: the platforms of a kept index, and the signatures, SBOMs and attestations (referrers) of a kept manifest. -- `dry_run_only`, `enabled`, `min_run_interval_seconds`, preview, run and - audit work as for NuGet repositories; audit rows carry the image as +- `dry_run_only`, `enabled`, `min_run_interval_seconds` or a cron + `schedule` in a `schedule_timezone`, `max_run_seconds`, preview, run and + audit work as for NuGet repositories ([Retention](retention.md)); audit rows carry the image as `package_name` and the tag or digest as `version`. Group repositories have no policy of their own. The repository's **Retention** screen in the web UI (Repositories → the repository → Retention) edits the rules, previews a @@ -379,7 +384,7 @@ A docker repository's retention policy (`PUT An image retention policy may also include a `size_budget`; it evicts least-recently used unprotected manifests and tags after its ordinary rules. -See [Retention size budgets](retention.md) for configuration and logical size +See [Retention](retention.md#size-budgets) for configuration and logical size accounting. ## Configuration diff --git a/docs/format-plugins.md b/docs/format-plugins.md index dfdff418..89b703ba 100644 --- a/docs/format-plugins.md +++ b/docs/format-plugins.md @@ -33,6 +33,7 @@ repositories ([docker.md](docker.md#helm-charts)). - [How it fits together](#how-it-fits-together) - [Files formats](#files-formats) - [Images formats](#images-formats) +- [Retention](#retention) - [Proxy repositories](#proxy-repositories) - [Using a plugin](#using-a-plugin) - [Security model](#security-model) @@ -102,7 +103,7 @@ The pieces: | Logins | Checks a password through the realm chain, or runs a login approved in the browser, and mints, lists and revokes the API tokens; accepts `Authorization: Bearer` API tokens | Says what a request asks (`auth`) and renders the outcome (`auth_response`) | | Proxies | Sends every upstream request (credentials only to the upstream's origin, bounded, cached), downloads and verifies packages, keeps the package delay, the not-found markers and the offline mode | `upstream`: which documents an operation needs, and what they say ([Proxy repositories](#proxy-repositories)) | | Groups | Merges members by priority; the dependency-confusion guards apply | Nothing: a group of plugin repositories just works | -| Retention | Evaluates policies, ordering versions with the plugin's `compare_versions` | Nothing more | +| Retention | Evaluates policies on a schedule, ordering versions with the plugin's `compare_versions` ([Retention](#retention)) | Optionally, a version's class (`version_class`) and the partition keep-N counts within (`retention_partition`) | | Files (a files format) | Keeps files under paths, content-addressed with their digests: serving, ranges and validators, uploads under the deployment policy, listings, proxies and groups, selectors on paths, retention by path ([Files formats](#files-formats)) | Which path a request names, which file or directory it needs, what the response looks like | A request to every non-Docker repository is served under @@ -351,6 +352,35 @@ requires; the token is the host's, cached for its lifetime. `testing::converse_answered` runs `upstream` against canned answers with their status and headers, as the Docker plugin's tests do. +## Retention + +The host runs every retention rule ([retention.md](retention.md)) for a +package format; the plugin only says what its versions mean. Ordering is +`compare_versions`, and two hooks, both with defaults, say the rest: + +| Hook | Says | Default | +| --- | --- | --- | +| `version_class(version)` | Whether a normalized version is a `Release`, a `Prerelease` or a `Snapshot` ([`VersionClass`](../crates/plugin-api/src/types.rs)). A rule's `release_type` filters on it, and `prerelease_superseded_for_days` deletes what is not a release once a listed release that `compare_versions` orders after it has been out long enough. | SemVer's: a `-` makes a prerelease | +| `retention_partition(id, version)` | The partition of a package's versions keep-N counts within (`keep_last_n`, `keep_n_most_downloaded`, `max_versions_per_package`): versions with the same key are counted together. | `None`: the package is one partition | + +NuGet and npm keep the defaults. A format whose versions differ overrides +them; these are the hooks the formats planned next need: + +- **Maven:** `33.0.0-jre` is a release and `1.0-SNAPSHOT` (with its + timestamped builds) a snapshot, so `version_class` reads Maven's rules + rather than the `-`. `retention_partition` returns the base `-SNAPSHOT` + version for a snapshot build, so `keep_last_n` keeps the newest N builds + of each snapshot version, and `None` for releases, which count per + `groupId:artifactId`. `compare_versions` orders a version's snapshots + before its release, so `prerelease_superseded_for_days` deletes them once + the release exists, and `release_type: snapshots` with `older_than_days` + deletes builds older than N days. +- **apt:** `retention_partition` returns a version's architecture, so keep-N + counts per package and architecture. + +Both run inline for every version and rule a run evaluates: keep them quick +and free of I/O. + ## Proxy repositories A plugin that lists `proxy` among its kinds implements the upstream side of @@ -623,9 +653,10 @@ to lift them: - **One package file per version.** Formats that publish several files per version (PyPI wheels per platform, Maven's jar, pom and sources) need multi-asset versions. -- **Prerelease is "contains a `-`".** This is right for SemVer formats; a - format with another rule (PEP 440's `1.0a1`) needs a contract function - for it. +- **Prerelease is "contains a `-`" for searches.** This is right for SemVer + formats. Retention asks the plugin (`version_class`, [Retention](#retention)), + but the host's search filter (`prerelease`) still reads the `-`; a format + with another rule (PEP 440's `1.0a1`) needs the search to ask it too. - **Listings are bounded.** A response may load at most 2,048 answers, so a listing of every package of a repository covers at most about 2,000 packages. diff --git a/docs/npm.md b/docs/npm.md index 16736e72..f79d5d6c 100644 --- a/docs/npm.md +++ b/docs/npm.md @@ -325,7 +325,8 @@ versions as kept. Package retention policies may include a `size_budget` to evict least-recently used eligible versions when a repository exceeds its logical size limit. See -[Retention size budgets](retention.md) for configuration and size accounting. +[Retention](retention.md#size-budgets) for configuration and size accounting. +The rules themselves, and their schedules, are in [Retention](retention.md). ## Limits diff --git a/docs/raw.md b/docs/raw.md index 514a05a4..e2c813ec 100644 --- a/docs/raw.md +++ b/docs/raw.md @@ -294,8 +294,9 @@ the file's path and SHA-256. ## Retention A raw repository's retention policy has the envelope of a package policy -(`version`, `enabled`, `dry_run_only`, `min_run_interval_seconds`) with file -rules: +(`version`, `enabled`, `dry_run_only`, `min_run_interval_seconds` or a cron +`schedule` in a `schedule_timezone`, `max_run_seconds`; see +[Retention](retention.md#when-a-policy-runs)) with file rules: | Field | | | --- | --- | @@ -328,7 +329,7 @@ policy and lists the audit log. Retention policies may also include a `size_budget` to evict least-recently used files when the repository exceeds a logical size limit. See -[Retention size budgets](retention.md) for configuration and size accounting. +[Retention](retention.md#size-budgets) for configuration and size accounting. ## Example: the pdfium binaries diff --git a/docs/retention.md b/docs/retention.md index ed966ec2..0e3ed9ad 100644 --- a/docs/retention.md +++ b/docs/retention.md @@ -1,12 +1,368 @@ -# Retention size budgets +# Retention -Hosted and proxy package repositories, raw file repositories, and Docker -repositories can set a logical size budget in their retention policy. Configure -it through the repository's **Retention** screen or -`PUT /admin/repositories//retention`; it is stored with the policy, not -in `forge.toml`. Group repositories do not have their own retention policy. +Retention removes what a repository no longer needs: old and unused +versions of packages, prereleases a release superseded, versions unlisted +long ago, cold proxy caches, stale image tags and files, and whatever a +size budget evicts. Each hosted or proxy repository can have one retention +policy; group repositories hold nothing of their own and take none. + +A policy is set on the repository's **Retention** screen in the web UI +(Repositories → the repository → Retention), or through the admin API: + +```bash +curl -u admin@example.com: -X PUT -H 'content-type: application/json' \ + https://forge.example.com/admin/repositories/nuget-ci/retention -d '{"policy": {…}}' +``` + +It is stored in the database with the repository, never in `forge.toml`. +**Admin → Retention** switches runs on or off for every repository and sets +how long the audit log is kept; `[retention]` in `forge.toml` only tunes the +worker (how often an instance looks for a due repository, 30 seconds by +default). + +## Contents + +- [The policy](#the-policy) +- [When a policy runs](#when-a-policy-runs) +- [Package rules](#package-rules) + - [Filters](#filters) + - [Protections](#protections) + - [Triggers](#triggers) + - [Examples](#examples) +- [Proxy rules](#proxy-rules) +- [Formats: what "newest" and "prerelease" mean](#formats-what-newest-and-prerelease-mean) +- [Raw file rules](#raw-file-rules) +- [Image rules](#image-rules) +- [Preview, runs and the audit log](#preview-runs-and-the-audit-log) +- [Size budgets](#size-budgets) + +## The policy + +Every policy has the same envelope, with rules of the repository's kind: +package rules for NuGet, npm and the other package formats, file rules for +raw repositories, image rules for Docker repositories (Helm charts +included). + +```json +{ + "version": 1, + "enabled": true, + "dry_run_only": false, + "schedule": "0 3 * * *", + "schedule_timezone": "Europe/Berlin", + "max_run_seconds": 3600, + "rules": [ … ] +} +``` + +| Field | | +| --- | --- | +| `version` | Always `1`. | +| `enabled` | Default `true`. A disabled policy is stored and never runs. | +| `dry_run_only` | Default `false`. Runs record in the audit log what they would remove, and remove nothing. | +| `min_run_interval_seconds` | Default 3600: the policy runs at most this often. Ignored when `schedule` is set. | +| `schedule` | A cron pattern of 5 fields: the policy runs at its times instead ([When a policy runs](#when-a-policy-runs)). | +| `schedule_timezone` | The IANA time zone of `schedule`'s times (`Europe/Berlin`, `America/New_York`); UTC without one. Only with a `schedule`. | +| `max_run_seconds` | How long a run may take, from 60 seconds to a week; without it a run takes as long as it needs. | +| `size_budget` | A logical size limit ([Size budgets](#size-budgets)). | +| `rules` | The rules. May be empty when a size budget is set. | + +Unknown fields are refused (422), so a misspelled one is never dropped +silently, and an invalid value is a 400 that names its rule and field. + +## When a policy runs + +The retention worker runs on every instance; each repository is evaluated by +one instance at a time. + +- **At an interval** (`min_run_interval_seconds`, the default): a run at most + every hour, or whatever the policy says. +- **On a schedule** (`schedule`): at the times of a 5-field cron pattern, + `minute hour day-of-month month day-of-week`, in `schedule_timezone`: + + | `schedule` | Runs | + | --- | --- | + | `0 3 * * *` | at 03:00 every day | + | `30 1 * * 6` | at 01:30 on Saturdays | + | `0 22 * * 1-5` | at 22:00 Monday to Friday | + | `0 */6 * * *` | every six hours, on the hour | + | `@daily`, `@weekly` | at midnight every day, every Sunday | + + Lists (`1,15`), ranges (`1-5`), steps (`*/15`, `0-30/10`) and names + (`MON-FRI`, `JAN`) work as in Vixie cron. The times keep their local time + across daylight-saving changes: `0 3 * * *` in `Europe/Berlin` runs at + 03:00 Berlin time all year, which is 02:00 UTC in winter and 01:00 UTC in + summer. A time that falls into the hour the clocks skip (`30 2 * * *` on + the last Sunday of March in Berlin) runs once, right after the change, at + 03:00; a time in the hour they repeat (`30 2 * * *` on the last Sunday of + October) runs once, at its first occurrence. + + A policy saved with a schedule waits for its next time: it does not run + as soon as it is saved. The worker never starts a run less than a minute + after the previous one started. A time missed while no worker ran (every + instance down, or runs switched off in **Admin → Retention**) is made up + once, as soon as the worker runs again; set `max_run_seconds` to keep such + a run short. +- **Run now** (`POST /admin/repositories//retention/run`) runs the + stored policy at the worker's next poll, whatever its schedule or interval + says. The run after it is at the schedule's next time again. + +The **Retention** screen shows the schedule and the next run, and a preview +of a draft with a schedule lists its next five run times. + +**`max_run_seconds`** ends a run at the end of a maintenance window. The +run applies its decisions in batches (up to 100 versions of one package, or +files of one directory, or tags and manifests of one image); once the time +is up, it stops before the next batch. The run summary counts what it left +as `deferred`. The next run evaluates the repository again and goes on with +what is left, so a cleanup larger than one window spreads over several runs. +The first batch of a run always goes, so a run makes progress even when +evaluating the repository alone takes longer than the window. -Add `size_budget` to the policy envelope: +## Package rules + +A package rule selects versions and unlists or deletes them: + +```json +{ + "id": "ci-prereleases", + "action": "hard_delete", + "release_type": "prereleases", + "package_name_prefix": "acme.", + "selector": "ci-feeds", + "version_pattern": "*-ci.*", + "exclude_version_pattern": "*-ci.0", + "criteria": { + "keep_last_n": 3, + "prerelease_superseded_for_days": 14 + } +} +``` + +| Field | | +| --- | --- | +| `id` | Recorded on every audit row the rule produces; unique in the policy. `size_budget` is reserved. | +| `action` | `unlist` (the version stays downloadable by its exact version) or `hard_delete` (the version and its files are gone). | +| `scope` | `hosted` (the default) on hosted repositories, `proxy_cached` on proxies ([Proxy rules](#proxy-rules)). | +| `release_type`, `package_name_prefix`, `selector`, `version_pattern`, `exclude_version_pattern` | [Filters](#filters): the versions the rule looks at. | +| `criteria` | [Protections](#protections) and [triggers](#triggers). | + +How rules combine: + +- A version goes when **any** rule selects it. A rule selects a version + when its filters admit it and **every** trigger it sets holds. +- What **any** rule protects is never unlisted or deleted, by any rule and + whatever trigger selects it. +- When two rules select a version, deleting wins over unlisting. +- A version a package tag names (npm's `latest`, `next`, …) is never + touched. +- Unlisting a version that is unlisted already does nothing, and is not + audited again. + +### Filters + +All optional; a rule without filters looks at every version. + +| Filter | Admits | +| --- | --- | +| `release_type` | `any` (default), `releases`, `prereleases` or `snapshots`: the class the repository's format gives the version ([Formats](#formats-what-newest-and-prerelease-mean)). For NuGet and npm a prerelease has a label after a dash (`2.0.0-beta.1`). | +| `package_name_prefix` | Packages whose id starts with it, ignoring case (`acme.`). | +| `selector` | Packages a [content selector](deployment.md#access-levels) (**Admin → Content selectors**) admits, by its name: its include and exclude globs on the package id, as a grant scoped by it reads them. The selector must exist when the policy is saved; one that is deleted since fails the policy's runs (nothing is removed), and a selector a policy names cannot be deleted until no rule names it. | +| `version_pattern` | Versions this glob matches, ignoring case: `*` any characters, `?` one, `[0-9]` a set, `{*-ci.*,*-pr.*}` either. The glob covers the whole normalized version. | +| `exclude_version_pattern` | Never the versions this glob matches. | + +### Protections + +| Protection | Keeps | +| --- | --- | +| `keep_last_n` | The newest N **listed** versions of each package, among the versions the rule's filters admit. Unlisted versions neither count nor are kept by it, so after a broken release is unlisted, the N newest installable versions are still kept. | +| `keep_n_most_downloaded` | The N versions of each package downloaded last, listed or not; never-downloaded versions rank last, newest first. | + +A protection counts within the rule's own filters: a `releases` rule with +`keep_last_n: 3` keeps the three newest releases, never prereleases. A rule +needs a trigger; a rule with protections only is refused. + +### Triggers + +| Trigger | Selects a version | +| --- | --- | +| `older_than_days` | published at least N days ago. | +| `not_downloaded_for_days` | not downloaded for N days; a version never downloaded counts from its publication. | +| `prerelease_superseded_for_days` | that is a prerelease (or a snapshot) once its package has had, for N days, a **listed release that comes after it**: for SemVer, a release of the same or a higher version, so `2.3.0-ci.417` goes 14 days after `2.3.0` (or `2.4.0`) was published. The clock starts at the first such release's publication. A release is never selected. | +| `unlisted_for_days` | that has been unlisted for N days (a listed version never). Only on `hard_delete` rules: the second step of "unlist, then delete later". | +| `max_versions_per_package` | that is not among the N newest versions of its package (listed or not) the rule's filters admit. A pure count: no age needed. | + +Every count and day value is at least 1. The audit log and the preview give +each decision its reason: the triggers that held, the release that +superseded a prerelease (`superseded_by`), and a version's place past the +cap (`rank`). + +**When a version was unlisted.** Every version records it +(`component.unlisted_at`), whichever way it was unlisted: by a client +(NuGet's delete, npm), by a retention rule, pushed or imported unlisted. +Listing it again clears it. Versions unlisted before Forge recorded it take +the time of their last change, which is never earlier than their unlisting. + +### Examples + +CI prereleases go 14 days after their release, and no package keeps more +than 50 prereleases at all: + +```json +{ + "version": 1, + "schedule": "0 2 * * *", + "schedule_timezone": "Europe/Berlin", + "rules": [ + { "id": "superseded", "action": "hard_delete", "release_type": "prereleases", + "criteria": { "prerelease_superseded_for_days": 14 } }, + { "id": "too-many", "action": "hard_delete", "release_type": "prereleases", + "criteria": { "max_versions_per_package": 50 } } + ] +} +``` + +Pull-request builds go after 7 days, except the newest of each package: + +```json +{ "id": "pr-builds", "action": "hard_delete", "version_pattern": "*-pr.*", + "criteria": { "older_than_days": 7, "keep_last_n": 1 } } +``` + +Unlist, then delete later: releases unused for a year are unlisted, and +deleted 90 days after that, unless someone relisted them meanwhile; the +three newest releases of each package stay: + +```json +{ + "version": 1, + "rules": [ + { "id": "unlist-cold", "action": "unlist", "release_type": "releases", + "criteria": { "not_downloaded_for_days": 365, "keep_last_n": 3 } }, + { "id": "delete-unlisted", "action": "hard_delete", + "criteria": { "unlisted_for_days": 90 } } + ] +} +``` + +Only the packages of the `ci-feeds` content selector, outside working hours, +an hour at most per night: + +```json +{ + "version": 1, + "schedule": "0 1 * * 1-5", + "schedule_timezone": "America/New_York", + "max_run_seconds": 3600, + "rules": [ + { "id": "ci", "action": "hard_delete", "selector": "ci-feeds", + "criteria": { "older_than_days": 30, "keep_last_n": 10 } } + ] +} +``` + +## Proxy rules + +A proxy repository's rules say `"scope": "proxy_cached"` and evict cached +versions: their bytes are deleted, the version stays listed, and its next +pull fetches it from the upstream again. Proxy versions are never unlisted. + +| Trigger | Evicts a cached version | +| --- | --- | +| `not_pulled_for_days` | not pulled for N days; one never pulled counts from when the proxy first saw it. | +| `max_versions_per_package` | not among the N newest cached versions of its package the rule's filters admit, however recently pulled. | + +The filters and protections work as on hosted rules, over each package's +cached versions: `keep_last_n` keeps the newest N cached versions the +upstream lists, `keep_n_most_downloaded` the N pulled last. The hosted +triggers (`older_than_days`, `not_downloaded_for_days`, +`prerelease_superseded_for_days`, `unlisted_for_days`) are refused here. + +```json +{ "id": "evict-cold", "action": "hard_delete", "scope": "proxy_cached", + "criteria": { "not_pulled_for_days": 30, "keep_last_n": 5, "keep_n_most_downloaded": 5 } } +``` + +keeps the 5 newest and the 5 most recently pulled cached versions of each +package and evicts any other one nobody pulled for 30 days. + +## Formats: what "newest" and "prerelease" mean + +Versions are ordered by the repository's format, never as text: NuGet orders +`1.10.0` after `1.9.0` and a prerelease before its release. Two retention +hooks of the format contract +([format plugins](format-plugins.md#retention)) say the rest: + +- **The version class**: release, prerelease or snapshot. `release_type` + filters on it, and `prerelease_superseded_for_days` deletes what is not a + release. +- **The partition** keep-N counts within (`keep_last_n`, + `keep_n_most_downloaded`, `max_versions_per_package`): the whole package + unless the format splits it. + +NuGet and npm use the generic rules: a version with a `-` is a prerelease, +any other a release, and a package is one partition. Helm charts are stored +in Docker repositories and follow the [image rules](#image-rules). Maven and +apt bring their own when they arrive: Maven's `-SNAPSHOT` builds are +snapshots counted per snapshot version (the newest N builds of each), its +releases per `groupId:artifactId`, and `prerelease_superseded_for_days` +removes a version's snapshots once its release exists; apt counts per +package and architecture. + +## Raw file rules + +Raw repositories have file rules: `path_prefix` (only files under it), +`keep_newest_per_directory` (the N most recently deployed files of each +directory stay), and the triggers `older_than_days` and +`not_downloaded_for_days`. See [Raw file repositories](raw.md#retention). + +```json +{ "id": "nightlies", "path_prefix": "nightly/", "keep_newest_per_directory": 5, + "older_than_days": 14 } +``` + +## Image rules + +Docker repositories have image rules: per image (`image_prefix`), tag rules +with `keep_last_n_tags`, `protected_tags`, `tag_older_than_days` and +`tag_not_pulled_for_days`, and `delete_untagged_after_days` for manifests +left without a tag. A tag rule looks at the tags `tag_pattern` matches and +`exclude_tag_pattern` does not (all tags without either); `protected_tags` +takes names and globs. See [Docker / OCI registry](docker.md#retention-and-garbage-collection). + +```json +{ "id": "pr-tags", "tag_pattern": "pr-*", "tag_older_than_days": 7, + "protected_tags": ["latest", "release-*"] } +``` + +deletes `pr-*` tags a week after their push, whatever else the image has. + +## Preview, runs and the audit log + +- `POST /admin/repositories//retention/preview` evaluates the stored + policy, or a draft sent as `{"policy": {…}}` (checked like a `PUT` of it, + never stored, evaluated even when disabled), and lists what a run would do + now, with the rule and the reason of each decision. It changes nothing. + A draft with a schedule also lists its next runs (`next_runs`). +- A run re-checks each batch before it applies it: it stops when the policy + changed or it lost its lease (the next run goes on under the policy as it + is then), and skips what the repository's current state no longer + warrants (a version downloaded, relisted or protected since). +- `GET /admin/repositories//retention` returns the policy and the run + state: the next run, the last run's times, error and summary (`evaluated`, + `unlisted`, `deleted`, `skipped`, `deferred`, the blobs queued for + deletion, and a size budget's sizes). +- `GET …/retention/audit` lists every decision of every run, dry runs + included, newest first, filtered by `run_id` and paged with `before`. + +Deleted content goes through the blob outbox: a blob is deleted from the +object store only once nothing references it any more. + +## Size budgets + +Hosted and proxy package repositories, raw file repositories, and Docker +repositories can set a logical size budget in their retention policy. Add +`size_budget` to the policy envelope: ```json { diff --git a/migrations/0011_retention_unlisted_at.sql b/migrations/0011_retention_unlisted_at.sql new file mode 100644 index 00000000..7fc8ccc4 --- /dev/null +++ b/migrations/0011_retention_unlisted_at.sql @@ -0,0 +1,45 @@ +-- --------------------------------------------------------------------------- +-- When a version was unlisted (#185) +-- +-- Retention's `unlisted_for_days` trigger deletes versions that have been +-- unlisted for N days (the second step of "unlist, then delete later"), so a +-- version records when it was unlisted: `component.unlisted_at`, NULL while +-- it is listed. +-- +-- A trigger keeps it, so every path that unlists records it, whatever it is: +-- a client's unlist (NuGet's DELETE, npm), a retention run's `unlist`, a +-- push or import of a version that comes unlisted, a redeploy that unlists. +-- It is set when `listed` becomes false (or a row is inserted unlisted), +-- kept while the version stays unlisted (unlisting it again changes +-- nothing), and cleared when it is listed again. A write that sets +-- `unlisted_at` itself on an unlisted version keeps that value. +-- +-- Versions unlisted before this migration take their `updated_at`: every +-- path that unlists a version sets `updated_at` to that moment, so it is +-- never earlier than the version was unlisted. A version changed again after +-- it was unlisted counts from that later change, which only defers its +-- deletion. +-- --------------------------------------------------------------------------- +ALTER TABLE component ADD COLUMN unlisted_at timestamptz NULL; + +UPDATE component SET unlisted_at = updated_at WHERE NOT listed; + +CREATE FUNCTION component_unlisted_at() RETURNS trigger +LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.listed THEN + NEW.unlisted_at := NULL; + ELSIF TG_OP = 'INSERT' OR OLD.listed THEN + NEW.unlisted_at := COALESCE(NEW.unlisted_at, now()); + ELSE + -- Still unlisted: it keeps the time it was unlisted, unless the write + -- sets another one. + NEW.unlisted_at := COALESCE(NEW.unlisted_at, OLD.unlisted_at, now()); + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER component_unlisted_at + BEFORE INSERT OR UPDATE OF listed, unlisted_at ON component + FOR EACH ROW EXECUTE FUNCTION component_unlisted_at(); diff --git a/web/src/api/retention.ts b/web/src/api/retention.ts index e2a36256..8e38429c 100644 --- a/web/src/api/retention.ts +++ b/web/src/api/retention.ts @@ -9,7 +9,7 @@ import { httpClient } from '~/lib/httpClient' * which arms the worker; `DELETE` removes it. * - `POST …/preview` dry-runs the stored policy, or a draft `policy` from * the body: checked like a `PUT` of it, evaluated even when disabled, and - * never stored. + * never stored. A policy with a `schedule` also lists its next runs. * - `POST …/run` makes the policy due now; the worker starts the run on its * next poll. 409 without a policy, or while retention runs are switched * off for every repository (`/admin/settings/retention`). @@ -20,15 +20,20 @@ import { httpClient } from '~/lib/httpClient' * (`crates/core/src/retention/{model,files,images}.rs`): package rules for * NuGet and other package formats (hosted rules, or on a proxy * `scope: proxy_cached` eviction rules), file rules for raw repositories, - * image rules for docker. Every shape has the same envelope. Unknown fields - * are refused (422); an invalid rule is a 400 naming it. + * image rules for docker. Every shape has the same envelope, with its + * scheduling: `min_run_interval_seconds`, or a cron `schedule` in a + * `schedule_timezone`, and `max_run_seconds`. Unknown fields are refused + * (422); an invalid rule is a 400 naming it. */ /** What a package rule does with a version it selects. */ export type RetentionAction = 'unlist' | 'hard_delete' -/** The versions a package rule looks at. */ -export type ReleaseType = 'any' | 'releases' | 'prereleases' +/** + * The versions a package rule looks at, by the class the format gives a + * version (snapshots: formats that have them, such as Maven). + */ +export type ReleaseType = 'any' | 'releases' | 'prereleases' | 'snapshots' /** A package rule's protections and triggers (all optional). */ export interface PackageCriteria { @@ -38,6 +43,12 @@ export interface PackageCriteria { not_downloaded_for_days?: number /** Proxy rules only. */ not_pulled_for_days?: number + /** Hosted rules only: a prerelease once a newer release is this old. */ + prerelease_superseded_for_days?: number + /** Hosted rules only, with `hard_delete`: unlisted this long. */ + unlisted_for_days?: number + /** Not among the newest N versions of its package (a pure count). */ + max_versions_per_package?: number } export interface PackageRule { @@ -47,6 +58,11 @@ export interface PackageRule { scope?: 'hosted' | 'proxy_cached' release_type?: ReleaseType package_name_prefix?: string | null + /** A content selector's name: only the packages it admits. */ + selector?: string + /** A glob on the version, ignoring case (`*-ci.*`). */ + version_pattern?: string + exclude_version_pattern?: string criteria: PackageCriteria } @@ -63,6 +79,9 @@ export interface FileRule { export interface ImageRule { id: string image_prefix?: string + /** A glob on the tag (`pr-*`): only the tags it matches. */ + tag_pattern?: string + exclude_tag_pattern?: string keep_last_n_tags?: number protected_tags?: string[] tag_older_than_days?: number @@ -86,6 +105,12 @@ export interface RetentionPolicy { dry_run_only?: boolean /** Default 3600. Runs are at least a minute apart whatever this says. */ min_run_interval_seconds?: number + /** A 5-field cron pattern; runs at its times instead of the interval. */ + schedule?: string + /** The IANA time zone of `schedule` (default UTC). */ + schedule_timezone?: string + /** A run stops after this long; the next one goes on with the rest. */ + max_run_seconds?: number size_budget?: SizeBudget rules: R[] } @@ -97,6 +122,8 @@ export interface RunSummary { deleted?: number blobs_enqueued?: number skipped?: number + /** Left for the next run: the run reached its `max_run_seconds`. */ + deferred?: number size_before?: number size_after?: number target_bytes?: number @@ -184,6 +211,8 @@ export interface PreviewView { size_after?: number target_bytes?: number target_reached?: boolean + /** With a `schedule`: its next runs (RFC 3339, UTC). */ + next_runs?: string[] } /** diff --git a/web/src/components/RetentionPolicyForm.test.ts b/web/src/components/RetentionPolicyForm.test.ts index 026d61c9..9a2fa631 100644 --- a/web/src/components/RetentionPolicyForm.test.ts +++ b/web/src/components/RetentionPolicyForm.test.ts @@ -4,6 +4,7 @@ import type { RetentionPolicy } from '~/api/retention' import { buildPolicy, describeReason, + describeSchedule, draftFromPolicy, intervalFromSeconds, newRule, @@ -193,4 +194,92 @@ describe('retention policy model', () => { 'repository exceeds 1 GiB, target 0.5 GiB', ) }) + + it('round-trips a schedule, a time limit and the new rule fields', () => { + const stored = { + version: 1, + enabled: true, + dry_run_only: false, + min_run_interval_seconds: 3600, + schedule: '0 3 * * 1-5', + schedule_timezone: 'Europe/Berlin', + max_run_seconds: 7200, + rules: [ + { + id: 'ci', + action: 'hard_delete', + scope: 'hosted', + release_type: 'snapshots', + selector: 'ci-feeds', + version_pattern: '*-ci.*', + exclude_version_pattern: '*-ci.0', + criteria: { + prerelease_superseded_for_days: 14, + unlisted_for_days: 90, + max_versions_per_package: 20, + }, + }, + ], + } as unknown as RetentionPolicy + const draft = draftFromPolicy('package', stored) + expect(draft.scheduleMode).toBe('cron') + expect([draft.schedule, draft.scheduleTimezone]).toEqual(['0 3 * * 1-5', 'Europe/Berlin']) + expect([draft.maxRun, draft.maxRunUnit]).toEqual(['2', 'hours']) + expect(draft.rules[0].values.selector).toBe('ci-feeds') + expect(draft.rules[0].values['criteria.prerelease_superseded_for_days']).toBe('14') + expect(validateDraft('package', draft)).toBeNull() + expect(buildPolicy('package', draft)).toEqual(stored) + + // Back to an interval: the schedule goes, the time limit stays. + const interval = buildPolicy('package', { ...draft, scheduleMode: 'interval' }) + expect(interval.schedule).toBeUndefined() + expect(interval.schedule_timezone).toBeUndefined() + expect(interval.max_run_seconds).toBe(7200) + // Without a time zone or a limit, neither is sent; nor an empty selector. + draft.scheduleTimezone = ' ' + draft.maxRun = '' + draft.rules[0].values.selector = '' + const plain = buildPolicy('package', draft) + expect(plain.schedule_timezone).toBeUndefined() + expect(plain.max_run_seconds).toBeUndefined() + expect('selector' in plain.rules[0]).toBe(false) + }) + + it('checks a cron pattern and a time limit', () => { + const draft = { ...draftFromPolicy('image', null), scheduleMode: 'cron' as const } + expect(validateDraft('image', draft)?.schedule).toMatch(/5 fields/) + for (const ok of ['0 3 * * *', '*/15 2-4 * * MON-FRI', '@daily']) { + expect(validateDraft('image', { ...draft, schedule: ok })).toBeNull() + } + expect(validateDraft('image', { ...draft, schedule: '0 3 * *' })?.schedule).toBeDefined() + const timed = { ...draft, schedule: '0 3 * * *', maxRunUnit: 'seconds' as const } + expect(validateDraft('image', { ...timed, maxRun: '30' })?.maxRun).toMatch(/minute/) + expect(validateDraft('image', { ...timed, maxRun: '60' })).toBeNull() + expect( + validateDraft('image', { ...timed, maxRun: '8', maxRunUnit: 'days' })?.maxRun, + ).toMatch(/week/) + }) + + it('says why the new triggers selected a version, and when a policy runs', () => { + expect( + describeReason({ prerelease_superseded_for_days: 14, superseded_by: '2.3.0' }), + ).toBe('superseded by 2.3.0 for over 14 days') + expect(describeReason({ unlisted_for_days: 90 })).toBe('unlisted for 90 days') + expect(describeReason({ max_versions_per_package: 3, rank: 5 })).toBe( + 'number 5 of its package, beyond the newest 3', + ) + const base = { version: 1, rules: [] } + expect(describeSchedule({ ...base, min_run_interval_seconds: 3600 })).toBe( + 'at most every 1 hour', + ) + expect( + describeSchedule({ + ...base, + schedule: '0 3 * * *', + schedule_timezone: 'Europe/Berlin', + max_run_seconds: 5400, + }), + ).toBe('0 3 * * * (Europe/Berlin); a run stops after 90 minutes') + expect(describeSchedule({ ...base, schedule: '@daily' })).toBe('@daily (UTC)') + }) }) diff --git a/web/src/components/RetentionPolicyForm.tsx b/web/src/components/RetentionPolicyForm.tsx index 5341079b..c0c2428f 100644 --- a/web/src/components/RetentionPolicyForm.tsx +++ b/web/src/components/RetentionPolicyForm.tsx @@ -44,7 +44,7 @@ export function policyShape(repo: { return repo.kind === 'proxy' ? 'proxy' : 'package' } -type FieldKind = 'count' | 'days' | 'text' | 'tags' | 'select' +type FieldKind = 'count' | 'days' | 'text' | 'tags' | 'select' | 'selector' interface FieldSpec { /** The field's place in the rule's JSON. */ @@ -85,11 +85,13 @@ const RELEASE_TYPE: FieldSpec = { path: ['release_type'], label: 'Versions', kind: 'select', - description: 'A prerelease has a label after a dash, such as 2.0.0-beta.1.', + description: + 'As the format classes a version: a prerelease has a label after a dash, such as 2.0.0-beta.1; snapshots are builds such as Maven’s -SNAPSHOT ones.', options: [ { value: 'any', label: 'Every version' }, { value: 'releases', label: 'Releases only' }, { value: 'prereleases', label: 'Prereleases only' }, + { value: 'snapshots', label: 'Snapshots only' }, ], } @@ -101,6 +103,47 @@ const PACKAGE_PREFIX: FieldSpec = { description: 'Only packages whose id starts with this, ignoring case. Empty: every package.', } +const SELECTOR: FieldSpec = { + path: ['selector'], + label: 'Content selector', + kind: 'selector', + description: + 'Only packages a content selector admits (Admin → Content selectors). Empty: every package.', +} + +const VERSION_PATTERN: FieldSpec = { + path: ['version_pattern'], + label: 'Version pattern', + kind: 'text', + placeholder: '*-ci.*', + description: + 'Only versions this glob matches, ignoring case; {*-ci.*,*-pr.*} for either. Empty: every version.', +} + +const EXCLUDE_VERSION_PATTERN: FieldSpec = { + path: ['exclude_version_pattern'], + label: 'Except versions matching', + kind: 'text', + placeholder: '*-rc.*', + description: 'Never the versions this glob matches.', +} + +const PACKAGE_FILTERS = [ + RELEASE_TYPE, + PACKAGE_PREFIX, + SELECTOR, + VERSION_PATTERN, + EXCLUDE_VERSION_PATTERN, +] + +const MAX_VERSIONS: FieldSpec = { + path: ['criteria', 'max_versions_per_package'], + label: 'More than N versions', + kind: 'count', + description: + 'Not among the newest N versions of its package (among the versions above), listed or not. Needs no age.', +} + const SHAPES: Record = { package: { rulesNote: @@ -118,11 +161,11 @@ const SHAPES: Record = { }, ], groups: [ - { legend: 'Versions it looks at', fields: [RELEASE_TYPE, PACKAGE_PREFIX] }, + { legend: 'Versions it looks at', fields: PACKAGE_FILTERS }, { legend: 'Always keep', description: - 'Per package, among the versions above. A version any rule keeps is never unlisted or deleted.', + 'Per package, among the versions above. A version any rule keeps is never unlisted or deleted, whatever selects it.', fields: [ { path: ['criteria', 'keep_last_n'], @@ -153,6 +196,21 @@ const SHAPES: Record = { kind: 'days', description: 'A version never downloaded counts from its publication.', }, + { + path: ['criteria', 'prerelease_superseded_for_days'], + label: 'Superseded by a release for (days)', + kind: 'days', + description: + 'A prerelease, once a release that comes after it has been out this long: 2.3.0-ci.417 once 2.3.0 is.', + }, + { + path: ['criteria', 'unlisted_for_days'], + label: 'Unlisted for (days)', + kind: 'days', + description: + 'Unlisted at least this long: with “Delete”, the second step after a rule that unlists.', + }, + MAX_VERSIONS, ], }, ], @@ -170,7 +228,7 @@ const SHAPES: Record = { 'A cached version is evicted when any rule selects it. What any rule keeps is never evicted, so each package can keep a working set however long nobody pulls it.', head: [], groups: [ - { legend: 'Versions it looks at', fields: [RELEASE_TYPE, PACKAGE_PREFIX] }, + { legend: 'Versions it looks at', fields: PACKAGE_FILTERS }, { legend: 'Always keep', description: @@ -201,6 +259,11 @@ const SHAPES: Record = { kind: 'days', description: 'A version never pulled counts from when the proxy first saw it.', }, + { + ...MAX_VERSIONS, + description: + 'Not among the newest N cached versions of its package (among the versions above), however recently pulled.', + }, ], }, ], @@ -282,6 +345,20 @@ const SHAPES: Record = { placeholder: 'team/', description: 'Only images whose name starts with this. Empty: every image.', }, + { + path: ['tag_pattern'], + label: 'Tag pattern', + kind: 'text', + placeholder: 'pr-*', + description: 'Only tags this glob matches, as written. Empty: every tag.', + }, + { + path: ['exclude_tag_pattern'], + label: 'Except tags matching', + kind: 'text', + placeholder: 'v*', + description: 'Never the tags this glob matches.', + }, ], }, { @@ -298,8 +375,9 @@ const SHAPES: Record = { path: ['protected_tags'], label: 'Protected tags', kind: 'tags', - placeholder: 'latest, stable', - description: 'Never deleted by this rule. Separate them with commas or spaces.', + placeholder: 'latest, release-*', + description: + 'Never deleted by this rule: names or globs. Separate them with commas or spaces.', }, ], }, @@ -344,6 +422,15 @@ const SHAPES: Record = { }, } +const UNIT_OPTIONS = ( + <> + + + + + +) + function fieldsOf(shape: PolicyShape): FieldSpec[] { const spec = SHAPES[shape] return [RULE_ID, ...spec.head, ...spec.groups.flatMap((g) => g.fields)] @@ -421,8 +508,16 @@ export interface PolicyDraft { base: Record enabled: boolean dryRunOnly: boolean + /** Runs at most every `interval`, or at the times of a cron `schedule`. */ + scheduleMode: 'interval' | 'cron' interval: string intervalUnit: IntervalUnit + /** A 5-field cron pattern, and its IANA time zone (empty: UTC). */ + schedule: string + scheduleTimezone: string + /** How long a run may take (empty: no limit), in `maxRunUnit`. */ + maxRun: string + maxRunUnit: IntervalUnit /** The size budget's maximum and target, in `sizeBudgetUnit`. */ sizeBudgetMax: string sizeBudgetTarget: string @@ -492,8 +587,13 @@ export function draftFromPolicy( base: { version: 1 }, enabled: true, dryRunOnly: true, + scheduleMode: 'interval', interval: '1', intervalUnit: 'hours', + schedule: '', + scheduleTimezone: '', + maxRun: '', + maxRunUnit: 'hours', sizeBudgetMax: '', sizeBudgetTarget: '', sizeBudgetUnit: 'GiB', @@ -505,12 +605,19 @@ export function draftFromPolicy( const budget = policy.size_budget const sizes = budget ? [budget.max_bytes, budget.target_bytes ?? 0] : [] const unit = unitOf(sizes) + const maxRun = + policy.max_run_seconds === undefined ? null : intervalFromSeconds(policy.max_run_seconds) return { base: structuredClone(base), enabled: policy.enabled ?? true, dryRunOnly: policy.dry_run_only ?? false, + scheduleMode: policy.schedule ? 'cron' : 'interval', interval: interval.value, intervalUnit: interval.unit, + schedule: policy.schedule ?? '', + scheduleTimezone: policy.schedule_timezone ?? '', + maxRun: maxRun?.value ?? '', + maxRunUnit: maxRun?.unit ?? 'hours', sizeBudgetMax: budget ? amountIn(budget.max_bytes, unit) : '', sizeBudgetTarget: budget?.target_bytes === undefined ? '' : amountIn(budget.target_bytes, unit), @@ -522,6 +629,8 @@ export function draftFromPolicy( /** What is wrong with the form, field by field. */ export interface DraftErrors { interval?: string + schedule?: string + maxRun?: string sizeBudgetMax?: string sizeBudgetTarget?: string /** A message for the rules as a whole. */ @@ -532,6 +641,11 @@ export interface DraftErrors { const WHOLE = /^\d+$/ const DECIMAL = /^\d+(\.\d+)?$/ +/** Five fields (the server checks each), or a nickname such as @daily. */ +const CRON = /^(@[a-z]+|\S+(\s+\S+){4})$/i +/** The bounds of `max_run_seconds`: a minute and a week. */ +const MIN_MAX_RUN = 60 +const MAX_MAX_RUN = 7 * 86400 /** The browser's checks; `null` when the form can be sent. */ export function validateDraft(shape: PolicyShape, draft: PolicyDraft): DraftErrors | null { @@ -542,6 +656,18 @@ export function validateDraft(shape: PolicyShape, draft: PolicyDraft): DraftErro errors.interval = 'Enter a whole number of at least 1.' failed = true } + if (draft.scheduleMode === 'cron' && !CRON.test(draft.schedule.trim())) { + errors.schedule = 'Enter a cron pattern of 5 fields, such as 0 3 * * *.' + failed = true + } + const maxRun = draft.maxRun.trim() + if (maxRun !== '') { + const seconds = Number(maxRun) * UNIT_SECONDS[draft.maxRunUnit] + if (!WHOLE.test(maxRun) || seconds < MIN_MAX_RUN || seconds > MAX_MAX_RUN) { + errors.maxRun = 'Enter a time between a minute and a week, or leave it empty.' + failed = true + } + } const max = draft.sizeBudgetMax.trim() const target = draft.sizeBudgetTarget.trim() const validMax = DECIMAL.test(max) && Number(max) > 0 @@ -607,7 +733,7 @@ export function buildPolicy(shape: PolicyShape, draft: PolicyDraft): RetentionPo } else if (field.kind === 'tags') { const tags = raw.split(/[\s,]+/).filter((t) => t !== '') value = tags.length > 0 ? tags : undefined - } else if (field.kind === 'text' && raw === '' && field !== RULE_ID) { + } else if ((field.kind === 'text' || field.kind === 'selector') && raw === '' && field !== RULE_ID) { value = undefined } else { value = raw @@ -617,6 +743,18 @@ export function buildPolicy(shape: PolicyShape, draft: PolicyDraft): RetentionPo return json as unknown as RetentionRule }) const base = structuredClone(draft.base) + if (draft.scheduleMode === 'cron') { + base.schedule = draft.schedule.trim().replace(/\s+/g, ' ') + const zone = draft.scheduleTimezone.trim() + if (zone === '') delete base.schedule_timezone + else base.schedule_timezone = zone + } else { + delete base.schedule + delete base.schedule_timezone + } + const maxRun = draft.maxRun.trim() + if (maxRun === '') delete base.max_run_seconds + else base.max_run_seconds = Number(maxRun) * UNIT_SECONDS[draft.maxRunUnit] const max = draft.sizeBudgetMax.trim() const target = draft.sizeBudgetTarget.trim() if (max === '') { @@ -646,6 +784,24 @@ export function policyDeletes(shape: PolicyShape, policy: RetentionPolicy): bool return policy.rules.some((r) => 'action' in r && r.action === 'hard_delete') } +/** When a stored policy runs: its schedule, or its interval. */ +export function describeSchedule(policy: RetentionPolicy): string { + const limit = + policy.max_run_seconds === undefined + ? '' + : `; a run stops after ${durationText(policy.max_run_seconds)}` + if (policy.schedule) { + return `${policy.schedule} (${policy.schedule_timezone ?? 'UTC'})${limit}` + } + return `at most every ${durationText(policy.min_run_interval_seconds ?? 3600)}${limit}` +} + +function durationText(seconds: number): string { + const { value, unit } = intervalFromSeconds(seconds) + const n = Number(value) + return `${n.toLocaleString()} ${n === 1 ? unit.replace(/s$/, '') : unit}` +} + function plural(n: number, one: string, many = `${one}s`): string { return `${n.toLocaleString()} ${n === 1 ? one : many}` } @@ -681,10 +837,19 @@ function days(n: number): string { return plural(n, 'day') } -const REASONS: Record string> = { +const REASONS: Record string> = { older_than_days: (n) => `older than ${days(n)}`, not_downloaded_for_days: (n) => `not downloaded for ${days(n)}`, not_pulled_for_days: (n) => `not pulled for ${days(n)}`, + prerelease_superseded_for_days: (n, r) => + typeof r.superseded_by === 'string' + ? `superseded by ${r.superseded_by} for over ${days(n)}` + : `superseded by a release for over ${days(n)}`, + unlisted_for_days: (n) => `unlisted for ${days(n)}`, + max_versions_per_package: (n, r) => + typeof r.rank === 'number' + ? `number ${r.rank} of its package, beyond the newest ${n}` + : `beyond the newest ${n} of its package`, keep_newest_per_directory: (n) => `not among the ${n} newest in its directory`, keep_last_n_tags: (n) => `not among the ${n} newest tags`, tag_older_than_days: (n) => `pushed over ${days(n)} ago`, @@ -704,7 +869,7 @@ export function describeReason(reason: DecisionReason | null | undefined): strin if (!reason) return '' return Object.entries(REASONS) .filter(([key]) => typeof reason[key] === 'number') - .map(([key, phrase]) => phrase(reason[key] as number)) + .map(([key, phrase]) => phrase(reason[key] as number, reason)) .join(', ') } @@ -718,12 +883,15 @@ export function RetentionPolicyForm({ draft, errors, currentSizeBytes = 0, + selectors, onChange, }: { shape: PolicyShape draft: PolicyDraft errors: DraftErrors | null currentSizeBytes?: number + /** The content selectors a rule may name; `undefined`: typed by name. */ + selectors?: string[] onChange: (draft: PolicyDraft) => void }) { const spec = SHAPES[shape] @@ -761,27 +929,82 @@ export function RetentionPolicyForm({ description="Runs record in the audit log what they would remove, and remove nothing." /> +
+ + onChange({ + ...draft, + scheduleMode: e.target.value as PolicyDraft['scheduleMode'], + }) + } + description="On a schedule, the worker runs the policy at its times only, such as outside working hours." + > + + + + {draft.scheduleMode === 'interval' ? ( + <> + onChange({ ...draft, interval: e.target.value })} + error={errors?.interval} + description="Run now does not wait for it. Runs are at least a minute apart." + /> + + onChange({ ...draft, intervalUnit: e.target.value as IntervalUnit }) + } + > + {UNIT_OPTIONS} + + + ) : ( + <> + onChange({ ...draft, schedule: e.target.value })} + error={errors?.schedule} + description="Minute, hour, day of month, month, day of week: 0 3 * * * is 03:00 every day, 0 3 * * 6 Saturdays. Run now does not wait for it." + /> + onChange({ ...draft, scheduleTimezone: e.target.value })} + description="An IANA time zone; UTC when empty. The schedule keeps its local time across daylight-saving changes." + /> + + )} +
onChange({ ...draft, interval: e.target.value })} - error={errors?.interval} - description="Run now does not wait for it. Runs are at least a minute apart." + value={draft.maxRun} + onChange={(e) => onChange({ ...draft, maxRun: e.target.value })} + error={errors?.maxRun} + description="A run still going then stops before its next batch, and the next run goes on with the rest. Empty: no limit." /> - onChange({ ...draft, intervalUnit: e.target.value as IntervalUnit }) - } + label="Time unit" + value={draft.maxRunUnit} + onChange={(e) => onChange({ ...draft, maxRunUnit: e.target.value as IntervalUnit })} > - - - - + {UNIT_OPTIONS}
@@ -834,6 +1057,7 @@ export function RetentionPolicyForm({ spec={spec} rule={rule} errors={errors?.fields[rule.key]} + selectors={selectors} onChange={(values) => setRule(rule.key, values)} onMove={(by) => move(i, by)} onRemove={() => @@ -871,6 +1095,7 @@ function RuleFields({ spec, rule, errors, + selectors, onChange, onMove, onRemove, @@ -880,6 +1105,7 @@ function RuleFields({ spec: ShapeSpec rule: RuleDraft errors?: Record + selectors?: string[] onChange: (values: Record) => void onMove: (by: -1 | 1) => void onRemove: () => void @@ -941,6 +1167,7 @@ function RuleFields({ field={f} rule={rule} error={errors?.[keyOf(f)]} + selectors={selectors} onChange={onChange} /> ))} @@ -955,16 +1182,37 @@ function Field({ field, rule, error, + selectors, onChange, }: { field: FieldSpec rule: RuleDraft error?: string + selectors?: string[] onChange: (values: Record) => void }) { const key = keyOf(field) const value = rule.values[key] const set = (v: string) => onChange({ ...rule.values, [key]: v }) + if (field.kind === 'selector' && selectors) { + return ( + set(e.target.value)} + description={field.description} + > + + {selectors.map((name) => ( + + ))} + {/* A selector deleted since stays as loaded; the server refuses it. */} + {value && !selectors.includes(value) && } + + ) + } if (field.kind === 'select') { return ( s.name).sort() + : undefined const [draft, setDraft] = useState(() => draftFromPolicy(shape, stored)) const [errors, setErrors] = useState(null) const [refused, setRefused] = useState(null) @@ -368,6 +376,7 @@ function PolicyPanel({ draft={draft} errors={errors} currentSizeBytes={currentSizeBytes} + selectors={selectors} onChange={change} /> {errors && ( @@ -582,6 +591,16 @@ function PreviewPanel({ nothing. )} + {previewed.result.next_runs && previewed.result.next_runs.length > 0 && ( +
+

Next runs

+
    + {previewed.result.next_runs.map((at) => ( +
  • {formatDateTime(at)}
    • + ))} +
+
+ )} {previewSections(shape, previewed.result) .filter((s) => s.rows.length > 0) .map((s) => ( @@ -699,6 +718,7 @@ function RunPanel({ ), }, { term: 'Next run', value: nextRun }, + ...(policy ? [{ term: 'Schedule', value: describeSchedule(policy) }] : []), { term: 'Last started', value: formatDateTime(ledger.started_at) }, { term: 'Last finished', value: formatDateTime(ledger.finished_at) }, ...(summary @@ -711,6 +731,14 @@ function RunPanel({ term: 'Blobs queued for deletion', value: String(summary.blobs_enqueued ?? 0), }, + ...(summary.deferred + ? [ + { + term: 'Left for the next run', + value: `${summary.deferred}: the run reached its time limit`, + }, + ] + : []), ] : []), ]} @@ -742,7 +770,7 @@ function RunPanel({

{policy - ? 'Runs the stored policy without waiting for the minimum time between runs.' + ? 'Runs the stored policy now, without waiting for its schedule or the minimum time between runs.' : 'Save a policy to run it.'}

diff --git a/web/src/routes/admin/repositories/retention.test.tsx b/web/src/routes/admin/repositories/retention.test.tsx index bc7f6dcc..3b641f84 100644 --- a/web/src/routes/admin/repositories/retention.test.tsx +++ b/web/src/routes/admin/repositories/retention.test.tsx @@ -178,6 +178,14 @@ function mockApi(h: Handlers) { const body = init?.body ? JSON.parse(init.body as string) : {} if (path.endsWith('/admin/me')) return Promise.resolve(json(adminMe)) + if (path.endsWith('/admin/content-selectors') && method === 'GET') { + return Promise.resolve( + json([ + { id: 1, name: 'ci-feeds', description: '', includes: ['acme.*'], excludes: [] }, + { id: 2, name: 'all-internal', description: '', includes: [], excludes: [] }, + ]), + ) + } if (path.endsWith('/retention/preview') && method === 'POST') { return Promise.resolve(h.onPreview ? h.onPreview(body) : json(preview({}))) } @@ -664,4 +672,94 @@ describe('retention screen', () => { }), ) }) + + it('edits a schedule and the new filters and triggers, and shows the next runs', async () => { + const scheduled: RetentionPolicy = { + ...storedPolicy, + schedule: '0 3 * * *', + schedule_timezone: 'Europe/Berlin', + max_run_seconds: 3600, + } + let previewed: { policy?: RetentionPolicy } | undefined + mockApi({ + repositories: [repo({})], + view: { + policy: scheduled, + last_run: { + ...ledger, + last_run_summary: { ...ledger!.last_run_summary!, deferred: 40 }, + }, + runs_enabled: true, + current_size_bytes: 0, + }, + onPreview: (body) => { + previewed = body + return json( + preview({ + decisions: [ + { + component_id: 21, + package_name: 'acme.core', + version: '2.3.0-ci.417', + action: 'hard_delete', + rule_id: 'stale-releases', + reason: { prerelease_superseded_for_days: 14, superseded_by: '2.3.0' }, + }, + ], + next_runs: ['2026-11-01T02:00:00Z', '2026-11-02T02:00:00Z'], + }), + ) + }, + }) + await renderAt('/manage/repositories/nuget-internal/retention') + expect(await screen.findByLabelText('When it runs')).toHaveValue('cron') + expect(screen.getByLabelText('Cron schedule')).toHaveValue('0 3 * * *') + expect(screen.getByLabelText(/^Time zone/)).toHaveValue('Europe/Berlin') + expect(screen.getByLabelText(/^Stop a run after/)).toHaveValue('1') + expect(screen.queryByLabelText('Minimum time between runs')).not.toBeInTheDocument() + const runs = within(screen.getByRole('complementary', { name: 'Runs' })) + expect( + runs.getByText('0 3 * * * (Europe/Berlin); a run stops after 1 hour'), + ).toBeInTheDocument() + expect(runs.getByText('40: the run reached its time limit')).toBeInTheDocument() + + // The content selectors are offered by name, once they are loaded. + await waitFor(() => + expect(rule(1).getByLabelText(/^Content selector/).tagName).toBe('SELECT'), + ) + const selector = rule(1).getByLabelText(/^Content selector/) + expect(within(selector).getByRole('option', { name: 'ci-feeds' })).toBeInTheDocument() + await userEvent.selectOptions(selector, 'ci-feeds') + await userEvent.type(rule(1).getByLabelText(/^Version pattern/), '*-ci.*') + await userEvent.type(rule(1).getByLabelText(/^Superseded by a release for/), '14') + await userEvent.type(rule(1).getByLabelText(/^More than N versions/), '50') + await userEvent.clear(screen.getByLabelText('Cron schedule')) + await userEvent.type(screen.getByLabelText('Cron schedule'), '30 2 * * 6') + await userEvent.click(screen.getByRole('button', { name: 'Preview' })) + + expect(await screen.findByText('superseded by 2.3.0 for over 14 days')).toBeInTheDocument() + expect(screen.getByRole('heading', { name: 'Next runs' })).toBeInTheDocument() + expect(previewed?.policy).toMatchObject({ + schedule: '30 2 * * 6', + schedule_timezone: 'Europe/Berlin', + max_run_seconds: 3600, + rules: [ + { + selector: 'ci-feeds', + version_pattern: '*-ci.*', + criteria: { + keep_last_n: 5, + older_than_days: 180, + prerelease_superseded_for_days: 14, + max_versions_per_package: 50, + }, + }, + ], + }) + + // Back to an interval. + await userEvent.selectOptions(screen.getByLabelText('When it runs'), 'interval') + expect(screen.getByLabelText('Minimum time between runs')).toHaveValue('1') + expect(screen.queryByLabelText('Cron schedule')).not.toBeInTheDocument() + }) })