From 09da4f94117f7026ce656a787c9aca19144c7775 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 30 Oct 2026 08:53:07 +0000 Subject: [PATCH] Import through the format's own protocol, named neutrally An import found what to copy through Nexus 3's REST API, so it could only import from a Nexus. It now reads its source through the format's own protocol, through the format plugin's upstream conversations, so a hosted repository imports from any server that speaks it: - NuGet: a v3 feed's service index. Its search (walked to an empty page: Nexus reports a page's hits as the total) names the package ids, the package base address lists every version, unlisted ones included, and serves the .nupkg, the registration gives the publish time and, where inline, the SHA-512 the download is verified against and a re-run compares without downloading. A new UpstreamOp::Packages walks the search past nuget.org's offset cap. - Docker: a registry's catalog (a new UpstreamOp::ImageCatalog), its tag lists, and the referrers API for each copied manifest; a registry without it keeps referrers under tags (cosign's .sig, the referrers tag schema), copied like any other. - An import can name its package ids or images instead, for a source without a search or catalog. - An imported version counts as downloaded at its import, so retention's not_downloaded_for_days does not delete it right after; nuget.org's 1900-01-01 is no publish time; a tag is created at its import. The module, tables (migration 0007), API view, web UI and docs drop the Nexus name. The previous request (a Nexus url and repository) is still accepted, and an import stored before the upgrade lists and resumes, its password still opening. Fixes #642 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012e6DJXRhLYxFjzrWiwKFKc --- ...a38c1c9f2483a89334487bd1cb8c196203b80.json | 17 - ...a5522d6d31cc44898a3d69f2b4c7fdb816382.json | 16 - ...3304967d5300fb2e1dc8252a1fce6ec02e78.json} | 34 +- ...3cf00faf57b77f147a3740cb46b8a8d8e0af6.json | 14 - ...00705407882063f5df1901be42b18a22a2b2a.json | 17 + ...20378689533f6cc9ed1ce98283c4470704c9a.json | 14 + ...dc45008bdac65951b6576fe55c51339b65085.json | 17 - ...43f1f435f702711e7b9e1c4a0246c59c72f7.json} | 6 +- ...db56b445baf0fbd06c0f2694cc4d31605a22.json} | 12 +- ...35b1889669d3f11a30eb5bf9db8bd220e8869.json | 15 + ...89698d5671ef4dbb4069ee79e15bebe547ea8.json | 14 + ...1dc6f50cba5847f5c970c6eae1b54660a8da2.json | 16 + ...c3b999e72258f0ef99aaa10bd8fa57c36ea3.json} | 8 +- ...fcd037828f51721884ce144e1585c30d77743.json | 14 - ...682d3d940e7d7dc575caaf1bccae9c49d5159.json | 14 - ...310a4e1813055046ab567f0de726b8df12570.json | 28 + ...278b7e198a4b3f4c71452eb964ce0f1e2cbe2.json | 22 + ...e4c4bda7f09eca3169e18aa9d18e272a5895.json} | 11 +- ...fdab40c67cb99c0ef2b5b2043cb90bfe062da.json | 30 - ...3d93c4345a6d47d512c9c52e79cde2ca82a1.json} | 6 +- ...f452b51384d82aeb35ef5fdec699ac93d9f21.json | 17 + ...9c9f91d9d89631712a7c6877c386849c132c4.json | 20 - ...7dcaa2726c343de9870b5b97a2c7253b596f.json} | 16 +- ...25c5fa2c6941903be30227fa3546b5e027218.json | 18 + ...9d93622dc0f39c69883e5f375d08dbd6b728.json} | 110 +- ...7c4914086eba14eca1441e4580df1450c7601.json | 22 - ...985a0220e34b780b144ec22d9e1e908d227bf.json | 14 - ...2e233cf506d7a1bf8c4cb4d8b215402c889c1.json | 21 + ...5f737b9b577dd404f7a4661c32235c2328f2.json} | 110 +- ...a4c8b5af033bb15b9cbb68c4d49a1297a4ef8.json | 31 + ...7ca0fe027527b85313cc532c68f92749b40c6.json | 14 + ...9bd142b8e2c86eb2a67d1112dd41fdbfa4d9.json} | 34 +- ...06717cf27e2048aaec1e0872db3981be69be.json} | 4 +- ...9289ce31989989e995e61c72f9a84a87edd7d.json | 14 + ...533bf723383b068c26b6f214d907d1762a9b.json} | 4 +- ...0133aa6305e6b9ec0fa570e3b5192ce5384dd.json | 15 - ...3408869c26546a1f77b1dcd0fa836866cd030.json | 15 + ...0117c00dfd6241855b08490cda7c1a1ce136.json} | 6 +- ...9869f241e62762f17773039410806fd7fd6d.json} | 6 +- ...22c752ad03e90239ba01e75c91ee6d68da98.json} | 4 +- ...281d784f0d9220cbcd18c3d3a9a7e86c29d2.json} | 14 +- ...8ce32a87eb4ffe41e499aad73e218dd739de0.json | 30 + README.md | 7 +- crates/core/src/repo/hosted.rs | 15 +- crates/core/src/repo/mod.rs | 2 +- crates/db/src/import_dao.rs | 153 +- crates/db/src/secret_dao.rs | 54 +- crates/db/tests/migrations.rs | 177 +- crates/e2e/tests/import.rs | 784 ++++++++ crates/e2e/tests/nexus_import.rs | 464 ----- crates/formats/docker/src/upstream.rs | 274 ++- crates/formats/npm/src/proxy.rs | 8 +- crates/formats/nuget/src/upstream.rs | 58 +- crates/formats/nuget/tests/upstream.rs | 24 + crates/plugin-api/src/images.rs | 9 + crates/plugin-api/src/types.rs | 19 +- crates/server/src/admin/imports.rs | 311 +-- crates/server/src/admin/mod.rs | 5 +- crates/server/src/images/import.rs | 106 +- crates/server/src/import/feed.rs | 332 ++++ crates/server/src/import/mod.rs | 1353 +++++++++++++ crates/server/src/lib.rs | 2 +- crates/server/src/nexus_import/mod.rs | 1084 ---------- crates/server/src/nexus_import/nexus.rs | 533 ----- crates/server/src/workers.rs | 20 +- crates/server/tests/import.rs | 1745 +++++++++++++++++ crates/server/tests/nexus_import.rs | 1227 ------------ docs/README.md | 7 +- docs/SUMMARY.md | 3 +- docs/changes/2.2.0/642.feature.md | 13 + docs/changes/2.2.0/642.upgrade.md | 12 + docs/docker.md | 7 +- docs/import.md | 167 ++ docs/nexus-import.md | 161 +- docs/raw.md | 4 +- migrations/0007_repository_import.sql | 73 + web/src/api/imports.ts | 42 +- web/src/routes/admin/repositories/Edit.tsx | 4 +- web/src/routes/admin/repositories/Import.tsx | 148 +- .../admin/repositories/imports.test.tsx | 117 +- web/tsconfig.tsbuildinfo | 1 + 81 files changed, 6160 insertions(+), 4229 deletions(-) delete mode 100644 .sqlx/query-02096c4b9d7da63d20ffa11be1aa38c1c9f2483a89334487bd1cb8c196203b80.json delete mode 100644 .sqlx/query-0321fa382dc868e2b659aaee5b9a5522d6d31cc44898a3d69f2b4c7fdb816382.json rename .sqlx/{query-d148cf931844bf52f505ccbde587d756e1a69b96bd96020da0f90f2a1b577ca8.json => query-03a52d53318f67093894ef691d0a3304967d5300fb2e1dc8252a1fce6ec02e78.json} (78%) delete mode 100644 .sqlx/query-0892948fb2d5b15fe0e8acd943a3cf00faf57b77f147a3740cb46b8a8d8e0af6.json create mode 100644 .sqlx/query-22132c561cf7d98feec907b61a000705407882063f5df1901be42b18a22a2b2a.json create mode 100644 .sqlx/query-24384fd673779f613494a3a2bcd20378689533f6cc9ed1ce98283c4470704c9a.json delete mode 100644 .sqlx/query-27ce5074610a907aa6cedbd9134dc45008bdac65951b6576fe55c51339b65085.json rename .sqlx/{query-aa70d0f56b9eebce1d354725f91d79f4aed0e0c61b5bac577cdc0feb68fa682c.json => query-2be9ffd4d8e233c91e4dcf2aab2c43f1f435f702711e7b9e1c4a0246c59c72f7.json} (79%) rename .sqlx/{query-7ed7bb86ff75aadcfbd46695f058f621775ee3b418f659c63b8237827cd453f7.json => query-32a681fad30926b74bbf1a6a49a7db56b445baf0fbd06c0f2694cc4d31605a22.json} (66%) create mode 100644 .sqlx/query-426f15afec440cec0f4f1dc397035b1889669d3f11a30eb5bf9db8bd220e8869.json create mode 100644 .sqlx/query-430f45430f035e0b7e728f7a44289698d5671ef4dbb4069ee79e15bebe547ea8.json create mode 100644 .sqlx/query-4fcb159341379dc599bf179e2721dc6f50cba5847f5c970c6eae1b54660a8da2.json rename .sqlx/{query-6deebbc753318b4d9542a4bdc25bb39500dc50f8d5e769bafd70ab13f73f60d4.json => query-4fd9d491b78faf8a40f16cf48ccdc3b999e72258f0ef99aaa10bd8fa57c36ea3.json} (66%) delete mode 100644 .sqlx/query-514969a81cf1fbe6b56a5e16b77fcd037828f51721884ce144e1585c30d77743.json delete mode 100644 .sqlx/query-52e5b03af038c311c1371b70575682d3d940e7d7dc575caaf1bccae9c49d5159.json create mode 100644 .sqlx/query-69ff060e1912e70357acfb8686f310a4e1813055046ab567f0de726b8df12570.json create mode 100644 .sqlx/query-6bb1cefc87f2512e2c69f00edaf278b7e198a4b3f4c71452eb964ce0f1e2cbe2.json rename .sqlx/{query-6ab60461c61652841ad8b7665046ba2f541ac9757cbe5aa08385753172586a64.json => query-6f9abe1f956bcac2d04dbf9b72c2e4c4bda7f09eca3169e18aa9d18e272a5895.json} (51%) delete mode 100644 .sqlx/query-75ae409c1803522bfb9d7349caefdab40c67cb99c0ef2b5b2043cb90bfe062da.json rename .sqlx/{query-4a31af4ec2a7e0ee7f4b2a1ca245e4ced1c380df88ca4f293ea949354b75e748.json => query-7be04e96c8e7d1ecbd1b1580a4df3d93c4345a6d47d512c9c52e79cde2ca82a1.json} (80%) create mode 100644 .sqlx/query-8a6deee6a71756a41027067c6c3f452b51384d82aeb35ef5fdec699ac93d9f21.json delete mode 100644 .sqlx/query-8b6f6a8fdad5aa9e5e98a581adb9c9f91d9d89631712a7c6877c386849c132c4.json rename .sqlx/{query-f2beb1ca7da114b1b63b853bf907b6d2fb07b1085064a01ca0a1eacb919b968f.json => query-8bae46d737dd65a10f034985beed7dcaa2726c343de9870b5b97a2c7253b596f.json} (69%) create mode 100644 .sqlx/query-8c90b5d5aa289dbd657480f0f7625c5fa2c6941903be30227fa3546b5e027218.json rename .sqlx/{query-b76fec429ab9b2826c95429795c138f7a72c1822f254b8315099035d6155bfeb.json => query-90727fe56ec79034408235b7f1319d93622dc0f39c69883e5f375d08dbd6b728.json} (71%) delete mode 100644 .sqlx/query-96aab5a8c7d6fee5601f80ab2d37c4914086eba14eca1441e4580df1450c7601.json delete mode 100644 .sqlx/query-97df002c3a51803fba427abfd3b985a0220e34b780b144ec22d9e1e908d227bf.json create mode 100644 .sqlx/query-9e18c564a49d360756f43c8f4a82e233cf506d7a1bf8c4cb4d8b215402c889c1.json rename .sqlx/{query-b4bc64780d014552c6c7d13101e3724e0bbc78d1e58793b56845d8e3109e9850.json => query-a597333c3c4a106ff419d7c1a3c75f737b9b577dd404f7a4661c32235c2328f2.json} (72%) create mode 100644 .sqlx/query-abcedd12415a3ff65f4bc84ab54a4c8b5af033bb15b9cbb68c4d49a1297a4ef8.json create mode 100644 .sqlx/query-c16bc17f5325a9e69bdbde3581c7ca0fe027527b85313cc532c68f92749b40c6.json rename .sqlx/{query-1217d97d8f397d87e27e6812a0929ec05d3d2cd8cf224a7a7fa8ebcad7edd7f4.json => query-ca60f5642b21b9598e45c13aa3e19bd142b8e2c86eb2a67d1112dd41fdbfa4d9.json} (78%) rename .sqlx/{query-b6951beb73f63f1829cf4034adc59bee26841388c176eebc9df5e5cc7b8b2461.json => query-cb435f03a1c8d8835dad30de6bc006717cf27e2048aaec1e0872db3981be69be.json} (61%) create mode 100644 .sqlx/query-cbf56582f6a6d29f724c916948d9289ce31989989e995e61c72f9a84a87edd7d.json rename .sqlx/{query-73e966747c8f9e8836c585a347b2896a1b67fa4811b229680ee30a300c001336.json => query-dc9a111531650d4a65b04b93c6e9533bf723383b068c26b6f214d907d1762a9b.json} (53%) delete mode 100644 .sqlx/query-e37a479c00daf5ec6ef04745ba90133aa6305e6b9ec0fa570e3b5192ce5384dd.json create mode 100644 .sqlx/query-e37c0a167cfaa3eee14c30633c63408869c26546a1f77b1dcd0fa836866cd030.json rename .sqlx/{query-8f94720def905b052e17c6fbcc3108c22eea77884ac5660079e4bbe1b8ca6663.json => query-e932ab583448f575388b9abfe0ba0117c00dfd6241855b08490cda7c1a1ce136.json} (66%) rename .sqlx/{query-7f7e92513dd74c34a940da96356ccef52fcc54fe48b3e52111158f505d55da7b.json => query-f17cac96289359ceb7e0c08868c49869f241e62762f17773039410806fd7fd6d.json} (60%) rename .sqlx/{query-182c705b9213322dcd113f316840d91291e82a00f19cb9243c0a9ed2005f790c.json => query-f9fd2bb73a05dc4932f6483282b222c752ad03e90239ba01e75c91ee6d68da98.json} (52%) rename .sqlx/{query-140e585934ee4f40b43117d5ac312fcb5390bf968547265fa2771a28b9380cd0.json => query-fdb3ed8114bfe2e9193b887f66c3281d784f0d9220cbcd18c3d3a9a7e86c29d2.json} (66%) create mode 100644 .sqlx/query-ff454fad056b994bafa995d317a8ce32a87eb4ffe41e499aad73e218dd739de0.json create mode 100644 crates/e2e/tests/import.rs delete mode 100644 crates/e2e/tests/nexus_import.rs create mode 100644 crates/server/src/import/feed.rs create mode 100644 crates/server/src/import/mod.rs delete mode 100644 crates/server/src/nexus_import/mod.rs delete mode 100644 crates/server/src/nexus_import/nexus.rs create mode 100644 crates/server/tests/import.rs delete mode 100644 crates/server/tests/nexus_import.rs create mode 100644 docs/changes/2.2.0/642.feature.md create mode 100644 docs/changes/2.2.0/642.upgrade.md create mode 100644 docs/import.md create mode 100644 migrations/0007_repository_import.sql create mode 100644 web/tsconfig.tsbuildinfo diff --git a/.sqlx/query-02096c4b9d7da63d20ffa11be1aa38c1c9f2483a89334487bd1cb8c196203b80.json b/.sqlx/query-02096c4b9d7da63d20ffa11be1aa38c1c9f2483a89334487bd1cb8c196203b80.json deleted file mode 100644 index 2fc51792..00000000 --- a/.sqlx/query-02096c4b9d7da63d20ffa11be1aa38c1c9f2483a89334487bd1cb8c196203b80.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nexus_import\n SET state = $3,\n last_error = $4,\n finished_at = now(),\n updated_at = now(),\n run_id = NULL,\n locked_until = NULL,\n cancel_requested = false,\n password_enc = CASE WHEN $3 = 'completed' THEN NULL ELSE password_enc END\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Uuid", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "02096c4b9d7da63d20ffa11be1aa38c1c9f2483a89334487bd1cb8c196203b80" -} diff --git a/.sqlx/query-0321fa382dc868e2b659aaee5b9a5522d6d31cc44898a3d69f2b4c7fdb816382.json b/.sqlx/query-0321fa382dc868e2b659aaee5b9a5522d6d31cc44898a3d69f2b4c7fdb816382.json deleted file mode 100644 index cbb06963..00000000 --- a/.sqlx/query-0321fa382dc868e2b659aaee5b9a5522d6d31cc44898a3d69f2b4c7fdb816382.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nexus_import SET locked_until = $3\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Uuid", - "Timestamptz" - ] - }, - "nullable": [] - }, - "hash": "0321fa382dc868e2b659aaee5b9a5522d6d31cc44898a3d69f2b4c7fdb816382" -} diff --git a/.sqlx/query-d148cf931844bf52f505ccbde587d756e1a69b96bd96020da0f90f2a1b577ca8.json b/.sqlx/query-03a52d53318f67093894ef691d0a3304967d5300fb2e1dc8252a1fce6ec02e78.json similarity index 78% rename from .sqlx/query-d148cf931844bf52f505ccbde587d756e1a69b96bd96020da0f90f2a1b577ca8.json rename to .sqlx/query-03a52d53318f67093894ef691d0a3304967d5300fb2e1dc8252a1fce6ec02e78.json index 1dd582a1..fa637eef 100644 --- a/.sqlx/query-d148cf931844bf52f505ccbde587d756e1a69b96bd96020da0f90f2a1b577ca8.json +++ b/.sqlx/query-03a52d53318f67093894ef691d0a3304967d5300fb2e1dc8252a1fce6ec02e78.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id, import_id, kind, name, version, state, detail, size, bytes_copied,\n published_at, downloaded_at, uploader, source, attempts, updated_at\n FROM nexus_import_item\n WHERE import_id = $1 AND ($2::text IS NULL OR state = $2) AND id > $3\n ORDER BY id\n LIMIT $4\n ", + "query": "\n SELECT id, import_id, kind, name, version, state, detail, size, bytes_copied,\n published_at, downloaded_at, uploader, source, attempts, updated_at\n FROM repository_import_item\n WHERE import_id = $1 AND kind = $2 AND state = 'pending' AND id > $3\n ORDER BY id\n LIMIT $4\n ", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "id" } } @@ -20,7 +20,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "import_id" } } @@ -31,7 +31,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "kind" } } @@ -42,7 +42,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "name" } } @@ -53,7 +53,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "version" } } @@ -64,7 +64,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "state" } } @@ -75,7 +75,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "detail" } } @@ -86,7 +86,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "size" } } @@ -97,7 +97,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "bytes_copied" } } @@ -108,7 +108,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "published_at" } } @@ -119,7 +119,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "downloaded_at" } } @@ -130,7 +130,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "uploader" } } @@ -141,7 +141,7 @@ "type_info": "Jsonb", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "source" } } @@ -152,7 +152,7 @@ "type_info": "Int4", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "attempts" } } @@ -163,7 +163,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "updated_at" } } @@ -195,5 +195,5 @@ false ] }, - "hash": "d148cf931844bf52f505ccbde587d756e1a69b96bd96020da0f90f2a1b577ca8" + "hash": "03a52d53318f67093894ef691d0a3304967d5300fb2e1dc8252a1fce6ec02e78" } diff --git a/.sqlx/query-0892948fb2d5b15fe0e8acd943a3cf00faf57b77f147a3740cb46b8a8d8e0af6.json b/.sqlx/query-0892948fb2d5b15fe0e8acd943a3cf00faf57b77f147a3740cb46b8a8d8e0af6.json deleted file mode 100644 index 4e37a47b..00000000 --- a/.sqlx/query-0892948fb2d5b15fe0e8acd943a3cf00faf57b77f147a3740cb46b8a8d8e0af6.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nexus_import\n SET state = CASE WHEN state = 'queued' THEN 'cancelled' ELSE state END,\n finished_at = CASE WHEN state = 'queued' THEN now() ELSE finished_at END,\n cancel_requested = (state = 'running'),\n updated_at = now()\n WHERE id = $1 AND state IN ('queued', 'running')\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "0892948fb2d5b15fe0e8acd943a3cf00faf57b77f147a3740cb46b8a8d8e0af6" -} diff --git a/.sqlx/query-22132c561cf7d98feec907b61a000705407882063f5df1901be42b18a22a2b2a.json b/.sqlx/query-22132c561cf7d98feec907b61a000705407882063f5df1901be42b18a22a2b2a.json new file mode 100644 index 00000000..55830d13 --- /dev/null +++ b/.sqlx/query-22132c561cf7d98feec907b61a000705407882063f5df1901be42b18a22a2b2a.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE repository_import_item\n SET state = $2, detail = $3, bytes_copied = bytes_copied + $4,\n attempts = attempts + 1, updated_at = now()\n WHERE id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "22132c561cf7d98feec907b61a000705407882063f5df1901be42b18a22a2b2a" +} diff --git a/.sqlx/query-24384fd673779f613494a3a2bcd20378689533f6cc9ed1ce98283c4470704c9a.json b/.sqlx/query-24384fd673779f613494a3a2bcd20378689533f6cc9ed1ce98283c4470704c9a.json new file mode 100644 index 00000000..1334ec90 --- /dev/null +++ b/.sqlx/query-24384fd673779f613494a3a2bcd20378689533f6cc9ed1ce98283c4470704c9a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE repository_import SET locked_until = now() - interval '1 second' WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "24384fd673779f613494a3a2bcd20378689533f6cc9ed1ce98283c4470704c9a" +} diff --git a/.sqlx/query-27ce5074610a907aa6cedbd9134dc45008bdac65951b6576fe55c51339b65085.json b/.sqlx/query-27ce5074610a907aa6cedbd9134dc45008bdac65951b6576fe55c51339b65085.json deleted file mode 100644 index dc94645f..00000000 --- a/.sqlx/query-27ce5074610a907aa6cedbd9134dc45008bdac65951b6576fe55c51339b65085.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nexus_import_item\n SET state = $2, detail = $3, bytes_copied = bytes_copied + $4,\n attempts = attempts + 1, updated_at = now()\n WHERE id = $1\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text", - "Text", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "27ce5074610a907aa6cedbd9134dc45008bdac65951b6576fe55c51339b65085" -} diff --git a/.sqlx/query-aa70d0f56b9eebce1d354725f91d79f4aed0e0c61b5bac577cdc0feb68fa682c.json b/.sqlx/query-2be9ffd4d8e233c91e4dcf2aab2c43f1f435f702711e7b9e1c4a0246c59c72f7.json similarity index 79% rename from .sqlx/query-aa70d0f56b9eebce1d354725f91d79f4aed0e0c61b5bac577cdc0feb68fa682c.json rename to .sqlx/query-2be9ffd4d8e233c91e4dcf2aab2c43f1f435f702711e7b9e1c4a0246c59c72f7.json index a1a573ec..ab8e02a8 100644 --- a/.sqlx/query-aa70d0f56b9eebce1d354725f91d79f4aed0e0c61b5bac577cdc0feb68fa682c.json +++ b/.sqlx/query-2be9ffd4d8e233c91e4dcf2aab2c43f1f435f702711e7b9e1c4a0246c59c72f7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT state,\n count(*) AS \"items!\",\n coalesce(sum(size), 0)::bigint AS \"size!\",\n coalesce(sum(bytes_copied), 0)::bigint AS \"bytes_copied!\"\n FROM nexus_import_item\n WHERE import_id = $1\n GROUP BY state\n ", + "query": "\n SELECT state,\n count(*) AS \"items!\",\n coalesce(sum(size), 0)::bigint AS \"size!\",\n coalesce(sum(bytes_copied), 0)::bigint AS \"bytes_copied!\"\n FROM repository_import_item\n WHERE import_id = $1\n GROUP BY state\n ", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "state" } } @@ -45,5 +45,5 @@ null ] }, - "hash": "aa70d0f56b9eebce1d354725f91d79f4aed0e0c61b5bac577cdc0feb68fa682c" + "hash": "2be9ffd4d8e233c91e4dcf2aab2c43f1f435f702711e7b9e1c4a0246c59c72f7" } diff --git a/.sqlx/query-7ed7bb86ff75aadcfbd46695f058f621775ee3b418f659c63b8237827cd453f7.json b/.sqlx/query-32a681fad30926b74bbf1a6a49a7db56b445baf0fbd06c0f2694cc4d31605a22.json similarity index 66% rename from .sqlx/query-7ed7bb86ff75aadcfbd46695f058f621775ee3b418f659c63b8237827cd453f7.json rename to .sqlx/query-32a681fad30926b74bbf1a6a49a7db56b445baf0fbd06c0f2694cc4d31605a22.json index 20992ffe..97b94364 100644 --- a/.sqlx/query-7ed7bb86ff75aadcfbd46695f058f621775ee3b418f659c63b8237827cd453f7.json +++ b/.sqlx/query-32a681fad30926b74bbf1a6a49a7db56b445baf0fbd06c0f2694cc4d31605a22.json @@ -1,16 +1,16 @@ { "db_name": "PostgreSQL", - "query": "SELECT base_url, registry_url FROM nexus_import WHERE id = $1 FOR UPDATE", + "query": "SELECT url, registry_url FROM repository_import WHERE id = $1 FOR UPDATE", "describe": { "columns": [ { "ordinal": 0, - "name": "base_url", + "name": "url", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", - "name": "base_url" + "table": "repository_import", + "name": "url" } } }, @@ -20,7 +20,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "registry_url" } } @@ -36,5 +36,5 @@ true ] }, - "hash": "7ed7bb86ff75aadcfbd46695f058f621775ee3b418f659c63b8237827cd453f7" + "hash": "32a681fad30926b74bbf1a6a49a7db56b445baf0fbd06c0f2694cc4d31605a22" } diff --git a/.sqlx/query-426f15afec440cec0f4f1dc397035b1889669d3f11a30eb5bf9db8bd220e8869.json b/.sqlx/query-426f15afec440cec0f4f1dc397035b1889669d3f11a30eb5bf9db8bd220e8869.json new file mode 100644 index 00000000..e6b36ef2 --- /dev/null +++ b/.sqlx/query-426f15afec440cec0f4f1dc397035b1889669d3f11a30eb5bf9db8bd220e8869.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE repository_import_item\n SET state = 'pending', detail = NULL, updated_at = now()\n WHERE import_id = $1 AND state = 'failed'\n AND ($2::bigint[] IS NULL OR id = ANY($2))\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "426f15afec440cec0f4f1dc397035b1889669d3f11a30eb5bf9db8bd220e8869" +} diff --git a/.sqlx/query-430f45430f035e0b7e728f7a44289698d5671ef4dbb4069ee79e15bebe547ea8.json b/.sqlx/query-430f45430f035e0b7e728f7a44289698d5671ef4dbb4069ee79e15bebe547ea8.json new file mode 100644 index 00000000..9938a228 --- /dev/null +++ b/.sqlx/query-430f45430f035e0b7e728f7a44289698d5671ef4dbb4069ee79e15bebe547ea8.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM repository_import WHERE id = $1 AND state NOT IN ('queued', 'running')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "430f45430f035e0b7e728f7a44289698d5671ef4dbb4069ee79e15bebe547ea8" +} diff --git a/.sqlx/query-4fcb159341379dc599bf179e2721dc6f50cba5847f5c970c6eae1b54660a8da2.json b/.sqlx/query-4fcb159341379dc599bf179e2721dc6f50cba5847f5c970c6eae1b54660a8da2.json new file mode 100644 index 00000000..db48abc8 --- /dev/null +++ b/.sqlx/query-4fcb159341379dc599bf179e2721dc6f50cba5847f5c970c6eae1b54660a8da2.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE repository_import SET locked_until = $3\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Uuid", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "4fcb159341379dc599bf179e2721dc6f50cba5847f5c970c6eae1b54660a8da2" +} diff --git a/.sqlx/query-6deebbc753318b4d9542a4bdc25bb39500dc50f8d5e769bafd70ab13f73f60d4.json b/.sqlx/query-4fd9d491b78faf8a40f16cf48ccdc3b999e72258f0ef99aaa10bd8fa57c36ea3.json similarity index 66% rename from .sqlx/query-6deebbc753318b4d9542a4bdc25bb39500dc50f8d5e769bafd70ab13f73f60d4.json rename to .sqlx/query-4fd9d491b78faf8a40f16cf48ccdc3b999e72258f0ef99aaa10bd8fa57c36ea3.json index 035afdd6..929d2f46 100644 --- a/.sqlx/query-6deebbc753318b4d9542a4bdc25bb39500dc50f8d5e769bafd70ab13f73f60d4.json +++ b/.sqlx/query-4fd9d491b78faf8a40f16cf48ccdc3b999e72258f0ef99aaa10bd8fa57c36ea3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, password_enc AS \"password_enc!\" FROM nexus_import WHERE password_enc IS NOT NULL", + "query": "SELECT id, password_enc AS \"password_enc!\" FROM repository_import WHERE password_enc IS NOT NULL", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "id" } } @@ -20,7 +20,7 @@ "type_info": "Bytea", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "password_enc" } } @@ -34,5 +34,5 @@ true ] }, - "hash": "6deebbc753318b4d9542a4bdc25bb39500dc50f8d5e769bafd70ab13f73f60d4" + "hash": "4fd9d491b78faf8a40f16cf48ccdc3b999e72258f0ef99aaa10bd8fa57c36ea3" } diff --git a/.sqlx/query-514969a81cf1fbe6b56a5e16b77fcd037828f51721884ce144e1585c30d77743.json b/.sqlx/query-514969a81cf1fbe6b56a5e16b77fcd037828f51721884ce144e1585c30d77743.json deleted file mode 100644 index ce68159e..00000000 --- a/.sqlx/query-514969a81cf1fbe6b56a5e16b77fcd037828f51721884ce144e1585c30d77743.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM nexus_import WHERE id = $1 AND state NOT IN ('queued', 'running')", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "514969a81cf1fbe6b56a5e16b77fcd037828f51721884ce144e1585c30d77743" -} diff --git a/.sqlx/query-52e5b03af038c311c1371b70575682d3d940e7d7dc575caaf1bccae9c49d5159.json b/.sqlx/query-52e5b03af038c311c1371b70575682d3d940e7d7dc575caaf1bccae9c49d5159.json deleted file mode 100644 index 8c8a0fec..00000000 --- a/.sqlx/query-52e5b03af038c311c1371b70575682d3d940e7d7dc575caaf1bccae9c49d5159.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE nexus_import SET locked_until = now() - interval '1 second' WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "52e5b03af038c311c1371b70575682d3d940e7d7dc575caaf1bccae9c49d5159" -} diff --git a/.sqlx/query-69ff060e1912e70357acfb8686f310a4e1813055046ab567f0de726b8df12570.json b/.sqlx/query-69ff060e1912e70357acfb8686f310a4e1813055046ab567f0de726b8df12570.json new file mode 100644 index 00000000..38b28078 --- /dev/null +++ b/.sqlx/query-69ff060e1912e70357acfb8686f310a4e1813055046ab567f0de726b8df12570.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM app_user WHERE username = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "app_user", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "69ff060e1912e70357acfb8686f310a4e1813055046ab567f0de726b8df12570" +} diff --git a/.sqlx/query-6bb1cefc87f2512e2c69f00edaf278b7e198a4b3f4c71452eb964ce0f1e2cbe2.json b/.sqlx/query-6bb1cefc87f2512e2c69f00edaf278b7e198a4b3f4c71452eb964ce0f1e2cbe2.json new file mode 100644 index 00000000..a8c5b810 --- /dev/null +++ b/.sqlx/query-6bb1cefc87f2512e2c69f00edaf278b7e198a4b3f4c71452eb964ce0f1e2cbe2.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO repository_import_item\n (import_id, kind, name, version, size, published_at, downloaded_at,\n uploader, source)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ON CONFLICT (import_id, kind, name, version) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text", + "Text", + "Int8", + "Timestamptz", + "Timestamptz", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "6bb1cefc87f2512e2c69f00edaf278b7e198a4b3f4c71452eb964ce0f1e2cbe2" +} diff --git a/.sqlx/query-6ab60461c61652841ad8b7665046ba2f541ac9757cbe5aa08385753172586a64.json b/.sqlx/query-6f9abe1f956bcac2d04dbf9b72c2e4c4bda7f09eca3169e18aa9d18e272a5895.json similarity index 51% rename from .sqlx/query-6ab60461c61652841ad8b7665046ba2f541ac9757cbe5aa08385753172586a64.json rename to .sqlx/query-6f9abe1f956bcac2d04dbf9b72c2e4c4bda7f09eca3169e18aa9d18e272a5895.json index deabe649..709cf099 100644 --- a/.sqlx/query-6ab60461c61652841ad8b7665046ba2f541ac9757cbe5aa08385753172586a64.json +++ b/.sqlx/query-6f9abe1f956bcac2d04dbf9b72c2e4c4bda7f09eca3169e18aa9d18e272a5895.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO nexus_import\n (repository_id, base_url, source_repository, registry_url, username,\n on_conflict, concurrency, dry_run, nexus_version, created_by)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)\n RETURNING id\n ", + "query": "\n INSERT INTO repository_import\n (repository_id, url, names, username, on_conflict, concurrency, dry_run,\n scan_listing, source_server, source_total, created_by)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)\n RETURNING id\n ", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "id" } } @@ -19,13 +19,14 @@ "Left": [ "Int8", "Text", - "Text", - "Text", + "TextArray", "Text", "Text", "Int4", "Bool", "Text", + "Text", + "Int8", "Int8" ] }, @@ -33,5 +34,5 @@ false ] }, - "hash": "6ab60461c61652841ad8b7665046ba2f541ac9757cbe5aa08385753172586a64" + "hash": "6f9abe1f956bcac2d04dbf9b72c2e4c4bda7f09eca3169e18aa9d18e272a5895" } diff --git a/.sqlx/query-75ae409c1803522bfb9d7349caefdab40c67cb99c0ef2b5b2043cb90bfe062da.json b/.sqlx/query-75ae409c1803522bfb9d7349caefdab40c67cb99c0ef2b5b2043cb90bfe062da.json deleted file mode 100644 index e338d0c0..00000000 --- a/.sqlx/query-75ae409c1803522bfb9d7349caefdab40c67cb99c0ef2b5b2043cb90bfe062da.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH due AS (\n SELECT id FROM nexus_import\n WHERE state IN ('queued', 'running')\n AND (locked_until IS NULL OR locked_until < $1)\n ORDER BY id\n FOR UPDATE SKIP LOCKED\n LIMIT 1\n )\n UPDATE nexus_import i\n SET state = 'running',\n run_id = $2,\n locked_until = $1 + make_interval(secs => $3),\n started_at = coalesce(i.started_at, $1),\n updated_at = $1\n FROM due\n WHERE i.id = due.id\n RETURNING i.id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8", - "origin": { - "Table": { - "table": "nexus_import", - "name": "id" - } - } - } - ], - "parameters": { - "Left": [ - "Timestamptz", - "Uuid", - "Float8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "75ae409c1803522bfb9d7349caefdab40c67cb99c0ef2b5b2043cb90bfe062da" -} diff --git a/.sqlx/query-4a31af4ec2a7e0ee7f4b2a1ca245e4ced1c380df88ca4f293ea949354b75e748.json b/.sqlx/query-7be04e96c8e7d1ecbd1b1580a4df3d93c4345a6d47d512c9c52e79cde2ca82a1.json similarity index 80% rename from .sqlx/query-4a31af4ec2a7e0ee7f4b2a1ca245e4ced1c380df88ca4f293ea949354b75e748.json rename to .sqlx/query-7be04e96c8e7d1ecbd1b1580a4df3d93c4345a6d47d512c9c52e79cde2ca82a1.json index e7560036..ca98e6ec 100644 --- a/.sqlx/query-4a31af4ec2a7e0ee7f4b2a1ca245e4ced1c380df88ca4f293ea949354b75e748.json +++ b/.sqlx/query-7be04e96c8e7d1ecbd1b1580a4df3d93c4345a6d47d512c9c52e79cde2ca82a1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT c.created_at, c.last_downloaded_at\n FROM component c JOIN repository r ON r.id = c.repository_id\n WHERE r.name = $1 AND c.name = 'acme.core'\n ", + "query": "\n SELECT c.created_at, c.last_downloaded_at\n FROM component c JOIN repository r ON r.id = c.repository_id\n WHERE r.name = $1 AND c.name = $2 AND c.version = $3\n ", "describe": { "columns": [ { @@ -28,6 +28,8 @@ ], "parameters": { "Left": [ + "Text", + "Text", "Text" ] }, @@ -36,5 +38,5 @@ true ] }, - "hash": "4a31af4ec2a7e0ee7f4b2a1ca245e4ced1c380df88ca4f293ea949354b75e748" + "hash": "7be04e96c8e7d1ecbd1b1580a4df3d93c4345a6d47d512c9c52e79cde2ca82a1" } diff --git a/.sqlx/query-8a6deee6a71756a41027067c6c3f452b51384d82aeb35ef5fdec699ac93d9f21.json b/.sqlx/query-8a6deee6a71756a41027067c6c3f452b51384d82aeb35ef5fdec699ac93d9f21.json new file mode 100644 index 00000000..1b6ff5d9 --- /dev/null +++ b/.sqlx/query-8a6deee6a71756a41027067c6c3f452b51384d82aeb35ef5fdec699ac93d9f21.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE repository_import\n SET state = $3,\n last_error = $4,\n finished_at = now(),\n updated_at = now(),\n run_id = NULL,\n locked_until = NULL,\n cancel_requested = false,\n password_enc = CASE WHEN $3 = 'completed' THEN NULL ELSE password_enc END\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Uuid", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "8a6deee6a71756a41027067c6c3f452b51384d82aeb35ef5fdec699ac93d9f21" +} diff --git a/.sqlx/query-8b6f6a8fdad5aa9e5e98a581adb9c9f91d9d89631712a7c6877c386849c132c4.json b/.sqlx/query-8b6f6a8fdad5aa9e5e98a581adb9c9f91d9d89631712a7c6877c386849c132c4.json deleted file mode 100644 index 8e42d73b..00000000 --- a/.sqlx/query-8b6f6a8fdad5aa9e5e98a581adb9c9f91d9d89631712a7c6877c386849c132c4.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nexus_import\n SET scan_listing = $3, cursor = $4, phase = $5,\n assets_seen = assets_seen + $6, bytes_seen = bytes_seen + $7,\n updated_at = now()\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Uuid", - "Text", - "Text", - "Text", - "Int8", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "8b6f6a8fdad5aa9e5e98a581adb9c9f91d9d89631712a7c6877c386849c132c4" -} diff --git a/.sqlx/query-f2beb1ca7da114b1b63b853bf907b6d2fb07b1085064a01ca0a1eacb919b968f.json b/.sqlx/query-8bae46d737dd65a10f034985beed7dcaa2726c343de9870b5b97a2c7253b596f.json similarity index 69% rename from .sqlx/query-f2beb1ca7da114b1b63b853bf907b6d2fb07b1085064a01ca0a1eacb919b968f.json rename to .sqlx/query-8bae46d737dd65a10f034985beed7dcaa2726c343de9870b5b97a2c7253b596f.json index c2e67703..8134162b 100644 --- a/.sqlx/query-f2beb1ca7da114b1b63b853bf907b6d2fb07b1085064a01ca0a1eacb919b968f.json +++ b/.sqlx/query-8bae46d737dd65a10f034985beed7dcaa2726c343de9870b5b97a2c7253b596f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id, base_url, registry_url, password_enc AS \"password_enc!\"\n FROM nexus_import WHERE password_enc IS NOT NULL\n ORDER BY id\n ", + "query": "\n SELECT id, url, registry_url, password_enc AS \"password_enc!\"\n FROM repository_import WHERE password_enc IS NOT NULL\n ORDER BY id\n ", "describe": { "columns": [ { @@ -9,19 +9,19 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "id" } } }, { "ordinal": 1, - "name": "base_url", + "name": "url", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", - "name": "base_url" + "table": "repository_import", + "name": "url" } } }, @@ -31,7 +31,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "registry_url" } } @@ -42,7 +42,7 @@ "type_info": "Bytea", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "password_enc" } } @@ -58,5 +58,5 @@ true ] }, - "hash": "f2beb1ca7da114b1b63b853bf907b6d2fb07b1085064a01ca0a1eacb919b968f" + "hash": "8bae46d737dd65a10f034985beed7dcaa2726c343de9870b5b97a2c7253b596f" } diff --git a/.sqlx/query-8c90b5d5aa289dbd657480f0f7625c5fa2c6941903be30227fa3546b5e027218.json b/.sqlx/query-8c90b5d5aa289dbd657480f0f7625c5fa2c6941903be30227fa3546b5e027218.json new file mode 100644 index 00000000..8644839a --- /dev/null +++ b/.sqlx/query-8c90b5d5aa289dbd657480f0f7625c5fa2c6941903be30227fa3546b5e027218.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE repository_import\n SET cursor = $3, phase = $4,\n source_total = CASE WHEN $5::bigint IS NULL THEN source_total\n ELSE greatest(coalesce(source_total, 0), $5) END,\n updated_at = now()\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Uuid", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "8c90b5d5aa289dbd657480f0f7625c5fa2c6941903be30227fa3546b5e027218" +} diff --git a/.sqlx/query-b76fec429ab9b2826c95429795c138f7a72c1822f254b8315099035d6155bfeb.json b/.sqlx/query-90727fe56ec79034408235b7f1319d93622dc0f39c69883e5f375d08dbd6b728.json similarity index 71% rename from .sqlx/query-b76fec429ab9b2826c95429795c138f7a72c1822f254b8315099035d6155bfeb.json rename to .sqlx/query-90727fe56ec79034408235b7f1319d93622dc0f39c69883e5f375d08dbd6b728.json index a7f519a1..0d3f7431 100644 --- a/.sqlx/query-b76fec429ab9b2826c95429795c138f7a72c1822f254b8315099035d6155bfeb.json +++ b/.sqlx/query-90727fe56ec79034408235b7f1319d93622dc0f39c69883e5f375d08dbd6b728.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id, repository_id, base_url, source_repository, registry_url, username,\n (password_enc IS NOT NULL) AS \"has_password!\",\n on_conflict, concurrency, dry_run, state, phase, scan_listing, cursor,\n cancel_requested, assets_seen, bytes_seen, nexus_version, last_error,\n created_by, created_at, started_at, finished_at, updated_at\n FROM nexus_import\n WHERE repository_id = $1\n ORDER BY id DESC\n LIMIT $2\n ", + "query": "\n SELECT id, repository_id, url, source_repository, registry_url, names, username,\n (password_enc IS NOT NULL) AS \"has_password!\",\n on_conflict, concurrency, dry_run, state, phase, scan_listing, cursor,\n cancel_requested, source_server, source_total,\n last_error, created_by, created_at, started_at, finished_at, updated_at\n FROM repository_import\n WHERE repository_id = $1\n ORDER BY id DESC\n LIMIT $2\n ", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "id" } } @@ -20,19 +20,19 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "repository_id" } } }, { "ordinal": 2, - "name": "base_url", + "name": "url", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", - "name": "base_url" + "table": "repository_import", + "name": "url" } } }, @@ -42,7 +42,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "source_repository" } } @@ -53,146 +53,146 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "registry_url" } } }, { "ordinal": 5, + "name": "names", + "type_info": "TextArray", + "origin": { + "Table": { + "table": "repository_import", + "name": "names" + } + } + }, + { + "ordinal": 6, "name": "username", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "username" } } }, { - "ordinal": 6, + "ordinal": 7, "name": "has_password!", "type_info": "Bool", "origin": "Expression" }, { - "ordinal": 7, + "ordinal": 8, "name": "on_conflict", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "on_conflict" } } }, { - "ordinal": 8, + "ordinal": 9, "name": "concurrency", "type_info": "Int4", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "concurrency" } } }, { - "ordinal": 9, + "ordinal": 10, "name": "dry_run", "type_info": "Bool", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "dry_run" } } }, { - "ordinal": 10, + "ordinal": 11, "name": "state", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "state" } } }, { - "ordinal": 11, + "ordinal": 12, "name": "phase", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "phase" } } }, { - "ordinal": 12, + "ordinal": 13, "name": "scan_listing", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "scan_listing" } } }, { - "ordinal": 13, + "ordinal": 14, "name": "cursor", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "cursor" } } }, { - "ordinal": 14, + "ordinal": 15, "name": "cancel_requested", "type_info": "Bool", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "cancel_requested" } } }, - { - "ordinal": 15, - "name": "assets_seen", - "type_info": "Int8", - "origin": { - "Table": { - "table": "nexus_import", - "name": "assets_seen" - } - } - }, { "ordinal": 16, - "name": "bytes_seen", - "type_info": "Int8", + "name": "source_server", + "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", - "name": "bytes_seen" + "table": "repository_import", + "name": "source_server" } } }, { "ordinal": 17, - "name": "nexus_version", - "type_info": "Text", + "name": "source_total", + "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", - "name": "nexus_version" + "table": "repository_import", + "name": "source_total" } } }, @@ -202,7 +202,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "last_error" } } @@ -213,7 +213,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "created_by" } } @@ -224,7 +224,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "created_at" } } @@ -235,7 +235,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "started_at" } } @@ -246,7 +246,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "finished_at" } } @@ -257,7 +257,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "updated_at" } } @@ -273,7 +273,8 @@ false, false, false, - false, + true, + true, true, true, null, @@ -285,8 +286,7 @@ false, true, false, - false, - false, + true, true, true, true, @@ -296,5 +296,5 @@ false ] }, - "hash": "b76fec429ab9b2826c95429795c138f7a72c1822f254b8315099035d6155bfeb" + "hash": "90727fe56ec79034408235b7f1319d93622dc0f39c69883e5f375d08dbd6b728" } diff --git a/.sqlx/query-96aab5a8c7d6fee5601f80ab2d37c4914086eba14eca1441e4580df1450c7601.json b/.sqlx/query-96aab5a8c7d6fee5601f80ab2d37c4914086eba14eca1441e4580df1450c7601.json deleted file mode 100644 index ca3f5193..00000000 --- a/.sqlx/query-96aab5a8c7d6fee5601f80ab2d37c4914086eba14eca1441e4580df1450c7601.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO nexus_import_item\n (import_id, kind, name, version, size, published_at, downloaded_at,\n uploader, source)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ON CONFLICT (import_id, kind, name, version) DO NOTHING\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text", - "Text", - "Text", - "Int8", - "Timestamptz", - "Timestamptz", - "Text", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "96aab5a8c7d6fee5601f80ab2d37c4914086eba14eca1441e4580df1450c7601" -} diff --git a/.sqlx/query-97df002c3a51803fba427abfd3b985a0220e34b780b144ec22d9e1e908d227bf.json b/.sqlx/query-97df002c3a51803fba427abfd3b985a0220e34b780b144ec22d9e1e908d227bf.json deleted file mode 100644 index c941e215..00000000 --- a/.sqlx/query-97df002c3a51803fba427abfd3b985a0220e34b780b144ec22d9e1e908d227bf.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nexus_import i\n SET state = 'queued', run_id = NULL, locked_until = NULL, finished_at = NULL,\n last_error = NULL, cancel_requested = false, updated_at = now()\n WHERE id = $1 AND state IN ('cancelled', 'failed', 'completed')\n AND NOT EXISTS (\n SELECT 1 FROM nexus_import o\n WHERE o.repository_id = i.repository_id AND o.id <> i.id\n AND o.state IN ('queued', 'running')\n )\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "97df002c3a51803fba427abfd3b985a0220e34b780b144ec22d9e1e908d227bf" -} diff --git a/.sqlx/query-9e18c564a49d360756f43c8f4a82e233cf506d7a1bf8c4cb4d8b215402c889c1.json b/.sqlx/query-9e18c564a49d360756f43c8f4a82e233cf506d7a1bf8c4cb4d8b215402c889c1.json new file mode 100644 index 00000000..84709c66 --- /dev/null +++ b/.sqlx/query-9e18c564a49d360756f43c8f4a82e233cf506d7a1bf8c4cb4d8b215402c889c1.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT coalesce(sum(a.download_count), 0)::bigint AS \"n!\"\n FROM asset a JOIN component c ON c.id = a.component_id\n JOIN repository r ON r.id = c.repository_id\n WHERE r.name = 'nuget-source'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8", + "origin": "Expression" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "9e18c564a49d360756f43c8f4a82e233cf506d7a1bf8c4cb4d8b215402c889c1" +} diff --git a/.sqlx/query-b4bc64780d014552c6c7d13101e3724e0bbc78d1e58793b56845d8e3109e9850.json b/.sqlx/query-a597333c3c4a106ff419d7c1a3c75f737b9b577dd404f7a4661c32235c2328f2.json similarity index 72% rename from .sqlx/query-b4bc64780d014552c6c7d13101e3724e0bbc78d1e58793b56845d8e3109e9850.json rename to .sqlx/query-a597333c3c4a106ff419d7c1a3c75f737b9b577dd404f7a4661c32235c2328f2.json index 4ef88075..1ff59db8 100644 --- a/.sqlx/query-b4bc64780d014552c6c7d13101e3724e0bbc78d1e58793b56845d8e3109e9850.json +++ b/.sqlx/query-a597333c3c4a106ff419d7c1a3c75f737b9b577dd404f7a4661c32235c2328f2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id, repository_id, base_url, source_repository, registry_url, username,\n (password_enc IS NOT NULL) AS \"has_password!\",\n on_conflict, concurrency, dry_run, state, phase, scan_listing, cursor,\n cancel_requested, assets_seen, bytes_seen, nexus_version, last_error,\n created_by, created_at, started_at, finished_at, updated_at\n FROM nexus_import\n WHERE id = $1\n ", + "query": "\n SELECT id, repository_id, url, source_repository, registry_url, names, username,\n (password_enc IS NOT NULL) AS \"has_password!\",\n on_conflict, concurrency, dry_run, state, phase, scan_listing, cursor,\n cancel_requested, source_server, source_total,\n last_error, created_by, created_at, started_at, finished_at, updated_at\n FROM repository_import\n WHERE id = $1\n ", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "id" } } @@ -20,19 +20,19 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "repository_id" } } }, { "ordinal": 2, - "name": "base_url", + "name": "url", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", - "name": "base_url" + "table": "repository_import", + "name": "url" } } }, @@ -42,7 +42,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "source_repository" } } @@ -53,146 +53,146 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "registry_url" } } }, { "ordinal": 5, + "name": "names", + "type_info": "TextArray", + "origin": { + "Table": { + "table": "repository_import", + "name": "names" + } + } + }, + { + "ordinal": 6, "name": "username", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "username" } } }, { - "ordinal": 6, + "ordinal": 7, "name": "has_password!", "type_info": "Bool", "origin": "Expression" }, { - "ordinal": 7, + "ordinal": 8, "name": "on_conflict", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "on_conflict" } } }, { - "ordinal": 8, + "ordinal": 9, "name": "concurrency", "type_info": "Int4", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "concurrency" } } }, { - "ordinal": 9, + "ordinal": 10, "name": "dry_run", "type_info": "Bool", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "dry_run" } } }, { - "ordinal": 10, + "ordinal": 11, "name": "state", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "state" } } }, { - "ordinal": 11, + "ordinal": 12, "name": "phase", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "phase" } } }, { - "ordinal": 12, + "ordinal": 13, "name": "scan_listing", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "scan_listing" } } }, { - "ordinal": 13, + "ordinal": 14, "name": "cursor", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "cursor" } } }, { - "ordinal": 14, + "ordinal": 15, "name": "cancel_requested", "type_info": "Bool", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "cancel_requested" } } }, - { - "ordinal": 15, - "name": "assets_seen", - "type_info": "Int8", - "origin": { - "Table": { - "table": "nexus_import", - "name": "assets_seen" - } - } - }, { "ordinal": 16, - "name": "bytes_seen", - "type_info": "Int8", + "name": "source_server", + "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", - "name": "bytes_seen" + "table": "repository_import", + "name": "source_server" } } }, { "ordinal": 17, - "name": "nexus_version", - "type_info": "Text", + "name": "source_total", + "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", - "name": "nexus_version" + "table": "repository_import", + "name": "source_total" } } }, @@ -202,7 +202,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "last_error" } } @@ -213,7 +213,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "created_by" } } @@ -224,7 +224,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "created_at" } } @@ -235,7 +235,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "started_at" } } @@ -246,7 +246,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "finished_at" } } @@ -257,7 +257,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "updated_at" } } @@ -272,7 +272,8 @@ false, false, false, - false, + true, + true, true, true, null, @@ -284,8 +285,7 @@ false, true, false, - false, - false, + true, true, true, true, @@ -295,5 +295,5 @@ false ] }, - "hash": "b4bc64780d014552c6c7d13101e3724e0bbc78d1e58793b56845d8e3109e9850" + "hash": "a597333c3c4a106ff419d7c1a3c75f737b9b577dd404f7a4661c32235c2328f2" } diff --git a/.sqlx/query-abcedd12415a3ff65f4bc84ab54a4c8b5af033bb15b9cbb68c4d49a1297a4ef8.json b/.sqlx/query-abcedd12415a3ff65f4bc84ab54a4c8b5af033bb15b9cbb68c4d49a1297a4ef8.json new file mode 100644 index 00000000..ce0e71d8 --- /dev/null +++ b/.sqlx/query-abcedd12415a3ff65f4bc84ab54a4c8b5af033bb15b9cbb68c4d49a1297a4ef8.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO repository_import\n (repository_id, url, source_repository, username, on_conflict, concurrency,\n dry_run, state, phase, scan_listing, source_server, created_by, last_error)\n VALUES ($1, $2, 'nuget-hosted', $3, 'skip', 2, false, 'failed', 'scan', 'search',\n 'Nexus/3.70.1-02 (OSS)', $4, 'Nexus is unreachable')\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "repository_import", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "abcedd12415a3ff65f4bc84ab54a4c8b5af033bb15b9cbb68c4d49a1297a4ef8" +} diff --git a/.sqlx/query-c16bc17f5325a9e69bdbde3581c7ca0fe027527b85313cc532c68f92749b40c6.json b/.sqlx/query-c16bc17f5325a9e69bdbde3581c7ca0fe027527b85313cc532c68f92749b40c6.json new file mode 100644 index 00000000..6fe8fef9 --- /dev/null +++ b/.sqlx/query-c16bc17f5325a9e69bdbde3581c7ca0fe027527b85313cc532c68f92749b40c6.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE repository_import\n SET state = CASE WHEN state = 'queued' THEN 'cancelled' ELSE state END,\n finished_at = CASE WHEN state = 'queued' THEN now() ELSE finished_at END,\n cancel_requested = (state = 'running'),\n updated_at = now()\n WHERE id = $1 AND state IN ('queued', 'running')\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "c16bc17f5325a9e69bdbde3581c7ca0fe027527b85313cc532c68f92749b40c6" +} diff --git a/.sqlx/query-1217d97d8f397d87e27e6812a0929ec05d3d2cd8cf224a7a7fa8ebcad7edd7f4.json b/.sqlx/query-ca60f5642b21b9598e45c13aa3e19bd142b8e2c86eb2a67d1112dd41fdbfa4d9.json similarity index 78% rename from .sqlx/query-1217d97d8f397d87e27e6812a0929ec05d3d2cd8cf224a7a7fa8ebcad7edd7f4.json rename to .sqlx/query-ca60f5642b21b9598e45c13aa3e19bd142b8e2c86eb2a67d1112dd41fdbfa4d9.json index 4592d1e8..a1b0f712 100644 --- a/.sqlx/query-1217d97d8f397d87e27e6812a0929ec05d3d2cd8cf224a7a7fa8ebcad7edd7f4.json +++ b/.sqlx/query-ca60f5642b21b9598e45c13aa3e19bd142b8e2c86eb2a67d1112dd41fdbfa4d9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id, import_id, kind, name, version, state, detail, size, bytes_copied,\n published_at, downloaded_at, uploader, source, attempts, updated_at\n FROM nexus_import_item\n WHERE import_id = $1 AND kind = $2 AND state = 'pending' AND id > $3\n ORDER BY id\n LIMIT $4\n ", + "query": "\n SELECT id, import_id, kind, name, version, state, detail, size, bytes_copied,\n published_at, downloaded_at, uploader, source, attempts, updated_at\n FROM repository_import_item\n WHERE import_id = $1 AND ($2::text IS NULL OR state = $2) AND id > $3\n ORDER BY id\n LIMIT $4\n ", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "id" } } @@ -20,7 +20,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "import_id" } } @@ -31,7 +31,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "kind" } } @@ -42,7 +42,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "name" } } @@ -53,7 +53,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "version" } } @@ -64,7 +64,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "state" } } @@ -75,7 +75,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "detail" } } @@ -86,7 +86,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "size" } } @@ -97,7 +97,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "bytes_copied" } } @@ -108,7 +108,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "published_at" } } @@ -119,7 +119,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "downloaded_at" } } @@ -130,7 +130,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "uploader" } } @@ -141,7 +141,7 @@ "type_info": "Jsonb", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "source" } } @@ -152,7 +152,7 @@ "type_info": "Int4", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "attempts" } } @@ -163,7 +163,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "updated_at" } } @@ -195,5 +195,5 @@ false ] }, - "hash": "1217d97d8f397d87e27e6812a0929ec05d3d2cd8cf224a7a7fa8ebcad7edd7f4" + "hash": "ca60f5642b21b9598e45c13aa3e19bd142b8e2c86eb2a67d1112dd41fdbfa4d9" } diff --git a/.sqlx/query-b6951beb73f63f1829cf4034adc59bee26841388c176eebc9df5e5cc7b8b2461.json b/.sqlx/query-cb435f03a1c8d8835dad30de6bc006717cf27e2048aaec1e0872db3981be69be.json similarity index 61% rename from .sqlx/query-b6951beb73f63f1829cf4034adc59bee26841388c176eebc9df5e5cc7b8b2461.json rename to .sqlx/query-cb435f03a1c8d8835dad30de6bc006717cf27e2048aaec1e0872db3981be69be.json index ba9be40b..c1ccfa99 100644 --- a/.sqlx/query-b6951beb73f63f1829cf4034adc59bee26841388c176eebc9df5e5cc7b8b2461.json +++ b/.sqlx/query-cb435f03a1c8d8835dad30de6bc006717cf27e2048aaec1e0872db3981be69be.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n WITH dek AS (\n SELECT 1 FROM encryption_key\n WHERE layer = 'dek' AND scope = $4 AND version = $5 AND state <> 'revoked'\n FOR SHARE\n )\n UPDATE nexus_import SET password_enc = $2\n WHERE id = $1 AND password_enc = $3 AND EXISTS (SELECT 1 FROM dek)\n ", + "query": "\n WITH dek AS (\n SELECT 1 FROM encryption_key\n WHERE layer = 'dek' AND scope = $4 AND version = $5 AND state <> 'revoked'\n FOR SHARE\n )\n UPDATE repository_import SET password_enc = $2\n WHERE id = $1 AND password_enc = $3 AND EXISTS (SELECT 1 FROM dek)\n ", "describe": { "columns": [], "parameters": { @@ -14,5 +14,5 @@ }, "nullable": [] }, - "hash": "b6951beb73f63f1829cf4034adc59bee26841388c176eebc9df5e5cc7b8b2461" + "hash": "cb435f03a1c8d8835dad30de6bc006717cf27e2048aaec1e0872db3981be69be" } diff --git a/.sqlx/query-cbf56582f6a6d29f724c916948d9289ce31989989e995e61c72f9a84a87edd7d.json b/.sqlx/query-cbf56582f6a6d29f724c916948d9289ce31989989e995e61c72f9a84a87edd7d.json new file mode 100644 index 00000000..54e528e6 --- /dev/null +++ b/.sqlx/query-cbf56582f6a6d29f724c916948d9289ce31989989e995e61c72f9a84a87edd7d.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE repository_import i\n SET state = 'queued', run_id = NULL, locked_until = NULL, finished_at = NULL,\n last_error = NULL, cancel_requested = false, updated_at = now()\n WHERE id = $1 AND state IN ('cancelled', 'failed', 'completed')\n AND NOT EXISTS (\n SELECT 1 FROM repository_import o\n WHERE o.repository_id = i.repository_id AND o.id <> i.id\n AND o.state IN ('queued', 'running')\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "cbf56582f6a6d29f724c916948d9289ce31989989e995e61c72f9a84a87edd7d" +} diff --git a/.sqlx/query-73e966747c8f9e8836c585a347b2896a1b67fa4811b229680ee30a300c001336.json b/.sqlx/query-dc9a111531650d4a65b04b93c6e9533bf723383b068c26b6f214d907d1762a9b.json similarity index 53% rename from .sqlx/query-73e966747c8f9e8836c585a347b2896a1b67fa4811b229680ee30a300c001336.json rename to .sqlx/query-dc9a111531650d4a65b04b93c6e9533bf723383b068c26b6f214d907d1762a9b.json index 8dfb8acd..190c5ef6 100644 --- a/.sqlx/query-73e966747c8f9e8836c585a347b2896a1b67fa4811b229680ee30a300c001336.json +++ b/.sqlx/query-dc9a111531650d4a65b04b93c6e9533bf723383b068c26b6f214d907d1762a9b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE nexus_import SET password_enc = $2 WHERE id = $1", + "query": "UPDATE repository_import SET password_enc = $2 WHERE id = $1", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "73e966747c8f9e8836c585a347b2896a1b67fa4811b229680ee30a300c001336" + "hash": "dc9a111531650d4a65b04b93c6e9533bf723383b068c26b6f214d907d1762a9b" } diff --git a/.sqlx/query-e37a479c00daf5ec6ef04745ba90133aa6305e6b9ec0fa570e3b5192ce5384dd.json b/.sqlx/query-e37a479c00daf5ec6ef04745ba90133aa6305e6b9ec0fa570e3b5192ce5384dd.json deleted file mode 100644 index ce2e392f..00000000 --- a/.sqlx/query-e37a479c00daf5ec6ef04745ba90133aa6305e6b9ec0fa570e3b5192ce5384dd.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nexus_import_item\n SET state = 'pending', detail = NULL, updated_at = now()\n WHERE import_id = $1 AND state = 'failed'\n AND ($2::bigint[] IS NULL OR id = ANY($2))\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8Array" - ] - }, - "nullable": [] - }, - "hash": "e37a479c00daf5ec6ef04745ba90133aa6305e6b9ec0fa570e3b5192ce5384dd" -} diff --git a/.sqlx/query-e37c0a167cfaa3eee14c30633c63408869c26546a1f77b1dcd0fa836866cd030.json b/.sqlx/query-e37c0a167cfaa3eee14c30633c63408869c26546a1f77b1dcd0fa836866cd030.json new file mode 100644 index 00000000..6d30388e --- /dev/null +++ b/.sqlx/query-e37c0a167cfaa3eee14c30633c63408869c26546a1f77b1dcd0fa836866cd030.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO repository_import_item (import_id, kind, name, version, state, source)\n VALUES ($1, 'component', 'Acme.Core', '1.0.0', 'pending', $2)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "e37c0a167cfaa3eee14c30633c63408869c26546a1f77b1dcd0fa836866cd030" +} diff --git a/.sqlx/query-8f94720def905b052e17c6fbcc3108c22eea77884ac5660079e4bbe1b8ca6663.json b/.sqlx/query-e932ab583448f575388b9abfe0ba0117c00dfd6241855b08490cda7c1a1ce136.json similarity index 66% rename from .sqlx/query-8f94720def905b052e17c6fbcc3108c22eea77884ac5660079e4bbe1b8ca6663.json rename to .sqlx/query-e932ab583448f575388b9abfe0ba0117c00dfd6241855b08490cda7c1a1ce136.json index 137f7dc5..44243eec 100644 --- a/.sqlx/query-8f94720def905b052e17c6fbcc3108c22eea77884ac5660079e4bbe1b8ca6663.json +++ b/.sqlx/query-e932ab583448f575388b9abfe0ba0117c00dfd6241855b08490cda7c1a1ce136.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT password_enc FROM nexus_import WHERE id = $1", + "query": "SELECT password_enc FROM repository_import WHERE id = $1", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Bytea", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "password_enc" } } @@ -24,5 +24,5 @@ true ] }, - "hash": "8f94720def905b052e17c6fbcc3108c22eea77884ac5660079e4bbe1b8ca6663" + "hash": "e932ab583448f575388b9abfe0ba0117c00dfd6241855b08490cda7c1a1ce136" } diff --git a/.sqlx/query-7f7e92513dd74c34a940da96356ccef52fcc54fe48b3e52111158f505d55da7b.json b/.sqlx/query-f17cac96289359ceb7e0c08868c49869f241e62762f17773039410806fd7fd6d.json similarity index 60% rename from .sqlx/query-7f7e92513dd74c34a940da96356ccef52fcc54fe48b3e52111158f505d55da7b.json rename to .sqlx/query-f17cac96289359ceb7e0c08868c49869f241e62762f17773039410806fd7fd6d.json index a5985164..8394f84a 100644 --- a/.sqlx/query-7f7e92513dd74c34a940da96356ccef52fcc54fe48b3e52111158f505d55da7b.json +++ b/.sqlx/query-f17cac96289359ceb7e0c08868c49869f241e62762f17773039410806fd7fd6d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT cancel_requested FROM nexus_import\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", + "query": "\n SELECT cancel_requested FROM repository_import\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Bool", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "cancel_requested" } } @@ -25,5 +25,5 @@ false ] }, - "hash": "7f7e92513dd74c34a940da96356ccef52fcc54fe48b3e52111158f505d55da7b" + "hash": "f17cac96289359ceb7e0c08868c49869f241e62762f17773039410806fd7fd6d" } diff --git a/.sqlx/query-182c705b9213322dcd113f316840d91291e82a00f19cb9243c0a9ed2005f790c.json b/.sqlx/query-f9fd2bb73a05dc4932f6483282b222c752ad03e90239ba01e75c91ee6d68da98.json similarity index 52% rename from .sqlx/query-182c705b9213322dcd113f316840d91291e82a00f19cb9243c0a9ed2005f790c.json rename to .sqlx/query-f9fd2bb73a05dc4932f6483282b222c752ad03e90239ba01e75c91ee6d68da98.json index 642d076f..33173041 100644 --- a/.sqlx/query-182c705b9213322dcd113f316840d91291e82a00f19cb9243c0a9ed2005f790c.json +++ b/.sqlx/query-f9fd2bb73a05dc4932f6483282b222c752ad03e90239ba01e75c91ee6d68da98.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM nexus_import_item WHERE id = $1", + "query": "DELETE FROM repository_import_item WHERE id = $1", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "182c705b9213322dcd113f316840d91291e82a00f19cb9243c0a9ed2005f790c" + "hash": "f9fd2bb73a05dc4932f6483282b222c752ad03e90239ba01e75c91ee6d68da98" } diff --git a/.sqlx/query-140e585934ee4f40b43117d5ac312fcb5390bf968547265fa2771a28b9380cd0.json b/.sqlx/query-fdb3ed8114bfe2e9193b887f66c3281d784f0d9220cbcd18c3d3a9a7e86c29d2.json similarity index 66% rename from .sqlx/query-140e585934ee4f40b43117d5ac312fcb5390bf968547265fa2771a28b9380cd0.json rename to .sqlx/query-fdb3ed8114bfe2e9193b887f66c3281d784f0d9220cbcd18c3d3a9a7e86c29d2.json index 353e8725..b74c635f 100644 --- a/.sqlx/query-140e585934ee4f40b43117d5ac312fcb5390bf968547265fa2771a28b9380cd0.json +++ b/.sqlx/query-fdb3ed8114bfe2e9193b887f66c3281d784f0d9220cbcd18c3d3a9a7e86c29d2.json @@ -1,16 +1,16 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT base_url, registry_url, password_enc\n FROM nexus_import WHERE id = $1\n ", + "query": "\n SELECT url, registry_url, password_enc\n FROM repository_import WHERE id = $1\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "base_url", + "name": "url", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", - "name": "base_url" + "table": "repository_import", + "name": "url" } } }, @@ -20,7 +20,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "registry_url" } } @@ -31,7 +31,7 @@ "type_info": "Bytea", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "password_enc" } } @@ -48,5 +48,5 @@ true ] }, - "hash": "140e585934ee4f40b43117d5ac312fcb5390bf968547265fa2771a28b9380cd0" + "hash": "fdb3ed8114bfe2e9193b887f66c3281d784f0d9220cbcd18c3d3a9a7e86c29d2" } diff --git a/.sqlx/query-ff454fad056b994bafa995d317a8ce32a87eb4ffe41e499aad73e218dd739de0.json b/.sqlx/query-ff454fad056b994bafa995d317a8ce32a87eb4ffe41e499aad73e218dd739de0.json new file mode 100644 index 00000000..9c1be629 --- /dev/null +++ b/.sqlx/query-ff454fad056b994bafa995d317a8ce32a87eb4ffe41e499aad73e218dd739de0.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH due AS (\n SELECT id FROM repository_import\n WHERE state IN ('queued', 'running')\n AND (locked_until IS NULL OR locked_until < $1)\n ORDER BY id\n FOR UPDATE SKIP LOCKED\n LIMIT 1\n )\n UPDATE repository_import i\n SET state = 'running',\n run_id = $2,\n locked_until = $1 + make_interval(secs => $3),\n started_at = coalesce(i.started_at, $1),\n updated_at = $1\n FROM due\n WHERE i.id = due.id\n RETURNING i.id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "repository_import", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Timestamptz", + "Uuid", + "Float8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ff454fad056b994bafa995d317a8ce32a87eb4ffe41e499aad73e218dd739de0" +} diff --git a/README.md b/README.md index 46f6defc..4c36647e 100644 --- a/README.md +++ b/README.md @@ -238,9 +238,10 @@ The guides below are also a site, with navigation and search: methods run side by side. - **[Format plugins](./docs/format-plugins.md)** — further package formats, built into the server: design, operation, and writing one. -- **[Importing from Nexus](./docs/nexus-import.md)** — copy a Nexus hosted - repository (NuGet, Docker) into Forge, with its publish dates: - dry run, conflicts, resuming, and the cutover. +- **[Importing repositories](./docs/import.md)** — copy a hosted repository + of Nexus, Artifactory, a registry or another Forge (NuGet, Docker) into + Forge through the format's own protocol: dry run, conflicts, resuming, and + the cutover. Run `cargo xtask help` for the complete command list. diff --git a/crates/core/src/repo/hosted.rs b/crates/core/src/repo/hosted.rs index 3488023d..12e3c6e0 100644 --- a/crates/core/src/repo/hosted.rs +++ b/crates/core/src/repo/hosted.rs @@ -742,10 +742,10 @@ impl HostedWrite { }) } - /// The SHA-256 of the package file of `(id, version)` in the hosted - /// repository `repository_id`, if the version exists: what an import - /// compares the source's checksum with before downloading anything. - pub async fn package_sha256( + /// The SHA-512 (base64) of the package file of `id` `version`, if the + /// repository holds that version and the digest was recorded: an import + /// compares it with what the source lists. + pub async fn package_sha512( &self, repository_id: i64, id: &str, @@ -761,7 +761,12 @@ impl HostedWrite { else { return Ok(None); }; - self.stored_package_sha256(&self.pool, comp.id).await + Ok( + asset_dao::find_asset(&self.pool, comp.id, self.format.package_asset_path()) + .await + .map_err(RepoError::from)? + .and_then(|asset| asset.sha512_base64), + ) } /// The SHA-256 of a component's package file. diff --git a/crates/core/src/repo/mod.rs b/crates/core/src/repo/mod.rs index 510419ad..66010398 100644 --- a/crates/core/src/repo/mod.rs +++ b/crates/core/src/repo/mod.rs @@ -212,7 +212,7 @@ pub struct ImportOptions { pub downloaded_at: Option>, /// Who published it at the source, for the audit trail. pub uploader: Option, - /// Where it came from, for the audit trail (` `). + /// Where it came from, for the audit trail (the source's URL). pub source: String, } diff --git a/crates/db/src/import_dao.rs b/crates/db/src/import_dao.rs index cc0ba43a..bd0b1e5c 100644 --- a/crates/db/src/import_dao.rs +++ b/crates/db/src/import_dao.rs @@ -1,5 +1,6 @@ -//! Data access for imports from Nexus (`nexus_import`, `nexus_import_item`): -//! an import's job row, its scan cursor and lease, and the items it found. +//! Data access for imports into hosted repositories (`repository_import`, +//! `repository_import_item`): an import's job row, its scan cursor and lease, +//! and the items it found. //! //! An import runs on one instance at a time: the worker claims a queued (or //! abandoned) import with `FOR UPDATE SKIP LOCKED` and holds it with a @@ -8,7 +9,7 @@ //! on its `run_id`, so an instance whose lease expired cannot overwrite the //! progress of the one that took over. //! -//! The Nexus password is not read or written here: it is a secret column of +//! The source's password is not read or written here: it is a secret column of //! [`crate::secret_dao`] ([`crate::secret_dao::set_import_password`]). use std::collections::HashMap; @@ -21,12 +22,20 @@ use uuid::Uuid; /// An import job, without its password. #[derive(Debug, Clone, Serialize)] -pub struct NexusImport { +pub struct RepositoryImport { pub id: i64, pub repository_id: i64, - pub base_url: String, - pub source_repository: String, + /// The source: a NuGet service index or a registry. For an import stored + /// before 2.2, the Nexus server that `source_repository` names a + /// repository of. + pub url: String, + /// The Nexus repository of an import stored before 2.2; `None` since. + pub source_repository: Option, + /// The registry API of a Docker import stored before 2.2. pub registry_url: Option, + /// The package ids or images the import copies; `None`: whatever the + /// source's search (or catalog) finds. + pub names: Option>, pub username: Option, /// Whether a password is stored. pub has_password: bool, @@ -38,14 +47,16 @@ pub struct NexusImport { pub state: String, /// `scan` or `copy`. pub phase: String, - /// `components` or `assets`. + /// What the scan walks: `search`, `catalog` or `names`. pub scan_listing: String, #[serde(skip)] pub cursor: Option, pub cancel_requested: bool, - pub assets_seen: i64, - pub bytes_seen: i64, - pub nexus_version: Option, + /// The `Server` header the source answered the preflight with. + pub source_server: Option, + /// How many packages the feed's search found: what it reported in the + /// preflight, or as many as the scan walked, whichever is more. + pub source_total: Option, pub last_error: Option, pub created_by: Option, pub created_at: DateTime, @@ -54,7 +65,7 @@ pub struct NexusImport { pub updated_at: DateTime, } -impl NexusImport { +impl RepositoryImport { /// Whether the import waits for, or is held by, a worker. pub fn is_active(&self) -> bool { matches!(self.state.as_str(), "queued" | "running") @@ -64,14 +75,16 @@ impl NexusImport { /// A new import. pub struct NewImport<'a> { pub repository_id: i64, - pub base_url: &'a str, - pub source_repository: &'a str, - pub registry_url: Option<&'a str>, + pub url: &'a str, + pub names: Option<&'a [String]>, pub username: Option<&'a str>, pub on_conflict: &'a str, pub concurrency: i32, pub dry_run: bool, - pub nexus_version: Option<&'a str>, + /// Where the scan starts: `search`, `catalog` or `names`. + pub scan_listing: &'a str, + pub source_server: Option<&'a str>, + pub source_total: Option, pub created_by: Option, } @@ -83,21 +96,22 @@ where { sqlx::query_scalar!( r#" - INSERT INTO nexus_import - (repository_id, base_url, source_repository, registry_url, username, - on_conflict, concurrency, dry_run, nexus_version, created_by) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + INSERT INTO repository_import + (repository_id, url, names, username, on_conflict, concurrency, dry_run, + scan_listing, source_server, source_total, created_by) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING id "#, new.repository_id, - new.base_url, - new.source_repository, - new.registry_url, + new.url, + new.names, new.username, new.on_conflict, new.concurrency, new.dry_run, - new.nexus_version, + new.scan_listing, + new.source_server, + new.source_total, new.created_by, ) .fetch_one(executor) @@ -105,19 +119,19 @@ where } /// The import `id`. -pub async fn get<'e, E>(executor: E, id: i64) -> sqlx::Result> +pub async fn get<'e, E>(executor: E, id: i64) -> sqlx::Result> where E: PgExecutor<'e>, { sqlx::query_as!( - NexusImport, + RepositoryImport, r#" - SELECT id, repository_id, base_url, source_repository, registry_url, username, + SELECT id, repository_id, url, source_repository, registry_url, names, username, (password_enc IS NOT NULL) AS "has_password!", on_conflict, concurrency, dry_run, state, phase, scan_listing, cursor, - cancel_requested, assets_seen, bytes_seen, nexus_version, last_error, - created_by, created_at, started_at, finished_at, updated_at - FROM nexus_import + cancel_requested, source_server, source_total, + last_error, created_by, created_at, started_at, finished_at, updated_at + FROM repository_import WHERE id = $1 "#, id, @@ -127,16 +141,20 @@ where } /// A repository's imports, newest first. -pub async fn list(pool: &PgPool, repository_id: i64, limit: i64) -> sqlx::Result> { +pub async fn list( + pool: &PgPool, + repository_id: i64, + limit: i64, +) -> sqlx::Result> { sqlx::query_as!( - NexusImport, + RepositoryImport, r#" - SELECT id, repository_id, base_url, source_repository, registry_url, username, + SELECT id, repository_id, url, source_repository, registry_url, names, username, (password_enc IS NOT NULL) AS "has_password!", on_conflict, concurrency, dry_run, state, phase, scan_listing, cursor, - cancel_requested, assets_seen, bytes_seen, nexus_version, last_error, - created_by, created_at, started_at, finished_at, updated_at - FROM nexus_import + cancel_requested, source_server, source_total, + last_error, created_by, created_at, started_at, finished_at, updated_at + FROM repository_import WHERE repository_id = $1 ORDER BY id DESC LIMIT $2 @@ -153,7 +171,7 @@ pub async fn list(pool: &PgPool, repository_id: i64, limit: i64) -> sqlx::Result pub struct StateCount { /// Items in the state. pub items: i64, - /// Their size in Nexus. + /// Their size at the source, where it says. pub size: i64, /// Bytes copied for them. pub bytes_copied: i64, @@ -167,7 +185,7 @@ pub async fn counts(pool: &PgPool, import_id: i64) -> sqlx::Result, now: DateTime, lease: Duration, -) -> sqlx::Result> { +) -> sqlx::Result> { let run_id = Uuid::new_v4(); let id = sqlx::query_scalar!( r#" WITH due AS ( - SELECT id FROM nexus_import + SELECT id FROM repository_import WHERE state IN ('queued', 'running') AND (locked_until IS NULL OR locked_until < $1) ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1 ) - UPDATE nexus_import i + UPDATE repository_import i SET state = 'running', run_id = $2, locked_until = $1 + make_interval(secs => $3), @@ -242,7 +260,7 @@ pub async fn renew_lease( ) -> sqlx::Result { let res = sqlx::query!( r#" - UPDATE nexus_import SET locked_until = $3 + UPDATE repository_import SET locked_until = $3 WHERE id = $1 AND run_id = $2 AND state = 'running' "#, id, @@ -259,7 +277,7 @@ pub async fn renew_lease( pub async fn poll(pool: &PgPool, id: i64, run_id: Uuid) -> sqlx::Result> { sqlx::query_scalar!( r#" - SELECT cancel_requested FROM nexus_import + SELECT cancel_requested FROM repository_import WHERE id = $1 AND run_id = $2 AND state = 'running' "#, id, @@ -269,35 +287,32 @@ pub async fn poll(pool: &PgPool, id: i64, run_id: Uuid) -> sqlx::Result, id: i64, run_id: Uuid, - listing: &str, cursor: Option<&str>, phase: &str, - assets: i64, - bytes: i64, + walked: Option, ) -> sqlx::Result { let res = sqlx::query!( r#" - UPDATE nexus_import - SET scan_listing = $3, cursor = $4, phase = $5, - assets_seen = assets_seen + $6, bytes_seen = bytes_seen + $7, + UPDATE repository_import + SET cursor = $3, phase = $4, + source_total = CASE WHEN $5::bigint IS NULL THEN source_total + ELSE greatest(coalesce(source_total, 0), $5) END, updated_at = now() WHERE id = $1 AND run_id = $2 AND state = 'running' "#, id, run_id, - listing, cursor, phase, - assets, - bytes, + walked, ) .execute(&mut **tx) .await?; @@ -317,7 +332,7 @@ pub async fn finish( ) -> sqlx::Result { let res = sqlx::query!( r#" - UPDATE nexus_import + UPDATE repository_import SET state = $3, last_error = $4, finished_at = now(), @@ -342,10 +357,10 @@ pub async fn finish( /// Cancel an import: a queued one at once, a running one at its worker's /// next check. Returns the import as it is now, `None` if there is none. -pub async fn cancel(pool: &PgPool, id: i64) -> sqlx::Result> { +pub async fn cancel(pool: &PgPool, id: i64) -> sqlx::Result> { sqlx::query!( r#" - UPDATE nexus_import + UPDATE repository_import SET state = CASE WHEN state = 'queued' THEN 'cancelled' ELSE state END, finished_at = CASE WHEN state = 'queued' THEN now() ELSE finished_at END, cancel_requested = (state = 'running'), @@ -372,12 +387,12 @@ pub async fn requeue( ) -> sqlx::Result> { let queued = sqlx::query!( r#" - UPDATE nexus_import i + UPDATE repository_import i SET state = 'queued', run_id = NULL, locked_until = NULL, finished_at = NULL, last_error = NULL, cancel_requested = false, updated_at = now() WHERE id = $1 AND state IN ('cancelled', 'failed', 'completed') AND NOT EXISTS ( - SELECT 1 FROM nexus_import o + SELECT 1 FROM repository_import o WHERE o.repository_id = i.repository_id AND o.id <> i.id AND o.state IN ('queued', 'running') ) @@ -394,7 +409,7 @@ pub async fn requeue( } let retried = sqlx::query!( r#" - UPDATE nexus_import_item + UPDATE repository_import_item SET state = 'pending', detail = NULL, updated_at = now() WHERE import_id = $1 AND state = 'failed' AND ($2::bigint[] IS NULL OR id = ANY($2)) @@ -411,7 +426,7 @@ pub async fn requeue( /// active or gone. pub async fn delete(pool: &PgPool, id: i64) -> sqlx::Result { let res = sqlx::query!( - r#"DELETE FROM nexus_import WHERE id = $1 AND state NOT IN ('queued', 'running')"#, + r#"DELETE FROM repository_import WHERE id = $1 AND state NOT IN ('queued', 'running')"#, id, ) .execute(pool) @@ -466,7 +481,7 @@ pub async fn insert_items( for item in items { sqlx::query!( r#" - INSERT INTO nexus_import_item + INSERT INTO repository_import_item (import_id, kind, name, version, size, published_at, downloaded_at, uploader, source) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) @@ -502,7 +517,7 @@ pub async fn pending_items( r#" SELECT id, import_id, kind, name, version, state, detail, size, bytes_copied, published_at, downloaded_at, uploader, source, attempts, updated_at - FROM nexus_import_item + FROM repository_import_item WHERE import_id = $1 AND kind = $2 AND state = 'pending' AND id > $3 ORDER BY id LIMIT $4 @@ -526,7 +541,7 @@ pub async fn finish_item( ) -> sqlx::Result<()> { sqlx::query!( r#" - UPDATE nexus_import_item + UPDATE repository_import_item SET state = $2, detail = $3, bytes_copied = bytes_copied + $4, attempts = attempts + 1, updated_at = now() WHERE id = $1 @@ -541,10 +556,10 @@ pub async fn finish_item( Ok(()) } -/// Forget an item: a manifest Nexus holds by digest that turned out not to -/// be a referrer of an imported image. +/// Forget an item: a manifest the source holds by digest that turned out not +/// to be a referrer of an imported image. pub async fn drop_item(pool: &PgPool, item_id: i64) -> sqlx::Result<()> { - sqlx::query!("DELETE FROM nexus_import_item WHERE id = $1", item_id) + sqlx::query!("DELETE FROM repository_import_item WHERE id = $1", item_id) .execute(pool) .await?; Ok(()) @@ -564,7 +579,7 @@ pub async fn list_items( r#" SELECT id, import_id, kind, name, version, state, detail, size, bytes_copied, published_at, downloaded_at, uploader, source, attempts, updated_at - FROM nexus_import_item + FROM repository_import_item WHERE import_id = $1 AND ($2::text IS NULL OR state = $2) AND id > $3 ORDER BY id LIMIT $4 diff --git a/crates/db/src/secret_dao.rs b/crates/db/src/secret_dao.rs index 78be88eb..6a157ae4 100644 --- a/crates/db/src/secret_dao.rs +++ b/crates/db/src/secret_dao.rs @@ -37,9 +37,10 @@ //! stored `config` //! ([`store_email_settings`], [`email_config`]). //! -//! An import from Nexus keeps the Nexus password in -//! `nexus_import.password_enc` until it completes, AAD-bound to -//! `('nexus_import', 'password_enc', )` plus the origins it is sent +//! An import keeps its source's password in +//! `repository_import.password_enc` until it completes, AAD-bound to +//! `('nexus_import', 'password_enc', )` (the table's name before 2.2, +//! kept so that passwords sealed then still open) plus the origins it is sent //! to ([`import_destination`]) ([`set_import_password`], //! [`get_import_password`]). //! @@ -434,7 +435,7 @@ pub async fn reseal_upstream_passwords_batch( /// Every DEK version still referenced by stored ciphertext, across all /// encrypted columns (`repository.upstream_password_enc`, -/// `auth_realm.secrets_enc`, `nexus_import.password_enc`, +/// `auth_realm.secrets_enc`, `repository_import.password_enc`, /// `email_settings.secrets_enc`, the sealed /// [`SECRET_SETTINGS`]; future secret columns add their scan here). Used /// by the sweep to decide which `retiring` DEKs are safe to revoke. @@ -499,7 +500,7 @@ async fn scan_referenced_dek_versions( } } let import_blobs = sqlx::query!( - r#"SELECT id, password_enc AS "password_enc!" FROM nexus_import WHERE password_enc IS NOT NULL"#, + r#"SELECT id, password_enc AS "password_enc!" FROM repository_import WHERE password_enc IS NOT NULL"#, ) .fetch_all(&mut *conn) .await?; @@ -1236,24 +1237,27 @@ pub async fn reseal_secret_settings( Ok(resealed) } -// ── Nexus import passwords ─────────────────────────────────────────────────── +// ── Import passwords ───────────────────────────────────────────────────────── -/// Owning table of an import's sealed Nexus password (AAD component). +/// Owning table of an import's sealed password (AAD component): the table's +/// name before 2.2 (`repository_import` since), kept so that a password +/// sealed then still opens. pub const IMPORT_PASSWORD_TABLE: &str = "nexus_import"; -/// Owning column of an import's sealed Nexus password (AAD component). +/// Owning column of an import's sealed password (AAD component). pub const IMPORT_PASSWORD_COLUMN: &str = "password_enc"; -/// Where an import's Nexus password may be sent: the origin of its -/// `base_url` and, when it is elsewhere, of its Docker `registry_url`. -pub fn import_destination(base_url: &str, registry_url: Option<&str>) -> String { - let base = upstream_destination(Some(base_url)); +/// Where an import's password may be sent: the origin of its `url` (the +/// source) and, for a Docker import stored before 2.2 whose registry is +/// elsewhere, of its `registry_url`. +pub fn import_destination(url: &str, registry_url: Option<&str>) -> String { + let base = upstream_destination(Some(url)); match registry_url.map(|url| upstream_destination(Some(url))) { Some(registry) if registry != base => format!("{base} {registry}"), _ => base, } } -/// The AAD binding of `nexus_import.password_enc` of one row. +/// The AAD binding of `repository_import.password_enc` of one row. fn import_binding(import_id: i64, destination: &str) -> AeadBinding<'_> { AeadBinding { table: IMPORT_PASSWORD_TABLE, @@ -1263,7 +1267,7 @@ fn import_binding(import_id: i64, destination: &str) -> AeadBinding<'_> { } } -/// Store an import's Nexus password, sealed (bound to the row and to +/// Store an import's password, sealed (bound to the row and to /// [`import_destination`]). Runs on the connection of the transaction that /// created the import. pub async fn set_import_password( @@ -1273,19 +1277,19 @@ pub async fn set_import_password( secret: &PlaintextSecret, ) -> Result<(), SecretDbError> { let row = sqlx::query!( - r#"SELECT base_url, registry_url FROM nexus_import WHERE id = $1 FOR UPDATE"#, + r#"SELECT url, registry_url FROM repository_import WHERE id = $1 FOR UPDATE"#, import_id, ) .fetch_optional(&mut *conn) .await? .ok_or(SecretDbError::ImportNotFound(import_id))?; - let destination = import_destination(&row.base_url, row.registry_url.as_deref()); + let destination = import_destination(&row.url, row.registry_url.as_deref()); let blob = envelope .seal(secret, import_binding(import_id, &destination)) .await?; lock_sealing_dek(conn, envelope, &blob).await?; sqlx::query!( - r#"UPDATE nexus_import SET password_enc = $2 WHERE id = $1"#, + r#"UPDATE repository_import SET password_enc = $2 WHERE id = $1"#, import_id, &blob, ) @@ -1294,7 +1298,7 @@ pub async fn set_import_password( Ok(()) } -/// An import's Nexus password, `None` when it has none (an anonymous +/// An import's password, `None` when it has none (an anonymous /// source, or the import completed). pub async fn get_import_password( pool: &PgPool, @@ -1303,8 +1307,8 @@ pub async fn get_import_password( ) -> Result, SecretDbError> { let row = sqlx::query!( r#" - SELECT base_url, registry_url, password_enc - FROM nexus_import WHERE id = $1 + SELECT url, registry_url, password_enc + FROM repository_import WHERE id = $1 "#, import_id, ) @@ -1314,7 +1318,7 @@ pub async fn get_import_password( let Some(blob) = row.password_enc else { return Ok(None); }; - let destination = import_destination(&row.base_url, row.registry_url.as_deref()); + let destination = import_destination(&row.url, row.registry_url.as_deref()); let secret = envelope .open(&blob, import_binding(import_id, &destination)) .await?; @@ -1330,8 +1334,8 @@ pub async fn reseal_import_passwords( ) -> Result { let rows = sqlx::query!( r#" - SELECT id, base_url, registry_url, password_enc AS "password_enc!" - FROM nexus_import WHERE password_enc IS NOT NULL + SELECT id, url, registry_url, password_enc AS "password_enc!" + FROM repository_import WHERE password_enc IS NOT NULL ORDER BY id "#, ) @@ -1339,7 +1343,7 @@ pub async fn reseal_import_passwords( .await?; let mut resealed = 0; for row in rows { - let destination = import_destination(&row.base_url, row.registry_url.as_deref()); + let destination = import_destination(&row.url, row.registry_url.as_deref()); let bound = import_binding(row.id, &destination); let Some(new_blob) = reseal_if_due(envelope, &row.password_enc, bound).await? else { continue; @@ -1353,7 +1357,7 @@ pub async fn reseal_import_passwords( WHERE layer = 'dek' AND scope = $4 AND version = $5 AND state <> 'revoked' FOR SHARE ) - UPDATE nexus_import SET password_enc = $2 + UPDATE repository_import SET password_enc = $2 WHERE id = $1 AND password_enc = $3 AND EXISTS (SELECT 1 FROM dek) "#, row.id, diff --git a/crates/db/tests/migrations.rs b/crates/db/tests/migrations.rs index e7c60818..30a0bd96 100644 --- a/crates/db/tests/migrations.rs +++ b/crates/db/tests/migrations.rs @@ -1,9 +1,19 @@ //! Integration test: spin up a real Postgres in a container, run the migrations, -//! and assert every expected table exists; and refuse a database an earlier -//! build created. +//! and assert every expected table exists; refuse a database an earlier +//! build created; and carry an earlier migration's rows through a later one. //! //! Requires Docker to be available on the host running the test. +use std::sync::Arc; + +use forge_crypto::envelope::{AeadBinding, Algorithm}; +use forge_crypto::kms::StaticRootProvider; +use forge_crypto::service::{EnvelopeService, EnvelopeServiceConfig}; +use forge_crypto::{ExposeSecret, KeyBytes, PlaintextSecret}; +use forge_db::encryption_key_dao::PgKeyStore; +use forge_db::import_dao::{self, NewImport}; +use forge_db::repository_dao::{self, NewRepository}; +use forge_db::secret_dao; use sqlx::Row; use testcontainers_modules::{ postgres::Postgres, @@ -781,6 +791,169 @@ async fn a_database_of_an_earlier_build_is_refused() { } } +/// An import stored before imports read the formats' protocols (0007): its +/// rows, its items and its sealed password come through the rename, and a +/// scan that walked Nexus' REST API starts over. +#[tokio::test] +async fn an_import_of_a_nexus_repository_survives_the_rename() { + let (server, _container) = server().await; + let url = fresh_database(&server, "import_rename").await; + let pool = forge_db::connect(&url).await.expect("connect"); + sqlx::migrate!("../../migrations") + .run_to(6, &pool) + .await + .expect("migrate to 0006"); + + let nuget = repository_dao::insert_repository(&pool, NewRepository::hosted("nuget-internal")) + .await + .unwrap(); + let images = repository_dao::insert_repository( + &pool, + NewRepository { + format: "docker".into(), + ..NewRepository::hosted("images") + }, + ) + .await + .unwrap(); + let insert = |repository_id: i64, + repository: &'static str, + registry: Option<&'static str>, + phase: &'static str, + listing: &'static str| { + let pool = pool.clone(); + async move { + sqlx::query_scalar::<_, i64>( + "INSERT INTO nexus_import (repository_id, base_url, source_repository, registry_url, \ + username, state, phase, scan_listing, cursor, nexus_version) \ + VALUES ($1, 'https://nexus.example.com', $2, $3, 'migrator', 'failed', $4, $5, \ + 'token-1', 'Nexus/3.70.1-02 (OSS)') RETURNING id", + ) + .bind(repository_id) + .bind(repository) + .bind(registry) + .bind(phase) + .bind(listing) + .fetch_one(&pool) + .await + .expect("insert an import") + } + }; + let scanning = insert(nuget.id, "nuget-hosted", None, "scan", "components").await; + let copying = insert( + images.id, + "docker-hosted", + Some("https://nexus.example.com:8082"), + "copy", + "assets", + ) + .await; + sqlx::query( + "INSERT INTO nexus_import_item (import_id, kind, name, version, source) \ + VALUES ($1, 'component', 'app', '1.0', '{\"assets\": []}')", + ) + .bind(copying) + .execute(&pool) + .await + .expect("insert an item"); + // The password, sealed as before the rename. + let envelope = EnvelopeService::new( + Arc::new(PgKeyStore::new(pool.clone())), + Arc::new(StaticRootProvider::new( + "env", + Algorithm::Aes256Gcm, + KeyBytes::generate(), + vec![], + )), + EnvelopeServiceConfig::default(), + ); + let destination = secret_dao::import_destination( + "https://nexus.example.com", + Some("https://nexus.example.com:8082"), + ); + let sealed = envelope + .seal( + &PlaintextSecret::from("nexus-secret".to_string()), + AeadBinding { + table: "nexus_import", + column: "password_enc", + row_pk: copying, + destination: Some(&destination), + }, + ) + .await + .expect("seal"); + sqlx::query("UPDATE nexus_import SET password_enc = $2 WHERE id = $1") + .bind(copying) + .bind(&sealed) + .execute(&pool) + .await + .expect("store the password"); + + forge_db::migrate(&pool).await.expect("migrate the rest"); + + let scanned = import_dao::get(&pool, scanning).await.unwrap().unwrap(); + assert_eq!(scanned.url, "https://nexus.example.com"); + assert_eq!(scanned.source_repository.as_deref(), Some("nuget-hosted")); + assert_eq!( + scanned.source_server.as_deref(), + Some("Nexus/3.70.1-02 (OSS)") + ); + assert_eq!( + (scanned.scan_listing.as_str(), scanned.cursor.as_deref()), + ("search", None), + "the scan starts over" + ); + assert_eq!((scanned.names, scanned.source_total), (None, None)); + let copied = import_dao::get(&pool, copying).await.unwrap().unwrap(); + assert_eq!(copied.scan_listing, "catalog"); + assert_eq!(copied.phase, "copy"); + assert_eq!( + copied.registry_url.as_deref(), + Some("https://nexus.example.com:8082") + ); + let items = import_dao::list_items(&pool, copying, None, 0, 10) + .await + .unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0].source, serde_json::json!({"assets": []})); + let password = secret_dao::get_import_password(&pool, &envelope, copying) + .await + .expect("the password opens") + .expect("a password"); + assert_eq!(password.expose_secret(), "nexus-secret"); + + // One active import per repository, as before. + let queued = || NewImport { + repository_id: nuget.id, + url: "https://feed.example.com/v3/index.json", + names: None, + username: None, + on_conflict: "skip", + concurrency: 4, + dry_run: false, + scan_listing: "search", + source_server: None, + source_total: Some(3), + created_by: None, + }; + import_dao::insert(&pool, queued()) + .await + .expect("a queued import"); + assert!(import_dao::insert(&pool, queued()).await.is_err()); + let leftovers: i64 = sqlx::query_scalar( + "SELECT (SELECT count(*) FROM pg_constraint WHERE conname LIKE 'nexus%') \ + + (SELECT count(*) FROM pg_class WHERE relname LIKE 'nexus%')", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + leftovers, 0, + "no constraint, index or sequence keeps the old name" + ); +} + /// A Postgres server for one test: a throwaway Postgres 18 container matching /// docker-compose (db/user/pw = forge), or, with FORGE_TEST_DATABASE_URL (no /// Docker), that server. Returns the URL of a database on it, and the diff --git a/crates/e2e/tests/import.rs b/crates/e2e/tests/import.rs new file mode 100644 index 00000000..23db1c3e --- /dev/null +++ b/crates/e2e/tests/import.rs @@ -0,0 +1,784 @@ +//! E2E: imports through the formats' own protocols, from real sources: +//! +//! * a Nexus Repository 3 OSS container (`sonatype/nexus3`): NuGet through +//! its hosted repository's v3 feed (`/repository//index.json`), and +//! Docker through its connector's catalog, holding more than 100 images, +//! with a cosign signature and an OCI 1.1 referrer pushed the way +//! `oras attach` pushes one to a registry without the referrers API (the +//! referrers tag schema); +//! * a `distribution` registry (`registry:2`): Docker through its catalog; +//! * another Forge: NuGet from a Forge NuGet repository, whose registration +//! lists each package's SHA-512, so a second run downloads nothing. +//! +//! After an import, bytes and digests are the source's and so are the NuGet +//! publish dates; a second run copies only what was added since. (Resuming +//! a killed import, the conflict policies, the dry run and a source without +//! a search or catalog are covered against a mock source in +//! `crates/server/tests/import.rs`.) Gated behind `forge-e2e`; Nexus takes a +//! minute or two to start. + +#![cfg(feature = "forge-e2e")] + +use std::time::Duration; + +use forge_e2e::{make_nupkg, StackBuilder, TestServer}; +use forge_server::import::ImportWorker; +use serde_json::{json, Value}; +use sha2::Digest; +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::runners::AsyncRunner; +use testcontainers::{GenericImage, ImageExt}; + +const NEXUS_IMAGE: &str = "mirror.gcr.io/sonatype/nexus3"; +const NEXUS_TAG: &str = "3.70.1"; +/// With `NEXUS_SECURITY_RANDOMPASSWORD=false`, Nexus' admin password. +const ADMIN: (&str, &str) = ("admin", "admin123"); + +const REGISTRY_IMAGE: &str = "mirror.gcr.io/library/registry"; +const REGISTRY_TAG: &str = "2"; + +/// More images than a catalog page holds (Nexus was reported to stop at +/// 100). +const BULK_IMAGES: usize = 105; + +const OCI_MANIFEST: &str = "application/vnd.oci.image.manifest.v1+json"; +const OCI_INDEX: &str = "application/vnd.oci.image.index.v1+json"; +const OCI_CONFIG: &str = "application/vnd.oci.image.config.v1+json"; +const OCI_EMPTY: &str = "application/vnd.oci.empty.v1+json"; +const LAYER: &str = "application/vnd.oci.image.layer.v1.tar+gzip"; + +fn sha256(bytes: &[u8]) -> String { + hex::encode(sha2::Sha256::digest(bytes)) +} + +fn digest(bytes: &[u8]) -> String { + format!("sha256:{}", sha256(bytes)) +} + +// ── Sources ───────────────────────────────────────────────────────────────── + +/// A Nexus with its REST API at `url` and its Docker connector at +/// `registry`. +struct Nexus { + url: String, + registry: Registry, + client: reqwest::Client, +} + +impl Nexus { + async fn start() -> (Self, testcontainers::ContainerAsync) { + let container = GenericImage::new(NEXUS_IMAGE, NEXUS_TAG) + .with_exposed_port(8081.tcp()) + .with_exposed_port(8082.tcp()) + .with_wait_for(WaitFor::message_on_stdout("Started Sonatype Nexus")) + .with_env_var("NEXUS_SECURITY_RANDOMPASSWORD", "false") + .with_env_var( + "INSTALL4J_ADD_VM_PARAMS", + "-Xms512m -Xmx1g -XX:MaxDirectMemorySize=1g", + ) + .with_startup_timeout(Duration::from_secs(600)) + .start() + .await + .expect("start nexus (is Docker running?)"); + let host = container.get_host().await.unwrap(); + let api = container.get_host_port_ipv4(8081).await.unwrap(); + let registry = container.get_host_port_ipv4(8082).await.unwrap(); + let client = reqwest::Client::new(); + let nexus = Nexus { + url: format!("http://{host}:{api}"), + registry: Registry { + url: format!("http://{host}:{registry}"), + auth: Some(ADMIN), + client: client.clone(), + }, + client, + }; + for _ in 0..300 { + let up = nexus + .client + .get(format!("{}/service/rest/v1/status/writable", nexus.url)) + .send() + .await + .is_ok_and(|r| r.status() == 200); + if up { + return (nexus, container); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + panic!("nexus never became writable"); + } + + fn api(&self, method: reqwest::Method, path: &str) -> reqwest::RequestBuilder { + self.client + .request(method, format!("{}/service/rest/v1/{path}", self.url)) + .basic_auth(ADMIN.0, Some(ADMIN.1)) + } + + async fn seed_repositories(&self) { + let resp = self + .api(reqwest::Method::PUT, "security/realms/active") + .json(&json!(["NexusAuthenticatingRealm", "DockerToken"])) + .send() + .await + .unwrap(); + assert!(resp.status().is_success(), "realms: {}", resp.status()); + let docker = json!({ + "name": "docker-hosted", "online": true, + "storage": {"blobStoreName": "default", "strictContentTypeValidation": true, "writePolicy": "allow"}, + "docker": {"v1Enabled": false, "forceBasicAuth": true, "httpPort": 8082}, + }); + let resp = self + .api(reqwest::Method::POST, "repositories/docker/hosted") + .json(&docker) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); + } + + /// Upload an asset through the components API (`.asset`). + async fn upload(&self, repository: &str, format: &str, file: &str, bytes: Vec) { + let form = reqwest::multipart::Form::new().part( + format!("{format}.asset"), + reqwest::multipart::Part::bytes(bytes).file_name(file.to_string()), + ); + let resp = self + .api( + reqwest::Method::POST, + &format!("components?repository={repository}"), + ) + .multipart(form) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 204, + "upload {file}: {}", + resp.text().await.unwrap() + ); + } + + /// The service index of the NuGet repository `name`. + fn feed(&self, name: &str) -> String { + format!("{}/repository/{name}/index.json", self.url) + } + + /// GET a document of the NuGet repository `name`'s v3 feed. + async fn feed_json(&self, name: &str, path: &str) -> Value { + self.client + .get(format!("{}/repository/{name}/{path}", self.url)) + .basic_auth(ADMIN.0, Some(ADMIN.1)) + .send() + .await + .unwrap() + .json() + .await + .unwrap() + } + + /// The URL of the resource of `kind` the feed `name` lists. + async fn resource(&self, name: &str, kind: &str) -> String { + let index = self.feed_json(name, "index.json").await; + index["resources"] + .as_array() + .unwrap() + .iter() + .find(|r| r["@type"].as_str().is_some_and(|t| t.starts_with(kind))) + .unwrap_or_else(|| panic!("{kind} in {index}"))["@id"] + .as_str() + .unwrap() + .to_string() + } + + /// When the feed's registration says `lower` `version` was published. + async fn published( + &self, + name: &str, + lower: &str, + version: &str, + ) -> chrono::DateTime { + let base = self.resource(name, "RegistrationsBaseUrl").await; + let url = format!("{}/{lower}/index.json", base.trim_end_matches('/')); + let registration: Value = self + .client + .get(&url) + .basic_auth(ADMIN.0, Some(ADMIN.1)) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let leaves = registration["items"] + .as_array() + .unwrap() + .iter() + .flat_map(|page| page["items"].as_array().cloned().unwrap_or_default()); + for leaf in leaves { + let entry = &leaf["catalogEntry"]; + if entry["version"].as_str() == Some(version) { + let published = entry["published"].as_str().expect("published"); + return chrono::DateTime::parse_from_rfc3339(published).unwrap(); + } + } + panic!("{version} not in {registration}"); + } + + /// Wait until the feed's search finds `n` packages: Nexus indexes an + /// upload for search a moment after it stored it. + async fn searchable(&self, name: &str, n: u64) { + let search = self.resource(name, "SearchQueryService").await; + for _ in 0..120 { + let found: Value = self + .client + .get(format!( + "{search}?q=&take=100&prerelease=true&semVerLevel=2.0.0" + )) + .basic_auth(ADMIN.0, Some(ADMIN.1)) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + if found["totalHits"].as_u64() >= Some(n) { + return; + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + panic!("Nexus' search never found {n} packages in {name}"); + } +} + +/// A registry API to push to. +struct Registry { + url: String, + auth: Option<(&'static str, &'static str)>, + client: reqwest::Client, +} + +impl Registry { + fn request(&self, method: reqwest::Method, url: &str) -> reqwest::RequestBuilder { + let request = self.client.request(method, url); + match self.auth { + Some((user, password)) => request.basic_auth(user, Some(password)), + None => request, + } + } + + async fn push_blob(&self, image: &str, bytes: &[u8]) { + let resp = self + .request( + reqwest::Method::POST, + &format!("{}/v2/{image}/blobs/uploads/", self.url), + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "start upload"); + let location = resp.headers()["location"].to_str().unwrap().to_string(); + let location = if location.starts_with("http") { + location + } else { + format!("{}{location}", self.url) + }; + let separator = if location.contains('?') { '&' } else { '?' }; + let resp = self + .request( + reqwest::Method::PUT, + &format!("{location}{separator}digest={}", digest(bytes)), + ) + .header("content-type", "application/octet-stream") + .body(bytes.to_vec()) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "finish upload"); + } + + async fn push_manifest(&self, image: &str, reference: &str, media_type: &str, bytes: &[u8]) { + let resp = self + .request( + reqwest::Method::PUT, + &format!("{}/v2/{image}/manifests/{reference}", self.url), + ) + .header("content-type", media_type) + .body(bytes.to_vec()) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 201, + "manifest {reference}: {}", + resp.text().await.unwrap() + ); + } + + /// A one-platform image of `image` with `layer`, pushed under `tag`. + async fn push_image(&self, image: &str, tag: &str, layer: &[u8]) -> Vec { + let config = json!({"architecture": "amd64", "os": "linux", "rootfs": {"type": "layers", "diff_ids": []}}) + .to_string() + .into_bytes(); + self.push_blob(image, &config).await; + self.push_blob(image, layer).await; + let manifest = json!({ + "schemaVersion": 2, "mediaType": OCI_MANIFEST, + "config": {"mediaType": OCI_CONFIG, "digest": digest(&config), "size": config.len()}, + "layers": [{"mediaType": LAYER, "digest": digest(layer), "size": layer.len()}], + }) + .to_string() + .into_bytes(); + self.push_manifest(image, tag, OCI_MANIFEST, &manifest) + .await; + manifest + } + + /// A two-platform image of `image` under `tag`, a cosign signature of + /// it (`sha256-.sig`) and an SBOM attached the way `oras attach` + /// attaches one to a registry without the referrers API: the referrer by + /// digest, and the index of the subject's referrers under the tag + /// `sha256-`. Returns the index, its children's digests, the + /// signature and the SBOM. + async fn push_signed_index( + &self, + image: &str, + tag: &str, + ) -> (Vec, Vec, Vec, Vec) { + let mut children = Vec::new(); + let mut descriptors = Vec::new(); + for arch in ["amd64", "arm64"] { + let config = json!({"architecture": arch, "os": "linux", "rootfs": {"type": "layers", "diff_ids": []}}) + .to_string() + .into_bytes(); + let layer = format!("the {arch} layer of {image}").into_bytes(); + self.push_blob(image, &config).await; + self.push_blob(image, &layer).await; + let manifest = json!({ + "schemaVersion": 2, "mediaType": OCI_MANIFEST, + "config": {"mediaType": OCI_CONFIG, "digest": digest(&config), "size": config.len()}, + "layers": [{"mediaType": LAYER, "digest": digest(&layer), "size": layer.len()}], + }) + .to_string() + .into_bytes(); + self.push_manifest(image, &digest(&manifest), OCI_MANIFEST, &manifest) + .await; + children.push(digest(&manifest)); + descriptors.push(json!({"mediaType": OCI_MANIFEST, "digest": digest(&manifest), "size": manifest.len(), "platform": {"architecture": arch, "os": "linux"}})); + } + let index = json!({"schemaVersion": 2, "mediaType": OCI_INDEX, "manifests": descriptors}) + .to_string() + .into_bytes(); + self.push_manifest(image, tag, OCI_INDEX, &index).await; + + let empty = b"{}".to_vec(); + self.push_blob(image, &empty).await; + let payload = format!("cosign signature payload of {image}").into_bytes(); + self.push_blob(image, &payload).await; + let signature = json!({ + "schemaVersion": 2, "mediaType": OCI_MANIFEST, + "config": {"mediaType": OCI_EMPTY, "digest": digest(&empty), "size": 2}, + "layers": [{"mediaType": "application/vnd.dev.cosign.simplesigning.v1+json", "digest": digest(&payload), "size": payload.len(), + "annotations": {"dev.cosignproject.cosign/signature": "MEUCIQ=="}}], + }) + .to_string() + .into_bytes(); + self.push_manifest( + image, + &format!("sha256-{}.sig", sha256(&index)), + OCI_MANIFEST, + &signature, + ) + .await; + + let sbom_body = + format!("{{\"spdxVersion\": \"SPDX-2.3\", \"name\": \"{image}\"}}").into_bytes(); + self.push_blob(image, &sbom_body).await; + let sbom = json!({ + "schemaVersion": 2, "mediaType": OCI_MANIFEST, "artifactType": "application/spdx+json", + "config": {"mediaType": OCI_EMPTY, "digest": digest(&empty), "size": 2}, + "layers": [{"mediaType": "application/spdx+json", "digest": digest(&sbom_body), "size": sbom_body.len()}], + "subject": {"mediaType": OCI_INDEX, "digest": digest(&index), "size": index.len()}, + }) + .to_string() + .into_bytes(); + self.push_manifest(image, &digest(&sbom), OCI_MANIFEST, &sbom) + .await; + let referrers = json!({"schemaVersion": 2, "mediaType": OCI_INDEX, "manifests": [ + {"mediaType": OCI_MANIFEST, "digest": digest(&sbom), "size": sbom.len(), "artifactType": "application/spdx+json"} + ]}) + .to_string() + .into_bytes(); + self.push_manifest( + image, + &format!("sha256-{}", sha256(&index)), + OCI_INDEX, + &referrers, + ) + .await; + (index, children, signature, sbom) + } +} + +// ── Forge ─────────────────────────────────────────────────────────────────── + +async fn admin( + srv: &TestServer, + method: reqwest::Method, + path: &str, + body: Option, +) -> reqwest::Response { + let (u, p) = srv.admin_basic(); + let mut req = srv + .client() + .request(method, format!("{}{path}", srv.base_url)) + .basic_auth(u, Some(p)); + if let Some(body) = body { + req = req.json(&body); + } + req.send().await.unwrap() +} + +/// Start an import into `repo` and run it to its end. +async fn import(srv: &TestServer, repo: &str, body: Value) -> Value { + let resp = admin( + srv, + reqwest::Method::POST, + &format!("/admin/repositories/{repo}/imports"), + Some(body), + ) + .await; + assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); + let id = resp.json::().await.unwrap()["id"].as_i64().unwrap(); + for _ in 0..1200 { + ImportWorker::new(srv.state.clone()).tick().await.unwrap(); + let view: Value = admin( + srv, + reqwest::Method::GET, + &format!("/admin/repositories/{repo}/imports/{id}"), + None, + ) + .await + .json() + .await + .unwrap(); + if !matches!(view["state"].as_str(), Some("queued" | "running")) { + return view; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("import into {repo} did not end"); +} + +async fn create_repo(srv: &TestServer, body: Value) { + let resp = admin( + srv, + reqwest::Method::POST, + "/admin/repositories", + Some(body), + ) + .await; + assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); +} + +/// The bytes of a Forge download, with the admin's credentials. +async fn download(srv: &TestServer, path: &str) -> Vec { + let (u, p) = srv.admin_basic(); + let resp = srv + .client() + .get(format!("{}{path}", srv.base_url)) + .basic_auth(u, Some(p)) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "GET {path}"); + resp.bytes().await.unwrap().to_vec() +} + +async fn repository_id(srv: &TestServer, repo: &str) -> i64 { + forge_db::repository_dao::find_repository_by_name(&srv.pool, repo) + .await + .unwrap() + .unwrap() + .id +} + +/// The image `image` of `repo` holds the signed index `index` with its +/// children, its signature tag, and the SBOM as a referrer. +async fn assert_signed_index( + srv: &TestServer, + repo: &str, + image: &str, + tag: &str, + (index, children, signature, sbom): &(Vec, Vec, Vec, Vec), +) { + let repo_id = repository_id(srv, repo).await; + let tagged = forge_db::oci_dao::find_tag(&srv.pool, repo_id, image, tag) + .await + .unwrap() + .unwrap_or_else(|| panic!("{image}:{tag}")); + assert_eq!(tagged.manifest.digest, digest(index)); + for child in children { + assert!( + forge_db::oci_dao::find_manifest(&srv.pool, repo_id, image, child) + .await + .unwrap() + .is_some(), + "child {child}" + ); + } + let sig = forge_db::oci_dao::find_tag( + &srv.pool, + repo_id, + image, + &format!("sha256-{}.sig", sha256(index)), + ) + .await + .unwrap() + .expect("the signature's tag"); + assert_eq!(sig.manifest.digest, digest(signature)); + let referrers: Vec = + forge_db::oci_dao::list_referrers(&srv.pool, repo_id, image, &digest(index)) + .await + .unwrap() + .into_iter() + .map(|m| m.digest) + .collect(); + assert_eq!(referrers, vec![digest(sbom)]); +} + +// ── Tests ─────────────────────────────────────────────────────────────────── + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn import_from_a_real_nexus() { + let (nexus, _container) = Nexus::start().await; + nexus.seed_repositories().await; + + // NuGet: two packages in the default nuget-hosted repository. + let core = make_nupkg("Acme.Core", "1.0.0"); + let web = make_nupkg("Acme.Web", "2.1.0"); + nexus + .upload( + "nuget-hosted", + "nuget", + "acme.core.1.0.0.nupkg", + core.clone(), + ) + .await; + nexus + .upload("nuget-hosted", "nuget", "acme.web.2.1.0.nupkg", web.clone()) + .await; + nexus.searchable("nuget-hosted", 2).await; + + // Docker: a signed two-platform image with an SBOM, and more images than + // a catalog page holds. + let signed = nexus.registry.push_signed_index("team/app", "1.0").await; + for i in 0..BULK_IMAGES { + nexus + .registry + .push_image( + &format!("bulk/img-{i:03}"), + "1.0", + format!("layer {i}").as_bytes(), + ) + .await; + } + + let srv = StackBuilder::default().start().await; + create_repo(&srv, json!({"name": "nuget-internal", "kind": "hosted"})).await; + create_repo( + &srv, + json!({"name": "images", "kind": "hosted", "format": "docker"}), + ) + .await; + + // NuGet: through the feed; the bytes and the publish dates are Nexus'. + let credentials = |url: String| json!({"url": url, "username": ADMIN.0, "password": ADMIN.1}); + let done = import( + &srv, + "nuget-internal", + credentials(nexus.feed("nuget-hosted")), + ) + .await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["source_total"], 2, "{done}"); + assert!( + done["source_server"] + .as_str() + .is_some_and(|s| s.starts_with("Nexus/")), + "{done}" + ); + assert_eq!(done["progress"]["imported"], 2, "{done}"); + assert_eq!( + download( + &srv, + "/repository/nuget-internal/flat/acme.core/1.0.0/acme.core.1.0.0.nupkg" + ) + .await, + core + ); + assert_eq!( + download( + &srv, + "/repository/nuget-internal/flat/acme.web/2.1.0/acme.web.2.1.0.nupkg" + ) + .await, + web + ); + let published = nexus.published("nuget-hosted", "acme.core", "1.0.0").await; + let created = sqlx::query_scalar!( + "SELECT c.created_at FROM component c JOIN repository r ON r.id = c.repository_id WHERE r.name = 'nuget-internal' AND c.name = 'acme.core'" + ) + .fetch_one(&srv.pool) + .await + .unwrap(); + assert_eq!(created, published); + + // Docker: through the connector's catalog, every image, the children, + // the signature tag and the SBOM. + let done = import(&srv, "images", credentials(nexus.registry.url.clone())).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["failed"], 0, "{done}"); + // Three tags of team/app (1.0, the signature, the referrers tag) and one + // of each bulk image. + assert_eq!( + done["progress"]["imported"], + 3 + BULK_IMAGES as i64, + "{done}" + ); + assert_signed_index(&srv, "images", "team/app", "1.0", &signed).await; + let repo_id = repository_id(&srv, "images").await; + for i in [0, 99, 100, BULK_IMAGES - 1] { + let image = format!("bulk/img-{i:03}"); + assert!( + forge_db::oci_dao::find_tag(&srv.pool, repo_id, &image, "1.0") + .await + .unwrap() + .is_some(), + "{image}" + ); + } + + // A second run, in the request form of a Nexus server and repository, + // copies only what is new in Nexus. + let next = make_nupkg("Acme.Core", "1.1.0"); + nexus + .upload( + "nuget-hosted", + "nuget", + "acme.core.1.1.0.nupkg", + next.clone(), + ) + .await; + let mut legacy = credentials(nexus.url.clone()); + legacy["repository"] = json!("nuget-hosted"); + let done = import(&srv, "nuget-internal", legacy).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["imported"], 1, "{done}"); + assert_eq!(done["progress"]["identical"], 2, "{done}"); + assert_eq!( + download( + &srv, + "/repository/nuget-internal/flat/acme.core/1.1.0/acme.core.1.1.0.nupkg" + ) + .await, + next + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn import_from_a_distribution_registry() { + let container = GenericImage::new(REGISTRY_IMAGE, REGISTRY_TAG) + .with_exposed_port(5000.tcp()) + .with_wait_for(WaitFor::message_on_stderr("listening on")) + .start() + .await + .expect("start registry:2 (is Docker running?)"); + let host = container.get_host().await.unwrap(); + let port = container.get_host_port_ipv4(5000).await.unwrap(); + let registry = Registry { + url: format!("http://{host}:{port}"), + auth: None, + client: reqwest::Client::new(), + }; + let signed = registry.push_signed_index("team/app", "1.0").await; + for image in ["team/base", "tools/cli", "tools/lint"] { + registry.push_image(image, "latest", image.as_bytes()).await; + } + + let srv = StackBuilder::default().start().await; + create_repo( + &srv, + json!({"name": "images", "kind": "hosted", "format": "docker"}), + ) + .await; + let done = import(&srv, "images", json!({"url": registry.url})).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["failed"], 0, "{done}"); + assert_eq!(done["progress"]["imported"], 6, "{done}"); + assert_signed_index(&srv, "images", "team/app", "1.0", &signed).await; + + let done = import(&srv, "images", json!({"url": registry.url})).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["identical"], 6, "{done}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn import_from_another_forge() { + let srv = StackBuilder::default().start().await; + create_repo(&srv, json!({"name": "nuget-source", "kind": "hosted"})).await; + create_repo(&srv, json!({"name": "nuget-copy", "kind": "hosted"})).await; + let packages = [ + make_nupkg("Acme.Core", "1.0.0"), + make_nupkg("Acme.Core", "1.1.0-beta.1"), + make_nupkg("Acme.Web", "2.0.0"), + ]; + for package in &packages { + assert_eq!( + srv.push_nupkg("nuget-source", package.clone()) + .await + .status(), + 201 + ); + } + let (user, token) = srv.admin_basic(); + let body = json!({ + "url": format!("{}/repository/nuget-source/index.json", srv.base_url), + "username": user, + "password": token, + }); + let done = import(&srv, "nuget-copy", body.clone()).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["imported"], 3, "{done}"); + assert_eq!(done["source_total"], 2, "{done}"); + assert_eq!( + download( + &srv, + "/repository/nuget-copy/flat/acme.core/1.1.0-beta.1/acme.core.1.1.0-beta.1.nupkg" + ) + .await, + packages[1] + ); + + // Forge lists each package's SHA-512: a second run downloads nothing. + let downloads = || async { + sqlx::query_scalar!( + r#" + SELECT coalesce(sum(a.download_count), 0)::bigint AS "n!" + FROM asset a JOIN component c ON c.id = a.component_id + JOIN repository r ON r.id = c.repository_id + WHERE r.name = 'nuget-source' + "# + ) + .fetch_one(&srv.pool) + .await + .unwrap() + }; + let before = downloads().await; + let done = import(&srv, "nuget-copy", body).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["identical"], 3, "{done}"); + assert_eq!(done["progress"]["bytes_copied"], 0, "{done}"); + assert_eq!(downloads().await, before); +} diff --git a/crates/e2e/tests/nexus_import.rs b/crates/e2e/tests/nexus_import.rs deleted file mode 100644 index a031b785..00000000 --- a/crates/e2e/tests/nexus_import.rs +++ /dev/null @@ -1,464 +0,0 @@ -//! E2E: importing from a real Nexus Repository 3 OSS container -//! (`sonatype/nexus3`) seeded with NuGet packages, a multi-platform image -//! with a cosign signature and an SBOM. After the import, -//! bytes and digests are Nexus' and so are the publish dates; a second run -//! copies only what was added to Nexus since. (Resuming a killed import, the -//! conflict policies and the dry run are covered against a mock Nexus in -//! `crates/server/tests/nexus_import.rs`.) Gated behind `forge-e2e`; Nexus -//! takes a minute or two to start. - -#![cfg(feature = "forge-e2e")] - -use std::time::Duration; - -use forge_e2e::{make_nupkg, StackBuilder, TestServer}; -use forge_server::nexus_import::ImportWorker; -use serde_json::{json, Value}; -use sha2::Digest; -use testcontainers::core::{IntoContainerPort, WaitFor}; -use testcontainers::runners::AsyncRunner; -use testcontainers::{GenericImage, ImageExt}; - -const NEXUS_IMAGE: &str = "mirror.gcr.io/sonatype/nexus3"; -const NEXUS_TAG: &str = "3.70.1"; -/// With `NEXUS_SECURITY_RANDOMPASSWORD=false`, Nexus' admin password. -const ADMIN: (&str, &str) = ("admin", "admin123"); - -const OCI_MANIFEST: &str = "application/vnd.oci.image.manifest.v1+json"; -const OCI_INDEX: &str = "application/vnd.oci.image.index.v1+json"; - -fn sha256(bytes: &[u8]) -> String { - hex::encode(sha2::Sha256::digest(bytes)) -} - -fn digest(bytes: &[u8]) -> String { - format!("sha256:{}", sha256(bytes)) -} - -/// A Nexus with its REST API at `url` and its Docker connector at -/// `registry`. -struct Nexus { - url: String, - registry: String, - client: reqwest::Client, -} - -impl Nexus { - async fn start() -> (Self, testcontainers::ContainerAsync) { - let container = GenericImage::new(NEXUS_IMAGE, NEXUS_TAG) - .with_exposed_port(8081.tcp()) - .with_exposed_port(8082.tcp()) - .with_wait_for(WaitFor::message_on_stdout("Started Sonatype Nexus")) - .with_env_var("NEXUS_SECURITY_RANDOMPASSWORD", "false") - .with_env_var( - "INSTALL4J_ADD_VM_PARAMS", - "-Xms512m -Xmx1g -XX:MaxDirectMemorySize=1g", - ) - .with_startup_timeout(Duration::from_secs(600)) - .start() - .await - .expect("start nexus (is Docker running?)"); - let host = container.get_host().await.unwrap(); - let api = container.get_host_port_ipv4(8081).await.unwrap(); - let registry = container.get_host_port_ipv4(8082).await.unwrap(); - let nexus = Nexus { - url: format!("http://{host}:{api}"), - registry: format!("http://{host}:{registry}"), - client: reqwest::Client::new(), - }; - for _ in 0..300 { - let up = nexus - .client - .get(format!("{}/service/rest/v1/status/writable", nexus.url)) - .send() - .await - .is_ok_and(|r| r.status() == 200); - if up { - return (nexus, container); - } - tokio::time::sleep(Duration::from_secs(1)).await; - } - panic!("nexus never became writable"); - } - - fn api(&self, method: reqwest::Method, path: &str) -> reqwest::RequestBuilder { - self.client - .request(method, format!("{}/service/rest/v1/{path}", self.url)) - .basic_auth(ADMIN.0, Some(ADMIN.1)) - } - - async fn seed_repositories(&self) { - let resp = self - .api(reqwest::Method::PUT, "security/realms/active") - .json(&json!(["NexusAuthenticatingRealm", "DockerToken"])) - .send() - .await - .unwrap(); - assert!(resp.status().is_success(), "realms: {}", resp.status()); - let docker = json!({ - "name": "docker-hosted", "online": true, - "storage": {"blobStoreName": "default", "strictContentTypeValidation": true, "writePolicy": "allow"}, - "docker": {"v1Enabled": false, "forceBasicAuth": true, "httpPort": 8082}, - }); - let resp = self - .api(reqwest::Method::POST, "repositories/docker/hosted") - .json(&docker) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); - } - - /// Upload an asset through the components API (`.asset`). - async fn upload(&self, repository: &str, format: &str, file: &str, bytes: Vec) { - let form = reqwest::multipart::Form::new().part( - format!("{format}.asset"), - reqwest::multipart::Part::bytes(bytes).file_name(file.to_string()), - ); - let resp = self - .api( - reqwest::Method::POST, - &format!("components?repository={repository}"), - ) - .multipart(form) - .send() - .await - .unwrap(); - assert_eq!( - resp.status(), - 204, - "upload {file}: {}", - resp.text().await.unwrap() - ); - } - - /// The components of `repository`, `(name, version)` → asset. - async fn components(&self, repository: &str) -> Vec { - let mut out = Vec::new(); - let mut token: Option = None; - loop { - let mut path = format!("components?repository={repository}"); - if let Some(t) = &token { - path.push_str(&format!("&continuationToken={t}")); - } - let page: Value = self - .api(reqwest::Method::GET, &path) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - out.extend(page["items"].as_array().unwrap().iter().cloned()); - match page["continuationToken"].as_str() { - Some(t) => token = Some(t.to_string()), - None => return out, - } - } - } - - /// Push a blob to the Docker connector. - async fn push_blob(&self, image: &str, bytes: &[u8]) { - let resp = self - .client - .post(format!("{}/v2/{image}/blobs/uploads/", self.registry)) - .basic_auth(ADMIN.0, Some(ADMIN.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), 202, "start upload"); - let location = resp.headers()["location"].to_str().unwrap().to_string(); - let location = if location.starts_with("http") { - location - } else { - format!("{}{location}", self.registry) - }; - let separator = if location.contains('?') { '&' } else { '?' }; - let resp = self - .client - .put(format!("{location}{separator}digest={}", digest(bytes))) - .basic_auth(ADMIN.0, Some(ADMIN.1)) - .header("content-type", "application/octet-stream") - .body(bytes.to_vec()) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), 201, "finish upload"); - } - - async fn push_manifest(&self, image: &str, reference: &str, media_type: &str, bytes: &[u8]) { - let resp = self - .client - .put(format!( - "{}/v2/{image}/manifests/{reference}", - self.registry - )) - .basic_auth(ADMIN.0, Some(ADMIN.1)) - .header("content-type", media_type) - .body(bytes.to_vec()) - .send() - .await - .unwrap(); - assert_eq!( - resp.status(), - 201, - "manifest {reference}: {}", - resp.text().await.unwrap() - ); - } -} - -async fn admin( - srv: &TestServer, - method: reqwest::Method, - path: &str, - body: Option, -) -> reqwest::Response { - let (u, p) = srv.admin_basic(); - let mut req = srv - .client() - .request(method, format!("{}{path}", srv.base_url)) - .basic_auth(u, Some(p)); - if let Some(body) = body { - req = req.json(&body); - } - req.send().await.unwrap() -} - -/// Start an import of `source` into `repo` and run it to its end. -async fn import(srv: &TestServer, nexus: &Nexus, repo: &str, source: &str) -> Value { - let resp = admin( - srv, - reqwest::Method::POST, - &format!("/admin/repositories/{repo}/imports"), - Some(json!({"url": nexus.url, "repository": source, "username": ADMIN.0, "password": ADMIN.1})), - ) - .await; - assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); - let id = resp.json::().await.unwrap()["id"].as_i64().unwrap(); - for _ in 0..600 { - ImportWorker::new(srv.state.clone()).tick().await.unwrap(); - let view: Value = admin( - srv, - reqwest::Method::GET, - &format!("/admin/repositories/{repo}/imports/{id}"), - None, - ) - .await - .json() - .await - .unwrap(); - if !matches!(view["state"].as_str(), Some("queued" | "running")) { - return view; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - panic!("import into {repo} did not end"); -} - -async fn create_repo(srv: &TestServer, body: Value) { - let resp = admin( - srv, - reqwest::Method::POST, - "/admin/repositories", - Some(body), - ) - .await; - assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); -} - -/// The bytes of a Forge download, with the admin's credentials. -async fn download(srv: &TestServer, path: &str) -> Vec { - let (u, p) = srv.admin_basic(); - let resp = srv - .client() - .get(format!("{}{path}", srv.base_url)) - .basic_auth(u, Some(p)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), 200, "GET {path}"); - resp.bytes().await.unwrap().to_vec() -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn import_from_a_real_nexus() { - let (nexus, _container) = Nexus::start().await; - nexus.seed_repositories().await; - - // NuGet: two packages in the default nuget-hosted repository. - let core = make_nupkg("Acme.Core", "1.0.0"); - let web = make_nupkg("Acme.Web", "2.1.0"); - nexus - .upload( - "nuget-hosted", - "nuget", - "acme.core.1.0.0.nupkg", - core.clone(), - ) - .await; - nexus - .upload("nuget-hosted", "nuget", "acme.web.2.1.0.nupkg", web.clone()) - .await; - - // Docker: a two-platform image, its cosign signature (a tag) and an - // SBOM pushed by digest (a referrer Nexus lists as an asset only). - let image = "team/app"; - let mut children = Vec::new(); - for arch in ["amd64", "arm64"] { - let config = json!({"architecture": arch, "os": "linux", "rootfs": {"type": "layers", "diff_ids": []}}).to_string().into_bytes(); - let layer = format!("the {arch} layer").into_bytes(); - nexus.push_blob(image, &config).await; - nexus.push_blob(image, &layer).await; - let manifest = json!({ - "schemaVersion": 2, "mediaType": OCI_MANIFEST, - "config": {"mediaType": "application/vnd.oci.image.config.v1+json", "digest": digest(&config), "size": config.len()}, - "layers": [{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": digest(&layer), "size": layer.len()}], - }) - .to_string() - .into_bytes(); - nexus - .push_manifest(image, &digest(&manifest), OCI_MANIFEST, &manifest) - .await; - children.push(json!({"mediaType": OCI_MANIFEST, "digest": digest(&manifest), "size": manifest.len(), "platform": {"architecture": arch, "os": "linux"}})); - } - let index = json!({"schemaVersion": 2, "mediaType": OCI_INDEX, "manifests": children}) - .to_string() - .into_bytes(); - nexus.push_manifest(image, "1.0", OCI_INDEX, &index).await; - let empty = b"{}".to_vec(); - nexus.push_blob(image, &empty).await; - let payload = b"cosign signature payload".to_vec(); - nexus.push_blob(image, &payload).await; - let subject = json!({"mediaType": OCI_INDEX, "digest": digest(&index), "size": index.len()}); - let signature = json!({ - "schemaVersion": 2, "mediaType": OCI_MANIFEST, - "config": {"mediaType": "application/vnd.oci.empty.v1+json", "digest": digest(&empty), "size": 2}, - "layers": [{"mediaType": "application/vnd.dev.cosign.simplesigning.v1+json", "digest": digest(&payload), "size": payload.len(), - "annotations": {"dev.cosignproject.cosign/signature": "MEUCIQ=="}}], - "subject": subject, - }) - .to_string() - .into_bytes(); - let sig_tag = format!("sha256-{}.sig", sha256(&index)); - nexus - .push_manifest(image, &sig_tag, OCI_MANIFEST, &signature) - .await; - let sbom = json!({ - "schemaVersion": 2, "mediaType": OCI_MANIFEST, "artifactType": "application/spdx+json", - "config": {"mediaType": "application/vnd.oci.empty.v1+json", "digest": digest(&empty), "size": 2}, - "layers": [{"mediaType": "application/vnd.oci.empty.v1+json", "digest": digest(&empty), "size": 2}], - "subject": subject, - }) - .to_string() - .into_bytes(); - nexus - .push_manifest(image, &digest(&sbom), OCI_MANIFEST, &sbom) - .await; - - let srv = StackBuilder::default().start().await; - create_repo(&srv, json!({"name": "nuget-internal", "kind": "hosted"})).await; - create_repo( - &srv, - json!({"name": "images", "kind": "hosted", "format": "docker"}), - ) - .await; - - // NuGet: the bytes and the publish dates are Nexus'. - let done = import(&srv, &nexus, "nuget-internal", "nuget-hosted").await; - assert_eq!(done["state"], "completed", "{done}"); - assert_eq!(done["progress"]["imported"], 2, "{done}"); - assert_eq!( - download( - &srv, - "/repository/nuget-internal/flat/acme.core/1.0.0/acme.core.1.0.0.nupkg" - ) - .await, - core - ); - assert_eq!( - download( - &srv, - "/repository/nuget-internal/flat/acme.web/2.1.0/acme.web.2.1.0.nupkg" - ) - .await, - web - ); - let listed = nexus.components("nuget-hosted").await; - let core_listed = listed.iter().find(|c| c["name"] == "Acme.Core").unwrap(); - let published = chrono::DateTime::parse_from_rfc3339( - core_listed["assets"][0]["blobCreated"].as_str().unwrap(), - ) - .unwrap(); - let created = sqlx::query_scalar!( - "SELECT c.created_at FROM component c JOIN repository r ON r.id = c.repository_id WHERE r.name = 'nuget-internal' AND c.name = 'acme.core'" - ) - .fetch_one(&srv.pool) - .await - .unwrap(); - assert_eq!(created, published); - - // Docker: digests, the children, the signature tag and both referrers. - let done = import(&srv, &nexus, "images", "docker-hosted").await; - assert_eq!(done["state"], "completed", "{done}"); - assert_eq!(done["progress"]["failed"], 0, "{done}"); - let repo_id = forge_db::repository_dao::find_repository_by_name(&srv.pool, "images") - .await - .unwrap() - .unwrap() - .id; - let tagged = forge_db::oci_dao::find_tag(&srv.pool, repo_id, image, "1.0") - .await - .unwrap() - .unwrap(); - assert_eq!(tagged.manifest.digest, digest(&index)); - for child in &children { - let d = child["digest"].as_str().unwrap(); - assert!( - forge_db::oci_dao::find_manifest(&srv.pool, repo_id, image, d) - .await - .unwrap() - .is_some(), - "child {d}" - ); - } - let sig = forge_db::oci_dao::find_tag(&srv.pool, repo_id, image, &sig_tag) - .await - .unwrap() - .unwrap(); - assert_eq!(sig.manifest.digest, digest(&signature)); - let mut referrers: Vec = - forge_db::oci_dao::list_referrers(&srv.pool, repo_id, image, &digest(&index)) - .await - .unwrap() - .into_iter() - .map(|m| m.digest) - .collect(); - referrers.sort(); - let mut expected = vec![digest(&signature), digest(&sbom)]; - expected.sort(); - assert_eq!(referrers, expected); - - // A second run copies only what is new in Nexus. - let next = make_nupkg("Acme.Core", "1.1.0"); - nexus - .upload( - "nuget-hosted", - "nuget", - "acme.core.1.1.0.nupkg", - next.clone(), - ) - .await; - let done = import(&srv, &nexus, "nuget-internal", "nuget-hosted").await; - assert_eq!(done["state"], "completed", "{done}"); - assert_eq!(done["progress"]["imported"], 1, "{done}"); - assert_eq!(done["progress"]["identical"], 2, "{done}"); - assert_eq!( - download( - &srv, - "/repository/nuget-internal/flat/acme.core/1.1.0/acme.core.1.1.0.nupkg" - ) - .await, - next - ); -} diff --git a/crates/formats/docker/src/upstream.rs b/crates/formats/docker/src/upstream.rs index a29cd1de..7d283b2c 100644 --- a/crates/formats/docker/src/upstream.rs +++ b/crates/formats/docker/src/upstream.rs @@ -12,10 +12,12 @@ //! to the upstream only) to fill one page of Forge's; an upstream that //! refuses a large `n` (400) is asked again without one. //! * Referrers come from the upstream's referrers API, where it has one. +//! * The catalog (`/v2/_catalog`, which an import walks) is paged like a +//! tag list; an upstream that does not serve it, or refuses it, has none. use std::collections::BTreeSet; -use forge_plugin_api::images::{ImageUrl, ManifestInfo, Reference, TagPage}; +use forge_plugin_api::images::{CatalogPage, ImageUrl, ManifestInfo, Reference, TagPage}; use forge_plugin_api::{ Fetch, FetchResult, Fetched, FormatError, Freshness, Remote, UpstreamAnswer, UpstreamOp, UpstreamStep, @@ -75,6 +77,9 @@ pub fn step( tags(&base, image, last.as_deref(), *n as usize, fetched) } UpstreamOp::ImageReferrers { image, digest } => referrers(&base, image, digest, fetched), + UpstreamOp::ImageCatalog { last, n } => { + catalog(&base, last.as_deref(), *n as usize, fetched) + } _ => Err(FormatError::Internal( "Docker proxies have only image operations".into(), )), @@ -117,12 +122,8 @@ fn get(url: String, image: &str, accept: Option<&str>) -> Fetch { } } -/// Up to `limit` of the upstream's tags of `image` after `last`: one more -/// is asked for, to know whether there are more, and the upstream's own -/// pages are followed for up to [`MAX_UPSTREAM_TAG_PAGES`] requests. An -/// upstream that refuses so large an `n` (400, as registries that cap it -/// answer) is asked again without one: its own page size and links take -/// over. The steps so far are in `fetched`. +/// Up to `limit` of the upstream's tags of `image` after `last` +/// ([`list`]). fn tags( base: &str, image: &str, @@ -130,6 +131,124 @@ fn tags( limit: usize, fetched: &[FetchResult], ) -> Result { + Ok( + match list(base, &Listing::Tags(image), last, limit, fetched) { + Listed::Fetch(fetch) => UpstreamStep::Fetch(vec![fetch]), + Listed::Unknown => UpstreamStep::Done(UpstreamAnswer::ImageTags(None)), + Listed::Failed(reason) => UpstreamStep::Done(UpstreamAnswer::Failed(reason)), + Listed::Page(tags, more) => { + UpstreamStep::Done(UpstreamAnswer::ImageTags(Some(TagPage { tags, more }))) + } + }, + ) +} + +/// Up to `limit` of the names of the upstream's images after `last`, from +/// its catalog (`/v2/_catalog`, [`list`]). The catalog is no part of the +/// distribution spec: an upstream that does not serve it (404), or refuses +/// it (401, 403: Docker Hub), has none. +fn catalog( + base: &str, + last: Option<&str>, + limit: usize, + fetched: &[FetchResult], +) -> Result { + if fetched + .first() + .is_some_and(|first| matches!(first.status, Some(401 | 403))) + { + return Ok(UpstreamStep::Done(UpstreamAnswer::ImageCatalog(None))); + } + Ok(match list(base, &Listing::Catalog, last, limit, fetched) { + Listed::Fetch(fetch) => UpstreamStep::Fetch(vec![fetch]), + Listed::Unknown => UpstreamStep::Done(UpstreamAnswer::ImageCatalog(None)), + Listed::Failed(reason) => UpstreamStep::Done(UpstreamAnswer::Failed(reason)), + Listed::Page(images, more) => { + UpstreamStep::Done(UpstreamAnswer::ImageCatalog(Some(CatalogPage { + images, + more, + }))) + } + }) +} + +/// A paged list of names the registry API serves. +enum Listing<'a> { + /// The tags of an image (`/v2//tags/list`). + Tags(&'a str), + /// The registry's images (`/v2/_catalog`). + Catalog, +} + +impl Listing<'_> { + fn url(&self, base: &str) -> String { + match self { + Listing::Tags(image) => format!("{base}/v2/{image}/tags/list"), + Listing::Catalog => format!("{base}/v2/_catalog"), + } + } + + /// The key of the document's name array. + fn key(&self) -> &'static str { + match self { + Listing::Tags(_) => "tags", + Listing::Catalog => "repositories", + } + } + + fn scope(&self) -> String { + match self { + Listing::Tags(image) => scope(image), + Listing::Catalog => "registry:catalog:*".to_string(), + } + } + + fn valid(&self, name: &str) -> bool { + match self { + Listing::Tags(_) => reference::valid_tag(name), + Listing::Catalog => reference::valid_name(name), + } + } + + fn what(&self) -> &'static str { + match self { + Listing::Tags(_) => "tag list", + Listing::Catalog => "catalog", + } + } + + fn get(&self, url: String) -> Fetch { + Fetch { + scope: Some(self.scope()), + ..Fetch::new(url, Freshness::Metadata) + } + } +} + +/// Where [`list`] stands. +enum Listed { + /// Fetch this page next. + Fetch(Fetch), + /// The upstream does not have the list (404). + Unknown, + Failed(String), + /// The names, and whether there are more after them. + Page(Vec, bool), +} + +/// Up to `limit` of the names of `listing` after `last`: one more is asked +/// for, to know whether there are more, and the upstream's own pages are +/// followed for up to [`MAX_UPSTREAM_TAG_PAGES`] requests. An upstream that +/// refuses so large an `n` (400, as registries that cap it answer) is asked +/// again without one: its own page size and links take over. The steps so +/// far are in `fetched`. +fn list( + base: &str, + listing: &Listing<'_>, + last: Option<&str>, + limit: usize, + fetched: &[FetchResult], +) -> Listed { let first = |n: Option| { let mut query = url::form_urlencoded::Serializer::new(String::new()); if let Some(n) = n { @@ -139,31 +258,29 @@ fn tags( query.append_pair("last", last); } let query = query.finish(); - let url = format!("{base}/v2/{image}/tags/list"); + let url = listing.url(base); let url = if query.is_empty() { url } else { format!("{url}?{query}") }; - get(url, image, None) + listing.get(url) }; if fetched.is_empty() { - return Ok(UpstreamStep::Fetch(vec![first(Some( - limit.saturating_add(1), - ))])); + return Listed::Fetch(first(Some(limit.saturating_add(1)))); } - let mut tags: BTreeSet = BTreeSet::new(); + let mut names: BTreeSet = BTreeSet::new(); let mut pages = 0; let mut left_out = false; for (i, result) in fetched.iter().enumerate() { let latest = i + 1 == fetched.len(); if pages == 0 { if result.outcome == Fetched::NotFound { - return Ok(UpstreamStep::Done(UpstreamAnswer::ImageTags(None))); + return Listed::Unknown; } if i == 0 && result.status == Some(400) { if latest { - return Ok(UpstreamStep::Fetch(vec![first(None)])); + return Listed::Fetch(first(None)); } continue; } @@ -171,52 +288,46 @@ fn tags( let body = match &result.outcome { Fetched::Found(body) => body, Fetched::NotFound => { - return Ok(UpstreamStep::Done(UpstreamAnswer::Failed( - "the upstream lost a page of the tag list".into(), - ))) - } - Fetched::Failed(reason) => { - return Ok(UpstreamStep::Done(UpstreamAnswer::Failed(reason.clone()))) + return Listed::Failed(format!( + "the upstream lost a page of the {}", + listing.what() + )) } + Fetched::Failed(reason) => return Listed::Failed(reason.clone()), }; pages += 1; - #[derive(serde::Deserialize)] - struct TagList { - #[serde(default)] - tags: Option>, - } - let Ok(list) = serde_json::from_slice::(body) else { - return Ok(UpstreamStep::Done(UpstreamAnswer::Failed( - "the upstream served an invalid tag list".into(), - ))); + let Ok(document) = serde_json::from_slice::(body) else { + return Listed::Failed(format!("the upstream served an invalid {}", listing.what())); + }; + let listed = match document.get(listing.key()) { + None | Some(serde_json::Value::Null) => Vec::new(), + Some(serde_json::Value::Array(listed)) => listed.clone(), + Some(_) => { + return Listed::Failed(format!("the upstream served an invalid {}", listing.what())) + } }; - tags.extend( - list.tags - .unwrap_or_default() + names.extend( + listed .into_iter() - .filter(|t| reference::valid_tag(t)) - .filter(|t| last.is_none_or(|l| t.as_bytes() > l.as_bytes())), + .filter_map(|name| name.as_str().map(str::to_string)) + .filter(|name| listing.valid(name)) + .filter(|name| last.is_none_or(|l| name.as_bytes() > l.as_bytes())), ); - if tags.len() > limit { + if names.len() > limit { break; } if latest { match next_link(result, base) { Some(next) if pages < MAX_UPSTREAM_TAG_PAGES => { - return Ok(UpstreamStep::Fetch(vec![get(next, image, None)])) + return Listed::Fetch(listing.get(next)) } Some(_) => left_out = true, None => {} } } } - let more = left_out || tags.len() > limit; - Ok(UpstreamStep::Done(UpstreamAnswer::ImageTags(Some( - TagPage { - tags: tags.into_iter().take(limit).collect(), - more, - }, - )))) + let more = left_out || names.len() > limit; + Listed::Page(names.into_iter().take(limit).collect(), more) } /// The `next` page a tag list's `Link` header names (RFC 8288), resolved @@ -404,6 +515,81 @@ mod tests { assert_eq!(conversed.fetches.len(), 2, "no link off the upstream"); } + #[test] + fn the_catalog_is_paged_like_a_tag_list() { + let mut pages = vec![ + ( + vec!["team/app", "team/db"], + Some("; rel=\"next\""), + ), + (vec!["Not A Name", "tools/cli"], None), + ] + .into_iter(); + let op = UpstreamOp::ImageCatalog { + last: Some("base".into()), + n: 10, + }; + let conversed = converse_answered(&Docker, &remote("http://up.test"), &op, |fetch| { + let (images, link) = pages.next().expect("no more pages"); + FetchResult { + fetch: fetch.clone(), + outcome: Fetched::Found( + serde_json::to_vec(&json!({ "repositories": images })).unwrap(), + ), + status: Some(200), + headers: link + .map(|l| vec![("link".to_string(), l.to_string())]) + .unwrap_or_default(), + } + }) + .unwrap(); + assert_eq!( + conversed.answer, + UpstreamAnswer::ImageCatalog(Some(CatalogPage { + images: vec!["team/app".into(), "team/db".into(), "tools/cli".into()], + more: false, + })), + "an invalid name is left out" + ); + assert_eq!( + conversed.fetches[0].url, + "http://up.test/v2/_catalog?n=11&last=base" + ); + assert_eq!( + conversed.fetches[0].scope.as_deref(), + Some("registry:catalog:*") + ); + assert_eq!( + conversed.fetches[1].url, + "http://up.test/v2/_catalog?n=2&last=team%2Fdb" + ); + } + + #[test] + fn a_registry_without_a_catalog_has_none() { + for (status, outcome) in [ + (404, Fetched::NotFound), + (401, Fetched::Failed("401".into())), + (403, Fetched::Failed("403".into())), + ] { + let op = UpstreamOp::ImageCatalog { last: None, n: 5 }; + let conversed = converse_answered(&Docker, &remote("http://up.test"), &op, |fetch| { + FetchResult { + fetch: fetch.clone(), + outcome: outcome.clone(), + status: Some(status), + headers: Vec::new(), + } + }) + .unwrap(); + assert_eq!( + conversed.answer, + UpstreamAnswer::ImageCatalog(None), + "{status}" + ); + } + } + #[test] fn an_upstream_refusing_a_large_n_is_asked_without_one() { let op = UpstreamOp::ImageTags { diff --git a/crates/formats/npm/src/proxy.rs b/crates/formats/npm/src/proxy.rs index c7c8304c..fa682631 100644 --- a/crates/formats/npm/src/proxy.rs +++ b/crates/formats/npm/src/proxy.rs @@ -274,13 +274,16 @@ pub fn upstream(remote: &Remote, op: &UpstreamOp, fetched: &[FetchResult]) -> Up } } UpstreamOp::Search(query) => search(remote, query, fetched), + // No import walks an npm registry yet. + UpstreamOp::Packages { .. } => done(UpstreamAnswer::Search(None)), // A registry holds packages: it has no files to fetch. UpstreamOp::File(_) => done(UpstreamAnswer::File(None)), UpstreamOp::ImageName(_) | UpstreamOp::ImageManifest { .. } | UpstreamOp::ImageBlob { .. } | UpstreamOp::ImageTags { .. } - | UpstreamOp::ImageReferrers { .. } => failed("a registry of packages holds no images"), + | UpstreamOp::ImageReferrers { .. } + | UpstreamOp::ImageCatalog { .. } => failed("a registry of packages holds no images"), } } @@ -382,6 +385,9 @@ fn metadata(doc: Option, wanted: &[String]) -> PackageMetadat created: time, summary, version, + // npm's integrity is checked from the version's dist + // when its tarball is located, not here. + sha512: None, } }) .collect(), diff --git a/crates/formats/nuget/src/upstream.rs b/crates/formats/nuget/src/upstream.rs index 1e5e8cc3..6f716694 100644 --- a/crates/formats/nuget/src/upstream.rs +++ b/crates/formats/nuget/src/upstream.rs @@ -136,7 +136,18 @@ pub(crate) fn step( UpstreamOp::Manifest(target) => manifest(&resources, target, fetched, limits), UpstreamOp::Metadata(wanted) => metadata(&resources, wanted, false, fetched), UpstreamOp::Dating(wanted) => metadata(&resources, wanted, true, fetched), - UpstreamOp::Search(query) => search(&resources, query, fetched), + UpstreamOp::Search(query) => search(&resources, query, true, fetched), + UpstreamOp::Packages { skip, take } => { + let query = SearchQuery { + text: String::new(), + skip: *skip, + take: *take, + prerelease: true, + semver2: true, + upstream: true, + }; + search(&resources, &query, false, fetched) + } // A feed holds packages: it has no files to fetch. UpstreamOp::File(_) => done(UpstreamAnswer::File(None)), // NuGet has no tags. @@ -145,7 +156,8 @@ pub(crate) fn step( | UpstreamOp::ImageManifest { .. } | UpstreamOp::ImageBlob { .. } | UpstreamOp::ImageTags { .. } - | UpstreamOp::ImageReferrers { .. } => failed("a feed holds no images"), + | UpstreamOp::ImageReferrers { .. } + | UpstreamOp::ImageCatalog { .. } => failed("a feed holds no images"), }) } @@ -289,6 +301,10 @@ struct Leaf { /// The `@id` of the version's catalog entry, which names when the /// version was created even when it is unlisted. catalog_url: Option, + /// The package's SHA-512 (base64), where the entry lists it inline (a + /// `packageHash` with `packageHashAlgorithm` `SHA512`, as Forge's + /// registration does; nuget.org's lists it in the catalog leaf only). + sha512: Option, } /// One page of a registration index. @@ -458,6 +474,7 @@ fn metadata( summary: nuspec::manifest_summary(&manifest) .ok() .and_then(|summary| serde_json::to_string(&summary).ok()), + sha512: leaf.sha512.clone(), } }) .collect(); @@ -534,6 +551,16 @@ fn registration_leaves(page: &Value) -> BTreeMap { published, groups, catalog_url: entry.get("@id").and_then(Value::as_str).map(str::to_string), + sha512: entry + .get("packageHash") + .and_then(Value::as_str) + .filter(|_| { + entry + .get("packageHashAlgorithm") + .and_then(Value::as_str) + .is_some_and(|algorithm| algorithm.eq_ignore_ascii_case("SHA512")) + }) + .map(str::to_string), }, ); } @@ -572,10 +599,18 @@ fn year(raw: &str) -> Option { } /// A page of the upstream's `SearchQueryService` results for `query`. -fn search(resources: &Resources, query: &SearchQuery, fetched: &[FetchResult]) -> UpstreamStep { +/// `capped`: a proxy's search, which does not ask for offsets beyond +/// nuget.org's; an import's walk asks for every offset the upstream allows +/// (Nexus: 10,000 packages). +fn search( + resources: &Resources, + query: &SearchQuery, + capped: bool, + fetched: &[FetchResult], +) -> UpstreamStep { // nuget.org limits skip to 3,000. Deeper pages use the proxy's local search // rather than requesting a rejected offset or silently changing the page. - if query.skip > 3_000 { + if capped && query.skip > 3_000 { return done(UpstreamAnswer::Search(None)); } let Some(service) = &resources.search_query_service else { @@ -771,4 +806,19 @@ mod tests { assert!(leaves["2.0.0"].listed); assert!(leaves["3.0.0"].listed); } + + #[test] + fn an_inline_sha512_package_hash_is_read() { + let page = serde_json::json!({ "items": [ + { "catalogEntry": { "version": "1.0.0", "packageHash": "c2hhNTEy", + "packageHashAlgorithm": "SHA512" } }, + { "catalogEntry": { "version": "2.0.0", "packageHash": "bWQ1", + "packageHashAlgorithm": "MD5" } }, + { "catalogEntry": { "version": "3.0.0", "packageHash": "c2hhNTEy" } }, + ]}); + let leaves = registration_leaves(&page); + assert_eq!(leaves["1.0.0"].sha512.as_deref(), Some("c2hhNTEy")); + assert_eq!(leaves["2.0.0"].sha512, None, "another algorithm"); + assert_eq!(leaves["3.0.0"].sha512, None, "no algorithm named"); + } } diff --git a/crates/formats/nuget/tests/upstream.rs b/crates/formats/nuget/tests/upstream.rs index 588875f5..b71afa16 100644 --- a/crates/formats/nuget/tests/upstream.rs +++ b/crates/formats/nuget/tests/upstream.rs @@ -368,3 +368,27 @@ fn search_offsets_beyond_nuget_orgs_limit_request_local_fallback() { assert!(done.fetches.is_empty()); } } + +#[test] +fn an_import_walks_every_package_beyond_nuget_orgs_offset_limit() { + let url = "https://search.example/query?q=&skip=5000&take=2&prerelease=true&semVerLevel=2.0.0"; + let page = found(json!({ "totalHits": 5002, "data": [ + { "id": "Contoso.Lib", "version": "1.0.0" }, + { "id": "Contoso.Web", "version": "2.0.0-beta.1" }, + ]})); + let done = run( + &remote(&resources()), + UpstreamOp::Packages { + skip: 5_000, + take: 2, + }, + &docs([(url.to_string(), page)]), + ); + assert_eq!(done.fetches[0].url, url); + let UpstreamAnswer::Search(Some(search)) = done.answer else { + panic!("{:?}", done.answer) + }; + assert_eq!(search.total, 5002); + let ids: Vec<&str> = search.hits.iter().map(|hit| hit.id.as_str()).collect(); + assert_eq!(ids, ["Contoso.Lib", "Contoso.Web"]); +} diff --git a/crates/plugin-api/src/images.rs b/crates/plugin-api/src/images.rs index 70a5c15c..de0b6445 100644 --- a/crates/plugin-api/src/images.rs +++ b/crates/plugin-api/src/images.rs @@ -273,6 +273,15 @@ pub struct TagPage { pub more: bool, } +/// A page of an upstream registry's catalog ([`crate::UpstreamOp::ImageCatalog`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CatalogPage { + /// Image names, in byte order. + pub images: Vec, + /// There are images after these the page leaves out. + pub more: bool, +} + /// An upload session. #[derive(Debug, Clone, PartialEq, Eq)] pub struct UploadState { diff --git a/crates/plugin-api/src/types.rs b/crates/plugin-api/src/types.rs index ba3e6e51..e22248fa 100644 --- a/crates/plugin-api/src/types.rs +++ b/crates/plugin-api/src/types.rs @@ -3,7 +3,7 @@ use std::fmt; use crate::images::{ - ImageAnswer, ImageName, ImageQuery, ImageUrl, ManifestInfo, Reference, TagPage, + CatalogPage, ImageAnswer, ImageName, ImageQuery, ImageUrl, ManifestInfo, Reference, TagPage, }; /// A dependency on another package of the same format. @@ -1031,6 +1031,12 @@ pub enum UpstreamOp { Dating(PackageVersions), /// A page of the upstream's search results. Search(SearchQuery), + /// Package formats: up to `take` of every package the upstream lists, + /// after the first `skip` ([`UpstreamAnswer::Search`]): what an import + /// walks. Unlike a proxy's [`UpstreamOp::Search`], every version counts + /// (prerelease, SemVer 2.0.0) and the offset is not capped below what + /// the upstream allows. `None`: the upstream lists no packages. + Packages { skip: u32, take: u32 }, /// The tags the upstream gives a package (canonical id). Tags(String), /// The URL of the file at a path (files formats). The host downloads it, @@ -1063,6 +1069,10 @@ pub enum UpstreamOp { /// Images formats: the upstream's referrers of a manifest /// ([`UpstreamAnswer::ImageReferrers`]). ImageReferrers { image: String, digest: String }, + /// Images formats: up to `n` of the names of the images the upstream + /// holds after `last`, in byte order ([`UpstreamAnswer::ImageCatalog`]): + /// what an import of a registry walks. + ImageCatalog { last: Option, n: u32 }, } /// How long the host may reuse a fetched document. @@ -1167,6 +1177,10 @@ pub struct VersionMetadata { /// /// [`FormatPlugin::manifest_summary`]: crate::FormatPlugin::manifest_summary pub summary: Option, + /// The SHA-512 of the version's package, base64, where the upstream's + /// metadata lists it: an import verifies the package against it, and + /// skips a version it holds with these bytes without downloading it. + pub sha512: Option, } /// The answer to [`UpstreamOp::Metadata`] and [`UpstreamOp::Dating`]. @@ -1247,6 +1261,9 @@ pub enum UpstreamAnswer { /// `None`: the upstream offers no referrers API (the host lists the /// referrers its cache holds). ImageReferrers(Option>), + /// `None`: the upstream serves no catalog (it is no part of the + /// distribution spec: Docker Hub, GHCR and ECR serve none), or refuses it. + ImageCatalog(Option), /// The upstream answered something unusable: the host serves what it /// has cached. Failed(String), diff --git a/crates/server/src/admin/imports.rs b/crates/server/src/admin/imports.rs index 44325f64..c1b45414 100644 --- a/crates/server/src/admin/imports.rs +++ b/crates/server/src/admin/imports.rs @@ -1,16 +1,19 @@ -//! Imports from Nexus into a hosted repository ([`crate::nexus_import`]). +//! Imports into a hosted repository from another server ([`crate::import`]): +//! a NuGet feed or a registry, read through the format's own protocol. //! **Admin-only.** //! //! * `GET /admin/repositories/{name}/imports` — the repository's imports, //! newest first ([`ImportView`]). //! * `POST /admin/repositories/{name}/imports` — start one //! ([`CreateImport`]). The source is checked first (the **preflight**): -//! Nexus answers its status endpoint, the Nexus repository exists, is -//! hosted and of the repository's format, and, for Docker, its registry -//! API answers. A failed check is a 400 (502 when Nexus is unreachable) -//! saying what is wrong; nothing is stored. A repository runs one import -//! at a time (409). `dry_run` decides every item without writing a -//! package: what would be copied, how much, and the conflicts. +//! a feed's service index resolves and its search answers (with how many +//! packages it finds), or a registry answers its API and serves a +//! catalog; an import that names its package ids or images needs no +//! search or catalog. A failed check is a 400 (502 when the source is +//! unreachable or failing) saying what is wrong; nothing is stored. A +//! repository runs one import at a time (409). `dry_run` decides every +//! item without writing a package: what would be copied, and the +//! conflicts. //! * `GET …/imports/{id}` — one import and its progress. //! * `GET …/imports/{id}/items?state=&after=&limit=` — its items in the //! order they were found ([`forge_db::import_dao::ImportItem`]), paged @@ -37,14 +40,19 @@ use axum::Extension; use chrono::{DateTime, Utc}; use forge_core::auth::principal::Principal; use forge_core::net::ssrf::validate_url; -use forge_db::import_dao::{self, NewImport, NexusImport}; +use forge_core::repo::proxy::Credentials; +use forge_db::import_dao::{self, NewImport, RepositoryImport}; use forge_db::model::{RepoKind, Repository}; use forge_db::{repository_dao, secret_dao}; use serde::{Deserialize, Serialize}; use super::db_error; -use crate::nexus_import::nexus::{Nexus, NexusError}; -use crate::nexus_import::{default_registry_url, password_secret, FORMATS}; +use crate::images::import::ImportSource; +use crate::import::feed::Feed; +use crate::import::{ + nexus_source_url, password_secret, source_url, Listing, SourceError, FORMATS, MAX_NAMES, + SEARCH_PAGE, +}; use crate::state::AppState; /// Imports listed per repository. @@ -85,20 +93,30 @@ fn is_unique_violation(e: &sqlx::Error) -> bool { #[derive(Deserialize)] #[serde(deny_unknown_fields)] pub struct CreateImport { - /// The Nexus server, e.g. `https://nexus.example.com` (with its context - /// path, if it has one). + /// The source: a NuGet feed's service index + /// (`https://nexus.example.com/repository/nuget-hosted/index.json`), or a + /// registry (`https://registry.example.com`). With `repository`, a Nexus + /// server instead (the request before imports read the format's + /// protocol). pub url: String, - /// The Nexus hosted repository to copy. - pub repository: String, - /// Nexus credentials (both, or neither for anonymous access). + /// The Nexus repository at `url` to read (with `url` naming the Nexus + /// server): its service index, `/repository//index.json`, or + /// its registry API, `registry_url` or `/repository/`. + #[serde(default)] + pub repository: Option, + /// With `repository`: the Nexus Docker repository's connector, e.g. + /// `http://nexus.example.com:8082`. + #[serde(default)] + pub registry_url: Option, + /// The package ids or images to copy, instead of every package the + /// feed's search finds (every image the registry's catalog lists). + #[serde(default)] + pub names: Option>, + /// The source's credentials (both, or neither for anonymous access). #[serde(default)] pub username: Option, #[serde(default)] pub password: Option, - /// Docker: the Nexus repository's registry API, e.g. its connector - /// `http://nexus.example.com:8082`. Default: `/repository/`. - #[serde(default)] - pub registry_url: Option, /// A version Forge holds with other bytes: `skip` (and report), /// `overwrite`, or `fail` the import. #[serde(default = "default_on_conflict")] @@ -123,7 +141,7 @@ fn default_concurrency() -> i32 { #[derive(Default, Deserialize)] #[serde(deny_unknown_fields)] pub struct ResumeImport { - /// The Nexus password, when the import no longer holds it (it + /// The source's password, when the import no longer holds it (it /// completed) or it changed. #[serde(default)] pub password: Option, @@ -156,13 +174,10 @@ pub struct Progress { pub failed: i64, /// Dry run: items that would be copied. pub planned: i64, - /// Their size in Nexus. + /// Their size, where the source says (a referrer's). pub planned_size: i64, - /// Bytes downloaded from Nexus. + /// Bytes downloaded from the source. pub bytes_copied: i64, - /// Every asset the scan saw in the Nexus repository, and their size. - pub assets: i64, - pub size: i64, } /// An import, without its password. @@ -170,10 +185,11 @@ pub struct Progress { pub struct ImportView { pub id: i64, pub repository: String, - /// The Nexus server and repository. + /// The source: a feed's service index or a registry. pub url: String, - pub source_repository: String, - pub registry_url: Option, + /// The package ids or images it copies; `null`: every one the source's + /// search or catalog lists. + pub names: Option>, pub username: Option, pub has_password: bool, pub on_conflict: String, @@ -184,7 +200,11 @@ pub struct ImportView { /// `scan` or `copy`. pub phase: String, pub cancel_requested: bool, - pub nexus_version: Option, + /// The `Server` header the source answered the preflight with. + pub source_server: Option, + /// How many packages the feed's search finds: what it reported in the + /// preflight, or as many as the scan walked, whichever is more. + pub source_total: Option, pub last_error: Option, pub created_by: Option, pub created_at: DateTime, @@ -197,7 +217,7 @@ pub struct ImportView { async fn view( state: &AppState, repo: &Repository, - import: NexusImport, + import: RepositoryImport, ) -> sqlx::Result { let counts = import_dao::counts(&state.pool, import.id).await?; let count = |s: &str| counts.get(s).map(|c| c.items).unwrap_or(0); @@ -212,15 +232,12 @@ async fn view( planned: count("planned"), planned_size: counts.get("planned").map(|c| c.size).unwrap_or(0), bytes_copied: counts.values().map(|c| c.bytes_copied).sum(), - assets: import.assets_seen, - size: import.bytes_seen, }; Ok(ImportView { id: import.id, repository: repo.name.clone(), - url: import.base_url, - source_repository: import.source_repository, - registry_url: import.registry_url, + url: source_url(&import, state.formats.stores_images(&repo.format)), + names: import.names, username: import.username, has_password: import.has_password, on_conflict: import.on_conflict, @@ -229,7 +246,8 @@ async fn view( state: import.state, phase: import.phase, cancel_requested: import.cancel_requested, - nexus_version: import.nexus_version, + source_server: import.source_server, + source_total: import.source_total, last_error: import.last_error, created_by: import.created_by, created_at: import.created_at, @@ -240,7 +258,7 @@ async fn view( }) } -async fn view_response(state: &AppState, repo: &Repository, import: NexusImport) -> Response { +async fn view_response(state: &AppState, repo: &Repository, import: RepositoryImport) -> Response { match view(state, repo, import).await { Ok(view) => Json(view).into_response(), Err(e) => db_error(e), @@ -260,7 +278,7 @@ async fn resolve( state: &AppState, name: &str, id: i64, -) -> Result<(Repository, NexusImport), Response> { +) -> Result<(Repository, RepositoryImport), Response> { let repo = resolve_repo(state, name).await?; match import_dao::get(&state.pool, id).await { Ok(Some(import)) if import.repository_id == repo.id => Ok((repo, import)), @@ -270,11 +288,11 @@ async fn resolve( } /// A preflight failure as the administrator sees it. -fn preflight_error(what: &str, e: NexusError) -> Response { - let status = match e { - NexusError::Unreachable(_) => StatusCode::BAD_GATEWAY, - NexusError::Status { status, .. } if status.is_server_error() => StatusCode::BAD_GATEWAY, - _ => StatusCode::BAD_REQUEST, +fn preflight_error(what: &str, e: SourceError) -> Response { + let status = if e.is_transient() { + StatusCode::BAD_GATEWAY + } else { + StatusCode::BAD_REQUEST }; (status, format!("{what}: {e}")).into_response() } @@ -324,21 +342,73 @@ pub async fn create( } if !FORMATS.contains(&repo.format.as_str()) { return bad_request(format!( - "importing {} repositories from Nexus is not supported; supported: {}", + "importing {} repositories is not supported; supported: {}", repo.format, FORMATS.join(", ") )); } + let images = state.formats.stores_images(&repo.format); + let Some(format) = state.formats.get(&repo.format) else { + return bad_request(format!("the {} format is not loaded", repo.format)); + }; // The request itself. let url = body.url.trim().trim_end_matches('/').to_string(); if let Err(e) = validate_url(&url) { return bad_request(format!("invalid url: {e}")); } - let source_repository = body.repository.trim().to_string(); - if source_repository.is_empty() || source_repository.contains('/') { - return bad_request("repository must name one Nexus repository"); - } + let registry_url = body + .registry_url + .map(|u| u.trim().trim_end_matches('/').to_string()) + .filter(|u| !u.is_empty()); + let source = match body.repository.map(|r| r.trim().to_string()) { + Some(repository) => { + if repository.is_empty() || repository.contains('/') { + return bad_request("repository must name one Nexus repository"); + } + if registry_url.is_some() && !images { + return bad_request("registry_url applies to Docker repositories only"); + } + if let Some(registry_url) = ®istry_url { + if let Err(e) = validate_url(registry_url) { + return bad_request(format!("invalid registry_url: {e}")); + } + } + nexus_source_url(&url, &repository, registry_url.as_deref(), images) + } + None if registry_url.is_some() => { + return bad_request( + "registry_url goes with repository (a Nexus server's url); give the registry as url", + ); + } + None => url, + }; + let names = match body.names { + None => None, + Some(names) => { + let mut seen = std::collections::HashSet::new(); + let mut kept = Vec::new(); + for name in names { + let name = name.trim().to_string(); + if name.is_empty() { + continue; + } + if let Err(e) = format.validate_id(&name) { + return bad_request(format!("names: {name:?} is no valid name: {e}")); + } + if seen.insert(format.lower_id(&name)) { + kept.push(name); + } + } + if kept.is_empty() { + return bad_request("names lists no package id or image; leave it out to copy every one the source lists"); + } + if kept.len() > MAX_NAMES { + return bad_request(format!("names lists more than {MAX_NAMES}")); + } + Some(kept) + } + }; let username = body.username.filter(|u| !u.is_empty()); let password = password_secret(body.password); if username.is_some() != password.is_some() { @@ -350,74 +420,76 @@ pub async fn create( if !(1..=16).contains(&body.concurrency) { return bad_request("concurrency must be between 1 and 16"); } - let images = state.formats.stores_images(&repo.format); - let registry_url = match body - .registry_url - .map(|u| u.trim().trim_end_matches('/').to_string()) - { - Some(_) if !images => { - return bad_request("registry_url applies to Docker repositories only") - } - Some(u) if u.is_empty() => None, - Some(u) => match validate_url(&u) { - Ok(()) => Some(u), - Err(e) => return bad_request(format!("invalid registry_url: {e}")), - }, - None => None, - }; - // The preflight: is it Nexus, and the right repository? - let nexus = Nexus::new( - state.upstream.clone(), - &url, - username.clone(), - password.clone(), - ); - let version = match nexus.status().await { - Ok(version) => version, - Err(e) => return preflight_error(&format!("Nexus at {url}"), e), - }; - match nexus.repository(&source_repository).await { - Ok(Some(source)) => { - if source.kind != "hosted" { - return bad_request(format!( - "{source_repository} is a {} repository in Nexus; only hosted repositories are imported (a proxy refills from its upstream, a group is configuration)", - source.kind - )); + // The preflight: does the source speak the format's protocol, and list + // what to copy? + let (server, total) = if images { + let credentials = match (&username, &password) { + (Some(username), Some(password)) => Some(Credentials::for_url( + &source, + username.clone(), + password.clone(), + )), + _ => None, + }; + let from = ImportSource::new(&source, credentials, "import:preflight".into()); + let registry = state.registry(); + let server = match registry.probe_source(&from).await { + Ok(server) => server, + Err(e) => { + return preflight_error( + &format!("the registry API does not answer at {source}/v2/"), + SourceError::of_image(e), + ) } - if source.format != repo.format { - return bad_request(format!( - "{source_repository} is a {} repository in Nexus, but {} holds {}", - source.format, repo.name, repo.format - )); + }; + if names.is_none() { + match registry.source_catalog(&from, None, 1).await { + Ok(Some(_)) => {} + Ok(None) => { + return bad_request(format!( + "the registry at {source} serves no catalog (/v2/_catalog); name the images to import" + )) + } + Err(e) => { + return preflight_error("reading the registry's catalog", SourceError::of_image(e)) + } } } - Ok(None) => { - return bad_request(format!( - "Nexus has no repository {source_repository}, or the credentials may not see it" - )); - } - Err(e) => return preflight_error("listing the Nexus repositories", e), - } - if images { - let registry = registry_url - .clone() - .unwrap_or_else(|| default_registry_url(&url, &source_repository)); - let probe = Nexus::new( + (server, None) + } else { + let feed = match Feed::open( state.upstream.clone(), - ®istry, + format, + &source, username.clone(), password.clone(), - ); - if let Err(e) = probe.probe_registry(&format!("{registry}/v2/")).await { - return preflight_error( - &format!( - "the registry API of {source_repository} does not answer at {registry}/v2/ (set registry_url to the repository's connector, e.g. http://nexus.example.com:8082)" - ), - e, - ); - } - } + ) + .await + { + Ok(feed) => feed, + Err(e) => return preflight_error(&format!("the NuGet feed at {source}"), e), + }; + let total = if names.is_none() { + // A page's worth: Nexus reports the hits of the page it answers + // as the search's total. + match feed.search(0, SEARCH_PAGE).await { + Ok(Some((total, ids))) => { + Some(i64::try_from(total.max(ids.len() as u64)).unwrap_or(i64::MAX)) + } + Ok(None) => { + return bad_request(format!( + "the NuGet feed at {source} offers no search; name the packages to import" + )) + } + Err(e) => return preflight_error("searching the feed", e), + } + } else { + None + }; + (feed.server().map(str::to_string), total) + }; + let listing = Listing::of(images, names.is_some()); let created = async { let mut tx = forge_db::begin(&state.pool).await?; @@ -425,14 +497,15 @@ pub async fn create( &mut *tx, NewImport { repository_id: repo.id, - base_url: &url, - source_repository: &source_repository, - registry_url: registry_url.as_deref(), + url: &source, + names: names.as_deref(), username: username.as_deref(), on_conflict: &body.on_conflict, concurrency: body.concurrency, dry_run: body.dry_run, - nexus_version: Some(&version), + scan_listing: listing.as_str(), + source_server: server.as_deref(), + source_total: total, created_by: Some(user_id), }, ) @@ -462,11 +535,11 @@ pub async fn create( tracing::info!( import = id, repository = %repo.name, - source = %url, - source_repository = %source_repository, - nexus = %version, + source = %source, + server = server.as_deref(), + total, dry_run = body.dry_run, - "nexus import queued" + "import queued" ); match import_dao::get(&state.pool, id).await { Ok(Some(import)) => match view(&state, &repo, import).await { @@ -563,11 +636,11 @@ pub async fn resume( } let password = password_secret(body.password); if password.is_some() && import.username.is_none() { - return bad_request("the import reads Nexus anonymously; it takes no password"); + return bad_request("the import reads its source anonymously; it takes no password"); } if import.username.is_some() && !import.has_password && password.is_none() { return bad_request( - "the import's password was deleted when it completed; give the Nexus password again", + "the import's password was deleted when it completed; give the source's password again", ); } let resumed = async { diff --git a/crates/server/src/admin/mod.rs b/crates/server/src/admin/mod.rs index c17790d8..af2dcdda 100644 --- a/crates/server/src/admin/mod.rs +++ b/crates/server/src/admin/mod.rs @@ -24,8 +24,9 @@ //! * [`blob_cache`] — the local blob cache's statistics. //! * [`proxy_cache`] — invalidating what a proxy repository cached from its //! upstream (its "not found" answers, listings and documents). -//! * [`imports`] — importing a Nexus hosted repository into a hosted -//! repository (start with a preflight, progress, cancel, resume, retry). +//! * [`imports`] — importing a repository from another server (a NuGet feed, +//! a registry) into a hosted repository (start with a preflight, progress, +//! cancel, resume, retry). //! * [`email`] — the email service's settings and a test send. //! * [`email_oauth`] — registering and authorizing Forge for the email //! service's SMTP with OAuth. diff --git a/crates/server/src/images/import.rs b/crates/server/src/images/import.rs index b44eccd1..2e7055e2 100644 --- a/crates/server/src/images/import.rs +++ b/crates/server/src/images/import.rs @@ -1,6 +1,8 @@ -//! Imports into a hosted repository from another registry's API (a Nexus -//! hosted repository's): a tag with everything it references, copied the -//! way `crane copy` copies it, and stored through the same checks as a push. +//! Imports into a hosted repository from another registry's API (Nexus, +//! Harbor, a `distribution` registry, another Forge): what the source lists +//! (its catalog, tag lists and referrers, [`Registry::source_catalog`] and +//! its siblings), and a tag with everything it references, copied the way +//! `crane copy` copies it, and stored through the same checks as a push. //! //! * Digests are kept: a manifest is stored as the source served it, once //! its bytes matched the digest it was asked for (a tag's: the source's @@ -36,9 +38,10 @@ use forge_core::repo::proxy::Credentials; use forge_db::model::{RepoKind, Repository}; use forge_db::oci_dao::{self, LinkOutcome, TagWrite}; use forge_plugin::api::images::{ - ImageError, ManifestBlob, ParsedManifest, Reference, MAX_MANIFEST_BYTES, + CatalogPage, ImageError, ManifestBlob, ManifestInfo, ParsedManifest, Reference, TagPage, + MAX_MANIFEST_BYTES, }; -use forge_plugin::api::UpstreamOp; +use forge_plugin::api::{UpstreamAnswer, UpstreamOp}; use futures_util::TryStreamExt; use reqwest::Method; @@ -80,6 +83,99 @@ impl ImportSource { } } +impl Registry { + /// Check that `source` answers the registry API (`GET /v2/`: 200, or 200 + /// once its token challenge is answered). Returns the `Server` header it + /// answered with. + pub async fn probe_source(&self, source: &ImportSource) -> ImgResult> { + let upstream = source.upstream(self); + let base = source.base.trim_end_matches('/'); + let url = format!("{}/v2/", base.strip_suffix("/v2").unwrap_or(base)); + let response = upstream + .send(Method::GET, &url, None, None, RequestClass::Metadata) + .await?; + if !response.status().is_success() { + return Err(source_status(response.status(), "/v2/")); + } + Ok(response + .headers() + .get(reqwest::header::SERVER) + .and_then(|v| v.to_str().ok()) + .map(str::to_string)) + } + + /// Up to `n` images of the source's catalog after `last`; `None` when it + /// serves no catalog. + pub async fn source_catalog( + &self, + source: &ImportSource, + last: Option<&str>, + n: u32, + ) -> ImgResult> { + let op = UpstreamOp::ImageCatalog { + last: last.map(str::to_string), + n, + }; + match source.upstream(self).converse(self, &op).await? { + UpstreamAnswer::ImageCatalog(page) => Ok(page), + UpstreamAnswer::Failed(reason) => Err(Failure::unavailable(reason)), + other => Err(Failure::internal( + "source catalog", + format!("the images format answered {other:?}"), + )), + } + } + + /// Up to `n` tags of `image` at the source after `last`; `None` when the + /// source does not know the image. + pub async fn source_tags( + &self, + source: &ImportSource, + image: &str, + last: Option<&str>, + n: u32, + ) -> ImgResult> { + let op = UpstreamOp::ImageTags { + image: image.to_string(), + last: last.map(str::to_string), + n, + }; + match source.upstream(self).converse(self, &op).await? { + UpstreamAnswer::ImageTags(page) => Ok(page), + UpstreamAnswer::Failed(reason) => Err(Failure::unavailable(reason)), + other => Err(Failure::internal( + "source tags", + format!("the images format answered {other:?}"), + )), + } + } + + /// The referrers the source lists for the manifest `digest` of `image` + /// (its referrers API); `None` when it has no such API: a referrer is + /// then reached through a tag (the referrers tag schema's + /// `sha256-`, cosign's `sha256-.sig`), which an import + /// copies like any other. + pub async fn source_referrers( + &self, + source: &ImportSource, + image: &str, + digest: &str, + ) -> ImgResult>> { + let op = UpstreamOp::ImageReferrers { + image: image.to_string(), + digest: digest.to_string(), + }; + match source.upstream(self).converse(self, &op).await? { + UpstreamAnswer::ImageReferrers(found) => Ok(found), + UpstreamAnswer::Failed(reason) => Err(Failure::unavailable(reason)), + other => Err(Failure::internal( + "source referrers", + format!("the images format answered {other:?}"), + )), + } + } +} + /// How an import writes an image. #[derive(Debug, Clone)] pub struct ImageImportOptions { diff --git a/crates/server/src/import/feed.rs b/crates/server/src/import/feed.rs new file mode 100644 index 00000000..7c68352b --- /dev/null +++ b/crates/server/src/import/feed.rs @@ -0,0 +1,332 @@ +//! A NuGet v3 feed as an import's source, read through the NuGet format's +//! own upstream protocol ([`Format::upstream`], the conversations a proxy +//! repository holds with its upstream): +//! +//! * the service index (`…/index.json`) resolves the feed's resources +//! ([`UpstreamOp::Index`]); +//! * its search (`SearchQueryService`) names the package ids, a page at a +//! time ([`Feed::search`]); +//! * its package base address lists each package's versions, unlisted ones +//! included ([`Feed::versions`]), and serves the `.nupkg` +//! ([`Feed::package_url`]); +//! * its registration says when each version was published, and its SHA-512 +//! where the feed lists one inline ([`Feed::metadata`]). +//! +//! Requests go only where the feed points: the origin of its service index, +//! and those its documents name ([`UpstreamOrigins`]). The credentials go to +//! the service index's origin alone, through the shared [`UpstreamHttp`] +//! client (timeouts, size caps, no cloud-metadata targets). + +use std::sync::{Arc, Mutex}; + +use forge_core::format::Format; +use forge_core::net::origins::{named_origins, UpstreamOrigins}; +use forge_core::net::upstream::{Origin, RequestClass, UpstreamAuth, UpstreamError, UpstreamHttp}; +use forge_crypto::{ExposeSecret, PlaintextSecret}; +use forge_plugin::api::{ + FetchResult, Fetched, PackageRef, PackageVersions, Remote, UpstreamAnswer, UpstreamOp, + UpstreamStep, VersionMetadata, +}; +use reqwest::StatusCode; + +use super::SourceError; + +/// Rounds of one conversation, and fetches in all: a version list or a +/// registration takes a few. +const MAX_ROUNDS: usize = 8; +const MAX_FETCHES: usize = 256; + +/// A NuGet v3 feed. +pub struct Feed { + http: UpstreamHttp, + format: Arc, + /// The service index, and the resources it lists (once resolved). + remote: Remote, + origins: Mutex, + username: Option, + password: Option, + /// The `Server` header the service index was served with. + server: Option, +} + +impl Feed { + /// The feed at `url` (its service index), resolved: its resources are + /// known once this returns. + pub async fn open( + http: UpstreamHttp, + format: Arc, + url: &str, + username: Option, + password: Option, + ) -> Result { + let mut feed = Feed { + http, + format, + remote: Remote { + url: url.to_string(), + index: None, + }, + origins: Mutex::new(UpstreamOrigins::new(url, [])), + username, + password, + server: None, + }; + // The service index once directly, for its `Server` header and a + // plain answer to an unreachable or refusing feed. + let response = feed + .http + .get(url, RequestClass::Metadata, feed.auth().as_ref()) + .await + .map_err(|e| SourceError::of(e, url))?; + if !response.status().is_success() { + return Err(SourceError::status(response.status(), url)); + } + feed.server = response + .headers() + .get(reqwest::header::SERVER) + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + match feed.converse(UpstreamOp::Index).await? { + UpstreamAnswer::Index(index) => feed.remote.index = Some(index), + other => { + return Err(SourceError::Invalid(format!( + "{url} is no NuGet v3 service index: {}", + answer_text(&other) + ))) + } + } + Ok(feed) + } + + /// The `Server` header the service index was served with. + pub fn server(&self) -> Option<&str> { + self.server.as_deref() + } + + /// The Basic credentials, for the service index's origin only. + fn auth(&self) -> Option> { + match (&self.username, &self.password) { + (Some(username), Some(password)) => Some(UpstreamAuth { + origin: Origin::of(&self.remote.url), + username, + password: password.expose_secret(), + bearer: false, + }), + _ => None, + } + } + + /// A page of the feed's search for every package (prerelease and SemVer + /// 2.0.0 versions included): its total and the ids, after `skip`. + /// `None` when the feed offers no search. + pub async fn search( + &self, + skip: u32, + take: u32, + ) -> Result)>, SourceError> { + match self.converse(UpstreamOp::Packages { skip, take }).await? { + UpstreamAnswer::Search(None) => Ok(None), + UpstreamAnswer::Search(Some(page)) => Ok(Some(( + page.total, + page.hits.into_iter().map(|hit| hit.id).collect(), + ))), + other => Err(unexpected("search", &other)), + } + } + + /// Every version of the package `id` the feed lists (normalized), + /// unlisted ones included; `None` when it does not know the package. + pub async fn versions(&self, id: &str) -> Result>, SourceError> { + let id = self.format.lower_id(id); + match self.converse(UpstreamOp::Versions(id)).await? { + UpstreamAnswer::Versions(versions) => Ok(versions), + other => Err(unexpected("versions", &other)), + } + } + + /// What the feed's registration says of `versions` of `id`: when each + /// was published, whether it is listed, its SHA-512 where listed. Best + /// effort: a version the registration does not have is left out. + pub async fn metadata( + &self, + id: &str, + versions: &[String], + ) -> Result, SourceError> { + let wanted = PackageVersions { + id: self.format.lower_id(id), + versions: versions.to_vec(), + }; + match self.converse(UpstreamOp::Metadata(wanted)).await? { + UpstreamAnswer::Metadata(metadata) => Ok(metadata.versions), + other => Err(unexpected("registration", &other)), + } + } + + /// Where the feed serves the package of `id` `version`: on an origin the + /// feed names, or an error. + pub async fn package_url(&self, id: &str, version: &str) -> Result { + let target = PackageRef { + id: self.format.lower_id(id), + version: version.to_string(), + }; + let url = match self.converse(UpstreamOp::Package(target)).await? { + UpstreamAnswer::Package(url) => url, + UpstreamAnswer::Located(location) => location.url, + other => return Err(unexpected("package", &other)), + }; + if !self.permits(&url) { + return Err(SourceError::Invalid(format!( + "the feed names the package at {url}, an origin it does not otherwise name" + ))); + } + Ok(url) + } + + /// GET a package (streamed by the caller). + pub async fn download(&self, url: &str) -> Result { + let response = self + .http + .get(url, RequestClass::Download, self.auth().as_ref()) + .await + .map_err(|e| SourceError::of(e, url))?; + if !response.status().is_success() { + return Err(SourceError::status(response.status(), url)); + } + Ok(response) + } + + fn permits(&self, url: &str) -> bool { + self.origins + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .permits(url) + } + + /// One conversation of the format with the feed about `op`: the format + /// names what to fetch, the feed's answers go back to it, until it + /// answers. A failed answer after a fetch that failed is that fetch's + /// failure (an unreachable or refusing feed). + async fn converse(&self, op: UpstreamOp) -> Result { + let mut fetched: Vec = Vec::new(); + let mut trouble: Option = None; + for _ in 0..MAX_ROUNDS { + let (format, remote, round) = (self.format.clone(), self.remote.clone(), op.clone()); + let (step, back) = tokio::task::spawn_blocking(move || { + let step = format.upstream(&remote, &round, &fetched); + (step, fetched) + }) + .await + .map_err(|e| SourceError::Invalid(format!("the NuGet protocol: {e}")))?; + fetched = back; + let wanted = + match step.map_err(|e| SourceError::Invalid(format!("the NuGet protocol: {e}")))? { + UpstreamStep::Done(UpstreamAnswer::Failed(reason)) => { + return Err(match trouble { + Some(trouble) => trouble, + None => SourceError::Invalid(reason), + }) + } + UpstreamStep::Done(answer) => return Ok(answer), + UpstreamStep::Fetch(wanted) => wanted, + }; + if wanted.is_empty() || fetched.len() + wanted.len() > MAX_FETCHES { + return Err(SourceError::Invalid(format!( + "the NuGet protocol asked for {} fetches after {}", + wanted.len(), + fetched.len() + ))); + } + for fetch in wanted { + let (outcome, status) = if !self.permits(&fetch.url) { + ( + Fetched::Failed(format!( + "{}: not sent, its origin is neither the service index's nor one \ + the feed names", + fetch.url + )), + None, + ) + } else { + let got = self + .http + .get_bytes_accepting( + &fetch.url, + self.auth().as_ref(), + fetch.accept.as_deref(), + ) + .await; + match got { + Ok(Some(body)) => { + self.origins + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .extend(named_origins(&body)); + (Fetched::Found(body), Some(200)) + } + Ok(None) => (Fetched::NotFound, Some(404)), + Err(e) => { + let error = SourceError::of(e, &fetch.url); + let status = error.status_code(); + let reason = error.to_string(); + trouble.get_or_insert(error); + (Fetched::Failed(reason), status) + } + } + }; + fetched.push(FetchResult { + fetch, + outcome, + status, + headers: Vec::new(), + }); + } + } + Err(SourceError::Invalid(format!( + "the NuGet protocol needed more than {MAX_ROUNDS} rounds" + ))) + } +} + +/// What an answer was, for an error message. +fn answer_text(answer: &UpstreamAnswer) -> String { + match answer { + UpstreamAnswer::Failed(reason) => reason.clone(), + other => format!("{other:?}"), + } +} + +fn unexpected(what: &str, answer: &UpstreamAnswer) -> SourceError { + SourceError::Invalid(format!("the feed's {what}: {}", answer_text(answer))) +} + +impl SourceError { + /// A transport or status error of the upstream client for `url`. + pub fn of(e: UpstreamError, url: &str) -> Self { + match e { + UpstreamError::Status { status, url } => SourceError::status(status, &url), + UpstreamError::Json { source, .. } => SourceError::Invalid(format!("{url}: {source}")), + other => SourceError::Unreachable(other.to_string()), + } + } + + /// An answer of `status` for `url` that is not the one asked for. + pub fn status(status: StatusCode, url: &str) -> Self { + if matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) { + SourceError::Refused(format!( + "the source refused the credentials ({status}) for {url}" + )) + } else { + SourceError::Status { + status, + url: url.to_string(), + } + } + } + + fn status_code(&self) -> Option { + match self { + SourceError::Status { status, .. } => Some(status.as_u16()), + _ => None, + } + } +} diff --git a/crates/server/src/import/mod.rs b/crates/server/src/import/mod.rs new file mode 100644 index 00000000..70399795 --- /dev/null +++ b/crates/server/src/import/mod.rs @@ -0,0 +1,1353 @@ +//! Importing a repository from another server into a Forge hosted +//! repository of the same format (NuGet, Docker), one repository at a time. +//! The source is read through the format's own protocol, so it may be any +//! server that speaks it: Nexus, Artifactory, ProGet, BaGet, Harbor, a +//! `distribution` registry or another Forge. +//! +//! * **NuGet:** a feed's service index (`…/index.json`, [`feed::Feed`]): +//! its search names the package ids, its package base address lists each +//! package's versions (unlisted ones included) and serves the `.nupkg`, +//! its registration says when each version was published and, where the +//! feed lists it inline, its SHA-512. +//! * **Docker:** a registry's API ([`ImportSource`]): its catalog +//! (`/v2/_catalog`) names the images, their tag lists the tags, and the +//! referrers API the signatures and SBOMs of each copied manifest. +//! +//! An import can name its package ids or images instead (`names`): a feed's +//! search may miss some (Nexus stops at 10,000 packages; a package whose +//! versions are all unlisted is in no search result), and Docker Hub, GHCR +//! and ECR serve no catalog. +//! +//! An administrator starts an import on a hosted repository +//! ([`crate::admin::imports`]); this worker runs it, on one instance at a +//! time: it claims a queued import (or one whose instance died) with a +//! lease in Postgres and renews the lease while it works, like the +//! retention worker. +//! +//! 1. **Scan.** The source's listing (the search, the catalog or the names) +//! is walked page by page; each page's versions or tags become pending +//! items and the page's cursor the import's, in one transaction, so a +//! scan resumes after a restart, a failure or a cancel at the page it +//! had not recorded. +//! 2. **Copy.** Pending items are copied, `concurrency` at a time: +//! * a package (NuGet) is streamed from the feed into staging, verified +//! against the SHA-512 the feed lists (where it lists one), and written +//! through the hosted write path ([`HostedWrite::import_staged`]): +//! format validation, the package selectors of the administrator who +//! started the import, the repository's upload limit, CAS dedup and +//! the audit trail (`action = "import"`). A version Forge holds with +//! the SHA-512 the feed lists is skipped without a download; without +//! one, the download is compared; +//! * a Docker tag is copied through the registry +//! ([`crate::images::Registry::import_tag`]): digests are kept, +//! multi-platform indexes arrive with every child manifest. The +//! referrers the source's referrers API lists for the tag's manifest +//! become items of their own, copied once their subject is in Forge +//! ([`crate::images::Registry::import_referrer`]); a source without +//! the API serves them under tags (the referrers tag schema's +//! `sha256-`, cosign's `sha256-.sig`), copied like any +//! other; +//! * what the source says is kept: a NuGet version's publish time becomes +//! its creation time (nuget.org's `1900-01-01` for an unlisted version +//! is no publish time). The protocols carry no last download, so an +//! imported version counts as downloaded at its import, and no push +//! time for a tag, so a tag is created at its import; +//! * a version Forge holds with other bytes is a conflict: skipped and +//! reported (`skip`), replaced (`overwrite`), or it fails the import +//! (`fail`). +//! +//! A dry run scans and decides every item (`planned`, `identical`, +//! `conflict`) without writing a package; it downloads no package. Running +//! an import again later copies only what is new at the source, so a +//! cutover is an import, then a final one just before clients switch to +//! Forge. +//! +//! An import stored before imports read the format's protocol names a +//! Nexus server and repository; it reads the same repository through its +//! protocol ([`source_url`]). +//! +//! A failed item can be retried on its own, and a cancelled or failed +//! import resumed ([`forge_db::import_dao::requeue`]). The source's +//! password is sealed with the secret encryption and deleted when the +//! import completes. + +pub mod feed; + +use std::collections::HashMap; +use std::panic::AssertUnwindSafe; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use chrono::{DateTime, Datelike, Utc}; +use forge_core::auth::principal::{AuthMethod, Principal}; +use forge_core::authz::Level; +use forge_core::repo::hosted::HostedWrite; +use forge_core::repo::proxy::Credentials; +use forge_core::repo::{ImportOptions, ImportOutcome, RepoKind, WriteError}; +use forge_crypto::PlaintextSecret; +use forge_db::import_dao::{self, ImportItem, NewItem, RepositoryImport}; +use forge_db::model::Repository; +use forge_db::{repository_dao, secret_dao, user_dao}; +use forge_plugin::api::images::ImageError; +use forge_plugin::api::VersionMetadata; +use forge_storage::StagedBlob; +use futures_util::future::BoxFuture; +use futures_util::{FutureExt, StreamExt, TryStreamExt}; +use reqwest::StatusCode; +use serde::{Deserialize, Serialize}; +use tokio::task::JoinHandle; +use uuid::Uuid; + +use crate::images::import::{ImageImportOptions, ImageOutcome, ImportSource}; +use crate::images::{Failure, Registry}; +use crate::state::AppState; +use feed::Feed; + +/// The formats an import copies: NuGet packages, and Docker images into +/// the image store. +pub const FORMATS: &[&str] = &["nuget", "docker"]; + +/// How often an idle instance looks for an import to run. +const POLL_INTERVAL: Duration = Duration::from_secs(5); + +/// The lease a running import is held with; renewed every third of it. +const LEASE_SECONDS: i64 = 120; + +/// Pending items read at a time. +const BATCH: i64 = 64; + +/// Items failing in a row that stop an import (the source went away, the +/// credentials stopped working): it fails, and can be resumed. +const MAX_CONSECUTIVE_FAILURES: usize = 25; + +/// Attempts at a read of the source's listing before the import fails. +const PAGE_ATTEMPTS: u32 = 4; + +/// How often a copy looks whether it was cancelled (or lost its lease). +const CHECK_INTERVAL: Duration = Duration::from_secs(5); + +/// Package ids a page of a feed's search names. +pub const SEARCH_PAGE: u32 = 100; + +/// Images a page of a registry's catalog names. +const CATALOG_PAGE: u32 = 100; + +/// Tags read at a time. +const TAGS_PAGE: u32 = 1000; + +/// Named package ids or images a scan page reads. +const NAMES_PAGE: usize = 20; + +/// The most package ids or images an import names. +pub const MAX_NAMES: usize = 10_000; + +/// What went wrong reading the source. +#[derive(Debug, thiserror::Error)] +pub enum SourceError { + /// The source could not be reached, or the transfer failed. + #[error("the source is unreachable: {0}")] + Unreachable(String), + /// The source cannot answer now (a registry's error, its rate limit). + #[error("{0}")] + Unavailable(String), + /// The credentials were refused (or may not see the resource). + #[error("{0}")] + Refused(String), + /// Any other answer than the one asked for. + #[error("the source answered {status} for {url}")] + Status { status: StatusCode, url: String }, + /// The answer was not what the protocol describes. + #[error("{0}")] + Invalid(String), +} + +impl SourceError { + /// Whether trying again later may succeed. + pub fn is_transient(&self) -> bool { + match self { + SourceError::Unreachable(_) | SourceError::Unavailable(_) => true, + SourceError::Status { status, .. } => { + status.is_server_error() || *status == StatusCode::TOO_MANY_REQUESTS + } + SourceError::Refused(_) | SourceError::Invalid(_) => false, + } + } + + /// A failure of the registry reading the source. + pub fn of_image(e: Failure) -> Self { + let text = e.message(); + match e.error { + ImageError::Unavailable(_) | ImageError::RateLimited => SourceError::Unavailable(text), + ImageError::InvalidCredentials | ImageError::Denied { .. } => { + SourceError::Refused(text) + } + _ => SourceError::Invalid(text), + } + } +} + +/// What an import's scan walks (its `scan_listing`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Listing { + /// A NuGet feed's search. + Search, + /// A registry's catalog. + Catalog, + /// The package ids or images the import names. + Names, +} + +impl Listing { + /// The listing of a new import. + pub fn of(images: bool, named: bool) -> Listing { + match (images, named) { + (_, true) => Listing::Names, + (true, false) => Listing::Catalog, + (false, false) => Listing::Search, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Listing::Search => "search", + Listing::Catalog => "catalog", + Listing::Names => "names", + } + } + + fn parse(s: &str) -> Listing { + match s { + "catalog" => Listing::Catalog, + "names" => Listing::Names, + _ => Listing::Search, + } + } +} + +/// Where in a Nexus server a repository is: `/repository/`. +/// Nexus serves a NuGet repository's service index below it, and a Docker +/// repository's registry API at it (for reads, on its own port), unless the +/// repository has a connector of its own (`registry_url`). +pub fn nexus_source_url( + url: &str, + repository: &str, + registry_url: Option<&str>, + images: bool, +) -> String { + let path = format!("{}/repository/{repository}", url.trim_end_matches('/')); + match registry_url { + Some(registry_url) if images => registry_url.to_string(), + _ if images => path, + _ => format!("{path}/index.json"), + } +} + +/// The source `import` reads: its `url`, or, for an import stored before +/// imports read the format's protocol (a Nexus server and repository), that +/// repository's service index or registry API ([`nexus_source_url`]). +pub fn source_url(import: &RepositoryImport, images: bool) -> String { + match &import.source_repository { + Some(repository) => nexus_source_url( + &import.url, + repository, + import.registry_url.as_deref(), + images, + ), + None => import.url.clone(), + } +} + +/// The worker running imports on this instance. +pub struct ImportWorker { + state: AppState, +} + +impl ImportWorker { + pub fn new(state: AppState) -> Self { + ImportWorker { state } + } + + /// Look for an import every few seconds and run each that is due. + pub fn spawn_loop(self) -> JoinHandle<()> { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(POLL_INTERVAL); + loop { + ticker.tick().await; + loop { + match self.tick().await { + Ok(Some(_)) => {} + Ok(None) => break, + Err(e) => { + tracing::error!(error = %e, "import tick failed"); + break; + } + } + } + } + }) + } + + /// Claim one import that is due and run it until it completes, fails, + /// is cancelled or loses its lease. Returns its id, `None` when no + /// import is due. + pub async fn tick(&self) -> sqlx::Result> { + let pool = &self.state.pool; + let claimed = { + let mut tx = forge_db::begin(pool).await?; + let now = forge_db::db_now(&mut *tx).await?; + let claimed = + import_dao::claim_next(&mut tx, now, chrono::Duration::seconds(LEASE_SECONDS)) + .await?; + tx.commit().await?; + claimed + }; + let Some((import, run_id)) = claimed else { + return Ok(None); + }; + let id = import.id; + tracing::info!( + import = id, + repository_id = import.repository_id, + source = %import.url, + source_repository = import.source_repository.as_deref(), + phase = %import.phase, + "import started" + ); + let lost = Arc::new(AtomicBool::new(false)); + let heartbeat = AbortOnDrop(spawn_heartbeat(pool.clone(), id, run_id, lost.clone())); + let run = Run { + state: &self.state, + import, + run_id, + lost, + stopping: AtomicBool::new(false), + last_search: std::sync::Mutex::new(Vec::new()), + }; + let end = AssertUnwindSafe(run.execute()) + .catch_unwind() + .await + .unwrap_or_else(|panic| { + End::Failed(format!("the import panicked: {}", panic_text(&*panic))) + }); + drop(heartbeat); + let (state, error) = match &end { + End::Completed => ("completed", None), + End::Cancelled => ("cancelled", None), + End::Failed(error) => ("failed", Some(error.as_str())), + End::LeaseLost => { + tracing::warn!( + import = id, + "import lease lost; another instance continues it" + ); + return Ok(Some(id)); + } + }; + if import_dao::finish(pool, id, run_id, state, error).await? { + tracing::info!(import = id, state, error, "import ended"); + } else { + tracing::warn!(import = id, "import lease lost before it ended"); + } + Ok(Some(id)) + } +} + +/// How a run ended. +enum End { + Completed, + Cancelled, + Failed(String), + /// Another instance holds the import now; it records the outcome. + LeaseLost, +} + +fn db_failed(e: sqlx::Error) -> End { + tracing::error!(error = %e, "import: database error"); + End::Failed(format!("database error: {e}")) +} + +/// Where the items of a run come from and go. +enum Target { + /// A package format: from a feed, through the hosted write path. + Packages { + feed: Box, + writer: Box, + }, + /// Docker: from a registry's API, through the registry. + Images { + registry: Registry, + source: ImportSource, + }, +} + +/// What a run works with. +struct Ctx { + repo: Repository, + principal: Principal, + target: Target, + /// The repository's upload limit. + limit: u64, + /// The source's URL, for the audit trail. + source: String, +} + +/// What the source says of an item beyond its name and version (an item's +/// `source`). An item an import stored before reads as none of it. +#[derive(Debug, Default, Serialize, Deserialize)] +struct ItemSource { + /// The package's SHA-512, base64, where the feed lists it. + #[serde(default, skip_serializing_if = "Option::is_none")] + sha512: Option, +} + +/// A version or tag the scan found. +struct Found { + name: String, + /// Empty for a named package or image the source does not have. + version: String, + published_at: Option>, + source: ItemSource, +} + +impl Found { + /// A named package or image the source does not have: an item that + /// fails, so the import reports it. + fn missing(name: &str) -> Found { + Found { + name: name.to_string(), + version: String::new(), + published_at: None, + source: ItemSource::default(), + } + } +} + +/// One page of the scan: what it found, and the cursor of the next page +/// (`None`: the scan is complete). +struct Page { + found: Vec, + next: Option, + /// A search: how many packages it has walked with this page. + walked: Option, +} + +/// What became of one item. +enum Copied { + /// It ends in `state`. + Item { + state: &'static str, + detail: Option, + bytes: u64, + }, + /// A manifest the source holds by digest that is no referrer: forgotten. + NotReferrer, +} + +/// How a recorded item bears on the run. +enum Recorded { + Done, + /// The run is ending: the item was left pending. + NotStarted, + Failed(String), + /// A conflict, and the import fails on conflicts. + Conflict(String), +} + +/// One run of one import, on this instance. +struct Run<'a> { + state: &'a AppState, + import: RepositoryImport, + run_id: Uuid, + /// Set by the heartbeat once the lease is lost. + lost: Arc, + /// Set once the run is to end: no further item is started. + stopping: AtomicBool, + /// The ids of the search page read last: a feed whose search ignores + /// `skip` answers the same page again. + last_search: std::sync::Mutex>, +} + +impl Run<'_> { + async fn execute(&self) -> End { + let ctx = match self.prepare().await { + Ok(ctx) => ctx, + Err(end) => return end, + }; + if self.import.phase == "scan" { + if let Err(end) = self.scan(&ctx).await { + return end; + } + } + match self.copy(&ctx).await { + Ok(()) => End::Completed, + Err(end) => end, + } + } + + /// The repository, the administrator the import runs as, and the + /// clients of the source. + async fn prepare(&self) -> Result { + let pool = &self.state.pool; + let import = &self.import; + let repo = repository_dao::find_repository_by_id(pool, import.repository_id) + .await + .map_err(db_failed)? + .ok_or_else(|| End::Failed("the repository was deleted".into()))?; + if repo.kind() != RepoKind::Hosted || !FORMATS.contains(&repo.format.as_str()) { + return Err(End::Failed(format!( + "imports go to a hosted NuGet or Docker repository; {} is a {} {} repository", + repo.name, repo.format, repo.kind + ))); + } + + // The import runs with the authority of the administrator who + // started it, as long as they are one. + let gone = + || End::Failed("the administrator who started the import no longer is one".into()); + let user_id = import.created_by.ok_or_else(gone)?; + let user = user_dao::find_user_by_id(pool, user_id) + .await + .map_err(db_failed)? + .filter(|user| user.active) + .ok_or_else(gone)?; + let principal = Principal::User { + user_id: user.id, + token_id: None, + method: AuthMethod::Session, + }; + if !self.state.authz.is_global_admin(&principal).await { + return Err(gone()); + } + + let password = secret_dao::get_import_password(pool, &self.state.envelope, import.id) + .await + .map_err(|e| End::Failed(format!("the import's password cannot be read: {e}")))?; + if import.username.is_some() && password.is_none() { + return Err(End::Failed( + "the import has no password; resume it with the source's password".into(), + )); + } + + let images = self.state.formats.stores_images(&repo.format); + let url = source_url(import, images); + let target = if images { + let credentials = match (&import.username, password) { + (Some(username), Some(password)) => { + Some(Credentials::for_url(&url, username.clone(), password)) + } + _ => None, + }; + Target::Images { + registry: self.state.registry(), + source: ImportSource::new(&url, credentials, format!("import:{}", import.id)), + } + } else { + let format = self + .state + .formats + .for_repository(&repo) + .map_err(|e| End::Failed(e.to_string()))?; + let feed = self + .retrying("reading the feed's service index", || { + let (http, format, url) = + (self.state.upstream.clone(), format.clone(), url.clone()); + let (username, password) = (import.username.clone(), password.clone()); + async move { Feed::open(http, format, &url, username, password).await }.boxed() + }) + .await?; + let authz = &self.state.authz; + let filter = authz + .package_filter_for(&principal, &repo, Level::Write) + .await; + let replace = authz + .package_filter_for(&principal, &repo, Level::Delete) + .await; + let role_filter = authz + .role_package_filter_for(&principal, &repo, Level::Write) + .await; + let role_replace = authz + .role_package_filter_for(&principal, &repo, Level::Delete) + .await; + let writer = Box::new(crate::dispatch::hosted_write( + self.state, + format, + repo.id, + repo.name.clone(), + principal.clone(), + filter, + replace, + role_filter, + role_replace, + )); + Target::Packages { + feed: Box::new(feed), + writer, + } + }; + Ok(Ctx { + limit: self.state.upload_limit(&repo), + source: url, + repo, + principal, + target, + }) + } + + /// Stop when the lease is lost or an administrator cancelled. + async fn check(&self) -> Result<(), End> { + if self.lost.load(Ordering::Relaxed) { + return Err(End::LeaseLost); + } + match import_dao::poll(&self.state.pool, self.import.id, self.run_id).await { + Ok(None) => Err(End::LeaseLost), + Ok(Some(true)) => Err(End::Cancelled), + Ok(Some(false)) => Ok(()), + Err(e) => Err(db_failed(e)), + } + } + + /// A read of the source, retried with a backoff while its failure may + /// pass. + async fn retrying<'f, T>( + &self, + what: &str, + mut read: impl FnMut() -> BoxFuture<'f, Result> + Send, + ) -> Result { + let mut attempt = 0; + loop { + attempt += 1; + match read().await { + Ok(read) => return Ok(read), + Err(e) if e.is_transient() && attempt < PAGE_ATTEMPTS => { + tracing::warn!(import = self.import.id, error = %e, attempt, "{what} failed; retrying"); + tokio::time::sleep(Duration::from_secs(1 << attempt)).await; + } + Err(e) => return Err(End::Failed(format!("{what}: {e}"))), + } + } + } + + /// Walk the listing from the stored cursor, recording each page's + /// items with the cursor of the next page. + async fn scan(&self, ctx: &Ctx) -> Result<(), End> { + let listing = Listing::parse(&self.import.scan_listing); + let mut cursor = self.import.cursor.clone(); + loop { + self.check().await?; + let what = format!("reading the source's {}", listing.as_str()); + let page = self + .retrying(&what, || self.page(ctx, listing, cursor.as_deref()).boxed()) + .await?; + let phase = if page.next.is_some() { "scan" } else { "copy" }; + let sources: Vec = page + .found + .iter() + .map(|found| serde_json::to_value(&found.source).unwrap_or_default()) + .collect(); + let new_items: Vec> = page + .found + .iter() + .zip(&sources) + .map(|(found, source)| NewItem { + kind: "component", + name: &found.name, + version: &found.version, + size: None, + published_at: found.published_at, + downloaded_at: None, + uploader: None, + source, + }) + .collect(); + let pool = &self.state.pool; + let mut tx = forge_db::begin(pool).await.map_err(db_failed)?; + import_dao::insert_items(&mut tx, self.import.id, &new_items) + .await + .map_err(db_failed)?; + let held = import_dao::save_scan_page( + &mut tx, + self.import.id, + self.run_id, + page.next.as_deref(), + phase, + page.walked, + ) + .await + .map_err(db_failed)?; + if !held { + return Err(End::LeaseLost); + } + tx.commit().await.map_err(db_failed)?; + match page.next { + Some(next) => cursor = Some(next), + None => return Ok(()), + } + } + } + + /// One page of the listing after `cursor`: its package ids' versions, + /// or its images' tags. + async fn page( + &self, + ctx: &Ctx, + listing: Listing, + cursor: Option<&str>, + ) -> Result { + let concurrency = self.import.concurrency.clamp(1, 16) as usize; + match (&ctx.target, listing) { + (_, Listing::Names) => { + let names = self.import.names.as_deref().unwrap_or_default(); + let start = cursor + .and_then(|c| c.parse::().ok()) + .unwrap_or(0) + .min(names.len()); + let end = (start + NAMES_PAGE).min(names.len()); + let found = self + .expand(ctx, &names[start..end], true, concurrency) + .await?; + Ok(Page { + found, + next: (end < names.len()).then(|| end.to_string()), + walked: None, + }) + } + (Target::Packages { feed, .. }, Listing::Search) => { + let skip = cursor.and_then(|c| c.parse::().ok()).unwrap_or(0); + let Some((_, ids)) = feed.search(skip, SEARCH_PAGE).await? else { + return Err(SourceError::Invalid( + "the feed offers no search; name the packages to import instead".into(), + )); + }; + // The walk ends at an empty page: a feed's total is no end + // (Nexus reports the hits of the page it answers), nor is a + // short page (a feed may answer fewer than it was asked for). + { + let mut last = self + .last_search + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if skip > 0 && !ids.is_empty() && *last == ids { + return Err(SourceError::Invalid(format!( + "the feed's search answers the same page at {skip} as before it: it \ + ignores skip; name the packages to import instead" + ))); + } + last.clone_from(&ids); + } + let walked = u64::from(skip) + ids.len() as u64; + let next = (!ids.is_empty()).then(|| walked.to_string()); + let found = self.expand(ctx, &ids, false, concurrency).await?; + Ok(Page { + found, + next, + walked: Some(i64::try_from(walked).unwrap_or(i64::MAX)), + }) + } + (Target::Images { registry, source }, Listing::Catalog) => { + let page = registry + .source_catalog(source, cursor, CATALOG_PAGE) + .await + .map_err(SourceError::of_image)? + .ok_or_else(|| { + SourceError::Invalid( + "the registry serves no catalog (/v2/_catalog); name the images to \ + import instead" + .into(), + ) + })?; + let next = page.more.then(|| page.images.last().cloned()).flatten(); + let found = self.expand(ctx, &page.images, false, concurrency).await?; + Ok(Page { + found, + next, + walked: None, + }) + } + _ => Err(SourceError::Invalid(format!( + "a {} repository's import does not walk a {}", + ctx.repo.format, + listing.as_str() + ))), + } + } + + /// The versions of the package ids `names`, or the tags of the images + /// `names`, in their order. A name the source does not have is left + /// out, or, when the import `named` it, found as missing. + async fn expand( + &self, + ctx: &Ctx, + names: &[String], + named: bool, + concurrency: usize, + ) -> Result, SourceError> { + // The reads are made up front: futures do nothing until polled. + let reads: Vec<_> = names + .iter() + .map(|name| expand_one(&ctx.target, name, named)) + .collect(); + let pages: Vec> = futures_util::stream::iter(reads) + .buffered(concurrency) + .try_collect() + .await?; + Ok(pages.into_iter().flatten().collect()) + } + + /// Copy the pending items: tags and versions first, then (Docker) the + /// referrers their copies found. + async fn copy(&self, ctx: &Ctx) -> Result<(), End> { + let failures = AtomicUsize::new(0); + let concurrency = self.import.concurrency.clamp(1, 16) as usize; + for kind in ["component", "manifest"] { + let mut after = 0; + loop { + self.check().await?; + let batch = + import_dao::pending_items(&self.state.pool, self.import.id, kind, after, BATCH) + .await + .map_err(db_failed)?; + let Some(last) = batch.last() else { + break; + }; + after = last.id; + let mut copies = futures_util::stream::iter(batch) + .map(|item| self.copy_item(ctx, item)) + .buffer_unordered(concurrency); + // Once the run is to end (a cancel, a lost lease, too many + // failures), the items in flight finish and the rest stay + // pending. + let mut end = None; + let mut checked = std::time::Instant::now(); + while let Some(recorded) = copies.next().await { + let mut stop = match recorded { + Err(e) => Some(db_failed(e)), + Ok(Recorded::Done) => { + failures.store(0, Ordering::Relaxed); + None + } + Ok(Recorded::NotStarted) => None, + Ok(Recorded::Failed(error)) => (failures.fetch_add(1, Ordering::Relaxed) + + 1 + >= MAX_CONSECUTIVE_FAILURES) + .then(|| { + End::Failed(format!( + "{MAX_CONSECUTIVE_FAILURES} items in a row failed; the last: {error}" + )) + }), + Ok(Recorded::Conflict(conflict)) => Some(End::Failed(conflict)), + }; + if end.is_none() && stop.is_none() && checked.elapsed() >= CHECK_INTERVAL { + checked = std::time::Instant::now(); + stop = self.check().await.err(); + } + if let (None, Some(stop)) = (&end, stop) { + self.stopping.store(true, Ordering::Relaxed); + end = Some(stop); + } + } + if let Some(end) = end { + return Err(end); + } + } + } + Ok(()) + } + + /// Copy one item and record what became of it. + async fn copy_item(&self, ctx: &Ctx, item: ImportItem) -> sqlx::Result { + if self.stopping.load(Ordering::Relaxed) { + return Ok(Recorded::NotStarted); + } + let pool = &self.state.pool; + let copied = if item.version.is_empty() { + Err(match ctx.target { + Target::Packages { .. } => "the source has no package of this id".to_string(), + Target::Images { .. } => "the source has no tag of this image".to_string(), + }) + } else { + match &ctx.target { + Target::Packages { feed, writer } => { + let source: ItemSource = + serde_json::from_value(item.source.clone()).unwrap_or_default(); + self.copy_package(ctx, feed, writer, &item, &source).await + } + Target::Images { registry, source } => { + self.copy_image(ctx, registry, source, &item).await + } + } + }; + match copied { + Ok(Copied::NotReferrer) => { + import_dao::drop_item(pool, item.id).await?; + Ok(Recorded::Done) + } + Ok(Copied::Item { + state, + detail, + bytes, + }) => { + import_dao::finish_item( + pool, + item.id, + state, + detail.as_deref(), + i64::try_from(bytes).unwrap_or(i64::MAX), + ) + .await?; + if state == "conflict" && self.import.on_conflict == "fail" { + return Ok(Recorded::Conflict(format!( + "{} {}: {}", + item.name, + item.version, + detail.unwrap_or_default() + ))); + } + Ok(Recorded::Done) + } + Err(error) => { + tracing::warn!(import = self.import.id, name = %item.name, version = %item.version, %error, "import item failed"); + import_dao::finish_item(pool, item.id, "failed", Some(&error), 0).await?; + Ok(Recorded::Failed(format!( + "{} {}: {error}", + item.name, item.version + ))) + } + } + } + + /// Copy a package version through the hosted write path. + async fn copy_package( + &self, + ctx: &Ctx, + feed: &Feed, + writer: &HostedWrite, + item: &ImportItem, + source: &ItemSource, + ) -> Result { + let overwrite = self.import.on_conflict == "overwrite"; + + // Nothing is downloaded for a version Forge holds with the bytes + // the feed lists, nor for a dry run (which decides from the + // checksums alone). + let stored = writer + .package_sha512(ctx.repo.id, &item.name, &item.version) + .await + .map_err(|e| e.to_string())?; + if let (Some(stored), Some(expected)) = (&stored, &source.sha512) { + if stored == expected { + return Ok(copied("identical", None, 0)); + } + } + if self.import.dry_run { + return Ok(match (stored, &source.sha512) { + (None, _) => copied("planned", None, 0), + (Some(_), None) => copied( + "planned", + Some("Forge has this version; the feed lists no SHA-512 to compare".into()), + 0, + ), + (Some(_), Some(_)) if overwrite => copied( + "planned", + Some("would replace the version Forge has with other bytes".into()), + 0, + ), + (Some(_), Some(_)) => conflict_item(), + }); + } + + let url = feed + .package_url(&item.name, &item.version) + .await + .map_err(|e| e.to_string())?; + let response = feed.download(&url).await.map_err(|e| e.to_string())?; + if response.content_length().is_some_and(|l| l > ctx.limit) { + return Err(WriteError::TooLarge { limit: ctx.limit }.to_string()); + } + let body = response.bytes_stream().map_err(std::io::Error::other); + let staged = self + .state + .cas + .stage(tokio_util::io::StreamReader::new(body), ctx.limit) + .await + .map_err(|e| WriteError::from(e).to_string())?; + verify(&staged, &url, source)?; + let bytes = staged.size; + let options = ImportOptions { + overwrite, + dry_run: false, + published_at: item.published_at, + downloaded_at: Some(item.downloaded_at.unwrap_or_else(Utc::now)), + uploader: item.uploader.clone(), + source: ctx.source.clone(), + }; + let outcome = writer + .import_staged(&ctx.repo.name, staged, ctx.limit, &options) + .await + .map_err(|e| e.to_string())?; + Ok(match outcome { + ImportOutcome::Created { .. } => copied("imported", None, bytes), + ImportOutcome::Replaced { .. } => copied( + "replaced", + Some("replaced the version Forge had with other bytes".into()), + bytes, + ), + ImportOutcome::Identical { .. } => copied("identical", None, bytes), + ImportOutcome::Conflict { .. } => conflict_item(), + ImportOutcome::Planned { .. } => copied("planned", None, bytes), + }) + } + + /// Copy a Docker tag (`component`) or referrer (`manifest`) through the + /// registry, and record the referrers the source lists for it. + async fn copy_image( + &self, + ctx: &Ctx, + registry: &Registry, + from: &ImportSource, + item: &ImportItem, + ) -> Result { + let options = ImageImportOptions { + principal: ctx.principal.clone(), + overwrite: self.import.on_conflict == "overwrite", + dry_run: self.import.dry_run, + published_at: item.published_at, + downloaded_at: Some(item.downloaded_at.unwrap_or_else(Utc::now)), + uploader: item.uploader.clone(), + source: ctx.source.clone(), + }; + let imported = if item.kind == "manifest" { + registry + .import_referrer(&ctx.repo, from, &item.name, &item.version, &options) + .await + } else { + registry + .import_tag(&ctx.repo, from, &item.name, &item.version, None, &options) + .await + } + .map_err(|e| e.message())?; + let bytes = imported.bytes_copied; + // Its referrers come along unless it stays out of Forge. + let held = matches!( + imported.outcome, + ImageOutcome::Created + | ImageOutcome::Replaced { .. } + | ImageOutcome::Identical + | ImageOutcome::Planned { .. } + ); + if let (true, Some(digest)) = (held, &imported.digest) { + self.record_referrers(registry, from, &item.name, digest) + .await + .map_err(|e| format!("copied, but listing its referrers failed: {e}"))?; + } + Ok(match imported.outcome { + ImageOutcome::Created => copied("imported", imported.digest, bytes), + ImageOutcome::Replaced { previous } => { + copied("replaced", Some(format!("moved from {previous}")), bytes) + } + ImageOutcome::Identical => copied("identical", imported.digest, bytes), + ImageOutcome::Conflict { + existing, + immutable, + } => copied( + "conflict", + Some(if immutable { + format!("the tag points at {existing} in Forge, and tags are immutable there") + } else { + format!("the tag points at {existing} in Forge") + }), + bytes, + ), + ImageOutcome::Planned { replaces } => copied( + "planned", + replaces.then(|| "would move the tag Forge has".to_string()), + bytes, + ), + ImageOutcome::NotReferrer => Copied::NotReferrer, + }) + } + + /// Record the referrers the source's referrers API lists for the + /// manifest `digest` of `image` as items, copied after the tags. A + /// source without the API lists none. + async fn record_referrers( + &self, + registry: &Registry, + from: &ImportSource, + image: &str, + digest: &str, + ) -> Result<(), String> { + let Some(referrers) = registry + .source_referrers(from, image, digest) + .await + .map_err(|e| e.message())? + else { + return Ok(()); + }; + if referrers.is_empty() { + return Ok(()); + } + let source = serde_json::json!({}); + let items: Vec> = referrers + .iter() + .map(|referrer| NewItem { + kind: "manifest", + name: image, + version: &referrer.digest, + size: i64::try_from(referrer.size).ok(), + published_at: None, + downloaded_at: None, + uploader: None, + source: &source, + }) + .collect(); + let recorded = async { + let mut tx = forge_db::begin(&self.state.pool).await?; + import_dao::insert_items(&mut tx, self.import.id, &items).await?; + tx.commit().await + }; + recorded.await.map_err(|e| format!("database error: {e}")) + } +} + +/// The versions of the package id `name`, or the tags of the image `name`. +async fn expand_one(target: &Target, name: &str, named: bool) -> Result, SourceError> { + match target { + Target::Packages { feed, .. } => package_versions(feed, name, named).await, + Target::Images { registry, source } => image_tags(registry, source, name, named).await, + } +} + +/// Every version the feed lists of the package `id`, with what its +/// registration says of each. A package the feed does not know is left +/// out, or found as missing when the import `named` it. +async fn package_versions(feed: &Feed, id: &str, named: bool) -> Result, SourceError> { + let versions = feed.versions(id).await?.unwrap_or_default(); + if versions.is_empty() { + return Ok(if named { + vec![Found::missing(id)] + } else { + Vec::new() + }); + } + // The registration is best effort: without it, a version has no + // publish time and no checksum. + let metadata = match feed.metadata(id, &versions).await { + Ok(metadata) => metadata, + Err(e) if e.is_transient() => return Err(e), + Err(e) => { + tracing::warn!(id, error = %e, "import: the feed's registration is unreadable"); + Vec::new() + } + }; + let metadata: HashMap = metadata + .into_iter() + .map(|m| (m.version.to_lowercase(), m)) + .collect(); + Ok(versions + .into_iter() + .map(|version| { + let meta = metadata.get(&version.to_lowercase()); + Found { + name: id.to_string(), + published_at: meta.and_then(|m| published_at(m.published.as_deref())), + source: ItemSource { + sha512: meta.and_then(|m| m.sha512.clone()), + }, + version, + } + }) + .collect()) +} + +/// Every tag of `image` at the source. An image the source does not have +/// (or has no tag of) is left out, or found as missing when the import +/// `named` it. +async fn image_tags( + registry: &Registry, + source: &ImportSource, + image: &str, + named: bool, +) -> Result, SourceError> { + let mut found = Vec::new(); + let mut last: Option = None; + loop { + let page = registry + .source_tags(source, image, last.as_deref(), TAGS_PAGE) + .await + .map_err(SourceError::of_image)?; + let Some(page) = page else { + break; + }; + let more = page.more && !page.tags.is_empty(); + last = page.tags.last().cloned().or(last); + found.extend(page.tags.into_iter().map(|tag| Found { + name: image.to_string(), + version: tag, + published_at: None, + source: ItemSource::default(), + })); + if !more { + break; + } + } + if found.is_empty() && named { + found.push(Found::missing(image)); + } + Ok(found) +} + +/// A publish time from a feed's registration. nuget.org gives an unlisted +/// version `1900-01-01`: no publish time. +fn published_at(published: Option<&str>) -> Option> { + DateTime::parse_from_rfc3339(published?) + .ok() + .map(|t| t.with_timezone(&Utc)) + .filter(|t| t.year() > 1900) +} + +fn copied(state: &'static str, detail: Option, bytes: u64) -> Copied { + Copied::Item { + state, + detail, + bytes, + } +} + +fn conflict_item() -> Copied { + copied( + "conflict", + Some("Forge has other bytes for this version".into()), + 0, + ) +} + +/// Check a staged download against the SHA-512 the feed lists for it. +fn verify(staged: &StagedBlob, url: &str, source: &ItemSource) -> Result<(), String> { + match &source.sha512 { + Some(expected) if *expected != staged.sha512_base64 => Err(format!( + "the download of {url} does not match the SHA-512 the feed lists for it" + )), + _ => Ok(()), + } +} + +/// Aborts the wrapped task when dropped. +struct AbortOnDrop(JoinHandle<()>); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.abort(); + } +} + +/// Renew the lease every third of its length until aborted; flags `lost` +/// and stops once the lease is gone. +fn spawn_heartbeat( + pool: sqlx::PgPool, + id: i64, + run_id: Uuid, + lost: Arc, +) -> JoinHandle<()> { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(Duration::from_secs((LEASE_SECONDS / 3) as u64)); + ticker.tick().await; + loop { + ticker.tick().await; + let renewed = match forge_db::db_now(&pool).await { + Ok(now) => { + import_dao::renew_lease( + &pool, + id, + run_id, + now + chrono::Duration::seconds(LEASE_SECONDS), + ) + .await + } + Err(e) => Err(e), + }; + match renewed { + Ok(true) => {} + Ok(false) => { + lost.store(true, Ordering::Relaxed); + return; + } + Err(e) => { + tracing::warn!(import = id, error = %e, "import lease renewal failed") + } + } + } + }) +} + +fn panic_text(payload: &(dyn std::any::Any + Send)) -> String { + payload + .downcast_ref::<&str>() + .map(|s| (*s).to_string()) + .or_else(|| payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "non-string panic payload".to_string()) +} + +/// The password an import is created or resumed with, if any. +pub fn password_secret(password: Option) -> Option { + password + .filter(|p| !p.is_empty()) + .map(PlaintextSecret::from) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_nuget_placeholder_date_is_no_publish_time() { + assert_eq!(published_at(Some("1900-01-01T00:00:00+00:00")), None); + assert_eq!(published_at(Some("not a date")), None); + assert_eq!(published_at(None), None); + assert_eq!( + published_at(Some("2024-05-06T07:08:09.123+02:00")), + Some("2024-05-06T05:08:09.123Z".parse().unwrap()) + ); + } + + #[test] + fn a_nexus_repository_reads_through_its_protocol() { + assert_eq!( + nexus_source_url("https://nexus.example.com/", "nuget-hosted", None, false), + "https://nexus.example.com/repository/nuget-hosted/index.json" + ); + assert_eq!( + nexus_source_url("https://nexus.example.com", "docker-hosted", None, true), + "https://nexus.example.com/repository/docker-hosted" + ); + assert_eq!( + nexus_source_url( + "https://nexus.example.com", + "docker-hosted", + Some("https://nexus.example.com:8082"), + true + ), + "https://nexus.example.com:8082" + ); + } + + #[test] + fn a_new_import_picks_its_listing() { + assert_eq!(Listing::of(false, false), Listing::Search); + assert_eq!(Listing::of(true, false), Listing::Catalog); + assert_eq!(Listing::of(true, true), Listing::Names); + for listing in [Listing::Search, Listing::Catalog, Listing::Names] { + assert_eq!(Listing::parse(listing.as_str()), listing); + } + } + + #[test] + fn an_item_stored_before_reads_as_no_checksum() { + let legacy = serde_json::json!({ + "name": "Sample.Lib", + "version": "1.0.0", + "assets": [{ "path": "Sample.Lib/1.0.0", "sha512": "abc" }] + }); + let source: ItemSource = serde_json::from_value(legacy).unwrap(); + assert_eq!(source.sha512, None); + } +} diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index 04ece27e..ef0198b2 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -27,11 +27,11 @@ pub mod feed; pub mod files; pub mod health; pub mod images; +pub mod import; pub mod invitations; pub mod me; pub mod me_tokens; pub mod middleware; -pub mod nexus_import; pub mod object_store; pub mod plugins; pub mod registry; diff --git a/crates/server/src/nexus_import/mod.rs b/crates/server/src/nexus_import/mod.rs deleted file mode 100644 index 05994338..00000000 --- a/crates/server/src/nexus_import/mod.rs +++ /dev/null @@ -1,1084 +0,0 @@ -//! Importing a Nexus hosted repository into a Forge hosted repository of the -//! same format, one repository at a time (NuGet, Docker). -//! -//! An administrator starts an import on a hosted repository -//! ([`crate::admin::imports`]); this worker runs it, on one instance at a -//! time: it claims a queued import (or one whose instance died) with a -//! lease in Postgres and renews the lease while it works, like the -//! retention worker. -//! -//! 1. **Scan.** The Nexus component listing is walked page by page; each -//! page's components become pending items and the page's -//! `continuationToken` becomes the import's cursor, in one transaction, -//! so a scan resumes after a restart, a failure or a cancel at the page -//! it had not recorded. For Docker the asset listing follows: manifests -//! Nexus holds by digest only (not listed as components) become items -//! too, so referrers (signatures, SBOMs) come along. The scan also -//! counts every asset of the repository and its size. -//! 2. **Copy.** Pending items are copied, `concurrency` at a time: -//! * a package (NuGet) is streamed from Nexus into staging, -//! verified against Nexus' checksums (SHA-256 and SHA-512, else SHA-1) -//! and size, and written through the hosted write path -//! ([`HostedWrite::import_staged`]): format validation, the package -//! selectors of the administrator who started the import, the -//! repository's upload limit, CAS dedup and the audit trail -//! (`action = "import"`). A version Forge holds with the same SHA-256 -//! is skipped without a download; -//! * a Docker tag is copied through the Nexus repository's registry API -//! ([`crate::images::Registry::import_tag`]): digests are kept, -//! multi-platform indexes arrive with every child manifest, and a -//! referrer is copied once its subject is in Forge; -//! * what Nexus recorded is kept: the publish time (`blobCreated`, for a -//! tag `lastModified`) becomes the version's creation time, the last -//! download its last download, the uploader goes to the audit trail; -//! * a version Forge holds with other bytes is a conflict: skipped and -//! reported (`skip`), replaced (`overwrite`), or it fails the import -//! (`fail`). -//! -//! A dry run scans and decides every item (`planned`, `identical`, -//! `conflict`) without writing a package; it downloads nothing that Nexus -//! lists a SHA-256 for. Running an import again later copies only what is -//! new in Nexus, so a cutover is an import, then a final one just before -//! clients switch to Forge. -//! -//! A failed item can be retried on its own, and a cancelled or failed -//! import resumed ([`forge_db::import_dao::requeue`]). The Nexus password -//! is sealed with the secret encryption and deleted when the import -//! completes. - -pub mod nexus; - -use std::panic::AssertUnwindSafe; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::Arc; -use std::time::Duration; - -use base64::Engine; -use forge_core::auth::principal::{AuthMethod, Principal}; -use forge_core::authz::Level; -use forge_core::repo::hosted::HostedWrite; -use forge_core::repo::proxy::Credentials; -use forge_core::repo::{ImportOptions, ImportOutcome, RepoKind, WriteError}; -use forge_crypto::PlaintextSecret; -use forge_db::import_dao::{self, ImportItem, NewItem, NexusImport}; -use forge_db::model::Repository; -use forge_db::{repository_dao, secret_dao, user_dao}; -use forge_storage::StagedBlob; -use futures_util::{FutureExt, StreamExt, TryStreamExt}; -use serde_json::Value as JsonValue; -use sha2::Digest as _; -use tokio::task::JoinHandle; -use uuid::Uuid; - -use crate::images::import::{ImageImportOptions, ImageOutcome, ImportSource}; -use crate::state::AppState; -use nexus::{Listing, Nexus, SourceAsset, SourceItem}; - -/// The formats an import copies, by the names Nexus and Forge share: NuGet -/// packages, and Docker images into the image store. -pub const FORMATS: &[&str] = &["nuget", "docker"]; - -/// How often an idle instance looks for an import to run. -const POLL_INTERVAL: Duration = Duration::from_secs(5); - -/// The lease a running import is held with; renewed every third of it. -const LEASE_SECONDS: i64 = 120; - -/// Pending items read at a time. -const BATCH: i64 = 64; - -/// Items failing in a row that stop an import (Nexus went away, the -/// credentials stopped working): it fails, and can be resumed. -const MAX_CONSECUTIVE_FAILURES: usize = 25; - -/// Attempts at a listing page before the import fails. -const PAGE_ATTEMPTS: u32 = 4; - -/// How often a copy looks whether it was cancelled (or lost its lease). -const CHECK_INTERVAL: Duration = Duration::from_secs(5); - -/// The worker running imports on this instance. -pub struct ImportWorker { - state: AppState, -} - -impl ImportWorker { - pub fn new(state: AppState) -> Self { - ImportWorker { state } - } - - /// Look for an import every few seconds and run each that is due. - pub fn spawn_loop(self) -> JoinHandle<()> { - tokio::spawn(async move { - let mut ticker = tokio::time::interval(POLL_INTERVAL); - loop { - ticker.tick().await; - loop { - match self.tick().await { - Ok(Some(_)) => {} - Ok(None) => break, - Err(e) => { - tracing::error!(error = %e, "nexus import tick failed"); - break; - } - } - } - } - }) - } - - /// Claim one import that is due and run it until it completes, fails, - /// is cancelled or loses its lease. Returns its id, `None` when no - /// import is due. - pub async fn tick(&self) -> sqlx::Result> { - let pool = &self.state.pool; - let claimed = { - let mut tx = forge_db::begin(pool).await?; - let now = forge_db::db_now(&mut *tx).await?; - let claimed = - import_dao::claim_next(&mut tx, now, chrono::Duration::seconds(LEASE_SECONDS)) - .await?; - tx.commit().await?; - claimed - }; - let Some((import, run_id)) = claimed else { - return Ok(None); - }; - let id = import.id; - tracing::info!( - import = id, - repository_id = import.repository_id, - source = %import.base_url, - source_repository = %import.source_repository, - phase = %import.phase, - "nexus import started" - ); - let lost = Arc::new(AtomicBool::new(false)); - let heartbeat = AbortOnDrop(spawn_heartbeat(pool.clone(), id, run_id, lost.clone())); - let run = Run { - state: &self.state, - import, - run_id, - lost, - stopping: AtomicBool::new(false), - }; - let end = AssertUnwindSafe(run.execute()) - .catch_unwind() - .await - .unwrap_or_else(|panic| { - End::Failed(format!("the import panicked: {}", panic_text(&*panic))) - }); - drop(heartbeat); - let (state, error) = match &end { - End::Completed => ("completed", None), - End::Cancelled => ("cancelled", None), - End::Failed(error) => ("failed", Some(error.as_str())), - End::LeaseLost => { - tracing::warn!( - import = id, - "nexus import lease lost; another instance continues it" - ); - return Ok(Some(id)); - } - }; - if import_dao::finish(pool, id, run_id, state, error).await? { - tracing::info!(import = id, state, error, "nexus import ended"); - } else { - tracing::warn!(import = id, "nexus import lease lost before it ended"); - } - Ok(Some(id)) - } -} - -/// How a run ended. -enum End { - Completed, - Cancelled, - Failed(String), - /// Another instance holds the import now; it records the outcome. - LeaseLost, -} - -fn db_failed(e: sqlx::Error) -> End { - tracing::error!(error = %e, "nexus import: database error"); - End::Failed(format!("database error: {e}")) -} - -/// Where the items of a run go. -enum Target { - /// A package format, through the hosted write path. - Packages(Box), - /// Docker, through the registry, from the Nexus repository's registry - /// API. - Images { - registry: crate::images::Registry, - source: ImportSource, - }, -} - -/// What a run works with. -struct Ctx { - repo: Repository, - principal: Principal, - nexus: Nexus, - target: Target, - /// The repository's upload limit. - limit: u64, - /// ` `, for the audit trail. - source: String, -} - -impl Ctx { - /// Whether the items are images (Docker). - fn images(&self) -> bool { - matches!(self.target, Target::Images { .. }) - } -} - -/// What became of one item. -enum Copied { - /// It ends in `state`. - Item { - state: &'static str, - detail: Option, - bytes: u64, - }, - /// A manifest Nexus holds by digest that is no referrer: forgotten. - NotReferrer, -} - -/// How a recorded item bears on the run. -enum Recorded { - Done, - /// The run is ending: the item was left pending. - NotStarted, - Failed(String), - /// A conflict, and the import fails on conflicts. - Conflict(String), -} - -/// One run of one import, on this instance. -struct Run<'a> { - state: &'a AppState, - import: NexusImport, - run_id: Uuid, - /// Set by the heartbeat once the lease is lost. - lost: Arc, - /// Set once the run is to end: no further item is started. - stopping: AtomicBool, -} - -impl Run<'_> { - async fn execute(&self) -> End { - let ctx = match self.prepare().await { - Ok(ctx) => ctx, - Err(end) => return end, - }; - if self.import.phase == "scan" { - if let Err(end) = self.scan(&ctx).await { - return end; - } - } - match self.copy(&ctx).await { - Ok(()) => End::Completed, - Err(end) => end, - } - } - - /// The repository, the administrator the import runs as, and the - /// clients of the source. - async fn prepare(&self) -> Result { - let pool = &self.state.pool; - let import = &self.import; - let repo = repository_dao::find_repository_by_id(pool, import.repository_id) - .await - .map_err(db_failed)? - .ok_or_else(|| End::Failed("the repository was deleted".into()))?; - if repo.kind() != RepoKind::Hosted || !FORMATS.contains(&repo.format.as_str()) { - return Err(End::Failed(format!( - "imports go to a hosted NuGet or Docker repository; {} is a {} {} repository", - repo.name, repo.format, repo.kind - ))); - } - - // The import runs with the authority of the administrator who - // started it, as long as they are one. - let gone = - || End::Failed("the administrator who started the import no longer is one".into()); - let user_id = import.created_by.ok_or_else(gone)?; - let user = user_dao::find_user_by_id(pool, user_id) - .await - .map_err(db_failed)? - .filter(|user| user.active) - .ok_or_else(gone)?; - let principal = Principal::User { - user_id: user.id, - token_id: None, - method: AuthMethod::Session, - }; - if !self.state.authz.is_global_admin(&principal).await { - return Err(gone()); - } - - let password = secret_dao::get_import_password(pool, &self.state.envelope, import.id) - .await - .map_err(|e| End::Failed(format!("the import's password cannot be read: {e}")))?; - if import.username.is_some() && password.is_none() { - return Err(End::Failed( - "the import has no password; resume it with the Nexus password".into(), - )); - } - let nexus = Nexus::new( - self.state.upstream.clone(), - &import.base_url, - import.username.clone(), - password.clone(), - ); - - let target = if self.state.formats.stores_images(&repo.format) { - let registry_url = import.registry_url.clone().unwrap_or_else(|| { - default_registry_url(&import.base_url, &import.source_repository) - }); - let credentials = match (&import.username, password) { - (Some(username), Some(password)) => Some(Credentials::for_url( - ®istry_url, - username.clone(), - password, - )), - _ => None, - }; - Target::Images { - registry: self.state.registry(), - source: ImportSource::new( - ®istry_url, - credentials, - format!("import:{}", import.id), - ), - } - } else { - let format = self - .state - .formats - .for_repository(&repo) - .map_err(|e| End::Failed(e.to_string()))?; - let authz = &self.state.authz; - let filter = authz - .package_filter_for(&principal, &repo, Level::Write) - .await; - let replace = authz - .package_filter_for(&principal, &repo, Level::Delete) - .await; - let role_filter = authz - .role_package_filter_for(&principal, &repo, Level::Write) - .await; - let role_replace = authz - .role_package_filter_for(&principal, &repo, Level::Delete) - .await; - Target::Packages(Box::new(crate::dispatch::hosted_write( - self.state, - format, - repo.id, - repo.name.clone(), - principal.clone(), - filter, - replace, - role_filter, - role_replace, - ))) - }; - Ok(Ctx { - limit: self.state.upload_limit(&repo), - source: format!("{} {}", import.base_url, import.source_repository), - repo, - principal, - nexus, - target, - }) - } - - /// Stop when the lease is lost or an administrator cancelled. - async fn check(&self) -> Result<(), End> { - if self.lost.load(Ordering::Relaxed) { - return Err(End::LeaseLost); - } - match import_dao::poll(&self.state.pool, self.import.id, self.run_id).await { - Ok(None) => Err(End::LeaseLost), - Ok(Some(true)) => Err(End::Cancelled), - Ok(Some(false)) => Ok(()), - Err(e) => Err(db_failed(e)), - } - } - - /// Walk the listings from the stored cursor, recording each page's - /// items with the cursor of the next page. - async fn scan(&self, ctx: &Ctx) -> Result<(), End> { - let mut listing = Listing::parse(&self.import.scan_listing); - let mut cursor = self.import.cursor.clone(); - loop { - self.check().await?; - let page = self.page(ctx, listing, cursor.as_deref()).await?; - let scanned = scanned_items(&ctx.repo.format, ctx.images(), listing, &page.items); - let (next_listing, next_cursor, phase) = match page.next { - Some(next) => (listing, Some(next), "scan"), - None if ctx.images() && listing == Listing::Components => { - (Listing::Assets, None, "scan") - } - None => (listing, None, "copy"), - }; - let sources: Vec = scanned - .items - .iter() - .map(|(_, item, _)| serde_json::to_value(item).unwrap_or(JsonValue::Null)) - .collect(); - let new_items: Vec> = scanned - .items - .iter() - .zip(&sources) - .map(|((kind, item, primary), source)| NewItem { - kind, - name: &item.name, - version: &item.version, - size: primary.and_then(|a| a.size), - published_at: primary.and_then(|a| published_at(ctx.images(), a)), - downloaded_at: primary.and_then(|a| a.downloaded), - uploader: primary.and_then(|a| a.uploader.as_deref()), - source, - }) - .collect(); - let pool = &self.state.pool; - let mut tx = forge_db::begin(pool).await.map_err(db_failed)?; - import_dao::insert_items(&mut tx, self.import.id, &new_items) - .await - .map_err(db_failed)?; - let held = import_dao::save_scan_page( - &mut tx, - self.import.id, - self.run_id, - next_listing.as_str(), - next_cursor.as_deref(), - phase, - scanned.assets, - scanned.bytes, - ) - .await - .map_err(db_failed)?; - if !held { - return Err(End::LeaseLost); - } - tx.commit().await.map_err(db_failed)?; - if phase == "copy" { - return Ok(()); - } - listing = next_listing; - cursor = next_cursor; - } - } - - /// One listing page, retried with a backoff while the failure may pass. - async fn page( - &self, - ctx: &Ctx, - listing: Listing, - cursor: Option<&str>, - ) -> Result { - let mut attempt = 0; - loop { - attempt += 1; - match ctx - .nexus - .page(listing, &self.import.source_repository, cursor) - .await - { - Ok(page) => return Ok(page), - Err(e) if e.is_transient() && attempt < PAGE_ATTEMPTS => { - tracing::warn!(import = self.import.id, error = %e, attempt, "nexus listing failed; retrying"); - tokio::time::sleep(Duration::from_secs(1 << attempt)).await; - } - Err(e) => { - return Err(End::Failed(format!( - "listing the {}: {e}", - listing.as_str() - ))) - } - } - } - } - - /// Copy the pending items: components first, then (Docker) the - /// manifests Nexus holds by digest, whose subjects the components - /// bring. - async fn copy(&self, ctx: &Ctx) -> Result<(), End> { - let failures = AtomicUsize::new(0); - let concurrency = self.import.concurrency.clamp(1, 16) as usize; - for kind in ["component", "manifest"] { - let mut after = 0; - loop { - self.check().await?; - let batch = - import_dao::pending_items(&self.state.pool, self.import.id, kind, after, BATCH) - .await - .map_err(db_failed)?; - let Some(last) = batch.last() else { - break; - }; - after = last.id; - let mut copies = futures_util::stream::iter(batch) - .map(|item| self.copy_item(ctx, item)) - .buffer_unordered(concurrency); - // Once the run is to end (a cancel, a lost lease, too many - // failures), the items in flight finish and the rest stay - // pending. - let mut end = None; - let mut checked = std::time::Instant::now(); - while let Some(recorded) = copies.next().await { - let mut stop = match recorded { - Err(e) => Some(db_failed(e)), - Ok(Recorded::Done) => { - failures.store(0, Ordering::Relaxed); - None - } - Ok(Recorded::NotStarted) => None, - Ok(Recorded::Failed(error)) => (failures.fetch_add(1, Ordering::Relaxed) - + 1 - >= MAX_CONSECUTIVE_FAILURES) - .then(|| { - End::Failed(format!( - "{MAX_CONSECUTIVE_FAILURES} items in a row failed; the last: {error}" - )) - }), - Ok(Recorded::Conflict(conflict)) => Some(End::Failed(conflict)), - }; - if end.is_none() && stop.is_none() && checked.elapsed() >= CHECK_INTERVAL { - checked = std::time::Instant::now(); - stop = self.check().await.err(); - } - if let (None, Some(stop)) = (&end, stop) { - self.stopping.store(true, Ordering::Relaxed); - end = Some(stop); - } - } - if let Some(end) = end { - return Err(end); - } - } - } - Ok(()) - } - - /// Copy one item and record what became of it. - async fn copy_item(&self, ctx: &Ctx, item: ImportItem) -> sqlx::Result { - if self.stopping.load(Ordering::Relaxed) { - return Ok(Recorded::NotStarted); - } - let pool = &self.state.pool; - let copied = match serde_json::from_value::(item.source.clone()) { - Err(e) => Err(format!("the item's recorded source is unreadable: {e}")), - Ok(source) => match &ctx.target { - Target::Packages(writer) => self.copy_package(ctx, writer, &source).await, - Target::Images { - registry, - source: from, - } => { - self.copy_image(ctx, registry, from, &item.kind, &source) - .await - } - }, - }; - match copied { - Ok(Copied::NotReferrer) => { - import_dao::drop_item(pool, item.id).await?; - Ok(Recorded::Done) - } - Ok(Copied::Item { - state, - detail, - bytes, - }) => { - import_dao::finish_item( - pool, - item.id, - state, - detail.as_deref(), - i64::try_from(bytes).unwrap_or(i64::MAX), - ) - .await?; - if state == "conflict" && self.import.on_conflict == "fail" { - return Ok(Recorded::Conflict(format!( - "{} {}: {}", - item.name, - item.version, - detail.unwrap_or_default() - ))); - } - Ok(Recorded::Done) - } - Err(error) => { - tracing::warn!(import = self.import.id, name = %item.name, version = %item.version, %error, "nexus import item failed"); - import_dao::finish_item(pool, item.id, "failed", Some(&error), 0).await?; - Ok(Recorded::Failed(format!( - "{} {}: {error}", - item.name, item.version - ))) - } - } - } - - /// Copy a package version through the hosted write path. - async fn copy_package( - &self, - ctx: &Ctx, - writer: &HostedWrite, - source: &SourceItem, - ) -> Result { - let asset = package_asset(&ctx.repo.format, &source.assets) - .ok_or("the component has no package file among its assets")?; - let overwrite = self.import.on_conflict == "overwrite"; - - // Nothing is downloaded for a version Forge holds with these bytes, - // nor for a dry run (which decides from the checksums alone). - let stored = writer - .package_sha256(ctx.repo.id, &source.name, &source.version) - .await - .map_err(|e| e.to_string())?; - if let (Some(stored), Some(expected)) = (&stored, &asset.sha256) { - if stored == expected { - return Ok(item("identical", None, 0)); - } - } - if self.import.dry_run { - return Ok(match (stored, &asset.sha256) { - (None, _) => item("planned", None, 0), - (Some(_), None) => item( - "planned", - Some("Forge has this version; Nexus lists no SHA-256 to compare".into()), - 0, - ), - (Some(_), Some(_)) if overwrite => item( - "planned", - Some("would replace the version Forge has with other bytes".into()), - 0, - ), - (Some(_), Some(_)) => conflict_item(), - }); - } - - if asset - .size - .is_some_and(|size| u64::try_from(size).unwrap_or(0) > ctx.limit) - { - return Err(WriteError::TooLarge { limit: ctx.limit }.to_string()); - } - let url = ctx - .nexus - .content_url(&self.import.source_repository, &asset.path); - let response = ctx.nexus.download(&url).await.map_err(|e| e.to_string())?; - if response.content_length().is_some_and(|l| l > ctx.limit) { - return Err(WriteError::TooLarge { limit: ctx.limit }.to_string()); - } - let body = response.bytes_stream().map_err(std::io::Error::other); - let staged = self - .state - .cas - .stage(tokio_util::io::StreamReader::new(body), ctx.limit) - .await - .map_err(|e| WriteError::from(e).to_string())?; - verify(&staged, asset).await?; - let bytes = staged.size; - let options = ImportOptions { - overwrite, - dry_run: false, - published_at: published_at(ctx.images(), asset), - downloaded_at: asset.downloaded, - uploader: asset.uploader.clone(), - source: ctx.source.clone(), - }; - let outcome = writer - .import_staged(&ctx.repo.name, staged, ctx.limit, &options) - .await - .map_err(|e| e.to_string())?; - Ok(match outcome { - ImportOutcome::Created { .. } => item("imported", None, bytes), - ImportOutcome::Replaced { .. } => item( - "replaced", - Some("replaced the version Forge had with other bytes".into()), - bytes, - ), - ImportOutcome::Identical { .. } => item("identical", None, bytes), - ImportOutcome::Conflict { .. } => conflict_item(), - ImportOutcome::Planned { .. } => item("planned", None, bytes), - }) - } - - /// Copy a Docker tag (`component`) or referrer (`manifest`) through the - /// registry. - async fn copy_image( - &self, - ctx: &Ctx, - registry: &crate::images::Registry, - from: &ImportSource, - kind: &str, - source: &SourceItem, - ) -> Result { - let asset = source - .assets - .iter() - .find(|a| a.path.contains("/manifests/")) - .or(source.assets.first()); - let options = ImageImportOptions { - principal: ctx.principal.clone(), - overwrite: self.import.on_conflict == "overwrite", - dry_run: self.import.dry_run, - published_at: asset.and_then(|a| published_at(ctx.images(), a)), - downloaded_at: asset.and_then(|a| a.downloaded), - uploader: asset.and_then(|a| a.uploader.clone()), - source: ctx.source.clone(), - }; - let imported = if kind == "manifest" { - registry - .import_referrer(&ctx.repo, from, &source.name, &source.version, &options) - .await - } else { - let expected = asset - .and_then(|a| a.sha256.as_ref()) - .map(|hex| format!("sha256:{hex}")); - registry - .import_tag( - &ctx.repo, - from, - &source.name, - &source.version, - expected.as_deref(), - &options, - ) - .await - } - .map_err(|e| e.message())?; - let bytes = imported.bytes_copied; - Ok(match imported.outcome { - ImageOutcome::Created => item("imported", imported.digest, bytes), - ImageOutcome::Replaced { previous } => { - item("replaced", Some(format!("moved from {previous}")), bytes) - } - ImageOutcome::Identical => item("identical", imported.digest, bytes), - ImageOutcome::Conflict { - existing, - immutable, - } => item( - "conflict", - Some(if immutable { - format!("the tag points at {existing} in Forge, and tags are immutable there") - } else { - format!("the tag points at {existing} in Forge") - }), - bytes, - ), - ImageOutcome::Planned { replaces } => item( - "planned", - replaces.then(|| "would move the tag Forge has".to_string()), - bytes, - ), - ImageOutcome::NotReferrer => Copied::NotReferrer, - }) - } -} - -fn item(state: &'static str, detail: Option, bytes: u64) -> Copied { - Copied::Item { - state, - detail, - bytes, - } -} - -fn conflict_item() -> Copied { - item( - "conflict", - Some("Forge has other bytes for this version".into()), - 0, - ) -} - -/// The registry API of the Nexus Docker repository `repository`, by path -/// (`/repository/`, which Nexus serves for reads on its own -/// port). -pub fn default_registry_url(base_url: &str, repository: &str) -> String { - format!("{}/repository/{repository}", base_url.trim_end_matches('/')) -} - -/// When Nexus says an asset was published: for a Docker tag when it last -/// moved (`lastModified`), for a package when its bytes were stored -/// (`blobCreated`). -fn published_at(images: bool, asset: &SourceAsset) -> Option> { - if images { - asset.modified.or(asset.created) - } else { - asset.created.or(asset.modified) - } -} - -/// The package file among a component's assets: NuGet's `.nupkg` (Nexus -/// keeps it under `/`), any other format's only asset. -fn package_asset<'a>(format: &str, assets: &'a [SourceAsset]) -> Option<&'a SourceAsset> { - match format { - "nuget" => assets - .iter() - .find(|a| a.path.ends_with(".nupkg")) - .or_else(|| assets.iter().find(|a| !a.path.ends_with(".nuspec"))), - _ => match assets { - [only] => Some(only), - _ => None, - }, - } -} - -/// What one scanned page holds. -struct Scanned<'a> { - /// `(kind, item, its primary asset)`. - items: Vec<(&'static str, SourceItem, Option<&'a SourceAsset>)>, - assets: i64, - bytes: i64, -} - -/// The items of a page: every component of a components listing; of a -/// (Docker) asset listing, the manifests held by digest. Every asset is -/// counted once: a package component's in the components listing, a -/// Docker repository's (blobs included) in the asset listing. -fn scanned_items<'a>( - format: &str, - images: bool, - listing: Listing, - page: &'a [SourceItem], -) -> Scanned<'a> { - let mut scanned = Scanned { - items: Vec::new(), - assets: 0, - bytes: 0, - }; - for item in page { - match listing { - Listing::Components => { - let primary = if images { - item.assets.iter().find(|a| a.path.contains("/manifests/")) - } else { - scanned.assets += item.assets.len() as i64; - scanned.bytes += item.assets.iter().filter_map(|a| a.size).sum::(); - package_asset(format, &item.assets) - }; - scanned.items.push(("component", item.clone(), primary)); - } - Listing::Assets => { - for asset in &item.assets { - scanned.assets += 1; - scanned.bytes += asset.size.unwrap_or(0); - if let Some((image, digest)) = manifest_by_digest(&asset.path) { - scanned.items.push(( - "manifest", - SourceItem { - name: image.to_string(), - version: digest.to_string(), - assets: vec![asset.clone()], - }, - Some(asset), - )); - } - } - } - } - } - scanned -} - -/// `(image, digest)` of a Docker asset path `v2//manifests/`. -fn manifest_by_digest(path: &str) -> Option<(&str, &str)> { - let rest = path.strip_prefix("v2/")?; - let (image, digest) = rest.rsplit_once("/manifests/")?; - (!image.is_empty() && digest.starts_with("sha256:")).then_some((image, digest)) -} - -/// Check a staged download against what Nexus lists for it: its size, its -/// SHA-256 and SHA-512, else its SHA-1. -async fn verify(staged: &StagedBlob, asset: &SourceAsset) -> Result<(), String> { - let mismatch = |what: &str| { - format!( - "the download of {} does not match its {what} in Nexus", - asset.path - ) - }; - if asset - .size - .is_some_and(|size| u64::try_from(size).ok() != Some(staged.size)) - { - return Err(mismatch("size")); - } - let mut checked = false; - if let Some(expected) = &asset.sha256 { - if staged.sha256 != *expected { - return Err(mismatch("SHA-256")); - } - checked = true; - } - if let Some(expected) = &asset.sha512 { - let actual = base64::engine::general_purpose::STANDARD - .decode(&staged.sha512_base64) - .map(hex::encode) - .map_err(|e| e.to_string())?; - if actual != *expected { - return Err(mismatch("SHA-512")); - } - checked = true; - } - if let (false, Some(expected)) = (checked, &asset.sha1) { - let path = staged.read_path(); - let actual = tokio::task::spawn_blocking(move || -> std::io::Result { - use std::io::Read; - let mut file = std::fs::File::open(path)?; - let mut hasher = sha1::Sha1::new(); - let mut buf = vec![0u8; 64 * 1024]; - loop { - let n = file.read(&mut buf)?; - if n == 0 { - break; - } - hasher.update(&buf[..n]); - } - Ok(hex::encode(hasher.finalize())) - }) - .await - .map_err(|e| e.to_string())? - .map_err(|e| e.to_string())?; - if actual != *expected { - return Err(mismatch("SHA-1")); - } - } - Ok(()) -} - -/// Aborts the wrapped task when dropped. -struct AbortOnDrop(JoinHandle<()>); - -impl Drop for AbortOnDrop { - fn drop(&mut self) { - self.0.abort(); - } -} - -/// Renew the lease every third of its length until aborted; flags `lost` -/// and stops once the lease is gone. -fn spawn_heartbeat( - pool: sqlx::PgPool, - id: i64, - run_id: Uuid, - lost: Arc, -) -> JoinHandle<()> { - tokio::spawn(async move { - let mut ticker = tokio::time::interval(Duration::from_secs((LEASE_SECONDS / 3) as u64)); - ticker.tick().await; - loop { - ticker.tick().await; - let renewed = match forge_db::db_now(&pool).await { - Ok(now) => { - import_dao::renew_lease( - &pool, - id, - run_id, - now + chrono::Duration::seconds(LEASE_SECONDS), - ) - .await - } - Err(e) => Err(e), - }; - match renewed { - Ok(true) => {} - Ok(false) => { - lost.store(true, Ordering::Relaxed); - return; - } - Err(e) => { - tracing::warn!(import = id, error = %e, "nexus import lease renewal failed") - } - } - } - }) -} - -fn panic_text(payload: &(dyn std::any::Any + Send)) -> String { - payload - .downcast_ref::<&str>() - .map(|s| (*s).to_string()) - .or_else(|| payload.downcast_ref::().cloned()) - .unwrap_or_else(|| "non-string panic payload".to_string()) -} - -/// The password an import is created or resumed with, if any. -pub fn password_secret(password: Option) -> Option { - password - .filter(|p| !p.is_empty()) - .map(PlaintextSecret::from) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn asset(path: &str) -> SourceAsset { - SourceAsset { - path: path.into(), - ..Default::default() - } - } - - #[test] - fn package_files_are_picked_per_format() { - let nuget = [asset("Sample.Lib/1.0.0")]; - assert_eq!( - package_asset("nuget", &nuget).unwrap().path, - "Sample.Lib/1.0.0" - ); - let two = [asset("app-1.0.0.tgz.prov"), asset("app-1.0.0.tgz")]; - assert!(package_asset("raw", &two).is_none()); - } - - #[test] - fn manifests_held_by_digest_are_recognized() { - assert_eq!( - manifest_by_digest("v2/team/app/manifests/sha256:abc"), - Some(("team/app", "sha256:abc")) - ); - assert_eq!(manifest_by_digest("v2/team/app/manifests/1.0"), None); - assert_eq!(manifest_by_digest("v2/-/blobs/sha256:abc"), None); - } - - #[test] - fn a_docker_asset_listing_counts_blobs_and_keeps_digest_manifests() { - let page = [ - SourceItem { - assets: vec![SourceAsset { - size: Some(100), - ..asset("v2/-/blobs/sha256:aa") - }], - ..Default::default() - }, - SourceItem { - assets: vec![SourceAsset { - size: Some(10), - ..asset("v2/app/manifests/sha256:bb") - }], - ..Default::default() - }, - SourceItem { - assets: vec![SourceAsset { - size: Some(10), - ..asset("v2/app/manifests/1.0") - }], - ..Default::default() - }, - ]; - let scanned = scanned_items("docker", true, Listing::Assets, &page); - assert_eq!(scanned.assets, 3); - assert_eq!(scanned.bytes, 120); - assert_eq!(scanned.items.len(), 1); - assert_eq!(scanned.items[0].0, "manifest"); - assert_eq!(scanned.items[0].1.name, "app"); - assert_eq!(scanned.items[0].1.version, "sha256:bb"); - } -} diff --git a/crates/server/src/nexus_import/nexus.rs b/crates/server/src/nexus_import/nexus.rs deleted file mode 100644 index 3eff1cd0..00000000 --- a/crates/server/src/nexus_import/nexus.rs +++ /dev/null @@ -1,533 +0,0 @@ -//! The parts of the Nexus Repository 3 REST API an import reads: -//! -//! * `GET /service/rest/v1/status`: the server is up; its `Server` header -//! names the version (`Nexus/3.70.1-02 (OSS)`); -//! * `GET /service/rest/v1/repositories`: name, format and type of each -//! repository the credentials may see; -//! * `GET /service/rest/v1/components?repository=` and -//! `…/assets?repository=`, paged with `continuationToken`; -//! * `/repository//`: an asset's content. -//! -//! Requests go through the shared [`UpstreamHttp`] client (timeouts, size -//! caps, no cloud-metadata targets); the credentials are sent to the Nexus -//! origin only. Download URLs are built from the import's base URL and the -//! asset's path rather than taken from Nexus' `downloadUrl`, which carries -//! Nexus' configured base URL and may name a host Forge does not reach the -//! server on. - -use chrono::{DateTime, Utc}; -use forge_core::net::upstream::{Origin, RequestClass, UpstreamAuth, UpstreamError, UpstreamHttp}; -use forge_crypto::{ExposeSecret, PlaintextSecret}; -use reqwest::StatusCode; -use serde::de::{Deserializer, MapAccess, Visitor}; -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value as JsonValue}; - -/// What went wrong talking to Nexus. -#[derive(Debug, thiserror::Error)] -pub enum NexusError { - /// The server could not be reached, or the transfer failed. - #[error("Nexus is unreachable: {0}")] - Unreachable(String), - /// The credentials were refused (or may not see the resource). - #[error("Nexus refused the credentials ({status}) for {url}")] - Refused { status: StatusCode, url: String }, - /// Any other answer than the one asked for. - #[error("Nexus answered {status} for {url}")] - Status { status: StatusCode, url: String }, - /// The answer was not the document the API describes. - #[error("Nexus answered an unexpected document for {url}: {error}")] - Invalid { url: String, error: String }, -} - -impl NexusError { - /// Whether trying again later may succeed. - pub fn is_transient(&self) -> bool { - match self { - NexusError::Unreachable(_) => true, - NexusError::Status { status, .. } => { - status.is_server_error() || *status == StatusCode::TOO_MANY_REQUESTS - } - NexusError::Refused { .. } | NexusError::Invalid { .. } => false, - } - } - - fn from_upstream(e: UpstreamError, url: &str) -> Self { - match e { - UpstreamError::Status { status, url } => status_error(status, url), - UpstreamError::Json { source, .. } => NexusError::Invalid { - url: url.to_string(), - error: source.to_string(), - }, - other => NexusError::Unreachable(other.to_string()), - } - } -} - -fn status_error(status: StatusCode, url: String) -> NexusError { - if matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) { - NexusError::Refused { status, url } - } else { - NexusError::Status { status, url } - } -} - -/// A repository as `GET /service/rest/v1/repositories` lists it. -#[derive(Debug, Clone, Deserialize)] -pub struct NexusRepository { - pub name: String, - /// `nuget`, `docker`, `helm`, `apt`, `maven2`, … - pub format: String, - /// `hosted`, `proxy` or `group`. - #[serde(rename = "type")] - pub kind: String, -} - -/// The two listings an import walks. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Listing { - Components, - Assets, -} - -impl Listing { - pub fn as_str(self) -> &'static str { - match self { - Listing::Components => "components", - Listing::Assets => "assets", - } - } - - pub fn parse(s: &str) -> Self { - match s { - "assets" => Listing::Assets, - _ => Listing::Components, - } - } -} - -/// An asset as an import keeps it (the item's `source`). -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct SourceAsset { - /// The path within the repository (`Sample.Lib/1.0.0`, - /// `v2/team/app/manifests/1.0`). - pub path: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub content_type: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub size: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub sha1: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub sha256: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub sha512: Option, - /// `blobCreated`: when these bytes were stored. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub created: Option>, - /// `lastModified`: when the path last changed (a tag last moved). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub modified: Option>, - /// `lastDownloaded`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub downloaded: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub uploader: Option, -} - -impl SourceAsset { - /// Read an asset object of either listing. `None` without a path. - fn from_object(object: &Map) -> Option { - let text = |key: &str| { - object - .get(key) - .and_then(JsonValue::as_str) - .filter(|s| !s.is_empty()) - .map(str::to_string) - }; - let time = |key: &str| { - object - .get(key) - .and_then(JsonValue::as_str) - .and_then(|s| DateTime::parse_from_rfc3339(s).ok()) - .map(|t| t.with_timezone(&Utc)) - }; - let checksum = |key: &str| { - object - .get("checksum") - .and_then(|c| c.get(key)) - .and_then(JsonValue::as_str) - .filter(|s| !s.is_empty()) - .map(str::to_ascii_lowercase) - }; - Some(SourceAsset { - path: text("path")?, - content_type: text("contentType"), - size: object.get("fileSize").and_then(JsonValue::as_i64), - sha1: checksum("sha1"), - sha256: checksum("sha256"), - sha512: checksum("sha512"), - created: time("blobCreated"), - modified: time("lastModified"), - downloaded: time("lastDownloaded"), - uploader: text("uploader"), - }) - } -} - -/// A component (or, for Docker, a manifest held by digest) as an import -/// keeps it. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct SourceItem { - /// The package id or image name. - pub name: String, - /// The version, tag, or manifest digest. - pub version: String, - pub assets: Vec, -} - -/// One page of a listing. -#[derive(Debug, Default)] -pub struct Page { - /// Components (a components listing), or every asset (an assets - /// listing) as one-asset items without a name or version. - pub items: Vec, - /// The token of the next page; `None` on the last. - pub next: Option, -} - -/// A client of one Nexus server. -pub struct Nexus { - http: UpstreamHttp, - base: String, - origin: Option, - username: Option, - password: Option, -} - -impl Nexus { - /// A client of the Nexus at `base_url` (a context path included, no - /// trailing `/`), with Basic credentials when both are given. - pub fn new( - http: UpstreamHttp, - base_url: &str, - username: Option, - password: Option, - ) -> Self { - let base = base_url.trim_end_matches('/').to_string(); - Nexus { - origin: Origin::of(&base), - http, - base, - username, - password, - } - } - - fn auth(&self) -> Option> { - match (&self.username, &self.password) { - (Some(username), Some(password)) => Some(UpstreamAuth { - origin: self.origin.clone(), - username, - password: password.expose_secret(), - bearer: false, - }), - _ => None, - } - } - - fn api(&self, path: &str, query: &[(&str, &str)]) -> String { - let mut url = format!("{}/service/rest/v1/{path}", self.base); - if !query.is_empty() { - url.push('?'); - url.push_str( - &url::form_urlencoded::Serializer::new(String::new()) - .extend_pairs(query) - .finish(), - ); - } - url - } - - /// GET a JSON document of the API and parse it as `T`. - async fn get(&self, url: &str) -> Result { - let bytes = self - .http - .get_bytes(url, self.auth().as_ref()) - .await - .map_err(|e| NexusError::from_upstream(e, url))? - .ok_or_else(|| NexusError::Status { - status: StatusCode::NOT_FOUND, - url: url.to_string(), - })?; - serde_json::from_slice(&bytes).map_err(|e| NexusError::Invalid { - url: url.to_string(), - error: e.to_string(), - }) - } - - /// The server's version (its `Server` header), checking that it is up - /// and is Nexus Repository 3. - pub async fn status(&self) -> Result { - let url = self.api("status", &[]); - let response = self - .http - .get(&url, RequestClass::Metadata, self.auth().as_ref()) - .await - .map_err(|e| NexusError::from_upstream(e, &url))?; - if !response.status().is_success() { - return Err(status_error(response.status(), url)); - } - let server = response - .headers() - .get(reqwest::header::SERVER) - .and_then(|v| v.to_str().ok()) - .unwrap_or_default() - .to_string(); - if !server.starts_with("Nexus/") { - return Err(NexusError::Invalid { - url, - error: format!("not a Nexus Repository 3 server (Server: {server:?})"), - }); - } - Ok(server) - } - - /// The repository `name`, if the credentials may see it. - pub async fn repository(&self, name: &str) -> Result, NexusError> { - let url = self.api("repositories", &[]); - let repositories: Vec = self.get(&url).await?; - Ok(repositories.into_iter().find(|r| r.name == name)) - } - - /// One page of `listing` of `repository`, after `cursor`. - pub async fn page( - &self, - listing: Listing, - repository: &str, - cursor: Option<&str>, - ) -> Result { - let mut query = vec![("repository", repository)]; - if let Some(cursor) = cursor { - query.push(("continuationToken", cursor)); - } - let url = self.api(listing.as_str(), &query); - let raw: RawPage = self.get(&url).await?; - let items = match listing { - Listing::Components => raw - .items - .iter() - .filter_map(|item| { - let component: RawComponent = - serde_json::from_value(JsonValue::Object(item.0.clone())).ok()?; - Some(SourceItem { - name: component.name?, - version: component.version?, - assets: component - .assets - .iter() - .filter_map(|asset| SourceAsset::from_object(&asset.0)) - .collect(), - }) - }) - .collect(), - Listing::Assets => raw - .items - .iter() - .filter_map(|item| SourceAsset::from_object(&item.0)) - .map(|asset| SourceItem { - name: String::new(), - version: String::new(), - assets: vec![asset], - }) - .collect(), - }; - Ok(Page { - items, - next: raw.continuation_token.filter(|t| !t.is_empty()), - }) - } - - /// The URL of the asset at `path` in `repository`. - pub fn content_url(&self, repository: &str, path: &str) -> String { - let mut url = format!("{}/repository/", self.base); - let segments = std::iter::once(repository).chain(path.split('/')); - let encoded: Vec = segments.map(percent_encode_segment).collect(); - url.push_str(&encoded.join("/")); - url - } - - /// GET the content of an asset (streamed by the caller). - pub async fn download(&self, url: &str) -> Result { - let response = self - .http - .get(url, RequestClass::Download, self.auth().as_ref()) - .await - .map_err(|e| NexusError::from_upstream(e, url))?; - if !response.status().is_success() { - return Err(status_error(response.status(), url.to_string())); - } - Ok(response) - } - - /// Whether `url` answers the registry API (`/v2/`): 200, or a 401 - /// challenge. - pub async fn probe_registry(&self, url: &str) -> Result<(), NexusError> { - let response = self - .http - .get(url, RequestClass::Metadata, self.auth().as_ref()) - .await - .map_err(|e| NexusError::from_upstream(e, url))?; - let status = response.status(); - let answers = status.is_success() - || (status == StatusCode::UNAUTHORIZED - && response - .headers() - .contains_key(reqwest::header::WWW_AUTHENTICATE)); - if answers { - Ok(()) - } else { - Err(status_error(status, url.to_string())) - } - } -} - -/// Percent-encode one path segment (RFC 3986 `pchar`, minus the -/// sub-delimiters a server might treat specially). -fn percent_encode_segment(segment: &str) -> String { - let mut out = String::with_capacity(segment.len()); - for byte in segment.bytes() { - match byte { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' | b':' | b'@' => { - out.push(byte as char) - } - _ => out.push_str(&format!("%{byte:02X}")), - } - } - out -} - -#[derive(Deserialize)] -struct RawPage { - #[serde(default)] - items: Vec, - #[serde(rename = "continuationToken", default)] - continuation_token: Option, -} - -#[derive(Deserialize)] -struct RawComponent { - #[serde(default)] - name: Option, - #[serde(default)] - version: Option, - #[serde(default)] - assets: Vec, -} - -/// A JSON object whose repeated keys keep their first non-null value. Nexus -/// 3.70 writes `"blobCreated": null` and then `"blobCreated": "