diff --git a/.sqlx/query-02096c4b9d7da63d20ffa11be1aa38c1c9f2483a89334487bd1cb8c196203b80.json b/.sqlx/query-02096c4b9d7da63d20ffa11be1aa38c1c9f2483a89334487bd1cb8c196203b80.json deleted file mode 100644 index 2fc51792..00000000 --- a/.sqlx/query-02096c4b9d7da63d20ffa11be1aa38c1c9f2483a89334487bd1cb8c196203b80.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nexus_import\n SET state = $3,\n last_error = $4,\n finished_at = now(),\n updated_at = now(),\n run_id = NULL,\n locked_until = NULL,\n cancel_requested = false,\n password_enc = CASE WHEN $3 = 'completed' THEN NULL ELSE password_enc END\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Uuid", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "02096c4b9d7da63d20ffa11be1aa38c1c9f2483a89334487bd1cb8c196203b80" -} diff --git a/.sqlx/query-0321fa382dc868e2b659aaee5b9a5522d6d31cc44898a3d69f2b4c7fdb816382.json b/.sqlx/query-0321fa382dc868e2b659aaee5b9a5522d6d31cc44898a3d69f2b4c7fdb816382.json deleted file mode 100644 index cbb06963..00000000 --- a/.sqlx/query-0321fa382dc868e2b659aaee5b9a5522d6d31cc44898a3d69f2b4c7fdb816382.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nexus_import SET locked_until = $3\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Uuid", - "Timestamptz" - ] - }, - "nullable": [] - }, - "hash": "0321fa382dc868e2b659aaee5b9a5522d6d31cc44898a3d69f2b4c7fdb816382" -} diff --git a/.sqlx/query-d148cf931844bf52f505ccbde587d756e1a69b96bd96020da0f90f2a1b577ca8.json b/.sqlx/query-03a52d53318f67093894ef691d0a3304967d5300fb2e1dc8252a1fce6ec02e78.json similarity index 78% rename from .sqlx/query-d148cf931844bf52f505ccbde587d756e1a69b96bd96020da0f90f2a1b577ca8.json rename to .sqlx/query-03a52d53318f67093894ef691d0a3304967d5300fb2e1dc8252a1fce6ec02e78.json index 1dd582a1..fa637eef 100644 --- a/.sqlx/query-d148cf931844bf52f505ccbde587d756e1a69b96bd96020da0f90f2a1b577ca8.json +++ b/.sqlx/query-03a52d53318f67093894ef691d0a3304967d5300fb2e1dc8252a1fce6ec02e78.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id, import_id, kind, name, version, state, detail, size, bytes_copied,\n published_at, downloaded_at, uploader, source, attempts, updated_at\n FROM nexus_import_item\n WHERE import_id = $1 AND ($2::text IS NULL OR state = $2) AND id > $3\n ORDER BY id\n LIMIT $4\n ", + "query": "\n SELECT id, import_id, kind, name, version, state, detail, size, bytes_copied,\n published_at, downloaded_at, uploader, source, attempts, updated_at\n FROM repository_import_item\n WHERE import_id = $1 AND kind = $2 AND state = 'pending' AND id > $3\n ORDER BY id\n LIMIT $4\n ", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "id" } } @@ -20,7 +20,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "import_id" } } @@ -31,7 +31,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "kind" } } @@ -42,7 +42,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "name" } } @@ -53,7 +53,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "version" } } @@ -64,7 +64,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "state" } } @@ -75,7 +75,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "detail" } } @@ -86,7 +86,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "size" } } @@ -97,7 +97,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "bytes_copied" } } @@ -108,7 +108,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "published_at" } } @@ -119,7 +119,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "downloaded_at" } } @@ -130,7 +130,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "uploader" } } @@ -141,7 +141,7 @@ "type_info": "Jsonb", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "source" } } @@ -152,7 +152,7 @@ "type_info": "Int4", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "attempts" } } @@ -163,7 +163,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "updated_at" } } @@ -195,5 +195,5 @@ false ] }, - "hash": "d148cf931844bf52f505ccbde587d756e1a69b96bd96020da0f90f2a1b577ca8" + "hash": "03a52d53318f67093894ef691d0a3304967d5300fb2e1dc8252a1fce6ec02e78" } diff --git a/.sqlx/query-0892948fb2d5b15fe0e8acd943a3cf00faf57b77f147a3740cb46b8a8d8e0af6.json b/.sqlx/query-0892948fb2d5b15fe0e8acd943a3cf00faf57b77f147a3740cb46b8a8d8e0af6.json deleted file mode 100644 index 4e37a47b..00000000 --- a/.sqlx/query-0892948fb2d5b15fe0e8acd943a3cf00faf57b77f147a3740cb46b8a8d8e0af6.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nexus_import\n SET state = CASE WHEN state = 'queued' THEN 'cancelled' ELSE state END,\n finished_at = CASE WHEN state = 'queued' THEN now() ELSE finished_at END,\n cancel_requested = (state = 'running'),\n updated_at = now()\n WHERE id = $1 AND state IN ('queued', 'running')\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "0892948fb2d5b15fe0e8acd943a3cf00faf57b77f147a3740cb46b8a8d8e0af6" -} diff --git a/.sqlx/query-22132c561cf7d98feec907b61a000705407882063f5df1901be42b18a22a2b2a.json b/.sqlx/query-22132c561cf7d98feec907b61a000705407882063f5df1901be42b18a22a2b2a.json new file mode 100644 index 00000000..55830d13 --- /dev/null +++ b/.sqlx/query-22132c561cf7d98feec907b61a000705407882063f5df1901be42b18a22a2b2a.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE repository_import_item\n SET state = $2, detail = $3, bytes_copied = bytes_copied + $4,\n attempts = attempts + 1, updated_at = now()\n WHERE id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "22132c561cf7d98feec907b61a000705407882063f5df1901be42b18a22a2b2a" +} diff --git a/.sqlx/query-24384fd673779f613494a3a2bcd20378689533f6cc9ed1ce98283c4470704c9a.json b/.sqlx/query-24384fd673779f613494a3a2bcd20378689533f6cc9ed1ce98283c4470704c9a.json new file mode 100644 index 00000000..1334ec90 --- /dev/null +++ b/.sqlx/query-24384fd673779f613494a3a2bcd20378689533f6cc9ed1ce98283c4470704c9a.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE repository_import SET locked_until = now() - interval '1 second' WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "24384fd673779f613494a3a2bcd20378689533f6cc9ed1ce98283c4470704c9a" +} diff --git a/.sqlx/query-27ce5074610a907aa6cedbd9134dc45008bdac65951b6576fe55c51339b65085.json b/.sqlx/query-27ce5074610a907aa6cedbd9134dc45008bdac65951b6576fe55c51339b65085.json deleted file mode 100644 index dc94645f..00000000 --- a/.sqlx/query-27ce5074610a907aa6cedbd9134dc45008bdac65951b6576fe55c51339b65085.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nexus_import_item\n SET state = $2, detail = $3, bytes_copied = bytes_copied + $4,\n attempts = attempts + 1, updated_at = now()\n WHERE id = $1\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text", - "Text", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "27ce5074610a907aa6cedbd9134dc45008bdac65951b6576fe55c51339b65085" -} diff --git a/.sqlx/query-aa70d0f56b9eebce1d354725f91d79f4aed0e0c61b5bac577cdc0feb68fa682c.json b/.sqlx/query-2be9ffd4d8e233c91e4dcf2aab2c43f1f435f702711e7b9e1c4a0246c59c72f7.json similarity index 79% rename from .sqlx/query-aa70d0f56b9eebce1d354725f91d79f4aed0e0c61b5bac577cdc0feb68fa682c.json rename to .sqlx/query-2be9ffd4d8e233c91e4dcf2aab2c43f1f435f702711e7b9e1c4a0246c59c72f7.json index a1a573ec..ab8e02a8 100644 --- a/.sqlx/query-aa70d0f56b9eebce1d354725f91d79f4aed0e0c61b5bac577cdc0feb68fa682c.json +++ b/.sqlx/query-2be9ffd4d8e233c91e4dcf2aab2c43f1f435f702711e7b9e1c4a0246c59c72f7.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT state,\n count(*) AS \"items!\",\n coalesce(sum(size), 0)::bigint AS \"size!\",\n coalesce(sum(bytes_copied), 0)::bigint AS \"bytes_copied!\"\n FROM nexus_import_item\n WHERE import_id = $1\n GROUP BY state\n ", + "query": "\n SELECT state,\n count(*) AS \"items!\",\n coalesce(sum(size), 0)::bigint AS \"size!\",\n coalesce(sum(bytes_copied), 0)::bigint AS \"bytes_copied!\"\n FROM repository_import_item\n WHERE import_id = $1\n GROUP BY state\n ", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "state" } } @@ -45,5 +45,5 @@ null ] }, - "hash": "aa70d0f56b9eebce1d354725f91d79f4aed0e0c61b5bac577cdc0feb68fa682c" + "hash": "2be9ffd4d8e233c91e4dcf2aab2c43f1f435f702711e7b9e1c4a0246c59c72f7" } diff --git a/.sqlx/query-7ed7bb86ff75aadcfbd46695f058f621775ee3b418f659c63b8237827cd453f7.json b/.sqlx/query-32a681fad30926b74bbf1a6a49a7db56b445baf0fbd06c0f2694cc4d31605a22.json similarity index 66% rename from .sqlx/query-7ed7bb86ff75aadcfbd46695f058f621775ee3b418f659c63b8237827cd453f7.json rename to .sqlx/query-32a681fad30926b74bbf1a6a49a7db56b445baf0fbd06c0f2694cc4d31605a22.json index 20992ffe..97b94364 100644 --- a/.sqlx/query-7ed7bb86ff75aadcfbd46695f058f621775ee3b418f659c63b8237827cd453f7.json +++ b/.sqlx/query-32a681fad30926b74bbf1a6a49a7db56b445baf0fbd06c0f2694cc4d31605a22.json @@ -1,16 +1,16 @@ { "db_name": "PostgreSQL", - "query": "SELECT base_url, registry_url FROM nexus_import WHERE id = $1 FOR UPDATE", + "query": "SELECT url, registry_url FROM repository_import WHERE id = $1 FOR UPDATE", "describe": { "columns": [ { "ordinal": 0, - "name": "base_url", + "name": "url", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", - "name": "base_url" + "table": "repository_import", + "name": "url" } } }, @@ -20,7 +20,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "registry_url" } } @@ -36,5 +36,5 @@ true ] }, - "hash": "7ed7bb86ff75aadcfbd46695f058f621775ee3b418f659c63b8237827cd453f7" + "hash": "32a681fad30926b74bbf1a6a49a7db56b445baf0fbd06c0f2694cc4d31605a22" } diff --git a/.sqlx/query-426f15afec440cec0f4f1dc397035b1889669d3f11a30eb5bf9db8bd220e8869.json b/.sqlx/query-426f15afec440cec0f4f1dc397035b1889669d3f11a30eb5bf9db8bd220e8869.json new file mode 100644 index 00000000..e6b36ef2 --- /dev/null +++ b/.sqlx/query-426f15afec440cec0f4f1dc397035b1889669d3f11a30eb5bf9db8bd220e8869.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE repository_import_item\n SET state = 'pending', detail = NULL, updated_at = now()\n WHERE import_id = $1 AND state = 'failed'\n AND ($2::bigint[] IS NULL OR id = ANY($2))\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "426f15afec440cec0f4f1dc397035b1889669d3f11a30eb5bf9db8bd220e8869" +} diff --git a/.sqlx/query-430f45430f035e0b7e728f7a44289698d5671ef4dbb4069ee79e15bebe547ea8.json b/.sqlx/query-430f45430f035e0b7e728f7a44289698d5671ef4dbb4069ee79e15bebe547ea8.json new file mode 100644 index 00000000..9938a228 --- /dev/null +++ b/.sqlx/query-430f45430f035e0b7e728f7a44289698d5671ef4dbb4069ee79e15bebe547ea8.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM repository_import WHERE id = $1 AND state NOT IN ('queued', 'running')", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "430f45430f035e0b7e728f7a44289698d5671ef4dbb4069ee79e15bebe547ea8" +} diff --git a/.sqlx/query-4fcb159341379dc599bf179e2721dc6f50cba5847f5c970c6eae1b54660a8da2.json b/.sqlx/query-4fcb159341379dc599bf179e2721dc6f50cba5847f5c970c6eae1b54660a8da2.json new file mode 100644 index 00000000..db48abc8 --- /dev/null +++ b/.sqlx/query-4fcb159341379dc599bf179e2721dc6f50cba5847f5c970c6eae1b54660a8da2.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE repository_import SET locked_until = $3\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Uuid", + "Timestamptz" + ] + }, + "nullable": [] + }, + "hash": "4fcb159341379dc599bf179e2721dc6f50cba5847f5c970c6eae1b54660a8da2" +} diff --git a/.sqlx/query-6deebbc753318b4d9542a4bdc25bb39500dc50f8d5e769bafd70ab13f73f60d4.json b/.sqlx/query-4fd9d491b78faf8a40f16cf48ccdc3b999e72258f0ef99aaa10bd8fa57c36ea3.json similarity index 66% rename from .sqlx/query-6deebbc753318b4d9542a4bdc25bb39500dc50f8d5e769bafd70ab13f73f60d4.json rename to .sqlx/query-4fd9d491b78faf8a40f16cf48ccdc3b999e72258f0ef99aaa10bd8fa57c36ea3.json index 035afdd6..929d2f46 100644 --- a/.sqlx/query-6deebbc753318b4d9542a4bdc25bb39500dc50f8d5e769bafd70ab13f73f60d4.json +++ b/.sqlx/query-4fd9d491b78faf8a40f16cf48ccdc3b999e72258f0ef99aaa10bd8fa57c36ea3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT id, password_enc AS \"password_enc!\" FROM nexus_import WHERE password_enc IS NOT NULL", + "query": "SELECT id, password_enc AS \"password_enc!\" FROM repository_import WHERE password_enc IS NOT NULL", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "id" } } @@ -20,7 +20,7 @@ "type_info": "Bytea", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "password_enc" } } @@ -34,5 +34,5 @@ true ] }, - "hash": "6deebbc753318b4d9542a4bdc25bb39500dc50f8d5e769bafd70ab13f73f60d4" + "hash": "4fd9d491b78faf8a40f16cf48ccdc3b999e72258f0ef99aaa10bd8fa57c36ea3" } diff --git a/.sqlx/query-514969a81cf1fbe6b56a5e16b77fcd037828f51721884ce144e1585c30d77743.json b/.sqlx/query-514969a81cf1fbe6b56a5e16b77fcd037828f51721884ce144e1585c30d77743.json deleted file mode 100644 index ce68159e..00000000 --- a/.sqlx/query-514969a81cf1fbe6b56a5e16b77fcd037828f51721884ce144e1585c30d77743.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM nexus_import WHERE id = $1 AND state NOT IN ('queued', 'running')", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "514969a81cf1fbe6b56a5e16b77fcd037828f51721884ce144e1585c30d77743" -} diff --git a/.sqlx/query-52e5b03af038c311c1371b70575682d3d940e7d7dc575caaf1bccae9c49d5159.json b/.sqlx/query-52e5b03af038c311c1371b70575682d3d940e7d7dc575caaf1bccae9c49d5159.json deleted file mode 100644 index 8c8a0fec..00000000 --- a/.sqlx/query-52e5b03af038c311c1371b70575682d3d940e7d7dc575caaf1bccae9c49d5159.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE nexus_import SET locked_until = now() - interval '1 second' WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "52e5b03af038c311c1371b70575682d3d940e7d7dc575caaf1bccae9c49d5159" -} diff --git a/.sqlx/query-69ff060e1912e70357acfb8686f310a4e1813055046ab567f0de726b8df12570.json b/.sqlx/query-69ff060e1912e70357acfb8686f310a4e1813055046ab567f0de726b8df12570.json new file mode 100644 index 00000000..38b28078 --- /dev/null +++ b/.sqlx/query-69ff060e1912e70357acfb8686f310a4e1813055046ab567f0de726b8df12570.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT id FROM app_user WHERE username = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "app_user", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "69ff060e1912e70357acfb8686f310a4e1813055046ab567f0de726b8df12570" +} diff --git a/.sqlx/query-6bb1cefc87f2512e2c69f00edaf278b7e198a4b3f4c71452eb964ce0f1e2cbe2.json b/.sqlx/query-6bb1cefc87f2512e2c69f00edaf278b7e198a4b3f4c71452eb964ce0f1e2cbe2.json new file mode 100644 index 00000000..a8c5b810 --- /dev/null +++ b/.sqlx/query-6bb1cefc87f2512e2c69f00edaf278b7e198a4b3f4c71452eb964ce0f1e2cbe2.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO repository_import_item\n (import_id, kind, name, version, size, published_at, downloaded_at,\n uploader, source)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ON CONFLICT (import_id, kind, name, version) DO NOTHING\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text", + "Text", + "Int8", + "Timestamptz", + "Timestamptz", + "Text", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "6bb1cefc87f2512e2c69f00edaf278b7e198a4b3f4c71452eb964ce0f1e2cbe2" +} diff --git a/.sqlx/query-6ab60461c61652841ad8b7665046ba2f541ac9757cbe5aa08385753172586a64.json b/.sqlx/query-6f9abe1f956bcac2d04dbf9b72c2e4c4bda7f09eca3169e18aa9d18e272a5895.json similarity index 51% rename from .sqlx/query-6ab60461c61652841ad8b7665046ba2f541ac9757cbe5aa08385753172586a64.json rename to .sqlx/query-6f9abe1f956bcac2d04dbf9b72c2e4c4bda7f09eca3169e18aa9d18e272a5895.json index deabe649..709cf099 100644 --- a/.sqlx/query-6ab60461c61652841ad8b7665046ba2f541ac9757cbe5aa08385753172586a64.json +++ b/.sqlx/query-6f9abe1f956bcac2d04dbf9b72c2e4c4bda7f09eca3169e18aa9d18e272a5895.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO nexus_import\n (repository_id, base_url, source_repository, registry_url, username,\n on_conflict, concurrency, dry_run, nexus_version, created_by)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)\n RETURNING id\n ", + "query": "\n INSERT INTO repository_import\n (repository_id, url, names, username, on_conflict, concurrency, dry_run,\n scan_listing, source_server, source_total, created_by)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)\n RETURNING id\n ", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "id" } } @@ -19,13 +19,14 @@ "Left": [ "Int8", "Text", - "Text", - "Text", + "TextArray", "Text", "Text", "Int4", "Bool", "Text", + "Text", + "Int8", "Int8" ] }, @@ -33,5 +34,5 @@ false ] }, - "hash": "6ab60461c61652841ad8b7665046ba2f541ac9757cbe5aa08385753172586a64" + "hash": "6f9abe1f956bcac2d04dbf9b72c2e4c4bda7f09eca3169e18aa9d18e272a5895" } diff --git a/.sqlx/query-75ae409c1803522bfb9d7349caefdab40c67cb99c0ef2b5b2043cb90bfe062da.json b/.sqlx/query-75ae409c1803522bfb9d7349caefdab40c67cb99c0ef2b5b2043cb90bfe062da.json deleted file mode 100644 index e338d0c0..00000000 --- a/.sqlx/query-75ae409c1803522bfb9d7349caefdab40c67cb99c0ef2b5b2043cb90bfe062da.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH due AS (\n SELECT id FROM nexus_import\n WHERE state IN ('queued', 'running')\n AND (locked_until IS NULL OR locked_until < $1)\n ORDER BY id\n FOR UPDATE SKIP LOCKED\n LIMIT 1\n )\n UPDATE nexus_import i\n SET state = 'running',\n run_id = $2,\n locked_until = $1 + make_interval(secs => $3),\n started_at = coalesce(i.started_at, $1),\n updated_at = $1\n FROM due\n WHERE i.id = due.id\n RETURNING i.id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8", - "origin": { - "Table": { - "table": "nexus_import", - "name": "id" - } - } - } - ], - "parameters": { - "Left": [ - "Timestamptz", - "Uuid", - "Float8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "75ae409c1803522bfb9d7349caefdab40c67cb99c0ef2b5b2043cb90bfe062da" -} diff --git a/.sqlx/query-4a31af4ec2a7e0ee7f4b2a1ca245e4ced1c380df88ca4f293ea949354b75e748.json b/.sqlx/query-7be04e96c8e7d1ecbd1b1580a4df3d93c4345a6d47d512c9c52e79cde2ca82a1.json similarity index 80% rename from .sqlx/query-4a31af4ec2a7e0ee7f4b2a1ca245e4ced1c380df88ca4f293ea949354b75e748.json rename to .sqlx/query-7be04e96c8e7d1ecbd1b1580a4df3d93c4345a6d47d512c9c52e79cde2ca82a1.json index e7560036..ca98e6ec 100644 --- a/.sqlx/query-4a31af4ec2a7e0ee7f4b2a1ca245e4ced1c380df88ca4f293ea949354b75e748.json +++ b/.sqlx/query-7be04e96c8e7d1ecbd1b1580a4df3d93c4345a6d47d512c9c52e79cde2ca82a1.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT c.created_at, c.last_downloaded_at\n FROM component c JOIN repository r ON r.id = c.repository_id\n WHERE r.name = $1 AND c.name = 'acme.core'\n ", + "query": "\n SELECT c.created_at, c.last_downloaded_at\n FROM component c JOIN repository r ON r.id = c.repository_id\n WHERE r.name = $1 AND c.name = $2 AND c.version = $3\n ", "describe": { "columns": [ { @@ -28,6 +28,8 @@ ], "parameters": { "Left": [ + "Text", + "Text", "Text" ] }, @@ -36,5 +38,5 @@ true ] }, - "hash": "4a31af4ec2a7e0ee7f4b2a1ca245e4ced1c380df88ca4f293ea949354b75e748" + "hash": "7be04e96c8e7d1ecbd1b1580a4df3d93c4345a6d47d512c9c52e79cde2ca82a1" } diff --git a/.sqlx/query-8a6deee6a71756a41027067c6c3f452b51384d82aeb35ef5fdec699ac93d9f21.json b/.sqlx/query-8a6deee6a71756a41027067c6c3f452b51384d82aeb35ef5fdec699ac93d9f21.json new file mode 100644 index 00000000..1b6ff5d9 --- /dev/null +++ b/.sqlx/query-8a6deee6a71756a41027067c6c3f452b51384d82aeb35ef5fdec699ac93d9f21.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE repository_import\n SET state = $3,\n last_error = $4,\n finished_at = now(),\n updated_at = now(),\n run_id = NULL,\n locked_until = NULL,\n cancel_requested = false,\n password_enc = CASE WHEN $3 = 'completed' THEN NULL ELSE password_enc END\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Uuid", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "8a6deee6a71756a41027067c6c3f452b51384d82aeb35ef5fdec699ac93d9f21" +} diff --git a/.sqlx/query-8b6f6a8fdad5aa9e5e98a581adb9c9f91d9d89631712a7c6877c386849c132c4.json b/.sqlx/query-8b6f6a8fdad5aa9e5e98a581adb9c9f91d9d89631712a7c6877c386849c132c4.json deleted file mode 100644 index 8e42d73b..00000000 --- a/.sqlx/query-8b6f6a8fdad5aa9e5e98a581adb9c9f91d9d89631712a7c6877c386849c132c4.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nexus_import\n SET scan_listing = $3, cursor = $4, phase = $5,\n assets_seen = assets_seen + $6, bytes_seen = bytes_seen + $7,\n updated_at = now()\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Uuid", - "Text", - "Text", - "Text", - "Int8", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "8b6f6a8fdad5aa9e5e98a581adb9c9f91d9d89631712a7c6877c386849c132c4" -} diff --git a/.sqlx/query-f2beb1ca7da114b1b63b853bf907b6d2fb07b1085064a01ca0a1eacb919b968f.json b/.sqlx/query-8bae46d737dd65a10f034985beed7dcaa2726c343de9870b5b97a2c7253b596f.json similarity index 69% rename from .sqlx/query-f2beb1ca7da114b1b63b853bf907b6d2fb07b1085064a01ca0a1eacb919b968f.json rename to .sqlx/query-8bae46d737dd65a10f034985beed7dcaa2726c343de9870b5b97a2c7253b596f.json index c2e67703..8134162b 100644 --- a/.sqlx/query-f2beb1ca7da114b1b63b853bf907b6d2fb07b1085064a01ca0a1eacb919b968f.json +++ b/.sqlx/query-8bae46d737dd65a10f034985beed7dcaa2726c343de9870b5b97a2c7253b596f.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id, base_url, registry_url, password_enc AS \"password_enc!\"\n FROM nexus_import WHERE password_enc IS NOT NULL\n ORDER BY id\n ", + "query": "\n SELECT id, url, registry_url, password_enc AS \"password_enc!\"\n FROM repository_import WHERE password_enc IS NOT NULL\n ORDER BY id\n ", "describe": { "columns": [ { @@ -9,19 +9,19 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "id" } } }, { "ordinal": 1, - "name": "base_url", + "name": "url", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", - "name": "base_url" + "table": "repository_import", + "name": "url" } } }, @@ -31,7 +31,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "registry_url" } } @@ -42,7 +42,7 @@ "type_info": "Bytea", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "password_enc" } } @@ -58,5 +58,5 @@ true ] }, - "hash": "f2beb1ca7da114b1b63b853bf907b6d2fb07b1085064a01ca0a1eacb919b968f" + "hash": "8bae46d737dd65a10f034985beed7dcaa2726c343de9870b5b97a2c7253b596f" } diff --git a/.sqlx/query-8c90b5d5aa289dbd657480f0f7625c5fa2c6941903be30227fa3546b5e027218.json b/.sqlx/query-8c90b5d5aa289dbd657480f0f7625c5fa2c6941903be30227fa3546b5e027218.json new file mode 100644 index 00000000..8644839a --- /dev/null +++ b/.sqlx/query-8c90b5d5aa289dbd657480f0f7625c5fa2c6941903be30227fa3546b5e027218.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE repository_import\n SET cursor = $3, phase = $4,\n source_total = CASE WHEN $5::bigint IS NULL THEN source_total\n ELSE greatest(coalesce(source_total, 0), $5) END,\n updated_at = now()\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Uuid", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "8c90b5d5aa289dbd657480f0f7625c5fa2c6941903be30227fa3546b5e027218" +} diff --git a/.sqlx/query-b76fec429ab9b2826c95429795c138f7a72c1822f254b8315099035d6155bfeb.json b/.sqlx/query-90727fe56ec79034408235b7f1319d93622dc0f39c69883e5f375d08dbd6b728.json similarity index 71% rename from .sqlx/query-b76fec429ab9b2826c95429795c138f7a72c1822f254b8315099035d6155bfeb.json rename to .sqlx/query-90727fe56ec79034408235b7f1319d93622dc0f39c69883e5f375d08dbd6b728.json index a7f519a1..0d3f7431 100644 --- a/.sqlx/query-b76fec429ab9b2826c95429795c138f7a72c1822f254b8315099035d6155bfeb.json +++ b/.sqlx/query-90727fe56ec79034408235b7f1319d93622dc0f39c69883e5f375d08dbd6b728.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id, repository_id, base_url, source_repository, registry_url, username,\n (password_enc IS NOT NULL) AS \"has_password!\",\n on_conflict, concurrency, dry_run, state, phase, scan_listing, cursor,\n cancel_requested, assets_seen, bytes_seen, nexus_version, last_error,\n created_by, created_at, started_at, finished_at, updated_at\n FROM nexus_import\n WHERE repository_id = $1\n ORDER BY id DESC\n LIMIT $2\n ", + "query": "\n SELECT id, repository_id, url, source_repository, registry_url, names, username,\n (password_enc IS NOT NULL) AS \"has_password!\",\n on_conflict, concurrency, dry_run, state, phase, scan_listing, cursor,\n cancel_requested, source_server, source_total,\n last_error, created_by, created_at, started_at, finished_at, updated_at\n FROM repository_import\n WHERE repository_id = $1\n ORDER BY id DESC\n LIMIT $2\n ", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "id" } } @@ -20,19 +20,19 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "repository_id" } } }, { "ordinal": 2, - "name": "base_url", + "name": "url", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", - "name": "base_url" + "table": "repository_import", + "name": "url" } } }, @@ -42,7 +42,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "source_repository" } } @@ -53,146 +53,146 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "registry_url" } } }, { "ordinal": 5, + "name": "names", + "type_info": "TextArray", + "origin": { + "Table": { + "table": "repository_import", + "name": "names" + } + } + }, + { + "ordinal": 6, "name": "username", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "username" } } }, { - "ordinal": 6, + "ordinal": 7, "name": "has_password!", "type_info": "Bool", "origin": "Expression" }, { - "ordinal": 7, + "ordinal": 8, "name": "on_conflict", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "on_conflict" } } }, { - "ordinal": 8, + "ordinal": 9, "name": "concurrency", "type_info": "Int4", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "concurrency" } } }, { - "ordinal": 9, + "ordinal": 10, "name": "dry_run", "type_info": "Bool", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "dry_run" } } }, { - "ordinal": 10, + "ordinal": 11, "name": "state", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "state" } } }, { - "ordinal": 11, + "ordinal": 12, "name": "phase", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "phase" } } }, { - "ordinal": 12, + "ordinal": 13, "name": "scan_listing", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "scan_listing" } } }, { - "ordinal": 13, + "ordinal": 14, "name": "cursor", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "cursor" } } }, { - "ordinal": 14, + "ordinal": 15, "name": "cancel_requested", "type_info": "Bool", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "cancel_requested" } } }, - { - "ordinal": 15, - "name": "assets_seen", - "type_info": "Int8", - "origin": { - "Table": { - "table": "nexus_import", - "name": "assets_seen" - } - } - }, { "ordinal": 16, - "name": "bytes_seen", - "type_info": "Int8", + "name": "source_server", + "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", - "name": "bytes_seen" + "table": "repository_import", + "name": "source_server" } } }, { "ordinal": 17, - "name": "nexus_version", - "type_info": "Text", + "name": "source_total", + "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", - "name": "nexus_version" + "table": "repository_import", + "name": "source_total" } } }, @@ -202,7 +202,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "last_error" } } @@ -213,7 +213,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "created_by" } } @@ -224,7 +224,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "created_at" } } @@ -235,7 +235,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "started_at" } } @@ -246,7 +246,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "finished_at" } } @@ -257,7 +257,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "updated_at" } } @@ -273,7 +273,8 @@ false, false, false, - false, + true, + true, true, true, null, @@ -285,8 +286,7 @@ false, true, false, - false, - false, + true, true, true, true, @@ -296,5 +296,5 @@ false ] }, - "hash": "b76fec429ab9b2826c95429795c138f7a72c1822f254b8315099035d6155bfeb" + "hash": "90727fe56ec79034408235b7f1319d93622dc0f39c69883e5f375d08dbd6b728" } diff --git a/.sqlx/query-96aab5a8c7d6fee5601f80ab2d37c4914086eba14eca1441e4580df1450c7601.json b/.sqlx/query-96aab5a8c7d6fee5601f80ab2d37c4914086eba14eca1441e4580df1450c7601.json deleted file mode 100644 index ca3f5193..00000000 --- a/.sqlx/query-96aab5a8c7d6fee5601f80ab2d37c4914086eba14eca1441e4580df1450c7601.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO nexus_import_item\n (import_id, kind, name, version, size, published_at, downloaded_at,\n uploader, source)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)\n ON CONFLICT (import_id, kind, name, version) DO NOTHING\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text", - "Text", - "Text", - "Int8", - "Timestamptz", - "Timestamptz", - "Text", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "96aab5a8c7d6fee5601f80ab2d37c4914086eba14eca1441e4580df1450c7601" -} diff --git a/.sqlx/query-97df002c3a51803fba427abfd3b985a0220e34b780b144ec22d9e1e908d227bf.json b/.sqlx/query-97df002c3a51803fba427abfd3b985a0220e34b780b144ec22d9e1e908d227bf.json deleted file mode 100644 index c941e215..00000000 --- a/.sqlx/query-97df002c3a51803fba427abfd3b985a0220e34b780b144ec22d9e1e908d227bf.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nexus_import i\n SET state = 'queued', run_id = NULL, locked_until = NULL, finished_at = NULL,\n last_error = NULL, cancel_requested = false, updated_at = now()\n WHERE id = $1 AND state IN ('cancelled', 'failed', 'completed')\n AND NOT EXISTS (\n SELECT 1 FROM nexus_import o\n WHERE o.repository_id = i.repository_id AND o.id <> i.id\n AND o.state IN ('queued', 'running')\n )\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [] - }, - "hash": "97df002c3a51803fba427abfd3b985a0220e34b780b144ec22d9e1e908d227bf" -} diff --git a/.sqlx/query-9e18c564a49d360756f43c8f4a82e233cf506d7a1bf8c4cb4d8b215402c889c1.json b/.sqlx/query-9e18c564a49d360756f43c8f4a82e233cf506d7a1bf8c4cb4d8b215402c889c1.json new file mode 100644 index 00000000..84709c66 --- /dev/null +++ b/.sqlx/query-9e18c564a49d360756f43c8f4a82e233cf506d7a1bf8c4cb4d8b215402c889c1.json @@ -0,0 +1,21 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT coalesce(sum(a.download_count), 0)::bigint AS \"n!\"\n FROM asset a JOIN component c ON c.id = a.component_id\n JOIN repository r ON r.id = c.repository_id\n WHERE r.name = 'nuget-source'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8", + "origin": "Expression" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "9e18c564a49d360756f43c8f4a82e233cf506d7a1bf8c4cb4d8b215402c889c1" +} diff --git a/.sqlx/query-b4bc64780d014552c6c7d13101e3724e0bbc78d1e58793b56845d8e3109e9850.json b/.sqlx/query-a597333c3c4a106ff419d7c1a3c75f737b9b577dd404f7a4661c32235c2328f2.json similarity index 72% rename from .sqlx/query-b4bc64780d014552c6c7d13101e3724e0bbc78d1e58793b56845d8e3109e9850.json rename to .sqlx/query-a597333c3c4a106ff419d7c1a3c75f737b9b577dd404f7a4661c32235c2328f2.json index 4ef88075..1ff59db8 100644 --- a/.sqlx/query-b4bc64780d014552c6c7d13101e3724e0bbc78d1e58793b56845d8e3109e9850.json +++ b/.sqlx/query-a597333c3c4a106ff419d7c1a3c75f737b9b577dd404f7a4661c32235c2328f2.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id, repository_id, base_url, source_repository, registry_url, username,\n (password_enc IS NOT NULL) AS \"has_password!\",\n on_conflict, concurrency, dry_run, state, phase, scan_listing, cursor,\n cancel_requested, assets_seen, bytes_seen, nexus_version, last_error,\n created_by, created_at, started_at, finished_at, updated_at\n FROM nexus_import\n WHERE id = $1\n ", + "query": "\n SELECT id, repository_id, url, source_repository, registry_url, names, username,\n (password_enc IS NOT NULL) AS \"has_password!\",\n on_conflict, concurrency, dry_run, state, phase, scan_listing, cursor,\n cancel_requested, source_server, source_total,\n last_error, created_by, created_at, started_at, finished_at, updated_at\n FROM repository_import\n WHERE id = $1\n ", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "id" } } @@ -20,19 +20,19 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "repository_id" } } }, { "ordinal": 2, - "name": "base_url", + "name": "url", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", - "name": "base_url" + "table": "repository_import", + "name": "url" } } }, @@ -42,7 +42,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "source_repository" } } @@ -53,146 +53,146 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "registry_url" } } }, { "ordinal": 5, + "name": "names", + "type_info": "TextArray", + "origin": { + "Table": { + "table": "repository_import", + "name": "names" + } + } + }, + { + "ordinal": 6, "name": "username", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "username" } } }, { - "ordinal": 6, + "ordinal": 7, "name": "has_password!", "type_info": "Bool", "origin": "Expression" }, { - "ordinal": 7, + "ordinal": 8, "name": "on_conflict", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "on_conflict" } } }, { - "ordinal": 8, + "ordinal": 9, "name": "concurrency", "type_info": "Int4", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "concurrency" } } }, { - "ordinal": 9, + "ordinal": 10, "name": "dry_run", "type_info": "Bool", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "dry_run" } } }, { - "ordinal": 10, + "ordinal": 11, "name": "state", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "state" } } }, { - "ordinal": 11, + "ordinal": 12, "name": "phase", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "phase" } } }, { - "ordinal": 12, + "ordinal": 13, "name": "scan_listing", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "scan_listing" } } }, { - "ordinal": 13, + "ordinal": 14, "name": "cursor", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "cursor" } } }, { - "ordinal": 14, + "ordinal": 15, "name": "cancel_requested", "type_info": "Bool", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "cancel_requested" } } }, - { - "ordinal": 15, - "name": "assets_seen", - "type_info": "Int8", - "origin": { - "Table": { - "table": "nexus_import", - "name": "assets_seen" - } - } - }, { "ordinal": 16, - "name": "bytes_seen", - "type_info": "Int8", + "name": "source_server", + "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", - "name": "bytes_seen" + "table": "repository_import", + "name": "source_server" } } }, { "ordinal": 17, - "name": "nexus_version", - "type_info": "Text", + "name": "source_total", + "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", - "name": "nexus_version" + "table": "repository_import", + "name": "source_total" } } }, @@ -202,7 +202,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "last_error" } } @@ -213,7 +213,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "created_by" } } @@ -224,7 +224,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "created_at" } } @@ -235,7 +235,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "started_at" } } @@ -246,7 +246,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "finished_at" } } @@ -257,7 +257,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "updated_at" } } @@ -272,7 +272,8 @@ false, false, false, - false, + true, + true, true, true, null, @@ -284,8 +285,7 @@ false, true, false, - false, - false, + true, true, true, true, @@ -295,5 +295,5 @@ false ] }, - "hash": "b4bc64780d014552c6c7d13101e3724e0bbc78d1e58793b56845d8e3109e9850" + "hash": "a597333c3c4a106ff419d7c1a3c75f737b9b577dd404f7a4661c32235c2328f2" } diff --git a/.sqlx/query-abcedd12415a3ff65f4bc84ab54a4c8b5af033bb15b9cbb68c4d49a1297a4ef8.json b/.sqlx/query-abcedd12415a3ff65f4bc84ab54a4c8b5af033bb15b9cbb68c4d49a1297a4ef8.json new file mode 100644 index 00000000..ce0e71d8 --- /dev/null +++ b/.sqlx/query-abcedd12415a3ff65f4bc84ab54a4c8b5af033bb15b9cbb68c4d49a1297a4ef8.json @@ -0,0 +1,31 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO repository_import\n (repository_id, url, source_repository, username, on_conflict, concurrency,\n dry_run, state, phase, scan_listing, source_server, created_by, last_error)\n VALUES ($1, $2, 'nuget-hosted', $3, 'skip', 2, false, 'failed', 'scan', 'search',\n 'Nexus/3.70.1-02 (OSS)', $4, 'Nexus is unreachable')\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "repository_import", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "abcedd12415a3ff65f4bc84ab54a4c8b5af033bb15b9cbb68c4d49a1297a4ef8" +} diff --git a/.sqlx/query-c16bc17f5325a9e69bdbde3581c7ca0fe027527b85313cc532c68f92749b40c6.json b/.sqlx/query-c16bc17f5325a9e69bdbde3581c7ca0fe027527b85313cc532c68f92749b40c6.json new file mode 100644 index 00000000..6fe8fef9 --- /dev/null +++ b/.sqlx/query-c16bc17f5325a9e69bdbde3581c7ca0fe027527b85313cc532c68f92749b40c6.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE repository_import\n SET state = CASE WHEN state = 'queued' THEN 'cancelled' ELSE state END,\n finished_at = CASE WHEN state = 'queued' THEN now() ELSE finished_at END,\n cancel_requested = (state = 'running'),\n updated_at = now()\n WHERE id = $1 AND state IN ('queued', 'running')\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "c16bc17f5325a9e69bdbde3581c7ca0fe027527b85313cc532c68f92749b40c6" +} diff --git a/.sqlx/query-1217d97d8f397d87e27e6812a0929ec05d3d2cd8cf224a7a7fa8ebcad7edd7f4.json b/.sqlx/query-ca60f5642b21b9598e45c13aa3e19bd142b8e2c86eb2a67d1112dd41fdbfa4d9.json similarity index 78% rename from .sqlx/query-1217d97d8f397d87e27e6812a0929ec05d3d2cd8cf224a7a7fa8ebcad7edd7f4.json rename to .sqlx/query-ca60f5642b21b9598e45c13aa3e19bd142b8e2c86eb2a67d1112dd41fdbfa4d9.json index 4592d1e8..a1b0f712 100644 --- a/.sqlx/query-1217d97d8f397d87e27e6812a0929ec05d3d2cd8cf224a7a7fa8ebcad7edd7f4.json +++ b/.sqlx/query-ca60f5642b21b9598e45c13aa3e19bd142b8e2c86eb2a67d1112dd41fdbfa4d9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT id, import_id, kind, name, version, state, detail, size, bytes_copied,\n published_at, downloaded_at, uploader, source, attempts, updated_at\n FROM nexus_import_item\n WHERE import_id = $1 AND kind = $2 AND state = 'pending' AND id > $3\n ORDER BY id\n LIMIT $4\n ", + "query": "\n SELECT id, import_id, kind, name, version, state, detail, size, bytes_copied,\n published_at, downloaded_at, uploader, source, attempts, updated_at\n FROM repository_import_item\n WHERE import_id = $1 AND ($2::text IS NULL OR state = $2) AND id > $3\n ORDER BY id\n LIMIT $4\n ", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "id" } } @@ -20,7 +20,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "import_id" } } @@ -31,7 +31,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "kind" } } @@ -42,7 +42,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "name" } } @@ -53,7 +53,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "version" } } @@ -64,7 +64,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "state" } } @@ -75,7 +75,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "detail" } } @@ -86,7 +86,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "size" } } @@ -97,7 +97,7 @@ "type_info": "Int8", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "bytes_copied" } } @@ -108,7 +108,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "published_at" } } @@ -119,7 +119,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "downloaded_at" } } @@ -130,7 +130,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "uploader" } } @@ -141,7 +141,7 @@ "type_info": "Jsonb", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "source" } } @@ -152,7 +152,7 @@ "type_info": "Int4", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "attempts" } } @@ -163,7 +163,7 @@ "type_info": "Timestamptz", "origin": { "Table": { - "table": "nexus_import_item", + "table": "repository_import_item", "name": "updated_at" } } @@ -195,5 +195,5 @@ false ] }, - "hash": "1217d97d8f397d87e27e6812a0929ec05d3d2cd8cf224a7a7fa8ebcad7edd7f4" + "hash": "ca60f5642b21b9598e45c13aa3e19bd142b8e2c86eb2a67d1112dd41fdbfa4d9" } diff --git a/.sqlx/query-b6951beb73f63f1829cf4034adc59bee26841388c176eebc9df5e5cc7b8b2461.json b/.sqlx/query-cb435f03a1c8d8835dad30de6bc006717cf27e2048aaec1e0872db3981be69be.json similarity index 61% rename from .sqlx/query-b6951beb73f63f1829cf4034adc59bee26841388c176eebc9df5e5cc7b8b2461.json rename to .sqlx/query-cb435f03a1c8d8835dad30de6bc006717cf27e2048aaec1e0872db3981be69be.json index ba9be40b..c1ccfa99 100644 --- a/.sqlx/query-b6951beb73f63f1829cf4034adc59bee26841388c176eebc9df5e5cc7b8b2461.json +++ b/.sqlx/query-cb435f03a1c8d8835dad30de6bc006717cf27e2048aaec1e0872db3981be69be.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n WITH dek AS (\n SELECT 1 FROM encryption_key\n WHERE layer = 'dek' AND scope = $4 AND version = $5 AND state <> 'revoked'\n FOR SHARE\n )\n UPDATE nexus_import SET password_enc = $2\n WHERE id = $1 AND password_enc = $3 AND EXISTS (SELECT 1 FROM dek)\n ", + "query": "\n WITH dek AS (\n SELECT 1 FROM encryption_key\n WHERE layer = 'dek' AND scope = $4 AND version = $5 AND state <> 'revoked'\n FOR SHARE\n )\n UPDATE repository_import SET password_enc = $2\n WHERE id = $1 AND password_enc = $3 AND EXISTS (SELECT 1 FROM dek)\n ", "describe": { "columns": [], "parameters": { @@ -14,5 +14,5 @@ }, "nullable": [] }, - "hash": "b6951beb73f63f1829cf4034adc59bee26841388c176eebc9df5e5cc7b8b2461" + "hash": "cb435f03a1c8d8835dad30de6bc006717cf27e2048aaec1e0872db3981be69be" } diff --git a/.sqlx/query-cbf56582f6a6d29f724c916948d9289ce31989989e995e61c72f9a84a87edd7d.json b/.sqlx/query-cbf56582f6a6d29f724c916948d9289ce31989989e995e61c72f9a84a87edd7d.json new file mode 100644 index 00000000..54e528e6 --- /dev/null +++ b/.sqlx/query-cbf56582f6a6d29f724c916948d9289ce31989989e995e61c72f9a84a87edd7d.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE repository_import i\n SET state = 'queued', run_id = NULL, locked_until = NULL, finished_at = NULL,\n last_error = NULL, cancel_requested = false, updated_at = now()\n WHERE id = $1 AND state IN ('cancelled', 'failed', 'completed')\n AND NOT EXISTS (\n SELECT 1 FROM repository_import o\n WHERE o.repository_id = i.repository_id AND o.id <> i.id\n AND o.state IN ('queued', 'running')\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "cbf56582f6a6d29f724c916948d9289ce31989989e995e61c72f9a84a87edd7d" +} diff --git a/.sqlx/query-73e966747c8f9e8836c585a347b2896a1b67fa4811b229680ee30a300c001336.json b/.sqlx/query-dc9a111531650d4a65b04b93c6e9533bf723383b068c26b6f214d907d1762a9b.json similarity index 53% rename from .sqlx/query-73e966747c8f9e8836c585a347b2896a1b67fa4811b229680ee30a300c001336.json rename to .sqlx/query-dc9a111531650d4a65b04b93c6e9533bf723383b068c26b6f214d907d1762a9b.json index 8dfb8acd..190c5ef6 100644 --- a/.sqlx/query-73e966747c8f9e8836c585a347b2896a1b67fa4811b229680ee30a300c001336.json +++ b/.sqlx/query-dc9a111531650d4a65b04b93c6e9533bf723383b068c26b6f214d907d1762a9b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE nexus_import SET password_enc = $2 WHERE id = $1", + "query": "UPDATE repository_import SET password_enc = $2 WHERE id = $1", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "73e966747c8f9e8836c585a347b2896a1b67fa4811b229680ee30a300c001336" + "hash": "dc9a111531650d4a65b04b93c6e9533bf723383b068c26b6f214d907d1762a9b" } diff --git a/.sqlx/query-e37a479c00daf5ec6ef04745ba90133aa6305e6b9ec0fa570e3b5192ce5384dd.json b/.sqlx/query-e37a479c00daf5ec6ef04745ba90133aa6305e6b9ec0fa570e3b5192ce5384dd.json deleted file mode 100644 index ce2e392f..00000000 --- a/.sqlx/query-e37a479c00daf5ec6ef04745ba90133aa6305e6b9ec0fa570e3b5192ce5384dd.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE nexus_import_item\n SET state = 'pending', detail = NULL, updated_at = now()\n WHERE import_id = $1 AND state = 'failed'\n AND ($2::bigint[] IS NULL OR id = ANY($2))\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8Array" - ] - }, - "nullable": [] - }, - "hash": "e37a479c00daf5ec6ef04745ba90133aa6305e6b9ec0fa570e3b5192ce5384dd" -} diff --git a/.sqlx/query-e37c0a167cfaa3eee14c30633c63408869c26546a1f77b1dcd0fa836866cd030.json b/.sqlx/query-e37c0a167cfaa3eee14c30633c63408869c26546a1f77b1dcd0fa836866cd030.json new file mode 100644 index 00000000..6d30388e --- /dev/null +++ b/.sqlx/query-e37c0a167cfaa3eee14c30633c63408869c26546a1f77b1dcd0fa836866cd030.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO repository_import_item (import_id, kind, name, version, state, source)\n VALUES ($1, 'component', 'Acme.Core', '1.0.0', 'pending', $2)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "e37c0a167cfaa3eee14c30633c63408869c26546a1f77b1dcd0fa836866cd030" +} diff --git a/.sqlx/query-8f94720def905b052e17c6fbcc3108c22eea77884ac5660079e4bbe1b8ca6663.json b/.sqlx/query-e932ab583448f575388b9abfe0ba0117c00dfd6241855b08490cda7c1a1ce136.json similarity index 66% rename from .sqlx/query-8f94720def905b052e17c6fbcc3108c22eea77884ac5660079e4bbe1b8ca6663.json rename to .sqlx/query-e932ab583448f575388b9abfe0ba0117c00dfd6241855b08490cda7c1a1ce136.json index 137f7dc5..44243eec 100644 --- a/.sqlx/query-8f94720def905b052e17c6fbcc3108c22eea77884ac5660079e4bbe1b8ca6663.json +++ b/.sqlx/query-e932ab583448f575388b9abfe0ba0117c00dfd6241855b08490cda7c1a1ce136.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT password_enc FROM nexus_import WHERE id = $1", + "query": "SELECT password_enc FROM repository_import WHERE id = $1", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Bytea", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "password_enc" } } @@ -24,5 +24,5 @@ true ] }, - "hash": "8f94720def905b052e17c6fbcc3108c22eea77884ac5660079e4bbe1b8ca6663" + "hash": "e932ab583448f575388b9abfe0ba0117c00dfd6241855b08490cda7c1a1ce136" } diff --git a/.sqlx/query-7f7e92513dd74c34a940da96356ccef52fcc54fe48b3e52111158f505d55da7b.json b/.sqlx/query-f17cac96289359ceb7e0c08868c49869f241e62762f17773039410806fd7fd6d.json similarity index 60% rename from .sqlx/query-7f7e92513dd74c34a940da96356ccef52fcc54fe48b3e52111158f505d55da7b.json rename to .sqlx/query-f17cac96289359ceb7e0c08868c49869f241e62762f17773039410806fd7fd6d.json index a5985164..8394f84a 100644 --- a/.sqlx/query-7f7e92513dd74c34a940da96356ccef52fcc54fe48b3e52111158f505d55da7b.json +++ b/.sqlx/query-f17cac96289359ceb7e0c08868c49869f241e62762f17773039410806fd7fd6d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT cancel_requested FROM nexus_import\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", + "query": "\n SELECT cancel_requested FROM repository_import\n WHERE id = $1 AND run_id = $2 AND state = 'running'\n ", "describe": { "columns": [ { @@ -9,7 +9,7 @@ "type_info": "Bool", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "cancel_requested" } } @@ -25,5 +25,5 @@ false ] }, - "hash": "7f7e92513dd74c34a940da96356ccef52fcc54fe48b3e52111158f505d55da7b" + "hash": "f17cac96289359ceb7e0c08868c49869f241e62762f17773039410806fd7fd6d" } diff --git a/.sqlx/query-182c705b9213322dcd113f316840d91291e82a00f19cb9243c0a9ed2005f790c.json b/.sqlx/query-f9fd2bb73a05dc4932f6483282b222c752ad03e90239ba01e75c91ee6d68da98.json similarity index 52% rename from .sqlx/query-182c705b9213322dcd113f316840d91291e82a00f19cb9243c0a9ed2005f790c.json rename to .sqlx/query-f9fd2bb73a05dc4932f6483282b222c752ad03e90239ba01e75c91ee6d68da98.json index 642d076f..33173041 100644 --- a/.sqlx/query-182c705b9213322dcd113f316840d91291e82a00f19cb9243c0a9ed2005f790c.json +++ b/.sqlx/query-f9fd2bb73a05dc4932f6483282b222c752ad03e90239ba01e75c91ee6d68da98.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "DELETE FROM nexus_import_item WHERE id = $1", + "query": "DELETE FROM repository_import_item WHERE id = $1", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "182c705b9213322dcd113f316840d91291e82a00f19cb9243c0a9ed2005f790c" + "hash": "f9fd2bb73a05dc4932f6483282b222c752ad03e90239ba01e75c91ee6d68da98" } diff --git a/.sqlx/query-140e585934ee4f40b43117d5ac312fcb5390bf968547265fa2771a28b9380cd0.json b/.sqlx/query-fdb3ed8114bfe2e9193b887f66c3281d784f0d9220cbcd18c3d3a9a7e86c29d2.json similarity index 66% rename from .sqlx/query-140e585934ee4f40b43117d5ac312fcb5390bf968547265fa2771a28b9380cd0.json rename to .sqlx/query-fdb3ed8114bfe2e9193b887f66c3281d784f0d9220cbcd18c3d3a9a7e86c29d2.json index 353e8725..b74c635f 100644 --- a/.sqlx/query-140e585934ee4f40b43117d5ac312fcb5390bf968547265fa2771a28b9380cd0.json +++ b/.sqlx/query-fdb3ed8114bfe2e9193b887f66c3281d784f0d9220cbcd18c3d3a9a7e86c29d2.json @@ -1,16 +1,16 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT base_url, registry_url, password_enc\n FROM nexus_import WHERE id = $1\n ", + "query": "\n SELECT url, registry_url, password_enc\n FROM repository_import WHERE id = $1\n ", "describe": { "columns": [ { "ordinal": 0, - "name": "base_url", + "name": "url", "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", - "name": "base_url" + "table": "repository_import", + "name": "url" } } }, @@ -20,7 +20,7 @@ "type_info": "Text", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "registry_url" } } @@ -31,7 +31,7 @@ "type_info": "Bytea", "origin": { "Table": { - "table": "nexus_import", + "table": "repository_import", "name": "password_enc" } } @@ -48,5 +48,5 @@ true ] }, - "hash": "140e585934ee4f40b43117d5ac312fcb5390bf968547265fa2771a28b9380cd0" + "hash": "fdb3ed8114bfe2e9193b887f66c3281d784f0d9220cbcd18c3d3a9a7e86c29d2" } diff --git a/.sqlx/query-ff454fad056b994bafa995d317a8ce32a87eb4ffe41e499aad73e218dd739de0.json b/.sqlx/query-ff454fad056b994bafa995d317a8ce32a87eb4ffe41e499aad73e218dd739de0.json new file mode 100644 index 00000000..9c1be629 --- /dev/null +++ b/.sqlx/query-ff454fad056b994bafa995d317a8ce32a87eb4ffe41e499aad73e218dd739de0.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH due AS (\n SELECT id FROM repository_import\n WHERE state IN ('queued', 'running')\n AND (locked_until IS NULL OR locked_until < $1)\n ORDER BY id\n FOR UPDATE SKIP LOCKED\n LIMIT 1\n )\n UPDATE repository_import i\n SET state = 'running',\n run_id = $2,\n locked_until = $1 + make_interval(secs => $3),\n started_at = coalesce(i.started_at, $1),\n updated_at = $1\n FROM due\n WHERE i.id = due.id\n RETURNING i.id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8", + "origin": { + "Table": { + "table": "repository_import", + "name": "id" + } + } + } + ], + "parameters": { + "Left": [ + "Timestamptz", + "Uuid", + "Float8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ff454fad056b994bafa995d317a8ce32a87eb4ffe41e499aad73e218dd739de0" +} diff --git a/README.md b/README.md index 46f6defc..4c36647e 100644 --- a/README.md +++ b/README.md @@ -238,9 +238,10 @@ The guides below are also a site, with navigation and search: methods run side by side. - **[Format plugins](./docs/format-plugins.md)** — further package formats, built into the server: design, operation, and writing one. -- **[Importing from Nexus](./docs/nexus-import.md)** — copy a Nexus hosted - repository (NuGet, Docker) into Forge, with its publish dates: - dry run, conflicts, resuming, and the cutover. +- **[Importing repositories](./docs/import.md)** — copy a hosted repository + of Nexus, Artifactory, a registry or another Forge (NuGet, Docker) into + Forge through the format's own protocol: dry run, conflicts, resuming, and + the cutover. Run `cargo xtask help` for the complete command list. diff --git a/crates/core/src/repo/hosted.rs b/crates/core/src/repo/hosted.rs index 3488023d..12e3c6e0 100644 --- a/crates/core/src/repo/hosted.rs +++ b/crates/core/src/repo/hosted.rs @@ -742,10 +742,10 @@ impl HostedWrite { }) } - /// The SHA-256 of the package file of `(id, version)` in the hosted - /// repository `repository_id`, if the version exists: what an import - /// compares the source's checksum with before downloading anything. - pub async fn package_sha256( + /// The SHA-512 (base64) of the package file of `id` `version`, if the + /// repository holds that version and the digest was recorded: an import + /// compares it with what the source lists. + pub async fn package_sha512( &self, repository_id: i64, id: &str, @@ -761,7 +761,12 @@ impl HostedWrite { else { return Ok(None); }; - self.stored_package_sha256(&self.pool, comp.id).await + Ok( + asset_dao::find_asset(&self.pool, comp.id, self.format.package_asset_path()) + .await + .map_err(RepoError::from)? + .and_then(|asset| asset.sha512_base64), + ) } /// The SHA-256 of a component's package file. diff --git a/crates/core/src/repo/mod.rs b/crates/core/src/repo/mod.rs index 510419ad..66010398 100644 --- a/crates/core/src/repo/mod.rs +++ b/crates/core/src/repo/mod.rs @@ -212,7 +212,7 @@ pub struct ImportOptions { pub downloaded_at: Option>, /// Who published it at the source, for the audit trail. pub uploader: Option, - /// Where it came from, for the audit trail (` `). + /// Where it came from, for the audit trail (the source's URL). pub source: String, } diff --git a/crates/db/src/import_dao.rs b/crates/db/src/import_dao.rs index cc0ba43a..bd0b1e5c 100644 --- a/crates/db/src/import_dao.rs +++ b/crates/db/src/import_dao.rs @@ -1,5 +1,6 @@ -//! Data access for imports from Nexus (`nexus_import`, `nexus_import_item`): -//! an import's job row, its scan cursor and lease, and the items it found. +//! Data access for imports into hosted repositories (`repository_import`, +//! `repository_import_item`): an import's job row, its scan cursor and lease, +//! and the items it found. //! //! An import runs on one instance at a time: the worker claims a queued (or //! abandoned) import with `FOR UPDATE SKIP LOCKED` and holds it with a @@ -8,7 +9,7 @@ //! on its `run_id`, so an instance whose lease expired cannot overwrite the //! progress of the one that took over. //! -//! The Nexus password is not read or written here: it is a secret column of +//! The source's password is not read or written here: it is a secret column of //! [`crate::secret_dao`] ([`crate::secret_dao::set_import_password`]). use std::collections::HashMap; @@ -21,12 +22,20 @@ use uuid::Uuid; /// An import job, without its password. #[derive(Debug, Clone, Serialize)] -pub struct NexusImport { +pub struct RepositoryImport { pub id: i64, pub repository_id: i64, - pub base_url: String, - pub source_repository: String, + /// The source: a NuGet service index or a registry. For an import stored + /// before 2.2, the Nexus server that `source_repository` names a + /// repository of. + pub url: String, + /// The Nexus repository of an import stored before 2.2; `None` since. + pub source_repository: Option, + /// The registry API of a Docker import stored before 2.2. pub registry_url: Option, + /// The package ids or images the import copies; `None`: whatever the + /// source's search (or catalog) finds. + pub names: Option>, pub username: Option, /// Whether a password is stored. pub has_password: bool, @@ -38,14 +47,16 @@ pub struct NexusImport { pub state: String, /// `scan` or `copy`. pub phase: String, - /// `components` or `assets`. + /// What the scan walks: `search`, `catalog` or `names`. pub scan_listing: String, #[serde(skip)] pub cursor: Option, pub cancel_requested: bool, - pub assets_seen: i64, - pub bytes_seen: i64, - pub nexus_version: Option, + /// The `Server` header the source answered the preflight with. + pub source_server: Option, + /// How many packages the feed's search found: what it reported in the + /// preflight, or as many as the scan walked, whichever is more. + pub source_total: Option, pub last_error: Option, pub created_by: Option, pub created_at: DateTime, @@ -54,7 +65,7 @@ pub struct NexusImport { pub updated_at: DateTime, } -impl NexusImport { +impl RepositoryImport { /// Whether the import waits for, or is held by, a worker. pub fn is_active(&self) -> bool { matches!(self.state.as_str(), "queued" | "running") @@ -64,14 +75,16 @@ impl NexusImport { /// A new import. pub struct NewImport<'a> { pub repository_id: i64, - pub base_url: &'a str, - pub source_repository: &'a str, - pub registry_url: Option<&'a str>, + pub url: &'a str, + pub names: Option<&'a [String]>, pub username: Option<&'a str>, pub on_conflict: &'a str, pub concurrency: i32, pub dry_run: bool, - pub nexus_version: Option<&'a str>, + /// Where the scan starts: `search`, `catalog` or `names`. + pub scan_listing: &'a str, + pub source_server: Option<&'a str>, + pub source_total: Option, pub created_by: Option, } @@ -83,21 +96,22 @@ where { sqlx::query_scalar!( r#" - INSERT INTO nexus_import - (repository_id, base_url, source_repository, registry_url, username, - on_conflict, concurrency, dry_run, nexus_version, created_by) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + INSERT INTO repository_import + (repository_id, url, names, username, on_conflict, concurrency, dry_run, + scan_listing, source_server, source_total, created_by) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) RETURNING id "#, new.repository_id, - new.base_url, - new.source_repository, - new.registry_url, + new.url, + new.names, new.username, new.on_conflict, new.concurrency, new.dry_run, - new.nexus_version, + new.scan_listing, + new.source_server, + new.source_total, new.created_by, ) .fetch_one(executor) @@ -105,19 +119,19 @@ where } /// The import `id`. -pub async fn get<'e, E>(executor: E, id: i64) -> sqlx::Result> +pub async fn get<'e, E>(executor: E, id: i64) -> sqlx::Result> where E: PgExecutor<'e>, { sqlx::query_as!( - NexusImport, + RepositoryImport, r#" - SELECT id, repository_id, base_url, source_repository, registry_url, username, + SELECT id, repository_id, url, source_repository, registry_url, names, username, (password_enc IS NOT NULL) AS "has_password!", on_conflict, concurrency, dry_run, state, phase, scan_listing, cursor, - cancel_requested, assets_seen, bytes_seen, nexus_version, last_error, - created_by, created_at, started_at, finished_at, updated_at - FROM nexus_import + cancel_requested, source_server, source_total, + last_error, created_by, created_at, started_at, finished_at, updated_at + FROM repository_import WHERE id = $1 "#, id, @@ -127,16 +141,20 @@ where } /// A repository's imports, newest first. -pub async fn list(pool: &PgPool, repository_id: i64, limit: i64) -> sqlx::Result> { +pub async fn list( + pool: &PgPool, + repository_id: i64, + limit: i64, +) -> sqlx::Result> { sqlx::query_as!( - NexusImport, + RepositoryImport, r#" - SELECT id, repository_id, base_url, source_repository, registry_url, username, + SELECT id, repository_id, url, source_repository, registry_url, names, username, (password_enc IS NOT NULL) AS "has_password!", on_conflict, concurrency, dry_run, state, phase, scan_listing, cursor, - cancel_requested, assets_seen, bytes_seen, nexus_version, last_error, - created_by, created_at, started_at, finished_at, updated_at - FROM nexus_import + cancel_requested, source_server, source_total, + last_error, created_by, created_at, started_at, finished_at, updated_at + FROM repository_import WHERE repository_id = $1 ORDER BY id DESC LIMIT $2 @@ -153,7 +171,7 @@ pub async fn list(pool: &PgPool, repository_id: i64, limit: i64) -> sqlx::Result pub struct StateCount { /// Items in the state. pub items: i64, - /// Their size in Nexus. + /// Their size at the source, where it says. pub size: i64, /// Bytes copied for them. pub bytes_copied: i64, @@ -167,7 +185,7 @@ pub async fn counts(pool: &PgPool, import_id: i64) -> sqlx::Result, now: DateTime, lease: Duration, -) -> sqlx::Result> { +) -> sqlx::Result> { let run_id = Uuid::new_v4(); let id = sqlx::query_scalar!( r#" WITH due AS ( - SELECT id FROM nexus_import + SELECT id FROM repository_import WHERE state IN ('queued', 'running') AND (locked_until IS NULL OR locked_until < $1) ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1 ) - UPDATE nexus_import i + UPDATE repository_import i SET state = 'running', run_id = $2, locked_until = $1 + make_interval(secs => $3), @@ -242,7 +260,7 @@ pub async fn renew_lease( ) -> sqlx::Result { let res = sqlx::query!( r#" - UPDATE nexus_import SET locked_until = $3 + UPDATE repository_import SET locked_until = $3 WHERE id = $1 AND run_id = $2 AND state = 'running' "#, id, @@ -259,7 +277,7 @@ pub async fn renew_lease( pub async fn poll(pool: &PgPool, id: i64, run_id: Uuid) -> sqlx::Result> { sqlx::query_scalar!( r#" - SELECT cancel_requested FROM nexus_import + SELECT cancel_requested FROM repository_import WHERE id = $1 AND run_id = $2 AND state = 'running' "#, id, @@ -269,35 +287,32 @@ pub async fn poll(pool: &PgPool, id: i64, run_id: Uuid) -> sqlx::Result, id: i64, run_id: Uuid, - listing: &str, cursor: Option<&str>, phase: &str, - assets: i64, - bytes: i64, + walked: Option, ) -> sqlx::Result { let res = sqlx::query!( r#" - UPDATE nexus_import - SET scan_listing = $3, cursor = $4, phase = $5, - assets_seen = assets_seen + $6, bytes_seen = bytes_seen + $7, + UPDATE repository_import + SET cursor = $3, phase = $4, + source_total = CASE WHEN $5::bigint IS NULL THEN source_total + ELSE greatest(coalesce(source_total, 0), $5) END, updated_at = now() WHERE id = $1 AND run_id = $2 AND state = 'running' "#, id, run_id, - listing, cursor, phase, - assets, - bytes, + walked, ) .execute(&mut **tx) .await?; @@ -317,7 +332,7 @@ pub async fn finish( ) -> sqlx::Result { let res = sqlx::query!( r#" - UPDATE nexus_import + UPDATE repository_import SET state = $3, last_error = $4, finished_at = now(), @@ -342,10 +357,10 @@ pub async fn finish( /// Cancel an import: a queued one at once, a running one at its worker's /// next check. Returns the import as it is now, `None` if there is none. -pub async fn cancel(pool: &PgPool, id: i64) -> sqlx::Result> { +pub async fn cancel(pool: &PgPool, id: i64) -> sqlx::Result> { sqlx::query!( r#" - UPDATE nexus_import + UPDATE repository_import SET state = CASE WHEN state = 'queued' THEN 'cancelled' ELSE state END, finished_at = CASE WHEN state = 'queued' THEN now() ELSE finished_at END, cancel_requested = (state = 'running'), @@ -372,12 +387,12 @@ pub async fn requeue( ) -> sqlx::Result> { let queued = sqlx::query!( r#" - UPDATE nexus_import i + UPDATE repository_import i SET state = 'queued', run_id = NULL, locked_until = NULL, finished_at = NULL, last_error = NULL, cancel_requested = false, updated_at = now() WHERE id = $1 AND state IN ('cancelled', 'failed', 'completed') AND NOT EXISTS ( - SELECT 1 FROM nexus_import o + SELECT 1 FROM repository_import o WHERE o.repository_id = i.repository_id AND o.id <> i.id AND o.state IN ('queued', 'running') ) @@ -394,7 +409,7 @@ pub async fn requeue( } let retried = sqlx::query!( r#" - UPDATE nexus_import_item + UPDATE repository_import_item SET state = 'pending', detail = NULL, updated_at = now() WHERE import_id = $1 AND state = 'failed' AND ($2::bigint[] IS NULL OR id = ANY($2)) @@ -411,7 +426,7 @@ pub async fn requeue( /// active or gone. pub async fn delete(pool: &PgPool, id: i64) -> sqlx::Result { let res = sqlx::query!( - r#"DELETE FROM nexus_import WHERE id = $1 AND state NOT IN ('queued', 'running')"#, + r#"DELETE FROM repository_import WHERE id = $1 AND state NOT IN ('queued', 'running')"#, id, ) .execute(pool) @@ -466,7 +481,7 @@ pub async fn insert_items( for item in items { sqlx::query!( r#" - INSERT INTO nexus_import_item + INSERT INTO repository_import_item (import_id, kind, name, version, size, published_at, downloaded_at, uploader, source) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) @@ -502,7 +517,7 @@ pub async fn pending_items( r#" SELECT id, import_id, kind, name, version, state, detail, size, bytes_copied, published_at, downloaded_at, uploader, source, attempts, updated_at - FROM nexus_import_item + FROM repository_import_item WHERE import_id = $1 AND kind = $2 AND state = 'pending' AND id > $3 ORDER BY id LIMIT $4 @@ -526,7 +541,7 @@ pub async fn finish_item( ) -> sqlx::Result<()> { sqlx::query!( r#" - UPDATE nexus_import_item + UPDATE repository_import_item SET state = $2, detail = $3, bytes_copied = bytes_copied + $4, attempts = attempts + 1, updated_at = now() WHERE id = $1 @@ -541,10 +556,10 @@ pub async fn finish_item( Ok(()) } -/// Forget an item: a manifest Nexus holds by digest that turned out not to -/// be a referrer of an imported image. +/// Forget an item: a manifest the source holds by digest that turned out not +/// to be a referrer of an imported image. pub async fn drop_item(pool: &PgPool, item_id: i64) -> sqlx::Result<()> { - sqlx::query!("DELETE FROM nexus_import_item WHERE id = $1", item_id) + sqlx::query!("DELETE FROM repository_import_item WHERE id = $1", item_id) .execute(pool) .await?; Ok(()) @@ -564,7 +579,7 @@ pub async fn list_items( r#" SELECT id, import_id, kind, name, version, state, detail, size, bytes_copied, published_at, downloaded_at, uploader, source, attempts, updated_at - FROM nexus_import_item + FROM repository_import_item WHERE import_id = $1 AND ($2::text IS NULL OR state = $2) AND id > $3 ORDER BY id LIMIT $4 diff --git a/crates/db/src/secret_dao.rs b/crates/db/src/secret_dao.rs index 78be88eb..6a157ae4 100644 --- a/crates/db/src/secret_dao.rs +++ b/crates/db/src/secret_dao.rs @@ -37,9 +37,10 @@ //! stored `config` //! ([`store_email_settings`], [`email_config`]). //! -//! An import from Nexus keeps the Nexus password in -//! `nexus_import.password_enc` until it completes, AAD-bound to -//! `('nexus_import', 'password_enc', )` plus the origins it is sent +//! An import keeps its source's password in +//! `repository_import.password_enc` until it completes, AAD-bound to +//! `('nexus_import', 'password_enc', )` (the table's name before 2.2, +//! kept so that passwords sealed then still open) plus the origins it is sent //! to ([`import_destination`]) ([`set_import_password`], //! [`get_import_password`]). //! @@ -434,7 +435,7 @@ pub async fn reseal_upstream_passwords_batch( /// Every DEK version still referenced by stored ciphertext, across all /// encrypted columns (`repository.upstream_password_enc`, -/// `auth_realm.secrets_enc`, `nexus_import.password_enc`, +/// `auth_realm.secrets_enc`, `repository_import.password_enc`, /// `email_settings.secrets_enc`, the sealed /// [`SECRET_SETTINGS`]; future secret columns add their scan here). Used /// by the sweep to decide which `retiring` DEKs are safe to revoke. @@ -499,7 +500,7 @@ async fn scan_referenced_dek_versions( } } let import_blobs = sqlx::query!( - r#"SELECT id, password_enc AS "password_enc!" FROM nexus_import WHERE password_enc IS NOT NULL"#, + r#"SELECT id, password_enc AS "password_enc!" FROM repository_import WHERE password_enc IS NOT NULL"#, ) .fetch_all(&mut *conn) .await?; @@ -1236,24 +1237,27 @@ pub async fn reseal_secret_settings( Ok(resealed) } -// ── Nexus import passwords ─────────────────────────────────────────────────── +// ── Import passwords ───────────────────────────────────────────────────────── -/// Owning table of an import's sealed Nexus password (AAD component). +/// Owning table of an import's sealed password (AAD component): the table's +/// name before 2.2 (`repository_import` since), kept so that a password +/// sealed then still opens. pub const IMPORT_PASSWORD_TABLE: &str = "nexus_import"; -/// Owning column of an import's sealed Nexus password (AAD component). +/// Owning column of an import's sealed password (AAD component). pub const IMPORT_PASSWORD_COLUMN: &str = "password_enc"; -/// Where an import's Nexus password may be sent: the origin of its -/// `base_url` and, when it is elsewhere, of its Docker `registry_url`. -pub fn import_destination(base_url: &str, registry_url: Option<&str>) -> String { - let base = upstream_destination(Some(base_url)); +/// Where an import's password may be sent: the origin of its `url` (the +/// source) and, for a Docker import stored before 2.2 whose registry is +/// elsewhere, of its `registry_url`. +pub fn import_destination(url: &str, registry_url: Option<&str>) -> String { + let base = upstream_destination(Some(url)); match registry_url.map(|url| upstream_destination(Some(url))) { Some(registry) if registry != base => format!("{base} {registry}"), _ => base, } } -/// The AAD binding of `nexus_import.password_enc` of one row. +/// The AAD binding of `repository_import.password_enc` of one row. fn import_binding(import_id: i64, destination: &str) -> AeadBinding<'_> { AeadBinding { table: IMPORT_PASSWORD_TABLE, @@ -1263,7 +1267,7 @@ fn import_binding(import_id: i64, destination: &str) -> AeadBinding<'_> { } } -/// Store an import's Nexus password, sealed (bound to the row and to +/// Store an import's password, sealed (bound to the row and to /// [`import_destination`]). Runs on the connection of the transaction that /// created the import. pub async fn set_import_password( @@ -1273,19 +1277,19 @@ pub async fn set_import_password( secret: &PlaintextSecret, ) -> Result<(), SecretDbError> { let row = sqlx::query!( - r#"SELECT base_url, registry_url FROM nexus_import WHERE id = $1 FOR UPDATE"#, + r#"SELECT url, registry_url FROM repository_import WHERE id = $1 FOR UPDATE"#, import_id, ) .fetch_optional(&mut *conn) .await? .ok_or(SecretDbError::ImportNotFound(import_id))?; - let destination = import_destination(&row.base_url, row.registry_url.as_deref()); + let destination = import_destination(&row.url, row.registry_url.as_deref()); let blob = envelope .seal(secret, import_binding(import_id, &destination)) .await?; lock_sealing_dek(conn, envelope, &blob).await?; sqlx::query!( - r#"UPDATE nexus_import SET password_enc = $2 WHERE id = $1"#, + r#"UPDATE repository_import SET password_enc = $2 WHERE id = $1"#, import_id, &blob, ) @@ -1294,7 +1298,7 @@ pub async fn set_import_password( Ok(()) } -/// An import's Nexus password, `None` when it has none (an anonymous +/// An import's password, `None` when it has none (an anonymous /// source, or the import completed). pub async fn get_import_password( pool: &PgPool, @@ -1303,8 +1307,8 @@ pub async fn get_import_password( ) -> Result, SecretDbError> { let row = sqlx::query!( r#" - SELECT base_url, registry_url, password_enc - FROM nexus_import WHERE id = $1 + SELECT url, registry_url, password_enc + FROM repository_import WHERE id = $1 "#, import_id, ) @@ -1314,7 +1318,7 @@ pub async fn get_import_password( let Some(blob) = row.password_enc else { return Ok(None); }; - let destination = import_destination(&row.base_url, row.registry_url.as_deref()); + let destination = import_destination(&row.url, row.registry_url.as_deref()); let secret = envelope .open(&blob, import_binding(import_id, &destination)) .await?; @@ -1330,8 +1334,8 @@ pub async fn reseal_import_passwords( ) -> Result { let rows = sqlx::query!( r#" - SELECT id, base_url, registry_url, password_enc AS "password_enc!" - FROM nexus_import WHERE password_enc IS NOT NULL + SELECT id, url, registry_url, password_enc AS "password_enc!" + FROM repository_import WHERE password_enc IS NOT NULL ORDER BY id "#, ) @@ -1339,7 +1343,7 @@ pub async fn reseal_import_passwords( .await?; let mut resealed = 0; for row in rows { - let destination = import_destination(&row.base_url, row.registry_url.as_deref()); + let destination = import_destination(&row.url, row.registry_url.as_deref()); let bound = import_binding(row.id, &destination); let Some(new_blob) = reseal_if_due(envelope, &row.password_enc, bound).await? else { continue; @@ -1353,7 +1357,7 @@ pub async fn reseal_import_passwords( WHERE layer = 'dek' AND scope = $4 AND version = $5 AND state <> 'revoked' FOR SHARE ) - UPDATE nexus_import SET password_enc = $2 + UPDATE repository_import SET password_enc = $2 WHERE id = $1 AND password_enc = $3 AND EXISTS (SELECT 1 FROM dek) "#, row.id, diff --git a/crates/db/tests/migrations.rs b/crates/db/tests/migrations.rs index e7c60818..30a0bd96 100644 --- a/crates/db/tests/migrations.rs +++ b/crates/db/tests/migrations.rs @@ -1,9 +1,19 @@ //! Integration test: spin up a real Postgres in a container, run the migrations, -//! and assert every expected table exists; and refuse a database an earlier -//! build created. +//! and assert every expected table exists; refuse a database an earlier +//! build created; and carry an earlier migration's rows through a later one. //! //! Requires Docker to be available on the host running the test. +use std::sync::Arc; + +use forge_crypto::envelope::{AeadBinding, Algorithm}; +use forge_crypto::kms::StaticRootProvider; +use forge_crypto::service::{EnvelopeService, EnvelopeServiceConfig}; +use forge_crypto::{ExposeSecret, KeyBytes, PlaintextSecret}; +use forge_db::encryption_key_dao::PgKeyStore; +use forge_db::import_dao::{self, NewImport}; +use forge_db::repository_dao::{self, NewRepository}; +use forge_db::secret_dao; use sqlx::Row; use testcontainers_modules::{ postgres::Postgres, @@ -781,6 +791,169 @@ async fn a_database_of_an_earlier_build_is_refused() { } } +/// An import stored before imports read the formats' protocols (0007): its +/// rows, its items and its sealed password come through the rename, and a +/// scan that walked Nexus' REST API starts over. +#[tokio::test] +async fn an_import_of_a_nexus_repository_survives_the_rename() { + let (server, _container) = server().await; + let url = fresh_database(&server, "import_rename").await; + let pool = forge_db::connect(&url).await.expect("connect"); + sqlx::migrate!("../../migrations") + .run_to(6, &pool) + .await + .expect("migrate to 0006"); + + let nuget = repository_dao::insert_repository(&pool, NewRepository::hosted("nuget-internal")) + .await + .unwrap(); + let images = repository_dao::insert_repository( + &pool, + NewRepository { + format: "docker".into(), + ..NewRepository::hosted("images") + }, + ) + .await + .unwrap(); + let insert = |repository_id: i64, + repository: &'static str, + registry: Option<&'static str>, + phase: &'static str, + listing: &'static str| { + let pool = pool.clone(); + async move { + sqlx::query_scalar::<_, i64>( + "INSERT INTO nexus_import (repository_id, base_url, source_repository, registry_url, \ + username, state, phase, scan_listing, cursor, nexus_version) \ + VALUES ($1, 'https://nexus.example.com', $2, $3, 'migrator', 'failed', $4, $5, \ + 'token-1', 'Nexus/3.70.1-02 (OSS)') RETURNING id", + ) + .bind(repository_id) + .bind(repository) + .bind(registry) + .bind(phase) + .bind(listing) + .fetch_one(&pool) + .await + .expect("insert an import") + } + }; + let scanning = insert(nuget.id, "nuget-hosted", None, "scan", "components").await; + let copying = insert( + images.id, + "docker-hosted", + Some("https://nexus.example.com:8082"), + "copy", + "assets", + ) + .await; + sqlx::query( + "INSERT INTO nexus_import_item (import_id, kind, name, version, source) \ + VALUES ($1, 'component', 'app', '1.0', '{\"assets\": []}')", + ) + .bind(copying) + .execute(&pool) + .await + .expect("insert an item"); + // The password, sealed as before the rename. + let envelope = EnvelopeService::new( + Arc::new(PgKeyStore::new(pool.clone())), + Arc::new(StaticRootProvider::new( + "env", + Algorithm::Aes256Gcm, + KeyBytes::generate(), + vec![], + )), + EnvelopeServiceConfig::default(), + ); + let destination = secret_dao::import_destination( + "https://nexus.example.com", + Some("https://nexus.example.com:8082"), + ); + let sealed = envelope + .seal( + &PlaintextSecret::from("nexus-secret".to_string()), + AeadBinding { + table: "nexus_import", + column: "password_enc", + row_pk: copying, + destination: Some(&destination), + }, + ) + .await + .expect("seal"); + sqlx::query("UPDATE nexus_import SET password_enc = $2 WHERE id = $1") + .bind(copying) + .bind(&sealed) + .execute(&pool) + .await + .expect("store the password"); + + forge_db::migrate(&pool).await.expect("migrate the rest"); + + let scanned = import_dao::get(&pool, scanning).await.unwrap().unwrap(); + assert_eq!(scanned.url, "https://nexus.example.com"); + assert_eq!(scanned.source_repository.as_deref(), Some("nuget-hosted")); + assert_eq!( + scanned.source_server.as_deref(), + Some("Nexus/3.70.1-02 (OSS)") + ); + assert_eq!( + (scanned.scan_listing.as_str(), scanned.cursor.as_deref()), + ("search", None), + "the scan starts over" + ); + assert_eq!((scanned.names, scanned.source_total), (None, None)); + let copied = import_dao::get(&pool, copying).await.unwrap().unwrap(); + assert_eq!(copied.scan_listing, "catalog"); + assert_eq!(copied.phase, "copy"); + assert_eq!( + copied.registry_url.as_deref(), + Some("https://nexus.example.com:8082") + ); + let items = import_dao::list_items(&pool, copying, None, 0, 10) + .await + .unwrap(); + assert_eq!(items.len(), 1); + assert_eq!(items[0].source, serde_json::json!({"assets": []})); + let password = secret_dao::get_import_password(&pool, &envelope, copying) + .await + .expect("the password opens") + .expect("a password"); + assert_eq!(password.expose_secret(), "nexus-secret"); + + // One active import per repository, as before. + let queued = || NewImport { + repository_id: nuget.id, + url: "https://feed.example.com/v3/index.json", + names: None, + username: None, + on_conflict: "skip", + concurrency: 4, + dry_run: false, + scan_listing: "search", + source_server: None, + source_total: Some(3), + created_by: None, + }; + import_dao::insert(&pool, queued()) + .await + .expect("a queued import"); + assert!(import_dao::insert(&pool, queued()).await.is_err()); + let leftovers: i64 = sqlx::query_scalar( + "SELECT (SELECT count(*) FROM pg_constraint WHERE conname LIKE 'nexus%') \ + + (SELECT count(*) FROM pg_class WHERE relname LIKE 'nexus%')", + ) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + leftovers, 0, + "no constraint, index or sequence keeps the old name" + ); +} + /// A Postgres server for one test: a throwaway Postgres 18 container matching /// docker-compose (db/user/pw = forge), or, with FORGE_TEST_DATABASE_URL (no /// Docker), that server. Returns the URL of a database on it, and the diff --git a/crates/e2e/tests/import.rs b/crates/e2e/tests/import.rs new file mode 100644 index 00000000..23db1c3e --- /dev/null +++ b/crates/e2e/tests/import.rs @@ -0,0 +1,784 @@ +//! E2E: imports through the formats' own protocols, from real sources: +//! +//! * a Nexus Repository 3 OSS container (`sonatype/nexus3`): NuGet through +//! its hosted repository's v3 feed (`/repository//index.json`), and +//! Docker through its connector's catalog, holding more than 100 images, +//! with a cosign signature and an OCI 1.1 referrer pushed the way +//! `oras attach` pushes one to a registry without the referrers API (the +//! referrers tag schema); +//! * a `distribution` registry (`registry:2`): Docker through its catalog; +//! * another Forge: NuGet from a Forge NuGet repository, whose registration +//! lists each package's SHA-512, so a second run downloads nothing. +//! +//! After an import, bytes and digests are the source's and so are the NuGet +//! publish dates; a second run copies only what was added since. (Resuming +//! a killed import, the conflict policies, the dry run and a source without +//! a search or catalog are covered against a mock source in +//! `crates/server/tests/import.rs`.) Gated behind `forge-e2e`; Nexus takes a +//! minute or two to start. + +#![cfg(feature = "forge-e2e")] + +use std::time::Duration; + +use forge_e2e::{make_nupkg, StackBuilder, TestServer}; +use forge_server::import::ImportWorker; +use serde_json::{json, Value}; +use sha2::Digest; +use testcontainers::core::{IntoContainerPort, WaitFor}; +use testcontainers::runners::AsyncRunner; +use testcontainers::{GenericImage, ImageExt}; + +const NEXUS_IMAGE: &str = "mirror.gcr.io/sonatype/nexus3"; +const NEXUS_TAG: &str = "3.70.1"; +/// With `NEXUS_SECURITY_RANDOMPASSWORD=false`, Nexus' admin password. +const ADMIN: (&str, &str) = ("admin", "admin123"); + +const REGISTRY_IMAGE: &str = "mirror.gcr.io/library/registry"; +const REGISTRY_TAG: &str = "2"; + +/// More images than a catalog page holds (Nexus was reported to stop at +/// 100). +const BULK_IMAGES: usize = 105; + +const OCI_MANIFEST: &str = "application/vnd.oci.image.manifest.v1+json"; +const OCI_INDEX: &str = "application/vnd.oci.image.index.v1+json"; +const OCI_CONFIG: &str = "application/vnd.oci.image.config.v1+json"; +const OCI_EMPTY: &str = "application/vnd.oci.empty.v1+json"; +const LAYER: &str = "application/vnd.oci.image.layer.v1.tar+gzip"; + +fn sha256(bytes: &[u8]) -> String { + hex::encode(sha2::Sha256::digest(bytes)) +} + +fn digest(bytes: &[u8]) -> String { + format!("sha256:{}", sha256(bytes)) +} + +// ── Sources ───────────────────────────────────────────────────────────────── + +/// A Nexus with its REST API at `url` and its Docker connector at +/// `registry`. +struct Nexus { + url: String, + registry: Registry, + client: reqwest::Client, +} + +impl Nexus { + async fn start() -> (Self, testcontainers::ContainerAsync) { + let container = GenericImage::new(NEXUS_IMAGE, NEXUS_TAG) + .with_exposed_port(8081.tcp()) + .with_exposed_port(8082.tcp()) + .with_wait_for(WaitFor::message_on_stdout("Started Sonatype Nexus")) + .with_env_var("NEXUS_SECURITY_RANDOMPASSWORD", "false") + .with_env_var( + "INSTALL4J_ADD_VM_PARAMS", + "-Xms512m -Xmx1g -XX:MaxDirectMemorySize=1g", + ) + .with_startup_timeout(Duration::from_secs(600)) + .start() + .await + .expect("start nexus (is Docker running?)"); + let host = container.get_host().await.unwrap(); + let api = container.get_host_port_ipv4(8081).await.unwrap(); + let registry = container.get_host_port_ipv4(8082).await.unwrap(); + let client = reqwest::Client::new(); + let nexus = Nexus { + url: format!("http://{host}:{api}"), + registry: Registry { + url: format!("http://{host}:{registry}"), + auth: Some(ADMIN), + client: client.clone(), + }, + client, + }; + for _ in 0..300 { + let up = nexus + .client + .get(format!("{}/service/rest/v1/status/writable", nexus.url)) + .send() + .await + .is_ok_and(|r| r.status() == 200); + if up { + return (nexus, container); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + panic!("nexus never became writable"); + } + + fn api(&self, method: reqwest::Method, path: &str) -> reqwest::RequestBuilder { + self.client + .request(method, format!("{}/service/rest/v1/{path}", self.url)) + .basic_auth(ADMIN.0, Some(ADMIN.1)) + } + + async fn seed_repositories(&self) { + let resp = self + .api(reqwest::Method::PUT, "security/realms/active") + .json(&json!(["NexusAuthenticatingRealm", "DockerToken"])) + .send() + .await + .unwrap(); + assert!(resp.status().is_success(), "realms: {}", resp.status()); + let docker = json!({ + "name": "docker-hosted", "online": true, + "storage": {"blobStoreName": "default", "strictContentTypeValidation": true, "writePolicy": "allow"}, + "docker": {"v1Enabled": false, "forceBasicAuth": true, "httpPort": 8082}, + }); + let resp = self + .api(reqwest::Method::POST, "repositories/docker/hosted") + .json(&docker) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); + } + + /// Upload an asset through the components API (`.asset`). + async fn upload(&self, repository: &str, format: &str, file: &str, bytes: Vec) { + let form = reqwest::multipart::Form::new().part( + format!("{format}.asset"), + reqwest::multipart::Part::bytes(bytes).file_name(file.to_string()), + ); + let resp = self + .api( + reqwest::Method::POST, + &format!("components?repository={repository}"), + ) + .multipart(form) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 204, + "upload {file}: {}", + resp.text().await.unwrap() + ); + } + + /// The service index of the NuGet repository `name`. + fn feed(&self, name: &str) -> String { + format!("{}/repository/{name}/index.json", self.url) + } + + /// GET a document of the NuGet repository `name`'s v3 feed. + async fn feed_json(&self, name: &str, path: &str) -> Value { + self.client + .get(format!("{}/repository/{name}/{path}", self.url)) + .basic_auth(ADMIN.0, Some(ADMIN.1)) + .send() + .await + .unwrap() + .json() + .await + .unwrap() + } + + /// The URL of the resource of `kind` the feed `name` lists. + async fn resource(&self, name: &str, kind: &str) -> String { + let index = self.feed_json(name, "index.json").await; + index["resources"] + .as_array() + .unwrap() + .iter() + .find(|r| r["@type"].as_str().is_some_and(|t| t.starts_with(kind))) + .unwrap_or_else(|| panic!("{kind} in {index}"))["@id"] + .as_str() + .unwrap() + .to_string() + } + + /// When the feed's registration says `lower` `version` was published. + async fn published( + &self, + name: &str, + lower: &str, + version: &str, + ) -> chrono::DateTime { + let base = self.resource(name, "RegistrationsBaseUrl").await; + let url = format!("{}/{lower}/index.json", base.trim_end_matches('/')); + let registration: Value = self + .client + .get(&url) + .basic_auth(ADMIN.0, Some(ADMIN.1)) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + let leaves = registration["items"] + .as_array() + .unwrap() + .iter() + .flat_map(|page| page["items"].as_array().cloned().unwrap_or_default()); + for leaf in leaves { + let entry = &leaf["catalogEntry"]; + if entry["version"].as_str() == Some(version) { + let published = entry["published"].as_str().expect("published"); + return chrono::DateTime::parse_from_rfc3339(published).unwrap(); + } + } + panic!("{version} not in {registration}"); + } + + /// Wait until the feed's search finds `n` packages: Nexus indexes an + /// upload for search a moment after it stored it. + async fn searchable(&self, name: &str, n: u64) { + let search = self.resource(name, "SearchQueryService").await; + for _ in 0..120 { + let found: Value = self + .client + .get(format!( + "{search}?q=&take=100&prerelease=true&semVerLevel=2.0.0" + )) + .basic_auth(ADMIN.0, Some(ADMIN.1)) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + if found["totalHits"].as_u64() >= Some(n) { + return; + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + panic!("Nexus' search never found {n} packages in {name}"); + } +} + +/// A registry API to push to. +struct Registry { + url: String, + auth: Option<(&'static str, &'static str)>, + client: reqwest::Client, +} + +impl Registry { + fn request(&self, method: reqwest::Method, url: &str) -> reqwest::RequestBuilder { + let request = self.client.request(method, url); + match self.auth { + Some((user, password)) => request.basic_auth(user, Some(password)), + None => request, + } + } + + async fn push_blob(&self, image: &str, bytes: &[u8]) { + let resp = self + .request( + reqwest::Method::POST, + &format!("{}/v2/{image}/blobs/uploads/", self.url), + ) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 202, "start upload"); + let location = resp.headers()["location"].to_str().unwrap().to_string(); + let location = if location.starts_with("http") { + location + } else { + format!("{}{location}", self.url) + }; + let separator = if location.contains('?') { '&' } else { '?' }; + let resp = self + .request( + reqwest::Method::PUT, + &format!("{location}{separator}digest={}", digest(bytes)), + ) + .header("content-type", "application/octet-stream") + .body(bytes.to_vec()) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "finish upload"); + } + + async fn push_manifest(&self, image: &str, reference: &str, media_type: &str, bytes: &[u8]) { + let resp = self + .request( + reqwest::Method::PUT, + &format!("{}/v2/{image}/manifests/{reference}", self.url), + ) + .header("content-type", media_type) + .body(bytes.to_vec()) + .send() + .await + .unwrap(); + assert_eq!( + resp.status(), + 201, + "manifest {reference}: {}", + resp.text().await.unwrap() + ); + } + + /// A one-platform image of `image` with `layer`, pushed under `tag`. + async fn push_image(&self, image: &str, tag: &str, layer: &[u8]) -> Vec { + let config = json!({"architecture": "amd64", "os": "linux", "rootfs": {"type": "layers", "diff_ids": []}}) + .to_string() + .into_bytes(); + self.push_blob(image, &config).await; + self.push_blob(image, layer).await; + let manifest = json!({ + "schemaVersion": 2, "mediaType": OCI_MANIFEST, + "config": {"mediaType": OCI_CONFIG, "digest": digest(&config), "size": config.len()}, + "layers": [{"mediaType": LAYER, "digest": digest(layer), "size": layer.len()}], + }) + .to_string() + .into_bytes(); + self.push_manifest(image, tag, OCI_MANIFEST, &manifest) + .await; + manifest + } + + /// A two-platform image of `image` under `tag`, a cosign signature of + /// it (`sha256-.sig`) and an SBOM attached the way `oras attach` + /// attaches one to a registry without the referrers API: the referrer by + /// digest, and the index of the subject's referrers under the tag + /// `sha256-`. Returns the index, its children's digests, the + /// signature and the SBOM. + async fn push_signed_index( + &self, + image: &str, + tag: &str, + ) -> (Vec, Vec, Vec, Vec) { + let mut children = Vec::new(); + let mut descriptors = Vec::new(); + for arch in ["amd64", "arm64"] { + let config = json!({"architecture": arch, "os": "linux", "rootfs": {"type": "layers", "diff_ids": []}}) + .to_string() + .into_bytes(); + let layer = format!("the {arch} layer of {image}").into_bytes(); + self.push_blob(image, &config).await; + self.push_blob(image, &layer).await; + let manifest = json!({ + "schemaVersion": 2, "mediaType": OCI_MANIFEST, + "config": {"mediaType": OCI_CONFIG, "digest": digest(&config), "size": config.len()}, + "layers": [{"mediaType": LAYER, "digest": digest(&layer), "size": layer.len()}], + }) + .to_string() + .into_bytes(); + self.push_manifest(image, &digest(&manifest), OCI_MANIFEST, &manifest) + .await; + children.push(digest(&manifest)); + descriptors.push(json!({"mediaType": OCI_MANIFEST, "digest": digest(&manifest), "size": manifest.len(), "platform": {"architecture": arch, "os": "linux"}})); + } + let index = json!({"schemaVersion": 2, "mediaType": OCI_INDEX, "manifests": descriptors}) + .to_string() + .into_bytes(); + self.push_manifest(image, tag, OCI_INDEX, &index).await; + + let empty = b"{}".to_vec(); + self.push_blob(image, &empty).await; + let payload = format!("cosign signature payload of {image}").into_bytes(); + self.push_blob(image, &payload).await; + let signature = json!({ + "schemaVersion": 2, "mediaType": OCI_MANIFEST, + "config": {"mediaType": OCI_EMPTY, "digest": digest(&empty), "size": 2}, + "layers": [{"mediaType": "application/vnd.dev.cosign.simplesigning.v1+json", "digest": digest(&payload), "size": payload.len(), + "annotations": {"dev.cosignproject.cosign/signature": "MEUCIQ=="}}], + }) + .to_string() + .into_bytes(); + self.push_manifest( + image, + &format!("sha256-{}.sig", sha256(&index)), + OCI_MANIFEST, + &signature, + ) + .await; + + let sbom_body = + format!("{{\"spdxVersion\": \"SPDX-2.3\", \"name\": \"{image}\"}}").into_bytes(); + self.push_blob(image, &sbom_body).await; + let sbom = json!({ + "schemaVersion": 2, "mediaType": OCI_MANIFEST, "artifactType": "application/spdx+json", + "config": {"mediaType": OCI_EMPTY, "digest": digest(&empty), "size": 2}, + "layers": [{"mediaType": "application/spdx+json", "digest": digest(&sbom_body), "size": sbom_body.len()}], + "subject": {"mediaType": OCI_INDEX, "digest": digest(&index), "size": index.len()}, + }) + .to_string() + .into_bytes(); + self.push_manifest(image, &digest(&sbom), OCI_MANIFEST, &sbom) + .await; + let referrers = json!({"schemaVersion": 2, "mediaType": OCI_INDEX, "manifests": [ + {"mediaType": OCI_MANIFEST, "digest": digest(&sbom), "size": sbom.len(), "artifactType": "application/spdx+json"} + ]}) + .to_string() + .into_bytes(); + self.push_manifest( + image, + &format!("sha256-{}", sha256(&index)), + OCI_INDEX, + &referrers, + ) + .await; + (index, children, signature, sbom) + } +} + +// ── Forge ─────────────────────────────────────────────────────────────────── + +async fn admin( + srv: &TestServer, + method: reqwest::Method, + path: &str, + body: Option, +) -> reqwest::Response { + let (u, p) = srv.admin_basic(); + let mut req = srv + .client() + .request(method, format!("{}{path}", srv.base_url)) + .basic_auth(u, Some(p)); + if let Some(body) = body { + req = req.json(&body); + } + req.send().await.unwrap() +} + +/// Start an import into `repo` and run it to its end. +async fn import(srv: &TestServer, repo: &str, body: Value) -> Value { + let resp = admin( + srv, + reqwest::Method::POST, + &format!("/admin/repositories/{repo}/imports"), + Some(body), + ) + .await; + assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); + let id = resp.json::().await.unwrap()["id"].as_i64().unwrap(); + for _ in 0..1200 { + ImportWorker::new(srv.state.clone()).tick().await.unwrap(); + let view: Value = admin( + srv, + reqwest::Method::GET, + &format!("/admin/repositories/{repo}/imports/{id}"), + None, + ) + .await + .json() + .await + .unwrap(); + if !matches!(view["state"].as_str(), Some("queued" | "running")) { + return view; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + panic!("import into {repo} did not end"); +} + +async fn create_repo(srv: &TestServer, body: Value) { + let resp = admin( + srv, + reqwest::Method::POST, + "/admin/repositories", + Some(body), + ) + .await; + assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); +} + +/// The bytes of a Forge download, with the admin's credentials. +async fn download(srv: &TestServer, path: &str) -> Vec { + let (u, p) = srv.admin_basic(); + let resp = srv + .client() + .get(format!("{}{path}", srv.base_url)) + .basic_auth(u, Some(p)) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "GET {path}"); + resp.bytes().await.unwrap().to_vec() +} + +async fn repository_id(srv: &TestServer, repo: &str) -> i64 { + forge_db::repository_dao::find_repository_by_name(&srv.pool, repo) + .await + .unwrap() + .unwrap() + .id +} + +/// The image `image` of `repo` holds the signed index `index` with its +/// children, its signature tag, and the SBOM as a referrer. +async fn assert_signed_index( + srv: &TestServer, + repo: &str, + image: &str, + tag: &str, + (index, children, signature, sbom): &(Vec, Vec, Vec, Vec), +) { + let repo_id = repository_id(srv, repo).await; + let tagged = forge_db::oci_dao::find_tag(&srv.pool, repo_id, image, tag) + .await + .unwrap() + .unwrap_or_else(|| panic!("{image}:{tag}")); + assert_eq!(tagged.manifest.digest, digest(index)); + for child in children { + assert!( + forge_db::oci_dao::find_manifest(&srv.pool, repo_id, image, child) + .await + .unwrap() + .is_some(), + "child {child}" + ); + } + let sig = forge_db::oci_dao::find_tag( + &srv.pool, + repo_id, + image, + &format!("sha256-{}.sig", sha256(index)), + ) + .await + .unwrap() + .expect("the signature's tag"); + assert_eq!(sig.manifest.digest, digest(signature)); + let referrers: Vec = + forge_db::oci_dao::list_referrers(&srv.pool, repo_id, image, &digest(index)) + .await + .unwrap() + .into_iter() + .map(|m| m.digest) + .collect(); + assert_eq!(referrers, vec![digest(sbom)]); +} + +// ── Tests ─────────────────────────────────────────────────────────────────── + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn import_from_a_real_nexus() { + let (nexus, _container) = Nexus::start().await; + nexus.seed_repositories().await; + + // NuGet: two packages in the default nuget-hosted repository. + let core = make_nupkg("Acme.Core", "1.0.0"); + let web = make_nupkg("Acme.Web", "2.1.0"); + nexus + .upload( + "nuget-hosted", + "nuget", + "acme.core.1.0.0.nupkg", + core.clone(), + ) + .await; + nexus + .upload("nuget-hosted", "nuget", "acme.web.2.1.0.nupkg", web.clone()) + .await; + nexus.searchable("nuget-hosted", 2).await; + + // Docker: a signed two-platform image with an SBOM, and more images than + // a catalog page holds. + let signed = nexus.registry.push_signed_index("team/app", "1.0").await; + for i in 0..BULK_IMAGES { + nexus + .registry + .push_image( + &format!("bulk/img-{i:03}"), + "1.0", + format!("layer {i}").as_bytes(), + ) + .await; + } + + let srv = StackBuilder::default().start().await; + create_repo(&srv, json!({"name": "nuget-internal", "kind": "hosted"})).await; + create_repo( + &srv, + json!({"name": "images", "kind": "hosted", "format": "docker"}), + ) + .await; + + // NuGet: through the feed; the bytes and the publish dates are Nexus'. + let credentials = |url: String| json!({"url": url, "username": ADMIN.0, "password": ADMIN.1}); + let done = import( + &srv, + "nuget-internal", + credentials(nexus.feed("nuget-hosted")), + ) + .await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["source_total"], 2, "{done}"); + assert!( + done["source_server"] + .as_str() + .is_some_and(|s| s.starts_with("Nexus/")), + "{done}" + ); + assert_eq!(done["progress"]["imported"], 2, "{done}"); + assert_eq!( + download( + &srv, + "/repository/nuget-internal/flat/acme.core/1.0.0/acme.core.1.0.0.nupkg" + ) + .await, + core + ); + assert_eq!( + download( + &srv, + "/repository/nuget-internal/flat/acme.web/2.1.0/acme.web.2.1.0.nupkg" + ) + .await, + web + ); + let published = nexus.published("nuget-hosted", "acme.core", "1.0.0").await; + let created = sqlx::query_scalar!( + "SELECT c.created_at FROM component c JOIN repository r ON r.id = c.repository_id WHERE r.name = 'nuget-internal' AND c.name = 'acme.core'" + ) + .fetch_one(&srv.pool) + .await + .unwrap(); + assert_eq!(created, published); + + // Docker: through the connector's catalog, every image, the children, + // the signature tag and the SBOM. + let done = import(&srv, "images", credentials(nexus.registry.url.clone())).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["failed"], 0, "{done}"); + // Three tags of team/app (1.0, the signature, the referrers tag) and one + // of each bulk image. + assert_eq!( + done["progress"]["imported"], + 3 + BULK_IMAGES as i64, + "{done}" + ); + assert_signed_index(&srv, "images", "team/app", "1.0", &signed).await; + let repo_id = repository_id(&srv, "images").await; + for i in [0, 99, 100, BULK_IMAGES - 1] { + let image = format!("bulk/img-{i:03}"); + assert!( + forge_db::oci_dao::find_tag(&srv.pool, repo_id, &image, "1.0") + .await + .unwrap() + .is_some(), + "{image}" + ); + } + + // A second run, in the request form of a Nexus server and repository, + // copies only what is new in Nexus. + let next = make_nupkg("Acme.Core", "1.1.0"); + nexus + .upload( + "nuget-hosted", + "nuget", + "acme.core.1.1.0.nupkg", + next.clone(), + ) + .await; + let mut legacy = credentials(nexus.url.clone()); + legacy["repository"] = json!("nuget-hosted"); + let done = import(&srv, "nuget-internal", legacy).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["imported"], 1, "{done}"); + assert_eq!(done["progress"]["identical"], 2, "{done}"); + assert_eq!( + download( + &srv, + "/repository/nuget-internal/flat/acme.core/1.1.0/acme.core.1.1.0.nupkg" + ) + .await, + next + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn import_from_a_distribution_registry() { + let container = GenericImage::new(REGISTRY_IMAGE, REGISTRY_TAG) + .with_exposed_port(5000.tcp()) + .with_wait_for(WaitFor::message_on_stderr("listening on")) + .start() + .await + .expect("start registry:2 (is Docker running?)"); + let host = container.get_host().await.unwrap(); + let port = container.get_host_port_ipv4(5000).await.unwrap(); + let registry = Registry { + url: format!("http://{host}:{port}"), + auth: None, + client: reqwest::Client::new(), + }; + let signed = registry.push_signed_index("team/app", "1.0").await; + for image in ["team/base", "tools/cli", "tools/lint"] { + registry.push_image(image, "latest", image.as_bytes()).await; + } + + let srv = StackBuilder::default().start().await; + create_repo( + &srv, + json!({"name": "images", "kind": "hosted", "format": "docker"}), + ) + .await; + let done = import(&srv, "images", json!({"url": registry.url})).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["failed"], 0, "{done}"); + assert_eq!(done["progress"]["imported"], 6, "{done}"); + assert_signed_index(&srv, "images", "team/app", "1.0", &signed).await; + + let done = import(&srv, "images", json!({"url": registry.url})).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["identical"], 6, "{done}"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn import_from_another_forge() { + let srv = StackBuilder::default().start().await; + create_repo(&srv, json!({"name": "nuget-source", "kind": "hosted"})).await; + create_repo(&srv, json!({"name": "nuget-copy", "kind": "hosted"})).await; + let packages = [ + make_nupkg("Acme.Core", "1.0.0"), + make_nupkg("Acme.Core", "1.1.0-beta.1"), + make_nupkg("Acme.Web", "2.0.0"), + ]; + for package in &packages { + assert_eq!( + srv.push_nupkg("nuget-source", package.clone()) + .await + .status(), + 201 + ); + } + let (user, token) = srv.admin_basic(); + let body = json!({ + "url": format!("{}/repository/nuget-source/index.json", srv.base_url), + "username": user, + "password": token, + }); + let done = import(&srv, "nuget-copy", body.clone()).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["imported"], 3, "{done}"); + assert_eq!(done["source_total"], 2, "{done}"); + assert_eq!( + download( + &srv, + "/repository/nuget-copy/flat/acme.core/1.1.0-beta.1/acme.core.1.1.0-beta.1.nupkg" + ) + .await, + packages[1] + ); + + // Forge lists each package's SHA-512: a second run downloads nothing. + let downloads = || async { + sqlx::query_scalar!( + r#" + SELECT coalesce(sum(a.download_count), 0)::bigint AS "n!" + FROM asset a JOIN component c ON c.id = a.component_id + JOIN repository r ON r.id = c.repository_id + WHERE r.name = 'nuget-source' + "# + ) + .fetch_one(&srv.pool) + .await + .unwrap() + }; + let before = downloads().await; + let done = import(&srv, "nuget-copy", body).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["identical"], 3, "{done}"); + assert_eq!(done["progress"]["bytes_copied"], 0, "{done}"); + assert_eq!(downloads().await, before); +} diff --git a/crates/e2e/tests/nexus_import.rs b/crates/e2e/tests/nexus_import.rs deleted file mode 100644 index a031b785..00000000 --- a/crates/e2e/tests/nexus_import.rs +++ /dev/null @@ -1,464 +0,0 @@ -//! E2E: importing from a real Nexus Repository 3 OSS container -//! (`sonatype/nexus3`) seeded with NuGet packages, a multi-platform image -//! with a cosign signature and an SBOM. After the import, -//! bytes and digests are Nexus' and so are the publish dates; a second run -//! copies only what was added to Nexus since. (Resuming a killed import, the -//! conflict policies and the dry run are covered against a mock Nexus in -//! `crates/server/tests/nexus_import.rs`.) Gated behind `forge-e2e`; Nexus -//! takes a minute or two to start. - -#![cfg(feature = "forge-e2e")] - -use std::time::Duration; - -use forge_e2e::{make_nupkg, StackBuilder, TestServer}; -use forge_server::nexus_import::ImportWorker; -use serde_json::{json, Value}; -use sha2::Digest; -use testcontainers::core::{IntoContainerPort, WaitFor}; -use testcontainers::runners::AsyncRunner; -use testcontainers::{GenericImage, ImageExt}; - -const NEXUS_IMAGE: &str = "mirror.gcr.io/sonatype/nexus3"; -const NEXUS_TAG: &str = "3.70.1"; -/// With `NEXUS_SECURITY_RANDOMPASSWORD=false`, Nexus' admin password. -const ADMIN: (&str, &str) = ("admin", "admin123"); - -const OCI_MANIFEST: &str = "application/vnd.oci.image.manifest.v1+json"; -const OCI_INDEX: &str = "application/vnd.oci.image.index.v1+json"; - -fn sha256(bytes: &[u8]) -> String { - hex::encode(sha2::Sha256::digest(bytes)) -} - -fn digest(bytes: &[u8]) -> String { - format!("sha256:{}", sha256(bytes)) -} - -/// A Nexus with its REST API at `url` and its Docker connector at -/// `registry`. -struct Nexus { - url: String, - registry: String, - client: reqwest::Client, -} - -impl Nexus { - async fn start() -> (Self, testcontainers::ContainerAsync) { - let container = GenericImage::new(NEXUS_IMAGE, NEXUS_TAG) - .with_exposed_port(8081.tcp()) - .with_exposed_port(8082.tcp()) - .with_wait_for(WaitFor::message_on_stdout("Started Sonatype Nexus")) - .with_env_var("NEXUS_SECURITY_RANDOMPASSWORD", "false") - .with_env_var( - "INSTALL4J_ADD_VM_PARAMS", - "-Xms512m -Xmx1g -XX:MaxDirectMemorySize=1g", - ) - .with_startup_timeout(Duration::from_secs(600)) - .start() - .await - .expect("start nexus (is Docker running?)"); - let host = container.get_host().await.unwrap(); - let api = container.get_host_port_ipv4(8081).await.unwrap(); - let registry = container.get_host_port_ipv4(8082).await.unwrap(); - let nexus = Nexus { - url: format!("http://{host}:{api}"), - registry: format!("http://{host}:{registry}"), - client: reqwest::Client::new(), - }; - for _ in 0..300 { - let up = nexus - .client - .get(format!("{}/service/rest/v1/status/writable", nexus.url)) - .send() - .await - .is_ok_and(|r| r.status() == 200); - if up { - return (nexus, container); - } - tokio::time::sleep(Duration::from_secs(1)).await; - } - panic!("nexus never became writable"); - } - - fn api(&self, method: reqwest::Method, path: &str) -> reqwest::RequestBuilder { - self.client - .request(method, format!("{}/service/rest/v1/{path}", self.url)) - .basic_auth(ADMIN.0, Some(ADMIN.1)) - } - - async fn seed_repositories(&self) { - let resp = self - .api(reqwest::Method::PUT, "security/realms/active") - .json(&json!(["NexusAuthenticatingRealm", "DockerToken"])) - .send() - .await - .unwrap(); - assert!(resp.status().is_success(), "realms: {}", resp.status()); - let docker = json!({ - "name": "docker-hosted", "online": true, - "storage": {"blobStoreName": "default", "strictContentTypeValidation": true, "writePolicy": "allow"}, - "docker": {"v1Enabled": false, "forceBasicAuth": true, "httpPort": 8082}, - }); - let resp = self - .api(reqwest::Method::POST, "repositories/docker/hosted") - .json(&docker) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); - } - - /// Upload an asset through the components API (`.asset`). - async fn upload(&self, repository: &str, format: &str, file: &str, bytes: Vec) { - let form = reqwest::multipart::Form::new().part( - format!("{format}.asset"), - reqwest::multipart::Part::bytes(bytes).file_name(file.to_string()), - ); - let resp = self - .api( - reqwest::Method::POST, - &format!("components?repository={repository}"), - ) - .multipart(form) - .send() - .await - .unwrap(); - assert_eq!( - resp.status(), - 204, - "upload {file}: {}", - resp.text().await.unwrap() - ); - } - - /// The components of `repository`, `(name, version)` → asset. - async fn components(&self, repository: &str) -> Vec { - let mut out = Vec::new(); - let mut token: Option = None; - loop { - let mut path = format!("components?repository={repository}"); - if let Some(t) = &token { - path.push_str(&format!("&continuationToken={t}")); - } - let page: Value = self - .api(reqwest::Method::GET, &path) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - out.extend(page["items"].as_array().unwrap().iter().cloned()); - match page["continuationToken"].as_str() { - Some(t) => token = Some(t.to_string()), - None => return out, - } - } - } - - /// Push a blob to the Docker connector. - async fn push_blob(&self, image: &str, bytes: &[u8]) { - let resp = self - .client - .post(format!("{}/v2/{image}/blobs/uploads/", self.registry)) - .basic_auth(ADMIN.0, Some(ADMIN.1)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), 202, "start upload"); - let location = resp.headers()["location"].to_str().unwrap().to_string(); - let location = if location.starts_with("http") { - location - } else { - format!("{}{location}", self.registry) - }; - let separator = if location.contains('?') { '&' } else { '?' }; - let resp = self - .client - .put(format!("{location}{separator}digest={}", digest(bytes))) - .basic_auth(ADMIN.0, Some(ADMIN.1)) - .header("content-type", "application/octet-stream") - .body(bytes.to_vec()) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), 201, "finish upload"); - } - - async fn push_manifest(&self, image: &str, reference: &str, media_type: &str, bytes: &[u8]) { - let resp = self - .client - .put(format!( - "{}/v2/{image}/manifests/{reference}", - self.registry - )) - .basic_auth(ADMIN.0, Some(ADMIN.1)) - .header("content-type", media_type) - .body(bytes.to_vec()) - .send() - .await - .unwrap(); - assert_eq!( - resp.status(), - 201, - "manifest {reference}: {}", - resp.text().await.unwrap() - ); - } -} - -async fn admin( - srv: &TestServer, - method: reqwest::Method, - path: &str, - body: Option, -) -> reqwest::Response { - let (u, p) = srv.admin_basic(); - let mut req = srv - .client() - .request(method, format!("{}{path}", srv.base_url)) - .basic_auth(u, Some(p)); - if let Some(body) = body { - req = req.json(&body); - } - req.send().await.unwrap() -} - -/// Start an import of `source` into `repo` and run it to its end. -async fn import(srv: &TestServer, nexus: &Nexus, repo: &str, source: &str) -> Value { - let resp = admin( - srv, - reqwest::Method::POST, - &format!("/admin/repositories/{repo}/imports"), - Some(json!({"url": nexus.url, "repository": source, "username": ADMIN.0, "password": ADMIN.1})), - ) - .await; - assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); - let id = resp.json::().await.unwrap()["id"].as_i64().unwrap(); - for _ in 0..600 { - ImportWorker::new(srv.state.clone()).tick().await.unwrap(); - let view: Value = admin( - srv, - reqwest::Method::GET, - &format!("/admin/repositories/{repo}/imports/{id}"), - None, - ) - .await - .json() - .await - .unwrap(); - if !matches!(view["state"].as_str(), Some("queued" | "running")) { - return view; - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - panic!("import into {repo} did not end"); -} - -async fn create_repo(srv: &TestServer, body: Value) { - let resp = admin( - srv, - reqwest::Method::POST, - "/admin/repositories", - Some(body), - ) - .await; - assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); -} - -/// The bytes of a Forge download, with the admin's credentials. -async fn download(srv: &TestServer, path: &str) -> Vec { - let (u, p) = srv.admin_basic(); - let resp = srv - .client() - .get(format!("{}{path}", srv.base_url)) - .basic_auth(u, Some(p)) - .send() - .await - .unwrap(); - assert_eq!(resp.status(), 200, "GET {path}"); - resp.bytes().await.unwrap().to_vec() -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn import_from_a_real_nexus() { - let (nexus, _container) = Nexus::start().await; - nexus.seed_repositories().await; - - // NuGet: two packages in the default nuget-hosted repository. - let core = make_nupkg("Acme.Core", "1.0.0"); - let web = make_nupkg("Acme.Web", "2.1.0"); - nexus - .upload( - "nuget-hosted", - "nuget", - "acme.core.1.0.0.nupkg", - core.clone(), - ) - .await; - nexus - .upload("nuget-hosted", "nuget", "acme.web.2.1.0.nupkg", web.clone()) - .await; - - // Docker: a two-platform image, its cosign signature (a tag) and an - // SBOM pushed by digest (a referrer Nexus lists as an asset only). - let image = "team/app"; - let mut children = Vec::new(); - for arch in ["amd64", "arm64"] { - let config = json!({"architecture": arch, "os": "linux", "rootfs": {"type": "layers", "diff_ids": []}}).to_string().into_bytes(); - let layer = format!("the {arch} layer").into_bytes(); - nexus.push_blob(image, &config).await; - nexus.push_blob(image, &layer).await; - let manifest = json!({ - "schemaVersion": 2, "mediaType": OCI_MANIFEST, - "config": {"mediaType": "application/vnd.oci.image.config.v1+json", "digest": digest(&config), "size": config.len()}, - "layers": [{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", "digest": digest(&layer), "size": layer.len()}], - }) - .to_string() - .into_bytes(); - nexus - .push_manifest(image, &digest(&manifest), OCI_MANIFEST, &manifest) - .await; - children.push(json!({"mediaType": OCI_MANIFEST, "digest": digest(&manifest), "size": manifest.len(), "platform": {"architecture": arch, "os": "linux"}})); - } - let index = json!({"schemaVersion": 2, "mediaType": OCI_INDEX, "manifests": children}) - .to_string() - .into_bytes(); - nexus.push_manifest(image, "1.0", OCI_INDEX, &index).await; - let empty = b"{}".to_vec(); - nexus.push_blob(image, &empty).await; - let payload = b"cosign signature payload".to_vec(); - nexus.push_blob(image, &payload).await; - let subject = json!({"mediaType": OCI_INDEX, "digest": digest(&index), "size": index.len()}); - let signature = json!({ - "schemaVersion": 2, "mediaType": OCI_MANIFEST, - "config": {"mediaType": "application/vnd.oci.empty.v1+json", "digest": digest(&empty), "size": 2}, - "layers": [{"mediaType": "application/vnd.dev.cosign.simplesigning.v1+json", "digest": digest(&payload), "size": payload.len(), - "annotations": {"dev.cosignproject.cosign/signature": "MEUCIQ=="}}], - "subject": subject, - }) - .to_string() - .into_bytes(); - let sig_tag = format!("sha256-{}.sig", sha256(&index)); - nexus - .push_manifest(image, &sig_tag, OCI_MANIFEST, &signature) - .await; - let sbom = json!({ - "schemaVersion": 2, "mediaType": OCI_MANIFEST, "artifactType": "application/spdx+json", - "config": {"mediaType": "application/vnd.oci.empty.v1+json", "digest": digest(&empty), "size": 2}, - "layers": [{"mediaType": "application/vnd.oci.empty.v1+json", "digest": digest(&empty), "size": 2}], - "subject": subject, - }) - .to_string() - .into_bytes(); - nexus - .push_manifest(image, &digest(&sbom), OCI_MANIFEST, &sbom) - .await; - - let srv = StackBuilder::default().start().await; - create_repo(&srv, json!({"name": "nuget-internal", "kind": "hosted"})).await; - create_repo( - &srv, - json!({"name": "images", "kind": "hosted", "format": "docker"}), - ) - .await; - - // NuGet: the bytes and the publish dates are Nexus'. - let done = import(&srv, &nexus, "nuget-internal", "nuget-hosted").await; - assert_eq!(done["state"], "completed", "{done}"); - assert_eq!(done["progress"]["imported"], 2, "{done}"); - assert_eq!( - download( - &srv, - "/repository/nuget-internal/flat/acme.core/1.0.0/acme.core.1.0.0.nupkg" - ) - .await, - core - ); - assert_eq!( - download( - &srv, - "/repository/nuget-internal/flat/acme.web/2.1.0/acme.web.2.1.0.nupkg" - ) - .await, - web - ); - let listed = nexus.components("nuget-hosted").await; - let core_listed = listed.iter().find(|c| c["name"] == "Acme.Core").unwrap(); - let published = chrono::DateTime::parse_from_rfc3339( - core_listed["assets"][0]["blobCreated"].as_str().unwrap(), - ) - .unwrap(); - let created = sqlx::query_scalar!( - "SELECT c.created_at FROM component c JOIN repository r ON r.id = c.repository_id WHERE r.name = 'nuget-internal' AND c.name = 'acme.core'" - ) - .fetch_one(&srv.pool) - .await - .unwrap(); - assert_eq!(created, published); - - // Docker: digests, the children, the signature tag and both referrers. - let done = import(&srv, &nexus, "images", "docker-hosted").await; - assert_eq!(done["state"], "completed", "{done}"); - assert_eq!(done["progress"]["failed"], 0, "{done}"); - let repo_id = forge_db::repository_dao::find_repository_by_name(&srv.pool, "images") - .await - .unwrap() - .unwrap() - .id; - let tagged = forge_db::oci_dao::find_tag(&srv.pool, repo_id, image, "1.0") - .await - .unwrap() - .unwrap(); - assert_eq!(tagged.manifest.digest, digest(&index)); - for child in &children { - let d = child["digest"].as_str().unwrap(); - assert!( - forge_db::oci_dao::find_manifest(&srv.pool, repo_id, image, d) - .await - .unwrap() - .is_some(), - "child {d}" - ); - } - let sig = forge_db::oci_dao::find_tag(&srv.pool, repo_id, image, &sig_tag) - .await - .unwrap() - .unwrap(); - assert_eq!(sig.manifest.digest, digest(&signature)); - let mut referrers: Vec = - forge_db::oci_dao::list_referrers(&srv.pool, repo_id, image, &digest(&index)) - .await - .unwrap() - .into_iter() - .map(|m| m.digest) - .collect(); - referrers.sort(); - let mut expected = vec![digest(&signature), digest(&sbom)]; - expected.sort(); - assert_eq!(referrers, expected); - - // A second run copies only what is new in Nexus. - let next = make_nupkg("Acme.Core", "1.1.0"); - nexus - .upload( - "nuget-hosted", - "nuget", - "acme.core.1.1.0.nupkg", - next.clone(), - ) - .await; - let done = import(&srv, &nexus, "nuget-internal", "nuget-hosted").await; - assert_eq!(done["state"], "completed", "{done}"); - assert_eq!(done["progress"]["imported"], 1, "{done}"); - assert_eq!(done["progress"]["identical"], 2, "{done}"); - assert_eq!( - download( - &srv, - "/repository/nuget-internal/flat/acme.core/1.1.0/acme.core.1.1.0.nupkg" - ) - .await, - next - ); -} diff --git a/crates/formats/docker/src/upstream.rs b/crates/formats/docker/src/upstream.rs index a29cd1de..7d283b2c 100644 --- a/crates/formats/docker/src/upstream.rs +++ b/crates/formats/docker/src/upstream.rs @@ -12,10 +12,12 @@ //! to the upstream only) to fill one page of Forge's; an upstream that //! refuses a large `n` (400) is asked again without one. //! * Referrers come from the upstream's referrers API, where it has one. +//! * The catalog (`/v2/_catalog`, which an import walks) is paged like a +//! tag list; an upstream that does not serve it, or refuses it, has none. use std::collections::BTreeSet; -use forge_plugin_api::images::{ImageUrl, ManifestInfo, Reference, TagPage}; +use forge_plugin_api::images::{CatalogPage, ImageUrl, ManifestInfo, Reference, TagPage}; use forge_plugin_api::{ Fetch, FetchResult, Fetched, FormatError, Freshness, Remote, UpstreamAnswer, UpstreamOp, UpstreamStep, @@ -75,6 +77,9 @@ pub fn step( tags(&base, image, last.as_deref(), *n as usize, fetched) } UpstreamOp::ImageReferrers { image, digest } => referrers(&base, image, digest, fetched), + UpstreamOp::ImageCatalog { last, n } => { + catalog(&base, last.as_deref(), *n as usize, fetched) + } _ => Err(FormatError::Internal( "Docker proxies have only image operations".into(), )), @@ -117,12 +122,8 @@ fn get(url: String, image: &str, accept: Option<&str>) -> Fetch { } } -/// Up to `limit` of the upstream's tags of `image` after `last`: one more -/// is asked for, to know whether there are more, and the upstream's own -/// pages are followed for up to [`MAX_UPSTREAM_TAG_PAGES`] requests. An -/// upstream that refuses so large an `n` (400, as registries that cap it -/// answer) is asked again without one: its own page size and links take -/// over. The steps so far are in `fetched`. +/// Up to `limit` of the upstream's tags of `image` after `last` +/// ([`list`]). fn tags( base: &str, image: &str, @@ -130,6 +131,124 @@ fn tags( limit: usize, fetched: &[FetchResult], ) -> Result { + Ok( + match list(base, &Listing::Tags(image), last, limit, fetched) { + Listed::Fetch(fetch) => UpstreamStep::Fetch(vec![fetch]), + Listed::Unknown => UpstreamStep::Done(UpstreamAnswer::ImageTags(None)), + Listed::Failed(reason) => UpstreamStep::Done(UpstreamAnswer::Failed(reason)), + Listed::Page(tags, more) => { + UpstreamStep::Done(UpstreamAnswer::ImageTags(Some(TagPage { tags, more }))) + } + }, + ) +} + +/// Up to `limit` of the names of the upstream's images after `last`, from +/// its catalog (`/v2/_catalog`, [`list`]). The catalog is no part of the +/// distribution spec: an upstream that does not serve it (404), or refuses +/// it (401, 403: Docker Hub), has none. +fn catalog( + base: &str, + last: Option<&str>, + limit: usize, + fetched: &[FetchResult], +) -> Result { + if fetched + .first() + .is_some_and(|first| matches!(first.status, Some(401 | 403))) + { + return Ok(UpstreamStep::Done(UpstreamAnswer::ImageCatalog(None))); + } + Ok(match list(base, &Listing::Catalog, last, limit, fetched) { + Listed::Fetch(fetch) => UpstreamStep::Fetch(vec![fetch]), + Listed::Unknown => UpstreamStep::Done(UpstreamAnswer::ImageCatalog(None)), + Listed::Failed(reason) => UpstreamStep::Done(UpstreamAnswer::Failed(reason)), + Listed::Page(images, more) => { + UpstreamStep::Done(UpstreamAnswer::ImageCatalog(Some(CatalogPage { + images, + more, + }))) + } + }) +} + +/// A paged list of names the registry API serves. +enum Listing<'a> { + /// The tags of an image (`/v2//tags/list`). + Tags(&'a str), + /// The registry's images (`/v2/_catalog`). + Catalog, +} + +impl Listing<'_> { + fn url(&self, base: &str) -> String { + match self { + Listing::Tags(image) => format!("{base}/v2/{image}/tags/list"), + Listing::Catalog => format!("{base}/v2/_catalog"), + } + } + + /// The key of the document's name array. + fn key(&self) -> &'static str { + match self { + Listing::Tags(_) => "tags", + Listing::Catalog => "repositories", + } + } + + fn scope(&self) -> String { + match self { + Listing::Tags(image) => scope(image), + Listing::Catalog => "registry:catalog:*".to_string(), + } + } + + fn valid(&self, name: &str) -> bool { + match self { + Listing::Tags(_) => reference::valid_tag(name), + Listing::Catalog => reference::valid_name(name), + } + } + + fn what(&self) -> &'static str { + match self { + Listing::Tags(_) => "tag list", + Listing::Catalog => "catalog", + } + } + + fn get(&self, url: String) -> Fetch { + Fetch { + scope: Some(self.scope()), + ..Fetch::new(url, Freshness::Metadata) + } + } +} + +/// Where [`list`] stands. +enum Listed { + /// Fetch this page next. + Fetch(Fetch), + /// The upstream does not have the list (404). + Unknown, + Failed(String), + /// The names, and whether there are more after them. + Page(Vec, bool), +} + +/// Up to `limit` of the names of `listing` after `last`: one more is asked +/// for, to know whether there are more, and the upstream's own pages are +/// followed for up to [`MAX_UPSTREAM_TAG_PAGES`] requests. An upstream that +/// refuses so large an `n` (400, as registries that cap it answer) is asked +/// again without one: its own page size and links take over. The steps so +/// far are in `fetched`. +fn list( + base: &str, + listing: &Listing<'_>, + last: Option<&str>, + limit: usize, + fetched: &[FetchResult], +) -> Listed { let first = |n: Option| { let mut query = url::form_urlencoded::Serializer::new(String::new()); if let Some(n) = n { @@ -139,31 +258,29 @@ fn tags( query.append_pair("last", last); } let query = query.finish(); - let url = format!("{base}/v2/{image}/tags/list"); + let url = listing.url(base); let url = if query.is_empty() { url } else { format!("{url}?{query}") }; - get(url, image, None) + listing.get(url) }; if fetched.is_empty() { - return Ok(UpstreamStep::Fetch(vec![first(Some( - limit.saturating_add(1), - ))])); + return Listed::Fetch(first(Some(limit.saturating_add(1)))); } - let mut tags: BTreeSet = BTreeSet::new(); + let mut names: BTreeSet = BTreeSet::new(); let mut pages = 0; let mut left_out = false; for (i, result) in fetched.iter().enumerate() { let latest = i + 1 == fetched.len(); if pages == 0 { if result.outcome == Fetched::NotFound { - return Ok(UpstreamStep::Done(UpstreamAnswer::ImageTags(None))); + return Listed::Unknown; } if i == 0 && result.status == Some(400) { if latest { - return Ok(UpstreamStep::Fetch(vec![first(None)])); + return Listed::Fetch(first(None)); } continue; } @@ -171,52 +288,46 @@ fn tags( let body = match &result.outcome { Fetched::Found(body) => body, Fetched::NotFound => { - return Ok(UpstreamStep::Done(UpstreamAnswer::Failed( - "the upstream lost a page of the tag list".into(), - ))) - } - Fetched::Failed(reason) => { - return Ok(UpstreamStep::Done(UpstreamAnswer::Failed(reason.clone()))) + return Listed::Failed(format!( + "the upstream lost a page of the {}", + listing.what() + )) } + Fetched::Failed(reason) => return Listed::Failed(reason.clone()), }; pages += 1; - #[derive(serde::Deserialize)] - struct TagList { - #[serde(default)] - tags: Option>, - } - let Ok(list) = serde_json::from_slice::(body) else { - return Ok(UpstreamStep::Done(UpstreamAnswer::Failed( - "the upstream served an invalid tag list".into(), - ))); + let Ok(document) = serde_json::from_slice::(body) else { + return Listed::Failed(format!("the upstream served an invalid {}", listing.what())); + }; + let listed = match document.get(listing.key()) { + None | Some(serde_json::Value::Null) => Vec::new(), + Some(serde_json::Value::Array(listed)) => listed.clone(), + Some(_) => { + return Listed::Failed(format!("the upstream served an invalid {}", listing.what())) + } }; - tags.extend( - list.tags - .unwrap_or_default() + names.extend( + listed .into_iter() - .filter(|t| reference::valid_tag(t)) - .filter(|t| last.is_none_or(|l| t.as_bytes() > l.as_bytes())), + .filter_map(|name| name.as_str().map(str::to_string)) + .filter(|name| listing.valid(name)) + .filter(|name| last.is_none_or(|l| name.as_bytes() > l.as_bytes())), ); - if tags.len() > limit { + if names.len() > limit { break; } if latest { match next_link(result, base) { Some(next) if pages < MAX_UPSTREAM_TAG_PAGES => { - return Ok(UpstreamStep::Fetch(vec![get(next, image, None)])) + return Listed::Fetch(listing.get(next)) } Some(_) => left_out = true, None => {} } } } - let more = left_out || tags.len() > limit; - Ok(UpstreamStep::Done(UpstreamAnswer::ImageTags(Some( - TagPage { - tags: tags.into_iter().take(limit).collect(), - more, - }, - )))) + let more = left_out || names.len() > limit; + Listed::Page(names.into_iter().take(limit).collect(), more) } /// The `next` page a tag list's `Link` header names (RFC 8288), resolved @@ -404,6 +515,81 @@ mod tests { assert_eq!(conversed.fetches.len(), 2, "no link off the upstream"); } + #[test] + fn the_catalog_is_paged_like_a_tag_list() { + let mut pages = vec![ + ( + vec!["team/app", "team/db"], + Some("; rel=\"next\""), + ), + (vec!["Not A Name", "tools/cli"], None), + ] + .into_iter(); + let op = UpstreamOp::ImageCatalog { + last: Some("base".into()), + n: 10, + }; + let conversed = converse_answered(&Docker, &remote("http://up.test"), &op, |fetch| { + let (images, link) = pages.next().expect("no more pages"); + FetchResult { + fetch: fetch.clone(), + outcome: Fetched::Found( + serde_json::to_vec(&json!({ "repositories": images })).unwrap(), + ), + status: Some(200), + headers: link + .map(|l| vec![("link".to_string(), l.to_string())]) + .unwrap_or_default(), + } + }) + .unwrap(); + assert_eq!( + conversed.answer, + UpstreamAnswer::ImageCatalog(Some(CatalogPage { + images: vec!["team/app".into(), "team/db".into(), "tools/cli".into()], + more: false, + })), + "an invalid name is left out" + ); + assert_eq!( + conversed.fetches[0].url, + "http://up.test/v2/_catalog?n=11&last=base" + ); + assert_eq!( + conversed.fetches[0].scope.as_deref(), + Some("registry:catalog:*") + ); + assert_eq!( + conversed.fetches[1].url, + "http://up.test/v2/_catalog?n=2&last=team%2Fdb" + ); + } + + #[test] + fn a_registry_without_a_catalog_has_none() { + for (status, outcome) in [ + (404, Fetched::NotFound), + (401, Fetched::Failed("401".into())), + (403, Fetched::Failed("403".into())), + ] { + let op = UpstreamOp::ImageCatalog { last: None, n: 5 }; + let conversed = converse_answered(&Docker, &remote("http://up.test"), &op, |fetch| { + FetchResult { + fetch: fetch.clone(), + outcome: outcome.clone(), + status: Some(status), + headers: Vec::new(), + } + }) + .unwrap(); + assert_eq!( + conversed.answer, + UpstreamAnswer::ImageCatalog(None), + "{status}" + ); + } + } + #[test] fn an_upstream_refusing_a_large_n_is_asked_without_one() { let op = UpstreamOp::ImageTags { diff --git a/crates/formats/npm/src/proxy.rs b/crates/formats/npm/src/proxy.rs index c7c8304c..fa682631 100644 --- a/crates/formats/npm/src/proxy.rs +++ b/crates/formats/npm/src/proxy.rs @@ -274,13 +274,16 @@ pub fn upstream(remote: &Remote, op: &UpstreamOp, fetched: &[FetchResult]) -> Up } } UpstreamOp::Search(query) => search(remote, query, fetched), + // No import walks an npm registry yet. + UpstreamOp::Packages { .. } => done(UpstreamAnswer::Search(None)), // A registry holds packages: it has no files to fetch. UpstreamOp::File(_) => done(UpstreamAnswer::File(None)), UpstreamOp::ImageName(_) | UpstreamOp::ImageManifest { .. } | UpstreamOp::ImageBlob { .. } | UpstreamOp::ImageTags { .. } - | UpstreamOp::ImageReferrers { .. } => failed("a registry of packages holds no images"), + | UpstreamOp::ImageReferrers { .. } + | UpstreamOp::ImageCatalog { .. } => failed("a registry of packages holds no images"), } } @@ -382,6 +385,9 @@ fn metadata(doc: Option, wanted: &[String]) -> PackageMetadat created: time, summary, version, + // npm's integrity is checked from the version's dist + // when its tarball is located, not here. + sha512: None, } }) .collect(), diff --git a/crates/formats/nuget/src/upstream.rs b/crates/formats/nuget/src/upstream.rs index 1e5e8cc3..6f716694 100644 --- a/crates/formats/nuget/src/upstream.rs +++ b/crates/formats/nuget/src/upstream.rs @@ -136,7 +136,18 @@ pub(crate) fn step( UpstreamOp::Manifest(target) => manifest(&resources, target, fetched, limits), UpstreamOp::Metadata(wanted) => metadata(&resources, wanted, false, fetched), UpstreamOp::Dating(wanted) => metadata(&resources, wanted, true, fetched), - UpstreamOp::Search(query) => search(&resources, query, fetched), + UpstreamOp::Search(query) => search(&resources, query, true, fetched), + UpstreamOp::Packages { skip, take } => { + let query = SearchQuery { + text: String::new(), + skip: *skip, + take: *take, + prerelease: true, + semver2: true, + upstream: true, + }; + search(&resources, &query, false, fetched) + } // A feed holds packages: it has no files to fetch. UpstreamOp::File(_) => done(UpstreamAnswer::File(None)), // NuGet has no tags. @@ -145,7 +156,8 @@ pub(crate) fn step( | UpstreamOp::ImageManifest { .. } | UpstreamOp::ImageBlob { .. } | UpstreamOp::ImageTags { .. } - | UpstreamOp::ImageReferrers { .. } => failed("a feed holds no images"), + | UpstreamOp::ImageReferrers { .. } + | UpstreamOp::ImageCatalog { .. } => failed("a feed holds no images"), }) } @@ -289,6 +301,10 @@ struct Leaf { /// The `@id` of the version's catalog entry, which names when the /// version was created even when it is unlisted. catalog_url: Option, + /// The package's SHA-512 (base64), where the entry lists it inline (a + /// `packageHash` with `packageHashAlgorithm` `SHA512`, as Forge's + /// registration does; nuget.org's lists it in the catalog leaf only). + sha512: Option, } /// One page of a registration index. @@ -458,6 +474,7 @@ fn metadata( summary: nuspec::manifest_summary(&manifest) .ok() .and_then(|summary| serde_json::to_string(&summary).ok()), + sha512: leaf.sha512.clone(), } }) .collect(); @@ -534,6 +551,16 @@ fn registration_leaves(page: &Value) -> BTreeMap { published, groups, catalog_url: entry.get("@id").and_then(Value::as_str).map(str::to_string), + sha512: entry + .get("packageHash") + .and_then(Value::as_str) + .filter(|_| { + entry + .get("packageHashAlgorithm") + .and_then(Value::as_str) + .is_some_and(|algorithm| algorithm.eq_ignore_ascii_case("SHA512")) + }) + .map(str::to_string), }, ); } @@ -572,10 +599,18 @@ fn year(raw: &str) -> Option { } /// A page of the upstream's `SearchQueryService` results for `query`. -fn search(resources: &Resources, query: &SearchQuery, fetched: &[FetchResult]) -> UpstreamStep { +/// `capped`: a proxy's search, which does not ask for offsets beyond +/// nuget.org's; an import's walk asks for every offset the upstream allows +/// (Nexus: 10,000 packages). +fn search( + resources: &Resources, + query: &SearchQuery, + capped: bool, + fetched: &[FetchResult], +) -> UpstreamStep { // nuget.org limits skip to 3,000. Deeper pages use the proxy's local search // rather than requesting a rejected offset or silently changing the page. - if query.skip > 3_000 { + if capped && query.skip > 3_000 { return done(UpstreamAnswer::Search(None)); } let Some(service) = &resources.search_query_service else { @@ -771,4 +806,19 @@ mod tests { assert!(leaves["2.0.0"].listed); assert!(leaves["3.0.0"].listed); } + + #[test] + fn an_inline_sha512_package_hash_is_read() { + let page = serde_json::json!({ "items": [ + { "catalogEntry": { "version": "1.0.0", "packageHash": "c2hhNTEy", + "packageHashAlgorithm": "SHA512" } }, + { "catalogEntry": { "version": "2.0.0", "packageHash": "bWQ1", + "packageHashAlgorithm": "MD5" } }, + { "catalogEntry": { "version": "3.0.0", "packageHash": "c2hhNTEy" } }, + ]}); + let leaves = registration_leaves(&page); + assert_eq!(leaves["1.0.0"].sha512.as_deref(), Some("c2hhNTEy")); + assert_eq!(leaves["2.0.0"].sha512, None, "another algorithm"); + assert_eq!(leaves["3.0.0"].sha512, None, "no algorithm named"); + } } diff --git a/crates/formats/nuget/tests/upstream.rs b/crates/formats/nuget/tests/upstream.rs index 588875f5..b71afa16 100644 --- a/crates/formats/nuget/tests/upstream.rs +++ b/crates/formats/nuget/tests/upstream.rs @@ -368,3 +368,27 @@ fn search_offsets_beyond_nuget_orgs_limit_request_local_fallback() { assert!(done.fetches.is_empty()); } } + +#[test] +fn an_import_walks_every_package_beyond_nuget_orgs_offset_limit() { + let url = "https://search.example/query?q=&skip=5000&take=2&prerelease=true&semVerLevel=2.0.0"; + let page = found(json!({ "totalHits": 5002, "data": [ + { "id": "Contoso.Lib", "version": "1.0.0" }, + { "id": "Contoso.Web", "version": "2.0.0-beta.1" }, + ]})); + let done = run( + &remote(&resources()), + UpstreamOp::Packages { + skip: 5_000, + take: 2, + }, + &docs([(url.to_string(), page)]), + ); + assert_eq!(done.fetches[0].url, url); + let UpstreamAnswer::Search(Some(search)) = done.answer else { + panic!("{:?}", done.answer) + }; + assert_eq!(search.total, 5002); + let ids: Vec<&str> = search.hits.iter().map(|hit| hit.id.as_str()).collect(); + assert_eq!(ids, ["Contoso.Lib", "Contoso.Web"]); +} diff --git a/crates/plugin-api/src/images.rs b/crates/plugin-api/src/images.rs index 70a5c15c..de0b6445 100644 --- a/crates/plugin-api/src/images.rs +++ b/crates/plugin-api/src/images.rs @@ -273,6 +273,15 @@ pub struct TagPage { pub more: bool, } +/// A page of an upstream registry's catalog ([`crate::UpstreamOp::ImageCatalog`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CatalogPage { + /// Image names, in byte order. + pub images: Vec, + /// There are images after these the page leaves out. + pub more: bool, +} + /// An upload session. #[derive(Debug, Clone, PartialEq, Eq)] pub struct UploadState { diff --git a/crates/plugin-api/src/types.rs b/crates/plugin-api/src/types.rs index ba3e6e51..e22248fa 100644 --- a/crates/plugin-api/src/types.rs +++ b/crates/plugin-api/src/types.rs @@ -3,7 +3,7 @@ use std::fmt; use crate::images::{ - ImageAnswer, ImageName, ImageQuery, ImageUrl, ManifestInfo, Reference, TagPage, + CatalogPage, ImageAnswer, ImageName, ImageQuery, ImageUrl, ManifestInfo, Reference, TagPage, }; /// A dependency on another package of the same format. @@ -1031,6 +1031,12 @@ pub enum UpstreamOp { Dating(PackageVersions), /// A page of the upstream's search results. Search(SearchQuery), + /// Package formats: up to `take` of every package the upstream lists, + /// after the first `skip` ([`UpstreamAnswer::Search`]): what an import + /// walks. Unlike a proxy's [`UpstreamOp::Search`], every version counts + /// (prerelease, SemVer 2.0.0) and the offset is not capped below what + /// the upstream allows. `None`: the upstream lists no packages. + Packages { skip: u32, take: u32 }, /// The tags the upstream gives a package (canonical id). Tags(String), /// The URL of the file at a path (files formats). The host downloads it, @@ -1063,6 +1069,10 @@ pub enum UpstreamOp { /// Images formats: the upstream's referrers of a manifest /// ([`UpstreamAnswer::ImageReferrers`]). ImageReferrers { image: String, digest: String }, + /// Images formats: up to `n` of the names of the images the upstream + /// holds after `last`, in byte order ([`UpstreamAnswer::ImageCatalog`]): + /// what an import of a registry walks. + ImageCatalog { last: Option, n: u32 }, } /// How long the host may reuse a fetched document. @@ -1167,6 +1177,10 @@ pub struct VersionMetadata { /// /// [`FormatPlugin::manifest_summary`]: crate::FormatPlugin::manifest_summary pub summary: Option, + /// The SHA-512 of the version's package, base64, where the upstream's + /// metadata lists it: an import verifies the package against it, and + /// skips a version it holds with these bytes without downloading it. + pub sha512: Option, } /// The answer to [`UpstreamOp::Metadata`] and [`UpstreamOp::Dating`]. @@ -1247,6 +1261,9 @@ pub enum UpstreamAnswer { /// `None`: the upstream offers no referrers API (the host lists the /// referrers its cache holds). ImageReferrers(Option>), + /// `None`: the upstream serves no catalog (it is no part of the + /// distribution spec: Docker Hub, GHCR and ECR serve none), or refuses it. + ImageCatalog(Option), /// The upstream answered something unusable: the host serves what it /// has cached. Failed(String), diff --git a/crates/server/src/admin/imports.rs b/crates/server/src/admin/imports.rs index 44325f64..c1b45414 100644 --- a/crates/server/src/admin/imports.rs +++ b/crates/server/src/admin/imports.rs @@ -1,16 +1,19 @@ -//! Imports from Nexus into a hosted repository ([`crate::nexus_import`]). +//! Imports into a hosted repository from another server ([`crate::import`]): +//! a NuGet feed or a registry, read through the format's own protocol. //! **Admin-only.** //! //! * `GET /admin/repositories/{name}/imports` — the repository's imports, //! newest first ([`ImportView`]). //! * `POST /admin/repositories/{name}/imports` — start one //! ([`CreateImport`]). The source is checked first (the **preflight**): -//! Nexus answers its status endpoint, the Nexus repository exists, is -//! hosted and of the repository's format, and, for Docker, its registry -//! API answers. A failed check is a 400 (502 when Nexus is unreachable) -//! saying what is wrong; nothing is stored. A repository runs one import -//! at a time (409). `dry_run` decides every item without writing a -//! package: what would be copied, how much, and the conflicts. +//! a feed's service index resolves and its search answers (with how many +//! packages it finds), or a registry answers its API and serves a +//! catalog; an import that names its package ids or images needs no +//! search or catalog. A failed check is a 400 (502 when the source is +//! unreachable or failing) saying what is wrong; nothing is stored. A +//! repository runs one import at a time (409). `dry_run` decides every +//! item without writing a package: what would be copied, and the +//! conflicts. //! * `GET …/imports/{id}` — one import and its progress. //! * `GET …/imports/{id}/items?state=&after=&limit=` — its items in the //! order they were found ([`forge_db::import_dao::ImportItem`]), paged @@ -37,14 +40,19 @@ use axum::Extension; use chrono::{DateTime, Utc}; use forge_core::auth::principal::Principal; use forge_core::net::ssrf::validate_url; -use forge_db::import_dao::{self, NewImport, NexusImport}; +use forge_core::repo::proxy::Credentials; +use forge_db::import_dao::{self, NewImport, RepositoryImport}; use forge_db::model::{RepoKind, Repository}; use forge_db::{repository_dao, secret_dao}; use serde::{Deserialize, Serialize}; use super::db_error; -use crate::nexus_import::nexus::{Nexus, NexusError}; -use crate::nexus_import::{default_registry_url, password_secret, FORMATS}; +use crate::images::import::ImportSource; +use crate::import::feed::Feed; +use crate::import::{ + nexus_source_url, password_secret, source_url, Listing, SourceError, FORMATS, MAX_NAMES, + SEARCH_PAGE, +}; use crate::state::AppState; /// Imports listed per repository. @@ -85,20 +93,30 @@ fn is_unique_violation(e: &sqlx::Error) -> bool { #[derive(Deserialize)] #[serde(deny_unknown_fields)] pub struct CreateImport { - /// The Nexus server, e.g. `https://nexus.example.com` (with its context - /// path, if it has one). + /// The source: a NuGet feed's service index + /// (`https://nexus.example.com/repository/nuget-hosted/index.json`), or a + /// registry (`https://registry.example.com`). With `repository`, a Nexus + /// server instead (the request before imports read the format's + /// protocol). pub url: String, - /// The Nexus hosted repository to copy. - pub repository: String, - /// Nexus credentials (both, or neither for anonymous access). + /// The Nexus repository at `url` to read (with `url` naming the Nexus + /// server): its service index, `/repository//index.json`, or + /// its registry API, `registry_url` or `/repository/`. + #[serde(default)] + pub repository: Option, + /// With `repository`: the Nexus Docker repository's connector, e.g. + /// `http://nexus.example.com:8082`. + #[serde(default)] + pub registry_url: Option, + /// The package ids or images to copy, instead of every package the + /// feed's search finds (every image the registry's catalog lists). + #[serde(default)] + pub names: Option>, + /// The source's credentials (both, or neither for anonymous access). #[serde(default)] pub username: Option, #[serde(default)] pub password: Option, - /// Docker: the Nexus repository's registry API, e.g. its connector - /// `http://nexus.example.com:8082`. Default: `/repository/`. - #[serde(default)] - pub registry_url: Option, /// A version Forge holds with other bytes: `skip` (and report), /// `overwrite`, or `fail` the import. #[serde(default = "default_on_conflict")] @@ -123,7 +141,7 @@ fn default_concurrency() -> i32 { #[derive(Default, Deserialize)] #[serde(deny_unknown_fields)] pub struct ResumeImport { - /// The Nexus password, when the import no longer holds it (it + /// The source's password, when the import no longer holds it (it /// completed) or it changed. #[serde(default)] pub password: Option, @@ -156,13 +174,10 @@ pub struct Progress { pub failed: i64, /// Dry run: items that would be copied. pub planned: i64, - /// Their size in Nexus. + /// Their size, where the source says (a referrer's). pub planned_size: i64, - /// Bytes downloaded from Nexus. + /// Bytes downloaded from the source. pub bytes_copied: i64, - /// Every asset the scan saw in the Nexus repository, and their size. - pub assets: i64, - pub size: i64, } /// An import, without its password. @@ -170,10 +185,11 @@ pub struct Progress { pub struct ImportView { pub id: i64, pub repository: String, - /// The Nexus server and repository. + /// The source: a feed's service index or a registry. pub url: String, - pub source_repository: String, - pub registry_url: Option, + /// The package ids or images it copies; `null`: every one the source's + /// search or catalog lists. + pub names: Option>, pub username: Option, pub has_password: bool, pub on_conflict: String, @@ -184,7 +200,11 @@ pub struct ImportView { /// `scan` or `copy`. pub phase: String, pub cancel_requested: bool, - pub nexus_version: Option, + /// The `Server` header the source answered the preflight with. + pub source_server: Option, + /// How many packages the feed's search finds: what it reported in the + /// preflight, or as many as the scan walked, whichever is more. + pub source_total: Option, pub last_error: Option, pub created_by: Option, pub created_at: DateTime, @@ -197,7 +217,7 @@ pub struct ImportView { async fn view( state: &AppState, repo: &Repository, - import: NexusImport, + import: RepositoryImport, ) -> sqlx::Result { let counts = import_dao::counts(&state.pool, import.id).await?; let count = |s: &str| counts.get(s).map(|c| c.items).unwrap_or(0); @@ -212,15 +232,12 @@ async fn view( planned: count("planned"), planned_size: counts.get("planned").map(|c| c.size).unwrap_or(0), bytes_copied: counts.values().map(|c| c.bytes_copied).sum(), - assets: import.assets_seen, - size: import.bytes_seen, }; Ok(ImportView { id: import.id, repository: repo.name.clone(), - url: import.base_url, - source_repository: import.source_repository, - registry_url: import.registry_url, + url: source_url(&import, state.formats.stores_images(&repo.format)), + names: import.names, username: import.username, has_password: import.has_password, on_conflict: import.on_conflict, @@ -229,7 +246,8 @@ async fn view( state: import.state, phase: import.phase, cancel_requested: import.cancel_requested, - nexus_version: import.nexus_version, + source_server: import.source_server, + source_total: import.source_total, last_error: import.last_error, created_by: import.created_by, created_at: import.created_at, @@ -240,7 +258,7 @@ async fn view( }) } -async fn view_response(state: &AppState, repo: &Repository, import: NexusImport) -> Response { +async fn view_response(state: &AppState, repo: &Repository, import: RepositoryImport) -> Response { match view(state, repo, import).await { Ok(view) => Json(view).into_response(), Err(e) => db_error(e), @@ -260,7 +278,7 @@ async fn resolve( state: &AppState, name: &str, id: i64, -) -> Result<(Repository, NexusImport), Response> { +) -> Result<(Repository, RepositoryImport), Response> { let repo = resolve_repo(state, name).await?; match import_dao::get(&state.pool, id).await { Ok(Some(import)) if import.repository_id == repo.id => Ok((repo, import)), @@ -270,11 +288,11 @@ async fn resolve( } /// A preflight failure as the administrator sees it. -fn preflight_error(what: &str, e: NexusError) -> Response { - let status = match e { - NexusError::Unreachable(_) => StatusCode::BAD_GATEWAY, - NexusError::Status { status, .. } if status.is_server_error() => StatusCode::BAD_GATEWAY, - _ => StatusCode::BAD_REQUEST, +fn preflight_error(what: &str, e: SourceError) -> Response { + let status = if e.is_transient() { + StatusCode::BAD_GATEWAY + } else { + StatusCode::BAD_REQUEST }; (status, format!("{what}: {e}")).into_response() } @@ -324,21 +342,73 @@ pub async fn create( } if !FORMATS.contains(&repo.format.as_str()) { return bad_request(format!( - "importing {} repositories from Nexus is not supported; supported: {}", + "importing {} repositories is not supported; supported: {}", repo.format, FORMATS.join(", ") )); } + let images = state.formats.stores_images(&repo.format); + let Some(format) = state.formats.get(&repo.format) else { + return bad_request(format!("the {} format is not loaded", repo.format)); + }; // The request itself. let url = body.url.trim().trim_end_matches('/').to_string(); if let Err(e) = validate_url(&url) { return bad_request(format!("invalid url: {e}")); } - let source_repository = body.repository.trim().to_string(); - if source_repository.is_empty() || source_repository.contains('/') { - return bad_request("repository must name one Nexus repository"); - } + let registry_url = body + .registry_url + .map(|u| u.trim().trim_end_matches('/').to_string()) + .filter(|u| !u.is_empty()); + let source = match body.repository.map(|r| r.trim().to_string()) { + Some(repository) => { + if repository.is_empty() || repository.contains('/') { + return bad_request("repository must name one Nexus repository"); + } + if registry_url.is_some() && !images { + return bad_request("registry_url applies to Docker repositories only"); + } + if let Some(registry_url) = ®istry_url { + if let Err(e) = validate_url(registry_url) { + return bad_request(format!("invalid registry_url: {e}")); + } + } + nexus_source_url(&url, &repository, registry_url.as_deref(), images) + } + None if registry_url.is_some() => { + return bad_request( + "registry_url goes with repository (a Nexus server's url); give the registry as url", + ); + } + None => url, + }; + let names = match body.names { + None => None, + Some(names) => { + let mut seen = std::collections::HashSet::new(); + let mut kept = Vec::new(); + for name in names { + let name = name.trim().to_string(); + if name.is_empty() { + continue; + } + if let Err(e) = format.validate_id(&name) { + return bad_request(format!("names: {name:?} is no valid name: {e}")); + } + if seen.insert(format.lower_id(&name)) { + kept.push(name); + } + } + if kept.is_empty() { + return bad_request("names lists no package id or image; leave it out to copy every one the source lists"); + } + if kept.len() > MAX_NAMES { + return bad_request(format!("names lists more than {MAX_NAMES}")); + } + Some(kept) + } + }; let username = body.username.filter(|u| !u.is_empty()); let password = password_secret(body.password); if username.is_some() != password.is_some() { @@ -350,74 +420,76 @@ pub async fn create( if !(1..=16).contains(&body.concurrency) { return bad_request("concurrency must be between 1 and 16"); } - let images = state.formats.stores_images(&repo.format); - let registry_url = match body - .registry_url - .map(|u| u.trim().trim_end_matches('/').to_string()) - { - Some(_) if !images => { - return bad_request("registry_url applies to Docker repositories only") - } - Some(u) if u.is_empty() => None, - Some(u) => match validate_url(&u) { - Ok(()) => Some(u), - Err(e) => return bad_request(format!("invalid registry_url: {e}")), - }, - None => None, - }; - // The preflight: is it Nexus, and the right repository? - let nexus = Nexus::new( - state.upstream.clone(), - &url, - username.clone(), - password.clone(), - ); - let version = match nexus.status().await { - Ok(version) => version, - Err(e) => return preflight_error(&format!("Nexus at {url}"), e), - }; - match nexus.repository(&source_repository).await { - Ok(Some(source)) => { - if source.kind != "hosted" { - return bad_request(format!( - "{source_repository} is a {} repository in Nexus; only hosted repositories are imported (a proxy refills from its upstream, a group is configuration)", - source.kind - )); + // The preflight: does the source speak the format's protocol, and list + // what to copy? + let (server, total) = if images { + let credentials = match (&username, &password) { + (Some(username), Some(password)) => Some(Credentials::for_url( + &source, + username.clone(), + password.clone(), + )), + _ => None, + }; + let from = ImportSource::new(&source, credentials, "import:preflight".into()); + let registry = state.registry(); + let server = match registry.probe_source(&from).await { + Ok(server) => server, + Err(e) => { + return preflight_error( + &format!("the registry API does not answer at {source}/v2/"), + SourceError::of_image(e), + ) } - if source.format != repo.format { - return bad_request(format!( - "{source_repository} is a {} repository in Nexus, but {} holds {}", - source.format, repo.name, repo.format - )); + }; + if names.is_none() { + match registry.source_catalog(&from, None, 1).await { + Ok(Some(_)) => {} + Ok(None) => { + return bad_request(format!( + "the registry at {source} serves no catalog (/v2/_catalog); name the images to import" + )) + } + Err(e) => { + return preflight_error("reading the registry's catalog", SourceError::of_image(e)) + } } } - Ok(None) => { - return bad_request(format!( - "Nexus has no repository {source_repository}, or the credentials may not see it" - )); - } - Err(e) => return preflight_error("listing the Nexus repositories", e), - } - if images { - let registry = registry_url - .clone() - .unwrap_or_else(|| default_registry_url(&url, &source_repository)); - let probe = Nexus::new( + (server, None) + } else { + let feed = match Feed::open( state.upstream.clone(), - ®istry, + format, + &source, username.clone(), password.clone(), - ); - if let Err(e) = probe.probe_registry(&format!("{registry}/v2/")).await { - return preflight_error( - &format!( - "the registry API of {source_repository} does not answer at {registry}/v2/ (set registry_url to the repository's connector, e.g. http://nexus.example.com:8082)" - ), - e, - ); - } - } + ) + .await + { + Ok(feed) => feed, + Err(e) => return preflight_error(&format!("the NuGet feed at {source}"), e), + }; + let total = if names.is_none() { + // A page's worth: Nexus reports the hits of the page it answers + // as the search's total. + match feed.search(0, SEARCH_PAGE).await { + Ok(Some((total, ids))) => { + Some(i64::try_from(total.max(ids.len() as u64)).unwrap_or(i64::MAX)) + } + Ok(None) => { + return bad_request(format!( + "the NuGet feed at {source} offers no search; name the packages to import" + )) + } + Err(e) => return preflight_error("searching the feed", e), + } + } else { + None + }; + (feed.server().map(str::to_string), total) + }; + let listing = Listing::of(images, names.is_some()); let created = async { let mut tx = forge_db::begin(&state.pool).await?; @@ -425,14 +497,15 @@ pub async fn create( &mut *tx, NewImport { repository_id: repo.id, - base_url: &url, - source_repository: &source_repository, - registry_url: registry_url.as_deref(), + url: &source, + names: names.as_deref(), username: username.as_deref(), on_conflict: &body.on_conflict, concurrency: body.concurrency, dry_run: body.dry_run, - nexus_version: Some(&version), + scan_listing: listing.as_str(), + source_server: server.as_deref(), + source_total: total, created_by: Some(user_id), }, ) @@ -462,11 +535,11 @@ pub async fn create( tracing::info!( import = id, repository = %repo.name, - source = %url, - source_repository = %source_repository, - nexus = %version, + source = %source, + server = server.as_deref(), + total, dry_run = body.dry_run, - "nexus import queued" + "import queued" ); match import_dao::get(&state.pool, id).await { Ok(Some(import)) => match view(&state, &repo, import).await { @@ -563,11 +636,11 @@ pub async fn resume( } let password = password_secret(body.password); if password.is_some() && import.username.is_none() { - return bad_request("the import reads Nexus anonymously; it takes no password"); + return bad_request("the import reads its source anonymously; it takes no password"); } if import.username.is_some() && !import.has_password && password.is_none() { return bad_request( - "the import's password was deleted when it completed; give the Nexus password again", + "the import's password was deleted when it completed; give the source's password again", ); } let resumed = async { diff --git a/crates/server/src/admin/mod.rs b/crates/server/src/admin/mod.rs index c17790d8..af2dcdda 100644 --- a/crates/server/src/admin/mod.rs +++ b/crates/server/src/admin/mod.rs @@ -24,8 +24,9 @@ //! * [`blob_cache`] — the local blob cache's statistics. //! * [`proxy_cache`] — invalidating what a proxy repository cached from its //! upstream (its "not found" answers, listings and documents). -//! * [`imports`] — importing a Nexus hosted repository into a hosted -//! repository (start with a preflight, progress, cancel, resume, retry). +//! * [`imports`] — importing a repository from another server (a NuGet feed, +//! a registry) into a hosted repository (start with a preflight, progress, +//! cancel, resume, retry). //! * [`email`] — the email service's settings and a test send. //! * [`email_oauth`] — registering and authorizing Forge for the email //! service's SMTP with OAuth. diff --git a/crates/server/src/images/import.rs b/crates/server/src/images/import.rs index b44eccd1..2e7055e2 100644 --- a/crates/server/src/images/import.rs +++ b/crates/server/src/images/import.rs @@ -1,6 +1,8 @@ -//! Imports into a hosted repository from another registry's API (a Nexus -//! hosted repository's): a tag with everything it references, copied the -//! way `crane copy` copies it, and stored through the same checks as a push. +//! Imports into a hosted repository from another registry's API (Nexus, +//! Harbor, a `distribution` registry, another Forge): what the source lists +//! (its catalog, tag lists and referrers, [`Registry::source_catalog`] and +//! its siblings), and a tag with everything it references, copied the way +//! `crane copy` copies it, and stored through the same checks as a push. //! //! * Digests are kept: a manifest is stored as the source served it, once //! its bytes matched the digest it was asked for (a tag's: the source's @@ -36,9 +38,10 @@ use forge_core::repo::proxy::Credentials; use forge_db::model::{RepoKind, Repository}; use forge_db::oci_dao::{self, LinkOutcome, TagWrite}; use forge_plugin::api::images::{ - ImageError, ManifestBlob, ParsedManifest, Reference, MAX_MANIFEST_BYTES, + CatalogPage, ImageError, ManifestBlob, ManifestInfo, ParsedManifest, Reference, TagPage, + MAX_MANIFEST_BYTES, }; -use forge_plugin::api::UpstreamOp; +use forge_plugin::api::{UpstreamAnswer, UpstreamOp}; use futures_util::TryStreamExt; use reqwest::Method; @@ -80,6 +83,99 @@ impl ImportSource { } } +impl Registry { + /// Check that `source` answers the registry API (`GET /v2/`: 200, or 200 + /// once its token challenge is answered). Returns the `Server` header it + /// answered with. + pub async fn probe_source(&self, source: &ImportSource) -> ImgResult> { + let upstream = source.upstream(self); + let base = source.base.trim_end_matches('/'); + let url = format!("{}/v2/", base.strip_suffix("/v2").unwrap_or(base)); + let response = upstream + .send(Method::GET, &url, None, None, RequestClass::Metadata) + .await?; + if !response.status().is_success() { + return Err(source_status(response.status(), "/v2/")); + } + Ok(response + .headers() + .get(reqwest::header::SERVER) + .and_then(|v| v.to_str().ok()) + .map(str::to_string)) + } + + /// Up to `n` images of the source's catalog after `last`; `None` when it + /// serves no catalog. + pub async fn source_catalog( + &self, + source: &ImportSource, + last: Option<&str>, + n: u32, + ) -> ImgResult> { + let op = UpstreamOp::ImageCatalog { + last: last.map(str::to_string), + n, + }; + match source.upstream(self).converse(self, &op).await? { + UpstreamAnswer::ImageCatalog(page) => Ok(page), + UpstreamAnswer::Failed(reason) => Err(Failure::unavailable(reason)), + other => Err(Failure::internal( + "source catalog", + format!("the images format answered {other:?}"), + )), + } + } + + /// Up to `n` tags of `image` at the source after `last`; `None` when the + /// source does not know the image. + pub async fn source_tags( + &self, + source: &ImportSource, + image: &str, + last: Option<&str>, + n: u32, + ) -> ImgResult> { + let op = UpstreamOp::ImageTags { + image: image.to_string(), + last: last.map(str::to_string), + n, + }; + match source.upstream(self).converse(self, &op).await? { + UpstreamAnswer::ImageTags(page) => Ok(page), + UpstreamAnswer::Failed(reason) => Err(Failure::unavailable(reason)), + other => Err(Failure::internal( + "source tags", + format!("the images format answered {other:?}"), + )), + } + } + + /// The referrers the source lists for the manifest `digest` of `image` + /// (its referrers API); `None` when it has no such API: a referrer is + /// then reached through a tag (the referrers tag schema's + /// `sha256-`, cosign's `sha256-.sig`), which an import + /// copies like any other. + pub async fn source_referrers( + &self, + source: &ImportSource, + image: &str, + digest: &str, + ) -> ImgResult>> { + let op = UpstreamOp::ImageReferrers { + image: image.to_string(), + digest: digest.to_string(), + }; + match source.upstream(self).converse(self, &op).await? { + UpstreamAnswer::ImageReferrers(found) => Ok(found), + UpstreamAnswer::Failed(reason) => Err(Failure::unavailable(reason)), + other => Err(Failure::internal( + "source referrers", + format!("the images format answered {other:?}"), + )), + } + } +} + /// How an import writes an image. #[derive(Debug, Clone)] pub struct ImageImportOptions { diff --git a/crates/server/src/import/feed.rs b/crates/server/src/import/feed.rs new file mode 100644 index 00000000..7c68352b --- /dev/null +++ b/crates/server/src/import/feed.rs @@ -0,0 +1,332 @@ +//! A NuGet v3 feed as an import's source, read through the NuGet format's +//! own upstream protocol ([`Format::upstream`], the conversations a proxy +//! repository holds with its upstream): +//! +//! * the service index (`…/index.json`) resolves the feed's resources +//! ([`UpstreamOp::Index`]); +//! * its search (`SearchQueryService`) names the package ids, a page at a +//! time ([`Feed::search`]); +//! * its package base address lists each package's versions, unlisted ones +//! included ([`Feed::versions`]), and serves the `.nupkg` +//! ([`Feed::package_url`]); +//! * its registration says when each version was published, and its SHA-512 +//! where the feed lists one inline ([`Feed::metadata`]). +//! +//! Requests go only where the feed points: the origin of its service index, +//! and those its documents name ([`UpstreamOrigins`]). The credentials go to +//! the service index's origin alone, through the shared [`UpstreamHttp`] +//! client (timeouts, size caps, no cloud-metadata targets). + +use std::sync::{Arc, Mutex}; + +use forge_core::format::Format; +use forge_core::net::origins::{named_origins, UpstreamOrigins}; +use forge_core::net::upstream::{Origin, RequestClass, UpstreamAuth, UpstreamError, UpstreamHttp}; +use forge_crypto::{ExposeSecret, PlaintextSecret}; +use forge_plugin::api::{ + FetchResult, Fetched, PackageRef, PackageVersions, Remote, UpstreamAnswer, UpstreamOp, + UpstreamStep, VersionMetadata, +}; +use reqwest::StatusCode; + +use super::SourceError; + +/// Rounds of one conversation, and fetches in all: a version list or a +/// registration takes a few. +const MAX_ROUNDS: usize = 8; +const MAX_FETCHES: usize = 256; + +/// A NuGet v3 feed. +pub struct Feed { + http: UpstreamHttp, + format: Arc, + /// The service index, and the resources it lists (once resolved). + remote: Remote, + origins: Mutex, + username: Option, + password: Option, + /// The `Server` header the service index was served with. + server: Option, +} + +impl Feed { + /// The feed at `url` (its service index), resolved: its resources are + /// known once this returns. + pub async fn open( + http: UpstreamHttp, + format: Arc, + url: &str, + username: Option, + password: Option, + ) -> Result { + let mut feed = Feed { + http, + format, + remote: Remote { + url: url.to_string(), + index: None, + }, + origins: Mutex::new(UpstreamOrigins::new(url, [])), + username, + password, + server: None, + }; + // The service index once directly, for its `Server` header and a + // plain answer to an unreachable or refusing feed. + let response = feed + .http + .get(url, RequestClass::Metadata, feed.auth().as_ref()) + .await + .map_err(|e| SourceError::of(e, url))?; + if !response.status().is_success() { + return Err(SourceError::status(response.status(), url)); + } + feed.server = response + .headers() + .get(reqwest::header::SERVER) + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + match feed.converse(UpstreamOp::Index).await? { + UpstreamAnswer::Index(index) => feed.remote.index = Some(index), + other => { + return Err(SourceError::Invalid(format!( + "{url} is no NuGet v3 service index: {}", + answer_text(&other) + ))) + } + } + Ok(feed) + } + + /// The `Server` header the service index was served with. + pub fn server(&self) -> Option<&str> { + self.server.as_deref() + } + + /// The Basic credentials, for the service index's origin only. + fn auth(&self) -> Option> { + match (&self.username, &self.password) { + (Some(username), Some(password)) => Some(UpstreamAuth { + origin: Origin::of(&self.remote.url), + username, + password: password.expose_secret(), + bearer: false, + }), + _ => None, + } + } + + /// A page of the feed's search for every package (prerelease and SemVer + /// 2.0.0 versions included): its total and the ids, after `skip`. + /// `None` when the feed offers no search. + pub async fn search( + &self, + skip: u32, + take: u32, + ) -> Result)>, SourceError> { + match self.converse(UpstreamOp::Packages { skip, take }).await? { + UpstreamAnswer::Search(None) => Ok(None), + UpstreamAnswer::Search(Some(page)) => Ok(Some(( + page.total, + page.hits.into_iter().map(|hit| hit.id).collect(), + ))), + other => Err(unexpected("search", &other)), + } + } + + /// Every version of the package `id` the feed lists (normalized), + /// unlisted ones included; `None` when it does not know the package. + pub async fn versions(&self, id: &str) -> Result>, SourceError> { + let id = self.format.lower_id(id); + match self.converse(UpstreamOp::Versions(id)).await? { + UpstreamAnswer::Versions(versions) => Ok(versions), + other => Err(unexpected("versions", &other)), + } + } + + /// What the feed's registration says of `versions` of `id`: when each + /// was published, whether it is listed, its SHA-512 where listed. Best + /// effort: a version the registration does not have is left out. + pub async fn metadata( + &self, + id: &str, + versions: &[String], + ) -> Result, SourceError> { + let wanted = PackageVersions { + id: self.format.lower_id(id), + versions: versions.to_vec(), + }; + match self.converse(UpstreamOp::Metadata(wanted)).await? { + UpstreamAnswer::Metadata(metadata) => Ok(metadata.versions), + other => Err(unexpected("registration", &other)), + } + } + + /// Where the feed serves the package of `id` `version`: on an origin the + /// feed names, or an error. + pub async fn package_url(&self, id: &str, version: &str) -> Result { + let target = PackageRef { + id: self.format.lower_id(id), + version: version.to_string(), + }; + let url = match self.converse(UpstreamOp::Package(target)).await? { + UpstreamAnswer::Package(url) => url, + UpstreamAnswer::Located(location) => location.url, + other => return Err(unexpected("package", &other)), + }; + if !self.permits(&url) { + return Err(SourceError::Invalid(format!( + "the feed names the package at {url}, an origin it does not otherwise name" + ))); + } + Ok(url) + } + + /// GET a package (streamed by the caller). + pub async fn download(&self, url: &str) -> Result { + let response = self + .http + .get(url, RequestClass::Download, self.auth().as_ref()) + .await + .map_err(|e| SourceError::of(e, url))?; + if !response.status().is_success() { + return Err(SourceError::status(response.status(), url)); + } + Ok(response) + } + + fn permits(&self, url: &str) -> bool { + self.origins + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .permits(url) + } + + /// One conversation of the format with the feed about `op`: the format + /// names what to fetch, the feed's answers go back to it, until it + /// answers. A failed answer after a fetch that failed is that fetch's + /// failure (an unreachable or refusing feed). + async fn converse(&self, op: UpstreamOp) -> Result { + let mut fetched: Vec = Vec::new(); + let mut trouble: Option = None; + for _ in 0..MAX_ROUNDS { + let (format, remote, round) = (self.format.clone(), self.remote.clone(), op.clone()); + let (step, back) = tokio::task::spawn_blocking(move || { + let step = format.upstream(&remote, &round, &fetched); + (step, fetched) + }) + .await + .map_err(|e| SourceError::Invalid(format!("the NuGet protocol: {e}")))?; + fetched = back; + let wanted = + match step.map_err(|e| SourceError::Invalid(format!("the NuGet protocol: {e}")))? { + UpstreamStep::Done(UpstreamAnswer::Failed(reason)) => { + return Err(match trouble { + Some(trouble) => trouble, + None => SourceError::Invalid(reason), + }) + } + UpstreamStep::Done(answer) => return Ok(answer), + UpstreamStep::Fetch(wanted) => wanted, + }; + if wanted.is_empty() || fetched.len() + wanted.len() > MAX_FETCHES { + return Err(SourceError::Invalid(format!( + "the NuGet protocol asked for {} fetches after {}", + wanted.len(), + fetched.len() + ))); + } + for fetch in wanted { + let (outcome, status) = if !self.permits(&fetch.url) { + ( + Fetched::Failed(format!( + "{}: not sent, its origin is neither the service index's nor one \ + the feed names", + fetch.url + )), + None, + ) + } else { + let got = self + .http + .get_bytes_accepting( + &fetch.url, + self.auth().as_ref(), + fetch.accept.as_deref(), + ) + .await; + match got { + Ok(Some(body)) => { + self.origins + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .extend(named_origins(&body)); + (Fetched::Found(body), Some(200)) + } + Ok(None) => (Fetched::NotFound, Some(404)), + Err(e) => { + let error = SourceError::of(e, &fetch.url); + let status = error.status_code(); + let reason = error.to_string(); + trouble.get_or_insert(error); + (Fetched::Failed(reason), status) + } + } + }; + fetched.push(FetchResult { + fetch, + outcome, + status, + headers: Vec::new(), + }); + } + } + Err(SourceError::Invalid(format!( + "the NuGet protocol needed more than {MAX_ROUNDS} rounds" + ))) + } +} + +/// What an answer was, for an error message. +fn answer_text(answer: &UpstreamAnswer) -> String { + match answer { + UpstreamAnswer::Failed(reason) => reason.clone(), + other => format!("{other:?}"), + } +} + +fn unexpected(what: &str, answer: &UpstreamAnswer) -> SourceError { + SourceError::Invalid(format!("the feed's {what}: {}", answer_text(answer))) +} + +impl SourceError { + /// A transport or status error of the upstream client for `url`. + pub fn of(e: UpstreamError, url: &str) -> Self { + match e { + UpstreamError::Status { status, url } => SourceError::status(status, &url), + UpstreamError::Json { source, .. } => SourceError::Invalid(format!("{url}: {source}")), + other => SourceError::Unreachable(other.to_string()), + } + } + + /// An answer of `status` for `url` that is not the one asked for. + pub fn status(status: StatusCode, url: &str) -> Self { + if matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) { + SourceError::Refused(format!( + "the source refused the credentials ({status}) for {url}" + )) + } else { + SourceError::Status { + status, + url: url.to_string(), + } + } + } + + fn status_code(&self) -> Option { + match self { + SourceError::Status { status, .. } => Some(status.as_u16()), + _ => None, + } + } +} diff --git a/crates/server/src/import/mod.rs b/crates/server/src/import/mod.rs new file mode 100644 index 00000000..70399795 --- /dev/null +++ b/crates/server/src/import/mod.rs @@ -0,0 +1,1353 @@ +//! Importing a repository from another server into a Forge hosted +//! repository of the same format (NuGet, Docker), one repository at a time. +//! The source is read through the format's own protocol, so it may be any +//! server that speaks it: Nexus, Artifactory, ProGet, BaGet, Harbor, a +//! `distribution` registry or another Forge. +//! +//! * **NuGet:** a feed's service index (`…/index.json`, [`feed::Feed`]): +//! its search names the package ids, its package base address lists each +//! package's versions (unlisted ones included) and serves the `.nupkg`, +//! its registration says when each version was published and, where the +//! feed lists it inline, its SHA-512. +//! * **Docker:** a registry's API ([`ImportSource`]): its catalog +//! (`/v2/_catalog`) names the images, their tag lists the tags, and the +//! referrers API the signatures and SBOMs of each copied manifest. +//! +//! An import can name its package ids or images instead (`names`): a feed's +//! search may miss some (Nexus stops at 10,000 packages; a package whose +//! versions are all unlisted is in no search result), and Docker Hub, GHCR +//! and ECR serve no catalog. +//! +//! An administrator starts an import on a hosted repository +//! ([`crate::admin::imports`]); this worker runs it, on one instance at a +//! time: it claims a queued import (or one whose instance died) with a +//! lease in Postgres and renews the lease while it works, like the +//! retention worker. +//! +//! 1. **Scan.** The source's listing (the search, the catalog or the names) +//! is walked page by page; each page's versions or tags become pending +//! items and the page's cursor the import's, in one transaction, so a +//! scan resumes after a restart, a failure or a cancel at the page it +//! had not recorded. +//! 2. **Copy.** Pending items are copied, `concurrency` at a time: +//! * a package (NuGet) is streamed from the feed into staging, verified +//! against the SHA-512 the feed lists (where it lists one), and written +//! through the hosted write path ([`HostedWrite::import_staged`]): +//! format validation, the package selectors of the administrator who +//! started the import, the repository's upload limit, CAS dedup and +//! the audit trail (`action = "import"`). A version Forge holds with +//! the SHA-512 the feed lists is skipped without a download; without +//! one, the download is compared; +//! * a Docker tag is copied through the registry +//! ([`crate::images::Registry::import_tag`]): digests are kept, +//! multi-platform indexes arrive with every child manifest. The +//! referrers the source's referrers API lists for the tag's manifest +//! become items of their own, copied once their subject is in Forge +//! ([`crate::images::Registry::import_referrer`]); a source without +//! the API serves them under tags (the referrers tag schema's +//! `sha256-`, cosign's `sha256-.sig`), copied like any +//! other; +//! * what the source says is kept: a NuGet version's publish time becomes +//! its creation time (nuget.org's `1900-01-01` for an unlisted version +//! is no publish time). The protocols carry no last download, so an +//! imported version counts as downloaded at its import, and no push +//! time for a tag, so a tag is created at its import; +//! * a version Forge holds with other bytes is a conflict: skipped and +//! reported (`skip`), replaced (`overwrite`), or it fails the import +//! (`fail`). +//! +//! A dry run scans and decides every item (`planned`, `identical`, +//! `conflict`) without writing a package; it downloads no package. Running +//! an import again later copies only what is new at the source, so a +//! cutover is an import, then a final one just before clients switch to +//! Forge. +//! +//! An import stored before imports read the format's protocol names a +//! Nexus server and repository; it reads the same repository through its +//! protocol ([`source_url`]). +//! +//! A failed item can be retried on its own, and a cancelled or failed +//! import resumed ([`forge_db::import_dao::requeue`]). The source's +//! password is sealed with the secret encryption and deleted when the +//! import completes. + +pub mod feed; + +use std::collections::HashMap; +use std::panic::AssertUnwindSafe; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use chrono::{DateTime, Datelike, Utc}; +use forge_core::auth::principal::{AuthMethod, Principal}; +use forge_core::authz::Level; +use forge_core::repo::hosted::HostedWrite; +use forge_core::repo::proxy::Credentials; +use forge_core::repo::{ImportOptions, ImportOutcome, RepoKind, WriteError}; +use forge_crypto::PlaintextSecret; +use forge_db::import_dao::{self, ImportItem, NewItem, RepositoryImport}; +use forge_db::model::Repository; +use forge_db::{repository_dao, secret_dao, user_dao}; +use forge_plugin::api::images::ImageError; +use forge_plugin::api::VersionMetadata; +use forge_storage::StagedBlob; +use futures_util::future::BoxFuture; +use futures_util::{FutureExt, StreamExt, TryStreamExt}; +use reqwest::StatusCode; +use serde::{Deserialize, Serialize}; +use tokio::task::JoinHandle; +use uuid::Uuid; + +use crate::images::import::{ImageImportOptions, ImageOutcome, ImportSource}; +use crate::images::{Failure, Registry}; +use crate::state::AppState; +use feed::Feed; + +/// The formats an import copies: NuGet packages, and Docker images into +/// the image store. +pub const FORMATS: &[&str] = &["nuget", "docker"]; + +/// How often an idle instance looks for an import to run. +const POLL_INTERVAL: Duration = Duration::from_secs(5); + +/// The lease a running import is held with; renewed every third of it. +const LEASE_SECONDS: i64 = 120; + +/// Pending items read at a time. +const BATCH: i64 = 64; + +/// Items failing in a row that stop an import (the source went away, the +/// credentials stopped working): it fails, and can be resumed. +const MAX_CONSECUTIVE_FAILURES: usize = 25; + +/// Attempts at a read of the source's listing before the import fails. +const PAGE_ATTEMPTS: u32 = 4; + +/// How often a copy looks whether it was cancelled (or lost its lease). +const CHECK_INTERVAL: Duration = Duration::from_secs(5); + +/// Package ids a page of a feed's search names. +pub const SEARCH_PAGE: u32 = 100; + +/// Images a page of a registry's catalog names. +const CATALOG_PAGE: u32 = 100; + +/// Tags read at a time. +const TAGS_PAGE: u32 = 1000; + +/// Named package ids or images a scan page reads. +const NAMES_PAGE: usize = 20; + +/// The most package ids or images an import names. +pub const MAX_NAMES: usize = 10_000; + +/// What went wrong reading the source. +#[derive(Debug, thiserror::Error)] +pub enum SourceError { + /// The source could not be reached, or the transfer failed. + #[error("the source is unreachable: {0}")] + Unreachable(String), + /// The source cannot answer now (a registry's error, its rate limit). + #[error("{0}")] + Unavailable(String), + /// The credentials were refused (or may not see the resource). + #[error("{0}")] + Refused(String), + /// Any other answer than the one asked for. + #[error("the source answered {status} for {url}")] + Status { status: StatusCode, url: String }, + /// The answer was not what the protocol describes. + #[error("{0}")] + Invalid(String), +} + +impl SourceError { + /// Whether trying again later may succeed. + pub fn is_transient(&self) -> bool { + match self { + SourceError::Unreachable(_) | SourceError::Unavailable(_) => true, + SourceError::Status { status, .. } => { + status.is_server_error() || *status == StatusCode::TOO_MANY_REQUESTS + } + SourceError::Refused(_) | SourceError::Invalid(_) => false, + } + } + + /// A failure of the registry reading the source. + pub fn of_image(e: Failure) -> Self { + let text = e.message(); + match e.error { + ImageError::Unavailable(_) | ImageError::RateLimited => SourceError::Unavailable(text), + ImageError::InvalidCredentials | ImageError::Denied { .. } => { + SourceError::Refused(text) + } + _ => SourceError::Invalid(text), + } + } +} + +/// What an import's scan walks (its `scan_listing`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Listing { + /// A NuGet feed's search. + Search, + /// A registry's catalog. + Catalog, + /// The package ids or images the import names. + Names, +} + +impl Listing { + /// The listing of a new import. + pub fn of(images: bool, named: bool) -> Listing { + match (images, named) { + (_, true) => Listing::Names, + (true, false) => Listing::Catalog, + (false, false) => Listing::Search, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Listing::Search => "search", + Listing::Catalog => "catalog", + Listing::Names => "names", + } + } + + fn parse(s: &str) -> Listing { + match s { + "catalog" => Listing::Catalog, + "names" => Listing::Names, + _ => Listing::Search, + } + } +} + +/// Where in a Nexus server a repository is: `/repository/`. +/// Nexus serves a NuGet repository's service index below it, and a Docker +/// repository's registry API at it (for reads, on its own port), unless the +/// repository has a connector of its own (`registry_url`). +pub fn nexus_source_url( + url: &str, + repository: &str, + registry_url: Option<&str>, + images: bool, +) -> String { + let path = format!("{}/repository/{repository}", url.trim_end_matches('/')); + match registry_url { + Some(registry_url) if images => registry_url.to_string(), + _ if images => path, + _ => format!("{path}/index.json"), + } +} + +/// The source `import` reads: its `url`, or, for an import stored before +/// imports read the format's protocol (a Nexus server and repository), that +/// repository's service index or registry API ([`nexus_source_url`]). +pub fn source_url(import: &RepositoryImport, images: bool) -> String { + match &import.source_repository { + Some(repository) => nexus_source_url( + &import.url, + repository, + import.registry_url.as_deref(), + images, + ), + None => import.url.clone(), + } +} + +/// The worker running imports on this instance. +pub struct ImportWorker { + state: AppState, +} + +impl ImportWorker { + pub fn new(state: AppState) -> Self { + ImportWorker { state } + } + + /// Look for an import every few seconds and run each that is due. + pub fn spawn_loop(self) -> JoinHandle<()> { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(POLL_INTERVAL); + loop { + ticker.tick().await; + loop { + match self.tick().await { + Ok(Some(_)) => {} + Ok(None) => break, + Err(e) => { + tracing::error!(error = %e, "import tick failed"); + break; + } + } + } + } + }) + } + + /// Claim one import that is due and run it until it completes, fails, + /// is cancelled or loses its lease. Returns its id, `None` when no + /// import is due. + pub async fn tick(&self) -> sqlx::Result> { + let pool = &self.state.pool; + let claimed = { + let mut tx = forge_db::begin(pool).await?; + let now = forge_db::db_now(&mut *tx).await?; + let claimed = + import_dao::claim_next(&mut tx, now, chrono::Duration::seconds(LEASE_SECONDS)) + .await?; + tx.commit().await?; + claimed + }; + let Some((import, run_id)) = claimed else { + return Ok(None); + }; + let id = import.id; + tracing::info!( + import = id, + repository_id = import.repository_id, + source = %import.url, + source_repository = import.source_repository.as_deref(), + phase = %import.phase, + "import started" + ); + let lost = Arc::new(AtomicBool::new(false)); + let heartbeat = AbortOnDrop(spawn_heartbeat(pool.clone(), id, run_id, lost.clone())); + let run = Run { + state: &self.state, + import, + run_id, + lost, + stopping: AtomicBool::new(false), + last_search: std::sync::Mutex::new(Vec::new()), + }; + let end = AssertUnwindSafe(run.execute()) + .catch_unwind() + .await + .unwrap_or_else(|panic| { + End::Failed(format!("the import panicked: {}", panic_text(&*panic))) + }); + drop(heartbeat); + let (state, error) = match &end { + End::Completed => ("completed", None), + End::Cancelled => ("cancelled", None), + End::Failed(error) => ("failed", Some(error.as_str())), + End::LeaseLost => { + tracing::warn!( + import = id, + "import lease lost; another instance continues it" + ); + return Ok(Some(id)); + } + }; + if import_dao::finish(pool, id, run_id, state, error).await? { + tracing::info!(import = id, state, error, "import ended"); + } else { + tracing::warn!(import = id, "import lease lost before it ended"); + } + Ok(Some(id)) + } +} + +/// How a run ended. +enum End { + Completed, + Cancelled, + Failed(String), + /// Another instance holds the import now; it records the outcome. + LeaseLost, +} + +fn db_failed(e: sqlx::Error) -> End { + tracing::error!(error = %e, "import: database error"); + End::Failed(format!("database error: {e}")) +} + +/// Where the items of a run come from and go. +enum Target { + /// A package format: from a feed, through the hosted write path. + Packages { + feed: Box, + writer: Box, + }, + /// Docker: from a registry's API, through the registry. + Images { + registry: Registry, + source: ImportSource, + }, +} + +/// What a run works with. +struct Ctx { + repo: Repository, + principal: Principal, + target: Target, + /// The repository's upload limit. + limit: u64, + /// The source's URL, for the audit trail. + source: String, +} + +/// What the source says of an item beyond its name and version (an item's +/// `source`). An item an import stored before reads as none of it. +#[derive(Debug, Default, Serialize, Deserialize)] +struct ItemSource { + /// The package's SHA-512, base64, where the feed lists it. + #[serde(default, skip_serializing_if = "Option::is_none")] + sha512: Option, +} + +/// A version or tag the scan found. +struct Found { + name: String, + /// Empty for a named package or image the source does not have. + version: String, + published_at: Option>, + source: ItemSource, +} + +impl Found { + /// A named package or image the source does not have: an item that + /// fails, so the import reports it. + fn missing(name: &str) -> Found { + Found { + name: name.to_string(), + version: String::new(), + published_at: None, + source: ItemSource::default(), + } + } +} + +/// One page of the scan: what it found, and the cursor of the next page +/// (`None`: the scan is complete). +struct Page { + found: Vec, + next: Option, + /// A search: how many packages it has walked with this page. + walked: Option, +} + +/// What became of one item. +enum Copied { + /// It ends in `state`. + Item { + state: &'static str, + detail: Option, + bytes: u64, + }, + /// A manifest the source holds by digest that is no referrer: forgotten. + NotReferrer, +} + +/// How a recorded item bears on the run. +enum Recorded { + Done, + /// The run is ending: the item was left pending. + NotStarted, + Failed(String), + /// A conflict, and the import fails on conflicts. + Conflict(String), +} + +/// One run of one import, on this instance. +struct Run<'a> { + state: &'a AppState, + import: RepositoryImport, + run_id: Uuid, + /// Set by the heartbeat once the lease is lost. + lost: Arc, + /// Set once the run is to end: no further item is started. + stopping: AtomicBool, + /// The ids of the search page read last: a feed whose search ignores + /// `skip` answers the same page again. + last_search: std::sync::Mutex>, +} + +impl Run<'_> { + async fn execute(&self) -> End { + let ctx = match self.prepare().await { + Ok(ctx) => ctx, + Err(end) => return end, + }; + if self.import.phase == "scan" { + if let Err(end) = self.scan(&ctx).await { + return end; + } + } + match self.copy(&ctx).await { + Ok(()) => End::Completed, + Err(end) => end, + } + } + + /// The repository, the administrator the import runs as, and the + /// clients of the source. + async fn prepare(&self) -> Result { + let pool = &self.state.pool; + let import = &self.import; + let repo = repository_dao::find_repository_by_id(pool, import.repository_id) + .await + .map_err(db_failed)? + .ok_or_else(|| End::Failed("the repository was deleted".into()))?; + if repo.kind() != RepoKind::Hosted || !FORMATS.contains(&repo.format.as_str()) { + return Err(End::Failed(format!( + "imports go to a hosted NuGet or Docker repository; {} is a {} {} repository", + repo.name, repo.format, repo.kind + ))); + } + + // The import runs with the authority of the administrator who + // started it, as long as they are one. + let gone = + || End::Failed("the administrator who started the import no longer is one".into()); + let user_id = import.created_by.ok_or_else(gone)?; + let user = user_dao::find_user_by_id(pool, user_id) + .await + .map_err(db_failed)? + .filter(|user| user.active) + .ok_or_else(gone)?; + let principal = Principal::User { + user_id: user.id, + token_id: None, + method: AuthMethod::Session, + }; + if !self.state.authz.is_global_admin(&principal).await { + return Err(gone()); + } + + let password = secret_dao::get_import_password(pool, &self.state.envelope, import.id) + .await + .map_err(|e| End::Failed(format!("the import's password cannot be read: {e}")))?; + if import.username.is_some() && password.is_none() { + return Err(End::Failed( + "the import has no password; resume it with the source's password".into(), + )); + } + + let images = self.state.formats.stores_images(&repo.format); + let url = source_url(import, images); + let target = if images { + let credentials = match (&import.username, password) { + (Some(username), Some(password)) => { + Some(Credentials::for_url(&url, username.clone(), password)) + } + _ => None, + }; + Target::Images { + registry: self.state.registry(), + source: ImportSource::new(&url, credentials, format!("import:{}", import.id)), + } + } else { + let format = self + .state + .formats + .for_repository(&repo) + .map_err(|e| End::Failed(e.to_string()))?; + let feed = self + .retrying("reading the feed's service index", || { + let (http, format, url) = + (self.state.upstream.clone(), format.clone(), url.clone()); + let (username, password) = (import.username.clone(), password.clone()); + async move { Feed::open(http, format, &url, username, password).await }.boxed() + }) + .await?; + let authz = &self.state.authz; + let filter = authz + .package_filter_for(&principal, &repo, Level::Write) + .await; + let replace = authz + .package_filter_for(&principal, &repo, Level::Delete) + .await; + let role_filter = authz + .role_package_filter_for(&principal, &repo, Level::Write) + .await; + let role_replace = authz + .role_package_filter_for(&principal, &repo, Level::Delete) + .await; + let writer = Box::new(crate::dispatch::hosted_write( + self.state, + format, + repo.id, + repo.name.clone(), + principal.clone(), + filter, + replace, + role_filter, + role_replace, + )); + Target::Packages { + feed: Box::new(feed), + writer, + } + }; + Ok(Ctx { + limit: self.state.upload_limit(&repo), + source: url, + repo, + principal, + target, + }) + } + + /// Stop when the lease is lost or an administrator cancelled. + async fn check(&self) -> Result<(), End> { + if self.lost.load(Ordering::Relaxed) { + return Err(End::LeaseLost); + } + match import_dao::poll(&self.state.pool, self.import.id, self.run_id).await { + Ok(None) => Err(End::LeaseLost), + Ok(Some(true)) => Err(End::Cancelled), + Ok(Some(false)) => Ok(()), + Err(e) => Err(db_failed(e)), + } + } + + /// A read of the source, retried with a backoff while its failure may + /// pass. + async fn retrying<'f, T>( + &self, + what: &str, + mut read: impl FnMut() -> BoxFuture<'f, Result> + Send, + ) -> Result { + let mut attempt = 0; + loop { + attempt += 1; + match read().await { + Ok(read) => return Ok(read), + Err(e) if e.is_transient() && attempt < PAGE_ATTEMPTS => { + tracing::warn!(import = self.import.id, error = %e, attempt, "{what} failed; retrying"); + tokio::time::sleep(Duration::from_secs(1 << attempt)).await; + } + Err(e) => return Err(End::Failed(format!("{what}: {e}"))), + } + } + } + + /// Walk the listing from the stored cursor, recording each page's + /// items with the cursor of the next page. + async fn scan(&self, ctx: &Ctx) -> Result<(), End> { + let listing = Listing::parse(&self.import.scan_listing); + let mut cursor = self.import.cursor.clone(); + loop { + self.check().await?; + let what = format!("reading the source's {}", listing.as_str()); + let page = self + .retrying(&what, || self.page(ctx, listing, cursor.as_deref()).boxed()) + .await?; + let phase = if page.next.is_some() { "scan" } else { "copy" }; + let sources: Vec = page + .found + .iter() + .map(|found| serde_json::to_value(&found.source).unwrap_or_default()) + .collect(); + let new_items: Vec> = page + .found + .iter() + .zip(&sources) + .map(|(found, source)| NewItem { + kind: "component", + name: &found.name, + version: &found.version, + size: None, + published_at: found.published_at, + downloaded_at: None, + uploader: None, + source, + }) + .collect(); + let pool = &self.state.pool; + let mut tx = forge_db::begin(pool).await.map_err(db_failed)?; + import_dao::insert_items(&mut tx, self.import.id, &new_items) + .await + .map_err(db_failed)?; + let held = import_dao::save_scan_page( + &mut tx, + self.import.id, + self.run_id, + page.next.as_deref(), + phase, + page.walked, + ) + .await + .map_err(db_failed)?; + if !held { + return Err(End::LeaseLost); + } + tx.commit().await.map_err(db_failed)?; + match page.next { + Some(next) => cursor = Some(next), + None => return Ok(()), + } + } + } + + /// One page of the listing after `cursor`: its package ids' versions, + /// or its images' tags. + async fn page( + &self, + ctx: &Ctx, + listing: Listing, + cursor: Option<&str>, + ) -> Result { + let concurrency = self.import.concurrency.clamp(1, 16) as usize; + match (&ctx.target, listing) { + (_, Listing::Names) => { + let names = self.import.names.as_deref().unwrap_or_default(); + let start = cursor + .and_then(|c| c.parse::().ok()) + .unwrap_or(0) + .min(names.len()); + let end = (start + NAMES_PAGE).min(names.len()); + let found = self + .expand(ctx, &names[start..end], true, concurrency) + .await?; + Ok(Page { + found, + next: (end < names.len()).then(|| end.to_string()), + walked: None, + }) + } + (Target::Packages { feed, .. }, Listing::Search) => { + let skip = cursor.and_then(|c| c.parse::().ok()).unwrap_or(0); + let Some((_, ids)) = feed.search(skip, SEARCH_PAGE).await? else { + return Err(SourceError::Invalid( + "the feed offers no search; name the packages to import instead".into(), + )); + }; + // The walk ends at an empty page: a feed's total is no end + // (Nexus reports the hits of the page it answers), nor is a + // short page (a feed may answer fewer than it was asked for). + { + let mut last = self + .last_search + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if skip > 0 && !ids.is_empty() && *last == ids { + return Err(SourceError::Invalid(format!( + "the feed's search answers the same page at {skip} as before it: it \ + ignores skip; name the packages to import instead" + ))); + } + last.clone_from(&ids); + } + let walked = u64::from(skip) + ids.len() as u64; + let next = (!ids.is_empty()).then(|| walked.to_string()); + let found = self.expand(ctx, &ids, false, concurrency).await?; + Ok(Page { + found, + next, + walked: Some(i64::try_from(walked).unwrap_or(i64::MAX)), + }) + } + (Target::Images { registry, source }, Listing::Catalog) => { + let page = registry + .source_catalog(source, cursor, CATALOG_PAGE) + .await + .map_err(SourceError::of_image)? + .ok_or_else(|| { + SourceError::Invalid( + "the registry serves no catalog (/v2/_catalog); name the images to \ + import instead" + .into(), + ) + })?; + let next = page.more.then(|| page.images.last().cloned()).flatten(); + let found = self.expand(ctx, &page.images, false, concurrency).await?; + Ok(Page { + found, + next, + walked: None, + }) + } + _ => Err(SourceError::Invalid(format!( + "a {} repository's import does not walk a {}", + ctx.repo.format, + listing.as_str() + ))), + } + } + + /// The versions of the package ids `names`, or the tags of the images + /// `names`, in their order. A name the source does not have is left + /// out, or, when the import `named` it, found as missing. + async fn expand( + &self, + ctx: &Ctx, + names: &[String], + named: bool, + concurrency: usize, + ) -> Result, SourceError> { + // The reads are made up front: futures do nothing until polled. + let reads: Vec<_> = names + .iter() + .map(|name| expand_one(&ctx.target, name, named)) + .collect(); + let pages: Vec> = futures_util::stream::iter(reads) + .buffered(concurrency) + .try_collect() + .await?; + Ok(pages.into_iter().flatten().collect()) + } + + /// Copy the pending items: tags and versions first, then (Docker) the + /// referrers their copies found. + async fn copy(&self, ctx: &Ctx) -> Result<(), End> { + let failures = AtomicUsize::new(0); + let concurrency = self.import.concurrency.clamp(1, 16) as usize; + for kind in ["component", "manifest"] { + let mut after = 0; + loop { + self.check().await?; + let batch = + import_dao::pending_items(&self.state.pool, self.import.id, kind, after, BATCH) + .await + .map_err(db_failed)?; + let Some(last) = batch.last() else { + break; + }; + after = last.id; + let mut copies = futures_util::stream::iter(batch) + .map(|item| self.copy_item(ctx, item)) + .buffer_unordered(concurrency); + // Once the run is to end (a cancel, a lost lease, too many + // failures), the items in flight finish and the rest stay + // pending. + let mut end = None; + let mut checked = std::time::Instant::now(); + while let Some(recorded) = copies.next().await { + let mut stop = match recorded { + Err(e) => Some(db_failed(e)), + Ok(Recorded::Done) => { + failures.store(0, Ordering::Relaxed); + None + } + Ok(Recorded::NotStarted) => None, + Ok(Recorded::Failed(error)) => (failures.fetch_add(1, Ordering::Relaxed) + + 1 + >= MAX_CONSECUTIVE_FAILURES) + .then(|| { + End::Failed(format!( + "{MAX_CONSECUTIVE_FAILURES} items in a row failed; the last: {error}" + )) + }), + Ok(Recorded::Conflict(conflict)) => Some(End::Failed(conflict)), + }; + if end.is_none() && stop.is_none() && checked.elapsed() >= CHECK_INTERVAL { + checked = std::time::Instant::now(); + stop = self.check().await.err(); + } + if let (None, Some(stop)) = (&end, stop) { + self.stopping.store(true, Ordering::Relaxed); + end = Some(stop); + } + } + if let Some(end) = end { + return Err(end); + } + } + } + Ok(()) + } + + /// Copy one item and record what became of it. + async fn copy_item(&self, ctx: &Ctx, item: ImportItem) -> sqlx::Result { + if self.stopping.load(Ordering::Relaxed) { + return Ok(Recorded::NotStarted); + } + let pool = &self.state.pool; + let copied = if item.version.is_empty() { + Err(match ctx.target { + Target::Packages { .. } => "the source has no package of this id".to_string(), + Target::Images { .. } => "the source has no tag of this image".to_string(), + }) + } else { + match &ctx.target { + Target::Packages { feed, writer } => { + let source: ItemSource = + serde_json::from_value(item.source.clone()).unwrap_or_default(); + self.copy_package(ctx, feed, writer, &item, &source).await + } + Target::Images { registry, source } => { + self.copy_image(ctx, registry, source, &item).await + } + } + }; + match copied { + Ok(Copied::NotReferrer) => { + import_dao::drop_item(pool, item.id).await?; + Ok(Recorded::Done) + } + Ok(Copied::Item { + state, + detail, + bytes, + }) => { + import_dao::finish_item( + pool, + item.id, + state, + detail.as_deref(), + i64::try_from(bytes).unwrap_or(i64::MAX), + ) + .await?; + if state == "conflict" && self.import.on_conflict == "fail" { + return Ok(Recorded::Conflict(format!( + "{} {}: {}", + item.name, + item.version, + detail.unwrap_or_default() + ))); + } + Ok(Recorded::Done) + } + Err(error) => { + tracing::warn!(import = self.import.id, name = %item.name, version = %item.version, %error, "import item failed"); + import_dao::finish_item(pool, item.id, "failed", Some(&error), 0).await?; + Ok(Recorded::Failed(format!( + "{} {}: {error}", + item.name, item.version + ))) + } + } + } + + /// Copy a package version through the hosted write path. + async fn copy_package( + &self, + ctx: &Ctx, + feed: &Feed, + writer: &HostedWrite, + item: &ImportItem, + source: &ItemSource, + ) -> Result { + let overwrite = self.import.on_conflict == "overwrite"; + + // Nothing is downloaded for a version Forge holds with the bytes + // the feed lists, nor for a dry run (which decides from the + // checksums alone). + let stored = writer + .package_sha512(ctx.repo.id, &item.name, &item.version) + .await + .map_err(|e| e.to_string())?; + if let (Some(stored), Some(expected)) = (&stored, &source.sha512) { + if stored == expected { + return Ok(copied("identical", None, 0)); + } + } + if self.import.dry_run { + return Ok(match (stored, &source.sha512) { + (None, _) => copied("planned", None, 0), + (Some(_), None) => copied( + "planned", + Some("Forge has this version; the feed lists no SHA-512 to compare".into()), + 0, + ), + (Some(_), Some(_)) if overwrite => copied( + "planned", + Some("would replace the version Forge has with other bytes".into()), + 0, + ), + (Some(_), Some(_)) => conflict_item(), + }); + } + + let url = feed + .package_url(&item.name, &item.version) + .await + .map_err(|e| e.to_string())?; + let response = feed.download(&url).await.map_err(|e| e.to_string())?; + if response.content_length().is_some_and(|l| l > ctx.limit) { + return Err(WriteError::TooLarge { limit: ctx.limit }.to_string()); + } + let body = response.bytes_stream().map_err(std::io::Error::other); + let staged = self + .state + .cas + .stage(tokio_util::io::StreamReader::new(body), ctx.limit) + .await + .map_err(|e| WriteError::from(e).to_string())?; + verify(&staged, &url, source)?; + let bytes = staged.size; + let options = ImportOptions { + overwrite, + dry_run: false, + published_at: item.published_at, + downloaded_at: Some(item.downloaded_at.unwrap_or_else(Utc::now)), + uploader: item.uploader.clone(), + source: ctx.source.clone(), + }; + let outcome = writer + .import_staged(&ctx.repo.name, staged, ctx.limit, &options) + .await + .map_err(|e| e.to_string())?; + Ok(match outcome { + ImportOutcome::Created { .. } => copied("imported", None, bytes), + ImportOutcome::Replaced { .. } => copied( + "replaced", + Some("replaced the version Forge had with other bytes".into()), + bytes, + ), + ImportOutcome::Identical { .. } => copied("identical", None, bytes), + ImportOutcome::Conflict { .. } => conflict_item(), + ImportOutcome::Planned { .. } => copied("planned", None, bytes), + }) + } + + /// Copy a Docker tag (`component`) or referrer (`manifest`) through the + /// registry, and record the referrers the source lists for it. + async fn copy_image( + &self, + ctx: &Ctx, + registry: &Registry, + from: &ImportSource, + item: &ImportItem, + ) -> Result { + let options = ImageImportOptions { + principal: ctx.principal.clone(), + overwrite: self.import.on_conflict == "overwrite", + dry_run: self.import.dry_run, + published_at: item.published_at, + downloaded_at: Some(item.downloaded_at.unwrap_or_else(Utc::now)), + uploader: item.uploader.clone(), + source: ctx.source.clone(), + }; + let imported = if item.kind == "manifest" { + registry + .import_referrer(&ctx.repo, from, &item.name, &item.version, &options) + .await + } else { + registry + .import_tag(&ctx.repo, from, &item.name, &item.version, None, &options) + .await + } + .map_err(|e| e.message())?; + let bytes = imported.bytes_copied; + // Its referrers come along unless it stays out of Forge. + let held = matches!( + imported.outcome, + ImageOutcome::Created + | ImageOutcome::Replaced { .. } + | ImageOutcome::Identical + | ImageOutcome::Planned { .. } + ); + if let (true, Some(digest)) = (held, &imported.digest) { + self.record_referrers(registry, from, &item.name, digest) + .await + .map_err(|e| format!("copied, but listing its referrers failed: {e}"))?; + } + Ok(match imported.outcome { + ImageOutcome::Created => copied("imported", imported.digest, bytes), + ImageOutcome::Replaced { previous } => { + copied("replaced", Some(format!("moved from {previous}")), bytes) + } + ImageOutcome::Identical => copied("identical", imported.digest, bytes), + ImageOutcome::Conflict { + existing, + immutable, + } => copied( + "conflict", + Some(if immutable { + format!("the tag points at {existing} in Forge, and tags are immutable there") + } else { + format!("the tag points at {existing} in Forge") + }), + bytes, + ), + ImageOutcome::Planned { replaces } => copied( + "planned", + replaces.then(|| "would move the tag Forge has".to_string()), + bytes, + ), + ImageOutcome::NotReferrer => Copied::NotReferrer, + }) + } + + /// Record the referrers the source's referrers API lists for the + /// manifest `digest` of `image` as items, copied after the tags. A + /// source without the API lists none. + async fn record_referrers( + &self, + registry: &Registry, + from: &ImportSource, + image: &str, + digest: &str, + ) -> Result<(), String> { + let Some(referrers) = registry + .source_referrers(from, image, digest) + .await + .map_err(|e| e.message())? + else { + return Ok(()); + }; + if referrers.is_empty() { + return Ok(()); + } + let source = serde_json::json!({}); + let items: Vec> = referrers + .iter() + .map(|referrer| NewItem { + kind: "manifest", + name: image, + version: &referrer.digest, + size: i64::try_from(referrer.size).ok(), + published_at: None, + downloaded_at: None, + uploader: None, + source: &source, + }) + .collect(); + let recorded = async { + let mut tx = forge_db::begin(&self.state.pool).await?; + import_dao::insert_items(&mut tx, self.import.id, &items).await?; + tx.commit().await + }; + recorded.await.map_err(|e| format!("database error: {e}")) + } +} + +/// The versions of the package id `name`, or the tags of the image `name`. +async fn expand_one(target: &Target, name: &str, named: bool) -> Result, SourceError> { + match target { + Target::Packages { feed, .. } => package_versions(feed, name, named).await, + Target::Images { registry, source } => image_tags(registry, source, name, named).await, + } +} + +/// Every version the feed lists of the package `id`, with what its +/// registration says of each. A package the feed does not know is left +/// out, or found as missing when the import `named` it. +async fn package_versions(feed: &Feed, id: &str, named: bool) -> Result, SourceError> { + let versions = feed.versions(id).await?.unwrap_or_default(); + if versions.is_empty() { + return Ok(if named { + vec![Found::missing(id)] + } else { + Vec::new() + }); + } + // The registration is best effort: without it, a version has no + // publish time and no checksum. + let metadata = match feed.metadata(id, &versions).await { + Ok(metadata) => metadata, + Err(e) if e.is_transient() => return Err(e), + Err(e) => { + tracing::warn!(id, error = %e, "import: the feed's registration is unreadable"); + Vec::new() + } + }; + let metadata: HashMap = metadata + .into_iter() + .map(|m| (m.version.to_lowercase(), m)) + .collect(); + Ok(versions + .into_iter() + .map(|version| { + let meta = metadata.get(&version.to_lowercase()); + Found { + name: id.to_string(), + published_at: meta.and_then(|m| published_at(m.published.as_deref())), + source: ItemSource { + sha512: meta.and_then(|m| m.sha512.clone()), + }, + version, + } + }) + .collect()) +} + +/// Every tag of `image` at the source. An image the source does not have +/// (or has no tag of) is left out, or found as missing when the import +/// `named` it. +async fn image_tags( + registry: &Registry, + source: &ImportSource, + image: &str, + named: bool, +) -> Result, SourceError> { + let mut found = Vec::new(); + let mut last: Option = None; + loop { + let page = registry + .source_tags(source, image, last.as_deref(), TAGS_PAGE) + .await + .map_err(SourceError::of_image)?; + let Some(page) = page else { + break; + }; + let more = page.more && !page.tags.is_empty(); + last = page.tags.last().cloned().or(last); + found.extend(page.tags.into_iter().map(|tag| Found { + name: image.to_string(), + version: tag, + published_at: None, + source: ItemSource::default(), + })); + if !more { + break; + } + } + if found.is_empty() && named { + found.push(Found::missing(image)); + } + Ok(found) +} + +/// A publish time from a feed's registration. nuget.org gives an unlisted +/// version `1900-01-01`: no publish time. +fn published_at(published: Option<&str>) -> Option> { + DateTime::parse_from_rfc3339(published?) + .ok() + .map(|t| t.with_timezone(&Utc)) + .filter(|t| t.year() > 1900) +} + +fn copied(state: &'static str, detail: Option, bytes: u64) -> Copied { + Copied::Item { + state, + detail, + bytes, + } +} + +fn conflict_item() -> Copied { + copied( + "conflict", + Some("Forge has other bytes for this version".into()), + 0, + ) +} + +/// Check a staged download against the SHA-512 the feed lists for it. +fn verify(staged: &StagedBlob, url: &str, source: &ItemSource) -> Result<(), String> { + match &source.sha512 { + Some(expected) if *expected != staged.sha512_base64 => Err(format!( + "the download of {url} does not match the SHA-512 the feed lists for it" + )), + _ => Ok(()), + } +} + +/// Aborts the wrapped task when dropped. +struct AbortOnDrop(JoinHandle<()>); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.abort(); + } +} + +/// Renew the lease every third of its length until aborted; flags `lost` +/// and stops once the lease is gone. +fn spawn_heartbeat( + pool: sqlx::PgPool, + id: i64, + run_id: Uuid, + lost: Arc, +) -> JoinHandle<()> { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(Duration::from_secs((LEASE_SECONDS / 3) as u64)); + ticker.tick().await; + loop { + ticker.tick().await; + let renewed = match forge_db::db_now(&pool).await { + Ok(now) => { + import_dao::renew_lease( + &pool, + id, + run_id, + now + chrono::Duration::seconds(LEASE_SECONDS), + ) + .await + } + Err(e) => Err(e), + }; + match renewed { + Ok(true) => {} + Ok(false) => { + lost.store(true, Ordering::Relaxed); + return; + } + Err(e) => { + tracing::warn!(import = id, error = %e, "import lease renewal failed") + } + } + } + }) +} + +fn panic_text(payload: &(dyn std::any::Any + Send)) -> String { + payload + .downcast_ref::<&str>() + .map(|s| (*s).to_string()) + .or_else(|| payload.downcast_ref::().cloned()) + .unwrap_or_else(|| "non-string panic payload".to_string()) +} + +/// The password an import is created or resumed with, if any. +pub fn password_secret(password: Option) -> Option { + password + .filter(|p| !p.is_empty()) + .map(PlaintextSecret::from) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_nuget_placeholder_date_is_no_publish_time() { + assert_eq!(published_at(Some("1900-01-01T00:00:00+00:00")), None); + assert_eq!(published_at(Some("not a date")), None); + assert_eq!(published_at(None), None); + assert_eq!( + published_at(Some("2024-05-06T07:08:09.123+02:00")), + Some("2024-05-06T05:08:09.123Z".parse().unwrap()) + ); + } + + #[test] + fn a_nexus_repository_reads_through_its_protocol() { + assert_eq!( + nexus_source_url("https://nexus.example.com/", "nuget-hosted", None, false), + "https://nexus.example.com/repository/nuget-hosted/index.json" + ); + assert_eq!( + nexus_source_url("https://nexus.example.com", "docker-hosted", None, true), + "https://nexus.example.com/repository/docker-hosted" + ); + assert_eq!( + nexus_source_url( + "https://nexus.example.com", + "docker-hosted", + Some("https://nexus.example.com:8082"), + true + ), + "https://nexus.example.com:8082" + ); + } + + #[test] + fn a_new_import_picks_its_listing() { + assert_eq!(Listing::of(false, false), Listing::Search); + assert_eq!(Listing::of(true, false), Listing::Catalog); + assert_eq!(Listing::of(true, true), Listing::Names); + for listing in [Listing::Search, Listing::Catalog, Listing::Names] { + assert_eq!(Listing::parse(listing.as_str()), listing); + } + } + + #[test] + fn an_item_stored_before_reads_as_no_checksum() { + let legacy = serde_json::json!({ + "name": "Sample.Lib", + "version": "1.0.0", + "assets": [{ "path": "Sample.Lib/1.0.0", "sha512": "abc" }] + }); + let source: ItemSource = serde_json::from_value(legacy).unwrap(); + assert_eq!(source.sha512, None); + } +} diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index 04ece27e..ef0198b2 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -27,11 +27,11 @@ pub mod feed; pub mod files; pub mod health; pub mod images; +pub mod import; pub mod invitations; pub mod me; pub mod me_tokens; pub mod middleware; -pub mod nexus_import; pub mod object_store; pub mod plugins; pub mod registry; diff --git a/crates/server/src/nexus_import/mod.rs b/crates/server/src/nexus_import/mod.rs deleted file mode 100644 index 05994338..00000000 --- a/crates/server/src/nexus_import/mod.rs +++ /dev/null @@ -1,1084 +0,0 @@ -//! Importing a Nexus hosted repository into a Forge hosted repository of the -//! same format, one repository at a time (NuGet, Docker). -//! -//! An administrator starts an import on a hosted repository -//! ([`crate::admin::imports`]); this worker runs it, on one instance at a -//! time: it claims a queued import (or one whose instance died) with a -//! lease in Postgres and renews the lease while it works, like the -//! retention worker. -//! -//! 1. **Scan.** The Nexus component listing is walked page by page; each -//! page's components become pending items and the page's -//! `continuationToken` becomes the import's cursor, in one transaction, -//! so a scan resumes after a restart, a failure or a cancel at the page -//! it had not recorded. For Docker the asset listing follows: manifests -//! Nexus holds by digest only (not listed as components) become items -//! too, so referrers (signatures, SBOMs) come along. The scan also -//! counts every asset of the repository and its size. -//! 2. **Copy.** Pending items are copied, `concurrency` at a time: -//! * a package (NuGet) is streamed from Nexus into staging, -//! verified against Nexus' checksums (SHA-256 and SHA-512, else SHA-1) -//! and size, and written through the hosted write path -//! ([`HostedWrite::import_staged`]): format validation, the package -//! selectors of the administrator who started the import, the -//! repository's upload limit, CAS dedup and the audit trail -//! (`action = "import"`). A version Forge holds with the same SHA-256 -//! is skipped without a download; -//! * a Docker tag is copied through the Nexus repository's registry API -//! ([`crate::images::Registry::import_tag`]): digests are kept, -//! multi-platform indexes arrive with every child manifest, and a -//! referrer is copied once its subject is in Forge; -//! * what Nexus recorded is kept: the publish time (`blobCreated`, for a -//! tag `lastModified`) becomes the version's creation time, the last -//! download its last download, the uploader goes to the audit trail; -//! * a version Forge holds with other bytes is a conflict: skipped and -//! reported (`skip`), replaced (`overwrite`), or it fails the import -//! (`fail`). -//! -//! A dry run scans and decides every item (`planned`, `identical`, -//! `conflict`) without writing a package; it downloads nothing that Nexus -//! lists a SHA-256 for. Running an import again later copies only what is -//! new in Nexus, so a cutover is an import, then a final one just before -//! clients switch to Forge. -//! -//! A failed item can be retried on its own, and a cancelled or failed -//! import resumed ([`forge_db::import_dao::requeue`]). The Nexus password -//! is sealed with the secret encryption and deleted when the import -//! completes. - -pub mod nexus; - -use std::panic::AssertUnwindSafe; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; -use std::sync::Arc; -use std::time::Duration; - -use base64::Engine; -use forge_core::auth::principal::{AuthMethod, Principal}; -use forge_core::authz::Level; -use forge_core::repo::hosted::HostedWrite; -use forge_core::repo::proxy::Credentials; -use forge_core::repo::{ImportOptions, ImportOutcome, RepoKind, WriteError}; -use forge_crypto::PlaintextSecret; -use forge_db::import_dao::{self, ImportItem, NewItem, NexusImport}; -use forge_db::model::Repository; -use forge_db::{repository_dao, secret_dao, user_dao}; -use forge_storage::StagedBlob; -use futures_util::{FutureExt, StreamExt, TryStreamExt}; -use serde_json::Value as JsonValue; -use sha2::Digest as _; -use tokio::task::JoinHandle; -use uuid::Uuid; - -use crate::images::import::{ImageImportOptions, ImageOutcome, ImportSource}; -use crate::state::AppState; -use nexus::{Listing, Nexus, SourceAsset, SourceItem}; - -/// The formats an import copies, by the names Nexus and Forge share: NuGet -/// packages, and Docker images into the image store. -pub const FORMATS: &[&str] = &["nuget", "docker"]; - -/// How often an idle instance looks for an import to run. -const POLL_INTERVAL: Duration = Duration::from_secs(5); - -/// The lease a running import is held with; renewed every third of it. -const LEASE_SECONDS: i64 = 120; - -/// Pending items read at a time. -const BATCH: i64 = 64; - -/// Items failing in a row that stop an import (Nexus went away, the -/// credentials stopped working): it fails, and can be resumed. -const MAX_CONSECUTIVE_FAILURES: usize = 25; - -/// Attempts at a listing page before the import fails. -const PAGE_ATTEMPTS: u32 = 4; - -/// How often a copy looks whether it was cancelled (or lost its lease). -const CHECK_INTERVAL: Duration = Duration::from_secs(5); - -/// The worker running imports on this instance. -pub struct ImportWorker { - state: AppState, -} - -impl ImportWorker { - pub fn new(state: AppState) -> Self { - ImportWorker { state } - } - - /// Look for an import every few seconds and run each that is due. - pub fn spawn_loop(self) -> JoinHandle<()> { - tokio::spawn(async move { - let mut ticker = tokio::time::interval(POLL_INTERVAL); - loop { - ticker.tick().await; - loop { - match self.tick().await { - Ok(Some(_)) => {} - Ok(None) => break, - Err(e) => { - tracing::error!(error = %e, "nexus import tick failed"); - break; - } - } - } - } - }) - } - - /// Claim one import that is due and run it until it completes, fails, - /// is cancelled or loses its lease. Returns its id, `None` when no - /// import is due. - pub async fn tick(&self) -> sqlx::Result> { - let pool = &self.state.pool; - let claimed = { - let mut tx = forge_db::begin(pool).await?; - let now = forge_db::db_now(&mut *tx).await?; - let claimed = - import_dao::claim_next(&mut tx, now, chrono::Duration::seconds(LEASE_SECONDS)) - .await?; - tx.commit().await?; - claimed - }; - let Some((import, run_id)) = claimed else { - return Ok(None); - }; - let id = import.id; - tracing::info!( - import = id, - repository_id = import.repository_id, - source = %import.base_url, - source_repository = %import.source_repository, - phase = %import.phase, - "nexus import started" - ); - let lost = Arc::new(AtomicBool::new(false)); - let heartbeat = AbortOnDrop(spawn_heartbeat(pool.clone(), id, run_id, lost.clone())); - let run = Run { - state: &self.state, - import, - run_id, - lost, - stopping: AtomicBool::new(false), - }; - let end = AssertUnwindSafe(run.execute()) - .catch_unwind() - .await - .unwrap_or_else(|panic| { - End::Failed(format!("the import panicked: {}", panic_text(&*panic))) - }); - drop(heartbeat); - let (state, error) = match &end { - End::Completed => ("completed", None), - End::Cancelled => ("cancelled", None), - End::Failed(error) => ("failed", Some(error.as_str())), - End::LeaseLost => { - tracing::warn!( - import = id, - "nexus import lease lost; another instance continues it" - ); - return Ok(Some(id)); - } - }; - if import_dao::finish(pool, id, run_id, state, error).await? { - tracing::info!(import = id, state, error, "nexus import ended"); - } else { - tracing::warn!(import = id, "nexus import lease lost before it ended"); - } - Ok(Some(id)) - } -} - -/// How a run ended. -enum End { - Completed, - Cancelled, - Failed(String), - /// Another instance holds the import now; it records the outcome. - LeaseLost, -} - -fn db_failed(e: sqlx::Error) -> End { - tracing::error!(error = %e, "nexus import: database error"); - End::Failed(format!("database error: {e}")) -} - -/// Where the items of a run go. -enum Target { - /// A package format, through the hosted write path. - Packages(Box), - /// Docker, through the registry, from the Nexus repository's registry - /// API. - Images { - registry: crate::images::Registry, - source: ImportSource, - }, -} - -/// What a run works with. -struct Ctx { - repo: Repository, - principal: Principal, - nexus: Nexus, - target: Target, - /// The repository's upload limit. - limit: u64, - /// ` `, for the audit trail. - source: String, -} - -impl Ctx { - /// Whether the items are images (Docker). - fn images(&self) -> bool { - matches!(self.target, Target::Images { .. }) - } -} - -/// What became of one item. -enum Copied { - /// It ends in `state`. - Item { - state: &'static str, - detail: Option, - bytes: u64, - }, - /// A manifest Nexus holds by digest that is no referrer: forgotten. - NotReferrer, -} - -/// How a recorded item bears on the run. -enum Recorded { - Done, - /// The run is ending: the item was left pending. - NotStarted, - Failed(String), - /// A conflict, and the import fails on conflicts. - Conflict(String), -} - -/// One run of one import, on this instance. -struct Run<'a> { - state: &'a AppState, - import: NexusImport, - run_id: Uuid, - /// Set by the heartbeat once the lease is lost. - lost: Arc, - /// Set once the run is to end: no further item is started. - stopping: AtomicBool, -} - -impl Run<'_> { - async fn execute(&self) -> End { - let ctx = match self.prepare().await { - Ok(ctx) => ctx, - Err(end) => return end, - }; - if self.import.phase == "scan" { - if let Err(end) = self.scan(&ctx).await { - return end; - } - } - match self.copy(&ctx).await { - Ok(()) => End::Completed, - Err(end) => end, - } - } - - /// The repository, the administrator the import runs as, and the - /// clients of the source. - async fn prepare(&self) -> Result { - let pool = &self.state.pool; - let import = &self.import; - let repo = repository_dao::find_repository_by_id(pool, import.repository_id) - .await - .map_err(db_failed)? - .ok_or_else(|| End::Failed("the repository was deleted".into()))?; - if repo.kind() != RepoKind::Hosted || !FORMATS.contains(&repo.format.as_str()) { - return Err(End::Failed(format!( - "imports go to a hosted NuGet or Docker repository; {} is a {} {} repository", - repo.name, repo.format, repo.kind - ))); - } - - // The import runs with the authority of the administrator who - // started it, as long as they are one. - let gone = - || End::Failed("the administrator who started the import no longer is one".into()); - let user_id = import.created_by.ok_or_else(gone)?; - let user = user_dao::find_user_by_id(pool, user_id) - .await - .map_err(db_failed)? - .filter(|user| user.active) - .ok_or_else(gone)?; - let principal = Principal::User { - user_id: user.id, - token_id: None, - method: AuthMethod::Session, - }; - if !self.state.authz.is_global_admin(&principal).await { - return Err(gone()); - } - - let password = secret_dao::get_import_password(pool, &self.state.envelope, import.id) - .await - .map_err(|e| End::Failed(format!("the import's password cannot be read: {e}")))?; - if import.username.is_some() && password.is_none() { - return Err(End::Failed( - "the import has no password; resume it with the Nexus password".into(), - )); - } - let nexus = Nexus::new( - self.state.upstream.clone(), - &import.base_url, - import.username.clone(), - password.clone(), - ); - - let target = if self.state.formats.stores_images(&repo.format) { - let registry_url = import.registry_url.clone().unwrap_or_else(|| { - default_registry_url(&import.base_url, &import.source_repository) - }); - let credentials = match (&import.username, password) { - (Some(username), Some(password)) => Some(Credentials::for_url( - ®istry_url, - username.clone(), - password, - )), - _ => None, - }; - Target::Images { - registry: self.state.registry(), - source: ImportSource::new( - ®istry_url, - credentials, - format!("import:{}", import.id), - ), - } - } else { - let format = self - .state - .formats - .for_repository(&repo) - .map_err(|e| End::Failed(e.to_string()))?; - let authz = &self.state.authz; - let filter = authz - .package_filter_for(&principal, &repo, Level::Write) - .await; - let replace = authz - .package_filter_for(&principal, &repo, Level::Delete) - .await; - let role_filter = authz - .role_package_filter_for(&principal, &repo, Level::Write) - .await; - let role_replace = authz - .role_package_filter_for(&principal, &repo, Level::Delete) - .await; - Target::Packages(Box::new(crate::dispatch::hosted_write( - self.state, - format, - repo.id, - repo.name.clone(), - principal.clone(), - filter, - replace, - role_filter, - role_replace, - ))) - }; - Ok(Ctx { - limit: self.state.upload_limit(&repo), - source: format!("{} {}", import.base_url, import.source_repository), - repo, - principal, - nexus, - target, - }) - } - - /// Stop when the lease is lost or an administrator cancelled. - async fn check(&self) -> Result<(), End> { - if self.lost.load(Ordering::Relaxed) { - return Err(End::LeaseLost); - } - match import_dao::poll(&self.state.pool, self.import.id, self.run_id).await { - Ok(None) => Err(End::LeaseLost), - Ok(Some(true)) => Err(End::Cancelled), - Ok(Some(false)) => Ok(()), - Err(e) => Err(db_failed(e)), - } - } - - /// Walk the listings from the stored cursor, recording each page's - /// items with the cursor of the next page. - async fn scan(&self, ctx: &Ctx) -> Result<(), End> { - let mut listing = Listing::parse(&self.import.scan_listing); - let mut cursor = self.import.cursor.clone(); - loop { - self.check().await?; - let page = self.page(ctx, listing, cursor.as_deref()).await?; - let scanned = scanned_items(&ctx.repo.format, ctx.images(), listing, &page.items); - let (next_listing, next_cursor, phase) = match page.next { - Some(next) => (listing, Some(next), "scan"), - None if ctx.images() && listing == Listing::Components => { - (Listing::Assets, None, "scan") - } - None => (listing, None, "copy"), - }; - let sources: Vec = scanned - .items - .iter() - .map(|(_, item, _)| serde_json::to_value(item).unwrap_or(JsonValue::Null)) - .collect(); - let new_items: Vec> = scanned - .items - .iter() - .zip(&sources) - .map(|((kind, item, primary), source)| NewItem { - kind, - name: &item.name, - version: &item.version, - size: primary.and_then(|a| a.size), - published_at: primary.and_then(|a| published_at(ctx.images(), a)), - downloaded_at: primary.and_then(|a| a.downloaded), - uploader: primary.and_then(|a| a.uploader.as_deref()), - source, - }) - .collect(); - let pool = &self.state.pool; - let mut tx = forge_db::begin(pool).await.map_err(db_failed)?; - import_dao::insert_items(&mut tx, self.import.id, &new_items) - .await - .map_err(db_failed)?; - let held = import_dao::save_scan_page( - &mut tx, - self.import.id, - self.run_id, - next_listing.as_str(), - next_cursor.as_deref(), - phase, - scanned.assets, - scanned.bytes, - ) - .await - .map_err(db_failed)?; - if !held { - return Err(End::LeaseLost); - } - tx.commit().await.map_err(db_failed)?; - if phase == "copy" { - return Ok(()); - } - listing = next_listing; - cursor = next_cursor; - } - } - - /// One listing page, retried with a backoff while the failure may pass. - async fn page( - &self, - ctx: &Ctx, - listing: Listing, - cursor: Option<&str>, - ) -> Result { - let mut attempt = 0; - loop { - attempt += 1; - match ctx - .nexus - .page(listing, &self.import.source_repository, cursor) - .await - { - Ok(page) => return Ok(page), - Err(e) if e.is_transient() && attempt < PAGE_ATTEMPTS => { - tracing::warn!(import = self.import.id, error = %e, attempt, "nexus listing failed; retrying"); - tokio::time::sleep(Duration::from_secs(1 << attempt)).await; - } - Err(e) => { - return Err(End::Failed(format!( - "listing the {}: {e}", - listing.as_str() - ))) - } - } - } - } - - /// Copy the pending items: components first, then (Docker) the - /// manifests Nexus holds by digest, whose subjects the components - /// bring. - async fn copy(&self, ctx: &Ctx) -> Result<(), End> { - let failures = AtomicUsize::new(0); - let concurrency = self.import.concurrency.clamp(1, 16) as usize; - for kind in ["component", "manifest"] { - let mut after = 0; - loop { - self.check().await?; - let batch = - import_dao::pending_items(&self.state.pool, self.import.id, kind, after, BATCH) - .await - .map_err(db_failed)?; - let Some(last) = batch.last() else { - break; - }; - after = last.id; - let mut copies = futures_util::stream::iter(batch) - .map(|item| self.copy_item(ctx, item)) - .buffer_unordered(concurrency); - // Once the run is to end (a cancel, a lost lease, too many - // failures), the items in flight finish and the rest stay - // pending. - let mut end = None; - let mut checked = std::time::Instant::now(); - while let Some(recorded) = copies.next().await { - let mut stop = match recorded { - Err(e) => Some(db_failed(e)), - Ok(Recorded::Done) => { - failures.store(0, Ordering::Relaxed); - None - } - Ok(Recorded::NotStarted) => None, - Ok(Recorded::Failed(error)) => (failures.fetch_add(1, Ordering::Relaxed) - + 1 - >= MAX_CONSECUTIVE_FAILURES) - .then(|| { - End::Failed(format!( - "{MAX_CONSECUTIVE_FAILURES} items in a row failed; the last: {error}" - )) - }), - Ok(Recorded::Conflict(conflict)) => Some(End::Failed(conflict)), - }; - if end.is_none() && stop.is_none() && checked.elapsed() >= CHECK_INTERVAL { - checked = std::time::Instant::now(); - stop = self.check().await.err(); - } - if let (None, Some(stop)) = (&end, stop) { - self.stopping.store(true, Ordering::Relaxed); - end = Some(stop); - } - } - if let Some(end) = end { - return Err(end); - } - } - } - Ok(()) - } - - /// Copy one item and record what became of it. - async fn copy_item(&self, ctx: &Ctx, item: ImportItem) -> sqlx::Result { - if self.stopping.load(Ordering::Relaxed) { - return Ok(Recorded::NotStarted); - } - let pool = &self.state.pool; - let copied = match serde_json::from_value::(item.source.clone()) { - Err(e) => Err(format!("the item's recorded source is unreadable: {e}")), - Ok(source) => match &ctx.target { - Target::Packages(writer) => self.copy_package(ctx, writer, &source).await, - Target::Images { - registry, - source: from, - } => { - self.copy_image(ctx, registry, from, &item.kind, &source) - .await - } - }, - }; - match copied { - Ok(Copied::NotReferrer) => { - import_dao::drop_item(pool, item.id).await?; - Ok(Recorded::Done) - } - Ok(Copied::Item { - state, - detail, - bytes, - }) => { - import_dao::finish_item( - pool, - item.id, - state, - detail.as_deref(), - i64::try_from(bytes).unwrap_or(i64::MAX), - ) - .await?; - if state == "conflict" && self.import.on_conflict == "fail" { - return Ok(Recorded::Conflict(format!( - "{} {}: {}", - item.name, - item.version, - detail.unwrap_or_default() - ))); - } - Ok(Recorded::Done) - } - Err(error) => { - tracing::warn!(import = self.import.id, name = %item.name, version = %item.version, %error, "nexus import item failed"); - import_dao::finish_item(pool, item.id, "failed", Some(&error), 0).await?; - Ok(Recorded::Failed(format!( - "{} {}: {error}", - item.name, item.version - ))) - } - } - } - - /// Copy a package version through the hosted write path. - async fn copy_package( - &self, - ctx: &Ctx, - writer: &HostedWrite, - source: &SourceItem, - ) -> Result { - let asset = package_asset(&ctx.repo.format, &source.assets) - .ok_or("the component has no package file among its assets")?; - let overwrite = self.import.on_conflict == "overwrite"; - - // Nothing is downloaded for a version Forge holds with these bytes, - // nor for a dry run (which decides from the checksums alone). - let stored = writer - .package_sha256(ctx.repo.id, &source.name, &source.version) - .await - .map_err(|e| e.to_string())?; - if let (Some(stored), Some(expected)) = (&stored, &asset.sha256) { - if stored == expected { - return Ok(item("identical", None, 0)); - } - } - if self.import.dry_run { - return Ok(match (stored, &asset.sha256) { - (None, _) => item("planned", None, 0), - (Some(_), None) => item( - "planned", - Some("Forge has this version; Nexus lists no SHA-256 to compare".into()), - 0, - ), - (Some(_), Some(_)) if overwrite => item( - "planned", - Some("would replace the version Forge has with other bytes".into()), - 0, - ), - (Some(_), Some(_)) => conflict_item(), - }); - } - - if asset - .size - .is_some_and(|size| u64::try_from(size).unwrap_or(0) > ctx.limit) - { - return Err(WriteError::TooLarge { limit: ctx.limit }.to_string()); - } - let url = ctx - .nexus - .content_url(&self.import.source_repository, &asset.path); - let response = ctx.nexus.download(&url).await.map_err(|e| e.to_string())?; - if response.content_length().is_some_and(|l| l > ctx.limit) { - return Err(WriteError::TooLarge { limit: ctx.limit }.to_string()); - } - let body = response.bytes_stream().map_err(std::io::Error::other); - let staged = self - .state - .cas - .stage(tokio_util::io::StreamReader::new(body), ctx.limit) - .await - .map_err(|e| WriteError::from(e).to_string())?; - verify(&staged, asset).await?; - let bytes = staged.size; - let options = ImportOptions { - overwrite, - dry_run: false, - published_at: published_at(ctx.images(), asset), - downloaded_at: asset.downloaded, - uploader: asset.uploader.clone(), - source: ctx.source.clone(), - }; - let outcome = writer - .import_staged(&ctx.repo.name, staged, ctx.limit, &options) - .await - .map_err(|e| e.to_string())?; - Ok(match outcome { - ImportOutcome::Created { .. } => item("imported", None, bytes), - ImportOutcome::Replaced { .. } => item( - "replaced", - Some("replaced the version Forge had with other bytes".into()), - bytes, - ), - ImportOutcome::Identical { .. } => item("identical", None, bytes), - ImportOutcome::Conflict { .. } => conflict_item(), - ImportOutcome::Planned { .. } => item("planned", None, bytes), - }) - } - - /// Copy a Docker tag (`component`) or referrer (`manifest`) through the - /// registry. - async fn copy_image( - &self, - ctx: &Ctx, - registry: &crate::images::Registry, - from: &ImportSource, - kind: &str, - source: &SourceItem, - ) -> Result { - let asset = source - .assets - .iter() - .find(|a| a.path.contains("/manifests/")) - .or(source.assets.first()); - let options = ImageImportOptions { - principal: ctx.principal.clone(), - overwrite: self.import.on_conflict == "overwrite", - dry_run: self.import.dry_run, - published_at: asset.and_then(|a| published_at(ctx.images(), a)), - downloaded_at: asset.and_then(|a| a.downloaded), - uploader: asset.and_then(|a| a.uploader.clone()), - source: ctx.source.clone(), - }; - let imported = if kind == "manifest" { - registry - .import_referrer(&ctx.repo, from, &source.name, &source.version, &options) - .await - } else { - let expected = asset - .and_then(|a| a.sha256.as_ref()) - .map(|hex| format!("sha256:{hex}")); - registry - .import_tag( - &ctx.repo, - from, - &source.name, - &source.version, - expected.as_deref(), - &options, - ) - .await - } - .map_err(|e| e.message())?; - let bytes = imported.bytes_copied; - Ok(match imported.outcome { - ImageOutcome::Created => item("imported", imported.digest, bytes), - ImageOutcome::Replaced { previous } => { - item("replaced", Some(format!("moved from {previous}")), bytes) - } - ImageOutcome::Identical => item("identical", imported.digest, bytes), - ImageOutcome::Conflict { - existing, - immutable, - } => item( - "conflict", - Some(if immutable { - format!("the tag points at {existing} in Forge, and tags are immutable there") - } else { - format!("the tag points at {existing} in Forge") - }), - bytes, - ), - ImageOutcome::Planned { replaces } => item( - "planned", - replaces.then(|| "would move the tag Forge has".to_string()), - bytes, - ), - ImageOutcome::NotReferrer => Copied::NotReferrer, - }) - } -} - -fn item(state: &'static str, detail: Option, bytes: u64) -> Copied { - Copied::Item { - state, - detail, - bytes, - } -} - -fn conflict_item() -> Copied { - item( - "conflict", - Some("Forge has other bytes for this version".into()), - 0, - ) -} - -/// The registry API of the Nexus Docker repository `repository`, by path -/// (`/repository/`, which Nexus serves for reads on its own -/// port). -pub fn default_registry_url(base_url: &str, repository: &str) -> String { - format!("{}/repository/{repository}", base_url.trim_end_matches('/')) -} - -/// When Nexus says an asset was published: for a Docker tag when it last -/// moved (`lastModified`), for a package when its bytes were stored -/// (`blobCreated`). -fn published_at(images: bool, asset: &SourceAsset) -> Option> { - if images { - asset.modified.or(asset.created) - } else { - asset.created.or(asset.modified) - } -} - -/// The package file among a component's assets: NuGet's `.nupkg` (Nexus -/// keeps it under `/`), any other format's only asset. -fn package_asset<'a>(format: &str, assets: &'a [SourceAsset]) -> Option<&'a SourceAsset> { - match format { - "nuget" => assets - .iter() - .find(|a| a.path.ends_with(".nupkg")) - .or_else(|| assets.iter().find(|a| !a.path.ends_with(".nuspec"))), - _ => match assets { - [only] => Some(only), - _ => None, - }, - } -} - -/// What one scanned page holds. -struct Scanned<'a> { - /// `(kind, item, its primary asset)`. - items: Vec<(&'static str, SourceItem, Option<&'a SourceAsset>)>, - assets: i64, - bytes: i64, -} - -/// The items of a page: every component of a components listing; of a -/// (Docker) asset listing, the manifests held by digest. Every asset is -/// counted once: a package component's in the components listing, a -/// Docker repository's (blobs included) in the asset listing. -fn scanned_items<'a>( - format: &str, - images: bool, - listing: Listing, - page: &'a [SourceItem], -) -> Scanned<'a> { - let mut scanned = Scanned { - items: Vec::new(), - assets: 0, - bytes: 0, - }; - for item in page { - match listing { - Listing::Components => { - let primary = if images { - item.assets.iter().find(|a| a.path.contains("/manifests/")) - } else { - scanned.assets += item.assets.len() as i64; - scanned.bytes += item.assets.iter().filter_map(|a| a.size).sum::(); - package_asset(format, &item.assets) - }; - scanned.items.push(("component", item.clone(), primary)); - } - Listing::Assets => { - for asset in &item.assets { - scanned.assets += 1; - scanned.bytes += asset.size.unwrap_or(0); - if let Some((image, digest)) = manifest_by_digest(&asset.path) { - scanned.items.push(( - "manifest", - SourceItem { - name: image.to_string(), - version: digest.to_string(), - assets: vec![asset.clone()], - }, - Some(asset), - )); - } - } - } - } - } - scanned -} - -/// `(image, digest)` of a Docker asset path `v2//manifests/`. -fn manifest_by_digest(path: &str) -> Option<(&str, &str)> { - let rest = path.strip_prefix("v2/")?; - let (image, digest) = rest.rsplit_once("/manifests/")?; - (!image.is_empty() && digest.starts_with("sha256:")).then_some((image, digest)) -} - -/// Check a staged download against what Nexus lists for it: its size, its -/// SHA-256 and SHA-512, else its SHA-1. -async fn verify(staged: &StagedBlob, asset: &SourceAsset) -> Result<(), String> { - let mismatch = |what: &str| { - format!( - "the download of {} does not match its {what} in Nexus", - asset.path - ) - }; - if asset - .size - .is_some_and(|size| u64::try_from(size).ok() != Some(staged.size)) - { - return Err(mismatch("size")); - } - let mut checked = false; - if let Some(expected) = &asset.sha256 { - if staged.sha256 != *expected { - return Err(mismatch("SHA-256")); - } - checked = true; - } - if let Some(expected) = &asset.sha512 { - let actual = base64::engine::general_purpose::STANDARD - .decode(&staged.sha512_base64) - .map(hex::encode) - .map_err(|e| e.to_string())?; - if actual != *expected { - return Err(mismatch("SHA-512")); - } - checked = true; - } - if let (false, Some(expected)) = (checked, &asset.sha1) { - let path = staged.read_path(); - let actual = tokio::task::spawn_blocking(move || -> std::io::Result { - use std::io::Read; - let mut file = std::fs::File::open(path)?; - let mut hasher = sha1::Sha1::new(); - let mut buf = vec![0u8; 64 * 1024]; - loop { - let n = file.read(&mut buf)?; - if n == 0 { - break; - } - hasher.update(&buf[..n]); - } - Ok(hex::encode(hasher.finalize())) - }) - .await - .map_err(|e| e.to_string())? - .map_err(|e| e.to_string())?; - if actual != *expected { - return Err(mismatch("SHA-1")); - } - } - Ok(()) -} - -/// Aborts the wrapped task when dropped. -struct AbortOnDrop(JoinHandle<()>); - -impl Drop for AbortOnDrop { - fn drop(&mut self) { - self.0.abort(); - } -} - -/// Renew the lease every third of its length until aborted; flags `lost` -/// and stops once the lease is gone. -fn spawn_heartbeat( - pool: sqlx::PgPool, - id: i64, - run_id: Uuid, - lost: Arc, -) -> JoinHandle<()> { - tokio::spawn(async move { - let mut ticker = tokio::time::interval(Duration::from_secs((LEASE_SECONDS / 3) as u64)); - ticker.tick().await; - loop { - ticker.tick().await; - let renewed = match forge_db::db_now(&pool).await { - Ok(now) => { - import_dao::renew_lease( - &pool, - id, - run_id, - now + chrono::Duration::seconds(LEASE_SECONDS), - ) - .await - } - Err(e) => Err(e), - }; - match renewed { - Ok(true) => {} - Ok(false) => { - lost.store(true, Ordering::Relaxed); - return; - } - Err(e) => { - tracing::warn!(import = id, error = %e, "nexus import lease renewal failed") - } - } - } - }) -} - -fn panic_text(payload: &(dyn std::any::Any + Send)) -> String { - payload - .downcast_ref::<&str>() - .map(|s| (*s).to_string()) - .or_else(|| payload.downcast_ref::().cloned()) - .unwrap_or_else(|| "non-string panic payload".to_string()) -} - -/// The password an import is created or resumed with, if any. -pub fn password_secret(password: Option) -> Option { - password - .filter(|p| !p.is_empty()) - .map(PlaintextSecret::from) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn asset(path: &str) -> SourceAsset { - SourceAsset { - path: path.into(), - ..Default::default() - } - } - - #[test] - fn package_files_are_picked_per_format() { - let nuget = [asset("Sample.Lib/1.0.0")]; - assert_eq!( - package_asset("nuget", &nuget).unwrap().path, - "Sample.Lib/1.0.0" - ); - let two = [asset("app-1.0.0.tgz.prov"), asset("app-1.0.0.tgz")]; - assert!(package_asset("raw", &two).is_none()); - } - - #[test] - fn manifests_held_by_digest_are_recognized() { - assert_eq!( - manifest_by_digest("v2/team/app/manifests/sha256:abc"), - Some(("team/app", "sha256:abc")) - ); - assert_eq!(manifest_by_digest("v2/team/app/manifests/1.0"), None); - assert_eq!(manifest_by_digest("v2/-/blobs/sha256:abc"), None); - } - - #[test] - fn a_docker_asset_listing_counts_blobs_and_keeps_digest_manifests() { - let page = [ - SourceItem { - assets: vec![SourceAsset { - size: Some(100), - ..asset("v2/-/blobs/sha256:aa") - }], - ..Default::default() - }, - SourceItem { - assets: vec![SourceAsset { - size: Some(10), - ..asset("v2/app/manifests/sha256:bb") - }], - ..Default::default() - }, - SourceItem { - assets: vec![SourceAsset { - size: Some(10), - ..asset("v2/app/manifests/1.0") - }], - ..Default::default() - }, - ]; - let scanned = scanned_items("docker", true, Listing::Assets, &page); - assert_eq!(scanned.assets, 3); - assert_eq!(scanned.bytes, 120); - assert_eq!(scanned.items.len(), 1); - assert_eq!(scanned.items[0].0, "manifest"); - assert_eq!(scanned.items[0].1.name, "app"); - assert_eq!(scanned.items[0].1.version, "sha256:bb"); - } -} diff --git a/crates/server/src/nexus_import/nexus.rs b/crates/server/src/nexus_import/nexus.rs deleted file mode 100644 index 3eff1cd0..00000000 --- a/crates/server/src/nexus_import/nexus.rs +++ /dev/null @@ -1,533 +0,0 @@ -//! The parts of the Nexus Repository 3 REST API an import reads: -//! -//! * `GET /service/rest/v1/status`: the server is up; its `Server` header -//! names the version (`Nexus/3.70.1-02 (OSS)`); -//! * `GET /service/rest/v1/repositories`: name, format and type of each -//! repository the credentials may see; -//! * `GET /service/rest/v1/components?repository=` and -//! `…/assets?repository=`, paged with `continuationToken`; -//! * `/repository//`: an asset's content. -//! -//! Requests go through the shared [`UpstreamHttp`] client (timeouts, size -//! caps, no cloud-metadata targets); the credentials are sent to the Nexus -//! origin only. Download URLs are built from the import's base URL and the -//! asset's path rather than taken from Nexus' `downloadUrl`, which carries -//! Nexus' configured base URL and may name a host Forge does not reach the -//! server on. - -use chrono::{DateTime, Utc}; -use forge_core::net::upstream::{Origin, RequestClass, UpstreamAuth, UpstreamError, UpstreamHttp}; -use forge_crypto::{ExposeSecret, PlaintextSecret}; -use reqwest::StatusCode; -use serde::de::{Deserializer, MapAccess, Visitor}; -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value as JsonValue}; - -/// What went wrong talking to Nexus. -#[derive(Debug, thiserror::Error)] -pub enum NexusError { - /// The server could not be reached, or the transfer failed. - #[error("Nexus is unreachable: {0}")] - Unreachable(String), - /// The credentials were refused (or may not see the resource). - #[error("Nexus refused the credentials ({status}) for {url}")] - Refused { status: StatusCode, url: String }, - /// Any other answer than the one asked for. - #[error("Nexus answered {status} for {url}")] - Status { status: StatusCode, url: String }, - /// The answer was not the document the API describes. - #[error("Nexus answered an unexpected document for {url}: {error}")] - Invalid { url: String, error: String }, -} - -impl NexusError { - /// Whether trying again later may succeed. - pub fn is_transient(&self) -> bool { - match self { - NexusError::Unreachable(_) => true, - NexusError::Status { status, .. } => { - status.is_server_error() || *status == StatusCode::TOO_MANY_REQUESTS - } - NexusError::Refused { .. } | NexusError::Invalid { .. } => false, - } - } - - fn from_upstream(e: UpstreamError, url: &str) -> Self { - match e { - UpstreamError::Status { status, url } => status_error(status, url), - UpstreamError::Json { source, .. } => NexusError::Invalid { - url: url.to_string(), - error: source.to_string(), - }, - other => NexusError::Unreachable(other.to_string()), - } - } -} - -fn status_error(status: StatusCode, url: String) -> NexusError { - if matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) { - NexusError::Refused { status, url } - } else { - NexusError::Status { status, url } - } -} - -/// A repository as `GET /service/rest/v1/repositories` lists it. -#[derive(Debug, Clone, Deserialize)] -pub struct NexusRepository { - pub name: String, - /// `nuget`, `docker`, `helm`, `apt`, `maven2`, … - pub format: String, - /// `hosted`, `proxy` or `group`. - #[serde(rename = "type")] - pub kind: String, -} - -/// The two listings an import walks. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Listing { - Components, - Assets, -} - -impl Listing { - pub fn as_str(self) -> &'static str { - match self { - Listing::Components => "components", - Listing::Assets => "assets", - } - } - - pub fn parse(s: &str) -> Self { - match s { - "assets" => Listing::Assets, - _ => Listing::Components, - } - } -} - -/// An asset as an import keeps it (the item's `source`). -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct SourceAsset { - /// The path within the repository (`Sample.Lib/1.0.0`, - /// `v2/team/app/manifests/1.0`). - pub path: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub content_type: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub size: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub sha1: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub sha256: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub sha512: Option, - /// `blobCreated`: when these bytes were stored. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub created: Option>, - /// `lastModified`: when the path last changed (a tag last moved). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub modified: Option>, - /// `lastDownloaded`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub downloaded: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub uploader: Option, -} - -impl SourceAsset { - /// Read an asset object of either listing. `None` without a path. - fn from_object(object: &Map) -> Option { - let text = |key: &str| { - object - .get(key) - .and_then(JsonValue::as_str) - .filter(|s| !s.is_empty()) - .map(str::to_string) - }; - let time = |key: &str| { - object - .get(key) - .and_then(JsonValue::as_str) - .and_then(|s| DateTime::parse_from_rfc3339(s).ok()) - .map(|t| t.with_timezone(&Utc)) - }; - let checksum = |key: &str| { - object - .get("checksum") - .and_then(|c| c.get(key)) - .and_then(JsonValue::as_str) - .filter(|s| !s.is_empty()) - .map(str::to_ascii_lowercase) - }; - Some(SourceAsset { - path: text("path")?, - content_type: text("contentType"), - size: object.get("fileSize").and_then(JsonValue::as_i64), - sha1: checksum("sha1"), - sha256: checksum("sha256"), - sha512: checksum("sha512"), - created: time("blobCreated"), - modified: time("lastModified"), - downloaded: time("lastDownloaded"), - uploader: text("uploader"), - }) - } -} - -/// A component (or, for Docker, a manifest held by digest) as an import -/// keeps it. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct SourceItem { - /// The package id or image name. - pub name: String, - /// The version, tag, or manifest digest. - pub version: String, - pub assets: Vec, -} - -/// One page of a listing. -#[derive(Debug, Default)] -pub struct Page { - /// Components (a components listing), or every asset (an assets - /// listing) as one-asset items without a name or version. - pub items: Vec, - /// The token of the next page; `None` on the last. - pub next: Option, -} - -/// A client of one Nexus server. -pub struct Nexus { - http: UpstreamHttp, - base: String, - origin: Option, - username: Option, - password: Option, -} - -impl Nexus { - /// A client of the Nexus at `base_url` (a context path included, no - /// trailing `/`), with Basic credentials when both are given. - pub fn new( - http: UpstreamHttp, - base_url: &str, - username: Option, - password: Option, - ) -> Self { - let base = base_url.trim_end_matches('/').to_string(); - Nexus { - origin: Origin::of(&base), - http, - base, - username, - password, - } - } - - fn auth(&self) -> Option> { - match (&self.username, &self.password) { - (Some(username), Some(password)) => Some(UpstreamAuth { - origin: self.origin.clone(), - username, - password: password.expose_secret(), - bearer: false, - }), - _ => None, - } - } - - fn api(&self, path: &str, query: &[(&str, &str)]) -> String { - let mut url = format!("{}/service/rest/v1/{path}", self.base); - if !query.is_empty() { - url.push('?'); - url.push_str( - &url::form_urlencoded::Serializer::new(String::new()) - .extend_pairs(query) - .finish(), - ); - } - url - } - - /// GET a JSON document of the API and parse it as `T`. - async fn get(&self, url: &str) -> Result { - let bytes = self - .http - .get_bytes(url, self.auth().as_ref()) - .await - .map_err(|e| NexusError::from_upstream(e, url))? - .ok_or_else(|| NexusError::Status { - status: StatusCode::NOT_FOUND, - url: url.to_string(), - })?; - serde_json::from_slice(&bytes).map_err(|e| NexusError::Invalid { - url: url.to_string(), - error: e.to_string(), - }) - } - - /// The server's version (its `Server` header), checking that it is up - /// and is Nexus Repository 3. - pub async fn status(&self) -> Result { - let url = self.api("status", &[]); - let response = self - .http - .get(&url, RequestClass::Metadata, self.auth().as_ref()) - .await - .map_err(|e| NexusError::from_upstream(e, &url))?; - if !response.status().is_success() { - return Err(status_error(response.status(), url)); - } - let server = response - .headers() - .get(reqwest::header::SERVER) - .and_then(|v| v.to_str().ok()) - .unwrap_or_default() - .to_string(); - if !server.starts_with("Nexus/") { - return Err(NexusError::Invalid { - url, - error: format!("not a Nexus Repository 3 server (Server: {server:?})"), - }); - } - Ok(server) - } - - /// The repository `name`, if the credentials may see it. - pub async fn repository(&self, name: &str) -> Result, NexusError> { - let url = self.api("repositories", &[]); - let repositories: Vec = self.get(&url).await?; - Ok(repositories.into_iter().find(|r| r.name == name)) - } - - /// One page of `listing` of `repository`, after `cursor`. - pub async fn page( - &self, - listing: Listing, - repository: &str, - cursor: Option<&str>, - ) -> Result { - let mut query = vec![("repository", repository)]; - if let Some(cursor) = cursor { - query.push(("continuationToken", cursor)); - } - let url = self.api(listing.as_str(), &query); - let raw: RawPage = self.get(&url).await?; - let items = match listing { - Listing::Components => raw - .items - .iter() - .filter_map(|item| { - let component: RawComponent = - serde_json::from_value(JsonValue::Object(item.0.clone())).ok()?; - Some(SourceItem { - name: component.name?, - version: component.version?, - assets: component - .assets - .iter() - .filter_map(|asset| SourceAsset::from_object(&asset.0)) - .collect(), - }) - }) - .collect(), - Listing::Assets => raw - .items - .iter() - .filter_map(|item| SourceAsset::from_object(&item.0)) - .map(|asset| SourceItem { - name: String::new(), - version: String::new(), - assets: vec![asset], - }) - .collect(), - }; - Ok(Page { - items, - next: raw.continuation_token.filter(|t| !t.is_empty()), - }) - } - - /// The URL of the asset at `path` in `repository`. - pub fn content_url(&self, repository: &str, path: &str) -> String { - let mut url = format!("{}/repository/", self.base); - let segments = std::iter::once(repository).chain(path.split('/')); - let encoded: Vec = segments.map(percent_encode_segment).collect(); - url.push_str(&encoded.join("/")); - url - } - - /// GET the content of an asset (streamed by the caller). - pub async fn download(&self, url: &str) -> Result { - let response = self - .http - .get(url, RequestClass::Download, self.auth().as_ref()) - .await - .map_err(|e| NexusError::from_upstream(e, url))?; - if !response.status().is_success() { - return Err(status_error(response.status(), url.to_string())); - } - Ok(response) - } - - /// Whether `url` answers the registry API (`/v2/`): 200, or a 401 - /// challenge. - pub async fn probe_registry(&self, url: &str) -> Result<(), NexusError> { - let response = self - .http - .get(url, RequestClass::Metadata, self.auth().as_ref()) - .await - .map_err(|e| NexusError::from_upstream(e, url))?; - let status = response.status(); - let answers = status.is_success() - || (status == StatusCode::UNAUTHORIZED - && response - .headers() - .contains_key(reqwest::header::WWW_AUTHENTICATE)); - if answers { - Ok(()) - } else { - Err(status_error(status, url.to_string())) - } - } -} - -/// Percent-encode one path segment (RFC 3986 `pchar`, minus the -/// sub-delimiters a server might treat specially). -fn percent_encode_segment(segment: &str) -> String { - let mut out = String::with_capacity(segment.len()); - for byte in segment.bytes() { - match byte { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' | b':' | b'@' => { - out.push(byte as char) - } - _ => out.push_str(&format!("%{byte:02X}")), - } - } - out -} - -#[derive(Deserialize)] -struct RawPage { - #[serde(default)] - items: Vec, - #[serde(rename = "continuationToken", default)] - continuation_token: Option, -} - -#[derive(Deserialize)] -struct RawComponent { - #[serde(default)] - name: Option, - #[serde(default)] - version: Option, - #[serde(default)] - assets: Vec, -} - -/// A JSON object whose repeated keys keep their first non-null value. Nexus -/// 3.70 writes `"blobCreated": null` and then `"blobCreated": "