diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 10a1ec79..a1706a9c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -725,6 +725,7 @@ jobs: docker pull mirror.gcr.io/versity/versitygw:v1.9.0 docker pull mirror.gcr.io/library/redis:5.0 docker pull mirror.gcr.io/sonatype/nexus3:3.70.1 + docker pull mirror.gcr.io/verdaccio/verdaccio:6.11.1 - name: Run e2e tests (forge-e2e feature) id: tests diff --git a/.sqlx/query-1633b8039e8f13812668b97d0b09ea4f43199955578f439fc23348a8a9a6a58d.json b/.sqlx/query-1633b8039e8f13812668b97d0b09ea4f43199955578f439fc23348a8a9a6a58d.json new file mode 100644 index 00000000..e44d3d26 --- /dev/null +++ b/.sqlx/query-1633b8039e8f13812668b97d0b09ea4f43199955578f439fc23348a8a9a6a58d.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE package_tag SET version = '1.0.0' WHERE repository_id = $1 AND package = 'alpha' AND tag = 'next'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1633b8039e8f13812668b97d0b09ea4f43199955578f439fc23348a8a9a6a58d" +} diff --git a/.sqlx/query-7f1c4828bf0ff818ceeb6815333c10042c2c391c8743083468a4d65b1ff5a6b2.json b/.sqlx/query-7f1c4828bf0ff818ceeb6815333c10042c2c391c8743083468a4d65b1ff5a6b2.json new file mode 100644 index 00000000..b9eebad0 --- /dev/null +++ b/.sqlx/query-7f1c4828bf0ff818ceeb6815333c10042c2c391c8743083468a4d65b1ff5a6b2.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT i.source\n FROM repository_import_item i\n JOIN repository_import r ON r.id = i.import_id\n WHERE r.repository_id = $1 AND r.id < $2 AND NOT r.dry_run\n AND i.kind = $3 AND i.name = $4 AND i.version = $5\n AND i.state IN ('imported', 'replaced', 'identical')\n ORDER BY r.id DESC\n LIMIT 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "source", + "type_info": "Jsonb", + "origin": { + "Table": { + "table": "repository_import_item", + "name": "source" + } + } + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7f1c4828bf0ff818ceeb6815333c10042c2c391c8743083468a4d65b1ff5a6b2" +} diff --git a/.sqlx/query-be57f461cb3eeac3b4de91e8a938e2813009c7dd4001c388e74c4ccd2bd816b5.json b/.sqlx/query-be57f461cb3eeac3b4de91e8a938e2813009c7dd4001c388e74c4ccd2bd816b5.json new file mode 100644 index 00000000..f1e2e507 --- /dev/null +++ b/.sqlx/query-be57f461cb3eeac3b4de91e8a938e2813009c7dd4001c388e74c4ccd2bd816b5.json @@ -0,0 +1,30 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT deprecation FROM component\n WHERE repository_id = $1 AND name = $2 AND version = $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "deprecation", + "type_info": "Text", + "origin": { + "Table": { + "table": "component", + "name": "deprecation" + } + } + } + ], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "be57f461cb3eeac3b4de91e8a938e2813009c7dd4001c388e74c4ccd2bd816b5" +} diff --git a/.sqlx/query-e94844cd18f33b4fce6277abd9c40b2083075887349cc02195750b79f505e7dd.json b/.sqlx/query-e94844cd18f33b4fce6277abd9c40b2083075887349cc02195750b79f505e7dd.json new file mode 100644 index 00000000..755f24b4 --- /dev/null +++ b/.sqlx/query-e94844cd18f33b4fce6277abd9c40b2083075887349cc02195750b79f505e7dd.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE component SET deprecation = NULL WHERE repository_id = $1 AND name = 'alpha' AND version = '1.0.0'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "e94844cd18f33b4fce6277abd9c40b2083075887349cc02195750b79f505e7dd" +} diff --git a/README.md b/README.md index 8a8241ee..67737ee4 100644 --- a/README.md +++ b/README.md @@ -239,8 +239,8 @@ The guides below are also a site, with navigation and search: - **[Format plugins](./docs/format-plugins.md)** — further package formats, built into the server: design, operation, and writing one. - **[Importing repositories](./docs/import.md)** — copy a hosted repository - of Nexus, Artifactory, a registry or another Forge (NuGet, Docker) into - Forge through the format's own protocol: dry run, conflicts, resuming, and + of Nexus, Artifactory, Verdaccio, a registry or another Forge (NuGet, npm, + Docker) into Forge through the format's own protocol: dry run, conflicts, resuming, and the cutover. Run `cargo xtask help` for the complete command list. diff --git a/crates/core/src/net/upstream.rs b/crates/core/src/net/upstream.rs index d0972f5c..b165eb5e 100644 --- a/crates/core/src/net/upstream.rs +++ b/crates/core/src/net/upstream.rs @@ -786,8 +786,11 @@ impl UpstreamHttp { } /// GET a metadata body, size-capped and spooled; a gzip-encoded body is - /// inflated (on the blocking pool) under the same cap. - async fn get_metadata_body( + /// inflated (on the blocking pool) under the same cap. `Ok(None)` on a + /// 404; any other non-2xx is an error. A large document (an npm + /// packument) is read from the spool as it streams by, rather than held + /// in memory whole. + pub async fn get_metadata_body( &self, url: &str, auth: Option<&UpstreamAuth<'_>>, diff --git a/crates/core/src/repo/hosted.rs b/crates/core/src/repo/hosted.rs index 12e3c6e0..b54b683c 100644 --- a/crates/core/src/repo/hosted.rs +++ b/crates/core/src/repo/hosted.rs @@ -827,6 +827,22 @@ impl HostedWrite { )) }; + // An import of a package the source named otherwise stores nothing. + if let Some((id, version)) = import.and_then(|options| options.expected.as_ref()) { + let same_version = self + .format + .normalize_version(version) + .is_ok_and(|v| v == parsed.version); + if self.format.lower_id(id) != parsed.id || !same_version { + return Err(WriteError::InvalidPackage( + crate::domain::FormatError::InvalidPackage(format!( + "the package is {} {}, not {id} {version}", + parsed.original_id, parsed.version + )), + )); + } + } + let mut publish_context = self.publish_policy .as_ref() diff --git a/crates/core/src/repo/mod.rs b/crates/core/src/repo/mod.rs index 66010398..a42a6c47 100644 --- a/crates/core/src/repo/mod.rs +++ b/crates/core/src/repo/mod.rs @@ -214,6 +214,12 @@ pub struct ImportOptions { pub uploader: Option, /// Where it came from, for the audit trail (the source's URL). pub source: String, + /// The package the source says this is, `(id, version)` as the format + /// writes them: a package that turns out to be another one is refused + /// before anything is written ([`crate::domain::FormatError`]). An npm + /// registry names a tarball in a package's packument, and the tarball's + /// `package.json` must agree. + pub expected: Option<(String, String)>, } /// What an import did with one package version. diff --git a/crates/db/src/component_dao.rs b/crates/db/src/component_dao.rs index 269d4b47..b8241b78 100644 --- a/crates/db/src/component_dao.rs +++ b/crates/db/src/component_dao.rs @@ -614,6 +614,30 @@ pub async fn set_imported_times( Ok(()) } +/// Why version `version` of `name` is deprecated: `None` when there is no +/// such version, `Some(None)` when it is not deprecated. +pub async fn deprecation<'e, E>( + executor: E, + repository_id: i64, + name: &str, + version: &str, +) -> sqlx::Result>> +where + E: sqlx::PgExecutor<'e>, +{ + sqlx::query_scalar!( + r#" + SELECT deprecation FROM component + WHERE repository_id = $1 AND name = $2 AND version = $3 + "#, + repository_id, + name, + version, + ) + .fetch_optional(executor) + .await +} + /// Deprecate version `version` of `name` with `message`, or (`None`) no /// longer. Returns whether the version exists. pub async fn set_deprecation<'e, E>( diff --git a/crates/db/src/import_dao.rs b/crates/db/src/import_dao.rs index bd0b1e5c..b5652f16 100644 --- a/crates/db/src/import_dao.rs +++ b/crates/db/src/import_dao.rs @@ -47,7 +47,8 @@ pub struct RepositoryImport { pub state: String, /// `scan` or `copy`. pub phase: String, - /// What the scan walks: `search`, `catalog` or `names`. + /// What the scan walks: `search`, `catalog`, `all` (npm's `/-/all`) or + /// `names`. pub scan_listing: String, #[serde(skip)] pub cursor: Option, @@ -81,7 +82,7 @@ pub struct NewImport<'a> { pub on_conflict: &'a str, pub concurrency: i32, pub dry_run: bool, - /// Where the scan starts: `search`, `catalog` or `names`. + /// Where the scan starts: `search`, `catalog`, `all` or `names`. pub scan_listing: &'a str, pub source_server: Option<&'a str>, pub source_total: Option, @@ -442,7 +443,9 @@ pub struct ImportItem { pub id: i64, #[serde(skip)] pub import_id: i64, - /// `component` or `manifest`. + /// `component` (a version, a Docker tag), `manifest` (a Docker referrer + /// held by digest) or `tag` (an npm dist-tag: `name` the package, + /// `version` the tag). pub kind: String, pub name: String, pub version: String, @@ -556,6 +559,40 @@ pub async fn finish_item( Ok(()) } +/// What the newest earlier import into repository `repository_id` (before +/// import `import_id`, a dry run none of them) recorded of the item `kind` +/// `name` `version` it copied or found identical: its `source`. Where Forge +/// still holds what that import set (a dist-tag where it pointed it, a +/// deprecation it gave), a later import follows the source. +pub async fn earlier_source( + pool: &PgPool, + repository_id: i64, + import_id: i64, + kind: &str, + name: &str, + version: &str, +) -> sqlx::Result> { + sqlx::query_scalar!( + r#" + SELECT i.source + FROM repository_import_item i + JOIN repository_import r ON r.id = i.import_id + WHERE r.repository_id = $1 AND r.id < $2 AND NOT r.dry_run + AND i.kind = $3 AND i.name = $4 AND i.version = $5 + AND i.state IN ('imported', 'replaced', 'identical') + ORDER BY r.id DESC + LIMIT 1 + "#, + repository_id, + import_id, + kind, + name, + version, + ) + .fetch_optional(pool) + .await +} + /// Forget an item: a manifest the source holds by digest that turned out not /// to be a referrer of an imported image. pub async fn drop_item(pool: &PgPool, item_id: i64) -> sqlx::Result<()> { diff --git a/crates/e2e/tests/import.rs b/crates/e2e/tests/import.rs index 23db1c3e..f7d4591b 100644 --- a/crates/e2e/tests/import.rs +++ b/crates/e2e/tests/import.rs @@ -8,19 +8,28 @@ //! referrers tag schema); //! * a `distribution` registry (`registry:2`): Docker through its catalog; //! * another Forge: NuGet from a Forge NuGet repository, whose registration -//! lists each package's SHA-512, so a second run downloads nothing. +//! lists each package's SHA-512, so a second run downloads nothing; +//! * npm from Nexus' npm hosted repository and from Verdaccio +//! (`verdaccio/verdaccio`), both through their `/-/all`: scoped and +//! unscoped packages, a prerelease, dist-tags and a deprecated version +//! arrive with the source's bytes and publish times, `npm install` from +//! Forge works (when npm is installed), and a second run copies only the +//! new release and moves `latest`. //! //! After an import, bytes and digests are the source's and so are the NuGet //! publish dates; a second run copies only what was added since. (Resuming //! a killed import, the conflict policies, the dry run and a source without //! a search or catalog are covered against a mock source in //! `crates/server/tests/import.rs`.) Gated behind `forge-e2e`; Nexus takes a -//! minute or two to start. +//! minute or two to start, and the tests that start one take turns. #![cfg(feature = "forge-e2e")] +use std::collections::BTreeMap; use std::time::Duration; +use base64::Engine; + use forge_e2e::{make_nupkg, StackBuilder, TestServer}; use forge_server::import::ImportWorker; use serde_json::{json, Value}; @@ -552,6 +561,7 @@ async fn assert_signed_index( #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn import_from_a_real_nexus() { + let _turn = NEXUS.lock().await; let (nexus, _container) = Nexus::start().await; nexus.seed_repositories().await; @@ -782,3 +792,461 @@ async fn import_from_another_forge() { assert_eq!(done["progress"]["bytes_copied"], 0, "{done}"); assert_eq!(downloads().await, before); } + +// ── npm ───────────────────────────────────────────────────────────────────── + +const VERDACCIO_IMAGE: &str = "mirror.gcr.io/verdaccio/verdaccio"; +const VERDACCIO_TAG: &str = "6.11.1"; + +/// Verdaccio without an uplink (it would refuse a publish while +/// registry.npmjs.org is out of reach), with a user database, and the +/// `@acme` scope readable by its users only: the import must send its +/// credentials. +const VERDACCIO_CONFIG: &str = "\ +storage: /verdaccio/storage/data +plugins: /verdaccio/plugins +auth: + htpasswd: + file: /verdaccio/storage/htpasswd + max_users: 100 +packages: + '@acme/*': + access: $authenticated + publish: $authenticated + unpublish: $authenticated + '**': + access: $all + publish: $authenticated + unpublish: $authenticated +log: + type: stdout + format: pretty + level: http +"; + +/// Nexus takes a few GiB: the tests that start one take turns. +static NEXUS: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +/// An npm registry to publish to, as npm does. +struct NpmSource { + /// The registry's URL, without a trailing `/`. + url: String, + /// The `Authorization` header npm sends. + auth: String, + client: reqwest::Client, +} + +/// A tarball of `name@version`. +fn npm_tgz(name: &str, version: &str) -> Vec { + let package_json = json!({ + "name": name, "version": version, "main": "index.js", + "description": format!("{name} for the import e2e"), + }) + .to_string(); + let index = format!("module.exports = {:?};\n", format!("{name}@{version}")); + forge_format_npm::tarball::build(&[ + ("package/package.json", package_json.as_bytes()), + ("package/index.js", index.as_bytes()), + ]) +} + +impl NpmSource { + fn escaped(name: &str) -> String { + name.replacen('/', "%2f", 1) + } + + /// `npm publish` of `tarball` as `name@version`, tagged `tag`. + async fn publish(&self, name: &str, version: &str, tag: &str, tarball: &[u8]) { + let digests = forge_format_npm::integrity::Digests::of(tarball).unwrap(); + let unscoped = name.rsplit('/').next().unwrap(); + let file = format!("{unscoped}-{version}.tgz"); + let body = json!({ + "_id": name, "name": name, + "description": format!("{name} for the import e2e"), + "dist-tags": {tag: version}, + "versions": {version: { + "name": name, "version": version, "main": "index.js", + "description": format!("{name} for the import e2e"), + "_id": format!("{name}@{version}"), + "dist": { + "integrity": digests.integrity(), "shasum": digests.sha1_hex, + "tarball": format!("{}/{name}/-/{file}", self.url), + }, + }}, + "access": null, + "_attachments": {file: { + "content_type": "application/octet-stream", + "data": base64::engine::general_purpose::STANDARD.encode(tarball), + "length": tarball.len(), + }}, + }); + let resp = self + .client + .put(format!("{}/{}", self.url, Self::escaped(name))) + .header("authorization", &self.auth) + .json(&body) + .send() + .await + .unwrap(); + assert!( + resp.status().is_success(), + "publish {name}@{version}: {} {}", + resp.status(), + resp.text().await.unwrap() + ); + } + + /// `npm dist-tag add name@version tag`. + async fn tag(&self, name: &str, version: &str, tag: &str) { + let resp = self + .client + .put(format!( + "{}/-/package/{}/dist-tags/{tag}", + self.url, + Self::escaped(name) + )) + .header("authorization", &self.auth) + .json(&json!(version)) + .send() + .await + .unwrap(); + assert!(resp.status().is_success(), "dist-tag: {}", resp.status()); + } + + /// The full packument of `name`. + async fn packument(&self, name: &str) -> Value { + let resp = self + .client + .get(format!("{}/{}", self.url, Self::escaped(name))) + .header("authorization", &self.auth) + .header("accept", "application/json") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 200, "packument of {name}"); + resp.json().await.unwrap() + } + + /// `npm deprecate name@version message`: the packument, written back + /// with the version's `deprecated`. + async fn deprecate(&self, name: &str, version: &str, message: &str) { + let mut doc = self.packument(name).await; + doc["versions"][version]["deprecated"] = json!(message); + if let Some(doc) = doc.as_object_mut() { + doc.remove("_attachments"); + } + let resp = self + .client + .put(format!("{}/{}", self.url, Self::escaped(name))) + .header("authorization", &self.auth) + .json(&doc) + .send() + .await + .unwrap(); + assert!( + resp.status().is_success(), + "deprecate {name}@{version}: {} {}", + resp.status(), + resp.text().await.unwrap() + ); + } + + /// The packages the e2e imports: scoped and unscoped, a prerelease + /// under `next`, a dist-tag of its own on an older version, and a + /// deprecated version. Returns the tarballs by `name@version`. + async fn seed(&self) -> BTreeMap> { + let mut tarballs = BTreeMap::new(); + for (name, version, tag) in [ + ("alpha", "1.0.0", "latest"), + ("alpha", "1.1.0", "latest"), + ("alpha", "2.0.0-beta.1", "next"), + ("@acme/ui", "1.0.0", "latest"), + ("@acme/ui", "1.0.1", "latest"), + ("@acme/core", "0.1.0", "latest"), + ] { + let tarball = npm_tgz(name, version); + self.publish(name, version, tag, &tarball).await; + tarballs.insert(format!("{name}@{version}"), tarball); + } + self.tag("@acme/ui", "1.0.0", "stable").await; + self.deprecate("alpha", "1.0.0", "use 1.1.0").await; + tarballs + } +} + +/// A Verdaccio, its user's credentials, and an npm client of it. +async fn verdaccio() -> ( + NpmSource, + (String, String), + testcontainers::ContainerAsync, +) { + let container = GenericImage::new(VERDACCIO_IMAGE, VERDACCIO_TAG) + .with_exposed_port(4873.tcp()) + .with_wait_for(WaitFor::message_on_stdout("http address")) + .with_copy_to( + "/verdaccio/conf/config.yaml", + VERDACCIO_CONFIG.as_bytes().to_vec(), + ) + .with_startup_timeout(Duration::from_secs(120)) + .start() + .await + .expect("start verdaccio (is Docker running?)"); + let host = container.get_host().await.unwrap(); + let port = container.get_host_port_ipv4(4873).await.unwrap(); + let url = format!("http://{host}:{port}"); + let client = reqwest::Client::new(); + let (user, password) = ("importer".to_string(), "importer-secret".to_string()); + let resp = client + .put(format!("{url}/-/user/org.couchdb.user:{user}")) + .json(&json!({"name": user, "password": password})) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "add the user"); + let token = resp.json::().await.unwrap()["token"] + .as_str() + .unwrap() + .to_string(); + ( + NpmSource { + url, + auth: format!("Bearer {token}"), + client, + }, + (user, password), + container, + ) +} + +/// The Forge packument of `name` in `repo`. +async fn forge_packument(srv: &TestServer, repo: &str, name: &str) -> Value { + serde_json::from_slice( + &download( + srv, + &format!("/repository/{repo}/{}", NpmSource::escaped(name)), + ) + .await, + ) + .unwrap() +} + +/// After an import of `source`'s packages into `repo`: their bytes, publish +/// times, dist-tags and deprecation are the source's. +async fn assert_npm_copied( + srv: &TestServer, + repo: &str, + source: &NpmSource, + tarballs: &BTreeMap>, +) { + for (spec, tarball) in tarballs { + let (name, version) = spec.rsplit_once('@').unwrap(); + let unscoped = name.rsplit('/').next().unwrap(); + assert_eq!( + &download( + srv, + &format!("/repository/{repo}/{name}/-/{unscoped}-{version}.tgz") + ) + .await, + tarball, + "{spec}" + ); + } + for name in ["alpha", "@acme/ui", "@acme/core"] { + let theirs = source.packument(name).await; + let ours = forge_packument(srv, repo, name).await; + assert_eq!(ours["dist-tags"], theirs["dist-tags"], "{name}"); + for (version, object) in theirs["versions"].as_object().unwrap() { + assert_eq!( + ours["versions"][version]["dist"]["integrity"], object["dist"]["integrity"], + "{name}@{version}" + ); + assert_eq!( + ours["versions"][version].get("deprecated"), + object.get("deprecated").filter(|m| m != &&json!("")), + "{name}@{version}" + ); + let time = |doc: &Value| { + chrono::DateTime::parse_from_rfc3339(doc["time"][version].as_str().unwrap()) + .unwrap() + }; + assert_eq!(time(&ours), time(&theirs), "{name}@{version}"); + } + } + assert_eq!( + forge_packument(srv, repo, "alpha").await["versions"]["1.0.0"]["deprecated"], + "use 1.1.0" + ); +} + +/// `npm install` of `specs` from Forge's `repo`, with the admin's token; +/// the versions installed, by package. `None` without npm. +fn npm_install(srv: &TestServer, repo: &str, specs: &[&str]) -> Option> { + let has_npm = std::process::Command::new("npm") + .arg("--version") + .output() + .is_ok_and(|o| o.status.success()); + if !has_npm { + eprintln!("npm is not installed: the npm install from Forge is skipped"); + return None; + } + let dir = tempfile::tempdir().unwrap(); + let registry = format!("{}/repository/{repo}/", srv.base_url); + let (_, token) = srv.admin_basic(); + let host = registry.trim_start_matches("http:"); + std::fs::write( + dir.path().join(".npmrc"), + format!("registry={registry}\n{host}:_authToken={token}\n"), + ) + .unwrap(); + std::fs::write( + dir.path().join("package.json"), + r#"{"name": "consumer", "version": "1.0.0", "private": true}"#, + ) + .unwrap(); + let out = std::process::Command::new("npm") + .args(["install", "--no-audit", "--no-fund", "--no-update-notifier"]) + .args(specs) + .current_dir(dir.path()) + .env("npm_config_cache", dir.path().join(".cache")) + .output() + .expect("run npm"); + assert!( + out.status.success(), + "npm install {specs:?}: {}\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + let mut installed = BTreeMap::new(); + for spec in specs { + let name = match spec.rfind('@') { + Some(at) if at > 0 => &spec[..at], + _ => spec, + }; + let manifest: Value = serde_json::from_slice( + &std::fs::read( + dir.path() + .join("node_modules") + .join(name) + .join("package.json"), + ) + .unwrap(), + ) + .unwrap(); + installed.insert( + name.to_string(), + manifest["version"].as_str().unwrap().to_string(), + ); + } + Some(installed) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn npm_import_from_a_real_nexus() { + let _turn = NEXUS.lock().await; + let (nexus, _container) = Nexus::start().await; + let hosted = json!({ + "name": "npm-hosted", "online": true, + "storage": {"blobStoreName": "default", "strictContentTypeValidation": true, "writePolicy": "allow_once"}, + }); + let resp = nexus + .api(reqwest::Method::POST, "repositories/npm/hosted") + .json(&hosted) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap()); + let basic = + base64::engine::general_purpose::STANDARD.encode(format!("{}:{}", ADMIN.0, ADMIN.1)); + let source = NpmSource { + url: format!("{}/repository/npm-hosted", nexus.url), + auth: format!("Basic {basic}"), + client: reqwest::Client::new(), + }; + let mut tarballs = source.seed().await; + + let srv = StackBuilder::default().start().await; + create_repo( + &srv, + json!({"name": "npm", "kind": "hosted", "format": "npm"}), + ) + .await; + let body = json!({"url": source.url, "username": ADMIN.0, "password": ADMIN.1}); + let done = import(&srv, "npm", body.clone()).await; + assert_eq!(done["state"], "completed", "{done}"); + // Nexus' search finds nothing without a text: its /-/all lists them. + assert_eq!(done["listing"], "all", "{done}"); + assert_eq!(done["source_total"], 3, "{done}"); + assert!( + done["source_server"] + .as_str() + .unwrap() + .starts_with("Nexus/"), + "{done}" + ); + // Six versions; alpha's latest and next, @acme/ui's latest and stable, + // @acme/core's latest. + assert_eq!(done["progress"]["failed"], 0, "{done}"); + assert_eq!(done["progress"]["imported"], 11, "{done}"); + assert_npm_copied(&srv, "npm", &source, &tarballs).await; + if let Some(installed) = npm_install(&srv, "npm", &["alpha", "@acme/ui@stable", "@acme/core"]) { + assert_eq!( + installed, + BTreeMap::from([ + ("alpha".to_string(), "1.1.0".to_string()), + ("@acme/ui".to_string(), "1.0.0".to_string()), + ("@acme/core".to_string(), "0.1.0".to_string()), + ]) + ); + } + + // A new release becomes latest at Nexus: a second run copies it alone + // and moves the tag. + let newer = npm_tgz("alpha", "1.2.0"); + source.publish("alpha", "1.2.0", "latest", &newer).await; + tarballs.insert("alpha@1.2.0".into(), newer.clone()); + let done = import(&srv, "npm", body).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["imported"], 1, "{done}"); + assert_eq!(done["progress"]["replaced"], 1, "{done}"); + assert_eq!(done["progress"]["identical"], 10, "{done}"); + assert_eq!(done["progress"]["bytes_copied"], newer.len(), "{done}"); + assert_npm_copied(&srv, "npm", &source, &tarballs).await; + if let Some(installed) = npm_install(&srv, "npm", &["alpha", "alpha@next"]) { + assert_eq!(installed["alpha"], "2.0.0-beta.1"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn npm_import_from_verdaccio() { + let (source, (user, password), _container) = verdaccio().await; + let tarballs = source.seed().await; + + let srv = StackBuilder::default().start().await; + create_repo( + &srv, + json!({"name": "npm", "kind": "hosted", "format": "npm"}), + ) + .await; + let body = json!({"url": source.url, "username": user, "password": password}); + let done = import(&srv, "npm", body.clone()).await; + assert_eq!(done["state"], "completed", "{done}"); + // Verdaccio's search needs a text: its /-/all lists them, the @acme + // scope to its users alone. + assert_eq!(done["listing"], "all", "{done}"); + assert_eq!(done["source_total"], 3, "{done}"); + assert_eq!(done["progress"]["failed"], 0, "{done}"); + assert_eq!(done["progress"]["imported"], 11, "{done}"); + assert_npm_copied(&srv, "npm", &source, &tarballs).await; + if let Some(installed) = npm_install(&srv, "npm", &["alpha@next", "@acme/ui"]) { + assert_eq!(installed["alpha"], "2.0.0-beta.1"); + assert_eq!(installed["@acme/ui"], "1.0.1"); + } + + // A scope named: its packages from /-/all. Everything is there already. + let mut named = body; + named["names"] = json!(["@acme"]); + let done = import(&srv, "npm", named).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["identical"], 6, "{done}"); + assert_eq!(done["progress"]["bytes_copied"], 0, "{done}"); +} diff --git a/crates/formats/npm/src/import.rs b/crates/formats/npm/src/import.rs new file mode 100644 index 00000000..d6909dfb --- /dev/null +++ b/crates/formats/npm/src/import.rs @@ -0,0 +1,414 @@ +//! What an import reads of another npm registry (Nexus, Verdaccio, +//! Artifactory, another Forge) to copy its packages into a hosted +//! repository. The host sends the requests; this module says where to, and +//! reads the answers. +//! +//! npm's protocol has no listing of every package, so the names come from +//! one of: +//! +//! * the search, `GET /-/v1/search?text=&size=&from=` ([`search_url`], +//! [`read_search`]), paged with `from`. With an empty `text` another Forge +//! lists every package; Nexus 3.70 finds nothing, and Verdaccio refuses +//! it. Nexus also answers a scoped package by its name alone (`ui` for +//! `@acme/ui`), without a `scope`; +//! * the legacy listing, `GET /-/all` ([`ALL_PATH`], [`read_all`]): an object +//! keyed by every package's name, which Nexus' hosted repositories and +//! Verdaccio still serve; +//! * the names, or scopes (`@acme`, [`parse_scope`]), the import gives. +//! +//! A package's versions come from its full packument ([`read_package`], +//! fetched with [`packument_url`]): each version's `dist.integrity` (its +//! SHA-512) and `dist.shasum`, its publish time (`time`), its `deprecated` +//! message, and the package's dist-tags. The tarball is fetched from the +//! import's URL ([`tarball_url`]), not from `dist.tarball`, whose host is +//! the source's configured base URL, which need not be one the importing +//! server reaches the source on. + +use std::collections::BTreeMap; +use std::fmt; +use std::io::Read; + +use chrono::{DateTime, Utc}; +use serde::de::{IgnoredAny, MapAccess, Visitor}; +use serde::Deserializer; +use serde_json::Value; + +use crate::integrity::sha512_of; +use crate::upstream::{self, Keep, Select}; +use crate::PackageName; + +/// The largest page of a search (npm's registry allows 250). +pub const MAX_SEARCH_SIZE: u32 = 250; + +/// The legacy listing of every package, below the registry's URL. +pub const ALL_PATH: &str = "/-/all"; + +/// `/-/v1/search` of the registry at `base` (no trailing `/`), for `text`. +pub fn search_url(base: &str, text: &str, from: u32, size: u32) -> String { + format!( + "{base}/-/v1/search?text={}&size={}&from={from}", + encode(text), + size.clamp(1, MAX_SEARCH_SIZE) + ) +} + +/// The packument of `name` at the registry at `base`. +pub fn packument_url(base: &str, name: &PackageName) -> String { + format!("{base}/{}", name.escaped()) +} + +/// The tarball `file` of `name` at the registry at `base`: +/// `//-/`, as npm's registry, Nexus, Verdaccio and Forge +/// serve it. +pub fn tarball_url(base: &str, name: &PackageName, file: &str) -> String { + format!("{base}/{name}/-/{file}") +} + +/// `s` percent-encoded for a query string. +pub(crate) fn encode(s: &str) -> String { + s.bytes() + .map(|b| match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => { + (b as char).to_string() + } + _ => format!("%{b:02X}"), + }) + .collect() +} + +/// The scope an import names, `@acme` (also written `@acme/` or +/// `@acme/*`), as `@acme`; `None` when `raw` names no scope. +pub fn parse_scope(raw: &str) -> Option { + let scope = raw.strip_prefix('@')?; + let scope = scope + .strip_suffix("/*") + .or_else(|| scope.strip_suffix('/')) + .unwrap_or(scope); + // A scope is what a scoped name's scope may be. + PackageName::parse(&format!("@{scope}/x")).ok()?; + Some(format!("@{scope}")) +} + +/// One page of a registry's search. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct SearchPage { + /// The matches the registry counts. + pub total: u64, + /// The names of the page's packages, in its order. + pub names: Vec, +} + +/// Read a page of `/-/v1/search`. A package named without its scope, with +/// the scope beside it (`"scope": "acme"`), is named with it; a name that is +/// none is left out. +pub fn read_search(body: &[u8]) -> Result { + let page: Value = + serde_json::from_slice(body).map_err(|e| format!("the search answer is no JSON: {e}"))?; + let objects = page + .get("objects") + .and_then(Value::as_array) + .ok_or("the search answer lists no objects")?; + let names = objects + .iter() + .filter_map(|object| { + let package = object.get("package")?; + let name = package.get("name")?.as_str()?; + let scope = package + .get("scope") + .and_then(Value::as_str) + .filter(|s| !s.is_empty() && *s != "unscoped"); + let name = match scope { + Some(scope) if !name.starts_with('@') => format!("@{scope}/{name}"), + _ => name.to_string(), + }; + PackageName::parse(&name).ok()?; + Some(name) + }) + .collect(); + Ok(SearchPage { + total: page.get("total").and_then(Value::as_u64).unwrap_or(0), + names, + }) +} + +/// Read `/-/all` from `reader`: the names of the packages it lists, sorted, +/// each once. What its entries say is skipped as it streams by. +pub fn read_all(reader: impl Read) -> Result, String> { + let mut de = serde_json::Deserializer::from_reader(std::io::BufReader::new(reader)); + let mut names = de + .deserialize_map(Keys) + .map_err(|e| format!("not a listing: {e}"))?; + de.end().map_err(|e| format!("not a listing: {e}"))?; + names.sort(); + names.dedup(); + Ok(names) +} + +/// The keys of an object that are package names (`_updated` is none). +struct Keys; + +impl<'de> Visitor<'de> for Keys { + type Value = Vec; + + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("an object of packages") + } + + fn visit_map>(self, mut map: A) -> Result { + let mut names = Vec::new(); + while let Some(key) = map.next_key::()? { + map.next_value::()?; + if PackageName::parse(&key).is_ok() { + names.push(key); + } + } + Ok(names) + } +} + +/// One version of a package at the source. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Release { + /// Normalized. + pub version: String, + /// When it was published (`time`), if the packument says. + pub published: Option>, + /// `dist.integrity`, an SRI string. + pub integrity: Option, + /// `dist.shasum`, hex SHA-1. + pub shasum: Option, + /// Why it is deprecated, if it is. + pub deprecated: Option, + /// The tarball's file name: the last segment of `dist.tarball` where it + /// is `…/-/`, otherwise `-.tgz`. + pub file: String, +} + +impl Release { + /// The SHA-512 of the tarball, base64, where `dist.integrity` lists one. + pub fn sha512(&self) -> Option { + self.integrity.as_deref().and_then(sha512_of) + } +} + +/// A package at the source: its versions, in ascending precedence, and its +/// dist-tags (normalized versions; a tag naming a version the packument +/// does not have is left out). +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct Package { + pub name: String, + pub dist_tags: BTreeMap, + pub releases: Vec, +} + +/// Read the full packument of `name` from `reader`: what an import copies of +/// every version. A packument of another package is refused. +pub fn read_package(reader: impl Read, name: &PackageName) -> Result { + let doc = upstream::read( + reader, + Select { + every: true, + keep: Keep::Release, + time: true, + ..Select::default() + }, + )?; + let expected = name.to_string(); + if !doc.name.is_empty() && doc.name != expected { + return Err(format!( + "the registry answered package {:?} for {expected:?}", + doc.name + )); + } + let releases: Vec = doc + .versions() + .into_iter() + .map(|v| Release { + file: tarball_file(v.tarball(), name, v.original), + published: v.published, + integrity: v.integrity().map(str::to_string), + shasum: v.shasum().map(str::to_ascii_lowercase), + deprecated: v.deprecated().map(str::to_string), + version: v.version, + }) + .collect(); + let dist_tags = doc + .normalized_dist_tags() + .into_iter() + .filter(|(_, version)| releases.iter().any(|r| &r.version == version)) + .collect(); + Ok(Package { + name: expected, + dist_tags, + releases, + }) +} + +/// The file name of a tarball: `dist.tarball`'s, where it is +/// `…/-/`, otherwise npm's `-.tgz`. +fn tarball_file(tarball: Option<&str>, name: &PackageName, version: &str) -> String { + let path = tarball.map(|t| t.split(['?', '#']).next().unwrap_or(t)); + path.and_then(|p| p.rsplit_once("/-/")) + .map(|(_, file)| file) + .filter(|file| !file.is_empty() && !file.contains('/') && file.ends_with(".tgz")) + .map(str::to_string) + .unwrap_or_else(|| name.tarball_file(version)) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn name(raw: &str) -> PackageName { + PackageName::parse(raw).unwrap() + } + + #[test] + fn urls_are_built_from_the_registry_url() { + let base = "http://nexus:8081/repository/npm-hosted"; + assert_eq!( + search_url(base, "", 500, 1000), + "http://nexus:8081/repository/npm-hosted/-/v1/search?text=&size=250&from=500" + ); + assert_eq!( + search_url(base, "@acme", 0, 100), + "http://nexus:8081/repository/npm-hosted/-/v1/search?text=%40acme&size=100&from=0" + ); + assert_eq!( + packument_url(base, &name("@acme/ui")), + "http://nexus:8081/repository/npm-hosted/@acme%2fui" + ); + assert_eq!( + tarball_url(base, &name("@acme/ui"), "ui-1.0.0.tgz"), + "http://nexus:8081/repository/npm-hosted/@acme/ui/-/ui-1.0.0.tgz" + ); + } + + #[test] + fn scopes_are_named_with_their_at() { + assert_eq!(parse_scope("@acme").as_deref(), Some("@acme")); + assert_eq!(parse_scope("@acme/").as_deref(), Some("@acme")); + assert_eq!(parse_scope("@acme/*").as_deref(), Some("@acme")); + assert_eq!(parse_scope("@acme/ui"), None); + assert_eq!(parse_scope("acme"), None); + assert_eq!(parse_scope("@"), None); + assert_eq!(parse_scope("@.acme"), None); + assert_eq!(parse_scope("@ac/me/"), None); + } + + #[test] + fn a_search_names_packages_with_their_scope() { + let body = serde_json::to_vec(&json!({ + "objects": [ + {"package": {"name": "alpha", "version": "1.0.0"}}, + {"package": {"name": "@acme/ui", "scope": "acme", "version": "1.0.0"}}, + {"package": {"name": "core", "scope": "acme", "version": "0.1.0"}}, + {"package": {"name": "beta", "scope": "unscoped", "version": "0.1.0"}}, + {"package": {"name": "../x", "version": "1.0.0"}}, + {"package": {"version": "1.0.0"}}, + ], + "total": 9 + })) + .unwrap(); + assert_eq!( + read_search(&body).unwrap(), + SearchPage { + total: 9, + names: vec![ + "alpha".into(), + "@acme/ui".into(), + "@acme/core".into(), + "beta".into() + ], + } + ); + assert!(read_search(b"{\"error\": \"text required\"}").is_err()); + assert!(read_search(b"").is_err()); + } + + #[test] + fn the_legacy_listing_names_every_package() { + let body = serde_json::to_vec(&json!({ + "_updated": 1793435634215u64, + "zeta": {"name": "zeta", "versions": {"1.0.0": "latest"}}, + "@acme/ui": {"name": "@acme/ui", "dist-tags": {"latest": "1.0.1"}}, + "alpha": {"name": "alpha"}, + })) + .unwrap(); + assert_eq!(read_all(&body[..]).unwrap(), ["@acme/ui", "alpha", "zeta"]); + assert!(read_all(&b"[]"[..]).is_err()); + assert!(read_all(&b"{\"a\": 1} trailing"[..]).is_err()); + } + + #[test] + fn a_packument_gives_every_version_with_its_checksums_time_and_deprecation() { + let body = serde_json::to_vec(&json!({ + "name": "@acme/ui", + "dist-tags": {"latest": "1.1.0", "next": "2.0.0-rc.1", "gone": "9.9.9"}, + "time": { + "created": "2020-01-01T00:00:00.000Z", + "1.0.0": "2020-01-01T00:00:00.000Z", + "1.1.0": "2021-02-03T04:05:06.789Z" + }, + "versions": { + "1.0.0": { + "name": "@acme/ui", "version": "1.0.0", "readme": "# big", + "deprecated": "use 1.1.0", + "dependencies": {"b": "^1"}, + "dist": { + "tarball": "https://registry.example.com/@acme/ui/-/ui-1.0.0.tgz", + "integrity": "sha512-AAAA sha1-BBBB", "shasum": "ABCD" + } + }, + "1.1.0": { + "name": "@acme/ui", "version": "1.1.0", "deprecated": "", + "dist": {"tarball": "https://cdn.example.com/download/ui/1.1.0", "shasum": "ef01"} + }, + "2.0.0-rc.1": {"name": "@acme/ui", "version": "2.0.0-rc.1", "dist": { + "tarball": "http://old-host/repository/npm/@acme/ui/-/ui-2.0.0-rc.1.tgz?x=1", + "integrity": "sha512-CCCC" + }} + } + })) + .unwrap(); + let package = read_package(&body[..], &name("@acme/ui")).unwrap(); + assert_eq!(package.name, "@acme/ui"); + assert_eq!( + package.dist_tags, + BTreeMap::from([ + ("latest".into(), "1.1.0".into()), + ("next".into(), "2.0.0-rc.1".into()) + ]) + ); + let versions: Vec<&str> = package + .releases + .iter() + .map(|r| r.version.as_str()) + .collect(); + assert_eq!(versions, ["1.0.0", "1.1.0", "2.0.0-rc.1"]); + let first = &package.releases[0]; + assert_eq!(first.sha512().as_deref(), Some("AAAA")); + assert_eq!(first.shasum.as_deref(), Some("abcd")); + assert_eq!(first.deprecated.as_deref(), Some("use 1.1.0")); + assert_eq!(first.file, "ui-1.0.0.tgz"); + assert_eq!( + first.published, + upstream::parse_time("2020-01-01T00:00:00.000Z") + ); + // An unconventional tarball URL: npm's file name. No integrity: no + // SHA-512; an empty deprecation is none. + let second = &package.releases[1]; + assert_eq!(second.file, "ui-1.1.0.tgz"); + assert_eq!(second.sha512(), None); + assert_eq!(second.shasum.as_deref(), Some("ef01")); + assert_eq!(second.deprecated, None); + // The file of a tarball on another host, its query left out. + let third = &package.releases[2]; + assert_eq!(third.file, "ui-2.0.0-rc.1.tgz"); + assert_eq!(third.published, None); + + assert!(read_package(&body[..], &name("@acme/other")).is_err()); + assert!(read_package(&b"[]"[..], &name("@acme/ui")).is_err()); + } +} diff --git a/crates/formats/npm/src/integrity.rs b/crates/formats/npm/src/integrity.rs index eb50b6fb..706ce488 100644 --- a/crates/formats/npm/src/integrity.rs +++ b/crates/formats/npm/src/integrity.rs @@ -6,6 +6,15 @@ use std::io::Read; use base64::Engine as _; use sha1::Digest as _; +/// The SHA-512 of an SRI `integrity` (`sha512-`), base64, if it +/// has one. +pub fn sha512_of(integrity: &str) -> Option { + integrity + .split_whitespace() + .find_map(|sri| sri.strip_prefix("sha512-")) + .map(|b64| b64.split('?').next().unwrap_or(b64).to_string()) +} + /// The digests of a tarball. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Digests { diff --git a/crates/formats/npm/src/lib.rs b/crates/formats/npm/src/lib.rs index c377a5ab..ea3668ef 100644 --- a/crates/formats/npm/src/lib.rs +++ b/crates/formats/npm/src/lib.rs @@ -7,8 +7,9 @@ //! search, publishing (the tarball comes inside a JSON document, //! [`publish`]), dist-tags, deprecations, unpublishing, the logins, and //! audits passed to an upstream. Proxies read another registry through -//! [`proxy`]. Forge keeps the rest: storage, authentication, the tags and -//! deprecations themselves, hosted/proxy/group semantics and retention. +//! [`proxy`], and an import copies one's packages through [`import`]. +//! Forge keeps the rest: storage, authentication, the tags and deprecations +//! themselves, hosted/proxy/group semantics and retention. //! //! The pure parts: [`name`] and [`version`] rules, [`tarball`] reading, //! [`integrity`] checks, the [`document`]s clients `PUT`, version @@ -16,6 +17,7 @@ //! [`upstream`] packuments. pub mod document; +pub mod import; pub mod integrity; pub mod manifest; pub mod name; diff --git a/crates/formats/npm/src/proxy.rs b/crates/formats/npm/src/proxy.rs index fa682631..badb493d 100644 --- a/crates/formats/npm/src/proxy.rs +++ b/crates/formats/npm/src/proxy.rs @@ -29,6 +29,8 @@ use forge_plugin_api::{ }; use serde_json::{Map, Value}; +use crate::import::encode; +use crate::integrity::sha512_of; use crate::packument::{timestamp, LATEST}; use crate::upstream::{self, Keep, Kept, Select, UpstreamPackument}; use crate::{manifest, version, PackageName}; @@ -149,26 +151,6 @@ fn with_packument( answer_from(packument(remote, id, full, fetched, select), answer) } -/// The SHA-512 of an SRI `integrity` (`sha512-`), if it has one. -fn sha512_of(integrity: &str) -> Option { - integrity - .split_whitespace() - .find_map(|sri| sri.strip_prefix("sha512-")) - .map(|b64| b64.split('?').next().unwrap_or(b64).to_string()) -} - -/// `s` percent-encoded for a query string. -fn encode(s: &str) -> String { - s.bytes() - .map(|b| match b { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'.' | b'_' | b'~' => { - (b as char).to_string() - } - _ => format!("%{b:02X}"), - }) - .collect() -} - /// One round of the upstream conversation for `op`. pub fn upstream(remote: &Remote, op: &UpstreamOp, fetched: &[FetchResult]) -> UpstreamStep { match op { @@ -274,7 +256,9 @@ pub fn upstream(remote: &Remote, op: &UpstreamOp, fetched: &[FetchResult]) -> Up } } UpstreamOp::Search(query) => search(remote, query, fetched), - // No import walks an npm registry yet. + // An import of a registry reads its listings and packuments through + // `crate::import` (a full packument per package), not through this + // conversation. UpstreamOp::Packages { .. } => done(UpstreamAnswer::Search(None)), // A registry holds packages: it has no files to fetch. UpstreamOp::File(_) => done(UpstreamAnswer::File(None)), diff --git a/crates/formats/npm/src/upstream.rs b/crates/formats/npm/src/upstream.rs index e0c5c512..628e1996 100644 --- a/crates/formats/npm/src/upstream.rs +++ b/crates/formats/npm/src/upstream.rs @@ -8,7 +8,9 @@ //! a stream and keeps only what the operation asks for ([`Select`]): the //! name and the dist-tags, which versions there are, the objects of the //! versions asked for (without their READMEs) or just their summaries, and -//! the publish times and the top-level README (bounded) when asked. +//! the publish times and the top-level README (bounded) when asked. An +//! import of another registry ([`crate::import`]) keeps what it copies of +//! every version ([`Keep::Release`]). use std::collections::{BTreeMap, HashSet}; use std::fmt; @@ -32,6 +34,8 @@ pub struct Select<'a> { /// The versions (normalized) whose objects are kept, as `keep` says; /// every other version is only listed ([`Kept::Listed`]). pub versions: &'a [String], + /// Keep the object of every version, not only of `versions`. + pub every: bool, pub keep: Keep, /// The publish times (`time`). pub time: bool, @@ -47,14 +51,21 @@ pub enum Keep { Manifest, /// Its summary ([`manifest::summary`]) alone, as JSON. Summary, + /// What an import copies of it, as a [`Kept::Manifest`]: its `name`, + /// `version`, `dist` and `deprecated`. + Release, } +/// The fields of a version object [`Keep::Release`] keeps. +const RELEASE_FIELDS: &[&str] = &["name", "version", "dist", "deprecated"]; + /// A version object as [`read`] kept it. #[derive(Debug, Clone, PartialEq)] pub enum Kept { /// Not kept: the version is listed, nothing more. Listed, - /// The manifest, without its README. + /// The manifest, without its README (or, with [`Keep::Release`], the + /// fields it keeps). Manifest(Map), /// Its summary, as JSON. Summary(String), @@ -117,6 +128,14 @@ impl<'a> UpstreamVersion<'a> { .and_then(Value::as_str) } + /// Why the version is deprecated, if it is (an empty message is none). + pub fn deprecated(&self) -> Option<&'a str> { + self.manifest()? + .get("deprecated") + .and_then(Value::as_str) + .filter(|m| !m.is_empty()) + } + fn dist(&self) -> Option<&'a Map> { self.manifest()?.get("dist").and_then(Value::as_object) } @@ -246,6 +265,7 @@ impl<'de> Visitor<'de> for TopVisitor<'_> { "versions" => { out.versions = map.next_value_seed(VersionsSeed { wanted: &self.wanted, + every: self.select.every, keep: self.select.keep, })? } @@ -265,10 +285,11 @@ impl<'de> Visitor<'de> for TopVisitor<'_> { } } -/// The `versions` object: the objects of the `wanted` versions (normalized) -/// kept as `keep` says, the others only listed. +/// The `versions` object: the objects of the `wanted` versions (normalized), +/// or of `every` version, kept as `keep` says, the others only listed. struct VersionsSeed<'s, 'a> { wanted: &'s HashSet<&'a str>, + every: bool, keep: Keep, } @@ -298,10 +319,8 @@ impl<'de> Visitor<'de> for VersionsSeed<'_, '_> { map.next_value::()?; continue; }; - let keep = self - .wanted - .contains(normalized.as_str()) - .then_some(self.keep); + let keep = + (self.every || self.wanted.contains(normalized.as_str())).then_some(self.keep); if let Some(kept) = map.next_value_seed(ObjectSeed(keep))? { out.insert(key, kept); } @@ -353,14 +372,18 @@ impl<'de> Visitor<'de> for ObjectSeed { }; let mut out = Map::new(); while let Some(key) = map.next_key::()? { - if key == "readme" || key == "readmeFilename" { + let skipped = match keep { + Keep::Release => !RELEASE_FIELDS.contains(&key.as_str()), + Keep::Manifest | Keep::Summary => key == "readme" || key == "readmeFilename", + }; + if skipped { map.next_value::()?; } else { out.insert(key, map.next_value::()?); } } Ok(Some(match keep { - Keep::Manifest => Kept::Manifest(out), + Keep::Manifest | Keep::Release => Kept::Manifest(out), Keep::Summary => Kept::Summary( serde_json::to_string(&manifest::into_summary(out)) .map_err(::custom)?, @@ -407,6 +430,7 @@ mod tests { &left_pad()[..], Select { versions: &versions, + every: false, keep: Keep::Manifest, time: true, readme: true, diff --git a/crates/server/src/admin/imports.rs b/crates/server/src/admin/imports.rs index c1b45414..454dd0bd 100644 --- a/crates/server/src/admin/imports.rs +++ b/crates/server/src/admin/imports.rs @@ -1,15 +1,16 @@ //! Imports into a hosted repository from another server ([`crate::import`]): -//! a NuGet feed or a registry, read through the format's own protocol. -//! **Admin-only.** +//! a NuGet feed, an npm registry or a registry API, read through the +//! format's own protocol. **Admin-only.** //! //! * `GET /admin/repositories/{name}/imports` — the repository's imports, //! newest first ([`ImportView`]). //! * `POST /admin/repositories/{name}/imports` — start one //! ([`CreateImport`]). The source is checked first (the **preflight**): //! a feed's service index resolves and its search answers (with how many -//! packages it finds), or a registry answers its API and serves a -//! catalog; an import that names its package ids or images needs no -//! search or catalog. A failed check is a 400 (502 when the source is +//! packages it finds), an npm registry lists its packages (its search, +//! or `/-/all`), or a registry answers its API and serves a catalog; an +//! import that names its package ids, npm packages or scopes, or images +//! needs no search or catalog. A failed check is a 400 (502 when the source is //! unreachable or failing) saying what is wrong; nothing is stored. A //! repository runs one import at a time (409). `dry_run` decides every //! item without writing a package: what would be copied, and the @@ -49,9 +50,10 @@ use serde::{Deserialize, Serialize}; use super::db_error; use crate::images::import::ImportSource; use crate::import::feed::Feed; +use crate::import::npm::NpmRegistry; use crate::import::{ - nexus_source_url, password_secret, source_url, Listing, SourceError, FORMATS, MAX_NAMES, - SEARCH_PAGE, + nexus_source_url, password_secret, source_url, Listing, Protocol, SourceError, FORMATS, + MAX_NAMES, NPM_PAGE, SEARCH_PAGE, }; use crate::state::AppState; @@ -94,14 +96,16 @@ fn is_unique_violation(e: &sqlx::Error) -> bool { #[serde(deny_unknown_fields)] pub struct CreateImport { /// The source: a NuGet feed's service index - /// (`https://nexus.example.com/repository/nuget-hosted/index.json`), or a + /// (`https://nexus.example.com/repository/nuget-hosted/index.json`), an + /// npm registry (`https://nexus.example.com/repository/npm-hosted`), or a /// registry (`https://registry.example.com`). With `repository`, a Nexus /// server instead (the request before imports read the format's /// protocol). pub url: String, /// The Nexus repository at `url` to read (with `url` naming the Nexus - /// server): its service index, `/repository//index.json`, or - /// its registry API, `registry_url` or `/repository/`. + /// server): its service index, `/repository//index.json`, its + /// npm registry, `/repository/`, or its registry API, + /// `registry_url` or `/repository/`. #[serde(default)] pub repository: Option, /// With `repository`: the Nexus Docker repository's connector, e.g. @@ -109,7 +113,8 @@ pub struct CreateImport { #[serde(default)] pub registry_url: Option, /// The package ids or images to copy, instead of every package the - /// feed's search finds (every image the registry's catalog lists). + /// feed's search finds (every image the registry's catalog lists). npm: + /// package names, or scopes (`@acme`: every package of the scope). #[serde(default)] pub names: Option>, /// The source's credentials (both, or neither for anonymous access). @@ -203,8 +208,12 @@ pub struct ImportView { /// The `Server` header the source answered the preflight with. pub source_server: Option, /// How many packages the feed's search finds: what it reported in the - /// preflight, or as many as the scan walked, whichever is more. + /// preflight, or as many as the scan walked, whichever is more. npm: what + /// the registry's search reports, or how many `/-/all` lists. pub source_total: Option, + /// What the scan walks: `search`, `catalog`, `all` (npm's `/-/all`) or + /// `names`. + pub listing: String, pub last_error: Option, pub created_by: Option, pub created_at: DateTime, @@ -236,7 +245,10 @@ async fn view( Ok(ImportView { id: import.id, repository: repo.name.clone(), - url: source_url(&import, state.formats.stores_images(&repo.format)), + url: source_url( + &import, + Protocol::of(&repo.format, state.formats.stores_images(&repo.format)), + ), names: import.names, username: import.username, has_password: import.has_password, @@ -248,6 +260,7 @@ async fn view( cancel_requested: import.cancel_requested, source_server: import.source_server, source_total: import.source_total, + listing: import.scan_listing, last_error: import.last_error, created_by: import.created_by, created_at: import.created_at, @@ -348,6 +361,7 @@ pub async fn create( )); } let images = state.formats.stores_images(&repo.format); + let protocol = Protocol::of(&repo.format, images); let Some(format) = state.formats.get(&repo.format) else { return bad_request(format!("the {} format is not loaded", repo.format)); }; @@ -366,7 +380,7 @@ pub async fn create( if repository.is_empty() || repository.contains('/') { return bad_request("repository must name one Nexus repository"); } - if registry_url.is_some() && !images { + if registry_url.is_some() && protocol != Protocol::Images { return bad_request("registry_url applies to Docker repositories only"); } if let Some(registry_url) = ®istry_url { @@ -374,7 +388,7 @@ pub async fn create( return bad_request(format!("invalid registry_url: {e}")); } } - nexus_source_url(&url, &repository, registry_url.as_deref(), images) + nexus_source_url(&url, &repository, registry_url.as_deref(), protocol) } None if registry_url.is_some() => { return bad_request( @@ -389,12 +403,21 @@ pub async fn create( let mut seen = std::collections::HashSet::new(); let mut kept = Vec::new(); for name in names { - let name = name.trim().to_string(); + let mut name = name.trim().to_string(); if name.is_empty() { continue; } - if let Err(e) = format.validate_id(&name) { - return bad_request(format!("names: {name:?} is no valid name: {e}")); + // npm: a scope stands for every package of it. + let scope = (protocol == Protocol::Npm) + .then(|| forge_format_npm::import::parse_scope(&name)) + .flatten(); + match scope { + Some(scope) => name = scope, + None => { + if let Err(e) = format.validate_id(&name) { + return bad_request(format!("names: {name:?} is no valid name: {e}")); + } + } } if seen.insert(format.lower_id(&name)) { kept.push(name); @@ -423,7 +446,53 @@ pub async fn create( // The preflight: does the source speak the format's protocol, and list // what to copy? - let (server, total) = if images { + let mut listing = Listing::of(images, names.is_some()); + let (server, total) = if protocol == Protocol::Npm { + let mut registry = NpmRegistry::new( + state.upstream.clone(), + &source, + username.clone(), + password.clone(), + ); + if let Err(e) = registry.ping().await { + return preflight_error(&format!("the npm registry at {source}"), e); + } + let total = if names.is_none() { + // The search for every package, then `/-/all`: Nexus' search + // finds nothing without a text, and Verdaccio refuses one. + let searched = match registry.search("", 0, NPM_PAGE).await { + Ok(searched) => searched, + // An answer that is no search result: no search to walk. + Err(SourceError::Invalid(_)) => None, + Err(e) => return preflight_error("searching the registry", e), + }; + match searched { + Some(page) if !page.names.is_empty() => { + Some(i64::try_from(page.total.max(page.names.len() as u64)).unwrap_or(i64::MAX)) + } + searched => match registry.all().await { + Ok(Some(all)) => { + listing = Listing::All; + Some(i64::try_from(all.len()).unwrap_or(i64::MAX)) + } + // A search that answers, and finds nothing: an empty + // registry. + Ok(None) if searched.is_some() => Some(0), + Ok(None) => { + return bad_request(format!( + "the npm registry at {source} lists no packages: it offers no search \ + for every package (/-/v1/search) and no /-/all; name the packages or \ + scopes to import" + )) + } + Err(e) => return preflight_error("reading the registry's /-/all", e), + }, + } + } else { + None + }; + (registry.server().map(str::to_string), total) + } else if images { let credentials = match (&username, &password) { (Some(username), Some(password)) => Some(Credentials::for_url( &source, @@ -489,7 +558,6 @@ pub async fn create( }; (feed.server().map(str::to_string), total) }; - let listing = Listing::of(images, names.is_some()); let created = async { let mut tx = forge_db::begin(&state.pool).await?; diff --git a/crates/server/src/import/mod.rs b/crates/server/src/import/mod.rs index 70399795..12286bef 100644 --- a/crates/server/src/import/mod.rs +++ b/crates/server/src/import/mod.rs @@ -1,22 +1,30 @@ //! Importing a repository from another server into a Forge hosted -//! repository of the same format (NuGet, Docker), one repository at a time. -//! The source is read through the format's own protocol, so it may be any -//! server that speaks it: Nexus, Artifactory, ProGet, BaGet, Harbor, a -//! `distribution` registry or another Forge. +//! repository of the same format (NuGet, npm, Docker), one repository at a +//! time. The source is read through the format's own protocol, so it may be +//! any server that speaks it: Nexus, Artifactory, ProGet, BaGet, Verdaccio, +//! Harbor, a `distribution` registry or another Forge. //! //! * **NuGet:** a feed's service index (`…/index.json`, [`feed::Feed`]): //! its search names the package ids, its package base address lists each //! package's versions (unlisted ones included) and serves the `.nupkg`, //! its registration says when each version was published and, where the //! feed lists it inline, its SHA-512. +//! * **npm:** a registry ([`npm::NpmRegistry`]): its search +//! (`/-/v1/search`), or where that lists nothing its legacy `/-/all`, +//! names the packages; each package's full packument lists its versions +//! with their SHA-512 (`dist.integrity`), publish times (`time`) and +//! deprecations, and its dist-tags. A tarball is downloaded from the +//! import's URL, whatever host `dist.tarball` names. //! * **Docker:** a registry's API ([`ImportSource`]): its catalog //! (`/v2/_catalog`) names the images, their tag lists the tags, and the //! referrers API the signatures and SBOMs of each copied manifest. //! //! An import can name its package ids or images instead (`names`): a feed's //! search may miss some (Nexus stops at 10,000 packages; a package whose -//! versions are all unlisted is in no search result), and Docker Hub, GHCR -//! and ECR serve no catalog. +//! versions are all unlisted is in no search result), Docker Hub, GHCR and +//! ECR serve no catalog, and an npm registry may serve neither a search +//! that lists every package nor `/-/all` (an npm import names scopes too, +//! `@acme`). //! //! An administrator starts an import on a hosted repository //! ([`crate::admin::imports`]); this worker runs it, on one instance at a @@ -54,7 +62,13 @@ //! time for a tag, so a tag is created at its import; //! * a version Forge holds with other bytes is a conflict: skipped and //! reported (`skip`), replaced (`overwrite`), or it fails the import -//! (`fail`). +//! (`fail`); +//! * npm: a version's deprecation comes along, and once the versions are +//! copied, the dist-tags are set (`tag` items). A dist-tag Forge points +//! at another version, or a deprecation Forge gives otherwise, is a +//! conflict too, unless an earlier import of the repository set what +//! Forge holds: then it follows the source (a tag the source moved +//! since moves along). //! //! A dry run scans and decides every item (`planned`, `identical`, //! `conflict`) without writing a package; it downloads no package. Running @@ -72,6 +86,7 @@ //! import completes. pub mod feed; +pub mod npm; use std::collections::HashMap; use std::panic::AssertUnwindSafe; @@ -81,14 +96,14 @@ use std::time::Duration; use chrono::{DateTime, Datelike, Utc}; use forge_core::auth::principal::{AuthMethod, Principal}; -use forge_core::authz::Level; +use forge_core::authz::{Level, PackageFilter}; use forge_core::repo::hosted::HostedWrite; use forge_core::repo::proxy::Credentials; use forge_core::repo::{ImportOptions, ImportOutcome, RepoKind, WriteError}; use forge_crypto::PlaintextSecret; use forge_db::import_dao::{self, ImportItem, NewItem, RepositoryImport}; use forge_db::model::Repository; -use forge_db::{repository_dao, secret_dao, user_dao}; +use forge_db::{component_dao, repository_dao, secret_dao, tag_dao, user_dao}; use forge_plugin::api::images::ImageError; use forge_plugin::api::VersionMetadata; use forge_storage::StagedBlob; @@ -103,10 +118,11 @@ use crate::images::import::{ImageImportOptions, ImageOutcome, ImportSource}; use crate::images::{Failure, Registry}; use crate::state::AppState; use feed::Feed; +use npm::NpmRegistry; -/// The formats an import copies: NuGet packages, and Docker images into -/// the image store. -pub const FORMATS: &[&str] = &["nuget", "docker"]; +/// The formats an import copies: NuGet and npm packages, and Docker images +/// into the image store. +pub const FORMATS: &[&str] = &["nuget", "npm", "docker"]; /// How often an idle instance looks for an import to run. const POLL_INTERVAL: Duration = Duration::from_secs(5); @@ -130,6 +146,10 @@ const CHECK_INTERVAL: Duration = Duration::from_secs(5); /// Package ids a page of a feed's search names. pub const SEARCH_PAGE: u32 = 100; +/// npm packages a scan page reads: a page of the registry's search, or of +/// its `/-/all`. +pub const NPM_PAGE: u32 = 100; + /// Images a page of a registry's catalog names. const CATALOG_PAGE: u32 = 100; @@ -194,6 +214,8 @@ pub enum Listing { Search, /// A registry's catalog. Catalog, + /// An npm registry's legacy listing of every package, `/-/all`. + All, /// The package ids or images the import names. Names, } @@ -212,6 +234,7 @@ impl Listing { match self { Listing::Search => "search", Listing::Catalog => "catalog", + Listing::All => "all", Listing::Names => "names", } } @@ -219,40 +242,65 @@ impl Listing { fn parse(s: &str) -> Listing { match s { "catalog" => Listing::Catalog, + "all" => Listing::All, "names" => Listing::Names, _ => Listing::Search, } } } +/// The protocol an import reads its source through. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Protocol { + /// A NuGet v3 feed. + NuGet, + /// An npm registry. + Npm, + /// A registry API (Docker / OCI images). + Images, +} + +impl Protocol { + /// The protocol of a repository of `format` (`images`: an images + /// format). + pub fn of(format: &str, images: bool) -> Protocol { + match format { + _ if images => Protocol::Images, + "npm" => Protocol::Npm, + _ => Protocol::NuGet, + } + } +} + /// Where in a Nexus server a repository is: `/repository/`. -/// Nexus serves a NuGet repository's service index below it, and a Docker -/// repository's registry API at it (for reads, on its own port), unless the -/// repository has a connector of its own (`registry_url`). +/// Nexus serves a NuGet repository's service index below it, an npm +/// repository's registry at it, and a Docker repository's registry API at +/// it (for reads, on its own port), unless the repository has a connector +/// of its own (`registry_url`). pub fn nexus_source_url( url: &str, repository: &str, registry_url: Option<&str>, - images: bool, + protocol: Protocol, ) -> String { let path = format!("{}/repository/{repository}", url.trim_end_matches('/')); - match registry_url { - Some(registry_url) if images => registry_url.to_string(), - _ if images => path, - _ => format!("{path}/index.json"), + match (protocol, registry_url) { + (Protocol::Images, Some(registry_url)) => registry_url.to_string(), + (Protocol::Images | Protocol::Npm, _) => path, + (Protocol::NuGet, _) => format!("{path}/index.json"), } } /// The source `import` reads: its `url`, or, for an import stored before /// imports read the format's protocol (a Nexus server and repository), that /// repository's service index or registry API ([`nexus_source_url`]). -pub fn source_url(import: &RepositoryImport, images: bool) -> String { +pub fn source_url(import: &RepositoryImport, protocol: Protocol) -> String { match &import.source_repository { Some(repository) => nexus_source_url( &import.url, repository, import.registry_url.as_deref(), - images, + protocol, ), None => import.url.clone(), } @@ -323,6 +371,7 @@ impl ImportWorker { lost, stopping: AtomicBool::new(false), last_search: std::sync::Mutex::new(Vec::new()), + npm_all: tokio::sync::Mutex::new(None), }; let end = AssertUnwindSafe(run.execute()) .catch_unwind() @@ -368,11 +417,19 @@ fn db_failed(e: sqlx::Error) -> End { /// Where the items of a run come from and go. enum Target { - /// A package format: from a feed, through the hosted write path. + /// NuGet: from a feed, through the hosted write path. Packages { feed: Box, writer: Box, }, + /// npm: from a registry, through the hosted write path; the dist-tags + /// and deprecations beside it, as the administrator's package + /// selectors (`filter`) allow. + Npm { + registry: Box, + writer: Box, + filter: PackageFilter, + }, /// Docker: from a registry's API, through the registry. Images { registry: Registry, @@ -398,10 +455,27 @@ struct ItemSource { /// The package's SHA-512, base64, where the feed lists it. #[serde(default, skip_serializing_if = "Option::is_none")] sha512: Option, + /// npm: `dist.integrity` (an SRI string) and `dist.shasum` (hex SHA-1), + /// checked where no SHA-512 is listed (an old package's SHA-1). + #[serde(default, skip_serializing_if = "Option::is_none")] + integrity: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + shasum: Option, + /// npm: where the tarball is downloaded, below the import's URL. + #[serde(default, skip_serializing_if = "Option::is_none")] + tarball: Option, + /// npm: why the version is deprecated, if it is. + #[serde(default, skip_serializing_if = "Option::is_none")] + deprecated: Option, + /// npm, a `tag` item: the version the dist-tag points at. + #[serde(default, skip_serializing_if = "Option::is_none")] + target: Option, } /// A version or tag the scan found. struct Found { + /// `component`, or `tag` (an npm dist-tag). + kind: &'static str, name: String, /// Empty for a named package or image the source does not have. version: String, @@ -414,6 +488,7 @@ impl Found { /// fails, so the import reports it. fn missing(name: &str) -> Found { Found { + kind: "component", name: name.to_string(), version: String::new(), published_at: None, @@ -465,6 +540,9 @@ struct Run<'a> { /// The ids of the search page read last: a feed whose search ignores /// `skip` answers the same page again. last_search: std::sync::Mutex>, + /// npm: the names `/-/all` lists, once read (`Some(None)`: the registry + /// does not serve it). + npm_all: tokio::sync::Mutex>>>>, } impl Run<'_> { @@ -495,7 +573,7 @@ impl Run<'_> { .ok_or_else(|| End::Failed("the repository was deleted".into()))?; if repo.kind() != RepoKind::Hosted || !FORMATS.contains(&repo.format.as_str()) { return Err(End::Failed(format!( - "imports go to a hosted NuGet or Docker repository; {} is a {} {} repository", + "imports go to a hosted NuGet, npm or Docker repository; {} is a {} {} repository", repo.name, repo.format, repo.kind ))); } @@ -529,7 +607,8 @@ impl Run<'_> { } let images = self.state.formats.stores_images(&repo.format); - let url = source_url(import, images); + let protocol = Protocol::of(&repo.format, images); + let url = source_url(import, protocol); let target = if images { let credentials = match (&import.username, password) { (Some(username), Some(password)) => { @@ -547,14 +626,21 @@ impl Run<'_> { .formats .for_repository(&repo) .map_err(|e| End::Failed(e.to_string()))?; - let feed = self - .retrying("reading the feed's service index", || { - let (http, format, url) = - (self.state.upstream.clone(), format.clone(), url.clone()); - let (username, password) = (import.username.clone(), password.clone()); - async move { Feed::open(http, format, &url, username, password).await }.boxed() - }) - .await?; + // An npm registry needs nothing resolved: its documents are at + // fixed URLs. + let feed = match protocol { + Protocol::Npm => None, + _ => Some( + self.retrying("reading the feed's service index", || { + let (http, format, url) = + (self.state.upstream.clone(), format.clone(), url.clone()); + let (username, password) = (import.username.clone(), password.clone()); + async move { Feed::open(http, format, &url, username, password).await } + .boxed() + }) + .await?, + ), + }; let authz = &self.state.authz; let filter = authz .package_filter_for(&principal, &repo, Level::Write) @@ -574,14 +660,26 @@ impl Run<'_> { repo.id, repo.name.clone(), principal.clone(), - filter, + filter.clone(), replace, role_filter, role_replace, )); - Target::Packages { - feed: Box::new(feed), - writer, + match feed { + Some(feed) => Target::Packages { + feed: Box::new(feed), + writer, + }, + None => Target::Npm { + registry: Box::new(NpmRegistry::new( + self.state.upstream.clone(), + &url, + import.username.clone(), + password, + )), + writer, + filter, + }, } }; Ok(Ctx { @@ -649,7 +747,7 @@ impl Run<'_> { .iter() .zip(&sources) .map(|(found, source)| NewItem { - kind: "component", + kind: found.kind, name: &found.name, version: &found.version, size: None, @@ -743,6 +841,61 @@ impl Run<'_> { walked: Some(i64::try_from(walked).unwrap_or(i64::MAX)), }) } + (Target::Npm { registry, .. }, Listing::Search) => { + let from = cursor.and_then(|c| c.parse::().ok()).unwrap_or(0); + let Some(page) = registry.search("", from, NPM_PAGE).await? else { + return Err(SourceError::Invalid( + "the registry no longer offers a search; name the packages or scopes to \ + import instead" + .into(), + )); + }; + let names = page.names; + { + let mut last = self + .last_search + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if from > 0 && !names.is_empty() && *last == names { + return Err(SourceError::Invalid(format!( + "the registry's search answers the same page at {from} as before it: \ + it ignores from; name the packages or scopes to import instead" + ))); + } + last.clone_from(&names); + } + let walked = u64::from(from) + names.len() as u64; + let next = (!names.is_empty()).then(|| walked.to_string()); + // A package the search lists but the registry serves no + // packument of is an item that fails: Nexus' search names a + // scoped package without its scope. + let found = self.expand(ctx, &names, true, concurrency).await?; + Ok(Page { + found, + next, + walked: Some(i64::try_from(walked).unwrap_or(i64::MAX)), + }) + } + (Target::Npm { registry, .. }, Listing::All) => { + let Some(names) = self.npm_all(registry).await? else { + return Err(SourceError::Invalid( + "the registry no longer serves /-/all; name the packages or scopes to \ + import instead" + .into(), + )); + }; + // The names are sorted: the cursor is the last one recorded. + let start = cursor.map_or(0, |last| names.partition_point(|n| n.as_str() <= last)); + let end = (start + NPM_PAGE as usize).min(names.len()); + let page = &names[start..end]; + let next = page.last().filter(|_| end < names.len()).cloned(); + let found = self.expand(ctx, page, true, concurrency).await?; + Ok(Page { + found, + next, + walked: None, + }) + } (Target::Images { registry, source }, Listing::Catalog) => { let page = registry .source_catalog(source, cursor, CATALOG_PAGE) @@ -784,7 +937,7 @@ impl Run<'_> { // The reads are made up front: futures do nothing until polled. let reads: Vec<_> = names .iter() - .map(|name| expand_one(&ctx.target, name, named)) + .map(|name| self.expand_one(ctx, name, named, concurrency)) .collect(); let pages: Vec> = futures_util::stream::iter(reads) .buffered(concurrency) @@ -793,12 +946,80 @@ impl Run<'_> { Ok(pages.into_iter().flatten().collect()) } + /// The versions of the package id `name`, or the tags of the image + /// `name`; for npm, the versions and dist-tags of the package `name`, or + /// of every package of the scope `name` (`@acme`). + async fn expand_one( + &self, + ctx: &Ctx, + name: &str, + named: bool, + concurrency: usize, + ) -> Result, SourceError> { + match &ctx.target { + Target::Packages { feed, .. } => package_versions(feed, name, named).await, + Target::Images { registry, source } => image_tags(registry, source, name, named).await, + Target::Npm { registry, .. } => { + let Some(scope) = forge_format_npm::import::parse_scope(name) else { + return npm_package(registry, name, named).await; + }; + let packages = self.npm_scope(registry, &scope).await?; + if packages.is_empty() { + return Ok(vec![Found::missing(name)]); + } + let reads: Vec<_> = packages + .iter() + .map(|package| npm_package(registry, package, true)) + .collect(); + let found: Vec> = futures_util::stream::iter(reads) + .buffered(concurrency) + .try_collect() + .await?; + Ok(found.into_iter().flatten().collect()) + } + } + } + + /// What `/-/all` lists, read once a run; `None` when the registry does + /// not serve it. + async fn npm_all( + &self, + registry: &NpmRegistry, + ) -> Result>>, SourceError> { + let mut all = self.npm_all.lock().await; + if let Some(all) = &*all { + return Ok(all.clone()); + } + let read = registry.all().await?.map(Arc::new); + *all = Some(read.clone()); + Ok(read) + } + + /// The packages of the scope `scope` (`@acme`): from `/-/all` where the + /// registry serves it, otherwise from its search for the scope. + async fn npm_scope( + &self, + registry: &NpmRegistry, + scope: &str, + ) -> Result, SourceError> { + let prefix = format!("{scope}/"); + match self.npm_all(registry).await? { + Some(all) => Ok(all + .iter() + .filter(|name| name.starts_with(&prefix)) + .cloned() + .collect()), + None => registry.search_scope(scope).await, + } + } + /// Copy the pending items: tags and versions first, then (Docker) the - /// referrers their copies found. + /// referrers their copies found, then (npm) the dist-tags, once their + /// versions are in Forge. async fn copy(&self, ctx: &Ctx) -> Result<(), End> { let failures = AtomicUsize::new(0); let concurrency = self.import.concurrency.clamp(1, 16) as usize; - for kind in ["component", "manifest"] { + for kind in ["component", "manifest", "tag"] { let mut after = 0; loop { self.check().await?; @@ -862,14 +1083,40 @@ impl Run<'_> { let copied = if item.version.is_empty() { Err(match ctx.target { Target::Packages { .. } => "the source has no package of this id".to_string(), + Target::Npm { .. } + if forge_format_npm::import::parse_scope(&item.name).is_some() => + { + "the source has no package of this scope".to_string() + } + Target::Npm { .. } => "the source has no package of this name".to_string(), Target::Images { .. } => "the source has no tag of this image".to_string(), }) } else { + let source: ItemSource = + serde_json::from_value(item.source.clone()).unwrap_or_default(); match &ctx.target { Target::Packages { feed, writer } => { - let source: ItemSource = - serde_json::from_value(item.source.clone()).unwrap_or_default(); - self.copy_package(ctx, feed, writer, &item, &source).await + self.copy_package(ctx, PackageSource::Feed(feed), writer, &item, &source) + .await + } + Target::Npm { filter, .. } if item.kind == "tag" => { + self.copy_dist_tag(ctx, filter, &item, &source).await + } + Target::Npm { + registry, + writer, + filter, + } => { + let copied = self + .copy_package(ctx, PackageSource::Npm(registry), writer, &item, &source) + .await; + match copied { + Ok(copied) => { + self.carry_deprecation(ctx, filter, &item, &source, copied) + .await + } + Err(e) => Err(e), + } } Target::Images { registry, source } => { self.copy_image(ctx, registry, source, &item).await @@ -919,7 +1166,7 @@ impl Run<'_> { async fn copy_package( &self, ctx: &Ctx, - feed: &Feed, + from: PackageSource<'_>, writer: &HostedWrite, item: &ImportItem, source: &ItemSource, @@ -943,7 +1190,10 @@ impl Run<'_> { (None, _) => copied("planned", None, 0), (Some(_), None) => copied( "planned", - Some("Forge has this version; the feed lists no SHA-512 to compare".into()), + Some(format!( + "Forge has this version; the {} lists no SHA-512 to compare", + from.noun() + )), 0, ), (Some(_), Some(_)) if overwrite => copied( @@ -955,11 +1205,24 @@ impl Run<'_> { }); } - let url = feed - .package_url(&item.name, &item.version) - .await - .map_err(|e| e.to_string())?; - let response = feed.download(&url).await.map_err(|e| e.to_string())?; + let (url, response) = match from { + PackageSource::Feed(feed) => { + let url = feed + .package_url(&item.name, &item.version) + .await + .map_err(|e| e.to_string())?; + let response = feed.download(&url).await.map_err(|e| e.to_string())?; + (url, response) + } + PackageSource::Npm(registry) => { + let url = source + .tarball + .clone() + .ok_or("the registry names no tarball of this version")?; + let response = registry.download(&url).await.map_err(|e| e.to_string())?; + (url, response) + } + }; if response.content_length().is_some_and(|l| l > ctx.limit) { return Err(WriteError::TooLarge { limit: ctx.limit }.to_string()); } @@ -970,7 +1233,7 @@ impl Run<'_> { .stage(tokio_util::io::StreamReader::new(body), ctx.limit) .await .map_err(|e| WriteError::from(e).to_string())?; - verify(&staged, &url, source)?; + verify(&staged, &url, source, from.noun()).await?; let bytes = staged.size; let options = ImportOptions { overwrite, @@ -979,6 +1242,10 @@ impl Run<'_> { downloaded_at: Some(item.downloaded_at.unwrap_or_else(Utc::now)), uploader: item.uploader.clone(), source: ctx.source.clone(), + // A registry names the tarball in the package's packument: it + // must be that package. + expected: matches!(from, PackageSource::Npm(_)) + .then(|| (item.name.clone(), item.version.clone())), }; let outcome = writer .import_staged(&ctx.repo.name, staged, ctx.limit, &options) @@ -997,6 +1264,200 @@ impl Run<'_> { }) } + /// What an earlier import of the repository recorded of `item` it + /// copied or found identical ([`import_dao::earlier_source`]). + async fn earlier(&self, ctx: &Ctx, item: &ImportItem) -> Result, String> { + let earlier = import_dao::earlier_source( + &self.state.pool, + ctx.repo.id, + self.import.id, + &item.kind, + &item.name, + &item.version, + ) + .await + .map_err(|e| format!("database error: {e}"))?; + Ok(earlier.map(|source| serde_json::from_value(source).unwrap_or_default())) + } + + /// npm: bring a copied version's deprecation along (`outcome`: what became + /// of its bytes). A version Forge holds with these bytes takes the + /// source's deprecation where Forge gives none, or the one an earlier + /// import gave; a deprecation Forge gives otherwise is a conflict + /// (replaced with `overwrite`). + async fn carry_deprecation( + &self, + ctx: &Ctx, + filter: &PackageFilter, + item: &ImportItem, + source: &ItemSource, + outcome: Copied, + ) -> Result { + let Copied::Item { + state, + detail, + bytes, + } = outcome + else { + return Ok(outcome); + }; + // A version that stays out of Forge (a conflict skipped) keeps + // Forge's; one a dry run plans is not there yet. + if !matches!(state, "imported" | "replaced" | "identical") { + return Ok(Copied::Item { + state, + detail, + bytes, + }); + } + let db = |e: sqlx::Error| format!("database error: {e}"); + let pool = &self.state.pool; + let Some(current) = + component_dao::deprecation(pool, ctx.repo.id, &item.name, &item.version) + .await + .map_err(db)? + else { + return Ok(Copied::Item { + state, + detail, + bytes, + }); + }; + let wanted = source.deprecated.clone(); + if current == wanted { + return Ok(Copied::Item { + state, + detail, + bytes, + }); + } + let own = match self.earlier(ctx, item).await? { + Some(earlier) => earlier.deprecated == current, + None => current.is_none(), + }; + let overwrite = self.import.on_conflict == "overwrite"; + if !(own || overwrite || state == "replaced") { + let conflict = match (¤t, &wanted) { + (Some(current), Some(wanted)) => format!( + "Forge deprecates this version ({current:?}); the source deprecates it \ + otherwise ({wanted:?})" + ), + (Some(current), None) => { + format!("Forge deprecates this version ({current:?}); the source does not") + } + (None, _) => format!( + "the source deprecates this version ({:?}); Forge no longer does", + wanted.unwrap_or_default() + ), + }; + return Ok(copied("conflict", Some(conflict), bytes)); + } + if !filter.allows(&item.name) { + return Err( + "the package selectors of the administrator who started the import do not \ + allow deprecating this package" + .into(), + ); + } + let said = match &wanted { + Some(message) => format!("deprecated: {message}"), + None => "no longer deprecated".to_string(), + }; + if self.import.dry_run { + return Ok(copied(state, Some(format!("would be {said}")), bytes)); + } + component_dao::set_deprecation( + pool, + ctx.repo.id, + &item.name, + &item.version, + wanted.as_deref(), + ) + .await + .map_err(db)?; + Ok(copied(state, Some(detail.unwrap_or(said)), bytes)) + } + + /// npm: set a dist-tag (`item.version`) of the package `item.name` to + /// the version the source points it at, once that version is in Forge. + /// A tag Forge points at another version is a conflict, unless an + /// earlier import pointed it there: then it moves with the source. + async fn copy_dist_tag( + &self, + ctx: &Ctx, + filter: &PackageFilter, + item: &ImportItem, + source: &ItemSource, + ) -> Result { + let db = |e: sqlx::Error| format!("database error: {e}"); + let pool = &self.state.pool; + let tag = &item.version; + let target = source + .target + .as_deref() + .ok_or("the source names no version for this dist-tag")?; + if !crate::plugins::valid_tag(tag) { + return Err(format!("{tag:?} is no dist-tag Forge stores")); + } + let tags = tag_dao::tags(pool, ctx.repo.id, &item.name) + .await + .map_err(db)?; + let current = tags.get(tag); + if current.map(String::as_str) == Some(target) { + return Ok(copied("identical", Some(format!("points at {target}")), 0)); + } + if let Some(current) = current { + let own = self + .earlier(ctx, item) + .await? + .is_some_and(|earlier| earlier.target.as_ref() == Some(current)); + if !own && self.import.on_conflict != "overwrite" { + return Ok(copied( + "conflict", + Some(format!( + "Forge points it at {current}; the source at {target}" + )), + 0, + )); + } + } + if !filter.allows(&item.name) { + return Err( + "the package selectors of the administrator who started the import do not \ + allow tagging this package" + .into(), + ); + } + if self.import.dry_run { + return Ok(copied( + "planned", + Some(match current { + Some(current) => format!("would move it from {current} to {target}"), + None => format!("would point it at {target}"), + }), + 0, + )); + } + if component_dao::find_component(pool, ctx.repo.id, &item.name, target) + .await + .map_err(db)? + .is_none() + { + return Err(format!("version {target} is not in Forge")); + } + tag_dao::set_tag(pool, ctx.repo.id, &item.name, tag, target) + .await + .map_err(db)?; + Ok(match current { + Some(current) => copied( + "replaced", + Some(format!("moved from {current} to {target}")), + 0, + ), + None => copied("imported", Some(format!("points at {target}")), 0), + }) + } + /// Copy a Docker tag (`component`) or referrer (`manifest`) through the /// registry, and record the referrers the source lists for it. async fn copy_image( @@ -1109,14 +1570,6 @@ impl Run<'_> { } } -/// The versions of the package id `name`, or the tags of the image `name`. -async fn expand_one(target: &Target, name: &str, named: bool) -> Result, SourceError> { - match target { - Target::Packages { feed, .. } => package_versions(feed, name, named).await, - Target::Images { registry, source } => image_tags(registry, source, name, named).await, - } -} - /// Every version the feed lists of the package `id`, with what its /// registration says of each. A package the feed does not know is left /// out, or found as missing when the import `named` it. @@ -1148,10 +1601,12 @@ async fn package_versions(feed: &Feed, id: &str, named: bool) -> Result Result Result, SourceError> { + let package = registry + .package(name) + .await? + .filter(|p| !p.releases.is_empty()); + let Some(package) = package else { + return Ok(if named { + vec![Found::missing(name)] + } else { + Vec::new() + }); + }; + let mut found: Vec = Vec::with_capacity(package.releases.len()); + for release in &package.releases { + found.push(Found { + kind: "component", + name: package.name.clone(), + version: release.version.clone(), + published_at: release.published, + source: ItemSource { + sha512: release.sha512(), + integrity: release.integrity.clone(), + shasum: release.shasum.clone(), + tarball: registry.tarball_url(&package.name, &release.file), + deprecated: release.deprecated.clone(), + target: None, + }, + }); + } + for (tag, version) in &package.dist_tags { + found.push(Found { + kind: "tag", + name: package.name.clone(), + version: tag.clone(), + published_at: None, + source: ItemSource { + target: Some(version.clone()), + ..ItemSource::default() + }, + }); + } + Ok(found) +} + /// Every tag of `image` at the source. An image the source does not have /// (or has no tag of) is left out, or found as missing when the import /// `named` it. @@ -1181,6 +1688,7 @@ async fn image_tags( let more = page.more && !page.tags.is_empty(); last = page.tags.last().cloned().or(last); found.extend(page.tags.into_iter().map(|tag| Found { + kind: "component", name: image.to_string(), version: tag, published_at: None, @@ -1221,13 +1729,52 @@ fn conflict_item() -> Copied { ) } -/// Check a staged download against the SHA-512 the feed lists for it. -fn verify(staged: &StagedBlob, url: &str, source: &ItemSource) -> Result<(), String> { - match &source.sha512 { - Some(expected) if *expected != staged.sha512_base64 => Err(format!( - "the download of {url} does not match the SHA-512 the feed lists for it" - )), - _ => Ok(()), +/// Check a staged download against the SHA-512 the source (a `lister`, the +/// feed or the registry) lists for it, or, for an npm tarball listed without +/// one, against its `dist.integrity` and `dist.shasum`. +async fn verify( + staged: &StagedBlob, + url: &str, + source: &ItemSource, + lister: &str, +) -> Result<(), String> { + if let Some(expected) = &source.sha512 { + if *expected != staged.sha512_base64 { + return Err(format!( + "the download of {url} does not match the SHA-512 the {lister} lists for it" + )); + } + return Ok(()); + } + if source.integrity.is_none() && source.shasum.is_none() { + return Ok(()); + } + let path = staged.read_path(); + let digests = tokio::task::spawn_blocking(move || { + std::fs::File::open(path).and_then(forge_format_npm::integrity::Digests::of) + }) + .await + .map_err(|e| format!("checking the download: {e}"))? + .map_err(|e| format!("checking the download: {e}"))?; + digests + .verify(source.integrity.as_deref(), source.shasum.as_deref()) + .map_err(|e| format!("the download of {url} does not match the {lister}'s checksums: {e}")) +} + +/// Where a package version is copied from. +#[derive(Clone, Copy)] +enum PackageSource<'a> { + Feed(&'a Feed), + Npm(&'a NpmRegistry), +} + +impl PackageSource<'_> { + /// What the source is called in a message. + fn noun(self) -> &'static str { + match self { + PackageSource::Feed(_) => "feed", + PackageSource::Npm(_) => "registry", + } } } @@ -1312,11 +1859,30 @@ mod tests { #[test] fn a_nexus_repository_reads_through_its_protocol() { assert_eq!( - nexus_source_url("https://nexus.example.com/", "nuget-hosted", None, false), + nexus_source_url( + "https://nexus.example.com/", + "nuget-hosted", + None, + Protocol::NuGet + ), "https://nexus.example.com/repository/nuget-hosted/index.json" ); assert_eq!( - nexus_source_url("https://nexus.example.com", "docker-hosted", None, true), + nexus_source_url( + "https://nexus.example.com", + "npm-hosted", + Some("ignored"), + Protocol::Npm + ), + "https://nexus.example.com/repository/npm-hosted" + ); + assert_eq!( + nexus_source_url( + "https://nexus.example.com", + "docker-hosted", + None, + Protocol::Images + ), "https://nexus.example.com/repository/docker-hosted" ); assert_eq!( @@ -1324,10 +1890,13 @@ mod tests { "https://nexus.example.com", "docker-hosted", Some("https://nexus.example.com:8082"), - true + Protocol::Images ), "https://nexus.example.com:8082" ); + assert_eq!(Protocol::of("npm", false), Protocol::Npm); + assert_eq!(Protocol::of("nuget", false), Protocol::NuGet); + assert_eq!(Protocol::of("docker", true), Protocol::Images); } #[test] @@ -1335,7 +1904,12 @@ mod tests { assert_eq!(Listing::of(false, false), Listing::Search); assert_eq!(Listing::of(true, false), Listing::Catalog); assert_eq!(Listing::of(true, true), Listing::Names); - for listing in [Listing::Search, Listing::Catalog, Listing::Names] { + for listing in [ + Listing::Search, + Listing::Catalog, + Listing::All, + Listing::Names, + ] { assert_eq!(Listing::parse(listing.as_str()), listing); } } @@ -1350,4 +1924,24 @@ mod tests { let source: ItemSource = serde_json::from_value(legacy).unwrap(); assert_eq!(source.sha512, None); } + + #[test] + fn an_item_source_stores_only_what_it_has() { + let nuget = ItemSource { + sha512: Some("abc".into()), + ..ItemSource::default() + }; + assert_eq!( + serde_json::to_value(&nuget).unwrap(), + serde_json::json!({"sha512": "abc"}) + ); + let tag = ItemSource { + target: Some("1.0.0".into()), + ..ItemSource::default() + }; + let stored = serde_json::to_value(&tag).unwrap(); + assert_eq!(stored, serde_json::json!({"target": "1.0.0"})); + let read: ItemSource = serde_json::from_value(stored).unwrap(); + assert_eq!(read.target.as_deref(), Some("1.0.0")); + } } diff --git a/crates/server/src/import/npm.rs b/crates/server/src/import/npm.rs new file mode 100644 index 00000000..aab3dcf1 --- /dev/null +++ b/crates/server/src/import/npm.rs @@ -0,0 +1,256 @@ +//! An npm registry as an import's source: Nexus' `…/repository/`, +//! Verdaccio, Artifactory or another Forge. Where to ask and how to read the +//! answers is the npm format's ([`forge_format_npm::import`], with the +//! packument reader npm proxies use); this sends the requests, through the +//! shared [`UpstreamHttp`] client (timeouts, size caps, no cloud-metadata +//! targets), with the credentials, as Basic authentication. +//! +//! Every request goes to the registry's URL, the one the import was given: +//! a tarball too ([`NpmRegistry::tarball_url`]), whatever host its +//! `dist.tarball` names. +//! So the credentials go to that origin alone. + +use forge_core::net::upstream::{Origin, RequestClass, UpstreamAuth, UpstreamError, UpstreamHttp}; +use forge_crypto::{ExposeSecret, PlaintextSecret}; +use forge_format_npm::import::{self as npm, Package, SearchPage}; +use forge_format_npm::proxy::{ABBREVIATED_ACCEPT, FULL_ACCEPT}; +use forge_format_npm::PackageName; +use reqwest::StatusCode; + +use super::SourceError; + +/// Pages of a search for a scope read at most. +const MAX_SCOPE_PAGES: u32 = 1000; + +/// An npm registry. +pub struct NpmRegistry { + http: UpstreamHttp, + /// Without a trailing `/`. + url: String, + username: Option, + password: Option, + /// The `Server` header the registry answered [`Self::ping`] with. + server: Option, +} + +/// A status by which a registry says it does not serve a listing: Verdaccio +/// refuses a search without text (400); a registry that dropped `/-/all` +/// may answer 405, 410 or 501 rather than 404. +fn not_served(status: StatusCode) -> bool { + matches!( + status, + StatusCode::BAD_REQUEST + | StatusCode::METHOD_NOT_ALLOWED + | StatusCode::GONE + | StatusCode::NOT_IMPLEMENTED + ) +} + +impl NpmRegistry { + /// The registry at `url`, read with the credentials when given. + pub fn new( + http: UpstreamHttp, + url: &str, + username: Option, + password: Option, + ) -> Self { + NpmRegistry { + http, + url: url.trim_end_matches('/').to_string(), + username, + password, + server: None, + } + } + + /// The `Server` header the registry answered [`Self::ping`] with. + pub fn server(&self) -> Option<&str> { + self.server.as_deref() + } + + /// The Basic credentials, for the registry's origin only. + fn auth(&self) -> Option> { + match (&self.username, &self.password) { + (Some(username), Some(password)) => Some(UpstreamAuth { + origin: Origin::of(&self.url), + username, + password: password.expose_secret(), + bearer: false, + }), + _ => None, + } + } + + /// `GET /-/ping`: an unreachable registry, or one that refuses the + /// credentials, is an error. One that does not serve the ping is let + /// through: its listing, or its packages, say more. Records the + /// `Server` header. + pub async fn ping(&mut self) -> Result<(), SourceError> { + let url = format!("{}/-/ping", self.url); + let response = self + .http + .get(&url, RequestClass::Metadata, self.auth().as_ref()) + .await + .map_err(|e| SourceError::of(e, &url))?; + self.server = response + .headers() + .get(reqwest::header::SERVER) + .and_then(|v| v.to_str().ok()) + .map(str::to_string); + let status = response.status(); + if matches!(status, StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN) + || status.is_server_error() + { + return Err(SourceError::status(status, &url)); + } + Ok(()) + } + + /// A metadata document (`Ok(None)`: the registry does not have it, or + /// does not serve it), streamed into a spool rather than held whole. + async fn document( + &self, + url: &str, + accept: &str, + ) -> Result, SourceError> { + match self + .http + .get_metadata_body(url, self.auth().as_ref(), Some(accept)) + .await + { + Ok(body) => Ok(body), + Err(UpstreamError::Status { status, .. }) if not_served(status) => Ok(None), + Err(e) => Err(SourceError::of(e, url)), + } + } + + /// A page of the registry's search for `text` (empty: every package), + /// `size` packages after the first `from`. `None` when the registry + /// offers no such search. + pub async fn search( + &self, + text: &str, + from: u32, + size: u32, + ) -> Result, SourceError> { + let url = npm::search_url(&self.url, text, from, size); + let Some(body) = self.document(&url, FULL_ACCEPT).await? else { + return Ok(None); + }; + read(body, move |reader| { + let mut bytes = Vec::new(); + std::io::Read::read_to_end(reader, &mut bytes).map_err(|e| e.to_string())?; + npm::read_search(&bytes) + }) + .await + .map(Some) + } + + /// Every package `/-/all` lists, sorted; `None` when the registry does + /// not serve it. + pub async fn all(&self) -> Result>, SourceError> { + let url = format!("{}{}", self.url, npm::ALL_PATH); + let Some(body) = self.document(&url, FULL_ACCEPT).await? else { + return Ok(None); + }; + read(body, |reader| npm::read_all(reader)).await.map(Some) + } + + /// The packages of `scope` (`@acme`) the registry's search finds, read + /// page by page up to an empty page (or the same page again: a search + /// that ignores `from`). + pub async fn search_scope(&self, scope: &str) -> Result, SourceError> { + let prefix = format!("{scope}/"); + let mut found: Vec = Vec::new(); + let mut last: Vec = Vec::new(); + // A registry may answer fewer than it was asked for: the next page + // starts after what it answered. + let mut from = 0u32; + for _ in 0..MAX_SCOPE_PAGES { + let Some(page) = self.search(scope, from, npm::MAX_SEARCH_SIZE).await? else { + break; + }; + if page.names.is_empty() || page.names == last { + break; + } + from = from.saturating_add(u32::try_from(page.names.len()).unwrap_or(u32::MAX)); + for name in &page.names { + if name.starts_with(&prefix) && !found.contains(name) { + found.push(name.clone()); + } + } + last = page.names; + } + found.sort(); + Ok(found) + } + + /// The package `name`, from its full packument; `None` when the registry + /// does not have it. A full packument larger than Forge reads + /// (`proxy.max_metadata_bytes`) is read abbreviated, without its publish + /// times. + pub async fn package(&self, name: &str) -> Result, SourceError> { + let name = PackageName::parse(name) + .map_err(|e| SourceError::Invalid(format!("{name:?} is no package name: {e}")))?; + let url = npm::packument_url(&self.url, &name); + let body = match self + .http + .get_metadata_body(&url, self.auth().as_ref(), Some(FULL_ACCEPT)) + .await + { + Ok(body) => body, + Err(UpstreamError::TooLarge { .. }) => { + tracing::warn!(package = %name, "import: the full packument is too large; reading it abbreviated, without publish times"); + self.http + .get_metadata_body(&url, self.auth().as_ref(), Some(ABBREVIATED_ACCEPT)) + .await + .map_err(|e| SourceError::of(e, &url))? + } + Err(e) => return Err(SourceError::of(e, &url)), + }; + let Some(body) = body else { + return Ok(None); + }; + read(body, move |reader| npm::read_package(reader, &name)) + .await + .map(Some) + } + + /// Where the import downloads the tarball `file` of `name`. + pub fn tarball_url(&self, name: &str, file: &str) -> Option { + let name = PackageName::parse(name).ok()?; + Some(npm::tarball_url(&self.url, &name, file)) + } + + /// GET a tarball (streamed by the caller), at the registry's origin. + pub async fn download(&self, url: &str) -> Result { + if Origin::of(url) != Origin::of(&self.url) { + return Err(SourceError::Invalid(format!( + "{url} is not at the registry's origin" + ))); + } + let response = self + .http + .get(url, RequestClass::Download, self.auth().as_ref()) + .await + .map_err(|e| SourceError::of(e, url))?; + if !response.status().is_success() { + return Err(SourceError::status(response.status(), url)); + } + Ok(response) + } +} + +/// Read a spooled document with `parse`, on the blocking pool. +async fn read( + body: forge_core::net::upstream::SpooledBody, + parse: impl FnOnce(&mut dyn std::io::Read) -> Result + Send + 'static, +) -> Result { + tokio::task::spawn_blocking(move || { + let mut reader = body.into_reader().map_err(|e| e.to_string())?; + parse(&mut reader) + }) + .await + .map_err(|e| SourceError::Invalid(format!("reading the registry's answer: {e}")))? + .map_err(SourceError::Invalid) +} diff --git a/crates/server/src/plugins.rs b/crates/server/src/plugins.rs index 7bea6684..340d2c4c 100644 --- a/crates/server/src/plugins.rs +++ b/crates/server/src/plugins.rs @@ -1927,7 +1927,7 @@ fn outcome(result: Result) -> Result<(), Response> { } /// A tag: a short word of letters, digits, `.`, `_` and `-`. -fn valid_tag(tag: &str) -> bool { +pub(crate) fn valid_tag(tag: &str) -> bool { !tag.is_empty() && tag.len() <= 64 && tag diff --git a/crates/server/tests/import.rs b/crates/server/tests/import.rs index 738fd623..8af57f4c 100644 --- a/crates/server/tests/import.rs +++ b/crates/server/tests/import.rs @@ -23,7 +23,16 @@ //! it, through the referrers tag schema, digests kept; //! * an import stored before imports read the formats' protocols lists and //! resumes; -//! * with `[crypto]`, the password is sealed at rest. +//! * with `[crypto]`, the password is sealed at rest; +//! * npm, from a mock registry at `/repository//`: a search that lists +//! every package (as another Forge's does), paged; `/-/all` where the +//! search lists nothing (Nexus) or refuses an empty text (Verdaccio); +//! named packages and scopes. Bytes, publish times, dist-tags and +//! deprecations arrive, tarballs come from the import's URL whatever +//! host `dist.tarball` names, and are checked against `dist.integrity` +//! (or `dist.shasum` alone). A second run downloads only what is new and +//! moves the tags the source moved; a tag or deprecation changed in Forge +//! is a conflict; the dry run downloads nothing. mod common; @@ -110,6 +119,44 @@ struct SourceState { downloads: HashMap, /// Slows every package download. delay: Duration, + /// npm registries, by repository. + npm: HashMap, +} + +/// A version of an npm package at a mock registry. +#[derive(Clone)] +struct NpmVersion { + bytes: Vec, + published: Option, + deprecated: Option, + /// The packument lists `dist.integrity` (old packages have only + /// `dist.shasum`). + integrity: bool, +} + +/// How a mock registry's search answers a search for every package (an +/// empty `text`). +#[derive(Clone, Copy, Default, PartialEq)] +enum NpmSearch { + /// Every package, scoped ones with their scope (another Forge). + #[default] + Lists, + /// Nothing; and a search with a text names a scoped package without its + /// scope (Nexus 3.70). + Nothing, + /// 400: a text is required (Verdaccio). + Refused, +} + +#[derive(Default)] +struct NpmSource { + /// name → version → version. + packages: BTreeMap>, + /// name → dist-tag → version. + tags: BTreeMap>, + search: NpmSearch, + /// Serves `/-/all`. + all: bool, } #[derive(Clone)] @@ -299,6 +346,9 @@ async fn answer(state: &Mutex, base: &str, req: Request) -> Respons return StatusCode::NOT_FOUND.into_response(); }; let (repo, inner) = rest.split_once('/').unwrap_or((rest, "")); + if state.lock().unwrap().npm.contains_key(repo) { + return npm_registry(state, repo, inner, &query); + } if let Some(api) = inner.strip_prefix("v2/") { return registry(state, repo, api, &query); } @@ -529,6 +579,198 @@ fn registry( StatusCode::NOT_FOUND.into_response() } +/// The host the mock registries' packuments name tarballs on, as a Nexus +/// configured with a base URL clients use names its own: the import does +/// not reach it. +const NPM_TARBALL_HOST: &str = "http://npm.internal.invalid"; + +/// An npm registry at `/repository//`. +fn npm_registry( + state: &Mutex, + repo: &str, + inner: &str, + query: &HashMap, +) -> Response { + let mut s = state.lock().unwrap(); + let page_size = s.page_size; + let registry = &s.npm[repo]; + match inner { + "-/ping" => return json_response(json!({})), + "-/all" if registry.all => { + let mut all = serde_json::Map::new(); + all.insert("_updated".into(), json!(1)); + for (name, versions) in ®istry.packages { + let tags = registry.tags.get(name).cloned().unwrap_or_default(); + all.insert( + name.clone(), + json!({"name": name, "dist-tags": tags, "versions": versions.keys().map(|v| (v.clone(), json!("latest"))).collect::>()}), + ); + } + return json_response(Value::Object(all)); + } + "-/v1/search" => { + let text = query.get("text").cloned().unwrap_or_default(); + if text.is_empty() && registry.search == NpmSearch::Refused { + return ( + StatusCode::BAD_REQUEST, + json!({"error": "'text' query parameter is required"}).to_string(), + ) + .into_response(); + } + let found: Vec<&String> = registry + .packages + .keys() + .filter(|name| match registry.search { + NpmSearch::Nothing if text.is_empty() => false, + _ => name.contains(text.as_str()), + }) + .collect(); + let from: usize = query.get("from").and_then(|v| v.parse().ok()).unwrap_or(0); + let size: usize = query.get("size").and_then(|v| v.parse().ok()).unwrap_or(20); + let objects: Vec = found + .iter() + .skip(from) + .take(size.min(page_size)) + .map(|name| match registry.search { + // Nexus: the name without its scope, and no scope. + NpmSearch::Nothing => { + json!({"package": {"name": name.rsplit('/').next().unwrap(), "version": "1.0.0"}}) + } + _ => { + let scope = name + .strip_prefix('@') + .and_then(|n| n.split_once('/')) + .map_or("unscoped", |(scope, _)| scope); + json!({"package": {"name": name, "scope": scope, "version": "1.0.0"}}) + } + }) + .collect(); + return json_response(json!({"objects": objects, "total": found.len()})); + } + _ => {} + } + // A tarball, `/-/`. + if let Some((name, file)) = inner.split_once("/-/") { + let unscoped = name.rsplit('/').next().unwrap(); + let Some(version) = file + .strip_suffix(".tgz") + .and_then(|f| f.strip_prefix(unscoped)) + .and_then(|f| f.strip_prefix('-')) + else { + return StatusCode::NOT_FOUND.into_response(); + }; + let Some(found) = registry + .packages + .get(name) + .and_then(|versions| versions.get(version)) + else { + return StatusCode::NOT_FOUND.into_response(); + }; + let bytes = found.bytes.clone(); + let key = format!("npm/{repo}/{name}/{version}"); + *s.downloads.entry(key.clone()).or_default() += 1; + return s + .corrupt + .get(&key) + .cloned() + .unwrap_or(bytes) + .into_response(); + } + // A packument, `` or `@scope%2fname`. + let name = inner.replacen("%2f", "/", 1).replacen("%2F", "/", 1); + let Some(versions) = registry.packages.get(&name) else { + return StatusCode::NOT_FOUND.into_response(); + }; + let unscoped = name.rsplit('/').next().unwrap(); + let mut time = serde_json::Map::new(); + let mut objects = serde_json::Map::new(); + for (version, v) in versions { + let digests = forge_format_npm::integrity::Digests::of(&v.bytes[..]).unwrap(); + let mut dist = json!({ + "shasum": digests.sha1_hex, + "tarball": format!("{NPM_TARBALL_HOST}/repository/{repo}/{name}/-/{unscoped}-{version}.tgz"), + }); + if v.integrity { + dist["integrity"] = json!(digests.integrity()); + } + let mut object = json!({"name": name, "version": version, "readme": "# big", "dist": dist}); + if let Some(message) = &v.deprecated { + object["deprecated"] = json!(message); + } + if let Some(published) = &v.published { + time.insert(version.clone(), json!(published)); + } + objects.insert(version.clone(), object); + } + json_response(json!({ + "_id": name, "name": name, + "dist-tags": registry.tags.get(&name).cloned().unwrap_or_default(), + "time": time, + "versions": objects, + })) +} + +/// A gzip-compressed npm tarball of `name@version`, with `content`. +fn tgz(name: &str, version: &str, content: &str) -> Vec { + let package_json = json!({"name": name, "version": version, "main": "index.js"}).to_string(); + forge_format_npm::tarball::build(&[ + ("package/package.json", package_json.as_bytes()), + ("package/index.js", content.as_bytes()), + ]) +} + +impl Source { + /// The npm registry `name`, set up by `f`. + fn npm(&self, name: &str, f: impl FnOnce(&mut NpmSource)) { + self.with(|s| f(s.npm.entry(name.into()).or_default())); + } + + /// Add (or replace) a version of an npm package, with its integrity. + fn npm_version( + &self, + registry: &str, + name: &str, + version: &str, + bytes: Vec, + published: &str, + ) { + self.npm(registry, |r| { + r.packages.entry(name.into()).or_default().insert( + version.into(), + NpmVersion { + bytes, + published: Some(published.into()), + deprecated: None, + integrity: true, + }, + ); + }); + } + + fn npm_tag(&self, registry: &str, name: &str, tag: &str, version: &str) { + self.npm(registry, |r| { + r.tags + .entry(name.into()) + .or_default() + .insert(tag.into(), version.into()); + }); + } + + /// The registry API of `name`: npm's. + fn npm_url(&self, name: &str) -> String { + format!("{}/repository/{name}", self.url) + } + + fn npm_downloads(&self, registry: &str, name: &str, version: &str) -> usize { + self.with(|s| { + s.downloads + .get(&format!("npm/{registry}/{name}/{version}")) + .copied() + .unwrap_or(0) + }) + } +} + // ── Forge helpers ─────────────────────────────────────────────────────────── /// A server over a database of its own, so no other test's worker runs its @@ -1743,3 +1985,584 @@ async fn the_password_is_sealed_at_rest() { assert_eq!(done["state"], "completed", "{done}"); assert_eq!(done["progress"]["imported"], 1); } + +// ── npm ───────────────────────────────────────────────────────────────────── + +/// A path of an npm repository of Forge, read as the administrator. +async fn npm_get(srv: &TestServer, repo: &str, path: &str) -> reqwest::Response { + let (u, p) = srv.admin_basic(); + srv.client() + .get(format!("{}/repository/{repo}/{path}", srv.base_url)) + .basic_auth(u, Some(p)) + .send() + .await + .unwrap() +} + +/// Forge's full packument of `name`. +async fn packument(srv: &TestServer, repo: &str, name: &str) -> Value { + let resp = npm_get(srv, repo, &name.replacen('/', "%2f", 1)).await; + assert_eq!(resp.status(), 200, "{name}"); + resp.json().await.unwrap() +} + +/// The tarball of `name@version` Forge serves, if it has it. +async fn npm_tarball(srv: &TestServer, repo: &str, name: &str, version: &str) -> Option> { + let unscoped = name.rsplit('/').next().unwrap(); + let resp = npm_get(srv, repo, &format!("{name}/-/{unscoped}-{version}.tgz")).await; + (resp.status() == 200).then_some(resp.bytes().await.unwrap().to_vec()) +} + +/// The item of an import of `kind` `name` `version`. +async fn item_of(srv: &TestServer, repo: &str, id: i64, name: &str, version: &str) -> Value { + items(srv, repo, id, None) + .await + .into_iter() + .find(|i| i["name"] == name && i["version"] == version) + .unwrap_or_else(|| panic!("no item {name} {version}")) +} + +fn parse_time(t: &str) -> DateTime { + DateTime::parse_from_rfc3339(t).unwrap().with_timezone(&Utc) +} + +#[tokio::test] +async fn npm_import_keeps_bytes_times_tags_and_deprecations_and_a_second_run_copies_only_what_is_new( +) { + let srv = forge().await; + let source = Source::start().await; + let reg = "npm-hosted"; + let alpha = tgz("alpha", "1.0.0", "one"); + source.npm_version( + reg, + "alpha", + "1.0.0", + alpha.clone(), + "2020-01-02T03:04:05.678Z", + ); + source.npm_version( + reg, + "alpha", + "1.1.0", + tgz("alpha", "1.1.0", "two"), + "2021-01-01T00:00:00.000Z", + ); + source.npm_version( + reg, + "alpha", + "2.0.0-beta.1", + tgz("alpha", "2.0.0-beta.1", "three"), + "2022-01-01T00:00:00.000Z", + ); + source.npm_tag(reg, "alpha", "latest", "1.1.0"); + source.npm_tag(reg, "alpha", "next", "2.0.0-beta.1"); + source.npm(reg, |r| { + r.packages + .get_mut("alpha") + .unwrap() + .get_mut("1.0.0") + .unwrap() + .deprecated = Some("use 1.1.0".into()) + }); + let ui = tgz("@acme/ui", "1.0.0", "ui"); + source.npm_version( + reg, + "@acme/ui", + "1.0.0", + ui.clone(), + "2023-05-06T07:08:09.000Z", + ); + source.npm_tag(reg, "@acme/ui", "latest", "1.0.0"); + // An old package, with `dist.shasum` alone. + source.npm_version( + reg, + "legacy", + "0.1.0", + tgz("legacy", "0.1.0", "old"), + "2015-01-01T00:00:00.000Z", + ); + source.npm(reg, |r| { + r.packages + .get_mut("legacy") + .unwrap() + .get_mut("0.1.0") + .unwrap() + .integrity = false + }); + source.npm_tag(reg, "legacy", "latest", "0.1.0"); + + let repo = repo_name("npm-import"); + create_repo( + &srv, + json!({"name": repo, "kind": "hosted", "format": "npm"}), + ) + .await; + let id = start(&srv, &repo, from(source.npm_url(reg))).await; + let queued = get_import(&srv, &repo, id).await; + assert_eq!(queued["url"], source.npm_url(reg)); + // Another Forge's search lists every package; it pages by two here. + assert_eq!(queued["listing"], "search"); + assert_eq!(queued["source_total"], 3); + assert_eq!(queued["source_server"], SERVER); + let done = run(&srv, &repo, id).await; + assert_eq!(done["state"], "completed", "{done}"); + // Five versions and four dist-tags. + assert_eq!(done["progress"]["found"], 9, "{done}"); + assert_eq!(done["progress"]["imported"], 9, "{done}"); + assert_eq!(done["has_password"], false); + + // The tarballs came from the import's URL (the packuments name another + // host), with their bytes; the publish times are the registry's. + assert_eq!( + npm_tarball(&srv, &repo, "alpha", "1.0.0").await.unwrap(), + alpha + ); + assert_eq!( + npm_tarball(&srv, &repo, "@acme/ui", "1.0.0").await.unwrap(), + ui + ); + assert_eq!(source.npm_downloads(reg, "alpha", "1.0.0"), 1); + let (created, _) = times(&srv, &repo, "alpha", "1.0.0").await; + assert_eq!(created, parse_time("2020-01-02T03:04:05.678Z")); + let (created, _) = times(&srv, &repo, "legacy", "0.1.0").await; + assert_eq!(created, parse_time("2015-01-01T00:00:00Z")); + let doc = packument(&srv, &repo, "alpha").await; + assert_eq!( + doc["dist-tags"], + json!({"latest": "1.1.0", "next": "2.0.0-beta.1"}) + ); + assert_eq!(doc["versions"]["1.0.0"]["deprecated"], "use 1.1.0"); + assert!(doc["versions"]["1.1.0"].get("deprecated").is_none()); + assert_eq!( + parse_time(doc["time"]["1.1.0"].as_str().unwrap()), + parse_time("2021-01-01T00:00:00Z") + ); + assert_eq!( + doc["versions"]["1.0.0"]["dist"]["integrity"], + forge_format_npm::integrity::Digests::of(&alpha[..]) + .unwrap() + .integrity() + ); + let doc = packument(&srv, &repo, "@acme/ui").await; + assert_eq!(doc["dist-tags"], json!({"latest": "1.0.0"})); + let tag = item_of(&srv, &repo, id, "alpha", "next").await; + assert_eq!(tag["kind"], "tag"); + assert_eq!(tag["detail"], "points at 2.0.0-beta.1"); + + // The source moves on: a new version becomes latest, a version is + // deprecated. + let newer = tgz("alpha", "1.2.0", "four"); + source.npm_version( + reg, + "alpha", + "1.2.0", + newer.clone(), + "2024-02-03T04:05:06.000Z", + ); + source.npm_tag(reg, "alpha", "latest", "1.2.0"); + source.npm(reg, |r| { + r.packages + .get_mut("@acme/ui") + .unwrap() + .get_mut("1.0.0") + .unwrap() + .deprecated = Some("gone".into()) + }); + let again = start(&srv, &repo, from(source.npm_url(reg))).await; + let done = run(&srv, &repo, again).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["found"], 10, "{done}"); + assert_eq!(done["progress"]["imported"], 1, "{done}"); + assert_eq!(done["progress"]["replaced"], 1, "the tag the source moved"); + assert_eq!(done["progress"]["identical"], 8, "{done}"); + // Only the new version is downloaded, and the one the registry lists no + // SHA-512 of, to compare. + assert_eq!(source.npm_downloads(reg, "alpha", "1.0.0"), 1); + assert_eq!(source.npm_downloads(reg, "alpha", "1.2.0"), 1); + assert_eq!(source.npm_downloads(reg, "@acme/ui", "1.0.0"), 1); + assert_eq!(source.npm_downloads(reg, "legacy", "0.1.0"), 2); + assert_eq!( + npm_tarball(&srv, &repo, "alpha", "1.2.0").await.unwrap(), + newer + ); + let doc = packument(&srv, &repo, "alpha").await; + assert_eq!(doc["dist-tags"]["latest"], "1.2.0"); + let moved = item_of(&srv, &repo, again, "alpha", "latest").await; + assert_eq!(moved["state"], "replaced"); + assert_eq!(moved["detail"], "moved from 1.1.0 to 1.2.0"); + let doc = packument(&srv, &repo, "@acme/ui").await; + assert_eq!(doc["versions"]["1.0.0"]["deprecated"], "gone"); + let deprecated = item_of(&srv, &repo, again, "@acme/ui", "1.0.0").await; + assert_eq!( + (deprecated["state"].as_str(), deprecated["detail"].as_str()), + (Some("identical"), Some("deprecated: gone")) + ); + + // Forge's own changes: a dist-tag pointed elsewhere and a deprecation + // lifted, in Forge. A run that skips conflicts keeps them, and says so. + let repo_id = repository_id(&srv, &repo).await; + sqlx::query!( + "UPDATE package_tag SET version = '1.0.0' WHERE repository_id = $1 AND package = 'alpha' AND tag = 'next'", + repo_id + ) + .execute(&srv.pool) + .await + .unwrap(); + sqlx::query!( + "UPDATE component SET deprecation = NULL WHERE repository_id = $1 AND name = 'alpha' AND version = '1.0.0'", + repo_id + ) + .execute(&srv.pool) + .await + .unwrap(); + let skipped = start(&srv, &repo, from(source.npm_url(reg))).await; + let done = run(&srv, &repo, skipped).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["conflict"], 2, "{done}"); + let tag = item_of(&srv, &repo, skipped, "alpha", "next").await; + assert_eq!( + tag["detail"], + "Forge points it at 1.0.0; the source at 2.0.0-beta.1" + ); + let version = item_of(&srv, &repo, skipped, "alpha", "1.0.0").await; + assert_eq!(version["state"], "conflict"); + assert!( + version["detail"] + .as_str() + .unwrap() + .contains("Forge no longer does"), + "{version}" + ); + let doc = packument(&srv, &repo, "alpha").await; + assert_eq!(doc["dist-tags"]["next"], "1.0.0"); + assert!(doc["versions"]["1.0.0"].get("deprecated").is_none()); + + // With overwrite, the source's. + let mut overwrite = from(source.npm_url(reg)); + overwrite["on_conflict"] = json!("overwrite"); + let overwritten = start(&srv, &repo, overwrite).await; + let done = run(&srv, &repo, overwritten).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["conflict"], 0, "{done}"); + assert_eq!(done["progress"]["replaced"], 1, "{done}"); + let doc = packument(&srv, &repo, "alpha").await; + assert_eq!(doc["dist-tags"]["next"], "2.0.0-beta.1"); + assert_eq!(doc["versions"]["1.0.0"]["deprecated"], "use 1.1.0"); + assert_eq!(source.npm_downloads(reg, "alpha", "1.0.0"), 1); +} + +#[tokio::test] +async fn npm_import_walks_all_where_the_search_lists_nothing_and_checks_every_tarball() { + let srv = forge().await; + let source = Source::start().await; + // Nexus: a search without a text finds nothing, `/-/all` lists all. + let reg = "npm-nexus"; + source.npm(reg, |r| { + r.search = NpmSearch::Nothing; + r.all = true; + }); + for (name, version) in [ + ("@acme/core", "0.1.0"), + ("@acme/ui", "1.0.0"), + ("zeta", "1.0.0"), + ] { + source.npm_version( + reg, + name, + version, + tgz(name, version, name), + "2022-02-02T00:00:00.000Z", + ); + source.npm_tag(reg, name, "latest", version); + } + let legacy = tgz("legacy", "0.1.0", "old"); + source.npm_version( + reg, + "legacy", + "0.1.0", + legacy.clone(), + "2015-01-01T00:00:00.000Z", + ); + source.npm(reg, |r| { + r.packages + .get_mut("legacy") + .unwrap() + .get_mut("0.1.0") + .unwrap() + .integrity = false + }); + // Two tarballs the registry serves corrupt: one listed with its + // integrity, one with its shasum alone. + source.with(|s| { + s.corrupt.insert( + format!("npm/{reg}/zeta/1.0.0"), + tgz("zeta", "1.0.0", "tampered"), + ); + s.corrupt.insert( + format!("npm/{reg}/legacy/0.1.0"), + tgz("legacy", "0.1.0", "tampered"), + ); + }); + + let repo = repo_name("npm-all"); + create_repo( + &srv, + json!({"name": repo, "kind": "hosted", "format": "npm"}), + ) + .await; + let id = start(&srv, &repo, from(source.npm_url(reg))).await; + let queued = get_import(&srv, &repo, id).await; + assert_eq!(queued["listing"], "all"); + assert_eq!(queued["source_total"], 4); + let done = run(&srv, &repo, id).await; + assert_eq!(done["state"], "completed", "{done}"); + // Four versions and three tags; zeta's version, and with it its tag, + // and legacy's version failed. + assert_eq!(done["progress"]["found"], 7, "{done}"); + assert_eq!(done["progress"]["imported"], 4, "{done}"); + assert_eq!(done["progress"]["failed"], 3, "{done}"); + let failed = items(&srv, &repo, id, Some("failed")).await; + let detail = |name: &str, version: &str| { + failed + .iter() + .find(|i| i["name"] == name && i["version"] == version) + .unwrap_or_else(|| panic!("{failed:?}"))["detail"] + .as_str() + .unwrap() + .to_string() + }; + assert!( + detail("zeta", "1.0.0").contains("does not match the SHA-512 the registry lists"), + "{failed:?}" + ); + assert!( + detail("legacy", "0.1.0").contains("dist.shasum"), + "{failed:?}" + ); + assert_eq!(detail("zeta", "latest"), "version 1.0.0 is not in Forge"); + assert!(npm_tarball(&srv, &repo, "zeta", "1.0.0").await.is_none()); + assert!(npm_tarball(&srv, &repo, "@acme/core", "0.1.0") + .await + .is_some()); + + // Retried once the registry serves them right: the versions, then the tag. + source.with(|s| s.corrupt.clear()); + let resp = admin( + &srv, + reqwest::Method::POST, + &format!("/admin/repositories/{repo}/imports/{id}/resume"), + Some(json!({"retry_failed": true, "password": PASSWORD})), + ) + .await; + assert_eq!(resp.status(), 200); + let done = run(&srv, &repo, id).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["failed"], 0, "{done}"); + assert_eq!(done["progress"]["imported"], 7, "{done}"); + assert_eq!( + npm_tarball(&srv, &repo, "legacy", "0.1.0").await.unwrap(), + legacy + ); + assert_eq!( + packument(&srv, &repo, "zeta").await["dist-tags"]["latest"], + "1.0.0" + ); +} + +#[tokio::test] +async fn npm_preflight_names_scopes_conflicts_and_the_dry_run() { + let srv = forge().await; + let source = Source::start().await; + // Verdaccio: a search needs a text, and there is no `/-/all`. + let reg = "npm-verdaccio"; + source.npm(reg, |r| r.search = NpmSearch::Refused); + for (name, version) in [ + ("@acme/core", "0.1.0"), + ("@acme/ui", "1.0.0"), + ("@other/x", "1.0.0"), + ("alpha", "1.0.0"), + ] { + source.npm_version( + reg, + name, + version, + tgz(name, version, name), + "2022-02-02T00:00:00.000Z", + ); + source.npm_tag(reg, name, "latest", version); + } + let repo = repo_name("npm-named"); + create_repo( + &srv, + json!({"name": repo, "kind": "hosted", "format": "npm"}), + ) + .await; + let attempt = |body: Value| { + let srv = &srv; + let repo = repo.clone(); + async move { + let resp = admin( + srv, + reqwest::Method::POST, + &format!("/admin/repositories/{repo}/imports"), + Some(body), + ) + .await; + (resp.status().as_u16(), resp.text().await.unwrap()) + } + }; + let (status, text) = attempt(from(source.npm_url(reg))).await; + assert_eq!(status, 400); + assert!(text.contains("lists no packages"), "{text}"); + let mut wrong = from(source.npm_url(reg)); + wrong["password"] = json!("wrong"); + wrong["names"] = json!(["alpha"]); + let (status, text) = attempt(wrong).await; + assert_eq!(status, 400); + assert!(text.contains("refused the credentials"), "{text}"); + let mut invalid = from(source.npm_url(reg)); + invalid["names"] = json!(["alpha", "../admin"]); + let (status, text) = attempt(invalid).await; + assert_eq!(status, 400); + assert!(text.contains("../admin"), "{text}"); + + // Named packages and scopes; a scope's packages come from the search + // for it. A dry run downloads nothing. + let mut named = from(source.npm_url(reg)); + named["names"] = json!(["@acme", "alpha", "missing", "@nobody/*"]); + named["dry_run"] = json!(true); + let id = start(&srv, &repo, named.clone()).await; + let queued = get_import(&srv, &repo, id).await; + assert_eq!(queued["listing"], "names"); + assert_eq!( + queued["names"], + json!(["@acme", "alpha", "missing", "@nobody"]) + ); + let done = run(&srv, &repo, id).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["planned"], 6, "{done}"); + assert_eq!(done["progress"]["failed"], 2, "{done}"); + let failed = items(&srv, &repo, id, Some("failed")).await; + let details: BTreeMap = failed + .iter() + .map(|i| { + ( + i["name"].as_str().unwrap().to_string(), + i["detail"].as_str().unwrap().to_string(), + ) + }) + .collect(); + assert_eq!( + details, + BTreeMap::from([ + ( + "@nobody".to_string(), + "the source has no package of this scope".to_string() + ), + ( + "missing".to_string(), + "the source has no package of this name".to_string() + ), + ]) + ); + assert_eq!(source.npm_downloads(reg, "alpha", "1.0.0"), 0); + assert!(npm_tarball(&srv, &repo, "alpha", "1.0.0").await.is_none()); + + named["dry_run"] = json!(false); + let id = start(&srv, &repo, named.clone()).await; + let done = run(&srv, &repo, id).await; + assert_eq!(done["progress"]["imported"], 6, "{done}"); + assert!(npm_tarball(&srv, &repo, "@acme/ui", "1.0.0") + .await + .is_some()); + assert_eq!(npm_get(&srv, &repo, "@other%2fx").await.status(), 404); + + // The source's alpha 1.0.0 is now another tarball: a dry run tells + // from the checksums alone, a run skips and reports it. + let other = tgz("alpha", "1.0.0", "republished"); + source.npm_version( + reg, + "alpha", + "1.0.0", + other.clone(), + "2022-02-02T00:00:00.000Z", + ); + named["names"] = json!(["alpha"]); + named["dry_run"] = json!(true); + let id = start(&srv, &repo, named.clone()).await; + let done = run(&srv, &repo, id).await; + assert_eq!(done["progress"]["conflict"], 1, "{done}"); + assert_eq!(done["progress"]["identical"], 1, "the tag: {done}"); + assert_eq!(source.npm_downloads(reg, "alpha", "1.0.0"), 1); + named["dry_run"] = json!(false); + let id = start(&srv, &repo, named.clone()).await; + let done = run(&srv, &repo, id).await; + assert_eq!(done["progress"]["conflict"], 1, "{done}"); + assert_eq!( + item_of(&srv, &repo, id, "alpha", "1.0.0").await["detail"], + "Forge has other bytes for this version" + ); + assert_ne!( + npm_tarball(&srv, &repo, "alpha", "1.0.0").await.unwrap(), + other + ); + named["on_conflict"] = json!("overwrite"); + let id = start(&srv, &repo, named).await; + let done = run(&srv, &repo, id).await; + assert_eq!(done["progress"]["replaced"], 1, "{done}"); + assert_eq!( + npm_tarball(&srv, &repo, "alpha", "1.0.0").await.unwrap(), + other + ); +} + +/// The previous request form names a Nexus server and its npm repository. +#[tokio::test] +async fn npm_import_names_a_nexus_repository_in_the_previous_request_form() { + let srv = forge().await; + let source = Source::start().await; + source.npm("npm-hosted", |r| { + r.search = NpmSearch::Nothing; + r.all = true; + }); + source.npm_version( + "npm-hosted", + "alpha", + "1.0.0", + tgz("alpha", "1.0.0", "a"), + "2022-02-02T00:00:00.000Z", + ); + // A packument whose tarball is another version: refused, whatever its + // checksums say. + source.npm_version( + "npm-hosted", + "beta", + "1.0.0", + tgz("beta", "9.9.9", "b"), + "2022-02-02T00:00:00.000Z", + ); + let repo = repo_name("npm-nexus-form"); + create_repo( + &srv, + json!({"name": repo, "kind": "hosted", "format": "npm"}), + ) + .await; + let mut body = from(source.url.clone()); + body["repository"] = json!("npm-hosted"); + let id = start(&srv, &repo, body).await; + let queued = get_import(&srv, &repo, id).await; + assert_eq!(queued["url"], source.npm_url("npm-hosted")); + let done = run(&srv, &repo, id).await; + assert_eq!(done["state"], "completed", "{done}"); + assert_eq!(done["progress"]["imported"], 1, "{done}"); + assert_eq!(done["progress"]["failed"], 1, "{done}"); + let refused = item_of(&srv, &repo, id, "beta", "1.0.0").await; + assert!( + refused["detail"] + .as_str() + .unwrap() + .contains("the package is beta 9.9.9, not beta 1.0.0"), + "{refused}" + ); + assert_eq!(npm_get(&srv, &repo, "beta").await.status(), 404); +} diff --git a/docs/README.md b/docs/README.md index 3b067bf3..da452b4b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,8 +22,8 @@ its [quickstart](../README.md#quickstart); - **[Retention size budgets](retention.md)**: keep a repository under a logical size: the budget, how size is counted, and what goes first. - **[Importing repositories](import.md)**: copy a hosted repository of - another server (a NuGet feed, a registry: Nexus, Artifactory, another - Forge) into Forge, with its publish dates. + another server (a NuGet feed, an npm registry, a registry: Nexus, + Artifactory, Verdaccio, another Forge) into Forge, with its publish dates. - **[Importing from Nexus](nexus-import.md)**: where Nexus serves its feeds and registries, its search's limits, and imports stored before 2.2. diff --git a/docs/changes/2.2.0/643.feature.md b/docs/changes/2.2.0/643.feature.md new file mode 100644 index 00000000..784f2764 --- /dev/null +++ b/docs/changes/2.2.0/643.feature.md @@ -0,0 +1,12 @@ +An import copies an npm registry's packages into a hosted npm repository: +Nexus' `/repository/`, Verdaccio, Artifactory or another Forge. The +packages come from the registry's search, or from its `/-/all` where the +search lists nothing (Nexus, Verdaccio), or from the packages and scopes +(`@acme`) the import names. Every version's tarball is downloaded from the +import's URL, whatever host `dist.tarball` names, and checked against +`dist.integrity` (or `dist.shasum` alone); its publish time comes from the +packument's `time`, and its deprecation comes along. The dist-tags are set +once their versions are in Forge. A second run downloads only what is new +and moves the dist-tags the source moved; a dist-tag or deprecation changed +in Forge follows `on_conflict`. The web UI offers the import on npm hosted +repositories. diff --git a/docs/import.md b/docs/import.md index ec7e7ec4..6da7ecae 100644 --- a/docs/import.md +++ b/docs/import.md @@ -9,17 +9,17 @@ content. An import reads the source through the format's own protocol, the one clients speak, so the source can be any server that speaks it: Nexus, -Artifactory, ProGet, BaGet, Harbor, a `distribution` registry or another -Forge. [Importing from Nexus](nexus-import.md) says where Nexus serves them, -and what to know of its search. +Artifactory, ProGet, BaGet, Verdaccio, Harbor, a `distribution` registry or +another Forge. [Importing from Nexus](nexus-import.md) says where Nexus +serves them, and what to know of its searches. | Format | The source | What is copied | | --- | --- | --- | | NuGet | A NuGet v3 feed: its service index, `…/index.json` | the `.nupkg` of every version, unlisted ones included | +| npm | An npm registry: the URL clients use as `--registry` | the tarball of every version, prereleases included, with its publish time and deprecation; the dist-tags | | Docker | A registry API | every tag with everything it references: config and layers, the child manifests of a multi-platform index; signatures and SBOMs (cosign's `sha256-.sig` tags, the referrers the registry's referrers API lists, and those an OCI 1.1 client stored under the referrers tag schema's `sha256-` tag) | -npm ([#643](https://github.com/forgepkg/forge/issues/643)) and apt -([#644](https://github.com/forgepkg/forge/issues/644)) follow. Proxy +apt ([#644](https://github.com/forgepkg/forge/issues/644)) follows. Proxy repositories are not imported (they refill from their upstream), nor are groups (recreate them as configuration), users, roles and privileges (credential forwarding covers sign-in) or cleanup policies (see retention). @@ -43,10 +43,10 @@ curl -u admin@example.com: -X POST \ | Field | | | --- | --- | -| `url` | The source: a NuGet feed's service index (`…/index.json`), or a registry (`https://registry.example.com`, or a path below which a server serves the registry API, as Nexus' `…/repository/`). | -| `names` | The package ids or images to copy (up to 10,000), instead of every package the feed's search finds or every image the registry's catalog lists. | +| `url` | The source: a NuGet feed's service index (`…/index.json`), an npm registry (`https://nexus.example.com/repository/npm-hosted`, `https://verdaccio.example.com`), or a registry (`https://registry.example.com`, or a path below which a server serves the registry API, as Nexus' `…/repository/`). | +| `names` | The package ids or images to copy (up to 10,000), instead of every package the feed's search finds or every image the registry's catalog lists. npm: package names (`left-pad`, `@acme/ui`) or scopes (`@acme`, every package of the scope). | | `username`, `password` | A user of the source that may read the repository; both, or neither for anonymous access. | -| `on_conflict` | A version Forge already has with other bytes: `skip` it and report it (default), `overwrite` it with the source's bytes, or `fail` the import. | +| `on_conflict` | A version Forge already has with other bytes: `skip` it and report it (default), `overwrite` it with the source's bytes, or `fail` the import. npm: likewise a dist-tag Forge points at another version, or a deprecation Forge gives otherwise ([below](#npm-dist-tags-and-deprecations)). | | `concurrency` | Items copied at a time, 1–16 (default 4), so the source is not overwhelmed. | | `dry_run` | Decide every item and write nothing. | @@ -59,10 +59,21 @@ Before an import is accepted, Forge checks the source (the **preflight**): - NuGet: the service index resolves, and its search answers. The import records how many packages the search reports (`source_total`), and the `Server` header the feed answered with (`source_server`). +- npm: the registry answers (`GET /-/ping`; one that does not serve it is let + through, but not one that refuses the credentials), and lists its + packages. npm's protocol has no listing of every package, so the preflight + tries, in turn, the registry's search with an empty text + (`/-/v1/search?text=`; another Forge lists every package so) and its legacy + `/-/all` (Nexus' hosted repositories and Verdaccio serve it; Nexus' search + finds nothing without a text, and Verdaccio refuses one). The import walks + the first that lists packages (`listing`: `search` or `all`), and records + how many it lists (`source_total`). A registry that serves neither is + refused: name the packages or scopes to import. - Docker: the registry answers its API (`GET /v2/`, with its token challenge answered), and serves a catalog (`/v2/_catalog`). -An import that names its packages or images needs no search or catalog. A +An import that names its packages, scopes or images needs no search or +catalog. A failed check answers 400 (502 when the source cannot be reached or fails) with what is wrong, and nothing is stored. A repository runs one import at a time. @@ -74,21 +85,26 @@ nuget.org, an import would walk nuget.org's search. The preflight's A **dry run** reports what would be copied and the conflicts with versions Forge already has, without writing anything. It downloads no package: a -version Forge has is compared through the SHA-512 the feed lists, and where -the feed lists none it is planned with a note saying so. +version Forge has is compared through the SHA-512 the feed or registry +lists, and where it lists none it is planned with a note saying so. ## What an import does 1. **Scan.** The source's listing is walked page by page: a feed's search - (with an empty query, prerelease and SemVer 2.0.0 versions included), a - registry's catalog, or the names the import gives. For each package id, - the feed's package base address lists its versions, unlisted ones - included; for each image, the registry lists its tags. Each page's + (with an empty query, prerelease and SemVer 2.0.0 versions included), an + npm registry's search or `/-/all`, a registry's catalog, or the names the + import gives. For each package id, the feed's package base address lists + its versions, unlisted ones included; for each npm package, its full + packument lists its versions and dist-tags (a named scope's packages come + from `/-/all`, or else from the search for the scope); for each image, + the registry lists its tags. Each page's versions or tags are recorded as the import's items, and its position as the import's cursor, in one transaction: an import resumes after a restart, a failure or a cancel at the page it had not recorded. A named package or image the source does not have is an item that fails, saying - so. + so; so is an npm package the registry's search or `/-/all` lists but + whose packument it does not serve (Nexus' search names a scoped package + without its scope). 2. **Copy.** The items are copied, `concurrency` at a time: - A package is streamed from the feed's package base address into staging, checked against the SHA-512 the feed's registration lists @@ -99,6 +115,15 @@ the feed lists none it is planned with a note saying so. version Forge has with the SHA-512 the feed lists is skipped without downloading it; where the feed lists none, it is downloaded and compared, and comes out `identical`. + - An npm tarball is downloaded from the import's URL + (`//-/`), not from where the packument's + `dist.tarball` points: that names the host the registry was configured + with (Nexus' base URL), which need not be one Forge reaches it on. It is + checked against `dist.integrity` (its SHA-512), or, for an old package + listed without one, against `dist.shasum`, and must be the package and + version the packument names. Then it is written through the same path + as a publish. A version Forge has with the SHA-512 the registry lists is + skipped without downloading it. - A Docker tag is copied like `crane copy`: digests are kept, and the manifests go through the same checks as a `docker push` (every blob and child present, the repository's immutable tags). A layer the repository @@ -114,7 +139,10 @@ the feed lists none it is planned with a note saying so. - A NuGet version's `published` time in the feed's registration becomes its creation time, which retention's `older_than_days` reads. nuget.org's `1900-01-01` for an unlisted version is no publish time: - such a version is created at its import. + such a version is created at its import. An npm version's publish time + comes from the packument's `time`. (A full packument larger than + `proxy.max_metadata_bytes` is read abbreviated, without `time`: its + versions are created at their import.) - No protocol carries a last download: an imported version counts as downloaded at its import, so `not_downloaded_for_days` counts from there. Download counts are not carried either, so @@ -126,6 +154,23 @@ the feed lists none it is planned with a note saying so. Every imported version is audited as an import: a `forge::audit` log line with `action = "import"`, the administrator and the source. +### npm: dist-tags and deprecations + +A version's `deprecated` message comes along with it. The dist-tags +(`latest`, `next` and the rest) are items of their own (`tag`, the package +and the tag), set after the versions, once the version a tag names is in +Forge: a tag whose version failed fails too, saying so. + +A dist-tag Forge already points at another version, or a deprecation Forge +gives otherwise (another message, or none where the source deprecates the +version, or one where it does not), follows `on_conflict`: `skip` keeps +Forge's and reports a conflict, `overwrite` takes the source's, `fail` fails +the import. Except where what Forge holds is what an earlier import of the +repository set: then it follows the source. So a final import before a +cutover moves `latest` to the version the source published since, and takes +a deprecation the source added, without `overwrite`; a tag or deprecation +changed in Forge since stays, and is reported. + ## Progress, cancel, resume, retry `GET /admin/repositories/{name}/imports` (and the web UI, which refreshes @@ -148,7 +193,8 @@ away, the credentials stopped working), on a conflict with one. Running an import again later copies only what is new at the source: -everything Forge already has is `identical`. A cutover is therefore an +everything Forge already has is `identical` (npm: a dist-tag the source moved +since moves along, as above). A cutover is therefore an import ahead of time, then a final one just before DNS or the clients switch to Forge. diff --git a/docs/nexus-import.md b/docs/nexus-import.md index c7266150..b9fe306c 100644 --- a/docs/nexus-import.md +++ b/docs/nexus-import.md @@ -6,6 +6,7 @@ serves for its hosted repositories: - **NuGet:** the feed `/repository//index.json` (Nexus 3.24 and later). +- **npm:** the registry `/repository/`, the URL npm clients use. - **Docker:** the repository's connector (`http://nexus.example.com:8082`, its HTTP port), or its path route `/repository/`, which serves reads on Nexus' own port. @@ -23,10 +24,19 @@ to as many packages as it walked. Nexus lists no SHA-512 in its registration: a version is checked against nothing but the `.nupkg`'s own format, and a second run downloads every version Forge has to compare it. +Nexus' npm search (3.70) finds nothing without a text, and names a scoped +package without its scope (`ui` for `@acme/ui`), so an npm import of a +Nexus hosted repository walks its `/-/all`, which lists every package by its +full name. Nexus' npm packuments list each version's `dist.integrity`: a +second run downloads nothing Forge has. Their `dist.tarball` names the base +URL Nexus is configured with (**Administration → System → Base URL**), where +one is set; the import downloads from its own URL instead, so a Nexus Forge +reaches under another name imports all the same. + The request of Forge before 2.2, a Nexus server and repository, is still accepted: `url` the Nexus server, `repository` the Nexus repository, and for Docker `registry_url` its connector (default: its path route). The import -reads that repository's feed or registry. +reads that repository's feed or registry (npm: `/repository/`). An import stored by Forge before 2.2 read Nexus' REST API. It still lists, with the feed or registry it now reads. Resumed, its scan starts again (its diff --git a/docs/npm.md b/docs/npm.md index 16736e72..ce7d496c 100644 --- a/docs/npm.md +++ b/docs/npm.md @@ -26,6 +26,7 @@ retention and the web UI are the same as for every other format. - [Installing and searching](#installing-and-searching) - [Proxy repositories](#proxy-repositories) - [Group repositories](#group-repositories) +- [Importing another registry](#importing-another-registry) - [Audits](#audits) - [Permissions](#permissions) - [Retention](#retention) @@ -268,6 +269,26 @@ A group serves its members, by priority: takes publishes, dist-tag changes, deprecations and unpublishing through the group; without one they are refused. +## Importing another registry + +An [import](import.md) copies the packages of another registry's hosted +repository (Nexus, Verdaccio, Artifactory, another Forge) into a hosted npm +repository: every version's tarball, its publish time and deprecation, and +the dist-tags. In the web UI, open the hosted repository under **Admin → +Repositories** and choose **Import**, or: + +```bash +api https://forge.example.com/admin/repositories/npm-hosted/imports \ + -d '{"url": "https://nexus.example.com/repository/npm-hosted", + "username": "migrator", "password": "…", "dry_run": true}' +``` + +The registry's search, or else its `/-/all`, names the packages; `names` +lists packages or scopes (`"names": ["@acme", "left-pad"]`) for a registry +that serves neither. Tarballs come from the import's URL and are checked +against `dist.integrity`; a second run copies only what is new, and moves the +dist-tags the source moved. See [Importing repositories](import.md#npm-dist-tags-and-deprecations). + ## Audits `npm audit` (and `pnpm audit`) asks the registry for advisories. A proxy diff --git a/migrations/0009_npm_import.sql b/migrations/0009_npm_import.sql new file mode 100644 index 00000000..87b6626a --- /dev/null +++ b/migrations/0009_npm_import.sql @@ -0,0 +1,22 @@ +-- --------------------------------------------------------------------------- +-- Imports of npm registries (#643). 0008 is the device authorization grant +-- (#649); the two touch different tables and apply in either order. +-- +-- An import copies an npm registry's packages into a hosted npm repository. +-- npm's protocol has no listing of every package: an import walks the +-- registry's search (`/-/v1/search`), or, where that lists nothing, its +-- legacy listing (`/-/all`), or the names and scopes it gives. +-- +-- * `scan_listing` takes `all`: the scan walks `/-/all`, its cursor the last +-- package name it recorded. +-- * An item may be a `tag`: an npm dist-tag (`latest`, `next`), its `name` +-- the package and its `version` the tag. It is set once the versions are +-- copied, after them. +-- --------------------------------------------------------------------------- +ALTER TABLE repository_import DROP CONSTRAINT repository_import_scan_listing_check; +ALTER TABLE repository_import ADD CONSTRAINT repository_import_scan_listing_check + CHECK (scan_listing IN ('search', 'catalog', 'all', 'names')); + +ALTER TABLE repository_import_item DROP CONSTRAINT repository_import_item_kind_check; +ALTER TABLE repository_import_item ADD CONSTRAINT repository_import_item_kind_check + CHECK (kind IN ('component', 'manifest', 'tag')); diff --git a/web/src/api/imports.ts b/web/src/api/imports.ts index cf502e44..9e7aa308 100644 --- a/web/src/api/imports.ts +++ b/web/src/api/imports.ts @@ -2,14 +2,15 @@ import { httpClient } from '~/lib/httpClient' /** * Imports into a hosted repository from another server, read through the - * format's own protocol: a NuGet feed or a registry + * format's own protocol: a NuGet feed, an npm registry or a registry API * (`crates/server/src/admin/imports.rs`). * * - `POST /admin/repositories/{name}/imports` checks the source first (the - * preflight: a feed's service index resolves and its search answers, or a + * preflight: a feed's service index resolves and its search answers, an + * npm registry lists its packages through its search or `/-/all`, or a * registry answers its API and serves a catalog; an import that names its - * packages or images needs neither) and answers 400/502 with the reason, - * or 409 while the repository has an import running. + * packages, scopes or images needs none of it) and answers 400/502 with + * the reason, or 409 while the repository has an import running. * - An import scans the source (`phase: scan`), then copies the items it * found (`phase: copy`). A dry run decides every item without writing a * package. @@ -18,7 +19,7 @@ import { httpClient } from '~/lib/httpClient' */ /** Formats an import copies (`crates/server/src/import/mod.rs`). */ -export const IMPORT_FORMATS = ['nuget', 'docker'] +export const IMPORT_FORMATS = ['nuget', 'npm', 'docker'] /** `queued` → `running` → `completed` | `failed` | `cancelled`. */ export type ImportState = @@ -38,9 +39,15 @@ export type ItemState = | 'failed' | 'planned' -/** A version Forge holds with other bytes. */ +/** A version Forge holds with other bytes (npm: or a dist-tag pointed elsewhere). */ export type OnConflict = 'skip' | 'overwrite' | 'fail' +/** + * What the scan walks: a feed's or an npm registry's `search`, a registry's + * `catalog`, an npm registry's legacy `/-/all`, or the `names` the import gives. + */ +export type ImportListing = 'search' | 'catalog' | 'all' | 'names' + export interface ImportProgress { /** Items the scan found so far. */ found: number @@ -74,8 +81,9 @@ export interface ImportView { cancel_requested: boolean /** The `Server` header the source answered the preflight with. */ source_server: string | null - /** How many packages the feed's search reported in the preflight. */ + /** How many packages the feed's search reported in the preflight (npm: the registry's search, or `/-/all`). */ source_total: number | null + listing: ImportListing last_error: string | null created_by: number | null created_at: string @@ -87,8 +95,11 @@ export interface ImportView { export interface ImportItem { id: number - /** `component` (a version, a tag), or a Docker `manifest` held by digest (a referrer). */ - kind: 'component' | 'manifest' + /** + * `component` (a version, a Docker tag), a Docker `manifest` held by digest + * (a referrer), or an npm dist-`tag` (`name` the package, `version` the tag). + */ + kind: 'component' | 'manifest' | 'tag' name: string version: string state: ItemState @@ -104,9 +115,12 @@ export interface ImportItem { /** `POST /admin/repositories/{name}/imports` (unknown fields are refused). */ export interface CreateImport { - /** A feed's service index (`…/index.json`), or a registry. */ + /** A feed's service index (`…/index.json`), an npm registry, or a registry. */ url: string - /** The package ids or images to copy, instead of every one the source lists. */ + /** + * The package ids or images to copy, instead of every one the source lists + * (npm: package names, or scopes such as `@acme`). + */ names?: string[] username?: string password?: string diff --git a/web/src/routes/admin/repositories/Import.tsx b/web/src/routes/admin/repositories/Import.tsx index e8cafa25..fa348d94 100644 --- a/web/src/routes/admin/repositories/Import.tsx +++ b/web/src/routes/admin/repositories/Import.tsx @@ -5,6 +5,7 @@ import { IMPORT_FORMATS, isActive, type CreateImport, + type ImportItem, type ImportState, type ImportView, type ItemState, @@ -72,9 +73,10 @@ const ITEM_TONES: Record = { /** * Import a repository from another server into this hosted repository (the - * admin API under `/admin/repositories/{name}/imports`): a NuGet feed or a - * registry, read through the format's own protocol, so Nexus, Artifactory, - * ProGet, BaGet, Harbor, a `distribution` registry or another Forge alike. + * admin API under `/admin/repositories/{name}/imports`): a NuGet feed, an npm + * registry or a registry API, read through the format's own protocol, so + * Nexus, Artifactory, ProGet, BaGet, Verdaccio, Harbor, a `distribution` + * registry or another Forge alike. * The form starts an import (or a dry run); the server checks the source * before it accepts one and says what is wrong. Below, the repository's imports with their progress, * re-read every two seconds while one runs: cancel, resume, retry the failed @@ -124,12 +126,12 @@ function ImportRepository() { crumbs={crumbs} kind={repo.data.kind} format={repo.data.format} - description="Copy the packages of a NuGet feed, or the images of a registry, into this repository, read through the format's own protocol: from Nexus, Artifactory, ProGet, BaGet, Harbor, a distribution registry or another Forge. Versions Forge already has with the same bytes are skipped, so running an import again copies only what is new: import once, then again just before clients switch to Forge." + description="Copy the packages of a NuGet feed or an npm registry, or the images of a registry, into this repository, read through the format's own protocol: from Nexus, Artifactory, ProGet, BaGet, Verdaccio, Harbor, a distribution registry or another Forge. Versions Forge already has with the same bytes are skipped, so running an import again copies only what is new: import once, then again just before clients switch to Forge." /> {!supported ? ( - Imports go to a hosted NuGet or Docker repository; this is a{' '} + Imports go to a hosted NuGet, npm or Docker repository; this is a{' '} {repo.data.format} {repo.data.kind} repository. ) : active ? ( @@ -140,7 +142,7 @@ function ImportRepository() { ) : null} @@ -162,7 +164,7 @@ function ImportRepository() { all.map((i, n) => ( {n > 0 &&
} - + )) )} @@ -177,13 +179,15 @@ function ImportRepository() { */ function StartImportForm({ repo, - docker, + format, previous, }: { repo: string - docker: boolean + format: string previous?: ImportView }) { + const docker = format === 'docker' + const npm = format === 'npm' const create = useCreateImport(repo) const [url, setUrl] = useState(previous?.url ?? '') const [names, setNames] = useState((previous?.names ?? []).join('\n')) @@ -205,7 +209,7 @@ function StartImportForm({ setFormError(null) if (!url.trim()) { setFormError( - docker + docker || npm ? 'Enter the URL of the registry to import from.' : 'Enter the URL of the feed to import from.', ) @@ -259,17 +263,28 @@ function StartImportForm({ >
setUrl(e.target.value)} placeholder={ docker ? 'https://registry.example.com' - : 'https://nexus.example.com/repository/nuget-hosted/index.json' + : npm + ? 'https://nexus.example.com/repository/npm-hosted' + : 'https://nexus.example.com/repository/nuget-hosted/index.json' } description={ - docker ? ( + npm ? ( + <> + The npm registry, as clients use it with{' '} + --registry. Nexus:{' '} + <Nexus URL>/repository/<name>. Tarballs are + downloaded from this URL, whatever host the registry’s metadata + names. Point it at a hosted repository: a group or a proxy lists its + upstream’s packages too. + + ) : docker ? ( <> The registry API. Nexus: the repository’s connector, e.g.{' '} http://nexus.example.com:8082, or its path route,{' '} @@ -286,7 +301,7 @@ function StartImportForm({ } />
@@ -318,7 +335,11 @@ function StartImportForm({
setOnConflict(e.target.value as OnConflict)} > @@ -361,7 +382,15 @@ function parseNames(text: string): string[] { } /** One import of the repository: its state, progress, counts and actions. */ -function ImportCard({ repo, imp }: { repo: string; imp: ImportView }) { +function ImportCard({ + repo, + format, + imp, +}: { + repo: string + format: string + imp: ImportView +}) { const cancel = useCancelImport(repo) const resume = useResumeImport(repo) const remove = useDeleteImport(repo) @@ -413,8 +442,7 @@ function ImportCard({ repo, imp }: { repo: string; imp: ImportView }) { : imp.phase === 'scan' && active ? `Scanning the source: ${p.found} found so far.` : `${done} of ${p.found} done.`}{' '} - {imp.source_total !== null && - `The feed's search found ${imp.source_total} packages. `} + {imp.source_total !== null && `${sourceTotal(imp, format)} `} Started {formatDateTime(imp.started_at ?? imp.created_at)} {imp.finished_at && `, ended ${formatDateTime(imp.finished_at)}`}. {imp.source_server && ` ${imp.source_server}.`} @@ -534,6 +562,22 @@ function ImportCard({ repo, imp }: { repo: string; imp: ImportView }) { ) } +/** What the preflight (and the scan) found at the source. */ +function sourceTotal(imp: ImportView, format: string): string { + const n = imp.source_total + if (imp.listing === 'all') return `The registry's /-/all lists ${n} packages.` + return format === 'npm' + ? `The registry's search found ${n} packages.` + : `The feed's search found ${n} packages.` +} + +/** An item as the lists name it: a version, an image's tag, an npm dist-tag. */ +function itemLabel(item: ImportItem): string { + if (!item.version) return item.name + if (item.kind === 'tag') return `${item.name} dist-tag ${item.version}` + return `${item.name} ${item.version}` +} + /** The items of an import in one state (its conflicts, its failures). */ function ItemList({ repo, @@ -562,9 +606,7 @@ function ItemList({ - - {item.version ? `${item.name} ${item.version}` : item.name} - + {itemLabel(item)} diff --git a/web/src/routes/admin/repositories/imports.test.tsx b/web/src/routes/admin/repositories/imports.test.tsx index 8c44865f..ff5c989c 100644 --- a/web/src/routes/admin/repositories/imports.test.tsx +++ b/web/src/routes/admin/repositories/imports.test.tsx @@ -24,7 +24,9 @@ import { searchOptions } from '~/lib/search' * the package ids it names, and shows the server's preflight refusal; * - an import shows its progress, its conflicts on demand, and a completed * import's failed items are retried with the password it no longer holds; - * the form starts from the last import. + * the form starts from the last import; + * - an npm hosted repository offers the import of a registry, with named + * packages and scopes, and shows its dist-tag conflicts. */ const adminMe: Me = { @@ -75,6 +77,7 @@ function imp(over: Partial): ImportView { cancel_requested: false, source_server: 'Nexus/3.70.1-02 (OSS)', source_total: 10, + listing: 'search', last_error: null, created_by: 1, created_at: '2026-10-22T10:00:00Z', @@ -317,6 +320,81 @@ describe('import', () => { expect(requests('POST', '/imports/7/resume')).toHaveLength(1) }) + it('offers the import on an npm hosted repository, with packages and scopes', async () => { + let created: unknown + mockApi({ + repositories: [repo({ name: 'npm-internal', format: 'npm' })], + imports: [ + imp({ + repository: 'npm-internal', + url: 'https://nexus.example.com/repository/npm-hosted', + username: null, + listing: 'all', + source_total: 3, + progress: { + found: 5, + pending: 0, + imported: 4, + replaced: 0, + identical: 0, + conflict: 1, + failed: 0, + planned: 0, + planned_size: 0, + bytes_copied: 2048, + }, + }), + ], + items: [ + item({ + kind: 'tag', + name: '@acme/ui', + version: 'latest', + detail: 'Forge points it at 1.0.0; the source at 1.1.0', + }), + ], + onCreate: (body) => { + created = body + return json(imp({ state: 'queued' }), 201) + }, + }) + const router = await renderAt('/manage/repositories/npm-internal') + await userEvent.click(await screen.findByRole('link', { name: 'Import' })) + await waitFor(() => + expect(router.state.location.pathname).toBe( + '/manage/repositories/npm-internal/import', + ), + ) + const card = await screen.findByRole('article', { name: 'Import 7' }) + expect( + within(card).getByText(/\/-\/all lists 3 packages/, { exact: false }), + ).toBeInTheDocument() + await userEvent.click(within(card).getByRole('button', { name: 'Show conflict' })) + expect( + await within(card).findByText('@acme/ui dist-tag latest'), + ).toBeInTheDocument() + expect( + within(card).getByText('Forge points it at 1.0.0; the source at 1.1.0'), + ).toBeInTheDocument() + + const url = screen.getByLabelText('Registry URL') + expect(url).toHaveValue('https://nexus.example.com/repository/npm-hosted') + await userEvent.type( + screen.getByLabelText(/Packages or scopes/), + '@acme\nleft-pad', + ) + await userEvent.click(screen.getByRole('button', { name: 'Start import' })) + await waitFor(() => + expect(created).toEqual({ + url: 'https://nexus.example.com/repository/npm-hosted', + names: ['@acme', 'left-pad'], + on_conflict: 'skip', + concurrency: 4, + dry_run: false, + }), + ) + }) + it('imports the named images of a registry without a catalog', async () => { let created: unknown mockApi({