diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88d9b4a023..27b25b776a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1259,8 +1259,15 @@ jobs: # Generate the overrides from the fixtures, including automatic Ctrl-C # isolation, so this configuration cannot drift from the runner. VP_SNAP_NEXTEST_CONFIG="$RUNNER_TEMP/snapshot-nextest.toml" \ + VP_SNAP_PREWARM_MANIFEST="$RUNNER_TEMP/snapshot-prewarm.json" \ cargo-nextest nextest list --archive-file windows-snapshot-tests.tar.zst --workspace-remap . \ > "$RUNNER_TEMP/snapshot-test-list.txt" + # Derive Bun archives from the same fixtures as the test list, so + # version changes need no matching CI edit. Only response bytes are shared. + node packages/tools/src/local-npm-registry.ts \ + --snapshot-cache "$VP_SNAP_REGISTRY_CACHE_DIR" \ + --request-log "$VP_SNAP_ARTIFACTS_DIR/prewarm/registry.jsonl" \ + --prewarm-manifest "$RUNNER_TEMP/snapshot-prewarm.json" nextest_run_args=( --test-threads 8 --config-file "$RUNNER_TEMP/snapshot-nextest.toml" --archive-file windows-snapshot-tests.tar.zst --workspace-remap . --no-fail-fast diff --git a/crates/vp_cli_snapshots/tests/cli_snapshots/README.md b/crates/vp_cli_snapshots/tests/cli_snapshots/README.md index 00d7e1b2ab..d12b5cfcd4 100644 --- a/crates/vp_cli_snapshots/tests/cli_snapshots/README.md +++ b/crates/vp_cli_snapshots/tests/cli_snapshots/README.md @@ -97,6 +97,7 @@ Environment overrides, mainly for CI: | `VP_SNAP_PACKAGES_DIR` | Run-scoped directory for sharing packed packages across test processes | | `VP_SNAP_ARTIFACTS_DIR` | Directory for phase timings and failure diagnostics; unset disables artifacts | | `VP_SNAP_NEXTEST_CONFIG` | With `--list`, write nextest overrides for the discovered isolated cases | +| `VP_SNAP_PREWARM_MANIFEST` | With `--list`, write a JSON array of Bun archive URLs derived from runnable cases | Windows CI generates its nextest configuration from the same case definitions. Exact nextest runs read only the selected fixture; listing and native shard @@ -140,6 +141,27 @@ including across nextest processes when `VP_SNAP_REGISTRY_CACHE_DIR` is set. Use a fresh directory per run; this is not a persistent dependency cache. Package-manager caches and installed packages remain isolated per case. +Windows CI generates a prewarm manifest while listing cases, then passes it to +`local-npm-registry.ts --prewarm-manifest ` before starting test commands. +The runner collects exact Bun pins from each case's package ancestry +(`packageManager` and `devEngines.packageManager`), case/step environment overrides, +`vpt json-edit` of package-manager declarations, `vpt write-file` of package files, +and `vp env default/use/exec` arguments. Versions are deduplicated and archives +match the runner platform, including the x64 baseline variant. Changing these +fixture pins automatically updates preparation; CI contains no separate version list. + +Ignored and platform-excluded cases do not contribute pins. Set `prewarm-bun = false` +for fake installations and configuration-only cases. Script-generated pins must use +declarative setup or environment overrides to participate; discovery does not execute +scripts or scan assertions. Floating tags and version ranges retain normal registry +resolution and are not prewarmed. The manifest includes all runnable cases before +nextest partitions them, so each shard can reuse the same preparation logic. + +Preparation retries +failed downloads and permits up to 180 seconds per attempt. Tests reuse the +cached response with their existing deadlines and isolated installations. +Preparation requests are logged in `prewarm/registry.jsonl` under the artifact root. + `local-registry = true` additionally serves the packed checkout packages. Published versions remain available for cases that deliberately install an older release. Their upstream metadata is fetched once per run and merged with the @@ -216,6 +238,7 @@ requires = ["bash"] # "sh" | "bash" | "zsh" | "cmd" | "fish" | "nu" | ignore = false # true: only runs with `-- --ignored` serial = false # true: run alone across cargo test/nextest workers seed-runtime = true # false: start from an empty VP_HOME +prewarm-bun = true # false: fake install or configuration-only case link-node-modules = false # true: expose the run-root node_modules as # the workspace's parent-dir node_modules, # for `../node_modules/vite-plus/...` paths diff --git a/crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/command_env_package_manager_modes/snapshots.toml b/crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/command_env_package_manager_modes/snapshots.toml index 7babda2948..0b0eccaa6c 100644 --- a/crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/command_env_package_manager_modes/snapshots.toml +++ b/crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/command_env_package_manager_modes/snapshots.toml @@ -1,5 +1,6 @@ [[case]] name = "command_env_package_manager_modes" +prewarm-bun = false # Inspect configuration without downloading Bun. vp = "global" seed-runtime = false steps = [ diff --git a/crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/pm_audit/mock-manifest.json b/crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/pm_audit/mock-manifest.json new file mode 100644 index 0000000000..ef79f1d9ac --- /dev/null +++ b/crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/pm_audit/mock-manifest.json @@ -0,0 +1,3 @@ +{ + "-/npm/v1/security/advisories/bulk": {} +} diff --git a/crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/pm_audit/snapshots.toml b/crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/pm_audit/snapshots.toml index 57b2923fd9..a96deaddc0 100644 --- a/crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/pm_audit/snapshots.toml +++ b/crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/pm_audit/snapshots.toml @@ -4,6 +4,8 @@ name = "pm_audit_bun" skip-platforms = ["windows"] vp = "global" cwd = "bun" +# mock-manifest.json keeps audit responses local so these command-routing checks +# do not depend on the availability or contents of npm's advisory service. steps = [ { argv = ["vp", "install", "--", "--silent"], comment = "should install packages first", continue-on-failure = true }, { argv = ["vp", "pm", "audit", "--production"], comment = "should audit production dependencies" }, diff --git a/crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/pm_ci/snapshots.toml b/crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/pm_ci/snapshots.toml index 7c0ae8a5c3..daa9ac5776 100644 --- a/crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/pm_ci/snapshots.toml +++ b/crates/vp_cli_snapshots/tests/cli_snapshots/fixtures/pm_ci/snapshots.toml @@ -1,5 +1,6 @@ [[case]] name = "pm_ci_package_managers" +prewarm-bun = false # This case installs fake package managers. vp = "global" comment = """ Covers `vp pm ci` command delegation for each supported package manager. diff --git a/crates/vp_cli_snapshots/tests/cli_snapshots/main.rs b/crates/vp_cli_snapshots/tests/cli_snapshots/main.rs index f7530e178b..35a0a7b280 100644 --- a/crates/vp_cli_snapshots/tests/cli_snapshots/main.rs +++ b/crates/vp_cli_snapshots/tests/cli_snapshots/main.rs @@ -17,6 +17,7 @@ #[path = "../../src/bin/vpt/exit_code.rs"] mod exit_code; mod flavor; +mod prewarm; mod redact; mod registry_pack; mod report; @@ -389,6 +390,9 @@ struct Case { /// Keep registry request counters private for tests of metadata caching. #[serde(default, rename = "isolated-registry")] isolated_registry: bool, + /// Disable archive preparation for fake installations and configuration-only cases. + #[serde(default = "default_true", rename = "prewarm-bun")] + prewarm_bun: bool, /// Seed the case's `VP_HOME` with an already-provisioned managed JS /// runtime (see `flavor::js_runtime_seed_dir`). Default true; runtime /// provisioning tests set false to start from a genuinely empty home. @@ -1972,6 +1976,9 @@ fn main() { let mut tests: Vec = Vec::new(); let mut isolated_trials = BTreeSet::new(); + let prewarm_manifest = + args.list.then(|| std::env::var_os("VP_SNAP_PREWARM_MANIFEST")).flatten(); + let mut bun_prewarm = prewarm::BunPrewarm::default(); for fixture_path in fixture_paths { let fixture_path: Arc = Arc::from(fixture_path.as_path()); let fixture_name: Arc = Arc::from(fixture_path.file_name().unwrap().to_str().unwrap()); @@ -2007,6 +2014,24 @@ fn main() { let ignored = case.ignore || required_tool_missing || (case.local_registry && !local_build_present); + if prewarm_manifest.is_some() && !ignored && case.prewarm_bun { + bun_prewarm + .env(case.env.iter().map(|(key, value)| (key.as_str(), value.as_str()))); + bun_prewarm.workspace(&fixture_path, &case.cwd).unwrap_or_else(|error| { + panic!("failed to read prewarm inputs for {trial_name}: {error}") + }); + for step in case.steps.iter().chain(&case.after) { + bun_prewarm + .workspace(&fixture_path, step.cwd.as_deref().unwrap_or(&case.cwd)) + .unwrap_or_else(|error| { + panic!("failed to read prewarm inputs for {trial_name}: {error}") + }); + bun_prewarm.env( + step.envs.iter().map(|(key, value)| (key.as_str(), value.as_str())), + ); + bun_prewarm.step(&step.argv); + } + } let isolated = case_needs_isolation(&case); if isolated { isolated_trials.insert(trial_name.clone()); @@ -2086,6 +2111,13 @@ fn main() { } } + if let Some(path) = prewarm_manifest { + let urls = bun_prewarm.urls(&prewarm::platform_package()); + std::fs::write(&path, serde_json::to_vec_pretty(&urls).unwrap()).unwrap_or_else(|error| { + panic!("failed to write prewarm manifest {}: {error}", Path::new(&path).display()) + }); + } + if args.list && let Some(path) = std::env::var_os("VP_SNAP_NEXTEST_CONFIG") { diff --git a/crates/vp_cli_snapshots/tests/cli_snapshots/prewarm.rs b/crates/vp_cli_snapshots/tests/cli_snapshots/prewarm.rs new file mode 100644 index 0000000000..0d47968c43 --- /dev/null +++ b/crates/vp_cli_snapshots/tests/cli_snapshots/prewarm.rs @@ -0,0 +1,141 @@ +use std::{ + collections::BTreeSet, + path::{Path, PathBuf}, + sync::LazyLock, +}; + +use serde_json::Value; + +/// Exact fixture pins only. Floating versions keep their normal registry resolution. +static EXACT_VERSION: LazyLock = LazyLock::new(|| { + regex::Regex::new(r"^v?(\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?)(?:\+[0-9A-Za-z.-]+)?$").unwrap() +}); + +#[derive(Default)] +pub struct BunPrewarm { + versions: BTreeSet, + packages: BTreeSet, +} + +impl BunPrewarm { + fn version(&mut self, spec: &str) { + if let Some(version) = EXACT_VERSION.captures(spec).and_then(|captures| captures.get(1)) { + self.versions.insert(version.as_str().to_owned()); + } + } + + fn package_manager(&mut self, value: &Value) { + if let Some(spec) = value.as_str().and_then(|value| value.strip_prefix("bun@")) { + self.version(spec); + } + } + + fn dev_engine(&mut self, value: &Value) { + if let Some(entries) = value.as_array() { + for entry in entries { + self.dev_engine(entry); + } + } else if value["name"] == "bun" + && let Some(version) = value["version"].as_str() + { + self.version(version); + } + } + + pub fn package_json(&mut self, value: &Value) { + self.package_manager(&value["packageManager"]); + self.dev_engine(&value["devEngines"]["packageManager"]); + } + + /// Read the same package ancestry a step sees, without scanning unrelated subprojects. + pub fn workspace(&mut self, fixture: &Path, cwd: &str) -> std::io::Result<()> { + let directory = fixture.join(cwd); + for ancestor in directory.ancestors().take_while(|path| path.starts_with(fixture)) { + let package = ancestor.join("package.json"); + if !self.packages.insert(package.clone()) { + continue; + } + match std::fs::read_to_string(package) { + Ok(text) => { + // Malformed package files are deliberate inputs in validation cases. + if let Ok(value) = serde_json::from_str(&text) { + self.package_json(&value); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + } + Ok(()) + } + + pub fn env<'a>(&mut self, entries: impl IntoIterator) { + for (key, value) in entries { + match key { + "VP_BUN_VERSION" => self.version(value), + "VP_PACKAGE_MANAGER" => self.package_manager(&Value::String(value.to_owned())), + _ => {} + } + } + } + + /// Inspect declarative setup, not comments, assertions, or arbitrary script source. + pub fn step(&mut self, argv: &[String]) { + let args: Vec<&str> = argv.iter().map(String::as_str).collect(); + match args.as_slice() { + ["vpt", "json-edit", file, key, raw] + if Path::new(file).file_name().is_some_and(|name| name == "package.json") => + { + let value = + serde_json::from_str(raw).unwrap_or_else(|_| Value::String((*raw).into())); + match *key { + "packageManager" => self.package_manager(&value), + "devEngines" => self.dev_engine(&value["packageManager"]), + "devEngines.packageManager" => self.dev_engine(&value), + _ => {} + } + } + ["vpt", "write-file", file, text] + if Path::new(file).file_name().is_some_and(|name| name == "package.json") => + { + if let Ok(value) = serde_json::from_str(text) { + self.package_json(&value); + } + } + ["vp", "env", "default" | "use" | "exec", specs @ ..] => { + for spec in specs { + self.package_manager(&Value::String((*spec).into())); + } + } + _ => {} + } + } + + pub fn urls(&self, package: &str) -> Vec { + self.versions + .iter() + .map(|version| { + format!("https://registry.npmjs.org/@oven/{package}/-/{package}-{version}.tgz") + }) + .collect() + } +} + +/// Match the native Bun archive selected by vp_pm_cli, including older x64 CPUs. +pub fn platform_package() -> String { + let platform = match (std::env::consts::OS, std::env::consts::ARCH) { + ("windows", "x86_64") => "windows-x64", + ("windows", "aarch64") => "windows-aarch64", + ("macos", "x86_64") => "darwin-x64", + ("macos", "aarch64") => "darwin-aarch64", + ("linux", "x86_64") => "linux-x64", + ("linux", "aarch64") => "linux-aarch64", + (os, arch) => panic!("unsupported Bun prewarm platform: {os}-{arch}"), + }; + let libc = if cfg!(all(target_os = "linux", target_env = "musl")) { "-musl" } else { "" }; + #[cfg(target_arch = "x86_64")] + let baseline = if std::arch::is_x86_feature_detected!("avx2") { "" } else { "-baseline" }; + #[cfg(not(target_arch = "x86_64"))] + let baseline = ""; + format!("bun-{platform}{libc}{baseline}") +} diff --git a/crates/vp_cli_snapshots/tests/prewarm_unit.rs b/crates/vp_cli_snapshots/tests/prewarm_unit.rs new file mode 100644 index 0000000000..443ca95e6f --- /dev/null +++ b/crates/vp_cli_snapshots/tests/prewarm_unit.rs @@ -0,0 +1,106 @@ +#![expect(clippy::disallowed_macros, reason = "standalone test uses std macros")] +#![expect(clippy::disallowed_types, reason = "standalone test uses std types")] + +#[path = "cli_snapshots/prewarm.rs"] +mod prewarm; + +use prewarm::BunPrewarm; +use serde_json::json; + +fn step(prewarm: &mut BunPrewarm, argv: &[&str]) { + prewarm.step(&argv.iter().map(|arg| (*arg).to_owned()).collect::>()); +} + +#[test] +fn package_changes_update_the_manifest_without_a_second_version_list() { + let fixture = tempfile::tempdir().unwrap(); + let package = fixture.path().join("package.json"); + std::fs::write(&package, r#"{"packageManager":"bun@1.2.3"}"#).unwrap(); + std::fs::create_dir_all(fixture.path().join("packages/app")).unwrap(); + let mut before = BunPrewarm::default(); + before.workspace(fixture.path(), "packages/app").unwrap(); + assert_eq!( + before.urls("bun-windows-x64"), + ["https://registry.npmjs.org/@oven/bun-windows-x64/-/bun-windows-x64-1.2.3.tgz"] + ); + + std::fs::write( + &package, + r#"{"devEngines":{"packageManager":{"name":"bun","version":"2.0.0"}}}"#, + ) + .unwrap(); + let mut after = BunPrewarm::default(); + after.workspace(fixture.path(), "packages/app").unwrap(); + assert_eq!( + after.urls("bun-windows-x64"), + ["https://registry.npmjs.org/@oven/bun-windows-x64/-/bun-windows-x64-2.0.0.tgz"] + ); +} + +#[test] +fn collects_setup_steps_and_overrides_without_duplicate_downloads() { + let mut manifest = BunPrewarm::default(); + manifest.package_json(&json!({"packageManager": "bun@1.2.3"})); + manifest.env([("VP_BUN_VERSION", "1.2.3"), ("VP_PACKAGE_MANAGER", "bun@2.0.0")]); + step(&mut manifest, &["vpt", "json-edit", "sub/package.json", "packageManager", "bun@3.0.0"]); + step( + &mut manifest, + &[ + "vpt", + "json-edit", + "package.json", + "devEngines.packageManager", + r#"{"name":"bun","version":"4.0.0"}"#, + ], + ); + step( + &mut manifest, + &["vpt", "write-file", "package.json", r#"{"packageManager":"bun@5.0.0"}"#], + ); + step(&mut manifest, &["vp", "env", "default", "bun@6.0.0"]); + let urls = manifest.urls("bun-windows-x64"); + assert_eq!(urls.len(), 6); + for (url, version) in urls.iter().zip(["1.2.3", "2.0.0", "3.0.0", "4.0.0", "5.0.0", "6.0.0"]) { + assert!(url.ends_with(&format!("-{version}.tgz"))); + } +} + +#[test] +fn changing_a_step_pin_changes_the_archive_to_prepare() { + let mut before = BunPrewarm::default(); + let mut after = BunPrewarm::default(); + step(&mut before, &["vpt", "json-edit", "package.json", "packageManager", "bun@1.0.0"]); + step(&mut after, &["vpt", "json-edit", "package.json", "packageManager", "bun@2.0.0"]); + assert!(before.urls("bun-windows-x64")[0].ends_with("-1.0.0.tgz")); + assert!(after.urls("bun-windows-x64")[0].ends_with("-2.0.0.tgz")); +} + +#[test] +fn ignores_assertions_other_managers_and_floating_specs() { + let mut manifest = BunPrewarm::default(); + manifest.package_json( + &json!({"packageManager": "pnpm@1.0.0", "devDependencies": {"bun":"99.0.0"}}), + ); + manifest.env([("EXPECTED_BUN_VERSION", "99.0.0"), ("VP_BUN_VERSION", "latest")]); + step(&mut manifest, &["vpt", "print", "bun@99.0.0"]); + step( + &mut manifest, + &["vpt", "write-file", "example.json", r#"{"packageManager":"bun@99.0.0"}"#], + ); + step(&mut manifest, &["node", "-e", "console.log('bun@99.0.0')"]); + for spec in ["bun@latest", "bun@^1.0.0", "bun@>=2", "bun@1.2"] { + manifest.package_json(&json!({"packageManager": spec})); + } + assert!(manifest.urls(&prewarm::platform_package()).is_empty()); +} + +#[test] +fn supports_dev_engine_arrays_and_integrity_pins() { + let mut manifest = BunPrewarm::default(); + manifest.package_json(&json!({"devEngines":{"packageManager":[{"name":"pnpm","version":"99.0.0"},{"name":"bun","version":"1.2.3"}]}})); + manifest.package_json(&json!({"packageManager":"bun@v2.0.0-beta.1+sha512.deadbeef"})); + let urls = manifest.urls("bun-windows-x64-baseline"); + assert_eq!(urls.len(), 2); + assert!(urls[0].ends_with("bun-windows-x64-baseline-1.2.3.tgz")); + assert!(urls[1].ends_with("bun-windows-x64-baseline-2.0.0-beta.1.tgz")); +} diff --git a/crates/vp_setup/src/install.rs b/crates/vp_setup/src/install.rs index 65626b0eea..98aa226ce3 100644 --- a/crates/vp_setup/src/install.rs +++ b/crates/vp_setup/src/install.rs @@ -1066,23 +1066,20 @@ mod tests { async fn fake_pnpm_runtime( version_dir: &AbsolutePath, ) -> (AbsolutePathBuf, AbsolutePathBuf, vp_js_runtime::JsRuntime, AbsolutePathBuf) { - use std::os::unix::fs::PermissionsExt; - let node_bin = version_dir.join("node").join("bin"); let pnpm_bin = version_dir.join("pnpm").join("bin"); tokio::fs::create_dir_all(&node_bin).await.unwrap(); tokio::fs::create_dir_all(&pnpm_bin).await.unwrap(); let node_binary = node_bin.join("node"); - tokio::fs::write( + // Reuse a checked-in executable: concurrent test processes can inherit a freshly + // written script's write descriptor and cause ETXTBSY on Linux. + tokio::fs::symlink( + concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/fake-node.sh"), &node_binary, - "#!/bin/sh\nprintf '%s\\n' \"$@\" > invocation.txt\nprintf '%s' \"$PATH\" > path.txt\nprintf '%s' \"$npm_config_registry\" > registry.txt\n", ) .await .unwrap(); - tokio::fs::set_permissions(&node_binary, std::fs::Permissions::from_mode(0o755)) - .await - .unwrap(); let pnpm_entry = pnpm_bin.join("pnpm.cjs"); tokio::fs::write(&pnpm_entry, "").await.unwrap(); let node_runtime = vp_js_runtime::JsRuntime::from_system(JsRuntimeType::Node, node_binary); diff --git a/crates/vp_setup/tests/fixtures/fake-node.sh b/crates/vp_setup/tests/fixtures/fake-node.sh new file mode 100755 index 0000000000..d3ceb6e50d --- /dev/null +++ b/crates/vp_setup/tests/fixtures/fake-node.sh @@ -0,0 +1,4 @@ +#!/bin/sh +printf '%s\n' "$@" > invocation.txt +printf '%s' "$PATH" > path.txt +printf '%s' "$npm_config_registry" > registry.txt diff --git a/packages/tools/src/__tests__/local-npm-registry.spec.ts b/packages/tools/src/__tests__/local-npm-registry.spec.ts index bc4f6b98ba..9e50c38ee9 100644 --- a/packages/tools/src/__tests__/local-npm-registry.spec.ts +++ b/packages/tools/src/__tests__/local-npm-registry.spec.ts @@ -1,11 +1,58 @@ -import { execFileSync, spawnSync } from 'node:child_process'; +import { execFile, execFileSync, spawnSync } from 'node:child_process'; import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { createServer } from 'node:http'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; import { expect, it } from 'vitest'; +import { RegistryCache } from '../registry-cache.ts'; + +it('prewarms every generated archive once and picks up changed fixture versions', async () => { + const script = fileURLToPath(new URL('../local-npm-registry.ts', import.meta.url)); + const root = mkdtempSync(path.join(tmpdir(), 'vp-prewarm-manifest-test-')); + const requests: string[] = []; + const server = createServer((req, res) => { + requests.push(req.url!); + res.end(Buffer.from([0, 255, 128])); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('Missing registry address'); + } + const registry = `http://127.0.0.1:${address.port}`; + const manifest = path.join(root, 'prewarm.json'); + const cache = path.join(root, 'cache'); + const first = `${registry}/bun-1.0.0.tgz`; + const second = `${registry}/bun-2.0.0.tgz`; + const run = () => + promisify(execFile)( + process.execPath, + [script, '--snapshot-cache', cache, '--prewarm-manifest', manifest], + { timeout: 10_000 }, + ); + writeFileSync(manifest, JSON.stringify([first, first])); + await run(); + writeFileSync(manifest, JSON.stringify([first, second])); + await run(); + expect(requests).toEqual(['/bun-1.0.0.tgz', '/bun-2.0.0.tgz']); + for (const url of [first, second]) { + expect((await new RegistryCache(cache).get(url, 'application/octet-stream')).body).toEqual( + Buffer.from([0, 255, 128]), + ); + } + expect(requests).toHaveLength(2); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + rmSync(root, { recursive: true, force: true }); + } +}); + it.each([false, true])( 'shares a registry with snapshot processes without changing their environment (packed: %s)', (packed) => { diff --git a/packages/tools/src/__tests__/registry-cache.spec.ts b/packages/tools/src/__tests__/registry-cache.spec.ts index b2cbe6535c..59cd1f01cf 100644 --- a/packages/tools/src/__tests__/registry-cache.spec.ts +++ b/packages/tools/src/__tests__/registry-cache.spec.ts @@ -103,6 +103,42 @@ it('can bypass response storage for a live-network benchmark', async () => { expect((await cache.get(url, 'application/json')).body.toString()).toBe('2'); }); +it('prepares an archive once for eight workers with shorter request budgets', async () => { + let requests = 0; + const bytes = Buffer.from([0, 255, 10, 128, 31, 139]); + const url = `${await upstream((_req, res) => { + requests++; + setTimeout(() => res.end(bytes), 150); + })}/tool.tgz`; + const dir = await directory(); + await new RegistryCache(dir).prewarm(url); + const responses = await Promise.all( + Array.from({ length: 8 }, () => + new RegistryCache(dir, true, undefined, 50).get(url, 'application/json'), + ), + ); + expect(requests).toBe(1); + for (const response of responses) { + expect(response.body).toEqual(bytes); + } +}); + +it('retries a failed preparation without caching the failed response', async () => { + let requests = 0; + const url = await upstream((_req, res) => { + if (++requests === 1) { + res.writeHead(503); + res.end('retry'); + } else { + res.end('archive'); + } + }); + const cache = new RegistryCache(await directory()); + await cache.prewarm(url); + expect((await cache.get(url, 'application/octet-stream')).body.toString()).toBe('archive'); + expect(requests).toBe(2); +}); + it('bounds an unfinished download and releases its lock for another registry', async () => { let complete = false; const url = await upstream((_req, res) => { diff --git a/packages/tools/src/local-npm-registry.ts b/packages/tools/src/local-npm-registry.ts index 52c16b6310..349574640e 100644 --- a/packages/tools/src/local-npm-registry.ts +++ b/packages/tools/src/local-npm-registry.ts @@ -36,6 +36,8 @@ // --serve keep the server running and print the registry URL // and env exports instead of wrapping a command // --snapshot-cache share successful upstream GETs for one snapshot run +// --prewarm download an archive into --snapshot-cache, then exit +// --prewarm-manifest prepare the runner's JSON array of archive URLs // --snapshot-fixtures serve recorded manifest.json and tarballs/ data // --snapshot-runner wrap a test runner with a shared registry; the runner // still gives each case its own package-manager caches @@ -81,6 +83,8 @@ let packTo: string | undefined; let serve = false; let snapshotRunner = false; let snapshotCacheDir: string | undefined; +let prewarmUrl: string | undefined; +let prewarmManifest: string | undefined; let snapshotFixtures: string | undefined; let requestLog: string | undefined; let live = false; @@ -95,6 +99,10 @@ for (let i = 0; i < flags.length; i++) { packTo = flags[++i]; } else if (flags[i] === '--snapshot-cache' && flags[i + 1]) { snapshotCacheDir = flags[++i]; + } else if (flags[i] === '--prewarm' && flags[i + 1]) { + prewarmUrl = flags[++i]; + } else if (flags[i] === '--prewarm-manifest' && flags[i + 1]) { + prewarmManifest = flags[++i]; } else if (flags[i] === '--snapshot-fixtures' && flags[i + 1]) { snapshotFixtures = flags[++i]; } else if (flags[i] === '--request-log' && flags[i + 1]) { @@ -128,6 +136,36 @@ if (requestLog) { mkdirSync(path.dirname(requestLog), { recursive: true }); } +if (prewarmUrl || prewarmManifest) { + if ( + !snapshotCacheDir || + command.length || + serve || + snapshotRunner || + pack || + packTo || + (prewarmUrl && prewarmManifest) + ) { + console.error( + 'local-npm-registry: prewarming requires --snapshot-cache, one input, and no runner command', + ); + process.exit(2); + } + // A large archive can take longer than the normal per-request budget. + // Prepare it once before eight test workers start their command timers. + const urls: unknown = prewarmManifest + ? JSON.parse(await readFile(prewarmManifest, 'utf8')) + : [prewarmUrl]; + if (!Array.isArray(urls) || urls.some((url) => typeof url !== 'string')) { + throw new Error('local-npm-registry: prewarm manifest must be a JSON array of archive URLs'); + } + const cache = new RegistryCache(snapshotCacheDir, true, requestLog, 180_000); + for (const url of new Set(urls)) { + await cache.prewarm(url); + } + process.exit(0); +} + // `--ps` / `--kill`: troubleshoot registry processes left behind by // interrupted runs (the wrapper and --serve clean up after themselves, but a // hard kill skips the handlers). A registry process is a `node` executable diff --git a/packages/tools/src/registry-cache.ts b/packages/tools/src/registry-cache.ts index 62e214749f..0169f1290c 100644 --- a/packages/tools/src/registry-cache.ts +++ b/packages/tools/src/registry-cache.ts @@ -88,6 +88,25 @@ export class RegistryCache { } } + // Prepare large tool archives before snapshot command deadlines start. + // Only HTTP response bytes are shared; installations remain case-local. + async prewarm(url: string): Promise { + for (let attempt = 0; ; attempt++) { + try { + const response = await this.get(url, 'application/octet-stream'); + if (response.status !== 200) { + throw new Error(`Cannot prepare registry response: ${url} (HTTP ${response.status})`); + } + return; + } catch (error) { + if (attempt === 2) { + throw error; + } + await setTimeout(1000 * 2 ** attempt); + } + } + } + async get(url: string, accept: string): Promise { this.record('request', url); if (!this.enabled) {