diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 926439e..40fef27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -127,7 +127,7 @@ jobs: run: gem install minitest --version '~> 5.25' --no-document - name: Test the Ruby evaluator - run: ruby crates/rb-gemfile/tests/ruby/evaluator_test.rb + run: ruby gems/rb-gemfile/test/all_test.rb - name: Test MicroGems run: ruby gems/rb-microgems/test/all_test.rb @@ -154,10 +154,10 @@ jobs: bundler: none - name: Test gem installation and build compatibility - run: cargo test -p rb-gem-installer -p rb-microgems + run: cargo test -p rb-gem-installer -p rb-microgems -p rb-packages - name: Build installer examples - run: cargo build -p rb-gem-installer --examples + run: cargo build -p rb-gem-installer -p rb-packages --examples test-rust: name: Rust Tests diff --git a/Cargo.lock b/Cargo.lock index 11f9092..3707d1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1373,6 +1373,32 @@ dependencies = [ "tempfile", ] +[[package]] +name = "rb-packages" +version = "0.3.1" +dependencies = [ + "anyhow", + "dunce", + "flate2", + "rb-compact-index", + "rb-gem-archive", + "rb-gem-installer", + "rb-gem-types", + "rb-gemfile", + "rb-lockfile", + "rb-microgems", + "rb-resolver", + "rb-task", + "serde", + "serde_json", + "serde_yaml_ng", + "sha2", + "tar", + "tempfile", + "toml", + "url", +] + [[package]] name = "rb-progress" version = "0.3.1" @@ -1529,9 +1555,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "once_cell", "ring", diff --git a/crates/rb-compact-index/README.md b/crates/rb-compact-index/README.md index e469010..255033a 100644 --- a/crates/rb-compact-index/README.md +++ b/crates/rb-compact-index/README.md @@ -15,6 +15,9 @@ for gem in client.info("rack")? { | API | Purpose | | --- | --- | +| `Client::names` | Discover registry gem names, with an offline cache. | +| `Client::with_credentials` | Apply HTTP Basic credentials to registry requests. | +| `Client::info_or_empty` | Look up candidates, allowing confirmed missing gems. | | `Client::info` | Fetch a gem's versions, dependency requirements and metadata. | | `Client::download` | Fetch an archive with an expected SHA-256 checksum. | | `parse` | Parse a compact-index response without HTTP or cache access. | diff --git a/crates/rb-compact-index/src/client.rs b/crates/rb-compact-index/src/client.rs index 0e00579..2f295fc 100644 --- a/crates/rb-compact-index/src/client.rs +++ b/crates/rb-compact-index/src/client.rs @@ -24,7 +24,7 @@ use std::{ pub struct Client { pub(crate) source: String, pub(crate) cache: PathBuf, - offline: bool, + pub(crate) offline: bool, pub(crate) client: HttpClient, } @@ -67,7 +67,38 @@ impl Client { &self.source } + pub fn with_credentials(mut self, credentials: &str) -> Result { + let mut headers = header::HeaderMap::new(); + let credentials = if credentials.contains(':') { + credentials.to_owned() + } else { + format!("{credentials}:") + }; + let mut authorization = header::HeaderValue::from_str(&format!( + "Basic {}", + base64::engine::general_purpose::STANDARD.encode(credentials) + ))?; + authorization.set_sensitive(true); + headers.insert(header::AUTHORIZATION, authorization); + self.client = HttpClient::builder() + .timeout(REQUEST_TIMEOUT) + .connect_timeout(CONNECT_TIMEOUT) + .user_agent(concat!("rb-compact-index/", env!("CARGO_PKG_VERSION"))) + .default_headers(headers) + .build()?; + Ok(self) + } + pub fn info(&self, name: &str) -> Result> { + self.fetch_info(name, false) + } + + /// Returns no candidates for a missing gem on a working compact-index server. + pub fn info_or_empty(&self, name: &str) -> Result> { + self.fetch_info(name, true) + } + + fn fetch_info(&self, name: &str, allow_missing: bool) -> Result> { validate_name(name)?; let directory = self .cache @@ -117,6 +148,17 @@ impl Client { if versions.status() == reqwest::StatusCode::NOT_FOUND { return self.full_info(name, &directory); } + versions.error_for_status()?; + if allow_missing { + atomic_write( + &path, + &serde_json::to_vec(&CachedInfo { + etag: None, + body: "---\n".into(), + })?, + )?; + return Ok(Vec::new()); + } } let response = response.error_for_status()?; let etag = response @@ -124,24 +166,7 @@ impl Client { .get(header::ETAG) .and_then(|v| v.to_str().ok()) .map(str::to_owned); - let digest = response - .headers() - .get("repr-digest") - .and_then(|v| v.to_str().ok()) - .map(str::to_owned); - let bytes = limited_read(response, MAX_COMPACT_INFO_BYTES)?; - if let Some(digest) = digest - && let Some(value) = digest - .split(',') - .find_map(|part| part.trim().strip_prefix("sha-256=")) - { - let expected = - base64::engine::general_purpose::STANDARD.decode(value.trim_matches([':', '"']))?; - ensure!( - sha2::Sha256::digest(&bytes).as_slice() == expected, - "Index digest mismatch for {name}" - ); - } + let bytes = read_index(response, MAX_COMPACT_INFO_BYTES)?; let body = String::from_utf8(bytes)?; let gems = parse(name, &body)?; atomic_write(&path, &serde_json::to_vec(&CachedInfo { etag, body })?)?; @@ -190,3 +215,25 @@ impl Client { Ok(path) } } + +pub(crate) fn read_index(response: reqwest::blocking::Response, limit: u64) -> Result> { + let digest = response + .headers() + .get("repr-digest") + .and_then(|v| v.to_str().ok()) + .map(str::to_owned); + let bytes = limited_read(response, limit)?; + if let Some(digest) = digest + && let Some(value) = digest + .split(',') + .find_map(|part| part.trim().strip_prefix("sha-256=")) + { + let expected = + base64::engine::general_purpose::STANDARD.decode(value.trim_matches([':', '"']))?; + ensure!( + sha2::Sha256::digest(&bytes).as_slice() == expected, + "Index digest mismatch" + ); + } + Ok(bytes) +} diff --git a/crates/rb-compact-index/src/constants.rs b/crates/rb-compact-index/src/constants.rs index 123d4ce..256385a 100644 --- a/crates/rb-compact-index/src/constants.rs +++ b/crates/rb-compact-index/src/constants.rs @@ -5,6 +5,7 @@ pub(crate) const CONNECT_TIMEOUT: Duration = Duration::from_secs(15); pub(crate) const ARCHIVE_BUFFER_BYTES: usize = 64 * 1024; pub(crate) const MAX_ARCHIVE_BYTES: u64 = 1024 * 1024 * 1024; +pub(crate) const MAX_VERSIONS_BYTES: u64 = 128 * 1024 * 1024; pub(crate) const MAX_COMPACT_INFO_BYTES: u64 = 32 * 1024 * 1024; pub(crate) const MAX_COMPRESSED_INDEX_BYTES: u64 = 16 * 1024 * 1024; pub(crate) const MAX_DECOMPRESSED_INDEX_BYTES: u64 = 32 * 1024 * 1024; @@ -12,6 +13,8 @@ pub(crate) const MAX_GEM_METADATA_BYTES: u64 = 16 * 1024 * 1024; pub(crate) const MAX_FALLBACK_ARCHIVE_BYTES: u64 = 256 * 1024 * 1024; pub(crate) const INDEX_CACHE_DIRECTORY: &str = "index"; +pub(crate) const REGISTRY_METADATA_DIRECTORY: &str = "registry"; +pub(crate) const VERSIONS_CACHE_FILE: &str = "versions.json"; pub(crate) const ARCHIVE_CACHE_DIRECTORY: &str = "gems"; pub(crate) const FULL_INFO_CACHE_SUFFIX: &str = ".full-platforms.json"; diff --git a/crates/rb-compact-index/src/lib.rs b/crates/rb-compact-index/src/lib.rs index 070ce1c..7833f24 100644 --- a/crates/rb-compact-index/src/lib.rs +++ b/crates/rb-compact-index/src/lib.rs @@ -6,6 +6,7 @@ mod constants; mod full_index; mod info; mod validation; +mod versions; pub use client::Client; pub use info::{GemInfo, parse}; diff --git a/crates/rb-compact-index/src/tests.rs b/crates/rb-compact-index/src/tests.rs index 53365c7..f13be2c 100644 --- a/crates/rb-compact-index/src/tests.rs +++ b/crates/rb-compact-index/src/tests.rs @@ -16,10 +16,12 @@ use std::{ thread, }; +mod authentication; mod download; mod fixtures; mod full_index; mod info; mod server; +mod versions; use server::Server; diff --git a/crates/rb-compact-index/src/tests/authentication.rs b/crates/rb-compact-index/src/tests/authentication.rs new file mode 100644 index 0000000..1b37577 --- /dev/null +++ b/crates/rb-compact-index/src/tests/authentication.rs @@ -0,0 +1,73 @@ +use super::*; +use base64::Engine; + +#[test] +fn credentials_authenticate_metadata_and_archives_without_entering_source_or_cache() { + let server = Server::new(); + let cache = tempfile::tempdir().unwrap(); + let archive = b"fixture archive"; + let sha = checksum(archive); + server.responses.lock().unwrap().extend([ + ( + "/info/example".into(), + format!("---\n1.0 |checksum:{sha}\n").into_bytes(), + ), + ("/versions".into(), b"---\nexample 1.0 abc\n".to_vec()), + ("/gems/example-1.0.gem".into(), archive.to_vec()), + ]); + let client = Client::new(&server.source, cache.path(), false) + .unwrap() + .with_credentials("user:secret-token") + .unwrap(); + assert_eq!(client.source(), server.source); + assert_eq!(client.info("example").unwrap().len(), 1); + assert!(client.names().unwrap().unwrap().contains("example")); + client.download("example", "1.0", &sha).unwrap(); + let expected = format!( + "authorization: basic {}", + base64::engine::general_purpose::STANDARD + .encode("user:secret-token") + .to_lowercase() + ); + for request in server.request_headers.lock().unwrap().iter() { + assert!(request.to_lowercase().contains(&expected)); + } + let info = cache + .path() + .join("index") + .join(checksum(server.source.as_bytes())) + .join("example.json"); + let cached = fs::read_to_string(info).unwrap(); + assert!(!cached.contains("secret-token")); + assert!(!cached.contains("authorization")); +} + +#[test] +fn credentials_are_not_forwarded_to_another_origin_on_redirect() { + let server = Server::new(); + let destination = Server::new(); + server + .statuses + .lock() + .unwrap() + .insert("/info/example".into(), "302 Found".into()); + server.headers.lock().unwrap().insert( + "/info/example".into(), + format!("Location: {}info/example\r\n", destination.source), + ); + destination.responses.lock().unwrap().insert( + "/info/example".into(), + format!("---\n1.0 |checksum:{}\n", "a".repeat(64)).into_bytes(), + ); + let cache = tempfile::tempdir().unwrap(); + let client = Client::new(&server.source, cache.path(), false) + .unwrap() + .with_credentials("user:secret-token") + .unwrap(); + client.info("example").unwrap(); + assert!( + !destination.request_headers.lock().unwrap()[0] + .to_lowercase() + .contains("authorization:") + ); +} diff --git a/crates/rb-compact-index/src/tests/info.rs b/crates/rb-compact-index/src/tests/info.rs index b95d59e..bed5a54 100644 --- a/crates/rb-compact-index/src/tests/info.rs +++ b/crates/rb-compact-index/src/tests/info.rs @@ -235,3 +235,81 @@ fn empty_or_unterminated_headers_are_rejected() { assert!(parse("app", body).is_err()); } } + +#[test] +fn missing_optional_lookup_is_cached_offline_and_rechecked_online() { + let server = Server::new(); + server + .responses + .lock() + .unwrap() + .insert("/versions".into(), b"---\n".to_vec()); + let cache = tempfile::tempdir().unwrap(); + let client = Client::new(&server.source, cache.path(), false).unwrap(); + assert!(client.info_or_empty("missing").unwrap().is_empty()); + assert_eq!( + *server.requests.lock().unwrap(), + ["/info/missing", "/versions"] + ); + let offline = Client::new(&server.source, cache.path(), true).unwrap(); + assert!(offline.info_or_empty("missing").unwrap().is_empty()); + server.responses.lock().unwrap().insert( + "/info/missing".into(), + format!("---\n1.0 |checksum:{}\n", "a".repeat(64)).into_bytes(), + ); + let found = client.info_or_empty("missing").unwrap(); + assert_eq!(found.len(), 1); + assert_eq!(found[0].version, "1.0"); + drop(server); + assert_eq!(offline.info_or_empty("missing").unwrap(), found); +} + +#[test] +fn optional_lookup_does_not_hide_access_server_or_protocol_errors() { + for status in [ + "401 Unauthorized", + "403 Forbidden", + "500 Internal Server Error", + ] { + let server = Server::new(); + server + .statuses + .lock() + .unwrap() + .insert("/info/private".into(), status.into()); + server + .responses + .lock() + .unwrap() + .insert("/versions".into(), b"---\n".to_vec()); + let cache = tempfile::tempdir().unwrap(); + let client = Client::new(&server.source, cache.path(), false).unwrap(); + assert!(client.info_or_empty("private").is_err()); + assert_eq!(*server.requests.lock().unwrap(), ["/info/private"]); + assert!( + Client::new(&server.source, cache.path(), true) + .unwrap() + .info_or_empty("private") + .is_err() + ); + } + let server = Server::new(); + server + .statuses + .lock() + .unwrap() + .insert("/versions".into(), "403 Forbidden".into()); + let cache = tempfile::tempdir().unwrap(); + assert!( + Client::new(&server.source, cache.path(), false) + .unwrap() + .info_or_empty("private") + .is_err() + ); + assert!( + Client::new(&server.source, cache.path(), true) + .unwrap() + .info_or_empty("private") + .is_err() + ); +} diff --git a/crates/rb-compact-index/src/tests/server.rs b/crates/rb-compact-index/src/tests/server.rs index d944c4f..bca529d 100644 --- a/crates/rb-compact-index/src/tests/server.rs +++ b/crates/rb-compact-index/src/tests/server.rs @@ -4,6 +4,7 @@ pub(super) struct Server { pub(super) source: String, pub(super) responses: Arc>>>, pub(super) requests: Arc>>, + pub(super) request_headers: Arc>>, pub(super) headers: Arc>>, pub(super) statuses: Arc>>, pub(super) revalidate: Arc, @@ -18,6 +19,8 @@ impl Server { let source = format!("http://{}/", listener.local_addr().unwrap()); let responses = Arc::new(Mutex::new(BTreeMap::>::new())); let requests = Arc::new(Mutex::new(Vec::new())); + let request_headers = Arc::new(Mutex::new(Vec::new())); + let captured_headers = request_headers.clone(); let headers = Arc::new(Mutex::new(BTreeMap::::new())); let statuses = Arc::new(Mutex::new(BTreeMap::::new())); let revalidate = Arc::new(AtomicBool::new(true)); @@ -33,6 +36,7 @@ impl Server { }; let request = read_request(&mut stream).unwrap(); let request = String::from_utf8_lossy(&request); + captured_headers.lock().unwrap().push(request.to_string()); let path = request.split_whitespace().nth(1).unwrap_or("").to_string(); log.lock().unwrap().push(path.clone()); let response = data.lock().unwrap().get(&path).cloned(); @@ -70,6 +74,7 @@ impl Server { source, responses, requests, + request_headers, headers, statuses, revalidate, diff --git a/crates/rb-compact-index/src/tests/versions.rs b/crates/rb-compact-index/src/tests/versions.rs new file mode 100644 index 0000000..bc98e9b --- /dev/null +++ b/crates/rb-compact-index/src/tests/versions.rs @@ -0,0 +1,128 @@ +use super::*; + +#[test] +fn names_are_scoped_cached_offline_and_do_not_collide_with_gem_metadata() { + let server = Server::new(); + server.responses.lock().unwrap().insert( + "/versions".into(), + b"created_at: fixture\n---\naddon 1.0 a\nchild 1.0 \naddon 2.0 c\n".to_vec(), + ); + server.responses.lock().unwrap().insert( + "/info/versions".into(), + format!("---\n1.0 |checksum:{}\n", "a".repeat(64)).into_bytes(), + ); + let cache = tempfile::tempdir().unwrap(); + let client = Client::new(&server.source, cache.path(), false).unwrap(); + let names = client.names().unwrap().unwrap(); + assert_eq!(names, ["addon".into(), "child".into()].into()); + assert_eq!(client.info("versions").unwrap()[0].version, "1.0"); + let offline = Client::new(&server.source, cache.path(), true).unwrap(); + drop(server); + assert_eq!(offline.names().unwrap().unwrap(), names); + assert_eq!(offline.info("versions").unwrap()[0].version, "1.0"); + assert!( + Client::new("https://other.example/", cache.path(), true) + .unwrap() + .names() + .is_err() + ); +} + +#[test] +fn unsupported_names_endpoint_is_remembered_without_hiding_other_failures() { + let server = Server::new(); + let cache = tempfile::tempdir().unwrap(); + let client = Client::new(&server.source, cache.path(), false).unwrap(); + assert!(client.names().unwrap().is_none()); + assert_eq!(*server.requests.lock().unwrap(), ["/versions"]); + drop(server); + assert!( + Client::new(client.source(), cache.path(), true) + .unwrap() + .names() + .unwrap() + .is_none() + ); + for status in [ + "401 Unauthorized", + "403 Forbidden", + "500 Internal Server Error", + ] { + let server = Server::new(); + server + .statuses + .lock() + .unwrap() + .insert("/versions".into(), status.into()); + let cache = tempfile::tempdir().unwrap(); + assert!( + Client::new(&server.source, cache.path(), false) + .unwrap() + .names() + .is_err() + ); + assert!( + Client::new(&server.source, cache.path(), true) + .unwrap() + .names() + .is_err() + ); + } +} + +#[test] +fn malformed_names_do_not_replace_the_cached_registry_listing() { + let server = Server::new(); + server.revalidate.store(false, Ordering::Relaxed); + let cache = tempfile::tempdir().unwrap(); + let client = Client::new(&server.source, cache.path(), false).unwrap(); + server + .responses + .lock() + .unwrap() + .insert("/versions".into(), b"---\naddon 1.0 a\n".to_vec()); + let expected = client.names().unwrap(); + for body in [ + "not a versions index", + "---\naddon", + "---\n../escape 1.0 a\n", + ] { + server + .responses + .lock() + .unwrap() + .insert("/versions".into(), body.as_bytes().to_vec()); + assert!(client.names().is_err()); + assert_eq!( + Client::new(&server.source, cache.path(), true) + .unwrap() + .names() + .unwrap(), + expected + ); + } + server + .responses + .lock() + .unwrap() + .insert("/versions".into(), b"---\nchanged 1.0 b\n".to_vec()); + server + .headers + .lock() + .unwrap() + .insert("/versions".into(), "Repr-Digest: sha-256=:AAAA:\r\n".into()); + assert!( + client + .names() + .unwrap_err() + .to_string() + .contains("Index digest mismatch") + ); + assert_eq!( + Client::new(&server.source, cache.path(), true) + .unwrap() + .names() + .unwrap(), + expected + ); +} diff --git a/crates/rb-compact-index/src/versions.rs b/crates/rb-compact-index/src/versions.rs new file mode 100644 index 0000000..a9f4080 --- /dev/null +++ b/crates/rb-compact-index/src/versions.rs @@ -0,0 +1,67 @@ +use crate::{ + Client, + cache::{atomic_write, checksum, lock}, + client::read_index, + constants::{ + INDEX_CACHE_DIRECTORY, MAX_VERSIONS_BYTES, REGISTRY_METADATA_DIRECTORY, + VERSIONS_CACHE_FILE, VERSIONS_PATH, + }, + validation::validate_name, +}; +use anyhow::{Context, Result, ensure}; +use std::{collections::BTreeSet, fs}; + +fn parse(body: &str) -> Result> { + let mut lines = body.lines(); + ensure!( + lines.by_ref().any(|line| line == "---"), + "Missing versions index header" + ); + let mut names = BTreeSet::new(); + for line in lines.filter(|line| !line.is_empty()) { + let fields: Vec<_> = line.split_whitespace().collect(); + ensure!( + (2..=3).contains(&fields.len()), + "Invalid versions index entry" + ); + validate_name(fields[0])?; + names.insert(fields[0].to_owned()); + } + Ok(names) +} + +impl Client { + /// Lists compact-index gem names, or returns None when that endpoint is unsupported. + pub fn names(&self) -> Result>> { + let directory = self + .cache + .join(INDEX_CACHE_DIRECTORY) + .join(checksum(self.source.as_bytes())) + .join(REGISTRY_METADATA_DIRECTORY); + fs::create_dir_all(&directory)?; + let path = directory.join(VERSIONS_CACHE_FILE); + let _guard = lock(&path.with_extension("lock"))?; + if self.offline { + let body: Option = serde_json::from_slice( + &fs::read(&path).context("No cached versions index (offline)")?, + )?; + return body.as_deref().map(parse).transpose(); + } + let response = self + .client + .get(format!("{}{VERSIONS_PATH}", self.source)) + .header(reqwest::header::ACCEPT_ENCODING, "identity") + .send()?; + let body = if response.status() == reqwest::StatusCode::NOT_FOUND { + None + } else { + Some(String::from_utf8(read_index( + response.error_for_status()?, + MAX_VERSIONS_BYTES, + )?)?) + }; + let names = body.as_deref().map(parse).transpose()?; + atomic_write(&path, &serde_json::to_vec(&body)?)?; + Ok(names) + } +} diff --git a/crates/rb-gem-installer/src/build_report.rs b/crates/rb-gem-installer/src/build_report.rs index 8c94e7e..f975de5 100644 --- a/crates/rb-gem-installer/src/build_report.rs +++ b/crates/rb-gem-installer/src/build_report.rs @@ -6,5 +6,6 @@ pub struct BuildReport { pub architecture: String, pub extensions: Vec, pub artifacts: Vec, + pub extension_directory: Option, pub elapsed: Duration, } diff --git a/crates/rb-gem-installer/src/native_builder.rs b/crates/rb-gem-installer/src/native_builder.rs index da627af..ebf915f 100644 --- a/crates/rb-gem-installer/src/native_builder.rs +++ b/crates/rb-gem-installer/src/native_builder.rs @@ -17,7 +17,7 @@ use std::{ pub struct NativeBuilder { pub(crate) options: BuildOptions, pub(crate) ruby: RubyConfig, - compatibility: Option<(PathBuf, PathBuf, String)>, + compatibility: Option<(PathBuf, PathBuf, String, String)>, } impl NativeBuilder { @@ -80,6 +80,7 @@ impl NativeBuilder { compatibility.load_path().to_path_buf(), compatibility.manifest_path().to_path_buf(), serde_json::to_string(&[&metadata.id.name, &metadata.id.version.to_string()])?, + serde_json::to_string(&configured.options.load_paths)?, )); configured.build_declared(metadata, &root, &mut events) } @@ -102,6 +103,7 @@ impl NativeBuilder { architecture: self.ruby.get("arch").into(), extensions: Vec::new(), artifacts: Vec::new(), + extension_directory: None, elapsed: Default::default(), }; if metadata.extensions.is_empty() { @@ -111,6 +113,7 @@ impl NativeBuilder { .join(self.ruby.get("arch")) .join(self.ruby.get("ruby_version")); let native_relative = payload_path::relative(&native_relative)?; + report.extension_directory = Some(native_relative.clone()); let destination = root.join(&native_relative); ensure!( !destination.exists(), @@ -231,11 +234,13 @@ impl NativeBuilder { command.env(format!("RBCONFIG_{key}"), value); } let mut paths = self.options.load_paths.clone(); - if let Some((directory, manifest, current)) = &self.compatibility { - paths.insert(0, directory.clone()); + if let Some((directory, manifest, current, load_paths)) = &self.compatibility { + paths.clear(); + paths.push(directory.clone()); command .env("RB_MICROGEMS_MANIFEST", manifest) .env("RB_MICROGEMS_CURRENT", current) + .env("RB_MICROGEMS_LOAD_PATHS", load_paths) .env("RUBYOPT", "--disable=gems -rmicrogems/compatibility"); } if let Ok(paths) = std::env::join_paths(&paths) { @@ -247,9 +252,6 @@ impl NativeBuilder { pub(crate) fn ruby_command(&self, directory: &Path) -> Command { let mut command = self.command(&self.ruby.executable, directory); command.arg("--disable=gems"); - for path in &self.options.load_paths { - command.arg("-I").arg(path); - } command } diff --git a/crates/rb-gem-installer/src/ruby_config.rs b/crates/rb-gem-installer/src/ruby_config.rs index 3368a88..9bed876 100644 --- a/crates/rb-gem-installer/src/ruby_config.rs +++ b/crates/rb-gem-installer/src/ruby_config.rs @@ -9,6 +9,20 @@ use std::{ }; const FILES: &[(&str, &str)] = &[ + ( + "lib/rb_gem_installer/source_specifications.rb", + include_str!( + "../../../gems/rb-gem-installer/lib/rb_gem_installer/source_specifications.rb" + ), + ), + ( + "lib/rb_gem_installer/runtime.rb", + include_str!("../../../gems/rb-gem-installer/lib/rb_gem_installer/runtime.rb"), + ), + ( + "lib/rb_gem_installer/specification.rb", + include_str!("../../../gems/rb-gem-installer/lib/rb_gem_installer/specification.rb"), + ), ( "lib/rb_gem_installer.rb", include_str!("../../../gems/rb-gem-installer/lib/rb_gem_installer.rb"), diff --git a/crates/rb-gem-installer/tests/building.rs b/crates/rb-gem-installer/tests/building.rs index 7c779e0..aa0135d 100644 --- a/crates/rb-gem-installer/tests/building.rs +++ b/crates/rb-gem-installer/tests/building.rs @@ -709,3 +709,30 @@ fn mkrf_builder_generates_and_runs_its_rakefile() { assert!(root.path().join("Rakefile").is_file()); require(root.path(), "require 'rake_output'; puts RAKE_BUILD_ANSWER"); } + +#[cfg(unix)] +#[test] +fn current_gem_cannot_shadow_compatibility_bootstrap_dependencies() { + let root = tempfile::tempdir().unwrap(); + copy_fixture("c", root.path()); + fs::create_dir(root.path().join("lib")).unwrap(); + fs::write( + root.path().join("lib/json.rb"), + "raise 'unfinished current gem loaded during bootstrap'\n", + ) + .unwrap(); + let mut metadata = metadata("extconf.rb"); + metadata.id.name = "json".into(); + let source = fs::read_to_string(root.path().join("extconf.rb")) + .unwrap() + .replace("fetch('dinner')", "fetch('json')"); + fs::write(root.path().join("extconf.rb"), source).unwrap(); + let report = builder().build(&metadata, root.path(), |_| {}).unwrap(); + let directory = report.extension_directory.as_ref().unwrap(); + assert!( + report + .artifacts + .iter() + .any(|path| path.starts_with(directory)) + ); +} diff --git a/crates/rb-gem-types/src/package_id.rs b/crates/rb-gem-types/src/package_id.rs index 71a9c97..bb169d9 100644 --- a/crates/rb-gem-types/src/package_id.rs +++ b/crates/rb-gem-types/src/package_id.rs @@ -1,9 +1,13 @@ use crate::{Platform, Version}; use serde::{Deserialize, Serialize}; -use std::fmt; +use std::{ + cmp::Ordering, + fmt, + hash::{Hash, Hasher}, +}; /// Identifies a gem archive by name, version and platform; source provenance is separate. -#[derive(Clone, Debug, Eq, PartialEq, Hash, Ord, PartialOrd, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct PackageId { pub name: String, pub version: Version, @@ -29,3 +33,33 @@ impl fmt::Display for PackageId { f.write_str(&self.full_name()) } } + +impl PartialEq for PackageId { + fn eq(&self, other: &Self) -> bool { + self.name == other.name + && self.version.as_str() == other.version.as_str() + && self.platform == other.platform + } +} +impl Eq for PackageId {} +impl Hash for PackageId { + fn hash(&self, state: &mut H) { + self.name.hash(state); + self.version.as_str().hash(state); + self.platform.hash(state); + } +} +impl Ord for PackageId { + fn cmp(&self, other: &Self) -> Ordering { + self.name + .cmp(&other.name) + .then_with(|| self.version.cmp(&other.version)) + .then_with(|| self.version.as_str().cmp(other.version.as_str())) + .then_with(|| self.platform.cmp(&other.platform)) + } +} +impl PartialOrd for PackageId { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} diff --git a/crates/rb-gem-types/src/version.rs b/crates/rb-gem-types/src/version.rs index 61486cb..5082906 100644 --- a/crates/rb-gem-types/src/version.rs +++ b/crates/rb-gem-types/src/version.rs @@ -72,6 +72,10 @@ fn trim_zeros(segments: &mut Vec) { } impl Version { + pub fn as_str(&self) -> &str { + &self.raw + } + pub fn prerelease(&self) -> bool { self.segments.iter().any(|s| matches!(s, Segment::Text(_))) } diff --git a/crates/rb-gem-types/tests/types.rs b/crates/rb-gem-types/tests/types.rs index 6f64903..93a7223 100644 --- a/crates/rb-gem-types/tests/types.rs +++ b/crates/rb-gem-types/tests/types.rs @@ -144,3 +144,21 @@ fn platform_display_round_trips_canonical_components() { assert_eq!(Platform::parse("x86_64-ruby").os, "unknown"); assert_eq!(Platform::parse("x86_64-haiku").os, "unknown"); } + +#[test] +fn archive_identity_preserves_equivalent_version_spellings() { + let releases: Vec<_> = ["2.0", "2.0.0"] + .into_iter() + .map(|value| PackageId { + name: "faker".into(), + version: version(value), + platform: Platform::parse("ruby"), + }) + .collect(); + assert_eq!(releases[0].version, releases[1].version); + assert_ne!(releases[0], releases[1]); + assert_eq!(releases.iter().cloned().collect::>().len(), 2); + assert_eq!(releases.iter().cloned().collect::>().len(), 2); + assert_eq!(releases[0].full_name(), "faker-2.0"); + assert_eq!(releases[1].full_name(), "faker-2.0.0"); +} diff --git a/crates/rb-gemfile/README.md b/crates/rb-gemfile/README.md index 5646c70..2566eab 100644 --- a/crates/rb-gemfile/README.md +++ b/crates/rb-gemfile/README.md @@ -1,6 +1,6 @@ # rb-gemfile -Evaluate trusted Gemfiles into dependency manifests without loading Bundler. An embedded Ruby evaluator collects declarations; Rust provides typed data, TOML/KDL export and optional caching. Ruby and RubyGems are still required. This crate does not resolve versions, install gems or generate lockfiles. +Evaluate trusted Gemfiles into dependency manifests without loading Bundler. The Ruby helper in `gems/rb-gemfile` collects declarations; Rust provides typed data, TOML/KDL export and optional caching. Ruby and RubyGems are still required. This crate does not resolve versions, install gems or generate lockfiles. ```rust,no_run use rb_gemfile::{Evaluator, write_project}; @@ -20,12 +20,12 @@ write_project(".rb/rbproject.toml", &project)?; | `SnapshotCache::evaluate` | Reuse deterministic evaluations when fingerprinted inputs are unchanged. | | `write_project` | Write a generated file atomically. | -Supports groups, platforms, conditions, require settings, registry/Git/path sources, included Gemfiles and local gemspecs. Repeated declarations remain separate in `Dependency::variants`; exports omit inactive declarations. Bundler compatibility helpers are limited: plugins are recorded, not installed or executed, and source `cooldown` is accepted but not enforced. +Supports groups, platforms, conditions, require settings, registry/Git/path sources, included Gemfiles and local gemspecs. Repeated declarations remain separate in `Dependency::variants`; exports omit inactive declarations. Bundler compatibility helpers are limited: plugins are recorded, not installed or executed, and source `cooldown` is accepted but not enforced. Git dependencies accept one selector: `branch`, `tag` or `ref`/`rev`. Gemfiles execute arbitrary Ruby: evaluate only trusted inputs. For caching, declare external files/directories with `input` and unread generated files with `output`. The project directory is fingerprinted automatically. Use uncached evaluation for time, network, subprocess results or side effects. The CLI exposes this through `rb export-gemfile`, with `--format kdl` for KDL output. Exported dependencies are not yet consumed by `rb-core`. -Tested against 1,477 snapshots from the public [Gemfile corpus](https://github.com/RubyElders/gemfile-corpus), using Ruby 3.4.1 and Bundler 2.6.2 as a comparison: 1,318 evaluated and exported to TOML successfully; 159 failed, of which 157 also failed Bundler. Most failures involved missing companion files. Two Git-option combinations accepted by Bundler remain unsupported. This checks evaluation/export, not installation or complete manifest equivalence. +Tested against 1,477 snapshots from the public [Gemfile corpus](https://github.com/RubyElders/gemfile-corpus), using Ruby 3.4.10: 1,318 evaluated and exported to TOML successfully; 159 failed. Most failures involved missing companion files. Combined Git selectors are intentionally rejected. This checks evaluation/export, not installation or complete manifest equivalence. -Run `cargo test -p rb-gemfile`. Direct Ruby tests use `ruby crates/rb-gemfile/tests/ruby/evaluator_test.rb` with Minitest 5. `RB_GEMFILE_RUBY` selects Ruby for Rust tests and the `evaluate` example. +Run `cargo test -p rb-gemfile`. Direct Ruby tests use `ruby gems/rb-gemfile/test/all_test.rb` with Minitest 5. `RB_GEMFILE_RUBY` selects Ruby for Rust tests and the `evaluate` example. diff --git a/crates/rb-gemfile/src/evaluator.rs b/crates/rb-gemfile/src/evaluator.rs index e514992..6bfb172 100644 --- a/crates/rb-gemfile/src/evaluator.rs +++ b/crates/rb-gemfile/src/evaluator.rs @@ -44,13 +44,15 @@ impl Evaluator { let directory = tempfile::tempdir()?; let result = directory.path().join("manifest.json"); for (name, source) in SOURCES { - std::fs::write(directory.path().join(name), source)?; + let path = directory.path().join(name); + std::fs::create_dir_all(path.parent().unwrap())?; + std::fs::write(path, source)?; } let ruby = self.ruby_executable()?; let output = Command::new(ruby) .envs(&self.environment) .env_remove("RUBYOPT") - .arg(directory.path().join("evaluate.rb")) + .arg(directory.path().join("exe/rb-evaluate-gemfile")) .arg(&gemfile) .arg(&result) .current_dir(gemfile.parent().unwrap()) @@ -106,12 +108,32 @@ pub(crate) fn gemfile_path(path: &Path) -> io::Result { } pub(crate) const SOURCES: &[(&str, &str)] = &[ - ("evaluate.rb", include_str!("../assets/evaluate.rb")), - ("gemfile.rb", include_str!("../assets/gemfile.rb")), - ("sources.rb", include_str!("../assets/sources.rb")), - ("gemspec.rb", include_str!("../assets/gemspec.rb")), ( - "compatibility.rb", - include_str!("../assets/compatibility.rb"), + "exe/rb-evaluate-gemfile", + include_str!("../../../gems/rb-gemfile/exe/rb-evaluate-gemfile"), + ), + ( + "lib/rb_gemfile.rb", + include_str!("../../../gems/rb-gemfile/lib/rb_gemfile.rb"), + ), + ( + "lib/rb_gemfile/version.rb", + include_str!("../../../gems/rb-gemfile/lib/rb_gemfile/version.rb"), + ), + ( + "lib/rb_gemfile/gemfile.rb", + include_str!("../../../gems/rb-gemfile/lib/rb_gemfile/gemfile.rb"), + ), + ( + "lib/rb_gemfile/sources.rb", + include_str!("../../../gems/rb-gemfile/lib/rb_gemfile/sources.rb"), + ), + ( + "lib/rb_gemfile/gemspec.rb", + include_str!("../../../gems/rb-gemfile/lib/rb_gemfile/gemspec.rb"), + ), + ( + "lib/rb_gemfile/compatibility.rb", + include_str!("../../../gems/rb-gemfile/lib/rb_gemfile/compatibility.rb"), ), ]; diff --git a/crates/rb-gemfile/src/snapshot.rs b/crates/rb-gemfile/src/snapshot.rs index ac773fc..e9654a9 100644 --- a/crates/rb-gemfile/src/snapshot.rs +++ b/crates/rb-gemfile/src/snapshot.rs @@ -18,6 +18,7 @@ pub struct SnapshotCache { path: PathBuf, inputs: Vec, outputs: Vec, + output_directories: Vec, } #[derive(Debug)] @@ -40,6 +41,7 @@ impl SnapshotCache { path: path.into(), inputs: Vec::new(), outputs: Vec::new(), + output_directories: Vec::new(), } } @@ -55,6 +57,11 @@ impl SnapshotCache { self } + pub fn output_directory(mut self, path: impl Into) -> Self { + self.output_directories.push(path.into()); + self + } + pub fn evaluate( &self, evaluator: &Evaluator, @@ -81,6 +88,21 @@ impl SnapshotCache { )); } let mut excluded = vec![path.clone()]; + for output in &self.output_directories { + let output = absolute(output)?; + if output == directory + || gemfile.starts_with(&output) + || ruby.starts_with(&output) + || inputs.iter().any(|input| input.starts_with(&output)) + || (output.exists() && !output.is_dir()) + { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Generated directory must not contain explicit inputs", + )); + } + excluded.push(output); + } for output in &self.outputs { let output = absolute(output)?; if output.is_dir() || output == gemfile || output == ruby || inputs.contains(&output) { @@ -126,7 +148,7 @@ impl SnapshotCache { } let evaluation = evaluator.evaluate(&gemfile)?; let covered = evaluation.inputs.iter().all(|path| { - !excluded.contains(path) + !excluded.iter().any(|output| path.starts_with(output)) && (path.starts_with(directory) || inputs .iter() diff --git a/crates/rb-gemfile/tests/evaluation.rs b/crates/rb-gemfile/tests/evaluation.rs index c5f04ad..00b2d7f 100644 --- a/crates/rb-gemfile/tests/evaluation.rs +++ b/crates/rb-gemfile/tests/evaluation.rs @@ -168,10 +168,6 @@ fn rejects_unsupported_or_ambiguous_declarations() { ), ("gem 'rake', git: 'url', path: '.'", "Conflicting sources"), ("gem 'rake', branch: 'main'", "Git options require"), - ( - "gem 'rake', git: 'url', branch: 'main', tag: 'v1'", - "Conflicting Git selectors", - ), ("eval_gemfile 'Gemfile'", "Recursive eval_gemfile"), ("source 'one'; source 'two'", "Multiple global sources"), ( @@ -703,3 +699,65 @@ fn compatibility_helpers_preserve_manifest_and_input_tracking() { evaluation.manifest ); } + +#[test] +fn rejects_combined_git_selectors_and_git_options_without_git() { + for declaration in [ + "gem 'fixture', git: 'url', branch: 'main', tag: 'v1'", + "gem 'fixture', git: 'url', branch: 'main', ref: 'abc'", + "gem 'fixture', git: 'url', tag: 'v1', ref: 'abc'", + "gem 'fixture', ref: 'abc'", + "gem 'fixture', tag: 'v1'", + "gem 'fixture', submodules: true", + ] { + let root = tempfile::tempdir().unwrap(); + write(root.path(), "Gemfile", declaration); + let error = evaluator() + .evaluate(root.path().join("Gemfile")) + .unwrap_err(); + let expected = if declaration.contains("git:") { + "Conflicting Git selectors" + } else { + "Git options require" + }; + assert!(error.to_string().contains(expected), "{error}"); + } + for options in [ + "branch='main',tag='v1'", + "branch='main',rev='abc'", + "tag='v1',rev='abc'", + ] { + let toml = format!("[dependencies]\nfixture={{git='url',{options}}}\n"); + assert!( + toml::from_str::(&toml) + .unwrap_err() + .to_string() + .contains("Specify only one") + ); + let kdl = format!( + "gem \"fixture\" git=\"url\" {}\n", + options.replace(',', " ").replace('\'', "\"") + ); + assert!( + Manifest::from_kdl(&kdl) + .unwrap_err() + .to_string() + .contains("Specify only one") + ); + } +} + +#[test] +fn plugin_dsl_guard_keeps_normal_gemfile_dependencies() { + let root = tempfile::tempdir().unwrap(); + write( + root.path(), + "Gemfile", + include_str!("fixtures/ruby/plugin_dsl_guard.rb"), + ); + let manifest = evaluator() + .evaluate(root.path().join("Gemfile")) + .unwrap() + .manifest; + assert_eq!(manifest.dependencies["fixture"].version, ["~> 3.0"]); +} diff --git a/crates/rb-gemfile/tests/fixtures/ruby/plugin_dsl_guard.rb b/crates/rb-gemfile/tests/fixtures/ruby/plugin_dsl_guard.rb new file mode 100644 index 0000000..a3e6037 --- /dev/null +++ b/crates/rb-gemfile/tests/fixtures/ruby/plugin_dsl_guard.rb @@ -0,0 +1,2 @@ +return if is_a?(Bundler::Plugin::DSL) +gem "fixture", "~> 3.0" diff --git a/crates/rb-gemfile/tests/snapshot.rs b/crates/rb-gemfile/tests/snapshot.rs index caf2c64..efaece8 100644 --- a/crates/rb-gemfile/tests/snapshot.rs +++ b/crates/rb-gemfile/tests/snapshot.rs @@ -422,3 +422,47 @@ fn changing_condition_inputs_updates_exported_dependencies() { } } } + +#[test] +fn generated_directory_is_excluded_but_cannot_hide_gemfile_inputs() { + let root = tempfile::tempdir().unwrap(); + write( + root.path(), + "Gemfile", + "source 'https://rubygems.org'\ngem 'rake'\n", + ); + let generated = root.path().join(".rb"); + let cache = SnapshotCache::new(generated.join("snapshot.json")).output_directory(&generated); + assert!( + !cache + .evaluate(&evaluator(), root.path().join("Gemfile")) + .unwrap() + .reused + ); + write(root.path(), ".rb/installed/data", "generated\n"); + assert!( + cache + .evaluate(&evaluator(), root.path().join("Gemfile")) + .unwrap() + .reused + ); + let invalid = SnapshotCache::new(generated.join("snapshot.json")).output_directory(root.path()); + assert!( + invalid + .evaluate(&evaluator(), root.path().join("Gemfile")) + .is_err() + ); + write( + root.path(), + "Gemfile", + "Bundler.read_file('.rb/installed/data')\nsource 'https://rubygems.org'\ngem 'rake'\n", + ); + for _ in 0..2 { + assert!( + !cache + .evaluate(&evaluator(), root.path().join("Gemfile")) + .unwrap() + .reused + ); + } +} diff --git a/crates/rb-packages/Cargo.toml b/crates/rb-packages/Cargo.toml new file mode 100644 index 0000000..c8e012e --- /dev/null +++ b/crates/rb-packages/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "rb-packages" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Standalone Ruby bundle planning and conventional installation" + +[dependencies] +anyhow = "1" +dunce = "1" +rb-compact-index = { path = "../rb-compact-index" } +rb-gem-archive = { path = "../rb-gem-archive" } +rb-gem-installer = { path = "../rb-gem-installer" } +rb-gem-types = { path = "../rb-gem-types" } +rb-gemfile = { path = "../rb-gemfile" } +rb-lockfile = { path = "../rb-lockfile" } +rb-microgems = { path = "../rb-microgems" } +rb-resolver = { path = "../rb-resolver" } +rb-task = { path = "../rb-task" } +serde = { version = "1", features = ["derive"] } +serde_json = "1" +serde_yaml_ng = "0.10" +sha2 = "0.10" +tempfile = "3" +toml = "0.8" +url = "2" + +flate2 = "1" +tar = "0.4" diff --git a/crates/rb-packages/README.md b/crates/rb-packages/README.md new file mode 100644 index 0000000..073301e --- /dev/null +++ b/crates/rb-packages/README.md @@ -0,0 +1,35 @@ +# rb-packages + +Standalone Ruby bundle preparation: read `rbproject.toml`/KDL or a trusted Gemfile, resolve dependencies, and install registry, Git and path gems into `.rb`, including native extensions. Work reports through `rb-task` events. The CLI is not connected yet. + +```rust,no_run +use rb_packages::{PrepareOptions, prepare}; +use rb_task::TaskEvents; + +let bundle = prepare( + PrepareOptions::new("my-app", "ruby", "gem-cache"), + TaskEvents::default(), +)?; +let environment = bundle.installed.environment(); +# Ok::<(), anyhow::Error>(()) +``` + +The returned environment supports real RubyGems and Bundler. Bundler does not resolve or install; RubyGems still handles runtime inspection and specification serialization. Gemfile conversion is cached under `.rb`; Gemfiles must not read generated bundle files. Path gems use snapshots - prepare again after edits. Git/path sources require gemspecs. + +Existing lockfile platforms are resolved together when the lock needs refreshing; only the local platform is installed. + +Transitive registry dependencies prefer the parent source, then fall back to the global source when absent. Competing scoped sources require an explicit declaration. + +Registry authentication shims Bundler's local config, environment and global config lookup, using source URL or hostname keys; Bundler is not loaded. Native command credentials take priority for an exact source URL. Configure them in `$XDG_CONFIG_HOME/rb/auth.toml` (default `~/.config/rb/auth.toml`; `%APPDATA%/rb/auth.toml` on Windows): + +```toml +[auth."https://rubygems.pkg.github.com/OWNER/"] +command = ["gh", "auth", "token", "--hostname", "github.com", "--user", "USERNAME"] +username = "USERNAME" +``` + +Commands run directly, without a shell, from the config directory. Stdout supplies the token for HTTP Basic authentication; credentials stay in memory for the invocation and out of source URLs and lockfiles. Offline preparation does not run credential commands. + +Credential stores, platform-specific Git/path sources, custom lock paths, plugins and MicroGems application runtime are not supported yet. + +Run `cargo test -p rb-packages`, or try `cargo run -p rb-packages --example prepare -- /path/to/app /path/to/ruby /path/to/cache`. diff --git a/crates/rb-packages/examples/corpus.rs b/crates/rb-packages/examples/corpus.rs new file mode 100644 index 0000000..83af6e5 --- /dev/null +++ b/crates/rb-packages/examples/corpus.rs @@ -0,0 +1,131 @@ +use anyhow::{Result, ensure}; +use rb_gemfile::Evaluator; +use serde_json::{Value, json}; +use std::{ + fs, + path::{Path, PathBuf}, + sync::{Arc, Mutex}, +}; + +fn files(root: &Path, output: &mut Vec) -> Result<()> { + for entry in fs::read_dir(root)? { + let path = entry?.path(); + if path.is_dir() { + files(&path, output)?; + } else { + output.push(path); + } + } + Ok(()) +} + +fn classify(error: &str) -> &'static str { + if error.contains("Git options require") || error.contains("Conflicting Git selectors") { + "unsupported Git option combination" + } else if error.contains("No such file") + || error.contains("gemspec") + || error.contains("cannot load such file") + { + "missing companion or gemspec" + } else if error.contains("Ruby versions") { + "older Ruby required" + } else if error.contains("exists?") { + "removed Ruby API" + } else if error.contains("database.yml") || error.contains("ENV") { + "application configuration required" + } else if error.contains("Bundler::Plugin") { + "plugin DSL unsupported" + } else { + "evaluation failure" + } +} + +fn canonical(mut lock: rb_lockfile::Lockfile) -> rb_lockfile::Lockfile { + lock.platforms.sort(); + for source in &mut lock.sources { + source.remotes.sort(); + source.packages.sort_by_key(|package| package.identity()); + } + lock.sources + .sort_by(|a, b| (a.kind, &a.remotes, &a.options).cmp(&(b.kind, &b.remotes, &b.options))); + lock +} + +fn main() -> Result<()> { + ensure!( + std::env::var_os("BUNDLE_GEMFILE").is_none(), + "Unset BUNDLE_GEMFILE for a corpus run" + ); + let mut arguments = std::env::args_os().skip(1); + let root = PathBuf::from(arguments.next().expect("Pass the gemfile-corpus directory")); + let ruby = PathBuf::from(arguments.next().unwrap_or_else(|| "ruby".into())); + let mut gemfiles = Vec::new(); + files(&root.join("gemfiles"), &mut gemfiles)?; + gemfiles.retain(|p| { + p.file_name() + .is_some_and(|n| n == "Gemfile" || n == "gems.rb") + }); + gemfiles.sort(); + let total = gemfiles.len(); + let input = Arc::new(Mutex::new(gemfiles.into_iter())); + let results = Arc::new(Mutex::new(Vec::new())); + std::thread::scope(|scope| { + for _ in 0..4 { + let input = input.clone(); + let results = results.clone(); + let ruby = ruby.clone(); + let root = root.clone(); + scope.spawn(move || loop { + let Some(path) = input.lock().unwrap().next() else { break; }; + let result = (|| -> Result { + let project = tempfile::tempdir()?; + fs::copy(&path, project.path().join(path.file_name().unwrap()))?; + let input = Evaluator::new(&ruby).evaluate(project.path().join(path.file_name().unwrap()))?; + let exported = input.manifest.to_toml()?; + let roundtrip: rb_gemfile::Manifest = toml::from_str(&exported)?; + ensure!(roundtrip.to_toml()? == exported, "Manifest roundtrip changed"); + let git = input.manifest.dependencies.values().any(|d| d.git.is_some() || d.variants.iter().any(|d| d.git.is_some())); + let local = !input.manifest.gemspecs.is_empty() || input.manifest.dependencies.values().any(|d| d.path.is_some() || d.variants.iter().any(|d| d.path.is_some())); + Ok(json!({"status":"loaded", "dependencies":input.manifest.dependencies.len(), "git":git, "path":local})) + })(); + let row = match result { + Ok(value) => json!({"path":path.strip_prefix(&root).unwrap(), "result":value}), + Err(error) => { let error = format!("{error:#}"); json!({"path":path.strip_prefix(&root).unwrap(), "result":{"status":"failed", "category":classify(&error), "error":error}}) } + }; + results.lock().unwrap().push(row); + }); + } + }); + let mut results = Arc::try_unwrap(results).unwrap().into_inner().unwrap(); + results.sort_by_key(|r| r["path"].as_str().unwrap().to_owned()); + let loaded = results + .iter() + .filter(|r| r["result"]["status"] == "loaded") + .count(); + let mut locks = Vec::new(); + files(&root.join("lockfiles"), &mut locks)?; + locks.sort(); + let mut lock_failures = Vec::new(); + for path in &locks { + let result = (|| -> Result<()> { + let lock = rb_lockfile::Lockfile::parse(&fs::read_to_string(path)?)?; + let rendered = lock.render()?; + ensure!( + canonical(rb_lockfile::Lockfile::parse(&rendered)?) == canonical(lock), + "Lockfile roundtrip changed" + ); + Ok(()) + })(); + if let Err(error) = result { + lock_failures + .push(json!({"path":path.strip_prefix(&root)?, "error":format!("{error:#}")})); + } + } + println!( + "{}", + serde_json::to_string_pretty( + &json!({"gemfiles":{"total":total,"loaded":loaded,"failed":total-loaded,"results":results},"lockfiles":{"total":locks.len(),"passed":locks.len()-lock_failures.len(),"failures":lock_failures}}) + )? + ); + Ok(()) +} diff --git a/crates/rb-packages/examples/prepare.rs b/crates/rb-packages/examples/prepare.rs new file mode 100644 index 0000000..a563b32 --- /dev/null +++ b/crates/rb-packages/examples/prepare.rs @@ -0,0 +1,59 @@ +use rb_packages::{PrepareOptions, prepare}; +use rb_task::{TaskEvent, TaskEventSink, TaskEvents}; +use std::{path::PathBuf, sync::Arc}; + +struct Output; +impl TaskEventSink for Output { + fn event(&self, event: TaskEvent) { + match event { + TaskEvent::Started { worker, label, .. } => eprintln!("worker {worker}: {label}"), + TaskEvent::Output { worker, line, .. } => eprintln!("worker {worker}: {line}"), + _ => {} + } + } +} + +fn main() -> anyhow::Result<()> { + let mut arguments = std::env::args_os().skip(1); + let project = PathBuf::from(arguments.next().unwrap_or_else(|| ".".into())); + let ruby = PathBuf::from(arguments.next().unwrap_or_else(|| "ruby".into())); + let cache = PathBuf::from( + arguments + .next() + .unwrap_or_else(|| project.join(".rb/cache").into()), + ); + let mut options = PrepareOptions::new(project, ruby, cache); + if let Ok(seconds) = std::env::var("RB_GEM_BUILD_TIMEOUT") { + options.install.build.timeout = std::time::Duration::from_secs(seconds.parse()?); + } + if let Ok(jobs) = std::env::var("RB_GEM_BUILD_JOBS") { + options.install.build.jobs = jobs.parse()?; + } + if let Some(path) = std::env::var_os("RB_GEM_BUILD_DEPENDENCIES") { + options.install.build.dependencies = serde_json::from_slice(&std::fs::read(path)?)?; + } + options.offline = std::env::var_os("RB_PACKAGES_OFFLINE").is_some(); + options.plan.mode = if std::env::var_os("RB_PACKAGES_LOCKED").is_some() { + rb_packages::LockMode::Locked + } else { + rb_packages::LockMode::Prefer + }; + options + .install + .build + .environment + .insert("CCACHE_DISABLE".into(), "1".into()); + let events = TaskEvents::default(); + events.subscribe(Arc::new(Output)); + let result = prepare(options, events)?; + println!( + "{} installed, {} reused in {}", + result.installed.installed, + result.installed.reused, + result.installed.gem_home.display() + ); + for (key, value) in result.installed.environment() { + println!("{key}={value}"); + } + Ok(()) +} diff --git a/crates/rb-packages/src/bundler_credentials.rs b/crates/rb-packages/src/bundler_credentials.rs new file mode 100644 index 0000000..452e645 --- /dev/null +++ b/crates/rb-packages/src/bundler_credentials.rs @@ -0,0 +1,206 @@ +use anyhow::{Context, Result}; +use std::{ + collections::BTreeMap, + fs, + path::{Path, PathBuf}, +}; + +pub(crate) struct Settings { + layers: Vec>, +} + +fn key(value: &str) -> String { + format!( + "BUNDLE_{}", + value.replace('.', "__").replace('-', "___").to_uppercase() + ) +} + +fn read(path: Option) -> Result> { + let Some(path) = path else { + return Ok(BTreeMap::new()); + }; + let text = match fs::read_to_string(&path) { + Ok(text) => text, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(BTreeMap::new()), + Err(error) => return Err(error).context("Cannot read Bundler credentials config"), + }; + if text.trim().is_empty() { + return Ok(BTreeMap::new()); + } + let values: BTreeMap = serde_yaml_ng::from_str(&text) + .map_err(|_| anyhow::anyhow!("Invalid Bundler credentials config"))?; + Ok(values + .into_iter() + .filter_map(|(mut name, value)| { + let value = value.as_str()?.to_owned(); + if (name.to_lowercase().contains("https:") || name.to_lowercase().contains("http:")) + && !name.ends_with('/') + { + name.push('/'); + } + Some((name.replace('.', "__").replace('-', "___"), value)) + }) + .collect()) +} + +impl Settings { + pub(crate) fn from_environment( + project: &Path, + environment: BTreeMap, + ) -> Result { + let setting = |name: &str| { + environment + .get(name) + .filter(|value| !value.is_empty()) + .map(PathBuf::from) + }; + let local = project + .join(setting("BUNDLE_APP_CONFIG").unwrap_or_else(|| ".bundle".into())) + .join("config"); + let global = setting("BUNDLE_CONFIG") + .or_else(|| setting("BUNDLE_USER_CONFIG")) + .or_else(|| setting("BUNDLE_USER_HOME").map(|home| home.join("config"))) + .or_else(|| { + setting("HOME") + .or_else(|| setting("USERPROFILE")) + .map(|home| home.join(".bundle/config")) + }); + let ignore = environment.contains_key("BUNDLE_IGNORE_CONFIG"); + Ok(Self { + layers: vec![ + read((!ignore).then_some(local))?, + environment, + read(if ignore { None } else { global })?, + ], + }) + } + + pub(crate) fn credentials(&self, source: &str) -> Result> { + let url = url::Url::parse(source)?; + for name in [source, url.host_str().context("Gem source has no host")?] { + let key = key(name); + if let Some(value) = self.layers.iter().find_map(|layer| layer.get(&key)) { + return Ok(Some(value)); + } + } + Ok(None) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn credentials_follow_bundler_precedence_and_source_scope() { + let root = tempfile::tempdir().unwrap(); + fs::create_dir(root.path().join(".bundle")).unwrap(); + fs::write( + root.path().join(".bundle/config"), + "BUNDLE_GEMS__EXAMPLE___COM: local\n", + ) + .unwrap(); + let global = root.path().join("global"); + fs::write(&global, "BUNDLE_GEMS__EXAMPLE___COM: global\nBUNDLE_HTTPS://GEMS__EXAMPLE___COM/TEAM/: user:source-token\n").unwrap(); + let mut env = BTreeMap::from([ + ("BUNDLE_CONFIG".into(), global.to_str().unwrap().into()), + ("BUNDLE_GEMS__EXAMPLE___COM".into(), "environment".into()), + ]); + let settings = Settings::from_environment(root.path(), env.clone()).unwrap(); + assert_eq!( + settings.credentials("https://gems.example-com/").unwrap(), + Some("local") + ); + assert_eq!( + settings + .credentials("https://gems.example-com/team/") + .unwrap(), + Some("user:source-token") + ); + assert_eq!( + settings.credentials("https://other.example/").unwrap(), + None + ); + env.insert("BUNDLE_IGNORE_CONFIG".into(), "1".into()); + let settings = Settings::from_environment(root.path(), env).unwrap(); + assert_eq!( + settings + .credentials("https://gems.example-com/team/") + .unwrap(), + Some("environment") + ); + } + + #[test] + fn malformed_config_does_not_disclose_its_contents() { + let root = tempfile::tempdir().unwrap(); + fs::create_dir(root.path().join(".bundle")).unwrap(); + fs::write(root.path().join(".bundle/config"), "[secret-token").unwrap(); + let error = Settings::from_environment(root.path(), BTreeMap::new()) + .err() + .unwrap(); + assert_eq!(error.to_string(), "Invalid Bundler credentials config"); + } + + #[test] + fn config_paths_and_environment_overrides_follow_bundler() { + let root = tempfile::tempdir().unwrap(); + let home = root.path().join("home"); + fs::create_dir_all(home.join(".bundle")).unwrap(); + fs::write( + home.join(".bundle/config"), + "BUNDLE_GEMS__EXAMPLE: user:global\n", + ) + .unwrap(); + let mut env = BTreeMap::from([("HOME".into(), home.to_str().unwrap().into())]); + let credentials = |env| { + Settings::from_environment(root.path(), env) + .unwrap() + .credentials("https://gems.example/") + .unwrap() + .map(str::to_owned) + }; + assert_eq!(credentials(env.clone()).as_deref(), Some("user:global")); + for variable in ["BUNDLE_USER_HOME", "BUNDLE_USER_CONFIG", "BUNDLE_CONFIG"] { + let directory = root.path().join(variable); + fs::create_dir(&directory).unwrap(); + let file = directory.join("config"); + fs::write(&file, format!("BUNDLE_GEMS__EXAMPLE: {variable}\n")).unwrap(); + env.insert( + variable.into(), + if variable == "BUNDLE_USER_HOME" { + directory + } else { + file + } + .to_str() + .unwrap() + .into(), + ); + assert_eq!(credentials(env.clone()).as_deref(), Some(variable)); + } + env.insert("BUNDLE_GEMS__EXAMPLE".into(), "user:environment".into()); + assert_eq!( + credentials(env.clone()).as_deref(), + Some("user:environment") + ); + for location in [ + PathBuf::from("custom-config"), + root.path().join("absolute-config"), + ] { + let directory = root.path().join(&location); + fs::create_dir(&directory).unwrap(); + fs::write( + directory.join("config"), + "BUNDLE_GEMS__EXAMPLE: user:local\n", + ) + .unwrap(); + env.insert( + "BUNDLE_APP_CONFIG".into(), + location.to_str().unwrap().into(), + ); + assert_eq!(credentials(env.clone()).as_deref(), Some("user:local")); + } + } +} diff --git a/crates/rb-packages/src/constants.rs b/crates/rb-packages/src/constants.rs new file mode 100644 index 0000000..0992510 --- /dev/null +++ b/crates/rb-packages/src/constants.rs @@ -0,0 +1,37 @@ +pub(crate) const DEFAULT_SOURCE: &str = "https://rubygems.org/"; +pub(crate) const STATE_DIRECTORY: &str = ".rb"; +pub(crate) const AUTH_CONFIG_DIRECTORY: &str = "rb"; +pub(crate) const AUTH_CONFIG_FILE: &str = "auth.toml"; +pub(crate) const GEM_HOME_DIRECTORY: &str = "bundle"; +pub(crate) const SNAPSHOT_FILE: &str = "gemfile.snapshot.json"; +pub(crate) const MANIFEST_FILE: &str = "rbproject.toml"; +pub(crate) const INSTALL_RECEIPT: &str = "installation.json"; +pub(crate) const DEFAULT_WORKERS: usize = 4; +pub(crate) const SPECIFICATIONS_DIRECTORY: &str = "specifications"; +pub(crate) const GEMS_DIRECTORY: &str = "gems"; +pub(crate) const EXTENSIONS_DIRECTORY: &str = "extensions"; +pub(crate) const BIN_DIRECTORY: &str = "bin"; +pub(crate) const BUILD_COMPLETE_MARKER: &str = "gem.build_complete"; +pub(crate) const PROJECT_LOCK: &str = "prepare.lock"; +pub(crate) const INSTALL_LOCK: &str = ".install.lock"; +pub(crate) const CHECKSUM_CHARACTERS: usize = 64; +pub(crate) const READ_BUFFER_BYTES: usize = 8192; +pub(crate) const MAX_OUTPUT_LINE_BYTES: usize = 1024 * 1024; + +pub(crate) const DEFAULT_GEMSPEC_GLOB: &str = "{,*,*/*}.gemspec"; + +pub(crate) const GIT_OBJECT_ID_LENGTHS: [usize; 2] = [40, 64]; +pub(crate) const GIT_SHORT_REVISION_LENGTH: usize = 12; +pub(crate) const GIT_CACHE_DIRECTORY: &str = "git"; +pub(crate) const GIT_REPOSITORY_DIRECTORY: &str = "repository.git"; +pub(crate) const GIT_CHECKOUTS_DIRECTORY: &str = "checkouts"; +pub(crate) const CACHE_LOCK_FILE: &str = "cache.lock"; +pub(crate) const DEFAULT_GIT_REFERENCE: &str = "refs/rb/default"; +pub(crate) const SOURCE_ARCHIVES_DIRECTORY: &str = "source-archives"; +pub(crate) const BUNDLER_DIRECTORY: &str = "bundler"; +pub(crate) const SOURCE_PATHS_DIRECTORY: &str = "paths"; +pub(crate) const SOURCE_STAMP_FILE: &str = ".rb-source"; +pub(crate) const RUNTIME_DIRECTORY: &str = "runtime"; +pub(crate) const GEMFILE_FILE: &str = "Gemfile"; +pub(crate) const GEM_METADATA_MEMBER: &str = "metadata.gz"; +pub(crate) const GEM_PAYLOAD_MEMBER: &str = "data.tar.gz"; diff --git a/crates/rb-packages/src/copy_tree.rs b/crates/rb-packages/src/copy_tree.rs new file mode 100644 index 0000000..c08e67e --- /dev/null +++ b/crates/rb-packages/src/copy_tree.rs @@ -0,0 +1,28 @@ +use anyhow::Result; +use std::{fs, path::Path}; + +pub(crate) fn copy_tree(source: &Path, destination: &Path) -> Result<()> { + fs::create_dir_all(destination)?; + for entry in fs::read_dir(source)? { + let entry = entry?; + let to = destination.join(entry.file_name()); + let kind = entry.file_type()?; + if kind.is_dir() { + copy_tree(&entry.path(), &to)?; + } else if kind.is_symlink() { + #[cfg(unix)] + std::os::unix::fs::symlink(fs::read_link(entry.path())?, to)?; + #[cfg(windows)] + { + if entry.path().is_dir() { + std::os::windows::fs::symlink_dir(fs::read_link(entry.path())?, to)?; + } else { + std::os::windows::fs::symlink_file(fs::read_link(entry.path())?, to)?; + } + } + } else { + fs::copy(entry.path(), to)?; + } + } + Ok(()) +} diff --git a/crates/rb-packages/src/credentials.rs b/crates/rb-packages/src/credentials.rs new file mode 100644 index 0000000..5822d5a --- /dev/null +++ b/crates/rb-packages/src/credentials.rs @@ -0,0 +1,127 @@ +use crate::{ + bundler_credentials, + constants::{AUTH_CONFIG_DIRECTORY, AUTH_CONFIG_FILE}, + provider::source_url, +}; +use anyhow::{Context, Result, ensure}; +use serde::Deserialize; +use std::{ + collections::BTreeMap, + fs, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct CommandCredential { + command: Vec, + username: String, +} + +#[derive(Default, Deserialize)] +struct Config { + #[serde(default)] + auth: BTreeMap, +} + +pub(crate) struct Credentials { + directory: Option, + commands: BTreeMap, + bundler: bundler_credentials::Settings, +} + +fn config_path(environment: &BTreeMap) -> Option { + let value = |name: &str| { + environment + .get(name) + .filter(|value| !value.is_empty()) + .map(PathBuf::from) + }; + let base = value("XDG_CONFIG_HOME").or_else(|| { + if cfg!(windows) { + value("APPDATA") + } else { + value("HOME").map(|home| home.join(".config")) + } + })?; + Some(base.join(AUTH_CONFIG_DIRECTORY).join(AUTH_CONFIG_FILE)) +} + +impl Credentials { + pub(crate) fn load(project: &Path) -> Result { + Self::from_environment( + project, + std::env::vars_os() + .filter_map(|(key, value)| { + Some((key.into_string().ok()?, value.into_string().ok()?)) + }) + .collect(), + ) + } + + fn from_environment(project: &Path, environment: BTreeMap) -> Result { + let path = config_path(&environment); + let config = if let Some(path) = &path { + match fs::read_to_string(path) { + Ok(text) => toml::from_str::(&text) + .map_err(|_| anyhow::anyhow!("Invalid rb authentication config"))?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Config::default(), + Err(error) => return Err(error).context("Cannot read rb authentication config"), + } + } else { + Config::default() + }; + let mut commands = BTreeMap::new(); + for (source, credential) in config.auth { + ensure!( + !credential.command.is_empty() && !credential.command[0].is_empty(), + "Credential command must not be empty" + ); + ensure!( + !credential.username.is_empty() && !credential.username.contains([':', '\r', '\n']), + "Invalid credential username" + ); + let source = + source_url(&source).map_err(|_| anyhow::anyhow!("Invalid credential source"))?; + ensure!( + commands.insert(source, credential).is_none(), + "Duplicate credential source" + ); + } + Ok(Self { + directory: path.and_then(|path| path.parent().map(Path::to_path_buf)), + commands, + bundler: bundler_credentials::Settings::from_environment(project, environment)?, + }) + } + + pub(crate) fn credentials(&self, source: &str) -> Result> { + let source = source_url(source)?; + let Some(credential) = self.commands.get(&source) else { + return Ok(self.bundler.credentials(&source)?.map(str::to_owned)); + }; + let output = Command::new(&credential.command[0]) + .args(&credential.command[1..]) + .current_dir( + self.directory + .as_ref() + .context("Missing authentication config directory")?, + ) + .stdin(Stdio::null()) + .output() + .map_err(|_| anyhow::anyhow!("Cannot run credential command"))?; + ensure!(output.status.success(), "Credential command failed"); + let token = std::str::from_utf8(&output.stdout) + .map_err(|_| anyhow::anyhow!("Credential command returned invalid UTF-8"))? + .trim_end_matches(['\r', '\n']); + ensure!( + !token.is_empty() && !token.contains(['\r', '\n']), + "Credential command must return one nonempty token line" + ); + Ok(Some(format!("{}:{token}", credential.username))) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/rb-packages/src/credentials/tests.rs b/crates/rb-packages/src/credentials/tests.rs new file mode 100644 index 0000000..a2ddf2e --- /dev/null +++ b/crates/rb-packages/src/credentials/tests.rs @@ -0,0 +1,137 @@ +use super::*; + +fn setup(code: &str) -> (tempfile::TempDir, BTreeMap) { + let root = tempfile::tempdir().unwrap(); + let directory = root.path().join("rb"); + fs::create_dir(&directory).unwrap(); + fs::write(directory.join("credential.rb"), code).unwrap(); + let ruby = std::env::var_os("RB_GEM_BUILD_RUBY").unwrap_or_else(|| "ruby".into()); + let text = toml::to_string(&serde_json::json!({ + "auth": { "https://gems.example/team": { + "command": [ruby.to_str().unwrap(), "credential.rb", "token; literal argument"], + "username": "account" + }} + })) + .unwrap(); + fs::write(directory.join(AUTH_CONFIG_FILE), text).unwrap(); + let env = BTreeMap::from([ + ( + "XDG_CONFIG_HOME".into(), + root.path().to_str().unwrap().into(), + ), + ("BUNDLE_GEMS__EXAMPLE".into(), "bundler:token".into()), + ]); + (root, env) +} + +#[test] +fn command_credentials_override_bundler_only_for_the_configured_source() { + let (root, env) = setup("print ARGV.fetch(0), \"\\r\\n\"\n"); + let credentials = Credentials::from_environment(root.path(), env).unwrap(); + assert_eq!( + credentials + .credentials("https://gems.example/team/") + .unwrap() + .as_deref(), + Some("account:token; literal argument") + ); + assert_eq!( + credentials + .credentials("https://gems.example/other/") + .unwrap() + .as_deref(), + Some("bundler:token") + ); + assert_eq!( + credentials + .credentials("https://unrelated.example/") + .unwrap(), + None + ); +} + +#[test] +fn commands_run_only_for_matched_sources_and_fail_without_disclosing_output() { + for (code, message) in [ + ( + "warn 'secret'; print 'secret'; exit 1", + "Credential command failed", + ), + ( + "print ''", + "Credential command must return one nonempty token line", + ), + ( + "print \"secret\\nsecond\\n\"", + "Credential command must return one nonempty token line", + ), + ( + "STDOUT.binmode; print [255].pack('C')", + "Credential command returned invalid UTF-8", + ), + ] { + let (root, env) = setup(code); + let credentials = Credentials::from_environment(root.path(), env).unwrap(); + assert_eq!( + credentials + .credentials("https://unrelated.example/") + .unwrap(), + None + ); + assert_eq!( + format!( + "{:#}", + credentials + .credentials("https://gems.example/team/") + .unwrap_err() + ), + message + ); + } +} + +#[test] +fn config_uses_rb_user_directory_and_rejects_invalid_entries() { + let home = BTreeMap::from([("HOME".into(), "/home/example".into())]); + if !cfg!(windows) { + assert_eq!( + config_path(&home), + Some(PathBuf::from("/home/example/.config/rb/auth.toml")) + ); + } + assert!(config_path(&BTreeMap::new()).is_none()); + for text in [ + "secret = [", + "[auth.\"https://gems.example/\"]\ncommand = []\nusername = 'user'", + "[auth.\"https://gems.example/\"]\ncommand = ['missing']\nusername = 'bad:user'", + "[auth.\"https://user:secret@gems.example/\"]\ncommand = ['missing']\nusername = 'user'", + ] { + let (root, env) = setup(""); + fs::write(root.path().join("rb/auth.toml"), text).unwrap(); + let error = Credentials::from_environment(root.path(), env) + .err() + .unwrap(); + assert!(!format!("{error:#}").contains("secret")); + } +} + +#[test] +fn commands_are_lazy_and_offline_registry_setup_needs_no_credentials() { + let (root, env) = setup("File.write('ran', 'yes'); puts 'token'"); + let credentials = Credentials::from_environment(root.path(), env).unwrap(); + assert!(!root.path().join("rb/ran").exists()); + assert_eq!( + credentials + .credentials("https://gems.example/team/") + .unwrap() + .as_deref(), + Some("account:token") + ); + assert!(root.path().join("rb/ran").exists()); + let manifest: rb_gemfile::Manifest = + toml::from_str("[bundle]\nsource = 'https://gems.example/'").unwrap(); + let mut index = crate::RegistryIndex::new(&manifest, root.path(), true).unwrap(); + fs::create_dir(root.path().join(".bundle")).unwrap(); + fs::write(root.path().join(".bundle/config"), "[invalid").unwrap(); + index.load_credentials(root.path(), true).unwrap(); +} diff --git a/crates/rb-packages/src/gemfile.rs b/crates/rb-packages/src/gemfile.rs new file mode 100644 index 0000000..2aafbcc --- /dev/null +++ b/crates/rb-packages/src/gemfile.rs @@ -0,0 +1,112 @@ +use anyhow::Result; +use rb_gemfile::{Manifest, Require}; + +pub(crate) fn render(manifest: &Manifest) -> Result { + let needs_default = manifest.dependencies.values().any(|dependency| { + std::iter::once(dependency) + .chain(&dependency.variants) + .any(|declaration| { + declaration.enabled + && declaration.source.is_none() + && declaration.git.is_none() + && declaration.path.is_none() + }) + }); + let mut output = String::new(); + if let Some(source) = manifest + .bundle + .source + .as_deref() + .or_else(|| needs_default.then_some(crate::constants::DEFAULT_SOURCE)) + { + output.push_str(&format!("source {}\n", literal(source))); + } + if !manifest.bundle.ruby.is_empty() { + let mut arguments: Vec<_> = manifest.bundle.ruby.iter().map(|v| literal(v)).collect(); + for (key, value) in [ + ("engine", &manifest.bundle.engine), + ("engine_version", &manifest.bundle.engine_version), + ("patchlevel", &manifest.bundle.patchlevel), + ] { + if let Some(value) = value { + arguments.push(format!("{key}: {}", literal(value))); + } + } + output.push_str(&format!("ruby {}\n", arguments.join(", "))); + } + for group in &manifest.bundle.optional_groups { + output.push_str(&format!( + "group {}, optional: true do\nend\n", + literal(group) + )); + } + for (name, dependency) in &manifest.dependencies { + for declaration in std::iter::once(dependency) + .chain(&dependency.variants) + .filter(|d| d.enabled) + { + declaration.validate().map_err(anyhow::Error::msg)?; + + let mut arguments = vec![literal(name)]; + arguments.extend(declaration.version.iter().map(|v| literal(v))); + arguments.push(format!( + "groups: [{}]", + declaration + .groups + .iter() + .map(|v| literal(v)) + .collect::>() + .join(", ") + )); + if !declaration.platforms.is_empty() { + arguments.push(format!( + "platforms: [{}]", + declaration + .platforms + .iter() + .map(|v| literal(v)) + .collect::>() + .join(", ") + )); + } + if let Some(source) = &declaration.source { + arguments.push(format!("source: {}", literal(source))); + } + for (key, value) in [ + ("git", &declaration.git), + ("path", &declaration.path), + ("branch", &declaration.branch), + ("tag", &declaration.tag), + ("ref", &declaration.rev), + ("glob", &declaration.glob), + ] { + if let Some(value) = value { + arguments.push(format!("{key}: {}", literal(value))); + } + } + if let Some(value) = declaration.submodules { + arguments.push(format!("submodules: {value}")); + } + if let Some(require) = &declaration.require { + let value = match require { + Require::Enabled(v) => v.to_string(), + Require::Name(v) => literal(v), + Require::Names(v) => format!( + "[{}]", + v.iter().map(|v| literal(v)).collect::>().join(", ") + ), + }; + arguments.push(format!("require: {value}")); + } + if let Some(force) = declaration.force_ruby_platform { + arguments.push(format!("force_ruby_platform: {force}")); + } + output.push_str(&format!("gem {}\n", arguments.join(", "))); + } + } + Ok(output) +} + +fn literal(value: &str) -> String { + format!("'{}'", value.replace('\\', "\\\\").replace('\'', "\\'")) +} diff --git a/crates/rb-packages/src/git_cache.rs b/crates/rb-packages/src/git_cache.rs new file mode 100644 index 0000000..bb72762 --- /dev/null +++ b/crates/rb-packages/src/git_cache.rs @@ -0,0 +1,196 @@ +use anyhow::{Context, Result, ensure}; +use sha2::{Digest, Sha256}; +use std::{ + fs, + path::{Path, PathBuf}, + process::Command, +}; + +pub(crate) fn run(command: &mut Command) -> Result { + let output = command + .env("GIT_TERMINAL_PROMPT", "0") + .output() + .context("Running source command")?; + ensure!( + output.status.success(), + "Source command failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + Ok(String::from_utf8(output.stdout)?.trim().to_owned()) +} + +pub(crate) fn checkout( + root: &Path, + remote: &str, + options: &std::collections::BTreeMap, + cache: &Path, + pinned: Option<&str>, + offline: bool, +) -> Result<(String, PathBuf)> { + ensure!( + !remote.is_empty() && !remote.starts_with('-') && !remote.chars().any(char::is_control), + "Invalid Git remote" + ); + let location = if Path::new(remote).is_absolute() || root.join(remote).exists() { + dunce::canonicalize(root.join(remote)) + .unwrap_or_else(|_| root.join(remote)) + .to_string_lossy() + .into_owned() + } else { + remote.to_owned() + }; + if !Path::new(&location).is_absolute() { + if let Ok(url) = url::Url::parse(&location) { + ensure!( + matches!(url.scheme(), "https" | "http" | "ssh" | "git" | "file"), + "Unsupported Git protocol" + ); + ensure!( + url.password().is_none() + && (!matches!(url.scheme(), "https" | "http") || url.username().is_empty()), + "Git URL credentials are not supported" + ); + } else { + ensure!( + location.contains(':') && !location.contains("::"), + "Invalid Git remote" + ); + } + } + let id = format!("{:x}", Sha256::digest(location.as_bytes())); + let directory = cache.join(crate::constants::GIT_CACHE_DIRECTORY).join(id); + fs::create_dir_all(&directory)?; + let guard = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(directory.join(crate::constants::CACHE_LOCK_FILE))?; + guard.lock()?; + let repository = directory.join(crate::constants::GIT_REPOSITORY_DIRECTORY); + if !repository.exists() { + ensure!(!offline, "Git repository is not cached for offline use"); + let temporary = tempfile::tempdir_in(&directory)?; + let target = temporary + .path() + .join(crate::constants::GIT_REPOSITORY_DIRECTORY); + run(Command::new("git") + .args(["clone", "--bare", "--"]) + .arg(&location) + .arg(&target))?; + fs::rename(target, &repository)?; + } + let git = |arguments: &[&str]| -> Result { + run(Command::new("git") + .arg("--git-dir") + .arg(&repository) + .args(arguments)) + }; + if pinned.is_none() && !offline { + git(&[ + "fetch", + "origin", + "+refs/heads/*:refs/heads/*", + "+refs/tags/*:refs/tags/*", + ])?; + } + let selector = pinned + .map(str::to_owned) + .or_else(|| options.get("ref").cloned()) + .or_else(|| { + options + .get("branch") + .map(|branch| format!("refs/heads/{branch}")) + }) + .or_else(|| options.get("tag").map(|tag| format!("refs/tags/{tag}"))) + .unwrap_or_else(|| crate::constants::DEFAULT_GIT_REFERENCE.into()); + ensure!( + !selector.starts_with('-') && !selector.chars().any(char::is_control), + "Invalid Git selector" + ); + if selector == crate::constants::DEFAULT_GIT_REFERENCE { + if !offline { + git(&["fetch", "origin", "HEAD"])?; + git(&[ + "update-ref", + crate::constants::DEFAULT_GIT_REFERENCE, + "FETCH_HEAD", + ])?; + } else if git(&[ + "rev-parse", + "--verify", + crate::constants::DEFAULT_GIT_REFERENCE, + ]) + .is_err() + { + git(&[ + "update-ref", + crate::constants::DEFAULT_GIT_REFERENCE, + "HEAD", + ])?; + } + } + let resolve = || git(&["rev-parse", "--verify", &format!("{selector}^{{commit}}")]); + let revision = match resolve() { + Ok(revision) => revision, + Err(error) if offline => return Err(error.context("Git commit is not cached")), + Err(_) => { + git(&["fetch", "origin", &selector])?; + resolve()? + } + }; + ensure!( + crate::constants::GIT_OBJECT_ID_LENGTHS.contains(&revision.len()) + && revision.bytes().all(|byte| byte.is_ascii_hexdigit()), + "Invalid Git object ID" + ); + git(&["update-ref", &format!("refs/rb/{revision}"), &revision])?; + let key = format!( + "{revision}-{}", + options.get("submodules").is_some_and(|v| v == "true") + ); + let checkout = directory + .join(crate::constants::GIT_CHECKOUTS_DIRECTORY) + .join(key); + if !checkout.exists() { + fs::create_dir_all(checkout.parent().unwrap())?; + let temporary = tempfile::tempdir_in(checkout.parent().unwrap())?; + let target = temporary.path().join("checkout"); + run(Command::new("git") + .args(["clone", "--shared", "--no-checkout", "--"]) + .arg(&repository) + .arg(&target))?; + run(Command::new("git") + .arg("-C") + .arg(&target) + .args(["checkout", "--detach", &revision]))?; + run(Command::new("git") + .arg("-C") + .arg(&target) + .args(["remote", "set-url", "origin", &location]))?; + if options.get("submodules").is_some_and(|v| v == "true") { + ensure!(!offline, "Submodule checkout is not cached for offline use"); + run(Command::new("git").arg("-C").arg(&target).args([ + "submodule", + "update", + "--init", + "--recursive", + ]))?; + } + fs::rename(target, &checkout)?; + } + let head = run(Command::new("git") + .arg("-C") + .arg(&checkout) + .args(["rev-parse", "HEAD"]))?; + let changes = run(Command::new("git").arg("-C").arg(&checkout).args([ + "status", + "--porcelain", + "--untracked-files=all", + ]))?; + ensure!( + head == revision && changes.is_empty(), + "Cached Git checkout has been modified" + ); + Ok((revision, checkout)) +} diff --git a/crates/rb-packages/src/input.rs b/crates/rb-packages/src/input.rs new file mode 100644 index 0000000..4bd5700 --- /dev/null +++ b/crates/rb-packages/src/input.rs @@ -0,0 +1,137 @@ +use crate::constants::{MANIFEST_FILE, SNAPSHOT_FILE, STATE_DIRECTORY}; +use anyhow::{Context, Result, ensure}; +use rb_gemfile::{Evaluator, Manifest, SnapshotCache}; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +#[derive(Clone, Debug)] +pub struct ProjectInput { + pub root: PathBuf, + pub manifest: Manifest, + pub gemfile: Option, + pub lockfile: PathBuf, + pub bundler_gemfile: PathBuf, + pub reused: bool, +} + +impl ProjectInput { + pub fn load(root: impl AsRef, ruby: impl AsRef) -> Result { + let root = dunce::canonicalize(root)?; + let toml = root.join("rbproject.toml"); + let kdl = root.join("rbproject.kdl"); + ensure!( + !(toml.exists() && kdl.exists()), + "Choose one rbproject format" + ); + let project = if toml.exists() { + Some(fs::read_to_string(toml)?) + } else { + None + }; + let gemfile = std::env::var_os("BUNDLE_GEMFILE") + .map(|name| root.join(name)) + .or_else(|| { + ["gems.rb", "Gemfile"] + .into_iter() + .map(|name| root.join(name)) + .find(|path| path.is_file()) + }); + let has_dependencies = project + .as_ref() + .map(|s| s.parse::()) + .transpose()? + .as_ref() + .is_some_and(|v| v.get("dependencies").is_some() || v.get("sources").is_some()); + let (mut manifest, reused, gemfile) = if kdl.exists() { + (Manifest::from_kdl(&fs::read_to_string(kdl)?)?, false, None) + } else if has_dependencies { + (parse_project(project.as_ref().unwrap())?, false, None) + } else { + let gemfile = gemfile.context("No rbproject dependencies, gems.rb or Gemfile found")?; + let state = root.join(STATE_DIRECTORY); + fs::create_dir_all(&state)?; + let snapshot = SnapshotCache::new(state.join(SNAPSHOT_FILE)).output_directory(&state); + let cached = snapshot.evaluate(&Evaluator::new(ruby.as_ref()), &gemfile)?; + let document = if let Some(settings) = project { + cached.evaluation.manifest.merge_project(&settings)? + } else { + cached.evaluation.manifest.to_toml()? + }; + rb_gemfile::write_project( + state.join(MANIFEST_FILE), + &format!("{}{document}", rb_gemfile::GENERATED_HEADER), + )?; + (parse_project(&document)?, cached.reused, Some(gemfile)) + }; + for dependency in manifest.dependencies.values_mut() { + for path in std::iter::once(&mut dependency.path) + .chain( + dependency + .variants + .iter_mut() + .map(|variant| &mut variant.path), + ) + .flatten() + { + *path = dunce::canonicalize(root.join(&*path))? + .to_string_lossy() + .into_owned(); + } + } + ensure!( + manifest.bundle.lockfile.is_none(), + "Custom lockfile paths are not supported by Bundler compatibility preparation" + ); + let bundler_gemfile = if let Some(path) = &gemfile { + path.clone() + } else { + let generated = root.join(STATE_DIRECTORY).join("Gemfile"); + rb_gemfile::write_project(&generated, &crate::gemfile::render(&manifest)?)?; + generated + }; + let lockfile = manifest + .bundle + .lockfile + .as_ref() + .map(|path| root.join(path)) + .unwrap_or_else(|| { + if gemfile + .as_ref() + .is_some_and(|p| p.file_name().is_some_and(|n| n == "gems.rb")) + { + root.join("gems.locked") + } else if let Some(gemfile) = &gemfile { + let mut name = gemfile.as_os_str().to_owned(); + name.push(".lock"); + PathBuf::from(name) + } else { + { + let mut path = bundler_gemfile.as_os_str().to_owned(); + path.push(".lock"); + PathBuf::from(path) + } + } + }); + Ok(Self { + root, + manifest, + gemfile, + lockfile, + bundler_gemfile, + reused, + }) + } +} + +fn parse_project(input: &str) -> Result { + let mut value: toml::Value = input.parse()?; + value + .as_table_mut() + .context("Project must be a table")? + .remove("project"); + value + .try_into() + .context("Reading project dependency declarations") +} diff --git a/crates/rb-packages/src/install.rs b/crates/rb-packages/src/install.rs new file mode 100644 index 0000000..4da72bd --- /dev/null +++ b/crates/rb-packages/src/install.rs @@ -0,0 +1,341 @@ +use crate::{ + InstallOptions, InstalledBundle, PackageIndex, PackagePlan, PackageRecord, RubyEnvironment, + constants::{GEM_HOME_DIRECTORY, INSTALL_RECEIPT, STATE_DIRECTORY}, + copy_tree::copy_tree, + package_install::install_slot, + receipt::{InstalledReceipt, Receipt}, + slot::Slot, + validation::{ensure_record, verify}, +}; +use anyhow::{Context, Result, ensure}; +use rb_task::{Executor, TaskAction, TaskEvents, TaskGraph}; +use std::{ + collections::BTreeMap, + fs, + path::Path, + sync::{Arc, Mutex}, +}; + +pub fn install( + project: &Path, + plan: &PackagePlan, + runtime: &RubyEnvironment, + index: Arc, + options: InstallOptions, + events: TaskEvents, +) -> Result { + let project = project.to_owned(); + let init_plan = plan.clone(); + let init_runtime = runtime.clone(); + let (bundle, home, lockfile, transaction, staged_home, runtime, options, slots) = + crate::task::run("prepare installation", events.clone(), move |_| { + let project = &project; + let plan = &init_plan; + let runtime = &init_runtime; + let options = options.clone(); + ensure!(options.workers > 0, "Worker count must be positive"); + let bundle = project.join(STATE_DIRECTORY).join(GEM_HOME_DIRECTORY); + let home = bundle.join(&runtime.engine).join( + runtime + .version + .to_string() + .split('.') + .take(2) + .collect::>() + .join(".") + + ".0", + ); + fs::create_dir_all(home.parent().context("Missing gem home parent")?)?; + let lockfile = fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(home.parent().unwrap().join(crate::constants::INSTALL_LOCK))?; + lockfile.lock()?; + let transaction = tempfile::Builder::new() + .prefix(".bundle-") + .tempdir_in(home.parent().unwrap())?; + let staged_home = transaction.path().join("home"); + if home.exists() { + copy_tree(&home, &staged_home)?; + } else { + fs::create_dir_all(&staged_home)?; + } + let staged_home = Arc::new(staged_home); + let runtime = Arc::new(runtime.clone()); + let options = Arc::new(options); + let records: BTreeMap<_, _> = plan + .packages + .iter() + .filter(|p| p.install) + .map(|p| (p.record.package.id.name.clone(), p.record.clone())) + .collect(); + let mut slots = BTreeMap::new(); + for planned in plan.packages.iter().filter(|p| p.install) { + let record = &planned.record; + record.validate()?; + let mut receipt = Receipt::new(record, &runtime, &options.build)?; + receipt.dependencies = dependencies(&records, &record.package.id.name) + .into_iter() + .map(|name| { + let dependency = &records[&name]; + ( + name, + ( + dependency.source.clone(), + dependency.archive_version.clone(), + dependency.checksum.clone(), + ), + ) + }) + .collect(); + let directory = staged_home + .join(crate::constants::GEMS_DIRECTORY) + .join(format!( + "{}-{}", + record.package.id.name, record.archive_version + )); + let registered = fs::read(directory.join(INSTALL_RECEIPT)) + .ok() + .and_then(|b| serde_json::from_slice::(&b).ok()); + let reused = registered.as_ref().is_some_and(|previous| { + previous.receipt == receipt + && previous + .files + .iter() + .all(|path| fs::symlink_metadata(directory.join(path)).is_ok()) + && previous.extension.as_ref().is_none_or(|path| { + staged_home + .join(path) + .join(crate::constants::BUILD_COMPLETE_MARKER) + .is_file() + }) + }) && staged_home + .join(crate::constants::SPECIFICATIONS_DIRECTORY) + .join(format!( + "{}-{}.gemspec", + record.package.id.name, record.archive_version + )) + .is_file(); + if directory.exists() && !reused { + fs::remove_dir_all(&directory)?; + } + let gem = if reused { + registered.map(|mut r| { + r.gem.root = directory.clone(); + r.gem + }) + } else { + None + }; + slots.insert( + record.package.id.name.clone(), + Arc::new(Mutex::new(Slot { + directory, + archive: None, + metadata: None, + staging: None, + reused, + receipt, + gem, + dependencies: dependencies(&records, &record.package.id.name), + })), + ); + } + let slots = Arc::new(slots); + Ok(( + bundle, + home, + lockfile, + transaction, + staged_home, + runtime, + options, + slots, + )) + })?; + let _installation_lock = lockfile; + let mut graph = TaskGraph::new(); + let mut actions: BTreeMap<_, TaskAction> = BTreeMap::new(); + for planned in plan.packages.iter().filter(|p| p.install) { + let record = planned.record.clone(); + let name = record.package.id.name.clone(); + let download = graph.add(format!("download {}", record.package.id), []); + let extract = graph.add(format!("extract {}", record.package.id), [download]); + let index = index.clone(); + let slot_download = slots[&name].clone(); + let download_record = record.clone(); + actions.insert( + download, + Arc::new(move |context| { + let mut slot = slot_download.lock().unwrap(); + if !slot.reused { + let path = index + .archive(&download_record) + .map_err(|e| format!("{e:#}"))?; + verify(&path, &download_record.checksum).map_err(|e| format!("{e:#}"))?; + slot.archive = Some(path); + } + context.progress( + "download", + 1, + 1, + Some(if slot.reused { + "reused".into() + } else { + download_record.package.id.to_string() + }), + ); + Ok(()) + }), + ); + let slot_extract = slots[&name].clone(); + let home_extract = staged_home.clone(); + actions.insert( + extract, + Arc::new(move |context| { + let mut slot = slot_extract.lock().unwrap(); + if slot.reused { + return Ok(()); + } + let staging = tempfile::Builder::new() + .prefix(".package-") + .tempdir_in(home_extract.as_path()) + .map_err(|e| e.to_string())?; + let directory = staging.path().join("gem"); + let metadata = rb_gem_installer::install( + fs::File::open(slot.archive.as_ref().unwrap()).map_err(|e| e.to_string())?, + &directory, + ) + .map_err(|e| format!("{e:#}"))?; + ensure_record(&metadata, &record).map_err(|e| format!("{e:#}"))?; + slot.staging = Some(staging); + slot.metadata = Some(metadata); + context.progress("extract", 1, 1, Some(name.clone())); + Ok(()) + }), + ); + } + Executor::new(options.workers, events.clone()).run(graph, actions)?; + let mut graph = TaskGraph::new(); + let mut actions: BTreeMap<_, TaskAction> = BTreeMap::new(); + let mut pending: BTreeMap<_, _> = plan + .packages + .iter() + .filter(|p| p.install) + .map(|p| (p.record.package.id.name.clone(), p.record.clone())) + .collect(); + let mut completed = BTreeMap::new(); + while !pending.is_empty() { + let ready: Vec<_> = pending + .iter() + .filter(|(name, _)| { + let slot = slots[*name].lock().unwrap(); + slot.reused + || slot + .metadata + .as_ref() + .is_some_and(|m| m.extensions.is_empty()) + || slot.dependencies.iter().all(|d| completed.contains_key(d)) + }) + .map(|(name, _)| name.clone()) + .collect(); + ensure!( + !ready.is_empty(), + "Native build dependency cycle: {}", + pending.keys().cloned().collect::>().join(", ") + ); + for name in ready { + let record = pending.remove(&name).unwrap(); + let dependency_ids: Vec<_> = { + let slot = slots[&name].lock().unwrap(); + if slot.reused + || slot + .metadata + .as_ref() + .is_some_and(|m| m.extensions.is_empty()) + { + Vec::new() + } else { + slot.dependencies + .iter() + .filter_map(|name| completed.get(name).copied()) + .collect() + } + }; + let id = graph.add(format!("install {}", record.package.id), dependency_ids); + completed.insert(name.clone(), id); + let slots = slots.clone(); + let home = staged_home.clone(); + let runtime = runtime.clone(); + let options = options.clone(); + actions.insert( + id, + Arc::new(move |context| { + install_slot(&slots, &name, &home, &runtime, &options, &context) + .map_err(|e| format!("{e:#}")) + }), + ); + } + } + Executor::new(options.workers, events.clone()).run(graph, actions)?; + let mut reused = 0; + for slot in slots.values() { + if slot.lock().unwrap().reused { + reused += 1; + } + } + let staged = staged_home.as_ref().clone(); + let destination = home.clone(); + let transaction_path = transaction.path().to_owned(); + let source_index = index.clone(); + let source_home = staged_home.as_ref().clone(); + let source_plan = plan.clone(); + let source_runtime = runtime.clone(); + let sources_changed = + crate::task::run("prepare source snapshots", events.clone(), move |_| { + source_index.stage_sources(&source_home, &source_plan, &source_runtime) + })?; + if reused != slots.len() || sources_changed { + crate::task::run("publish bundle", events, move |_| { + let backup = transaction_path.join("previous"); + if destination.exists() { + fs::rename(&destination, &backup)?; + } + if let Err(error) = fs::rename(&staged, &destination) { + if backup.exists() { + fs::rename(backup, &destination)?; + } + return Err(error.into()); + } + Ok(()) + })?; + } + Ok(InstalledBundle { + gem_home: home, + bundle_path: bundle, + installed: slots.len() - reused, + reused, + without: plan.without.iter().cloned().collect(), + with: plan.with.iter().cloned().collect(), + gemfile: None, + }) +} + +fn dependencies( + records: &BTreeMap, + name: &str, +) -> std::collections::BTreeSet { + let mut seen = std::collections::BTreeSet::new(); + let mut pending: Vec<_> = records[name].package.dependencies.keys().cloned().collect(); + while let Some(dependency) = pending.pop() { + if dependency != name + && seen.insert(dependency.clone()) + && let Some(record) = records.get(&dependency) + { + pending.extend(record.package.dependencies.keys().cloned()); + } + } + seen +} diff --git a/crates/rb-packages/src/install_options.rs b/crates/rb-packages/src/install_options.rs new file mode 100644 index 0000000..24c2c5e --- /dev/null +++ b/crates/rb-packages/src/install_options.rs @@ -0,0 +1,18 @@ +use crate::constants::DEFAULT_WORKERS; +use rb_gem_installer::BuildOptions; +use std::path::PathBuf; + +#[derive(Clone, Debug)] +pub struct InstallOptions { + pub workers: usize, + pub build: BuildOptions, +} + +impl InstallOptions { + pub fn new(ruby: impl Into) -> Self { + Self { + workers: DEFAULT_WORKERS, + build: BuildOptions::new(ruby), + } + } +} diff --git a/crates/rb-packages/src/installed_bundle.rs b/crates/rb-packages/src/installed_bundle.rs new file mode 100644 index 0000000..c6178ed --- /dev/null +++ b/crates/rb-packages/src/installed_bundle.rs @@ -0,0 +1,42 @@ +use std::{collections::BTreeMap, path::PathBuf}; + +#[derive(Clone, Debug)] +pub struct InstalledBundle { + pub gem_home: PathBuf, + pub bundle_path: PathBuf, + pub installed: usize, + pub reused: usize, + pub without: Vec, + pub with: Vec, + pub gemfile: Option, +} + +impl InstalledBundle { + pub fn environment(&self) -> BTreeMap { + let mut environment = BTreeMap::from([ + ("BUNDLE_WITHOUT".into(), self.without.join(":")), + ("BUNDLE_WITH".into(), self.with.join(":")), + ( + "GEM_HOME".into(), + self.gem_home.to_string_lossy().into_owned(), + ), + ( + "GEM_PATH".into(), + self.gem_home.to_string_lossy().into_owned(), + ), + ( + "BUNDLE_PATH".into(), + self.bundle_path.to_string_lossy().into_owned(), + ), + ("BUNDLE_DISABLE_SHARED_GEMS".into(), "true".into()), + ("BUNDLE_IGNORE_CONFIG".into(), "true".into()), + ]); + if let Some(gemfile) = &self.gemfile { + environment.insert( + "BUNDLE_GEMFILE".into(), + gemfile.to_string_lossy().into_owned(), + ); + } + environment + } +} diff --git a/crates/rb-packages/src/lib.rs b/crates/rb-packages/src/lib.rs new file mode 100644 index 0000000..c5f44a6 --- /dev/null +++ b/crates/rb-packages/src/lib.rs @@ -0,0 +1,48 @@ +#![doc = include_str!("../README.md")] + +mod bundler_credentials; +mod constants; +mod copy_tree; +mod credentials; +mod gemfile; +mod git_cache; +mod input; +mod install; +mod install_options; +mod installed_bundle; +mod output_lines; +mod package_install; +mod package_source; +mod plan; +mod prepare; +mod provider; +mod receipt; +mod registry_index; +mod runtime; +mod slot; +mod source_archive; +mod source_compatibility; +mod source_files; +mod source_index; +mod source_install; +mod source_package; +mod source_pin; +mod source_specs; +mod specification; +mod target_plan; +mod task; +mod validation; +mod wrappers; + +pub use input::ProjectInput; +pub use install::install; +pub use install_options::InstallOptions; +pub use installed_bundle::InstalledBundle; +pub use plan::{LockMode, PackagePlan, PlanOptions, PlannedPackage, plan}; +pub use prepare::{PrepareOptions, PreparedBundle, prepare}; +pub use provider::{MemoryIndex, PackageIndex, PackageRecord}; +pub use registry_index::RegistryIndex; +pub use runtime::RubyEnvironment; + +pub use package_source::PackageSource; +pub use source_index::SourceIndex; diff --git a/crates/rb-packages/src/output_lines.rs b/crates/rb-packages/src/output_lines.rs new file mode 100644 index 0000000..0455daa --- /dev/null +++ b/crates/rb-packages/src/output_lines.rs @@ -0,0 +1,40 @@ +#[derive(Default)] +pub(crate) struct OutputLines(Vec); + +impl OutputLines { + pub(crate) fn push(&mut self, bytes: &[u8], mut output: impl FnMut(String)) { + self.0.extend_from_slice(bytes); + while let Some(position) = self.0.iter().position(|byte| *byte == b'\n') { + let line: Vec<_> = self.0.drain(..=position).collect(); + output( + String::from_utf8_lossy(&line[..line.len() - 1]) + .trim_end_matches('\r') + .to_owned(), + ); + } + if self.0.len() >= crate::constants::MAX_OUTPUT_LINE_BYTES { + self.finish(output); + } + } + + pub(crate) fn finish(&mut self, mut output: impl FnMut(String)) { + if !self.0.is_empty() { + output(String::from_utf8_lossy(&self.0).into_owned()); + self.0.clear(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn read_boundaries_do_not_split_utf8_or_create_extra_lines() { + let mut lines = OutputLines::default(); + let mut output = Vec::new(); + lines.push(&[b'a', 0xc3], |line| output.push(line)); + lines.push(&[0xa9, b'\r', b'\n', b'b'], |line| output.push(line)); + lines.finish(|line| output.push(line)); + assert_eq!(output, ["a\u{e9}", "b"]); + } +} diff --git a/crates/rb-packages/src/package_install.rs b/crates/rb-packages/src/package_install.rs new file mode 100644 index 0000000..6ca20a6 --- /dev/null +++ b/crates/rb-packages/src/package_install.rs @@ -0,0 +1,163 @@ +use crate::{ + InstallOptions, RubyEnvironment, constants::INSTALL_RECEIPT, receipt::InstalledReceipt, + slot::Slots, wrappers::wrappers, +}; +use anyhow::{Context, Result, ensure}; +use rb_gem_installer::{BuildEvent, NativeBuilder}; +use rb_task::TaskContext; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +pub(crate) fn install_slot( + slots: &Slots, + name: &str, + home: &Path, + runtime: &RubyEnvironment, + options: &InstallOptions, + context: &TaskContext, +) -> Result<()> { + let mut slot = slots[name].lock().unwrap(); + if !slot.reused { + let root = slot.staging.as_ref().unwrap().path().join("gem"); + let metadata = slot.metadata.as_ref().unwrap(); + let payload_files = crate::source_files::payload_files(&root)?; + ensure!( + metadata.rubygems.matches(&runtime.rubygems), + "Gem requires RubyGems {}", + metadata.rubygems + ); + let mut build = options.build.clone(); + build.ruby = runtime.executable.clone(); + for dependency in slot + .dependencies + .iter() + .filter(|_| !metadata.extensions.is_empty()) + { + let other = slots + .get(dependency) + .context("Missing build dependency")? + .lock() + .unwrap(); + if !metadata.extensions.is_empty() { + build.dependencies.push( + other + .gem + .clone() + .context("Build dependency is incomplete")?, + ); + } + } + let report = if metadata.extensions.is_empty() { + None + } else { + let mut stdout = crate::output_lines::OutputLines::default(); + let mut stderr = crate::output_lines::OutputLines::default(); + let result = NativeBuilder::new(build)?.build(metadata, &root, |event| { + if let BuildEvent::Output { bytes, stream, .. } = event { + let lines = if stream == rb_gem_installer::OutputStream::Stdout { + &mut stdout + } else { + &mut stderr + }; + lines.push(&bytes, |line| context.output(line)); + } + }); + stdout.finish(|line| context.output(line)); + stderr.finish(|line| context.output(line)); + Some(result?) + }; + let artifacts = report + .as_ref() + .map(|report| report.artifacts.as_slice()) + .unwrap_or(&[]); + let staged_home = slot.staging.as_ref().unwrap().path(); + let specification = crate::specification::write( + &runtime.executable, + metadata, + &slot.receipt.archive_version, + staged_home, + )?; + let extension_name = format!("{}-{}", metadata.id.name, slot.receipt.archive_version); + let extension = staged_home + .join(crate::constants::EXTENSIONS_DIRECTORY) + .join(&runtime.platform) + .join(&runtime.abi) + .join(&extension_name); + if !metadata.extensions.is_empty() { + fs::create_dir_all(&extension)?; + for artifact in artifacts { + if let Some(relative) = report + .as_ref() + .and_then(|report| report.extension_directory.as_ref()) + .and_then(|directory| artifact.strip_prefix(directory).ok()) + { + let target = extension.join(relative); + fs::create_dir_all(target.parent().unwrap())?; + fs::copy(root.join(artifact), target)?; + } + } + fs::write(extension.join(crate::constants::BUILD_COMPLETE_MARKER), b"")?; + } + let gem = installed_gem(metadata, &slot.directory); + fs::write( + root.join(INSTALL_RECEIPT), + serde_json::to_vec(&InstalledReceipt { + receipt: slot.receipt.clone(), + gem: gem.clone(), + files: payload_files + .into_iter() + .chain(artifacts.iter().cloned()) + .collect(), + build_artifacts: artifacts.to_vec(), + extension: (!metadata.extensions.is_empty()).then(|| { + PathBuf::from(crate::constants::EXTENSIONS_DIRECTORY) + .join(&runtime.platform) + .join(&runtime.abi) + .join(&extension_name) + }), + })?, + )?; + fs::create_dir_all(home.join(crate::constants::GEMS_DIRECTORY))?; + fs::create_dir_all(home.join(crate::constants::SPECIFICATIONS_DIRECTORY))?; + fs::rename(&root, &slot.directory)?; + if extension.exists() { + let destination = home + .join(crate::constants::EXTENSIONS_DIRECTORY) + .join(&runtime.platform) + .join(&runtime.abi) + .join(&extension_name); + fs::create_dir_all(destination.parent().unwrap())?; + if destination.exists() { + fs::remove_dir_all(&destination)?; + } + fs::rename(extension, destination)?; + } + let destination_spec = home + .join(crate::constants::SPECIFICATIONS_DIRECTORY) + .join(specification.file_name().unwrap()); + rb_gemfile::write_project(destination_spec, &fs::read_to_string(specification)?)?; + wrappers(home, metadata)?; + slot.gem = Some(gem); + slot.staging = None; + } + context.progress("install", 1, 1, Some(name.into())); + Ok(()) +} + +fn installed_gem( + metadata: &rb_gem_archive::GemMetadata, + root: &Path, +) -> rb_microgems::InstalledGem { + rb_microgems::InstalledGem { + name: metadata.id.name.clone(), + version: metadata.id.version.to_string(), + platform: metadata.id.platform.to_string(), + root: root.into(), + require_paths: metadata.require_paths.clone(), + metadata: metadata.metadata.clone(), + executables: metadata.executables.clone(), + bindir: metadata.bindir.clone(), + } +} diff --git a/crates/rb-packages/src/package_source.rs b/crates/rb-packages/src/package_source.rs new file mode 100644 index 0000000..e6a5291 --- /dev/null +++ b/crates/rb-packages/src/package_source.rs @@ -0,0 +1,180 @@ +use anyhow::{Context, Result, ensure}; +use rb_lockfile::{Source, SourceKind}; +use serde::{Deserialize, Serialize}; +use std::{ + collections::BTreeMap, + path::{Path, PathBuf}, +}; + +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +pub enum PackageSource { + Registry(String), + Git { + remote: String, + revision: String, + options: BTreeMap, + }, + Path { + directory: PathBuf, + glob: Option, + }, +} + +impl From for PackageSource { + fn from(value: String) -> Self { + Self::Registry(value) + } +} +impl From<&str> for PackageSource { + fn from(value: &str) -> Self { + Self::Registry(value.into()) + } +} + +impl PackageSource { + pub fn lock_source(&self) -> Source { + match self { + Self::Registry(url) => Source::new(SourceKind::Registry, url), + Self::Git { + remote, + revision, + options, + } => { + let mut source = Source::new(SourceKind::Git, remote); + source.options = options.clone(); + source.options.insert("revision".into(), revision.clone()); + source + } + Self::Path { directory, glob } => { + let mut source = Source::new(SourceKind::Path, directory.to_string_lossy()); + if let Some(glob) = glob { + source.options.insert("glob".into(), glob.clone()); + } + source + } + } + } + + pub(crate) fn from_lock(source: &Source, remote: &str, root: Option<&Path>) -> Result { + Ok(match source.kind { + SourceKind::Registry => Self::Registry(crate::provider::source_url(remote)?), + SourceKind::Path => Self::Path { + directory: { + let path = root + .map(Path::to_owned) + .unwrap_or(std::env::current_dir()?) + .join(remote); + dunce::canonicalize(&path).unwrap_or(path) + }, + glob: normalized_glob(source.options.get("glob").map(String::as_str)), + }, + SourceKind::Git => { + let mut options = source.options.clone(); + let revision = options + .remove("revision") + .context("Git lock has no revision")?; + if options + .get("glob") + .is_some_and(|value| value == crate::constants::DEFAULT_GEMSPEC_GLOB) + { + options.remove("glob"); + } + Self::Git { + remote: remote.trim_end_matches('/').into(), + revision, + options, + } + } + }) + } + + pub(crate) fn matches( + &self, + dependency: &rb_gemfile::Dependency, + default: Option<&str>, + ) -> Result { + Ok(match self { + Self::Registry(url) => { + dependency.git.is_none() + && dependency.path.is_none() + && *url + == crate::provider::source_url( + dependency + .source + .as_deref() + .or(default) + .unwrap_or(crate::constants::DEFAULT_SOURCE), + )? + } + Self::Git { + remote, options, .. + } => { + dependency + .git + .as_ref() + .is_some_and(|git| git.trim_end_matches('/') == remote) + && *options == git_options(dependency) + } + Self::Path { directory, glob } => { + dependency + .path + .as_ref() + .is_some_and(|path| Path::new(path) == directory) + && glob == &normalized_glob(dependency.glob.as_deref()) + } + }) + } + + pub(crate) fn validate(&self) -> Result<()> { + match self { + Self::Registry(url) => ensure!( + crate::provider::source_url(url)? == *url, + "Registry source must be normalized" + ), + Self::Git { + remote, revision, .. + } => { + ensure!( + !remote.is_empty() + && !remote.starts_with('-') + && !remote.chars().any(char::is_control), + "Invalid Git remote" + ); + ensure!( + crate::constants::GIT_OBJECT_ID_LENGTHS.contains(&revision.len()) + && revision.bytes().all(|b| b.is_ascii_hexdigit()), + "Invalid Git revision" + ); + } + Self::Path { directory, .. } => { + ensure!(directory.is_absolute(), "Path source must be absolute") + } + } + Ok(()) + } +} + +pub(crate) fn git_options(dependency: &rb_gemfile::Dependency) -> BTreeMap { + let mut options = BTreeMap::new(); + for (name, value) in [ + ("branch", &dependency.branch), + ("tag", &dependency.tag), + ("ref", &dependency.rev), + ("glob", &dependency.glob), + ] { + if let Some(value) = value + && (name != "glob" || value != crate::constants::DEFAULT_GEMSPEC_GLOB) + { + options.insert(name.into(), value.clone()); + } + } + if dependency.submodules == Some(true) { + options.insert("submodules".into(), "true".into()); + } + options +} + +pub(crate) fn normalized_glob(glob: Option<&str>) -> Option { + glob.filter(|value| *value != crate::constants::DEFAULT_GEMSPEC_GLOB) + .map(str::to_owned) +} diff --git a/crates/rb-packages/src/plan.rs b/crates/rb-packages/src/plan.rs new file mode 100644 index 0000000..5949290 --- /dev/null +++ b/crates/rb-packages/src/plan.rs @@ -0,0 +1,448 @@ +use crate::{PackageIndex, PackageRecord, RubyEnvironment, provider::source_url}; +use anyhow::{Context, Result, ensure}; +use rb_gem_types::{PackageId, Platform, Requirement}; +use rb_lockfile::{LockedPackage, Lockfile, Source, SourceKind}; +use rb_resolver::{Package, PackageProvider, ResolveRequest}; +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Mutex, +}; + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum LockMode { + #[default] + Prefer, + Locked, + Update, +} + +#[derive(Clone, Debug, Default)] +pub struct PlanOptions { + pub mode: LockMode, + pub without: BTreeSet, + pub with: BTreeSet, + pub allow_prereleases: bool, + pub source_root: Option, +} + +#[derive(Clone, Debug)] +pub struct PlannedPackage { + pub record: PackageRecord, + pub install: bool, +} + +#[derive(Clone, Debug)] +pub struct PackagePlan { + pub packages: Vec, + pub lockfile: Lockfile, + pub lock_changed: bool, + pub without: BTreeSet, + pub with: BTreeSet, +} + +struct Provider<'a> { + index: &'a dyn PackageIndex, + records: Mutex>>, + sources: BTreeMap, + default_source: Option, + pins: BTreeMap>, + strict: bool, + runtime: &'a RubyEnvironment, +} + +impl PackageProvider for Provider<'_> { + type Error = anyhow::Error; + fn candidates(&self, name: &str) -> Result> { + if self.strict && !self.pins.contains_key(name) { + return Ok(Vec::new()); + } + let mut records = self.index.candidates(name)?; + for record in &records { + record.validate()?; + } + records.retain(|record| { + record + .rubygems + .as_ref() + .is_none_or(|requirement| requirement.matches(&self.runtime.rubygems)) + && (name != "bundler" + || self + .runtime + .bundler + .as_ref() + .is_none_or(|version| &record.package.id.version == version)) + }); + if let Some(source) = self.sources.get(name) { + records = records + .into_iter() + .filter_map(|record| { + match record + .source + .matches(source, self.default_source.as_deref()) + { + Ok(true) => Some(Ok(record)), + Ok(false) => None, + Err(error) => Some(Err(error)), + } + }) + .collect::>()?; + } + if self.strict { + records.retain(|record| { + self.pins + .get(name) + .is_some_and(|pins| pins.iter().any(|pin| pin.matches(record))) + }); + for record in &records { + ensure!( + self.pins[name] + .iter() + .any(|pin| pin.matches(record) && pin.dependencies_match(record)), + "Locked dependency metadata changed for {}", + record.package.id + ); + } + } + let mut identities = BTreeMap::new(); + for record in &records { + if let Some(previous) = identities.insert(record.package.id.clone(), record) { + ensure!( + previous == record, + "Ambiguous source or metadata for {}", + record.package.id + ); + } + } + let packages = records.iter().map(|r| r.package.clone()).collect(); + self.records.lock().unwrap().insert(name.into(), records); + Ok(packages) + } +} + +pub fn plan( + manifest: &rb_gemfile::Manifest, + runtime: &RubyEnvironment, + previous: Option<&Lockfile>, + index: &dyn PackageIndex, + options: &PlanOptions, +) -> Result { + let ruby_requirement: Requirement = if manifest.bundle.ruby.is_empty() { + ">= 0".parse()? + } else { + manifest.bundle.ruby.join(", ").parse()? + }; + ensure!( + ruby_requirement.matches(&runtime.version), + "Project requires Ruby {ruby_requirement}, selected Ruby is {}", + runtime.version + ); + if let Some(engine) = &manifest.bundle.engine { + ensure!( + engine == &runtime.engine, + "Project requires Ruby engine {engine}" + ); + } + if let Some(version) = &manifest.bundle.engine_version { + ensure!( + version.parse::()? == runtime.engine_version, + "Project Ruby engine version does not match selected Ruby" + ); + } + if let Some(patchlevel) = &manifest.bundle.patchlevel { + ensure!( + patchlevel.parse::()? == runtime.patchlevel, + "Project Ruby patchlevel does not match selected Ruby" + ); + } + ensure!( + manifest.bundle.plugins.is_empty(), + "Bundler plugins are not supported by this coordinator" + ); + ensure!( + options.mode != LockMode::Locked || previous.is_some(), + "Locked mode requires an existing lockfile" + ); + if options.mode == LockMode::Prefer && previous.is_some() { + let strict = PlanOptions { + mode: LockMode::Locked, + ..options.clone() + }; + if let Ok(plan) = plan(manifest, runtime, previous, index, &strict) { + return Ok(plan); + } + } + let mut targets = vec![runtime.clone()]; + if options.mode != LockMode::Locked + && let Some(previous) = previous + { + for platform in &previous.platforms { + if targets.iter().any(|target| &target.platform == platform) { + continue; + } + let mut target = runtime.clone(); + target.platform = platform.clone(); + target.engine = if target.target().os == "java" { + "jruby".into() + } else if runtime.engine == "jruby" { + "ruby".into() + } else { + runtime.engine.clone() + }; + targets.push(target); + } + } + let target_plans = targets + .iter() + .map(|target| crate::target_plan::TargetPlan::new(manifest, target, options)) + .collect::>>()?; + let declarations = target_plans[0].declarations.clone(); + let install_roots = target_plans[0].install_roots.clone(); + let mut sources = BTreeMap::new(); + for target in &target_plans { + for (name, source) in &target.sources { + if let Some(previous) = sources.insert(name.clone(), source.clone()) { + ensure!( + previous.source == source.source + && previous.git == source.git + && previous.path == source.path + && crate::package_source::git_options(&previous) + == crate::package_source::git_options(source), + "Conflicting sources for {name}" + ); + } + } + } + let mut locked = Vec::new(); + let mut pins: BTreeMap> = BTreeMap::new(); + if let Some(previous) = previous { + for source in &previous.sources { + for package in &source.packages { + let id = PackageId { + name: package.name.clone(), + version: package.version.clone(), + platform: Platform::parse(&package.platform), + }; + locked.push(id.clone()); + let checksum = previous + .checksums + .as_ref() + .and_then(|checksums| checksums.get(&package.identity())) + .cloned() + .flatten(); + for remote in &source.remotes { + pins.entry(id.name.clone()).or_default().push( + crate::source_pin::SourcePin::new( + id.clone(), + crate::PackageSource::from_lock( + source, + remote, + options.source_root.as_deref(), + )?, + checksum.clone(), + package, + )?, + ); + } + } + } + if options.mode == LockMode::Locked { + ensure!( + previous.dependencies == declarations, + "Dependency declarations changed in locked mode" + ); + } + } + let provider = Provider { + index, + records: Mutex::new(BTreeMap::new()), + sources, + default_source: manifest.bundle.source.clone(), + pins, + strict: options.mode == LockMode::Locked, + runtime, + }; + let requests: Vec<_> = targets + .iter() + .zip(&target_plans) + .map(|(target, input)| { + let mut request = + ResolveRequest::new(input.roots.clone(), target.version.clone(), target.target()); + if options.mode != LockMode::Update { + request.locked = locked.clone(); + } + request.allow_prereleases = options.allow_prereleases; + request.force_ruby = input.force_ruby.clone(); + request + }) + .collect(); + let resolved = rb_resolver::resolve_targets(&provider, &requests)?; + let fetched = provider.records.into_inner().unwrap(); + let mut records = BTreeMap::new(); + let mut locked_records = BTreeMap::new(); + for (target, packages) in resolved.into_iter().enumerate() { + for package in packages { + let record = fetched[&package.id.name] + .iter() + .find(|r| r.package == package) + .context("Resolved package metadata is missing")? + .clone(); + locked_records.insert(package.id.clone(), record.clone()); + if target == 0 { + records.insert(package.id.name.clone(), record); + } + } + } + if options.mode != LockMode::Update + && let Some(previous) = previous + { + for record in locked_records.values() { + for source in &previous.sources { + if !source.remotes.iter().any(|remote| { + crate::PackageSource::from_lock(source, remote, options.source_root.as_deref()) + .is_ok_and(|identity| identity == record.source) + }) { + continue; + } + for package in &source.packages { + if package.name == record.package.id.name + && package.archive_version() == record.archive_version + && let Some(Some(checksum)) = previous + .checksums + .as_ref() + .and_then(|sums| sums.get(&package.identity())) + { + ensure!( + checksum.eq_ignore_ascii_case(&record.checksum), + "Locked checksum changed for {}", + record.package.id + ); + } + } + } + } + } + let mut installed = BTreeSet::new(); + let mut pending: Vec<_> = install_roots.into_iter().collect(); + while let Some(name) = pending.pop() { + if installed.insert(name.clone()) { + pending.extend(records[&name].package.dependencies.keys().cloned()); + } + } + let mut lock = Lockfile { + dependencies: declarations, + platforms: targets + .iter() + .map(|target| target.platform.clone()) + .collect(), + bundled_with: runtime.bundler.clone(), + ruby_version: (!manifest.bundle.ruby.is_empty()).then(|| { + let mut ruby = format!("ruby {}", runtime.version); + if runtime.patchlevel >= 0 { + ruby.push_str(&format!("p{}", runtime.patchlevel)); + } + if runtime.engine == "ruby" { + ruby + } else { + format!("{ruby} ({} {})", runtime.engine, runtime.engine_version) + } + }), + ..Default::default() + }; + let mut sections: BTreeMap = BTreeMap::new(); + let mut checksums = BTreeMap::new(); + for record in locked_records.values() { + let (_, platform) = record + .archive_version + .split_once('-') + .unwrap_or((&record.archive_version, "ruby")); + let package = LockedPackage { + name: record.package.id.name.clone(), + version: record.package.id.version.clone(), + platform: platform.into(), + dependencies: record + .package + .dependencies + .iter() + .map(|(n, r)| (n.clone(), vec![r.clone()])) + .collect(), + }; + checksums.insert( + package.identity(), + if matches!(record.source, crate::PackageSource::Registry(_)) { + Some(record.checksum.clone()) + } else { + None + }, + ); + sections + .entry(record.source.clone()) + .or_insert_with(|| record.source.lock_source()) + .packages + .push(package); + } + if let Some(source) = &manifest.bundle.source { + let source = source_url(source)?; + sections + .entry(source.clone().into()) + .or_insert_with(|| Source::new(SourceKind::Registry, source)); + } else if manifest.dependencies.values().all(|dependency| { + std::iter::once(dependency) + .chain(&dependency.variants) + .all(|declaration| { + !declaration.enabled + || declaration.source.is_some() + || declaration.git.is_some() + || declaration.path.is_some() + }) + }) { + let mut source = Source::new(SourceKind::Registry, ""); + source.remotes.clear(); + sections.insert(crate::PackageSource::Registry(String::new()), source); + } + lock.sources = sections.into_values().collect(); + lock.checksums = Some(checksums); + if let Some(previous) = previous + && options.mode == LockMode::Locked + { + ensure!( + previous + .sources + .iter() + .all(|source| source.remotes.iter().all(|remote| { + crate::PackageSource::from_lock(source, remote, options.source_root.as_deref()) + .is_ok() + })), + "Invalid locked source" + ); + } + if options.mode == LockMode::Locked { + for record in records.values() { + ensure!( + previous.unwrap().sources.iter().any(|source| source + .packages + .iter() + .any(|package| package.name == record.package.id.name + && package.archive_version() == record.archive_version)), + "Locked archive spelling changed for {}", + record.package.id + ); + } + } + let lock_changed = previous.is_none_or(|previous| previous != &lock); + if options.mode == LockMode::Locked { + lock = previous.unwrap().clone(); + } + Ok(PackagePlan { + packages: records + .into_iter() + .map(|(name, record)| PlannedPackage { + record, + install: installed.contains(&name), + }) + .collect(), + lockfile: lock, + lock_changed: lock_changed && options.mode != LockMode::Locked, + without: options.without.clone(), + with: options.with.clone(), + }) +} diff --git a/crates/rb-packages/src/prepare.rs b/crates/rb-packages/src/prepare.rs new file mode 100644 index 0000000..5119ee6 --- /dev/null +++ b/crates/rb-packages/src/prepare.rs @@ -0,0 +1,135 @@ +use crate::{ + InstallOptions, InstalledBundle, LockMode, PackagePlan, PlanOptions, ProjectInput, + RubyEnvironment, SourceIndex, +}; +use anyhow::{Result, ensure}; +use rb_task::TaskEvents; +use std::{path::PathBuf, sync::Arc}; + +#[derive(Clone, Debug)] +pub struct PrepareOptions { + pub project: PathBuf, + pub ruby: PathBuf, + pub cache: PathBuf, + pub offline: bool, + pub plan: PlanOptions, + pub install: InstallOptions, +} + +impl PrepareOptions { + pub fn new( + project: impl Into, + ruby: impl Into, + cache: impl Into, + ) -> Self { + let ruby = ruby.into(); + Self { + project: project.into(), + cache: cache.into(), + offline: false, + plan: PlanOptions::default(), + install: InstallOptions::new(&ruby), + ruby, + } + } +} + +pub struct PreparedBundle { + pub input: ProjectInput, + pub plan: PackagePlan, + pub installed: InstalledBundle, +} + +pub fn prepare(options: PrepareOptions, events: TaskEvents) -> Result { + let project = options.project.clone(); + let _project_lock = crate::task::run("lock project", events.clone(), move |_| { + let state = project.join(crate::constants::STATE_DIRECTORY); + std::fs::create_dir_all(&state)?; + let lock = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(state.join(crate::constants::PROJECT_LOCK))?; + lock.lock()?; + Ok(lock) + })?; + let ruby = options.ruby.clone(); + let runtime = crate::task::run("inspect Ruby", events.clone(), move |_| { + RubyEnvironment::inspect(&ruby) + })?; + let project = options.project.clone(); + let ruby = options.ruby.clone(); + let input = crate::task::run("load dependencies", events.clone(), move |_| { + ProjectInput::load(&project, &ruby) + })?; + let source_input = input.clone(); + let source_runtime = runtime.clone(); + let mut plan_options = options.plan.clone(); + plan_options.source_root = Some(input.root.clone()); + let cache = options.cache.clone(); + let offline = options.offline; + let (plan, index) = crate::task::run("resolve dependencies", events.clone(), move |_| { + let previous = match std::fs::read_to_string(&source_input.lockfile) { + Ok(text) => Some(rb_lockfile::Lockfile::parse(&text)?), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, + Err(error) => return Err(error.into()), + }; + let index = Arc::new(SourceIndex::new( + &source_input, + &source_runtime, + &cache, + offline, + previous.as_ref(), + plan_options.mode, + )?); + let plan = crate::plan( + &source_input.manifest, + &source_runtime, + previous.as_ref(), + index.as_ref(), + &plan_options, + )?; + Ok((plan, index)) + })?; + ensure!( + options.plan.mode != LockMode::Locked || !plan.lock_changed, + "Lockfile changed in locked mode" + ); + let mut installed = crate::install( + &input.root, + &plan, + &runtime, + index.clone(), + options.install, + events.clone(), + )?; + let compatibility_input = input.clone(); + let compatibility_plan = plan.clone(); + let compatibility_bundle = installed.clone(); + installed.gemfile = Some(crate::task::run( + "prepare Bundler compatibility", + events.clone(), + move |_| { + crate::source_compatibility::write( + &compatibility_input, + &compatibility_plan, + &compatibility_bundle, + index.as_ref(), + ) + }, + )?); + if plan.lock_changed { + let lockfile = input.lockfile.clone(); + let rendered = plan.lockfile.render()?; + crate::task::run("write lockfile", events, move |_| { + rb_gemfile::write_project(&lockfile, &rendered)?; + Ok(()) + })?; + } + Ok(PreparedBundle { + input, + plan, + installed, + }) +} diff --git a/crates/rb-packages/src/provider.rs b/crates/rb-packages/src/provider.rs new file mode 100644 index 0000000..a534f0a --- /dev/null +++ b/crates/rb-packages/src/provider.rs @@ -0,0 +1,146 @@ +use anyhow::{Context, Result, ensure}; +use rb_compact_index::GemInfo; +use rb_gem_types::{PackageId, Platform}; +use rb_resolver::Package; +use std::{ + collections::BTreeMap, + path::{Path, PathBuf}, +}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct PackageRecord { + pub package: Package, + pub source: crate::PackageSource, + pub archive_version: String, + pub checksum: String, + pub rubygems: Option, +} + +impl PackageRecord { + pub(crate) fn validate(&self) -> Result<()> { + for value in [&self.package.id.name, &self.archive_version] { + ensure!( + !value.is_empty() + && value != "." + && value != ".." + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() + || matches!(byte, b'_' | b'-' | b'.')), + "Invalid package archive identity" + ); + } + let (version, platform) = self + .archive_version + .split_once('-') + .unwrap_or((&self.archive_version, "ruby")); + ensure!( + version.parse::()? == self.package.id.version + && Platform::parse(platform) == self.package.id.platform, + "Archive filename does not match package identity" + ); + ensure!( + self.checksum.len() == crate::constants::CHECKSUM_CHARACTERS + && self.checksum.bytes().all(|byte| byte.is_ascii_hexdigit()), + "Invalid archive checksum" + ); + self.source.validate()?; + Ok(()) + } + + pub fn from_info(name: &str, source: &str, record: GemInfo) -> Result { + let (version, platform) = record + .version + .split_once('-') + .unwrap_or((&record.version, "ruby")); + Ok(Self { + package: Package { + id: PackageId { + name: name.into(), + version: version.parse()?, + platform: Platform::parse(platform), + }, + dependencies: record + .dependencies + .iter() + .map(|(name, value)| Ok((name.clone(), value.replace('&', ", ").parse()?))) + .collect::>()?, + ruby: record + .metadata + .get("ruby") + .map(|v| v.replace('&', ", ").parse()) + .transpose()? + .unwrap_or_default(), + }, + rubygems: record + .metadata + .get("rubygems") + .map(|v| v.replace('&', ", ").parse()) + .transpose()?, + source: source_url(source)?.into(), + archive_version: record.version.clone(), + checksum: record + .metadata + .get("checksum") + .context("Missing archive checksum")? + .clone(), + }) + } +} + +pub trait PackageIndex: Send + Sync { + fn candidates(&self, name: &str) -> Result>; + fn archive(&self, record: &PackageRecord) -> Result; + fn stage_sources( + &self, + _home: &Path, + plan: &crate::PackagePlan, + _runtime: &crate::RubyEnvironment, + ) -> Result { + ensure!( + plan.packages + .iter() + .all(|package| matches!(package.record.source, crate::PackageSource::Registry(_))), + "This index does not provide source snapshots" + ); + Ok(false) + } + fn compatibility_path(&self, _record: &PackageRecord, _home: &Path) -> Result> { + Ok(None) + } +} + +#[derive(Default)] +pub struct MemoryIndex { + pub packages: BTreeMap>, + pub archives: BTreeMap, +} + +impl PackageIndex for MemoryIndex { + fn candidates(&self, name: &str) -> Result> { + Ok(self.packages.get(name).cloned().unwrap_or_default()) + } + fn archive(&self, record: &PackageRecord) -> Result { + self.archives + .get(&record.checksum) + .cloned() + .context("Archive is not available in the memory index") + } +} + +pub(crate) fn source_url(source: &str) -> Result { + let mut url = url::Url::parse(source)?; + ensure!( + matches!(url.scheme(), "http" | "https") + && url.host_str().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none(), + "Invalid registry source" + ); + if !url.path().ends_with('/') { + url.set_path(&format!("{}/", url.path())); + } + Ok(url.to_string()) +} diff --git a/crates/rb-packages/src/receipt.rs b/crates/rb-packages/src/receipt.rs new file mode 100644 index 0000000..3130d1b --- /dev/null +++ b/crates/rb-packages/src/receipt.rs @@ -0,0 +1,94 @@ +use crate::{PackageRecord, RubyEnvironment}; +use anyhow::{Context, Result}; +use rb_gem_installer::BuildOptions; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{collections::BTreeMap, path::PathBuf}; + +#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)] +pub(crate) struct Receipt { + pub(crate) checksum: String, + pub(crate) archive_version: String, + pub(crate) source: crate::PackageSource, + pub(crate) ruby: PathBuf, + pub(crate) version: String, + pub(crate) platform: String, + pub(crate) abi: String, + pub(crate) configuration_hash: String, + pub(crate) dependencies: BTreeMap, +} + +impl Receipt { + pub(crate) fn new( + record: &PackageRecord, + runtime: &RubyEnvironment, + build: &BuildOptions, + ) -> Result { + Ok(Self { + checksum: record.checksum.clone(), + archive_version: record.archive_version.clone(), + source: record.source.clone(), + ruby: runtime.executable.clone(), + version: runtime.version.to_string(), + platform: runtime.platform.clone(), + abi: runtime.abi.clone(), + configuration_hash: configuration_hash(build)?, + dependencies: BTreeMap::new(), + }) + } +} + +#[derive(Serialize, Deserialize)] +pub(crate) struct InstalledReceipt { + pub(crate) receipt: Receipt, + pub(crate) gem: rb_microgems::InstalledGem, + pub(crate) files: Vec, + pub(crate) build_artifacts: Vec, + pub(crate) extension: Option, +} + +fn configuration_hash(build: &BuildOptions) -> Result { + let mut environment: BTreeMap<_, _> = [ + "CC", + "CXX", + "CFLAGS", + "CXXFLAGS", + "CPPFLAGS", + "LDFLAGS", + "LIBS", + "MAKE", + "RUSTFLAGS", + "CARGO_ENCODED_RUSTFLAGS", + "PKG_CONFIG_PATH", + "SDKROOT", + ] + .into_iter() + .filter_map(|key| std::env::var_os(key).map(|value| (key.to_owned(), value))) + .collect(); + for (key, value) in &build.environment { + environment.insert( + key.to_str() + .context("Environment name is not UTF-8")? + .to_owned(), + value.clone(), + ); + } + let environment: BTreeMap<_, _> = environment + .iter() + .map(|(key, value)| { + Ok(( + key, + value.to_str().context("Environment value is not UTF-8")?, + )) + }) + .collect::>()?; + let arguments: Vec<_> = build + .arguments + .iter() + .map(|value| value.to_str().context("Build argument is not UTF-8")) + .collect::>()?; + let bytes = serde_json::to_vec( + &serde_json::json!({"arguments":arguments,"environment":environment,"load_paths":build.load_paths,"dependencies":build.dependencies,"make":build.make,"cargo":build.cargo,"cmake":build.cmake}), + )?; + Ok(format!("{:x}", Sha256::digest(bytes))) +} diff --git a/crates/rb-packages/src/registry_index.rs b/crates/rb-packages/src/registry_index.rs new file mode 100644 index 0000000..31a7ff9 --- /dev/null +++ b/crates/rb-packages/src/registry_index.rs @@ -0,0 +1,197 @@ +use crate::{PackageIndex, PackageRecord, constants::DEFAULT_SOURCE, provider::source_url}; +use anyhow::{Context, Result, ensure}; +use rb_compact_index::Client; +use std::{ + collections::{BTreeMap, BTreeSet}, + path::{Path, PathBuf}, + sync::Mutex, +}; + +#[derive(Default)] +struct Metadata { + discovered: bool, + names: BTreeMap>>, + routes: BTreeMap, + records: BTreeMap<(String, String), Vec>, +} + +pub struct RegistryIndex { + clients: BTreeMap, + default: Option, + roots: BTreeMap, + explicit: BTreeSet, + metadata: Mutex, +} + +impl RegistryIndex { + pub(crate) fn load_credentials(&mut self, project: &Path, offline: bool) -> Result<()> { + if offline { + return Ok(()); + } + let settings = crate::credentials::Credentials::load(project)?; + for (source, client) in &mut self.clients { + if let Some(credentials) = settings.credentials(source)? { + *client = client.clone().with_credentials(&credentials)?; + } + } + Ok(()) + } + + pub fn new(manifest: &rb_gemfile::Manifest, cache: &Path, offline: bool) -> Result { + let locations: BTreeSet<_> = manifest + .dependencies + .values() + .flat_map(|dependency| std::iter::once(dependency).chain(&dependency.variants)) + .filter(|dependency| dependency.enabled) + .filter_map(|dependency| dependency.source.as_deref()) + .map(source_url) + .collect::>()?; + let default = if let Some(source) = &manifest.bundle.source { + Some(source_url(source)?) + } else if locations.len() == 1 { + locations.first().cloned() + } else if locations.is_empty() { + Some(source_url(DEFAULT_SOURCE)?) + } else { + None + }; + let mut clients = BTreeMap::new(); + if let Some(source) = &default { + clients.insert(source.clone(), Client::new(source, cache, offline)?); + } + let mut roots = BTreeMap::new(); + let mut explicit = BTreeSet::new(); + for (name, dependency) in &manifest.dependencies { + for declaration in std::iter::once(dependency) + .chain(&dependency.variants) + .filter(|d| d.enabled) + { + explicit.insert(name.clone()); + if declaration.git.is_some() || declaration.path.is_some() { + continue; + } + let source = source_url( + declaration + .source + .as_deref() + .or(default.as_deref()) + .context("Dependency needs an explicit source")?, + )?; + if let Some(previous) = roots.insert(name.clone(), source.clone()) { + ensure!(previous == source, "Conflicting sources for {name}"); + } + if !clients.contains_key(&source) { + clients.insert(source.clone(), Client::new(&source, cache, offline)?); + } + } + } + let metadata = Metadata { + routes: roots.clone(), + ..Default::default() + }; + Ok(Self { + clients, + default, + roots, + explicit, + metadata: Mutex::new(metadata), + }) + } + + fn fetch( + &self, + metadata: &mut Metadata, + source: &str, + name: &str, + ) -> Result> { + let key = (source.to_owned(), name.to_owned()); + if let Some(records) = metadata.records.get(&key) { + return Ok(records.clone()); + } + let records = self.clients[source] + .info_or_empty(name)? + .into_iter() + .map(|info| PackageRecord::from_info(name, source, info)) + .collect::>>()?; + metadata.records.insert(key, records.clone()); + Ok(records) + } + + fn discover_scoped_sources(&self, metadata: &mut Metadata) -> Result<()> { + if metadata.discovered { + return Ok(()); + } + let mut pending: BTreeSet<_> = self + .roots + .iter() + .filter(|(_, source)| Some(source.as_str()) != self.default.as_deref()) + .map(|(name, source)| (source.clone(), name.clone())) + .collect(); + let mut visited = BTreeSet::new(); + while let Some((source, name)) = pending.pop_first() { + if !visited.insert((source.clone(), name.clone())) { + continue; + } + if !self.explicit.contains(&name) { + if !metadata.names.contains_key(&source) { + metadata + .names + .insert(source.clone(), self.clients[&source].names()?); + } + if metadata.names[&source] + .as_ref() + .is_some_and(|names| !names.contains(&name)) + { + continue; + } + } + let records = self.fetch(metadata, &source, &name)?; + if records.is_empty() { + continue; + } + if !self.explicit.contains(&name) + && let Some(previous) = metadata.routes.insert(name.clone(), source.clone()) + { + ensure!(previous == source, "Ambiguous transitive source for {name}"); + } + for dependency in records + .iter() + .flat_map(|record| record.package.dependencies.keys()) + { + if !self.explicit.contains(dependency) { + pending.insert((source.clone(), dependency.clone())); + } + } + } + metadata.discovered = true; + Ok(()) + } +} + +impl PackageIndex for RegistryIndex { + fn candidates(&self, name: &str) -> Result> { + let mut metadata = self.metadata.lock().unwrap(); + self.discover_scoped_sources(&mut metadata)?; + let source = metadata + .routes + .get(name) + .cloned() + .or_else(|| self.default.clone()) + .context("Transitive dependency has no unambiguous source")?; + self.fetch(&mut metadata, &source, name) + } + + fn archive(&self, record: &PackageRecord) -> Result { + let crate::PackageSource::Registry(source) = &record.source else { + anyhow::bail!("Not a registry archive"); + }; + self.clients + .get(source) + .context("Unknown package source")? + .download( + &record.package.id.name, + &record.archive_version, + &record.checksum, + ) + } +} diff --git a/crates/rb-packages/src/runtime.rs b/crates/rb-packages/src/runtime.rs new file mode 100644 index 0000000..41e62b5 --- /dev/null +++ b/crates/rb-packages/src/runtime.rs @@ -0,0 +1,51 @@ +use anyhow::{Context, Result, ensure}; +use rb_gem_types::{Platform, Version}; +use serde::Deserialize; +use std::{ + path::{Path, PathBuf}, + process::Command, +}; + +#[derive(Clone, Debug, Deserialize)] +pub struct RubyEnvironment { + pub executable: PathBuf, + pub version: Version, + pub platform: String, + pub engine: String, + pub engine_version: Version, + pub patchlevel: i32, + pub abi: String, + pub rubygems: Version, + pub bundler: Option, +} + +impl RubyEnvironment { + pub fn inspect(ruby: impl AsRef) -> Result { + let helper = tempfile::Builder::new().suffix(".rb").tempfile()?; + std::fs::write( + helper.path(), + include_str!("../../../gems/rb-gem-installer/lib/rb_gem_installer/runtime.rb"), + )?; + let output = Command::new(ruby.as_ref()) + .env_remove("RUBYOPT") + .env_remove("RUBYLIB") + .args(["-rjson", "-r"]) + .arg(helper.path()) + .args([ + "-e", + "puts JSON.generate(RubyButler::GemInstaller::Runtime.read)", + ]) + .output() + .context("Inspecting Ruby")?; + ensure!( + output.status.success(), + "Ruby inspection failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + serde_json::from_slice(&output.stdout).context("Reading Ruby environment") + } + + pub fn target(&self) -> Platform { + Platform::parse(&self.platform) + } +} diff --git a/crates/rb-packages/src/slot.rs b/crates/rb-packages/src/slot.rs new file mode 100644 index 0000000..f0cc965 --- /dev/null +++ b/crates/rb-packages/src/slot.rs @@ -0,0 +1,19 @@ +use crate::receipt::Receipt; +use std::{ + collections::BTreeMap, + path::PathBuf, + sync::{Arc, Mutex}, +}; + +pub(crate) type Slots = BTreeMap>>; + +pub(crate) struct Slot { + pub(crate) directory: PathBuf, + pub(crate) archive: Option, + pub(crate) metadata: Option, + pub(crate) staging: Option, + pub(crate) reused: bool, + pub(crate) receipt: Receipt, + pub(crate) gem: Option, + pub(crate) dependencies: std::collections::BTreeSet, +} diff --git a/crates/rb-packages/src/source_archive.rs b/crates/rb-packages/src/source_archive.rs new file mode 100644 index 0000000..d5cf300 --- /dev/null +++ b/crates/rb-packages/src/source_archive.rs @@ -0,0 +1,33 @@ +use anyhow::Result; +use flate2::{Compression, write::GzEncoder}; +use std::{io::Write, path::Path}; + +pub(crate) fn archive( + root: &Path, + metadata: &str, + files: &[std::path::PathBuf], +) -> Result> { + let mut payload = tar::Builder::new(GzEncoder::new(Vec::new(), Compression::default())); + payload.follow_symlinks(false); + for path in files { + payload.append_path_with_name(root.join(path), path)?; + } + let payload = payload.into_inner()?.finish()?; + let mut metadata_gzip = GzEncoder::new(Vec::new(), Compression::default()); + metadata_gzip.write_all(metadata.as_bytes())?; + let mut outer = tar::Builder::new(Vec::new()); + for (name, bytes) in [ + ( + crate::constants::GEM_METADATA_MEMBER, + metadata_gzip.finish()?, + ), + (crate::constants::GEM_PAYLOAD_MEMBER, payload), + ] { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + outer.append_data(&mut header, name, bytes.as_slice())?; + } + Ok(outer.into_inner()?) +} diff --git a/crates/rb-packages/src/source_compatibility.rs b/crates/rb-packages/src/source_compatibility.rs new file mode 100644 index 0000000..97bb53b --- /dev/null +++ b/crates/rb-packages/src/source_compatibility.rs @@ -0,0 +1,77 @@ +use crate::{InstalledBundle, PackageIndex, PackagePlan, PackageSource, ProjectInput}; +use anyhow::{Context, Result}; +use std::path::PathBuf; + +pub(crate) fn write( + input: &ProjectInput, + plan: &PackagePlan, + bundle: &InstalledBundle, + index: &dyn PackageIndex, +) -> Result { + if !plan + .packages + .iter() + .any(|p| matches!(p.record.source, PackageSource::Path { .. })) + { + return Ok(input.bundler_gemfile.clone()); + } + let mut manifest = input.manifest.clone(); + let mut lock = plan.lockfile.clone(); + let mut paths = std::collections::BTreeMap::new(); + for package in &plan.packages { + if matches!(package.record.source, PackageSource::Path { .. }) { + let runtime = index + .compatibility_path(&package.record, &bundle.gem_home)? + .context("Path source has no compatibility snapshot")?; + paths.insert( + package.record.source.clone(), + runtime.to_string_lossy().into_owned(), + ); + } + } + fn rewrite( + dependency: &mut rb_gemfile::Dependency, + paths: &std::collections::BTreeMap, + ) -> Result<()> { + if let Some(path) = &mut dependency.path { + let source = PackageSource::Path { + directory: PathBuf::from(&*path), + glob: crate::package_source::normalized_glob(dependency.glob.as_deref()), + }; + *path = paths.get(&source).context("Missing path snapshot")?.clone(); + } + for variant in &mut dependency.variants { + rewrite(variant, paths)?; + } + Ok(()) + } + for dependency in manifest.dependencies.values_mut() { + rewrite(dependency, &paths)?; + } + for source in &mut lock.sources { + if source.kind == rb_lockfile::SourceKind::Path { + let identity = PackageSource::from_lock( + source, + source + .remotes + .first() + .context("Path lock has no location")?, + Some(&input.root), + )?; + source.remotes = vec![ + paths + .get(&identity) + .context("Missing locked path snapshot")? + .clone(), + ]; + } + } + let gemfile = input + .root + .join(crate::constants::STATE_DIRECTORY) + .join(crate::constants::RUNTIME_DIRECTORY) + .join(crate::constants::GEMFILE_FILE); + rb_gemfile::write_project(&gemfile, &crate::gemfile::render(&manifest)?)?; + rb_gemfile::write_project(gemfile.with_file_name("Gemfile.lock"), &lock.render()?)?; + Ok(gemfile) +} diff --git a/crates/rb-packages/src/source_files.rs b/crates/rb-packages/src/source_files.rs new file mode 100644 index 0000000..be15659 --- /dev/null +++ b/crates/rb-packages/src/source_files.rs @@ -0,0 +1,101 @@ +use anyhow::{Result, ensure}; +use sha2::{Digest, Sha256}; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +pub(crate) fn files(root: &Path) -> Result> { + enumerate(root, true) +} +pub(crate) fn payload_files(root: &Path) -> Result> { + enumerate(root, false) +} + +fn enumerate(root: &Path, source: bool) -> Result> { + fn visit(root: &Path, path: &Path, output: &mut Vec, source: bool) -> Result<()> { + let mut entries = fs::read_dir(path)?.collect::>>()?; + entries.sort_by_key(|entry| entry.file_name()); + for entry in entries { + if source + && matches!( + entry.file_name().to_str(), + Some(".git" | ".rb" | ".rb-native") + ) + { + continue; + } + let path = entry.path(); + let kind = entry.file_type()?; + if kind.is_dir() { + visit(root, &path, output, source)?; + } else { + ensure!( + kind.is_file() || kind.is_symlink(), + "Unsupported source file" + ); + let relative = path.strip_prefix(root)?; + ensure!( + !relative.components().any(|part| part + .as_os_str() + .as_encoded_bytes() + .iter() + .any(|byte| matches!(byte, b'\\' | b':'))), + "Nonportable source file" + ); + if source && kind.is_symlink() { + ensure!( + dunce::canonicalize(&path)?.starts_with(dunce::canonicalize(root)?), + "Source link escapes its directory" + ); + } + output.push(relative.to_owned()); + } + } + Ok(()) + } + let mut output = Vec::new(); + visit(root, root, &mut output, source)?; + Ok(output) +} + +pub(crate) fn fingerprint(root: &Path) -> Result { + let mut hash = Sha256::new(); + for path in files(root)? { + let name = path.as_os_str().as_encoded_bytes(); + hash.update(name.len().to_le_bytes()); + hash.update(name); + let absolute = root.join(&path); + let metadata = fs::symlink_metadata(&absolute)?; + if metadata.file_type().is_symlink() { + hash.update(b"link"); + hash.update(fs::read_link(absolute)?.as_os_str().as_encoded_bytes()); + } else { + hash.update(b"file"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + hash.update(metadata.permissions().mode().to_le_bytes()); + } + hash.update(metadata.len().to_le_bytes()); + hash.update(fs::read(absolute)?); + } + } + Ok(format!("{:x}", hash.finalize())) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn native_filesystem_separators_work_in_nested_payloads() { + let root = tempfile::tempdir().unwrap(); + fs::create_dir(root.path().join("lib")).unwrap(); + fs::write(root.path().join("lib").join("fixture.rb"), "answer=42\n").unwrap(); + assert_eq!( + payload_files(root.path()).unwrap(), + [PathBuf::from("lib").join("fixture.rb")] + ); + } +} diff --git a/crates/rb-packages/src/source_index.rs b/crates/rb-packages/src/source_index.rs new file mode 100644 index 0000000..f67b829 --- /dev/null +++ b/crates/rb-packages/src/source_index.rs @@ -0,0 +1,237 @@ +use crate::{ + LockMode, PackageIndex, PackageRecord, PackageSource, ProjectInput, RegistryIndex, + RubyEnvironment, source_package::SourcePackage, +}; +use anyhow::{Context, Result, ensure}; +use std::{ + collections::BTreeMap, + path::{Path, PathBuf}, +}; + +pub struct SourceIndex { + registry: RegistryIndex, + pub(crate) packages: BTreeMap>, + roots: BTreeMap, + affinity: std::sync::Mutex>, +} + +impl SourceIndex { + pub fn new( + input: &ProjectInput, + runtime: &RubyEnvironment, + cache: &Path, + offline: bool, + previous: Option<&rb_lockfile::Lockfile>, + mode: LockMode, + ) -> Result { + ensure!( + input.manifest.dependencies.values().all(|dependency| { + std::iter::once(dependency) + .chain(&dependency.variants) + .all(|declaration| { + !declaration.enabled + || declaration.platforms.is_empty() + || (declaration.git.is_none() && declaration.path.is_none()) + }) + }), + "Platform-specific Git/path sources require multi-target source routing" + ); + let mut packages: BTreeMap> = BTreeMap::new(); + let mut seen = std::collections::BTreeSet::new(); + for declaration in input + .manifest + .dependencies + .values() + .flat_map(|d| std::iter::once(d).chain(&d.variants)) + .filter(|d| d.enabled && (d.git.is_some() || d.path.is_some())) + { + let key = serde_json::to_string( + &serde_json::json!({"git":declaration.git,"path":declaration.path,"options":crate::package_source::git_options(declaration)}), + )?; + if !seen.insert(key) { + continue; + } + let (source, directory) = if let Some(remote) = &declaration.git { + let options = crate::package_source::git_options(declaration); + let pinned = if mode == LockMode::Update { + None + } else { + previous + .and_then(|lock| { + lock.sources.iter().find(|s| { + let mut old = s.options.clone(); + old.remove("revision"); + if old.get("glob").is_some_and(|value| { + value == crate::constants::DEFAULT_GEMSPEC_GLOB + }) { + old.remove("glob"); + } + s.kind == rb_lockfile::SourceKind::Git + && s.remotes.iter().any(|previous| { + previous.trim_end_matches('/') + == remote.trim_end_matches('/') + }) + && old == options + }) + }) + .and_then(|s| s.options.get("revision")) + .map(String::as_str) + }; + let (revision, directory) = crate::git_cache::checkout( + &input.root, + remote, + &options, + cache, + pinned, + offline, + )?; + ( + PackageSource::Git { + remote: remote.trim_end_matches('/').into(), + revision, + options, + }, + directory, + ) + } else { + let directory = + dunce::canonicalize(input.root.join(declaration.path.as_ref().unwrap()))?; + ( + PackageSource::Path { + directory: directory.clone(), + glob: crate::package_source::normalized_glob(declaration.glob.as_deref()), + }, + directory, + ) + }; + let snapshot = crate::source_files::fingerprint(&directory)?; + let specifications = crate::source_specs::read( + &runtime.executable, + &directory, + declaration.glob.as_deref(), + )?; + ensure!( + crate::source_files::fingerprint(&directory)? == snapshot, + "Gemspec evaluation modified its source directory" + ); + for spec in specifications { + let package = SourcePackage::new(spec, &source, &directory, cache, &snapshot)?; + packages + .entry(package.record.package.id.name.clone()) + .or_default() + .push(package); + } + } + let mut registry = RegistryIndex::new(&input.manifest, cache, offline)?; + registry.load_credentials(&input.root, offline)?; + Ok(Self { + registry, + packages, + roots: input + .manifest + .dependencies + .iter() + .filter_map(|(name, d)| { + std::iter::once(d) + .chain(&d.variants) + .find(|d| d.enabled) + .map(|d| (name.clone(), d.clone())) + }) + .collect(), + affinity: std::sync::Mutex::new(BTreeMap::new()), + }) + } + + pub(crate) fn local(&self, record: &PackageRecord) -> Result<&SourcePackage> { + self.packages + .get(&record.package.id.name) + .and_then(|entries| entries.iter().find(|entry| entry.record == *record)) + .context("Source package is unavailable") + } +} + +impl PackageIndex for SourceIndex { + fn candidates(&self, name: &str) -> Result> { + let records = if let Some(root) = self.roots.get(name) { + if root.git.is_none() && root.path.is_none() { + return self.registry.candidates(name); + } + self.packages + .get(name) + .map(|entries| { + entries + .iter() + .filter_map(|p| match p.record.source.matches(root, None) { + Ok(true) => Some(Ok(p.record.clone())), + Ok(false) => None, + Err(error) => Some(Err(error)), + }) + .collect::>>() + }) + .transpose()? + .unwrap_or_default() + } else if let Some(source) = self.affinity.lock().unwrap().get(name).cloned() { + self.packages + .get(name) + .map(|entries| { + entries + .iter() + .filter(|p| p.record.source == source) + .map(|p| p.record.clone()) + .collect() + }) + .unwrap_or_default() + } else { + return self.registry.candidates(name); + }; + let mut affinity = self.affinity.lock().unwrap(); + for record in &records { + for dependency in record + .package + .dependencies + .keys() + .filter(|name| !self.roots.contains_key(*name)) + { + if self + .packages + .get(dependency) + .is_some_and(|entries| entries.iter().any(|p| p.record.source == record.source)) + { + if let Some(previous) = affinity.get(dependency) { + ensure!( + previous == &record.source, + "Ambiguous source for {dependency}" + ); + } + affinity.insert(dependency.clone(), record.source.clone()); + } + } + } + Ok(records) + } + fn archive(&self, record: &PackageRecord) -> Result { + if matches!(record.source, PackageSource::Registry(_)) { + self.registry.archive(record) + } else { + Ok(self.local(record)?.archive.clone()) + } + } + fn stage_sources( + &self, + home: &Path, + plan: &crate::PackagePlan, + runtime: &RubyEnvironment, + ) -> Result { + crate::source_install::stage(self, home, plan, runtime) + } + fn compatibility_path(&self, record: &PackageRecord, home: &Path) -> Result> { + if matches!(record.source, PackageSource::Path { .. }) { + Ok(Some(crate::source_install::directory( + self.local(record)?, + home, + )?)) + } else { + Ok(None) + } + } +} diff --git a/crates/rb-packages/src/source_install.rs b/crates/rb-packages/src/source_install.rs new file mode 100644 index 0000000..c099812 --- /dev/null +++ b/crates/rb-packages/src/source_install.rs @@ -0,0 +1,175 @@ +use crate::{ + PackagePlan, PackageSource, RubyEnvironment, source_index::SourceIndex, + source_package::SourcePackage, +}; +use anyhow::{Result, ensure}; +use sha2::{Digest, Sha256}; +use std::{ + collections::BTreeSet, + fs, + path::{Path, PathBuf}, +}; + +pub(crate) fn directory(package: &SourcePackage, home: &Path) -> Result { + Ok(match &package.record.source { + PackageSource::Git { + remote, revision, .. + } => { + let basename = if Path::new(remote).is_absolute() { + Path::new(remote).file_name().unwrap().to_string_lossy() + } else { + std::borrow::Cow::Borrowed( + remote + .trim_end_matches('/') + .rsplit(['/', ':']) + .next() + .unwrap(), + ) + }; + let base = basename.strip_suffix(".git").unwrap_or(&basename); + ensure!( + !base.is_empty() + && base + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_' | b'.')), + "Nonportable Git repository name" + ); + home.join(crate::constants::BUNDLER_DIRECTORY) + .join(crate::constants::GEMS_DIRECTORY) + .join(format!( + "{base}-{}", + &revision[..crate::constants::GIT_SHORT_REVISION_LENGTH] + )) + } + PackageSource::Path { .. } => { + let id = format!( + "{:x}", + Sha256::digest(serde_json::to_vec(&( + &package.record.source, + &package.snapshot + ))?) + ); + home.join(crate::constants::BUNDLER_DIRECTORY) + .join(crate::constants::SOURCE_PATHS_DIRECTORY) + .join(id) + } + PackageSource::Registry(_) => anyhow::bail!("Registry package has no source snapshot"), + }) +} + +fn copy_snapshot(source: &Path, destination: &Path, files: &BTreeSet) -> Result<()> { + fs::create_dir_all(destination)?; + for relative in files { + let from = source.join(relative); + let to = destination.join(relative); + fs::create_dir_all(to.parent().unwrap())?; + if fs::symlink_metadata(&from)?.file_type().is_symlink() { + let target = fs::read_link(&from)?; + ensure!(!target.is_absolute(), "Source links must be relative"); + #[cfg(unix)] + std::os::unix::fs::symlink(target, to)?; + #[cfg(windows)] + { + if from.is_dir() { + std::os::windows::fs::symlink_dir(target, to)?; + } else { + std::os::windows::fs::symlink_file(target, to)?; + } + } + } else { + fs::copy(from, to)?; + } + } + Ok(()) +} + +pub(crate) fn stage( + index: &SourceIndex, + home: &Path, + plan: &PackagePlan, + runtime: &RubyEnvironment, +) -> Result { + let mut copied = BTreeSet::new(); + let mut extension_targets = BTreeSet::new(); + let mut changed = false; + for planned in plan + .packages + .iter() + .filter(|p| !matches!(p.record.source, PackageSource::Registry(_))) + { + let package = index.local(&planned.record)?; + let destination = directory(package, home)?; + if copied.insert(destination.clone()) { + let stamp = destination.join(crate::constants::SOURCE_STAMP_FILE); + let files: BTreeSet<_> = index + .packages + .values() + .flatten() + .filter(|entry| entry.record.source == package.record.source) + .flat_map(|entry| { + entry + .payload + .iter() + .map(|path| entry.gemspec.parent().unwrap().join(path)) + }) + .collect(); + let complete = files.iter().all(|path| destination.join(path).exists()); + if !complete || fs::read_to_string(&stamp).ok().as_deref() != Some(&package.snapshot) { + if destination.exists() { + fs::remove_dir_all(&destination)?; + } + copy_snapshot(&package.directory, &destination, &files)?; + fs::write(&stamp, &package.snapshot)?; + changed = true; + } + } + let spec = destination.join(&package.gemspec); + if fs::read_to_string(&spec).ok().as_deref() != Some(&package.serialized) { + rb_gemfile::write_project(&spec, &package.serialized)?; + changed = true; + } + let full_name = format!( + "{}-{}", + planned.record.package.id.name, planned.record.archive_version + ); + if !planned.install { + continue; + } + let installed = home.join(crate::constants::GEMS_DIRECTORY).join(&full_name); + let receipt = fs::read(installed.join(crate::constants::INSTALL_RECEIPT))?; + let metadata: crate::receipt::InstalledReceipt = serde_json::from_slice(&receipt)?; + if let Some(extension) = &metadata.extension { + let gem_directory = destination.join(package.gemspec.parent().unwrap()); + for relative in &metadata.build_artifacts { + let target = gem_directory.join(relative); + fs::create_dir_all(target.parent().unwrap())?; + fs::copy(installed.join(relative), target)?; + } + let source_extensions = home.join(extension); + let unique = if matches!(package.record.source, PackageSource::Git { .. }) { + let scope = destination.file_name().unwrap().to_string_lossy(); + let name = gem_directory.file_name().unwrap().to_string_lossy(); + if scope == name { + scope.into_owned() + } else { + format!("{scope}-{name}") + } + } else { + full_name.clone() + }; + let target = gem_directory + .parent() + .unwrap() + .join(crate::constants::EXTENSIONS_DIRECTORY) + .join(&runtime.platform) + .join(&runtime.abi) + .join(unique); + if extension_targets.insert(target.clone()) && target.exists() { + fs::remove_dir_all(&target)?; + } + crate::copy_tree::copy_tree(&source_extensions, &target)?; + changed = true; + } + } + Ok(changed) +} diff --git a/crates/rb-packages/src/source_package.rs b/crates/rb-packages/src/source_package.rs new file mode 100644 index 0000000..db51c95 --- /dev/null +++ b/crates/rb-packages/src/source_package.rs @@ -0,0 +1,68 @@ +use crate::{PackageRecord, PackageSource, source_specs::SourceSpecification}; +use anyhow::{Result, ensure}; +use sha2::{Digest, Sha256}; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +pub(crate) struct SourcePackage { + pub(crate) record: PackageRecord, + pub(crate) archive: PathBuf, + pub(crate) directory: PathBuf, + pub(crate) gemspec: PathBuf, + pub(crate) serialized: String, + pub(crate) snapshot: String, + pub(crate) payload: Vec, +} + +impl SourcePackage { + pub(crate) fn new( + spec: SourceSpecification, + source: &PackageSource, + directory: &Path, + cache: &Path, + snapshot: &str, + ) -> Result { + let gem_root = directory.join(spec.gemspec.parent().unwrap()); + ensure!( + gem_root.starts_with(directory), + "Gemspec directory escapes source" + ); + let payload = spec.payload(&gem_root)?; + let bytes = crate::source_archive::archive(&gem_root, &spec.metadata, &payload)?; + let checksum = format!("{:x}", Sha256::digest(&bytes)); + let metadata = rb_gem_archive::read(bytes.as_slice())?; + let record = PackageRecord { + archive_version: metadata.id.archive_version(), + package: rb_resolver::Package { + id: metadata.id.clone(), + dependencies: metadata.dependencies.clone(), + ruby: metadata.ruby.clone(), + }, + source: source.clone(), + checksum: checksum.clone(), + rubygems: Some(metadata.rubygems), + }; + record.validate()?; + let artifact = cache + .join(crate::constants::SOURCE_ARCHIVES_DIRECTORY) + .join(format!("{checksum}.gem")); + fs::create_dir_all(artifact.parent().unwrap())?; + if crate::validation::verify(&artifact, &checksum).is_err() { + use std::io::Write; + let mut temporary = tempfile::NamedTempFile::new_in(artifact.parent().unwrap())?; + temporary.write_all(&bytes)?; + temporary.persist(&artifact).map_err(|error| error.error)?; + } + Ok(Self { + record, + archive: artifact, + directory: directory.to_owned(), + gemspec: spec.gemspec, + serialized: spec.specification, + snapshot: snapshot.into(), + payload, + }) + } +} diff --git a/crates/rb-packages/src/source_pin.rs b/crates/rb-packages/src/source_pin.rs new file mode 100644 index 0000000..9c87332 --- /dev/null +++ b/crates/rb-packages/src/source_pin.rs @@ -0,0 +1,62 @@ +use crate::{PackageRecord, PackageSource}; +use anyhow::Result; +use rb_gem_types::{PackageId, Requirement}; +use std::collections::BTreeMap; + +pub(crate) struct SourcePin { + pub(crate) id: PackageId, + pub(crate) source: PackageSource, + pub(crate) checksum: Option, + dependencies: BTreeMap, +} + +impl SourcePin { + pub(crate) fn new( + id: PackageId, + source: PackageSource, + checksum: Option, + package: &rb_lockfile::LockedPackage, + ) -> Result { + let dependencies = package + .dependencies + .iter() + .map(|(name, requirements)| { + Ok(( + name.clone(), + requirements + .iter() + .map(ToString::to_string) + .collect::>() + .join(", ") + .parse()?, + )) + }) + .collect::>()?; + Ok(Self { + id, + source, + checksum, + dependencies, + }) + } + pub(crate) fn matches(&self, record: &PackageRecord) -> bool { + self.id == record.package.id + && self.source == record.source + && self + .checksum + .as_ref() + .is_none_or(|sum| sum.eq_ignore_ascii_case(&record.checksum)) + } + pub(crate) fn dependencies_match(&self, record: &PackageRecord) -> bool { + self.dependencies.len() == record.package.dependencies.len() + && self.dependencies.iter().all(|(name, requirement)| { + record + .package + .dependencies + .get(name) + .is_some_and(|expected| { + crate::validation::same_requirement(requirement, expected) + }) + }) + } +} diff --git a/crates/rb-packages/src/source_specs.rs b/crates/rb-packages/src/source_specs.rs new file mode 100644 index 0000000..224420f --- /dev/null +++ b/crates/rb-packages/src/source_specs.rs @@ -0,0 +1,119 @@ +use anyhow::{Context, Result, ensure}; +use serde::Deserialize; +use std::{ + path::{Path, PathBuf}, + process::Command, +}; + +#[derive(Deserialize)] +pub(crate) struct SourceSpecification { + pub(crate) gemspec: PathBuf, + pub(crate) metadata: String, + pub(crate) specification: String, + pub(crate) files: Vec, + pub(crate) require_paths: Vec, + pub(crate) extensions: Vec, +} + +impl SourceSpecification { + pub(crate) fn payload(&self, root: &Path) -> Result> { + let mut selected = if self.files.is_empty() { + self.require_paths.clone() + } else { + self.files.clone() + }; + for extension in &self.extensions { + selected.push( + extension + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or(extension) + .to_owned(), + ); + } + for path in &selected { + ensure!( + path.components().all(|part| matches!( + part, + std::path::Component::Normal(_) | std::path::Component::CurDir + )), + "Source specification path escapes its package" + ); + } + Ok(crate::source_files::files(root)? + .into_iter() + .filter(|path| { + selected.iter().any(|selection| { + path == selection || path.starts_with(selection) || selection == Path::new(".") + }) + }) + .collect()) + } +} + +pub(crate) fn read( + ruby: &Path, + directory: &Path, + glob: Option<&str>, +) -> Result> { + let helper = tempfile::Builder::new().suffix(".rb").tempfile()?; + std::fs::write( + helper.path(), + include_str!( + "../../../gems/rb-gem-installer/lib/rb_gem_installer/source_specifications.rb" + ), + )?; + let result = tempfile::NamedTempFile::new()?; + let request = tempfile::NamedTempFile::new()?; + std::fs::write( + request.path(), + serde_json::to_vec(&serde_json::json!({ + "directory": directory, + "glob": glob.unwrap_or(crate::constants::DEFAULT_GEMSPEC_GLOB), + "output": result.path(), + }))?, + )?; + let output = Command::new(ruby).env_remove("RUBYOPT").env_remove("RUBYLIB").env_remove("BUNDLE_GEMFILE") + .args(["-rjson", "-r"]).arg(helper.path()).args(["-e", "request = JSON.parse(File.read(ARGV.fetch(0))); specs = RubyButler::GemInstaller::SourceSpecifications.read(request.fetch('directory'), request.fetch('glob')); File.write(request.fetch('output'), JSON.generate(specs))"]) + .arg(request.path()) + .current_dir(directory).output().context("Evaluating source gemspecs")?; + ensure!( + output.status.success(), + "Source gemspec evaluation failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + let specs: Vec = serde_json::from_slice(&std::fs::read(result.path())?) + .context("Reading source gemspec response")?; + ensure!(!specs.is_empty(), "Source contains no matching gemspecs"); + Ok(specs) +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::{fs, os::unix::fs::PermissionsExt}; + + #[test] + fn wildcard_expansion_cannot_shift_helper_arguments() { + let root = tempfile::tempdir().unwrap(); + for name in ["one", "two"] { + fs::write(root.path().join(format!("{name}.gemspec")), format!("Gem::Specification.new {{ |s| s.name='{name}'; s.version='1.0'; s.summary='Fixture'; s.authors=['Fixture'] }}\n")).unwrap(); + } + let ruby = root.path().join("ruby-shim"); + fs::write( + &ruby, + include_str!("../tests/fixtures/expand_ruby_arguments.rb"), + ) + .unwrap(); + fs::set_permissions(&ruby, fs::Permissions::from_mode(0o755)).unwrap(); + let specs = read(&ruby, root.path(), None).unwrap(); + assert_eq!(specs.len(), 2); + assert_eq!( + specs + .iter() + .map(|spec| spec.gemspec.clone()) + .collect::>(), + [PathBuf::from("one.gemspec"), PathBuf::from("two.gemspec")] + ); + } +} diff --git a/crates/rb-packages/src/specification.rs b/crates/rb-packages/src/specification.rs new file mode 100644 index 0000000..006bf79 --- /dev/null +++ b/crates/rb-packages/src/specification.rs @@ -0,0 +1,122 @@ +use anyhow::{Context, Result, ensure}; +use rb_gem_archive::GemMetadata; +use serde_json::json; +use std::{ + path::{Path, PathBuf}, + process::Command, +}; + +pub(crate) fn write( + ruby: &Path, + metadata: &GemMetadata, + archive_version: &str, + home: &Path, +) -> Result { + let directory = home.join(crate::constants::SPECIFICATIONS_DIRECTORY); + std::fs::create_dir_all(&directory)?; + let filename = directory.join(format!("{}-{archive_version}.gemspec", metadata.id.name)); + let (_, platform) = archive_version + .split_once('-') + .unwrap_or((archive_version, "ruby")); + let value = json!({ + "name": metadata.id.name, "version": metadata.id.version.to_string(), "platform": platform, + "ruby": metadata.ruby.to_string(), "rubygems": metadata.rubygems.to_string(), + "dependencies": metadata.dependencies.iter().map(|(n,r)| (n.clone(),r.to_string())).collect::>(), + "require_paths": metadata.require_paths, "executables": metadata.executables, "bindir": metadata.bindir, + "extensions": metadata.extensions, "files": metadata.files, "metadata": metadata.metadata, + }); + let data = tempfile::NamedTempFile::new()?; + std::fs::write(data.path(), serde_json::to_vec(&value)?)?; + let helper = tempfile::Builder::new().suffix(".rb").tempfile()?; + std::fs::write( + helper.path(), + include_str!("../../../gems/rb-gem-installer/lib/rb_gem_installer/specification.rb"), + )?; + let output = Command::new(ruby) + .env_remove("RUBYOPT") + .env_remove("RUBYLIB") + .args([ + "-rrubygems", + "-rjson", + "-e", + "require ARGV.shift; RubyButler::GemInstaller::Specification.write(JSON.parse(File.read(ARGV[0])), ARGV[1])", + ]) + .arg(helper.path()) + .arg(data.path()) + .arg(&filename) + .output() + .context("Writing installed specification")?; + ensure!( + output.status.success(), + "Writing specification failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + Ok(filename) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + fs, + sync::{Arc, Mutex}, + }; + + #[test] + #[ignore = "requires RB_GEM_CORPUS and RubyGems for the archive oracle"] + fn specifications_match_the_archive_corpus() { + fn archives(root: &Path, files: &mut Vec) { + for entry in fs::read_dir(root).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + archives(&path, files); + } else if path.extension().is_some_and(|extension| extension == "gem") { + files.push(path); + } + } + } + let root = std::env::var_os("RB_GEM_CORPUS").expect("Set RB_GEM_CORPUS"); + let ruby = std::env::var_os("RB_GEM_BUILD_RUBY").unwrap_or_else(|| "ruby".into()); + let mut paths = Vec::new(); + archives(Path::new(&root), &mut paths); + paths.sort(); + assert!(!paths.is_empty()); + let paths = Arc::new(Mutex::new(paths.into_iter())); + let failures = Arc::new(Mutex::new(Vec::new())); + let count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + std::thread::scope(|scope| { + for _ in 0..4 { + let paths = paths.clone(); + let failures = failures.clone(); + let count = count.clone(); + let ruby = ruby.clone(); + scope.spawn(move || loop { + let Some(archive) = paths.lock().unwrap().next() else { break; }; + let result = (|| -> Result<()> { + let metadata = rb_gem_archive::read(fs::File::open(&archive)?)?; + let home = tempfile::tempdir()?; + let specification = write(Path::new(&ruby), &metadata, &metadata.id.archive_version(), home.path())?; + let output = Command::new(&ruby).env_remove("RUBYOPT").env_remove("RUBYLIB") + .args(["-rrubygems/package", "-e", "a = Gem::Package.new(ARGV[0]).spec; b = Gem::Specification.load(ARGV[1]); fields = [:name, :version, :platform, :required_ruby_version, :required_rubygems_version, :raw_require_paths, :executables, :bindir, :extensions, :metadata]; fields.each { |field| raise field.to_s + ': ' + [a.public_send(field), b.public_send(field)].inspect unless a.public_send(field) == b.public_send(field) }; raise 'dependencies differ' unless a.runtime_dependencies.sort_by(&:name) == b.runtime_dependencies.sort_by(&:name)"]) + .arg(&archive).arg(specification).output()?; + ensure!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr)); + Ok(()) + })(); + count.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + if let Err(error) = result { failures.lock().unwrap().push(format!("{}: {error:#}", archive.display())); } + }); + } + }); + let failures = failures.lock().unwrap(); + println!( + "Checked {} archive specifications", + count.load(std::sync::atomic::Ordering::Relaxed) + ); + assert!( + failures.is_empty(), + "{} failures:\n{}", + failures.len(), + failures.join("\n") + ); + } +} diff --git a/crates/rb-packages/src/target_plan.rs b/crates/rb-packages/src/target_plan.rs new file mode 100644 index 0000000..2ec1c08 --- /dev/null +++ b/crates/rb-packages/src/target_plan.rs @@ -0,0 +1,137 @@ +use crate::RubyEnvironment; +use anyhow::{Result, ensure}; +use rb_gem_types::Requirement; +use rb_lockfile::Dependency; +use std::collections::{BTreeMap, BTreeSet}; + +pub(crate) struct TargetPlan { + pub(crate) roots: BTreeMap, + pub(crate) install_roots: BTreeSet, + pub(crate) sources: BTreeMap, + pub(crate) force_ruby: BTreeSet, + pub(crate) declarations: BTreeMap, +} + +impl TargetPlan { + pub(crate) fn new( + manifest: &rb_gemfile::Manifest, + runtime: &RubyEnvironment, + options: &crate::PlanOptions, + ) -> Result { + let mut roots: BTreeMap = BTreeMap::new(); + let mut install_roots = BTreeSet::new(); + let mut sources = BTreeMap::new(); + let mut force_ruby = BTreeSet::new(); + let mut declarations = BTreeMap::new(); + for (name, dependency) in &manifest.dependencies { + for declaration in std::iter::once(dependency).chain(&dependency.variants) { + declaration.validate().map_err(anyhow::Error::msg)?; + if !declaration.enabled { + continue; + } + let applies = platform_applies(&declaration.platforms, runtime)?; + + let constraint: Requirement = declaration.version.join(", ").parse()?; + let value = if let Some(previous) = roots.get(name) { + format!("{previous}, {constraint}").parse()? + } else { + constraint.clone() + }; + if let Some(previous) = declarations.get(name) { + let previous: &Dependency = previous; + ensure!( + previous.requirement == constraint, + "Different requirements across declarations for {name} are not supported" + ); + } + declarations.insert( + name.clone(), + Dependency { + requirement: constraint, + pinned: declaration.source.is_some() + || declaration.git.is_some() + || declaration.path.is_some(), + }, + ); + if !applies { + continue; + } + roots.insert(name.clone(), value); + if let Some(previous) = sources.insert(name.clone(), declaration.clone()) { + ensure!( + previous.git == declaration.git + && previous.path == declaration.path + && previous.source == declaration.source + && crate::package_source::git_options(&previous) + == crate::package_source::git_options(declaration), + "Conflicting sources for {name}" + ); + } + if declaration.force_ruby_platform == Some(true) { + force_ruby.insert(name.clone()); + } + if declaration.groups.iter().any(|g| { + !options.without.contains(g) + && (!manifest.bundle.optional_groups.contains(g) + || options.with.contains(g)) + }) { + install_roots.insert(name.clone()); + } + } + } + Ok(Self { + roots, + install_roots, + sources, + force_ruby, + declarations, + }) + } +} + +fn platform_applies(platforms: &[String], runtime: &RubyEnvironment) -> Result { + if platforms.is_empty() { + return Ok(true); + } + for selector in platforms { + let (name, version) = selector + .rsplit_once('_') + .filter(|(_, suffix)| suffix.chars().all(|c| c.is_ascii_digit())) + .map_or((selector.as_str(), None), |(name, version)| { + (name, Some(version)) + }); + let platform = runtime.target(); + let windows = platform.os.starts_with("mingw") + || platform.os.starts_with("mswin") + || platform.os == "cygwin"; + let applies = match name { + "ruby" => { + !windows + && matches!( + runtime.engine.as_str(), + "ruby" | "rbx" | "maglev" | "truffleruby" + ) + } + "mri" => !windows && runtime.engine == "ruby", + "jruby" => runtime.engine == "jruby", + "truffleruby" => runtime.engine == "truffleruby", + "rbx" => !windows && runtime.engine == "rbx", + "maglev" => runtime.engine == "maglev", + "windows" | "mingw" | "x64_mingw" | "mswin" | "mswin64" => windows, + _ => anyhow::bail!("Unsupported Gemfile platform selector {selector}"), + }; + let version_matches = version.is_none_or(|version| { + runtime + .version + .to_string() + .split('.') + .take(if version.len() == 1 { 1 } else { 2 }) + .collect::() + == version + }); + if applies && version_matches { + return Ok(true); + } + } + Ok(false) +} diff --git a/crates/rb-packages/src/task.rs b/crates/rb-packages/src/task.rs new file mode 100644 index 0000000..66117b8 --- /dev/null +++ b/crates/rb-packages/src/task.rs @@ -0,0 +1,27 @@ +use anyhow::{Result, anyhow}; +use rb_task::{Executor, TaskAction, TaskContext, TaskEvents, TaskGraph}; +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, +}; + +pub(crate) fn run( + label: &str, + events: TaskEvents, + work: impl Fn(TaskContext) -> Result + Send + Sync + 'static, +) -> Result { + let output = Arc::new(Mutex::new(None)); + let result = output.clone(); + let mut graph = TaskGraph::new(); + let id = graph.add(label, []); + let action: TaskAction = Arc::new(move |context| { + *result.lock().unwrap() = Some(work(context).map_err(|e| format!("{e:#}"))?); + Ok(()) + }); + Executor::new(1, events).run(graph, BTreeMap::from([(id, action)]))?; + output + .lock() + .unwrap() + .take() + .ok_or_else(|| anyhow!("Task did not return a result")) +} diff --git a/crates/rb-packages/src/validation.rs b/crates/rb-packages/src/validation.rs new file mode 100644 index 0000000..b625985 --- /dev/null +++ b/crates/rb-packages/src/validation.rs @@ -0,0 +1,62 @@ +use crate::PackageRecord; +use anyhow::{Result, ensure}; +use sha2::{Digest, Sha256}; +use std::{fs, io::Read, path::Path}; + +pub(crate) fn verify(path: &Path, checksum: &str) -> Result<()> { + let mut input = fs::File::open(path)?; + let mut hash = Sha256::new(); + let mut buffer = [0; crate::constants::READ_BUFFER_BYTES]; + loop { + let count = input.read(&mut buffer)?; + if count == 0 { + break; + } + hash.update(&buffer[..count]); + } + ensure!( + format!("{:x}", hash.finalize()) == checksum, + "Archive checksum mismatch" + ); + Ok(()) +} + +pub(crate) fn ensure_record( + metadata: &rb_gem_archive::GemMetadata, + record: &PackageRecord, +) -> Result<()> { + ensure!( + metadata.id == record.package.id, + "Archive identity differs from resolution" + ); + ensure!( + metadata.dependencies.len() == record.package.dependencies.len() + && metadata + .dependencies + .iter() + .all(|(name, requirement)| record + .package + .dependencies + .get(name) + .is_some_and(|expected| same_requirement(requirement, expected))) + && same_requirement(&metadata.ruby, &record.package.ruby), + "Archive metadata differs from resolution for {}: dependencies {:?} / {:?}, Ruby {} / {}", + record.package.id, + metadata.dependencies, + record.package.dependencies, + metadata.ruby, + record.package.ruby + ); + Ok(()) +} + +pub(crate) fn same_requirement( + left: &rb_gem_types::Requirement, + right: &rb_gem_types::Requirement, +) -> bool { + left.constraints() + .all(|constraint| right.constraints().any(|other| constraint == other)) + && right + .constraints() + .all(|constraint| left.constraints().any(|other| constraint == other)) +} diff --git a/crates/rb-packages/src/wrappers.rs b/crates/rb-packages/src/wrappers.rs new file mode 100644 index 0000000..bf2434d --- /dev/null +++ b/crates/rb-packages/src/wrappers.rs @@ -0,0 +1,36 @@ +use anyhow::{Result, ensure}; +use std::{fs, path::Path}; + +pub(crate) fn wrappers(home: &Path, metadata: &rb_gem_archive::GemMetadata) -> Result<()> { + fs::create_dir_all(home.join(crate::constants::BIN_DIRECTORY))?; + for executable in &metadata.executables { + ensure!( + !executable.is_empty() + && Path::new(executable).components().count() == 1 + && executable + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.')), + "Invalid executable name" + ); + let name = serde_json::to_string(&metadata.id.name)?; + let version = serde_json::to_string(&metadata.id.version.to_string())?; + let executable_literal = serde_json::to_string(executable)?; + let script = format!( + "#!/usr/bin/env ruby\nrequire 'rubygems'\nload Gem.activate_bin_path({name}, {executable_literal}, '= ' + {version})\n" + ); + let path = home.join(crate::constants::BIN_DIRECTORY).join(executable); + fs::write(&path, script)?; + #[cfg(windows)] + fs::write( + home.join(crate::constants::BIN_DIRECTORY) + .join(format!("{executable}.bat")), + "@ECHO OFF\r\nruby.exe \"%~dpn0\" %*\r\n", + )?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755))?; + } + } + Ok(()) +} diff --git a/crates/rb-packages/tests/common/git_server.rs b/crates/rb-packages/tests/common/git_server.rs new file mode 100644 index 0000000..6eb7196 --- /dev/null +++ b/crates/rb-packages/tests/common/git_server.rs @@ -0,0 +1,88 @@ +use std::{ + fs, + io::{Read, Write}, + net::TcpListener, + path::{Component, Path, PathBuf}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, + thread, + time::Duration, +}; + +pub struct GitServer { + pub url: String, + stop: Arc, + thread: Option>, +} + +impl GitServer { + pub fn start(root: &Path) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let url = format!("http://{}/", listener.local_addr().unwrap()); + let root = root.to_owned(); + let stop = Arc::new(AtomicBool::new(false)); + let stopped = stop.clone(); + let thread = thread::spawn(move || { + while !stopped.load(Ordering::Relaxed) { + let (mut socket, _) = match listener.accept() { + Ok(value) => value, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(5)); + continue; + } + Err(error) => panic!("{error}"), + }; + socket.set_nonblocking(false).unwrap(); + socket + .set_read_timeout(Some(Duration::from_secs(5))) + .unwrap(); + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + while !request.windows(4).any(|bytes| bytes == b"\r\n\r\n") { + let count = socket.read(&mut buffer).unwrap(); + if count == 0 { + break; + } + request.extend_from_slice(&buffer[..count]); + } + let request = String::from_utf8(request).unwrap(); + let path = PathBuf::from( + request + .split_whitespace() + .nth(1) + .unwrap() + .split('?') + .next() + .unwrap() + .trim_start_matches('/'), + ); + assert!( + path.components() + .all(|part| matches!(part, Component::Normal(_))) + ); + let (status, body) = match fs::read(root.join(path)) { + Ok(bytes) => ("200 OK", bytes), + Err(_) => ("404 Not Found", Vec::new()), + }; + if write!(socket, "HTTP/1.1 {status}\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).is_ok() { let _ = socket.write_all(&body); } + } + }); + Self { + url, + stop, + thread: Some(thread), + } + } +} +impl Drop for GitServer { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + let result = self.thread.take().unwrap().join(); + if !std::thread::panicking() { + result.unwrap(); + } + } +} diff --git a/crates/rb-packages/tests/common/mod.rs b/crates/rb-packages/tests/common/mod.rs new file mode 100644 index 0000000..02f47f2 --- /dev/null +++ b/crates/rb-packages/tests/common/mod.rs @@ -0,0 +1,118 @@ +#![allow(dead_code)] +pub mod server; +use flate2::{Compression, write::GzEncoder}; +use rb_gem_types::{PackageId, Platform}; +use rb_packages::{MemoryIndex, PackageRecord, RubyEnvironment}; +use rb_resolver::Package; +use sha2::{Digest, Sha256}; +use std::{ + io::Write, + path::{Path, PathBuf}, + process::Command, +}; + +pub fn ruby() -> PathBuf { + std::env::var_os("RB_GEM_BUILD_RUBY") + .unwrap_or_else(|| "ruby".into()) + .into() +} +pub fn runtime() -> RubyEnvironment { + RubyEnvironment::inspect(ruby()).unwrap() +} +pub fn manifest(source: &str, dependencies: &str) -> rb_gemfile::Manifest { + toml::from_str(&format!( + "[bundle]\nsource = {source:?}\n[dependencies]\n{dependencies}\n" + )) + .unwrap() +} +pub fn record(name: &str, version: &str, dependencies: &[(&str, &str)]) -> PackageRecord { + PackageRecord { + package: Package { + id: PackageId { + name: name.into(), + version: version.parse().unwrap(), + platform: Platform::parse("ruby"), + }, + dependencies: dependencies + .iter() + .map(|(n, r)| (n.to_string(), r.parse().unwrap())) + .collect(), + ruby: Default::default(), + }, + source: "https://rubygems.org/".into(), + archive_version: version.into(), + checksum: "a".repeat(64), + rubygems: None, + } +} +pub fn add(index: &mut MemoryIndex, record: PackageRecord) { + index + .packages + .entry(record.package.id.name.clone()) + .or_default() + .push(record); +} +pub fn add_archive( + index: &mut MemoryIndex, + directory: &Path, + name: &str, + dependencies: &[(&str, &str)], + extensions: &[&str], + files: &[(&str, &[u8])], +) { + let metadata = serde_json::json!({ "name": name, "version": "1.0.0", "platform": "ruby", "dependencies": dependencies.iter().map(|(n,r)| serde_json::json!({"name":n,"requirement": {"requirements": [[">=", {"version":r.trim_start_matches(">= ")}]]}, "type":"runtime"})).collect::>(), "require_paths": ["lib"], "extensions": extensions, "files": files.iter().map(|(p,_)| *p).collect::>(), "executables": if files.iter().any(|(p,_)| p.starts_with("bin/")) { vec![name] } else { vec![] }, "bindir":"bin" }); + let payload = tar(files); + let archive = tar(&[ + ( + "metadata.gz", + &gzip(&serde_json::to_vec(&metadata).unwrap()), + ), + ("data.tar.gz", &gzip(&payload)), + ]); + let checksum = format!("{:x}", Sha256::digest(&archive)); + let path = directory.join(format!("{name}.gem")); + std::fs::write(&path, archive).unwrap(); + let mut record = record(name, "1.0.0", dependencies); + record.checksum = checksum.clone(); + index.archives.insert(checksum, path); + add(index, record); +} +fn tar(files: &[(&str, &[u8])]) -> Vec { + let mut archive = tar::Builder::new(Vec::new()); + for (name, data) in files { + let mut header = tar::Header::new_gnu(); + header.set_mode(0o644); + header.set_size(data.len() as u64); + header.set_cksum(); + archive.append_data(&mut header, name, *data).unwrap(); + } + archive.into_inner().unwrap() +} +fn gzip(data: &[u8]) -> Vec { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(data).unwrap(); + encoder.finish().unwrap() +} +pub fn run( + runtime: &RubyEnvironment, + bundle: &rb_packages::InstalledBundle, + project: &Path, + code: &str, +) -> std::process::Output { + Command::new(&runtime.executable) + .envs(bundle.environment()) + .env_remove("RUBYOPT") + .env_remove("RUBYLIB") + .env_remove("BUNDLE_GEMFILE") + .env("BUNDLE_IGNORE_CONFIG", "true") + .current_dir(project) + .args(["-e", code]) + .output() + .unwrap() +} + +pub fn stdout(output: &std::process::Output) -> String { + String::from_utf8(output.stdout.clone()) + .unwrap() + .replace("\r\n", "\n") +} diff --git a/crates/rb-packages/tests/common/server.rs b/crates/rb-packages/tests/common/server.rs new file mode 100644 index 0000000..9db46a8 --- /dev/null +++ b/crates/rb-packages/tests/common/server.rs @@ -0,0 +1,91 @@ +use std::{ + collections::BTreeMap, + io::{Read, Write}, + net::TcpListener, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, Ordering}, + }, + thread, + time::Duration, +}; + +pub struct Server { + pub url: String, + pub requests: Arc>>, + stop: Arc, + thread: Option>, +} + +impl Server { + pub fn new(responses: BTreeMap>) -> Self { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let url = format!("http://{}/", listener.local_addr().unwrap()); + let requests = Arc::new(Mutex::new(Vec::new())); + let stop = Arc::new(AtomicBool::new(false)); + let recorded = requests.clone(); + let finished = stop.clone(); + let thread = thread::spawn(move || { + while !finished.load(Ordering::Relaxed) { + let (mut socket, _) = match listener.accept() { + Ok(pair) => pair, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(5)); + continue; + } + Err(error) => panic!("{error}"), + }; + socket.set_nonblocking(false).unwrap(); + socket + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + let mut request = Vec::new(); + let mut buffer = [0; 1024]; + while !request.windows(4).any(|v| v == b"\r\n\r\n") { + let count = socket.read(&mut buffer).unwrap(); + if count == 0 { + break; + } + request.extend_from_slice(&buffer[..count]); + } + let path = String::from_utf8(request) + .unwrap() + .split_whitespace() + .nth(1) + .unwrap() + .to_owned(); + recorded.lock().unwrap().push(path.clone()); + let status = if responses.contains_key(&path) { + "200 OK" + } else { + "404 Not Found" + }; + let body = responses.get(&path).map_or(&[][..], Vec::as_slice); + write!( + socket, + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + .unwrap(); + socket.write_all(body).unwrap(); + } + }); + Self { + url, + requests, + stop, + thread: Some(thread), + } + } +} + +impl Drop for Server { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + let result = self.thread.take().unwrap().join(); + if !std::thread::panicking() { + result.unwrap(); + } + } +} diff --git a/crates/rb-packages/tests/corpus.rs b/crates/rb-packages/tests/corpus.rs new file mode 100644 index 0000000..da98299 --- /dev/null +++ b/crates/rb-packages/tests/corpus.rs @@ -0,0 +1,62 @@ +mod common; +use rb_packages::{MemoryIndex, PackageRecord, PlanOptions, plan}; +use std::{fs, path::Path}; + +#[test] +fn frozen_gemfile_projections_resolve_reference_lock_closures() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../rb-resolver/tests/fixtures/corpus"); + let cases: Vec = + serde_json::from_str(&fs::read_to_string(root.join("cases.json")).unwrap()).unwrap(); + let mut index = MemoryIndex::default(); + for file in fs::read_dir(root.join("index")).unwrap() { + let path = file.unwrap().path(); + let name = path.file_stem().unwrap().to_str().unwrap(); + for info in rb_compact_index::parse(name, &fs::read_to_string(&path).unwrap()).unwrap() { + common::add( + &mut index, + PackageRecord::from_info(name, "https://rubygems.org/", info).unwrap(), + ); + } + } + for case in cases { + let name = case["name"].as_str().unwrap(); + let expected: serde_json::Value = serde_json::from_str( + &fs::read_to_string(root.join(name).join("expected.json")).unwrap(), + ) + .unwrap(); + let mut manifest: rb_gemfile::Manifest = + toml::from_str(&fs::read_to_string(root.join(name).join("rbproject.toml")).unwrap()) + .unwrap(); + manifest.bundle.ruby.clear(); + manifest.dependencies.retain(|_, d| { + d.enabled && d.platforms.is_empty() && d.groups.iter().any(|g| g == "default") + }); + manifest.dependencies.insert( + "bundler".into(), + rb_gemfile::Dependency { + version: vec![format!("= {}", expected["bundler"].as_str().unwrap())], + ..Default::default() + }, + ); + let mut runtime = common::runtime(); + runtime.version = expected["ruby"].as_str().unwrap().parse().unwrap(); + runtime.platform = expected["target"].as_str().unwrap().into(); + runtime.bundler = Some(expected["bundler"].as_str().unwrap().parse().unwrap()); + runtime.rubygems = expected["rubygems"].as_str().unwrap().parse().unwrap(); + let mut lock = rb_lockfile::Lockfile::parse( + &fs::read_to_string(root.join(name).join("Gemfile.lock")).unwrap(), + ) + .unwrap(); + lock.platforms = vec![runtime.platform.clone()]; + let result = plan( + &manifest, + &runtime, + Some(&lock), + &index, + &PlanOptions::default(), + ) + .unwrap_or_else(|e| panic!("{name}: {e:#}")); + let actual: Vec<_> = result.packages.iter().map(|p| serde_json::json!({"name":p.record.package.id.name,"version":p.record.package.id.version.to_string(),"platform":p.record.package.id.platform.to_string()})).collect(); + assert_eq!(serde_json::json!(actual), expected["packages"], "{name}"); + } +} diff --git a/crates/rb-packages/tests/fixtures/README.md b/crates/rb-packages/tests/fixtures/README.md new file mode 100644 index 0000000..07b6b4c --- /dev/null +++ b/crates/rb-packages/tests/fixtures/README.md @@ -0,0 +1 @@ +`faker.info` contains the two historical records from https://rubygems.org/info/faker that exposed equivalent versions with distinct archive checksums. Both releases must remain valid candidates: version requirements compare `2.0` and `2.0.0` as equal, while archive selection and lockfile pins must preserve the release spelling. The tests supply stubbed dependency records and perform no network requests. diff --git a/crates/rb-packages/tests/fixtures/dependencies.rb b/crates/rb-packages/tests/fixtures/dependencies.rb new file mode 100644 index 0000000..dae7e89 --- /dev/null +++ b/crates/rb-packages/tests/fixtures/dependencies.rb @@ -0,0 +1,5 @@ +source 'https://rubygems.org' +gem 'teacup' +group :test do + gem 'test-helper' +end diff --git a/crates/rb-packages/tests/fixtures/expand_ruby_arguments.rb b/crates/rb-packages/tests/fixtures/expand_ruby_arguments.rb new file mode 100644 index 0000000..ad5916d --- /dev/null +++ b/crates/rb-packages/tests/fixtures/expand_ruby_arguments.rb @@ -0,0 +1,8 @@ +#!/usr/bin/env ruby +require 'rbconfig' + +arguments = ARGV.flat_map do |argument| + matches = Dir.glob(argument) if argument.include?('*') + matches && !matches.empty? ? matches.sort : [argument] +end +exec ENV.fetch('RB_GEM_BUILD_RUBY', RbConfig.ruby), *arguments diff --git a/crates/rb-packages/tests/fixtures/faker.info b/crates/rb-packages/tests/fixtures/faker.info new file mode 100644 index 0000000..6c32971 --- /dev/null +++ b/crates/rb-packages/tests/fixtures/faker.info @@ -0,0 +1,3 @@ +--- +2.0 i18n:>= 0.7|checksum:d4d52230559b1bc67a53a8c5d75d6861ec2ad0179d050d4d8e4b180b6a616d94,ruby:>= 2.3,created_at:2019-07-31T08:04:16Z +2.0.0 i18n:>= 0.7|checksum:2b7132895b69736c54f5e86d221228e27b5076299bb89fce5a115ce0429f4cf5,ruby:>= 2.3,created_at:2019-07-31T08:22:34Z diff --git a/crates/rb-packages/tests/fixtures/native/extconf.rb b/crates/rb-packages/tests/fixtures/native/extconf.rb new file mode 100644 index 0000000..cb3a2a2 --- /dev/null +++ b/crates/rb-packages/tests/fixtures/native/extconf.rb @@ -0,0 +1,4 @@ +require 'mkmf' +require 'middle' +raise 'transitive build dependency is missing' unless LEAF == 42 +create_makefile('package_native') diff --git a/crates/rb-packages/tests/fixtures/native/package_native.c b/crates/rb-packages/tests/fixtures/native/package_native.c new file mode 100644 index 0000000..ea2776b --- /dev/null +++ b/crates/rb-packages/tests/fixtures/native/package_native.c @@ -0,0 +1,10 @@ +#include + +static VALUE answer(VALUE self) { + return INT2NUM(42); +} + +void Init_package_native(void) { + VALUE module = rb_define_module("PackageNative"); + rb_define_singleton_method(module, "answer", answer, 0); +} diff --git a/crates/rb-packages/tests/fixtures/source-priority/README.md b/crates/rb-packages/tests/fixtures/source-priority/README.md new file mode 100644 index 0000000..8c2a731 --- /dev/null +++ b/crates/rb-packages/tests/fixtures/source-priority/README.md @@ -0,0 +1 @@ +These stub compact indexes model an optional scoped dependency whose children must be searched in the parent registry before the global registry. A newer global child must not replace a scoped child. A missing scoped child falls back to the global registry. Explicit declarations override inherited routes, and two scoped registries supplying an unpinned child are ambiguous. The `/versions` listing identifies available scoped names before probing child metadata. This models registries that error on unknown names. Omitted HTTP entries return 404; registries without a versions listing can confirm missing gems through the metadata lookup. diff --git a/crates/rb-packages/tests/fixtures/source-priority/application.info b/crates/rb-packages/tests/fixtures/source-priority/application.info new file mode 100644 index 0000000..3a8eb09 --- /dev/null +++ b/crates/rb-packages/tests/fixtures/source-priority/application.info @@ -0,0 +1,2 @@ +--- +1.0 public:>= 1|checksum:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/crates/rb-packages/tests/fixtures/source-priority/child.info b/crates/rb-packages/tests/fixtures/source-priority/child.info new file mode 100644 index 0000000..ddc4ccc --- /dev/null +++ b/crates/rb-packages/tests/fixtures/source-priority/child.info @@ -0,0 +1,2 @@ +--- +1.0 leaf:>= 1|checksum:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/crates/rb-packages/tests/fixtures/source-priority/leaf.info b/crates/rb-packages/tests/fixtures/source-priority/leaf.info new file mode 100644 index 0000000..86851df --- /dev/null +++ b/crates/rb-packages/tests/fixtures/source-priority/leaf.info @@ -0,0 +1,2 @@ +--- +1.0 |checksum:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/crates/rb-packages/tests/fixtures/source-priority/newer-child.info b/crates/rb-packages/tests/fixtures/source-priority/newer-child.info new file mode 100644 index 0000000..811d4a9 --- /dev/null +++ b/crates/rb-packages/tests/fixtures/source-priority/newer-child.info @@ -0,0 +1,2 @@ +--- +99.0 |checksum:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/crates/rb-packages/tests/fixtures/source-priority/parent.info b/crates/rb-packages/tests/fixtures/source-priority/parent.info new file mode 100644 index 0000000..64e91ee --- /dev/null +++ b/crates/rb-packages/tests/fixtures/source-priority/parent.info @@ -0,0 +1,2 @@ +--- +1.0 child:>= 1,public:>= 1|checksum:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/crates/rb-packages/tests/fixtures/source-priority/public.info b/crates/rb-packages/tests/fixtures/source-priority/public.info new file mode 100644 index 0000000..86851df --- /dev/null +++ b/crates/rb-packages/tests/fixtures/source-priority/public.info @@ -0,0 +1,2 @@ +--- +1.0 |checksum:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa diff --git a/crates/rb-packages/tests/fixtures/sources/extconf.rb b/crates/rb-packages/tests/fixtures/sources/extconf.rb new file mode 100644 index 0000000..dca1232 --- /dev/null +++ b/crates/rb-packages/tests/fixtures/sources/extconf.rb @@ -0,0 +1,2 @@ +require 'mkmf' +create_makefile('source_native') diff --git a/crates/rb-packages/tests/fixtures/sources/fixture.gemspec b/crates/rb-packages/tests/fixtures/sources/fixture.gemspec new file mode 100644 index 0000000..7c7e332 --- /dev/null +++ b/crates/rb-packages/tests/fixtures/sources/fixture.gemspec @@ -0,0 +1,11 @@ +require_relative 'lib/fixture/version' + +Gem::Specification.new do |spec| + spec.name = 'fixture' + spec.version = Fixture::VERSION + spec.summary = 'Source installation fixture' + spec.authors = ['Fixture'] + spec.files = Dir.glob('{lib,ext}/**/*').select { |path| File.file?(path) } + spec.require_paths = ['lib'] + spec.extensions = Dir.glob('ext/**/extconf.rb') +end diff --git a/crates/rb-packages/tests/fixtures/sources/helper.gemspec b/crates/rb-packages/tests/fixtures/sources/helper.gemspec new file mode 100644 index 0000000..eae21f9 --- /dev/null +++ b/crates/rb-packages/tests/fixtures/sources/helper.gemspec @@ -0,0 +1,8 @@ +Gem::Specification.new do |spec| + spec.name = 'helper' + spec.version = '1.0.0' + spec.summary = 'Source helper fixture' + spec.authors = ['Fixture'] + spec.files = ['lib/helper.rb'] + spec.require_paths = ['lib'] +end diff --git a/crates/rb-packages/tests/fixtures/sources/lib/fixture.rb b/crates/rb-packages/tests/fixtures/sources/lib/fixture.rb new file mode 100644 index 0000000..338fbf3 --- /dev/null +++ b/crates/rb-packages/tests/fixtures/sources/lib/fixture.rb @@ -0,0 +1 @@ +require_relative 'fixture/version' diff --git a/crates/rb-packages/tests/fixtures/sources/lib/fixture/version.rb b/crates/rb-packages/tests/fixtures/sources/lib/fixture/version.rb new file mode 100644 index 0000000..046a29e --- /dev/null +++ b/crates/rb-packages/tests/fixtures/sources/lib/fixture/version.rb @@ -0,0 +1,3 @@ +module Fixture + VERSION = '1.0.0'.freeze +end diff --git a/crates/rb-packages/tests/fixtures/sources/source_native.c b/crates/rb-packages/tests/fixtures/sources/source_native.c new file mode 100644 index 0000000..1218042 --- /dev/null +++ b/crates/rb-packages/tests/fixtures/sources/source_native.c @@ -0,0 +1,10 @@ +#include + +static VALUE answer(VALUE self) { + return INT2NUM(42); +} + +void Init_source_native(void) { + VALUE fixture = rb_define_module("Fixture"); + rb_define_singleton_method(fixture, "answer", answer, 0); +} diff --git a/crates/rb-packages/tests/input.rs b/crates/rb-packages/tests/input.rs new file mode 100644 index 0000000..e0fa938 --- /dev/null +++ b/crates/rb-packages/tests/input.rs @@ -0,0 +1,53 @@ +mod common; +use rb_packages::ProjectInput; +use std::fs; + +#[test] +fn gemfile_snapshot_ignores_generated_bundle_but_tracks_source_edits() { + let root = tempfile::tempdir().unwrap(); + fs::write( + root.path().join("Gemfile"), + "source 'https://rubygems.org'\ngem 'rake'\n", + ) + .unwrap(); + let input = ProjectInput::load(root.path(), common::ruby()).unwrap(); + assert!(!input.reused); + fs::create_dir_all(root.path().join(".rb/bundle/gems")).unwrap(); + fs::write(root.path().join(".rb/bundle/gems/generated"), "data").unwrap(); + let reused = ProjectInput::load(root.path(), common::ruby()).unwrap(); + assert!(reused.reused); + fs::write( + root.path().join("Gemfile"), + "source 'https://rubygems.org'\ngem 'rack'\n", + ) + .unwrap(); + let changed = ProjectInput::load(root.path(), common::ruby()).unwrap(); + assert!(!changed.reused); + assert!(changed.manifest.dependencies.contains_key("rack")); +} + +#[test] +fn explicit_project_dependencies_are_authoritative_without_running_gemfile() { + let root = tempfile::tempdir().unwrap(); + fs::write(root.path().join("Gemfile"), "raise 'must not execute'\n").unwrap(); + fs::write( + root.path().join("rbproject.toml"), + "[project]\nname='example'\n[dependencies]\nrake='>= 0'\n", + ) + .unwrap(); + let input = ProjectInput::load(root.path(), "missing-ruby").unwrap(); + assert!(input.manifest.dependencies.contains_key("rake")); + assert!(input.gemfile.is_none()); +} + +#[test] +fn custom_lockfile_path_cannot_silently_diverge_from_bundler() { + let root = tempfile::tempdir().unwrap(); + fs::write( + root.path().join("rbproject.toml"), + "[bundle]\nlockfile='custom.lock'\n[dependencies]\nrake='>= 0'\n", + ) + .unwrap(); + let error = ProjectInput::load(root.path(), "missing-ruby").unwrap_err(); + assert!(error.to_string().contains("Custom lockfile paths")); +} diff --git a/crates/rb-packages/tests/installation.rs b/crates/rb-packages/tests/installation.rs new file mode 100644 index 0000000..a2a140a --- /dev/null +++ b/crates/rb-packages/tests/installation.rs @@ -0,0 +1,666 @@ +mod common; +use common::*; +use rb_packages::{InstallOptions, MemoryIndex, PlanOptions, install, plan}; +use rb_task::TaskEvents; +use std::{fs, sync::Arc}; + +#[test] +fn installed_bundle_runs_with_rubygems_bundler_and_executable_and_reuses() { + let root = tempfile::tempdir().unwrap(); + let mut index = MemoryIndex::default(); + add_archive( + &mut index, + root.path(), + "teacup", + &[], + &[], + &[ + ("lib/teacup.rb", b"TEACUP = 42\n"), + ("bin/teacup", b"require 'teacup'; puts TEACUP\n"), + ], + ); + fs::write( + root.path().join("Gemfile"), + "source 'https://rubygems.org'\ngem 'teacup'\n", + ) + .unwrap(); + let runtime = runtime(); + let plan = plan( + &manifest("https://rubygems.org", "teacup = '>= 0'"), + &runtime, + None, + &index, + &PlanOptions::default(), + ) + .unwrap(); + let index = Arc::new(index); + let bundle = install( + root.path(), + &plan, + &runtime, + index.clone(), + InstallOptions::new(&runtime.executable), + TaskEvents::default(), + ) + .unwrap(); + fs::write( + root.path().join("Gemfile.lock"), + plan.lockfile.render().unwrap(), + ) + .unwrap(); + for code in [ + "require 'rubygems'; require 'teacup'; puts TEACUP", + "require 'bundler/setup'; require 'teacup'; puts TEACUP", + ] { + let output = run(&runtime, &bundle, root.path(), code); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(stdout(&output), "42\n"); + } + let code = format!( + "load {}", + serde_json::to_string(&bundle.gem_home.join("bin/teacup").to_string_lossy()).unwrap() + ); + let output = run(&runtime, &bundle, root.path(), &code); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(stdout(&output), "42\n"); + let reused = install( + root.path(), + &plan, + &runtime, + index, + InstallOptions::new(&runtime.executable), + TaskEvents::default(), + ) + .unwrap(); + assert_eq!(bundle.installed, 1); + assert_eq!(reused.installed, 0); + assert_eq!(reused.reused, 1); +} + +#[test] +fn failed_build_does_not_publish_a_bundle() { + let root = tempfile::tempdir().unwrap(); + let mut index = MemoryIndex::default(); + add_archive( + &mut index, + root.path(), + "broken", + &[], + &["ext/extconf.rb"], + &[("ext/extconf.rb", b"raise 'missing prerequisite'\n")], + ); + let runtime = runtime(); + let plan = plan( + &manifest("https://rubygems.org", "broken = '>= 0'"), + &runtime, + None, + &index, + &PlanOptions::default(), + ) + .unwrap(); + let result = install( + root.path(), + &plan, + &runtime, + Arc::new(index), + InstallOptions::new(&runtime.executable), + TaskEvents::default(), + ); + assert!(format!("{:#}", result.unwrap_err()).contains("missing prerequisite")); + assert!( + !root + .path() + .join(".rb/bundle/ruby") + .join(runtime.abi) + .exists() + ); +} + +#[test] +fn runtime_dependency_cycles_do_not_become_task_cycles() { + let root = tempfile::tempdir().unwrap(); + let mut index = MemoryIndex::default(); + add_archive( + &mut index, + root.path(), + "one", + &[("two", ">= 0")], + &[], + &[("lib/one.rb", b"ONE = 1")], + ); + add_archive( + &mut index, + root.path(), + "two", + &[("one", ">= 0")], + &[], + &[("lib/two.rb", b"TWO = 2")], + ); + let runtime = runtime(); + let plan = plan( + &manifest("https://rubygems.org", "one = '>= 0'"), + &runtime, + None, + &index, + &PlanOptions::default(), + ) + .unwrap(); + let result = install( + root.path(), + &plan, + &runtime, + Arc::new(index), + InstallOptions::new(&runtime.executable), + TaskEvents::default(), + ) + .unwrap(); + assert_eq!(result.installed, 2); +} + +#[test] +fn excluded_groups_can_be_consumed_by_bundler() { + let root = tempfile::tempdir().unwrap(); + let mut index = MemoryIndex::default(); + add_archive( + &mut index, + root.path(), + "teacup", + &[], + &[], + &[("lib/teacup.rb", b"TEACUP=42\n")], + ); + add_archive( + &mut index, + root.path(), + "test-helper", + &[], + &[], + &[("lib/test-helper.rb", b"TEST_HELPER=true\n")], + ); + fs::write( + root.path().join("Gemfile"), + include_str!("fixtures/dependencies.rb"), + ) + .unwrap(); + let runtime = runtime(); + let plan = plan( + &manifest( + "https://rubygems.org", + "teacup='>= 0'\ntest-helper={groups=['test']}", + ), + &runtime, + None, + &index, + &PlanOptions { + without: ["test".into()].into(), + ..Default::default() + }, + ) + .unwrap(); + let bundle = install( + root.path(), + &plan, + &runtime, + Arc::new(index), + InstallOptions::new(&runtime.executable), + TaskEvents::default(), + ) + .unwrap(); + fs::write( + root.path().join("Gemfile.lock"), + plan.lockfile.render().unwrap(), + ) + .unwrap(); + let output = run( + &runtime, + &bundle, + root.path(), + "require 'bundler/setup'; require 'teacup'; raise if Gem.loaded_specs.key?('test-helper'); puts TEACUP", + ); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(stdout(&output), "42\n"); +} + +#[cfg(unix)] +#[test] +fn native_bundle_uses_transitive_dependencies_and_rubygems_extension_layout() { + let root = tempfile::tempdir().unwrap(); + let mut index = MemoryIndex::default(); + add_archive( + &mut index, + root.path(), + "leaf", + &[], + &[], + &[("lib/leaf.rb", b"LEAF=42\n")], + ); + add_archive( + &mut index, + root.path(), + "middle", + &[("leaf", ">= 0")], + &[], + &[("lib/middle.rb", b"require 'leaf'\n")], + ); + add_archive( + &mut index, + root.path(), + "native", + &[("middle", ">= 0")], + &["ext/extconf.rb"], + &[ + ( + "ext/extconf.rb", + include_bytes!("fixtures/native/extconf.rb"), + ), + ( + "ext/package_native.c", + include_bytes!("fixtures/native/package_native.c"), + ), + ("lib/native.rb", b"require 'package_native'\n"), + ], + ); + let runtime = runtime(); + let plan = plan( + &manifest("https://rubygems.org", "native='>= 0'"), + &runtime, + None, + &index, + &PlanOptions::default(), + ) + .unwrap(); + let mut options = InstallOptions::new(&runtime.executable); + options + .build + .environment + .insert("CCACHE_DISABLE".into(), "1".into()); + let index = Arc::new(index); + let bundle = install( + root.path(), + &plan, + &runtime, + index.clone(), + options.clone(), + TaskEvents::default(), + ) + .unwrap(); + let marker = bundle + .gem_home + .join("extensions") + .join(&runtime.platform) + .join(&runtime.abi) + .join("native-1.0.0/gem.build_complete"); + assert!(marker.is_file()); + let output = run( + &runtime, + &bundle, + root.path(), + "require 'rubygems'; require 'native'; raise if Gem.loaded_specs.fetch('native').missing_extensions?; puts PackageNative.answer", + ); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(stdout(&output), "42\n"); + fs::remove_file(marker).unwrap(); + let repaired = install( + root.path(), + &plan, + &runtime, + index, + options, + TaskEvents::default(), + ) + .unwrap(); + assert_eq!(repaired.installed, 1); + assert_eq!(repaired.reused, 2); +} + +#[test] +fn native_project_generates_a_gemfile_that_bundler_can_consume() { + let root = tempfile::tempdir().unwrap(); + let mut index = MemoryIndex::default(); + add_archive( + &mut index, + root.path(), + "teacup", + &[], + &[], + &[("lib/teacup.rb", b"TEACUP=42\n")], + ); + fs::write( + root.path().join("rbproject.toml"), + "[dependencies]\nteacup='>= 0'\n", + ) + .unwrap(); + fs::create_dir(root.path().join(".bundle")).unwrap(); + fs::write( + root.path().join(".bundle/config"), + "---\nBUNDLE_PATH: missing-bundle\n", + ) + .unwrap(); + let input = rb_packages::ProjectInput::load(root.path(), ruby()).unwrap(); + assert!(input.gemfile.is_none()); + let runtime = runtime(); + let plan = plan( + &input.manifest, + &runtime, + None, + &index, + &PlanOptions::default(), + ) + .unwrap(); + let mut bundle = install( + root.path(), + &plan, + &runtime, + Arc::new(index), + InstallOptions::new(&runtime.executable), + TaskEvents::default(), + ) + .unwrap(); + bundle.gemfile = Some(input.bundler_gemfile); + fs::write(input.lockfile, plan.lockfile.render().unwrap()).unwrap(); + let output = std::process::Command::new(&runtime.executable) + .envs(bundle.environment()) + .env_remove("RUBYOPT") + .current_dir(root.path()) + .args(["-rbundler/setup", "-rteacup", "-e", "puts TEACUP"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(stdout(&output), "42\n"); +} + +#[test] +fn failed_replacement_preserves_existing_bundle() { + let root = tempfile::tempdir().unwrap(); + let mut index = MemoryIndex::default(); + add_archive( + &mut index, + root.path(), + "teacup", + &[], + &[], + &[("lib/teacup.rb", b"TEACUP=42\n")], + ); + let runtime = runtime(); + let initial = plan( + &manifest("https://rubygems.org", "teacup='>= 0'"), + &runtime, + None, + &index, + &PlanOptions::default(), + ) + .unwrap(); + let bundle = install( + root.path(), + &initial, + &runtime, + Arc::new(index), + InstallOptions::new(&runtime.executable), + TaskEvents::default(), + ) + .unwrap(); + let original = fs::read(bundle.gem_home.join("gems/teacup-1.0.0/lib/teacup.rb")).unwrap(); + let mut index = MemoryIndex::default(); + add_archive( + &mut index, + root.path(), + "teacup", + &[], + &["ext/extconf.rb"], + &[ + ("lib/teacup.rb", b"TEACUP=99\n"), + ("ext/extconf.rb", b"raise 'failed replacement'\n"), + ], + ); + let replacement = plan( + &manifest("https://rubygems.org", "teacup='>= 0'"), + &runtime, + None, + &index, + &PlanOptions::default(), + ) + .unwrap(); + assert!( + install( + root.path(), + &replacement, + &runtime, + Arc::new(index), + InstallOptions::new(&runtime.executable), + TaskEvents::default() + ) + .is_err() + ); + assert_eq!( + fs::read(bundle.gem_home.join("gems/teacup-1.0.0/lib/teacup.rb")).unwrap(), + original + ); +} + +#[test] +fn build_configuration_changes_invalidate_reuse_without_persisting_secret_values() { + let root = tempfile::tempdir().unwrap(); + let mut index = MemoryIndex::default(); + add_archive( + &mut index, + root.path(), + "teacup", + &[], + &[], + &[("lib/teacup.rb", b"TEACUP=42\n")], + ); + let runtime = runtime(); + let plan = plan( + &manifest("https://rubygems.org", "teacup='>= 0'"), + &runtime, + None, + &index, + &PlanOptions::default(), + ) + .unwrap(); + let index = Arc::new(index); + let first = install( + root.path(), + &plan, + &runtime, + index.clone(), + InstallOptions::new(&runtime.executable), + TaskEvents::default(), + ) + .unwrap(); + let mut options = InstallOptions::new(&runtime.executable); + options + .build + .environment + .insert("BUILD_TOKEN".into(), "fixture-secret-value".into()); + let changed = install( + root.path(), + &plan, + &runtime, + index, + options, + TaskEvents::default(), + ) + .unwrap(); + assert_eq!(first.installed, 1); + assert_eq!(changed.installed, 1); + assert_eq!(changed.reused, 0); + let receipt = + fs::read_to_string(changed.gem_home.join("gems/teacup-1.0.0/installation.json")).unwrap(); + assert!(!receipt.contains("fixture-secret-value")); +} + +#[test] +fn declared_ruby_lock_is_accepted_by_frozen_bundler_without_edits() { + let root = tempfile::tempdir().unwrap(); + let mut index = MemoryIndex::default(); + add_archive( + &mut index, + root.path(), + "teacup", + &[], + &[], + &[("lib/teacup.rb", b"TEACUP=42\n")], + ); + let runtime = runtime(); + let mut manifest = manifest("https://rubygems.org", "teacup='>= 0'"); + manifest.bundle.ruby = vec![">= 3.3".into()]; + let plan = plan(&manifest, &runtime, None, &index, &PlanOptions::default()).unwrap(); + fs::write( + root.path().join("Gemfile"), + "source 'https://rubygems.org'\nruby '>= 3.3'\ngem 'teacup'\n", + ) + .unwrap(); + let lock = plan.lockfile.render().unwrap(); + fs::write(root.path().join("Gemfile.lock"), &lock).unwrap(); + let bundle = install( + root.path(), + &plan, + &runtime, + Arc::new(index), + InstallOptions::new(&runtime.executable), + TaskEvents::default(), + ) + .unwrap(); + let output = std::process::Command::new(&runtime.executable) + .envs(bundle.environment()) + .env_remove("RUBYOPT") + .env("BUNDLE_FROZEN", "true") + .env("BUNDLE_IGNORE_CONFIG", "true") + .current_dir(root.path()) + .args(["-rbundler/setup", "-rteacup", "-e", "puts TEACUP"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(stdout(&output), "42\n"); + assert_eq!( + fs::read_to_string(root.path().join("Gemfile.lock")).unwrap(), + lock + ); +} + +#[test] +fn ruby_output_assertions_accept_windows_line_endings() { + let output = std::process::Command::new(ruby()) + .env_remove("RUBYOPT") + .env_remove("RUBYLIB") + .args(["-e", "STDOUT.binmode; print \"42\\r\\n\""]) + .output() + .unwrap(); + assert!(output.status.success()); + assert_eq!(stdout(&output), "42\n"); +} + +#[test] +fn refreshed_multi_target_lock_runs_with_frozen_bundler_without_installing_foreign_roots() { + let root = tempfile::tempdir().unwrap(); + let mut index = MemoryIndex::default(); + add_archive( + &mut index, + root.path(), + "teacup", + &[], + &[], + &[("lib/teacup.rb", b"TEACUP = 42\n")], + ); + add_archive( + &mut index, + root.path(), + "foreign-helper", + &[], + &[], + &[( + "lib/foreign-helper.rb", + b"raise 'foreign dependency loaded'\n", + )], + ); + let runtime = runtime(); + let windows = + runtime.target().os.starts_with("mingw") || runtime.target().os.starts_with("mswin"); + let (selector, foreign) = if windows { + ("ruby", "x86_64-linux") + } else { + ("windows", "x64-mingw-ucrt") + }; + let manifest = manifest( + "https://rubygems.org", + &format!("teacup = '>= 0'\nforeign-helper = {{platforms=['{selector}']}}"), + ); + fs::write(root.path().join("Gemfile"), format!("source 'https://rubygems.org'\ngem 'teacup'\nplatforms :{selector} do\n gem 'foreign-helper'\nend\n")).unwrap(); + let previous = rb_lockfile::Lockfile { + platforms: vec![runtime.platform.clone(), foreign.into()], + ..Default::default() + }; + let plan = plan( + &manifest, + &runtime, + Some(&previous), + &index, + &PlanOptions { + mode: rb_packages::LockMode::Update, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(plan.packages.len(), 1); + assert_eq!(plan.lockfile.platforms.len(), 2); + assert!( + plan.lockfile + .sources + .iter() + .flat_map(|source| &source.packages) + .any(|package| package.name == "foreign-helper") + ); + let lock = plan.lockfile.render().unwrap(); + fs::write(root.path().join("Gemfile.lock"), &lock).unwrap(); + let bundle = install( + root.path(), + &plan, + &runtime, + Arc::new(index), + InstallOptions::new(&runtime.executable), + TaskEvents::default(), + ) + .unwrap(); + assert_eq!(bundle.installed, 1); + let output = run( + &runtime, + &bundle, + root.path(), + "require 'bundler'; Bundler.settings.set_local(:frozen, true); require 'bundler/setup'; require 'teacup'; raise 'foreign gem activated' if Gem.loaded_specs.key?('foreign-helper'); puts TEACUP", + ); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(stdout(&output), "42\n"); + assert_eq!( + fs::read_to_string(root.path().join("Gemfile.lock")).unwrap(), + lock + ); +} diff --git a/crates/rb-packages/tests/planning.rs b/crates/rb-packages/tests/planning.rs new file mode 100644 index 0000000..362b43e --- /dev/null +++ b/crates/rb-packages/tests/planning.rs @@ -0,0 +1,446 @@ +mod common; +use common::*; +use rb_packages::{LockMode, MemoryIndex, PlanOptions, plan}; + +#[test] +fn locks_prefer_existing_versions_updates_choose_new_versions_and_removal_prunes() { + let mut index = MemoryIndex::default(); + add(&mut index, record("tea", "1.0", &[])); + add(&mut index, record("tea", "2.0", &[])); + let runtime = runtime(); + let request = manifest("https://rubygems.org", "tea = '= 1.0'"); + let first = plan(&request, &runtime, None, &index, &PlanOptions::default()).unwrap(); + let request = manifest("https://rubygems.org", "tea = '>= 0'"); + let preferred = plan( + &request, + &runtime, + Some(&first.lockfile), + &index, + &PlanOptions::default(), + ) + .unwrap(); + assert_eq!( + preferred.packages[0].record.package.id.version.to_string(), + "1.0" + ); + let updated = plan( + &request, + &runtime, + Some(&first.lockfile), + &index, + &PlanOptions { + mode: LockMode::Update, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + updated.packages[0].record.package.id.version.to_string(), + "2.0" + ); + assert!( + plan( + &request, + &runtime, + Some(&first.lockfile), + &index, + &PlanOptions { + mode: LockMode::Locked, + ..Default::default() + } + ) + .is_err() + ); + let empty = manifest("https://rubygems.org", ""); + let removed = plan( + &empty, + &runtime, + Some(&first.lockfile), + &index, + &PlanOptions::default(), + ) + .unwrap(); + assert!(removed.packages.is_empty()); + assert_eq!(removed.lockfile.sources.len(), 1); + assert_eq!( + removed.lockfile.sources[0].remotes, + ["https://rubygems.org/"] + ); + assert!(removed.lockfile.sources[0].packages.is_empty()); + assert!(removed.lock_changed); +} + +#[test] +fn excluded_groups_stay_in_lock_but_not_installation() { + let mut index = MemoryIndex::default(); + add(&mut index, record("tea", "1.0", &[("water", ">= 1")])); + add(&mut index, record("water", "1.0", &[])); + add(&mut index, record("test-helper", "1.0", &[])); + let manifest = manifest( + "https://rubygems.org", + "tea = '>= 0'\ntest-helper = {groups=['test']}", + ); + let plan = plan( + &manifest, + &runtime(), + None, + &index, + &PlanOptions { + without: ["test".into()].into(), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(plan.packages.len(), 3); + assert!( + !plan + .packages + .iter() + .find(|p| p.record.package.id.name == "test-helper") + .unwrap() + .install + ); + assert!( + plan.packages + .iter() + .filter(|p| p.record.package.id.name != "test-helper") + .all(|p| p.install) + ); + assert_eq!( + rb_lockfile::Lockfile::parse(&plan.lockfile.render().unwrap()).unwrap(), + plan.lockfile + ); +} + +#[test] +fn locked_mode_keeps_original_lock_and_rejects_missing_candidates() { + let mut index = MemoryIndex::default(); + add(&mut index, record("tea", "1.0", &[])); + let request = manifest("https://rubygems.org", "tea = '>= 0'"); + let runtime = runtime(); + let first = plan(&request, &runtime, None, &index, &PlanOptions::default()).unwrap(); + let options = PlanOptions { + mode: LockMode::Locked, + ..Default::default() + }; + let locked = plan(&request, &runtime, Some(&first.lockfile), &index, &options).unwrap(); + assert_eq!(locked.lockfile, first.lockfile); + assert!(!locked.lock_changed); + assert!( + plan( + &request, + &runtime, + Some(&first.lockfile), + &MemoryIndex::default(), + &options + ) + .is_err() + ); +} + +#[test] +fn source_ambiguity_never_substitutes_package_provenance() { + let mut index = MemoryIndex::default(); + let mut first = record("tea", "1.0", &[]); + first.source = "https://one.example/".into(); + let mut second = first.clone(); + second.source = "https://two.example/".into(); + add(&mut index, first); + add(&mut index, second); + let request = manifest("https://one.example", "tea = '>= 0'"); + let result = plan(&request, &runtime(), None, &index, &PlanOptions::default()).unwrap(); + assert_eq!( + result.packages[0].record.source, + "https://one.example/".into() + ); +} + +#[test] +fn locked_source_and_checksum_are_enforced_for_transitive_packages() { + let mut index = MemoryIndex::default(); + add(&mut index, record("tea", "1.0", &[("water", ">= 1")])); + add(&mut index, record("water", "1.0", &[])); + let runtime = runtime(); + let manifest = manifest("https://rubygems.org", "tea='>= 0'"); + let first = plan(&manifest, &runtime, None, &index, &PlanOptions::default()).unwrap(); + let options = PlanOptions { + mode: LockMode::Locked, + ..Default::default() + }; + index.packages.get_mut("water").unwrap()[0].source = "https://other.example/".into(); + assert!(plan(&manifest, &runtime, Some(&first.lockfile), &index, &options).is_err()); + index.packages.get_mut("water").unwrap()[0].source = "https://rubygems.org/".into(); + index.packages.get_mut("water").unwrap()[0].checksum = "b".repeat(64); + assert!(plan(&manifest, &runtime, Some(&first.lockfile), &index, &options).is_err()); +} + +#[test] +fn valid_multi_platform_lock_is_preserved_without_dropping_other_targets() { + let mut index = MemoryIndex::default(); + add(&mut index, record("tea", "1.0", &[])); + let runtime = runtime(); + let manifest = manifest("https://rubygems.org", "tea='>= 0'"); + let mut first = plan(&manifest, &runtime, None, &index, &PlanOptions::default()).unwrap(); + first.lockfile.platforms.push("arm64-darwin-24".into()); + let result = plan( + &manifest, + &runtime, + Some(&first.lockfile), + &index, + &PlanOptions::default(), + ) + .unwrap(); + assert_eq!(result.lockfile, first.lockfile); + assert!(!result.lock_changed); +} + +#[test] +fn preferred_lock_does_not_silently_accept_changed_archive_checksums() { + let mut index = MemoryIndex::default(); + add(&mut index, record("tea", "1.0", &[])); + let runtime = runtime(); + let manifest = manifest("https://rubygems.org", "tea='>= 0'"); + let original = plan(&manifest, &runtime, None, &index, &PlanOptions::default()).unwrap(); + index.packages.get_mut("tea").unwrap()[0].checksum = "b".repeat(64); + let error = plan( + &manifest, + &runtime, + Some(&original.lockfile), + &index, + &PlanOptions::default(), + ) + .unwrap_err(); + assert!(format!("{error:#}").contains("Locked checksum changed")); +} + +#[test] +fn declared_ruby_and_engine_are_checked_and_recorded_in_lock() { + let mut runtime = runtime(); + runtime.version = "3.4.10".parse().unwrap(); + runtime.engine_version = runtime.version.clone(); + runtime.patchlevel = 1; + let mut manifest = manifest("https://rubygems.org", ""); + manifest.bundle.ruby = vec![">= 3.3".into()]; + let result = plan( + &manifest, + &runtime, + None, + &MemoryIndex::default(), + &PlanOptions::default(), + ) + .unwrap(); + assert_eq!( + result.lockfile.ruby_version.as_deref(), + Some("ruby 3.4.10p1") + ); + manifest.bundle.engine_version = Some("3.3.7".into()); + assert!( + plan( + &manifest, + &runtime, + None, + &MemoryIndex::default(), + &PlanOptions::default() + ) + .is_err() + ); + manifest.bundle.engine_version = None; + manifest.bundle.patchlevel = Some("0".into()); + assert!( + plan( + &manifest, + &runtime, + None, + &MemoryIndex::default(), + &PlanOptions::default() + ) + .is_err() + ); +} + +#[test] +fn archive_names_from_custom_providers_cannot_escape_installation() { + let mut index = MemoryIndex::default(); + let mut package = record("tea", "1.0", &[]); + package.archive_version = "1.0/../../outside".into(); + add(&mut index, package); + assert!( + plan( + &manifest("https://rubygems.org", "tea='>= 0'"), + &runtime(), + None, + &index, + &PlanOptions::default() + ) + .is_err() + ); +} + +#[test] +fn platform_and_major_version_selectors_choose_only_applicable_roots() { + let mut index = MemoryIndex::default(); + add(&mut index, record("tea", "1.0", &[])); + add(&mut index, record("windows-helper", "1.0", &[])); + let request = manifest( + "https://rubygems.org", + "tea={platforms=['ruby_3']}\nwindows-helper={platforms=['mswin']}", + ); + let mut runtime = runtime(); + runtime.version = "3.4.10".parse().unwrap(); + runtime.platform = "x86_64-linux".into(); + let unix = plan(&request, &runtime, None, &index, &PlanOptions::default()).unwrap(); + assert_eq!(unix.packages[0].record.package.id.name, "tea"); + assert_eq!(unix.packages.len(), 1); + runtime.platform = "x64-mingw-ucrt".into(); + let windows = plan(&request, &runtime, None, &index, &PlanOptions::default()).unwrap(); + assert_eq!(windows.packages[0].record.package.id.name, "windows-helper"); + assert_eq!(windows.packages.len(), 1); +} + +#[test] +fn equivalent_faker_releases_keep_their_archives_and_locked_checksums() { + let mut index = MemoryIndex::default(); + for info in rb_compact_index::parse("faker", include_str!("fixtures/faker.info")).unwrap() { + add( + &mut index, + rb_packages::PackageRecord::from_info("faker", "https://rubygems.org", info).unwrap(), + ); + } + add(&mut index, record("i18n", "1.0", &[])); + let request = manifest("https://rubygems.org", "faker = '= 2.0'"); + let runtime = runtime(); + for spelling in ["2.0", "2.0.0"] { + let mut pinned = MemoryIndex { + packages: index.packages.clone(), + ..Default::default() + }; + pinned + .packages + .get_mut("faker") + .unwrap() + .retain(|r| r.archive_version == spelling); + let initial = plan(&request, &runtime, None, &pinned, &PlanOptions::default()).unwrap(); + for mode in [LockMode::Prefer, LockMode::Locked] { + let result = plan( + &request, + &runtime, + Some(&initial.lockfile), + &index, + &PlanOptions { + mode, + ..Default::default() + }, + ) + .unwrap(); + let faker = result + .packages + .iter() + .find(|p| p.record.package.id.name == "faker") + .unwrap(); + assert_eq!(faker.record.archive_version, spelling); + assert_eq!( + faker.record.checksum, + initial + .packages + .iter() + .find(|p| p.record.package.id.name == "faker") + .unwrap() + .record + .checksum + ); + assert!(!result.lock_changed); + } + } + add(&mut index, record("faker", "3.0", &[])); + let result = plan( + &manifest("https://rubygems.org", "faker = '>= 3'"), + &runtime, + None, + &index, + &PlanOptions::default(), + ) + .unwrap(); + assert_eq!(result.packages.len(), 1); + assert_eq!(result.packages[0].record.archive_version, "3.0"); +} + +#[test] +fn updating_multiple_targets_resolves_native_dependencies_and_installs_only_local_packages() { + let mut index = MemoryIndex::default(); + for version in ["1", "2"] { + for (platform, dependency) in [ + ("x86_64-linux", "linux-helper"), + ("arm64-darwin", "darwin-helper"), + ] { + let mut gem = record("native", version, &[(dependency, &format!("= {version}"))]); + gem.package.id.platform = rb_gem_types::Platform::parse(platform); + gem.archive_version = format!("{version}-{platform}"); + add(&mut index, gem); + } + add(&mut index, record("linux-helper", version, &[])); + add(&mut index, record("darwin-helper", version, &[])); + } + let mut runtime = runtime(); + runtime.platform = "x86_64-linux".into(); + let request = manifest("https://rubygems.org", "native = '>= 1'"); + let mut original = plan( + &manifest("https://rubygems.org", "native = '= 1'"), + &runtime, + None, + &index, + &PlanOptions::default(), + ) + .unwrap() + .lockfile; + original.platforms.push("arm64-darwin".into()); + let updated = plan( + &request, + &runtime, + Some(&original), + &index, + &PlanOptions { + mode: LockMode::Update, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(updated.lockfile.platforms.len(), 2); + assert!(updated.lockfile.platforms.contains(&"arm64-darwin".into())); + let native: Vec<_> = updated + .lockfile + .sources + .iter() + .flat_map(|s| &s.packages) + .filter(|p| p.name == "native") + .collect(); + assert_eq!(native.len(), 2); + assert!(native.iter().all(|p| p.version.to_string() == "2")); + assert!( + updated + .lockfile + .sources + .iter() + .flat_map(|s| &s.packages) + .any(|p| p.name == "darwin-helper") + ); + assert_eq!(updated.packages.len(), 2); + assert!( + updated + .packages + .iter() + .all(|p| p.install && p.record.package.id.name != "darwin-helper") + ); + let frozen = plan( + &request, + &runtime, + Some(&updated.lockfile), + &index, + &PlanOptions { + mode: LockMode::Locked, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(frozen.lockfile, updated.lockfile); + assert!(!frozen.lock_changed); +} diff --git a/crates/rb-packages/tests/registry.rs b/crates/rb-packages/tests/registry.rs new file mode 100644 index 0000000..2a75b05 --- /dev/null +++ b/crates/rb-packages/tests/registry.rs @@ -0,0 +1,195 @@ +mod common; +use common::server::Server; +use rb_task::{TaskEvent, TaskEventSink, TaskEvents}; +use std::{ + collections::BTreeMap, + fs, + io::{Read, Write}, + sync::{Arc, Mutex}, + thread, + time::Duration, +}; + +#[derive(Default)] +struct Events(Mutex>); +impl TaskEventSink for Events { + fn event(&self, event: TaskEvent) { + self.0.lock().unwrap().push(event); + } +} + +#[test] +fn prepares_from_gemfile_then_reuses_offline_without_contacting_registry() { + let project = tempfile::tempdir().unwrap(); + let archives = tempfile::tempdir().unwrap(); + let mut index = rb_packages::MemoryIndex::default(); + common::add_archive( + &mut index, + archives.path(), + "teacup", + &[], + &[], + &[("lib/teacup.rb", b"TEACUP=42\n")], + ); + let record = &index.packages["teacup"][0]; + let server = Server::new(BTreeMap::from([ + ( + "/info/teacup".into(), + format!("---\n1.0.0 |checksum:{}\n", record.checksum).into_bytes(), + ), + ( + "/gems/teacup-1.0.0.gem".into(), + fs::read(index.archives.values().next().unwrap()).unwrap(), + ), + ])); + fs::write( + project.path().join("Gemfile"), + format!("source '{}'\ngem 'teacup'\n", server.url), + ) + .unwrap(); + let events = TaskEvents::default(); + let collected = Arc::new(Events::default()); + events.subscribe(collected.clone()); + let mut options = rb_packages::PrepareOptions::new( + project.path(), + common::ruby(), + project.path().join(".rb/cache"), + ); + let result = rb_packages::prepare(options.clone(), events.clone()).unwrap(); + assert_eq!(result.installed.installed, 1); + assert!(result.input.bundler_gemfile.is_file()); + let count = server.requests.lock().unwrap().len(); + assert_eq!(count, 2); + options.offline = true; + let reused = rb_packages::prepare(options, events).unwrap(); + assert_eq!(reused.installed.reused, 1); + assert!(!reused.plan.lock_changed); + assert_eq!(server.requests.lock().unwrap().len(), count); + let labels: Vec<_> = collected + .0 + .lock() + .unwrap() + .iter() + .filter_map(|event| { + if let TaskEvent::Started { label, worker, .. } = event { + assert!(*worker < 4); + Some(label.clone()) + } else { + None + } + }) + .collect(); + for label in [ + "lock project", + "inspect Ruby", + "load dependencies", + "resolve dependencies", + "prepare installation", + "download teacup-1.0.0", + "extract teacup-1.0.0", + "install teacup-1.0.0", + "publish bundle", + "write lockfile", + ] { + assert!( + labels.iter().any(|value| value == label), + "Missing worker task {label}" + ); + } +} + +#[test] +fn explicit_registry_blocks_are_usable_by_frozen_bundler() { + let project = tempfile::tempdir().unwrap(); + let archives = tempfile::tempdir().unwrap(); + let mut index = rb_packages::MemoryIndex::default(); + common::add_archive( + &mut index, + archives.path(), + "teacup", + &[], + &[], + &[("lib/teacup.rb", b"TEACUP=42\n")], + ); + common::add_archive( + &mut index, + archives.path(), + "saucer", + &[], + &[], + &[("lib/saucer.rb", b"SAUCER=1\n")], + ); + let make_server = |name: &str| { + let record = &index.packages[name][0]; + Server::new(BTreeMap::from([ + ( + format!("/info/{name}"), + format!("---\n1.0.0 |checksum:{}\n", record.checksum).into_bytes(), + ), + ( + format!("/gems/{name}-1.0.0.gem"), + fs::read(&index.archives[&record.checksum]).unwrap(), + ), + ])) + }; + let one = make_server("teacup"); + let two = make_server("saucer"); + let mut manifest = common::manifest(&one.url, "teacup='>= 0'\nsaucer='>= 0'"); + manifest.bundle.source = None; + manifest.dependencies.get_mut("teacup").unwrap().source = Some(one.url.clone()); + manifest.dependencies.get_mut("saucer").unwrap().source = Some(two.url.clone()); + fs::write( + project.path().join("rbproject.toml"), + manifest.to_toml().unwrap(), + ) + .unwrap(); + let options = rb_packages::PrepareOptions::new( + project.path(), + common::ruby(), + project.path().join(".rb/cache"), + ); + let result = rb_packages::prepare(options, TaskEvents::default()).unwrap(); + let output = std::process::Command::new(common::ruby()) + .envs(result.installed.environment()) + .env_remove("RUBYOPT") + .env("BUNDLE_FROZEN", "true") + .env("BUNDLE_IGNORE_CONFIG", "true") + .current_dir(project.path()) + .args([ + "-rbundler/setup", + "-rteacup", + "-rsaucer", + "-e", + "puts TEACUP + SAUCER", + ]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(common::stdout(&output), "43\n"); +} + +#[test] +fn registry_fixture_accepts_a_request_after_connection_establishment() { + let server = Server::new(BTreeMap::from([("/probe".into(), b"ready".to_vec())])); + let address = server + .url + .trim_start_matches("http://") + .trim_end_matches('/'); + let mut socket = std::net::TcpStream::connect(address).unwrap(); + socket + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + thread::sleep(Duration::from_millis(100)); + socket + .write_all(b"GET /probe HTTP/1.1\r\nHost: localhost\r\n\r\n") + .unwrap(); + let mut response = String::new(); + socket.read_to_string(&mut response).unwrap(); + assert!(response.starts_with("HTTP/1.1 200 OK\r\n")); + assert!(response.ends_with("\r\n\r\nready")); + assert_eq!(*server.requests.lock().unwrap(), ["/probe"]); +} diff --git a/crates/rb-packages/tests/registry_sources.rs b/crates/rb-packages/tests/registry_sources.rs new file mode 100644 index 0000000..d27f2a4 --- /dev/null +++ b/crates/rb-packages/tests/registry_sources.rs @@ -0,0 +1,337 @@ +mod common; +use common::server::Server; +use rb_packages::{LockMode, PackageIndex, PlanOptions, RegistryIndex, plan}; +use std::collections::BTreeMap; + +fn info(name: &str) -> Vec { + std::fs::read(format!( + "{}/tests/fixtures/source-priority/{name}.info", + env!("CARGO_MANIFEST_DIR") + )) + .unwrap() +} + +fn versions(names: &[&str]) -> Vec { + format!( + "---\n{}", + names + .iter() + .map(|name| format!("{name} 1.0 {}\n", "a".repeat(32))) + .collect::() + ) + .into_bytes() +} + +fn scoped() -> Server { + Server::new(BTreeMap::from([ + ("/versions".into(), versions(&["parent", "child", "leaf"])), + ("/info/parent".into(), info("parent")), + ("/info/child".into(), info("child")), + ("/info/leaf".into(), info("leaf")), + ])) +} + +#[test] +fn optional_scoped_children_take_priority_before_global_candidates_and_reuse_offline() { + let global = Server::new(BTreeMap::from([ + ("/info/application".into(), info("application")), + ("/info/public".into(), info("public")), + ("/info/child".into(), info("newer-child")), + ])); + let scoped = scoped(); + let mut manifest = common::manifest( + &global.url, + "application='>= 0'\nparent={groups=['optional']}", + ); + manifest.bundle.optional_groups.push("optional".into()); + manifest.dependencies.get_mut("parent").unwrap().source = Some(scoped.url.clone()); + let cache = tempfile::tempdir().unwrap(); + let index = RegistryIndex::new(&manifest, cache.path(), false).unwrap(); + let child = index.candidates("child").unwrap(); + assert_eq!(child.len(), 1); + assert_eq!(child[0].archive_version, "1.0"); + assert_eq!(child[0].source, scoped.url.clone().into()); + assert_eq!( + index.candidates("leaf").unwrap()[0].source, + scoped.url.clone().into() + ); + let runtime = common::runtime(); + let result = plan(&manifest, &runtime, None, &index, &PlanOptions::default()).unwrap(); + assert_eq!(result.packages.len(), 5); + let installed: Vec<_> = result + .packages + .iter() + .filter(|p| p.install) + .map(|p| p.record.package.id.name.as_str()) + .collect(); + assert_eq!(installed, ["application", "public"]); + assert_eq!( + result + .packages + .iter() + .find(|p| p.record.package.id.name == "public") + .unwrap() + .record + .source, + global.url.clone().into() + ); + assert!( + !global + .requests + .lock() + .unwrap() + .contains(&"/info/child".into()) + ); + assert!( + !scoped + .requests + .lock() + .unwrap() + .contains(&"/info/public".into()) + ); + for path in ["/info/parent", "/info/child", "/info/leaf"] { + assert_eq!( + scoped + .requests + .lock() + .unwrap() + .iter() + .filter(|p| p.as_str() == path) + .count(), + 1 + ); + } + drop(index); + drop(global); + drop(scoped); + let offline = RegistryIndex::new(&manifest, cache.path(), true).unwrap(); + let frozen = plan( + &manifest, + &runtime, + Some(&result.lockfile), + &offline, + &PlanOptions { + mode: LockMode::Locked, + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(frozen.lockfile, result.lockfile); + assert!(!frozen.lock_changed); + assert_eq!(frozen.packages.iter().filter(|p| p.install).count(), 2); +} + +#[test] +fn competing_scoped_children_require_an_explicit_source() { + let one = Server::new(BTreeMap::from([ + ("/versions".into(), versions(&["alpha", "child", "leaf"])), + ("/info/alpha".into(), info("parent")), + ("/info/child".into(), info("child")), + ("/info/leaf".into(), info("leaf")), + ])); + let two = Server::new(BTreeMap::from([ + ("/versions".into(), versions(&["omega", "child"])), + ("/info/omega".into(), info("parent")), + ("/info/child".into(), info("newer-child")), + ])); + let global = Server::new(BTreeMap::from([("/info/public".into(), info("public"))])); + let mut manifest = common::manifest(&global.url, "alpha='>= 0'\nomega='>= 0'"); + manifest.dependencies.get_mut("alpha").unwrap().source = Some(one.url.clone()); + manifest.dependencies.get_mut("omega").unwrap().source = Some(two.url.clone()); + let cache = tempfile::tempdir().unwrap(); + let index = RegistryIndex::new(&manifest, cache.path(), false).unwrap(); + let error = index.candidates("alpha").unwrap_err(); + assert!( + error + .to_string() + .contains("Ambiguous transitive source for child") + ); + manifest.dependencies.insert( + "child".into(), + rb_gemfile::Dependency { + source: Some(one.url.clone()), + ..Default::default() + }, + ); + let explicit = RegistryIndex::new(&manifest, cache.path(), false).unwrap(); + let result = plan( + &manifest, + &common::runtime(), + None, + &explicit, + &PlanOptions::default(), + ) + .unwrap(); + assert_eq!( + result + .packages + .iter() + .find(|p| p.record.package.id.name == "child") + .unwrap() + .record + .source, + one.url.clone().into() + ); + assert_eq!( + result + .packages + .iter() + .find(|p| p.record.package.id.name == "child") + .unwrap() + .record + .archive_version, + "1.0" + ); +} + +#[test] +fn explicit_missing_sources_do_not_fall_back_and_global_roots_override_inheritance() { + let private = scoped(); + let global = Server::new(BTreeMap::from([ + ("/info/child".into(), info("newer-child")), + ("/info/public".into(), info("public")), + ("/info/absent".into(), info("leaf")), + ])); + let mut manifest = common::manifest(&global.url, "parent='>= 0'\nchild='>= 0'\nabsent='>= 0'"); + manifest.dependencies.get_mut("parent").unwrap().source = Some(private.url.clone()); + manifest.dependencies.get_mut("absent").unwrap().source = Some(private.url.clone()); + let cache = tempfile::tempdir().unwrap(); + let index = RegistryIndex::new(&manifest, cache.path(), false).unwrap(); + assert!(index.candidates("absent").unwrap().is_empty()); + assert!( + !global + .requests + .lock() + .unwrap() + .contains(&"/info/absent".into()) + ); + let child = index.candidates("child").unwrap(); + assert_eq!(child[0].archive_version, "99.0"); + assert_eq!(child[0].source, global.url.clone().into()); + assert!( + !private + .requests + .lock() + .unwrap() + .contains(&"/info/child".into()) + ); +} + +#[test] +fn optional_scoped_archives_are_downloaded_only_after_opt_in_and_run_with_frozen_bundler() { + let project = tempfile::tempdir().unwrap(); + let archives = tempfile::tempdir().unwrap(); + let mut index = rb_packages::MemoryIndex::default(); + for (name, dependencies, file, content) in [ + ( + "application", + vec![("helper", ">= 1.0.0")], + "lib/application.rb", + "APPLICATION = 1\n", + ), + ("helper", vec![], "lib/helper.rb", "HELPER = 2\n"), + ( + "addon", + vec![("license", ">= 1.0.0"), ("helper", ">= 1.0.0")], + "lib/addon.rb", + "ADDON = 3\n", + ), + ("license", vec![], "lib/license.rb", "LICENSE = 4\n"), + ] { + common::add_archive( + &mut index, + archives.path(), + name, + &dependencies, + &[], + &[(file, content.as_bytes())], + ); + } + let responses = |names: &[&str]| { + let mut responses = BTreeMap::from([("/versions".into(), versions(names))]); + for name in names { + let record = &index.packages[*name][0]; + let dependencies = record + .package + .dependencies + .iter() + .map(|(n, r)| format!("{n}:{r}")) + .collect::>() + .join(","); + responses.insert( + format!("/info/{name}"), + format!("---\n1.0.0 {dependencies}|checksum:{}\n", record.checksum).into_bytes(), + ); + responses.insert( + format!("/gems/{name}-1.0.0.gem"), + std::fs::read(&index.archives[&record.checksum]).unwrap(), + ); + } + responses + }; + let global = Server::new(responses(&["application", "helper"])); + let scoped = Server::new(responses(&["addon", "license"])); + std::fs::write(project.path().join("Gemfile"), format!("source '{}'\ngem 'application'\ngroup :addon, optional: true do\n source '{}' do\n gem 'addon'\n end\nend\n", global.url, scoped.url)).unwrap(); + let options = || { + rb_packages::PrepareOptions::new( + project.path(), + common::ruby(), + project.path().join(".rb/cache"), + ) + }; + let first = rb_packages::prepare(options(), rb_task::TaskEvents::default()).unwrap(); + assert_eq!(first.installed.installed, 2); + assert!( + scoped + .requests + .lock() + .unwrap() + .iter() + .all(|path| !path.starts_with("/gems/")) + ); + assert!(!first.installed.gem_home.join("gems/addon-1.0.0").exists()); + let output = common::run( + &common::runtime(), + &first.installed, + project.path(), + "require 'bundler'; Bundler.settings.set_local(:frozen, true); require 'bundler/setup'; require 'application'; raise 'optional activated' if Gem.loaded_specs.key?('addon'); puts APPLICATION", + ); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(common::stdout(&output), "1\n"); + let mut enabled = options(); + enabled.plan.with.insert("addon".into()); + let second = rb_packages::prepare(enabled, rb_task::TaskEvents::default()).unwrap(); + assert!(!second.plan.lock_changed); + assert_eq!(second.installed.installed, 2); + assert_eq!(second.installed.reused, 2); + assert!( + scoped + .requests + .lock() + .unwrap() + .contains(&"/gems/addon-1.0.0.gem".into()) + ); + assert!( + scoped + .requests + .lock() + .unwrap() + .contains(&"/gems/license-1.0.0.gem".into()) + ); + let output = common::run( + &common::runtime(), + &second.installed, + project.path(), + "require 'bundler'; Bundler.settings.set_local(:frozen, true); require 'bundler/setup'; require 'application'; require 'addon'; require 'license'; puts APPLICATION + ADDON + LICENSE", + ); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(common::stdout(&output), "8\n"); +} diff --git a/crates/rb-packages/tests/sources.rs b/crates/rb-packages/tests/sources.rs new file mode 100644 index 0000000..a073f31 --- /dev/null +++ b/crates/rb-packages/tests/sources.rs @@ -0,0 +1,546 @@ +mod common; +use rb_packages::{LockMode, PrepareOptions, PreparedBundle, prepare}; +use rb_task::TaskEvents; +use std::{ + fs, + path::{Path, PathBuf}, + process::Command, +}; + +fn source(root: &Path) -> PathBuf { + let directory = root.join("source-repository"); + fs::create_dir_all(directory.join("lib/fixture")).unwrap(); + fs::write( + directory.join("fixture.gemspec"), + include_str!("fixtures/sources/fixture.gemspec"), + ) + .unwrap(); + fs::write( + directory.join("lib/fixture.rb"), + include_str!("fixtures/sources/lib/fixture.rb"), + ) + .unwrap(); + fs::write( + directory.join("lib/fixture/version.rb"), + include_str!("fixtures/sources/lib/fixture/version.rb"), + ) + .unwrap(); + directory +} + +fn git(directory: &Path, arguments: &[&str]) -> String { + let output = Command::new("git") + .arg("-C") + .arg(directory) + .args(arguments) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap().trim().into() +} + +fn commit(directory: &Path) -> String { + git(directory, &["add", "."]); + git( + directory, + &[ + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.test", + "commit", + "-m", + "Fixture", + ], + ); + git(directory, &["rev-parse", "HEAD"]) +} + +fn options(root: &Path) -> PrepareOptions { + let mut options = PrepareOptions::new(root, common::ruby(), root.join(".rb/cache")); + options + .install + .build + .environment + .insert("CCACHE_DISABLE".into(), "1".into()); + options +} + +fn require(bundle: &PreparedBundle, code: &str) -> String { + let output = Command::new(common::ruby()) + .envs(bundle.installed.environment()) + .env_remove("RUBYOPT") + .env_remove("RUBYLIB") + .env("BUNDLE_FROZEN", "true") + .current_dir(&bundle.input.root) + .args(["-rbundler/setup", "-e", code]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + common::stdout(&output) +} + +#[test] +fn git_source_locks_commit_installs_for_bundler_and_reuses_offline() { + let root = tempfile::tempdir().unwrap(); + let source = source(root.path()); + git(&source, &["init", "--initial-branch=main"]); + let revision = commit(&source); + fs::write( + root.path().join("Gemfile"), + format!( + "gem 'fixture', git: {}, branch: 'main'\n", + serde_json::to_string(&source.to_string_lossy()).unwrap() + ), + ) + .unwrap(); + let first = prepare(options(root.path()), TaskEvents::default()).unwrap(); + let git_source = first + .plan + .lockfile + .sources + .iter() + .find(|s| s.kind == rb_lockfile::SourceKind::Git) + .unwrap(); + assert_eq!(git_source.options["revision"], revision); + assert_eq!(git_source.options["branch"], "main"); + assert_eq!( + require(&first, "require 'fixture'; puts Fixture::VERSION"), + "1.0.0\n" + ); + let mut offline = options(root.path()); + offline.offline = true; + offline.plan.mode = LockMode::Locked; + let second = prepare(offline, TaskEvents::default()).unwrap(); + assert_eq!(second.installed.reused, 1); + assert!(!second.plan.lock_changed); + assert_eq!( + require(&second, "require 'fixture'; puts Fixture::VERSION"), + "1.0.0\n" + ); + fs::write( + source.join("lib/fixture/version.rb"), + "module Fixture; VERSION='1.0.1'; end\n", + ) + .unwrap(); + let changed_revision = commit(&source); + let preferred = prepare(options(root.path()), TaskEvents::default()).unwrap(); + assert_eq!( + require(&preferred, "require 'fixture'; puts Fixture::VERSION"), + "1.0.0\n" + ); + let mut update = options(root.path()); + update.plan.mode = LockMode::Update; + let changed = prepare(update, TaskEvents::default()).unwrap(); + assert_eq!( + require(&changed, "require 'fixture'; puts Fixture::VERSION"), + "1.0.1\n" + ); + assert_eq!( + changed + .plan + .lockfile + .sources + .iter() + .find(|s| s.kind == rb_lockfile::SourceKind::Git) + .unwrap() + .options["revision"], + changed_revision + ); +} + +#[test] +fn path_source_is_snapshotted_without_mutating_project_and_refreshes_on_edit() { + let root = tempfile::tempdir().unwrap(); + let source = source(root.path()); + fs::write( + root.path().join("Gemfile"), + "gem 'fixture', path: 'source-repository'\n", + ) + .unwrap(); + let first = prepare(options(root.path()), TaskEvents::default()).unwrap(); + assert_eq!( + require(&first, "require 'fixture'; puts Fixture::VERSION"), + "1.0.0\n" + ); + assert!(first.input.lockfile.is_file()); + assert_eq!( + first + .plan + .lockfile + .sources + .iter() + .find(|s| s.kind == rb_lockfile::SourceKind::Path) + .unwrap() + .remotes, + [dunce::canonicalize(&source) + .unwrap() + .to_string_lossy() + .into_owned()] + ); + let second = prepare(options(root.path()), TaskEvents::default()).unwrap(); + assert_eq!(second.installed.reused, 1); + fs::write( + source.join("lib/fixture.rb"), + "require_relative 'fixture/version'; CHANGED=true\n", + ) + .unwrap(); + let changed = prepare(options(root.path()), TaskEvents::default()).unwrap(); + assert_eq!(changed.installed.installed, 1); + assert_eq!( + require(&changed, "require 'fixture'; puts CHANGED"), + "true\n" + ); + assert!(!source.join("installation.json").exists()); +} + +#[cfg(unix)] +#[test] +fn native_sources_compile_and_load_with_frozen_bundler() { + for kind in ["git", "path"] { + let root = tempfile::tempdir().unwrap(); + let source = source(root.path()); + fs::create_dir_all(source.join("ext/fixture")).unwrap(); + fs::write( + source.join("ext/fixture/extconf.rb"), + include_str!("fixtures/sources/extconf.rb"), + ) + .unwrap(); + fs::write( + source.join("ext/fixture/source_native.c"), + include_str!("fixtures/sources/source_native.c"), + ) + .unwrap(); + fs::write( + source.join("lib/fixture.rb"), + "require_relative 'fixture/version'; require 'source_native'\n", + ) + .unwrap(); + if kind == "git" { + git(&source, &["init", "--initial-branch=main"]); + commit(&source); + } + fs::write( + root.path().join("Gemfile"), + format!( + "gem 'fixture', {kind}: {}\n", + serde_json::to_string(&source.to_string_lossy()).unwrap() + ), + ) + .unwrap(); + let bundle = prepare(options(root.path()), TaskEvents::default()).unwrap(); + assert_eq!( + require( + &bundle, + "require 'fixture'; raise if Gem.loaded_specs.fetch('fixture').missing_extensions?; puts Fixture.answer" + ), + "42\n", + "{kind}" + ); + assert!(!source.join("ext/fixture/Makefile").exists()); + } +} + +#[test] +fn git_ref_and_tag_select_exact_commits_and_custom_glob() { + let root = tempfile::tempdir().unwrap(); + let source = source(root.path()); + git(&source, &["init", "--initial-branch=main"]); + let old = commit(&source); + git(&source, &["tag", "v1"]); + fs::write( + source.join("lib/fixture/version.rb"), + "module Fixture; VERSION='2.0.0'; end\n", + ) + .unwrap(); + commit(&source); + for selector in ["tag: 'v1'".to_owned(), format!("ref: '{old}'")] { + fs::write( + root.path().join("Gemfile"), + format!( + "gem 'fixture', git: {}, {selector}, glob: 'fixture.gemspec'\n", + serde_json::to_string(&source.to_string_lossy()).unwrap() + ), + ) + .unwrap(); + let bundle = prepare(options(root.path()), TaskEvents::default()).unwrap(); + assert_eq!( + require(&bundle, "require 'fixture'; puts Fixture::VERSION"), + "1.0.0\n" + ); + let lock = bundle + .plan + .lockfile + .sources + .iter() + .find(|s| s.kind == rb_lockfile::SourceKind::Git) + .unwrap(); + assert_eq!(lock.options["revision"], old); + assert_eq!(lock.options["glob"], "fixture.gemspec"); + } +} + +#[test] +fn source_gems_from_one_repository_resolve_transitive_siblings() { + let root = tempfile::tempdir().unwrap(); + let source = source(root.path()); + let spec = fs::read_to_string(source.join("fixture.gemspec")) + .unwrap() + .replace( + "spec.name = 'fixture'", + "spec.name = 'fixture'; spec.add_runtime_dependency('helper', '>= 1')", + ); + fs::write(source.join("fixture.gemspec"), spec).unwrap(); + fs::write( + source.join("helper.gemspec"), + include_str!("fixtures/sources/helper.gemspec"), + ) + .unwrap(); + fs::write(source.join("lib/helper.rb"), "HELPER=42\n").unwrap(); + fs::write( + source.join("lib/fixture.rb"), + "require 'helper'; require_relative 'fixture/version'\n", + ) + .unwrap(); + git(&source, &["init", "--initial-branch=main"]); + commit(&source); + fs::write( + root.path().join("Gemfile"), + format!( + "gem 'fixture', git: {}\n", + serde_json::to_string(&source.to_string_lossy()).unwrap() + ), + ) + .unwrap(); + let bundle = prepare(options(root.path()), TaskEvents::default()).unwrap(); + assert_eq!(bundle.plan.packages.len(), 2); + assert_eq!(require(&bundle, "require 'fixture'; puts HELPER"), "42\n"); + assert!( + bundle + .plan + .packages + .iter() + .all(|p| matches!(p.record.source, rb_packages::PackageSource::Git { .. })) + ); +} + +#[cfg(unix)] +#[test] +fn source_symlink_cannot_escape_the_package_directory() { + let root = tempfile::tempdir().unwrap(); + let source = source(root.path()); + fs::write(root.path().join("outside"), "private\n").unwrap(); + std::os::unix::fs::symlink(root.path().join("outside"), source.join("leak")).unwrap(); + fs::write( + root.path().join("Gemfile"), + "gem 'fixture', path: 'source-repository'\n", + ) + .unwrap(); + let error = prepare(options(root.path()), TaskEvents::default()) + .err() + .unwrap(); + assert!(format!("{error:#}").contains("Source link escapes")); +} + +#[test] +fn snapshots_omit_unlisted_project_files_and_repair_missing_runtime_files() { + let root = tempfile::tempdir().unwrap(); + let source = source(root.path()); + fs::write(source.join(".env"), "TOKEN=fixture-secret\n").unwrap(); + fs::write( + root.path().join("Gemfile"), + "gem 'fixture', path: 'source-repository'\n", + ) + .unwrap(); + let first = prepare(options(root.path()), TaskEvents::default()).unwrap(); + assert!( + !first + .installed + .gem_home + .join("gems/fixture-1.0.0/.env") + .exists() + ); + assert_eq!( + require( + &first, + "puts File.exist?(File.join(Gem.loaded_specs.fetch('fixture').full_gem_path, '.env'))" + ), + "false\n" + ); + let loaded = require( + &first, + "puts Gem.loaded_specs.fetch('fixture').full_gem_path", + ); + fs::remove_file(Path::new(loaded.trim()).join("lib/fixture.rb")).unwrap(); + let repaired = prepare(options(root.path()), TaskEvents::default()).unwrap(); + assert_eq!( + require(&repaired, "require 'fixture'; puts Fixture::VERSION"), + "1.0.0\n" + ); +} + +#[test] +fn locked_path_rejects_changed_dependency_metadata_without_fetching_new_packages() { + let root = tempfile::tempdir().unwrap(); + let source = source(root.path()); + fs::write( + root.path().join("Gemfile"), + "gem 'fixture', path: 'source-repository'\n", + ) + .unwrap(); + prepare(options(root.path()), TaskEvents::default()).unwrap(); + let old = fs::read_to_string(source.join("fixture.gemspec")).unwrap(); + fs::write( + source.join("fixture.gemspec"), + old.replace( + "spec.name = 'fixture'", + "spec.name = 'fixture'; spec.add_runtime_dependency('unavailable', '>= 0')", + ), + ) + .unwrap(); + let mut locked = options(root.path()); + locked.offline = true; + locked.plan.mode = LockMode::Locked; + let error = prepare(locked, TaskEvents::default()).err().unwrap(); + assert!(format!("{error:#}").contains("Locked dependency metadata changed")); +} + +#[test] +fn cached_git_checkout_modifications_are_rejected() { + let root = tempfile::tempdir().unwrap(); + let source = source(root.path()); + git(&source, &["init", "--initial-branch=main"]); + commit(&source); + fs::write( + root.path().join("Gemfile"), + format!( + "gem 'fixture', git: {}\n", + serde_json::to_string(&source.to_string_lossy()).unwrap() + ), + ) + .unwrap(); + let first = prepare(options(root.path()), TaskEvents::default()).unwrap(); + let cache = root.path().join(".rb/cache/git"); + let repository = fs::read_dir(cache).unwrap().next().unwrap().unwrap().path(); + let checkout = fs::read_dir(repository.join("checkouts")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + fs::write(checkout.join("lib/fixture.rb"), "raise 'tampered'\n").unwrap(); + let mut offline = options(root.path()); + offline.offline = true; + let error = prepare(offline, TaskEvents::default()).err().unwrap(); + assert!(format!("{error:#}").contains("Cached Git checkout has been modified")); + assert_eq!( + require(&first, "require 'fixture'; puts Fixture::VERSION"), + "1.0.0\n" + ); +} + +#[path = "common/git_server.rs"] +mod git_server; + +#[test] +fn git_submodules_are_materialized_and_reused_offline() { + let root = tempfile::tempdir().unwrap(); + let module = root.path().join("metadata-module"); + fs::create_dir(&module).unwrap(); + git(&module, &["init", "--initial-branch=main"]); + fs::write( + module.join("version.rb"), + include_str!("fixtures/sources/lib/fixture/version.rb"), + ) + .unwrap(); + commit(&module); + let bare = root.path().join("metadata-module.git"); + git( + root.path(), + &[ + "clone", + "--bare", + module.to_str().unwrap(), + bare.to_str().unwrap(), + ], + ); + git(&bare, &["update-server-info"]); + let server = git_server::GitServer::start(&bare); + let source = source(root.path()); + git(&source, &["init", "--initial-branch=main"]); + git(&source, &["submodule", "add", &server.url, "vendor"]); + fs::write( + source.join("lib/fixture/version.rb"), + "require_relative '../../vendor/version'\n", + ) + .unwrap(); + let spec = fs::read_to_string(source.join("fixture.gemspec")) + .unwrap() + .replace("{lib,ext}", "{lib,ext,vendor}"); + fs::write(source.join("fixture.gemspec"), spec).unwrap(); + commit(&source); + fs::write( + root.path().join("Gemfile"), + format!( + "gem 'fixture', git: {}, submodules: true\n", + serde_json::to_string(&source.to_string_lossy()).unwrap() + ), + ) + .unwrap(); + let first = prepare(options(root.path()), TaskEvents::default()).unwrap(); + assert_eq!( + require(&first, "require 'fixture'; puts Fixture::VERSION"), + "1.0.0\n" + ); + drop(server); + let mut offline = options(root.path()); + offline.offline = true; + offline.plan.mode = LockMode::Locked; + let second = prepare(offline, TaskEvents::default()).unwrap(); + assert_eq!(second.installed.reused, 1); + assert_eq!( + require(&second, "require 'fixture'; puts Fixture::VERSION"), + "1.0.0\n" + ); +} + +#[test] +fn path_sources_with_distinct_globs_keep_distinct_compatibility_locations() { + let root = tempfile::tempdir().unwrap(); + let source = source(root.path()); + fs::write( + source.join("helper.gemspec"), + include_str!("fixtures/sources/helper.gemspec"), + ) + .unwrap(); + fs::write(source.join("lib/helper.rb"), "HELPER=42\n").unwrap(); + fs::write(root.path().join("Gemfile"), "gem 'fixture', path: 'source-repository', glob: 'fixture.gemspec'\ngem 'helper', path: 'source-repository', glob: 'helper.gemspec'\n").unwrap(); + let bundle = prepare(options(root.path()), TaskEvents::default()).unwrap(); + assert_eq!( + require(&bundle, "require 'fixture'; require 'helper'; puts HELPER"), + "42\n" + ); +} + +#[test] +fn platform_specific_git_source_is_rejected_before_acquisition() { + let root = tempfile::tempdir().unwrap(); + fs::write( + root.path().join("Gemfile"), + "gem 'fixture', git: 'https://unavailable.example.test/fixture.git', platforms: :jruby\n", + ) + .unwrap(); + let mut options = options(root.path()); + options.offline = true; + let error = prepare(options, TaskEvents::default()).err().unwrap(); + assert!(format!("{error:#}").contains("Platform-specific Git/path sources")); + assert!(!root.path().join(".rb/cache/git").exists()); +} diff --git a/crates/rb-resolver/README.md b/crates/rb-resolver/README.md index 885e2bc..d768555 100644 --- a/crates/rb-resolver/README.md +++ b/crates/rb-resolver/README.md @@ -30,6 +30,10 @@ locked identity, then a locked version, then the newest version with the best platform score. Locks are preferences, so conflicts can move to another version. Results are sorted by gem name, independently of provider order. +`resolve_targets` accepts multiple requests and returns one closure per target, +with a shared release for each gem and platform-specific dependencies. +`force_ruby` selects source gems for the named dependencies in a request. + Prereleases are eligible when requested explicitly, locked, enabled through `allow_prereleases`, or when all Ruby/platform-compatible candidates are prereleases. Provider results are fetched once per gem per resolution; identical diff --git a/crates/rb-resolver/src/lib.rs b/crates/rb-resolver/src/lib.rs index 605748a..f39c198 100644 --- a/crates/rb-resolver/src/lib.rs +++ b/crates/rb-resolver/src/lib.rs @@ -10,4 +10,4 @@ pub use error::ResolveError; pub use package::Package; pub use provider::{InMemoryIndex, PackageProvider}; pub use request::ResolveRequest; -pub use solver::resolve; +pub use solver::{resolve, resolve_targets}; diff --git a/crates/rb-resolver/src/request.rs b/crates/rb-resolver/src/request.rs index 0e63c27..3ef5e01 100644 --- a/crates/rb-resolver/src/request.rs +++ b/crates/rb-resolver/src/request.rs @@ -1,5 +1,5 @@ use rb_gem_types::{PackageId, Platform, Requirement, Version}; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; /// Resolution inputs and locked package preferences. #[derive(Clone, Debug)] @@ -9,6 +9,7 @@ pub struct ResolveRequest { pub target: Platform, pub locked: Vec, pub allow_prereleases: bool, + pub force_ruby: BTreeSet, } impl ResolveRequest { @@ -19,6 +20,7 @@ impl ResolveRequest { target, locked: Vec::new(), allow_prereleases: false, + force_ruby: BTreeSet::new(), } } } diff --git a/crates/rb-resolver/src/solver.rs b/crates/rb-resolver/src/solver.rs index b894515..80df7f3 100644 --- a/crates/rb-resolver/src/solver.rs +++ b/crates/rb-resolver/src/solver.rs @@ -3,49 +3,53 @@ use pubgrub::{ DefaultStringReporter, Dependencies, DependencyProvider, PackageResolutionStatistics, PubGrubError, Ranges, Reporter, }; -use rb_gem_types::{PackageId, Platform, Requirement, Version}; +use rb_gem_types::{PackageId, Requirement, Version}; use std::{cell::RefCell, collections::BTreeMap, fmt}; #[derive(Clone, Debug, Eq, PartialEq, Hash)] enum Name { Root, - Gem(String), + Gem(usize, String), + Release(String), } impl fmt::Display for Name { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Root => f.write_str(""), - Self::Gem(name) => name.fmt(f), + Self::Gem(_, name) | Self::Release(name) => name.fmt(f), } } } #[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] -struct Candidate { - version: Version, - platform: Platform, +enum Candidate { + Root, + Release(Version), + Package(PackageId), } impl Candidate { fn package(package: &Package) -> Self { - Self { - version: package.id.version.clone(), - platform: package.id.platform.clone(), - } + Self::Package(package.id.clone()) + } + + fn release(package: &Package) -> Self { + Self::Release(package.id.version.clone()) } fn root() -> Self { - Self { - version: "0".parse().unwrap(), - platform: Platform::parse("ruby"), - } + Self::Root } } impl fmt::Display for Candidate { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{} [{}]", self.version, self.platform) + match self { + Self::Root => f.write_str(""), + Self::Release(version) => version.fmt(f), + Self::Package(id) => write!(f, "{} [{}]", id.version, id.platform), + } } } @@ -53,25 +57,39 @@ pub fn resolve( provider: &P, request: &ResolveRequest, ) -> Result, ResolveError> { + resolve_targets(provider, std::slice::from_ref(request)).map(|mut results| results.remove(0)) +} + +/// Resolves platform-specific closures with one shared release per gem name. +pub fn resolve_targets( + provider: &P, + requests: &[ResolveRequest], +) -> Result>, ResolveError> { let solver = Solver { provider, - request, + requests, + available: RefCell::new(BTreeMap::new()), packages: RefCell::new(BTreeMap::new()), }; let solution = pubgrub::resolve(&solver, Name::Root, Candidate::root()).map_err(|error| match error { PubGrubError::NoSolution(mut tree) => { tree.collapse_no_versions(); - let roots = request - .roots + let environments = requests .iter() - .map(|(name, requirement)| format!("{name} ({requirement})")) + .map(|request| { + let roots = request + .roots + .iter() + .map(|(name, requirement)| format!("{name} ({requirement})")) + .collect::>() + .join(", "); + format!("Ruby {} on {} with {roots}", request.ruby, request.target) + }) .collect::>() - .join(", "); + .join("; "); ResolveError::NoSolution(format!( - "Dependency resolution failed for Ruby {} on {} with {roots}: {}", - request.ruby, - request.target, + "Dependency resolution failed for {environments}: {}", DefaultStringReporter::report(&tree) )) } @@ -80,29 +98,32 @@ pub fn resolve( | PubGrubError::ErrorInShouldCancel(source) => source, })?; let packages = solver.packages.borrow(); - let mut result = Vec::new(); + let mut result = vec![Vec::new(); requests.len()]; for (name, candidate) in solution { - if let Name::Gem(name) = name { - let package = packages[&name] + if let Name::Gem(target, name) = name { + let package = packages[&(target, name)] .iter() .find(|package| Candidate::package(package) == candidate) .unwrap(); - result.push(package.clone()); + result[target].push(package.clone()); } } - result.sort_by(|a, b| a.id.name.cmp(&b.id.name)); + for packages in &mut result { + packages.sort_by(|a, b| a.id.name.cmp(&b.id.name)); + } Ok(result) } struct Solver<'a, P> { provider: &'a P, - request: &'a ResolveRequest, - packages: RefCell>>, + requests: &'a [ResolveRequest], + available: RefCell>>, + packages: RefCell>>, } impl Solver<'_, P> { - fn fetch(&self, name: &str) -> Result<(), ResolveError> { - if self.packages.borrow().contains_key(name) { + fn fetch_available(&self, name: &str) -> Result<(), ResolveError> { + if self.available.borrow().contains_key(name) { return Ok(()); } let mut unique = BTreeMap::::new(); @@ -121,9 +142,7 @@ impl Solver<'_, P> { }); } if let Some(previous) = unique.get(&package.id) { - if previous != &package - || previous.id.version.to_string() != package.id.version.to_string() - { + if previous != &package { return Err(ResolveError::InvalidMetadata { package: name.into(), reason: format!("conflicting records for {}", package.id), @@ -133,50 +152,69 @@ impl Solver<'_, P> { unique.insert(package.id.clone(), package); } } - let mut packages: Vec<_> = unique - .into_values() + self.available + .borrow_mut() + .insert(name.into(), unique.into_values().collect()); + Ok(()) + } + + fn fetch(&self, target: usize, name: &str) -> Result<(), ResolveError> { + let key = (target, name.to_owned()); + if self.packages.borrow().contains_key(&key) { + return Ok(()); + } + self.fetch_available(name)?; + let request = &self.requests[target]; + let mut packages: Vec<_> = self.available.borrow()[name] + .iter() .filter(|package| { - package.id.platform.matches(&self.request.target) - && package.ruby.matches(&self.request.ruby) + package.id.platform.matches(&request.target) + && package.ruby.matches(&request.ruby) + && (!request.force_ruby.contains(name) || package.id.platform.os == "ruby") }) + .cloned() .collect(); packages.sort_by(|a, b| { b.id.version .cmp(&a.id.version) .then_with(|| { a.id.platform - .score(&self.request.target) - .cmp(&b.id.platform.score(&self.request.target)) + .score(&request.target) + .cmp(&b.id.platform.score(&request.target)) }) .then_with(|| a.id.platform.cmp(&b.id.platform)) + .then_with(|| b.id.version.as_str().cmp(a.id.version.as_str())) }); - self.packages.borrow_mut().insert(name.into(), packages); + self.packages.borrow_mut().insert(key, packages); Ok(()) } - fn locked_version(&self, id: &PackageId) -> bool { - self.request + fn locked_version(&self, target: usize, id: &PackageId) -> bool { + self.requests[target] .locked .iter() - .any(|locked| locked.name == id.name && locked.version == id.version) + .any(|locked| locked.name == id.name && locked.version.as_str() == id.version.as_str()) } fn range( &self, + target: usize, name: &str, requirement: &Requirement, ) -> Result, ResolveError> { - self.fetch(name)?; + self.fetch(target, name)?; + let request = &self.requests[target]; let packages = self.packages.borrow(); - let only_prereleases = packages[name] + let packages = &packages[&(target, name.to_owned())]; + let only_prereleases = packages .iter() .all(|package| package.id.version.prerelease()); let mut range = Ranges::empty(); - for package in &packages[name] { - let prerelease_allowed = self.request.allow_prereleases + for package in packages { + let prerelease_allowed = request.allow_prereleases || requirement.prerelease() || only_prereleases - || self.locked_version(&package.id); + || self.locked_version(target, &package.id); if requirement.matches(&package.id.version) && (!package.id.version.prerelease() || prerelease_allowed) { @@ -204,26 +242,37 @@ impl DependencyProvider for Solver<'_, P> { name: &Name, range: &Self::VS, ) -> Result, ResolveError> { - let Name::Gem(name) = name else { - return Ok(Some(Candidate::root())); - }; - self.fetch(name)?; - let packages = self.packages.borrow(); - let eligible: Vec<_> = packages[name] - .iter() - .filter(|package| range.contains(&Candidate::package(package))) - .collect(); - let locked = &self.request.locked; - let chosen = eligible - .iter() - .find(|package| locked.contains(&package.id)) - .or_else(|| { - eligible + match name { + Name::Root => Ok(Some(Candidate::root())), + Name::Release(name) => { + self.fetch_available(name)?; + Ok(self.available.borrow()[name] .iter() - .find(|package| self.locked_version(&package.id)) - }) - .or_else(|| eligible.first()); - Ok(chosen.map(|package| Candidate::package(package))) + .map(Candidate::release) + .filter(|candidate| range.contains(candidate)) + .max()) + } + Name::Gem(target, name) => { + let target = *target; + self.fetch(target, name)?; + let packages = self.packages.borrow(); + let eligible: Vec<_> = packages[&(target, name.clone())] + .iter() + .filter(|package| range.contains(&Candidate::package(package))) + .collect(); + let locked = &self.requests[target].locked; + let chosen = eligible + .iter() + .find(|package| locked.contains(&package.id)) + .or_else(|| { + eligible + .iter() + .find(|package| self.locked_version(target, &package.id)) + }) + .or_else(|| eligible.first()); + Ok(chosen.map(|package| Candidate::package(package))) + } + } } fn get_dependencies( @@ -231,21 +280,39 @@ impl DependencyProvider for Solver<'_, P> { name: &Name, version: &Candidate, ) -> Result, ResolveError> { - let dependencies = match name { - Name::Root => self.request.roots.clone(), - Name::Gem(name) => { - self.fetch(name)?; - self.packages.borrow()[name] + let mut result = Vec::new(); + match name { + Name::Root => { + for (target, request) in self.requests.iter().enumerate() { + for (name, requirement) in &request.roots { + result.push(( + Name::Gem(target, name.clone()), + self.range(target, name, requirement)?, + )); + } + } + } + Name::Release(_) => {} + Name::Gem(target, name) => { + self.fetch(*target, name)?; + let package = self.packages.borrow()[&(*target, name.clone())] .iter() .find(|package| Candidate::package(package) == *version) .unwrap() - .dependencies - .clone() + .clone(); + for (dependency, requirement) in &package.dependencies { + result.push(( + Name::Gem(*target, dependency.clone()), + self.range(*target, dependency, requirement)?, + )); + } + if self.requests.len() > 1 { + result.push(( + Name::Release(name.clone()), + Ranges::singleton(Candidate::release(&package)), + )); + } } - }; - let mut result = Vec::new(); - for (name, requirement) in dependencies { - result.push((Name::Gem(name.clone()), self.range(&name, &requirement)?)); } Ok(Dependencies::Available(result.into_iter().collect())) } diff --git a/crates/rb-resolver/tests/providers.rs b/crates/rb-resolver/tests/providers.rs index 37ed010..cbe2a6e 100644 --- a/crates/rb-resolver/tests/providers.rs +++ b/crates/rb-resolver/tests/providers.rs @@ -119,10 +119,14 @@ fn mismatched_provider_names_are_errors_not_unsatisfiable_requirements() { } #[test] -fn equal_versions_with_different_archive_spellings_are_rejected() { +fn equal_versions_with_different_archive_spellings_remain_individually_lockable() { let index = InMemoryIndex::new([package("app", "1", &[]), package("app", "1.0", &[])]); - let error = resolve(&index, &request(&[("app", ">= 0")])).unwrap_err(); - assert!(matches!(error, ResolveError::InvalidMetadata { .. })); + for spelling in ["1", "1.0"] { + let mut input = request(&[("app", ">= 0")]); + input.locked.push(package("app", spelling, &[]).id); + let result = resolve(&index, &input).unwrap(); + assert_eq!(result[0].id.version.to_string(), spelling); + } } #[test] diff --git a/crates/rb-resolver/tests/resolution.rs b/crates/rb-resolver/tests/resolution.rs index 03b1d90..4e04e38 100644 --- a/crates/rb-resolver/tests/resolution.rs +++ b/crates/rb-resolver/tests/resolution.rs @@ -220,3 +220,22 @@ fn pessimistic_requirements_exclude_the_next_release_prerelease() { "1.9" ); } + +#[test] +fn equivalent_releases_can_backtrack_without_merging_dependencies() { + let records = [ + package("faker", "2.0", &[("i18n", "= 1")]), + package("faker", "2.0.0", &[("i18n", "= 2")]), + package("i18n", "1", &[]), + package("i18n", "2", &[]), + ]; + for candidates in [records.to_vec(), records.into_iter().rev().collect()] { + let index = InMemoryIndex::new(candidates); + let result = resolve(&index, &request(&[("faker", "= 2.0"), ("i18n", "= 1")])).unwrap(); + assert_eq!(result[0].id.version.to_string(), "2.0"); + let result = resolve(&index, &request(&[("faker", "= 2.0"), ("i18n", "= 2")])).unwrap(); + assert_eq!(result[0].id.version.to_string(), "2.0.0"); + let result = resolve(&index, &request(&[("faker", "= 2.0")])).unwrap(); + assert_eq!(result[0].id.version.to_string(), "2.0.0"); + } +} diff --git a/crates/rb-resolver/tests/targets.rs b/crates/rb-resolver/tests/targets.rs new file mode 100644 index 0000000..c5531c0 --- /dev/null +++ b/crates/rb-resolver/tests/targets.rs @@ -0,0 +1,103 @@ +mod common; +use common::{names, package, request}; +use rb_gem_types::Platform; +use rb_resolver::{InMemoryIndex, Package, ResolveError, resolve_targets}; + +fn native(name: &str, version: &str, platform: &str, dependencies: &[(&str, &str)]) -> Package { + let mut candidate = package(name, version, dependencies); + candidate.id.platform = Platform::parse(platform); + candidate +} + +#[test] +fn targets_backtrack_to_a_shared_release_and_keep_native_dependency_closures() { + let records = [ + native("native", "3", "x86_64-linux", &[("shared", "= 3")]), + native("native", "2", "x86_64-linux", &[("shared", "= 2")]), + native("native", "2", "arm64-darwin", &[("shared", "= 1")]), + native("native", "1", "x86_64-linux", &[("shared", "= 1")]), + native( + "native", + "1", + "arm64-darwin", + &[("shared", "= 1"), ("darwin-helper", ">= 0")], + ), + package("shared", "1", &[]), + package("shared", "2", &[]), + package("shared", "3", &[]), + package("darwin-helper", "1", &[]), + ]; + let linux = request(&[("native", ">= 0")]); + let mut darwin = linux.clone(); + darwin.target = Platform::parse("arm64-darwin"); + for candidates in [records.to_vec(), records.into_iter().rev().collect()] { + let result = resolve_targets( + &InMemoryIndex::new(candidates), + &[linux.clone(), darwin.clone()], + ) + .unwrap(); + assert_eq!(result.len(), 2); + for packages in &result { + assert!( + names(packages.clone()) + .values() + .all(|version| version == "1") + ); + } + assert_eq!(result[0].len(), 2); + assert_eq!(result[1].len(), 3); + assert_eq!(result[0][0].id.platform.to_string(), "x86_64-linux"); + assert_eq!(result[1][0].id.name, "darwin-helper"); + assert_eq!(result[1][1].id.platform.to_string(), "arm64-darwin"); + } +} + +#[test] +fn incompatible_target_roots_fail_instead_of_locking_unrelated_versions() { + let index = InMemoryIndex::new([package("shared", "1", &[]), package("shared", "2", &[])]); + let linux = request(&[("shared", "= 1")]); + let mut darwin = request(&[("shared", "= 2")]); + darwin.target = Platform::parse("arm64-darwin"); + let error = resolve_targets(&index, &[linux, darwin]).unwrap_err(); + assert!(matches!(error, ResolveError::NoSolution(_))); + let message = error.to_string(); + assert!(message.contains("shared")); + assert!(message.contains("arm64-darwin")); + assert!(message.contains("x86_64-linux")); +} + +#[test] +fn force_ruby_is_local_to_each_target_and_empty_requests_do_not_resolve_packages() { + let index = InMemoryIndex::new([ + package("native", "1", &[]), + native("native", "1", "x86_64-linux", &[("linux-helper", ">= 0")]), + native("native", "1", "arm64-darwin", &[("darwin-helper", ">= 0")]), + package("linux-helper", "1", &[]), + package("darwin-helper", "1", &[]), + ]); + let mut linux = request(&[("native", ">= 0")]); + linux.force_ruby.insert("native".into()); + let mut darwin = request(&[("native", ">= 0")]); + darwin.target = Platform::parse("arm64-darwin"); + let result = resolve_targets(&index, &[linux, darwin]).unwrap(); + assert_eq!(result[0].len(), 1); + assert_eq!(result[0][0].id.platform.os, "ruby"); + assert_eq!(result[1].len(), 2); + assert_eq!(result[1][0].id.name, "darwin-helper"); + assert!(resolve_targets(&index, &[]).unwrap().is_empty()); +} + +#[test] +fn equivalent_versions_share_a_requirement_without_merging_target_archives() { + let index = InMemoryIndex::new([ + native("native", "2.0", "x86_64-linux", &[]), + native("native", "2.0.0", "arm64-darwin", &[]), + ]); + let linux = request(&[("native", "= 2")]); + let mut darwin = linux.clone(); + darwin.target = Platform::parse("arm64-darwin"); + let result = resolve_targets(&index, &[linux, darwin]).unwrap(); + assert_eq!(result[0][0].id.version, result[1][0].id.version); + assert_eq!(result[0][0].id.full_name(), "native-2.0-x86_64-linux"); + assert_eq!(result[1][0].id.full_name(), "native-2.0.0-arm64-darwin"); +} diff --git a/gems/.rubocop.yml b/gems/.rubocop.yml index f86b138..84aeaf3 100644 --- a/gems/.rubocop.yml +++ b/gems/.rubocop.yml @@ -19,3 +19,7 @@ Layout/LineLength: Security: Enabled: true + +Security/Eval: + Exclude: + - 'rb-gemfile/lib/rb_gemfile/gemspec.rb' diff --git a/gems/Rakefile b/gems/Rakefile index ae1197c..6531c5e 100644 --- a/gems/Rakefile +++ b/gems/Rakefile @@ -1,4 +1,5 @@ task :test do + ruby File.join(__dir__, 'rb-gemfile/test/all_test.rb') ruby File.join(__dir__, 'rb-microgems/test/all_test.rb') ruby File.join(__dir__, 'rb-gem-installer/test/all_test.rb') end diff --git a/gems/rb-gem-installer/README.md b/gems/rb-gem-installer/README.md index 201277c..c40b0d3 100644 --- a/gems/rb-gem-installer/README.md +++ b/gems/rb-gem-installer/README.md @@ -3,3 +3,5 @@ The Ruby helper for the Rust installer. `RubyButler::GemInstaller::RubyConfig.read` returns the selected interpreter's build configuration. `ruby --disable=gems -Ilib exe/rb-ruby-config` emits it as JSON without RubyGems or Bundler. Run `ruby test/all_test.rb` or `rake test`. Minitest and rake are development dependencies only. Build with `gem build rb-gem-installer.gemspec`; the Rust installer embeds and materializes these files directly. + +`Runtime.read` reports Ruby and available RubyGems/Bundler versions. `Specification.write` serializes verified runtime metadata into an installed gemspec. `SourceSpecifications.read` evaluates trusted source gemspecs, including relative project helpers, and returns their metadata and file lists. These compatibility helpers use RubyGems when called; loading the library and using `RubyConfig` still work with RubyGems disabled. diff --git a/gems/rb-gem-installer/lib/rb_gem_installer.rb b/gems/rb-gem-installer/lib/rb_gem_installer.rb index 9f13c98..e36ffac 100644 --- a/gems/rb-gem-installer/lib/rb_gem_installer.rb +++ b/gems/rb-gem-installer/lib/rb_gem_installer.rb @@ -1,4 +1,7 @@ require_relative 'rb_gem_installer/ruby_config' +require_relative 'rb_gem_installer/specification' +require_relative 'rb_gem_installer/runtime' +require_relative 'rb_gem_installer/source_specifications' module RubyButler module GemInstaller diff --git a/gems/rb-gem-installer/lib/rb_gem_installer/runtime.rb b/gems/rb-gem-installer/lib/rb_gem_installer/runtime.rb new file mode 100644 index 0000000..7a00209 --- /dev/null +++ b/gems/rb-gem-installer/lib/rb_gem_installer/runtime.rb @@ -0,0 +1,16 @@ +require 'rbconfig' + +module RubyButler + module GemInstaller + module Runtime + def self.read + require 'rubygems' + { executable: RbConfig.ruby, version: RUBY_VERSION, engine: RUBY_ENGINE, + engine_version: RUBY_ENGINE_VERSION, patchlevel: RUBY_PATCHLEVEL, + platform: Gem::Platform.local.to_s, abi: Gem.extension_api_version, + rubygems: Gem::VERSION, + bundler: Gem::Specification.find_all_by_name('bundler').map(&:version).max&.to_s } + end + end + end +end diff --git a/gems/rb-gem-installer/lib/rb_gem_installer/source_specifications.rb b/gems/rb-gem-installer/lib/rb_gem_installer/source_specifications.rb new file mode 100644 index 0000000..6b6e229 --- /dev/null +++ b/gems/rb-gem-installer/lib/rb_gem_installer/source_specifications.rb @@ -0,0 +1,22 @@ +module RubyButler + module GemInstaller + module SourceSpecifications + DEFAULT_GLOB = "{,*,*/*}.gemspec".freeze + + def self.read(directory, glob = DEFAULT_GLOB) + require 'rubygems' + root = File.realpath(directory) + Dir.glob(glob, base: root).sort.map do |path| + absolute = File.realpath(File.join(root, path)) + raise "Gemspec escapes source directory" unless absolute.start_with?("#{root}/") + + specification = Dir.chdir(File.dirname(absolute)) { Gem::Specification.load(absolute) } + raise "Unable to load #{path}" unless specification + + { gemspec: absolute.delete_prefix("#{root}/"), metadata: specification.to_yaml, specification: specification.to_ruby, + files: specification.files, require_paths: specification.raw_require_paths, extensions: specification.extensions } + end + end + end + end +end diff --git a/gems/rb-gem-installer/lib/rb_gem_installer/specification.rb b/gems/rb-gem-installer/lib/rb_gem_installer/specification.rb new file mode 100644 index 0000000..85fd9fe --- /dev/null +++ b/gems/rb-gem-installer/lib/rb_gem_installer/specification.rb @@ -0,0 +1,23 @@ +module RubyButler + module GemInstaller + module Specification + def self.write(record, path) + spec = Gem::Specification.new do |gem| + gem.name = record.fetch('name') + gem.version = record.fetch('version') + gem.platform = record.fetch('platform') + gem.required_ruby_version = record.fetch('ruby').split(', ') + gem.required_rubygems_version = record.fetch('rubygems').split(', ') + gem.require_paths = record.fetch('require_paths') + gem.executables = record.fetch('executables') + gem.bindir = record.fetch('bindir') + gem.extensions = record.fetch('extensions') + gem.files = record.fetch('files') + gem.metadata = record.fetch('metadata') + record.fetch('dependencies').each { |name, requirement| gem.add_runtime_dependency(name, *requirement.split(', ')) } + end + File.write(path, spec.to_ruby) + end + end + end +end diff --git a/gems/rb-gem-installer/test/runtime_test.rb b/gems/rb-gem-installer/test/runtime_test.rb new file mode 100644 index 0000000..be3411d --- /dev/null +++ b/gems/rb-gem-installer/test/runtime_test.rb @@ -0,0 +1,17 @@ +require 'minitest/autorun' +require_relative '../lib/rb_gem_installer' + +class RuntimeTest < Minitest::Test + def test_reports_selected_ruby_and_available_compatibility_runtime + runtime = RubyButler::GemInstaller::Runtime.read + assert_equal RbConfig.ruby, runtime.fetch(:executable) + assert_equal RUBY_VERSION, runtime.fetch(:version) + assert_equal RUBY_ENGINE, runtime.fetch(:engine) + assert_equal RUBY_ENGINE_VERSION, runtime.fetch(:engine_version) + assert_equal RUBY_PATCHLEVEL, runtime.fetch(:patchlevel) + assert_equal Gem::Platform.local.to_s, runtime.fetch(:platform) + assert_equal Gem.extension_api_version, runtime.fetch(:abi) + assert_equal Gem::VERSION, runtime.fetch(:rubygems) + assert_equal Gem::Specification.find_all_by_name('bundler').map(&:version).max&.to_s, runtime.fetch(:bundler) + end +end diff --git a/gems/rb-gem-installer/test/source_specifications_test.rb b/gems/rb-gem-installer/test/source_specifications_test.rb new file mode 100644 index 0000000..736d4be --- /dev/null +++ b/gems/rb-gem-installer/test/source_specifications_test.rb @@ -0,0 +1,31 @@ +require 'minitest/autorun' +require 'tmpdir' +require 'fileutils' +require_relative '../lib/rb_gem_installer' + +class SourceSpecificationsTest < Minitest::Test + def test_reads_gemspec_with_relative_project_helpers + Dir.mktmpdir do |directory| + fixtures = File.expand_path('../../../crates/rb-packages/tests/fixtures/sources', __dir__) + FileUtils.cp_r(File.join(fixtures, '.'), directory) + records = RubyButler::GemInstaller::SourceSpecifications.read(directory, 'fixture.gemspec') + assert_equal 1, records.size + assert_equal 'fixture.gemspec', records.first.fetch(:gemspec) + specification = Gem::Specification.from_yaml(records.first.fetch(:metadata)) + assert_equal 'fixture', specification.name + assert_equal '1.0.0', specification.version.to_s + assert_includes records.first.fetch(:specification), 'Gem::Specification.new' + end + end + + def test_rejects_gemspec_glob_outside_the_source + Dir.mktmpdir do |directory| + FileUtils.mkdir_p(File.join(directory, 'source')) + File.write(File.join(directory, 'outside.gemspec'), "raise 'must not execute'\n") + error = assert_raises(RuntimeError) do + RubyButler::GemInstaller::SourceSpecifications.read(File.join(directory, 'source'), '../outside.gemspec') + end + assert_equal 'Gemspec escapes source directory', error.message + end + end +end diff --git a/gems/rb-gem-installer/test/specification_test.rb b/gems/rb-gem-installer/test/specification_test.rb new file mode 100644 index 0000000..f79f310 --- /dev/null +++ b/gems/rb-gem-installer/test/specification_test.rb @@ -0,0 +1,26 @@ +require 'minitest/autorun' +require 'tmpdir' +require_relative '../lib/rb_gem_installer' + +class SpecificationTest < Minitest::Test + def test_installed_specification_preserves_runtime_contract + record = { 'name' => 'fixture', 'version' => '1.2.3', 'platform' => 'ruby', + 'ruby' => '>= 3.3', 'rubygems' => '>= 0', 'require_paths' => ['lib'], + 'executables' => ['fixture'], 'bindir' => 'bin', 'extensions' => ['ext/extconf.rb'], + 'files' => ['lib/fixture.rb'], 'metadata' => { 'purpose' => 'test' }, + 'dependencies' => { 'rake' => '>= 13, < 14' } } + Dir.mktmpdir do |directory| + path = File.join(directory, 'fixture.gemspec') + RubyButler::GemInstaller::Specification.write(record, path) + specification = Gem::Specification.load(path) + assert_equal 'fixture', specification.name + assert_equal '1.2.3', specification.version.to_s + assert_equal ['lib'], specification.raw_require_paths + assert_equal ['ext/extconf.rb'], specification.extensions + assert_equal ['fixture'], specification.executables + assert_equal 'bin', specification.bindir + assert_equal({ 'purpose' => 'test' }, specification.metadata) + assert_equal Gem::Requirement.new('>= 13', '< 14'), specification.dependencies.first.requirement + end + end +end diff --git a/gems/rb-gemfile/README.md b/gems/rb-gemfile/README.md new file mode 100644 index 0000000..6d3a268 --- /dev/null +++ b/gems/rb-gemfile/README.md @@ -0,0 +1,5 @@ +# rb-gemfile + +The Ruby evaluator embedded by the Rust `rb-gemfile` crate. Executes trusted Gemfiles and returns dependency declarations as JSON without loading Bundler. Allows one Git selector (`branch`, `tag` or `ref`/`rev`) per dependency. + +Run `ruby test/all_test.rb` or `rake test`. Build with `gem build rb-gemfile.gemspec`. diff --git a/gems/rb-gemfile/Rakefile b/gems/rb-gemfile/Rakefile new file mode 100644 index 0000000..2bce5fd --- /dev/null +++ b/gems/rb-gemfile/Rakefile @@ -0,0 +1,12 @@ +require 'rake/testtask' + +Rake::TestTask.new do |task| + task.libs << 'lib' + task.pattern = 'test/*_test.rb' +end + +task :lint do + sh 'rubocop', '--config', File.expand_path('../.rubocop.yml', __dir__), __dir__ +end + +task default: :test diff --git a/crates/rb-gemfile/assets/evaluate.rb b/gems/rb-gemfile/exe/rb-evaluate-gemfile old mode 100644 new mode 100755 similarity index 77% rename from crates/rb-gemfile/assets/evaluate.rb rename to gems/rb-gemfile/exe/rb-evaluate-gemfile index 2fbd271..178375f --- a/crates/rb-gemfile/assets/evaluate.rb +++ b/gems/rb-gemfile/exe/rb-evaluate-gemfile @@ -1,7 +1,5 @@ -require_relative "gemfile" -require_relative "sources" -require_relative "gemspec" -require_relative "compatibility" +#!/usr/bin/env ruby +require_relative '../lib/rb_gemfile' begin evaluator = RubyButler::Gemfile.new diff --git a/gems/rb-gemfile/lib/rb_gemfile.rb b/gems/rb-gemfile/lib/rb_gemfile.rb new file mode 100644 index 0000000..8e82154 --- /dev/null +++ b/gems/rb-gemfile/lib/rb_gemfile.rb @@ -0,0 +1,5 @@ +require_relative 'rb_gemfile/version' +require_relative 'rb_gemfile/gemfile' +require_relative 'rb_gemfile/sources' +require_relative 'rb_gemfile/gemspec' +require_relative 'rb_gemfile/compatibility' diff --git a/crates/rb-gemfile/assets/compatibility.rb b/gems/rb-gemfile/lib/rb_gemfile/compatibility.rb similarity index 94% rename from crates/rb-gemfile/assets/compatibility.rb rename to gems/rb-gemfile/lib/rb_gemfile/compatibility.rb index 72e5279..c98fbe3 100644 --- a/crates/rb-gemfile/assets/compatibility.rb +++ b/gems/rb-gemfile/lib/rb_gemfile/compatibility.rb @@ -10,6 +10,8 @@ def self.load_gemspec(path, validate = false) end module Plugin + DSL = Class.new + def self.index self end diff --git a/crates/rb-gemfile/assets/gemfile.rb b/gems/rb-gemfile/lib/rb_gemfile/gemfile.rb similarity index 99% rename from crates/rb-gemfile/assets/gemfile.rb rename to gems/rb-gemfile/lib/rb_gemfile/gemfile.rb index 9b3e8d2..3cdb7ca 100644 --- a/crates/rb-gemfile/assets/gemfile.rb +++ b/gems/rb-gemfile/lib/rb_gemfile/gemfile.rb @@ -32,6 +32,7 @@ def dependencies def plugin(name, *requirements, **options) raise ArgumentError, "Plugin options are unsupported" unless options.empty? + requirements = Gem::Requirement.new(requirements.flatten.compact).as_list @bundle[:plugins] ||= [] @bundle[:plugins] << { name: name, version: requirements } @@ -45,6 +46,7 @@ def manifest def eval_gemfile(path, contents = nil) path = File.expand_path(path, @stack.empty? ? Dir.pwd : File.dirname(@stack.last)) raise ArgumentError, "Recursive eval_gemfile: #{path}" if @stack.include?(path) + @inputs << path unless @inputs.include?(path) @stack.push(path) begin @@ -74,10 +76,12 @@ def gem(name, *requirements, **options) @git_sources.each do |key, expansion| if options.key?(key.to_sym) raise ArgumentError, "Conflicting Git sources for #{name}" if options[:git] + expansion = expansion.call(options.delete(key.to_sym)) expansion = { git: expansion } if expansion.is_a?(String) expansion = expansion.transform_keys(&:to_sym) raise ArgumentError, "Conflicting Git shortcut options for #{name}" unless (expansion.keys & options.keys).empty? + options.merge!(expansion) end end @@ -87,6 +91,7 @@ def gem(name, *requirements, **options) raise ArgumentError, "Gem names must be strings" unless name.is_a?(String) raise ArgumentError, "Empty gem name" if name.empty? raise ArgumentError, "Gem names cannot contain whitespace" if name.match?(/\s/) + @declarations.delete(name) unless @dependencies.any? { |dependency| dependency.name == name } options[:require] = false if options.key?(:require) && options[:require].nil? requirements = requirements.flatten.compact.map(&:to_s) @@ -109,6 +114,7 @@ def gem(name, *requirements, **options) end dependency.merge!(options.reject { |key, _| %i[group groups platform platforms install_if].include?(key) }) raise ArgumentError, "Specify ref or rev, not both for #{name}" if dependency[:ref] && dependency[:rev] + dependency[:rev] = dependency.delete(:ref) if dependency.key?(:ref) if dependency[:path] dependency[:path] = File.expand_path(dependency[:path], File.dirname(@stack.last)) @@ -118,6 +124,7 @@ def gem(name, *requirements, **options) if !dependency[:git] && %i[branch tag rev submodules].any? { |key| dependency.key?(key) } raise ArgumentError, "Git options require a Git source for #{name}" end + dependency.merge!(version: requirements, groups: groups.empty? ? ["default"] : groups.map(&:to_s).uniq, platforms: platforms.map(&:to_s).uniq, enabled: (@context[:conditions] + Array(options[:install_if])).all? { |value| condition(value) } && environment_enabled?) @@ -125,10 +132,12 @@ def gem(name, *requirements, **options) unless Gem::Requirement.new(current[:version]) == Gem::Requirement.new(dependency[:version]) raise ArgumentError, "Conflicting declarations for #{name}" end + source_keys = %i[source git path branch tag rev glob submodules] unless source_keys.all? { |key| current[key] == dependency[key] } raise ArgumentError, "Conflicting sources for #{name}" end + primary = current.reject { |key, _| key == :variants } unless primary == dependency || Array(current[:variants]).include?(dependency) (current[:variants] ||= []) << dependency @@ -164,6 +173,7 @@ def environment_enabled? value = @context[:environment] return true unless value return !!ENV[value.to_s] unless value.is_a?(Hash) + value.all? do |name, expected| actual = ENV[name.to_s] actual && (expected.is_a?(Regexp) ? expected.match?(actual) : actual == expected) @@ -176,6 +186,7 @@ def condition(value) def scoped(values) raise ArgumentError, "This declaration requires a block" unless block_given? + previous = @context @context = previous.merge(values) begin diff --git a/crates/rb-gemfile/assets/gemspec.rb b/gems/rb-gemfile/lib/rb_gemfile/gemspec.rb similarity index 99% rename from crates/rb-gemfile/assets/gemspec.rb rename to gems/rb-gemfile/lib/rb_gemfile/gemspec.rb index 80c6ea0..48b10ad 100644 --- a/crates/rb-gemfile/assets/gemspec.rb +++ b/gems/rb-gemfile/lib/rb_gemfile/gemspec.rb @@ -5,9 +5,11 @@ def ruby(*versions, **options) options = options.transform_keys(&:to_sym) unknown = options.keys - %i[engine engine_version file patchlevel] raise ArgumentError, "Unsupported ruby options: #{unknown.join(', ')}" unless unknown.empty? + engine, engine_version, file, patchlevel = options.values_at(:engine, :engine_version, :file, :patchlevel) if file raise ArgumentError, "ruby accepts versions or file, not both" unless versions.empty? + location = File.expand_path(file, File.dirname(@stack.last)) @inputs << location unless @inputs.include?(location) text = read_file(location) @@ -15,12 +17,14 @@ def ruby(*versions, **options) versions = [match ? match.captures.compact.first : text.strip] end raise ArgumentError, "ruby requires a version" if versions.empty? + versions = versions.flatten Gem::Requirement.new(versions) raise ArgumentError, "engine and engine_version must be specified together" if engine.nil? != engine_version.nil? if engine == "ruby" && engine_version && versions != Array(engine_version) raise ArgumentError, "Ruby requirements must match engine_version for MRI" end + @bundle.merge!(ruby: versions.map(&:to_s), engine: engine, engine_version: engine_version, patchlevel: patchlevel&.to_s) end @@ -28,6 +32,7 @@ def gemspec(options = {}, **keywords) options = options.merge(keywords).transform_keys(&:to_sym) unknown = options.keys - %i[path name glob development_group require] raise ArgumentError, "Unsupported gemspec options: #{unknown.join(', ')}" unless unknown.empty? + path = options.fetch(:path, ".") name = options[:name] glob = options[:glob] @@ -37,6 +42,7 @@ def gemspec(options = {}, **keywords) specs.select! { |spec| spec.name == name } if name groups = specs.group_by { |spec| [spec.name, spec.version] } raise ArgumentError, "Expected one gemspec in #{directory}, found #{groups.length}" unless groups.length == 1 + choices = groups.values.first spec = choices.find { |candidate| candidate.platform.to_s == Gem::Platform.local.to_s } || choices.find { |candidate| candidate.platform.to_s == "ruby" } || choices.first @@ -74,6 +80,7 @@ def load_gemspec(path, validate = false) end end raise ArgumentError, "Unable to evaluate #{path}" unless spec.is_a?(Gem::Specification) + spec.loaded_from = path spec.validate if validate spec diff --git a/crates/rb-gemfile/assets/sources.rb b/gems/rb-gemfile/lib/rb_gemfile/sources.rb similarity index 99% rename from crates/rb-gemfile/assets/sources.rb rename to gems/rb-gemfile/lib/rb_gemfile/sources.rb index f47974b..a940d7b 100644 --- a/crates/rb-gemfile/assets/sources.rb +++ b/gems/rb-gemfile/lib/rb_gemfile/sources.rb @@ -4,16 +4,19 @@ def source(url, options = {}, **keywords, &block) options = options.merge(keywords).transform_keys(&:to_sym) unknown = options.keys - [:cooldown] raise ArgumentError, "Unsupported source options: #{unknown.join(', ')}" unless unknown.empty? + cooldown = options[:cooldown] unless cooldown.nil? || (cooldown.is_a?(Integer) && cooldown >= 0) raise ArgumentError, "Expected cooldown to be a non-negative integer" end + if block scoped(source_context.merge(source: url.to_s), &block) else if @bundle[:source] && @bundle[:source] != url.to_s raise ArgumentError, "Multiple global sources are unsupported; use source blocks" end + @bundle[:source] = url.to_s end end @@ -21,18 +24,21 @@ def source(url, options = {}, **keywords, &block) def git(url, **options, &block) unknown = options.keys - %i[branch tag ref rev glob submodules] raise ArgumentError, "Unsupported Git options: #{unknown.join(', ')}" unless unknown.empty? + scoped(source_context.merge(options).merge(git: url.to_s), &block) end def path(directory, **options, &block) unknown = options.keys - [:glob] raise ArgumentError, "Unsupported path options: #{unknown.join(', ')}" unless unknown.empty? + directory = File.expand_path(directory, File.dirname(@stack.last)) scoped(source_context.merge(options).merge(path: directory), &block) end def git_source(name, &block) raise ArgumentError, "git_source requires a block" unless block + @git_sources[name.to_s] = block end @@ -41,6 +47,7 @@ def github(repository, **options, &block) expanded = { git: expanded } if expanded.is_a?(String) expanded = expanded.transform_keys(&:to_sym) raise ArgumentError, "Conflicting Git shortcut options" unless (expanded.keys & options.keys).empty? + url = expanded.delete(:git) git(url, **expanded.merge(options), &block) end diff --git a/gems/rb-gemfile/lib/rb_gemfile/version.rb b/gems/rb-gemfile/lib/rb_gemfile/version.rb new file mode 100644 index 0000000..0fbe01e --- /dev/null +++ b/gems/rb-gemfile/lib/rb_gemfile/version.rb @@ -0,0 +1,5 @@ +module RubyButler + class Gemfile + VERSION = '0.3.1' + end +end diff --git a/gems/rb-gemfile/rb-gemfile.gemspec b/gems/rb-gemfile/rb-gemfile.gemspec new file mode 100644 index 0000000..24fea39 --- /dev/null +++ b/gems/rb-gemfile/rb-gemfile.gemspec @@ -0,0 +1,17 @@ +require_relative 'lib/rb_gemfile/version' + +Gem::Specification.new do |spec| + spec.name = 'rb-gemfile' + spec.version = RubyButler::Gemfile::VERSION + spec.authors = ['RubyElders'] + spec.summary = 'Gemfile evaluation without Bundler' + spec.homepage = 'https://github.com/RubyElders/ruby-butler' + spec.licenses = ['MIT', 'Apache-2.0'] + spec.required_ruby_version = '>= 3.3' + spec.files = Dir.chdir(__dir__) { Dir['lib/**/*.rb', 'exe/*', 'README.md'] } + spec.require_paths = ['lib'] + spec.bindir = 'exe' + spec.executables = ['rb-evaluate-gemfile'] + spec.add_development_dependency 'minitest', '>= 5.20', '< 7' + spec.add_development_dependency 'rake', '>= 13', '< 14' +end diff --git a/gems/rb-gemfile/test/all_test.rb b/gems/rb-gemfile/test/all_test.rb new file mode 100644 index 0000000..e330269 --- /dev/null +++ b/gems/rb-gemfile/test/all_test.rb @@ -0,0 +1,3 @@ +Dir.glob(File.join(__dir__, '*_test.rb')).each do |path| + require path unless path == File.expand_path(__FILE__) +end diff --git a/crates/rb-gemfile/tests/ruby/evaluator_test.rb b/gems/rb-gemfile/test/evaluator_test.rb similarity index 87% rename from crates/rb-gemfile/tests/ruby/evaluator_test.rb rename to gems/rb-gemfile/test/evaluator_test.rb index 43ed825..cef557e 100644 --- a/crates/rb-gemfile/tests/ruby/evaluator_test.rb +++ b/gems/rb-gemfile/test/evaluator_test.rb @@ -3,14 +3,11 @@ require "fileutils" require "open3" require "rbconfig" -require_relative "../../assets/gemfile" -require_relative "../../assets/sources" -require_relative "../../assets/gemspec" -require_relative "../../assets/compatibility" +require_relative '../lib/rb_gemfile' class EvaluatorTest < Minitest::Test - FIXTURES = File.expand_path("../fixtures", __dir__) - ENTRYPOINT = File.expand_path("../../assets/evaluate.rb", __dir__) + FIXTURES = File.expand_path("../../../crates/rb-gemfile/tests/fixtures", __dir__) + ENTRYPOINT = File.expand_path("../exe/rb-evaluate-gemfile", __dir__) def setup @evaluator = RubyButler::Gemfile.new @@ -35,6 +32,27 @@ def test_plugin_index_does_not_load_plugins refute Gem.loaded_specs.key?("bundler-inject") end + def test_rejects_combined_git_selectors + [{ branch: "main", tag: "v1" }, { branch: "main", ref: "abc" }, { tag: "v1", ref: "abc" }].each do |options| + error = assert_raises(ArgumentError) { @evaluator.gem("fixture", git: "https://example.test/fixture.git", **options) } + assert_match "Conflicting Git selectors", error.message + end + end + + def test_git_options_require_a_git_source + [{ branch: "main" }, { ref: "abc" }, { tag: "v1" }, { submodules: true }].each do |options| + error = assert_raises(ArgumentError) { @evaluator.gem("fixture", **options) } + assert_match "Git options require", error.message + end + end + + def test_plugin_dsl_guard_does_not_skip_normal_dependency_evaluation + @evaluator.eval_gemfile(File.join(FIXTURES, "ruby/plugin_dsl_guard.rb")) + assert_equal ["~> 3.0"], @evaluator.manifest.fetch(:dependencies).fetch("fixture").fetch(:version) + refute @evaluator.is_a?(Bundler::Plugin::DSL) + refute Gem.loaded_specs.key?("bundler") + end + def test_groups_sources_and_conditions @evaluator.eval_gemfile(File.join(FIXTURES, "format/evaluate-1.rb")) dependencies = @evaluator.manifest.fetch(:dependencies) diff --git a/gems/rb-gemfile/test/package_test.rb b/gems/rb-gemfile/test/package_test.rb new file mode 100644 index 0000000..c669b33 --- /dev/null +++ b/gems/rb-gemfile/test/package_test.rb @@ -0,0 +1,35 @@ +require 'minitest/autorun' +require 'tmpdir' +require 'rubygems/package' +require 'open3' +require 'json' +require 'rbconfig' +require_relative '../lib/rb_gemfile/version' + +class PackageTest < Minitest::Test + def test_packaged_evaluator_runs_without_repository_files + root = File.expand_path('..', __dir__) + spec = Gem::Specification.load(File.join(root, 'rb-gemfile.gemspec')) + assert_equal RubyButler::Gemfile::VERSION, spec.version.to_s + assert_empty spec.runtime_dependencies + assert_equal ['rb-evaluate-gemfile'], spec.executables + Dir.mktmpdir do |directory| + archive = File.join(directory, 'rb-gemfile.gem') + capture_io { Dir.chdir(root) { Gem::Package.build(spec, false, true, archive) } } + destination = File.join(directory, 'extracted') + Gem::Package.new(archive).extract_files(destination) + files = Dir.chdir(root) { Dir['lib/**/*.rb', 'exe/*'].sort } + assert_equal files, Dir.chdir(destination) { Dir['lib/**/*.rb', 'exe/*'].sort } + files.each do |path| + assert_equal File.binread(File.join(root, path)), File.binread(File.join(destination, path)), path + end + gemfile = File.join(directory, 'Gemfile') + result = File.join(directory, 'manifest.json') + File.write(gemfile, "gem 'rake', '~> 13.0'\n") + stdout, stderr, status = Open3.capture3(RbConfig.ruby, File.join(destination, 'exe/rb-evaluate-gemfile'), gemfile, result) + assert status.success?, stderr + assert_empty stdout + assert_equal ['~> 13.0'], JSON.parse(File.read(result)).fetch('manifest').fetch('dependencies').fetch('rake').fetch('version') + end + end +end diff --git a/gems/rb-microgems/lib/microgems.rb b/gems/rb-microgems/lib/microgems.rb index 6289a4a..7d2cbb8 100644 --- a/gems/rb-microgems/lib/microgems.rb +++ b/gems/rb-microgems/lib/microgems.rb @@ -93,6 +93,13 @@ def default_exec_format = '%s' def suffixes = ['', '.rb', ".#{RbConfig::CONFIG.fetch('DLEXT')}"] def binary_mode = 'rb' + # Datadog needs Gem.dir to calculate relative native library paths. + def dir + home = ENV['GEM_HOME'] || File.join(RbConfig::CONFIG.fetch('rubylibprefix'), 'gems', RbConfig::CONFIG.fetch('ruby_version')) + home = home.tr(File::ALT_SEPARATOR, File::SEPARATOR) if File::ALT_SEPARATOR + File.directory?(home) ? File.realpath(home) : home + end + def bin_path(name, executable, *requirements) spec = Specification.find_by_name(name, *requirements) raise MicroGems::Exception, diff --git a/gems/rb-microgems/lib/microgems/compatibility.rb b/gems/rb-microgems/lib/microgems/compatibility.rb index 12711fb..00642f5 100644 --- a/gems/rb-microgems/lib/microgems/compatibility.rb +++ b/gems/rb-microgems/lib/microgems/compatibility.rb @@ -14,6 +14,12 @@ def gem(name, *requirements) = MicroGems.activate(name, *requirements) MicroGems.configure(JSON.parse(File.read(ENV.fetch('RB_MICROGEMS_MANIFEST')))) end +if ENV['RB_MICROGEMS_LOAD_PATHS'] + JSON.parse(ENV.fetch('RB_MICROGEMS_LOAD_PATHS')).reverse_each do |path| + $LOAD_PATH.unshift(path) unless $LOAD_PATH.include?(path) + end +end + if ENV['RB_MICROGEMS_CURRENT'] name, version = JSON.parse(ENV.fetch('RB_MICROGEMS_CURRENT')) MicroGems.activate(name, "= #{version}") diff --git a/gems/rb-microgems/test/compatibility_test.rb b/gems/rb-microgems/test/compatibility_test.rb index 33a626b..09f9693 100644 --- a/gems/rb-microgems/test/compatibility_test.rb +++ b/gems/rb-microgems/test/compatibility_test.rb @@ -24,6 +24,17 @@ def test_build_api_contracts assert_includes output, 'contracts passed' end + def test_gem_directory_matches_rubygems_with_default_and_custom_homes + [nil, @root, File.join(@root, '.'), File.join(@root, 'missing'), 'relative-gem-home'].each do |home| + environment = { 'GEM_HOME' => home } + expected, error, status = Open3.capture3(environment, RbConfig.ruby, '-rrubygems', '-e', 'puts Gem.dir') + assert status.success?, error + output, error, status = ruby_fixture('gem_directory.rb', environment: environment, compatibility: true) + assert status.success?, error + assert_equal expected, output + end + end + def test_core_does_not_install_the_compatibility_namespace output, error, status = ruby_fixture('namespaces.rb', environment: @environment, compatibility: true) assert status.success?, error diff --git a/gems/rb-microgems/test/fixtures/gem_directory.rb b/gems/rb-microgems/test/fixtures/gem_directory.rb new file mode 100644 index 0000000..68916ab --- /dev/null +++ b/gems/rb-microgems/test/fixtures/gem_directory.rb @@ -0,0 +1,5 @@ +require 'microgems/compatibility' + +raise 'Full RubyGems loaded' if $LOADED_FEATURES.any? { |file| file.include?('rubygems/core_ext') } + +puts Gem.dir