From 953c0f53fb5443263a7ed5d50478729077aba57e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20=C5=A0im=C3=A1nek?= Date: Sun, 11 Oct 2026 17:08:53 +0200 Subject: [PATCH] Add standalone `rb-gem-installer` Compile native extensions with explicit dependencies supplied through `rb-microgems` and publish successful installations without replacing existing directories. Keep the Ruby helpers in root gems with independent lint and test jobs. --- .github/workflows/ci.yml | 50 +- Cargo.lock | 28 + crates/rb-gem-archive/Cargo.toml | 1 + crates/rb-gem-archive/src/checksums.rs | 57 +- crates/rb-gem-archive/tests/compatibility.rs | 38 + crates/rb-gem-installer/Cargo.toml | 26 + crates/rb-gem-installer/README.md | 32 + crates/rb-gem-installer/examples/compile.rs | 44 ++ crates/rb-gem-installer/examples/install.rs | 12 + .../rb-gem-installer/src/build_diagnostics.rs | 20 + crates/rb-gem-installer/src/build_event.rs | 35 + crates/rb-gem-installer/src/build_options.rs | 33 + crates/rb-gem-installer/src/build_outputs.rs | 88 +++ crates/rb-gem-installer/src/build_process.rs | 215 ++++++ crates/rb-gem-installer/src/build_report.rs | 10 + crates/rb-gem-installer/src/cargo_builder.rs | 121 +++ crates/rb-gem-installer/src/cmake_builder.rs | 43 ++ crates/rb-gem-installer/src/constants.rs | 13 + crates/rb-gem-installer/src/extract.rs | 106 +++ crates/rb-gem-installer/src/install.rs | 61 ++ crates/rb-gem-installer/src/lib.rs | 26 + crates/rb-gem-installer/src/make_builder.rs | 93 +++ crates/rb-gem-installer/src/native_builder.rs | 273 +++++++ crates/rb-gem-installer/src/payload_links.rs | 89 +++ crates/rb-gem-installer/src/payload_path.rs | 74 ++ crates/rb-gem-installer/src/publish.rs | 72 ++ crates/rb-gem-installer/src/rake_builder.rs | 34 + crates/rb-gem-installer/src/ruby_config.rs | 68 ++ crates/rb-gem-installer/tests/building.rs | 711 ++++++++++++++++++ crates/rb-gem-installer/tests/corpus.rs | 39 + .../tests/fixtures/database_cleaner-2.0.2.gem | Bin 0 -> 4608 bytes .../fixtures/database_cleaner-2.0.2.json | 8 + .../tests/fixtures/native/c/butler_native.c | 10 + .../tests/fixtures/native/c/extconf.rb | 8 + .../tests/fixtures/native/cargo/Cargo.lock | 5 + .../tests/fixtures/native/cargo/Cargo.toml | 8 + .../tests/fixtures/native/cargo/lib.rs | 10 + .../fixtures/native/cmake/CMakeLists.txt | 11 + .../fixtures/native/compatibility/extconf.rb | 9 + .../tests/fixtures/native/configure/configure | 10 + .../tests/fixtures/native/cpp/butler_cpp.cpp | 14 + .../tests/fixtures/native/cpp/extconf.rb | 3 + .../fixtures/native/dependent/extconf.rb | 6 + .../tests/fixtures/native/failing/extconf.rb | 1 + .../tests/fixtures/native/mkrf/mkrf_conf.rb | 2 + .../tests/fixtures/native/rake/Rakefile | 6 + .../tests/fixtures/native/runtime/dinner.rb | 7 + crates/rb-gem-installer/tests/installation.rs | 363 +++++++++ crates/rb-gem-installer/tests/oracle.rs | 39 + crates/rb-microgems/Cargo.toml | 11 + crates/rb-microgems/README.md | 11 + crates/rb-microgems/src/environment.rs | 128 ++++ crates/rb-microgems/src/installed_gem.rs | 21 + crates/rb-microgems/src/lib.rs | 7 + crates/rb-microgems/tests/compatibility.rs | 231 ++++++ docker/native/Dockerfile | 49 ++ docker/native/Dockerfile.dockerignore | 8 + docker/native/README.md | 56 ++ docker/native/heavy.rb | 39 + docker/native/smoke.rb | 18 + gems/.rubocop.yml | 21 + gems/Gemfile | 4 + gems/Gemfile.lock | 45 ++ gems/README.md | 5 + gems/Rakefile | 10 + gems/rb-gem-installer/README.md | 5 + gems/rb-gem-installer/Rakefile | 12 + gems/rb-gem-installer/exe/rb-ruby-config | 5 + gems/rb-gem-installer/lib/rb_gem_installer.rb | 7 + .../lib/rb_gem_installer/ruby_config.rb | 18 + .../rb-gem-installer/rb-gem-installer.gemspec | 17 + gems/rb-gem-installer/test/all_test.rb | 3 + gems/rb-gem-installer/test/package_test.rb | 25 + .../rb-gem-installer/test/ruby_config_test.rb | 53 ++ gems/rb-microgems/README.md | 5 + gems/rb-microgems/Rakefile | 12 + gems/rb-microgems/lib/microgems.rb | 104 +++ gems/rb-microgems/lib/microgems/command.rb | 5 + .../lib/microgems/compatibility.rb | 20 + .../rb-microgems/lib/microgems/ext/builder.rb | 40 + gems/rb-microgems/lib/microgems/package.rb | 159 ++++ gems/rb-microgems/lib/microgems/platform.rb | 67 ++ .../rb-microgems/lib/microgems/requirement.rb | 45 ++ .../lib/microgems/specification.rb | 42 ++ gems/rb-microgems/lib/microgems/version.rb | 81 ++ gems/rb-microgems/lib/rubygems.rb | 1 + gems/rb-microgems/lib/rubygems/command.rb | 2 + gems/rb-microgems/lib/rubygems/ext.rb | 1 + gems/rb-microgems/lib/rubygems/ext/builder.rb | 2 + gems/rb-microgems/lib/rubygems/package.rb | 2 + gems/rb-microgems/lib/rubygems/platform.rb | 2 + gems/rb-microgems/lib/rubygems/requirement.rb | 2 + .../lib/rubygems/specification.rb | 2 + gems/rb-microgems/lib/rubygems/version.rb | 2 + gems/rb-microgems/rb-microgems.gemspec | 15 + gems/rb-microgems/test/all_test.rb | 3 + gems/rb-microgems/test/compatibility_test.rb | 46 ++ gems/rb-microgems/test/fixtures/contracts.rb | 61 ++ .../test/fixtures/existing_rubygems.rb | 14 + gems/rb-microgems/test/fixtures/namespaces.rb | 25 + gems/rb-microgems/test/fixtures/shims.rb | 10 + .../test/fixtures/version_oracle.rb | 24 + gems/rb-microgems/test/package_test.rb | 22 + gems/rb-microgems/test/test_helper.rb | 14 + gems/rb-microgems/test/value_test.rb | 51 ++ 105 files changed, 4701 insertions(+), 21 deletions(-) create mode 100644 crates/rb-gem-installer/Cargo.toml create mode 100644 crates/rb-gem-installer/README.md create mode 100644 crates/rb-gem-installer/examples/compile.rs create mode 100644 crates/rb-gem-installer/examples/install.rs create mode 100644 crates/rb-gem-installer/src/build_diagnostics.rs create mode 100644 crates/rb-gem-installer/src/build_event.rs create mode 100644 crates/rb-gem-installer/src/build_options.rs create mode 100644 crates/rb-gem-installer/src/build_outputs.rs create mode 100644 crates/rb-gem-installer/src/build_process.rs create mode 100644 crates/rb-gem-installer/src/build_report.rs create mode 100644 crates/rb-gem-installer/src/cargo_builder.rs create mode 100644 crates/rb-gem-installer/src/cmake_builder.rs create mode 100644 crates/rb-gem-installer/src/constants.rs create mode 100644 crates/rb-gem-installer/src/extract.rs create mode 100644 crates/rb-gem-installer/src/install.rs create mode 100644 crates/rb-gem-installer/src/lib.rs create mode 100644 crates/rb-gem-installer/src/make_builder.rs create mode 100644 crates/rb-gem-installer/src/native_builder.rs create mode 100644 crates/rb-gem-installer/src/payload_links.rs create mode 100644 crates/rb-gem-installer/src/payload_path.rs create mode 100644 crates/rb-gem-installer/src/publish.rs create mode 100644 crates/rb-gem-installer/src/rake_builder.rs create mode 100644 crates/rb-gem-installer/src/ruby_config.rs create mode 100644 crates/rb-gem-installer/tests/building.rs create mode 100644 crates/rb-gem-installer/tests/corpus.rs create mode 100644 crates/rb-gem-installer/tests/fixtures/database_cleaner-2.0.2.gem create mode 100644 crates/rb-gem-installer/tests/fixtures/database_cleaner-2.0.2.json create mode 100644 crates/rb-gem-installer/tests/fixtures/native/c/butler_native.c create mode 100644 crates/rb-gem-installer/tests/fixtures/native/c/extconf.rb create mode 100644 crates/rb-gem-installer/tests/fixtures/native/cargo/Cargo.lock create mode 100644 crates/rb-gem-installer/tests/fixtures/native/cargo/Cargo.toml create mode 100644 crates/rb-gem-installer/tests/fixtures/native/cargo/lib.rs create mode 100644 crates/rb-gem-installer/tests/fixtures/native/cmake/CMakeLists.txt create mode 100644 crates/rb-gem-installer/tests/fixtures/native/compatibility/extconf.rb create mode 100644 crates/rb-gem-installer/tests/fixtures/native/configure/configure create mode 100644 crates/rb-gem-installer/tests/fixtures/native/cpp/butler_cpp.cpp create mode 100644 crates/rb-gem-installer/tests/fixtures/native/cpp/extconf.rb create mode 100644 crates/rb-gem-installer/tests/fixtures/native/dependent/extconf.rb create mode 100644 crates/rb-gem-installer/tests/fixtures/native/failing/extconf.rb create mode 100644 crates/rb-gem-installer/tests/fixtures/native/mkrf/mkrf_conf.rb create mode 100644 crates/rb-gem-installer/tests/fixtures/native/rake/Rakefile create mode 100644 crates/rb-gem-installer/tests/fixtures/native/runtime/dinner.rb create mode 100644 crates/rb-gem-installer/tests/installation.rs create mode 100644 crates/rb-gem-installer/tests/oracle.rs create mode 100644 crates/rb-microgems/Cargo.toml create mode 100644 crates/rb-microgems/README.md create mode 100644 crates/rb-microgems/src/environment.rs create mode 100644 crates/rb-microgems/src/installed_gem.rs create mode 100644 crates/rb-microgems/src/lib.rs create mode 100644 crates/rb-microgems/tests/compatibility.rs create mode 100644 docker/native/Dockerfile create mode 100644 docker/native/Dockerfile.dockerignore create mode 100644 docker/native/README.md create mode 100644 docker/native/heavy.rb create mode 100644 docker/native/smoke.rb create mode 100644 gems/.rubocop.yml create mode 100644 gems/Gemfile create mode 100644 gems/Gemfile.lock create mode 100644 gems/README.md create mode 100644 gems/Rakefile create mode 100644 gems/rb-gem-installer/README.md create mode 100644 gems/rb-gem-installer/Rakefile create mode 100755 gems/rb-gem-installer/exe/rb-ruby-config create mode 100644 gems/rb-gem-installer/lib/rb_gem_installer.rb create mode 100644 gems/rb-gem-installer/lib/rb_gem_installer/ruby_config.rb create mode 100644 gems/rb-gem-installer/rb-gem-installer.gemspec create mode 100644 gems/rb-gem-installer/test/all_test.rb create mode 100644 gems/rb-gem-installer/test/package_test.rb create mode 100644 gems/rb-gem-installer/test/ruby_config_test.rb create mode 100644 gems/rb-microgems/README.md create mode 100644 gems/rb-microgems/Rakefile create mode 100644 gems/rb-microgems/lib/microgems.rb create mode 100644 gems/rb-microgems/lib/microgems/command.rb create mode 100644 gems/rb-microgems/lib/microgems/compatibility.rb create mode 100644 gems/rb-microgems/lib/microgems/ext/builder.rb create mode 100644 gems/rb-microgems/lib/microgems/package.rb create mode 100644 gems/rb-microgems/lib/microgems/platform.rb create mode 100644 gems/rb-microgems/lib/microgems/requirement.rb create mode 100644 gems/rb-microgems/lib/microgems/specification.rb create mode 100644 gems/rb-microgems/lib/microgems/version.rb create mode 100644 gems/rb-microgems/lib/rubygems.rb create mode 100644 gems/rb-microgems/lib/rubygems/command.rb create mode 100644 gems/rb-microgems/lib/rubygems/ext.rb create mode 100644 gems/rb-microgems/lib/rubygems/ext/builder.rb create mode 100644 gems/rb-microgems/lib/rubygems/package.rb create mode 100644 gems/rb-microgems/lib/rubygems/platform.rb create mode 100644 gems/rb-microgems/lib/rubygems/requirement.rb create mode 100644 gems/rb-microgems/lib/rubygems/specification.rb create mode 100644 gems/rb-microgems/lib/rubygems/version.rb create mode 100644 gems/rb-microgems/rb-microgems.gemspec create mode 100644 gems/rb-microgems/test/all_test.rb create mode 100644 gems/rb-microgems/test/compatibility_test.rb create mode 100644 gems/rb-microgems/test/fixtures/contracts.rb create mode 100644 gems/rb-microgems/test/fixtures/existing_rubygems.rb create mode 100644 gems/rb-microgems/test/fixtures/namespaces.rb create mode 100644 gems/rb-microgems/test/fixtures/shims.rb create mode 100644 gems/rb-microgems/test/fixtures/version_oracle.rb create mode 100644 gems/rb-microgems/test/package_test.rb create mode 100644 gems/rb-microgems/test/test_helper.rb create mode 100644 gems/rb-microgems/test/value_test.rb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 239b1fe..926439e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,8 +91,24 @@ jobs: - name: Run yamllint run: make lint-yaml - test-gemfile-ruby: - name: Gemfile Ruby Tests (${{ matrix.ruby }}, ${{ matrix.os }}) + lint-ruby: + name: Lint (Ruby) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ env.RUBY_TEST_VERSION }} + bundler-cache: true + working-directory: gems + + - name: Lint Ruby gems + working-directory: gems + run: bundle exec rake lint + + test-ruby: + name: Ruby Tests (${{ matrix.ruby }}, ${{ matrix.os }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -113,6 +129,36 @@ jobs: - name: Test the Ruby evaluator run: ruby crates/rb-gemfile/tests/ruby/evaluator_test.rb + - name: Test MicroGems + run: ruby gems/rb-microgems/test/all_test.rb + + - name: Test the installer Ruby helper + run: ruby gems/rb-gem-installer/test/all_test.rb + + test-native-builds: + name: Native Build Tests (Ruby ${{ matrix.ruby }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + ruby: ['3.3', '3.4', '4.0'] + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + run: rustup install + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler: none + + - name: Test gem installation and build compatibility + run: cargo test -p rb-gem-installer -p rb-microgems + + - name: Build installer examples + run: cargo build -p rb-gem-installer --examples + test-rust: name: Rust Tests runs-on: ${{ matrix.os }} diff --git a/Cargo.lock b/Cargo.lock index a9f21c2..11f9092 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1305,6 +1305,7 @@ name = "rb-gem-archive" version = "0.3.1" dependencies = [ "anyhow", + "base64 0.22.1", "flate2", "rb-gem-types", "serde_json", @@ -1314,6 +1315,24 @@ dependencies = [ "tar", ] +[[package]] +name = "rb-gem-installer" +version = "0.3.1" +dependencies = [ + "anyhow", + "flate2", + "rb-gem-archive", + "rb-gem-types", + "rb-microgems", + "rustix 1.0.8", + "serde", + "serde_json", + "sha2", + "tar", + "tempfile", + "windows-sys 0.61.2", +] + [[package]] name = "rb-gem-types" version = "0.3.1" @@ -1345,6 +1364,15 @@ dependencies = [ "rb-gem-types", ] +[[package]] +name = "rb-microgems" +version = "0.3.1" +dependencies = [ + "serde", + "serde_json", + "tempfile", +] + [[package]] name = "rb-progress" version = "0.3.1" diff --git a/crates/rb-gem-archive/Cargo.toml b/crates/rb-gem-archive/Cargo.toml index ccd744a..d405d4a 100644 --- a/crates/rb-gem-archive/Cargo.toml +++ b/crates/rb-gem-archive/Cargo.toml @@ -7,6 +7,7 @@ description = "Standalone Ruby gem archive metadata reading" [dependencies] anyhow = "1.0" +base64 = "0.22" flate2 = "1.1" rb-gem-types = { path = "../rb-gem-types" } serde_yaml_ng = "0.10" diff --git a/crates/rb-gem-archive/src/checksums.rs b/crates/rb-gem-archive/src/checksums.rs index 93b6a11..e2c49da 100644 --- a/crates/rb-gem-archive/src/checksums.rs +++ b/crates/rb-gem-archive/src/checksums.rs @@ -3,9 +3,10 @@ use crate::{ metadata, }; use anyhow::{Context, Result, ensure}; +use base64::{Engine, engine::general_purpose::STANDARD}; use sha1::Sha1; use sha2::{Digest, Sha256, Sha512}; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; #[derive(Default)] pub(crate) struct Digests { @@ -40,34 +41,29 @@ pub(crate) fn verify( .as_mapping() .context("Invalid gem checksum map")?; ensure!(!algorithms.is_empty(), "Empty gem checksum map"); + let mut seen_algorithms = BTreeSet::new(); for (algorithm, values) in algorithms { - let algorithm = metadata::untag(algorithm) - .as_str() - .context("Invalid checksum algorithm")?; + let algorithm = text(algorithm)?; ensure!( - matches!(algorithm, "SHA1" | "SHA256" | "SHA512"), + seen_algorithms.insert(algorithm.clone()), + "Duplicate checksum algorithm" + ); + ensure!( + matches!(algorithm.as_str(), "SHA1" | "SHA256" | "SHA512"), "Unsupported checksum algorithm: {algorithm}" ); let values = metadata::untag(values) .as_mapping() .context("Invalid member checksum map")?; - for name in [METADATA_MEMBER, DATA_MEMBER] { - ensure!( - values.contains_key(serde_yaml_ng::Value::String(name.into())), - "Missing member checksum" - ); - } + let mut names = BTreeSet::new(); for (name, expected) in values { - let name = metadata::untag(name) - .as_str() - .context("Invalid checksum member")?; - let expected = metadata::untag(expected) - .as_str() - .context("Invalid member checksum")?; + let name = text(name)?; + ensure!(names.insert(name.clone()), "Duplicate checksum member"); + let expected = text(expected)?; let actual = hashes - .get(name) + .get(&name) .context("Checksum references missing member")? - .get(algorithm) + .get(algorithm.as_str()) .unwrap(); ensure!( expected.len() == actual.len() && expected.bytes().all(|c| c.is_ascii_hexdigit()), @@ -78,6 +74,29 @@ pub(crate) fn verify( "Gem member checksum mismatch: {name} ({algorithm})" ); } + ensure!( + names.contains(METADATA_MEMBER) && names.contains(DATA_MEMBER), + "Missing member checksum" + ); } Ok(()) } + +fn text(value: &serde_yaml_ng::Value) -> Result { + let content = metadata::untag(value) + .as_str() + .context("Invalid checksum text")?; + if let serde_yaml_ng::Value::Tagged(tagged) = value + && (tagged.tag == "binary" || tagged.tag == "tag:yaml.org,2002:binary") + { + let encoded: String = content + .chars() + .filter(|c| !c.is_ascii_whitespace()) + .collect(); + let bytes = STANDARD + .decode(encoded) + .context("Invalid binary checksum encoding")?; + return String::from_utf8(bytes).context("Non-UTF8 checksum text"); + } + Ok(content.to_owned()) +} diff --git a/crates/rb-gem-archive/tests/compatibility.rs b/crates/rb-gem-archive/tests/compatibility.rs index 5f6bdef..8dfc538 100644 --- a/crates/rb-gem-archive/tests/compatibility.rs +++ b/crates/rb-gem-archive/tests/compatibility.rs @@ -94,3 +94,41 @@ fn signature_members_must_stay_bounded_and_belong_to_known_members() { members.push(("other.sig", b"signature bytes".to_vec())); assert!(read(common::container(&members).as_slice()).is_err()); } + +#[test] +fn binary_tagged_legacy_checksums_are_decoded_and_verified() { + use base64::{Engine, engine::general_purpose::STANDARD}; + let mut members = common::members(include_bytes!("fixtures/metadata.yml")); + let checksum = format!( + "!binary U0hBMjU2:\n metadata.gz: !binary {}\n data.tar.gz: !binary {}\n", + STANDARD.encode(format!("{:x}", sha2::Sha256::digest(&members[0].1))), + STANDARD.encode(format!("{:x}", sha2::Sha256::digest(&members[1].1))) + ); + members[2].1 = common::gzip(checksum.as_bytes()); + assert_eq!( + read(common::container(&members).as_slice()) + .unwrap() + .id + .name, + "dinner" + ); + members[1].1.push(0); + assert!(read(common::container(&members).as_slice()).is_err()); +} + +#[test] +fn binary_checksum_names_cannot_hide_duplicate_entries() { + let mut members = common::members(include_bytes!("fixtures/metadata.yml")); + let metadata = format!("{:x}", sha2::Sha256::digest(&members[0].1)); + let data = format!("{:x}", sha2::Sha256::digest(&members[1].1)); + let values = format!(" metadata.gz: {metadata}\n data.tar.gz: {data}\n"); + members[2].1 = common::gzip(format!("SHA256:\n{values}!binary U0hBMjU2:\n{values}").as_bytes()); + let error = read(common::container(&members).as_slice()).unwrap_err(); + assert!(error.to_string().contains("Duplicate checksum algorithm")); + + members[2].1 = common::gzip( + format!("SHA256:\n{values} !binary bWV0YWRhdGEuZ3o=: {metadata}\n").as_bytes(), + ); + let error = read(common::container(&members).as_slice()).unwrap_err(); + assert!(error.to_string().contains("Duplicate checksum member")); +} diff --git a/crates/rb-gem-installer/Cargo.toml b/crates/rb-gem-installer/Cargo.toml new file mode 100644 index 0000000..4b182a5 --- /dev/null +++ b/crates/rb-gem-installer/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "rb-gem-installer" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Standalone staged Ruby gem payload installation" + +[dependencies] +anyhow = "1.0" +flate2 = "1.1" +rb-gem-archive = { path = "../rb-gem-archive" } +rb-gem-types = { path = "../rb-gem-types" } +rb-microgems = { path = "../rb-microgems" } +serde = { version = "1", features = ["derive"] } +serde_json = "1.0" +tar = "0.4" +tempfile = "3" + +[target.'cfg(unix)'.dependencies] +rustix = { version = "1", features = ["fs", "process"] } + +[target.'cfg(windows)'.dependencies] +windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } + +[dev-dependencies] +sha2 = "0.10" diff --git a/crates/rb-gem-installer/README.md b/crates/rb-gem-installer/README.md new file mode 100644 index 0000000..b16122a --- /dev/null +++ b/crates/rb-gem-installer/README.md @@ -0,0 +1,32 @@ +# rb-gem-installer + +Verifies a `.gem` archive and extracts its payload into a new directory. The parent directory must exist and be controlled by the caller; installation does not replace existing gems. Extraction happens in a private staging directory and failed installs are removed. + +```rust,no_run +let archive = std::fs::File::open("rake-13.2.1.gem")?; +let metadata = rb_gem_installer::install(archive, "gems/rake-13.2.1")?; +println!("Installed {}", metadata.id); +# Ok::<(), anyhow::Error>(()) +``` + +Regular files and directories are supported. Relative symlinks to payload files and directories are supported, including chains. Links outside the installation, cycles, hard links and special files are rejected. Identical repeated files are deduplicated; conflicting paths are rejected. Archive size, expanded size and entry count are bounded. File permissions are preserved on Unix subject to the process umask, without special permission bits. Signature trust follows `rb-gem-archive` and is not verified. + +Native compilation is opt-in. `install_with_builder` builds in staging and publishes only after every declared extension succeeds. `NativeBuilder::build` can also build an already extracted directory; callers manage staging when using it directly. + +```rust,no_run +use rb_gem_installer::{BuildOptions, NativeBuilder, install_with_builder}; + +let builder = NativeBuilder::new(BuildOptions::new("ruby"))?; +let archive = std::fs::File::open("json.gem")?; +let (metadata, report) = install_with_builder(archive, "gems/json", &builder, |_| {})?; +println!("Built {} outputs for {}", report.artifacts.len(), metadata.id); +# Ok::<(), anyhow::Error>(()) +``` + +Builders support extconf/make, rake and mkrf, configure, CMake and Cargo. Ruby configuration comes from the selected interpreter; its development headers and the required build tools must be available. Build arguments, dependency load paths, environment, tool paths, job count and per-command timeout are caller-controlled. Structured events stream stdout, stderr and command stages without a reporting dependency. Build logs retained in errors are bounded; callbacks receive the full stream. + +Collected outputs are stored under `.rb-native//` and copied into the first require path. Reports also include native libraries written directly into gem require paths or extension directories. This crate does not resolve build dependencies, generate executable wrappers or activate gems for application runtime. Build subprocesses preload `rb-microgems` for minimal RubyGems API compatibility. Supply exact installed build dependencies through `BuildOptions::dependencies`; missing gems and unsupported APIs fail explicitly. Bundler and full RubyGems are not loaded. + +The regular tests include a public gem checked against RubyGems extraction. To run the larger local corpus, set `RB_GEM_CORPUS` to its absolute `gems` directory and run `cargo test -p rb-gem-installer --test corpus -- --ignored --nocapture`. + +The Ruby configuration helper is a standalone gem in `gems/rb-gem-installer`. Its files are embedded in this crate and materialized for the selected interpreter without a gem installation. Run its Ruby tests with `ruby gems/rb-gem-installer/test/all_test.rb` from the repository root. diff --git a/crates/rb-gem-installer/examples/compile.rs b/crates/rb-gem-installer/examples/compile.rs new file mode 100644 index 0000000..9e606a7 --- /dev/null +++ b/crates/rb-gem-installer/examples/compile.rs @@ -0,0 +1,44 @@ +use rb_gem_installer::{BuildEvent, BuildOptions, NativeBuilder}; +use std::{io::Write, time::Duration}; + +fn main() -> anyhow::Result<()> { + let mut arguments = std::env::args_os().skip(1); + let archive = arguments + .next() + .ok_or_else(|| anyhow::anyhow!("Expected archive path"))?; + let destination = arguments + .next() + .ok_or_else(|| anyhow::anyhow!("Expected destination path"))?; + let ruby = arguments.next().unwrap_or_else(|| "ruby".into()); + let mut options = BuildOptions::new(ruby); + options.arguments = arguments.collect(); + if let Ok(seconds) = std::env::var("RB_GEM_BUILD_TIMEOUT") { + options.timeout = Duration::from_secs(seconds.parse()?); + } + if let Ok(jobs) = std::env::var("RB_GEM_BUILD_JOBS") { + options.jobs = jobs.parse()?; + } + if let Some(paths) = std::env::var_os("RB_GEM_BUILD_LOAD_PATHS") { + options.load_paths = std::env::split_paths(&paths).collect(); + } + if let Some(path) = std::env::var_os("RB_GEM_BUILD_DEPENDENCIES") { + options.dependencies = serde_json::from_slice(&std::fs::read(path)?)?; + } + let builder = NativeBuilder::new(options)?; + let (metadata, report) = rb_gem_installer::install_with_builder( + std::fs::File::open(archive)?, + destination, + &builder, + |event| { + if let BuildEvent::Output { bytes, .. } = event { + let _ = std::io::stderr().write_all(&bytes); + } + }, + )?; + println!( + "Installed {} with {} build outputs", + metadata.id, + report.artifacts.len() + ); + Ok(()) +} diff --git a/crates/rb-gem-installer/examples/install.rs b/crates/rb-gem-installer/examples/install.rs new file mode 100644 index 0000000..1c60dcf --- /dev/null +++ b/crates/rb-gem-installer/examples/install.rs @@ -0,0 +1,12 @@ +fn main() -> anyhow::Result<()> { + let mut args = std::env::args_os().skip(1); + let archive = args + .next() + .ok_or_else(|| anyhow::anyhow!("Expected archive path"))?; + let destination = args + .next() + .ok_or_else(|| anyhow::anyhow!("Expected destination path"))?; + let metadata = rb_gem_installer::install(std::fs::File::open(archive)?, destination)?; + println!("Installed {}", metadata.id); + Ok(()) +} diff --git a/crates/rb-gem-installer/src/build_diagnostics.rs b/crates/rb-gem-installer/src/build_diagnostics.rs new file mode 100644 index 0000000..4077c02 --- /dev/null +++ b/crates/rb-gem-installer/src/build_diagnostics.rs @@ -0,0 +1,20 @@ +use crate::constants::MAX_BUILD_OUTPUT_BYTES; +use std::{ + fs::File, + io::{Read, Seek, SeekFrom}, + path::Path, +}; + +pub(crate) fn mkmf(directory: Option<&Path>) -> Option { + let mut file = File::open(directory?.join("mkmf.log")).ok()?; + let bytes = file.metadata().ok()?.len(); + file.seek(SeekFrom::Start( + bytes.saturating_sub(MAX_BUILD_OUTPUT_BYTES as u64), + )) + .ok()?; + let mut tail = Vec::new(); + file.take(MAX_BUILD_OUTPUT_BYTES as u64) + .read_to_end(&mut tail) + .ok()?; + Some(String::from_utf8_lossy(&tail).into_owned()) +} diff --git a/crates/rb-gem-installer/src/build_event.rs b/crates/rb-gem-installer/src/build_event.rs new file mode 100644 index 0000000..2ce9b2a --- /dev/null +++ b/crates/rb-gem-installer/src/build_event.rs @@ -0,0 +1,35 @@ +use std::{ffi::OsString, path::PathBuf, time::Duration}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum BuildStage { + Configure, + Compile, + Install, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum OutputStream { + Stdout, + Stderr, +} + +#[derive(Clone, Debug)] +pub enum BuildEvent { + Started { + extension: PathBuf, + stage: BuildStage, + program: OsString, + arguments: Vec, + }, + Output { + extension: PathBuf, + stage: BuildStage, + stream: OutputStream, + bytes: Vec, + }, + Finished { + extension: PathBuf, + stage: BuildStage, + elapsed: Duration, + }, +} diff --git a/crates/rb-gem-installer/src/build_options.rs b/crates/rb-gem-installer/src/build_options.rs new file mode 100644 index 0000000..89e6ed7 --- /dev/null +++ b/crates/rb-gem-installer/src/build_options.rs @@ -0,0 +1,33 @@ +use crate::constants::{DEFAULT_BUILD_JOBS, DEFAULT_BUILD_TIMEOUT}; +use std::{collections::BTreeMap, ffi::OsString, path::PathBuf, time::Duration}; + +#[derive(Clone, Debug)] +pub struct BuildOptions { + pub ruby: PathBuf, + pub jobs: usize, + pub arguments: Vec, + pub environment: BTreeMap, + pub load_paths: Vec, + pub dependencies: Vec, + pub timeout: Duration, + pub make: Option, + pub cargo: OsString, + pub cmake: OsString, +} + +impl BuildOptions { + pub fn new(ruby: impl Into) -> Self { + Self { + ruby: ruby.into(), + jobs: DEFAULT_BUILD_JOBS, + arguments: Vec::new(), + environment: BTreeMap::new(), + load_paths: Vec::new(), + dependencies: Vec::new(), + timeout: DEFAULT_BUILD_TIMEOUT, + make: None, + cargo: "cargo".into(), + cmake: "cmake".into(), + } + } +} diff --git a/crates/rb-gem-installer/src/build_outputs.rs b/crates/rb-gem-installer/src/build_outputs.rs new file mode 100644 index 0000000..6b94aca --- /dev/null +++ b/crates/rb-gem-installer/src/build_outputs.rs @@ -0,0 +1,88 @@ +use crate::{ + constants::{MAX_FILES, MAX_PAYLOAD_BYTES}, + payload_path, +}; +use anyhow::{Context, Result, ensure}; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +pub(crate) fn copy(source: &Path, destination: &Path) -> Result> { + let mut files = Vec::new(); + visit(source, source, &mut files)?; + ensure!(files.len() <= MAX_FILES, "Too many build outputs"); + let mut bytes = 0_u64; + for relative in &files { + let from = source.join(relative); + bytes = bytes + .checked_add(fs::metadata(&from)?.len()) + .context("Build output size overflow")?; + ensure!( + bytes <= MAX_PAYLOAD_BYTES, + "Build outputs exceed size limit" + ); + let to = destination.join(relative); + fs::create_dir_all(to.parent().context("Missing output parent")?)?; + if fs::symlink_metadata(&to).is_ok() { + ensure!( + !fs::symlink_metadata(&to)?.file_type().is_symlink(), + "Build destination is a symlink" + ); + } + fs::copy(from, to)?; + } + Ok(files) +} + +fn visit(root: &Path, directory: &Path, files: &mut Vec) -> Result<()> { + for entry in fs::read_dir(directory)? { + let entry = entry?; + let kind = entry.file_type()?; + let path = entry.path(); + ensure!( + !kind.is_symlink(), + "Build output is a symlink: {}", + path.display() + ); + if kind.is_dir() { + visit(root, &path, files)?; + } else { + ensure!(kind.is_file(), "Build output is not a regular file"); + files.push(payload_path::relative(path.strip_prefix(root)?)?); + ensure!(files.len() <= MAX_FILES, "Too many build outputs"); + } + } + Ok(()) +} + +pub(crate) fn native_files( + root: &Path, + directory: &Path, + suffix: &str, + files: &mut Vec, +) -> Result<()> { + if !directory.is_dir() { + return Ok(()); + } + for entry in fs::read_dir(directory)? { + let entry = entry?; + let path = entry.path(); + let kind = entry.file_type()?; + if kind.is_dir() { + native_files(root, &path, suffix, files)?; + } else if path.is_file() + && path.extension().is_some_and(|extension| { + extension == suffix + || extension == "so" + || extension == "dll" + || extension == "dylib" + || extension == "bundle" + }) + { + files.push(path.strip_prefix(root)?.to_path_buf()); + ensure!(files.len() <= MAX_FILES, "Too many native artifacts"); + } + } + Ok(()) +} diff --git a/crates/rb-gem-installer/src/build_process.rs b/crates/rb-gem-installer/src/build_process.rs new file mode 100644 index 0000000..3b14a81 --- /dev/null +++ b/crates/rb-gem-installer/src/build_process.rs @@ -0,0 +1,215 @@ +use crate::{ + BuildEvent, BuildStage, OutputStream, + constants::{ + BUILD_CHANNEL_CAPACITY, BUILD_POLL_INTERVAL, MAX_BUILD_OUTPUT_BYTES, READ_BUFFER_BYTES, + }, +}; +use anyhow::{Context, Result, ensure}; +use std::{ + io::Read, + path::Path, + process::{Child, Command, ExitStatus, Stdio}, + sync::mpsc, + thread, + time::{Duration, Instant}, +}; + +pub(crate) struct ProcessOutput { + pub(crate) stdout: Vec, +} + +fn append(tail: &mut Vec, bytes: &[u8]) { + tail.extend_from_slice(bytes); + if tail.len() > MAX_BUILD_OUTPUT_BYTES { + tail.drain(..tail.len() - MAX_BUILD_OUTPUT_BYTES); + } +} + +fn terminate(child: &mut Child) { + #[cfg(unix)] + { + if let Some(pid) = rustix::process::Pid::from_raw(child.id() as i32) { + let _ = rustix::process::kill_process_group(pid, rustix::process::Signal::KILL); + } + } + #[cfg(windows)] + { + let _ = Command::new("taskkill") + .args(["/F", "/T", "/PID", &child.id().to_string()]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + let _ = child.kill(); + let _ = child.wait(); +} + +struct RunningChild { + child: Child, + finished: bool, +} + +impl std::ops::Deref for RunningChild { + type Target = Child; + fn deref(&self) -> &Child { + &self.child + } +} + +impl std::ops::DerefMut for RunningChild { + fn deref_mut(&mut self) -> &mut Child { + &mut self.child + } +} + +impl Drop for RunningChild { + fn drop(&mut self) { + if !self.finished { + terminate(&mut self.child); + } + } +} + +pub(crate) fn run( + mut command: Command, + extension: &Path, + stage: BuildStage, + timeout: Duration, + events: &mut impl FnMut(BuildEvent), +) -> Result { + let program = command.get_program().to_os_string(); + let arguments = command.get_args().map(|arg| arg.to_os_string()).collect(); + command + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + command.process_group(0); + } + let started = Instant::now(); + events(BuildEvent::Started { + extension: extension.to_path_buf(), + stage, + program: program.clone(), + arguments, + }); + let child = command + .spawn() + .with_context(|| format!("Cannot start build tool {}", program.to_string_lossy()))?; + let mut child = RunningChild { + child, + finished: false, + }; + let stdout = child.stdout.take().context("Missing child stdout")?; + let stderr = child.stderr.take().context("Missing child stderr")?; + let (sender, receiver) = mpsc::sync_channel(BUILD_CHANNEL_CAPACITY); + let streams: [(OutputStream, Box); 2] = [ + (OutputStream::Stdout, Box::new(stdout)), + (OutputStream::Stderr, Box::new(stderr)), + ]; + let readers: Vec<_> = streams + .into_iter() + .map(|(stream, mut reader)| { + let sender = sender.clone(); + thread::spawn(move || { + let mut bytes = [0; READ_BUFFER_BYTES]; + loop { + match reader.read(&mut bytes) { + Ok(0) => break, + Ok(size) => { + if sender.send((stream, Ok(bytes[..size].to_vec()))).is_err() { + break; + } + } + Err(error) if error.kind() == std::io::ErrorKind::Interrupted => continue, + Err(error) => { + let _ = sender.send((stream, Err(error))); + break; + } + } + } + }) + }) + .collect(); + drop(sender); + let mut output = Vec::new(); + let mut errors = Vec::new(); + let result = (|| -> Result { + let mut status = None; + loop { + if started.elapsed() >= timeout { + anyhow::bail!( + "Build tool {} timed out after {:.1}s", + program.to_string_lossy(), + timeout.as_secs_f64() + ); + } + match receiver.recv_timeout(BUILD_POLL_INTERVAL) { + Ok((stream, bytes)) => { + let bytes = bytes?; + append( + if stream == OutputStream::Stdout { + &mut output + } else { + &mut errors + }, + &bytes, + ); + events(BuildEvent::Output { + extension: extension.to_path_buf(), + stage, + stream, + bytes, + }); + } + Err(mpsc::RecvTimeoutError::Timeout) => {} + Err(mpsc::RecvTimeoutError::Disconnected) => break, + } + if status.is_none() { + status = child.try_wait()?; + } + } + loop { + if let Some(status) = status { + return Ok(status); + } + if started.elapsed() >= timeout { + anyhow::bail!("Build tool {} timed out", program.to_string_lossy()); + } + status = child.try_wait()?; + if status.is_none() { + thread::sleep(BUILD_POLL_INTERVAL); + } + } + })(); + if result.is_err() { + terminate(&mut child); + } + child.finished = true; + drop(receiver); + for reader in readers { + let _ = reader.join(); + } + let status = result.with_context(|| { + format!( + "{}\n{}", + String::from_utf8_lossy(&output), + String::from_utf8_lossy(&errors) + ) + })?; + ensure!( + status.success(), + "Build tool {} failed ({status})\n{}\n{}", + program.to_string_lossy(), + String::from_utf8_lossy(&output), + String::from_utf8_lossy(&errors) + ); + events(BuildEvent::Finished { + extension: extension.to_path_buf(), + stage, + elapsed: started.elapsed(), + }); + Ok(ProcessOutput { stdout: output }) +} diff --git a/crates/rb-gem-installer/src/build_report.rs b/crates/rb-gem-installer/src/build_report.rs new file mode 100644 index 0000000..8c94e7e --- /dev/null +++ b/crates/rb-gem-installer/src/build_report.rs @@ -0,0 +1,10 @@ +use std::{path::PathBuf, time::Duration}; + +#[derive(Clone, Debug)] +pub struct BuildReport { + pub ruby_version: String, + pub architecture: String, + pub extensions: Vec, + pub artifacts: Vec, + pub elapsed: Duration, +} diff --git a/crates/rb-gem-installer/src/cargo_builder.rs b/crates/rb-gem-installer/src/cargo_builder.rs new file mode 100644 index 0000000..b3810f2 --- /dev/null +++ b/crates/rb-gem-installer/src/cargo_builder.rs @@ -0,0 +1,121 @@ +use crate::{BuildEvent, BuildStage, NativeBuilder}; +use anyhow::{Context, Result, ensure}; +use std::{fs, path::Path}; + +pub(crate) fn build( + builder: &NativeBuilder, + manifest: &Path, + extension: &Path, + output: &Path, + events: &mut impl FnMut(BuildEvent), +) -> Result<()> { + let directory = manifest.parent().context("Missing Cargo directory")?; + let mut command = builder.command(&builder.options.cargo, directory); + command + .args([ + "metadata", + "--format-version", + "1", + "--no-deps", + "--manifest-path", + ]) + .arg(manifest); + let metadata = builder.run(command, extension, BuildStage::Configure, events)?; + let metadata: serde_json::Value = + serde_json::from_slice(&metadata.stdout).context("Reading Cargo metadata")?; + let package = metadata["packages"] + .as_array() + .context("Missing Cargo packages")? + .iter() + .find(|package| { + package["manifest_path"].as_str().is_some_and(|path| { + Path::new(path).canonicalize().ok().as_deref() == Some(manifest) + }) + }) + .context("Missing extension Cargo package")?; + let package_id = package["id"] + .as_str() + .context("Missing Cargo package identity")?; + let target = output.join("cargo"); + let mut command = builder.command(&builder.options.cargo, directory); + command + .args([ + "rustc", + "--lib", + "--release", + "--locked", + "--message-format=json-render-diagnostics", + "--manifest-path", + ]) + .arg(manifest) + .arg("--target-dir") + .arg(&target) + .args(&builder.options.arguments) + .arg("--") + .arg("--crate-type=cdylib"); + command + .arg("-L") + .arg(format!("native={}", builder.ruby.get("libdir"))); + if let Some(linker) = builder.ruby.linker.first() { + command.arg("-C").arg(format!("linker={linker}")); + } + for argument in builder + .ruby + .linker + .iter() + .skip(1) + .chain(builder.ruby.libraries.iter()) + { + command.arg("-C").arg(format!("link-arg={argument}")); + } + if builder.ruby.get("target_os").contains("darwin") { + command + .arg("-C") + .arg("link-arg=-Wl,-undefined,dynamic_lookup"); + } + let compiled = builder.run(command, extension, BuildStage::Compile, events)?; + let mut found = false; + for line in compiled.stdout.split(|byte| *byte == b'\n') { + let Ok(message) = serde_json::from_slice::(line) else { + continue; + }; + if message["reason"] != "compiler-artifact" || message["package_id"] != package_id { + continue; + } + let name = message["target"]["name"] + .as_str() + .context("Missing Cargo library name")?; + for file in message["filenames"] + .as_array() + .context("Missing Cargo artifact filenames")? + { + let path = Path::new(file.as_str().context("Invalid Cargo artifact filename")?); + if path + .extension() + .is_some_and(|suffix| suffix == "so" || suffix == "dylib" || suffix == "dll") + { + let canonical = path.canonicalize()?; + ensure!( + canonical.starts_with(target.canonicalize()?), + "Cargo artifact is outside build directory" + ); + let nesting = extension + .parent() + .unwrap_or(Path::new("")) + .components() + .skip(1) + .collect::(); + let destination = output.join(nesting); + fs::create_dir_all(&destination)?; + fs::copy( + path, + destination.join(format!("{name}.{}", builder.ruby.get("DLEXT"))), + )?; + found = true; + } + } + } + ensure!(found, "Cargo did not produce an extension library"); + fs::remove_dir_all(target)?; + Ok(()) +} diff --git a/crates/rb-gem-installer/src/cmake_builder.rs b/crates/rb-gem-installer/src/cmake_builder.rs new file mode 100644 index 0000000..8eefb46 --- /dev/null +++ b/crates/rb-gem-installer/src/cmake_builder.rs @@ -0,0 +1,43 @@ +use crate::{BuildEvent, BuildStage, NativeBuilder}; +use anyhow::{Context, Result}; +use std::{fs, path::Path}; + +pub(crate) fn build( + builder: &NativeBuilder, + path: &Path, + extension: &Path, + output: &Path, + events: &mut impl FnMut(BuildEvent), +) -> Result<()> { + let directory = path.parent().context("Missing CMake directory")?; + let build = output.join("cmake"); + let mut command = builder.command(&builder.options.cmake, directory); + command + .arg("-S") + .arg(directory) + .arg("-B") + .arg(&build) + .arg(format!( + "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={}", + output.display() + )) + .arg(format!( + "-DCMAKE_RUNTIME_OUTPUT_DIRECTORY={}", + output.display() + )) + .arg(format!( + "-DRuby_EXECUTABLE={}", + builder.ruby.executable.display() + )) + .args(&builder.options.arguments); + builder.run(command, extension, BuildStage::Configure, events)?; + let mut command = builder.command(&builder.options.cmake, directory); + command + .arg("--build") + .arg(&build) + .args(["--config", "Release", "--parallel"]) + .arg(builder.options.jobs.to_string()); + builder.run(command, extension, BuildStage::Compile, events)?; + fs::remove_dir_all(build)?; + Ok(()) +} diff --git a/crates/rb-gem-installer/src/constants.rs b/crates/rb-gem-installer/src/constants.rs new file mode 100644 index 0000000..4691ba6 --- /dev/null +++ b/crates/rb-gem-installer/src/constants.rs @@ -0,0 +1,13 @@ +pub(crate) const MAX_ARCHIVE_BYTES: u64 = 1024 * 1024 * 1024; +pub(crate) const MAX_PAYLOAD_BYTES: u64 = 1024 * 1024 * 1024; +pub(crate) const MAX_FILES: usize = 100_000; +pub(crate) const DATA_MEMBER: &str = "data.tar.gz"; +pub(crate) const READ_BUFFER_BYTES: usize = 8192; +pub(crate) const MAX_LINK_DEPTH: usize = 40; +pub(crate) const MAX_BUILD_OUTPUT_BYTES: usize = 1024 * 1024; +pub(crate) const BUILD_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_millis(25); +pub(crate) const DEFAULT_BUILD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300); +pub(crate) const DEFAULT_BUILD_JOBS: usize = 2; +pub(crate) const BUILD_CHANNEL_CAPACITY: usize = 64; +pub(crate) const NATIVE_DIRECTORY: &str = ".rb-native"; +pub(crate) const BUILD_COMPLETE_MARKER: &str = "gem.build_complete"; diff --git a/crates/rb-gem-installer/src/extract.rs b/crates/rb-gem-installer/src/extract.rs new file mode 100644 index 0000000..307cc28 --- /dev/null +++ b/crates/rb-gem-installer/src/extract.rs @@ -0,0 +1,106 @@ +use crate::{ + constants::{DATA_MEMBER, MAX_FILES, MAX_PAYLOAD_BYTES, READ_BUFFER_BYTES}, + payload_links::PayloadLinks, + payload_path, +}; +use anyhow::{Context, Result, ensure}; +use flate2::read::GzDecoder; +use std::{ + collections::BTreeMap, + fs::{File, OpenOptions}, + io::{self, Read}, + path::Path, +}; + +pub(crate) fn extract(reader: impl Read, destination: &Path) -> Result<()> { + let mut outer = tar::Archive::new(reader); + for member in outer.entries()? { + let member = member?; + if member.path()?.as_ref() != Path::new(DATA_MEMBER) { + continue; + } + let mut decoder = GzDecoder::new(member).take(MAX_PAYLOAD_BYTES + 1); + let mut links = PayloadLinks::new(); + { + let mut payload = tar::Archive::new(&mut decoder); + let mut paths = BTreeMap::new(); + let mut total = 0_u64; + for (index, entry) in payload.entries()?.enumerate() { + ensure!(index < MAX_FILES, "Payload exceeds file count limit"); + let mut entry = entry?; + let path = payload_path::normalize(&entry.path()?)?; + let kind = entry.header().entry_type(); + let mode = entry.header().mode()? & 0o777; + ensure!( + kind.is_file() || kind.is_dir() || kind.is_symlink(), + "Unsupported payload entry: {}", + path.display() + ); + total = total + .checked_add(entry.size()) + .context("Payload size overflow")?; + ensure!(total <= MAX_PAYLOAD_BYTES, "Payload exceeds size limit"); + let target = destination.join(&path); + if let Some(previous) = paths.insert(path.clone(), (kind, mode)) { + ensure!( + previous == (kind, mode) && kind.is_file(), + "Conflicting payload entry: {}", + path.display() + ); + identical(File::open(&target)?, &mut entry)?; + continue; + } + if kind.is_dir() { + std::fs::create_dir_all(&target)?; + } else if kind.is_symlink() { + let link = entry + .link_name()? + .context("Missing symlink target")? + .into_owned(); + links.add(path, link)?; + } else { + std::fs::create_dir_all(target.parent().context("Missing payload parent")?)?; + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(mode); + } + let mut file = options.open(&target)?; + io::copy(&mut entry, &mut file)?; + } + } + } + io::copy(&mut decoder, &mut io::sink())?; + ensure!( + decoder.limit() > 0, + "Payload exceeds decompressed size limit" + ); + links.create(destination)?; + return Ok(()); + } + anyhow::bail!("Gem is missing data.tar.gz") +} + +fn identical(mut existing: File, mut entry: impl Read) -> Result<()> { + let mut old = [0; READ_BUFFER_BYTES]; + let mut new = [0; READ_BUFFER_BYTES]; + loop { + let size = existing.read(&mut old)?; + if size == 0 { + ensure!( + entry.read(&mut new[..1])? == 0, + "Conflicting duplicate payload content" + ); + return Ok(()); + } + entry + .read_exact(&mut new[..size]) + .context("Conflicting duplicate payload size")?; + ensure!( + old[..size] == new[..size], + "Conflicting duplicate payload content" + ); + } +} diff --git a/crates/rb-gem-installer/src/install.rs b/crates/rb-gem-installer/src/install.rs new file mode 100644 index 0000000..3198ca0 --- /dev/null +++ b/crates/rb-gem-installer/src/install.rs @@ -0,0 +1,61 @@ +use crate::{constants::MAX_ARCHIVE_BYTES, extract}; +use anyhow::{Context, Result, ensure}; +use rb_gem_archive::GemMetadata; +use std::{ + fs, + io::{self, Read, Seek, SeekFrom}, + path::Path, +}; + +/// Verifies an archive and installs its payload into a new directory. +pub fn install(reader: impl Read, destination: impl AsRef) -> Result { + staged(reader, destination.as_ref(), |_, _| Ok(())).map(|(metadata, ())| metadata) +} + +pub fn install_with_builder( + reader: impl Read, + destination: impl AsRef, + builder: &crate::NativeBuilder, + events: impl FnMut(crate::BuildEvent), +) -> Result<(GemMetadata, crate::BuildReport)> { + staged(reader, destination.as_ref(), |metadata, root| { + builder.build(metadata, root, events) + }) +} + +fn staged( + reader: impl Read, + destination: &Path, + work: impl FnOnce(&GemMetadata, &Path) -> Result, +) -> Result<(GemMetadata, T)> { + ensure!( + destination.file_name().is_some(), + "Installation needs a directory name" + ); + let parent = destination + .parent() + .filter(|path| !path.as_os_str().is_empty()) + .unwrap_or(Path::new(".")); + ensure!(parent.is_dir(), "Installation parent must exist"); + ensure!( + fs::symlink_metadata(destination) + .is_err_and(|error| error.kind() == io::ErrorKind::NotFound), + "Installation destination already exists or is inaccessible" + ); + let mut snapshot = tempfile::tempfile()?; + let copied = io::copy(&mut reader.take(MAX_ARCHIVE_BYTES + 1), &mut snapshot)?; + ensure!( + copied <= MAX_ARCHIVE_BYTES, + "Gem archive exceeds size limit" + ); + snapshot.seek(SeekFrom::Start(0))?; + let metadata = rb_gem_archive::read(&mut snapshot)?; + snapshot.seek(SeekFrom::Start(0))?; + let staging = tempfile::Builder::new() + .prefix(".rb-gem-") + .tempdir_in(parent)?; + extract::extract(&mut snapshot, staging.path())?; + let result = work(&metadata, staging.path())?; + crate::publish::publish(staging.path(), destination).context("Cannot publish installed gem")?; + Ok((metadata, result)) +} diff --git a/crates/rb-gem-installer/src/lib.rs b/crates/rb-gem-installer/src/lib.rs new file mode 100644 index 0000000..ceb9f8e --- /dev/null +++ b/crates/rb-gem-installer/src/lib.rs @@ -0,0 +1,26 @@ +#![doc = include_str!("../README.md")] + +mod build_diagnostics; +mod build_event; +mod build_options; +mod build_outputs; +mod build_process; +mod build_report; +mod cargo_builder; +mod cmake_builder; +mod constants; +mod extract; +mod install; +mod make_builder; +mod native_builder; +mod payload_links; +mod payload_path; +mod publish; +mod rake_builder; +mod ruby_config; + +pub use build_event::{BuildEvent, BuildStage, OutputStream}; +pub use build_options::BuildOptions; +pub use build_report::BuildReport; +pub use install::{install, install_with_builder}; +pub use native_builder::NativeBuilder; diff --git a/crates/rb-gem-installer/src/make_builder.rs b/crates/rb-gem-installer/src/make_builder.rs new file mode 100644 index 0000000..e0e34b3 --- /dev/null +++ b/crates/rb-gem-installer/src/make_builder.rs @@ -0,0 +1,93 @@ +use crate::{BuildEvent, BuildStage, NativeBuilder}; +use anyhow::{Context, Result}; +use std::{ffi::OsString, path::Path}; + +pub(crate) fn build( + builder: &NativeBuilder, + path: &Path, + extension: &Path, + output: &Path, + events: &mut impl FnMut(BuildEvent), +) -> Result<()> { + let directory = path.parent().context("Missing build directory")?; + if path.file_name().is_some_and(|name| name == "configure") { + let mut command = builder.command("sh", directory); + command + .arg("./configure") + .arg(format!( + "--prefix=./{}", + output + .file_name() + .context("Missing temporary output name")? + .to_string_lossy() + )) + .args(&builder.options.arguments); + builder.run(command, extension, BuildStage::Configure, events)?; + make(builder, directory, output, extension, events) + } else { + let mut command = builder.ruby_command(directory); + command + .arg(path.file_name().context("Missing extension filename")?) + .args(&builder.options.arguments); + builder.run(command, extension, BuildStage::Configure, events)?; + if directory.join("Makefile").is_file() { + make(builder, directory, output, extension, events)?; + } + Ok(()) + } +} + +fn make( + builder: &NativeBuilder, + directory: &Path, + output: &Path, + extension: &Path, + events: &mut impl FnMut(BuildEvent), +) -> Result<()> { + let program = builder + .options + .make + .clone() + .or_else(|| { + builder + .options + .environment + .get(std::ffi::OsStr::new("MAKE")) + .cloned() + }) + .or_else(|| std::env::var_os("MAKE")) + .unwrap_or_else(|| { + OsString::from(if builder.ruby.get("target_os").contains("mswin") { + "nmake" + } else { + "make" + }) + }); + let is_nmake = program.to_string_lossy().contains("nmake"); + for (stage, target) in [ + (BuildStage::Compile, None), + (BuildStage::Install, Some("install")), + ] { + let mut command = builder.command(&program, directory); + if !is_nmake { + command + .arg(format!("-j{}", builder.options.jobs)) + .arg("DESTDIR="); + } + let relative = format!( + "./{}", + output + .file_name() + .context("Missing temporary output name")? + .to_string_lossy() + ); + command + .arg(format!("sitearchdir={relative}")) + .arg(format!("sitelibdir={relative}")); + if let Some(target) = target { + command.arg(target); + } + builder.run(command, extension, stage, events)?; + } + Ok(()) +} diff --git a/crates/rb-gem-installer/src/native_builder.rs b/crates/rb-gem-installer/src/native_builder.rs new file mode 100644 index 0000000..da627af --- /dev/null +++ b/crates/rb-gem-installer/src/native_builder.rs @@ -0,0 +1,273 @@ +use crate::{ + BuildEvent, BuildOptions, BuildReport, BuildStage, build_outputs, build_process, cargo_builder, + constants::{BUILD_COMPLETE_MARKER, NATIVE_DIRECTORY}, + payload_path, + ruby_config::RubyConfig, +}; +use anyhow::{Context, Result, ensure}; +use rb_gem_archive::GemMetadata; +use std::{ + fs, + path::{Component, Path, PathBuf}, + process::Command, + time::Instant, +}; + +#[derive(Clone, Debug)] +pub struct NativeBuilder { + pub(crate) options: BuildOptions, + pub(crate) ruby: RubyConfig, + compatibility: Option<(PathBuf, PathBuf, String)>, +} + +impl NativeBuilder { + pub fn new(options: BuildOptions) -> Result { + ensure!(options.jobs > 0, "Build jobs must be positive"); + ensure!(!options.timeout.is_zero(), "Build timeout must be positive"); + std::env::join_paths(&options.load_paths).context("Invalid dependency load path")?; + let ruby = RubyConfig::read(&options)?; + Ok(Self { + options, + ruby, + compatibility: None, + }) + } + + pub fn build( + &self, + metadata: &GemMetadata, + root: impl AsRef, + mut events: impl FnMut(BuildEvent), + ) -> Result { + let root = root.as_ref().canonicalize()?; + let mut configured = self.clone(); + let paths = if metadata.require_paths.is_empty() { + vec!["lib".to_owned()] + } else { + metadata.require_paths.clone() + }; + let paths = paths + .iter() + .map(|path| { + let path = Path::new(path); + if path.components().all(|part| part == Component::CurDir) { + Ok(root.clone()) + } else { + payload_path::normalize(path).map(|path| root.join(path)) + } + }) + .collect::>>()?; + configured.options.load_paths.splice(0..0, paths); + std::env::join_paths(&configured.options.load_paths).context("Invalid gem load path")?; + let mut gems = configured.options.dependencies.clone(); + gems.retain(|gem| { + !(gem.name == metadata.id.name + && gem.version == metadata.id.version.to_string() + && gem.platform == metadata.id.platform.to_string()) + }); + gems.push(rb_microgems::InstalledGem { + name: metadata.id.name.clone(), + version: metadata.id.version.to_string(), + platform: metadata.id.platform.to_string(), + root: root.clone(), + require_paths: metadata.require_paths.clone(), + metadata: metadata.metadata.clone(), + executables: metadata.executables.clone(), + bindir: metadata.bindir.clone(), + }); + let compatibility = rb_microgems::Environment::new(&gems, &configured.options.arguments)?; + configured.compatibility = Some(( + compatibility.load_path().to_path_buf(), + compatibility.manifest_path().to_path_buf(), + serde_json::to_string(&[&metadata.id.name, &metadata.id.version.to_string()])?, + )); + configured.build_declared(metadata, &root, &mut events) + } + + fn build_declared( + &self, + metadata: &GemMetadata, + root: &Path, + events: &mut impl FnMut(BuildEvent), + ) -> Result { + ensure!( + metadata.ruby.matches(&self.ruby.version.parse()?), + "Gem requires Ruby {}, selected Ruby is {}", + metadata.ruby, + self.ruby.version + ); + let started = Instant::now(); + let mut report = BuildReport { + ruby_version: self.ruby.version.clone(), + architecture: self.ruby.get("arch").into(), + extensions: Vec::new(), + artifacts: Vec::new(), + elapsed: Default::default(), + }; + if metadata.extensions.is_empty() { + return Ok(report); + } + let native_relative = PathBuf::from(NATIVE_DIRECTORY) + .join(self.ruby.get("arch")) + .join(self.ruby.get("ruby_version")); + let native_relative = payload_path::relative(&native_relative)?; + let destination = root.join(&native_relative); + ensure!( + !destination.exists(), + "Native build destination already exists" + ); + fs::create_dir_all(&destination)?; + let lib = self + .options + .load_paths + .first() + .context("Missing gem load path")?; + fs::create_dir_all(lib)?; + ensure!( + lib.canonicalize()?.starts_with(root), + "Gem load path escapes installation" + ); + for extension in &metadata.extensions { + let relative = payload_path::normalize(Path::new(extension))?; + let path = root + .join(&relative) + .canonicalize() + .with_context(|| format!("Missing extension {extension}"))?; + ensure!( + path.starts_with(root) && path.is_file(), + "Extension is outside installation or is not a file" + ); + let directory = path.parent().context("Missing extension directory")?; + let output = tempfile::Builder::new() + .prefix(".rb-build-") + .tempdir_in(directory)?; + let name = path + .file_name() + .and_then(|name| name.to_str()) + .context("Non-UTF8 builder name")?; + match name { + "extconf.rb" | "configure" => { + crate::make_builder::build(self, &path, &relative, output.path(), events)? + } + "Rakefile" | "rakefile" | "mkrf_conf.rb" => { + crate::rake_builder::build(self, &path, &relative, output.path(), events)? + } + "CMakeLists.txt" => { + crate::cmake_builder::build(self, &path, &relative, output.path(), events)? + } + "Cargo.toml" => { + cargo_builder::build(self, &path, &relative, output.path(), events)? + } + _ => anyhow::bail!("Unsupported extension builder: {extension}"), + } + let files = build_outputs::copy(output.path(), &destination) + .with_context(|| format!("Collecting build outputs for {extension}"))?; + build_outputs::copy(output.path(), lib)?; + report + .artifacts + .extend(files.into_iter().map(|file| native_relative.join(file))); + report.extensions.push(relative); + } + for path in self + .options + .load_paths + .iter() + .take(metadata.require_paths.len().max(1)) + { + build_outputs::native_files(root, path, self.ruby.get("DLEXT"), &mut report.artifacts)?; + } + for extension in &metadata.extensions { + let path = root.join(payload_path::normalize(Path::new(extension))?); + let directory = path.parent().context("Missing extension directory")?; + build_outputs::native_files( + root, + directory, + self.ruby.get("DLEXT"), + &mut report.artifacts, + )?; + } + report.artifacts.sort(); + report.artifacts.dedup(); + fs::write(destination.join(BUILD_COMPLETE_MARKER), b"")?; + report.elapsed = started.elapsed(); + Ok(report) + } + + pub(crate) fn command( + &self, + program: impl AsRef, + directory: &Path, + ) -> Command { + let mut command = Command::new(program); + command + .current_dir(directory) + .env_remove("RUBYLIB") + .envs(&self.options.environment); + command + .env_remove("RUBYGEMS_GEMDEPS") + .env_remove("RUBY_BOX") + .env_remove("DESTDIR"); + command + .env("RUBY", &self.ruby.executable) + .env("RUBYOPT", "--disable=gems"); + let mut make_flags = self + .options + .environment + .get(std::ffi::OsStr::new("MAKEFLAGS")) + .cloned() + .or_else(|| std::env::var_os("MAKEFLAGS")) + .unwrap_or_default(); + if !make_flags.is_empty() { + make_flags.push(" "); + } + make_flags.push(format!("-j{}", self.options.jobs)); + command + .env("MAKEFLAGS", make_flags) + .env("NJOBS", self.options.jobs.to_string()) + .env("CARGO_BUILD_JOBS", self.options.jobs.to_string()) + .env("CMAKE_BUILD_PARALLEL_LEVEL", self.options.jobs.to_string()) + .env("GRPC_RUBY_BUILD_PROCS", self.options.jobs.to_string()); + for (key, value) in &self.ruby.config { + command.env(format!("RBCONFIG_{key}"), value); + } + let mut paths = self.options.load_paths.clone(); + if let Some((directory, manifest, current)) = &self.compatibility { + paths.insert(0, directory.clone()); + command + .env("RB_MICROGEMS_MANIFEST", manifest) + .env("RB_MICROGEMS_CURRENT", current) + .env("RUBYOPT", "--disable=gems -rmicrogems/compatibility"); + } + if let Ok(paths) = std::env::join_paths(&paths) { + command.env("RUBYLIB", paths); + } + command + } + + pub(crate) fn ruby_command(&self, directory: &Path) -> Command { + let mut command = self.command(&self.ruby.executable, directory); + command.arg("--disable=gems"); + for path in &self.options.load_paths { + command.arg("-I").arg(path); + } + command + } + + pub(crate) fn run( + &self, + command: Command, + extension: &Path, + stage: BuildStage, + events: &mut impl FnMut(BuildEvent), + ) -> Result { + let directory = command.get_current_dir().map(Path::to_path_buf); + build_process::run(command, extension, stage, self.options.timeout, events) + .map_err( + |error| match crate::build_diagnostics::mkmf(directory.as_deref()) { + Some(log) => error.context(format!("mkmf.log:\n{log}")), + None => error, + }, + ) + .with_context(|| format!("Building {} ({stage:?})", extension.display())) + } +} diff --git a/crates/rb-gem-installer/src/payload_links.rs b/crates/rb-gem-installer/src/payload_links.rs new file mode 100644 index 0000000..dc22d9c --- /dev/null +++ b/crates/rb-gem-installer/src/payload_links.rs @@ -0,0 +1,89 @@ +use crate::{constants::MAX_LINK_DEPTH, payload_path}; +use anyhow::{Context, Result, ensure}; +use std::{ + collections::{BTreeMap, BTreeSet, VecDeque}, + fs, + path::{Component, Path, PathBuf}, +}; + +pub(crate) struct PayloadLinks(BTreeMap); + +impl PayloadLinks { + pub(crate) fn new() -> Self { + Self(BTreeMap::new()) + } + + pub(crate) fn add(&mut self, path: PathBuf, target: PathBuf) -> Result<()> { + let resolved = payload_path::link_target(&path, &target)?; + self.0.insert(path, (target, resolved)); + Ok(()) + } + + fn resolve(&self, target: &Path) -> Result { + let mut pending: VecDeque<_> = target + .components() + .filter_map(|part| match part { + Component::Normal(name) => Some(name.to_os_string()), + _ => None, + }) + .collect(); + let mut resolved = PathBuf::new(); + let mut followed = BTreeSet::new(); + while let Some(part) = pending.pop_front() { + resolved.push(part); + if let Some((_, target)) = self.0.get(&resolved) { + ensure!(followed.insert(resolved.clone()), "Cyclic payload symlink"); + let mut replacement: VecDeque<_> = target + .components() + .filter_map(|part| match part { + Component::Normal(name) => Some(name.to_os_string()), + _ => None, + }) + .collect(); + ensure!( + followed.len() <= MAX_LINK_DEPTH, + "Payload symlink chain exceeds depth limit" + ); + replacement.append(&mut pending); + pending = replacement; + resolved.clear(); + } + } + Ok(resolved) + } + + pub(crate) fn create(self, root: &Path) -> Result<()> { + let mut directories = BTreeMap::new(); + for (link, (_, target)) in &self.0 { + for parent in link.ancestors().skip(1) { + ensure!( + !self.0.contains_key(parent), + "Payload symlink cannot be a link parent" + ); + } + let resolved = self.resolve(target)?; + let metadata = fs::symlink_metadata(root.join(&resolved)) + .with_context(|| format!("Missing link target: {}", resolved.display()))?; + ensure!( + metadata.is_file() || metadata.is_dir(), + "Link target must be a payload file or directory" + ); + directories.insert(link.clone(), metadata.is_dir()); + } + for (path, (target, _)) in self.0 { + let destination = root.join(&path); + fs::create_dir_all(destination.parent().context("Missing link parent")?)?; + #[cfg(unix)] + std::os::unix::fs::symlink(target, destination)?; + #[cfg(windows)] + if directories[&path] { + std::os::windows::fs::symlink_dir(target, destination)?; + } else { + std::os::windows::fs::symlink_file(target, destination)?; + } + #[cfg(not(any(unix, windows)))] + anyhow::bail!("Symlinks are not supported on this platform"); + } + Ok(()) + } +} diff --git a/crates/rb-gem-installer/src/payload_path.rs b/crates/rb-gem-installer/src/payload_path.rs new file mode 100644 index 0000000..3463ab0 --- /dev/null +++ b/crates/rb-gem-installer/src/payload_path.rs @@ -0,0 +1,74 @@ +use anyhow::{Result, ensure}; +use std::path::{Component, Path, PathBuf}; + +pub(crate) fn normalize(path: &Path) -> Result { + ensure!( + portable(path), + "Nonportable payload path: {}", + path.display() + ); + relative(path) +} + +pub(crate) fn relative(path: &Path) -> Result { + ensure!( + path.components() + .all(|part| matches!(part, Component::Normal(_) | Component::CurDir)), + "Unsafe payload path: {}", + path.display() + ); + let normalized: PathBuf = path + .components() + .filter_map(|part| match part { + Component::Normal(name) => Some(name), + _ => None, + }) + .collect(); + ensure!(!normalized.as_os_str().is_empty(), "Empty payload path"); + Ok(normalized) +} + +pub(crate) fn link_target(path: &Path, target: &Path) -> Result { + ensure!(portable(target), "Nonportable link target"); + ensure!(!target.as_os_str().is_empty(), "Empty link target"); + let mut resolved = path.parent().unwrap_or(Path::new("")).to_path_buf(); + for part in target.components() { + match part { + Component::Normal(name) => resolved.push(name), + Component::CurDir => {} + Component::ParentDir => ensure!(resolved.pop(), "Link target escapes installation"), + _ => anyhow::bail!("Absolute link target"), + } + } + ensure!( + !resolved.as_os_str().is_empty(), + "Link target is installation root" + ); + Ok(resolved) +} + +fn portable(path: &Path) -> bool { + !path + .as_os_str() + .as_encoded_bytes() + .iter() + .any(|byte| matches!(byte, b'\\' | b':')) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn filesystem_paths_accept_native_separators() { + let path = PathBuf::from(".rb-native").join("architecture").join("abi"); + assert_eq!(relative(&path).unwrap(), path); + assert!(relative(Path::new("../outside")).is_err()); + assert!(relative(&std::env::temp_dir()).is_err()); + } + + #[test] + fn archive_paths_reject_nonportable_separators() { + assert!(normalize(Path::new("lib\\nested.rb")).is_err()); + } +} diff --git a/crates/rb-gem-installer/src/publish.rs b/crates/rb-gem-installer/src/publish.rs new file mode 100644 index 0000000..083f393 --- /dev/null +++ b/crates/rb-gem-installer/src/publish.rs @@ -0,0 +1,72 @@ +use anyhow::Result; +use std::path::Path; + +pub(crate) fn publish(source: &Path, destination: &Path) -> Result<()> { + #[cfg(any( + target_os = "linux", + target_os = "android", + target_os = "macos", + target_os = "ios" + ))] + rustix::fs::renameat_with( + rustix::fs::CWD, + source, + rustix::fs::CWD, + destination, + rustix::fs::RenameFlags::NOREPLACE, + )?; + #[cfg(windows)] + { + use std::os::windows::ffi::OsStrExt; + let source: Vec<_> = source.as_os_str().encode_wide().chain(Some(0)).collect(); + let destination: Vec<_> = destination + .as_os_str() + .encode_wide() + .chain(Some(0)) + .collect(); + anyhow::ensure!( + !source[..source.len() - 1].contains(&0) + && !destination[..destination.len() - 1].contains(&0), + "Installation path contains a null character" + ); + let moved = unsafe { + windows_sys::Win32::Storage::FileSystem::MoveFileExW( + source.as_ptr(), + destination.as_ptr(), + 0, + ) + }; + if moved == 0 { + return Err(std::io::Error::last_os_error().into()); + } + } + #[cfg(not(any( + target_os = "linux", + target_os = "android", + target_os = "macos", + target_os = "ios", + windows + )))] + anyhow::bail!( + "Atomic installation is not supported on this platform: {} -> {}", + source.display(), + destination.display() + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + #[test] + fn publication_cannot_replace_an_empty_destination() { + let root = tempfile::tempdir().unwrap(); + let source = root.path().join("source"); + let destination = root.path().join("destination"); + std::fs::create_dir(&source).unwrap(); + std::fs::write(source.join("file"), b"content").unwrap(); + std::fs::create_dir(&destination).unwrap(); + assert!(super::publish(&source, &destination).is_err()); + assert!(source.join("file").is_file()); + assert_eq!(std::fs::read_dir(&destination).unwrap().count(), 0); + } +} diff --git a/crates/rb-gem-installer/src/rake_builder.rs b/crates/rb-gem-installer/src/rake_builder.rs new file mode 100644 index 0000000..7e914a2 --- /dev/null +++ b/crates/rb-gem-installer/src/rake_builder.rs @@ -0,0 +1,34 @@ +use crate::{BuildEvent, BuildStage, NativeBuilder}; +use anyhow::{Context, Result}; +use std::path::Path; + +pub(crate) fn build( + builder: &NativeBuilder, + path: &Path, + extension: &Path, + output: &Path, + events: &mut impl FnMut(BuildEvent), +) -> Result<()> { + let directory = path.parent().context("Missing rake directory")?; + let name = path.file_name().context("Missing rake filename")?; + let configure = name == "mkrf_conf.rb"; + if configure { + let mut command = builder.ruby_command(directory); + command.arg(name).args(&builder.options.arguments); + builder.run(command, extension, BuildStage::Configure, events)?; + } + let mut command = builder.ruby_command(directory); + command.args(["-e", "require 'rake'; Rake.application.run", "--", "-f"]); + command.arg(if configure { + std::ffi::OsStr::new("Rakefile") + } else { + name + }); + let output = output.display().to_string().replace('\\', "/"); + command + .arg(format!("RUBYARCHDIR={output}")) + .arg(format!("RUBYLIBDIR={output}")) + .args(&builder.options.arguments); + builder.run(command, extension, BuildStage::Compile, events)?; + Ok(()) +} diff --git a/crates/rb-gem-installer/src/ruby_config.rs b/crates/rb-gem-installer/src/ruby_config.rs new file mode 100644 index 0000000..3368a88 --- /dev/null +++ b/crates/rb-gem-installer/src/ruby_config.rs @@ -0,0 +1,68 @@ +use crate::{BuildOptions, BuildStage, build_process}; +use anyhow::{Context, Result}; +use serde::Deserialize; +use std::{ + collections::BTreeMap, + fs, + path::{Path, PathBuf}, + process::Command, +}; + +const FILES: &[(&str, &str)] = &[ + ( + "lib/rb_gem_installer.rb", + include_str!("../../../gems/rb-gem-installer/lib/rb_gem_installer.rb"), + ), + ( + "lib/rb_gem_installer/ruby_config.rb", + include_str!("../../../gems/rb-gem-installer/lib/rb_gem_installer/ruby_config.rb"), + ), + ( + "exe/rb-ruby-config", + include_str!("../../../gems/rb-gem-installer/exe/rb-ruby-config"), + ), +]; + +#[derive(Clone, Debug, Deserialize)] +pub(crate) struct RubyConfig { + pub(crate) executable: PathBuf, + pub(crate) version: String, + pub(crate) config: BTreeMap, + pub(crate) linker: Vec, + pub(crate) libraries: Vec, +} + +impl RubyConfig { + pub(crate) fn read(options: &BuildOptions) -> Result { + let library = tempfile::Builder::new() + .prefix("rb-ruby-config-") + .tempdir()?; + for (name, source) in FILES { + let path = library.path().join(name); + fs::create_dir_all(path.parent().context("Missing Ruby helper directory")?)?; + fs::write(path, source)?; + } + let mut command = Command::new(&options.ruby); + command + .args(["--disable=gems", "-I"]) + .arg(library.path().join("lib")) + .arg(library.path().join("exe/rb-ruby-config")); + command + .envs(&options.environment) + .env_remove("RUBYOPT") + .env_remove("RUBYLIB") + .env_remove("RUBYGEMS_GEMDEPS"); + let output = build_process::run( + command, + Path::new(""), + BuildStage::Configure, + options.timeout, + &mut |_| {}, + )?; + serde_json::from_slice(&output.stdout).context("Reading Ruby build configuration") + } + + pub(crate) fn get(&self, key: &str) -> &str { + self.config.get(key).map_or("", String::as_str) + } +} diff --git a/crates/rb-gem-installer/tests/building.rs b/crates/rb-gem-installer/tests/building.rs new file mode 100644 index 0000000..7c779e0 --- /dev/null +++ b/crates/rb-gem-installer/tests/building.rs @@ -0,0 +1,711 @@ +use flate2::{Compression, write::GzEncoder}; +use rb_gem_installer::{BuildEvent, BuildOptions, NativeBuilder}; +use std::{ + fs, + io::Write, + path::{Path, PathBuf}, + process::Command, + time::Duration, +}; + +fn ruby() -> PathBuf { + std::env::var_os("RB_GEM_BUILD_RUBY") + .unwrap_or_else(|| "ruby".into()) + .into() +} +fn builder() -> NativeBuilder { + let mut options = BuildOptions::new(ruby()); + options + .environment + .insert("CCACHE_DISABLE".into(), "1".into()); + NativeBuilder::new(options).unwrap() +} +fn metadata(extension: &str) -> rb_gem_archive::GemMetadata { + let root = tempfile::tempdir().unwrap(); + let archive = archive(extension, &[]); + rb_gem_installer::install(archive.as_slice(), root.path().join("gem")).unwrap() +} +fn gzip(bytes: &[u8]) -> Vec { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(bytes).unwrap(); + encoder.finish().unwrap() +} +fn archive(extension: &str, files: &[(&str, &[u8])]) -> Vec { + archive_extensions(&[extension], files) +} +fn archive_extensions(extensions: &[&str], files: &[(&str, &[u8])]) -> Vec { + fn tar(files: &[(&str, Vec)]) -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + for (path, bytes) in files { + let mut header = tar::Header::new_gnu(); + header.set_mode(0o644); + header.set_size(bytes.len() as u64); + header.set_cksum(); + builder + .append_data(&mut header, path, bytes.as_slice()) + .unwrap(); + } + builder.into_inner().unwrap() + } + let data = tar(&files + .iter() + .map(|(path, bytes)| (*path, bytes.to_vec())) + .collect::>()); + let extensions = serde_json::to_string(extensions).unwrap(); + tar(&[("metadata.gz", gzip(format!("name: dinner\nversion: 1.2.3\nplatform: ruby\nextensions: {extensions}\nrequire_paths: [lib]\n").as_bytes())), ("data.tar.gz", gzip(&data))]) +} +fn copy_fixture(kind: &str, root: &Path) { + let source = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/native") + .join(kind); + for entry in fs::read_dir(source).unwrap() { + let entry = entry.unwrap(); + fs::copy(entry.path(), root.join(entry.file_name())).unwrap(); + } +} +fn rake_options() -> BuildOptions { + let output = Command::new(ruby()).args(["--disable=gems", "-rrbconfig", "-e", "puts Dir.glob(File.join(RbConfig::CONFIG.fetch('rubylibprefix'), 'gems', RbConfig::CONFIG.fetch('ruby_version'), 'gems', 'rake-*', 'lib')).last"]).output().unwrap(); + assert!(output.status.success()); + let path = PathBuf::from(String::from_utf8(output.stdout).unwrap().trim()); + let version = Command::new(ruby()) + .args(["--disable=gems", "-I"]) + .arg(&path) + .args(["-rrake/version", "-e", "puts Rake::VERSION"]) + .output() + .unwrap(); + assert!(version.status.success()); + let mut options = BuildOptions::new(ruby()); + options.dependencies.push(rb_microgems::InstalledGem { + name: "rake".into(), + version: String::from_utf8(version.stdout).unwrap().trim().into(), + platform: "ruby".into(), + root: path.parent().unwrap().into(), + require_paths: vec!["lib".into()], + metadata: Default::default(), + executables: vec!["rake".into()], + bindir: "exe".into(), + }); + options +} + +fn require(root: &Path, program: &str) { + let output = Command::new(ruby()) + .args(["--disable=gems", "-I"]) + .arg(root.join("lib")) + .args(["-e", program]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "42"); +} + +#[test] +fn failed_build_rolls_back_and_streams_diagnostics() { + let root = tempfile::tempdir().unwrap(); + let archive = archive( + "extconf.rb", + &[("extconf.rb", b"warn 'missing dependency'; exit 1\n")], + ); + let mut output = Vec::new(); + let error = rb_gem_installer::install_with_builder( + archive.as_slice(), + root.path().join("gem"), + &builder(), + |event| { + if let BuildEvent::Output { bytes, .. } = event { + output.extend(bytes); + } + }, + ) + .unwrap_err(); + assert!(format!("{error:#}").contains("missing dependency")); + assert!(String::from_utf8_lossy(&output).contains("missing dependency")); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); +} + +#[test] +fn timed_out_build_rolls_back() { + let root = tempfile::tempdir().unwrap(); + let archive = archive( + "extconf.rb", + &[("extconf.rb", b"puts 'started'; STDOUT.flush; sleep 10\n")], + ); + let mut options = BuildOptions::new(ruby()); + options.timeout = Duration::from_secs(1); + let builder = NativeBuilder::new(options).unwrap(); + let error = rb_gem_installer::install_with_builder( + archive.as_slice(), + root.path().join("gem"), + &builder, + |_| {}, + ) + .unwrap_err(); + assert!(format!("{error:#}").contains("timed out")); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); +} + +#[test] +fn unsupported_builders_and_escaping_declarations_are_rejected() { + for extension in ["script.py", "../extconf.rb"] { + let root = tempfile::tempdir().unwrap(); + fs::write(root.path().join("script.py"), "").unwrap(); + assert!( + builder() + .build(&metadata(extension), root.path(), |_| {}) + .is_err() + ); + } +} + +#[test] +fn configure_builder_installs_outputs() { + let root = tempfile::tempdir().unwrap(); + copy_fixture("configure", root.path()); + let mut configure_arguments = Vec::new(); + let report = builder() + .build(&metadata("configure"), root.path(), |event| { + if let BuildEvent::Started { + stage: rb_gem_installer::BuildStage::Configure, + arguments, + .. + } = event + { + configure_arguments = arguments; + } + }) + .unwrap(); + assert!( + configure_arguments[1] + .to_string_lossy() + .starts_with("--prefix=./") + ); + assert_eq!(report.extensions, [PathBuf::from("configure")]); + assert!(!report.artifacts.is_empty()); + require( + root.path(), + "require 'configure_output'; puts CONFIGURE_BUILD_ANSWER", + ); +} + +#[cfg(unix)] +#[test] +fn compiles_and_requires_a_c_extension_with_spaces_in_the_path() { + let root = tempfile::tempdir().unwrap(); + let directory = root.path().join("gem with spaces"); + fs::create_dir(&directory).unwrap(); + copy_fixture("c", &directory); + let mut started = Vec::new(); + let report = builder() + .build(&metadata("extconf.rb"), &directory, |event| { + if let BuildEvent::Started { stage, .. } = event { + started.push(stage); + } + }) + .unwrap(); + assert_eq!( + started, + [ + rb_gem_installer::BuildStage::Configure, + rb_gem_installer::BuildStage::Compile, + rb_gem_installer::BuildStage::Install + ] + ); + assert!(!report.artifacts.is_empty()); + assert!( + report + .artifacts + .iter() + .any(|path| path.parent() == Some(Path::new(""))) + ); + assert!( + report + .artifacts + .iter() + .all(|path| directory.join(path).is_file()) + ); + require( + &directory, + "require 'butler_native'; puts ButlerNative.answer", + ); + let library = fs::read_dir(directory.join("lib")) + .unwrap() + .next() + .unwrap() + .unwrap() + .path(); + assert!( + report + .artifacts + .contains(&library.strip_prefix(&directory).unwrap().to_path_buf()) + ); +} + +#[cfg(unix)] +#[test] +fn compiles_and_requires_a_rust_extension() { + let root = tempfile::tempdir().unwrap(); + copy_fixture("cargo", root.path()); + let report = builder() + .build(&metadata("Cargo.toml"), root.path(), |_| {}) + .unwrap(); + assert!(!report.artifacts.is_empty()); + require( + root.path(), + "require 'butler_rust'; abort unless defined?(ButlerRust); puts 42", + ); +} + +#[test] +fn rake_builder_uses_explicit_dependency_load_paths() { + let root = tempfile::tempdir().unwrap(); + copy_fixture("rake", root.path()); + let output = Command::new(ruby()).args(["--disable=gems", "-rrbconfig", "-e", "puts Dir.glob(File.join(RbConfig::CONFIG.fetch('rubylibprefix'), 'gems', RbConfig::CONFIG.fetch('ruby_version'), 'gems', 'rake-*', 'lib')).last"]).output().unwrap(); + assert!(output.status.success()); + let path = String::from_utf8(output.stdout).unwrap(); + assert!( + !path.trim().is_empty(), + "Test Ruby must include its bundled rake gem" + ); + let mut options = BuildOptions::new(ruby()); + options.load_paths.push(path.trim().into()); + let builder = NativeBuilder::new(options).unwrap(); + let report = builder + .build(&metadata("Rakefile"), root.path(), |_| {}) + .unwrap(); + assert!(!report.artifacts.is_empty()); + require(root.path(), "require 'rake_output'; puts RAKE_BUILD_ANSWER"); +} + +#[cfg(unix)] +#[test] +fn cmake_builder_compiles_an_extension() { + let root = tempfile::tempdir().unwrap(); + fs::create_dir(root.path().join("ext")).unwrap(); + copy_fixture("cmake", &root.path().join("ext")); + fs::write( + root.path().join("butler_native.c"), + include_bytes!("fixtures/native/c/butler_native.c"), + ) + .unwrap(); + let report = builder() + .build(&metadata("ext/CMakeLists.txt"), root.path(), |_| {}) + .unwrap(); + assert!(!report.artifacts.is_empty()); + require( + root.path(), + "require 'butler_native'; puts ButlerNative.answer", + ); +} + +#[cfg(unix)] +#[test] +fn panicking_reporters_do_not_leave_build_processes_or_partial_installs() { + let root = tempfile::tempdir().unwrap(); + let archive = archive( + "extconf.rb", + &[("extconf.rb", b"puts Process.pid; STDOUT.flush; sleep 10\n")], + ); + let mut pid = None; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + rb_gem_installer::install_with_builder( + archive.as_slice(), + root.path().join("gem"), + &builder(), + |event| { + if let BuildEvent::Output { bytes, .. } = event { + pid = String::from_utf8_lossy(&bytes).trim().parse::().ok(); + panic!("reporter failed"); + } + }, + ) + })); + assert!(result.is_err()); + let pid = rustix::process::Pid::from_raw(pid.unwrap()).unwrap(); + assert!(rustix::process::test_kill_process(pid).is_err()); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); +} + +#[test] +fn large_stdout_and_stderr_do_not_deadlock_or_lose_final_errors() { + let root = tempfile::tempdir().unwrap(); + let archive = archive("extconf.rb", &[("extconf.rb", b"STDOUT.write('x' * 2000000); STDERR.write('y' * 2000000); STDERR.puts('final failure'); exit 1\n")]); + let mut bytes = 0; + let error = rb_gem_installer::install_with_builder( + archive.as_slice(), + root.path().join("gem"), + &builder(), + |event| { + if let BuildEvent::Output { bytes: output, .. } = event { + bytes += output.len(); + } + }, + ) + .unwrap_err(); + assert!(format!("{error:#}").contains("final failure")); + assert!(bytes >= 4_000_000); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); +} + +#[test] +fn failed_builds_preserve_mkmf_diagnostics_before_rollback() { + let root = tempfile::tempdir().unwrap(); + let archive = archive( + "extconf.rb", + &[( + "extconf.rb", + b"File.write('mkmf.log', 'missing compiler header'); exit 1\n", + )], + ); + let error = rb_gem_installer::install_with_builder( + archive.as_slice(), + root.path().join("gem"), + &builder(), + |_| {}, + ) + .unwrap_err(); + assert!(format!("{error:#}").contains("missing compiler header")); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); +} + +#[test] +fn missing_build_tools_roll_back() { + let root = tempfile::tempdir().unwrap(); + let archive = archive( + "extconf.rb", + &[("extconf.rb", b"File.write('Makefile', '')\n")], + ); + let mut options = BuildOptions::new(ruby()); + options.make = Some(root.path().join("missing-make").into_os_string()); + let error = rb_gem_installer::install_with_builder( + archive.as_slice(), + root.path().join("gem"), + &NativeBuilder::new(options).unwrap(), + |_| {}, + ) + .unwrap_err(); + assert!(format!("{error:#}").contains("Cannot start build tool")); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); +} + +#[cfg(unix)] +#[test] +fn empty_require_paths_install_into_the_gem_root() { + let root = tempfile::tempdir().unwrap(); + copy_fixture("c", root.path()); + let mut metadata = metadata("extconf.rb"); + metadata.require_paths = vec![String::new()]; + let report = builder().build(&metadata, root.path(), |_| {}).unwrap(); + assert!(!report.artifacts.is_empty()); + let output = Command::new(ruby()) + .args(["--disable=gems", "-I"]) + .arg(root.path()) + .args(["-e", "require 'butler_native'; puts ButlerNative.answer"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "42"); +} + +#[cfg(unix)] +#[test] +fn dot_slash_require_paths_are_the_gem_root() { + let root = tempfile::tempdir().unwrap(); + copy_fixture("c", root.path()); + let mut metadata = metadata("extconf.rb"); + metadata.require_paths = vec!["./".to_owned()]; + let report = builder().build(&metadata, root.path(), |_| {}).unwrap(); + assert!(!report.artifacts.is_empty()); + let output = Command::new(ruby()) + .args(["--disable=gems", "-I"]) + .arg(root.path()) + .args(["-e", "require 'butler_native'; puts ButlerNative.answer"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "42"); +} + +#[test] +fn make_environment_override_is_used() { + let root = tempfile::tempdir().unwrap(); + let archive = archive( + "extconf.rb", + &[("extconf.rb", b"File.write('Makefile', '')\n")], + ); + let mut options = BuildOptions::new(ruby()); + options.environment.insert( + "MAKE".into(), + root.path().join("missing-make").into_os_string(), + ); + let error = rb_gem_installer::install_with_builder( + archive.as_slice(), + root.path().join("gem"), + &NativeBuilder::new(options).unwrap(), + |_| {}, + ) + .unwrap_err(); + assert!(format!("{error:#}").contains("Cannot start build tool")); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); +} + +#[cfg(unix)] +#[test] +fn explicit_build_dependency_is_activated_without_external_rubygems() { + let root = tempfile::tempdir().unwrap(); + let dependency = tempfile::tempdir().unwrap(); + fs::create_dir(dependency.path().join("lib")).unwrap(); + fs::write( + dependency.path().join("lib/helper_api.rb"), + "HELPER_ANSWER = 42\n", + ) + .unwrap(); + copy_fixture("c", root.path()); + copy_fixture("compatibility", root.path()); + let mut options = BuildOptions::new(ruby()); + options + .environment + .insert("CCACHE_DISABLE".into(), "1".into()); + options.arguments.push("--with-extra=a b".into()); + options.environment.insert("MAKEFLAGS".into(), "-s".into()); + options.dependencies.push(rb_microgems::InstalledGem { + name: "build-helper".into(), + version: "2.1.0".into(), + platform: "ruby".into(), + root: dependency.path().into(), + require_paths: vec!["lib".into()], + metadata: Default::default(), + executables: vec![], + bindir: "bin".into(), + }); + let builder = NativeBuilder::new(options).unwrap(); + let report = builder + .build(&metadata("extconf.rb"), root.path(), |_| {}) + .unwrap(); + assert!(!report.artifacts.is_empty()); + require( + root.path(), + "require 'butler_native'; puts ButlerNative.answer", + ); +} + +#[cfg(unix)] +#[test] +fn native_build_can_load_a_completed_native_dependency() { + let dependency = tempfile::tempdir().unwrap(); + copy_fixture("c", dependency.path()); + let dependency_metadata = metadata("extconf.rb"); + builder() + .build(&dependency_metadata, dependency.path(), |_| {}) + .unwrap(); + let root = tempfile::tempdir().unwrap(); + copy_fixture("c", root.path()); + copy_fixture("dependent", root.path()); + let mut consumer = metadata("extconf.rb"); + consumer.id.name = "guest".into(); + let mut options = BuildOptions::new(ruby()); + options + .environment + .insert("CCACHE_DISABLE".into(), "1".into()); + options.environment.insert( + "EXPECTED_DEPENDENCY_ROOT".into(), + dependency.path().as_os_str().into(), + ); + options.dependencies.push(rb_microgems::InstalledGem { + name: "dinner".into(), + version: "1.2.3".into(), + platform: "ruby".into(), + root: dependency.path().into(), + require_paths: vec!["lib".into()], + metadata: Default::default(), + executables: vec![], + bindir: "bin".into(), + }); + let report = NativeBuilder::new(options) + .unwrap() + .build(&consumer, root.path(), |_| {}) + .unwrap(); + assert!(!report.artifacts.is_empty()); + require( + root.path(), + "require 'butler_native'; puts ButlerNative.answer", + ); +} + +#[cfg(unix)] +#[test] +fn rake_builder_activates_an_explicit_dependency_without_load_path_overrides() { + let root = tempfile::tempdir().unwrap(); + copy_fixture("rake", root.path()); + let options = rake_options(); + let report = NativeBuilder::new(options) + .unwrap() + .build(&metadata("Rakefile"), root.path(), |_| {}) + .unwrap(); + assert!(!report.artifacts.is_empty()); + require(root.path(), "require 'rake_output'; puts RAKE_BUILD_ANSWER"); +} + +#[cfg(unix)] +#[test] +fn cpp_extension_uses_the_callers_compiler_flags() { + let root = tempfile::tempdir().unwrap(); + copy_fixture("cpp", root.path()); + let mut options = BuildOptions::new(ruby()); + options + .environment + .insert("CCACHE_DISABLE".into(), "1".into()); + options.arguments.push("--with-cxxflags=-std=c++17".into()); + let report = NativeBuilder::new(options) + .unwrap() + .build(&metadata("extconf.rb"), root.path(), |_| {}) + .unwrap(); + assert!(!report.artifacts.is_empty()); + require(root.path(), "require 'butler_cpp'; puts ButlerCpp.answer"); +} + +#[cfg(unix)] +#[test] +fn staged_native_installation_loads_through_manifest_activation() { + let root = tempfile::tempdir().unwrap(); + let destination = root.path().join("installed gem"); + let archive = archive( + "ext/native/extconf.rb", + &[ + ( + "ext/native/extconf.rb", + include_bytes!("fixtures/native/c/extconf.rb"), + ), + ( + "ext/native/butler_native.c", + include_bytes!("fixtures/native/c/butler_native.c"), + ), + ( + "lib/dinner.rb", + include_bytes!("fixtures/native/runtime/dinner.rb"), + ), + ], + ); + let (metadata, report) = rb_gem_installer::install_with_builder( + archive.as_slice(), + &destination, + &builder(), + |_| {}, + ) + .unwrap(); + assert_eq!(report.extensions, [PathBuf::from("ext/native/extconf.rb")]); + assert!( + report + .artifacts + .iter() + .all(|path| destination.join(path).is_file()) + ); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 1); + let gem = rb_microgems::InstalledGem { + name: metadata.id.name, + version: metadata.id.version.to_string(), + platform: metadata.id.platform.to_string(), + root: destination.clone(), + require_paths: metadata.require_paths, + metadata: metadata.metadata, + executables: metadata.executables, + bindir: metadata.bindir, + }; + let environment = rb_microgems::Environment::new(&[gem], &[]).unwrap(); + let output = Command::new(ruby()) + .args(["--disable=gems", "-I"]) + .arg(environment.load_path()) + .args(["-rmicrogems/compatibility", "-e", "require 'dinner'"]) + .env("RB_MICROGEMS_MANIFEST", environment.manifest_path()) + .env("EXPECTED_GEM_ROOT", &destination) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "42"); +} + +#[cfg(unix)] +#[test] +fn a_later_extension_failure_rolls_back_completed_extensions() { + let root = tempfile::tempdir().unwrap(); + let archive = archive_extensions( + &["ext/first/extconf.rb", "ext/second/extconf.rb"], + &[ + ( + "ext/first/extconf.rb", + include_bytes!("fixtures/native/c/extconf.rb"), + ), + ( + "ext/first/butler_native.c", + include_bytes!("fixtures/native/c/butler_native.c"), + ), + ( + "ext/second/extconf.rb", + include_bytes!("fixtures/native/failing/extconf.rb"), + ), + ], + ); + let mut completed = Vec::new(); + let error = rb_gem_installer::install_with_builder( + archive.as_slice(), + root.path().join("gem"), + &builder(), + |event| { + if let BuildEvent::Finished { + extension, + stage: rb_gem_installer::BuildStage::Install, + .. + } = event + { + completed.push(extension); + } + }, + ) + .unwrap_err(); + assert_eq!(completed, [PathBuf::from("ext/first/extconf.rb")]); + assert!(format!("{error:#}").contains("second extension failed")); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); +} + +#[test] +fn mkrf_builder_generates_and_runs_its_rakefile() { + let root = tempfile::tempdir().unwrap(); + copy_fixture("rake", root.path()); + fs::rename(root.path().join("Rakefile"), root.path().join("build.rake")).unwrap(); + copy_fixture("mkrf", root.path()); + let mut stages = Vec::new(); + let report = NativeBuilder::new(rake_options()) + .unwrap() + .build(&metadata("mkrf_conf.rb"), root.path(), |event| { + if let BuildEvent::Finished { stage, .. } = event { + stages.push(stage); + } + }) + .unwrap(); + assert_eq!( + stages, + [ + rb_gem_installer::BuildStage::Configure, + rb_gem_installer::BuildStage::Compile + ] + ); + assert_eq!(report.extensions, [PathBuf::from("mkrf_conf.rb")]); + assert!(!report.artifacts.is_empty()); + assert!(root.path().join("Rakefile").is_file()); + require(root.path(), "require 'rake_output'; puts RAKE_BUILD_ANSWER"); +} diff --git a/crates/rb-gem-installer/tests/corpus.rs b/crates/rb-gem-installer/tests/corpus.rs new file mode 100644 index 0000000..d129e8d --- /dev/null +++ b/crates/rb-gem-installer/tests/corpus.rs @@ -0,0 +1,39 @@ +use std::{fs, path::Path}; + +fn archives(root: &Path, found: &mut Vec) { + for entry in fs::read_dir(root).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + archives(&path, found); + } else if path.extension().is_some_and(|extension| extension == "gem") { + found.push(path); + } + } +} + +#[test] +#[ignore = "requires RB_GEM_CORPUS pointing to the public archive corpus"] +fn installs_the_archive_corpus() { + let root = std::env::var_os("RB_GEM_CORPUS").expect("Set RB_GEM_CORPUS"); + let mut found = Vec::new(); + archives(Path::new(&root), &mut found); + assert!(!found.is_empty(), "Corpus contains no archives"); + found.sort(); + let mut failures = Vec::new(); + for path in &found { + let root = tempfile::tempdir().unwrap(); + if let Err(error) = + rb_gem_installer::install(fs::File::open(path).unwrap(), root.path().join("gem")) + { + failures.push(format!("{}: {error:#}", path.display())); + } + } + assert!( + failures.is_empty(), + "{} of {} failed:\n{}", + failures.len(), + found.len(), + failures.join("\n") + ); + println!("Installed {} archives", found.len()); +} diff --git a/crates/rb-gem-installer/tests/fixtures/database_cleaner-2.0.2.gem b/crates/rb-gem-installer/tests/fixtures/database_cleaner-2.0.2.gem new file mode 100644 index 0000000000000000000000000000000000000000..a0c2aec18580110d5ee41c7ae7e8f88731689c2a GIT binary patch literal 4608 zcmc~zElEsCEJ@T$uVSDTFaQD*6B7my4Fu@4p`nQZgQ1CufsvWHxuLl+gMp#3iHRwL zf&r}@glukUaY8f@({lvJcp)zuyW`Q}rLjo0Wnbcl(HZC)tKt(9`~+}tBq`V*{|{;Xa%xw7QeDi2X1CF4s!3g3OT z{4ST?wP?ZmLkh=Fot9r9w434djZ}WG{4GMB)$VgP9LP|fG(m|eoUNtVAy3iePMZIb z`nlUVmR4WbwC1$x%U%y@d7Z_UZ5@iGb}nvQzZhj=xjB{l?Ah22E2~s*#S7$pSkts_ zqOZHYgS&y{IrblxFBp0}-b%?z9F1S3pt8lf#oEl+%`c;` z?QXAA=Kbd$}Ptlk@;$7gwOkIqr8p}4NuTJM$pYgR?g-R1a9c=hMp12=o>r!+{cop7k;(67&; z8z;E1Jze%Nw^&!={wKv3;!D>r`M7D5lA28YrHBaz(;^jK*80Tkp3!1>$Ehchapsoa zE1ft0TH9ZtAGCe-p}dU`cmDk|bNbbd+VWMOPE4#fyL`~=>!TS?+-CoRrEX`>eEqfK z+$tO0_{l47zsaj(R#yJ_ul@jI=Zb?-g2(okS{CKhU3{bGE_bK+TfT3D+lz&4gEiHU zx(gNp;IbcB0uEsLZ(w9(hF<;~o0*T6|FAfsC5r2KQx986- zRE>Ep3^bMn@7>2*zInrJC5@kEkxz8v6jYBVF7ld~CKsHzYmTVb9lS#=*Y=HwZD zWe@Pu{OTVE^y2^j%nWABL(&;IXzAZkvr$7JIU_YWySOyBSg$fMH-|Q*0H|q%r~L<% zL(l&vW=6)N`5!f=Xv0I~|I`UPCyF=-uzcN`BpZ>mYx(`l7cOjEu;0;1G3-!PoAAMg zQzzD3fBp5xzqi_(YHa46pZ~e9ue|2x$KQ{4W>=iI6Y{=!eZ|vjBF+M)A|4*bUY0N0 zv|b4(1y~w(ryqV=yXoG?*OSAa$;_R*!{vbZ!B+S0iykGlwVjxJRN`6CHqVSgt6M8m zKK*%pvf+QF)GW@GIqI*Swm&Kr`~E;K<(;p~sfeiS`jbB1nsoHlm4v5Dvm2*6xn%sF zIoE+(qh`8Br8`&DeNmoM8!vF1-CVolCflR?ev74q!x!Ayq_T4Lwu8JyYnTf-yP6me z1R0+;kN@o`KXLDwnx87$yN~>s`o8hnTVC_EH(Afu=H1BS44f0z`%wA0%uJ!r$K@mC Ye@?u@NSlx!HFY!uMnhmU1O_$)0FyXD4FCWD literal 0 HcmV?d00001 diff --git a/crates/rb-gem-installer/tests/fixtures/database_cleaner-2.0.2.json b/crates/rb-gem-installer/tests/fixtures/database_cleaner-2.0.2.json new file mode 100644 index 0000000..89ed429 --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/database_cleaner-2.0.2.json @@ -0,0 +1,8 @@ +{ + "source": "https://rubygems.org/downloads/database_cleaner-2.0.2.gem", + "sha256": "e356500b35c7113a16bd79adf6e6a3c1da8b615cbd3a67af06ed60f677860794", + "oracle": "RubyGems 4.0.20", + "files": { + "lib/database_cleaner.rb": "231cc228fac7fbebe036204514c42882b1ce1d5ae8bbd9a667a910755118e571" + } +} diff --git a/crates/rb-gem-installer/tests/fixtures/native/c/butler_native.c b/crates/rb-gem-installer/tests/fixtures/native/c/butler_native.c new file mode 100644 index 0000000..933ed11 --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/native/c/butler_native.c @@ -0,0 +1,10 @@ +#include + +static VALUE answer(VALUE self) { + return INT2NUM(42); +} + +void Init_butler_native(void) { + VALUE module = rb_define_module("ButlerNative"); + rb_define_singleton_method(module, "answer", answer, 0); +} diff --git a/crates/rb-gem-installer/tests/fixtures/native/c/extconf.rb b/crates/rb-gem-installer/tests/fixtures/native/c/extconf.rb new file mode 100644 index 0000000..6e02161 --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/native/c/extconf.rb @@ -0,0 +1,8 @@ +require 'mkmf' +raise 'Missing nested build job limit' unless ENV.fetch('NJOBS') == '2' && ENV.fetch('MAKEFLAGS') == '-j2' +raise 'Full RubyGems was loaded' if $LOADED_FEATURES.any? { |path| path.include?('rubygems/core_ext/kernel_require') } +raise 'Bundler was loaded' if defined?(Bundler) +require 'open3' +output, status = Open3.capture2e(Gem.ruby, '-e', "print Gem.loaded_specs.fetch('dinner').version") +raise "Nested Ruby lost compatibility: #{output}" unless status.success? && output == '1.2.3' +create_makefile('butler_native') diff --git a/crates/rb-gem-installer/tests/fixtures/native/cargo/Cargo.lock b/crates/rb-gem-installer/tests/fixtures/native/cargo/Cargo.lock new file mode 100644 index 0000000..d325526 --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/native/cargo/Cargo.lock @@ -0,0 +1,5 @@ +version = 4 + +[[package]] +name = "butler-rust" +version = "0.1.0" diff --git a/crates/rb-gem-installer/tests/fixtures/native/cargo/Cargo.toml b/crates/rb-gem-installer/tests/fixtures/native/cargo/Cargo.toml new file mode 100644 index 0000000..1a80b80 --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/native/cargo/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "butler-rust" +version = "0.1.0" +edition = "2021" + +[lib] +path = "lib.rs" +crate-type = ["cdylib"] diff --git a/crates/rb-gem-installer/tests/fixtures/native/cargo/lib.rs b/crates/rb-gem-installer/tests/fixtures/native/cargo/lib.rs new file mode 100644 index 0000000..9f95c8c --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/native/cargo/lib.rs @@ -0,0 +1,10 @@ +use std::ffi::c_char; + +extern "C" { + fn rb_define_module(name: *const c_char) -> usize; +} + +#[no_mangle] +pub unsafe extern "C" fn Init_butler_rust() { + rb_define_module(b"ButlerRust\0".as_ptr().cast()); +} diff --git a/crates/rb-gem-installer/tests/fixtures/native/cmake/CMakeLists.txt b/crates/rb-gem-installer/tests/fixtures/native/cmake/CMakeLists.txt new file mode 100644 index 0000000..d533ef8 --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/native/cmake/CMakeLists.txt @@ -0,0 +1,11 @@ +cmake_minimum_required(VERSION 3.18) +project(butler_cmake C) +find_package(Ruby REQUIRED) +add_library(butler_native MODULE ../butler_native.c) +target_include_directories(butler_native PRIVATE ${Ruby_INCLUDE_DIRS}) +set_target_properties(butler_native PROPERTIES PREFIX "") +if(APPLE) + target_link_options(butler_native PRIVATE "-undefined" "dynamic_lookup") +endif() +execute_process(COMMAND ${Ruby_EXECUTABLE} --disable=gems -rrbconfig -e "print RbConfig::CONFIG['DLEXT']" OUTPUT_VARIABLE RUBY_DLEXT) +set_target_properties(butler_native PROPERTIES SUFFIX ".${RUBY_DLEXT}") diff --git a/crates/rb-gem-installer/tests/fixtures/native/compatibility/extconf.rb b/crates/rb-gem-installer/tests/fixtures/native/compatibility/extconf.rb new file mode 100644 index 0000000..b1003bd --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/native/compatibility/extconf.rb @@ -0,0 +1,9 @@ +require 'mkmf' +raise 'Caller make flags were lost' unless ENV.fetch('MAKEFLAGS') == '-s -j2' +require 'rubygems/command' +raise 'Build arguments changed' unless Gem::Command.build_args == ['--with-extra=a b'] && ARGV.include?('--with-extra=a b') +require 'helper_api' +raise 'Wrong helper API' unless HELPER_ANSWER == 42 +raise 'Dependency was not activated' unless Gem.loaded_specs.fetch('build-helper').version.to_s == '2.1.0' +raise 'Wrong current gem root' unless Gem.loaded_specs.fetch('dinner').full_gem_path == File.realpath(__dir__) +create_makefile('butler_native') diff --git a/crates/rb-gem-installer/tests/fixtures/native/configure/configure b/crates/rb-gem-installer/tests/fixtures/native/configure/configure new file mode 100644 index 0000000..d9e20cf --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/native/configure/configure @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +prefix=${1#--prefix=} +cat > Makefile < "$prefix/configure_output.rb" +MAKE diff --git a/crates/rb-gem-installer/tests/fixtures/native/cpp/butler_cpp.cpp b/crates/rb-gem-installer/tests/fixtures/native/cpp/butler_cpp.cpp new file mode 100644 index 0000000..f9b9b5c --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/native/cpp/butler_cpp.cpp @@ -0,0 +1,14 @@ +#include +#include + +static_assert(__cplusplus >= 201703L, "C++17 is required"); + +static VALUE answer(VALUE self) { + constexpr std::array values{20, 22}; + return INT2NUM(values[0] + values[1]); +} + +extern "C" void Init_butler_cpp(void) { + VALUE module = rb_define_module("ButlerCpp"); + rb_define_singleton_method(module, "answer", answer, 0); +} diff --git a/crates/rb-gem-installer/tests/fixtures/native/cpp/extconf.rb b/crates/rb-gem-installer/tests/fixtures/native/cpp/extconf.rb new file mode 100644 index 0000000..b078792 --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/native/cpp/extconf.rb @@ -0,0 +1,3 @@ +require 'mkmf' +raise 'Missing caller C++ flags' unless with_config('cxxflags') == '-std=c++17' +create_makefile('butler_cpp') diff --git a/crates/rb-gem-installer/tests/fixtures/native/dependent/extconf.rb b/crates/rb-gem-installer/tests/fixtures/native/dependent/extconf.rb new file mode 100644 index 0000000..2a97b5b --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/native/dependent/extconf.rb @@ -0,0 +1,6 @@ +require 'mkmf' +gem 'dinner', '= 1.2.3' +require 'butler_native' +raise 'Compiled dependency could not be used' unless ButlerNative.answer == 42 +raise 'Wrong installed dependency root' unless Gem.loaded_specs.fetch('dinner').full_gem_path == ENV.fetch('EXPECTED_DEPENDENCY_ROOT') +create_makefile('butler_native') diff --git a/crates/rb-gem-installer/tests/fixtures/native/failing/extconf.rb b/crates/rb-gem-installer/tests/fixtures/native/failing/extconf.rb new file mode 100644 index 0000000..85b3418 --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/native/failing/extconf.rb @@ -0,0 +1 @@ +raise 'second extension failed' diff --git a/crates/rb-gem-installer/tests/fixtures/native/mkrf/mkrf_conf.rb b/crates/rb-gem-installer/tests/fixtures/native/mkrf/mkrf_conf.rb new file mode 100644 index 0000000..1a2bdec --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/native/mkrf/mkrf_conf.rb @@ -0,0 +1,2 @@ +require 'fileutils' +FileUtils.cp('build.rake', 'Rakefile') diff --git a/crates/rb-gem-installer/tests/fixtures/native/rake/Rakefile b/crates/rb-gem-installer/tests/fixtures/native/rake/Rakefile new file mode 100644 index 0000000..0f36a25 --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/native/rake/Rakefile @@ -0,0 +1,6 @@ +require 'fileutils' + +task :default do + FileUtils.mkdir_p(ENV.fetch('RUBYARCHDIR')) + File.write(File.join(ENV.fetch('RUBYARCHDIR'), 'rake_output.rb'), 'RAKE_BUILD_ANSWER = 42') +end diff --git a/crates/rb-gem-installer/tests/fixtures/native/runtime/dinner.rb b/crates/rb-gem-installer/tests/fixtures/native/runtime/dinner.rb new file mode 100644 index 0000000..0013b5b --- /dev/null +++ b/crates/rb-gem-installer/tests/fixtures/native/runtime/dinner.rb @@ -0,0 +1,7 @@ +spec = Gem.loaded_specs.fetch('dinner') +raise 'Wrong runtime gem root' unless spec.full_gem_path == ENV.fetch('EXPECTED_GEM_ROOT') +raise 'Wrong runtime require paths' unless spec.require_paths == ['lib'] +raise 'Bundler was loaded' if defined?(Bundler) +raise 'Full RubyGems was loaded' if $LOADED_FEATURES.any? { |path| path.include?('rubygems/core_ext') } +require 'butler_native' +puts ButlerNative.answer diff --git a/crates/rb-gem-installer/tests/installation.rs b/crates/rb-gem-installer/tests/installation.rs new file mode 100644 index 0000000..293346d --- /dev/null +++ b/crates/rb-gem-installer/tests/installation.rs @@ -0,0 +1,363 @@ +use flate2::{Compression, write::GzEncoder}; +use std::{fs, io::Write}; + +fn gzip(bytes: &[u8]) -> Vec { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(bytes).unwrap(); + encoder.finish().unwrap() +} + +fn payload(entries: &[(&str, tar::EntryType, &[u8], u32)]) -> Vec { + let mut archive = tar::Builder::new(Vec::new()); + for (path, kind, bytes, mode) in entries { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(*mode); + header.set_entry_type(*kind); + if kind.is_symlink() || kind.is_hard_link() { + header.set_link_name("../../outside").unwrap(); + } + let path_bytes = path.as_bytes(); + header.as_mut_bytes()[..path_bytes.len()].copy_from_slice(path_bytes); + header.set_cksum(); + archive.append(&header, *bytes).unwrap(); + } + archive.into_inner().unwrap() +} + +fn gem(payload: &[u8]) -> Vec { + let mut archive = tar::Builder::new(Vec::new()); + for (name, bytes) in [ + ( + "metadata.gz", + gzip(b"name: dinner\nversion: 1.2.3\nplatform: ruby\n"), + ), + ("data.tar.gz", gzip(payload)), + ] { + let mut header = tar::Header::new_gnu(); + header.set_size(bytes.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + archive + .append_data(&mut header, name, bytes.as_slice()) + .unwrap(); + } + archive.into_inner().unwrap() +} + +#[test] +fn installs_files_directories_and_executables() { + let root = tempfile::tempdir().unwrap(); + let destination = root.path().join("dinner"); + let archive = gem(&payload(&[ + ("lib/", tar::EntryType::Directory, b"", 0o755), + ( + "lib/dinner.rb", + tar::EntryType::Regular, + b"puts :dinner\n", + 0o644, + ), + ( + "bin/dinner", + tar::EntryType::Regular, + b"#!/usr/bin/env ruby\n", + 0o6755, + ), + ])); + let metadata = rb_gem_installer::install(archive.as_slice(), &destination).unwrap(); + assert_eq!(metadata.id.full_name(), "dinner-1.2.3"); + assert_eq!( + fs::read(destination.join("lib/dinner.rb")).unwrap(), + b"puts :dinner\n" + ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + fs::metadata(destination.join("bin/dinner")) + .unwrap() + .permissions() + .mode() + & 0o7777, + 0o755 + ); + } + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 1); +} + +#[test] +fn rejects_unsafe_paths_and_links_without_partial_installation() { + for (path, kind) in [ + ("../escape", tar::EntryType::Regular), + ("/escape", tar::EntryType::Regular), + ("C:\\escape", tar::EntryType::Regular), + ("..\\escape", tar::EntryType::Regular), + ("lib/../../escape", tar::EntryType::Regular), + ("link", tar::EntryType::Symlink), + ("link", tar::EntryType::Link), + ("pipe", tar::EntryType::Fifo), + ] { + let root = tempfile::tempdir().unwrap(); + let archive = gem(&payload(&[ + ("first", tar::EntryType::Regular, b"ok", 0o644), + (path, kind, b"", 0o644), + ])); + assert!( + rb_gem_installer::install(archive.as_slice(), root.path().join("gem")).is_err(), + "{path}" + ); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0, "{path}"); + } +} + +#[test] +fn rejects_duplicates_and_file_directory_collisions() { + for paths in [ + ["file", "file"], + ["file", "file/nested"], + ["./file", "file"], + ] { + let root = tempfile::tempdir().unwrap(); + let archive = gem(&payload(&[ + (paths[0], tar::EntryType::Regular, b"x".as_slice(), 0o644), + (paths[1], tar::EntryType::Regular, b"y".as_slice(), 0o644), + ])); + assert!(rb_gem_installer::install(archive.as_slice(), root.path().join("gem")).is_err()); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); + } +} + +#[test] +fn leaves_existing_installations_untouched() { + let root = tempfile::tempdir().unwrap(); + let destination = root.path().join("gem"); + fs::create_dir(&destination).unwrap(); + fs::write(destination.join("existing"), b"original").unwrap(); + let archive = gem(&payload(&[("new", tar::EntryType::Regular, b"new", 0o644)])); + assert!(rb_gem_installer::install(archive.as_slice(), &destination).is_err()); + assert_eq!(fs::read(destination.join("existing")).unwrap(), b"original"); + assert!(!destination.join("new").exists()); +} + +#[test] +fn corrupt_gzip_does_not_publish_a_directory() { + let root = tempfile::tempdir().unwrap(); + let mut archive = gem(&payload(&[("file", tar::EntryType::Regular, b"x", 0o644)])); + let marker = archive + .windows(3) + .rposition(|window| window == [0x1f, 0x8b, 8]) + .unwrap(); + archive[marker + 10] ^= 0xff; + assert!(rb_gem_installer::install(archive.as_slice(), root.path().join("gem")).is_err()); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); +} + +#[test] +fn oversized_payload_header_is_rejected_before_writing() { + let mut header = tar::Header::new_gnu(); + header.set_path("large").unwrap(); + header.set_size(1024 * 1024 * 1024 + 1); + header.set_mode(0o644); + header.set_cksum(); + let archive = gem(header.as_bytes()); + let root = tempfile::tempdir().unwrap(); + let error = rb_gem_installer::install(archive.as_slice(), root.path().join("gem")).unwrap_err(); + assert!(error.to_string().contains("size limit"), "{error:#}"); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); +} + +#[test] +fn concurrent_installers_publish_only_one_complete_payload() { + let root = tempfile::tempdir().unwrap(); + let destination = root.path().join("gem"); + let archive = gem(&payload(&[( + "file", + tar::EntryType::Regular, + b"content", + 0o644, + )])); + let barrier = std::sync::Barrier::new(8); + let successes = std::thread::scope(|scope| { + let handles: Vec<_> = (0..8) + .map(|_| { + scope.spawn(|| { + barrier.wait(); + rb_gem_installer::install(archive.as_slice(), &destination).is_ok() + }) + }) + .collect(); + handles + .into_iter() + .map(|handle| usize::from(handle.join().unwrap())) + .sum::() + }); + assert_eq!(successes, 1); + assert_eq!(fs::read(destination.join("file")).unwrap(), b"content"); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 1); +} + +#[test] +fn missing_parent_and_invalid_archives_leave_no_installation() { + let root = tempfile::tempdir().unwrap(); + let archive = gem(&payload(&[])); + assert!( + rb_gem_installer::install(archive.as_slice(), root.path().join("missing/gem")).is_err() + ); + assert!(rb_gem_installer::install(b"not a gem".as_slice(), root.path().join("gem")).is_err()); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); +} + +#[cfg(unix)] +#[test] +fn dangling_destination_symlinks_are_not_followed() { + let root = tempfile::tempdir().unwrap(); + let destination = root.path().join("gem"); + std::os::unix::fs::symlink("missing", &destination).unwrap(); + let archive = gem(&payload(&[( + "file", + tar::EntryType::Regular, + b"content", + 0o644, + )])); + assert!(rb_gem_installer::install(archive.as_slice(), &destination).is_err()); + assert_eq!( + fs::read_link(destination).unwrap(), + std::path::Path::new("missing") + ); + assert!(!root.path().join("missing").exists()); +} + +#[cfg(unix)] +#[test] +fn payload_permissions_respect_the_process_umask() { + use std::os::unix::fs::PermissionsExt; + let root = tempfile::tempdir().unwrap(); + let reference = root.path().join("reference"); + fs::write(&reference, b"reference").unwrap(); + let expected = fs::metadata(reference).unwrap().permissions().mode() & 0o664; + let archive = gem(&payload(&[( + "file", + tar::EntryType::Regular, + b"content", + 0o664, + )])); + let destination = root.path().join("gem"); + rb_gem_installer::install(archive.as_slice(), &destination).unwrap(); + assert_eq!( + fs::metadata(destination.join("file")) + .unwrap() + .permissions() + .mode() + & 0o777, + expected + ); +} + +#[test] +fn identical_files_with_dot_prefixes_are_deduplicated() { + let root = tempfile::tempdir().unwrap(); + let archive = gem(&payload(&[ + ("./file", tar::EntryType::Regular, b"content", 0o644), + ("file", tar::EntryType::Regular, b"content", 0o644), + ])); + let destination = root.path().join("gem"); + rb_gem_installer::install(archive.as_slice(), &destination).unwrap(); + assert_eq!(fs::read(destination.join("file")).unwrap(), b"content"); +} + +#[cfg(unix)] +#[test] +fn relative_symlinks_can_reference_regular_payload_files() { + let mut builder = tar::Builder::new(Vec::new()); + let mut file = tar::Header::new_gnu(); + file.set_size(7); + file.set_mode(0o644); + file.set_cksum(); + builder + .append_data(&mut file, "Cargo.lock", b"content".as_slice()) + .unwrap(); + let mut link = tar::Header::new_gnu(); + link.set_size(0); + link.set_mode(0o777); + link.set_entry_type(tar::EntryType::Symlink); + link.set_link_name("../../Cargo.lock").unwrap(); + link.set_cksum(); + builder + .append_data(&mut link, "ext/native/Cargo.lock", b"".as_slice()) + .unwrap(); + let archive = gem(&builder.into_inner().unwrap()); + let root = tempfile::tempdir().unwrap(); + let destination = root.path().join("gem"); + rb_gem_installer::install(archive.as_slice(), &destination).unwrap(); + assert_eq!( + fs::read(destination.join("ext/native/Cargo.lock")).unwrap(), + b"content" + ); + assert_eq!( + fs::read_link(destination.join("ext/native/Cargo.lock")).unwrap(), + std::path::Path::new("../../Cargo.lock") + ); +} + +#[cfg(unix)] +fn symlink_archive(links: &[(&str, &str)]) -> Vec { + let mut builder = tar::Builder::new(Vec::new()); + let mut header = tar::Header::new_gnu(); + header.set_size(7); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "real/file", b"content".as_slice()) + .unwrap(); + for (path, target) in links { + let mut header = tar::Header::new_gnu(); + header.set_size(0); + header.set_mode(0o777); + header.set_entry_type(tar::EntryType::Symlink); + header.set_link_name(target).unwrap(); + header.set_cksum(); + builder + .append_data(&mut header, path, b"".as_slice()) + .unwrap(); + } + gem(&builder.into_inner().unwrap()) +} + +#[cfg(unix)] +#[test] +fn directory_symlinks_and_forward_chains_remain_inside_the_payload() { + let archive = symlink_archive(&[ + ("directory", "real"), + ("file", "next"), + ("next", "directory/file"), + ]); + let root = tempfile::tempdir().unwrap(); + let destination = root.path().join("gem"); + rb_gem_installer::install(archive.as_slice(), &destination).unwrap(); + assert_eq!(fs::read(destination.join("file")).unwrap(), b"content"); + assert_eq!( + fs::read(destination.join("directory/file")).unwrap(), + b"content" + ); + assert_eq!( + fs::read_link(destination.join("file")).unwrap(), + std::path::Path::new("next") + ); +} + +#[cfg(unix)] +#[test] +fn cycles_dangling_links_and_link_parents_roll_back() { + for links in [ + vec![("a", "b"), ("b", "a")], + vec![("a", "missing")], + vec![("a", "real"), ("a/b", "file")], + vec![("a", "/outside")], + vec![("a", "../outside")], + ] { + let root = tempfile::tempdir().unwrap(); + let archive = symlink_archive(&links); + assert!(rb_gem_installer::install(archive.as_slice(), root.path().join("gem")).is_err()); + assert_eq!(fs::read_dir(root.path()).unwrap().count(), 0); + } +} diff --git a/crates/rb-gem-installer/tests/oracle.rs b/crates/rb-gem-installer/tests/oracle.rs new file mode 100644 index 0000000..7cd407e --- /dev/null +++ b/crates/rb-gem-installer/tests/oracle.rs @@ -0,0 +1,39 @@ +use sha2::{Digest, Sha256}; +use std::{collections::BTreeMap, fs, path::Path}; + +fn files(root: &Path, directory: &Path, found: &mut BTreeMap) { + for entry in fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + files(root, &path, found); + } else { + let name = path + .strip_prefix(root) + .unwrap() + .to_str() + .unwrap() + .replace('\\', "/"); + found.insert( + name, + format!("{:x}", Sha256::digest(fs::read(path).unwrap())), + ); + } + } +} + +#[test] +fn installed_files_match_rubygems() { + let archive = include_bytes!("fixtures/database_cleaner-2.0.2.gem"); + let oracle: serde_json::Value = + serde_json::from_str(include_str!("fixtures/database_cleaner-2.0.2.json")).unwrap(); + assert_eq!(format!("{:x}", Sha256::digest(archive)), oracle["sha256"]); + let root = tempfile::tempdir().unwrap(); + let destination = root.path().join("gem"); + let metadata = rb_gem_installer::install(archive.as_slice(), &destination).unwrap(); + assert_eq!(metadata.id.full_name(), "database_cleaner-2.0.2"); + let mut actual = BTreeMap::new(); + files(&destination, &destination, &mut actual); + let expected: BTreeMap = + serde_json::from_value(oracle["files"].clone()).unwrap(); + assert_eq!(actual, expected); +} diff --git a/crates/rb-microgems/Cargo.toml b/crates/rb-microgems/Cargo.toml new file mode 100644 index 0000000..3c4981d --- /dev/null +++ b/crates/rb-microgems/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "rb-microgems" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Minimal RubyGems API compatibility for isolated gem builds" + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1.0" +tempfile = "3" diff --git a/crates/rb-microgems/README.md b/crates/rb-microgems/README.md new file mode 100644 index 0000000..7b41f27 --- /dev/null +++ b/crates/rb-microgems/README.md @@ -0,0 +1,11 @@ +# rb-microgems + +Minimal RubyGems compatibility for isolated build processes running with `--disable=gems`. The caller provides exact installed gem identities and directories; this crate does not resolve, download or install gems. + +`Environment::new` creates a temporary Ruby load path and JSON manifest. Preload `microgems/compatibility` and set `RB_MICROGEMS_MANIFEST` to its manifest path. The environment must remain alive while its subprocesses run. + +The implementation lives in `MicroGems` under `microgems/`. The `microgems/compatibility` entry point aliases `Gem` to `MicroGems` and installs the `gem` and `require` hooks. Files under `rubygems/` only forward compatibility imports to the implementation. Loading `microgems` directly leaves the `Gem` namespace and Kernel hooks untouched. + +The supported surface includes versions, requirements, platforms, explicit gem activation, specification lookup, tar reading and build helpers. Registered dependencies can activate through `gem` or `require`. Remote fetching, dependency resolution and gem installation are outside this compatibility layer. Unsupported RubyGems feature imports fail explicitly. Bundler and full RubyGems are not loaded. + +The Ruby gem lives in `gems/rb-microgems`. Its `lib/` files are embedded in the Rust crate; no RubyGems installation is needed to materialize them. Run its independent Ruby tests with `ruby gems/rb-microgems/test/all_test.rb` from the repository root. diff --git a/crates/rb-microgems/src/environment.rs b/crates/rb-microgems/src/environment.rs new file mode 100644 index 0000000..433c2c4 --- /dev/null +++ b/crates/rb-microgems/src/environment.rs @@ -0,0 +1,128 @@ +use crate::InstalledGem; +use serde::Serialize; +use std::{ + ffi::OsString, + io, + path::{Path, PathBuf}, +}; + +const FILES: &[(&str, &str)] = &[ + ( + "microgems.rb", + include_str!("../../../gems/rb-microgems/lib/microgems.rb"), + ), + ( + "microgems/command.rb", + include_str!("../../../gems/rb-microgems/lib/microgems/command.rb"), + ), + ( + "microgems/ext/builder.rb", + include_str!("../../../gems/rb-microgems/lib/microgems/ext/builder.rb"), + ), + ( + "microgems/package.rb", + include_str!("../../../gems/rb-microgems/lib/microgems/package.rb"), + ), + ( + "microgems/platform.rb", + include_str!("../../../gems/rb-microgems/lib/microgems/platform.rb"), + ), + ( + "microgems/requirement.rb", + include_str!("../../../gems/rb-microgems/lib/microgems/requirement.rb"), + ), + ( + "microgems/specification.rb", + include_str!("../../../gems/rb-microgems/lib/microgems/specification.rb"), + ), + ( + "microgems/version.rb", + include_str!("../../../gems/rb-microgems/lib/microgems/version.rb"), + ), + ( + "microgems/compatibility.rb", + include_str!("../../../gems/rb-microgems/lib/microgems/compatibility.rb"), + ), + ( + "rubygems.rb", + include_str!("../../../gems/rb-microgems/lib/rubygems.rb"), + ), + ( + "rubygems/command.rb", + include_str!("../../../gems/rb-microgems/lib/rubygems/command.rb"), + ), + ( + "rubygems/ext.rb", + include_str!("../../../gems/rb-microgems/lib/rubygems/ext.rb"), + ), + ( + "rubygems/ext/builder.rb", + include_str!("../../../gems/rb-microgems/lib/rubygems/ext/builder.rb"), + ), + ( + "rubygems/package.rb", + include_str!("../../../gems/rb-microgems/lib/rubygems/package.rb"), + ), + ( + "rubygems/platform.rb", + include_str!("../../../gems/rb-microgems/lib/rubygems/platform.rb"), + ), + ( + "rubygems/requirement.rb", + include_str!("../../../gems/rb-microgems/lib/rubygems/requirement.rb"), + ), + ( + "rubygems/specification.rb", + include_str!("../../../gems/rb-microgems/lib/rubygems/specification.rb"), + ), + ( + "rubygems/version.rb", + include_str!("../../../gems/rb-microgems/lib/rubygems/version.rb"), + ), +]; + +pub struct Environment { + directory: tempfile::TempDir, + manifest: PathBuf, +} + +#[derive(Serialize)] +struct Manifest<'a> { + gems: &'a [InstalledGem], + arguments: Vec<&'a str>, +} + +impl Environment { + pub fn new(gems: &[InstalledGem], arguments: &[OsString]) -> io::Result { + let arguments = arguments + .iter() + .map(|value| { + value.to_str().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidInput, "Build argument is not UTF-8") + }) + }) + .collect::>>()?; + let directory = tempfile::Builder::new().prefix("rb-microgems-").tempdir()?; + for (name, source) in FILES { + let path = directory.path().join(name); + std::fs::create_dir_all(path.parent().unwrap())?; + std::fs::write(path, source)?; + } + let manifest = directory.path().join("manifest.json"); + std::fs::write( + &manifest, + serde_json::to_vec(&Manifest { gems, arguments }).map_err(io::Error::other)?, + )?; + Ok(Self { + directory, + manifest, + }) + } + + pub fn load_path(&self) -> &Path { + self.directory.path() + } + pub fn manifest_path(&self) -> &Path { + &self.manifest + } +} diff --git a/crates/rb-microgems/src/installed_gem.rs b/crates/rb-microgems/src/installed_gem.rs new file mode 100644 index 0000000..48da0a4 --- /dev/null +++ b/crates/rb-microgems/src/installed_gem.rs @@ -0,0 +1,21 @@ +use serde::{Deserialize, Serialize}; +use std::{collections::BTreeMap, path::PathBuf}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct InstalledGem { + pub name: String, + pub version: String, + pub platform: String, + pub root: PathBuf, + pub require_paths: Vec, + #[serde(default)] + pub metadata: BTreeMap, + #[serde(default)] + pub executables: Vec, + #[serde(default = "default_bindir")] + pub bindir: String, +} + +fn default_bindir() -> String { + "bin".into() +} diff --git a/crates/rb-microgems/src/lib.rs b/crates/rb-microgems/src/lib.rs new file mode 100644 index 0000000..74ab7b4 --- /dev/null +++ b/crates/rb-microgems/src/lib.rs @@ -0,0 +1,7 @@ +#![doc = include_str!("../README.md")] + +mod environment; +mod installed_gem; + +pub use environment::Environment; +pub use installed_gem::InstalledGem; diff --git a/crates/rb-microgems/tests/compatibility.rs b/crates/rb-microgems/tests/compatibility.rs new file mode 100644 index 0000000..fad8962 --- /dev/null +++ b/crates/rb-microgems/tests/compatibility.rs @@ -0,0 +1,231 @@ +use rb_microgems::{Environment, InstalledGem}; +use std::{collections::BTreeMap, ffi::OsString, process::Command}; + +fn ruby() -> OsString { + std::env::var_os("RB_GEM_BUILD_RUBY").unwrap_or_else(|| "ruby".into()) +} + +#[test] +fn build_api_contracts_without_rubygems() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir(root.path().join("lib")).unwrap(); + std::fs::write(root.path().join("lib/fixture_api.rb"), "FIXTURE_API = 42\n").unwrap(); + let gem = InstalledGem { + name: "fixture".into(), + version: "2.1.3".into(), + platform: "ruby".into(), + root: root.path().into(), + require_paths: vec!["lib".into()], + metadata: BTreeMap::from([("purpose".into(), "test".into())]), + executables: vec![], + bindir: "bin".into(), + }; + let environment = Environment::new(&[gem], &["--with-extra=a b".into()]).unwrap(); + let output = Command::new(ruby()) + .args(["--disable=gems", "-I"]) + .arg(environment.load_path()) + .arg(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../gems/rb-microgems/test/fixtures/contracts.rb" + )) + .env("RB_MICROGEMS_MANIFEST", environment.manifest_path()) + .env("FIXTURE_ROOT", root.path().join(".")) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("contracts passed")); +} + +#[test] +fn environment_is_removed_on_drop() { + let environment = Environment::new(&[], &[]).unwrap(); + let path = environment.load_path().to_owned(); + assert!(environment.manifest_path().is_file()); + drop(environment); + assert!(!path.exists()); +} + +#[test] +fn version_requirement_and_platform_match_rubygems() { + let fixture = concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../gems/rb-microgems/test/fixtures/version_oracle.rb" + ); + let canonical = Command::new(ruby()) + .arg("-rrubygems") + .arg(fixture) + .output() + .unwrap(); + assert!( + canonical.status.success(), + "{}", + String::from_utf8_lossy(&canonical.stderr) + ); + let environment = Environment::new(&[], &[]).unwrap(); + let compatible = Command::new(ruby()) + .args(["--disable=gems", "-I"]) + .arg(environment.load_path()) + .arg("-rmicrogems/compatibility") + .arg(fixture) + .output() + .unwrap(); + assert!( + compatible.status.success(), + "{}", + String::from_utf8_lossy(&compatible.stderr) + ); + let expected: serde_json::Value = serde_json::from_slice(&canonical.stdout).unwrap(); + let actual: serde_json::Value = serde_json::from_slice(&compatible.stdout).unwrap(); + for key in [ + "comparisons", + "requirements", + "versions", + "platforms", + "generic_platforms", + ] { + assert_eq!(actual[key], expected[key], "RubyGems mismatch for {key}"); + } +} + +#[cfg(unix)] +#[test] +fn rejects_non_utf8_build_arguments() { + use std::os::unix::ffi::OsStringExt; + let error = Environment::new(&[], &[OsString::from_vec(vec![0xff])]) + .err() + .unwrap(); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidInput); +} + +#[test] +fn core_namespace_works_without_the_gem_alias() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir(root.path().join("lib")).unwrap(); + std::fs::write(root.path().join("lib/fixture_api.rb"), "FIXTURE_API = 42\n").unwrap(); + let gem = InstalledGem { + name: "fixture".into(), + version: "2.1.3".into(), + platform: "ruby".into(), + root: root.path().into(), + require_paths: vec!["lib".into()], + metadata: BTreeMap::new(), + executables: vec![], + bindir: "bin".into(), + }; + let environment = Environment::new(&[gem], &[]).unwrap(); + let output = Command::new(ruby()) + .args(["--disable=gems", "-I"]) + .arg(environment.load_path()) + .arg(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../gems/rb-microgems/test/fixtures/namespaces.rb" + )) + .env("RB_MICROGEMS_MANIFEST", environment.manifest_path()) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + "namespaces passed" + ); +} + +#[test] +fn compatibility_imports_work_as_the_first_require() { + let environment = Environment::new(&[], &[]).unwrap(); + for feature in [ + "rubygems", + "rubygems/version", + "rubygems/requirement", + "rubygems/platform", + "rubygems/specification", + "rubygems/command", + "rubygems/package", + "rubygems/ext", + "rubygems/ext/builder", + ] { + let output = Command::new(ruby()) + .args(["--disable=gems", "-I"]) + .arg(environment.load_path()) + .arg(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../gems/rb-microgems/test/fixtures/shims.rb" + )) + .arg(feature) + .output() + .unwrap(); + assert!( + output.status.success(), + "{feature}: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + "shims passed" + ); + } +} + +#[test] +fn compatibility_alias_does_not_replace_loaded_rubygems() { + let environment = Environment::new(&[], &[]).unwrap(); + let output = Command::new(ruby()) + .arg(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../gems/rb-microgems/test/fixtures/existing_rubygems.rb" + )) + .env("MICROGEMS_PATH", environment.load_path()) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&output.stdout).trim(), + "existing RubyGems preserved" + ); +} + +#[test] +fn materialized_library_matches_the_gem_sources() { + fn files( + root: &std::path::Path, + directory: &std::path::Path, + found: &mut BTreeMap>, + ) { + for entry in std::fs::read_dir(directory).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + files(root, &path, found); + } else { + found.insert( + path.strip_prefix(root).unwrap().to_owned(), + std::fs::read(path).unwrap(), + ); + } + } + } + let source = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../gems/rb-microgems/lib"); + let mut expected = BTreeMap::new(); + files(&source, &source, &mut expected); + let environment = Environment::new(&[], &[]).unwrap(); + let mut actual = BTreeMap::new(); + files( + environment.load_path(), + environment.load_path(), + &mut actual, + ); + actual.remove(std::path::Path::new("manifest.json")); + assert_eq!(actual, expected); +} diff --git a/docker/native/Dockerfile b/docker/native/Dockerfile new file mode 100644 index 0000000..2508fa4 --- /dev/null +++ b/docker/native/Dockerfile @@ -0,0 +1,49 @@ +FROM rust:1.98.1-trixie + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates curl xz-utils git build-essential pkg-config \ + autoconf automake libtool bison cmake ninja-build python3 \ + clang-19 libclang-19-dev llvm-19-dev \ + libssl-dev libreadline-dev zlib1g-dev libyaml-dev libffi-dev \ + libgdbm-dev libncurses-dev libicu-dev libyajl-dev \ + imagemagick libmagickwand-dev libpq-dev default-libmysqlclient-dev \ + libsqlite3-dev libxml2-dev libxslt1-dev libidn-dev libcurl4-openssl-dev \ + libbz2-dev liblzma-dev patch unzip \ + && rm -rf /var/lib/apt/lists/* + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ARG RV_VERSION=0.5.3 +ARG RUBY_VERSION=4.0.7 +RUN mkdir -p /tmp/rv-install /opt/rubies \ + && curl -fsSL "https://github.com/spinel-coop/rv/releases/download/v${RV_VERSION}/rv-x86_64-unknown-linux-gnu.tar.xz" \ + | tar -xJ --strip-components=1 -C /tmp/rv-install \ + && install -m 0755 /tmp/rv-install/rv /usr/local/bin/rv \ + && RV_UPDATE_MODE=none rv ruby install --install-dir /opt/rubies "$RUBY_VERSION" \ + && ln -s "/opt/rubies/ruby-${RUBY_VERSION}" /opt/ruby \ + && rm -rf /tmp/rv-install + +ENV PATH="/opt/ruby/bin:${PATH}" \ + LIBCLANG_PATH=/usr/lib/llvm-19/lib \ + CARGO_NET_OFFLINE=false \ + RB_GEM_BUILD_TIMEOUT=1800 \ + RB_GEM_BUILD_JOBS=2 \ + RB_GEM_BUILD_DEPENDENCIES=/opt/build-dependencies.json + +RUN gem install --no-document --bindir /usr/local/bin rake rake-compiler rake-compiler-dock rb_sys mini_portile2 ffi-compiler libyajl2 pkg-config \ + && gem install --no-document --bindir /usr/local/bin libdatadog --version '~> 7.0.0.1.0' \ + && ruby -rjson -e 'File.write("/opt/build-dependencies.json", JSON.pretty_generate(Gem::Specification.map { |s| {name: s.name, version: s.version.to_s, platform: s.platform.to_s, root: s.full_gem_path, require_paths: s.full_require_paths, executables: s.executables, bindir: s.bindir} }))' + +WORKDIR /opt/butler +COPY Cargo.toml Cargo.lock rust-toolchain.toml ./ +COPY crates ./crates +COPY gems ./gems +RUN cargo build --locked --release -p rb-cli -p rb-gem-installer --examples --bin rb \ + && install -m 0755 target/release/rb /usr/local/bin/rb \ + && install -m 0755 target/release/examples/compile /usr/local/bin/butler-compile \ + && install -m 0755 target/release/examples/install /usr/local/bin/butler-install + +ENV RB_RUBIES_DIR=/opt/rubies + +WORKDIR /work +CMD ["bash"] diff --git a/docker/native/Dockerfile.dockerignore b/docker/native/Dockerfile.dockerignore new file mode 100644 index 0000000..d0eca9e --- /dev/null +++ b/docker/native/Dockerfile.dockerignore @@ -0,0 +1,8 @@ +** +!Cargo.toml +!Cargo.lock +!rust-toolchain.toml +!crates/ +!crates/** +!gems/ +!gems/** diff --git a/docker/native/README.md b/docker/native/README.md new file mode 100644 index 0000000..da0fd52 --- /dev/null +++ b/docker/native/README.md @@ -0,0 +1,56 @@ +# Native gem build snapshot + +Build from the repository root. The image includes the current working-tree crate sources, Ruby installed through rv, Rust/Cargo, native build tools, and development libraries for ImageMagick, ICU, YAJL, PostgreSQL, MySQL, SQLite, XML and SSL. Clang/LLVM 19 is provided for Rust extensions that use Bindgen. + +```sh +docker build -f docker/native/Dockerfile -t butler-native:snapshot . +docker run --rm -it -v "$PWD/../gems-corpus:/work/corpus:ro" butler-native:snapshot +``` + +Inside the container: + +```sh +rb version +ruby --version +butler-compile /work/corpus/gems/rmagick/rmagick-7.0.4.gem /work/rmagick ruby +butler-compile /work/corpus/gems/ffi-yajl/ffi-yajl-2.7.11.gem /work/ffi-yajl ruby +butler-compile /path/to/charlock_holmes.gem /work/charlock ruby --with-cxxflags=-std=c++17 +``` + +Use the actual archive paths from the corpus manifest. Destinations must not already exist. `butler-compile` and `butler-install` are the standalone installer examples; the CLI snapshot is not wired to that installer yet. + +RubyGems installs the build helpers, the Ruby `pkg-config` gem, `libyajl2`, and Libdatadog 7 during image preparation. Their metadata is exported to `/opt/build-dependencies.json` for microgems; native build subprocesses use the minimal compatibility layer. Add gem-specific dependencies to that registry when needed. Cargo can download missing crates, and keeps its cache under `/usr/local/cargo`; mount a named volume there to retain downloads between containers. + +Run the smoke check against the local corpus: + +```sh +docker run --rm \ + -v "$PWD/../gems-corpus:/corpus:ro" \ + -v "$PWD/docker/native/smoke.rb:/smoke.rb:ro" \ + butler-native:snapshot ruby /smoke.rb +``` + +It compiles RMagick, FFI YAJL, Charlock Holmes, CommonMarker, Prometheus Client MMAP and Rubydex through Butler, then loads RMagick and FFI YAJL and exercises their APIs. Pass gem names after `/smoke.rb` to select a smaller set. The existing corpus runners supply their own dependency registry and force offline Cargo; use this check for the image's additional prerequisites and online Cargo downloads. + +For the heavy builds, run this in two terminals with `job=grpc` and `job=libv8-node`. These require enough memory and CPU for both containers; the settings below allow eight jobs and up to 20 GiB per container. gRPC builds Protobuf first. MiniRacer builds after Libv8 succeeds. The runner saves timings and logs and checks runtime loading without RubyGems. + +```sh +job=grpc +mkdir -p ../gems-corpus/docker-parallel +docker run --rm --cpus=8 --memory=20g --memory-swap=20g \ + -e RB_GEM_BUILD_JOBS=8 -e RB_GEM_BUILD_TIMEOUT=1800 \ + -v "$PWD/../gems-corpus:/corpus:ro" \ + -v "$PWD/../gems-corpus/docker-parallel:/results" \ + -v "$PWD/docker/native/heavy.rb:/heavy.rb:ro" \ + butler-native:snapshot ruby /heavy.rb "$job" +``` + +Use a fresh results directory for another run; successful installation destinations cannot be overwritten. + +The eight-job parallel run built gRPC 1.81.1 in 25 minutes 41 seconds and passed its load check with RubyGems disabled. Libv8 Node 24.12.0.1 still exceeded the 30-minute command limit while compiling, so MiniRacer was not attempted. These timings describe cold source builds on the local machine, not a guaranteed duration on other hardware. + +Libv8 Node subsequently exceeded a 90-minute limit. A checkpoint of its extracted source and objects was resumed through `NativeBuilder::build` with a three-hour limit and completed in another 83 minutes 42 seconds. The two attempts consumed about 2 hours 54 minutes of command time, including recompilation after regenerating build inputs; this is not a fresh-build timing or a successful 90-minute installation. MiniRacer then compiled in 7 seconds and evaluated JavaScript through microgems with RubyGems disabled. Logs and the separate continuation report are in `../gems-corpus/docker-v8-90m/`. Runtime checks use manifest activation rather than adding every dependency directory directly to Ruby's load path. + +The default Ruby is 4.0.7. Build with `--build-arg RUBY_VERSION=3.3.7` for the older Ddtrace version; its matching Libdatadog prerequisite is included. Sass Embedded's remote installer API remains unsupported. Heavy source builds can still exceed the default 30-minute command limit; override `RB_GEM_BUILD_TIMEOUT` or `RB_GEM_BUILD_JOBS` when running the container. + +The image is currently for x86_64 Linux. It retains sources and Cargo for further compatibility testing. It does not include the corpus or publish an image remotely. diff --git a/docker/native/heavy.rb b/docker/native/heavy.rb new file mode 100644 index 0000000..39a4e2c --- /dev/null +++ b/docker/native/heavy.rb @@ -0,0 +1,39 @@ +require 'json' +require 'fileutils' +require 'rubygems/package' + +$stdout.sync = true +job = ARGV.fetch(0) +steps = case job + when 'grpc' then %w[google-protobuf grpc] + when 'libv8-node' then %w[libv8-node mini_racer] + else abort 'Expected grpc or libv8-node' + end +rows = JSON.parse(File.read('/corpus/manifest.json')).fetch('gems') +records = JSON.parse(File.read(ENV.fetch('RB_GEM_BUILD_DEPENDENCIES'))) +dependencies = "/results/#{job}-dependencies.json" +report = {ruby: RUBY_DESCRIPTION, jobs: ENV.fetch('RB_GEM_BUILD_JOBS'), timeout: ENV.fetch('RB_GEM_BUILD_TIMEOUT'), results: []} +steps.each do |name| + row = rows.find { |gem| gem.fetch('name') == name } or abort "Missing #{name}" + archive = "/corpus/#{row.fetch('path')}" + destination = "/results/builds/#{name}" + FileUtils.mkdir_p(File.dirname(destination)) + File.write(dependencies, JSON.pretty_generate(records)) + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + passed = File.open("/results/#{name}.log", 'w') do |log| + system({'RB_GEM_BUILD_DEPENDENCIES' => dependencies}, 'butler-compile', archive, destination, 'ruby', out: log, err: log) + end + result = {name: name, passed: passed, seconds: (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round(3), partial_installation: !passed && File.exist?(destination)} + report[:results] << result + File.write("/results/#{job}.json", JSON.pretty_generate(report) + "\n") + puts JSON.generate(result) + abort "Failed #{name}; dependent step skipped" unless passed + spec = Gem::Package.new(archive).spec + records << {'name' => spec.name, 'version' => spec.version.to_s, 'platform' => spec.platform.to_s, 'root' => destination, 'require_paths' => spec.require_paths, 'metadata' => spec.metadata, 'executables' => spec.executables, 'bindir' => spec.bindir} +end +File.write("/results/#{job}-runtime.json", JSON.pretty_generate(gems: records)) +code = job == 'grpc' ? 'require "grpc"; raise unless Gem.loaded_specs.key?("grpc"); puts GRPC::VERSION' : 'require "mini_racer"; raise unless Gem.loaded_specs.key?("mini_racer"); raise unless MiniRacer::Context.new.eval("6 * 7") == 42; puts "MiniRacer runtime passed"' +runtime = system({'RB_MICROGEMS_MANIFEST' => "/results/#{job}-runtime.json"}, 'ruby', '--disable=gems', '-I/opt/butler/gems/rb-microgems/lib', '-rmicrogems/compatibility', '-e', code) +report[:runtime_passed] = runtime +File.write("/results/#{job}.json", JSON.pretty_generate(report) + "\n") +abort 'Runtime check failed' unless runtime diff --git a/docker/native/smoke.rb b/docker/native/smoke.rb new file mode 100644 index 0000000..aa2c9c2 --- /dev/null +++ b/docker/native/smoke.rb @@ -0,0 +1,18 @@ +require 'json' + +puts RUBY_DESCRIPTION +abort unless system('rb', 'version') +rows = JSON.parse(File.read('/corpus/manifest.json')).fetch('gems') +names = ARGV.empty? ? %w[rmagick ffi-yajl charlock_holmes commonmarker prometheus-client-mmap rubydex] : ARGV +names.each do |name| + row = rows.find { |gem| gem.fetch('name') == name } or abort "Missing #{name}" + arguments = name == 'charlock_holmes' ? ['--with-cxxflags=-std=c++17'] : [] + abort "FAILED #{name}" unless system('butler-compile', "/corpus/#{row.fetch('path')}", "/work/#{name}", 'ruby', *arguments) + puts "PASSED #{name}" +end +if names.include?('rmagick') + abort unless system('ruby', '-I/work/rmagick/lib', '-rrmagick', '-e', 'raise unless Magick::Image.new(1, 1).columns == 1; puts "RMagick runtime passed"') +end +if names.include?('ffi-yajl') + abort unless system('ruby', '-I/work/ffi-yajl/lib', '-rffi_yajl', '-e', 'raise unless FFI_Yajl::Parser.parse(%q({"ok":true}))["ok"]; puts "ffi-yajl runtime passed"') +end diff --git a/gems/.rubocop.yml b/gems/.rubocop.yml new file mode 100644 index 0000000..f86b138 --- /dev/null +++ b/gems/.rubocop.yml @@ -0,0 +1,21 @@ +AllCops: + TargetRubyVersion: 3.3 + NewCops: enable + DisabledByDefault: true + Include: + - '**/*.rb' + - '**/*.gemspec' + - '**/Rakefile' + - '**/exe/*' + +Lint: + Enabled: true + +Layout: + Enabled: true + +Layout/LineLength: + Enabled: false + +Security: + Enabled: true diff --git a/gems/Gemfile b/gems/Gemfile new file mode 100644 index 0000000..ae178ee --- /dev/null +++ b/gems/Gemfile @@ -0,0 +1,4 @@ +source 'https://rubygems.org' + +gem 'rake', '~> 13.0' +gem 'rubocop', '~> 1.92.0' diff --git a/gems/Gemfile.lock b/gems/Gemfile.lock new file mode 100644 index 0000000..b3494d7 --- /dev/null +++ b/gems/Gemfile.lock @@ -0,0 +1,45 @@ +GEM + remote: https://rubygems.org/ + specs: + ast (2.4.3) + json (3.0.2) + language_server-protocol (3.17.0.6) + lint_roller (1.1.0) + parallel (2.3.0) + parser (3.3.12.0) + ast (~> 2.4.1) + racc + prism (1.9.0) + racc (1.8.1) + rainbow (3.1.1) + rake (13.2.1) + regexp_parser (2.13.1) + rubocop (1.92.0) + json (>= 2.3) + language_server-protocol (~> 3.17.0.2) + lint_roller (~> 1.1.0) + parallel (>= 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.11.3, < 3.0) + rubocop-ast (>= 1.49.0, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 4.0) + rubocop-ast (1.50.0) + parser (>= 3.3.7.2) + prism (~> 1.7) + ruby-progressbar (1.13.0) + unicode-display_width (3.3.0) + unicode-emoji (~> 4.3) + unicode-emoji (4.3.0) + +PLATFORMS + ruby + x86_64-linux + +DEPENDENCIES + rake (~> 13.0) + rubocop (~> 1.92.0) + +BUNDLED WITH + 2.6.9 diff --git a/gems/README.md b/gems/README.md new file mode 100644 index 0000000..69a0381 --- /dev/null +++ b/gems/README.md @@ -0,0 +1,5 @@ +# Ruby gems + +`rb-microgems` provides the native build compatibility layer. `rb-gem-installer` provides the selected Ruby's configuration helper. Rust embeds their runtime files into Butler. + +Install test tools with `gem install rake minitest`. From this directory, run `rake test` for both suites. Install lint tools with `bundle install`, then run `bundle exec rake lint`; `Gemfile.lock` pins the tooling versions. The same tasks are available inside each gem. Lint checks Ruby 3.3 syntax, correctness, layout and security rules. CI runs lint separately from the Ruby test matrix. diff --git a/gems/Rakefile b/gems/Rakefile new file mode 100644 index 0000000..ae1197c --- /dev/null +++ b/gems/Rakefile @@ -0,0 +1,10 @@ +task :test do + ruby File.join(__dir__, 'rb-microgems/test/all_test.rb') + ruby File.join(__dir__, 'rb-gem-installer/test/all_test.rb') +end + +task :lint do + sh 'rubocop', '--config', File.join(__dir__, '.rubocop.yml'), __dir__ +end + +task default: :test diff --git a/gems/rb-gem-installer/README.md b/gems/rb-gem-installer/README.md new file mode 100644 index 0000000..201277c --- /dev/null +++ b/gems/rb-gem-installer/README.md @@ -0,0 +1,5 @@ +# rb-gem-installer + +The Ruby helper for the Rust installer. `RubyButler::GemInstaller::RubyConfig.read` returns the selected interpreter's build configuration. `ruby --disable=gems -Ilib exe/rb-ruby-config` emits it as JSON without RubyGems or Bundler. + +Run `ruby test/all_test.rb` or `rake test`. Minitest and rake are development dependencies only. Build with `gem build rb-gem-installer.gemspec`; the Rust installer embeds and materializes these files directly. diff --git a/gems/rb-gem-installer/Rakefile b/gems/rb-gem-installer/Rakefile new file mode 100644 index 0000000..4866170 --- /dev/null +++ b/gems/rb-gem-installer/Rakefile @@ -0,0 +1,12 @@ +require 'rake/testtask' + +Rake::TestTask.new do |task| + task.libs = ['test'] + task.pattern = 'test/*_test.rb' +end + +task :lint do + sh 'rubocop', '--config', File.expand_path('../.rubocop.yml', __dir__), __dir__ +end + +task default: :test diff --git a/gems/rb-gem-installer/exe/rb-ruby-config b/gems/rb-gem-installer/exe/rb-ruby-config new file mode 100755 index 0000000..c679a0f --- /dev/null +++ b/gems/rb-gem-installer/exe/rb-ruby-config @@ -0,0 +1,5 @@ +#!/usr/bin/env ruby +require 'json' +require 'rb_gem_installer' + +puts JSON.generate(RubyButler::GemInstaller::RubyConfig.read) diff --git a/gems/rb-gem-installer/lib/rb_gem_installer.rb b/gems/rb-gem-installer/lib/rb_gem_installer.rb new file mode 100644 index 0000000..9f13c98 --- /dev/null +++ b/gems/rb-gem-installer/lib/rb_gem_installer.rb @@ -0,0 +1,7 @@ +require_relative 'rb_gem_installer/ruby_config' + +module RubyButler + module GemInstaller + VERSION = '0.3.1' + end +end diff --git a/gems/rb-gem-installer/lib/rb_gem_installer/ruby_config.rb b/gems/rb-gem-installer/lib/rb_gem_installer/ruby_config.rb new file mode 100644 index 0000000..169e7c7 --- /dev/null +++ b/gems/rb-gem-installer/lib/rb_gem_installer/ruby_config.rb @@ -0,0 +1,18 @@ +require 'rbconfig' +require 'shellwords' + +module RubyButler + module GemInstaller + module RubyConfig + def self.read(config: RbConfig::CONFIG, executable: RbConfig.ruby, version: RUBY_VERSION) + { + executable: executable, + version: version, + config: config, + linker: Shellwords.split(config.fetch('CC', 'cc')), + libraries: Shellwords.split(config.fetch('LIBRUBYARG_SHARED', '') + ' ' + config.fetch('LIBS', '')) + } + end + end + end +end diff --git a/gems/rb-gem-installer/rb-gem-installer.gemspec b/gems/rb-gem-installer/rb-gem-installer.gemspec new file mode 100644 index 0000000..3f34334 --- /dev/null +++ b/gems/rb-gem-installer/rb-gem-installer.gemspec @@ -0,0 +1,17 @@ +require_relative 'lib/rb_gem_installer' + +Gem::Specification.new do |spec| + spec.name = 'rb-gem-installer' + spec.version = RubyButler::GemInstaller::VERSION + spec.authors = ['RubyElders'] + spec.summary = 'Ruby configuration helper for native gem builds' + spec.homepage = 'https://github.com/RubyElders/ruby-butler' + spec.licenses = ['MIT', 'Apache-2.0'] + spec.required_ruby_version = '>= 3.3' + spec.files = Dir.chdir(__dir__) { Dir['lib/**/*.rb', 'exe/*', 'README.md'] } + spec.require_paths = ['lib'] + spec.bindir = 'exe' + spec.executables = ['rb-ruby-config'] + spec.add_development_dependency 'minitest', '>= 5.20', '< 7' + spec.add_development_dependency 'rake', '>= 13', '< 14' +end diff --git a/gems/rb-gem-installer/test/all_test.rb b/gems/rb-gem-installer/test/all_test.rb new file mode 100644 index 0000000..e330269 --- /dev/null +++ b/gems/rb-gem-installer/test/all_test.rb @@ -0,0 +1,3 @@ +Dir.glob(File.join(__dir__, '*_test.rb')).each do |path| + require path unless path == File.expand_path(__FILE__) +end diff --git a/gems/rb-gem-installer/test/package_test.rb b/gems/rb-gem-installer/test/package_test.rb new file mode 100644 index 0000000..9f6c8f7 --- /dev/null +++ b/gems/rb-gem-installer/test/package_test.rb @@ -0,0 +1,25 @@ +require 'minitest/autorun' +require 'tmpdir' +require 'rubygems/package' +require_relative '../lib/rb_gem_installer' + +class PackageTest < Minitest::Test + def test_built_gem_contains_the_configuration_helper + root = File.expand_path('..', __dir__) + spec = Gem::Specification.load(File.join(root, 'rb-gem-installer.gemspec')) + assert_equal RubyButler::GemInstaller::VERSION, spec.version.to_s + assert_empty spec.runtime_dependencies + assert_equal ['rb-ruby-config'], spec.executables + Dir.mktmpdir('installer-package-') do |directory| + archive = File.join(directory, 'rb-gem-installer.gem') + capture_io { Dir.chdir(root) { Gem::Package.build(spec, false, true, archive) } } + destination = File.join(directory, 'extracted') + Gem::Package.new(archive).extract_files(destination) + files = Dir.chdir(root) { Dir['lib/**/*.rb', 'exe/*'].sort } + assert_equal files, Dir.chdir(destination) { Dir['lib/**/*.rb', 'exe/*'].sort } + files.each { |path| + assert_equal File.binread(File.join(root, path)), File.binread(File.join(destination, path)), path + } + end + end +end diff --git a/gems/rb-gem-installer/test/ruby_config_test.rb b/gems/rb-gem-installer/test/ruby_config_test.rb new file mode 100644 index 0000000..702ee38 --- /dev/null +++ b/gems/rb-gem-installer/test/ruby_config_test.rb @@ -0,0 +1,53 @@ +require 'minitest/autorun' +require 'open3' +require 'json' +require_relative '../lib/rb_gem_installer' + +class RubyConfigTest < Minitest::Test + def test_selected_interpreter_configuration + result = RubyButler::GemInstaller::RubyConfig.read + assert_equal RbConfig.ruby, result.fetch(:executable) + assert_equal RUBY_VERSION, result.fetch(:version) + assert_equal RbConfig::CONFIG, result.fetch(:config) + assert_equal Shellwords.split(RbConfig::CONFIG.fetch('CC', 'cc')), result.fetch(:linker) + assert_equal Shellwords.split(RbConfig::CONFIG.fetch('LIBRUBYARG_SHARED', '') + ' ' + RbConfig::CONFIG.fetch('LIBS', '')), + result.fetch(:libraries) + end + + def test_quoted_toolchain_arguments + config = { 'CC' => '"/compiler with spaces/cc" -target example', + 'LIBRUBYARG_SHARED' => '-lruby -L"/ruby libraries"', 'LIBS' => '-lm -pthread' } + result = RubyButler::GemInstaller::RubyConfig.read(config: config, executable: '/ruby with spaces/ruby', + version: '3.4.0') + assert_equal ['/compiler with spaces/cc', '-target', 'example'], result.fetch(:linker) + assert_equal ['-lruby', '-L/ruby libraries', '-lm', '-pthread'], result.fetch(:libraries) + assert_equal '/ruby with spaces/ruby', result.fetch(:executable) + assert_equal '3.4.0', result.fetch(:version) + assert_equal config, result.fetch(:config) + end + + def test_optional_toolchain_fields + result = RubyButler::GemInstaller::RubyConfig.read(config: {}) + assert_equal ['cc'], result.fetch(:linker) + assert_empty result.fetch(:libraries) + end + + def test_cli_emits_only_configuration_json_without_rubygems + root = File.expand_path('..', __dir__) + output, error, status = Open3.capture3({ 'RUBYOPT' => nil, 'RUBYLIB' => nil }, RbConfig.ruby, + '--disable=gems', '-I', File.join(root, 'lib'), File.join(root, 'exe/rb-ruby-config')) + assert status.success?, error + assert_empty error + expected = JSON.parse(JSON.generate(RubyButler::GemInstaller::RubyConfig.read)) + assert_equal expected, JSON.parse(output) + end + + def test_import_has_no_output_and_needs_no_rubygems + lib = File.expand_path('../lib', __dir__) + program = 'require "rb_gem_installer"; raise if defined?(Gem) || defined?(Bundler)' + output, error, status = Open3.capture3(RbConfig.ruby, '--disable=gems', '-I', lib, '-e', program) + assert status.success?, error + assert_empty output + assert_empty error + end +end diff --git a/gems/rb-microgems/README.md b/gems/rb-microgems/README.md new file mode 100644 index 0000000..9036b04 --- /dev/null +++ b/gems/rb-microgems/README.md @@ -0,0 +1,5 @@ +# rb-microgems + +Minimal gem APIs under `MicroGems`. Load `microgems` for the core or preload `microgems/compatibility` with RubyGems disabled for `Gem`, `gem` and `require` compatibility. The caller supplies installed dependencies through `RB_MICROGEMS_MANIFEST`. + +Run `ruby test/all_test.rb` or `rake test`. Minitest and rake are development dependencies only. Build with `gem build rb-microgems.gemspec`; native builds consume the embedded library files directly and do not install this gem. diff --git a/gems/rb-microgems/Rakefile b/gems/rb-microgems/Rakefile new file mode 100644 index 0000000..4866170 --- /dev/null +++ b/gems/rb-microgems/Rakefile @@ -0,0 +1,12 @@ +require 'rake/testtask' + +Rake::TestTask.new do |task| + task.libs = ['test'] + task.pattern = 'test/*_test.rb' +end + +task :lint do + sh 'rubocop', '--config', File.expand_path('../.rubocop.yml', __dir__), __dir__ +end + +task default: :test diff --git a/gems/rb-microgems/lib/microgems.rb b/gems/rb-microgems/lib/microgems.rb new file mode 100644 index 0000000..6289a4a --- /dev/null +++ b/gems/rb-microgems/lib/microgems.rb @@ -0,0 +1,104 @@ +require 'rbconfig' +require 'json' + +module MicroGems + VERSION = '0.3.1' + WIN_PATTERNS = [/mswin/, /mingw/, /cygwin/].freeze + class Exception < StandardError; end + class InstallError < Exception; end + + class LoadError < ::LoadError + attr_accessor :name, :requirement + end + + class MissingSpecError < LoadError; end + class MissingSpecVersionError < MissingSpecError; end +end + +require_relative 'microgems/version' +require_relative 'microgems/requirement' +require_relative 'microgems/platform' +require_relative 'microgems/specification' + +module MicroGems + FEATURES = %w[rubygems rubygems/version rubygems/requirement rubygems/platform rubygems/specification rubygems/ext + rubygems/ext/builder rubygems/command rubygems/package].freeze + + def self.configure(manifest) + @specs = manifest.fetch('gems', []).map { |record| MicroGems::Specification.new(record) } + @loaded = {} + @arguments = manifest.fetch('arguments', []) + duplicates = @specs.group_by { |spec| + [spec.name, spec.version.to_s, spec.platform.to_s] + }.select { |_, specs| specs.length > 1 } + raise ArgumentError, 'Duplicate installed gem identities' unless duplicates.empty? + end + + def self.specs = @specs || [] + def self.loaded = @loaded ||= {} + def self.arguments = @arguments || [] + + def self.activate(name, *requirements) + requirement = MicroGems::Requirement.new(*requirements) + if (active = loaded[name.to_s]) + raise MicroGems::LoadError, + "#{name} #{active.version} is already activated; requested #{requirement}" unless requirement.satisfied_by?(active.version) + + return false + end + spec = MicroGems::Specification.find_by_name(name, requirement) + spec.full_require_paths.reverse_each { |path| $LOAD_PATH.unshift(path) unless $LOAD_PATH.include?(path) } + loaded[spec.name] = spec + true + end + + module Requires + def require(feature) + name = feature.to_s.delete_suffix('.rb') + if name.start_with?('rubygems/') && !MicroGems::FEATURES.include?(name) + raise ::LoadError, "Unsupported RubyGems compatibility feature: #{feature}" + end + + super + rescue ::LoadError => error + raise unless error.respond_to?(:path) && error.path == feature.to_s + + candidates = MicroGems.specs.select do |spec| + MicroGems::Platform.match(spec.platform) && spec.full_require_paths.any? do |path| + MicroGems.suffixes.any? { |suffix| File.file?(File.join(path, feature.to_s + suffix)) } + end + end.sort.reverse + spec = candidates.find { |candidate| + !MicroGems.loaded.key?(candidate.name) || MicroGems.loaded[candidate.name].version == candidate.version + } + raise unless spec + + MicroGems.activate(spec.name, "= #{spec.version}") + super(feature) + end + end +end + +module MicroGems + class << self + def win_platform? = WIN_PATTERNS.any? { |pattern| pattern.match?(RbConfig::CONFIG.fetch('host_os', RUBY_PLATFORM)) } + def ruby = RbConfig.ruby + def ruby_version = Version.new(RUBY_VERSION) + def target_rbconfig = RbConfig + def loaded_specs = MicroGems.loaded + def platforms = [Platform::RUBY, Platform.local] + def install_extension_in_lib = true + def configuration = @configuration ||= {} + def default_exec_format = '%s' + def suffixes = ['', '.rb', ".#{RbConfig::CONFIG.fetch('DLEXT')}"] + def binary_mode = 'rb' + + def bin_path(name, executable, *requirements) + spec = Specification.find_by_name(name, *requirements) + raise MicroGems::Exception, + "Missing executable #{executable} for #{name}" unless spec.executables.include?(executable) + + File.join(spec.full_gem_path, spec.bindir, executable) + end + end +end diff --git a/gems/rb-microgems/lib/microgems/command.rb b/gems/rb-microgems/lib/microgems/command.rb new file mode 100644 index 0000000..dd3e935 --- /dev/null +++ b/gems/rb-microgems/lib/microgems/command.rb @@ -0,0 +1,5 @@ +module MicroGems + class Command + def self.build_args = MicroGems.arguments + end +end diff --git a/gems/rb-microgems/lib/microgems/compatibility.rb b/gems/rb-microgems/lib/microgems/compatibility.rb new file mode 100644 index 0000000..12711fb --- /dev/null +++ b/gems/rb-microgems/lib/microgems/compatibility.rb @@ -0,0 +1,20 @@ +require_relative '../microgems' + +raise ::LoadError, 'MicroGems requires RubyGems to be disabled' if defined?(Gem) && !Gem.equal?(MicroGems) + +Gem = MicroGems unless defined?(Gem) + +module Kernel + def gem(name, *requirements) = MicroGems.activate(name, *requirements) + private :gem + prepend MicroGems::Requires +end + +if ENV['RB_MICROGEMS_MANIFEST'] + MicroGems.configure(JSON.parse(File.read(ENV.fetch('RB_MICROGEMS_MANIFEST')))) +end + +if ENV['RB_MICROGEMS_CURRENT'] + name, version = JSON.parse(ENV.fetch('RB_MICROGEMS_CURRENT')) + MicroGems.activate(name, "= #{version}") +end diff --git a/gems/rb-microgems/lib/microgems/ext/builder.rb b/gems/rb-microgems/lib/microgems/ext/builder.rb new file mode 100644 index 0000000..176897a --- /dev/null +++ b/gems/rb-microgems/lib/microgems/ext/builder.rb @@ -0,0 +1,40 @@ +require 'open3' +require 'shellwords' + +module MicroGems + module Ext + class Builder + class NoMakefileError < MicroGems::InstallError; end + class << self + def shellsplit(value) = Shellwords.split(value) + alias shellsplit_command shellsplit + def ruby = [RbConfig.ruby] + + def run(command, results, name = nil, directory = Dir.pwd, environment = {}) + raise ArgumentError, 'Build command must be an argv array' unless command.is_a?(Array) && !command.empty? + + results << "current directory: #{directory}" << Shellwords.join(command) + output = +'' + status = nil + Open3.popen2e(environment, *command, chdir: directory) do |input, stream, wait| + input.close + loop do + chunk = stream.readpartial(8192) + $stdout.write(chunk) + $stdout.flush + output << chunk + output = output.byteslice(-1_048_576, 1_048_576) if output.bytesize > 1_048_576 + rescue EOFError + break + end + status = wait.value + end + results << output + raise MicroGems::InstallError, "#{name || command.first} failed (#{status}): #{output}" unless status.success? + + results + end + end + end + end +end diff --git a/gems/rb-microgems/lib/microgems/package.rb b/gems/rb-microgems/lib/microgems/package.rb new file mode 100644 index 0000000..ec7a5b8 --- /dev/null +++ b/gems/rb-microgems/lib/microgems/package.rb @@ -0,0 +1,159 @@ +module MicroGems + class Package + class TarInvalidError < StandardError; end + class TooLongFileName < StandardError; end + + module TarBlock + def new(*args) + object = super + return object unless block_given? + + begin + yield object + ensure + object.close + end + nil + end + end + + class TarHeader + FIELDS = { name: 100, mode: 8, uid: 8, gid: 8, size: 12, mtime: 12, + checksum: 8, typeflag: 1, linkname: 100, magic: 6, version: 2, + uname: 32, gname: 32, devmajor: 8, devminor: 8, prefix: 155 }.freeze + NUMBERS = %i[mode uid gid size mtime checksum devmajor devminor].freeze + attr_accessor(*FIELDS.keys) + + def initialize(values) + defaults = { name: '', mode: 0, uid: 0, gid: 0, size: 0, mtime: 0, + checksum: 0, typeflag: '0', linkname: '', magic: "ustar\0", + version: '00', uname: '', gname: '', devmajor: 0, devminor: 0, prefix: '' } + defaults.merge(values).each { |key, value| public_send("#{key}=", value) } + self.mtime = mtime.to_i + end + + def self.from(io) + bytes = io.read(512) + raise TarInvalidError, 'Truncated tar header' unless bytes && bytes.bytesize == 512 + return new(name: '', empty: true) if bytes == "\0" * 512 + + values = {} + offset = 0 + FIELDS.each do |key, width| + value = bytes.byteslice(offset, width) + offset += width + if NUMBERS.include?(key) + value = value.delete("\0 ") + raise TarInvalidError, "Invalid tar number: #{key}" unless value.match?(/\A[0-7]*\z/) + + value = value.empty? ? 0 : value.to_i(8) + else + value = value.split("\0", 2).first || '' + end + values[key] = value + end + checked = bytes.dup + checked[148, 8] = ' ' * 8 + raise TarInvalidError, 'Invalid tar checksum' unless checked.bytes.sum == values[:checksum] + + new(values) + end + + attr_writer :empty + + def empty? = !!@empty + def full_name = prefix.empty? ? name : "#{prefix}/#{name}" + + def to_s + bytes = FIELDS.map do |key, width| + value = public_send(key) + if key == :checksum + ' ' * width + elsif NUMBERS.include?(key) + value = Integer(value).to_s(8) + raise TarInvalidError, "Tar number too large: #{key}" if value.size >= width || value.start_with?('-') + + value.rjust(width - 1, '0') + "\0" + else + value = value.to_s.b + raise TooLongFileName, "Tar field too long: #{key}" if value.bytesize > width + + value.ljust(width, "\0") + end + end.join.ljust(512, "\0") + self.checksum = bytes.bytes.sum + bytes[148, 8] = format("%06o\0 ", checksum) + bytes + end + end + + class TarReader + include Enumerable + extend TarBlock + + class Entry + attr_reader :header + + def initialize(header, io) + @header, @io, @read, @closed = header, io, 0, false + end + + def full_name = header.full_name + def size = header.size + def file? = ['0', ''].include?(header.typeflag) + def directory? = header.typeflag == '5' + def symlink? = header.typeflag == '2' + def closed? = @closed + def eof? = @read == size + + def read(length = nil) + raise IOError, 'Closed tar entry' if closed? + raise ArgumentError, 'Negative read size' if length && length < 0 + return nil if length && length > 0 && eof? + + count = length ? [length, size - @read].min : size - @read + value = @io.read(count) + raise TarInvalidError, 'Truncated tar entry' unless value && value.bytesize == count + + @read += count + value + end + + def close + return if closed? + + read([size - @read, 16384].min) until eof? + padding = (-size) % 512 + value = @io.read(padding) + raise TarInvalidError, 'Truncated tar padding' unless value && value.bytesize == padding + + @closed = true + end + end + + def initialize(io) + @io, @start = io, io.pos + end + + def close; end + def rewind = @io.pos = @start + + def each + return enum_for(__method__) unless block_given? + + until @io.eof? + header = TarHeader.from(@io) + break if header.empty? + + entry = Entry.new(header, @io) + begin + yield entry + ensure + entry.close + end + end + end + alias each_entry each + end + end +end diff --git a/gems/rb-microgems/lib/microgems/platform.rb b/gems/rb-microgems/lib/microgems/platform.rb new file mode 100644 index 0000000..d60f89e --- /dev/null +++ b/gems/rb-microgems/lib/microgems/platform.rb @@ -0,0 +1,67 @@ +module MicroGems + class Platform + RUBY = 'ruby' + CURRENT = 'current' + attr_reader :cpu, :os, :version + + def self.new(value) + return RUBY if value.nil? || value == '' || value == RUBY + return local if value == CURRENT + + super + end + + def self.local + arch = RUBY_ENGINE == 'jruby' ? 'java' : RbConfig::CONFIG.fetch('arch') + arch += '_60' if arch.end_with?('mswin32', 'mswin64') + new(arch) + end + + def self.match(platform) = platform.to_s == RUBY || new(platform) === local + def self.match_spec?(spec) = match(spec.platform) + def self.match_gem?(platform, _name) = match(platform) + + def initialize(value) + if value.is_a?(Array) + @cpu, @os, @version = value + return + end + parts = value.to_s.sub(/-+\z/, '').split('-', 2) + @cpu, system = parts.length == 1 ? [nil, parts.first] : parts + @cpu = 'x86' if @cpu&.match?(/\Ai\d86\z/) + match = system.match(/\A(mswin32|mswin64|mingw32|mingw|netbsdelf|freebsd|openbsd|solaris|darwin|linux|cygwin|macruby|dalvik|dotnet|java|aix|wasi)(.*)\z/) + @os = match ? match[1] : system + @os = 'java' if @os == 'jruby' + @version = match && match[2].sub(/\A[-_]+/, '') + @version = case @os + when 'cygwin', 'mingw32', 'netbsdelf', 'wasi' then nil + when 'aix', 'darwin', 'freebsd', 'dalvik', 'mswin32', 'mswin64' then @version&.slice(/\A\d+/) + when 'openbsd', 'solaris', 'macruby', 'java', 'dotnet' then @version&.slice(/\A\d+(?:\.\d+)*/) + else @version + end + @version = nil if @version == '' + @cpu ||= 'x86' if @os == 'mswin32' + end + + def to_a = [cpu, os, version] + alias deconstruct to_a + def to_s = to_a.compact.join(cpu ? '-' : '') + def ==(other) = other.respond_to?(:cpu) && [cpu, os, version] == [other.cpu, other.os, other.version] + alias eql? == + def hash = [cpu, os, version].hash + + def ===(other) + return false unless other.is_a?(Platform) + return true if [cpu, other.cpu].include?('universal') && os.start_with?('mingw') && other.os.start_with?('mingw') + return false unless os == other.os && (!cpu || !other.cpu || cpu == other.cpu || [cpu, other.cpu].include?('universal') || (cpu == 'arm' && other.cpu.start_with?('armv'))) + return !version || !other.version || version == other.version unless os == 'linux' + + normalize = ->(v) { v.to_s.sub(/\Agnu/, '').sub(/eabi(?:hf)?\z/, '') } + normalize.call(version) == normalize.call(other.version) || %w[musl musleabi musleabihf].any? { |libc| + other.version == libc + version.to_s + } + end + + def =~(other) = self === (other.is_a?(Platform) ? other : self.class.new(other)) + end +end diff --git a/gems/rb-microgems/lib/microgems/requirement.rb b/gems/rb-microgems/lib/microgems/requirement.rb new file mode 100644 index 0000000..4f8f41c --- /dev/null +++ b/gems/rb-microgems/lib/microgems/requirement.rb @@ -0,0 +1,45 @@ +module MicroGems + class Requirement + attr_reader :requirements + + def self.create(value) = value.is_a?(self) ? value : new(*Array(value)) + def self.default = new('>= 0') + + def initialize(*values) + values = values.flatten.compact.flat_map { |v| + v.is_a?(Requirement) ? v.requirements.map { |op, version| + "#{op} #{version}" + } : v.to_s.split(',') + } + values = ['>= 0'] if values.empty? + @requirements = values.map do |value| + match = /\A\s*(~>|>=|<=|!=|=|>|<)?\s*(\S+)\s*\z/.match(value) + raise ArgumentError, "Malformed requirement #{value.inspect}" unless match + + [match[1] || '=', Version.new(match[2])] + end.uniq + end + + def satisfied_by?(version) + version = Version.new(version) + requirements.all? do |op, target| + case op + when '=' then version == target + when '!=' then version != target + when '>' then version > target + when '<' then version < target + when '>=' then version >= target + when '<=' then version <= target + when '~>' then version >= target && version.release < target.bump + end + end + end + alias === satisfied_by? + def prerelease? = requirements.any? { |_, version| version.prerelease? } + def exact? = requirements.size == 1 && requirements.first.first == '=' + def none? = self == self.class.default + def to_s = requirements.map { |op, version| "#{op} #{version}" }.join(', ') + def as_list = requirements.map { |op, version| "#{op} #{version}" } + def ==(other) = other.is_a?(Requirement) && requirements.sort_by(&:to_s) == other.requirements.sort_by(&:to_s) + end +end diff --git a/gems/rb-microgems/lib/microgems/specification.rb b/gems/rb-microgems/lib/microgems/specification.rb new file mode 100644 index 0000000..6c66c0f --- /dev/null +++ b/gems/rb-microgems/lib/microgems/specification.rb @@ -0,0 +1,42 @@ +module MicroGems + class Specification + include Comparable + + attr_reader :name, :version, :platform, :require_paths, :metadata, :executables, :bindir + + def initialize(record) + @name = record.fetch('name') + @version = Version.new(record.fetch('version')) + @platform = Platform.new(record.fetch('platform', 'ruby')) + @root = File.expand_path(record.fetch('root')) + @require_paths = record.fetch('require_paths', ['lib']) + @metadata = record.fetch('metadata', {}) + @executables = record.fetch('executables', []) + @bindir = record.fetch('bindir', 'bin') + end + + def full_gem_path = @root + alias gem_dir full_gem_path + def full_require_paths = require_paths.map { |path| File.expand_path(path, full_gem_path) } + alias raw_require_paths require_paths + def full_name = [name, version.to_s, platform.to_s == 'ruby' ? nil : platform.to_s].compact.join('-') + def <=>(other) = version <=> other.version + def activate = MicroGems.activate(name, "= #{version}") + + def self.to_a = MicroGems.specs + def self.each(&block) = to_a.each(&block) + + def self.find_all_by_name(name, *requirements) + requirement = Requirement.new(*requirements) + to_a.select { |spec| + spec.name == name.to_s && requirement.satisfied_by?(spec.version) && Platform.match(spec.platform) + }.sort.reverse + end + + def self.find_by_name(name, *requirements) + find_all_by_name(name, + *requirements).first || raise(MissingSpecError, + "Missing installed gem #{name} (#{Requirement.new(*requirements)})") + end + end +end diff --git a/gems/rb-microgems/lib/microgems/version.rb b/gems/rb-microgems/lib/microgems/version.rb new file mode 100644 index 0000000..8c9b192 --- /dev/null +++ b/gems/rb-microgems/lib/microgems/version.rb @@ -0,0 +1,81 @@ +module MicroGems + class Version + include Comparable + + VERSION_PATTERN = '[0-9]+(?:\.[0-9a-zA-Z]+)*(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?'.freeze + attr_reader :version + + def self.new(value) + return value if value.is_a?(self) + + super + end + + def self.correct?(value) + /\A\s*(?:#{VERSION_PATTERN})?\s*\z/.match?(value.to_s) + end + + def self.create(value) + new(value) + end + + def initialize(value) + raise ArgumentError, "Malformed version #{value.inspect}" unless self.class.correct?(value) + + @version = value.to_s.strip.gsub('-', '.pre.') + @version = '0' if @version.empty? + end + + def segments + @segments ||= version.scan(/[0-9]+|[a-zA-Z]+/).map { |part| /\A\d+\z/.match?(part) ? part.to_i : part }.freeze + end + + def canonical_segments + boundary = segments.index { |part| part.is_a?(String) } || segments.size + [segments.take(boundary), segments.drop(boundary)].flat_map do |parts| + parts.pop while parts.last == 0 + parts + end + end + + def <=>(other) + other = self.class.new(other) if other.is_a?(String) + return unless other.is_a?(Version) + + left, right = canonical_segments, other.canonical_segments + [left.size, right.size].max.times do |i| + a, b = left.fetch(i, 0), right.fetch(i, 0) + next if a == b + return a <=> b if a.class == b.class + + return a.is_a?(String) ? -1 : 1 + end + 0 + end + + def prerelease? = segments.any?(String) + + def release + self.class.new(segments.take_while { |part| part.is_a?(Integer) }.join('.')) + end + + def bump + parts = release.segments.dup + parts.pop if parts.size > 1 + parts[-1] += 1 + self.class.new(parts.join('.')) + end + + def approximate_recommendation + parts = release.segments.first(2) + parts << 0 if parts.length == 1 + "~> #{parts.join('.')}#{prerelease? ? '.a' : ''}" + end + + def to_s = version + alias to_str to_s + def inspect = "#" + def eql?(other) = other.is_a?(Version) && version == other.version + def hash = canonical_segments.hash + end +end diff --git a/gems/rb-microgems/lib/rubygems.rb b/gems/rb-microgems/lib/rubygems.rb new file mode 100644 index 0000000..e1bd3ab --- /dev/null +++ b/gems/rb-microgems/lib/rubygems.rb @@ -0,0 +1 @@ +require_relative 'microgems/compatibility' diff --git a/gems/rb-microgems/lib/rubygems/command.rb b/gems/rb-microgems/lib/rubygems/command.rb new file mode 100644 index 0000000..7f12d4d --- /dev/null +++ b/gems/rb-microgems/lib/rubygems/command.rb @@ -0,0 +1,2 @@ +require_relative '../microgems/compatibility' +require_relative '../microgems/command' diff --git a/gems/rb-microgems/lib/rubygems/ext.rb b/gems/rb-microgems/lib/rubygems/ext.rb new file mode 100644 index 0000000..87bed26 --- /dev/null +++ b/gems/rb-microgems/lib/rubygems/ext.rb @@ -0,0 +1 @@ +require_relative 'ext/builder' diff --git a/gems/rb-microgems/lib/rubygems/ext/builder.rb b/gems/rb-microgems/lib/rubygems/ext/builder.rb new file mode 100644 index 0000000..a98188b --- /dev/null +++ b/gems/rb-microgems/lib/rubygems/ext/builder.rb @@ -0,0 +1,2 @@ +require_relative '../../microgems/compatibility' +require_relative '../../microgems/ext/builder' diff --git a/gems/rb-microgems/lib/rubygems/package.rb b/gems/rb-microgems/lib/rubygems/package.rb new file mode 100644 index 0000000..5cd41b5 --- /dev/null +++ b/gems/rb-microgems/lib/rubygems/package.rb @@ -0,0 +1,2 @@ +require_relative '../microgems/compatibility' +require_relative '../microgems/package' diff --git a/gems/rb-microgems/lib/rubygems/platform.rb b/gems/rb-microgems/lib/rubygems/platform.rb new file mode 100644 index 0000000..8c4867e --- /dev/null +++ b/gems/rb-microgems/lib/rubygems/platform.rb @@ -0,0 +1,2 @@ +require_relative '../microgems/compatibility' +require_relative '../microgems/platform' diff --git a/gems/rb-microgems/lib/rubygems/requirement.rb b/gems/rb-microgems/lib/rubygems/requirement.rb new file mode 100644 index 0000000..ca47592 --- /dev/null +++ b/gems/rb-microgems/lib/rubygems/requirement.rb @@ -0,0 +1,2 @@ +require_relative '../microgems/compatibility' +require_relative '../microgems/requirement' diff --git a/gems/rb-microgems/lib/rubygems/specification.rb b/gems/rb-microgems/lib/rubygems/specification.rb new file mode 100644 index 0000000..3c6d659 --- /dev/null +++ b/gems/rb-microgems/lib/rubygems/specification.rb @@ -0,0 +1,2 @@ +require_relative '../microgems/compatibility' +require_relative '../microgems/specification' diff --git a/gems/rb-microgems/lib/rubygems/version.rb b/gems/rb-microgems/lib/rubygems/version.rb new file mode 100644 index 0000000..23a0906 --- /dev/null +++ b/gems/rb-microgems/lib/rubygems/version.rb @@ -0,0 +1,2 @@ +require_relative '../microgems/compatibility' +require_relative '../microgems/version' diff --git a/gems/rb-microgems/rb-microgems.gemspec b/gems/rb-microgems/rb-microgems.gemspec new file mode 100644 index 0000000..3fc7cbe --- /dev/null +++ b/gems/rb-microgems/rb-microgems.gemspec @@ -0,0 +1,15 @@ +require_relative 'lib/microgems' + +Gem::Specification.new do |spec| + spec.name = 'rb-microgems' + spec.version = MicroGems::VERSION + spec.authors = ['RubyElders'] + spec.summary = 'Minimal RubyGems compatibility for isolated native builds' + spec.homepage = 'https://github.com/RubyElders/ruby-butler' + spec.licenses = ['MIT', 'Apache-2.0'] + spec.required_ruby_version = '>= 3.3' + spec.files = Dir.chdir(__dir__) { Dir['lib/**/*.rb', 'README.md'] } + spec.require_paths = ['lib'] + spec.add_development_dependency 'minitest', '>= 5.20', '< 7' + spec.add_development_dependency 'rake', '>= 13', '< 14' +end diff --git a/gems/rb-microgems/test/all_test.rb b/gems/rb-microgems/test/all_test.rb new file mode 100644 index 0000000..e330269 --- /dev/null +++ b/gems/rb-microgems/test/all_test.rb @@ -0,0 +1,3 @@ +Dir.glob(File.join(__dir__, '*_test.rb')).each do |path| + require path unless path == File.expand_path(__FILE__) +end diff --git a/gems/rb-microgems/test/compatibility_test.rb b/gems/rb-microgems/test/compatibility_test.rb new file mode 100644 index 0000000..33a626b --- /dev/null +++ b/gems/rb-microgems/test/compatibility_test.rb @@ -0,0 +1,46 @@ +require_relative 'test_helper' + +class CompatibilityTest < Minitest::Test + include RubyProcess + + def setup + @root = Dir.mktmpdir('microgems-test-') + FileUtils.mkdir_p(File.join(@root, 'lib')) + File.write(File.join(@root, 'lib', 'fixture_api.rb'), "FIXTURE_API = 42\n") + @manifest = File.join(@root, 'manifest.json') + gem = { name: 'fixture', version: '2.1.3', platform: 'ruby', root: @root, + require_paths: ['lib'], metadata: { purpose: 'test' }, executables: [], bindir: 'bin' } + File.write(@manifest, JSON.generate(gems: [gem], arguments: ['--with-extra=a b'])) + @environment = { 'RB_MICROGEMS_MANIFEST' => @manifest, 'FIXTURE_ROOT' => @root } + end + + def teardown + FileUtils.remove_entry(@root) + end + + def test_build_api_contracts + output, error, status = ruby_fixture('contracts.rb', environment: @environment, compatibility: true) + assert status.success?, error + assert_includes output, 'contracts passed' + end + + def test_core_does_not_install_the_compatibility_namespace + output, error, status = ruby_fixture('namespaces.rb', environment: @environment, compatibility: true) + assert status.success?, error + assert_equal "namespaces passed\n", output + end + + def test_each_shim_can_be_the_first_import + MicroGems::FEATURES.each do |feature| + output, error, status = ruby_fixture('shims.rb', feature, compatibility: true) + assert status.success?, "#{feature}: #{error}" + assert_equal "shims passed\n", output + end + end + + def test_loaded_rubygems_is_preserved + output, error, status = ruby_fixture('existing_rubygems.rb', environment: { 'MICROGEMS_PATH' => RubyProcess::LIB }) + assert status.success?, error + assert_equal "existing RubyGems preserved\n", output + end +end diff --git a/gems/rb-microgems/test/fixtures/contracts.rb b/gems/rb-microgems/test/fixtures/contracts.rb new file mode 100644 index 0000000..27c21f4 --- /dev/null +++ b/gems/rb-microgems/test/fixtures/contracts.rb @@ -0,0 +1,61 @@ +require 'microgems/compatibility' +require 'stringio' +require 'rubygems/package' +require 'rubygems/ext' +require 'rubygems/command' + +def check(value, message) + raise message unless value +end + +def raises(type) + begin + yield + rescue type + return + end + raise "Expected #{type}" +end + +root = File.expand_path(ENV.fetch('FIXTURE_ROOT')) + +check(!defined?(Bundler), 'Bundler loaded') +check(!$LOADED_FEATURES.any? { |file| file.include?('rubygems/core_ext') }, 'Full RubyGems loaded') +check(Gem::Command.build_args == ['--with-extra=a b'], 'Build arguments changed') +check(gem('fixture', '~> 2.1'), 'Activation failed') +check(!gem('fixture', '>= 2'), 'Repeated activation changed state') +check(Gem.loaded_specs.fetch('fixture').metadata.fetch('purpose') == 'test', 'Metadata missing') +check(Gem.loaded_specs.fetch('fixture').full_gem_path == root, 'Root changed') +raises(Gem::LoadError) { gem('fixture', '< 2') } +raises(Gem::MissingSpecError) { gem('missing') } +raises(LoadError) { require 'rubygems/remote_fetcher' } +raises(NoMethodError) { Gem.unsupported_api } + +header = Gem::Package::TarHeader.new(name: 'agent/file', size: 3) +tar = header.to_s + "abc" + ("\0" * 509) + ("\0" * 1024) +entries = [] +Gem::Package::TarReader.new(StringIO.new(tar)) do |reader| + reader.each { |entry| entries << [entry.full_name, entry.read(1), entry.read] } +end +check(entries == [['agent/file', 'a', 'bc']], 'Tar contents changed') +raises(Gem::Package::TarInvalidError) { Gem::Package::TarReader.new(StringIO.new(tar[0, 514])).each(&:read) } +corrupt = tar.dup +corrupt[0] = 'X' +raises(Gem::Package::TarInvalidError) { Gem::Package::TarReader.new(StringIO.new(corrupt)).each(&:read) } +results = [] +Gem::Ext::Builder.run([Gem.ruby, '--disable=gems', '-e', 'print ARGV.fetch(0)', 'a; echo incorrect'], results) +check(results.last == 'a; echo incorrect', 'Command arguments interpreted by shell') +raises(Gem::InstallError) { Gem::Ext::Builder.run([Gem.ruby, '--disable=gems', '-e', 'exit 3'], []) } + +record = { 'name' => 'choice', 'version' => '1.0', 'root' => root, 'platform' => 'ruby' } +MicroGems.configure('gems' => [record, record.merge('version' => '2.0')]) +check(Gem::Specification.find_by_name('choice', '< 2').version.to_s == '1.0', 'Requirement lookup ignored') +check(Gem::Specification.find_by_name('choice').version.to_s == '2.0', 'Latest version not chosen') +raises(Gem::MissingSpecError) { Gem::Specification.find_by_name('choice', '> 3') } +raises(ArgumentError) { MicroGems.configure('gems' => [record, record]) } + +$LOAD_PATH.delete(File.join(root, 'lib')) +MicroGems.configure('gems' => [record.merge('name' => 'auto')]) +require 'fixture_api' +check(FIXTURE_API == 42 && Gem.loaded_specs.key?('auto'), 'Require did not activate explicit dependency') +puts 'contracts passed' diff --git a/gems/rb-microgems/test/fixtures/existing_rubygems.rb b/gems/rb-microgems/test/fixtures/existing_rubygems.rb new file mode 100644 index 0000000..5e7ec07 --- /dev/null +++ b/gems/rb-microgems/test/fixtures/existing_rubygems.rb @@ -0,0 +1,14 @@ +original = Gem +version = Gem::Version +$LOAD_PATH.unshift(ENV.fetch('MICROGEMS_PATH')) +begin + require 'microgems/compatibility' + raise 'Full RubyGems was silently replaced' +rescue LoadError => error + raise unless error.message.include?('RubyGems to be disabled') +end +raise 'Gem namespace changed' unless Gem.equal?(original) +raise 'RubyGems version class changed' unless Gem::Version.equal?(version) +raise 'Implementation was defined under Gem' if MicroGems::Version.equal?(version) + +puts 'existing RubyGems preserved' diff --git a/gems/rb-microgems/test/fixtures/namespaces.rb b/gems/rb-microgems/test/fixtures/namespaces.rb new file mode 100644 index 0000000..ad808f2 --- /dev/null +++ b/gems/rb-microgems/test/fixtures/namespaces.rb @@ -0,0 +1,25 @@ +require 'microgems' +require 'microgems/package' +require 'microgems/ext/builder' +require 'microgems/command' + +raise 'Gem alias installed by the core' if defined?(Gem) +raise 'Kernel hook installed by the core' if Kernel.ancestors.include?(MicroGems::Requires) + +%w[Version Requirement Platform Specification Command Package Ext::Builder LoadError MissingSpecError].each do |name| + raise "Wrong implementation namespace: #{name}" unless MicroGems.const_get(name).name == "MicroGems::#{name}" +end + +MicroGems.configure(JSON.parse(File.read(ENV.fetch('RB_MICROGEMS_MANIFEST')))) +Kernel.prepend(MicroGems::Requires) +require 'fixture_api' +raise 'Core activation failed' unless FIXTURE_API == 42 && MicroGems.loaded_specs.fetch('fixture').version.to_s == '2.1.3' +raise 'Core require installed Gem' if defined?(Gem) + +require 'microgems/compatibility' +raise 'Compatibility namespace is not an alias' unless Gem.equal?(MicroGems) +raise 'Specification registries differ' unless Gem.loaded_specs.equal?(MicroGems.loaded_specs) +raise 'Exception classes differ' unless Gem::LoadError.equal?(MicroGems::LoadError) +raise 'Version classes differ' unless Gem::Version.equal?(MicroGems::Version) + +puts 'namespaces passed' diff --git a/gems/rb-microgems/test/fixtures/shims.rb b/gems/rb-microgems/test/fixtures/shims.rb new file mode 100644 index 0000000..71248b6 --- /dev/null +++ b/gems/rb-microgems/test/fixtures/shims.rb @@ -0,0 +1,10 @@ +require ARGV.fetch(0) +raise 'Gem is not the compatibility alias' unless Gem.equal?(MicroGems) +raise 'Version is not owned by MicroGems' unless Gem::Version.name == 'MicroGems::Version' + +MicroGems::FEATURES.each { |feature| require feature } +raise 'Package is not owned by MicroGems' unless Gem::Package.name == 'MicroGems::Package' +raise 'Builder is not owned by MicroGems' unless Gem::Ext::Builder.name == 'MicroGems::Ext::Builder' +raise 'Full RubyGems loaded' if $LOADED_FEATURES.any? { |file| file.include?('rubygems/core_ext') } + +puts 'shims passed' diff --git a/gems/rb-microgems/test/fixtures/version_oracle.rb b/gems/rb-microgems/test/fixtures/version_oracle.rb new file mode 100644 index 0000000..4c211f0 --- /dev/null +++ b/gems/rb-microgems/test/fixtures/version_oracle.rb @@ -0,0 +1,24 @@ +require 'json' +versions = %w[0 1 1.0 1.0.0 1.0.a 1.0.a.0 1.0.rc1 1.0-rc1 1.2.3 1.2.4 2 2.0.pre 10.4] +requirements = ['>= 0', '~> 1.2', '~> 1.2.3', '>= 1', '< 2', '!= 1.0', '= 1', '~> 1.0.a', '> 1.0.a'] +platforms = %w[x86_64-linux x86_64-linux-gnu aarch64-linux-musl arm-linux i686-linux x86_64-darwin-23 arm64-darwin-24 x64-mingw-ucrt x86-mswin32-60 java] +puts JSON.generate( + comparisons: versions.product(versions).map do |a, b| + [Gem::Version.new(a) <=> Gem::Version.new(b), Gem::Version.new(a).eql?(Gem::Version.new(b))] + end, + requirements: requirements.product(versions).map do |a, b| + Gem::Requirement.new(a).satisfied_by?(Gem::Version.new(b)) + end, + versions: versions.map do |v| + x = Gem::Version.new(v) + [x.to_s, x.release.to_s, x.bump.to_s, x.prerelease?, x.approximate_recommendation] + end, + generic_platforms: [nil, '', 'ruby', 'current'].map do |p| + x = Gem::Platform.new(p) + [x.to_s, x.is_a?(String)] + end, + platforms: platforms.map do |p| + x = Gem::Platform.new(p) + [x.to_a, x.to_s] + end +) diff --git a/gems/rb-microgems/test/package_test.rb b/gems/rb-microgems/test/package_test.rb new file mode 100644 index 0000000..3d9d48b --- /dev/null +++ b/gems/rb-microgems/test/package_test.rb @@ -0,0 +1,22 @@ +require_relative 'test_helper' +require 'rubygems/package' + +class PackageTest < Minitest::Test + def test_built_gem_contains_the_runtime_library + root = File.expand_path('..', __dir__) + spec = Gem::Specification.load(File.join(root, 'rb-microgems.gemspec')) + assert_equal MicroGems::VERSION, spec.version.to_s + assert_empty spec.runtime_dependencies + Dir.mktmpdir('microgems-package-') do |directory| + archive = File.join(directory, 'rb-microgems.gem') + capture_io { Dir.chdir(root) { Gem::Package.build(spec, false, true, archive) } } + destination = File.join(directory, 'extracted') + Gem::Package.new(archive).extract_files(destination) + files = Dir.chdir(root) { Dir['lib/**/*.rb'].sort } + assert_equal files, Dir.chdir(destination) { Dir['lib/**/*.rb'].sort } + files.each { |path| + assert_equal File.binread(File.join(root, path)), File.binread(File.join(destination, path)), path + } + end + end +end diff --git a/gems/rb-microgems/test/test_helper.rb b/gems/rb-microgems/test/test_helper.rb new file mode 100644 index 0000000..5f86a2a --- /dev/null +++ b/gems/rb-microgems/test/test_helper.rb @@ -0,0 +1,14 @@ +require 'minitest/autorun' +require 'tmpdir' +require 'fileutils' +require 'open3' +require_relative '../lib/microgems' + +module RubyProcess + LIB = File.expand_path('../lib', __dir__) + + def ruby_fixture(name, *arguments, environment: {}, compatibility: false) + flags = compatibility ? ['--disable=gems', '-I', LIB] : [] + Open3.capture3(environment, RbConfig.ruby, *flags, File.join(__dir__, 'fixtures', name), *arguments) + end +end diff --git a/gems/rb-microgems/test/value_test.rb b/gems/rb-microgems/test/value_test.rb new file mode 100644 index 0000000..5fa9372 --- /dev/null +++ b/gems/rb-microgems/test/value_test.rb @@ -0,0 +1,51 @@ +require_relative 'test_helper' + +class ValueTest < Minitest::Test + VERSIONS = %w[0 1 1.0 1.0.0 1.0.a 1.0.a.0 1.0.rc1 1.0-rc1 1.2.3 1.2.4 2 2.0.pre 10.4].freeze + REQUIREMENTS = ['>= 0', '~> 1.2', '~> 1.2.3', '>= 1', '< 2', '!= 1.0', '= 1', '~> 1.0.a', '> 1.0.a'].freeze + + def test_version_order_and_identity_match_rubygems + VERSIONS.product(VERSIONS).each do |left, right| + expected = Gem::Version.new(left) + actual = MicroGems::Version.new(left) + assert_equal expected <=> Gem::Version.new(right), actual <=> MicroGems::Version.new(right) + assert_equal expected.eql?(Gem::Version.new(right)), actual.eql?(MicroGems::Version.new(right)) + end + end + + def test_version_operations_match_rubygems + VERSIONS.each do |version| + expected = Gem::Version.new(version) + actual = MicroGems::Version.new(version) + assert_equal expected.release.to_s, actual.release.to_s + assert_equal expected.bump.to_s, actual.bump.to_s + assert_equal expected.prerelease?, actual.prerelease? + assert_equal expected.approximate_recommendation, actual.approximate_recommendation + end + assert_raises(ArgumentError) { MicroGems::Version.new('not-a-version') } + end + + def test_requirements_match_rubygems + REQUIREMENTS.product(VERSIONS).each do |requirement, version| + assert_equal Gem::Requirement.new(requirement).satisfied_by?(Gem::Version.new(version)), + MicroGems::Requirement.new(requirement).satisfied_by?(MicroGems::Version.new(version)) + end + constraint = MicroGems::Requirement.new('>= 1', '< 2') + assert constraint.satisfied_by?('1.5') + refute constraint.satisfied_by?('2') + assert_raises(ArgumentError) { MicroGems::Requirement.new('? 1') } + end + + def test_platform_fields_match_rubygems + %w[x86_64-linux x86_64-linux-gnu aarch64-linux-musl arm-linux i686-linux x86_64-darwin-23 arm64-darwin-24 + x64-mingw-ucrt x86-mswin32-60 java].each do |platform| + expected = Gem::Platform.new(platform) + actual = MicroGems::Platform.new(platform) + assert_equal expected.to_a, actual.to_a + assert_equal expected.to_s, actual.to_s + end + [nil, '', 'ruby', 'current'].each do |platform| + assert_equal Gem::Platform.new(platform).to_s, MicroGems::Platform.new(platform).to_s + end + end +end