Add rb gem installer

.github/workflows/ci.yml

@@ -91,8 +91,24 @@ jobs:
     - name: Run yamllint
       run: make lint-yaml
 
-  test-gemfile-ruby:
-    name: Gemfile Ruby Tests (${{ matrix.ruby }}, ${{ matrix.os }})
+  lint-ruby:
+    name: Lint (Ruby)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ env.RUBY_TEST_VERSION }}
+        bundler-cache: true
+        working-directory: gems
+
+    - name: Lint Ruby gems
+      working-directory: gems
+      run: bundle exec rake lint
+
+  test-ruby:
+    name: Ruby Tests (${{ matrix.ruby }}, ${{ matrix.os }})
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -113,6 +129,36 @@ jobs:
     - name: Test the Ruby evaluator
       run: ruby crates/rb-gemfile/tests/ruby/evaluator_test.rb
 
+    - name: Test MicroGems
+      run: ruby gems/rb-microgems/test/all_test.rb
+
+    - name: Test the installer Ruby helper
+      run: ruby gems/rb-gem-installer/test/all_test.rb
+
+  test-native-builds:
+    name: Native Build Tests (Ruby ${{ matrix.ruby }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ['3.3', '3.4', '4.0']
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Install Rust
+      run: rustup install
+
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler: none
+
+    - name: Test gem installation and build compatibility
+      run: cargo test -p rb-gem-installer -p rb-microgems
+
+    - name: Build installer examples
+      run: cargo build -p rb-gem-installer --examples
+
   test-rust:
     name: Rust Tests
     runs-on: ${{ matrix.os }}

crates/rb-gem-archive/Cargo.toml

@@ -7,6 +7,7 @@ description = "Standalone Ruby gem archive metadata reading"
 
 [dependencies]
 anyhow = "1.0"
+base64 = "0.22"
 flate2 = "1.1"
 rb-gem-types = { path = "../rb-gem-types" }
 serde_yaml_ng = "0.10"

crates/rb-gem-archive/src/checksums.rs

@@ -3,9 +3,10 @@ use crate::{
     metadata,
 };
 use anyhow::{Context, Result, ensure};
+use base64::{Engine, engine::general_purpose::STANDARD};
 use sha1::Sha1;
 use sha2::{Digest, Sha256, Sha512};
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 
 #[derive(Default)]
 pub(crate) struct Digests {
@@ -40,34 +41,29 @@ pub(crate) fn verify(
         .as_mapping()
         .context("Invalid gem checksum map")?;
     ensure!(!algorithms.is_empty(), "Empty gem checksum map");
+    let mut seen_algorithms = BTreeSet::new();
     for (algorithm, values) in algorithms {
-        let algorithm = metadata::untag(algorithm)
-            .as_str()
-            .context("Invalid checksum algorithm")?;
+        let algorithm = text(algorithm)?;
         ensure!(
-            matches!(algorithm, "SHA1" | "SHA256" | "SHA512"),
+            seen_algorithms.insert(algorithm.clone()),
+            "Duplicate checksum algorithm"
+        );
+        ensure!(
+            matches!(algorithm.as_str(), "SHA1" | "SHA256" | "SHA512"),
             "Unsupported checksum algorithm: {algorithm}"
         );
         let values = metadata::untag(values)
             .as_mapping()
             .context("Invalid member checksum map")?;
-        for name in [METADATA_MEMBER, DATA_MEMBER] {
-            ensure!(
-                values.contains_key(serde_yaml_ng::Value::String(name.into())),
-                "Missing member checksum"
-            );
-        }
+        let mut names = BTreeSet::new();
         for (name, expected) in values {
-            let name = metadata::untag(name)
-                .as_str()
-                .context("Invalid checksum member")?;
-            let expected = metadata::untag(expected)
-                .as_str()
-                .context("Invalid member checksum")?;
+            let name = text(name)?;
+            ensure!(names.insert(name.clone()), "Duplicate checksum member");
+            let expected = text(expected)?;
             let actual = hashes
-                .get(name)
+                .get(&name)
                 .context("Checksum references missing member")?
-                .get(algorithm)
+                .get(algorithm.as_str())
                 .unwrap();
             ensure!(
                 expected.len() == actual.len() && expected.bytes().all(|c| c.is_ascii_hexdigit()),
@@ -78,6 +74,29 @@ pub(crate) fn verify(
                 "Gem member checksum mismatch: {name} ({algorithm})"
             );
         }
+        ensure!(
+            names.contains(METADATA_MEMBER) && names.contains(DATA_MEMBER),
+            "Missing member checksum"
+        );
     }
     Ok(())
 }
+
+fn text(value: &serde_yaml_ng::Value) -> Result<String> {
+    let content = metadata::untag(value)
+        .as_str()
+        .context("Invalid checksum text")?;
+    if let serde_yaml_ng::Value::Tagged(tagged) = value
+        && (tagged.tag == "binary" || tagged.tag == "tag:yaml.org,2002:binary")
+    {
+        let encoded: String = content
+            .chars()
+            .filter(|c| !c.is_ascii_whitespace())
+            .collect();
+        let bytes = STANDARD
+            .decode(encoded)
+            .context("Invalid binary checksum encoding")?;
+        return String::from_utf8(bytes).context("Non-UTF8 checksum text");
+    }
+    Ok(content.to_owned())
+}

crates/rb-gem-archive/tests/compatibility.rs

@@ -94,3 +94,41 @@ fn signature_members_must_stay_bounded_and_belong_to_known_members() {
     members.push(("other.sig", b"signature bytes".to_vec()));
     assert!(read(common::container(&members).as_slice()).is_err());
 }
+
+#[test]
+fn binary_tagged_legacy_checksums_are_decoded_and_verified() {
+    use base64::{Engine, engine::general_purpose::STANDARD};
+    let mut members = common::members(include_bytes!("fixtures/metadata.yml"));
+    let checksum = format!(
+        "!binary U0hBMjU2:\n  metadata.gz: !binary {}\n  data.tar.gz: !binary {}\n",
+        STANDARD.encode(format!("{:x}", sha2::Sha256::digest(&members[0].1))),
+        STANDARD.encode(format!("{:x}", sha2::Sha256::digest(&members[1].1)))
+    );
+    members[2].1 = common::gzip(checksum.as_bytes());
+    assert_eq!(
+        read(common::container(&members).as_slice())
+            .unwrap()
+            .id
+            .name,
+        "dinner"
+    );
+    members[1].1.push(0);
+    assert!(read(common::container(&members).as_slice()).is_err());
+}
+
+#[test]
+fn binary_checksum_names_cannot_hide_duplicate_entries() {
+    let mut members = common::members(include_bytes!("fixtures/metadata.yml"));
+    let metadata = format!("{:x}", sha2::Sha256::digest(&members[0].1));
+    let data = format!("{:x}", sha2::Sha256::digest(&members[1].1));
+    let values = format!("  metadata.gz: {metadata}\n  data.tar.gz: {data}\n");
+    members[2].1 = common::gzip(format!("SHA256:\n{values}!binary U0hBMjU2:\n{values}").as_bytes());
+    let error = read(common::container(&members).as_slice()).unwrap_err();
+    assert!(error.to_string().contains("Duplicate checksum algorithm"));
+
+    members[2].1 = common::gzip(
+        format!("SHA256:\n{values}  !binary bWV0YWRhdGEuZ3o=: {metadata}\n").as_bytes(),
+    );
+    let error = read(common::container(&members).as_slice()).unwrap_err();
+    assert!(error.to_string().contains("Duplicate checksum member"));
+}

crates/rb-gem-installer/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "rb-gem-installer"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+description = "Standalone staged Ruby gem payload installation"
+
+[dependencies]
+anyhow = "1.0"
+flate2 = "1.1"
+rb-gem-archive = { path = "../rb-gem-archive" }
+rb-gem-types = { path = "../rb-gem-types" }
+rb-microgems = { path = "../rb-microgems" }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1.0"
+tar = "0.4"
+tempfile = "3"
+
+[target.'cfg(unix)'.dependencies]
+rustix = { version = "1", features = ["fs", "process"] }
+
+[target.'cfg(windows)'.dependencies]
+windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] }
+
+[dev-dependencies]
+sha2 = "0.10"

crates/rb-gem-installer/README.md

@@ -0,0 +1,32 @@
+# rb-gem-installer
+
+Verifies a `.gem` archive and extracts its payload into a new directory. The parent directory must exist and be controlled by the caller; installation does not replace existing gems. Extraction happens in a private staging directory and failed installs are removed.
+
+```rust,no_run
+let archive = std::fs::File::open("rake-13.2.1.gem")?;
+let metadata = rb_gem_installer::install(archive, "gems/rake-13.2.1")?;
+println!("Installed {}", metadata.id);
+# Ok::<(), anyhow::Error>(())
+```
+
+Regular files and directories are supported. Relative symlinks to payload files and directories are supported, including chains. Links outside the installation, cycles, hard links and special files are rejected. Identical repeated files are deduplicated; conflicting paths are rejected. Archive size, expanded size and entry count are bounded. File permissions are preserved on Unix subject to the process umask, without special permission bits. Signature trust follows `rb-gem-archive` and is not verified.
+
+Native compilation is opt-in. `install_with_builder` builds in staging and publishes only after every declared extension succeeds. `NativeBuilder::build` can also build an already extracted directory; callers manage staging when using it directly.
+
+```rust,no_run
+use rb_gem_installer::{BuildOptions, NativeBuilder, install_with_builder};
+
+let builder = NativeBuilder::new(BuildOptions::new("ruby"))?;
+let archive = std::fs::File::open("json.gem")?;
+let (metadata, report) = install_with_builder(archive, "gems/json", &builder, |_| {})?;
+println!("Built {} outputs for {}", report.artifacts.len(), metadata.id);
+# Ok::<(), anyhow::Error>(())
+```
+
+Builders support extconf/make, rake and mkrf, configure, CMake and Cargo. Ruby configuration comes from the selected interpreter; its development headers and the required build tools must be available. Build arguments, dependency load paths, environment, tool paths, job count and per-command timeout are caller-controlled. Structured events stream stdout, stderr and command stages without a reporting dependency. Build logs retained in errors are bounded; callbacks receive the full stream.
+
+Collected outputs are stored under `.rb-native/<architecture>/<Ruby ABI>` and copied into the first require path. Reports also include native libraries written directly into gem require paths or extension directories. This crate does not resolve build dependencies, generate executable wrappers or activate gems for application runtime. Build subprocesses preload `rb-microgems` for minimal RubyGems API compatibility. Supply exact installed build dependencies through `BuildOptions::dependencies`; missing gems and unsupported APIs fail explicitly. Bundler and full RubyGems are not loaded.
+
+The regular tests include a public gem checked against RubyGems extraction. To run the larger local corpus, set `RB_GEM_CORPUS` to its absolute `gems` directory and run `cargo test -p rb-gem-installer --test corpus -- --ignored --nocapture`.
+
+The Ruby configuration helper is a standalone gem in `gems/rb-gem-installer`. Its files are embedded in this crate and materialized for the selected interpreter without a gem installation. Run its Ruby tests with `ruby gems/rb-gem-installer/test/all_test.rb` from the repository root.

crates/rb-gem-installer/examples/compile.rs

@@ -0,0 +1,44 @@
+use rb_gem_installer::{BuildEvent, BuildOptions, NativeBuilder};
+use std::{io::Write, time::Duration};
+
+fn main() -> anyhow::Result<()> {
+    let mut arguments = std::env::args_os().skip(1);
+    let archive = arguments
+        .next()
+        .ok_or_else(|| anyhow::anyhow!("Expected archive path"))?;
+    let destination = arguments
+        .next()
+        .ok_or_else(|| anyhow::anyhow!("Expected destination path"))?;
+    let ruby = arguments.next().unwrap_or_else(|| "ruby".into());
+    let mut options = BuildOptions::new(ruby);
+    options.arguments = arguments.collect();
+    if let Ok(seconds) = std::env::var("RB_GEM_BUILD_TIMEOUT") {
+        options.timeout = Duration::from_secs(seconds.parse()?);
+    }
+    if let Ok(jobs) = std::env::var("RB_GEM_BUILD_JOBS") {
+        options.jobs = jobs.parse()?;
+    }
+    if let Some(paths) = std::env::var_os("RB_GEM_BUILD_LOAD_PATHS") {
+        options.load_paths = std::env::split_paths(&paths).collect();
+    }
+    if let Some(path) = std::env::var_os("RB_GEM_BUILD_DEPENDENCIES") {
+        options.dependencies = serde_json::from_slice(&std::fs::read(path)?)?;
+    }
+    let builder = NativeBuilder::new(options)?;
+    let (metadata, report) = rb_gem_installer::install_with_builder(
+        std::fs::File::open(archive)?,
+        destination,
+        &builder,
+        |event| {
+            if let BuildEvent::Output { bytes, .. } = event {
+                let _ = std::io::stderr().write_all(&bytes);
+            }
+        },
+    )?;
+    println!(
+        "Installed {} with {} build outputs",
+        metadata.id,
+        report.artifacts.len()
+    );
+    Ok(())
+}

crates/rb-gem-installer/examples/install.rs

@@ -0,0 +1,12 @@
+fn main() -> anyhow::Result<()> {
+    let mut args = std::env::args_os().skip(1);
+    let archive = args
+        .next()
+        .ok_or_else(|| anyhow::anyhow!("Expected archive path"))?;
+    let destination = args
+        .next()
+        .ok_or_else(|| anyhow::anyhow!("Expected destination path"))?;
+    let metadata = rb_gem_installer::install(std::fs::File::open(archive)?, destination)?;
+    println!("Installed {}", metadata.id);
+    Ok(())
+}

crates/rb-gem-installer/src/build_diagnostics.rs

@@ -0,0 +1,20 @@
+use crate::constants::MAX_BUILD_OUTPUT_BYTES;
+use std::{
+    fs::File,
+    io::{Read, Seek, SeekFrom},
+    path::Path,
+};
+
+pub(crate) fn mkmf(directory: Option<&Path>) -> Option<String> {
+    let mut file = File::open(directory?.join("mkmf.log")).ok()?;
+    let bytes = file.metadata().ok()?.len();
+    file.seek(SeekFrom::Start(
+        bytes.saturating_sub(MAX_BUILD_OUTPUT_BYTES as u64),
+    ))
+    .ok()?;
+    let mut tail = Vec::new();
+    file.take(MAX_BUILD_OUTPUT_BYTES as u64)
+        .read_to_end(&mut tail)
+        .ok()?;
+    Some(String::from_utf8_lossy(&tail).into_owned())
+}