Add standalone rb-packages bundle preparation

.github/workflows/ci.yml

@@ -127,7 +127,7 @@ jobs:
       run: gem install minitest --version '~> 5.25' --no-document
 
     - name: Test the Ruby evaluator
-      run: ruby crates/rb-gemfile/tests/ruby/evaluator_test.rb
+      run: ruby gems/rb-gemfile/test/all_test.rb
 
     - name: Test MicroGems
       run: ruby gems/rb-microgems/test/all_test.rb
@@ -154,10 +154,10 @@ jobs:
         bundler: none
 
     - name: Test gem installation and build compatibility
-      run: cargo test -p rb-gem-installer -p rb-microgems
+      run: cargo test -p rb-gem-installer -p rb-microgems -p rb-packages
 
     - name: Build installer examples
-      run: cargo build -p rb-gem-installer --examples
+      run: cargo build -p rb-gem-installer -p rb-packages --examples
 
   test-rust:
     name: Rust Tests

crates/rb-compact-index/README.md

@@ -15,6 +15,9 @@ for gem in client.info("rack")? {
 
 | API | Purpose |
 | --- | --- |
+| `Client::names` | Discover registry gem names, with an offline cache. |
+| `Client::with_credentials` | Apply HTTP Basic credentials to registry requests. |
+| `Client::info_or_empty` | Look up candidates, allowing confirmed missing gems. |
 | `Client::info` | Fetch a gem's versions, dependency requirements and metadata. |
 | `Client::download` | Fetch an archive with an expected SHA-256 checksum. |
 | `parse` | Parse a compact-index response without HTTP or cache access. |

crates/rb-compact-index/src/client.rs

@@ -24,7 +24,7 @@ use std::{
 pub struct Client {
     pub(crate) source: String,
     pub(crate) cache: PathBuf,
-    offline: bool,
+    pub(crate) offline: bool,
     pub(crate) client: HttpClient,
 }
 
@@ -67,7 +67,38 @@ impl Client {
         &self.source
     }
 
+    pub fn with_credentials(mut self, credentials: &str) -> Result<Self> {
+        let mut headers = header::HeaderMap::new();
+        let credentials = if credentials.contains(':') {
+            credentials.to_owned()
+        } else {
+            format!("{credentials}:")
+        };
+        let mut authorization = header::HeaderValue::from_str(&format!(
+            "Basic {}",
+            base64::engine::general_purpose::STANDARD.encode(credentials)
+        ))?;
+        authorization.set_sensitive(true);
+        headers.insert(header::AUTHORIZATION, authorization);
+        self.client = HttpClient::builder()
+            .timeout(REQUEST_TIMEOUT)
+            .connect_timeout(CONNECT_TIMEOUT)
+            .user_agent(concat!("rb-compact-index/", env!("CARGO_PKG_VERSION")))
+            .default_headers(headers)
+            .build()?;
+        Ok(self)
+    }
+
     pub fn info(&self, name: &str) -> Result<Vec<GemInfo>> {
+        self.fetch_info(name, false)
+    }
+
+    /// Returns no candidates for a missing gem on a working compact-index server.
+    pub fn info_or_empty(&self, name: &str) -> Result<Vec<GemInfo>> {
+        self.fetch_info(name, true)
+    }
+
+    fn fetch_info(&self, name: &str, allow_missing: bool) -> Result<Vec<GemInfo>> {
         validate_name(name)?;
         let directory = self
             .cache
@@ -117,31 +148,25 @@ impl Client {
             if versions.status() == reqwest::StatusCode::NOT_FOUND {
                 return self.full_info(name, &directory);
             }
+            versions.error_for_status()?;
+            if allow_missing {
+                atomic_write(
+                    &path,
+                    &serde_json::to_vec(&CachedInfo {
+                        etag: None,
+                        body: "---\n".into(),
+                    })?,
+                )?;
+                return Ok(Vec::new());
+            }
         }
         let response = response.error_for_status()?;
         let etag = response
             .headers()
             .get(header::ETAG)
             .and_then(|v| v.to_str().ok())
             .map(str::to_owned);
-        let digest = response
-            .headers()
-            .get("repr-digest")
-            .and_then(|v| v.to_str().ok())
-            .map(str::to_owned);
-        let bytes = limited_read(response, MAX_COMPACT_INFO_BYTES)?;
-        if let Some(digest) = digest
-            && let Some(value) = digest
-                .split(',')
-                .find_map(|part| part.trim().strip_prefix("sha-256="))
-        {
-            let expected =
-                base64::engine::general_purpose::STANDARD.decode(value.trim_matches([':', '"']))?;
-            ensure!(
-                sha2::Sha256::digest(&bytes).as_slice() == expected,
-                "Index digest mismatch for {name}"
-            );
-        }
+        let bytes = read_index(response, MAX_COMPACT_INFO_BYTES)?;
         let body = String::from_utf8(bytes)?;
         let gems = parse(name, &body)?;
         atomic_write(&path, &serde_json::to_vec(&CachedInfo { etag, body })?)?;
@@ -190,3 +215,25 @@ impl Client {
         Ok(path)
     }
 }
+
+pub(crate) fn read_index(response: reqwest::blocking::Response, limit: u64) -> Result<Vec<u8>> {
+    let digest = response
+        .headers()
+        .get("repr-digest")
+        .and_then(|v| v.to_str().ok())
+        .map(str::to_owned);
+    let bytes = limited_read(response, limit)?;
+    if let Some(digest) = digest
+        && let Some(value) = digest
+            .split(',')
+            .find_map(|part| part.trim().strip_prefix("sha-256="))
+    {
+        let expected =
+            base64::engine::general_purpose::STANDARD.decode(value.trim_matches([':', '"']))?;
+        ensure!(
+            sha2::Sha256::digest(&bytes).as_slice() == expected,
+            "Index digest mismatch"
+        );
+    }
+    Ok(bytes)
+}

crates/rb-compact-index/src/constants.rs

@@ -5,13 +5,16 @@ pub(crate) const CONNECT_TIMEOUT: Duration = Duration::from_secs(15);
 
 pub(crate) const ARCHIVE_BUFFER_BYTES: usize = 64 * 1024;
 pub(crate) const MAX_ARCHIVE_BYTES: u64 = 1024 * 1024 * 1024;
+pub(crate) const MAX_VERSIONS_BYTES: u64 = 128 * 1024 * 1024;
 pub(crate) const MAX_COMPACT_INFO_BYTES: u64 = 32 * 1024 * 1024;
 pub(crate) const MAX_COMPRESSED_INDEX_BYTES: u64 = 16 * 1024 * 1024;
 pub(crate) const MAX_DECOMPRESSED_INDEX_BYTES: u64 = 32 * 1024 * 1024;
 pub(crate) const MAX_GEM_METADATA_BYTES: u64 = 16 * 1024 * 1024;
 pub(crate) const MAX_FALLBACK_ARCHIVE_BYTES: u64 = 256 * 1024 * 1024;
 
 pub(crate) const INDEX_CACHE_DIRECTORY: &str = "index";
+pub(crate) const REGISTRY_METADATA_DIRECTORY: &str = "registry";
+pub(crate) const VERSIONS_CACHE_FILE: &str = "versions.json";
 pub(crate) const ARCHIVE_CACHE_DIRECTORY: &str = "gems";
 pub(crate) const FULL_INFO_CACHE_SUFFIX: &str = ".full-platforms.json";
 

crates/rb-compact-index/src/lib.rs

@@ -6,6 +6,7 @@ mod constants;
 mod full_index;
 mod info;
 mod validation;
+mod versions;
 
 pub use client::Client;
 pub use info::{GemInfo, parse};

crates/rb-compact-index/src/tests.rs

@@ -16,10 +16,12 @@ use std::{
     thread,
 };
 
+mod authentication;
 mod download;
 mod fixtures;
 mod full_index;
 mod info;
 mod server;
+mod versions;
 
 use server::Server;

crates/rb-compact-index/src/tests/authentication.rs

@@ -0,0 +1,73 @@
+use super::*;
+use base64::Engine;
+
+#[test]
+fn credentials_authenticate_metadata_and_archives_without_entering_source_or_cache() {
+    let server = Server::new();
+    let cache = tempfile::tempdir().unwrap();
+    let archive = b"fixture archive";
+    let sha = checksum(archive);
+    server.responses.lock().unwrap().extend([
+        (
+            "/info/example".into(),
+            format!("---\n1.0 |checksum:{sha}\n").into_bytes(),
+        ),
+        ("/versions".into(), b"---\nexample 1.0 abc\n".to_vec()),
+        ("/gems/example-1.0.gem".into(), archive.to_vec()),
+    ]);
+    let client = Client::new(&server.source, cache.path(), false)
+        .unwrap()
+        .with_credentials("user:secret-token")
+        .unwrap();
+    assert_eq!(client.source(), server.source);
+    assert_eq!(client.info("example").unwrap().len(), 1);
+    assert!(client.names().unwrap().unwrap().contains("example"));
+    client.download("example", "1.0", &sha).unwrap();
+    let expected = format!(
+        "authorization: basic {}",
+        base64::engine::general_purpose::STANDARD
+            .encode("user:secret-token")
+            .to_lowercase()
+    );
+    for request in server.request_headers.lock().unwrap().iter() {
+        assert!(request.to_lowercase().contains(&expected));
+    }
+    let info = cache
+        .path()
+        .join("index")
+        .join(checksum(server.source.as_bytes()))
+        .join("example.json");
+    let cached = fs::read_to_string(info).unwrap();
+    assert!(!cached.contains("secret-token"));
+    assert!(!cached.contains("authorization"));
+}
+
+#[test]
+fn credentials_are_not_forwarded_to_another_origin_on_redirect() {
+    let server = Server::new();
+    let destination = Server::new();
+    server
+        .statuses
+        .lock()
+        .unwrap()
+        .insert("/info/example".into(), "302 Found".into());
+    server.headers.lock().unwrap().insert(
+        "/info/example".into(),
+        format!("Location: {}info/example\r\n", destination.source),
+    );
+    destination.responses.lock().unwrap().insert(
+        "/info/example".into(),
+        format!("---\n1.0 |checksum:{}\n", "a".repeat(64)).into_bytes(),
+    );
+    let cache = tempfile::tempdir().unwrap();
+    let client = Client::new(&server.source, cache.path(), false)
+        .unwrap()
+        .with_credentials("user:secret-token")
+        .unwrap();
+    client.info("example").unwrap();
+    assert!(
+        !destination.request_headers.lock().unwrap()[0]
+            .to_lowercase()
+            .contains("authorization:")
+    );
+}

crates/rb-compact-index/src/tests/info.rs

@@ -235,3 +235,81 @@ fn empty_or_unterminated_headers_are_rejected() {
         assert!(parse("app", body).is_err());
     }
 }
+
+#[test]
+fn missing_optional_lookup_is_cached_offline_and_rechecked_online() {
+    let server = Server::new();
+    server
+        .responses
+        .lock()
+        .unwrap()
+        .insert("/versions".into(), b"---\n".to_vec());
+    let cache = tempfile::tempdir().unwrap();
+    let client = Client::new(&server.source, cache.path(), false).unwrap();
+    assert!(client.info_or_empty("missing").unwrap().is_empty());
+    assert_eq!(
+        *server.requests.lock().unwrap(),
+        ["/info/missing", "/versions"]
+    );
+    let offline = Client::new(&server.source, cache.path(), true).unwrap();
+    assert!(offline.info_or_empty("missing").unwrap().is_empty());
+    server.responses.lock().unwrap().insert(
+        "/info/missing".into(),
+        format!("---\n1.0 |checksum:{}\n", "a".repeat(64)).into_bytes(),
+    );
+    let found = client.info_or_empty("missing").unwrap();
+    assert_eq!(found.len(), 1);
+    assert_eq!(found[0].version, "1.0");
+    drop(server);
+    assert_eq!(offline.info_or_empty("missing").unwrap(), found);
+}
+
+#[test]
+fn optional_lookup_does_not_hide_access_server_or_protocol_errors() {
+    for status in [
+        "401 Unauthorized",
+        "403 Forbidden",
+        "500 Internal Server Error",
+    ] {
+        let server = Server::new();
+        server
+            .statuses
+            .lock()
+            .unwrap()
+            .insert("/info/private".into(), status.into());
+        server
+            .responses
+            .lock()
+            .unwrap()
+            .insert("/versions".into(), b"---\n".to_vec());
+        let cache = tempfile::tempdir().unwrap();
+        let client = Client::new(&server.source, cache.path(), false).unwrap();
+        assert!(client.info_or_empty("private").is_err());
+        assert_eq!(*server.requests.lock().unwrap(), ["/info/private"]);
+        assert!(
+            Client::new(&server.source, cache.path(), true)
+                .unwrap()
+                .info_or_empty("private")
+                .is_err()
+        );
+    }
+    let server = Server::new();
+    server
+        .statuses
+        .lock()
+        .unwrap()
+        .insert("/versions".into(), "403 Forbidden".into());
+    let cache = tempfile::tempdir().unwrap();
+    assert!(
+        Client::new(&server.source, cache.path(), false)
+            .unwrap()
+            .info_or_empty("private")
+            .is_err()
+    );
+    assert!(
+        Client::new(&server.source, cache.path(), true)
+            .unwrap()
+            .info_or_empty("private")
+            .is_err()
+    );
+}