diff --git a/crates/storage/opendal/src/azdls.rs b/crates/storage/opendal/src/azdls.rs index 88a197fea8..dbeccfeec9 100644 --- a/crates/storage/opendal/src/azdls.rs +++ b/crates/storage/opendal/src/azdls.rs @@ -324,10 +324,12 @@ fn validate_storage_and_scheme( #[cfg(test)] mod tests { use std::collections::HashMap; + use std::sync::Arc; use opendal::services::AzdlsConfig; use super::{AzureStoragePath, AzureStorageScheme, azdls_config_parse, azdls_create_operator}; + use crate::{OpenDalClientConfig, OpenDalStorage}; #[test] fn test_azdls_config_parse() { @@ -586,4 +588,23 @@ mod tests { assert_eq!(endpoint, expected, "Test case: {name}"); } } + + #[test] + fn test_relativize_path_azdls() { + let storage = OpenDalStorage::Azdls { + config: Arc::new(AzdlsConfig { + account_name: Some("myaccount".to_string()), + endpoint: Some("https://myaccount.dfs.core.windows.net".to_string()), + ..Default::default() + }), + client_config: OpenDalClientConfig::default(), + }; + + assert_eq!( + storage + .relativize_path("abfss://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet") + .unwrap(), + "path/to/file.parquet" + ); + } } diff --git a/crates/storage/opendal/src/fs.rs b/crates/storage/opendal/src/fs.rs index 1d830605a8..67ea06a772 100644 --- a/crates/storage/opendal/src/fs.rs +++ b/crates/storage/opendal/src/fs.rs @@ -28,3 +28,26 @@ pub(crate) fn fs_config_build() -> Result { Operator::from_config(cfg).map_err(from_opendal_error) } + +#[cfg(test)] +mod tests { + use crate::{OpenDalClientConfig, OpenDalStorage}; + + #[test] + fn test_relativize_path_fs() { + let storage = OpenDalStorage::LocalFs { + client_config: OpenDalClientConfig::default(), + }; + + assert_eq!( + storage + .relativize_path("file:/tmp/data/file.parquet") + .unwrap(), + "tmp/data/file.parquet" + ); + assert_eq!( + storage.relativize_path("/tmp/data/file.parquet").unwrap(), + "tmp/data/file.parquet" + ); + } +} diff --git a/crates/storage/opendal/src/gcs.rs b/crates/storage/opendal/src/gcs.rs index a00282cc22..09b4749811 100644 --- a/crates/storage/opendal/src/gcs.rs +++ b/crates/storage/opendal/src/gcs.rs @@ -84,3 +84,41 @@ pub(crate) fn gcs_config_build(cfg: &GcsConfig, path: &str) -> Result cfg.bucket = bucket.to_string(); Operator::from_config(cfg).map_err(from_opendal_error) } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use opendal::services::GcsConfig; + + use crate::{OpenDalClientConfig, OpenDalStorage}; + + #[test] + fn test_relativize_path_gcs() { + let storage = OpenDalStorage::Gcs { + config: Arc::new(GcsConfig::default()), + client_config: OpenDalClientConfig::default(), + }; + + assert_eq!( + storage + .relativize_path("gs://my-bucket/path/to/file.parquet") + .unwrap(), + "path/to/file.parquet" + ); + } + + #[test] + fn test_relativize_path_gcs_invalid_scheme() { + let storage = OpenDalStorage::Gcs { + config: Arc::new(GcsConfig::default()), + client_config: OpenDalClientConfig::default(), + }; + + assert!( + storage + .relativize_path("s3://my-bucket/path/to/file.parquet") + .is_err() + ); + } +} diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 693f4c18b3..8d3d48580b 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -1006,44 +1006,6 @@ mod tests { assert!(err.contains("{ timeout: 45 }"), "{err}"); } - #[cfg(feature = "opendal-s3")] - #[derive(Debug)] - struct EmptyCredentialLoader; - - #[cfg(feature = "opendal-s3")] - impl ProvideCredential for EmptyCredentialLoader { - type Credential = AwsCredential; - - async fn provide_credential( - &self, - _ctx: &reqsign_core::Context, - ) -> reqsign_core::Result> { - Ok(None) - } - } - - #[cfg(feature = "opendal-s3")] - #[test] - fn test_s3_factory_custom_credential_loader_serialization_fails() { - let file_io = iceberg::io::FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { - customized_credential_load: Some(CustomAwsCredentialLoader::new(EmptyCredentialLoader)), - })) - .build(); - - let err = file_io.serialize_all().unwrap_err(); - assert!( - err.to_string() - .contains("custom AWS credential loaders cannot be serialized") - ); - } - - #[cfg(feature = "opendal-memory")] - #[test] - fn test_default_memory_operator() { - let op = default_memory_operator(); - assert_eq!(op.info().scheme().to_string(), "memory"); - } - #[cfg(feature = "opendal-memory")] #[tokio::test] async fn test_writer_close_returns_stored_size() { @@ -1078,146 +1040,4 @@ mod tests { assert_eq!(metadata.size, storage.metadata(path).await.unwrap().size); } } - - #[cfg(feature = "opendal-memory")] - #[test] - fn test_relativize_path_memory() { - let storage = OpenDalStorage::Memory { - operator: default_memory_operator(), - client_config: OpenDalClientConfig::default(), - }; - - assert_eq!( - storage.relativize_path("memory:/path/to/file").unwrap(), - "path/to/file" - ); - // Without the scheme prefix, falls back to stripping the leading slash - assert_eq!( - storage.relativize_path("/path/to/file").unwrap(), - "path/to/file" - ); - } - - #[cfg(feature = "opendal-fs")] - #[test] - fn test_relativize_path_fs() { - let storage = OpenDalStorage::LocalFs { - client_config: OpenDalClientConfig::default(), - }; - - assert_eq!( - storage - .relativize_path("file:/tmp/data/file.parquet") - .unwrap(), - "tmp/data/file.parquet" - ); - assert_eq!( - storage.relativize_path("/tmp/data/file.parquet").unwrap(), - "tmp/data/file.parquet" - ); - } - - #[cfg(feature = "opendal-s3")] - #[test] - fn test_relativize_path_s3() { - let storage = OpenDalStorage::S3 { - config: Arc::new(S3Config::default()), - customized_credential_load: None, - client_config: OpenDalClientConfig::default(), - }; - - // All S3-family schemes are accepted by the same storage instance. - // Custom schemes for S3-compatible stores (e.g., `minio://`) are also - // accepted because the path's scheme is used as-is for prefix matching. - for scheme in ["s3", "s3a", "s3n", "minio"] { - assert_eq!( - storage - .relativize_path(&format!("{scheme}://my-bucket/path/to/file.parquet")) - .unwrap(), - "path/to/file.parquet" - ); - } - } - - #[cfg(feature = "opendal-gcs")] - #[test] - fn test_relativize_path_gcs() { - let storage = OpenDalStorage::Gcs { - config: Arc::new(GcsConfig::default()), - client_config: OpenDalClientConfig::default(), - }; - - assert_eq!( - storage - .relativize_path("gs://my-bucket/path/to/file.parquet") - .unwrap(), - "path/to/file.parquet" - ); - } - - #[cfg(feature = "opendal-gcs")] - #[test] - fn test_relativize_path_gcs_invalid_scheme() { - let storage = OpenDalStorage::Gcs { - config: Arc::new(GcsConfig::default()), - client_config: OpenDalClientConfig::default(), - }; - - assert!( - storage - .relativize_path("s3://my-bucket/path/to/file.parquet") - .is_err() - ); - } - - #[cfg(feature = "opendal-oss")] - #[test] - fn test_relativize_path_oss() { - let storage = OpenDalStorage::Oss { - config: Arc::new(OssConfig::default()), - client_config: OpenDalClientConfig::default(), - }; - - assert_eq!( - storage - .relativize_path("oss://my-bucket/path/to/file.parquet") - .unwrap(), - "path/to/file.parquet" - ); - } - - #[cfg(feature = "opendal-oss")] - #[test] - fn test_relativize_path_oss_invalid_scheme() { - let storage = OpenDalStorage::Oss { - config: Arc::new(OssConfig::default()), - client_config: OpenDalClientConfig::default(), - }; - - assert!( - storage - .relativize_path("s3://my-bucket/path/to/file.parquet") - .is_err() - ); - } - - #[cfg(feature = "opendal-azdls")] - #[test] - fn test_relativize_path_azdls() { - let storage = OpenDalStorage::Azdls { - config: Arc::new(AzdlsConfig { - account_name: Some("myaccount".to_string()), - endpoint: Some("https://myaccount.dfs.core.windows.net".to_string()), - ..Default::default() - }), - client_config: OpenDalClientConfig::default(), - }; - - assert_eq!( - storage - .relativize_path("abfss://myfs@myaccount.dfs.core.windows.net/path/to/file.parquet") - .unwrap(), - "path/to/file.parquet" - ); - } } diff --git a/crates/storage/opendal/src/memory.rs b/crates/storage/opendal/src/memory.rs index 746e927161..e61e06e14c 100644 --- a/crates/storage/opendal/src/memory.rs +++ b/crates/storage/opendal/src/memory.rs @@ -24,3 +24,32 @@ use crate::utils::from_opendal_error; pub(crate) fn memory_config_build() -> Result { Operator::from_config(MemoryConfig::default()).map_err(from_opendal_error) } + +#[cfg(test)] +mod tests { + use crate::{OpenDalClientConfig, OpenDalStorage, default_memory_operator}; + + #[test] + fn test_default_memory_operator() { + let op = default_memory_operator(); + assert_eq!(op.info().scheme().to_string(), "memory"); + } + + #[test] + fn test_relativize_path_memory() { + let storage = OpenDalStorage::Memory { + operator: default_memory_operator(), + client_config: OpenDalClientConfig::default(), + }; + + assert_eq!( + storage.relativize_path("memory:/path/to/file").unwrap(), + "path/to/file" + ); + // Without the scheme prefix, falls back to stripping the leading slash + assert_eq!( + storage.relativize_path("/path/to/file").unwrap(), + "path/to/file" + ); + } +} diff --git a/crates/storage/opendal/src/oss.rs b/crates/storage/opendal/src/oss.rs index a1cf44549c..f31a8594cb 100644 --- a/crates/storage/opendal/src/oss.rs +++ b/crates/storage/opendal/src/oss.rs @@ -55,3 +55,41 @@ pub(crate) fn oss_config_build(cfg: &OssConfig, path: &str) -> Result Operator::new(builder).map_err(from_opendal_error) } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use opendal::services::OssConfig; + + use crate::{OpenDalClientConfig, OpenDalStorage}; + + #[test] + fn test_relativize_path_oss() { + let storage = OpenDalStorage::Oss { + config: Arc::new(OssConfig::default()), + client_config: OpenDalClientConfig::default(), + }; + + assert_eq!( + storage + .relativize_path("oss://my-bucket/path/to/file.parquet") + .unwrap(), + "path/to/file.parquet" + ); + } + + #[test] + fn test_relativize_path_oss_invalid_scheme() { + let storage = OpenDalStorage::Oss { + config: Arc::new(OssConfig::default()), + client_config: OpenDalClientConfig::default(), + }; + + assert!( + storage + .relativize_path("s3://my-bucket/path/to/file.parquet") + .is_err() + ); + } +} diff --git a/crates/storage/opendal/src/s3.rs b/crates/storage/opendal/src/s3.rs index 2997fa81d9..7dad826a85 100644 --- a/crates/storage/opendal/src/s3.rs +++ b/crates/storage/opendal/src/s3.rs @@ -183,10 +183,13 @@ impl CustomAwsCredentialLoader { #[cfg(test)] mod tests { use std::collections::HashMap; + use std::sync::Arc; use iceberg::io::S3_PATH_STYLE_ACCESS; + use opendal::services::S3Config; - use super::s3_config_parse; + use super::{AwsCredential, CustomAwsCredentialLoader, ProvideCredential, s3_config_parse}; + use crate::{OpenDalClientConfig, OpenDalStorage, OpenDalStorageFactory}; fn parse_with(prop: Option<&str>) -> bool { let mut props = HashMap::new(); @@ -196,6 +199,20 @@ mod tests { s3_config_parse(props).unwrap().enable_virtual_host_style } + #[derive(Debug)] + struct EmptyCredentialLoader; + + impl ProvideCredential for EmptyCredentialLoader { + type Credential = AwsCredential; + + async fn provide_credential( + &self, + _ctx: &reqsign_core::Context, + ) -> reqsign_core::Result> { + Ok(None) + } + } + #[test] fn s3_config_parse_path_style_access() { // Match Iceberg S3FileIOProperties.PATH_STYLE_ACCESS_DEFAULT = false. @@ -203,4 +220,39 @@ mod tests { assert!(parse_with(Some("false"))); assert!(!parse_with(Some("true"))); } + + #[test] + fn test_relativize_path_s3() { + let storage = OpenDalStorage::S3 { + config: Arc::new(S3Config::default()), + customized_credential_load: None, + client_config: OpenDalClientConfig::default(), + }; + + // All S3-family schemes are accepted by the same storage instance. + // Custom schemes for S3-compatible stores (e.g., `minio://`) are also + // accepted because the path's scheme is used as-is for prefix matching. + for scheme in ["s3", "s3a", "s3n", "minio"] { + assert_eq!( + storage + .relativize_path(&format!("{scheme}://my-bucket/path/to/file.parquet")) + .unwrap(), + "path/to/file.parquet" + ); + } + } + + #[test] + fn test_s3_factory_custom_credential_loader_serialization_fails() { + let file_io = iceberg::io::FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: Some(CustomAwsCredentialLoader::new(EmptyCredentialLoader)), + })) + .build(); + + let err = file_io.serialize_all().unwrap_err(); + assert!( + err.to_string() + .contains("custom AWS credential loaders cannot be serialized") + ); + } }