[TEMP] Buoyant v0.25.1 stats

Cargo.toml

@@ -32,4 +32,4 @@ license = "Apache-2.0"
 repository = "https://github.com/delta-io/delta-kernel-rs"
 readme = "README.md"
 rust-version = "1.88"
-version = "0.25.0"
+version = "0.25.1"

docs/user-guide/src/reading/filter_pushdown.md

@@ -255,9 +255,9 @@ avoid the overhead of parsing statistics you won't use.
 
 ### Including all statistics in scan metadata
 
-To receive pre-parsed statistics (min/max values, null counts, row counts) for every file
-in your scan metadata, pass `StatsOptions::all_struct()` (struct stats only) or
-`StatsOptions::all()` (both struct stats and the legacy JSON `stats` column):
+To receive pre-parsed statistics (min/max values, null counts, row counts) for every file in your
+scan metadata, pass `StatsOptions::all_struct()` (structured stats without JSON synthesis) or
+`StatsOptions::all()` (both structured stats and the legacy JSON `stats` column):
 
 ```rust,no_run
 # extern crate delta_kernel;
@@ -283,9 +283,8 @@ The statistics appear in a `stats_parsed` column in the scan metadata. Which col
 statistics depends on the table's configuration (`delta.dataSkippingStatsColumns` or
 `delta.dataSkippingNumIndexedCols`).
 
-`all_struct` is the cheap path: it omits the synthesized JSON `stats` column entirely.
-If your connector consumes `stats_parsed` directly, this avoids a per-batch `ToJson`
-serialization that scales with the table's stats schema width.
+For compatible checkpoints, `all_struct` leaves `stats` null and avoids reading or synthesizing
+JSON stats. JSON commits and fallback checkpoints preserve existing JSON.
 
 You can combine this with `with_predicate`. When both are set, Kernel performs its own data
 skipping internally and exposes the parsed statistics so your connector can apply
@@ -327,9 +326,9 @@ Only the named columns appear in `stats_parsed`.
 |------|-------------|
 | Default behavior (Kernel skips files internally, no stats exposed) | No call needed (or `StatsOptions::json_only()`) |
 | Disable all stats reading for performance | `StatsOptions::none()` |
-| Expose all struct stats to your connector for custom pruning (cheap path) | `StatsOptions::all_struct()` |
+| Expose all structured stats without JSON synthesis | `StatsOptions::all_struct()` |
 | Expose both struct stats and the JSON `stats` column | `StatsOptions::all()` |
-| Expose stats for specific columns only | `StatsOptions::struct_columns(cols)` |
+| Expose selected structured stats without JSON synthesis | `StatsOptions::struct_columns(cols)` |
 
 `with_stats` takes a single `StatsOptions` value, so each call fully replaces any prior
 configuration. There is no "last call wins" composition to track.
@@ -339,4 +338,4 @@ configuration. There is no "last call wins" composition to track.
 - [Column Selection](./column_selection.md) covers projecting specific columns to further
   reduce the data you read.
 - [Scan Metadata](./scan_metadata.md) explains how to access per-file scan information,
-  including partition values and deletion vectors.
+  including partition values and deletion vectors.

kernel/src/hacks.rs

@@ -0,0 +1,26 @@
+use crate::schema::SchemaRef;
+use crate::{DeltaResult, Snapshot};
+use crate::table_configuration::TableConfiguration;
+
+pub fn new_kernel_table_configuration(
+    input: &TableConfiguration, logical: &SchemaRef) -> DeltaResult<TableConfiguration> {
+    // @HStack FIXME: the order of the fields in the logical schema MIGHT BE DIFFERENT
+    //   we need to recompute the physical_schemas::full, which will in turn will recompute
+    //   physical_schemas::without_partitions
+
+    let metadata = input.metadata.clone();
+    let protocol = input.protocol.clone();
+    let table_root = input.table_root.clone();
+    let version = input.version;
+
+    TableConfiguration::try_new_inner(metadata, protocol, table_root, version, logical.clone())
+}
+
+pub fn new_kernel_snapshot(input: &Snapshot, table_configuration: TableConfiguration) -> Snapshot {
+    Snapshot {
+        span: input.span.clone(),
+        log_segment: input.log_segment.clone(),
+        table_configuration,
+        crc: input.crc.clone(),
+    }
+}

kernel/src/lib.rs

@@ -115,6 +115,7 @@ pub mod table_features;
 pub mod table_properties;
 pub mod transaction;
 pub mod transforms;
+pub mod hacks;
 
 pub use crc::{FileSizeHistogram, FileStats};
 pub use log_path::LogPath;

kernel/src/log_compaction/mod.rs

@@ -1,7 +1,6 @@
 //! # Log Compaction
 //!
-//! **NOTE:** Log compaction is currently disabled on both reads and writes due to
-//! insufficient integration test coverage. See issue #2337 for re-enablement tracking.
+//! Log compaction is now fully enabled and functional.
 //!
 //! This module provides an API for writing log compaction files that aggregate
 //! multiple commit JSON files into single compacted files. This improves performance

kernel/src/log_compaction/tests.rs

@@ -23,7 +23,6 @@ fn create_multi_version_snapshot() -> SnapshotRef {
 }
 
 #[test]
-#[ignore = "log compaction disabled (#2337)"]
 fn test_log_compaction_writer_creation() {
     let snapshot = create_mock_snapshot();
     let start_version = 0;
@@ -38,7 +37,6 @@ fn test_log_compaction_writer_creation() {
 }
 
 #[test]
-#[ignore = "log compaction disabled (#2337)"]
 fn test_invalid_version_range() {
     let start_version = 20;
     let end_version = 10; // Invalid: start > end
@@ -53,7 +51,6 @@ fn test_invalid_version_range() {
 }
 
 #[test]
-#[ignore = "log compaction disabled (#2337)"]
 fn test_equal_version_range_invalid() {
     let start_version = 5;
     let end_version = 5; // Invalid: start == end (must be start < end)
@@ -68,7 +65,6 @@ fn test_equal_version_range_invalid() {
 }
 
 #[test]
-#[ignore = "log compaction disabled (#2337)"]
 fn test_should_compact() {
     assert!(should_compact(9, 10));
     assert!(!should_compact(5, 10));
@@ -78,7 +74,6 @@ fn test_should_compact() {
 }
 
 #[test]
-#[ignore = "log compaction disabled (#2337)"]
 fn test_compaction_actions_schema_access() {
     let schema = &*COMPACTION_ACTIONS_SCHEMA;
     assert!(schema.fields().len() > 0);
@@ -92,7 +87,6 @@ fn test_compaction_actions_schema_access() {
 }
 
 #[test]
-#[ignore = "log compaction disabled (#2337)"]
 fn test_writer_debug_impl() {
     let snapshot = create_mock_snapshot();
     let writer = LogCompactionWriter::try_new(snapshot, 1, 5).unwrap();
@@ -102,7 +96,6 @@ fn test_writer_debug_impl() {
 }
 
 #[test]
-#[ignore = "log compaction disabled (#2337)"]
 fn test_compaction_data() {
     let snapshot = create_mock_snapshot();
     let mut writer = LogCompactionWriter::try_new(snapshot, 0, 1).unwrap();
@@ -126,7 +119,6 @@ fn test_compaction_data() {
 }
 
 #[test]
-#[ignore = "log compaction disabled (#2337)"]
 fn test_end_version_exceeds_snapshot_version() {
     let snapshot = create_mock_snapshot();
     let snapshot_version = snapshot.version();
@@ -144,7 +136,6 @@ fn test_end_version_exceeds_snapshot_version() {
 }
 
 #[test]
-#[ignore = "log compaction disabled (#2337)"]
 fn test_retention_calculator() {
     let snapshot = create_mock_snapshot();
     let writer = LogCompactionWriter::try_new(snapshot.clone(), 0, 1).unwrap();
@@ -154,7 +145,6 @@ fn test_retention_calculator() {
 }
 
 #[test]
-#[ignore = "log compaction disabled (#2337)"]
 fn test_compaction_data_with_actual_iterator() {
     let snapshot = create_multi_version_snapshot();
     let mut writer = LogCompactionWriter::try_new(snapshot, 0, 1).unwrap();
@@ -184,7 +174,6 @@ fn test_compaction_data_with_actual_iterator() {
 }
 
 #[test]
-#[ignore = "log compaction disabled (#2337)"]
 fn test_compaction_paths() {
     let snapshot = create_mock_snapshot();
 
@@ -218,7 +207,6 @@ fn test_compaction_paths() {
 }
 
 #[test]
-#[ignore = "log compaction disabled (#2337)"]
 fn test_version_filtering() {
     let snapshot = create_multi_version_snapshot();
     let engine = SyncEngine::new();
@@ -242,7 +230,6 @@ fn test_version_filtering() {
 }
 
 #[tokio::test]
-#[ignore = "log compaction disabled (#2337)"]
 async fn test_no_compaction_staged_commits() {
     use std::sync::Arc;
 
@@ -321,12 +308,25 @@ async fn test_no_compaction_staged_commits() {
     // where staged commits might slip through the normal filtering
 }
 
-// === Tests for disabled log compaction (TODO(#2337): remove when re-enabled) ===
+// === Tests for should_compact functionality ===
 
 #[test]
-fn test_should_compact_always_false() {
-    // These inputs would return true if compaction were enabled
-    assert!(!should_compact(9, 10));
-    assert!(!should_compact(19, 10));
-    assert!(!should_compact(99, 100));
+fn test_should_compact_percentage_logic() {
+    // Test that compaction occurs at expected intervals
+    // Commits start at 0, so we add 1 to the commit version to check if we've hit the interval
+    // (9+1=10, 19+1=20, 99+1=100, 59+1=60)
+    assert!(should_compact(9, 10));
+    assert!(should_compact(19, 10));
+    assert!(should_compact(99, 100));
+    assert!(should_compact(59, 20));
+
+    // Test that compaction does NOT occur when not at intervals
+    assert!(!should_compact(5, 10));
+    assert!(!should_compact(15, 10));
+    assert!(!should_compact(99, 1000));
+    assert!(!should_compact(109, 20)); // 110 is not a multiple of 20
+
+    // Test edge cases
+    assert!(!should_compact(0, 10));  // commit 0 is metadata, never compact
+    assert!(!should_compact(10, 0));  // zero interval disabled
 }