feat: enable the semantic type contract on new 2.3 tables

python/python/tests/test_dataset.py

@@ -4307,7 +4307,9 @@ def _assert_mixed_version_reads(dataset, expected):
         ("2.0", "2.1", "2.2", "2.3"),
         ("2.1", "2.2", "2.3", "2.0"),
         ("2.2", "2.3", "2.0", "2.1"),
-        ("2.3", "2.0", "2.1", "2.2"),
+        # A table created on 2.3 follows the semantic type contract and only
+        # takes data files of version 2.1 or later.
+        ("2.3", "2.1", "2.2", "2.1"),
     ],
 )
 @pytest.mark.parametrize("mode", ["reencode", "try_binary_copy"])

rust/lance-arrow/src/scalar.rs

@@ -94,6 +94,15 @@ pub fn encode_scalar_value_buffer(scalar: &ArrayRef) -> Result<Vec<u8>> {
     Ok(out)
 }
 
+/// The lengths of the Arrow buffers a scalar value buffer holds, in order.
+pub fn scalar_value_buffer_lengths(value_buffer: &[u8]) -> Result<Vec<usize>> {
+    let mut offset = 0;
+    let num_buffers = read_u32(value_buffer, &mut offset)? as usize;
+    (0..num_buffers)
+        .map(|_| read_u32(value_buffer, &mut offset).map(|len| len as usize))
+        .collect()
+}
+
 pub fn decode_scalar_from_value_buffer(
     data_type: &DataType,
     value_buffer: &[u8],

rust/lance-core/src/datatypes/schema.rs

@@ -14,6 +14,7 @@ use arrow_array::RecordBatch;
 use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema};
 use lance_arrow::*;
 
+use super::OUTPUT_ENCODING_META_KEY;
 use super::field::{Field, OnTypeMismatch, SchemaCompareOptions};
 use crate::{
     Error, ROW_ADDR, ROW_ADDR_FIELD, ROW_CREATED_AT_VERSION, ROW_CREATED_AT_VERSION_FIELD, ROW_ID,
@@ -215,6 +216,102 @@ impl Schema {
         })
     }
 
+    /// Reject [`OUTPUT_ENCODING_META_KEY`] entries that this schema sets to a
+    /// value that is unknown or invalid for the field's semantic type.
+    ///
+    /// An entry that `previous` already holds with the same value, for the
+    /// field with the same ID, is not being set and passes: readers ignore
+    /// such entries, and a newer build may have written them.
+    pub fn check_output_encoding_entries(&self, previous: Option<&Self>) -> Result<()> {
+        for field in self.fields_pre_order() {
+            let Some(value) = field.metadata.get(OUTPUT_ENCODING_META_KEY) else {
+                continue;
+            };
+            if field.recorded_output_encoding().is_some() {
+                continue;
+            }
+            let unchanged = previous
+                .and_then(|previous| previous.field_by_id(field.id))
+                .and_then(|previous| previous.metadata.get(OUTPUT_ENCODING_META_KEY))
+                == Some(value);
+            if !unchanged {
+                return Err(Error::invalid_input(format!(
+                    "cannot set {OUTPUT_ENCODING_META_KEY} of field '{}' to '{value}': it is not an output encoding of type '{}'",
+                    field.name, field.logical_type
+                )));
+            }
+        }
+        Ok(())
+    }
+
+    /// This schema as a data file schema records it. See
+    /// [`Field::to_data_file_field`].
+    pub fn to_data_file_schema(&self) -> Result<Self> {
+        Ok(Self {
+            fields: self
+                .fields
+                .iter()
+                .map(Field::to_data_file_field)
+                .collect::<Result<_>>()?,
+            metadata: self.metadata.clone(),
+        })
+    }
+
+    /// Record the view layouts of `input`, the Arrow schema of data written
+    /// with this schema. See [`Field::record_view_layouts`].
+    pub fn record_view_layouts(&mut self, input: &ArrowSchema) {
+        for input_field in input.fields() {
+            if let Some(field) = self
+                .fields
+                .iter_mut()
+                .find(|f| &f.name == input_field.name())
+            {
+                field.record_view_layouts(input_field);
+            }
+        }
+    }
+
+    /// This write schema with the Arrow layouts of `input`, the schema of the
+    /// data being written. Fields `input` does not hold keep their layout.
+    /// See [`Field::with_input_layout`].
+    pub fn with_input_layouts(&self, input: &Self) -> Result<Self> {
+        let fields = self
+            .fields
+            .iter()
+            .map(|field| match input.field(&field.name) {
+                Some(input_field) => field.with_input_layout(input_field),
+                None => Ok(field.clone()),
+            })
+            .collect::<Result<_>>()?;
+        Ok(Self {
+            fields,
+            metadata: self.metadata.clone(),
+        })
+    }
+
+    /// The fields of this table schema that `input` writes, with `input`'s
+    /// Arrow layouts. See [`Field::with_input_layout`].
+    pub fn project_for_write(&self, input: &Self) -> Result<Self> {
+        let fields = input
+            .fields
+            .iter()
+            .map(|input_field| {
+                self.field(&input_field.name)
+                    .ok_or_else(|| {
+                        Error::schema(format!(
+                            "field '{}' does not exist in the table schema",
+                            input_field.name
+                        ))
+                    })?
+                    .with_input_layout(input_field)
+            })
+            .collect::<Result<_>>()?;
+        Ok(Self {
+            fields,
+            metadata: self.metadata.clone(),
+        })
+    }
+
     pub fn check_compatible(&self, expected: &Self, options: &SchemaCompareOptions) -> Result<()> {
         if !self.compare_with_options(expected, options) {
             let difference = self.explain_difference(expected, options);

rust/lance-core/src/datatypes/semantic.rs

@@ -19,6 +19,7 @@ use std::sync::Arc;
 use arrow_schema::{DataType, Field as ArrowField};
 
 use super::LogicalType;
+use crate::deepsize::DeepSizeOf;
 use crate::{Error, Result};
 
 /// Field metadata entry naming the Arrow layout that reads return for a field
@@ -197,6 +198,15 @@ pub enum OutputEncoding {
     LanceJson,
 }
 
+impl DeepSizeOf for OutputEncoding {
+    fn deep_size_of_children(&self, _context: &mut crate::deepsize::Context) -> usize {
+        match self {
+            Self::Dictionary { value, .. } => std::mem::size_of::<Self>() + value.deep_size_of(),
+            _ => 0,
+        }
+    }
+}
+
 impl OutputEncoding {
     /// The layout of an Arrow type that holds representation-only values, if
     /// it has one.