From 35d9e86d3a56b3b66fb4c521849c62a4521ce81b Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Wed, 7 Oct 2026 14:47:15 -0700 Subject: [PATCH 01/20] test --- python/lib/sift_client/sift_types/data_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/sift_client/sift_types/data_import.py b/python/lib/sift_client/sift_types/data_import.py index a01fcffe7b..0cf50005df 100644 --- a/python/lib/sift_client/sift_types/data_import.py +++ b/python/lib/sift_client/sift_types/data_import.py @@ -1063,7 +1063,7 @@ class McapComplexTypesImportMode(Enum): class McapDataColumn(DataColumnBase): """A single MCAP channel selection. - Channels are selected by topic and flattened field path, as returned by + Channels are selected by topicf and flattened field path, as returned by ``detect_config``. A variable-cardinality field is selected whole by its base path and imports per its ``data_type``: ``BYTES`` (Arrow IPC) or ``STRING`` (JSON), which ``complex_types_import_mode`` must allow. From 80448b0be9612bfc5b355f02f255af8ccce02cb2 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Fri, 9 Oct 2026 14:37:39 -0700 Subject: [PATCH 02/20] python(feat): detect flat dataset Parquet config client-side Detect PARQUET_FLATDATASET import configs locally from the Parquet footer schema with pyarrow instead of calling the server DetectConfig endpoint. Detection matches the server: struct fields are flattened, the first top-level time typed column is the time column, and configs embedded by Sift exports are used directly. The time column is excluded from the data columns, and an INT64/UINT64 column named ts, timestamp, or time is used when no time typed column exists. Add relative_start_time to detect_config and import_from_path. It applies to Parquet only and warns when ignored for other formats or absolute time formats. Add a parquet extra for pyarrow. Co-Authored-By: Claude Opus 5.5 --- .../lib/sift_client/_internal/util/parquet.py | 314 +++++++++ .../_tests/_internal/test_parquet.py | 594 ++++++++++++++++++ .../lib/sift_client/resources/data_imports.py | 67 +- .../resources/sync_stubs/__init__.pyi | 12 + python/pyproject.toml | 1 + 5 files changed, 986 insertions(+), 2 deletions(-) create mode 100644 python/lib/sift_client/_internal/util/parquet.py create mode 100644 python/lib/sift_client/_tests/_internal/test_parquet.py diff --git a/python/lib/sift_client/_internal/util/parquet.py b/python/lib/sift_client/_internal/util/parquet.py new file mode 100644 index 0000000000..1238e95006 --- /dev/null +++ b/python/lib/sift_client/_internal/util/parquet.py @@ -0,0 +1,314 @@ +"""Detect the import configuration of flat dataset Parquet files. +Reads only the Parquet footer schema, flattens struct fields into one channel +per leaf, and picks the first top-level time typed column as the time column. +Files exported with embedded channel configs return that config directly. +""" + +from __future__ import annotations + +import json +import warnings +from datetime import datetime, timezone +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq +from google.protobuf import json_format +from google.protobuf.timestamp_pb2 import Timestamp as TimestampPb +from sift.common.type.v1.channel_config_pb2 import ChannelConfig as ChannelConfigProto + +from sift_client._internal.util.file import extract_parquet_footer +from sift_client.resources.data_imports import _infer_time_column +from sift_client.sift_types.channel import ChannelDataType +from sift_client.sift_types.data_import import ( + ParquetDataColumn, + ParquetFlatDatasetImportConfig, + ParquetTimeColumn, + TimeFormat, + TimeFormatProto, + _enum_types_from_proto, +) + +# Separator used in column paths to address nested struct fields. Channel +# names replace it with "." since Sift forbids "|" in channel names. +PATH_SEPARATOR = "|" + +# Arrow field metadata keys written when a file is exported with +# embed_channel_configs. +EMBEDDED_TIME_CONFIG_KEY = b"sift_time_config" +EMBEDDED_CHANNEL_CONFIG_KEY = b"sift_channel_config" + +_TIMESTAMP_UNIT_TO_FORMAT: dict[str, TimeFormat] = { + "s": TimeFormat.ABSOLUTE_UNIX_SECONDS, + "ms": TimeFormat.ABSOLUTE_UNIX_MILLISECONDS, + "us": TimeFormat.ABSOLUTE_UNIX_MICROSECONDS, + "ns": TimeFormat.ABSOLUTE_UNIX_NANOSECONDS, +} + +_RELATIVE_UNIT_TO_FORMAT: dict[str, TimeFormat] = { + "s": TimeFormat.RELATIVE_SECONDS, + "ms": TimeFormat.RELATIVE_MILLISECONDS, + "us": TimeFormat.RELATIVE_MICROSECONDS, + "ns": TimeFormat.RELATIVE_NANOSECONDS, +} + + +def _is_time_type(t: pa.DataType) -> bool: + # Durations are excluded: the server's Parquet reader sees them as plain + # int64 columns, so they are never time columns. + return ( + pa.types.is_timestamp(t) + or pa.types.is_time32(t) + or pa.types.is_time64(t) + or pa.types.is_date32(t) + or pa.types.is_date64(t) + ) + + +def _is_list_type(t: pa.DataType) -> bool: + # The server reads large lists as lists. Fixed size lists are unsupported. + return pa.types.is_list(t) or pa.types.is_large_list(t) + + +def _arrow_to_sift_type(t: pa.DataType) -> ChannelDataType | None: + """Map an Arrow type to the channel data type it is ingested as. + + Returns None for unsupported types and structs, which are flattened. + Smaller ints widen to 32-bit and float16 widens to float. + """ + if pa.types.is_boolean(t): + return ChannelDataType.BOOL + if pa.types.is_float16(t) or pa.types.is_float32(t): + return ChannelDataType.FLOAT + if pa.types.is_float64(t): + return ChannelDataType.DOUBLE + if pa.types.is_int8(t) or pa.types.is_int16(t) or pa.types.is_int32(t): + return ChannelDataType.INT_32 + if pa.types.is_int64(t) or pa.types.is_duration(t): + return ChannelDataType.INT_64 + if pa.types.is_uint8(t) or pa.types.is_uint16(t) or pa.types.is_uint32(t): + return ChannelDataType.UINT_32 + if pa.types.is_uint64(t): + return ChannelDataType.UINT_64 + if pa.types.is_string(t) or pa.types.is_large_string(t) or pa.types.is_string_view(t): + return ChannelDataType.STRING + if ( + pa.types.is_binary(t) + or pa.types.is_large_binary(t) + or pa.types.is_binary_view(t) + or pa.types.is_fixed_size_binary(t) + ): + return ChannelDataType.BYTES + # All time types are ingested as int64 unix nanoseconds. + if _is_time_type(t): + return ChannelDataType.INT_64 + # Ingested as BYTES by default. The import can also add a STRING channel + # depending on complex_types_import_mode. + if _is_list_type(t) or pa.types.is_map(t): + return ChannelDataType.BYTES + return None + + +def _is_supported_type(t: pa.DataType) -> bool: + """Arrow types supported for Parquet ingestion.""" + return pa.types.is_struct(t) or _arrow_to_sift_type(t) is not None + + +def _add_fields(columns: list[ParquetDataColumn], field: pa.Field, parent_path: str = "") -> None: + """Append a column per leaf of ``field``, flattening structs. + + Nested paths are joined with ``PATH_SEPARATOR``; channel names use ``.``. + Duplicate channel names are not rejected here. + """ + full_path = f"{parent_path}{PATH_SEPARATOR}{field.name}" if parent_path else field.name + + if pa.types.is_struct(field.type): + for i in range(field.type.num_fields): + _add_fields(columns, field.type.field(i), full_path) + return + + name = full_path.replace(PATH_SEPARATOR, ".") + data_type = _arrow_to_sift_type(field.type) + if data_type is None: + raise ValueError(f"unsupported data type for column '{full_path}': {field.type}") + columns.append(ParquetDataColumn(name=name, data_type=data_type, path=full_path)) + + +def _detect_time_column( + field: pa.Field, relative_start_time: datetime | None = None +) -> ParquetTimeColumn | None: + """Return a time column for ``field`` if it has a time type. + + ``relative_start_time`` is only set on relative (time32/time64) columns. + """ + t = field.type + if not _is_time_type(t): + return None + + if pa.types.is_time32(t) or pa.types.is_time64(t): + return ParquetTimeColumn( + path=field.name, + format=_RELATIVE_UNIT_TO_FORMAT[t.unit], + relative_start_time=relative_start_time, + ) + if pa.types.is_timestamp(t): + return ParquetTimeColumn(path=field.name, format=_TIMESTAMP_UNIT_TO_FORMAT[t.unit]) + # Dates are converted to unix nanoseconds. + return ParquetTimeColumn(path=field.name, format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS) + + +def _has_embedded_config(schema: pa.Schema) -> bool: + return any( + field.metadata + and ( + EMBEDDED_TIME_CONFIG_KEY in field.metadata + or EMBEDDED_CHANNEL_CONFIG_KEY in field.metadata + ) + for field in schema + ) + + +def _parse_embedded_time_column(path: str, raw: bytes) -> ParquetTimeColumn: + """Parse the ``sift_time_config`` JSON written by the export service. + + Like the server, the format must be a ``TimeFormat`` enum name. + """ + try: + config = json.loads(raw) + except json.JSONDecodeError as e: + raise ValueError("failed to parse embedded time config") from e + format_name = config.get("time_format", "") if isinstance(config, dict) else None + if not isinstance(format_name, str): + raise ValueError("failed to parse embedded time config") + + try: + format_value = TimeFormatProto.Value(format_name) + except ValueError as e: + raise ValueError(f"invalid embedded time format: {format_name}") from e + fmt = TimeFormat(format_value) if format_value else None + + relative_start_time = None + raw_start = config.get("relative_start_time") + if raw_start: + start = TimestampPb() + start.FromJsonString(raw_start) + relative_start_time = start.ToDatetime(tzinfo=timezone.utc) + + return ParquetTimeColumn(path=path, format=fmt, relative_start_time=relative_start_time) + + +def _parse_embedded_data_column(path: str, raw: bytes) -> ParquetDataColumn: + """Parse the ``sift_channel_config`` JSON (a ChannelConfig) written by the export service. + + Like the server, unknown fields and enum values are rejected. + """ + try: + config = json_format.Parse(raw, ChannelConfigProto()) + data_type = ChannelDataType(config.data_type) if config.data_type else None + except (json_format.ParseError, ValueError) as e: + raise ValueError(f"failed to parse embedded channel config for column {path}") from e + if data_type is None: + raise ValueError(f"missing data type in embedded channel config for column {path}") + return ParquetDataColumn( + path=path, + name=config.name, + data_type=data_type, + units=config.units, + description=config.description, + enum_types=_enum_types_from_proto(config), + ) + + +def _embedded_flat_dataset_config( + schema: pa.Schema, +) -> tuple[ParquetTimeColumn, list[ParquetDataColumn]]: + """Build the time and data columns from sift-exported configs embedded in field metadata.""" + time_column: ParquetTimeColumn | None = None + data_columns: list[ParquetDataColumn] = [] + + for field in schema: + metadata = field.metadata or {} + + raw_time = metadata.get(EMBEDDED_TIME_CONFIG_KEY) + if raw_time is not None: + if time_column is not None: + raise ValueError("found multiple embedded time column configs") + time_column = _parse_embedded_time_column(field.name, raw_time) + continue + + raw_channel = metadata.get(EMBEDDED_CHANNEL_CONFIG_KEY) + if raw_channel is not None: + data_columns.append(_parse_embedded_data_column(field.name, raw_channel)) + + if time_column is None: + raise ValueError("no embedded time config found in parquet file") + if not data_columns: + raise ValueError("no embedded channel configs found in parquet file") + return time_column, data_columns + + +def detect_flat_parquet_config( + file_path: str | Path, relative_start_time: datetime | None = None +) -> ParquetFlatDatasetImportConfig: + """Detect the flat dataset import configuration of a Parquet file. + + Every supported column except the time column becomes a data column. + Struct fields are flattened. The first top-level time typed column is used + as the time column. If none exists, an INT64 or UINT64 column named ``ts``, + ``timestamp``, or ``time`` is used instead. Otherwise ``time_column.path`` + is empty. + + Args: + file_path: The path to the Parquet file. + relative_start_time: Start time for a relative (time32/time64) time + column. Required when the detected time column is relative. + Ignored for absolute time columns and embedded configs. + + Returns: + The detected configuration with footer offset and length populated. + + Raises: + ValueError: If the file is not a valid Parquet file, a struct holds an + unsupported type, or an embedded config is malformed. + """ + path = Path(file_path) + footer_bytes, footer_offset = extract_parquet_footer(path) + schema = pq.read_schema(path) + + if _has_embedded_config(schema): + time_column, data_columns = _embedded_flat_dataset_config(schema) + else: + time_column = ParquetTimeColumn(path="") + data_columns = [] + for field in schema: + if not _is_supported_type(field.type): + warnings.warn( + f"skipping unsupported data type for column '{field.name}': {field.type}", + stacklevel=2, + ) + continue + + try: + _add_fields(data_columns, field) + except ValueError as e: + raise ValueError(f"unable to add field {field.name}: {e}") from e + + # Select the first encountered time column, if it exists. + if not time_column.path: + detected = _detect_time_column(field, relative_start_time) + if detected is not None: + time_column = detected + + if not time_column.path: + inferred = _infer_time_column((dc.name, dc.data_type, dc.path) for dc in data_columns) + if inferred: + time_column = ParquetTimeColumn(path=inferred) + data_columns = [dc for dc in data_columns if dc.path != time_column.path] + + return ParquetFlatDatasetImportConfig( + asset_name="", + time_column=time_column, + data_columns=data_columns, + footer_offset=footer_offset, + footer_length=len(footer_bytes), + ) diff --git a/python/lib/sift_client/_tests/_internal/test_parquet.py b/python/lib/sift_client/_tests/_internal/test_parquet.py new file mode 100644 index 0000000000..5b709185f8 --- /dev/null +++ b/python/lib/sift_client/_tests/_internal/test_parquet.py @@ -0,0 +1,594 @@ +"""Tests for flat dataset Parquet config detection.""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest +from pydantic import ValidationError + +from sift_client._internal.util.parquet import detect_flat_parquet_config +from sift_client.resources import DataImportAPIAsync +from sift_client.resources.data_imports import _apply_relative_start_time +from sift_client.sift_types.channel import ChannelDataType +from sift_client.sift_types.data_import import ( + CsvImportConfig, + CsvTimeColumn, + DataTypeKey, + TimeFormat, +) + + +@pytest.fixture +def create_parquet_file(tmp_path): + """Return a helper that writes a table to a Parquet file and returns its path.""" + file_path = tmp_path / "test.parquet" + + def _create(table: pa.Table): + pq.write_table(table, file_path) + return file_path + + return _create + + +def _columns(config) -> dict[str, tuple[str, ChannelDataType]]: + return {dc.path: (dc.name, dc.data_type) for dc in config.data_columns} + + +def _with_time(arr: pa.Array) -> pa.Table: + return pa.table({"value": pa.array([1.0, 2.0]), "t": arr}) + + +def _embedded_field(name: str, arrow_type: pa.DataType, key: str, config: dict) -> pa.Field: + return pa.field(name, arrow_type, metadata={key: json.dumps(config)}) + + +def _embedded_table(fields: list[pa.Field]) -> pa.Table: + return pa.table( + {f.name: pa.array([1, 2]).cast(f.type) for f in fields}, schema=pa.schema(fields) + ) + + +_TIME_CONFIG = "sift_time_config" +_CHANNEL_CONFIG = "sift_channel_config" + + +def _time_field(name: str, time_format: str = "TIME_FORMAT_ABSOLUTE_UNIX_NANOSECONDS") -> pa.Field: + return _embedded_field(name, pa.int64(), _TIME_CONFIG, {"time_format": time_format}) + + +def _channel_field(name: str, data_type: str | None = "CHANNEL_DATA_TYPE_DOUBLE") -> pa.Field: + config = {"name": name} if data_type is None else {"name": name, "data_type": data_type} + return _embedded_field(name, pa.float64(), _CHANNEL_CONFIG, config) + + +class TestDataTypes: + def test_scalar_types(self, create_parquet_file): + table = pa.table( + { + "bool": pa.array([True]), + "f16": pa.array([1.0], pa.float16()), + "f32": pa.array([1.0], pa.float32()), + "f64": pa.array([1.0], pa.float64()), + "i8": pa.array([1], pa.int8()), + "i16": pa.array([1], pa.int16()), + "i32": pa.array([1], pa.int32()), + "i64": pa.array([1], pa.int64()), + "u8": pa.array([1], pa.uint8()), + "u16": pa.array([1], pa.uint16()), + "u32": pa.array([1], pa.uint32()), + "u64": pa.array([1], pa.uint64()), + "string": pa.array(["a"], pa.string()), + "large_string": pa.array(["a"], pa.large_string()), + "string_view": pa.array(["a"], pa.string_view()), + "binary": pa.array([b"a"], pa.binary()), + "large_binary": pa.array([b"a"], pa.large_binary()), + "binary_view": pa.array([b"a"], pa.binary_view()), + "fixed_binary": pa.array([b"ab"], pa.binary(2)), + } + ) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert {path: data_type for path, (_, data_type) in _columns(config).items()} == { + "bool": ChannelDataType.BOOL, + "f16": ChannelDataType.FLOAT, + "f32": ChannelDataType.FLOAT, + "f64": ChannelDataType.DOUBLE, + "i8": ChannelDataType.INT_32, + "i16": ChannelDataType.INT_32, + "i32": ChannelDataType.INT_32, + "i64": ChannelDataType.INT_64, + "u8": ChannelDataType.UINT_32, + "u16": ChannelDataType.UINT_32, + "u32": ChannelDataType.UINT_32, + "u64": ChannelDataType.UINT_64, + "string": ChannelDataType.STRING, + "large_string": ChannelDataType.STRING, + "string_view": ChannelDataType.STRING, + "binary": ChannelDataType.BYTES, + "large_binary": ChannelDataType.BYTES, + "binary_view": ChannelDataType.BYTES, + "fixed_binary": ChannelDataType.BYTES, + } + + def test_time_types_are_int64(self, create_parquet_file): + # The absolute timestamp comes first so it is the time column; the + # relative time64 column is then only a data column. + table = pa.table( + { + "ts": pa.array([0], pa.timestamp("ns")), + "d32": pa.array([0], pa.date32()), + "t64": pa.array([0], pa.time64("ns")), + "dur": pa.array([0], pa.duration("us")), + } + ) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert all(dc.data_type == ChannelDataType.INT_64 for dc in config.data_columns) + + def test_complex_types_are_bytes(self, create_parquet_file): + table = pa.table( + { + "list": pa.array([[1, 2]]), + "large_list": pa.array([[1, 2]], pa.large_list(pa.int64())), + "map": pa.array([[("k", 1)]], pa.map_(pa.string(), pa.int64())), + "list_of_struct": pa.array([[{"x": 1}]]), + } + ) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert _columns(config) == { + "list": ("list", ChannelDataType.BYTES), + "large_list": ("large_list", ChannelDataType.BYTES), + "map": ("map", ChannelDataType.BYTES), + "list_of_struct": ("list_of_struct", ChannelDataType.BYTES), + } + + @pytest.mark.parametrize( + ("column", "array"), + [ + ("dictionary", pa.array(["a"]).dictionary_encode()), + ("decimal", pa.array([1], pa.decimal128(10, 2))), + ("null", pa.array([None], pa.null())), + ("fixed_list", pa.array([[1, 2]], pa.list_(pa.int64(), 2))), + ], + ) + def test_unsupported_type_is_skipped_with_warning(self, create_parquet_file, column, array): + table = pa.table({"value": pa.array([1.0]), column: array}) + + with pytest.warns(UserWarning, match=f"column '{column}'"): + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert list(_columns(config)) == ["value"] + + def test_unsupported_type_inside_struct_raises(self, create_parquet_file): + struct = pa.StructArray.from_arrays([pa.array([1], pa.decimal128(10, 2))], names=["d"]) + table = pa.table({"s": struct}) + + with pytest.raises(ValueError, match="unable to add field s"): + detect_flat_parquet_config(create_parquet_file(table)) + + +class TestStructs: + def test_nested_fields_are_flattened(self, create_parquet_file): + table = pa.table({"s": pa.array([{"a": 1, "b": {"c": "x"}}])}) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert _columns(config) == { + "s|a": ("s.a", ChannelDataType.INT_64), + "s|b|c": ("s.b.c", ChannelDataType.STRING), + } + + def test_pipe_in_column_name_becomes_dot(self, create_parquet_file): + table = pa.table({"x|y": pa.array([1.0])}) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert _columns(config) == {"x|y": ("x.y", ChannelDataType.DOUBLE)} + + def test_duplicate_channel_names_are_kept(self, create_parquet_file): + """Matches the server: names that collide after flattening are not rejected.""" + table = pa.table({"a.b": pa.array([1]), "a": pa.array([{"b": 1}])}) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert _columns(config) == { + "a.b": ("a.b", ChannelDataType.INT_64), + "a|b": ("a.b", ChannelDataType.INT_64), + } + + +class TestTimeColumn: + @pytest.mark.parametrize( + ("arrow_type", "expected"), + [ + # Parquet has no second-resolution timestamp, so it is stored as ms. + (pa.timestamp("s"), TimeFormat.ABSOLUTE_UNIX_MILLISECONDS), + (pa.timestamp("ms"), TimeFormat.ABSOLUTE_UNIX_MILLISECONDS), + (pa.timestamp("us"), TimeFormat.ABSOLUTE_UNIX_MICROSECONDS), + (pa.timestamp("ns"), TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + (pa.timestamp("us", tz="UTC"), TimeFormat.ABSOLUTE_UNIX_MICROSECONDS), + (pa.date32(), TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + (pa.date64(), TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + ], + ) + def test_absolute_format_from_type(self, create_parquet_file, arrow_type, expected): + config = detect_flat_parquet_config( + create_parquet_file(_with_time(pa.array([0, 1], arrow_type))) + ) + + assert config.time_column.path == "t" + assert config.time_column.format == expected + + @pytest.mark.parametrize( + "arrow_type", [pa.time32("s"), pa.time32("ms"), pa.time64("us"), pa.time64("ns")] + ) + def test_relative_format_without_start_time_raises(self, create_parquet_file, arrow_type): + """A relative time column has no start time in the file, which the config rejects.""" + path = create_parquet_file(_with_time(pa.array([0, 1], arrow_type))) + + with pytest.raises(ValidationError, match="relative_start_time"): + detect_flat_parquet_config(path) + + @pytest.mark.parametrize( + ("arrow_type", "expected"), + [ + (pa.time32("ms"), TimeFormat.RELATIVE_MILLISECONDS), + (pa.time64("us"), TimeFormat.RELATIVE_MICROSECONDS), + (pa.time64("ns"), TimeFormat.RELATIVE_NANOSECONDS), + ], + ) + def test_relative_format_with_start_time(self, create_parquet_file, arrow_type, expected): + start = datetime(2024, 1, 1, tzinfo=timezone.utc) + path = create_parquet_file(_with_time(pa.array([0, 1], arrow_type))) + + config = detect_flat_parquet_config(path, relative_start_time=start) + + assert config.time_column.format == expected + assert config.time_column.relative_start_time == start + + @pytest.mark.parametrize("arrow_type", [pa.timestamp("ns"), pa.date32()]) + def test_absolute_format_ignores_start_time(self, create_parquet_file, arrow_type): + path = create_parquet_file(_with_time(pa.array([0, 1], arrow_type))) + + config = detect_flat_parquet_config( + path, relative_start_time=datetime(2024, 1, 1, tzinfo=timezone.utc) + ) + + assert config.time_column.relative_start_time is None + + @pytest.mark.parametrize("unit", ["s", "ms", "us", "ns"]) + def test_duration_is_int64_data_not_time(self, create_parquet_file, unit): + config = detect_flat_parquet_config( + create_parquet_file(_with_time(pa.array([0, 1], pa.duration(unit)))) + ) + + assert config.time_column.path == "" + assert _columns(config)["t"] == ("t", ChannelDataType.INT_64) + + def test_first_time_column_wins(self, create_parquet_file): + table = pa.table( + { + "t1": pa.array([0], pa.timestamp("ms")), + "t2": pa.array([0], pa.timestamp("ns")), + } + ) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert config.time_column.path == "t1" + + def test_time_column_is_not_a_data_column(self, create_parquet_file): + table = pa.table({"t": pa.array([0], pa.timestamp("ns")), "v": pa.array([1.0])}) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert list(_columns(config)) == ["v"] + + def test_no_time_column_leaves_path_empty(self, create_parquet_file): + config = detect_flat_parquet_config(create_parquet_file(pa.table({"v": [1.0]}))) + + assert config.time_column.path == "" + assert config.time_column.format is None + + @pytest.mark.parametrize("arrow_type", [pa.int64(), pa.uint64()]) + def test_infers_named_integer_time_column(self, create_parquet_file, arrow_type): + table = pa.table({"v": pa.array([1.0]), "TimeStamp": pa.array([0], arrow_type)}) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert config.time_column.path == "TimeStamp" + assert config.time_column.format is None + assert list(_columns(config)) == ["v"] + + def test_time_typed_column_wins_over_named_integer(self, create_parquet_file): + table = pa.table( + {"timestamp": pa.array([0], pa.int64()), "t": pa.array([0], pa.timestamp("ns"))} + ) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert config.time_column.path == "t" + assert list(_columns(config)) == ["timestamp"] + + @pytest.mark.parametrize( + "column", + [ + pytest.param({"count": pa.array([0], pa.int64())}, id="unmatched_name"), + pytest.param({"timestamp": pa.array([0], pa.int32())}, id="int32"), + ], + ) + def test_does_not_infer_other_integer_columns(self, create_parquet_file, column): + config = detect_flat_parquet_config(create_parquet_file(pa.table(column))) + + assert config.time_column.path == "" + assert list(_columns(config)) == list(column) + + def test_nested_time_field_is_not_selected(self, create_parquet_file): + struct = pa.StructArray.from_arrays([pa.array([0], pa.timestamp("ns"))], names=["t"]) + config = detect_flat_parquet_config(create_parquet_file(pa.table({"s": struct}))) + + assert config.time_column.path == "" + assert _columns(config) == {"s|t": ("s.t", ChannelDataType.INT_64)} + + +class TestEmbeddedConfig: + def test_uses_embedded_configs(self, create_parquet_file): + table = _embedded_table( + [ + _embedded_field( + "time", + pa.int64(), + _TIME_CONFIG, + { + "time_format": "TIME_FORMAT_RELATIVE_SECONDS", + "relative_start_time": "2024-01-01T00:00:00Z", + }, + ), + _embedded_field( + "x", + pa.float64(), + _CHANNEL_CONFIG, + { + "name": "x.renamed", + "data_type": "CHANNEL_DATA_TYPE_DOUBLE", + "units": "m", + "description": "position", + }, + ), + _embedded_field( + "state", + pa.uint32(), + _CHANNEL_CONFIG, + { + "name": "state", + "data_type": "CHANNEL_DATA_TYPE_ENUM", + "enum_types": [{"name": "on", "key": 1}], + }, + ), + ] + ) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert config.time_column.path == "time" + assert config.time_column.format == TimeFormat.RELATIVE_SECONDS + assert config.time_column.relative_start_time == datetime(2024, 1, 1, tzinfo=timezone.utc) + assert _columns(config) == { + "x": ("x.renamed", ChannelDataType.DOUBLE), + "state": ("state", ChannelDataType.ENUM), + } + assert config["x.renamed"].units == "m" + assert config["x.renamed"].description == "position" + assert config["state"].enum_types == {"on": 1} + + def test_camel_case_keys_and_numeric_enums(self, create_parquet_file): + """Protobuf JSON allows lowerCamelCase field names and enum numbers.""" + table = _embedded_table( + [ + _embedded_field( + "t", + pa.int64(), + _TIME_CONFIG, + {"time_format": "TIME_FORMAT_ABSOLUTE_UNIX_MILLISECONDS"}, + ), + _embedded_field( + "x", + pa.uint32(), + _CHANNEL_CONFIG, + { + "name": "x", + "dataType": ChannelDataType.ENUM.value, + "enumTypes": [{"name": "on", "key": 1}], + }, + ), + ] + ) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert config.time_column.format == TimeFormat.ABSOLUTE_UNIX_MILLISECONDS + assert config["x"].data_type == ChannelDataType.ENUM + assert config["x"].enum_types == {"on": 1} + + @pytest.mark.parametrize( + ("fields", "match"), + [ + pytest.param( + [_time_field("t"), _channel_field("x", data_type="BOGUS")], + "failed to parse embedded channel config for column x", + id="unknown_channel_data_type", + ), + pytest.param( + [ + _time_field("t"), + _embedded_field( + "x", + pa.float64(), + _CHANNEL_CONFIG, + {"name": "x", "data_type": "CHANNEL_DATA_TYPE_DOUBLE", "bogus": 1}, + ), + ], + "failed to parse embedded channel config for column x", + id="unknown_channel_config_field", + ), + pytest.param( + [_time_field("t"), _channel_field("x", data_type=None)], + "missing data type .* column x", + id="missing_channel_data_type", + ), + pytest.param( + [_channel_field("x")], + "no embedded time config found in parquet file", + id="missing_time_config", + ), + pytest.param( + [_time_field("t1"), _time_field("t2")], + "multiple embedded time column configs", + id="multiple_time_configs", + ), + pytest.param( + [_time_field("t", time_format="BOGUS"), _channel_field("x", data_type=None)], + "invalid embedded time format: BOGUS", + id="invalid_time_format", + ), + pytest.param( + [_embedded_field("t", pa.int64(), _TIME_CONFIG, {}), _channel_field("x")], + "invalid embedded time format: $", + id="missing_time_format", + ), + pytest.param( + [ + _embedded_field("t", pa.int64(), _TIME_CONFIG, {"time_format": 11}), + _channel_field("x"), + ], + "failed to parse embedded time config", + id="numeric_time_format", + ), + ], + ) + def test_invalid_embedded_config_raises(self, create_parquet_file, fields, match): + with pytest.raises(ValueError, match=match): + detect_flat_parquet_config(create_parquet_file(_embedded_table(fields))) + + def test_unconfigured_columns_are_ignored(self, create_parquet_file): + table = _embedded_table( + [_time_field("time"), _channel_field("x"), pa.field("plain", pa.float64())] + ) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert list(_columns(config)) == ["x"] + + +class TestFile: + def test_footer_fields_populated(self, create_parquet_file): + path = create_parquet_file(pa.table({"v": [1.0]})) + config = detect_flat_parquet_config(path) + + assert config.footer_length > 0 + assert config.footer_offset + config.footer_length + 8 == path.stat().st_size + + def test_empty_schema(self, create_parquet_file): + config = detect_flat_parquet_config(create_parquet_file(pa.table({}))) + + assert config.data_columns == [] + assert config.time_column.path == "" + + def test_invalid_file_raises(self, tmp_path): + path = tmp_path / "bad.parquet" + path.write_bytes(b"not a parquet file") + + with pytest.raises(ValueError, match="Invalid Parquet file"): + detect_flat_parquet_config(path) + + def test_truncated_file_raises(self, create_parquet_file): + path = create_parquet_file(pa.table({"v": [1.0]})) + data = path.read_bytes() + # Keep the trailer but drop the start of the footer metadata. + path.write_bytes(data[:4] + data[-8:]) + + with pytest.raises(Exception): # noqa: B017, PT011 + detect_flat_parquet_config(path) + + +class TestDetectConfigForType: + @pytest.mark.asyncio + async def test_detects_locally_and_drops_time_column(self, create_parquet_file): + table = pa.table({"t": pa.array([0], pa.timestamp("ms")), "v": pa.array([1.0])}) + api = DataImportAPIAsync(MagicMock()) + api._low_level_client = MagicMock() + api._low_level_client.detect_config = AsyncMock() + + config = await api._detect_config_for_type( + create_parquet_file(table), DataTypeKey.PARQUET_FLATDATASET + ) + + api._low_level_client.detect_config.assert_not_called() + assert config.time_column.path == "t" + assert config.time_column.format == TimeFormat.ABSOLUTE_UNIX_MILLISECONDS + assert [dc.path for dc in config.data_columns] == ["v"] + + @pytest.mark.asyncio + async def test_infers_int64_time_column_by_name(self, create_parquet_file): + table = pa.table({"timestamp": pa.array([0], pa.int64()), "v": pa.array([1.0])}) + api = DataImportAPIAsync(MagicMock()) + + config = await api._detect_config_for_type( + create_parquet_file(table), DataTypeKey.PARQUET_FLATDATASET + ) + + assert config.time_column.path == "timestamp" + assert [dc.path for dc in config.data_columns] == ["v"] + + +class TestDetectConfigRelativeStartTime: + START = datetime(2024, 1, 1, tzinfo=timezone.utc) + + @pytest.mark.asyncio + async def test_relative_column_uses_start_time(self, create_parquet_file): + path = create_parquet_file(_with_time(pa.array([0, 1], pa.time64("us")))) + api = DataImportAPIAsync(MagicMock()) + + config = await api.detect_config( + path, data_type=DataTypeKey.PARQUET_FLATDATASET, relative_start_time=self.START + ) + + assert config.time_column.format == TimeFormat.RELATIVE_MICROSECONDS + assert config.time_column.relative_start_time == self.START + + @pytest.mark.asyncio + async def test_relative_time_format_override_uses_start_time(self, create_parquet_file): + table = pa.table({"timestamp": pa.array([0, 1], pa.int64()), "v": pa.array([1.0, 2.0])}) + api = DataImportAPIAsync(MagicMock()) + + config = await api.detect_config( + create_parquet_file(table), + data_type=DataTypeKey.PARQUET_FLATDATASET, + time_format=TimeFormat.RELATIVE_SECONDS, + relative_start_time=self.START, + ) + + assert config.time_column.path == "timestamp" + assert config.time_column.format == TimeFormat.RELATIVE_SECONDS + assert config.time_column.relative_start_time == self.START + + @pytest.mark.asyncio + async def test_absolute_column_warns_and_ignores_start_time(self, create_parquet_file): + path = create_parquet_file(_with_time(pa.array([0, 1], pa.timestamp("ns")))) + api = DataImportAPIAsync(MagicMock()) + + with pytest.warns(UserWarning, match="relative_start_time"): + config = await api.detect_config( + path, data_type=DataTypeKey.PARQUET_FLATDATASET, relative_start_time=self.START + ) + + assert config.time_column.relative_start_time is None + + def test_non_parquet_config_warns_and_ignores_start_time(self): + config = CsvImportConfig( + asset_name="a", + time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339), + data_columns=[], + ) + config.time_column.format = TimeFormat.RELATIVE_SECONDS + + with pytest.warns(UserWarning, match="not supported for CsvImportConfig"): + _apply_relative_start_time(config, self.START) + + assert config.time_column.relative_start_time is None diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index 2f16125791..1f1e242ef9 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -1,11 +1,14 @@ from __future__ import annotations +import warnings from pathlib import Path from typing import TYPE_CHECKING from sift_client._internal.low_level_wrappers.data_imports import DataImportsLowLevelClient from sift_client._internal.util.executor import run_sync_function from sift_client._internal.util.file import extract_parquet_footer, upload_file +from sift_client._internal.util.util import caller_stacklevel +from sift_client.errors import SiftIgnoredInputWarning from sift_client.resources._base import ResourceBase from sift_client.sift_types.asset import Asset from sift_client.sift_types.channel import ChannelDataType @@ -23,6 +26,7 @@ ParquetTimeColumn, TdmsImportConfig, TimeFormat, + TimeFormatProto, UlogImportConfig, ) from sift_client.sift_types.run import Run @@ -30,6 +34,7 @@ if TYPE_CHECKING: from collections.abc import Iterable + from datetime import datetime from sift_client.client import SiftClient from sift_client.sift_types.data_import import DataImportStatus @@ -58,6 +63,7 @@ async def import_from_path( config: ImportConfig | None = None, data_type: DataTypeKey | None = None, time_format: TimeFormat | None = None, + relative_start_time: datetime | None = None, run: Run | str | None = None, run_name: str | None = None, show_progress: bool | None = None, @@ -124,6 +130,11 @@ async def import_from_path( ``TimeFormat.ABSOLUTE_UNIX_NANOSECONDS``. TDMS keeps its detected/default time handling. Only used when ``config`` is not provided. + relative_start_time: Start time for a relative time format. + Parquet only; ignored with a warning for other formats or + when the time format is absolute. Required for Parquet files + with a relative time column. Only used when ``config`` is not + provided. run: ``Run`` object or run ID string to import into an existing run. Mutually exclusive with ``run_name``. run_name: Name for a new run. Defaults to the filename if @@ -148,6 +159,7 @@ async def import_from_path( file_path, data_type=data_type, time_format=time_format, + relative_start_time=relative_start_time, ) if isinstance(config, (UlogImportConfig, McapImportConfig)): # An empty channel list imports every channel @@ -321,6 +333,8 @@ async def detect_config( file_path: str | Path, data_type: DataTypeKey | None = None, time_format: TimeFormat | None = None, + *, + relative_start_time: datetime | None = None, ) -> ImportConfig: """Auto-detect import configuration from a file. @@ -379,6 +393,10 @@ async def detect_config( HDF5 use the detected format if available, otherwise ``TimeFormat.ABSOLUTE_UNIX_NANOSECONDS``. TDMS keeps its detected/default time handling. + relative_start_time: Start time for a relative time format. + Parquet only; ignored with a warning for other formats or + when the time format is absolute. Required for Parquet files + with a relative time column. Returns: The detected import config. @@ -394,7 +412,7 @@ async def detect_config( raise FileNotFoundError(f"File not found: {file_path}") data_type_key = _resolve_data_type_key(path.suffix.lower(), data_type) - config = await self._detect_config_for_type(path, data_type_key) + config = await self._detect_config_for_type(path, data_type_key, relative_start_time) if time_format is not None: _apply_time_format(config, time_format) elif ( @@ -402,12 +420,15 @@ async def detect_config( and _get_time_format(config) is None ): _apply_time_format(config, TimeFormat.ABSOLUTE_UNIX_NANOSECONDS) + if relative_start_time is not None: + _apply_relative_start_time(config, relative_start_time) return config async def _detect_config_for_type( self, path: Path, data_type_key: DataTypeKey, + relative_start_time: datetime | None = None, ) -> ImportConfig: if data_type_key in ( DataTypeKey.HDF5_ONE_D, @@ -449,7 +470,17 @@ async def _detect_config_for_type( "Install them via `pip install sift-stack-py[mcap]`." ) from e return await run_sync_function(lambda: detect_mcap_config(path)) - + if data_type_key == DataTypeKey.PARQUET_FLATDATASET: + try: + from sift_client._internal.util.parquet import detect_flat_parquet_config + except ImportError as e: + raise RuntimeError( + "pyarrow is required for Parquet import. " + "Install it via `pip install sift-stack-py[parquet]`." + ) from e + return await run_sync_function( + lambda: detect_flat_parquet_config(path, relative_start_time) + ) is_parquet = data_type_key in ( DataTypeKey.PARQUET_FLATDATASET, DataTypeKey.PARQUET_SINGLE_CHANNEL_PER_ROW, @@ -506,6 +537,34 @@ def _apply_time_format(config: ImportConfig, time_format: TimeFormat) -> None: config.time_format = time_format +def _apply_relative_start_time(config: ImportConfig, relative_start_time: datetime) -> None: + """Set the relative start time on a detected Parquet config with a relative time format. + + Warns and leaves the config unchanged for other formats, or when the time + format is absolute or unset. + """ + if not isinstance( + config, (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig) + ): + warnings.warn( + "'relative_start_time' is ignored because it is not supported for " + f"{type(config).__name__}.", + SiftIgnoredInputWarning, + stacklevel=caller_stacklevel(), + ) + return + time_format = config.time_column.format + if time_format is None or not time_format.name.startswith("RELATIVE_"): + warnings.warn( + "'relative_start_time' is ignored because the time format " + f"({time_format.name if time_format else 'unset'}) is not relative.", + SiftIgnoredInputWarning, + stacklevel=caller_stacklevel(), + ) + return + config.time_column.relative_start_time = relative_start_time + + def _get_time_format(config: ImportConfig) -> TimeFormat | None: """Read the current time format off a config, regardless of where it's stored.""" if isinstance( @@ -544,6 +603,10 @@ def _resolve_data_type_key(ext: str, data_type: DataTypeKey | None) -> DataTypeK def _parse_csv_detect_response(proto) -> CsvImportConfig: """Parse a CSV DetectConfig response into a config.""" + if proto.time_column.format == TimeFormatProto.TIME_FORMAT_UNSPECIFIED: + raise ValueError( + "timestamp format is not supported. Please specify the time format explicitly." + ) csv_config = CsvImportConfig._from_proto(proto) time_col = csv_config.time_column.column csv_config.data_columns = [dc for dc in csv_config.data_columns if dc.column != time_col] diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index 792947a475..e01442ad16 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -1253,6 +1253,8 @@ class DataImportAPI: file_path: str | Path, data_type: DataTypeKey | None = None, time_format: TimeFormat | None = None, + *, + relative_start_time: datetime | None = None, ) -> ImportConfig: """Auto-detect import configuration from a file. @@ -1311,6 +1313,10 @@ class DataImportAPI: HDF5 use the detected format if available, otherwise ``TimeFormat.ABSOLUTE_UNIX_NANOSECONDS``. TDMS keeps its detected/default time handling. + relative_start_time: Start time for a relative time format. + Parquet only; ignored with a warning for other formats or + when the time format is absolute. Required for Parquet files + with a relative time column. Returns: The detected import config. @@ -1379,6 +1385,7 @@ class DataImportAPI: config: ImportConfig | None = None, data_type: DataTypeKey | None = None, time_format: TimeFormat | None = None, + relative_start_time: datetime | None = None, run: Run | str | None = None, run_name: str | None = None, show_progress: bool | None = None, @@ -1445,6 +1452,11 @@ class DataImportAPI: ``TimeFormat.ABSOLUTE_UNIX_NANOSECONDS``. TDMS keeps its detected/default time handling. Only used when ``config`` is not provided. + relative_start_time: Start time for a relative time format. + Parquet only; ignored with a warning for other formats or + when the time format is absolute. Required for Parquet files + with a relative time column. Only used when ``config`` is not + provided. run: ``Run`` object or run ID string to import into an existing run. Mutually exclusive with ``run_name``. run_name: Name for a new run. Defaults to the filename if diff --git a/python/pyproject.toml b/python/pyproject.toml index fce2e43eb4..09a4d94262 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -265,6 +265,7 @@ hdf5 = ["h5py~=3.11", "polars~=1.8"] # polars is only used by sift_py; remove o ulog = ["pyulog~=1.2.2"] mcap = ["mcap~=1.4", "mcap-ros2-support~=0.5.7"] data-review = ["pyarrow>=17.0.0"] +parquet = ["pyarrow>=17.0.0"] [tool.sift.extras.combine] # Combined extras configurations and aliases can be defined here From 63ec9f202155d8659ccbcd0b66c865f82bc8eb8d Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Fri, 9 Oct 2026 14:46:16 -0700 Subject: [PATCH 03/20] python(fix): lint pyproject.toml and update uv.lock --- python/pyproject.toml | 3 + python/uv.lock | 558 +++++++++++++++++++++--------------------- 2 files changed, 285 insertions(+), 276 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 09a4d94262..901d337cfb 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -205,6 +205,9 @@ openssl = [ 'pyOpenSSL<24.0.0', 'types-pyOpenSSL<24.0.0', ] +parquet = [ + 'pyarrow>=17.0.0', +] rosbags = [ "rosbags~=0.0 ; python_full_version >= '3.8.2'", ] diff --git a/python/uv.lock b/python/uv.lock index e3a4caee80..1543402ff2 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -1,10 +1,7 @@ version = 1 -revision = 3 +revision = 5 requires-python = ">=3.8" resolution-markers = [ - "python_full_version >= '3.8.2' and python_full_version < '3.9'", - "python_full_version < '3.8.2'", - "python_full_version == '3.9.*'", "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -12,6 +9,9 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", + "python_full_version >= '3.8.2' and python_full_version < '3.9'", + "python_full_version < '3.8.2'", ] supported-markers = [ "python_full_version < '3.9'", @@ -37,8 +37,8 @@ resolution-markers = [ "python_full_version < '3.8.2'", ] dependencies = [ - { name = "about-time", marker = "python_full_version < '3.9'" }, - { name = "grapheme", marker = "python_full_version < '3.9'" }, + { name = "about-time" }, + { name = "grapheme" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6a/cf/de25c4f6123c3b3eb5acc87144d3e017df25b32c16806b14572a259939ac/alive-progress-3.1.5.tar.gz", hash = "sha256:42e399a66c8150dc507602dff7b7953f105ef11faf97ddaa6d27b1cbf45c4c98", size = 111891, upload-time = "2023-11-08T23:23:54.559Z" } wheels = [ @@ -50,7 +50,6 @@ name = "alive-progress" version = "3.3.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -58,10 +57,11 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", ] dependencies = [ - { name = "about-time", marker = "python_full_version >= '3.9'" }, - { name = "graphemeu", marker = "python_full_version >= '3.9'" }, + { name = "about-time" }, + { name = "graphemeu" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9a/26/d43128764a6f8fe1668c4f87aba6b1fe52bea81d05a35c84a70d3c70b6f7/alive-progress-3.3.0.tar.gz", hash = "sha256:457dd2428b48dacd49854022a46448d236a48f1b7277874071c39395307e830c", size = 116281, upload-time = "2025-07-20T02:10:39.07Z" } wheels = [ @@ -189,8 +189,8 @@ name = "astunparse" version = "1.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "six", marker = "python_full_version < '3.9'" }, - { name = "wheel", marker = "python_full_version < '3.9'" }, + { name = "six" }, + { name = "wheel" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f3/af/4182184d3c338792894f34a62672919db7ca008c89abee9b564dd34d8029/astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872", size = 18290, upload-time = "2019-12-22T18:12:13.129Z" } wheels = [ @@ -233,8 +233,8 @@ name = "beautifulsoup4" version = "4.14.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "soupsieve", marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "soupsieve" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/c3/b0/1c6a16426d389813b48d95e26898aff79abbde42ad353958ad95cc8c9b21/beautifulsoup4-4.14.3.tar.gz", hash = "sha256:6292b1c5186d356bba669ef9f7f051757099565ad9ada5dd630bd9de5fa7fb86", size = 627737, upload-time = "2025-11-30T15:08:26.084Z" } wheels = [ @@ -246,7 +246,7 @@ name = "bleach" version = "6.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "webencodings", marker = "python_full_version >= '3.10'" }, + { name = "webencodings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/07/18/3c8523962314be6bf4c8989c79ad9531c825210dd13a8669f6b84336e8bd/bleach-6.3.0.tar.gz", hash = "sha256:6f3b91b1c0a02bb9a78b5a454c92506aa0fdf197e1d5e114d2e00c6f64306d22", size = 203533, upload-time = "2025-10-27T17:57:39.211Z" } wheels = [ @@ -255,7 +255,7 @@ wheels = [ [package.optional-dependencies] css = [ - { name = "tinycss2", marker = "python_full_version >= '3.10'" }, + { name = "tinycss2" }, ] [[package]] @@ -522,7 +522,7 @@ name = "click" version = "8.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/23/e4/796662cd90cf80e3a363c99db2b88e0e394b988a575f60a17e16440cd011/click-8.4.0.tar.gz", hash = "sha256:638f1338fe1235c8f4e008e4a8a254fb5c5fbdcbb40ece3c9142ebb78e792973", size = 350843, upload-time = "2026-05-17T00:47:58.425Z" } wheels = [ @@ -679,7 +679,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9' or python_full_version >= '3.11'" }, { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9' and python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } @@ -753,7 +753,7 @@ name = "ghp-import" version = "2.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "python-dateutil", marker = "python_full_version >= '3.10'" }, + { name = "python-dateutil" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d9/29/d40217cbe2f6b1359e00c6c307bb3fc876ba74068cbab3dde77f03ca0dc4/ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343", size = 10943, upload-time = "2022-05-02T15:47:16.11Z" } wheels = [ @@ -769,7 +769,7 @@ resolution-markers = [ "python_full_version < '3.8.2'", ] dependencies = [ - { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/99/96/a0205167fa0154f4a542fd6925bdc63d039d88dab3588b875078107e6f06/googleapis_common_protos-1.73.0.tar.gz", hash = "sha256:778d07cd4fbeff84c6f7c72102f0daf98fa2bfd3fa8bea426edc545588da0b5a", size = 147323, upload-time = "2026-03-06T21:53:09.727Z" } wheels = [ @@ -781,7 +781,6 @@ name = "googleapis-common-protos" version = "1.75.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -789,9 +788,10 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", ] dependencies = [ - { name = "protobuf", version = "6.33.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "protobuf", version = "6.33.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/b5/c8/f439cffde755cffa462bfbb156278fa6f9d09119719af9814b858fd4f81f/googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd", size = 151035, upload-time = "2026-05-07T08:04:49.423Z" } wheels = [ @@ -818,7 +818,7 @@ name = "griffe-pydantic" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "griffelib", marker = "python_full_version >= '3.10'" }, + { name = "griffelib" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5a/bd/d2eaeaf3f9910c9cd72793af0de18ee3d3a3a27bb30ab01cfd7659c08dc4/griffe_pydantic-1.3.1.tar.gz", hash = "sha256:f7caedfa0effedb22893bf01cc411fd567614f7b4de7ce0c1f4293eb7acb5c44", size = 38176, upload-time = "2026-02-21T09:38:49.674Z" } wheels = [ @@ -905,7 +905,6 @@ name = "grpcio" version = "1.80.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -913,9 +912,10 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", ] dependencies = [ - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/b7/48/af6173dbca4454f4637a4678b67f52ca7e0c1ed7d5894d89d434fecede05/grpcio-1.80.0.tar.gz", hash = "sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257", size = 12978905, upload-time = "2026-03-30T08:49:10.502Z" } wheels = [ @@ -990,8 +990,8 @@ resolution-markers = [ "python_full_version < '3.8.2'", ] dependencies = [ - { name = "grpcio", version = "1.70.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "grpcio", version = "1.70.0", source = { registry = "https://pypi.org/simple" } }, + { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/7d/43/b15d838a3927e5f4f33f61376ba52b1d870ffe7f397bc65d0ac2102d2df1/grpcio_testing-1.70.0.tar.gz", hash = "sha256:5eb54db74ce05bc97ee15bb3ae00eddfa002b83f5806813572fd9c798003e075", size = 22503, upload-time = "2025-01-23T18:00:35.6Z" } wheels = [ @@ -1003,7 +1003,6 @@ name = "grpcio-testing" version = "1.80.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -1011,10 +1010,11 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", ] dependencies = [ - { name = "grpcio", version = "1.80.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, - { name = "protobuf", version = "6.33.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "grpcio", version = "1.80.0", source = { registry = "https://pypi.org/simple" } }, + { name = "protobuf", version = "6.33.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/ff/59/f94e2c9ef532a42a805ccb7f632a825b401be8b4b59d356fb404a9cc261a/grpcio_testing-1.80.0.tar.gz", hash = "sha256:fb5ad57c40ffdfa9773043dfa38e8c37739af8e3471f1670a21e78051cd19afd", size = 23139, upload-time = "2026-03-30T08:54:40.239Z" } wheels = [ @@ -1030,7 +1030,7 @@ resolution-markers = [ "python_full_version < '3.8.2'", ] dependencies = [ - { name = "numpy", version = "1.24.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "numpy", version = "1.24.4", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/52/8f/e557819155a282da36fb21f8de4730cfd10a964b52b3ae8d20157ac1c668/h5py-3.11.0.tar.gz", hash = "sha256:7b7e8f78072a2edec87c9836f25f34203fd492a4475709a18b417a33cfb21fa9", size = 406519, upload-time = "2024-04-10T10:52:39.585Z" } wheels = [ @@ -1064,7 +1064,7 @@ resolution-markers = [ "python_full_version == '3.9.*'", ] dependencies = [ - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/5d/57/dfb3c5c3f1bf5f5ef2e59a22dec4ff1f3d7408b55bfcefcfb0ea69ef21c6/h5py-3.14.0.tar.gz", hash = "sha256:2372116b2e0d5d3e5e705b7f663f7c8d96fa79a4052d250484ef91d24d6a08f4", size = 424323, upload-time = "2025-06-06T14:06:15.01Z" } wheels = [ @@ -1109,7 +1109,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/db/33/acd0ce6863b6c0d7735007df01815403f5589a21ff8c2e1ee2587a38f548/h5py-3.16.0.tar.gz", hash = "sha256:a0dbaad796840ccaa67a4c144a0d0c8080073c34c76d5a6941d6818678ef2738", size = 446526, upload-time = "2026-03-06T13:49:08.07Z" } @@ -1190,7 +1190,7 @@ resolution-markers = [ "python_full_version < '3.8.2'", ] dependencies = [ - { name = "zipp", version = "3.20.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "zipp", version = "3.20.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/12/33e59336dca5be0c398a7482335911a33aa0e20776128f038019f1a95f1b/importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7", size = 55304, upload-time = "2024-09-11T14:56:08.937Z" } wheels = [ @@ -1205,7 +1205,7 @@ resolution-markers = [ "python_full_version == '3.9.*'", ] dependencies = [ - { name = "zipp", version = "3.23.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "zipp", version = "3.23.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/f3/49/3b30cad09e7771a4982d9975a8cbf64f00d4a1ececb53297f1d9a7be1b10/importlib_metadata-8.7.1.tar.gz", hash = "sha256:49fef1ae6440c182052f407c8d34a68f72efc36db9ca90dc0113398f2fdde8bb", size = 57107, upload-time = "2025-12-21T10:00:19.278Z" } wheels = [ @@ -1226,7 +1226,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "zipp", version = "4.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "zipp", version = "4.1.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/01/15bb152d77b21318514a96f43af312635eb2500c96b55398d020c93d86ea/importlib_metadata-9.0.0.tar.gz", hash = "sha256:a4f57ab599e6a2e3016d7595cfd72eb4661a5106e787a95bcc90c7105b831efc", size = 56405, upload-time = "2026-03-20T06:42:56.999Z" } wheels = [ @@ -1247,9 +1247,9 @@ name = "iniconfig" version = "2.1.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ + "python_full_version == '3.9.*'", "python_full_version >= '3.8.2' and python_full_version < '3.9'", "python_full_version < '3.8.2'", - "python_full_version == '3.9.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } wheels = [ @@ -1279,20 +1279,20 @@ name = "ipykernel" version = "6.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "appnope", marker = "python_full_version >= '3.10' and sys_platform == 'darwin'" }, - { name = "comm", marker = "python_full_version >= '3.10'" }, - { name = "debugpy", marker = "python_full_version >= '3.10'" }, - { name = "ipython", version = "8.39.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "appnope", marker = "sys_platform == 'darwin'" }, + { name = "comm" }, + { name = "debugpy" }, + { name = "ipython", version = "8.39.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "ipython", version = "9.13.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "jupyter-client", marker = "python_full_version >= '3.10'" }, - { name = "jupyter-core", marker = "python_full_version >= '3.10'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.10'" }, - { name = "nest-asyncio", marker = "python_full_version >= '3.10'" }, - { name = "packaging", marker = "python_full_version >= '3.10'" }, - { name = "psutil", marker = "python_full_version >= '3.10'" }, - { name = "pyzmq", marker = "python_full_version >= '3.10'" }, - { name = "tornado", marker = "python_full_version >= '3.10'" }, - { name = "traitlets", marker = "python_full_version >= '3.10'" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "matplotlib-inline" }, + { name = "nest-asyncio" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a5/1d/d5ba6edbfe6fae4c3105bca3a9c889563cc752c7f2de45e333164c7f4846/ipykernel-6.31.0.tar.gz", hash = "sha256:2372ce8bc1ff4f34e58cafed3a0feb2194b91fc7cad0fc72e79e47b45ee9e8f6", size = 167493, upload-time = "2025-10-20T11:42:39.948Z" } wheels = [ @@ -1307,17 +1307,17 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "colorama", marker = "python_full_version == '3.10.*' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version == '3.10.*'" }, - { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, - { name = "jedi", marker = "python_full_version == '3.10.*'" }, - { name = "matplotlib-inline", marker = "python_full_version == '3.10.*'" }, - { name = "pexpect", marker = "python_full_version == '3.10.*' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version == '3.10.*'" }, - { name = "pygments", version = "2.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "stack-data", marker = "python_full_version == '3.10.*'" }, - { name = "traitlets", marker = "python_full_version == '3.10.*'" }, - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "exceptiongroup" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "pygments", version = "2.20.0", source = { registry = "https://pypi.org/simple" } }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/40/18/f8598d287006885e7136451fdea0755af4ebcbfe342836f24deefaed1164/ipython-8.39.0.tar.gz", hash = "sha256:4110ae96012c379b8b6db898a07e186c40a2a1ef5d57a7fa83166047d9da7624", size = 5513971, upload-time = "2026-03-27T10:02:13.94Z" } wheels = [ @@ -1337,18 +1337,18 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.11' and sys_platform == 'win32'" }, - { name = "decorator", marker = "python_full_version >= '3.11'" }, - { name = "ipython-pygments-lexers", marker = "python_full_version >= '3.11'" }, - { name = "jedi", marker = "python_full_version >= '3.11'" }, - { name = "matplotlib-inline", marker = "python_full_version >= '3.11'" }, - { name = "pexpect", marker = "python_full_version >= '3.11' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "prompt-toolkit", marker = "python_full_version >= '3.11'" }, - { name = "psutil", marker = "python_full_version >= '3.11'" }, - { name = "pygments", version = "2.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "stack-data", marker = "python_full_version >= '3.11'" }, - { name = "traitlets", marker = "python_full_version >= '3.11'" }, - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "decorator" }, + { name = "ipython-pygments-lexers" }, + { name = "jedi" }, + { name = "matplotlib-inline" }, + { name = "pexpect", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "prompt-toolkit" }, + { name = "psutil" }, + { name = "pygments", version = "2.20.0", source = { registry = "https://pypi.org/simple" } }, + { name = "stack-data" }, + { name = "traitlets" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/87cda5842cf5c31837c06ddb588e11c3c35d8ece89b7a0108c06b8c9b00a/ipython-9.13.0.tar.gz", hash = "sha256:7e834b6afc99f020e3f05966ced34792f40267d64cb1ea9043886dab0dde5967", size = 4430549, upload-time = "2026-04-24T12:24:55.221Z" } wheels = [ @@ -1360,7 +1360,7 @@ name = "ipython-pygments-lexers" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pygments", version = "2.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "pygments", version = "2.20.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/4c/5dd1d8af08107f88c7f741ead7a40854b8ac24ddf9ae850afbcf698aa552/ipython_pygments_lexers-1.1.1.tar.gz", hash = "sha256:09c0138009e56b6854f9535736f4171d855c8c08a563a0dcd8022f78355c7e81", size = 8393, upload-time = "2025-01-17T11:24:34.505Z" } wheels = [ @@ -1372,7 +1372,7 @@ name = "jedi" version = "0.20.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "parso", marker = "python_full_version >= '3.10'" }, + { name = "parso" }, ] sdist = { url = "https://files.pythonhosted.org/packages/46/b7/a3635f6a2d7cf5b5dd98064fc1d5fbbafcb25477bcea204a3a92145d158b/jedi-0.20.0.tar.gz", hash = "sha256:c3f4ccbd276696f4b19c54618d4fb18f9fc24b0aef02acf704b23f487daa1011", size = 3119416, upload-time = "2026-05-01T23:38:47.814Z" } wheels = [ @@ -1397,10 +1397,10 @@ name = "jsonschema" version = "4.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "attrs", marker = "python_full_version >= '3.10'" }, - { name = "jsonschema-specifications", marker = "python_full_version >= '3.10'" }, - { name = "referencing", marker = "python_full_version >= '3.10'" }, - { name = "rpds-py", marker = "python_full_version >= '3.10'" }, + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } wheels = [ @@ -1412,7 +1412,7 @@ name = "jsonschema-specifications" version = "2025.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "referencing", marker = "python_full_version >= '3.10'" }, + { name = "referencing" }, ] sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } wheels = [ @@ -1424,11 +1424,11 @@ name = "jupyter-client" version = "8.8.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jupyter-core", marker = "python_full_version >= '3.10'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.10'" }, - { name = "pyzmq", marker = "python_full_version >= '3.10'" }, - { name = "tornado", marker = "python_full_version >= '3.10'" }, - { name = "traitlets", marker = "python_full_version >= '3.10'" }, + { name = "jupyter-core" }, + { name = "python-dateutil" }, + { name = "pyzmq" }, + { name = "tornado" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/05/e4/ba649102a3bc3fbca54e7239fb924fd434c766f855693d86de0b1f2bec81/jupyter_client-8.8.0.tar.gz", hash = "sha256:d556811419a4f2d96c869af34e854e3f059b7cc2d6d01a9cd9c85c267691be3e", size = 348020, upload-time = "2026-01-08T13:55:47.938Z" } wheels = [ @@ -1440,8 +1440,8 @@ name = "jupyter-core" version = "5.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "platformdirs", version = "4.9.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "traitlets", marker = "python_full_version >= '3.10'" }, + { name = "platformdirs", version = "4.9.6", source = { registry = "https://pypi.org/simple" } }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/02/49/9d1284d0dc65e2c757b74c6687b6d319b02f822ad039e5c512df9194d9dd/jupyter_core-5.9.1.tar.gz", hash = "sha256:4d09aaff303b9566c3ce657f580bd089ff5c91f5f89cf7d8846c3cdf465b5508", size = 89814, upload-time = "2025-10-16T19:19:18.444Z" } wheels = [ @@ -1462,12 +1462,12 @@ name = "jupytext" version = "1.19.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markdown-it-py", marker = "python_full_version >= '3.10'" }, - { name = "mdit-py-plugins", marker = "python_full_version >= '3.10'" }, - { name = "nbformat", marker = "python_full_version >= '3.10'" }, - { name = "packaging", marker = "python_full_version >= '3.10'" }, - { name = "pyyaml", marker = "python_full_version >= '3.10'" }, - { name = "tomli", marker = "python_full_version == '3.10.*'" }, + { name = "markdown-it-py" }, + { name = "mdit-py-plugins" }, + { name = "nbformat" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ef/2d/15624c3d9440d85a280ff13d2d23afd989802f25470ac59932f4fef6f0c6/jupytext-1.19.3.tar.gz", hash = "sha256:713c3ed4441afe0f31474d28ea2e6b61a268c04c40fd78e5ccfd7f7ac9e9f766", size = 4305350, upload-time = "2026-05-17T09:09:29.294Z" } wheels = [ @@ -1526,7 +1526,6 @@ name = "lz4" version = "4.4.5" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -1534,6 +1533,7 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/57/51/f1b86d93029f418033dddf9b9f79c8d2641e7454080478ee2aab5123173e/lz4-4.4.5.tar.gz", hash = "sha256:5f0b9e53c1e82e88c10d7c180069363980136b9d7a8306c4dca4f760d60c39f0", size = 172886, upload-time = "2025-11-03T13:02:36.061Z" } wheels = [ @@ -1609,7 +1609,7 @@ name = "markdown-it-py" version = "4.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mdurl", marker = "python_full_version >= '3.10'" }, + { name = "mdurl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } wheels = [ @@ -1683,7 +1683,6 @@ name = "markupsafe" version = "3.0.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -1691,6 +1690,7 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } wheels = [ @@ -1789,7 +1789,7 @@ name = "matplotlib-inline" version = "0.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "traitlets", marker = "python_full_version >= '3.10'" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bd/c0/9f7c9a46090390368a4d7bcb76bb87a4a36c421e4c0792cdb53486ffac7a/matplotlib_inline-0.2.2.tar.gz", hash = "sha256:72f3fe8fce36b70d4a5b612f899090cd0401deddc4ea90e1572b9f4bfb058c79", size = 8150, upload-time = "2026-05-08T17:33:33.49Z" } wheels = [ @@ -1828,7 +1828,7 @@ name = "mdit-py-plugins" version = "0.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markdown-it-py", marker = "python_full_version >= '3.10'" }, + { name = "markdown-it-py" }, ] sdist = { url = "https://files.pythonhosted.org/packages/59/fc/f8d0863f8862f25602c0404d75568e89fb6b4109804645e5cdfb1be5cf56/mdit_py_plugins-0.6.1.tar.gz", hash = "sha256:a2bca0f039f39dbd35fb74ae1b5f998608c437463371f0ff7f49a19a17a114d0", size = 56114, upload-time = "2026-05-13T09:03:38.91Z" } wheels = [ @@ -1858,14 +1858,14 @@ name = "mike" version = "2.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "importlib-metadata", version = "9.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "importlib-resources", marker = "python_full_version >= '3.10'" }, - { name = "jinja2", marker = "python_full_version >= '3.10'" }, - { name = "mkdocs", marker = "python_full_version >= '3.10'" }, - { name = "pyparsing", marker = "python_full_version >= '3.10'" }, - { name = "pyyaml", marker = "python_full_version >= '3.10'" }, - { name = "pyyaml-env-tag", marker = "python_full_version >= '3.10'" }, - { name = "verspec", marker = "python_full_version >= '3.10'" }, + { name = "importlib-metadata", version = "9.0.0", source = { registry = "https://pypi.org/simple" } }, + { name = "importlib-resources" }, + { name = "jinja2" }, + { name = "mkdocs" }, + { name = "pyparsing" }, + { name = "pyyaml" }, + { name = "pyyaml-env-tag" }, + { name = "verspec" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ab/f7/2933f1a1fb0e0f077d5d6a92c6c7f8a54e6128241f116dff4df8b6050bbf/mike-2.1.3.tar.gz", hash = "sha256:abd79b8ea483fb0275b7972825d3082e5ae67a41820f8d8a0dc7a3f49944e810", size = 38119, upload-time = "2024-08-13T05:02:14.167Z" } wheels = [ @@ -1877,7 +1877,7 @@ name = "mistune" version = "3.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ca/84/620cc3f7e3adf6f5067e10f4dbae71295d8f9e16d5d3f9ef97c40f2f592c/mistune-3.2.1.tar.gz", hash = "sha256:7c8e5501d38bac1582e067e46c8343f17d57ea1aaa735823f3aba1fd59c88a28", size = 98003, upload-time = "2026-05-03T14:33:22.312Z" } wheels = [ @@ -1889,19 +1889,19 @@ name = "mkdocs" version = "1.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click", marker = "python_full_version >= '3.10'" }, - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, - { name = "ghp-import", marker = "python_full_version >= '3.10'" }, - { name = "jinja2", marker = "python_full_version >= '3.10'" }, - { name = "markdown", marker = "python_full_version >= '3.10'" }, - { name = "markupsafe", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "mergedeep", marker = "python_full_version >= '3.10'" }, - { name = "mkdocs-get-deps", marker = "python_full_version >= '3.10'" }, - { name = "packaging", marker = "python_full_version >= '3.10'" }, - { name = "pathspec", marker = "python_full_version >= '3.10'" }, - { name = "pyyaml", marker = "python_full_version >= '3.10'" }, - { name = "pyyaml-env-tag", marker = "python_full_version >= '3.10'" }, - { name = "watchdog", marker = "python_full_version >= '3.10'" }, + { name = "click" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "ghp-import" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "markupsafe", version = "3.0.3", source = { registry = "https://pypi.org/simple" } }, + { name = "mergedeep" }, + { name = "mkdocs-get-deps" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "pyyaml" }, + { name = "pyyaml-env-tag" }, + { name = "watchdog" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bc/c6/bbd4f061bd16b378247f12953ffcb04786a618ce5e904b8c5a01a0309061/mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2", size = 3889159, upload-time = "2024-08-30T12:24:06.899Z" } wheels = [ @@ -1913,9 +1913,9 @@ name = "mkdocs-api-autonav" version = "0.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mkdocs", marker = "python_full_version >= '3.10'" }, - { name = "mkdocstrings-python", marker = "python_full_version >= '3.10'" }, - { name = "pyyaml", marker = "python_full_version >= '3.10'" }, + { name = "mkdocs" }, + { name = "mkdocstrings-python" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6d/b0/20960ee733a419a349877d09712d02e8ec2bda031672e5f0d0a15fc020b3/mkdocs_api_autonav-0.4.0.tar.gz", hash = "sha256:3527b0e5cf1b682bd374a3ce699ac12d6288f5fcaf93877f34a6b14c79740637", size = 17987, upload-time = "2025-09-09T12:42:02.216Z" } wheels = [ @@ -1927,9 +1927,9 @@ name = "mkdocs-autorefs" version = "1.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markdown", marker = "python_full_version >= '3.10'" }, - { name = "markupsafe", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "mkdocs", marker = "python_full_version >= '3.10'" }, + { name = "markdown" }, + { name = "markupsafe", version = "3.0.3", source = { registry = "https://pypi.org/simple" } }, + { name = "mkdocs" }, ] sdist = { url = "https://files.pythonhosted.org/packages/52/c0/f641843de3f612a6b48253f39244165acff36657a91cc903633d456ae1ac/mkdocs_autorefs-1.4.4.tar.gz", hash = "sha256:d54a284f27a7346b9c38f1f852177940c222da508e66edc816a0fa55fc6da197", size = 56588, upload-time = "2026-02-10T15:23:55.105Z" } wheels = [ @@ -1941,9 +1941,9 @@ name = "mkdocs-get-deps" version = "0.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mergedeep", marker = "python_full_version >= '3.10'" }, - { name = "platformdirs", version = "4.9.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "pyyaml", marker = "python_full_version >= '3.10'" }, + { name = "mergedeep" }, + { name = "platformdirs", version = "4.9.6", source = { registry = "https://pypi.org/simple" } }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ce/25/b3cccb187655b9393572bde9b09261d267c3bf2f2cdabe347673be5976a6/mkdocs_get_deps-0.2.2.tar.gz", hash = "sha256:8ee8d5f316cdbbb2834bc1df6e69c08fe769a83e040060de26d3c19fad3599a1", size = 11047, upload-time = "2026-03-10T02:46:33.632Z" } wheels = [ @@ -1955,8 +1955,8 @@ name = "mkdocs-include-markdown-plugin" version = "7.1.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mkdocs", marker = "python_full_version >= '3.10'" }, - { name = "wcmatch", marker = "python_full_version >= '3.10'" }, + { name = "mkdocs" }, + { name = "wcmatch" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2c/17/988d97ac6849b196f54d45ca9c60ca894880c160a512785f03834704b3d9/mkdocs_include_markdown_plugin-7.1.6.tar.gz", hash = "sha256:a0753cb82704c10a287f1e789fc9848f82b6beb8749814b24b03dd9f67816677", size = 23391, upload-time = "2025-06-13T18:25:51.193Z" } wheels = [ @@ -1968,12 +1968,12 @@ name = "mkdocs-jupyter" version = "0.25.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ipykernel", marker = "python_full_version >= '3.10'" }, - { name = "jupytext", marker = "python_full_version >= '3.10'" }, - { name = "mkdocs", marker = "python_full_version >= '3.10'" }, - { name = "mkdocs-material", marker = "python_full_version >= '3.10'" }, - { name = "nbconvert", marker = "python_full_version >= '3.10'" }, - { name = "pygments", version = "2.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "ipykernel" }, + { name = "jupytext" }, + { name = "mkdocs" }, + { name = "mkdocs-material" }, + { name = "nbconvert" }, + { name = "pygments", version = "2.20.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/6c/23/6ffb8d2fd2117aa860a04c6fe2510b21bc3c3c085907ffdd851caba53152/mkdocs_jupyter-0.25.1.tar.gz", hash = "sha256:0e9272ff4947e0ec683c92423a4bfb42a26477c103ab1a6ab8277e2dcc8f7afe", size = 1626747, upload-time = "2024-10-15T14:56:32.373Z" } wheels = [ @@ -1985,17 +1985,17 @@ name = "mkdocs-material" version = "9.7.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "babel", marker = "python_full_version >= '3.10'" }, - { name = "backrefs", marker = "python_full_version >= '3.10'" }, - { name = "colorama", marker = "python_full_version >= '3.10'" }, - { name = "jinja2", marker = "python_full_version >= '3.10'" }, - { name = "markdown", marker = "python_full_version >= '3.10'" }, - { name = "mkdocs", marker = "python_full_version >= '3.10'" }, - { name = "mkdocs-material-extensions", marker = "python_full_version >= '3.10'" }, - { name = "paginate", marker = "python_full_version >= '3.10'" }, - { name = "pygments", version = "2.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "pymdown-extensions", marker = "python_full_version >= '3.10'" }, - { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "babel" }, + { name = "backrefs" }, + { name = "colorama" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "mkdocs" }, + { name = "mkdocs-material-extensions" }, + { name = "paginate" }, + { name = "pygments", version = "2.20.0", source = { registry = "https://pypi.org/simple" } }, + { name = "pymdown-extensions" }, + { name = "requests", version = "2.34.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/74/76/5c202fecdc45d53e83e03a85bae70c48b6c81e9f87f0bc19a9e9c723bdc0/mkdocs_material-9.7.5.tar.gz", hash = "sha256:f76bdab532bad1d9c57ca7187b37eccf64dd12e1586909307f8856db3be384ea", size = 4097749, upload-time = "2026-03-10T15:43:22.809Z" } wheels = [ @@ -2016,12 +2016,12 @@ name = "mkdocstrings" version = "1.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jinja2", marker = "python_full_version >= '3.10'" }, - { name = "markdown", marker = "python_full_version >= '3.10'" }, - { name = "markupsafe", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "mkdocs", marker = "python_full_version >= '3.10'" }, - { name = "mkdocs-autorefs", marker = "python_full_version >= '3.10'" }, - { name = "pymdown-extensions", marker = "python_full_version >= '3.10'" }, + { name = "jinja2" }, + { name = "markdown" }, + { name = "markupsafe", version = "3.0.3", source = { registry = "https://pypi.org/simple" } }, + { name = "mkdocs" }, + { name = "mkdocs-autorefs" }, + { name = "pymdown-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/46/62/0dfc5719514115bf1781f44b1d7f2a0923fcc01e9c5d7990e48a05c9ae5d/mkdocstrings-1.0.3.tar.gz", hash = "sha256:ab670f55040722b49bb45865b2e93b824450fb4aef638b00d7acb493a9020434", size = 100946, upload-time = "2026-02-07T14:31:40.973Z" } wheels = [ @@ -2030,7 +2030,7 @@ wheels = [ [package.optional-dependencies] python = [ - { name = "mkdocstrings-python", marker = "python_full_version >= '3.10'" }, + { name = "mkdocstrings-python" }, ] [[package]] @@ -2038,10 +2038,10 @@ name = "mkdocstrings-python" version = "2.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "griffelib", marker = "python_full_version >= '3.10'" }, - { name = "mkdocs-autorefs", marker = "python_full_version >= '3.10'" }, - { name = "mkdocstrings", marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "griffelib" }, + { name = "mkdocs-autorefs" }, + { name = "mkdocstrings" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/29/33/c225eaf898634bdda489a6766fc35d1683c640bffe0e0acd10646b13536d/mkdocstrings_python-2.0.3.tar.gz", hash = "sha256:c518632751cc869439b31c9d3177678ad2bfa5c21b79b863956ad68fc92c13b8", size = 199083, upload-time = "2026-02-20T10:38:36.368Z" } wheels = [ @@ -2102,10 +2102,10 @@ name = "nbclient" version = "0.10.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jupyter-client", marker = "python_full_version >= '3.10'" }, - { name = "jupyter-core", marker = "python_full_version >= '3.10'" }, - { name = "nbformat", marker = "python_full_version >= '3.10'" }, - { name = "traitlets", marker = "python_full_version >= '3.10'" }, + { name = "jupyter-client" }, + { name = "jupyter-core" }, + { name = "nbformat" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/56/91/1c1d5a4b9a9ebba2b4e32b8c852c2975c872aec1fe42ab5e516b2cecd193/nbclient-0.10.4.tar.gz", hash = "sha256:1e54091b16e6da39e297b0ece3e10f6f29f4ac4e8ee515d29f8a7099bd6553c9", size = 62554, upload-time = "2025-12-23T07:45:46.369Z" } wheels = [ @@ -2117,20 +2117,20 @@ name = "nbconvert" version = "7.17.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "beautifulsoup4", marker = "python_full_version >= '3.10'" }, - { name = "bleach", extra = ["css"], marker = "python_full_version >= '3.10'" }, - { name = "defusedxml", marker = "python_full_version >= '3.10'" }, - { name = "jinja2", marker = "python_full_version >= '3.10'" }, - { name = "jupyter-core", marker = "python_full_version >= '3.10'" }, - { name = "jupyterlab-pygments", marker = "python_full_version >= '3.10'" }, - { name = "markupsafe", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "mistune", marker = "python_full_version >= '3.10'" }, - { name = "nbclient", marker = "python_full_version >= '3.10'" }, - { name = "nbformat", marker = "python_full_version >= '3.10'" }, - { name = "packaging", marker = "python_full_version >= '3.10'" }, - { name = "pandocfilters", marker = "python_full_version >= '3.10'" }, - { name = "pygments", version = "2.20.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "traitlets", marker = "python_full_version >= '3.10'" }, + { name = "beautifulsoup4" }, + { name = "bleach", extra = ["css"] }, + { name = "defusedxml" }, + { name = "jinja2" }, + { name = "jupyter-core" }, + { name = "jupyterlab-pygments" }, + { name = "markupsafe", version = "3.0.3", source = { registry = "https://pypi.org/simple" } }, + { name = "mistune" }, + { name = "nbclient" }, + { name = "nbformat" }, + { name = "packaging" }, + { name = "pandocfilters" }, + { name = "pygments", version = "2.20.0", source = { registry = "https://pypi.org/simple" } }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/01/b1/708e53fe2e429c103c6e6e159106bcf0357ac41aa4c28772bd8402339051/nbconvert-7.17.1.tar.gz", hash = "sha256:34d0d0a7e73ce3cbab6c5aae8f4f468797280b01fd8bd2ca746da8569eddd7d2", size = 865311, upload-time = "2026-04-08T00:44:14.914Z" } wheels = [ @@ -2142,10 +2142,10 @@ name = "nbformat" version = "5.10.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "fastjsonschema", marker = "python_full_version >= '3.10'" }, - { name = "jsonschema", marker = "python_full_version >= '3.10'" }, - { name = "jupyter-core", marker = "python_full_version >= '3.10'" }, - { name = "traitlets", marker = "python_full_version >= '3.10'" }, + { name = "fastjsonschema" }, + { name = "jsonschema" }, + { name = "jupyter-core" }, + { name = "traitlets" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6d/fd/91545e604bc3dad7dca9ed03284086039b294c6b3d75c0d2fa45f9e9caf3/nbformat-5.10.4.tar.gz", hash = "sha256:322168b14f937a5d11362988ecac2a4952d3d8e3a2cbeb2319584631226d5b3a", size = 142749, upload-time = "2024-04-04T11:20:37.371Z" } wheels = [ @@ -2455,10 +2455,10 @@ resolution-markers = [ "python_full_version < '3.8.2'", ] dependencies = [ - { name = "numpy", version = "1.24.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "python-dateutil", marker = "python_full_version < '3.9'" }, - { name = "pytz", marker = "python_full_version < '3.9'" }, - { name = "tzdata", marker = "python_full_version < '3.9'" }, + { name = "numpy", version = "1.24.4", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/a7/824332581e258b5aa4f3763ecb2a797e5f9a54269044ba2e50ac19936b32/pandas-2.0.3.tar.gz", hash = "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c", size = 5284455, upload-time = "2023-06-28T23:19:33.371Z" } wheels = [ @@ -2493,15 +2493,15 @@ name = "pandas" version = "2.3.3" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", ] dependencies = [ - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.10.*'" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.9' and python_full_version < '3.11'" }, - { name = "pytz", marker = "python_full_version >= '3.9' and python_full_version < '3.11'" }, - { name = "tzdata", marker = "python_full_version >= '3.9' and python_full_version < '3.11'" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -2574,9 +2574,9 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "python-dateutil", marker = "python_full_version >= '3.11'" }, - { name = "tzdata", marker = "(python_full_version >= '3.11' and sys_platform == 'emscripten') or (python_full_version >= '3.11' and sys_platform == 'win32')" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } wheels = [ @@ -2638,8 +2638,8 @@ resolution-markers = [ "python_full_version < '3.8.2'", ] dependencies = [ - { name = "numpy", version = "1.24.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "types-pytz", version = "2024.2.0.20241221", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "numpy", version = "1.24.4", source = { registry = "https://pypi.org/simple" } }, + { name = "types-pytz", version = "2024.2.0.20241221", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/46/55/223223da2938e60e0bf3f1fde843e0e0680471992d27dd9d8d0f11fed72e/pandas_stubs-2.0.2.230605.tar.gz", hash = "sha256:624c7bb06d38145a44b61be459ccd19b038e0bf20364a025ecaab78fea65e858", size = 98797, upload-time = "2023-06-05T12:43:28.394Z" } wheels = [ @@ -2654,8 +2654,8 @@ resolution-markers = [ "python_full_version == '3.9.*'", ] dependencies = [ - { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, - { name = "types-pytz", version = "2025.2.0.20251108", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" } }, + { name = "types-pytz", version = "2025.2.0.20251108", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/1f/df/0da95bc75c76f1e012e0bc0b76da31faaf4254e94b9870f25e6311145e98/pandas_stubs-2.2.2.240807.tar.gz", hash = "sha256:64a559725a57a449f46225fbafc422520b7410bff9252b661a225b5559192a93", size = 103095, upload-time = "2024-08-07T12:30:54.538Z" } wheels = [ @@ -2670,8 +2670,8 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, - { name = "types-pytz", version = "2026.2.0.20260518", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "types-pytz", version = "2026.2.0.20260518", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/92/5d/be23854a73fda69f1dbdda7bc10fbd6f930bd1fa87aaec389f00c901c1e8/pandas_stubs-2.3.3.260113.tar.gz", hash = "sha256:076e3724bcaa73de78932b012ec64b3010463d377fa63116f4e6850643d93800", size = 116131, upload-time = "2026-01-13T22:30:16.704Z" } wheels = [ @@ -2691,7 +2691,7 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/27/1d/297ff2c7ea50a768a2247621d6451abb2a07c0e9be7ca6d36ebe371658e5/pandas_stubs-3.0.0.260204.tar.gz", hash = "sha256:bf9294b76352effcffa9cb85edf0bed1339a7ec0c30b8e1ac3d66b4228f1fbc3", size = 109383, upload-time = "2026-02-04T15:17:17.247Z" } wheels = [ @@ -2747,7 +2747,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess", marker = "(python_full_version == '3.10.*' and sys_platform == 'emscripten') or (python_full_version == '3.10.*' and sys_platform == 'win32') or (python_full_version >= '3.10' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "ptyprocess" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -2802,7 +2802,6 @@ name = "pluggy" version = "1.6.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -2810,6 +2809,7 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } wheels = [ @@ -2841,7 +2841,7 @@ resolution-markers = [ "python_full_version == '3.9.*'", ] dependencies = [ - { name = "polars-runtime-32", version = "1.36.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "polars-runtime-32", version = "1.36.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/9f/dc/56f2a90c79a2cb13f9e956eab6385effe54216ae7a2068b3a6406bae4345/polars-1.36.1.tar.gz", hash = "sha256:12c7616a2305559144711ab73eaa18814f7aa898c522e7645014b68f1432d54c", size = 711993, upload-time = "2025-12-10T01:14:53.033Z" } wheels = [ @@ -2862,7 +2862,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "polars-runtime-32", version = "1.40.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "polars-runtime-32", version = "1.40.1", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/b3/8c/bc9bc948058348ed43117cecc3007cd608f395915dae8a00974579a5dab1/polars-1.40.1.tar.gz", hash = "sha256:ab2694134b137596b5a59bfd7b4c54ebbc9b59f9403127f18e32d363777552e8", size = 733574, upload-time = "2026-04-22T19:15:55.507Z" } wheels = [ @@ -2916,11 +2916,11 @@ name = "pre-commit" version = "4.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cfgv", marker = "python_full_version >= '3.9'" }, - { name = "identify", marker = "python_full_version >= '3.9'" }, - { name = "nodeenv", marker = "python_full_version >= '3.9'" }, - { name = "pyyaml", marker = "python_full_version >= '3.9'" }, - { name = "virtualenv", marker = "python_full_version >= '3.9'" }, + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ff/29/7cf5bbc236333876e4b41f56e06857a87937ce4bf91e117a6991a2dbb02a/pre_commit-4.3.0.tar.gz", hash = "sha256:499fe450cc9d42e9d58e606262795ecb64dd05438943c62b66f6a8673da30b16", size = 193792, upload-time = "2025-08-09T18:56:14.651Z" } wheels = [ @@ -2932,7 +2932,7 @@ name = "prompt-toolkit" version = "3.0.52" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "wcwidth", marker = "python_full_version >= '3.10'" }, + { name = "wcwidth" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } wheels = [ @@ -2966,7 +2966,6 @@ name = "protobuf" version = "6.33.6" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -2974,6 +2973,7 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } wheels = [ @@ -3067,7 +3067,7 @@ resolution-markers = [ "python_full_version < '3.8.2'", ] dependencies = [ - { name = "numpy", version = "1.24.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "numpy", version = "1.24.4", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/27/4e/ea6d43f324169f8aec0e57569443a38bab4b398d09769ca64f7b4d467de3/pyarrow-17.0.0.tar.gz", hash = "sha256:4beca9521ed2c0921c1023e68d097d0299b62c362639ea315572a58f3f50fd28", size = 1112479, upload-time = "2024-07-17T10:41:25.092Z" } wheels = [ @@ -3232,9 +3232,9 @@ name = "pycparser" version = "2.23" source = { registry = "https://pypi.org/simple" } resolution-markers = [ + "python_full_version == '3.9.*'", "python_full_version >= '3.8.2' and python_full_version < '3.9'", "python_full_version < '3.8.2'", - "python_full_version == '3.9.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/fe/cf/d2d3b9f5699fb1e4615c8e32ff220203e43b248e1dfcc6736ad9057731ca/pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2", size = 173734, upload-time = "2025-09-09T13:23:47.91Z" } wheels = [ @@ -3268,9 +3268,9 @@ resolution-markers = [ "python_full_version < '3.8.2'", ] dependencies = [ - { name = "annotated-types", marker = "python_full_version < '3.9'" }, - { name = "pydantic-core", version = "2.27.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, - { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "annotated-types" }, + { name = "pydantic-core", version = "2.27.2", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/b7/ae/d5220c5c52b158b1de7ca89fc5edb72f304a70a4c540c84c8844bf4008de/pydantic-2.10.6.tar.gz", hash = "sha256:ca5daa827cce33de7a42be142548b0096bf05a7e7b365aebfa5f8eeec7128236", size = 761681, upload-time = "2025-01-24T01:42:12.693Z" } wheels = [ @@ -3282,7 +3282,6 @@ name = "pydantic" version = "2.13.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -3290,12 +3289,13 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", ] dependencies = [ - { name = "annotated-types", marker = "python_full_version >= '3.9'" }, - { name = "pydantic-core", version = "2.46.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, - { name = "typing-inspection", marker = "python_full_version >= '3.9'" }, + { name = "annotated-types" }, + { name = "pydantic-core", version = "2.46.4", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-inspection" }, ] sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } wheels = [ @@ -3311,7 +3311,7 @@ resolution-markers = [ "python_full_version < '3.8.2'", ] dependencies = [ - { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "typing-extensions", version = "4.13.2", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/fc/01/f3e5ac5e7c25833db5eb555f7b7ab24cd6f8c322d3a3ad2d67a952dc0abc/pydantic_core-2.27.2.tar.gz", hash = "sha256:eb026e5a4c1fee05726072337ff51d1efb6f59090b7da90d30ea58625b1ffb39", size = 413443, upload-time = "2024-12-18T11:31:54.917Z" } wheels = [ @@ -3421,7 +3421,6 @@ name = "pydantic-core" version = "2.46.4" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -3429,9 +3428,10 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", ] dependencies = [ - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } wheels = [ @@ -3574,7 +3574,6 @@ name = "pygments" version = "2.20.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -3582,6 +3581,7 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } wheels = [ @@ -3593,8 +3593,8 @@ name = "pymdown-extensions" version = "10.21.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markdown", marker = "python_full_version >= '3.10'" }, - { name = "pyyaml", marker = "python_full_version >= '3.10'" }, + { name = "markdown" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9e/26/d1015444da4d952a1ca487a236b522eb979766f0295a0bd0c5fc089989a9/pymdown_extensions-10.21.3.tar.gz", hash = "sha256:72cfcf55f07aea0d4af2c4f11dd4e52466ddfb1bb819673146398e0bd3a77354", size = 854140, upload-time = "2026-05-13T12:57:32.267Z" } wheels = [ @@ -3747,9 +3747,9 @@ name = "python-discovery" version = "1.4.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock", version = "3.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "filelock", version = "3.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "filelock", version = "3.29.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "platformdirs", version = "4.9.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4c/81/58c70036dffeccb7fe7d79d6260c69f7a28272bbd3909c29a01ea9422744/python_discovery-1.4.4.tar.gz", hash = "sha256:5cad33982d412c1f3ffb8f9ca4ea292c9680bca3942451d30b69c37fce53a4a3", size = 72212, upload-time = "2026-07-08T23:06:50.691Z" } @@ -3909,7 +3909,7 @@ name = "pyyaml-env-tag" version = "1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyyaml", marker = "python_full_version >= '3.10'" }, + { name = "pyyaml" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/2e/79c822141bfd05a853236b504869ebc6b70159afc570e1d5a20641782eaa/pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff", size = 5737, upload-time = "2025-05-13T15:24:01.64Z" } wheels = [ @@ -3921,7 +3921,7 @@ name = "pyzmq" version = "27.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "python_full_version >= '3.10' and implementation_name == 'pypy'" }, + { name = "cffi", marker = "implementation_name == 'pypy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/04/0b/3c9baedbdf613ecaa7aa07027780b8867f57b6293b6ee50de316c9f3222b/pyzmq-27.1.0.tar.gz", hash = "sha256:ac0765e3d44455adb6ddbf4417dcce460fc40a05978c08efdf2948072f6db540", size = 281750, upload-time = "2025-09-08T23:10:18.157Z" } wheels = [ @@ -4094,9 +4094,9 @@ name = "referencing" version = "0.37.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "attrs", marker = "python_full_version >= '3.10'" }, - { name = "rpds-py", marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10' and python_full_version < '3.13'" }, + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } wheels = [ @@ -4112,10 +4112,10 @@ resolution-markers = [ "python_full_version < '3.8.2'", ] dependencies = [ - { name = "certifi", marker = "python_full_version < '3.9'" }, - { name = "charset-normalizer", marker = "python_full_version < '3.9'" }, - { name = "idna", marker = "python_full_version < '3.9'" }, - { name = "urllib3", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3", version = "2.2.3", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/e1/0a/929373653770d8a0d7ea76c37de6e41f11eb07559b103b1c02cafb3f7cf8/requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422", size = 135258, upload-time = "2025-06-09T16:43:07.34Z" } wheels = [ @@ -4130,10 +4130,10 @@ resolution-markers = [ "python_full_version == '3.9.*'", ] dependencies = [ - { name = "certifi", marker = "python_full_version == '3.9.*'" }, - { name = "charset-normalizer", marker = "python_full_version == '3.9.*'" }, - { name = "idna", marker = "python_full_version == '3.9.*'" }, - { name = "urllib3", version = "2.6.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3", version = "2.6.3", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/c9/74/b3ff8e6c8446842c3f5c837e9c3dfcfe2018ea6ecef224c710c85ef728f4/requests-2.32.5.tar.gz", hash = "sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf", size = 134517, upload-time = "2025-08-18T20:46:02.573Z" } wheels = [ @@ -4154,10 +4154,10 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "certifi", marker = "python_full_version >= '3.10'" }, - { name = "charset-normalizer", marker = "python_full_version >= '3.10'" }, - { name = "idna", marker = "python_full_version >= '3.10'" }, - { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ @@ -4183,17 +4183,17 @@ name = "rosbags" version = "0.9.23" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.8.2' and python_full_version < '3.9'", "python_full_version == '3.9.*'", + "python_full_version >= '3.8.2' and python_full_version < '3.9'", ] dependencies = [ - { name = "lz4", version = "4.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8.2' and python_full_version < '3.9'" }, + { name = "lz4", version = "4.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.9.*'" }, { name = "lz4", version = "4.4.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, - { name = "numpy", version = "1.24.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8.2' and python_full_version < '3.9'" }, + { name = "numpy", version = "1.24.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.9.*'" }, { name = "numpy", version = "2.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, - { name = "ruamel-yaml", version = "0.18.16", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8.2' and python_full_version < '3.9'" }, + { name = "ruamel-yaml", version = "0.18.16", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.9.*'" }, { name = "ruamel-yaml", version = "0.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, - { name = "zstandard", version = "0.23.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8.2' and python_full_version < '3.9'" }, + { name = "zstandard", version = "0.23.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.9.*'" }, { name = "zstandard", version = "0.25.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6a/da/8d90a8c2e26111515bd8565b75d5d926ac2540a41b0e575ac2c40bff9193/rosbags-0.9.23.tar.gz", hash = "sha256:89229ca8fbb6d5e1cc452045229e6e47028dc10a8724fbe86c93eda477296dc2", size = 165057, upload-time = "2024-04-12T08:48:40.672Z" } @@ -4215,13 +4215,13 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "apsw", marker = "python_full_version >= '3.10'" }, - { name = "lz4", version = "4.4.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "apsw" }, + { name = "lz4", version = "4.4.5", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "ruamel-yaml", version = "0.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "zstandard", version = "0.25.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "ruamel-yaml", version = "0.19.1", source = { registry = "https://pypi.org/simple" } }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" } }, + { name = "zstandard", version = "0.25.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/ba/59/8e5337c7091d5b353cae62d1b51d1f8b4ffae7280c784eb50bd1c587017f/rosbags-0.11.2.tar.gz", hash = "sha256:d6c0d78794270fe21c779b857a8ac9eca525818e36b5b8d6134eb69b5efc3997", size = 274863, upload-time = "2026-05-11T07:52:08.565Z" } wheels = [ @@ -4358,7 +4358,7 @@ resolution-markers = [ "python_full_version >= '3.8.2' and python_full_version < '3.9'", ] dependencies = [ - { name = "ruamel-yaml-clib", marker = "python_full_version >= '3.8.2' and python_full_version < '3.9' and platform_python_implementation == 'CPython'" }, + { name = "ruamel-yaml-clib", marker = "platform_python_implementation == 'CPython'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9f/c7/ee630b29e04a672ecfc9b63227c87fd7a37eb67c1bf30fe95376437f897c/ruamel.yaml-0.18.16.tar.gz", hash = "sha256:a6e587512f3c998b2225d68aa1f35111c29fad14aed561a26e73fab729ec5e5a", size = 147269, upload-time = "2025-10-22T17:54:02.346Z" } wheels = [ @@ -4370,7 +4370,6 @@ name = "ruamel-yaml" version = "0.19.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -4378,6 +4377,7 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/c7/3b/ebda527b56beb90cb7652cb1c7e4f91f48649fbcd8d2eb2fb6e77cd3329b/ruamel_yaml-0.19.1.tar.gz", hash = "sha256:53eb66cd27849eff968ebf8f0bf61f46cdac2da1d1f3576dd4ccee9b25c31993", size = 142709, upload-time = "2026-01-02T16:50:31.84Z" } wheels = [ @@ -4690,6 +4690,11 @@ openssl = [ { name = "pyopenssl" }, { name = "types-pyopenssl" }, ] +parquet = [ + { name = "pyarrow", version = "17.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "pyarrow", version = "24.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, +] rosbags = [ { name = "rosbags", version = "0.9.23", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8.2' and python_full_version < '3.10'" }, { name = "rosbags", version = "0.11.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, @@ -4786,6 +4791,7 @@ requires-dist = [ { name = "pyarrow", marker = "extra == 'data-review'", specifier = ">=17.0.0" }, { name = "pyarrow", marker = "extra == 'dev-all'", specifier = ">=17.0.0" }, { name = "pyarrow", marker = "extra == 'docs-build'", specifier = ">=17.0.0" }, + { name = "pyarrow", marker = "extra == 'parquet'", specifier = ">=17.0.0" }, { name = "pydantic", specifier = "~=2.10" }, { name = "pydantic-core", specifier = "~=2.3" }, { name = "pyopenssl", marker = "extra == 'all'", specifier = "<24.0.0" }, @@ -4857,7 +4863,7 @@ requires-dist = [ { name = "types-requests", specifier = "~=2.25" }, { name = "typing-extensions", specifier = "~=4.6" }, ] -provides-extras = ["all", "build", "data-review", "dev", "dev-all", "development", "docs", "docs-build", "file-imports", "hdf5", "mcap", "openssl", "rosbags", "sift-stream", "sift-stream-bindings", "tdms", "ulog"] +provides-extras = ["all", "build", "data-review", "dev", "dev-all", "development", "docs", "docs-build", "file-imports", "hdf5", "mcap", "openssl", "parquet", "rosbags", "sift-stream", "sift-stream-bindings", "tdms", "ulog"] [[package]] name = "sift-stream-bindings" @@ -4939,9 +4945,9 @@ name = "stack-data" version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "asttokens", marker = "python_full_version >= '3.10'" }, - { name = "executing", marker = "python_full_version >= '3.10'" }, - { name = "pure-eval", marker = "python_full_version >= '3.10'" }, + { name = "asttokens" }, + { name = "executing" }, + { name = "pure-eval" }, ] sdist = { url = "https://files.pythonhosted.org/packages/28/e3/55dcc2cfbc3ca9c29519eb6884dd1415ecb53b0e934862d3559ddcb7e20b/stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9", size = 44707, upload-time = "2023-09-30T13:58:05.479Z" } wheels = [ @@ -4953,7 +4959,7 @@ name = "tinycss2" version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "webencodings", marker = "python_full_version >= '3.10'" }, + { name = "webencodings" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/fd/7a5ee21fd08ff70d3d33a5781c255cbe779659bd03278feb98b19ee550f4/tinycss2-1.4.0.tar.gz", hash = "sha256:10c0972f6fc0fbee87c3edb76549357415e94548c1ae10ebccdea16fb404a9b7", size = 87085, upload-time = "2024-10-24T14:58:29.895Z" } wheels = [ @@ -5193,7 +5199,7 @@ resolution-markers = [ "python_full_version < '3.8.2'", ] dependencies = [ - { name = "urllib3", version = "2.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "urllib3", version = "2.2.3", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/3c/4f2a430c01a22abd49a583b6b944173e39e7d01b688190a5618bd59a2e22/types-requests-2.32.0.20241016.tar.gz", hash = "sha256:0d9cad2f27515d0e3e3da7134a1b6f28fb97129d86b867f24d9c726452634d95", size = 18065, upload-time = "2024-10-16T02:46:10.818Z" } wheels = [ @@ -5208,7 +5214,7 @@ resolution-markers = [ "python_full_version == '3.9.*'", ] dependencies = [ - { name = "urllib3", version = "2.6.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "urllib3", version = "2.6.3", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/f3/a0663907082280664d745929205a89d41dffb29e89a50f753af7d57d0a96/types_requests-2.32.4.20260107.tar.gz", hash = "sha256:018a11ac158f801bfa84857ddec1650750e393df8a004a8a9ae2a9bec6fcb24f", size = 23165, upload-time = "2026-01-07T03:20:54.091Z" } wheels = [ @@ -5229,7 +5235,7 @@ resolution-markers = [ "python_full_version == '3.10.*'", ] dependencies = [ - { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "urllib3", version = "2.7.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/e0/01/c5a19253fe1ac159159ddf9a3a07cec8bb5e486ec4d9002ad2821da0e5d2/types_requests-2.33.0.20260518.tar.gz", hash = "sha256:df7bd3bfe0ca8402dfb841e7d9be714bb5578203283d66d7dc4ef69343449a5e", size = 24752, upload-time = "2026-05-18T06:07:37.966Z" } wheels = [ @@ -5254,7 +5260,6 @@ name = "typing-extensions" version = "4.15.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -5262,6 +5267,7 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } wheels = [ @@ -5273,7 +5279,7 @@ name = "typing-inspection" version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" } }, ] sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } wheels = [ @@ -5346,13 +5352,13 @@ name = "virtualenv" version = "21.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "distlib", marker = "python_full_version >= '3.9'" }, - { name = "filelock", version = "3.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "distlib" }, + { name = "filelock", version = "3.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "filelock", version = "3.29.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "platformdirs", version = "4.9.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "python-discovery", marker = "python_full_version >= '3.9'" }, - { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9' and python_full_version < '3.11'" }, + { name = "python-discovery" }, + { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/34/d9/b477fddb68840b570af8b22afe9b035cbc277b5fb7b33dea390617a8b10f/virtualenv-21.6.1.tar.gz", hash = "sha256:15f978b7cd329f24855ff4a0c4b4899cc7678589f49adbdcbbb4d3232e641128", size = 5526620, upload-time = "2026-07-10T19:33:53.312Z" } wheels = [ @@ -5401,7 +5407,7 @@ name = "wcmatch" version = "10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "bracex", marker = "python_full_version >= '3.10'" }, + { name = "bracex" }, ] sdist = { url = "https://files.pythonhosted.org/packages/79/3e/c0bdc27cf06f4e47680bd5803a07cb3dfd17de84cde92dd217dcb9e05253/wcmatch-10.1.tar.gz", hash = "sha256:f11f94208c8c8484a16f4f48638a85d771d9513f4ab3f37595978801cb9465af", size = 117421, upload-time = "2025-06-22T19:14:02.49Z" } wheels = [ @@ -5487,7 +5493,7 @@ resolution-markers = [ "python_full_version < '3.8.2'", ] dependencies = [ - { name = "cffi", marker = "python_full_version < '3.9' and platform_python_implementation == 'PyPy'" }, + { name = "cffi", marker = "platform_python_implementation == 'PyPy'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ed/f6/2ac0287b442160a89d726b17a9184a4c615bb5237db763791a7fd16d9df1/zstandard-0.23.0.tar.gz", hash = "sha256:b2d8c62d08e7255f68f7a740bae85b3c9b8e5466baa9cbf7f57f1cde0ac6bc09", size = 681701, upload-time = "2024-07-15T00:18:06.141Z" } wheels = [ @@ -5594,7 +5600,6 @@ name = "zstandard" version = "0.25.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version == '3.9.*'", "python_full_version >= '3.14' and sys_platform == 'win32'", "python_full_version >= '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", @@ -5602,6 +5607,7 @@ resolution-markers = [ "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform == 'emscripten'", "python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", "python_full_version == '3.10.*'", + "python_full_version == '3.9.*'", ] sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } wheels = [ From bdde4a0b65208a30b82c409efd4fc9ecc34058f5 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Fri, 9 Oct 2026 15:33:23 -0700 Subject: [PATCH 04/20] python(test): make Parquet detection tests pass on Python 3.8 Python 3.8 resolves pyarrow 17, which cannot build float16 arrays from Python floats or write view types to Parquet. Build the float16 column from numpy values and move the view types into a test that is skipped when the installed pyarrow cannot write them. --- .../_tests/_internal/test_parquet.py | 35 ++++++++++++++++--- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/python/lib/sift_client/_tests/_internal/test_parquet.py b/python/lib/sift_client/_tests/_internal/test_parquet.py index 5b709185f8..2bf77a7cd6 100644 --- a/python/lib/sift_client/_tests/_internal/test_parquet.py +++ b/python/lib/sift_client/_tests/_internal/test_parquet.py @@ -6,6 +6,7 @@ from datetime import datetime, timezone from unittest.mock import AsyncMock, MagicMock +import numpy as np import pyarrow as pa import pyarrow.parquet as pq import pytest @@ -66,12 +67,22 @@ def _channel_field(name: str, data_type: str | None = "CHANNEL_DATA_TYPE_DOUBLE" return _embedded_field(name, pa.float64(), _CHANNEL_CONFIG, config) +def _parquet_writes_view_types() -> bool: + """Whether the installed pyarrow can write view types (pyarrow 17 on Python 3.8 cannot).""" + try: + pq.write_table(pa.table({"s": pa.array([""], pa.string_view())}), pa.BufferOutputStream()) + except pa.ArrowNotImplementedError: + return False + return True + + class TestDataTypes: def test_scalar_types(self, create_parquet_file): table = pa.table( { "bool": pa.array([True]), - "f16": pa.array([1.0], pa.float16()), + # pyarrow 17 (installed on Python 3.8) only builds float16 from numpy values. + "f16": pa.array(np.array([1.0], dtype=np.float16)), "f32": pa.array([1.0], pa.float32()), "f64": pa.array([1.0], pa.float64()), "i8": pa.array([1], pa.int8()), @@ -84,10 +95,8 @@ def test_scalar_types(self, create_parquet_file): "u64": pa.array([1], pa.uint64()), "string": pa.array(["a"], pa.string()), "large_string": pa.array(["a"], pa.large_string()), - "string_view": pa.array(["a"], pa.string_view()), "binary": pa.array([b"a"], pa.binary()), "large_binary": pa.array([b"a"], pa.large_binary()), - "binary_view": pa.array([b"a"], pa.binary_view()), "fixed_binary": pa.array([b"ab"], pa.binary(2)), } ) @@ -108,13 +117,29 @@ def test_scalar_types(self, create_parquet_file): "u64": ChannelDataType.UINT_64, "string": ChannelDataType.STRING, "large_string": ChannelDataType.STRING, - "string_view": ChannelDataType.STRING, "binary": ChannelDataType.BYTES, "large_binary": ChannelDataType.BYTES, - "binary_view": ChannelDataType.BYTES, "fixed_binary": ChannelDataType.BYTES, } + @pytest.mark.skipif( + not _parquet_writes_view_types(), + reason="this pyarrow version cannot write view types to Parquet", + ) + def test_view_types(self, create_parquet_file): + table = pa.table( + { + "string_view": pa.array(["a"], pa.string_view()), + "binary_view": pa.array([b"a"], pa.binary_view()), + } + ) + config = detect_flat_parquet_config(create_parquet_file(table)) + + assert {path: data_type for path, (_, data_type) in _columns(config).items()} == { + "string_view": ChannelDataType.STRING, + "binary_view": ChannelDataType.BYTES, + } + def test_time_types_are_int64(self, create_parquet_file): # The absolute timestamp comes first so it is the time column; the # relative time64 column is then only a data column. From 27dc582f83a2810c87eaaa30ba4ac3279fb79f11 Mon Sep 17 00:00:00 2001 From: alden-sift Date: Mon, 12 Oct 2026 10:03:34 -0700 Subject: [PATCH 05/20] Update python/lib/sift_client/sift_types/data_import.py Co-authored-by: Alex Luck --- python/lib/sift_client/sift_types/data_import.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lib/sift_client/sift_types/data_import.py b/python/lib/sift_client/sift_types/data_import.py index 0cf50005df..a01fcffe7b 100644 --- a/python/lib/sift_client/sift_types/data_import.py +++ b/python/lib/sift_client/sift_types/data_import.py @@ -1063,7 +1063,7 @@ class McapComplexTypesImportMode(Enum): class McapDataColumn(DataColumnBase): """A single MCAP channel selection. - Channels are selected by topicf and flattened field path, as returned by + Channels are selected by topic and flattened field path, as returned by ``detect_config``. A variable-cardinality field is selected whole by its base path and imports per its ``data_type``: ``BYTES`` (Arrow IPC) or ``STRING`` (JSON), which ``complex_types_import_mode`` must allow. From 9a4b196cc7b5a227453f943d2af2451ca857e548 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 12 Oct 2026 10:47:31 -0700 Subject: [PATCH 06/20] python(fix): scope relative_start_time to flat dataset Parquet configs --- .../_tests/_internal/test_parquet.py | 20 +++++++++++++++ .../lib/sift_client/resources/data_imports.py | 25 +++++++++---------- .../resources/sync_stubs/__init__.pyi | 14 +++++------ 3 files changed, 39 insertions(+), 20 deletions(-) diff --git a/python/lib/sift_client/_tests/_internal/test_parquet.py b/python/lib/sift_client/_tests/_internal/test_parquet.py index 2bf77a7cd6..0e29a7ee6b 100644 --- a/python/lib/sift_client/_tests/_internal/test_parquet.py +++ b/python/lib/sift_client/_tests/_internal/test_parquet.py @@ -20,6 +20,9 @@ CsvImportConfig, CsvTimeColumn, DataTypeKey, + ParquetSingleChannelConfig, + ParquetSingleChannelPerRowImportConfig, + ParquetTimeColumn, TimeFormat, ) @@ -617,3 +620,20 @@ def test_non_parquet_config_warns_and_ignores_start_time(self): _apply_relative_start_time(config, self.START) assert config.time_column.relative_start_time is None + + def test_single_channel_per_row_config_warns_and_ignores_start_time(self): + config = ParquetSingleChannelPerRowImportConfig( + asset_name="a", + time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), + single_channel=ParquetSingleChannelConfig( + data_path="v", name="v", data_type=ChannelDataType.DOUBLE + ), + ) + config.time_column.format = TimeFormat.RELATIVE_SECONDS + + with pytest.warns( + UserWarning, match="not supported for ParquetSingleChannelPerRowImportConfig" + ): + _apply_relative_start_time(config, self.START) + + assert config.time_column.relative_start_time is None diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index 1f1e242ef9..db73a4f162 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -131,10 +131,10 @@ async def import_from_path( detected/default time handling. Only used when ``config`` is not provided. relative_start_time: Start time for a relative time format. - Parquet only; ignored with a warning for other formats or - when the time format is absolute. Required for Parquet files - with a relative time column. Only used when ``config`` is not - provided. + Flat dataset Parquet only; ignored with a warning for other + formats or when the time format is absolute. Required for + Parquet files with a relative time column. Only used when + ``config`` is not provided. run: ``Run`` object or run ID string to import into an existing run. Mutually exclusive with ``run_name``. run_name: Name for a new run. Defaults to the filename if @@ -394,9 +394,9 @@ async def detect_config( ``TimeFormat.ABSOLUTE_UNIX_NANOSECONDS``. TDMS keeps its detected/default time handling. relative_start_time: Start time for a relative time format. - Parquet only; ignored with a warning for other formats or - when the time format is absolute. Required for Parquet files - with a relative time column. + Flat dataset Parquet only; ignored with a warning for other + formats or when the time format is absolute. Required for + Parquet files with a relative time column. Returns: The detected import config. @@ -538,14 +538,13 @@ def _apply_time_format(config: ImportConfig, time_format: TimeFormat) -> None: def _apply_relative_start_time(config: ImportConfig, relative_start_time: datetime) -> None: - """Set the relative start time on a detected Parquet config with a relative time format. + """Set the relative start time on a detected flat dataset Parquet config. - Warns and leaves the config unchanged for other formats, or when the time - format is absolute or unset. + Warns and leaves the config unchanged for other formats, including + single channel per row Parquet, or when the time format is absolute or + unset. """ - if not isinstance( - config, (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig) - ): + if not isinstance(config, ParquetFlatDatasetImportConfig): warnings.warn( "'relative_start_time' is ignored because it is not supported for " f"{type(config).__name__}.", diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index e01442ad16..39cbdc6adf 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -1314,9 +1314,9 @@ class DataImportAPI: ``TimeFormat.ABSOLUTE_UNIX_NANOSECONDS``. TDMS keeps its detected/default time handling. relative_start_time: Start time for a relative time format. - Parquet only; ignored with a warning for other formats or - when the time format is absolute. Required for Parquet files - with a relative time column. + Flat dataset Parquet only; ignored with a warning for other + formats or when the time format is absolute. Required for + Parquet files with a relative time column. Returns: The detected import config. @@ -1453,10 +1453,10 @@ class DataImportAPI: detected/default time handling. Only used when ``config`` is not provided. relative_start_time: Start time for a relative time format. - Parquet only; ignored with a warning for other formats or - when the time format is absolute. Required for Parquet files - with a relative time column. Only used when ``config`` is not - provided. + Flat dataset Parquet only; ignored with a warning for other + formats or when the time format is absolute. Required for + Parquet files with a relative time column. Only used when + ``config`` is not provided. run: ``Run`` object or run ID string to import into an existing run. Mutually exclusive with ``run_name``. run_name: Name for a new run. Defaults to the filename if From c54c883e9ffa68fa96f35e0b0e4e897f044f9dbc Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 12 Oct 2026 10:47:42 -0700 Subject: [PATCH 07/20] python(refactor): move infer_time_column into _internal.util --- .../lib/sift_client/_internal/util/parquet.py | 4 +-- .../sift_client/_internal/util/time_column.py | 29 +++++++++++++++++++ .../_tests/resources/test_data_imports.py | 12 ++++---- .../lib/sift_client/resources/data_imports.py | 28 ++---------------- 4 files changed, 40 insertions(+), 33 deletions(-) create mode 100644 python/lib/sift_client/_internal/util/time_column.py diff --git a/python/lib/sift_client/_internal/util/parquet.py b/python/lib/sift_client/_internal/util/parquet.py index 1238e95006..bc5ad819a4 100644 --- a/python/lib/sift_client/_internal/util/parquet.py +++ b/python/lib/sift_client/_internal/util/parquet.py @@ -18,7 +18,7 @@ from sift.common.type.v1.channel_config_pb2 import ChannelConfig as ChannelConfigProto from sift_client._internal.util.file import extract_parquet_footer -from sift_client.resources.data_imports import _infer_time_column +from sift_client._internal.util.time_column import infer_time_column from sift_client.sift_types.channel import ChannelDataType from sift_client.sift_types.data_import import ( ParquetDataColumn, @@ -300,7 +300,7 @@ def detect_flat_parquet_config( time_column = detected if not time_column.path: - inferred = _infer_time_column((dc.name, dc.data_type, dc.path) for dc in data_columns) + inferred = infer_time_column((dc.name, dc.data_type, dc.path) for dc in data_columns) if inferred: time_column = ParquetTimeColumn(path=inferred) data_columns = [dc for dc in data_columns if dc.path != time_column.path] diff --git a/python/lib/sift_client/_internal/util/time_column.py b/python/lib/sift_client/_internal/util/time_column.py new file mode 100644 index 0000000000..d0c7b8dace --- /dev/null +++ b/python/lib/sift_client/_internal/util/time_column.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sift_client.sift_types.channel import ChannelDataType + +if TYPE_CHECKING: + from collections.abc import Iterable + +_TIME_COLUMN_NAMES: frozenset[str] = frozenset({"ts", "timestamp", "time"}) +_TIME_COLUMN_TYPES: frozenset[ChannelDataType] = frozenset( + {ChannelDataType.INT_64, ChannelDataType.UINT_64} +) + + +def infer_time_column( + columns: Iterable[tuple[str, ChannelDataType, str]], +) -> str | None: + """Pick a likely time column when the server couldn't identify one. + + Returns the path of an INT64 or UINT64 column whose name + (case-insensitive) matches one of ``ts``, ``timestamp``, or ``time``. + Returns None otherwise. + """ + data_columns = sorted(columns, key=lambda c: c[0].lower()) + for name, data_type, path in data_columns: + if data_type in _TIME_COLUMN_TYPES and name.lower() in _TIME_COLUMN_NAMES: + return path + return None diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index cd2e9b4f04..77dd2a93fb 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -25,9 +25,9 @@ ParquetTimeColumn as ParquetTimeColumnProto, ) +from sift_client._internal.util.time_column import infer_time_column from sift_client.resources import DataImportAPI, DataImportAPIAsync from sift_client.resources.data_imports import ( - _infer_time_column, _parse_parquet_detect_response, _resolve_data_type_key, ) @@ -754,7 +754,7 @@ def test_unknown_extension_raises(self): class TestInferTimeColumn: def test_picks_canonical_skips_other_columns(self): - path = _infer_time_column( + path = infer_time_column( [ ("delta_time", ChannelDataType.INT_64, "delta_time"), ("voltage", ChannelDataType.DOUBLE, "voltage"), @@ -764,15 +764,15 @@ def test_picks_canonical_skips_other_columns(self): assert path == "timestamp" def test_accepts_uint64(self): - path = _infer_time_column([("time", ChannelDataType.UINT_64, "time")]) + path = infer_time_column([("time", ChannelDataType.UINT_64, "time")]) assert path == "time" def test_case_insensitive(self): - path = _infer_time_column([("TimeStamp", ChannelDataType.INT_64, "TimeStamp")]) + path = infer_time_column([("TimeStamp", ChannelDataType.INT_64, "TimeStamp")]) assert path == "TimeStamp" def test_multiple_candidates_sorted_alphabetically(self): - path = _infer_time_column( + path = infer_time_column( [ ("timestamp", ChannelDataType.INT_64, "timestamp"), ("time", ChannelDataType.INT_64, "time"), @@ -782,7 +782,7 @@ def test_multiple_candidates_sorted_alphabetically(self): assert path == "time" def test_returns_none_when_no_canonical_int_column(self): - path = _infer_time_column( + path = infer_time_column( [ ("timestamp", ChannelDataType.DOUBLE, "timestamp"), ("event_time", ChannelDataType.INT_64, "event_time"), diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index db73a4f162..431d5d7514 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -7,6 +7,7 @@ from sift_client._internal.low_level_wrappers.data_imports import DataImportsLowLevelClient from sift_client._internal.util.executor import run_sync_function from sift_client._internal.util.file import extract_parquet_footer, upload_file +from sift_client._internal.util.time_column import infer_time_column from sift_client._internal.util.util import caller_stacklevel from sift_client.errors import SiftIgnoredInputWarning from sift_client.resources._base import ResourceBase @@ -33,7 +34,6 @@ from sift_client.util import cel_utils as cel if TYPE_CHECKING: - from collections.abc import Iterable from datetime import datetime from sift_client.client import SiftClient @@ -612,28 +612,6 @@ def _parse_csv_detect_response(proto) -> CsvImportConfig: return csv_config -_TIME_COLUMN_NAMES: frozenset[str] = frozenset({"ts", "timestamp", "time"}) -_TIME_COLUMN_TYPES: frozenset[ChannelDataType] = frozenset( - {ChannelDataType.INT_64, ChannelDataType.UINT_64} -) - - -def _infer_time_column( - columns: Iterable[tuple[str, ChannelDataType, str]], -) -> str | None: - """Pick a likely time column when the server couldn't identify one. - - Returns the path of an INT64 or UINT64 column whose name - (case-insensitive) matches one of ``ts``, ``timestamp``, or ``time``. - Returns None otherwise. - """ - data_columns = sorted(columns, key=lambda c: c[0].lower()) - for name, data_type, path in data_columns: - if data_type in _TIME_COLUMN_TYPES and name.lower() in _TIME_COLUMN_NAMES: - return path - return None - - def _parse_parquet_detect_response( proto, filename: str, footer_offset: int, footer_length: int ) -> ParquetFlatDatasetImportConfig | ParquetSingleChannelPerRowImportConfig: @@ -644,7 +622,7 @@ def _parse_parquet_detect_response( ) time_path: str | None = parquet_config.time_column.path if not time_path: - time_path = _infer_time_column( + time_path = infer_time_column( (dc.name, dc.data_type, dc.path) for dc in parquet_config.data_columns ) if time_path: @@ -659,7 +637,7 @@ def _parse_parquet_detect_response( proto, footer_offset=footer_offset, footer_length=footer_length ) if not scpr_config.time_column.path: - time_path = _infer_time_column( + time_path = infer_time_column( (col.column_config.name, ChannelDataType(col.column_config.data_type), col.path) for col in proto.single_channel_per_row.columns ) From 0b773e142afed619abac7a8952d383309387a3a9 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 12 Oct 2026 10:47:45 -0700 Subject: [PATCH 08/20] python(refactor): make optional Parquet detection args keyword-only --- python/lib/sift_client/_internal/util/parquet.py | 12 +++++++----- python/lib/sift_client/resources/data_imports.py | 7 +++++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/python/lib/sift_client/_internal/util/parquet.py b/python/lib/sift_client/_internal/util/parquet.py index bc5ad819a4..83aa9cd8af 100644 --- a/python/lib/sift_client/_internal/util/parquet.py +++ b/python/lib/sift_client/_internal/util/parquet.py @@ -114,7 +114,9 @@ def _is_supported_type(t: pa.DataType) -> bool: return pa.types.is_struct(t) or _arrow_to_sift_type(t) is not None -def _add_fields(columns: list[ParquetDataColumn], field: pa.Field, parent_path: str = "") -> None: +def _add_fields( + columns: list[ParquetDataColumn], field: pa.Field, *, parent_path: str = "" +) -> None: """Append a column per leaf of ``field``, flattening structs. Nested paths are joined with ``PATH_SEPARATOR``; channel names use ``.``. @@ -124,7 +126,7 @@ def _add_fields(columns: list[ParquetDataColumn], field: pa.Field, parent_path: if pa.types.is_struct(field.type): for i in range(field.type.num_fields): - _add_fields(columns, field.type.field(i), full_path) + _add_fields(columns, field.type.field(i), parent_path=full_path) return name = full_path.replace(PATH_SEPARATOR, ".") @@ -135,7 +137,7 @@ def _add_fields(columns: list[ParquetDataColumn], field: pa.Field, parent_path: def _detect_time_column( - field: pa.Field, relative_start_time: datetime | None = None + field: pa.Field, *, relative_start_time: datetime | None = None ) -> ParquetTimeColumn | None: """Return a time column for ``field`` if it has a time type. @@ -248,7 +250,7 @@ def _embedded_flat_dataset_config( def detect_flat_parquet_config( - file_path: str | Path, relative_start_time: datetime | None = None + file_path: str | Path, *, relative_start_time: datetime | None = None ) -> ParquetFlatDatasetImportConfig: """Detect the flat dataset import configuration of a Parquet file. @@ -295,7 +297,7 @@ def detect_flat_parquet_config( # Select the first encountered time column, if it exists. if not time_column.path: - detected = _detect_time_column(field, relative_start_time) + detected = _detect_time_column(field, relative_start_time=relative_start_time) if detected is not None: time_column = detected diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index 431d5d7514..b2fb1f6679 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -412,7 +412,9 @@ async def detect_config( raise FileNotFoundError(f"File not found: {file_path}") data_type_key = _resolve_data_type_key(path.suffix.lower(), data_type) - config = await self._detect_config_for_type(path, data_type_key, relative_start_time) + config = await self._detect_config_for_type( + path, data_type_key, relative_start_time=relative_start_time + ) if time_format is not None: _apply_time_format(config, time_format) elif ( @@ -428,6 +430,7 @@ async def _detect_config_for_type( self, path: Path, data_type_key: DataTypeKey, + *, relative_start_time: datetime | None = None, ) -> ImportConfig: if data_type_key in ( @@ -479,7 +482,7 @@ async def _detect_config_for_type( "Install it via `pip install sift-stack-py[parquet]`." ) from e return await run_sync_function( - lambda: detect_flat_parquet_config(path, relative_start_time) + lambda: detect_flat_parquet_config(path, relative_start_time=relative_start_time) ) is_parquet = data_type_key in ( DataTypeKey.PARQUET_FLATDATASET, From a0cf8a4f1b23a7005da794669f25cada48ad04a7 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 12 Oct 2026 10:47:47 -0700 Subject: [PATCH 09/20] python(build): include parquet in the file-imports extra --- python/pyproject.toml | 3 ++- python/uv.lock | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 901d337cfb..f412d788ef 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -189,6 +189,7 @@ file-imports = [ 'mcap~=1.4', 'npTDMS~=1.9', 'polars~=1.8', + 'pyarrow>=17.0.0', 'pyulog~=1.2.2', "rosbags~=0.0 ; python_full_version >= '3.8.2'", ] @@ -278,7 +279,7 @@ dev = ["development"] sift-stream-bindings = ["sift-stream"] # combinations -file-imports = ["tdms", "rosbags", "hdf5", "ulog", "mcap"] +file-imports = ["tdms", "rosbags", "hdf5", "ulog", "mcap", "parquet"] all = ["openssl", "sift-stream", "file-imports", "data-review"] dev-all = ["development", "all", "build"] diff --git a/python/uv.lock b/python/uv.lock index 1543402ff2..c628989ba3 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -4669,6 +4669,9 @@ file-imports = [ { name = "polars", version = "1.8.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, { name = "polars", version = "1.36.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, { name = "polars", version = "1.40.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pyarrow", version = "17.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "pyarrow", version = "21.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "pyarrow", version = "24.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pyulog" }, { name = "rosbags", version = "0.9.23", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.8.2' and python_full_version < '3.10'" }, { name = "rosbags", version = "0.11.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, @@ -4791,6 +4794,7 @@ requires-dist = [ { name = "pyarrow", marker = "extra == 'data-review'", specifier = ">=17.0.0" }, { name = "pyarrow", marker = "extra == 'dev-all'", specifier = ">=17.0.0" }, { name = "pyarrow", marker = "extra == 'docs-build'", specifier = ">=17.0.0" }, + { name = "pyarrow", marker = "extra == 'file-imports'", specifier = ">=17.0.0" }, { name = "pyarrow", marker = "extra == 'parquet'", specifier = ">=17.0.0" }, { name = "pydantic", specifier = "~=2.10" }, { name = "pydantic-core", specifier = "~=2.3" }, From bca29900bb235ea50a6a875f76139c9db7787a87 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 12 Oct 2026 11:03:03 -0700 Subject: [PATCH 10/20] python(test): drop redundant single channel per row relative_start_time test --- .../_tests/_internal/test_parquet.py | 20 ------------------- 1 file changed, 20 deletions(-) diff --git a/python/lib/sift_client/_tests/_internal/test_parquet.py b/python/lib/sift_client/_tests/_internal/test_parquet.py index 0e29a7ee6b..2bf77a7cd6 100644 --- a/python/lib/sift_client/_tests/_internal/test_parquet.py +++ b/python/lib/sift_client/_tests/_internal/test_parquet.py @@ -20,9 +20,6 @@ CsvImportConfig, CsvTimeColumn, DataTypeKey, - ParquetSingleChannelConfig, - ParquetSingleChannelPerRowImportConfig, - ParquetTimeColumn, TimeFormat, ) @@ -620,20 +617,3 @@ def test_non_parquet_config_warns_and_ignores_start_time(self): _apply_relative_start_time(config, self.START) assert config.time_column.relative_start_time is None - - def test_single_channel_per_row_config_warns_and_ignores_start_time(self): - config = ParquetSingleChannelPerRowImportConfig( - asset_name="a", - time_column=ParquetTimeColumn(path="t", format=TimeFormat.ABSOLUTE_UNIX_NANOSECONDS), - single_channel=ParquetSingleChannelConfig( - data_path="v", name="v", data_type=ChannelDataType.DOUBLE - ), - ) - config.time_column.format = TimeFormat.RELATIVE_SECONDS - - with pytest.warns( - UserWarning, match="not supported for ParquetSingleChannelPerRowImportConfig" - ): - _apply_relative_start_time(config, self.START) - - assert config.time_column.relative_start_time is None From a6c19e3bc0cf9d329ff25d36a2921f21670ad0bb Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 12 Oct 2026 12:41:18 -0700 Subject: [PATCH 11/20] python(feat): support nanosecond precision for relative_start_time Import config fields accept a pd.Timestamp alongside datetime, and configs read from protos return a pd.Timestamp so nanoseconds survive a round trip. detect_config and import_from_path also accept an ISO 8601 string, parsed with pd.Timestamp. --- .../lib/sift_client/_internal/util/parquet.py | 8 ++- .../_tests/_internal/test_parquet.py | 48 +++++++++++++++++- .../_tests/resources/test_data_imports.py | 23 +++++++++ .../lib/sift_client/resources/data_imports.py | 13 ++++- .../resources/sync_stubs/__init__.pyi | 8 ++- .../lib/sift_client/sift_types/data_import.py | 50 +++++++++---------- 6 files changed, 117 insertions(+), 33 deletions(-) diff --git a/python/lib/sift_client/_internal/util/parquet.py b/python/lib/sift_client/_internal/util/parquet.py index 83aa9cd8af..b69dacac48 100644 --- a/python/lib/sift_client/_internal/util/parquet.py +++ b/python/lib/sift_client/_internal/util/parquet.py @@ -8,8 +8,8 @@ import json import warnings -from datetime import datetime, timezone from pathlib import Path +from typing import TYPE_CHECKING import pyarrow as pa import pyarrow.parquet as pq @@ -17,6 +17,7 @@ from google.protobuf.timestamp_pb2 import Timestamp as TimestampPb from sift.common.type.v1.channel_config_pb2 import ChannelConfig as ChannelConfigProto +from sift_client._internal.time import to_timestamp_nanos from sift_client._internal.util.file import extract_parquet_footer from sift_client._internal.util.time_column import infer_time_column from sift_client.sift_types.channel import ChannelDataType @@ -29,6 +30,9 @@ _enum_types_from_proto, ) +if TYPE_CHECKING: + from datetime import datetime + # Separator used in column paths to address nested struct fields. Channel # names replace it with "." since Sift forbids "|" in channel names. PATH_SEPARATOR = "|" @@ -194,7 +198,7 @@ def _parse_embedded_time_column(path: str, raw: bytes) -> ParquetTimeColumn: if raw_start: start = TimestampPb() start.FromJsonString(raw_start) - relative_start_time = start.ToDatetime(tzinfo=timezone.utc) + relative_start_time = to_timestamp_nanos(start) return ParquetTimeColumn(path=path, format=fmt, relative_start_time=relative_start_time) diff --git a/python/lib/sift_client/_tests/_internal/test_parquet.py b/python/lib/sift_client/_tests/_internal/test_parquet.py index 2bf77a7cd6..e0d96f850b 100644 --- a/python/lib/sift_client/_tests/_internal/test_parquet.py +++ b/python/lib/sift_client/_tests/_internal/test_parquet.py @@ -7,6 +7,7 @@ from unittest.mock import AsyncMock, MagicMock import numpy as np +import pandas as pd import pyarrow as pa import pyarrow.parquet as pq import pytest @@ -365,7 +366,7 @@ def test_uses_embedded_configs(self, create_parquet_file): _TIME_CONFIG, { "time_format": "TIME_FORMAT_RELATIVE_SECONDS", - "relative_start_time": "2024-01-01T00:00:00Z", + "relative_start_time": "2024-01-01T00:00:00.000000001Z", }, ), _embedded_field( @@ -395,7 +396,9 @@ def test_uses_embedded_configs(self, create_parquet_file): assert config.time_column.path == "time" assert config.time_column.format == TimeFormat.RELATIVE_SECONDS - assert config.time_column.relative_start_time == datetime(2024, 1, 1, tzinfo=timezone.utc) + assert config.time_column.relative_start_time == pd.Timestamp( + "2024-01-01T00:00:00.000000001Z" + ) assert _columns(config) == { "x": ("x.renamed", ChannelDataType.DOUBLE), "state": ("state", ChannelDataType.ENUM), @@ -593,6 +596,47 @@ async def test_relative_time_format_override_uses_start_time(self, create_parque assert config.time_column.format == TimeFormat.RELATIVE_SECONDS assert config.time_column.relative_start_time == self.START + @pytest.mark.asyncio + async def test_string_start_time_keeps_nanoseconds(self, create_parquet_file): + path = create_parquet_file(_with_time(pa.array([0, 1], pa.time64("us")))) + api = DataImportAPIAsync(MagicMock()) + + config = await api.detect_config( + path, + data_type=DataTypeKey.PARQUET_FLATDATASET, + relative_start_time="2024-01-01T00:00:00.000000001Z", + ) + + start = config.time_column.relative_start_time + assert start == pd.Timestamp("2024-01-01T00:00:00.000000001Z") + assert config.time_column._to_proto().relative_start_time.nanos == 1 + + @pytest.mark.asyncio + async def test_string_start_time_with_time_format_override(self, create_parquet_file): + table = pa.table({"timestamp": pa.array([0, 1], pa.int64()), "v": pa.array([1.0, 2.0])}) + api = DataImportAPIAsync(MagicMock()) + + config = await api.detect_config( + create_parquet_file(table), + data_type=DataTypeKey.PARQUET_FLATDATASET, + time_format=TimeFormat.RELATIVE_SECONDS, + relative_start_time="2024-01-01T00:00:00.000000001Z", + ) + + assert config.time_column._to_proto().relative_start_time.nanos == 1 + + @pytest.mark.asyncio + async def test_invalid_string_start_time_raises(self, create_parquet_file): + path = create_parquet_file(_with_time(pa.array([0, 1], pa.time64("us")))) + api = DataImportAPIAsync(MagicMock()) + + with pytest.raises(ValueError, match="not a timestamp"): + await api.detect_config( + path, + data_type=DataTypeKey.PARQUET_FLATDATASET, + relative_start_time="not a timestamp", + ) + @pytest.mark.asyncio async def test_absolute_column_warns_and_ignores_start_time(self, create_parquet_file): path = create_parquet_file(_with_time(pa.array([0, 1], pa.timestamp("ns")))) diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 77dd2a93fb..29a527f4c2 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, cast from unittest.mock import AsyncMock, MagicMock, patch +import pandas as pd import pytest from sift.common.type.v1.channel_config_pb2 import ChannelConfig as ChannelConfigProto from sift.data_imports.v2.data_imports_pb2 import ( @@ -396,6 +397,17 @@ def test_relative_start_time_round_trips(self): restored = UlogImportConfig._from_proto(proto) assert restored.relative_start_time == config.relative_start_time + def test_relative_start_time_round_trips_nanoseconds(self): + config = UlogImportConfig( + asset_name="a", + relative_start_time=pd.Timestamp("2026-01-01T00:00:00.000000001Z"), + ) + proto = config._to_proto() + assert proto.relative_start_time.nanos == 1 + restored = UlogImportConfig._from_proto(proto) + assert isinstance(restored.relative_start_time, pd.Timestamp) + assert restored.relative_start_time == config.relative_start_time + def test_from_proto_round_trip(self): config = self._config() restored = UlogImportConfig._from_proto(config._to_proto()) @@ -540,6 +552,17 @@ def test_relative_start_time_round_trips(self): restored = McapImportConfig._from_proto(proto) assert restored.relative_start_time == config.relative_start_time + def test_relative_start_time_round_trips_nanoseconds(self): + config = McapImportConfig( + asset_name="a", + relative_start_time=pd.Timestamp("2026-01-01T00:00:00.000000001Z"), + ) + proto = config._to_proto() + assert proto.relative_start_time.nanos == 1 + restored = McapImportConfig._from_proto(proto) + assert isinstance(restored.relative_start_time, pd.Timestamp) + assert restored.relative_start_time == config.relative_start_time + def test_from_proto_round_trip(self): config = self._config() restored = McapImportConfig._from_proto(config._to_proto()) diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index b2fb1f6679..e7f4602c3e 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING from sift_client._internal.low_level_wrappers.data_imports import DataImportsLowLevelClient +from sift_client._internal.time import to_timestamp_nanos from sift_client._internal.util.executor import run_sync_function from sift_client._internal.util.file import extract_parquet_footer, upload_file from sift_client._internal.util.time_column import infer_time_column @@ -36,6 +37,8 @@ if TYPE_CHECKING: from datetime import datetime + import pandas as pd + from sift_client.client import SiftClient from sift_client.sift_types.data_import import DataImportStatus from sift_client.sift_types.job import Job @@ -63,7 +66,7 @@ async def import_from_path( config: ImportConfig | None = None, data_type: DataTypeKey | None = None, time_format: TimeFormat | None = None, - relative_start_time: datetime | None = None, + relative_start_time: pd.Timestamp | datetime | str | None = None, run: Run | str | None = None, run_name: str | None = None, show_progress: bool | None = None, @@ -131,6 +134,8 @@ async def import_from_path( detected/default time handling. Only used when ``config`` is not provided. relative_start_time: Start time for a relative time format. + A string is parsed with ``pd.Timestamp``; a string or + ``pd.Timestamp`` keeps nanosecond precision. Flat dataset Parquet only; ignored with a warning for other formats or when the time format is absolute. Required for Parquet files with a relative time column. Only used when @@ -334,7 +339,7 @@ async def detect_config( data_type: DataTypeKey | None = None, time_format: TimeFormat | None = None, *, - relative_start_time: datetime | None = None, + relative_start_time: pd.Timestamp | datetime | str | None = None, ) -> ImportConfig: """Auto-detect import configuration from a file. @@ -394,6 +399,8 @@ async def detect_config( ``TimeFormat.ABSOLUTE_UNIX_NANOSECONDS``. TDMS keeps its detected/default time handling. relative_start_time: Start time for a relative time format. + A string is parsed with ``pd.Timestamp``; a string or + ``pd.Timestamp`` keeps nanosecond precision. Flat dataset Parquet only; ignored with a warning for other formats or when the time format is absolute. Required for Parquet files with a relative time column. @@ -412,6 +419,8 @@ async def detect_config( raise FileNotFoundError(f"File not found: {file_path}") data_type_key = _resolve_data_type_key(path.suffix.lower(), data_type) + if isinstance(relative_start_time, str): + relative_start_time = to_timestamp_nanos(relative_start_time) config = await self._detect_config_for_type( path, data_type_key, relative_start_time=relative_start_time ) diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index 39cbdc6adf..e13c932c14 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -1254,7 +1254,7 @@ class DataImportAPI: data_type: DataTypeKey | None = None, time_format: TimeFormat | None = None, *, - relative_start_time: datetime | None = None, + relative_start_time: pd.Timestamp | datetime | str | None = None, ) -> ImportConfig: """Auto-detect import configuration from a file. @@ -1314,6 +1314,8 @@ class DataImportAPI: ``TimeFormat.ABSOLUTE_UNIX_NANOSECONDS``. TDMS keeps its detected/default time handling. relative_start_time: Start time for a relative time format. + A string is parsed with ``pd.Timestamp``; a string or + ``pd.Timestamp`` keeps nanosecond precision. Flat dataset Parquet only; ignored with a warning for other formats or when the time format is absolute. Required for Parquet files with a relative time column. @@ -1385,7 +1387,7 @@ class DataImportAPI: config: ImportConfig | None = None, data_type: DataTypeKey | None = None, time_format: TimeFormat | None = None, - relative_start_time: datetime | None = None, + relative_start_time: pd.Timestamp | datetime | str | None = None, run: Run | str | None = None, run_name: str | None = None, show_progress: bool | None = None, @@ -1453,6 +1455,8 @@ class DataImportAPI: detected/default time handling. Only used when ``config`` is not provided. relative_start_time: Start time for a relative time format. + A string is parsed with ``pd.Timestamp``; a string or + ``pd.Timestamp`` keeps nanosecond precision. Flat dataset Parquet only; ignored with a warning for other formats or when the time format is absolute. Required for Parquet files with a relative time column. Only used when diff --git a/python/lib/sift_client/sift_types/data_import.py b/python/lib/sift_client/sift_types/data_import.py index a01fcffe7b..513503cf55 100644 --- a/python/lib/sift_client/sift_types/data_import.py +++ b/python/lib/sift_client/sift_types/data_import.py @@ -5,7 +5,8 @@ from enum import Enum from typing import TYPE_CHECKING, Union -from pydantic import BaseModel, model_validator +import pandas as pd # noqa: TC002 +from pydantic import BaseModel, ConfigDict, model_validator from sift.common.type.v1.channel_config_pb2 import ChannelConfig as ChannelConfigProto from sift.common.type.v1.channel_enum_type_pb2 import ChannelEnumType as ChannelEnumTypeProto from sift.data_imports.v2.data_imports_pb2 import ( @@ -65,7 +66,7 @@ from sift.data_imports.v2.data_imports_pb2 import UlogConfig as UlogConfigProto from sift.data_imports.v2.data_imports_pb2 import UlogDataConfig as UlogDataConfigProto -from sift_client._internal.time import to_timestamp_pb +from sift_client._internal.time import to_timestamp_nanos, to_timestamp_pb from sift_client.sift_types._base import BaseType from sift_client.sift_types.channel import ChannelDataType @@ -131,10 +132,14 @@ class TimeColumnBase(BaseModel, ABC): Attributes: format: The time format used in this column. relative_start_time: Required when using a relative time format. + Pass a ``pd.Timestamp`` for nanosecond precision. """ + # pd.Timestamp has no pydantic schema, so it is validated with isinstance. + model_config = ConfigDict(arbitrary_types_allowed=True) + format: TimeFormat - relative_start_time: datetime | None = None + relative_start_time: pd.Timestamp | datetime | None = None @model_validator(mode="after") def _check_relative_start_time(self) -> TimeColumnBase: @@ -210,6 +215,10 @@ class ImportConfigBase(BaseModel, ABC): run_id: ID of an existing run to append data to. """ + # pd.Timestamp (relative_start_time) has no pydantic schema, so it is + # validated with isinstance. + model_config = ConfigDict(arbitrary_types_allowed=True) + asset_name: str run_name: str | None = None run_id: str | None = None @@ -286,11 +295,7 @@ def _from_proto(cls, proto: CsvConfigProto) -> CsvImportConfig: """Create from a proto CsvConfig (e.g. from DetectConfig response).""" relative_start_time = None if proto.time_column.HasField("relative_start_time"): - from datetime import timezone - - relative_start_time = proto.time_column.relative_start_time.ToDatetime( - tzinfo=timezone.utc - ) + relative_start_time = to_timestamp_nanos(proto.time_column.relative_start_time) time_column = CsvTimeColumn( column=proto.time_column.column_number, format=TimeFormat(proto.time_column.format), @@ -362,9 +367,7 @@ def _to_proto(self) -> ParquetTimeColumnProto: def _from_proto(cls, proto: ParquetTimeColumnProto) -> ParquetTimeColumn: relative_start_time = None if proto.HasField("relative_start_time"): - from datetime import timezone - - relative_start_time = proto.relative_start_time.ToDatetime(tzinfo=timezone.utc) + relative_start_time = to_timestamp_nanos(proto.relative_start_time) fmt = TimeFormat(proto.format) if proto.format else None return cls( @@ -675,6 +678,7 @@ class TdmsImportConfig(ImportConfigBase): fallback_method: How to handle channels with missing timing information. time_format: Time format for time channels not using the TDMS timestamp type. relative_start_time: Relative start time for channels using a non-standard time channel. + Pass a ``pd.Timestamp`` for nanosecond precision. import_file_properties: If true, imports TDMS file properties as run metadata. """ @@ -682,7 +686,7 @@ class TdmsImportConfig(ImportConfigBase): data: list[TdmsDataColumn] = [] fallback_method: TdmsFallbackMethod = TdmsFallbackMethod.FAIL_ON_ERROR time_format: TimeFormat | None = None - relative_start_time: datetime | None = None + relative_start_time: pd.Timestamp | datetime | None = None import_file_properties: bool = False def __getitem__(self, name: str) -> TdmsDataColumn: @@ -732,9 +736,7 @@ def _from_proto(cls, proto: TDMSConfigProto) -> TdmsImportConfig: relative_start_time = None if proto.HasField("relative_start_time"): - from datetime import timezone - - relative_start_time = proto.relative_start_time.ToDatetime(tzinfo=timezone.utc) + relative_start_time = to_timestamp_nanos(proto.relative_start_time) data = [] for d in proto.data: @@ -812,11 +814,12 @@ class Hdf5ImportConfig(ImportConfigBase): so the caller must set this before importing. Importing without it raises ``ValueError``. relative_start_time: Required when using a relative time format. + Pass a ``pd.Timestamp`` for nanosecond precision. """ data: list[Hdf5DataColumn] time_format: TimeFormat | None = None - relative_start_time: datetime | None = None + relative_start_time: pd.Timestamp | datetime | None = None def __getitem__(self, name: str) -> Hdf5DataColumn: """Look up a data column by channel name. @@ -941,6 +944,7 @@ class UlogImportConfig(ImportConfigBase): are relative to boot time. When set, this anchors the timeline and takes precedence over the log's GPS fix; otherwise the GPS fix anchors the timeline. The import fails if neither is available. + Pass a ``pd.Timestamp`` for nanosecond precision. info_keys: Info (``I``/``M``) message keys to import as run metadata, stored as ``info.``. Requires ``run_name`` or ``run_id``. param_keys: Parameter (``P``) names to import as run metadata, stored @@ -950,7 +954,7 @@ class UlogImportConfig(ImportConfigBase): """ data: list[UlogDataColumn] = [] - relative_start_time: datetime | None = None + relative_start_time: pd.Timestamp | datetime | None = None info_keys: list[str] = [] param_keys: list[str] = [] parse_error_policy: UlogParseErrorPolicy = UlogParseErrorPolicy.FAIL_ON_ERROR @@ -994,9 +998,7 @@ def _from_proto(cls, proto: UlogConfigProto) -> UlogImportConfig: """Create from a proto UlogConfig (e.g. from a GetDataImport response).""" relative_start_time = None if proto.HasField("relative_start_time"): - from datetime import timezone - - relative_start_time = proto.relative_start_time.ToDatetime(tzinfo=timezone.utc) + relative_start_time = to_timestamp_nanos(proto.relative_start_time) parse_error_policy = UlogParseErrorPolicy.FAIL_ON_ERROR if proto.parse_error_policy == ULOG_PARSE_ERROR_POLICY_IGNORE_ERROR: @@ -1108,7 +1110,7 @@ class McapImportConfig(ImportConfigBase): channels. relative_start_time: Log-start UTC, only for logs on a non-Unix epoch. When set, ``log_time`` is reinterpreted as elapsed nanoseconds from - this start. + this start. Pass a ``pd.Timestamp`` for nanosecond precision. metadata_records: Metadata records to import as run metadata. Every key of each named record is stored as ``.``. Empty imports none. @@ -1121,7 +1123,7 @@ class McapImportConfig(ImportConfigBase): """ data: list[McapDataColumn] = [] - relative_start_time: datetime | None = None + relative_start_time: pd.Timestamp | datetime | None = None metadata_records: list[str] = [] parse_error_policy: McapParseErrorPolicy = McapParseErrorPolicy.FAIL_ON_ERROR complex_types_import_mode: McapComplexTypesImportMode = McapComplexTypesImportMode.BOTH @@ -1203,9 +1205,7 @@ def _from_proto(cls, proto: McapConfigProto) -> McapImportConfig: """Create from a proto McapConfig (e.g. from a GetDataImport response).""" relative_start_time = None if proto.HasField("relative_start_time"): - from datetime import timezone - - relative_start_time = proto.relative_start_time.ToDatetime(tzinfo=timezone.utc) + relative_start_time = to_timestamp_nanos(proto.relative_start_time) parse_error_policy = McapParseErrorPolicy.FAIL_ON_ERROR if proto.parse_error_policy == MCAP_PARSE_ERROR_POLICY_IGNORE_ERROR: From 11c7d84bf6919b7b7e9c53341faca58abf2fb58a Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 12 Oct 2026 12:52:09 -0700 Subject: [PATCH 12/20] python(fix): fall back when the time format is unspecified TimeFormat gains UNSPECIFIED, matching the proto's TIME_FORMAT_UNSPECIFIED. A CSV DetectConfig response without a detected format no longer raises; detect_config applies the caller's time_format or ABSOLUTE_UNIX_NANOSECONDS instead. Importing with an unspecified format raises the existing "format must be set" errors. --- .../_tests/resources/test_data_imports.py | 67 +++++++++++++++++++ .../lib/sift_client/resources/data_imports.py | 12 +--- .../lib/sift_client/sift_types/data_import.py | 11 ++- 3 files changed, 79 insertions(+), 11 deletions(-) diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 29a527f4c2..59d4d95f00 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -13,11 +13,15 @@ DATA_IMPORT_STATUS_FAILED, DATA_IMPORT_STATUS_SUCCEEDED, DATA_IMPORT_STATUS_UNSPECIFIED, + TIME_FORMAT_UNSPECIFIED, + CsvConfig, + DetectConfigResponse, ParquetColumn, ParquetConfig, ParquetFlatDatasetConfig, ParquetSingleChannelPerRowConfig, ) +from sift.data_imports.v2.data_imports_pb2 import CsvTimeColumn as CsvTimeColumnProto from sift.data_imports.v2.data_imports_pb2 import DataImport as DataImportProto from sift.data_imports.v2.data_imports_pb2 import ( ParquetDataColumn as ParquetDataColumnProto, @@ -68,6 +72,9 @@ from sift.data_imports.v2.data_imports_pb2 import ( DataImportStatus as DataImportStatusProto, ) + from sift.data_imports.v2.data_imports_pb2 import ( + TimeFormat as TimeFormatProto, + ) @pytest.mark.integration @@ -902,6 +909,66 @@ def test_scpr_infers_int64_time_column(self): assert config.time_column.path == "timestamp" +def _csv_detect_response(time_format: TimeFormatProto.ValueType) -> DetectConfigResponse: + return DetectConfigResponse( + csv_config=CsvConfig( + first_data_row=2, + time_column=CsvTimeColumnProto(column_number=1, format=time_format), + data_columns={ + 2: ChannelConfigProto(name="voltage", data_type=ChannelDataType.DOUBLE.value) + }, + ) + ) + + +class TestUnspecifiedTimeFormat: + def test_maps_from_proto_zero(self): + assert TimeFormat(TIME_FORMAT_UNSPECIFIED) == TimeFormat.UNSPECIFIED + + @pytest.mark.asyncio + async def test_csv_detect_falls_back_to_unix_nanoseconds(self, tmp_path): + path = tmp_path / "data.csv" + path.write_text("time,voltage\n2026-01-01,1.0\n") + api = _data_imports_api(MagicMock()) + api._low_level_client.detect_config = AsyncMock( + return_value=_csv_detect_response(TIME_FORMAT_UNSPECIFIED) + ) + + config = await api.detect_config(path) + + assert isinstance(config, CsvImportConfig) + assert config.time_column.format == TimeFormat.ABSOLUTE_UNIX_NANOSECONDS + assert [dc.name for dc in config.data_columns] == ["voltage"] + + @pytest.mark.asyncio + async def test_csv_detect_uses_caller_time_format(self, tmp_path): + path = tmp_path / "data.csv" + path.write_text("time,voltage\n2026-01-01,1.0\n") + api = _data_imports_api(MagicMock()) + api._low_level_client.detect_config = AsyncMock( + return_value=_csv_detect_response(TIME_FORMAT_UNSPECIFIED) + ) + + config = await api.detect_config(path, time_format=TimeFormat.ABSOLUTE_RFC3339) + + assert config.time_column.format == TimeFormat.ABSOLUTE_RFC3339 + + def test_csv_to_proto_rejects_unspecified(self): + col = CsvTimeColumn(column=1, format=TimeFormat.UNSPECIFIED) + with pytest.raises(ValueError, match="CsvTimeColumn.format must be set"): + col._to_proto() + + def test_parquet_to_proto_rejects_unspecified(self): + col = ParquetTimeColumn(path="timestamp", format=TimeFormat.UNSPECIFIED) + with pytest.raises(ValueError, match="ParquetTimeColumn.format must be set"): + col._to_proto() + + def test_hdf5_to_proto_rejects_unspecified(self): + config = Hdf5ImportConfig(asset_name="a", data=[], time_format=TimeFormat.UNSPECIFIED) + with pytest.raises(ValueError, match="time_format is required"): + config._to_proto() + + def _data_import_proto( data_import_id: str = "data-import-1", status: DataImportStatusProto.ValueType = DATA_IMPORT_STATUS_SUCCEEDED, diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index e7f4602c3e..9e8a9796d4 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -28,7 +28,6 @@ ParquetTimeColumn, TdmsImportConfig, TimeFormat, - TimeFormatProto, UlogImportConfig, ) from sift_client.sift_types.run import Run @@ -426,10 +425,9 @@ async def detect_config( ) if time_format is not None: _apply_time_format(config, time_format) - elif ( - not isinstance(config, (TdmsImportConfig, UlogImportConfig, McapImportConfig)) - and _get_time_format(config) is None - ): + elif not isinstance( + config, (TdmsImportConfig, UlogImportConfig, McapImportConfig) + ) and _get_time_format(config) in (None, TimeFormat.UNSPECIFIED): _apply_time_format(config, TimeFormat.ABSOLUTE_UNIX_NANOSECONDS) if relative_start_time is not None: _apply_relative_start_time(config, relative_start_time) @@ -614,10 +612,6 @@ def _resolve_data_type_key(ext: str, data_type: DataTypeKey | None) -> DataTypeK def _parse_csv_detect_response(proto) -> CsvImportConfig: """Parse a CSV DetectConfig response into a config.""" - if proto.time_column.format == TimeFormatProto.TIME_FORMAT_UNSPECIFIED: - raise ValueError( - "timestamp format is not supported. Please specify the time format explicitly." - ) csv_config = CsvImportConfig._from_proto(proto) time_col = csv_config.time_column.column csv_config.data_columns = [dc for dc in csv_config.data_columns if dc.column != time_col] diff --git a/python/lib/sift_client/sift_types/data_import.py b/python/lib/sift_client/sift_types/data_import.py index 513503cf55..1d4034339c 100644 --- a/python/lib/sift_client/sift_types/data_import.py +++ b/python/lib/sift_client/sift_types/data_import.py @@ -77,6 +77,7 @@ class TimeFormat(Enum): """Supported time formats for data import columns.""" + UNSPECIFIED = TimeFormatProto.TIME_FORMAT_UNSPECIFIED RELATIVE_NANOSECONDS = TimeFormatProto.TIME_FORMAT_RELATIVE_NANOSECONDS RELATIVE_MICROSECONDS = TimeFormatProto.TIME_FORMAT_RELATIVE_MICROSECONDS RELATIVE_MILLISECONDS = TimeFormatProto.TIME_FORMAT_RELATIVE_MILLISECONDS @@ -234,6 +235,12 @@ class CsvTimeColumn(TimeColumnBase): column: int def _to_proto(self) -> CsvTimeColumnProto: + if self.format == TimeFormat.UNSPECIFIED: + raise ValueError( + "CsvTimeColumn.format must be set before importing. " + "Pass time_format to detect_config/import_from_path, or set " + "config.time_column.format explicitly." + ) proto = CsvTimeColumnProto( column_number=self.column, format=self.format.value, @@ -349,7 +356,7 @@ class ParquetTimeColumn(TimeColumnBase): def _to_proto(self) -> ParquetTimeColumnProto: if not self.path: raise ValueError("ParquetTimeColumn.path must be set before importing.") - if self.format is None: + if self.format is None or self.format == TimeFormat.UNSPECIFIED: raise ValueError( "ParquetTimeColumn.format must be set before importing. " "Pass time_format to detect_config/import_from_path, or set " @@ -846,7 +853,7 @@ def _check_relative_start_time(self) -> Hdf5ImportConfig: return self def _to_proto(self) -> Hdf5ConfigProto: - if self.time_format is None: + if self.time_format is None or self.time_format == TimeFormat.UNSPECIFIED: raise ValueError( "time_format is required to import HDF5 files. Set " "config.time_format explicitly (e.g. TimeFormat.ABSOLUTE_UNIX_NANOSECONDS " From 9dbd2582e1aa9c315b8458a9a4fd2f42376d42b9 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 12 Oct 2026 13:02:42 -0700 Subject: [PATCH 13/20] python(refactor): added is_relative property to TimeFormat for easier classification --- python/lib/sift_client/resources/data_imports.py | 2 +- python/lib/sift_client/sift_types/data_import.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index 9e8a9796d4..b61708154f 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -563,7 +563,7 @@ def _apply_relative_start_time(config: ImportConfig, relative_start_time: dateti ) return time_format = config.time_column.format - if time_format is None or not time_format.name.startswith("RELATIVE_"): + if time_format is None or not time_format.is_relative: warnings.warn( "'relative_start_time' is ignored because the time format " f"({time_format.name if time_format else 'unset'}) is not relative.", diff --git a/python/lib/sift_client/sift_types/data_import.py b/python/lib/sift_client/sift_types/data_import.py index 1d4034339c..ba94356435 100644 --- a/python/lib/sift_client/sift_types/data_import.py +++ b/python/lib/sift_client/sift_types/data_import.py @@ -91,6 +91,11 @@ class TimeFormat(Enum): ABSOLUTE_UNIX_MICROSECONDS = TimeFormatProto.TIME_FORMAT_ABSOLUTE_UNIX_MICROSECONDS ABSOLUTE_UNIX_NANOSECONDS = TimeFormatProto.TIME_FORMAT_ABSOLUTE_UNIX_NANOSECONDS + @property + def is_relative(self) -> bool: + """Checks if the time format is relative based on name prefix.""" + return self.name.startswith("RELATIVE_") + class DataTypeKey(Enum): """Supported file types and layouts for data import detection.""" @@ -144,11 +149,7 @@ class TimeColumnBase(BaseModel, ABC): @model_validator(mode="after") def _check_relative_start_time(self) -> TimeColumnBase: - if ( - self.format is not None - and self.format.name.startswith("RELATIVE_") - and self.relative_start_time is None - ): + if self.format is not None and self.format.is_relative and self.relative_start_time is None: raise ValueError( f"'relative_start_time' is required when using a relative time format ({self.format.name})." ) @@ -844,7 +845,7 @@ def __getitem__(self, name: str) -> Hdf5DataColumn: def _check_relative_start_time(self) -> Hdf5ImportConfig: if ( self.time_format is not None - and self.time_format.name.startswith("RELATIVE_") + and self.time_format.is_relative and self.relative_start_time is None ): raise ValueError( From 906b4ad2d49d888b0728ac157c1692b1eac15c17 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 12 Oct 2026 13:27:43 -0700 Subject: [PATCH 14/20] python(refactor): represent an undetected time format as None Drop UNSPECIFIED from the public TimeFormat enum. CsvImportConfig maps the proto's TIME_FORMAT_UNSPECIFIED to None, matching Parquet, TDMS, and HDF5. TimeColumnBase.format now defaults to None, so CSV and Parquet time columns share the same rule: optional at construction, required at import. --- .../_tests/resources/test_data_imports.py | 19 +++++++------ .../lib/sift_client/resources/data_imports.py | 7 ++--- .../lib/sift_client/sift_types/data_import.py | 27 ++++++++++--------- 3 files changed, 29 insertions(+), 24 deletions(-) diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 59d4d95f00..4b427e79dc 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -922,8 +922,11 @@ def _csv_detect_response(time_format: TimeFormatProto.ValueType) -> DetectConfig class TestUnspecifiedTimeFormat: - def test_maps_from_proto_zero(self): - assert TimeFormat(TIME_FORMAT_UNSPECIFIED) == TimeFormat.UNSPECIFIED + def test_csv_from_proto_maps_unspecified_to_none(self): + config = CsvImportConfig._from_proto( + _csv_detect_response(TIME_FORMAT_UNSPECIFIED).csv_config + ) + assert config.time_column.format is None @pytest.mark.asyncio async def test_csv_detect_falls_back_to_unix_nanoseconds(self, tmp_path): @@ -953,18 +956,18 @@ async def test_csv_detect_uses_caller_time_format(self, tmp_path): assert config.time_column.format == TimeFormat.ABSOLUTE_RFC3339 - def test_csv_to_proto_rejects_unspecified(self): - col = CsvTimeColumn(column=1, format=TimeFormat.UNSPECIFIED) + def test_csv_to_proto_rejects_missing_format(self): + col = CsvTimeColumn(column=1, format=None) with pytest.raises(ValueError, match="CsvTimeColumn.format must be set"): col._to_proto() - def test_parquet_to_proto_rejects_unspecified(self): - col = ParquetTimeColumn(path="timestamp", format=TimeFormat.UNSPECIFIED) + def test_parquet_to_proto_rejects_missing_format(self): + col = ParquetTimeColumn(path="timestamp", format=None) with pytest.raises(ValueError, match="ParquetTimeColumn.format must be set"): col._to_proto() - def test_hdf5_to_proto_rejects_unspecified(self): - config = Hdf5ImportConfig(asset_name="a", data=[], time_format=TimeFormat.UNSPECIFIED) + def test_hdf5_to_proto_rejects_missing_format(self): + config = Hdf5ImportConfig(asset_name="a", data=[], time_format=None) with pytest.raises(ValueError, match="time_format is required"): config._to_proto() diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index b61708154f..c090351a55 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -425,9 +425,10 @@ async def detect_config( ) if time_format is not None: _apply_time_format(config, time_format) - elif not isinstance( - config, (TdmsImportConfig, UlogImportConfig, McapImportConfig) - ) and _get_time_format(config) in (None, TimeFormat.UNSPECIFIED): + elif ( + not isinstance(config, (TdmsImportConfig, UlogImportConfig, McapImportConfig)) + and _get_time_format(config) is None + ): _apply_time_format(config, TimeFormat.ABSOLUTE_UNIX_NANOSECONDS) if relative_start_time is not None: _apply_relative_start_time(config, relative_start_time) diff --git a/python/lib/sift_client/sift_types/data_import.py b/python/lib/sift_client/sift_types/data_import.py index ba94356435..df2eae9580 100644 --- a/python/lib/sift_client/sift_types/data_import.py +++ b/python/lib/sift_client/sift_types/data_import.py @@ -77,7 +77,6 @@ class TimeFormat(Enum): """Supported time formats for data import columns.""" - UNSPECIFIED = TimeFormatProto.TIME_FORMAT_UNSPECIFIED RELATIVE_NANOSECONDS = TimeFormatProto.TIME_FORMAT_RELATIVE_NANOSECONDS RELATIVE_MICROSECONDS = TimeFormatProto.TIME_FORMAT_RELATIVE_MICROSECONDS RELATIVE_MILLISECONDS = TimeFormatProto.TIME_FORMAT_RELATIVE_MILLISECONDS @@ -136,7 +135,11 @@ class TimeColumnBase(BaseModel, ABC): """Base class for time column configurations. Attributes: - format: The time format used in this column. + format: The time format used in this column. Optional at construction + so that ``detect_config`` can apply the standard precedence chain + (caller-passed ``time_format`` > server-detected format > + ``ABSOLUTE_UNIX_NANOSECONDS`` fallback). Importing without a + format set raises ``ValueError``. relative_start_time: Required when using a relative time format. Pass a ``pd.Timestamp`` for nanosecond precision. """ @@ -144,7 +147,7 @@ class TimeColumnBase(BaseModel, ABC): # pd.Timestamp has no pydantic schema, so it is validated with isinstance. model_config = ConfigDict(arbitrary_types_allowed=True) - format: TimeFormat + format: TimeFormat | None = None relative_start_time: pd.Timestamp | datetime | None = None @model_validator(mode="after") @@ -236,7 +239,7 @@ class CsvTimeColumn(TimeColumnBase): column: int def _to_proto(self) -> CsvTimeColumnProto: - if self.format == TimeFormat.UNSPECIFIED: + if self.format is None: raise ValueError( "CsvTimeColumn.format must be set before importing. " "Pass time_format to detect_config/import_from_path, or set " @@ -304,9 +307,13 @@ def _from_proto(cls, proto: CsvConfigProto) -> CsvImportConfig: relative_start_time = None if proto.time_column.HasField("relative_start_time"): relative_start_time = to_timestamp_nanos(proto.time_column.relative_start_time) + try: + time_format = TimeFormat(proto.time_column.format) + except ValueError: + time_format = None time_column = CsvTimeColumn( column=proto.time_column.column_number, - format=TimeFormat(proto.time_column.format), + format=time_format, relative_start_time=relative_start_time, ) data_columns = [ @@ -344,20 +351,14 @@ class ParquetTimeColumn(TimeColumnBase): Attributes: path: The column path in the Parquet schema (e.g. ``"timestamp"``). - format: The time format. Optional at construction so that - ``detect_config`` / ``import_from_path`` can apply the standard - precedence chain (caller-passed ``time_format`` > server-detected - format > ``ABSOLUTE_UNIX_NANOSECONDS`` fallback). Importing - without a format set raises ``ValueError``. """ path: str - format: TimeFormat | None = None # type: ignore[assignment] def _to_proto(self) -> ParquetTimeColumnProto: if not self.path: raise ValueError("ParquetTimeColumn.path must be set before importing.") - if self.format is None or self.format == TimeFormat.UNSPECIFIED: + if self.format is None: raise ValueError( "ParquetTimeColumn.format must be set before importing. " "Pass time_format to detect_config/import_from_path, or set " @@ -854,7 +855,7 @@ def _check_relative_start_time(self) -> Hdf5ImportConfig: return self def _to_proto(self) -> Hdf5ConfigProto: - if self.time_format is None or self.time_format == TimeFormat.UNSPECIFIED: + if self.time_format is None: raise ValueError( "time_format is required to import HDF5 files. Set " "config.time_format explicitly (e.g. TimeFormat.ABSOLUTE_UNIX_NANOSECONDS " From f731566eefce24b1b90fd1fae47e0c3b98815619 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 12 Oct 2026 15:33:51 -0700 Subject: [PATCH 15/20] python(feat): support relative_start_time for all import formats --- .../_internal/util/data_imports.py | 80 +++++++ .../_tests/_internal/test_parquet.py | 16 -- .../_tests/resources/test_data_imports.py | 225 ++++++++++++++++++ .../lib/sift_client/resources/data_imports.py | 74 +++--- .../resources/sync_stubs/__init__.pyi | 14 +- 5 files changed, 344 insertions(+), 65 deletions(-) create mode 100644 python/lib/sift_client/_internal/util/data_imports.py diff --git a/python/lib/sift_client/_internal/util/data_imports.py b/python/lib/sift_client/_internal/util/data_imports.py new file mode 100644 index 0000000000..772bd07b66 --- /dev/null +++ b/python/lib/sift_client/_internal/util/data_imports.py @@ -0,0 +1,80 @@ +"""Helpers for applying ``relative_start_time`` to detected import configs.""" + +from __future__ import annotations + +import warnings +from typing import TYPE_CHECKING + +from sift_client._internal.time import to_timestamp_pb +from sift_client._internal.util.util import caller_stacklevel +from sift_client.errors import SiftIgnoredInputWarning +from sift_client.sift_types.data_import import ( + Hdf5ImportConfig, + McapImportConfig, + TdmsImportConfig, + TimeFormat, + UlogImportConfig, +) + +if TYPE_CHECKING: + from datetime import datetime + + from sift_client.sift_types.data_import import ImportConfig + + +def apply_relative_start_time( + config: ImportConfig, relative_start_time: datetime, time_format: TimeFormat | None +) -> None: + """Set the relative start time on a detected config, dispatching by config type. + + ULog and MCAP always take it as the log start time. Other formats take it + only when ``time_format``, the config's current format, is relative; + otherwise it is ignored with a warning. + """ + if isinstance(config, (UlogImportConfig, McapImportConfig)): + config.relative_start_time = relative_start_time + return + if time_format is None or not time_format.is_relative: + warnings.warn( + "'relative_start_time' is ignored because the time format " + f"({time_format.name if time_format else 'unset'}) is not relative.", + SiftIgnoredInputWarning, + stacklevel=caller_stacklevel(), + ) + return + if isinstance(config, (TdmsImportConfig, Hdf5ImportConfig)): + config.relative_start_time = relative_start_time + else: + config.time_column.relative_start_time = relative_start_time + + +def check_relative_start_time(config: ImportConfig, time_format: TimeFormat | None) -> None: + """Raise if ``time_format``, the config's current format, is relative with no start time. + + Assigning a time format bypasses the config validators, so a + ``time_format`` override is checked here before an auto-detected config + is imported. Only formats whose configs require a start time are checked. + """ + if isinstance(config, (TdmsImportConfig, UlogImportConfig, McapImportConfig)): + return + if isinstance(config, Hdf5ImportConfig): + relative_start_time = config.relative_start_time + else: + relative_start_time = config.time_column.relative_start_time + if time_format is not None and time_format.is_relative and relative_start_time is None: + raise ValueError( + f"'relative_start_time' is required when using a relative time format ({time_format.name})." + ) + + +_RELATIVE_FORMAT_VALUES = {f.value for f in TimeFormat if f.is_relative} + + +def set_detected_start_time(time_column, relative_start_time: datetime | None) -> None: + """Set the start time on a detected relative time column proto. + + Without it, a relative format detected by the server fails config + validation before ``relative_start_time`` can be applied. + """ + if relative_start_time is not None and time_column.format in _RELATIVE_FORMAT_VALUES: + time_column.relative_start_time.CopyFrom(to_timestamp_pb(relative_start_time)) diff --git a/python/lib/sift_client/_tests/_internal/test_parquet.py b/python/lib/sift_client/_tests/_internal/test_parquet.py index e0d96f850b..16bc587245 100644 --- a/python/lib/sift_client/_tests/_internal/test_parquet.py +++ b/python/lib/sift_client/_tests/_internal/test_parquet.py @@ -15,11 +15,8 @@ from sift_client._internal.util.parquet import detect_flat_parquet_config from sift_client.resources import DataImportAPIAsync -from sift_client.resources.data_imports import _apply_relative_start_time from sift_client.sift_types.channel import ChannelDataType from sift_client.sift_types.data_import import ( - CsvImportConfig, - CsvTimeColumn, DataTypeKey, TimeFormat, ) @@ -648,16 +645,3 @@ async def test_absolute_column_warns_and_ignores_start_time(self, create_parquet ) assert config.time_column.relative_start_time is None - - def test_non_parquet_config_warns_and_ignores_start_time(self): - config = CsvImportConfig( - asset_name="a", - time_column=CsvTimeColumn(column=1, format=TimeFormat.ABSOLUTE_RFC3339), - data_columns=[], - ) - config.time_column.format = TimeFormat.RELATIVE_SECONDS - - with pytest.warns(UserWarning, match="not supported for CsvImportConfig"): - _apply_relative_start_time(config, self.START) - - assert config.time_column.relative_start_time is None diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 4b427e79dc..64eec84fd2 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -2,6 +2,7 @@ from __future__ import annotations +import warnings from datetime import datetime, timezone from typing import TYPE_CHECKING, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -13,6 +14,8 @@ DATA_IMPORT_STATUS_FAILED, DATA_IMPORT_STATUS_SUCCEEDED, DATA_IMPORT_STATUS_UNSPECIFIED, + TIME_FORMAT_ABSOLUTE_RFC3339, + TIME_FORMAT_RELATIVE_SECONDS, TIME_FORMAT_UNSPECIFIED, CsvConfig, DetectConfigResponse, @@ -30,6 +33,7 @@ ParquetTimeColumn as ParquetTimeColumnProto, ) +from sift_client._internal.util.data_imports import apply_relative_start_time from sift_client._internal.util.time_column import infer_time_column from sift_client.resources import DataImportAPI, DataImportAPIAsync from sift_client.resources.data_imports import ( @@ -972,6 +976,227 @@ def test_hdf5_to_proto_rejects_missing_format(self): config._to_proto() +class TestDetectConfigRelativeStartTime: + START = pd.Timestamp("2026-01-01T00:00:00.000000001Z") + + @pytest.fixture + def csv_path(self, tmp_path): + path = tmp_path / "data.csv" + path.write_text("time,voltage\n0,1.0\n") + return path + + @pytest.mark.asyncio + async def test_csv_detected_relative_format_uses_start_time(self, csv_path): + api = _data_imports_api(MagicMock()) + api._low_level_client.detect_config = AsyncMock( + return_value=_csv_detect_response(TIME_FORMAT_RELATIVE_SECONDS) + ) + + config = await api.detect_config(csv_path, relative_start_time=self.START) + + assert config.time_column.format == TimeFormat.RELATIVE_SECONDS + assert config.time_column.relative_start_time == self.START + + @pytest.mark.asyncio + async def test_csv_detected_relative_format_without_start_time_raises(self, csv_path): + api = _data_imports_api(MagicMock()) + api._low_level_client.detect_config = AsyncMock( + return_value=_csv_detect_response(TIME_FORMAT_RELATIVE_SECONDS) + ) + + with pytest.raises(ValueError, match="relative_start_time"): + await api.detect_config(csv_path) + + @pytest.mark.asyncio + async def test_csv_relative_time_format_override_uses_start_time(self, csv_path): + api = _data_imports_api(MagicMock()) + api._low_level_client.detect_config = AsyncMock( + return_value=_csv_detect_response(TIME_FORMAT_ABSOLUTE_RFC3339) + ) + + config = await api.detect_config( + csv_path, time_format=TimeFormat.RELATIVE_SECONDS, relative_start_time=self.START + ) + + assert config.time_column.format == TimeFormat.RELATIVE_SECONDS + assert config.time_column._to_proto().relative_start_time.nanos == 1 + + @pytest.mark.asyncio + async def test_relative_time_format_override_without_start_time_can_be_set_later( + self, csv_path + ): + # Without relative_start_time, the caller sets it on the detected config. + api = _data_imports_api(MagicMock()) + api._low_level_client.detect_config = AsyncMock( + return_value=_csv_detect_response(TIME_FORMAT_ABSOLUTE_RFC3339) + ) + + config = await api.detect_config(csv_path, time_format=TimeFormat.RELATIVE_SECONDS) + assert config.time_column.relative_start_time is None + + config.time_column.relative_start_time = self.START + assert config._to_proto().time_column.relative_start_time.nanos == 1 + + @pytest.mark.asyncio + async def test_import_relative_time_format_without_start_time_raises(self, csv_path): + api = _data_imports_api(MagicMock()) + api._low_level_client.detect_config = AsyncMock( + return_value=_csv_detect_response(TIME_FORMAT_ABSOLUTE_RFC3339) + ) + api._low_level_client.create_from_upload = AsyncMock() + + with pytest.raises(ValueError, match="relative_start_time"): + await api.import_from_path(csv_path, asset="a", time_format=TimeFormat.RELATIVE_SECONDS) + api._low_level_client.create_from_upload.assert_not_called() + + @pytest.mark.asyncio + async def test_csv_absolute_format_warns_and_ignores_start_time(self, csv_path): + api = _data_imports_api(MagicMock()) + api._low_level_client.detect_config = AsyncMock( + return_value=_csv_detect_response(TIME_FORMAT_ABSOLUTE_RFC3339) + ) + + with pytest.warns(UserWarning, match="ABSOLUTE_RFC3339"): + config = await api.detect_config(csv_path, relative_start_time=self.START) + + assert config.time_column.relative_start_time is None + + def test_single_channel_per_row_detected_relative_format_uses_start_time(self): + proto = ParquetConfig( + single_channel_per_row=ParquetSingleChannelPerRowConfig( + time_column=ParquetTimeColumnProto( + path="timestamp", format=TIME_FORMAT_RELATIVE_SECONDS + ) + ) + ) + + config = _parse_parquet_detect_response( + proto, "file.parquet", 0, 0, relative_start_time=self.START + ) + + assert config.time_column.relative_start_time == self.START + + @pytest.mark.parametrize("config_cls", [Hdf5ImportConfig, TdmsImportConfig]) + def test_relative_time_format_uses_start_time(self, config_cls): + config = config_cls(asset_name="a", data=[]) + config.time_format = TimeFormat.RELATIVE_SECONDS + + apply_relative_start_time(config, self.START, config.time_format) + + assert config.relative_start_time == self.START + + @pytest.mark.parametrize("config_cls", [Hdf5ImportConfig, TdmsImportConfig]) + def test_unset_time_format_warns_and_ignores_start_time(self, config_cls): + config = config_cls(asset_name="a", data=[]) + + with pytest.warns(UserWarning, match="unset"): + apply_relative_start_time(config, self.START, config.time_format) + + assert config.relative_start_time is None + + @pytest.mark.parametrize("config_cls", [UlogImportConfig, McapImportConfig]) + def test_log_formats_always_use_start_time(self, config_cls): + config = config_cls(asset_name="a") + + apply_relative_start_time(config, self.START, None) + + assert config.relative_start_time == self.START + + +_DETECTORS = { + "hdf5": "sift_client._internal.util.hdf5.detect_hdf5_config", + "tdms": "sift_client._internal.util.tdms.detect_tdms_config", + "ulg": "sift_client._internal.util.ulog.detect_ulog_config", + "mcap": "sift_client._internal.util.mcap.detect_mcap_config", +} + + +class TestDetectConfigRelativeStartTimeByFormat: + """``relative_start_time`` through ``detect_config`` for client-detected formats.""" + + START = pd.Timestamp("2026-01-01T00:00:00.000000001Z") + + async def _detect(self, tmp_path, monkeypatch, ext, detected, **kwargs): + path = tmp_path / f"data.{ext}" + path.write_bytes(b"") + monkeypatch.setattr(_DETECTORS[ext], lambda *a, **k: detected) + data_type = DataTypeKey.HDF5_ONE_D if ext == "hdf5" else None + return await DataImportAPIAsync(MagicMock()).detect_config( + path, data_type=data_type, **kwargs + ) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("ext", "detected"), + [ + ("hdf5", Hdf5ImportConfig(asset_name="a", data=[])), + ("tdms", TdmsImportConfig(asset_name="a")), + ], + ) + async def test_relative_time_format_uses_start_time(self, tmp_path, monkeypatch, ext, detected): + config = await self._detect( + tmp_path, + monkeypatch, + ext, + detected, + time_format=TimeFormat.RELATIVE_SECONDS, + relative_start_time="2026-01-01T00:00:00.000000001Z", + ) + + assert config.relative_start_time == self.START + assert config._to_proto().relative_start_time.nanos == 1 + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("ext", "detected"), + [("ulg", UlogImportConfig(asset_name="a")), ("mcap", McapImportConfig(asset_name="a"))], + ) + async def test_log_formats_use_start_time(self, tmp_path, monkeypatch, ext, detected): + config = await self._detect( + tmp_path, monkeypatch, ext, detected, relative_start_time=self.START + ) + + assert config.relative_start_time == self.START + assert config._to_proto().relative_start_time.nanos == 1 + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("ext", "detected"), + [ + ("hdf5", Hdf5ImportConfig(asset_name="a", data=[])), + ("tdms", TdmsImportConfig(asset_name="a")), + ("ulg", UlogImportConfig(asset_name="a")), + ("mcap", McapImportConfig(asset_name="a")), + ], + ) + async def test_without_start_time_is_unchanged(self, tmp_path, monkeypatch, ext, detected): + with warnings.catch_warnings(): + warnings.simplefilter("error") + config = await self._detect(tmp_path, monkeypatch, ext, detected) + + assert config.relative_start_time is None + + @pytest.mark.asyncio + async def test_import_from_path_passes_start_time(self, tmp_path, monkeypatch): + monkeypatch.setattr( + "sift_client.resources.data_imports.upload_file", lambda *a, **k: {"jobId": "j1"} + ) + path = tmp_path / "log.ulg" + path.write_bytes(b"") + monkeypatch.setattr(_DETECTORS["ulg"], lambda *a, **k: UlogImportConfig(asset_name="")) + api = DataImportAPIAsync(MagicMock()) + api._low_level_client = MagicMock() + api._low_level_client.create_from_upload = AsyncMock(return_value=("import_1", "url")) + api.client.async_.jobs.get = AsyncMock(return_value=MagicMock()) + + await api.import_from_path( + path, asset="a", relative_start_time=self.START, show_progress=False + ) + + (sent_config,) = api._low_level_client.create_from_upload.call_args.args + assert sent_config.relative_start_time == self.START + + def _data_import_proto( data_import_id: str = "data-import-1", status: DataImportStatusProto.ValueType = DATA_IMPORT_STATUS_SUCCEEDED, diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index c090351a55..c463daa142 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -1,16 +1,18 @@ from __future__ import annotations -import warnings from pathlib import Path from typing import TYPE_CHECKING from sift_client._internal.low_level_wrappers.data_imports import DataImportsLowLevelClient from sift_client._internal.time import to_timestamp_nanos +from sift_client._internal.util.data_imports import ( + apply_relative_start_time, + check_relative_start_time, + set_detected_start_time, +) from sift_client._internal.util.executor import run_sync_function from sift_client._internal.util.file import extract_parquet_footer, upload_file from sift_client._internal.util.time_column import infer_time_column -from sift_client._internal.util.util import caller_stacklevel -from sift_client.errors import SiftIgnoredInputWarning from sift_client.resources._base import ResourceBase from sift_client.sift_types.asset import Asset from sift_client.sift_types.channel import ChannelDataType @@ -135,10 +137,10 @@ async def import_from_path( relative_start_time: Start time for a relative time format. A string is parsed with ``pd.Timestamp``; a string or ``pd.Timestamp`` keeps nanosecond precision. - Flat dataset Parquet only; ignored with a warning for other - formats or when the time format is absolute. Required for - Parquet files with a relative time column. Only used when - ``config`` is not provided. + For ULog and MCAP, this is the log start time. For other + formats, it is required with a relative time format and + ignored with a warning otherwise. Only used when ``config`` + is not provided. run: ``Run`` object or run ID string to import into an existing run. Mutually exclusive with ``run_name``. run_name: Name for a new run. Defaults to the filename if @@ -165,6 +167,7 @@ async def import_from_path( time_format=time_format, relative_start_time=relative_start_time, ) + check_relative_start_time(config, _get_time_format(config)) if isinstance(config, (UlogImportConfig, McapImportConfig)): # An empty channel list imports every channel config.data = [] @@ -400,9 +403,9 @@ async def detect_config( relative_start_time: Start time for a relative time format. A string is parsed with ``pd.Timestamp``; a string or ``pd.Timestamp`` keeps nanosecond precision. - Flat dataset Parquet only; ignored with a warning for other - formats or when the time format is absolute. Required for - Parquet files with a relative time column. + For ULog and MCAP, this is the log start time. For other + formats, it is required with a relative time format and + ignored with a warning otherwise. Returns: The detected import config. @@ -431,7 +434,7 @@ async def detect_config( ): _apply_time_format(config, TimeFormat.ABSOLUTE_UNIX_NANOSECONDS) if relative_start_time is not None: - _apply_relative_start_time(config, relative_start_time) + apply_relative_start_time(config, relative_start_time, _get_time_format(config)) return config async def _detect_config_for_type( @@ -519,11 +522,15 @@ def _read_sample() -> bytes: ) if response.HasField("csv_config"): - return _parse_csv_detect_response(response.csv_config) + return _parse_csv_detect_response(response.csv_config, relative_start_time) if response.HasField("parquet_config"): return _parse_parquet_detect_response( - response.parquet_config, path.name, footer_offset, footer_length + response.parquet_config, + path.name, + footer_offset, + footer_length, + relative_start_time=relative_start_time, ) raise ValueError( @@ -548,33 +555,6 @@ def _apply_time_format(config: ImportConfig, time_format: TimeFormat) -> None: config.time_format = time_format -def _apply_relative_start_time(config: ImportConfig, relative_start_time: datetime) -> None: - """Set the relative start time on a detected flat dataset Parquet config. - - Warns and leaves the config unchanged for other formats, including - single channel per row Parquet, or when the time format is absolute or - unset. - """ - if not isinstance(config, ParquetFlatDatasetImportConfig): - warnings.warn( - "'relative_start_time' is ignored because it is not supported for " - f"{type(config).__name__}.", - SiftIgnoredInputWarning, - stacklevel=caller_stacklevel(), - ) - return - time_format = config.time_column.format - if time_format is None or not time_format.is_relative: - warnings.warn( - "'relative_start_time' is ignored because the time format " - f"({time_format.name if time_format else 'unset'}) is not relative.", - SiftIgnoredInputWarning, - stacklevel=caller_stacklevel(), - ) - return - config.time_column.relative_start_time = relative_start_time - - def _get_time_format(config: ImportConfig) -> TimeFormat | None: """Read the current time format off a config, regardless of where it's stored.""" if isinstance( @@ -611,8 +591,11 @@ def _resolve_data_type_key(ext: str, data_type: DataTypeKey | None) -> DataTypeK return EXTENSION_TO_DATA_TYPE_KEY[ext] -def _parse_csv_detect_response(proto) -> CsvImportConfig: +def _parse_csv_detect_response( + proto, relative_start_time: datetime | None = None +) -> CsvImportConfig: """Parse a CSV DetectConfig response into a config.""" + set_detected_start_time(proto.time_column, relative_start_time) csv_config = CsvImportConfig._from_proto(proto) time_col = csv_config.time_column.column csv_config.data_columns = [dc for dc in csv_config.data_columns if dc.column != time_col] @@ -620,10 +603,16 @@ def _parse_csv_detect_response(proto) -> CsvImportConfig: def _parse_parquet_detect_response( - proto, filename: str, footer_offset: int, footer_length: int + proto, + filename: str, + footer_offset: int, + footer_length: int, + *, + relative_start_time: datetime | None = None, ) -> ParquetFlatDatasetImportConfig | ParquetSingleChannelPerRowImportConfig: """Parse a Parquet DetectConfig response into a config.""" if proto.HasField("flat_dataset"): + set_detected_start_time(proto.flat_dataset.time_column, relative_start_time) parquet_config = ParquetFlatDatasetImportConfig._from_proto( proto, footer_offset=footer_offset, footer_length=footer_length ) @@ -640,6 +629,7 @@ def _parse_parquet_detect_response( ] return parquet_config elif proto.HasField("single_channel_per_row"): + set_detected_start_time(proto.single_channel_per_row.time_column, relative_start_time) scpr_config = ParquetSingleChannelPerRowImportConfig._from_proto( proto, footer_offset=footer_offset, footer_length=footer_length ) diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index e13c932c14..125c9cb246 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -1316,9 +1316,9 @@ class DataImportAPI: relative_start_time: Start time for a relative time format. A string is parsed with ``pd.Timestamp``; a string or ``pd.Timestamp`` keeps nanosecond precision. - Flat dataset Parquet only; ignored with a warning for other - formats or when the time format is absolute. Required for - Parquet files with a relative time column. + For ULog and MCAP, this is the log start time. For other + formats, it is required with a relative time format and + ignored with a warning otherwise. Returns: The detected import config. @@ -1457,10 +1457,10 @@ class DataImportAPI: relative_start_time: Start time for a relative time format. A string is parsed with ``pd.Timestamp``; a string or ``pd.Timestamp`` keeps nanosecond precision. - Flat dataset Parquet only; ignored with a warning for other - formats or when the time format is absolute. Required for - Parquet files with a relative time column. Only used when - ``config`` is not provided. + For ULog and MCAP, this is the log start time. For other + formats, it is required with a relative time format and + ignored with a warning otherwise. Only used when ``config`` + is not provided. run: ``Run`` object or run ID string to import into an existing run. Mutually exclusive with ``run_name``. run_name: Name for a new run. Defaults to the filename if From 69dda853c3d235acc6e3d24fa3665cc9cf1be3ac Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 12 Oct 2026 15:33:56 -0700 Subject: [PATCH 16/20] python(fix): raise when CSV detection finds no time column --- .../_tests/resources/test_data_imports.py | 12 ++++++++++++ python/lib/sift_client/resources/data_imports.py | 9 +++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 64eec84fd2..84bd572cb3 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -960,6 +960,18 @@ async def test_csv_detect_uses_caller_time_format(self, tmp_path): assert config.time_column.format == TimeFormat.ABSOLUTE_RFC3339 + @pytest.mark.asyncio + async def test_csv_detect_raises_without_time_column(self, tmp_path): + path = tmp_path / "data.csv" + path.write_text("elapsed,voltage\n0.5,1.0\n") + response = _csv_detect_response(TIME_FORMAT_UNSPECIFIED) + response.csv_config.time_column.column_number = 0 + api = _data_imports_api(MagicMock()) + api._low_level_client.detect_config = AsyncMock(return_value=response) + + with pytest.raises(ValueError, match=r"No time column detected in 'data\.csv'"): + await api.detect_config(path, time_format=TimeFormat.RELATIVE_SECONDS) + def test_csv_to_proto_rejects_missing_format(self): col = CsvTimeColumn(column=1, format=None) with pytest.raises(ValueError, match="CsvTimeColumn.format must be set"): diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index c463daa142..995a6bd037 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -522,7 +522,7 @@ def _read_sample() -> bytes: ) if response.HasField("csv_config"): - return _parse_csv_detect_response(response.csv_config, relative_start_time) + return _parse_csv_detect_response(response.csv_config, path.name, relative_start_time) if response.HasField("parquet_config"): return _parse_parquet_detect_response( @@ -592,9 +592,14 @@ def _resolve_data_type_key(ext: str, data_type: DataTypeKey | None) -> DataTypeK def _parse_csv_detect_response( - proto, relative_start_time: datetime | None = None + proto, filename: str, relative_start_time: datetime | None = None ) -> CsvImportConfig: """Parse a CSV DetectConfig response into a config.""" + if proto.time_column.column_number < 1: + raise ValueError( + f"No time column detected in '{filename}'. Build a CsvImportConfig with " + "time_column set and pass it to import_from_path as config." + ) set_detected_start_time(proto.time_column, relative_start_time) csv_config = CsvImportConfig._from_proto(proto) time_col = csv_config.time_column.column From f8ed1c6ec3b51714dde1889af86eb8c98d09ea18 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Mon, 12 Oct 2026 15:34:00 -0700 Subject: [PATCH 17/20] python(docs): clarify relative_start_time semantics for MCAP --- python/lib/sift_client/_internal/util/data_imports.py | 8 +++++--- python/lib/sift_client/resources/data_imports.py | 10 ++++++++-- .../lib/sift_client/resources/sync_stubs/__init__.pyi | 10 ++++++++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/python/lib/sift_client/_internal/util/data_imports.py b/python/lib/sift_client/_internal/util/data_imports.py index 772bd07b66..c53b4388a1 100644 --- a/python/lib/sift_client/_internal/util/data_imports.py +++ b/python/lib/sift_client/_internal/util/data_imports.py @@ -27,9 +27,11 @@ def apply_relative_start_time( ) -> None: """Set the relative start time on a detected config, dispatching by config type. - ULog and MCAP always take it as the log start time. Other formats take it - only when ``time_format``, the config's current format, is relative; - otherwise it is ignored with a warning. + ULog and MCAP always take it, regardless of ``time_format``: ULog as the + log start time, MCAP as the start that non-Unix-epoch ``log_time`` values + are measured from. Other formats take it only when ``time_format``, the + config's current format, is relative; otherwise it is ignored with a + warning. """ if isinstance(config, (UlogImportConfig, McapImportConfig)): config.relative_start_time = relative_start_time diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index 995a6bd037..a004ba43fb 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -137,7 +137,10 @@ async def import_from_path( relative_start_time: Start time for a relative time format. A string is parsed with ``pd.Timestamp``; a string or ``pd.Timestamp`` keeps nanosecond precision. - For ULog and MCAP, this is the log start time. For other + For ULog, this is the log start time and takes precedence + over the log's GPS fix. For MCAP, set it only for logs whose + ``log_time`` is not on the Unix epoch; ``log_time`` is then + read as nanoseconds elapsed from this time. For other formats, it is required with a relative time format and ignored with a warning otherwise. Only used when ``config`` is not provided. @@ -403,7 +406,10 @@ async def detect_config( relative_start_time: Start time for a relative time format. A string is parsed with ``pd.Timestamp``; a string or ``pd.Timestamp`` keeps nanosecond precision. - For ULog and MCAP, this is the log start time. For other + For ULog, this is the log start time and takes precedence + over the log's GPS fix. For MCAP, set it only for logs whose + ``log_time`` is not on the Unix epoch; ``log_time`` is then + read as nanoseconds elapsed from this time. For other formats, it is required with a relative time format and ignored with a warning otherwise. diff --git a/python/lib/sift_client/resources/sync_stubs/__init__.pyi b/python/lib/sift_client/resources/sync_stubs/__init__.pyi index 125c9cb246..d6e7f3b732 100644 --- a/python/lib/sift_client/resources/sync_stubs/__init__.pyi +++ b/python/lib/sift_client/resources/sync_stubs/__init__.pyi @@ -1316,7 +1316,10 @@ class DataImportAPI: relative_start_time: Start time for a relative time format. A string is parsed with ``pd.Timestamp``; a string or ``pd.Timestamp`` keeps nanosecond precision. - For ULog and MCAP, this is the log start time. For other + For ULog, this is the log start time and takes precedence + over the log's GPS fix. For MCAP, set it only for logs whose + ``log_time`` is not on the Unix epoch; ``log_time`` is then + read as nanoseconds elapsed from this time. For other formats, it is required with a relative time format and ignored with a warning otherwise. @@ -1457,7 +1460,10 @@ class DataImportAPI: relative_start_time: Start time for a relative time format. A string is parsed with ``pd.Timestamp``; a string or ``pd.Timestamp`` keeps nanosecond precision. - For ULog and MCAP, this is the log start time. For other + For ULog, this is the log start time and takes precedence + over the log's GPS fix. For MCAP, set it only for logs whose + ``log_time`` is not on the Unix epoch; ``log_time`` is then + read as nanoseconds elapsed from this time. For other formats, it is required with a relative time format and ignored with a warning otherwise. Only used when ``config`` is not provided. From efe86ef919de818fba12af96808291c9b0d08d46 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Tue, 13 Oct 2026 15:12:25 -0700 Subject: [PATCH 18/20] python(refactor): move time format and relative_start_time handling onto import configs --- .../_internal/util/data_imports.py | 161 ++++++++++++------ .../_tests/resources/test_data_imports.py | 41 +++-- .../lib/sift_client/resources/data_imports.py | 158 ++--------------- .../lib/sift_client/sift_types/data_import.py | 151 +++++++++++++++- 4 files changed, 289 insertions(+), 222 deletions(-) diff --git a/python/lib/sift_client/_internal/util/data_imports.py b/python/lib/sift_client/_internal/util/data_imports.py index c53b4388a1..abbd28b7bd 100644 --- a/python/lib/sift_client/_internal/util/data_imports.py +++ b/python/lib/sift_client/_internal/util/data_imports.py @@ -1,72 +1,28 @@ -"""Helpers for applying ``relative_start_time`` to detected import configs.""" +"""Helpers for detecting and preparing data import configs.""" from __future__ import annotations -import warnings from typing import TYPE_CHECKING from sift_client._internal.time import to_timestamp_pb -from sift_client._internal.util.util import caller_stacklevel -from sift_client.errors import SiftIgnoredInputWarning +from sift_client._internal.util.executor import run_sync_function +from sift_client._internal.util.file import extract_parquet_footer +from sift_client._internal.util.time_column import infer_time_column +from sift_client.sift_types.channel import ChannelDataType from sift_client.sift_types.data_import import ( - Hdf5ImportConfig, - McapImportConfig, - TdmsImportConfig, + EXTENSION_TO_DATA_TYPE_KEY, + CsvImportConfig, + ParquetFlatDatasetImportConfig, + ParquetSingleChannelPerRowImportConfig, + ParquetTimeColumn, TimeFormat, - UlogImportConfig, ) if TYPE_CHECKING: from datetime import datetime + from pathlib import Path - from sift_client.sift_types.data_import import ImportConfig - - -def apply_relative_start_time( - config: ImportConfig, relative_start_time: datetime, time_format: TimeFormat | None -) -> None: - """Set the relative start time on a detected config, dispatching by config type. - - ULog and MCAP always take it, regardless of ``time_format``: ULog as the - log start time, MCAP as the start that non-Unix-epoch ``log_time`` values - are measured from. Other formats take it only when ``time_format``, the - config's current format, is relative; otherwise it is ignored with a - warning. - """ - if isinstance(config, (UlogImportConfig, McapImportConfig)): - config.relative_start_time = relative_start_time - return - if time_format is None or not time_format.is_relative: - warnings.warn( - "'relative_start_time' is ignored because the time format " - f"({time_format.name if time_format else 'unset'}) is not relative.", - SiftIgnoredInputWarning, - stacklevel=caller_stacklevel(), - ) - return - if isinstance(config, (TdmsImportConfig, Hdf5ImportConfig)): - config.relative_start_time = relative_start_time - else: - config.time_column.relative_start_time = relative_start_time - - -def check_relative_start_time(config: ImportConfig, time_format: TimeFormat | None) -> None: - """Raise if ``time_format``, the config's current format, is relative with no start time. - - Assigning a time format bypasses the config validators, so a - ``time_format`` override is checked here before an auto-detected config - is imported. Only formats whose configs require a start time are checked. - """ - if isinstance(config, (TdmsImportConfig, UlogImportConfig, McapImportConfig)): - return - if isinstance(config, Hdf5ImportConfig): - relative_start_time = config.relative_start_time - else: - relative_start_time = config.time_column.relative_start_time - if time_format is not None and time_format.is_relative and relative_start_time is None: - raise ValueError( - f"'relative_start_time' is required when using a relative time format ({time_format.name})." - ) + from sift_client.sift_types.data_import import DataTypeKey _RELATIVE_FORMAT_VALUES = {f.value for f in TimeFormat if f.is_relative} @@ -80,3 +36,96 @@ def set_detected_start_time(time_column, relative_start_time: datetime | None) - """ if relative_start_time is not None and time_column.format in _RELATIVE_FORMAT_VALUES: time_column.relative_start_time.CopyFrom(to_timestamp_pb(relative_start_time)) + + +def resolve_data_type_key(ext: str, data_type: DataTypeKey | None) -> DataTypeKey: + """Resolve the data type key from file extension and explicit override.""" + if data_type is not None: + return data_type + if ext in (".parquet", ".pqt"): + raise ValueError( + "Parquet files require 'data_type' to be specified. " + "Use DataTypeKey.PARQUET_FLATDATASET or DataTypeKey.PARQUET_SINGLE_CHANNEL_PER_ROW." + ) + if ext in (".h5", ".hdf5"): + raise ValueError( + "HDF5 files require 'data_type' to be specified. " + "Use DataTypeKey.HDF5_ONE_D, DataTypeKey.HDF5_TWO_D, " + "or DataTypeKey.HDF5_COMPOUND." + ) + if ext not in EXTENSION_TO_DATA_TYPE_KEY: + raise ValueError( + f"Unsupported file extension '{ext}'. " + f"Supported: {', '.join(sorted(EXTENSION_TO_DATA_TYPE_KEY))}. " + "You can also specify 'data_type' explicitly using a DataTypeKey value." + ) + return EXTENSION_TO_DATA_TYPE_KEY[ext] + + +def parse_csv_detect_response( + proto, filename: str, relative_start_time: datetime | None = None +) -> CsvImportConfig: + """Parse a CSV DetectConfig response into a config.""" + if proto.time_column.column_number < 1: + raise ValueError( + f"No time column detected in '{filename}'. Build a CsvImportConfig with " + "time_column set and pass it to import_from_path as config." + ) + set_detected_start_time(proto.time_column, relative_start_time) + csv_config = CsvImportConfig._from_proto(proto) + time_col = csv_config.time_column.column + csv_config.data_columns = [dc for dc in csv_config.data_columns if dc.column != time_col] + return csv_config + + +def parse_parquet_detect_response( + proto, + filename: str, + footer_offset: int, + footer_length: int, + *, + relative_start_time: datetime | None = None, +) -> ParquetFlatDatasetImportConfig | ParquetSingleChannelPerRowImportConfig: + """Parse a Parquet DetectConfig response into a config.""" + if proto.HasField("flat_dataset"): + set_detected_start_time(proto.flat_dataset.time_column, relative_start_time) + parquet_config = ParquetFlatDatasetImportConfig._from_proto( + proto, footer_offset=footer_offset, footer_length=footer_length + ) + time_path: str | None = parquet_config.time_column.path + if not time_path: + time_path = infer_time_column( + (dc.name, dc.data_type, dc.path) for dc in parquet_config.data_columns + ) + if time_path: + parquet_config.time_column = ParquetTimeColumn(path=time_path) + if time_path: + parquet_config.data_columns = [ + dc for dc in parquet_config.data_columns if dc.path != time_path + ] + return parquet_config + elif proto.HasField("single_channel_per_row"): + set_detected_start_time(proto.single_channel_per_row.time_column, relative_start_time) + scpr_config = ParquetSingleChannelPerRowImportConfig._from_proto( + proto, footer_offset=footer_offset, footer_length=footer_length + ) + if not scpr_config.time_column.path: + time_path = infer_time_column( + (col.column_config.name, ChannelDataType(col.column_config.data_type), col.path) + for col in proto.single_channel_per_row.columns + ) + if time_path is not None: + scpr_config.time_column = ParquetTimeColumn(path=time_path) + return scpr_config + raise ValueError(f"Unsupported parquet layout in DetectConfig response for '{filename}'.") + + +async def prepare_parquet_config( + config: ParquetFlatDatasetImportConfig | ParquetSingleChannelPerRowImportConfig, + path: Path, +) -> None: + """Populate parquet footer fields on the config if not already set.""" + if config.footer_offset == 0 and config.footer_length == 0: + footer_bytes, footer_offset = await run_sync_function(lambda: extract_parquet_footer(path)) + config.footer_offset = footer_offset + config.footer_length = len(footer_bytes) diff --git a/python/lib/sift_client/_tests/resources/test_data_imports.py b/python/lib/sift_client/_tests/resources/test_data_imports.py index 84bd572cb3..f3bb9c2ea6 100644 --- a/python/lib/sift_client/_tests/resources/test_data_imports.py +++ b/python/lib/sift_client/_tests/resources/test_data_imports.py @@ -33,13 +33,12 @@ ParquetTimeColumn as ParquetTimeColumnProto, ) -from sift_client._internal.util.data_imports import apply_relative_start_time +from sift_client._internal.util.data_imports import ( + parse_parquet_detect_response, + resolve_data_type_key, +) from sift_client._internal.util.time_column import infer_time_column from sift_client.resources import DataImportAPI, DataImportAPIAsync -from sift_client.resources.data_imports import ( - _parse_parquet_detect_response, - _resolve_data_type_key, -) from sift_client.sift_types.channel import ChannelDataType from sift_client.sift_types.data_import import ( CsvDataColumn, @@ -758,32 +757,32 @@ def test_missing_format_raises(self): class TestResolveDataTypeKey: def test_parquet_requires_data_type(self): with pytest.raises(ValueError, match="data_type"): - _resolve_data_type_key(".parquet", None) + resolve_data_type_key(".parquet", None) def test_parquet_with_explicit_data_type(self): - result = _resolve_data_type_key(".parquet", DataTypeKey.PARQUET_FLATDATASET) + result = resolve_data_type_key(".parquet", DataTypeKey.PARQUET_FLATDATASET) assert result == DataTypeKey.PARQUET_FLATDATASET def test_pqt_requires_data_type(self): with pytest.raises(ValueError, match="data_type"): - _resolve_data_type_key(".pqt", None) + resolve_data_type_key(".pqt", None) def test_known_extension_uses_map(self): - assert _resolve_data_type_key(".csv", None) == DataTypeKey.CSV + assert resolve_data_type_key(".csv", None) == DataTypeKey.CSV def test_ulog_extension_uses_map(self): - assert _resolve_data_type_key(".ulg", None) == DataTypeKey.ULOG + assert resolve_data_type_key(".ulg", None) == DataTypeKey.ULOG def test_mcap_extension_uses_map(self): - assert _resolve_data_type_key(".mcap", None) == DataTypeKey.MCAP + assert resolve_data_type_key(".mcap", None) == DataTypeKey.MCAP def test_explicit_data_type_overrides_extension(self): - result = _resolve_data_type_key(".csv", DataTypeKey.TDMS) + result = resolve_data_type_key(".csv", DataTypeKey.TDMS) assert result == DataTypeKey.TDMS def test_unknown_extension_raises(self): with pytest.raises(ValueError, match="Unsupported file extension"): - _resolve_data_type_key(".xyz", None) + resolve_data_type_key(".xyz", None) class TestInferTimeColumn: @@ -873,7 +872,7 @@ def test_flat_dataset_infers_int64_time_column(self): ("status", ChannelDataType.INT_32.value), ], ) - config = _parse_parquet_detect_response(proto, "file.parquet", 0, 0) + config = parse_parquet_detect_response(proto, "file.parquet", 0, 0) assert isinstance(config, ParquetFlatDatasetImportConfig) assert config.time_column.path == "timestamp" assert [dc.path for dc in config.data_columns] == ["voltage", "status"] @@ -887,7 +886,7 @@ def test_flat_dataset_keeps_server_time_column_when_set(self): ("voltage", ChannelDataType.DOUBLE.value), ], ) - config = _parse_parquet_detect_response(proto, "file.parquet", 0, 0) + config = parse_parquet_detect_response(proto, "file.parquet", 0, 0) assert config.time_column.path == "server_ts" assert [dc.path for dc in config.data_columns] == ["timestamp", "voltage"] @@ -896,7 +895,7 @@ def test_flat_dataset_no_int64_match_leaves_time_empty(self): time_path="", data_columns=[("voltage", ChannelDataType.DOUBLE.value)], ) - config = _parse_parquet_detect_response(proto, "file.parquet", 0, 0) + config = parse_parquet_detect_response(proto, "file.parquet", 0, 0) assert config.time_column.path == "" assert [dc.path for dc in config.data_columns] == ["voltage"] @@ -908,7 +907,7 @@ def test_scpr_infers_int64_time_column(self): ("timestamp", ChannelDataType.INT_64.value), ], ) - config = _parse_parquet_detect_response(proto, "file.parquet", 0, 0) + config = parse_parquet_detect_response(proto, "file.parquet", 0, 0) assert isinstance(config, ParquetSingleChannelPerRowImportConfig) assert config.time_column.path == "timestamp" @@ -1082,7 +1081,7 @@ def test_single_channel_per_row_detected_relative_format_uses_start_time(self): ) ) - config = _parse_parquet_detect_response( + config = parse_parquet_detect_response( proto, "file.parquet", 0, 0, relative_start_time=self.START ) @@ -1093,7 +1092,7 @@ def test_relative_time_format_uses_start_time(self, config_cls): config = config_cls(asset_name="a", data=[]) config.time_format = TimeFormat.RELATIVE_SECONDS - apply_relative_start_time(config, self.START, config.time_format) + config._apply_relative_start_time(self.START) assert config.relative_start_time == self.START @@ -1102,7 +1101,7 @@ def test_unset_time_format_warns_and_ignores_start_time(self, config_cls): config = config_cls(asset_name="a", data=[]) with pytest.warns(UserWarning, match="unset"): - apply_relative_start_time(config, self.START, config.time_format) + config._apply_relative_start_time(self.START) assert config.relative_start_time is None @@ -1110,7 +1109,7 @@ def test_unset_time_format_warns_and_ignores_start_time(self, config_cls): def test_log_formats_always_use_start_time(self, config_cls): config = config_cls(asset_name="a") - apply_relative_start_time(config, self.START, None) + config._apply_relative_start_time(self.START) assert config.relative_start_time == self.START diff --git a/python/lib/sift_client/resources/data_imports.py b/python/lib/sift_client/resources/data_imports.py index a004ba43fb..727e6cc9ab 100644 --- a/python/lib/sift_client/resources/data_imports.py +++ b/python/lib/sift_client/resources/data_imports.py @@ -6,28 +6,23 @@ from sift_client._internal.low_level_wrappers.data_imports import DataImportsLowLevelClient from sift_client._internal.time import to_timestamp_nanos from sift_client._internal.util.data_imports import ( - apply_relative_start_time, - check_relative_start_time, - set_detected_start_time, + parse_csv_detect_response, + parse_parquet_detect_response, + prepare_parquet_config, + resolve_data_type_key, ) from sift_client._internal.util.executor import run_sync_function from sift_client._internal.util.file import extract_parquet_footer, upload_file -from sift_client._internal.util.time_column import infer_time_column from sift_client.resources._base import ResourceBase from sift_client.sift_types.asset import Asset -from sift_client.sift_types.channel import ChannelDataType from sift_client.sift_types.data_import import ( DATA_TYPE_KEY_TO_PROTO, - EXTENSION_TO_DATA_TYPE_KEY, - CsvImportConfig, DataImport, DataTypeKey, - Hdf5ImportConfig, ImportConfig, McapImportConfig, ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig, - ParquetTimeColumn, TdmsImportConfig, TimeFormat, UlogImportConfig, @@ -170,7 +165,7 @@ async def import_from_path( time_format=time_format, relative_start_time=relative_start_time, ) - check_relative_start_time(config, _get_time_format(config)) + config._validate_relative_start_time() if isinstance(config, (UlogImportConfig, McapImportConfig)): # An empty channel list imports every channel config.data = [] @@ -191,7 +186,7 @@ async def import_from_path( if isinstance( config, (ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig) ): - await _prepare_parquet_config(config, path) + await prepare_parquet_config(config, path) if show_progress is None: show_progress = self._show_progress() @@ -426,21 +421,21 @@ async def detect_config( if not path.is_file(): raise FileNotFoundError(f"File not found: {file_path}") - data_type_key = _resolve_data_type_key(path.suffix.lower(), data_type) + data_type_key = resolve_data_type_key(path.suffix.lower(), data_type) if isinstance(relative_start_time, str): relative_start_time = to_timestamp_nanos(relative_start_time) config = await self._detect_config_for_type( path, data_type_key, relative_start_time=relative_start_time ) if time_format is not None: - _apply_time_format(config, time_format) + config._set_time_format(time_format) elif ( not isinstance(config, (TdmsImportConfig, UlogImportConfig, McapImportConfig)) - and _get_time_format(config) is None + and config._get_time_format() is None ): - _apply_time_format(config, TimeFormat.ABSOLUTE_UNIX_NANOSECONDS) + config._set_time_format(TimeFormat.ABSOLUTE_UNIX_NANOSECONDS) if relative_start_time is not None: - apply_relative_start_time(config, relative_start_time, _get_time_format(config)) + config._apply_relative_start_time(relative_start_time) return config async def _detect_config_for_type( @@ -501,11 +496,7 @@ async def _detect_config_for_type( return await run_sync_function( lambda: detect_flat_parquet_config(path, relative_start_time=relative_start_time) ) - is_parquet = data_type_key in ( - DataTypeKey.PARQUET_FLATDATASET, - DataTypeKey.PARQUET_SINGLE_CHANNEL_PER_ROW, - ) - + is_parquet = data_type_key == DataTypeKey.PARQUET_SINGLE_CHANNEL_PER_ROW footer_offset = 0 footer_length = 0 @@ -528,10 +519,10 @@ def _read_sample() -> bytes: ) if response.HasField("csv_config"): - return _parse_csv_detect_response(response.csv_config, path.name, relative_start_time) + return parse_csv_detect_response(response.csv_config, path.name, relative_start_time) if response.HasField("parquet_config"): - return _parse_parquet_detect_response( + return parse_parquet_detect_response( response.parquet_config, path.name, footer_offset, @@ -543,124 +534,3 @@ def _read_sample() -> bytes: f"No supported configuration detected for '{path.name}'. " "Only CSV, Parquet, HDF5, TDMS, ULog, and MCAP are supported by auto-detection." ) - - -def _apply_time_format(config: ImportConfig, time_format: TimeFormat) -> None: - """Set the time format on a detected config, dispatching by config type. - - CSV and Parquet store the format under ``time_column.format``. TDMS and - HDF5 store it on ``time_format``. ULog and MCAP have no configurable - time format. - """ - if isinstance( - config, - (CsvImportConfig, ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig), - ): - config.time_column.format = time_format - elif isinstance(config, (TdmsImportConfig, Hdf5ImportConfig)): - config.time_format = time_format - - -def _get_time_format(config: ImportConfig) -> TimeFormat | None: - """Read the current time format off a config, regardless of where it's stored.""" - if isinstance( - config, - (CsvImportConfig, ParquetFlatDatasetImportConfig, ParquetSingleChannelPerRowImportConfig), - ): - return config.time_column.format - if isinstance(config, (TdmsImportConfig, Hdf5ImportConfig)): - return config.time_format - return None - - -def _resolve_data_type_key(ext: str, data_type: DataTypeKey | None) -> DataTypeKey: - """Resolve the data type key from file extension and explicit override.""" - if data_type is not None: - return data_type - if ext in (".parquet", ".pqt"): - raise ValueError( - "Parquet files require 'data_type' to be specified. " - "Use DataTypeKey.PARQUET_FLATDATASET or DataTypeKey.PARQUET_SINGLE_CHANNEL_PER_ROW." - ) - if ext in (".h5", ".hdf5"): - raise ValueError( - "HDF5 files require 'data_type' to be specified. " - "Use DataTypeKey.HDF5_ONE_D, DataTypeKey.HDF5_TWO_D, " - "or DataTypeKey.HDF5_COMPOUND." - ) - if ext not in EXTENSION_TO_DATA_TYPE_KEY: - raise ValueError( - f"Unsupported file extension '{ext}'. " - f"Supported: {', '.join(sorted(EXTENSION_TO_DATA_TYPE_KEY))}. " - "You can also specify 'data_type' explicitly using a DataTypeKey value." - ) - return EXTENSION_TO_DATA_TYPE_KEY[ext] - - -def _parse_csv_detect_response( - proto, filename: str, relative_start_time: datetime | None = None -) -> CsvImportConfig: - """Parse a CSV DetectConfig response into a config.""" - if proto.time_column.column_number < 1: - raise ValueError( - f"No time column detected in '{filename}'. Build a CsvImportConfig with " - "time_column set and pass it to import_from_path as config." - ) - set_detected_start_time(proto.time_column, relative_start_time) - csv_config = CsvImportConfig._from_proto(proto) - time_col = csv_config.time_column.column - csv_config.data_columns = [dc for dc in csv_config.data_columns if dc.column != time_col] - return csv_config - - -def _parse_parquet_detect_response( - proto, - filename: str, - footer_offset: int, - footer_length: int, - *, - relative_start_time: datetime | None = None, -) -> ParquetFlatDatasetImportConfig | ParquetSingleChannelPerRowImportConfig: - """Parse a Parquet DetectConfig response into a config.""" - if proto.HasField("flat_dataset"): - set_detected_start_time(proto.flat_dataset.time_column, relative_start_time) - parquet_config = ParquetFlatDatasetImportConfig._from_proto( - proto, footer_offset=footer_offset, footer_length=footer_length - ) - time_path: str | None = parquet_config.time_column.path - if not time_path: - time_path = infer_time_column( - (dc.name, dc.data_type, dc.path) for dc in parquet_config.data_columns - ) - if time_path: - parquet_config.time_column = ParquetTimeColumn(path=time_path) - if time_path: - parquet_config.data_columns = [ - dc for dc in parquet_config.data_columns if dc.path != time_path - ] - return parquet_config - elif proto.HasField("single_channel_per_row"): - set_detected_start_time(proto.single_channel_per_row.time_column, relative_start_time) - scpr_config = ParquetSingleChannelPerRowImportConfig._from_proto( - proto, footer_offset=footer_offset, footer_length=footer_length - ) - if not scpr_config.time_column.path: - time_path = infer_time_column( - (col.column_config.name, ChannelDataType(col.column_config.data_type), col.path) - for col in proto.single_channel_per_row.columns - ) - if time_path is not None: - scpr_config.time_column = ParquetTimeColumn(path=time_path) - return scpr_config - raise ValueError(f"Unsupported parquet layout in DetectConfig response for '{filename}'.") - - -async def _prepare_parquet_config( - config: ParquetFlatDatasetImportConfig | ParquetSingleChannelPerRowImportConfig, - path: Path, -) -> None: - """Populate parquet footer fields on the config if not already set.""" - if config.footer_offset == 0 and config.footer_length == 0: - footer_bytes, footer_offset = await run_sync_function(lambda: extract_parquet_footer(path)) - config.footer_offset = footer_offset - config.footer_length = len(footer_bytes) diff --git a/python/lib/sift_client/sift_types/data_import.py b/python/lib/sift_client/sift_types/data_import.py index df2eae9580..86688177bc 100644 --- a/python/lib/sift_client/sift_types/data_import.py +++ b/python/lib/sift_client/sift_types/data_import.py @@ -1,6 +1,7 @@ from __future__ import annotations -from abc import ABC +import warnings +from abc import ABC, abstractmethod from datetime import datetime, timezone from enum import Enum from typing import TYPE_CHECKING, Union @@ -67,6 +68,8 @@ from sift.data_imports.v2.data_imports_pb2 import UlogDataConfig as UlogDataConfigProto from sift_client._internal.time import to_timestamp_nanos, to_timestamp_pb +from sift_client._internal.util.util import caller_stacklevel +from sift_client.errors import SiftIgnoredInputWarning from sift_client.sift_types._base import BaseType from sift_client.sift_types.channel import ChannelDataType @@ -228,6 +231,56 @@ class ImportConfigBase(BaseModel, ABC): run_name: str | None = None run_id: str | None = None + @abstractmethod + def _get_time_format(self) -> TimeFormat | None: + """Return the time format, wherever this config type stores it.""" + + @abstractmethod + def _set_time_format(self, time_format: TimeFormat) -> None: + """Set the time format. A no-op for formats with no configurable time format.""" + + @abstractmethod + def _get_relative_start_time(self) -> pd.Timestamp | datetime | None: + """Return the relative start time, wherever this config type stores it.""" + + @abstractmethod + def _set_relative_start_time(self, relative_start_time: pd.Timestamp | datetime) -> None: + """Set the relative start time, wherever this config type stores it.""" + + def _apply_relative_start_time(self, relative_start_time: pd.Timestamp | datetime) -> None: + """Set the relative start time on a detected config. + + It is set only when the current time format is relative; otherwise it + is ignored with a warning. + """ + time_format = self._get_time_format() + if time_format is None or not time_format.is_relative: + warnings.warn( + "'relative_start_time' is ignored because the time format " + f"({time_format.name if time_format else 'unset'}) is not relative.", + SiftIgnoredInputWarning, + stacklevel=caller_stacklevel(), + ) + return + self._set_relative_start_time(relative_start_time) + + def _validate_relative_start_time(self) -> None: + """Raise if the time format is relative with no start time. + + Assigning a time format bypasses the config validators, so a + ``time_format`` override is checked here before an auto-detected + config is imported. + """ + time_format = self._get_time_format() + if ( + time_format is not None + and time_format.is_relative + and self._get_relative_start_time() is None + ): + raise ValueError( + f"'relative_start_time' is required when using a relative time format ({time_format.name})." + ) + class CsvTimeColumn(TimeColumnBase): """Time column configuration for CSV imports. @@ -289,6 +342,18 @@ def __getitem__(self, name: str) -> CsvDataColumn: return dc raise KeyError(f"No data column named '{name}'") + def _get_time_format(self) -> TimeFormat | None: + return self.time_column.format + + def _set_time_format(self, time_format: TimeFormat) -> None: + self.time_column.format = time_format + + def _get_relative_start_time(self) -> pd.Timestamp | datetime | None: + return self.time_column.relative_start_time + + def _set_relative_start_time(self, relative_start_time: pd.Timestamp | datetime) -> None: + self.time_column.relative_start_time = relative_start_time + def _to_proto(self) -> CsvConfigProto: if not self.data_columns: raise ValueError("Config has no data columns. Add at least one before importing.") @@ -429,6 +494,18 @@ def __getitem__(self, name: str) -> ParquetDataColumn: return dc raise KeyError(f"No data column named '{name}'") + def _get_time_format(self) -> TimeFormat | None: + return self.time_column.format + + def _set_time_format(self, time_format: TimeFormat) -> None: + self.time_column.format = time_format + + def _get_relative_start_time(self) -> pd.Timestamp | datetime | None: + return self.time_column.relative_start_time + + def _set_relative_start_time(self, relative_start_time: pd.Timestamp | datetime) -> None: + self.time_column.relative_start_time = relative_start_time + def _to_proto(self) -> ParquetConfigProto: if not self.data_columns: raise ValueError("Config has no data columns. Add at least one before importing.") @@ -544,6 +621,18 @@ def _check_channel_config(self) -> ParquetSingleChannelPerRowImportConfig: ) return self + def _get_time_format(self) -> TimeFormat | None: + return self.time_column.format + + def _set_time_format(self, time_format: TimeFormat) -> None: + self.time_column.format = time_format + + def _get_relative_start_time(self) -> pd.Timestamp | datetime | None: + return self.time_column.relative_start_time + + def _set_relative_start_time(self, relative_start_time: pd.Timestamp | datetime) -> None: + self.time_column.relative_start_time = relative_start_time + def _to_proto(self) -> ParquetConfigProto: if self.single_channel is None and self.multi_channel is None: raise ValueError( @@ -705,6 +794,22 @@ def __getitem__(self, name: str) -> TdmsDataColumn: return d raise KeyError(f"No data column named '{name}'") + def _get_time_format(self) -> TimeFormat | None: + return self.time_format + + def _set_time_format(self, time_format: TimeFormat) -> None: + self.time_format = time_format + + def _get_relative_start_time(self) -> pd.Timestamp | datetime | None: + return self.relative_start_time + + def _set_relative_start_time(self, relative_start_time: pd.Timestamp | datetime) -> None: + self.relative_start_time = relative_start_time + + def _validate_relative_start_time(self) -> None: + # TDMS uses the start time only for non-standard time channels, so it is never required. + pass + def _to_proto(self) -> TDMSConfigProto: proto = TDMSConfigProto( asset_name=self.asset_name, @@ -854,6 +959,18 @@ def _check_relative_start_time(self) -> Hdf5ImportConfig: ) return self + def _get_time_format(self) -> TimeFormat | None: + return self.time_format + + def _set_time_format(self, time_format: TimeFormat) -> None: + self.time_format = time_format + + def _get_relative_start_time(self) -> pd.Timestamp | datetime | None: + return self.relative_start_time + + def _set_relative_start_time(self, relative_start_time: pd.Timestamp | datetime) -> None: + self.relative_start_time = relative_start_time + def _to_proto(self) -> Hdf5ConfigProto: if self.time_format is None: raise ValueError( @@ -980,6 +1097,22 @@ def __getitem__(self, name: str) -> UlogDataColumn: return dc raise KeyError(f"No data column named '{name}'") + def _get_time_format(self) -> TimeFormat | None: + return None + + def _set_time_format(self, time_format: TimeFormat) -> None: + pass + + def _get_relative_start_time(self) -> pd.Timestamp | datetime | None: + return self.relative_start_time + + def _set_relative_start_time(self, relative_start_time: pd.Timestamp | datetime) -> None: + self.relative_start_time = relative_start_time + + def _apply_relative_start_time(self, relative_start_time: pd.Timestamp | datetime) -> None: + # Always used, as the log start time, regardless of time format. + self._set_relative_start_time(relative_start_time) + def _to_proto(self) -> UlogConfigProto: proto = UlogConfigProto( asset_name=self.asset_name, @@ -1149,6 +1282,22 @@ def __getitem__(self, name: str) -> McapDataColumn: return dc raise KeyError(f"No data column named '{name}'") + def _get_time_format(self) -> TimeFormat | None: + return None + + def _set_time_format(self, time_format: TimeFormat) -> None: + pass + + def _get_relative_start_time(self) -> pd.Timestamp | datetime | None: + return self.relative_start_time + + def _set_relative_start_time(self, relative_start_time: pd.Timestamp | datetime) -> None: + self.relative_start_time = relative_start_time + + def _apply_relative_start_time(self, relative_start_time: pd.Timestamp | datetime) -> None: + # Always used, as the start non-Unix-epoch log_time values are measured from. + self._set_relative_start_time(relative_start_time) + def _to_proto(self) -> McapConfigProto: proto = McapConfigProto( asset_name=self.asset_name, From c456562b80c17ad3be96f4d479957f7c520def39 Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Wed, 14 Oct 2026 15:18:56 -0700 Subject: [PATCH 19/20] python(build): define data-review extra as an alias of parquet --- python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index f412d788ef..296dbdd1d4 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -268,7 +268,6 @@ sift-stream = ["sift-stream-bindings==0.5.2"] hdf5 = ["h5py~=3.11", "polars~=1.8"] # polars is only used by sift_py; remove once sift_py is fully deprecated ulog = ["pyulog~=1.2.2"] mcap = ["mcap~=1.4", "mcap-ros2-support~=0.5.7"] -data-review = ["pyarrow>=17.0.0"] parquet = ["pyarrow>=17.0.0"] [tool.sift.extras.combine] @@ -277,6 +276,7 @@ parquet = ["pyarrow>=17.0.0"] # aliases dev = ["development"] sift-stream-bindings = ["sift-stream"] +data-review = ["parquet"] # combinations file-imports = ["tdms", "rosbags", "hdf5", "ulog", "mcap", "parquet"] From a41a330ed0ef1be486ae0d1e5d60ed165d2b540c Mon Sep 17 00:00:00 2001 From: Alden Liu Olyniec Date: Wed, 14 Oct 2026 16:55:36 -0700 Subject: [PATCH 20/20] python(docs): add changelog fragment --- python/changelog.d/819.feat.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 python/changelog.d/819.feat.md diff --git a/python/changelog.d/819.feat.md b/python/changelog.d/819.feat.md new file mode 100644 index 0000000000..b88e8a01de --- /dev/null +++ b/python/changelog.d/819.feat.md @@ -0,0 +1,5 @@ +`detect_config` and `import_from_path` detect flat dataset Parquet configs on the client, so wide Parquet files no longer fail detection with `RESOURCE_EXHAUSTED` when their footer exceeds the server's 20 MiB request limit. + +Flat dataset Parquet detection now requires pyarrow. Install it with `pip install sift-stack-py[parquet]`. The `file-imports` extra includes it. + +`detect_config` and `import_from_path` also accept `relative_start_time`, so files with a relative time format can be detected and imported without editing the config.