From 00cdac76bdcf6c6a1d42457438ddd3daba380295 Mon Sep 17 00:00:00 2001 From: Vishal Rane Date: Thu, 9 Jul 2026 15:21:09 +0530 Subject: [PATCH 01/12] Add pandas 3 support --- pyproject.toml | 11 +- spotfire/__init__.py | 3 +- spotfire/_metadata.py | 185 ++++++++++++++++++ spotfire/data_function.py | 42 ++-- spotfire/public.py | 92 +++++---- spotfire/sbdf.pyx | 77 ++++---- .../files/data_function/table_metadata.txt | 3 - spotfire/test/test_data_function.py | 76 +++---- spotfire/test/test_sbdf.py | 37 ++-- spotfire/version.py | 2 +- 10 files changed, 348 insertions(+), 180 deletions(-) create mode 100644 spotfire/_metadata.py diff --git a/pyproject.toml b/pyproject.toml index 2d11110..8870827 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,12 +14,11 @@ description = "Package for Building Python Extensions to Spotfire" readme = "README.md" authors = [{ name = "Cloud Software Group, Inc." }] maintainers = [{ name="Spotfire Python Package Support", email = "spotfirepython@tibco.com" }] -requires-python = ">= 3.9" +requires-python = ">= 3.11" dependencies = [ # DataFrame support - "pandas >= 2.2.2, < 3.0.0", - "numpy >= 1.23.5; python_version < '3.12'", - "numpy >= 1.26.0; python_version >= '3.12'", + "pandas >= 3.0.0", + "numpy >= 1.26.0", # Package interactions "packaging", "pip >= 21.2", @@ -32,8 +31,6 @@ classifiers = [ "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", @@ -102,7 +99,7 @@ jobs = 1 limit-inference-results = 100 # load-plugins = persistent = true -py-version = "3.9" +py-version = "3.11" # recursive = # source-roots = # unsafe-load-any-extension = diff --git a/spotfire/__init__.py b/spotfire/__init__.py index 6934d82..9cbbfc8 100644 --- a/spotfire/__init__.py +++ b/spotfire/__init__.py @@ -4,4 +4,5 @@ """User visible utility functions.""" -from spotfire.public import copy_metadata, get_spotfire_types, set_spotfire_types, set_geocoding_table +from spotfire.public import (copy_metadata, get_spotfire_types, set_spotfire_types, set_geocoding_table, + get_table_metadata, set_table_metadata, get_column_metadata, set_column_metadata) diff --git a/spotfire/_metadata.py b/spotfire/_metadata.py new file mode 100644 index 0000000..01ed75f --- /dev/null +++ b/spotfire/_metadata.py @@ -0,0 +1,185 @@ +# Copyright © 2026. Cloud Software Group, Inc. +# This file is subject to the license terms contained +# in the license file that is distributed with this file. + +"""Accessor helpers for Spotfire metadata on DataFrames. + +Metadata is stored in ``df.attrs`` at the DataFrame level (never per-column). +This survives copy, drop, rename, concat, groupby, and inplace operations +in pandas 3. Only ``merge()`` loses ``df.attrs``. + +For backward compatibility, the old monkey-patched ``df.spotfire_table_metadata`` +pattern is also checked (with a deprecation warning). + +For the old ``df['col'].spotfire_column_metadata`` pattern, a Series accessor +is registered that raises a clear error — this pattern is silently broken +under pandas 3 Copy-on-Write. + +This module is the single abstraction point for metadata storage on +pandas DataFrames. +""" + +import copy +import warnings + +_TABLE_METADATA_KEY = 'spotfire_table_metadata' +_COLUMN_METADATA_KEY = 'spotfire_column_metadata' +_SPOTFIRE_TYPES_KEY = 'spotfire_types' +_ALL_KEYS = (_TABLE_METADATA_KEY, _COLUMN_METADATA_KEY, _SPOTFIRE_TYPES_KEY) + +# Keys whose values grow with column count and need CoW optimization. +# Table metadata is excluded: it is small and passed to Cython's +# ``_export_metadata(dict md, ...)`` which rejects dict subclasses. +_COW_KEYS = (_COLUMN_METADATA_KEY, _SPOTFIRE_TYPES_KEY) + + +class _CowDict(dict): + """Copy-on-Write dict for pandas 3 ``deepcopy(df.attrs)`` performance. + + Pandas 3 calls ``deepcopy(df.attrs)`` on every ``df[col]`` access. + With large metadata dicts (one entry per column), the recursive deep + copy is O(n) per access, making column loops O(n^2). + + This subclass applies Copy-on-Write: ``__deepcopy__`` returns ``self`` + (O(1) — zero cost on read), and the ``_metadata`` setters detach by + shallow-copying on first write when the dict is shared. This gives + the speed of ``return self`` with the isolation of a full copy. + """ + + _shared = False + + def __deepcopy__(self, memo): + self._shared = True + return self + + def __copy__(self): + self._shared = True + return self + + def is_shared(self): + """Return whether this dict has been handed out via deepcopy.""" + return self._shared + + def detach(self): + """Create an independent shallow copy and clear the shared flag.""" + clone = _CowDict(self) + clone._shared = False # pylint: disable=protected-access + return clone + + +def _detach_if_shared(dataframe, key): + """If the dict for *key* is shared, replace it with an independent shallow copy.""" + val = dataframe.attrs.get(key) + if val is not None and isinstance(val, _CowDict) and val.is_shared(): + val = val.detach() + dataframe.attrs[key] = val + return val + + +def _get(dataframe, key, default=None): + """Read a metadata key from ``df.attrs``, with legacy ``__dict__`` fallback.""" + val = dataframe.attrs.get(key) + if val is not None: + return val + + # Fallback: legacy monkey-patched attribute in __dict__ + val = dataframe.__dict__.get(key) + if val is not None: + warnings.warn( + f"Accessing metadata via df.{key} is deprecated. " + f"Use the spotfire.get_table_metadata() / spotfire.set_table_metadata() " + f"functions instead.", + DeprecationWarning, + stacklevel=3 + ) + return val + + return default + + +def _set(dataframe, key, value): + """Write a metadata key to ``df.attrs``.""" + if key in _COW_KEYS and isinstance(value, dict) and not isinstance(value, _CowDict): + value = _CowDict(value) + dataframe.attrs[key] = value + + +# --- Public API --- + +def get_table_metadata(dataframe): + """Return the table-level Spotfire metadata dict, or ``{}`` if none.""" + return _get(dataframe, _TABLE_METADATA_KEY, {}) + + +def set_table_metadata(dataframe, metadata): + """Set the table-level Spotfire metadata dict.""" + _set(dataframe, _TABLE_METADATA_KEY, metadata) + + +def get_column_metadata(dataframe, col): + """Return the Spotfire metadata dict for *col*, or ``{}`` if none.""" + return _get(dataframe, _COLUMN_METADATA_KEY, {}).get(col, {}) + + +def set_column_metadata(dataframe, col, metadata): + """Set the Spotfire metadata dict for *col*.""" + col_meta = _detach_if_shared(dataframe, _COLUMN_METADATA_KEY) + if col_meta is None: + col_meta = _CowDict() + _set(dataframe, _COLUMN_METADATA_KEY, col_meta) + col_meta[col] = metadata + + +def get_spotfire_type(dataframe, col): + """Return the Spotfire type name for *col*, or ``None`` if not set.""" + return _get(dataframe, _SPOTFIRE_TYPES_KEY, {}).get(col) + + +def set_spotfire_type(dataframe, col, typename): + """Set the Spotfire type name for *col*.""" + types = _detach_if_shared(dataframe, _SPOTFIRE_TYPES_KEY) + if types is None: + types = _CowDict() + _set(dataframe, _SPOTFIRE_TYPES_KEY, types) + types[col] = typename + + +def get_all_spotfire_types(dataframe): + """Return a dict mapping column name -> Spotfire type name for all columns that have one.""" + return dict(_get(dataframe, _SPOTFIRE_TYPES_KEY, {})) + + +def copy_all_metadata(source, destination): + """Copy all Spotfire metadata from *source* to *destination*.""" + for key in _ALL_KEYS: + val = _get(source, key) + if val: + _set(destination, key, copy.deepcopy(val)) + + +# --- Deprecation helpers for old per-column patterns --- + +def _register_deprecated_accessors(): + """Register pandas accessor that raises clear error for old per-column metadata pattern. + + ``df['col'].spotfire_column_metadata = {...}`` is silently broken under pandas 3 + Copy-on-Write (writes are discarded). This accessor intercepts READ attempts + and raises a helpful error pointing to the new API. + """ + try: + import pandas as pd # pylint: disable=import-outside-toplevel + + @pd.api.extensions.register_series_accessor('spotfire_column_metadata') + class _DeprecatedColumnMetadata: # pylint: disable=too-few-public-methods + def __init__(self, series): + raise AttributeError( + "df['col'].spotfire_column_metadata is not supported under pandas 3 " + "Copy-on-Write (writes are silently discarded). " + "Use spotfire.get_column_metadata(df, 'col') and " + "spotfire.set_column_metadata(df, 'col', metadata) instead." + ) + except Exception: # pylint: disable=broad-exception-caught + pass + + +_register_deprecated_accessors() diff --git a/spotfire/data_function.py b/spotfire/data_function.py index ce64edd..5e98641 100644 --- a/spotfire/data_function.py +++ b/spotfire/data_function.py @@ -13,7 +13,9 @@ import typing import re -from spotfire import sbdf, _utils +import pandas as pd + +from spotfire import sbdf, _utils, _metadata _ExceptionInfo = typing.Union[ @@ -131,12 +133,10 @@ def read(self, globals_dict: _Globals, debug_fn: _LogFunction) -> None: debug_fn(f"read {dataframe.shape[0]} rows {dataframe.shape[1]} columns") # Table metadata - try: - if dataframe.spotfire_table_metadata: - table_meta = f"\n {pprint.pformat(dataframe.spotfire_table_metadata)}" - else: - table_meta = " (no table metadata present)" - except AttributeError: + table_md = _metadata.get_table_metadata(dataframe) + if table_md: + table_meta = f"\n {pprint.pformat(table_md)}" + else: table_meta = " (no table metadata present)" debug_fn(f"table metadata:{table_meta}") @@ -144,15 +144,13 @@ def read(self, globals_dict: _Globals, debug_fn: _LogFunction) -> None: column_blank = False pretty_column = io.StringIO() for col in dataframe.columns: - try: - if pretty_column.tell() > _COLUMN_METADATA_TRUNCATE_THRESHOLD: - pretty_column.write("\n (truncated due to length)") - break - if dataframe[col].spotfire_column_metadata: - pretty_column.write(f"\n {col}: {pprint.pformat(dataframe[col].spotfire_column_metadata)}") - else: - column_blank = True - except AttributeError: + if pretty_column.tell() > _COLUMN_METADATA_TRUNCATE_THRESHOLD: + pretty_column.write("\n (truncated due to length)") + break + col_md = _metadata.get_column_metadata(dataframe, col) + if col_md: + pretty_column.write(f"\n {col}: {pprint.pformat(col_md)}") + else: column_blank = True if pretty_column.tell(): column_meta = pretty_column.getvalue() @@ -168,14 +166,14 @@ def read(self, globals_dict: _Globals, debug_fn: _LogFunction) -> None: dataframe = dataframe[dataframe.columns[0]] if self._type == "value": value = dataframe.at[0, dataframe.columns[0]] - if type(value).__module__ == "numpy": - dataframe = value.tolist() - elif type(value).__module__ == "pandas._libs.tslibs.timedeltas": - dataframe = value.to_pytimedelta() - elif type(value).__module__ == "pandas._libs.tslibs.timestamps": + if isinstance(value, pd.Timestamp): dataframe = value.to_pydatetime() - elif type(value).__module__ == "pandas._libs.tslibs.nattype": + elif isinstance(value, pd.Timedelta): + dataframe = value.to_pytimedelta() + elif pd.isna(value): dataframe = None + elif type(value).__module__ == "numpy": + dataframe = value.tolist() else: dataframe = value diff --git a/spotfire/public.py b/spotfire/public.py index bf13af4..0603318 100644 --- a/spotfire/public.py +++ b/spotfire/public.py @@ -8,7 +8,7 @@ import pandas as pd -from spotfire import sbdf +from spotfire import sbdf, _metadata try: import geopandas as gpd @@ -21,6 +21,43 @@ # Table and column metadata functions +def get_table_metadata(dataframe): + """Get the table-level Spotfire metadata from a DataFrame. + + :param dataframe: the DataFrame to get the table metadata from + :returns: a dict containing the table metadata, or an empty dict if none + """ + return _metadata.get_table_metadata(dataframe) + + +def set_table_metadata(dataframe, metadata): + """Set the table-level Spotfire metadata on a DataFrame. + + :param dataframe: the DataFrame to set the table metadata on + :param metadata: a dict containing the table metadata + """ + _metadata.set_table_metadata(dataframe, metadata) + + +def get_column_metadata(dataframe, col): + """Get the Spotfire column metadata for a specific column. + + :param dataframe: the DataFrame containing the column + :param col: the name of the column + :returns: a dict containing the column metadata, or an empty dict if none + """ + return _metadata.get_column_metadata(dataframe, col) + + +def set_column_metadata(dataframe, col, metadata): + """Set the Spotfire column metadata for a specific column. + + :param dataframe: the DataFrame containing the column + :param col: the name of the column + :param metadata: a dict containing the column metadata + """ + _metadata.set_column_metadata(dataframe, col, metadata) + def copy_metadata(source, destination) -> None: """Copy the table and column metadata from a Pandas object to another. @@ -34,27 +71,8 @@ def copy_metadata(source, destination) -> None: if isinstance(source, pd.Series) and not isinstance(destination, pd.Series): raise TypeError("both source and destination must be Series") - # Handle DataFrames - if isinstance(source, pd.DataFrame): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - try: - destination.spotfire_table_metadata = source.spotfire_table_metadata - except AttributeError: - pass - for col in source.columns: - try: - source1 = source[col] - destination1 = destination[col] - destination1.spotfire_column_metadata = source1.spotfire_column_metadata - except AttributeError: - pass - # Handle Series - elif isinstance(source, pd.Series): - try: - destination.spotfire_column_metadata = source.spotfire_column_metadata - except AttributeError: - pass + if isinstance(source, (pd.DataFrame, pd.Series)): + _metadata.copy_all_metadata(source, destination) # Spotfire type functions @@ -67,12 +85,8 @@ def get_spotfire_types(dataframe: pd.DataFrame) -> pd.Series: """ if not isinstance(dataframe, pd.DataFrame): raise TypeError("dataframe is not a DataFrame") - spotfire_types = {} - for col in dataframe.columns: - if 'spotfire_type' in dataframe[col].attrs: - spotfire_types[col] = dataframe[col].attrs['spotfire_type'] - else: - spotfire_types[col] = None + all_types = _metadata.get_all_spotfire_types(dataframe) + spotfire_types = {col: all_types.get(col) for col in dataframe.columns} return pd.Series(spotfire_types) @@ -92,7 +106,7 @@ def set_spotfire_types(dataframe: pd.DataFrame, column_types: _ColumnTypes) -> N if not sbdf.spotfire_typename_to_valuetype_id(spotfire_type): warnings.warn(f"Spotfire type '{spotfire_type}' for column '{col}' not recognized", sbdf.SBDFWarning) continue - dataframe[col].attrs['spotfire_type'] = spotfire_type + _metadata.set_spotfire_type(dataframe, col, spotfire_type) # Spotfire geocoding table functions @@ -117,16 +131,18 @@ def set_geocoding_table(dataframe: "gpd.GeoDataFrame") -> None: # Create columns from geometry bounds = dataframe.geometry.bounds centroid = dataframe.geometry.centroid - dataframe.assign(XMin=bounds["minx"], XMax=bounds["maxx"], - YMin=bounds["miny"], YMax=bounds["maxy"], - XCenter=centroid.x, YCenter=centroid.y) + dataframe["XMin"] = bounds["minx"] + dataframe["XMax"] = bounds["maxx"] + dataframe["YMin"] = bounds["miny"] + dataframe["YMax"] = bounds["maxy"] + dataframe["XCenter"] = centroid.x + dataframe["YCenter"] = centroid.y for col in columns: - dataframe[col].spotfire_column_metadata = {"MapChart.ColumnTypeId": [col]} + _metadata.set_column_metadata(dataframe, col, {"MapChart.ColumnTypeId": [col]}) # Set table metadata - try: - table_metadata = dataframe.spotfire_table_metadata - except AttributeError: + table_metadata = _metadata.get_table_metadata(dataframe) + if not table_metadata: table_metadata = {} table_metadata["MapChart.IsGeocodingTable"] = True @@ -147,6 +163,4 @@ def set_geocoding_table(dataframe: "gpd.GeoDataFrame") -> None: else: raise sbdf.SBDFError(f"geocoding tables cannot contain unknown geometry types ('{geom_type}')") - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - dataframe.spotfire_table_metadata = table_metadata + _metadata.set_table_metadata(dataframe, table_metadata) diff --git a/spotfire/sbdf.pyx b/spotfire/sbdf.pyx index 2f005bf..18aa1af 100644 --- a/spotfire/sbdf.pyx +++ b/spotfire/sbdf.pyx @@ -15,7 +15,7 @@ import warnings import cython import spotfire -from spotfire import _utils +from spotfire import _utils, _metadata import numpy as np import pandas as pd @@ -392,6 +392,8 @@ cdef class _ImportContext: return "float32" elif self.numpy_type_num == np_c.NPY_FLOAT64: return "float64" + elif self.value_type.id == sbdf_c.SBDF_STRINGTYPEID: + return "str" else: return "object" @@ -782,10 +784,10 @@ def import_data(sbdf_file): name=column_names[i]) column_series.loc[importer_contexts[i].get_invalid_array()] = None imported_columns.append(column_series) - dataframe = pd.concat(imported_columns, axis=1) + dataframe = pd.concat(imported_columns, axis=1, sort=False) for i in range(num_columns): - dataframe[column_names[i]].spotfire_column_metadata = column_metadata[i] - dataframe[column_names[i]].attrs['spotfire_type'] = importer_contexts[i].get_spotfire_type_name() + _metadata.set_column_metadata(dataframe, column_names[i], column_metadata[i]) + _metadata.set_spotfire_type(dataframe, column_names[i], importer_contexts[i].get_spotfire_type_name()) if gpd is not None and table_metadata.get('MapChart.IsGeocodingTable'): # Turn the DataFrame into a GeoDataFrame if geopandas was detected and the table metadata # indicates geocoding is present in the SBDF data @@ -797,7 +799,7 @@ def import_data(sbdf_file): geometry.append(shapely.wkb.loads(x)) dataframe = dataframe.drop(columns='Geometry') gdf = gpd.GeoDataFrame(dataframe, geometry=geometry) - spotfire.copy_metadata(dataframe, gdf) + _metadata.copy_all_metadata(dataframe, gdf) # Determine the correct CRS to use if 'MapChart.GeographicCrs' in table_metadata.keys() and table_metadata['MapChart.GeographicCrs'] != "": proj = table_metadata['MapChart.GeographicCrs'][0] @@ -809,9 +811,7 @@ def import_data(sbdf_file): except AttributeError: pass dataframe = gdf - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - dataframe.spotfire_table_metadata = table_metadata + _metadata.set_table_metadata(dataframe, table_metadata) return dataframe finally: @@ -915,45 +915,45 @@ cdef _export_obj_dataframe(obj): if len(set(obj.keys().to_list())) != len(obj.columns): raise SBDFError("obj does not have unique column names") - # Table/column metadata and column information - try: - table_metadata = obj.spotfire_table_metadata - except AttributeError: - table_metadata = {} + # Pre-extract metadata and dtypes before the column loop to avoid + # repeated df.attrs lookups. Column access is cached in a local + # variable (one df[col] per column) and fillna replaces the slower + # replace(dict) for NA handling. + table_metadata = _metadata.get_table_metadata(obj) export_column_names = obj.columns.tolist() + all_dtypes = obj.dtypes + all_sf_types = {c: _metadata.get_spotfire_type(obj, c) for c in export_column_names} + all_col_meta = {c: _metadata.get_column_metadata(obj, c) for c in export_column_names} + column_names = [] column_metadata = [] exporter_contexts = [] for col in export_column_names: - if obj[col].dtype == 'geometry': + if all_dtypes[col] == 'geometry': # Special case for the 'geometry' dtype from geopandas _export_obj_geodataframe_geometry(obj[col], obj.crs, table_metadata, column_names, column_metadata, exporter_contexts) else: # Normal columns column_names.append(col) + series = obj[col] context = _ExportContext() - if 'spotfire_type' in obj[col].attrs: - context.set_valuetype_id(_export_infer_valuetype_from_spotfire_typename(obj[col], f"column '{col}'")) + sf_type = all_sf_types[col] + if sf_type is not None: + context.set_valuetype_id( + _export_infer_valuetype_from_spotfire_typename(series, f"column '{col}'", sf_type)) else: - context.set_valuetype_id(_export_infer_valuetype_from_pandas_dtype(obj[col], f"column '{col}'")) + context.set_valuetype_id(_export_infer_valuetype_from_pandas_dtype(series, f"column '{col}'")) na_value = context.get_numpy_na_value() - nas = {None: na_value, - np.nan: na_value, - pd.NA: na_value, - pd.NaT: na_value, - } - if obj[col].dtype == "object": - values = obj[col].replace(nas).to_numpy() + invalids = pd.isnull(series) + numpy_dtype = context.get_numpy_dtype() + if numpy_dtype is not None: + values = series.fillna(na_value).to_numpy(dtype=numpy_dtype, copy=False) else: - values = obj[col].replace(nas).to_numpy(dtype=context.get_numpy_dtype()) - invalids = pd.isnull(obj[col]) + values = series.fillna(na_value).to_numpy(copy=False) context.set_arrays(values, invalids) exporter_contexts.append(context) - try: - column_metadata.append(obj[col].spotfire_column_metadata) - except AttributeError: - column_metadata.append({}) + column_metadata.append(all_col_meta[col]) return table_metadata, column_names, column_metadata, exporter_contexts @@ -1016,16 +1016,14 @@ cdef _export_obj_series(obj, default_column_name): # Column metadata and information context = _ExportContext() - if 'spotfire_type' in obj.attrs: - context.set_valuetype_id(_export_infer_valuetype_from_spotfire_typename(obj, description)) + sf_type = _metadata.get_spotfire_type(obj, column_name) + if sf_type is not None: + context.set_valuetype_id(_export_infer_valuetype_from_spotfire_typename(obj, description, sf_type)) else: context.set_valuetype_id(_export_infer_valuetype_from_pandas_dtype(obj, description)) context.set_arrays(obj.to_numpy(context.get_numpy_dtype(), na_value=context.get_numpy_na_value()), _export_infer_invalids(obj)) - try: - column_metadata = obj.spotfire_column_metadata - except AttributeError: - column_metadata = {} + column_metadata = _metadata.get_column_metadata(obj, column_name) return {}, [column_name], [column_metadata], [context] @@ -1537,7 +1535,7 @@ cdef int _export_infer_valuetype_from_pandas_dtype(series, series_description): return sbdf_c.SBDF_DATETIMETYPEID elif dtype.startswith("timedelta64["): return sbdf_c.SBDF_TIMESPANTYPEID - elif dtype == "string": + elif dtype in ("string", "str") or dtype.startswith("string["): return sbdf_c.SBDF_STRINGTYPEID else: raise SBDFError(f"unknown dtype '{dtype}' in {series_description}") @@ -1548,16 +1546,15 @@ cdef object _VT_CONVERSIONS_NUMERIC = [sbdf_c.SBDF_BOOLTYPEID, sbdf_c.SBDF_INTTY sbdf_c.SBDF_FLOATTYPEID, sbdf_c.SBDF_DOUBLETYPEID] -cdef int _export_infer_valuetype_from_spotfire_typename(series, series_description): +cdef int _export_infer_valuetype_from_spotfire_typename(series, series_description, typename): """Determine a value type for a data set based on the name of the Spotfire type. :param series: the values to infer the value type of :param series_description: description of series (for error reporting) + :param typename: the Spotfire type name :return: the integer value type id representing the type of series :raise SBDFError: if the types of series are inconvertible, mixed, all missing, or unknown """ - # Determine if a type has been specified. - typename = series.attrs['spotfire_type'] specified_vt = spotfire_typename_to_valuetype_id(typename) # Verify the specified type is allowed to be converted from. diff --git a/spotfire/test/files/data_function/table_metadata.txt b/spotfire/test/files/data_function/table_metadata.txt index 622f55f..e69de29 100644 --- a/spotfire/test/files/data_function/table_metadata.txt +++ b/spotfire/test/files/data_function/table_metadata.txt @@ -1,3 +0,0 @@ - -Standard error: -:3: UserWarning: Pandas doesn't allow columns to be created via a new attribute name - see https://pandas.pydata.org/pandas-docs/stable/indexing.html#attribute-access diff --git a/spotfire/test/test_data_function.py b/spotfire/test/test_data_function.py index d5e61f1..1b01da0 100644 --- a/spotfire/test/test_data_function.py +++ b/spotfire/test/test_data_function.py @@ -5,12 +5,11 @@ import os.path import sys import unittest -import warnings import pandas as pd import pandas.testing as pdtest -from spotfire import sbdf, data_function as datafn, _utils +from spotfire import sbdf, data_function as datafn, _utils, _metadata from spotfire.test import utils as testutils @@ -115,15 +114,13 @@ def _run_analytic(self, script, inputs, outputs, success, expected_result, spec_ data_frame = sbdf.import_data(output.file) print(data_frame) pdtest.assert_frame_equal(outputs[output.name], data_frame) - try: - print(f"test: table metadata:\n{data_frame.spotfire_table_metadata!r}") - except AttributeError: - pass + table_md = _metadata.get_table_metadata(data_frame) + if table_md: + print(f"test: table metadata:\n{table_md!r}") for col in data_frame.columns: - try: - print(f"test: column '{col}' metadata:\n{data_frame[col].spotfire_column_metadata!r}") - except AttributeError: - pass + col_md = _metadata.get_column_metadata(data_frame, col) + if col_md: + print(f"test: column '{col}' metadata:\n{col_md!r}") self._assert_table_metadata_equal(outputs[output.name], data_frame) except AssertionError: raise @@ -143,28 +140,14 @@ def _process_log_message(msg): def _assert_table_metadata_equal(self, first, second, msg=None): """Test that two data frames have the same metadata.""" - # Test the table metadata - try: - first_meta = first.spotfire_table_metadata - except AttributeError: - first_meta = {} - try: - second_meta = second.spotfire_table_metadata - except AttributeError: - second_meta = {} + first_meta = _metadata.get_table_metadata(first) + second_meta = _metadata.get_table_metadata(second) self.assertEqual(first_meta, second_meta, msg) - # Test the column metadata pdtest.assert_index_equal(first.columns, second.columns) for col in first.columns: - try: - first_colmeta = first[col].spotfire_column_metadata - except AttributeError: - first_colmeta = {} - try: - second_colmeta = second[col].spotfire_column_metadata - except AttributeError: - second_colmeta = {} + first_colmeta = _metadata.get_column_metadata(first, col) + second_colmeta = _metadata.get_column_metadata(second, col) self.assertEqual(first_colmeta, second_colmeta, msg) def test_value_input(self): @@ -302,33 +285,30 @@ def test_output_not_defined(self): def test_table_metadata(self): """Test that table metadata goes in and comes out.""" in1_df = pd.DataFrame({"a": [1.0, 2.0, 3.0]}) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - in1_df.spotfire_table_metadata = { # type: ignore[attr-defined] - 'bravo': ['The second letter of the phonetic alphabet.'] - } - out_md_df = pd.DataFrame(in1_df.spotfire_table_metadata) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - out_md_df.spotfire_table_metadata = {'a': ['Alpha']} # type: ignore[attr-defined] + _metadata.set_table_metadata(in1_df, { + 'bravo': ['The second letter of the phonetic alphabet.'] + }) + out_md_df = pd.DataFrame(_metadata.get_table_metadata(in1_df)) + _metadata.set_table_metadata(out_md_df, {'a': ['Alpha']}) expected = _PythonVersionedExpectedValue("table_metadata") self._run_analytic("""import pandas as pd -out_md = pd.DataFrame(in1.spotfire_table_metadata) -out_md.spotfire_table_metadata = {'a': ['Alpha']}""", {"in1": in1_df}, {"out_md": out_md_df}, True, expected) +import spotfire +out_md = pd.DataFrame(spotfire.get_table_metadata(in1)) +spotfire.set_table_metadata(out_md, {'a': ['Alpha']})""", {"in1": in1_df}, {"out_md": out_md_df}, True, expected) def test_column_metadata(self): """Test that column metadata goes in and comes out.""" in1_df = pd.DataFrame({"a": [1.0, 2.0, 3.0]}) - in1_df['a'].spotfire_column_metadata = {'a': ['Alpha']} # type: ignore[attr-defined] + _metadata.set_column_metadata(in1_df, 'a', {'a': ['Alpha']}) out1_df = pd.DataFrame({"a": [1.0, 2.0, 3.0], "b": [1.0, 2.0, 3.0]}) - out1_df['a'].spotfire_column_metadata = {'a': ['Alpha']} # type: ignore[attr-defined] - out1_df['b'].spotfire_column_metadata = {'b': ['Bravo']} # type: ignore[attr-defined] + _metadata.set_column_metadata(out1_df, 'a', {'a': ['Alpha']}) + _metadata.set_column_metadata(out1_df, 'b', {'b': ['Bravo']}) self._run_analytic("""import pandas as pd import spotfire b = pd.Series([1.0, 2.0, 3.0], name='b') out1 = pd.concat([in1, b], axis=1) spotfire.copy_metadata(in1, out1) -out1['b'].spotfire_column_metadata = {'b': ['Bravo']}""", {'in1': in1_df}, {'out1': out1_df}, True, None) +spotfire.set_column_metadata(out1, 'b', {'b': ['Bravo']})""", {'in1': in1_df}, {'out1': out1_df}, True, None) def test_column_rename(self): """Test that a column renamed in a data function processes correctly.""" @@ -358,10 +338,8 @@ def _debug_log(spec): def test_debug_log(self): """Test that the debug log can be enabled""" in1_df = pd.DataFrame({"a": [1, 2, 3, 4, 5]}) - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - in1_df.spotfire_table_metadata = {"tbl_1": [1]} # type: ignore[attr-defined] - in1_df["a"].spotfire_column_metadata = {"col_a_1": [10]} # type: ignore[attr-defined] + _metadata.set_table_metadata(in1_df, {"tbl_1": [1]}) + _metadata.set_column_metadata(in1_df, "a", {"col_a_1": [10]}) expected = _PythonVersionedExpectedValue("debug_log") self._run_analytic("in1", {"in1": in1_df}, {}, True, expected, spec_adjust=self._debug_log) @@ -374,7 +352,7 @@ def test_debug_log_omit(self): # Some columns have no metadata in2_df = pd.DataFrame({"a": [1, 2, 3, 4, 5], "b": [6, 7, 8, 9, 10]}) - in2_df['b'].spotfire_column_metadata = {"col_b_1": [11]} # type: ignore[attr-defined] + _metadata.set_column_metadata(in2_df, 'b', {"col_b_1": [11]}) expected = _PythonVersionedExpectedValue("debug_log_omit_2") self._run_analytic("in2", {"in2": in2_df}, {}, True, expected, spec_adjust=self._debug_log) @@ -385,7 +363,7 @@ def test_debug_log_truncate(self): in1_dict[f"num{i}"] = [i, i+1, i+2] in1_df = pd.DataFrame(in1_dict) for i in range(10000): - in1_df[f"num{i}"].spotfire_column_metadata = {f"col_num{i}_1": [i]} # type: ignore[attr-defined] + _metadata.set_column_metadata(in1_df, f"num{i}", {f"col_num{i}_1": [i]}) expected = _PythonVersionedExpectedValue("debug_log_truncate") self._run_analytic("in1", {"in1": in1_df}, {}, True, expected, spec_adjust=self._debug_log) diff --git a/spotfire/test/test_sbdf.py b/spotfire/test/test_sbdf.py index de89774..6bb34f7 100644 --- a/spotfire/test/test_sbdf.py +++ b/spotfire/test/test_sbdf.py @@ -18,7 +18,7 @@ from packaging import version import spotfire -from spotfire import sbdf +from spotfire import sbdf, _metadata from spotfire.test import utils @@ -49,20 +49,20 @@ def verify(dict_, pre, post): self.assertEqual(dict_[f"{pre}MetaBinary{post}"][0], b"\x01") # Check table metadata - verify(dataframe.spotfire_table_metadata, "SbdfTest.Table", "") + verify(_metadata.get_table_metadata(dataframe), "SbdfTest.Table", "") # Check column metadata - verify(dataframe["Boolean"].spotfire_column_metadata, "SbdfTest.Column", "0") - verify(dataframe["Integer"].spotfire_column_metadata, "SbdfTest.Column", "1") - verify(dataframe["Long"].spotfire_column_metadata, "SbdfTest.Column", "2") - verify(dataframe["Float"].spotfire_column_metadata, "SbdfTest.Column", "3") - verify(dataframe["Double"].spotfire_column_metadata, "SbdfTest.Column", "4") - verify(dataframe["DateTime"].spotfire_column_metadata, "SbdfTest.Column", "5") - verify(dataframe["Date"].spotfire_column_metadata, "SbdfTest.Column", "6") - verify(dataframe["Time"].spotfire_column_metadata, "SbdfTest.Column", "7") - verify(dataframe["TimeSpan"].spotfire_column_metadata, "SbdfTest.Column", "8") - verify(dataframe["String"].spotfire_column_metadata, "SbdfTest.Column", "9") - verify(dataframe["Decimal"].spotfire_column_metadata, "SbdfTest.Column", "10") - verify(dataframe["Binary"].spotfire_column_metadata, "SbdfTest.Column", "11") + verify(_metadata.get_column_metadata(dataframe, "Boolean"), "SbdfTest.Column", "0") + verify(_metadata.get_column_metadata(dataframe, "Integer"), "SbdfTest.Column", "1") + verify(_metadata.get_column_metadata(dataframe, "Long"), "SbdfTest.Column", "2") + verify(_metadata.get_column_metadata(dataframe, "Float"), "SbdfTest.Column", "3") + verify(_metadata.get_column_metadata(dataframe, "Double"), "SbdfTest.Column", "4") + verify(_metadata.get_column_metadata(dataframe, "DateTime"), "SbdfTest.Column", "5") + verify(_metadata.get_column_metadata(dataframe, "Date"), "SbdfTest.Column", "6") + verify(_metadata.get_column_metadata(dataframe, "Time"), "SbdfTest.Column", "7") + verify(_metadata.get_column_metadata(dataframe, "TimeSpan"), "SbdfTest.Column", "8") + verify(_metadata.get_column_metadata(dataframe, "String"), "SbdfTest.Column", "9") + verify(_metadata.get_column_metadata(dataframe, "Decimal"), "SbdfTest.Column", "10") + verify(_metadata.get_column_metadata(dataframe, "Binary"), "SbdfTest.Column", "11") def test_read_1(self): """Reading simple SBDF files should work.""" @@ -99,8 +99,9 @@ def test_read_100(self): for i, j in zip(dataframe.get("Double")[0:9].dropna().tolist(), [116.18, 122.46, 125.6, 128.74, 131.88, 135.02]): self.assertAlmostEqual(i, j) - self.assertEqual(dataframe.get("String")[0:5].tolist(), - ["The", "quick", None, None, "jumps"]) + string_col = dataframe.get("String")[0:5] + self.assertEqual(string_col.dropna().tolist(), ["The", "quick", "jumps"]) + self.assertEqual(string_col.isna().tolist(), [False, False, True, True, False]) self.assertEqual(dataframe.get("Decimal")[0:4].tolist(), [decimal.Decimal("1438.1565"), None, None, decimal.Decimal("1538.493")]) @@ -262,7 +263,7 @@ def test_invalid_export_type(self): spotfire.set_spotfire_types(dataframe, {"x": "Unknown"}) # force set it and see expect it to be ignored - dataframe["x"].attrs["spotfire_type"] = "Unknown" + _metadata.set_spotfire_type(dataframe, "x", "Unknown") new_df = self._roundtrip_dataframe(dataframe) new_df_types = spotfire.get_spotfire_types(new_df) self.assertEqual(new_df_types["x"], "LongInteger") @@ -520,7 +521,7 @@ def test_export_import_unicode_path(self): pd.testing.assert_frame_equal(imported[["col", "txt"]], dataframe, check_dtype=False) # Check dtype of the column self.assertEqual(dataframe["col"].dtype, "int64") - self.assertEqual(dataframe["txt"].dtype, "object") + self.assertTrue(pd.api.types.is_string_dtype(dataframe["txt"])) @staticmethod def _roundtrip_dataframe(dataframe: typing.Any) -> pd.DataFrame: diff --git a/spotfire/version.py b/spotfire/version.py index 143f979..bfc8a64 100644 --- a/spotfire/version.py +++ b/spotfire/version.py @@ -3,4 +3,4 @@ # in the license file that is distributed with this file. # pylint: skip-file -__version__="2.5.0.dev0" +__version__="3.0.0.dev0" From 81939f0e70d2f6928590e3b8164c09b7317252fe Mon Sep 17 00:00:00 2001 From: Vishal Rane Date: Thu, 9 Jul 2026 15:25:09 +0530 Subject: [PATCH 02/12] Remove Python 3.9/3.10 from CI matrix (pandas 3 requires 3.11+) --- .github/python-versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/python-versions.json b/.github/python-versions.json index 53aeae8..b65f372 100644 --- a/.github/python-versions.json +++ b/.github/python-versions.json @@ -1 +1 @@ -[ "3.14", "3.13", "3.12", "3.11", "3.10", "3.9" ] +[ "3.14", "3.13", "3.12", "3.11" ] From 7ce460621509f38548b060471ce0546b7df06a54 Mon Sep 17 00:00:00 2001 From: Vishal Rane Date: Thu, 9 Jul 2026 15:44:16 +0530 Subject: [PATCH 03/12] Refactor array creation in test_sbdf.py to use explicit type annotations --- spotfire/test/test_sbdf.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/spotfire/test/test_sbdf.py b/spotfire/test/test_sbdf.py index 6bb34f7..f15576a 100644 --- a/spotfire/test/test_sbdf.py +++ b/spotfire/test/test_sbdf.py @@ -446,7 +446,8 @@ def test_numpy_datetime_resolution(self): } for resolution, timestamp in inputs.items(): with self.subTest(resolution=resolution): - array = np.array([[0], [timestamp]]).astype(f"datetime64[{resolution}]") + array: np.ndarray = np.array([[0], [timestamp]]) + array = array.astype(f"datetime64[{resolution}]") dataframe = pd.DataFrame(array, columns=["x"]) df2 = self._roundtrip_dataframe(dataframe) val = df2.at[1, 'x'] @@ -463,7 +464,8 @@ def test_numpy_timedelta_resolution(self): } for resolution, timestamp in inputs.items(): with self.subTest(resolution=resolution): - array = np.array([[0], [timestamp]]).astype(f"timedelta64[{resolution}]") + array: np.ndarray = np.array([[0], [timestamp]]) + array = array.astype(f"timedelta64[{resolution}]") dataframe = pd.DataFrame(array, columns=["x"]) df2 = self._roundtrip_dataframe(dataframe) val = df2.at[1, 'x'] From 85a7c8e37a97e1b85bdb124135dacf03ebfa807b Mon Sep 17 00:00:00 2001 From: Vishal Rane Date: Thu, 9 Jul 2026 15:51:12 +0530 Subject: [PATCH 04/12] Remove unused import of spotfire from sbdf.pyx --- spotfire/sbdf.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/spotfire/sbdf.pyx b/spotfire/sbdf.pyx index 18aa1af..d720ffa 100644 --- a/spotfire/sbdf.pyx +++ b/spotfire/sbdf.pyx @@ -14,7 +14,6 @@ import warnings import cython -import spotfire from spotfire import _utils, _metadata import numpy as np From 1555c4b6843f313fa2dd5fef3df2bf9dedee880d Mon Sep 17 00:00:00 2001 From: Vishal Rane Date: Wed, 22 Jul 2026 13:42:21 +0530 Subject: [PATCH 05/12] Update numpy version to >= 2.0.0 --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8870827..df13ee3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "setuptools >= 40.8", "wheel", "Cython >= 3.0.4", - "numpy >= 2.0.0rc1", + "numpy >= 2.0.0", ] build-backend = "setuptools.build_meta" @@ -18,7 +18,7 @@ requires-python = ">= 3.11" dependencies = [ # DataFrame support "pandas >= 3.0.0", - "numpy >= 1.26.0", + "numpy >= 2.0.0", # Package interactions "packaging", "pip >= 21.2", From 0800b44ee15d24c7f821dcefe2357e3dc55d80d0 Mon Sep 17 00:00:00 2001 From: Vishal Rane Date: Thu, 13 Aug 2026 13:44:09 +0530 Subject: [PATCH 06/12] Add tzdata dependency for time zone support in minimal Linux containers --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index df13ee3..2ceea5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,10 @@ dependencies = [ # DataFrame support "pandas >= 3.0.0", "numpy >= 2.0.0", + # Time zone data - pandas 3 only pulls tzdata on Windows; bundle it on all + # platforms so zoneinfo resolves named zones in minimal Linux containers + # (e.g. debian:13-slim) that do not ship /usr/share/zoneinfo. + "tzdata >= 2022.7", # Package interactions "packaging", "pip >= 21.2", From f70968706d406471276e0f0f76cf1663e0f0ede8 Mon Sep 17 00:00:00 2001 From: vrane-tibco <158255115+vrane-tibco@users.noreply.github.com> Date: Wed, 26 Aug 2026 00:09:15 +0530 Subject: [PATCH 07/12] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- spotfire/_metadata.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/spotfire/_metadata.py b/spotfire/_metadata.py index 01ed75f..de1dc72 100644 --- a/spotfire/_metadata.py +++ b/spotfire/_metadata.py @@ -85,10 +85,13 @@ def _get(dataframe, key, default=None): # Fallback: legacy monkey-patched attribute in __dict__ val = dataframe.__dict__.get(key) if val is not None: + api_hint = { + _TABLE_METADATA_KEY: "spotfire.get_table_metadata() / spotfire.set_table_metadata()", + _COLUMN_METADATA_KEY: "spotfire.get_column_metadata(df, col) / spotfire.set_column_metadata(df, col, metadata)", + _SPOTFIRE_TYPES_KEY: "spotfire.get_spotfire_types(df) / spotfire.set_spotfire_types(df, column_types)", + }.get(key, "the Spotfire metadata APIs") warnings.warn( - f"Accessing metadata via df.{key} is deprecated. " - f"Use the spotfire.get_table_metadata() / spotfire.set_table_metadata() " - f"functions instead.", + f"Accessing metadata via df.{key} is deprecated. Use {api_hint} instead.", DeprecationWarning, stacklevel=3 ) From 7f53ea5c81b76b13468ff750a5859f826710e9b8 Mon Sep 17 00:00:00 2001 From: Vishal Rane Date: Wed, 26 Aug 2026 00:31:14 +0530 Subject: [PATCH 08/12] Add support for exporting Spotfire types from standalone Series --- spotfire/_metadata.py | 19 +++++++++++++++++++ spotfire/sbdf.pyx | 2 +- spotfire/test/test_sbdf.py | 21 +++++++++++++++++++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/spotfire/_metadata.py b/spotfire/_metadata.py index de1dc72..da1bb91 100644 --- a/spotfire/_metadata.py +++ b/spotfire/_metadata.py @@ -25,6 +25,12 @@ _TABLE_METADATA_KEY = 'spotfire_table_metadata' _COLUMN_METADATA_KEY = 'spotfire_column_metadata' _SPOTFIRE_TYPES_KEY = 'spotfire_types' + +# Scalar key set directly on a standalone Series' attrs (``series.attrs['spotfire_type']``). +# There is no DataFrame to hang a keyed dict off of in that case, so this remains +# the supported way to force a Spotfire type when exporting a bare Series. +_SERIES_TYPE_KEY = 'spotfire_type' + _ALL_KEYS = (_TABLE_METADATA_KEY, _COLUMN_METADATA_KEY, _SPOTFIRE_TYPES_KEY) # Keys whose values grow with column count and need CoW optimization. @@ -138,6 +144,19 @@ def get_spotfire_type(dataframe, col): return _get(dataframe, _SPOTFIRE_TYPES_KEY, {}).get(col) +def get_series_spotfire_type(series, col): + """Return the Spotfire type name to use when exporting a standalone ``Series``, or ``None`` if not set. + + Looks in the DataFrame-style ``spotfire_types`` dict first (present when the metadata was + copied from a DataFrame), then falls back to the scalar ``series.attrs['spotfire_type']`` + that callers set directly on a bare Series. + """ + typename = get_spotfire_type(series, col) + if typename is None: + typename = series.attrs.get(_SERIES_TYPE_KEY) + return typename + + def set_spotfire_type(dataframe, col, typename): """Set the Spotfire type name for *col*.""" types = _detach_if_shared(dataframe, _SPOTFIRE_TYPES_KEY) diff --git a/spotfire/sbdf.pyx b/spotfire/sbdf.pyx index d720ffa..5a4a660 100644 --- a/spotfire/sbdf.pyx +++ b/spotfire/sbdf.pyx @@ -1015,7 +1015,7 @@ cdef _export_obj_series(obj, default_column_name): # Column metadata and information context = _ExportContext() - sf_type = _metadata.get_spotfire_type(obj, column_name) + sf_type = _metadata.get_series_spotfire_type(obj, column_name) if sf_type is not None: context.set_valuetype_id(_export_infer_valuetype_from_spotfire_typename(obj, description, sf_type)) else: diff --git a/spotfire/test/test_sbdf.py b/spotfire/test/test_sbdf.py index f15576a..aad642a 100644 --- a/spotfire/test/test_sbdf.py +++ b/spotfire/test/test_sbdf.py @@ -407,6 +407,27 @@ def test_column_promotion(self): self.assertEqual(exported_types['large'], 'LongInteger') self.assertEqual(exported_types['small'], 'Integer') + def test_series_spotfire_type(self): + """Verify a Spotfire type set on a bare ``Series`` is honored when exporting.""" + # Without an override the type is inferred from the dtype. + series = pd.Series([1, 2, 3], name="x") + self.assertEqual(spotfire.get_spotfire_types(self._roundtrip_dataframe(series))["x"], "LongInteger") + + # The scalar attrs key set directly on the Series overrides the inferred type. + series = pd.Series([1, 2, 3], name="x") + series.attrs["spotfire_type"] = "Integer" + self.assertEqual(spotfire.get_spotfire_types(self._roundtrip_dataframe(series))["x"], "Integer") + + # An unnamed Series is exported under the default column name. + series = pd.Series([1, 2, 3]) + series.attrs["spotfire_type"] = "Integer" + self.assertEqual(spotfire.get_spotfire_types(self._roundtrip_dataframe(series))["x"], "Integer") + + # A type carried in the DataFrame-style dict is honored as well. + series = pd.Series([1, 2, 3], name="x") + _metadata.set_spotfire_type(series, "x", "Integer") + self.assertEqual(spotfire.get_spotfire_types(self._roundtrip_dataframe(series))["x"], "Integer") + def test_non_str_column_name(self): """Verify non-string column names export properly.""" dataframe = pd.DataFrame({ From 10e589ec3607117e8c6819302ed97a0d068e03a7 Mon Sep 17 00:00:00 2001 From: Vishal Rane Date: Wed, 26 Aug 2026 00:36:11 +0530 Subject: [PATCH 09/12] Format long string in column metadata hint for lint fix --- spotfire/_metadata.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spotfire/_metadata.py b/spotfire/_metadata.py index da1bb91..33d4501 100644 --- a/spotfire/_metadata.py +++ b/spotfire/_metadata.py @@ -93,7 +93,8 @@ def _get(dataframe, key, default=None): if val is not None: api_hint = { _TABLE_METADATA_KEY: "spotfire.get_table_metadata() / spotfire.set_table_metadata()", - _COLUMN_METADATA_KEY: "spotfire.get_column_metadata(df, col) / spotfire.set_column_metadata(df, col, metadata)", + _COLUMN_METADATA_KEY: "spotfire.get_column_metadata(df, col) / " + "spotfire.set_column_metadata(df, col, metadata)", _SPOTFIRE_TYPES_KEY: "spotfire.get_spotfire_types(df) / spotfire.set_spotfire_types(df, column_types)", }.get(key, "the Spotfire metadata APIs") warnings.warn( From 5eccf9b5f08f4abbf28ca8ea9cb51939582db31b Mon Sep 17 00:00:00 2001 From: Vishal Rane Date: Wed, 26 Aug 2026 00:53:00 +0530 Subject: [PATCH 10/12] Refactor metadata retrieval to avoid cython-lint crashes by using explicit loops --- spotfire/sbdf.pyx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/spotfire/sbdf.pyx b/spotfire/sbdf.pyx index 5a4a660..37c28fa 100644 --- a/spotfire/sbdf.pyx +++ b/spotfire/sbdf.pyx @@ -921,8 +921,13 @@ cdef _export_obj_dataframe(obj): table_metadata = _metadata.get_table_metadata(obj) export_column_names = obj.columns.tolist() all_dtypes = obj.dtypes - all_sf_types = {c: _metadata.get_spotfire_type(obj, c) for c in export_column_names} - all_col_meta = {c: _metadata.get_column_metadata(obj, c) for c in export_column_names} + # Note: built with explicit loops instead of dict comprehensions, since cython-lint + # crashes when traversing dict comprehensions (DictComprehensionAppendNode). + all_sf_types = {} + all_col_meta = {} + for name in export_column_names: + all_sf_types[name] = _metadata.get_spotfire_type(obj, name) + all_col_meta[name] = _metadata.get_column_metadata(obj, name) column_names = [] column_metadata = [] From 5cc3b5c9d65d9eaf1740a002c357d431b5aed0c7 Mon Sep 17 00:00:00 2001 From: Vishal Rane Date: Wed, 26 Aug 2026 00:53:19 +0530 Subject: [PATCH 11/12] Enhance detach method documentation to clarify shallow copy behavior and its implications for metadata mapping --- spotfire/_metadata.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/spotfire/_metadata.py b/spotfire/_metadata.py index 33d4501..9d71897 100644 --- a/spotfire/_metadata.py +++ b/spotfire/_metadata.py @@ -49,7 +49,9 @@ class _CowDict(dict): This subclass applies Copy-on-Write: ``__deepcopy__`` returns ``self`` (O(1) — zero cost on read), and the ``_metadata`` setters detach by shallow-copying on first write when the dict is shared. This gives - the speed of ``return self`` with the isolation of a full copy. + the speed of ``return self`` with the isolation of a full copy for the + metadata mapping itself — see ``detach`` for what that deliberately + does not cover. """ _shared = False @@ -67,7 +69,17 @@ def is_shared(self): return self._shared def detach(self): - """Create an independent shallow copy and clear the shared flag.""" + """Create an independent shallow copy and clear the shared flag. + + The copy is deliberately shallow. It isolates the mapping, which is all + the setters need: ``set_column_metadata`` and ``set_spotfire_type`` + rebind a whole entry, so a write through one DataFrame cannot be seen by + another that shares this dict. Nested values stay shared, so mutating a + dict returned by ``get_column_metadata`` in place is still visible to + those copies; go through the setters instead. Deep-copying here would + make each detach O(n) in the column count and reintroduce the O(n^2) + column loop this class exists to avoid. + """ clone = _CowDict(self) clone._shared = False # pylint: disable=protected-access return clone From 60783d14459fbf8aefc8fbbf55758bc0b23bd4c9 Mon Sep 17 00:00:00 2001 From: Vishal Rane Date: Wed, 26 Aug 2026 01:03:43 +0530 Subject: [PATCH 12/12] Simplify numpy array conversion by consolidating dtype handling in export process --- spotfire/sbdf.pyx | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/spotfire/sbdf.pyx b/spotfire/sbdf.pyx index 37c28fa..b7fd44c 100644 --- a/spotfire/sbdf.pyx +++ b/spotfire/sbdf.pyx @@ -950,11 +950,7 @@ cdef _export_obj_dataframe(obj): context.set_valuetype_id(_export_infer_valuetype_from_pandas_dtype(series, f"column '{col}'")) na_value = context.get_numpy_na_value() invalids = pd.isnull(series) - numpy_dtype = context.get_numpy_dtype() - if numpy_dtype is not None: - values = series.fillna(na_value).to_numpy(dtype=numpy_dtype, copy=False) - else: - values = series.fillna(na_value).to_numpy(copy=False) + values = series.fillna(na_value).to_numpy(dtype=context.get_numpy_dtype(), copy=False) context.set_arrays(values, invalids) exporter_contexts.append(context) column_metadata.append(all_col_meta[col])