Add pandas 3 support Status: Open. #96

.github/python-versions.json

@@ -1 +1 @@
-[ "3.14", "3.13", "3.12", "3.11", "3.10", "3.9" ]
+[ "3.14", "3.13", "3.12", "3.11" ]

pyproject.toml

@@ -3,7 +3,7 @@ requires = [
     "setuptools >= 40.8",
     "wheel",
     "Cython >= 3.0.4",
-    "numpy >= 2.0.0rc1",
+    "numpy >= 2.0.0",
 ]
 build-backend = "setuptools.build_meta"
 
@@ -14,12 +14,15 @@ description = "Package for Building Python Extensions to Spotfire"
 readme = "README.md"
 authors = [{ name = "Cloud Software Group, Inc." }]
 maintainers = [{ name="Spotfire Python Package Support", email = "spotfirepython@tibco.com" }]
-requires-python = ">= 3.9"
+requires-python = ">= 3.11"
 dependencies = [
     # DataFrame support
-    "pandas >= 2.2.2, < 3.0.0",
-    "numpy >= 1.23.5; python_version < '3.12'",
-    "numpy >= 1.26.0; python_version >= '3.12'",
+    "pandas >= 3.0.0",
+    "numpy >= 2.0.0",
+    # Time zone data - pandas 3 only pulls tzdata on Windows; bundle it on all
+    # platforms so zoneinfo resolves named zones in minimal Linux containers
+    # (e.g. debian:13-slim) that do not ship /usr/share/zoneinfo.
+    "tzdata >= 2022.7",
     # Package interactions
     "packaging",
     "pip >= 21.2",
@@ -32,8 +35,6 @@ classifiers = [
     "Operating System :: OS Independent",
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.9",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
@@ -102,7 +103,7 @@ jobs = 1
 limit-inference-results = 100
 # load-plugins =
 persistent = true
-py-version = "3.9"
+py-version = "3.11"
 # recursive =
 # source-roots =
 # unsafe-load-any-extension =

spotfire/__init__.py

@@ -4,4 +4,5 @@
 
 """User visible utility functions."""
 
-from spotfire.public import copy_metadata, get_spotfire_types, set_spotfire_types, set_geocoding_table
+from spotfire.public import (copy_metadata, get_spotfire_types, set_spotfire_types, set_geocoding_table,
+                             get_table_metadata, set_table_metadata, get_column_metadata, set_column_metadata)

spotfire/_metadata.py

@@ -0,0 +1,220 @@
+# Copyright © 2026. Cloud Software Group, Inc.
+# This file is subject to the license terms contained
+# in the license file that is distributed with this file.
+
+"""Accessor helpers for Spotfire metadata on DataFrames.
+
+Metadata is stored in ``df.attrs`` at the DataFrame level (never per-column).
+This survives copy, drop, rename, concat, groupby, and inplace operations
+in pandas 3.  Only ``merge()`` loses ``df.attrs``.
+
+For backward compatibility, the old monkey-patched ``df.spotfire_table_metadata``
+pattern is also checked (with a deprecation warning).
+
+For the old ``df['col'].spotfire_column_metadata`` pattern, a Series accessor
+is registered that raises a clear error — this pattern is silently broken
+under pandas 3 Copy-on-Write.
+
+This module is the single abstraction point for metadata storage on
+pandas DataFrames.
+"""
+
+import copy
+import warnings
+
+_TABLE_METADATA_KEY = 'spotfire_table_metadata'
+_COLUMN_METADATA_KEY = 'spotfire_column_metadata'
+_SPOTFIRE_TYPES_KEY = 'spotfire_types'
+
+# Scalar key set directly on a standalone Series' attrs (``series.attrs['spotfire_type']``).
+# There is no DataFrame to hang a keyed dict off of in that case, so this remains
+# the supported way to force a Spotfire type when exporting a bare Series.
+_SERIES_TYPE_KEY = 'spotfire_type'
+
+_ALL_KEYS = (_TABLE_METADATA_KEY, _COLUMN_METADATA_KEY, _SPOTFIRE_TYPES_KEY)
+
+# Keys whose values grow with column count and need CoW optimization.
+# Table metadata is excluded: it is small and passed to Cython's
+# ``_export_metadata(dict md, ...)`` which rejects dict subclasses.
+_COW_KEYS = (_COLUMN_METADATA_KEY, _SPOTFIRE_TYPES_KEY)
+
+
+class _CowDict(dict):
+    """Copy-on-Write dict for pandas 3 ``deepcopy(df.attrs)`` performance.
+
+    Pandas 3 calls ``deepcopy(df.attrs)`` on every ``df[col]`` access.
+    With large metadata dicts (one entry per column), the recursive deep
+    copy is O(n) per access, making column loops O(n^2).
+
+    This subclass applies Copy-on-Write: ``__deepcopy__`` returns ``self``
+    (O(1) — zero cost on read), and the ``_metadata`` setters detach by
+    shallow-copying on first write when the dict is shared.  This gives
+    the speed of ``return self`` with the isolation of a full copy for the
+    metadata mapping itself — see ``detach`` for what that deliberately
+    does not cover.
+    """
+
+    _shared = False
+
+    def __deepcopy__(self, memo):
+        self._shared = True
+        return self
+
+    def __copy__(self):
+        self._shared = True
+        return self
+
+    def is_shared(self):
+        """Return whether this dict has been handed out via deepcopy."""
+        return self._shared
+
+    def detach(self):
+        """Create an independent shallow copy and clear the shared flag.
+
+        The copy is deliberately shallow.  It isolates the mapping, which is all
+        the setters need: ``set_column_metadata`` and ``set_spotfire_type``
+        rebind a whole entry, so a write through one DataFrame cannot be seen by
+        another that shares this dict.  Nested values stay shared, so mutating a
+        dict returned by ``get_column_metadata`` in place is still visible to
+        those copies; go through the setters instead.  Deep-copying here would
+        make each detach O(n) in the column count and reintroduce the O(n^2)
+        column loop this class exists to avoid.
+        """
+        clone = _CowDict(self)
+        clone._shared = False  # pylint: disable=protected-access
+        return clone
+
+
+def _detach_if_shared(dataframe, key):
+    """If the dict for *key* is shared, replace it with an independent shallow copy."""
+    val = dataframe.attrs.get(key)
+    if val is not None and isinstance(val, _CowDict) and val.is_shared():
+        val = val.detach()
+        dataframe.attrs[key] = val
+    return val
+
+
+def _get(dataframe, key, default=None):
+    """Read a metadata key from ``df.attrs``, with legacy ``__dict__`` fallback."""
+    val = dataframe.attrs.get(key)
+    if val is not None:
+        return val
+
+    # Fallback: legacy monkey-patched attribute in __dict__
+    val = dataframe.__dict__.get(key)
+    if val is not None:
+        api_hint = {
+            _TABLE_METADATA_KEY: "spotfire.get_table_metadata() / spotfire.set_table_metadata()",
+            _COLUMN_METADATA_KEY: "spotfire.get_column_metadata(df, col) / "
+                                  "spotfire.set_column_metadata(df, col, metadata)",
+            _SPOTFIRE_TYPES_KEY: "spotfire.get_spotfire_types(df) / spotfire.set_spotfire_types(df, column_types)",
+        }.get(key, "the Spotfire metadata APIs")
+        warnings.warn(
+            f"Accessing metadata via df.{key} is deprecated. Use {api_hint} instead.",
+            DeprecationWarning,
+            stacklevel=3
+        )
+        return val
+
+    return default
+
+
+def _set(dataframe, key, value):
+    """Write a metadata key to ``df.attrs``."""
+    if key in _COW_KEYS and isinstance(value, dict) and not isinstance(value, _CowDict):
+        value = _CowDict(value)
+    dataframe.attrs[key] = value
+
+
+# --- Public API ---
+
+def get_table_metadata(dataframe):
+    """Return the table-level Spotfire metadata dict, or ``{}`` if none."""
+    return _get(dataframe, _TABLE_METADATA_KEY, {})
+
+
+def set_table_metadata(dataframe, metadata):
+    """Set the table-level Spotfire metadata dict."""
+    _set(dataframe, _TABLE_METADATA_KEY, metadata)
+
+
+def get_column_metadata(dataframe, col):
+    """Return the Spotfire metadata dict for *col*, or ``{}`` if none."""
+    return _get(dataframe, _COLUMN_METADATA_KEY, {}).get(col, {})
+
+
+def set_column_metadata(dataframe, col, metadata):
+    """Set the Spotfire metadata dict for *col*."""
+    col_meta = _detach_if_shared(dataframe, _COLUMN_METADATA_KEY)
+    if col_meta is None:
+        col_meta = _CowDict()
+        _set(dataframe, _COLUMN_METADATA_KEY, col_meta)
+    col_meta[col] = metadata
+
+
+def get_spotfire_type(dataframe, col):
+    """Return the Spotfire type name for *col*, or ``None`` if not set."""
+    return _get(dataframe, _SPOTFIRE_TYPES_KEY, {}).get(col)
+
+
+def get_series_spotfire_type(series, col):
+    """Return the Spotfire type name to use when exporting a standalone ``Series``, or ``None`` if not set.
+
+    Looks in the DataFrame-style ``spotfire_types`` dict first (present when the metadata was
+    copied from a DataFrame), then falls back to the scalar ``series.attrs['spotfire_type']``
+    that callers set directly on a bare Series.
+    """
+    typename = get_spotfire_type(series, col)
+    if typename is None:
+        typename = series.attrs.get(_SERIES_TYPE_KEY)
+    return typename
+
+
+def set_spotfire_type(dataframe, col, typename):
+    """Set the Spotfire type name for *col*."""
+    types = _detach_if_shared(dataframe, _SPOTFIRE_TYPES_KEY)
+    if types is None:
+        types = _CowDict()
+        _set(dataframe, _SPOTFIRE_TYPES_KEY, types)
+    types[col] = typename
+
+
+def get_all_spotfire_types(dataframe):
+    """Return a dict mapping column name -> Spotfire type name for all columns that have one."""
+    return dict(_get(dataframe, _SPOTFIRE_TYPES_KEY, {}))
+
+
+def copy_all_metadata(source, destination):
+    """Copy all Spotfire metadata from *source* to *destination*."""
+    for key in _ALL_KEYS:
+        val = _get(source, key)
+        if val:
+            _set(destination, key, copy.deepcopy(val))
+
+
+# --- Deprecation helpers for old per-column patterns ---
+
+def _register_deprecated_accessors():
+    """Register pandas accessor that raises clear error for old per-column metadata pattern.
+
+    ``df['col'].spotfire_column_metadata = {...}`` is silently broken under pandas 3
+    Copy-on-Write (writes are discarded). This accessor intercepts READ attempts
+    and raises a helpful error pointing to the new API.
+    """
+    try:
+        import pandas as pd  # pylint: disable=import-outside-toplevel
+
+        @pd.api.extensions.register_series_accessor('spotfire_column_metadata')
+        class _DeprecatedColumnMetadata:  # pylint: disable=too-few-public-methods
+            def __init__(self, series):
+                raise AttributeError(
+                    "df['col'].spotfire_column_metadata is not supported under pandas 3 "
+                    "Copy-on-Write (writes are silently discarded). "
+                    "Use spotfire.get_column_metadata(df, 'col') and "
+                    "spotfire.set_column_metadata(df, 'col', metadata) instead."
+                )
+    except Exception:  # pylint: disable=broad-exception-caught
+        pass
+
+
+_register_deprecated_accessors()

spotfire/data_function.py

@@ -14,7 +14,9 @@
 import re
 import warnings
 
-from spotfire import sbdf, _utils
+import pandas as pd
+
+from spotfire import sbdf, _utils, _metadata
 
 
 _ExceptionInfo = typing.Union[
@@ -133,28 +135,24 @@ def read(self, globals_dict: _Globals, debug_fn: _LogFunction) -> None:
         debug_fn(f"read {dataframe.shape[0]} rows {dataframe.shape[1]} columns")
 
         # Table metadata
-        try:
-            if dataframe.spotfire_table_metadata:
-                table_meta = f"\n {pprint.pformat(dataframe.spotfire_table_metadata)}"
-            else:
-                table_meta = " (no table metadata present)"
-        except AttributeError:
+        table_md = _metadata.get_table_metadata(dataframe)
+        if table_md:
+            table_meta = f"\n {pprint.pformat(table_md)}"
+        else:
             table_meta = " (no table metadata present)"
         debug_fn(f"table metadata:{table_meta}")
 
         # Column metadata
         column_blank = False
         pretty_column = io.StringIO()
         for col in dataframe.columns:
-            try:
-                if pretty_column.tell() > _COLUMN_METADATA_TRUNCATE_THRESHOLD:
-                    pretty_column.write("\n (truncated due to length)")
-                    break
-                if dataframe[col].spotfire_column_metadata:
-                    pretty_column.write(f"\n {col}: {pprint.pformat(dataframe[col].spotfire_column_metadata)}")
-                else:
-                    column_blank = True
-            except AttributeError:
+            if pretty_column.tell() > _COLUMN_METADATA_TRUNCATE_THRESHOLD:
+                pretty_column.write("\n (truncated due to length)")
+                break
+            col_md = _metadata.get_column_metadata(dataframe, col)
+            if col_md:
+                pretty_column.write(f"\n {col}: {pprint.pformat(col_md)}")
+            else:
                 column_blank = True
         if pretty_column.tell():
             column_meta = pretty_column.getvalue()
@@ -170,14 +168,14 @@ def read(self, globals_dict: _Globals, debug_fn: _LogFunction) -> None:
             dataframe = dataframe[dataframe.columns[0]]
         if self._type == "value":
             value = dataframe.at[0, dataframe.columns[0]]
-            if type(value).__module__ == "numpy":
-                dataframe = value.tolist()
-            elif type(value).__module__ == "pandas._libs.tslibs.timedeltas":
-                dataframe = value.to_pytimedelta()
-            elif type(value).__module__ == "pandas._libs.tslibs.timestamps":
+            if isinstance(value, pd.Timestamp):
                 dataframe = value.to_pydatetime()
-            elif type(value).__module__ == "pandas._libs.tslibs.nattype":
+            elif isinstance(value, pd.Timedelta):
+                dataframe = value.to_pytimedelta()
+            elif pd.isna(value):
                 dataframe = None
+            elif type(value).__module__ == "numpy":
+                dataframe = value.tolist()
             else:
                 dataframe = value