Added Pandas 3 Support

imap_processing/ccsds/excel_to_xtce.py

@@ -457,7 +457,7 @@ def _add_state_conversion(self, row: pd.Series, parameter_type: Et.Element) -> N
             enumeration.attrib["value"] = str(valid_state["value"]).strip()
             enumeration.attrib["label"] = str(valid_state["state"]).strip()
 
-    def _ensure_state_value_is_int(self, state: dict) -> dict:
+    def _ensure_state_value_is_int(self, input_state: pd.Series | dict) -> dict:
         """
         Ensure the telemetry state value is an integer.
 
@@ -468,14 +468,18 @@ def _ensure_state_value_is_int(self, state: dict) -> dict:
 
         Parameters
         ----------
-        state : dict
-            Dictionary with telemetry state and value.
+        input_state : pandas.Series or dict
+            Telemetry state and value.
 
         Returns
         -------
         dict
             The dictionary for the state.
         """
+        # Excel rows are homogeneous pandas Series.  Convert to a plain mapping
+        # before replacing a hexadecimal string with an integer so the operation
+        # does not depend on the Series' inferred dtype.
+        state = dict(input_state)
         value = state["value"]
         # return if already an int
         if isinstance(value, int):

imap_processing/cdf/utils.py

@@ -10,6 +10,7 @@
 
 import imap_data_access
 import numpy as np
+import pandas as pd
 import xarray as xr
 from cdflib.logging import logger as cdflib_logger
 from cdflib.xarray import cdf_to_xarray, xarray_to_cdf
@@ -23,6 +24,43 @@
 logger = logging.getLogger(__name__)
 
 
+def _cdf_compatible_dataset(dataset: xr.Dataset) -> xr.Dataset:
+    """Return a shallow copy whose extension arrays are NumPy-backed.
+
+    ``cdflib`` expects array-valued variables to be backed by NumPy arrays.  In
+    particular, it cannot serialize the string extension arrays that pandas 3
+    uses by default.  Converting at the CDF boundary keeps the in-memory dataset
+    unchanged and also supports explicitly created extension arrays on pandas 2.
+
+    Parameters
+    ----------
+    dataset : xarray.Dataset
+        Dataset to prepare for serialization by ``cdflib``.
+
+    Returns
+    -------
+    xarray.Dataset
+        Shallow copy with extension-array variables converted to NumPy arrays.
+    """
+    converted = dataset.copy(deep=False)
+    for name, variable in dataset.variables.items():
+        if not isinstance(variable.data, pd.api.extensions.ExtensionArray):
+            continue
+
+        numpy_variable = xr.Variable(
+            variable.dims,
+            variable.data.to_numpy(copy=True),
+            attrs=variable.attrs,
+        )
+        numpy_variable.encoding = variable.encoding.copy()
+        if name in dataset.coords:
+            converted = converted.assign_coords({name: numpy_variable})
+        else:
+            converted[name] = numpy_variable
+
+    return converted
+
+
 def load_cdf(
     file_path: Path | str, remove_xarray_attrs: bool = True, **kwargs: dict
 ) -> xr.Dataset:
@@ -185,7 +223,11 @@ def write_cdf(
             # strict ISTP compliance
             logger.info("Disabling cdflib ISTP logging for level 1 data products")
             cdflib_logger.setLevel(logging.ERROR)
-        xarray_to_cdf(dataset, str(file_path), **extra_cdf_kwargs)
+        xarray_to_cdf(
+            _cdf_compatible_dataset(dataset),
+            str(file_path),
+            **extra_cdf_kwargs,
+        )
     finally:
         # Set back to the previous logging level
         cdflib_logger.setLevel(prev_cdflib_level)

imap_processing/ialirt/l0/process_swe.py

@@ -474,9 +474,8 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list
     accumulated_data["met"] = met
 
     # Drop any off-nominal SWE groups
-    nominal_data = accumulated_data.where(
-        accumulated_data["swe_nom_flag"] != 0,
-        drop=True,
+    nominal_data = accumulated_data.isel(
+        epoch=(accumulated_data["swe_nom_flag"] != 0).values
     )
 
     # Get total full cycle data available for processing.
@@ -502,8 +501,8 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list
         grouped = grouped_data.sel(epoch=group_mask)
 
         # Split into Q1 & Q2 (swe_seq 0-29) and Q3 & Q4 (swe_seq 30-59)
-        first_half = grouped.where(grouped["swe_seq"] < 30, drop=True)
-        second_half = grouped.where(grouped["swe_seq"] >= 30, drop=True)
+        first_half = grouped.isel(epoch=(grouped["swe_seq"] < 30).values)
+        second_half = grouped.isel(epoch=(grouped["swe_seq"] >= 30).values)
 
         # Prepare raw counts separately for both halves
         raw_counts_first_half = prepare_raw_counts(first_half)

imap_processing/ialirt/utils/grouping.py

@@ -39,9 +39,8 @@ def filter_valid_groups(grouped_data: xr.Dataset) -> xr.Dataset:
         else:
             logger.info(f"src_seq_ctr_diff != 1 for group {group}.")
 
-    filtered_data = grouped_data.where(
-        xr.DataArray(np.isin(grouped_data["group"], valid_groups), dims="epoch"),
-        drop=True,
+    filtered_data = grouped_data.isel(
+        epoch=np.isin(grouped_data["group"], valid_groups)
     )
 
     return filtered_data
@@ -105,9 +104,11 @@ def find_groups(
 
     # Filter data before the sequence_range=0
     # and after the last value of sequence_range.
-    grouped_data = sorted_data.where(
-        (sorted_data[time_name] >= start_time) & (sorted_data[time_name] <= end_time),
-        drop=True,
+    grouped_data = sorted_data.isel(
+        epoch=(
+            (sorted_data[time_name] >= start_time)
+            & (sorted_data[time_name] <= end_time)
+        ).values
     )
 
     # Assign labels based on the start_times.

imap_processing/idex/idex_l1b.py

@@ -443,6 +443,13 @@ def compute_trigger_values(
             vectorize=True,
             output_dtypes=[object, float],
         )
+        # Allocate the object array explicitly.  Otherwise pandas 3 string
+        # inference converts the no-trigger None values to NaN.
+        mode_values = np.asarray(mode_array.data, dtype=object)
+        mode_values[pd.isna(mode_values)] = None
+        object_mode_array = xr.full_like(modes, None, dtype=object)
+        object_mode_array.data[:] = mode_values
+        mode_array = object_mode_array
         # There should be an array of modes and threshold levels for each channel.
         # write each of them out as separate variables because there may be
         # multiple channels that can trigger an event. The trigger origin variable

imap_processing/tests/cdf/test_utils.py

@@ -4,6 +4,7 @@
 
 import imap_data_access
 import numpy as np
+import pandas as pd
 import pytest
 import xarray as xr
 
@@ -140,6 +141,24 @@ def test_write_cdf_extra_cdf_kwargs(test_dataset):
         assert xarray_to_cdf.call_args.kwargs["compression"] == 9
 
 
+def test_write_cdf_converts_extension_array(test_dataset):
+    """Convert extension arrays for cdflib without changing the input dataset."""
+    test_dataset["labels"] = (
+        "label",
+        pd.array(["first", "second"], dtype="string"),
+    )
+
+    with mock.patch(
+        "imap_processing.cdf.utils.xarray_to_cdf", autospec=True
+    ) as xarray_to_cdf:
+        write_cdf(test_dataset)
+
+    converted_dataset = xarray_to_cdf.call_args.args[0]
+    assert isinstance(converted_dataset["labels"].data, np.ndarray)
+    np.testing.assert_array_equal(converted_dataset["labels"], ["first", "second"])
+    assert isinstance(test_dataset["labels"].data, pd.api.extensions.ExtensionArray)
+
+
 @pytest.mark.parametrize(
     "test_str, compare_dict",
     [

imap_processing/tests/ialirt/unit/test_process_swapi.py

@@ -224,7 +224,7 @@ def test_optimize_parameters():
             f"{imap_module_directory}/tests/ialirt/data/l0/"
             f"{test_data[test_set]['file_name']}",
         )
-        count_rates = energy_data["Count Rates [Hz]"].to_numpy()
+        count_rates = energy_data["Count Rates [Hz]"].to_numpy(copy=True)
         count_rates[0] = 0.0
         count_rates_errors = energy_data["Count Rates Error [Hz]"].to_numpy()
 
@@ -264,7 +264,7 @@ def test_optimize_parameters_exception_handling():
     energy_data = pd.read_csv(
         f"{imap_module_directory}/tests/ialirt/data/l0/{file_name}"
     )
-    count_rates = energy_data["Count Rates [Hz]"].to_numpy()
+    count_rates = energy_data["Count Rates [Hz]"].to_numpy(copy=True)
     count_rates[0] = 0.0
     count_rates = np.tile(count_rates, (2, 1))
     count_rates_errors = energy_data["Count Rates Error [Hz]"].to_numpy()
@@ -306,7 +306,7 @@ def test_optimize_parameters_bad_fit_handling():
     energy_data = pd.read_csv(
         f"{imap_module_directory}/tests/ialirt/data/l0/{file_name}"
     )
-    count_rates = energy_data["Count Rates [Hz]"].to_numpy()
+    count_rates = energy_data["Count Rates [Hz]"].to_numpy(copy=True)
     count_rates[0] = 0.0
     count_rates_errors = energy_data["Count Rates Error [Hz]"].to_numpy()
 
@@ -343,9 +343,9 @@ def test_optimize_parameters_bad_covariance_handling():
     energy_data = pd.read_csv(
         f"{imap_module_directory}/tests/ialirt/data/l0/{file_name}"
     )
-    count_rates = energy_data["Count Rates [Hz]"].to_numpy()
+    count_rates = energy_data["Count Rates [Hz]"].to_numpy(copy=True)
     count_rates[0] = 0.0
-    count_rates_errors = energy_data["Count Rates Error [Hz]"].to_numpy()
+    count_rates_errors = energy_data["Count Rates Error [Hz]"].to_numpy(copy=True)
 
     # setting errors to 0 results in infinite covariance
     count_rates_errors *= 0

imap_processing/tests/mag/test_mag_l2.py

@@ -270,7 +270,7 @@ def test_offset_application(norm_dataset, mag_test_l2_data):
     new_timeshift[1] = -0.00001
     new_timeshift[2] = 1e-9
 
-    expected_timeshift = norm_dataset["epoch"].data
+    expected_timeshift = norm_dataset["epoch"].data.copy()
     # Timeshift is provided in seconds, epoch is in nanoseconds
     expected_timeshift[0] = expected_timeshift[0] + 10000
     expected_timeshift[1] = expected_timeshift[1] - 10000

imap_processing/tests/swapi/test_swapi_l1.py

@@ -94,7 +94,10 @@ def test_find_sweep_starts():
     time = np.arange(26)
     sequence_number = time % 12
     ds = xr.Dataset(
-        {"seq_number": sequence_number, "shcoarse": np.arange(1, 27, 1)},
+        {
+            "seq_number": ("epoch", sequence_number),
+            "shcoarse": ("epoch", np.arange(1, 27, 1)),
+        },
         coords={"epoch": met_to_ttj2000ns(time)},
     )
 
@@ -107,7 +110,7 @@ def test_find_sweep_starts():
 
     # Creating test data that doesn't have start sequence.
     # Sequence number range is 0-11.
-    ds["seq_number"] = np.arange(3, 29)
+    ds["seq_number"] = ("epoch", np.arange(3, 29))
     start_indices = find_sweep_starts(ds)
     np.testing.assert_array_equal(start_indices, [])
 

imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py

@@ -359,10 +359,10 @@ def test_flag_low_voltage(test_data):
     n_spins = 20
     mock_status_dataset = xr.Dataset(
         data_vars={
-            "shcoarse": np.arange(n_spins),
+            "shcoarse": ("epoch", np.arange(n_spins)),
             # Set Voltage below threshold
-            "rightdeflection_v": np.full(n_spins, 0.5),
-            "leftdeflection_v": np.full(n_spins, 1.5),
+            "rightdeflection_v": ("epoch", np.full(n_spins, 0.5)),
+            "leftdeflection_v": ("epoch", np.full(n_spins, 1.5)),
         }
     )
     spins = np.arange(n_spins)
@@ -452,15 +452,15 @@ def test_get_energy_and_spin_dependent_rejection_mask():
     ]  # Example energy bin edges (4 edges = 3 bins)
     goodtimes_dataset = xr.Dataset(
         data_vars={
-            "spin_number": np.arange(n_spins),
-            "quality_low_voltage": np.full(n_spins, 0),
-            "quality_high_energy": np.full(n_spins, 0),
-            "quality_statistics": np.full(n_spins, 0),
-            "energy_range_flags": energy_range_flags,
-            "energy_range_edges": energy_range_edges,
-            "quality_upstream_ion_1": np.full(n_spins, 0),
-            "quality_upstream_ion_2": np.full(n_spins, 0),
-            "quality_spectral": np.full(n_spins, 0),
+            "spin_number": ("spin", np.arange(n_spins)),
+            "quality_low_voltage": ("spin", np.full(n_spins, 0)),
+            "quality_high_energy": ("spin", np.full(n_spins, 0)),
+            "quality_statistics": ("spin", np.full(n_spins, 0)),
+            "energy_range_flags": ("energy_bin", energy_range_flags),
+            "energy_range_edges": ("energy_edge", energy_range_edges),
+            "quality_upstream_ion_1": ("spin", np.full(n_spins, 0)),
+            "quality_upstream_ion_2": ("spin", np.full(n_spins, 0)),
+            "quality_spectral": ("spin", np.full(n_spins, 0)),
         }
     )
     # update quality flags to test that events get rejected

imap_processing/ultra/l1b/lookup_utils.py

@@ -149,9 +149,9 @@ def get_energy_norm(
     norm_composite_energy : np.ndarray
         Normalized composite energy.
     """
-    row_number = ssd * 4096 + composite_energy
+    row_number = (ssd * 4096 + composite_energy).astype(np.intp)
     norm_lookup = pd.read_csv(ancillary_files["l1b-egynorm-lookup"])
-    return norm_lookup["NormEnergy"].iloc[row_number]
+    return norm_lookup["NormEnergy"].to_numpy(copy=True)[row_number]
 
 
 def get_image_params(image: str, sensor: str, ancillary_files: dict) -> np.float64: