From 5cd19928a0959a16bab2e297ff4a0f1b6e40e7ea Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Tue, 11 Aug 2026 16:55:12 -0400 Subject: [PATCH 1/3] imap_processing/tests/ialirt/unit/test_process_swapi.py: added to_numpy(copy=True) when array is mutated, supporting Pandas 3 --- .../tests/ialirt/unit/test_process_swapi.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/imap_processing/tests/ialirt/unit/test_process_swapi.py b/imap_processing/tests/ialirt/unit/test_process_swapi.py index e1f0a04b2d..a59ba4a791 100644 --- a/imap_processing/tests/ialirt/unit/test_process_swapi.py +++ b/imap_processing/tests/ialirt/unit/test_process_swapi.py @@ -224,7 +224,7 @@ def test_optimize_parameters(): f"{imap_module_directory}/tests/ialirt/data/l0/" f"{test_data[test_set]['file_name']}", ) - count_rates = energy_data["Count Rates [Hz]"].to_numpy() + count_rates = energy_data["Count Rates [Hz]"].to_numpy(copy=True) count_rates[0] = 0.0 count_rates_errors = energy_data["Count Rates Error [Hz]"].to_numpy() @@ -264,7 +264,7 @@ def test_optimize_parameters_exception_handling(): energy_data = pd.read_csv( f"{imap_module_directory}/tests/ialirt/data/l0/{file_name}" ) - count_rates = energy_data["Count Rates [Hz]"].to_numpy() + count_rates = energy_data["Count Rates [Hz]"].to_numpy(copy=True) count_rates[0] = 0.0 count_rates = np.tile(count_rates, (2, 1)) count_rates_errors = energy_data["Count Rates Error [Hz]"].to_numpy() @@ -306,7 +306,7 @@ def test_optimize_parameters_bad_fit_handling(): energy_data = pd.read_csv( f"{imap_module_directory}/tests/ialirt/data/l0/{file_name}" ) - count_rates = energy_data["Count Rates [Hz]"].to_numpy() + count_rates = energy_data["Count Rates [Hz]"].to_numpy(copy=True) count_rates[0] = 0.0 count_rates_errors = energy_data["Count Rates Error [Hz]"].to_numpy() @@ -343,9 +343,9 @@ def test_optimize_parameters_bad_covariance_handling(): energy_data = pd.read_csv( f"{imap_module_directory}/tests/ialirt/data/l0/{file_name}" ) - count_rates = energy_data["Count Rates [Hz]"].to_numpy() + count_rates = energy_data["Count Rates [Hz]"].to_numpy(copy=True) count_rates[0] = 0.0 - count_rates_errors = energy_data["Count Rates Error [Hz]"].to_numpy() + count_rates_errors = energy_data["Count Rates Error [Hz]"].to_numpy(copy=True) # setting errors to 0 results in infinite covariance count_rates_errors *= 0 From f1e0ccadcce1c4c7ab497f9c2ee3b263beb3bb7f Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Wed, 12 Aug 2026 13:17:26 -0400 Subject: [PATCH 2/3] imap_processing: fixed additional files to ensure compatibility with pandas 2 and 3 --- imap_processing/ccsds/excel_to_xtce.py | 10 +++-- imap_processing/cdf/utils.py | 44 ++++++++++++++++++- imap_processing/ialirt/l0/process_swe.py | 9 ++-- imap_processing/ialirt/utils/grouping.py | 13 +++--- imap_processing/idex/idex_l1b.py | 7 +++ imap_processing/tests/cdf/test_utils.py | 19 ++++++++ imap_processing/tests/mag/test_mag_l2.py | 2 +- imap_processing/tests/swapi/test_swapi_l1.py | 7 ++- .../ultra/unit/test_ultra_l1b_culling.py | 24 +++++----- imap_processing/ultra/l1b/lookup_utils.py | 4 +- 10 files changed, 107 insertions(+), 32 deletions(-) diff --git a/imap_processing/ccsds/excel_to_xtce.py b/imap_processing/ccsds/excel_to_xtce.py index 79a3a36cc8..d80f50c8a3 100644 --- a/imap_processing/ccsds/excel_to_xtce.py +++ b/imap_processing/ccsds/excel_to_xtce.py @@ -457,7 +457,7 @@ def _add_state_conversion(self, row: pd.Series, parameter_type: Et.Element) -> N enumeration.attrib["value"] = str(valid_state["value"]).strip() enumeration.attrib["label"] = str(valid_state["state"]).strip() - def _ensure_state_value_is_int(self, state: dict) -> dict: + def _ensure_state_value_is_int(self, input_state: pd.Series | dict) -> dict: """ Ensure the telemetry state value is an integer. @@ -468,14 +468,18 @@ def _ensure_state_value_is_int(self, state: dict) -> dict: Parameters ---------- - state : dict - Dictionary with telemetry state and value. + input_state : pandas.Series or dict + Telemetry state and value. Returns ------- dict The dictionary for the state. """ + # Excel rows are homogeneous pandas Series. Convert to a plain mapping + # before replacing a hexadecimal string with an integer so the operation + # does not depend on the Series' inferred dtype. + state = dict(input_state) value = state["value"] # return if already an int if isinstance(value, int): diff --git a/imap_processing/cdf/utils.py b/imap_processing/cdf/utils.py index 82a73a187a..001470f246 100644 --- a/imap_processing/cdf/utils.py +++ b/imap_processing/cdf/utils.py @@ -10,6 +10,7 @@ import imap_data_access import numpy as np +import pandas as pd import xarray as xr from cdflib.logging import logger as cdflib_logger from cdflib.xarray import cdf_to_xarray, xarray_to_cdf @@ -23,6 +24,43 @@ logger = logging.getLogger(__name__) +def _cdf_compatible_dataset(dataset: xr.Dataset) -> xr.Dataset: + """Return a shallow copy whose extension arrays are NumPy-backed. + + ``cdflib`` expects array-valued variables to be backed by NumPy arrays. In + particular, it cannot serialize the string extension arrays that pandas 3 + uses by default. Converting at the CDF boundary keeps the in-memory dataset + unchanged and also supports explicitly created extension arrays on pandas 2. + + Parameters + ---------- + dataset : xarray.Dataset + Dataset to prepare for serialization by ``cdflib``. + + Returns + ------- + xarray.Dataset + Shallow copy with extension-array variables converted to NumPy arrays. + """ + converted = dataset.copy(deep=False) + for name, variable in dataset.variables.items(): + if not isinstance(variable.data, pd.api.extensions.ExtensionArray): + continue + + numpy_variable = xr.Variable( + variable.dims, + variable.data.to_numpy(copy=True), + attrs=variable.attrs, + ) + numpy_variable.encoding = variable.encoding.copy() + if name in dataset.coords: + converted = converted.assign_coords({name: numpy_variable}) + else: + converted[name] = numpy_variable + + return converted + + def load_cdf( file_path: Path | str, remove_xarray_attrs: bool = True, **kwargs: dict ) -> xr.Dataset: @@ -185,7 +223,11 @@ def write_cdf( # strict ISTP compliance logger.info("Disabling cdflib ISTP logging for level 1 data products") cdflib_logger.setLevel(logging.ERROR) - xarray_to_cdf(dataset, str(file_path), **extra_cdf_kwargs) + xarray_to_cdf( + _cdf_compatible_dataset(dataset), + str(file_path), + **extra_cdf_kwargs, + ) finally: # Set back to the previous logging level cdflib_logger.setLevel(prev_cdflib_level) diff --git a/imap_processing/ialirt/l0/process_swe.py b/imap_processing/ialirt/l0/process_swe.py index 3ae639ae5e..0e57563fab 100644 --- a/imap_processing/ialirt/l0/process_swe.py +++ b/imap_processing/ialirt/l0/process_swe.py @@ -474,9 +474,8 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list accumulated_data["met"] = met # Drop any off-nominal SWE groups - nominal_data = accumulated_data.where( - accumulated_data["swe_nom_flag"] != 0, - drop=True, + nominal_data = accumulated_data.isel( + epoch=(accumulated_data["swe_nom_flag"] != 0).values ) # Get total full cycle data available for processing. @@ -502,8 +501,8 @@ def process_swe(accumulated_data: xr.Dataset, in_flight_cal_files: list) -> list grouped = grouped_data.sel(epoch=group_mask) # Split into Q1 & Q2 (swe_seq 0-29) and Q3 & Q4 (swe_seq 30-59) - first_half = grouped.where(grouped["swe_seq"] < 30, drop=True) - second_half = grouped.where(grouped["swe_seq"] >= 30, drop=True) + first_half = grouped.isel(epoch=(grouped["swe_seq"] < 30).values) + second_half = grouped.isel(epoch=(grouped["swe_seq"] >= 30).values) # Prepare raw counts separately for both halves raw_counts_first_half = prepare_raw_counts(first_half) diff --git a/imap_processing/ialirt/utils/grouping.py b/imap_processing/ialirt/utils/grouping.py index 30fdf5a009..13936c9c67 100644 --- a/imap_processing/ialirt/utils/grouping.py +++ b/imap_processing/ialirt/utils/grouping.py @@ -39,9 +39,8 @@ def filter_valid_groups(grouped_data: xr.Dataset) -> xr.Dataset: else: logger.info(f"src_seq_ctr_diff != 1 for group {group}.") - filtered_data = grouped_data.where( - xr.DataArray(np.isin(grouped_data["group"], valid_groups), dims="epoch"), - drop=True, + filtered_data = grouped_data.isel( + epoch=np.isin(grouped_data["group"], valid_groups) ) return filtered_data @@ -105,9 +104,11 @@ def find_groups( # Filter data before the sequence_range=0 # and after the last value of sequence_range. - grouped_data = sorted_data.where( - (sorted_data[time_name] >= start_time) & (sorted_data[time_name] <= end_time), - drop=True, + grouped_data = sorted_data.isel( + epoch=( + (sorted_data[time_name] >= start_time) + & (sorted_data[time_name] <= end_time) + ).values ) # Assign labels based on the start_times. diff --git a/imap_processing/idex/idex_l1b.py b/imap_processing/idex/idex_l1b.py index b8751230dc..c3f9a3db2f 100644 --- a/imap_processing/idex/idex_l1b.py +++ b/imap_processing/idex/idex_l1b.py @@ -443,6 +443,13 @@ def compute_trigger_values( vectorize=True, output_dtypes=[object, float], ) + # Allocate the object array explicitly. Otherwise pandas 3 string + # inference converts the no-trigger None values to NaN. + mode_values = np.asarray(mode_array.data, dtype=object) + mode_values[pd.isna(mode_values)] = None + object_mode_array = xr.full_like(modes, None, dtype=object) + object_mode_array.data[:] = mode_values + mode_array = object_mode_array # There should be an array of modes and threshold levels for each channel. # write each of them out as separate variables because there may be # multiple channels that can trigger an event. The trigger origin variable diff --git a/imap_processing/tests/cdf/test_utils.py b/imap_processing/tests/cdf/test_utils.py index 49214d30bb..73683c0c72 100644 --- a/imap_processing/tests/cdf/test_utils.py +++ b/imap_processing/tests/cdf/test_utils.py @@ -4,6 +4,7 @@ import imap_data_access import numpy as np +import pandas as pd import pytest import xarray as xr @@ -140,6 +141,24 @@ def test_write_cdf_extra_cdf_kwargs(test_dataset): assert xarray_to_cdf.call_args.kwargs["compression"] == 9 +def test_write_cdf_converts_extension_array(test_dataset): + """Convert extension arrays for cdflib without changing the input dataset.""" + test_dataset["labels"] = ( + "label", + pd.array(["first", "second"], dtype="string"), + ) + + with mock.patch( + "imap_processing.cdf.utils.xarray_to_cdf", autospec=True + ) as xarray_to_cdf: + write_cdf(test_dataset) + + converted_dataset = xarray_to_cdf.call_args.args[0] + assert isinstance(converted_dataset["labels"].data, np.ndarray) + np.testing.assert_array_equal(converted_dataset["labels"], ["first", "second"]) + assert isinstance(test_dataset["labels"].data, pd.api.extensions.ExtensionArray) + + @pytest.mark.parametrize( "test_str, compare_dict", [ diff --git a/imap_processing/tests/mag/test_mag_l2.py b/imap_processing/tests/mag/test_mag_l2.py index 0520b62cf3..fbdbf15c40 100644 --- a/imap_processing/tests/mag/test_mag_l2.py +++ b/imap_processing/tests/mag/test_mag_l2.py @@ -270,7 +270,7 @@ def test_offset_application(norm_dataset, mag_test_l2_data): new_timeshift[1] = -0.00001 new_timeshift[2] = 1e-9 - expected_timeshift = norm_dataset["epoch"].data + expected_timeshift = norm_dataset["epoch"].data.copy() # Timeshift is provided in seconds, epoch is in nanoseconds expected_timeshift[0] = expected_timeshift[0] + 10000 expected_timeshift[1] = expected_timeshift[1] - 10000 diff --git a/imap_processing/tests/swapi/test_swapi_l1.py b/imap_processing/tests/swapi/test_swapi_l1.py index 36437afbf3..1054b1ecb3 100644 --- a/imap_processing/tests/swapi/test_swapi_l1.py +++ b/imap_processing/tests/swapi/test_swapi_l1.py @@ -94,7 +94,10 @@ def test_find_sweep_starts(): time = np.arange(26) sequence_number = time % 12 ds = xr.Dataset( - {"seq_number": sequence_number, "shcoarse": np.arange(1, 27, 1)}, + { + "seq_number": ("epoch", sequence_number), + "shcoarse": ("epoch", np.arange(1, 27, 1)), + }, coords={"epoch": met_to_ttj2000ns(time)}, ) @@ -107,7 +110,7 @@ def test_find_sweep_starts(): # Creating test data that doesn't have start sequence. # Sequence number range is 0-11. - ds["seq_number"] = np.arange(3, 29) + ds["seq_number"] = ("epoch", np.arange(3, 29)) start_indices = find_sweep_starts(ds) np.testing.assert_array_equal(start_indices, []) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py index 94852f9b9b..f487039def 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py @@ -359,10 +359,10 @@ def test_flag_low_voltage(test_data): n_spins = 20 mock_status_dataset = xr.Dataset( data_vars={ - "shcoarse": np.arange(n_spins), + "shcoarse": ("epoch", np.arange(n_spins)), # Set Voltage below threshold - "rightdeflection_v": np.full(n_spins, 0.5), - "leftdeflection_v": np.full(n_spins, 1.5), + "rightdeflection_v": ("epoch", np.full(n_spins, 0.5)), + "leftdeflection_v": ("epoch", np.full(n_spins, 1.5)), } ) spins = np.arange(n_spins) @@ -452,15 +452,15 @@ def test_get_energy_and_spin_dependent_rejection_mask(): ] # Example energy bin edges (4 edges = 3 bins) goodtimes_dataset = xr.Dataset( data_vars={ - "spin_number": np.arange(n_spins), - "quality_low_voltage": np.full(n_spins, 0), - "quality_high_energy": np.full(n_spins, 0), - "quality_statistics": np.full(n_spins, 0), - "energy_range_flags": energy_range_flags, - "energy_range_edges": energy_range_edges, - "quality_upstream_ion_1": np.full(n_spins, 0), - "quality_upstream_ion_2": np.full(n_spins, 0), - "quality_spectral": np.full(n_spins, 0), + "spin_number": ("spin", np.arange(n_spins)), + "quality_low_voltage": ("spin", np.full(n_spins, 0)), + "quality_high_energy": ("spin", np.full(n_spins, 0)), + "quality_statistics": ("spin", np.full(n_spins, 0)), + "energy_range_flags": ("energy_bin", energy_range_flags), + "energy_range_edges": ("energy_edge", energy_range_edges), + "quality_upstream_ion_1": ("spin", np.full(n_spins, 0)), + "quality_upstream_ion_2": ("spin", np.full(n_spins, 0)), + "quality_spectral": ("spin", np.full(n_spins, 0)), } ) # update quality flags to test that events get rejected diff --git a/imap_processing/ultra/l1b/lookup_utils.py b/imap_processing/ultra/l1b/lookup_utils.py index fb9dad25c8..2181905d79 100644 --- a/imap_processing/ultra/l1b/lookup_utils.py +++ b/imap_processing/ultra/l1b/lookup_utils.py @@ -149,9 +149,9 @@ def get_energy_norm( norm_composite_energy : np.ndarray Normalized composite energy. """ - row_number = ssd * 4096 + composite_energy + row_number = (ssd * 4096 + composite_energy).astype(np.intp) norm_lookup = pd.read_csv(ancillary_files["l1b-egynorm-lookup"]) - return norm_lookup["NormEnergy"].iloc[row_number] + return norm_lookup["NormEnergy"].to_numpy(copy=True)[row_number] def get_image_params(image: str, sensor: str, ancillary_files: dict) -> np.float64: From b4fa1f40a3c9dda392a18baae541c6ff3f1b11e5 Mon Sep 17 00:00:00 2001 From: Leo Werneck Date: Fri, 14 Aug 2026 14:21:34 -0400 Subject: [PATCH 3/3] .github/workflows/test.yml: added workflow to test pandas 3 support --- .github/workflows/test.yml | 46 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d6b6b30d4c..3b402c291e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,3 +72,49 @@ jobs: uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} + + pandas-3-compatibility: + name: Pandas 3 compatibility (Python ${{ matrix.python-version }}) + # This is an advisory compatibility check and must not block pull requests. + continue-on-error: true + runs-on: ubuntu-latest + strategy: + # Run both supported Python versions even if one compatibility job fails. + fail-fast: false + matrix: + # The first Python version below is the oldest version with Pandas 3 + # support that is also supported by IMAP, while the second is the + # newest Python version supported by IMAP. + python-version: ['3.11', '3.12'] + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@v4 + with: + # We need the full history to generate the proper version number. + fetch-depth: 0 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "2.3.4" + + - name: Install dependencies, app, and Pandas 3 + run: | + poetry self add "poetry-dynamic-versioning[plugin]" + poetry install --extras "test" + poetry run python -m pip install "pandas>=3,<4" + poetry run python -c "import pandas; print(f'Pandas {pandas.__version__}')" + + # Run tests without external data. + - name: Test with Pandas 3 + run: | + # Coverage is produced and uploaded by the blocking primary matrix. Do + # not upload duplicate advisory results or affect the coverage report. + poetry run pytest -vvv -n auto --color=yes --log-disable=root \ + -m "not external_kernel and not external_test_data"