Skip to content

fix(bigquery): handle overlapping dataframe index and column names #18834

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AliAbouelazm wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(bigquery): handle overlapping dataframe index and column names #18834

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions packages/google-cloud-bigquery/google/cloud/bigquery/_pandas_helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -506,7 +506,6 @@ def dataframe_to_bq_schema(dataframe, bq_schema):

bq_schema_out = []
unknown_type_columns = []
dataframe_reset_index = dataframe.reset_index()
for column, dtype in list_columns_and_indexes(dataframe):
# Step 1: use provided type from schema, if present.
bq_field = bq_schema_index.get(column)
Expand All @@ -519,7 +518,8 @@ def dataframe_to_bq_schema(dataframe, bq_schema):
# pandas dtype.
bq_type = _PANDAS_DTYPE_TO_BQ.get(dtype.name)
if bq_type is None:
sample_data = _first_valid(dataframe_reset_index[column])
values = get_column_or_index(dataframe, column)
sample_data = _first_valid(values)
Comment on lines 519 to +522

@gemini-code-assist gemini-code-assist Bot Oct 23, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

critical

If bq_type is not None, the if bq_type is None: block is skipped, which means values is never defined. When the execution falls through to Step 3, calling _get_schema_by_pyarrow(column, values) will raise an UnboundLocalError. To fix this, define values unconditionally before checking bq_type.

Suggested change
bq_type = _PANDAS_DTYPE_TO_BQ.get(dtype.name)
if bq_type is None:
sample_data = _first_valid(dataframe_reset_index[column])
values = get_column_or_index(dataframe, column)
sample_data = _first_valid(values)
values = get_column_or_index(dataframe, column)
bq_type = _PANDAS_DTYPE_TO_BQ.get(dtype.name)
if bq_type is None:
sample_data = _first_valid(values)

if (
isinstance(sample_data, _BaseGeometry)
and sample_data is not None # Paranoia
Expand All @@ -530,7 +530,7 @@ def dataframe_to_bq_schema(dataframe, bq_schema):
continue

# Step 3: try with pyarrow if available
bq_field = _get_schema_by_pyarrow(column, dataframe_reset_index[column])
bq_field = _get_schema_by_pyarrow(column, values)
if bq_field is not None:
bq_schema_out.append(bq_field)
continue
Expand Down
93 changes: 93 additions & 0 deletions packages/google-cloud-bigquery/tests/unit/test__pandas_helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1316,6 +1316,99 @@ def test_dataframe_to_bq_schema_returns_schema_with_pandas_gbq(
assert got is not None


@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
@pytest.mark.parametrize(
"values, dtype, field_type, mode",
[
(["hello", "world"], "object", "STRING", "NULLABLE"),
([1, 2], "int64", "INTEGER", "NULLABLE"),
([1, 2], "object", "INT64", "NULLABLE"),
([], "int64", "INTEGER", "NULLABLE"),
([None, None], "float64", "FLOAT", "NULLABLE"),
(
[[], [datetime.datetime(2023, 1, 1)]],
"object",
"DATETIME",
"REPEATED",
),
],
)
def test_dataframe_to_bq_schema_overlapping_index_prefers_column(
module_under_test, monkeypatch, values, dtype, field_type, mode
):
monkeypatch.setattr(module_under_test, "pandas_gbq", None)
dataframe = pandas.DataFrame({"value": pandas.Series(values, dtype=dtype)})
# Different values and duplicate labels check both column precedence and
# positional access when sampling values for type inference.
dataframe.index = pandas.Index(["index value"] * len(values), name="value")
original = dataframe.copy(deep=True)

with pytest.warns(FutureWarning, match="pandas-gbq"):
returned_schema = module_under_test.dataframe_to_bq_schema(dataframe, [])

assert returned_schema == (schema.SchemaField("value", field_type, mode),)
pandas.testing.assert_frame_equal(dataframe, original)


@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
@pytest.mark.parametrize("values", [[], [None, None]])
def test_dataframe_to_bq_schema_overlapping_index_unknown_type(
module_under_test, monkeypatch, values
):
monkeypatch.setattr(module_under_test, "pandas_gbq", None)
dataframe = pandas.DataFrame({"value": pandas.Series(values, dtype=object)})
dataframe.index.name = "value"
original = dataframe.copy(deep=True)

with pytest.warns(FutureWarning, match="pandas-gbq"):
with pytest.warns(UserWarning, match="Could not determine the type.*value"):
returned_schema = module_under_test.dataframe_to_bq_schema(dataframe, [])

assert returned_schema is None
pandas.testing.assert_frame_equal(dataframe, original)


@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
def test_dataframe_to_bq_schema_overlapping_multiindex(module_under_test, monkeypatch):
monkeypatch.setattr(module_under_test, "pandas_gbq", None)
dataframe = pandas.DataFrame({"value": ["hello", "world"]}, dtype=object)
dataframe.index = pandas.MultiIndex.from_arrays(
[[1, 1], ["a", "a"], [False, True]], names=["value", "row", None]
)
original = dataframe.copy(deep=True)

with pytest.warns(FutureWarning, match="pandas-gbq"):
returned_schema = module_under_test.dataframe_to_bq_schema(dataframe, [])

assert returned_schema == (
schema.SchemaField("row", "STRING"),
schema.SchemaField("value", "STRING"),
)
pandas.testing.assert_frame_equal(dataframe, original)


@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
@pytest.mark.parametrize("values", [[], [None, None]])
def test_dataframe_to_bq_schema_overlapping_index_with_schema(
module_under_test, monkeypatch, values
):
monkeypatch.setattr(module_under_test, "pandas_gbq", None)
dataframe = pandas.DataFrame({"value": pandas.Series(values, dtype=object)})
dataframe.index.name = "value"
original = dataframe.copy(deep=True)
expected_schema = (schema.SchemaField("value", "INTEGER", "REQUIRED"),)

with pytest.warns(FutureWarning, match="pandas-gbq"):
returned_schema = module_under_test.dataframe_to_bq_schema(
dataframe, expected_schema
)

assert returned_schema == expected_schema
pandas.testing.assert_frame_equal(dataframe, original)


@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
def test_dataframe_to_bq_schema_w_named_index(module_under_test, monkeypatch):
monkeypatch.setattr(module_under_test, "pandas_gbq", None)
Expand Down
8 changes: 7 additions & 1 deletion packages/pandas-gbq/pandas_gbq/schema/pandas_to_bigquery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,13 @@ def dataframe_to_bigquery_fields(
continue

# Try to automatically determine the type based on a few rows of the data.
values = dataframe.reset_index()[column]
# Select the column or index level directly so that an index with the
# same name as a column is ignored, just as in list_columns_and_indexes.
if column in dataframe.columns:
values = dataframe[column]
else:
values = dataframe.index.get_level_values(column).to_series()
values = values.reset_index(drop=True)
bq_field = values_to_bigquery_field(column, values, default_type=default_type)

if bq_field:
Expand Down
71 changes: 71 additions & 0 deletions packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,77 @@ def module_under_test():
return pandas_to_bigquery


@pytest.mark.parametrize("index", [False, True])
@pytest.mark.parametrize(
"values, dtype, field_type, mode",
[
(["hello", "world"], "object", "STRING", "NULLABLE"),
([1, 2], "int64", "INTEGER", "NULLABLE"),
([1, 2], "object", "INTEGER", "NULLABLE"),
([], "object", "STRING", "NULLABLE"),
([None, None], "object", "STRING", "NULLABLE"),
(
[[], [datetime.datetime(2023, 1, 1)]],
"object",
"DATETIME",
"REPEATED",
),
],
)
def test_dataframe_to_bigquery_fields_overlapping_index_prefers_column(
module_under_test, index, values, dtype, field_type, mode
):
dataframe = pandas.DataFrame({"value": pandas.Series(values, dtype=dtype)})
# Different values and duplicate labels check both column precedence and
# positional access when sampling values for type inference.
dataframe.index = pandas.Index(["index value"] * len(values), name="value")
original = dataframe.copy(deep=True)

returned_schema = module_under_test.dataframe_to_bigquery_fields(
dataframe, index=index
)

assert returned_schema == (schema.SchemaField("value", field_type, mode),)
pandas.testing.assert_frame_equal(dataframe, original)


@pytest.mark.parametrize("index", [False, True])
def test_dataframe_to_bigquery_fields_overlapping_multiindex(module_under_test, index):
dataframe = pandas.DataFrame({"value": ["hello", "world"]}, dtype=object)
dataframe.index = pandas.MultiIndex.from_arrays(
[[1, 1], ["a", "a"], [False, True]], names=["value", "row", None]
)
original = dataframe.copy(deep=True)

returned_schema = module_under_test.dataframe_to_bigquery_fields(
dataframe, index=index
)

expected_schema = (schema.SchemaField("value", "STRING"),)
if index:
expected_schema = (schema.SchemaField("row", "STRING"),) + expected_schema
assert returned_schema == expected_schema
pandas.testing.assert_frame_equal(dataframe, original)


@pytest.mark.parametrize("index", [False, True])
@pytest.mark.parametrize("values", [[], [None, None]])
def test_dataframe_to_bigquery_fields_overlapping_index_with_schema(
module_under_test, index, values
):
dataframe = pandas.DataFrame({"value": pandas.Series(values, dtype=object)})
dataframe.index.name = "value"
original = dataframe.copy(deep=True)
expected_schema = (schema.SchemaField("value", "INTEGER", "REQUIRED"),)

returned_schema = module_under_test.dataframe_to_bigquery_fields(
dataframe, override_bigquery_fields=expected_schema, index=index
)

assert returned_schema == expected_schema
pandas.testing.assert_frame_equal(dataframe, original)


def test_dataframe_to_bigquery_fields_w_named_index(module_under_test):
df_data = collections.OrderedDict(
[
Expand Down
Loading