diff --git a/packages/google-cloud-bigquery/google/cloud/bigquery/_pandas_helpers.py b/packages/google-cloud-bigquery/google/cloud/bigquery/_pandas_helpers.py index 7bd9f99b625f..77c0aec0e191 100644 --- a/packages/google-cloud-bigquery/google/cloud/bigquery/_pandas_helpers.py +++ b/packages/google-cloud-bigquery/google/cloud/bigquery/_pandas_helpers.py @@ -506,7 +506,6 @@ def dataframe_to_bq_schema(dataframe, bq_schema): bq_schema_out = [] unknown_type_columns = [] - dataframe_reset_index = dataframe.reset_index() for column, dtype in list_columns_and_indexes(dataframe): # Step 1: use provided type from schema, if present. bq_field = bq_schema_index.get(column) @@ -519,7 +518,8 @@ def dataframe_to_bq_schema(dataframe, bq_schema): # pandas dtype. bq_type = _PANDAS_DTYPE_TO_BQ.get(dtype.name) if bq_type is None: - sample_data = _first_valid(dataframe_reset_index[column]) + values = get_column_or_index(dataframe, column) + sample_data = _first_valid(values) if ( isinstance(sample_data, _BaseGeometry) and sample_data is not None # Paranoia @@ -530,7 +530,7 @@ def dataframe_to_bq_schema(dataframe, bq_schema): continue # Step 3: try with pyarrow if available - bq_field = _get_schema_by_pyarrow(column, dataframe_reset_index[column]) + bq_field = _get_schema_by_pyarrow(column, values) if bq_field is not None: bq_schema_out.append(bq_field) continue diff --git a/packages/google-cloud-bigquery/tests/unit/test__pandas_helpers.py b/packages/google-cloud-bigquery/tests/unit/test__pandas_helpers.py index 34da6370e039..2849b75b5062 100644 --- a/packages/google-cloud-bigquery/tests/unit/test__pandas_helpers.py +++ b/packages/google-cloud-bigquery/tests/unit/test__pandas_helpers.py @@ -1316,6 +1316,99 @@ def test_dataframe_to_bq_schema_returns_schema_with_pandas_gbq( assert got is not None +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") +@pytest.mark.parametrize( + "values, dtype, field_type, mode", + [ + (["hello", "world"], "object", "STRING", "NULLABLE"), + ([1, 2], "int64", "INTEGER", "NULLABLE"), + ([1, 2], "object", "INT64", "NULLABLE"), + ([], "int64", "INTEGER", "NULLABLE"), + ([None, None], "float64", "FLOAT", "NULLABLE"), + ( + [[], [datetime.datetime(2023, 1, 1)]], + "object", + "DATETIME", + "REPEATED", + ), + ], +) +def test_dataframe_to_bq_schema_overlapping_index_prefers_column( + module_under_test, monkeypatch, values, dtype, field_type, mode +): + monkeypatch.setattr(module_under_test, "pandas_gbq", None) + dataframe = pandas.DataFrame({"value": pandas.Series(values, dtype=dtype)}) + # Different values and duplicate labels check both column precedence and + # positional access when sampling values for type inference. + dataframe.index = pandas.Index(["index value"] * len(values), name="value") + original = dataframe.copy(deep=True) + + with pytest.warns(FutureWarning, match="pandas-gbq"): + returned_schema = module_under_test.dataframe_to_bq_schema(dataframe, []) + + assert returned_schema == (schema.SchemaField("value", field_type, mode),) + pandas.testing.assert_frame_equal(dataframe, original) + + +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +@pytest.mark.parametrize("values", [[], [None, None]]) +def test_dataframe_to_bq_schema_overlapping_index_unknown_type( + module_under_test, monkeypatch, values +): + monkeypatch.setattr(module_under_test, "pandas_gbq", None) + dataframe = pandas.DataFrame({"value": pandas.Series(values, dtype=object)}) + dataframe.index.name = "value" + original = dataframe.copy(deep=True) + + with pytest.warns(FutureWarning, match="pandas-gbq"): + with pytest.warns(UserWarning, match="Could not determine the type.*value"): + returned_schema = module_under_test.dataframe_to_bq_schema(dataframe, []) + + assert returned_schema is None + pandas.testing.assert_frame_equal(dataframe, original) + + +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +@pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`") +def test_dataframe_to_bq_schema_overlapping_multiindex(module_under_test, monkeypatch): + monkeypatch.setattr(module_under_test, "pandas_gbq", None) + dataframe = pandas.DataFrame({"value": ["hello", "world"]}, dtype=object) + dataframe.index = pandas.MultiIndex.from_arrays( + [[1, 1], ["a", "a"], [False, True]], names=["value", "row", None] + ) + original = dataframe.copy(deep=True) + + with pytest.warns(FutureWarning, match="pandas-gbq"): + returned_schema = module_under_test.dataframe_to_bq_schema(dataframe, []) + + assert returned_schema == ( + schema.SchemaField("row", "STRING"), + schema.SchemaField("value", "STRING"), + ) + pandas.testing.assert_frame_equal(dataframe, original) + + +@pytest.mark.skipif(pandas is None, reason="Requires `pandas`") +@pytest.mark.parametrize("values", [[], [None, None]]) +def test_dataframe_to_bq_schema_overlapping_index_with_schema( + module_under_test, monkeypatch, values +): + monkeypatch.setattr(module_under_test, "pandas_gbq", None) + dataframe = pandas.DataFrame({"value": pandas.Series(values, dtype=object)}) + dataframe.index.name = "value" + original = dataframe.copy(deep=True) + expected_schema = (schema.SchemaField("value", "INTEGER", "REQUIRED"),) + + with pytest.warns(FutureWarning, match="pandas-gbq"): + returned_schema = module_under_test.dataframe_to_bq_schema( + dataframe, expected_schema + ) + + assert returned_schema == expected_schema + pandas.testing.assert_frame_equal(dataframe, original) + + @pytest.mark.skipif(pandas is None, reason="Requires `pandas`") def test_dataframe_to_bq_schema_w_named_index(module_under_test, monkeypatch): monkeypatch.setattr(module_under_test, "pandas_gbq", None) diff --git a/packages/pandas-gbq/pandas_gbq/schema/pandas_to_bigquery.py b/packages/pandas-gbq/pandas_gbq/schema/pandas_to_bigquery.py index 6ea8fc5b512a..05583403e311 100644 --- a/packages/pandas-gbq/pandas_gbq/schema/pandas_to_bigquery.py +++ b/packages/pandas-gbq/pandas_gbq/schema/pandas_to_bigquery.py @@ -112,7 +112,13 @@ def dataframe_to_bigquery_fields( continue # Try to automatically determine the type based on a few rows of the data. - values = dataframe.reset_index()[column] + # Select the column or index level directly so that an index with the + # same name as a column is ignored, just as in list_columns_and_indexes. + if column in dataframe.columns: + values = dataframe[column] + else: + values = dataframe.index.get_level_values(column).to_series() + values = values.reset_index(drop=True) bq_field = values_to_bigquery_field(column, values, default_type=default_type) if bq_field: diff --git a/packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py b/packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py index 5557f2f0a402..08f7200cce05 100644 --- a/packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py +++ b/packages/pandas-gbq/tests/unit/schema/test_pandas_to_bigquery.py @@ -19,6 +19,77 @@ def module_under_test(): return pandas_to_bigquery +@pytest.mark.parametrize("index", [False, True]) +@pytest.mark.parametrize( + "values, dtype, field_type, mode", + [ + (["hello", "world"], "object", "STRING", "NULLABLE"), + ([1, 2], "int64", "INTEGER", "NULLABLE"), + ([1, 2], "object", "INTEGER", "NULLABLE"), + ([], "object", "STRING", "NULLABLE"), + ([None, None], "object", "STRING", "NULLABLE"), + ( + [[], [datetime.datetime(2023, 1, 1)]], + "object", + "DATETIME", + "REPEATED", + ), + ], +) +def test_dataframe_to_bigquery_fields_overlapping_index_prefers_column( + module_under_test, index, values, dtype, field_type, mode +): + dataframe = pandas.DataFrame({"value": pandas.Series(values, dtype=dtype)}) + # Different values and duplicate labels check both column precedence and + # positional access when sampling values for type inference. + dataframe.index = pandas.Index(["index value"] * len(values), name="value") + original = dataframe.copy(deep=True) + + returned_schema = module_under_test.dataframe_to_bigquery_fields( + dataframe, index=index + ) + + assert returned_schema == (schema.SchemaField("value", field_type, mode),) + pandas.testing.assert_frame_equal(dataframe, original) + + +@pytest.mark.parametrize("index", [False, True]) +def test_dataframe_to_bigquery_fields_overlapping_multiindex(module_under_test, index): + dataframe = pandas.DataFrame({"value": ["hello", "world"]}, dtype=object) + dataframe.index = pandas.MultiIndex.from_arrays( + [[1, 1], ["a", "a"], [False, True]], names=["value", "row", None] + ) + original = dataframe.copy(deep=True) + + returned_schema = module_under_test.dataframe_to_bigquery_fields( + dataframe, index=index + ) + + expected_schema = (schema.SchemaField("value", "STRING"),) + if index: + expected_schema = (schema.SchemaField("row", "STRING"),) + expected_schema + assert returned_schema == expected_schema + pandas.testing.assert_frame_equal(dataframe, original) + + +@pytest.mark.parametrize("index", [False, True]) +@pytest.mark.parametrize("values", [[], [None, None]]) +def test_dataframe_to_bigquery_fields_overlapping_index_with_schema( + module_under_test, index, values +): + dataframe = pandas.DataFrame({"value": pandas.Series(values, dtype=object)}) + dataframe.index.name = "value" + original = dataframe.copy(deep=True) + expected_schema = (schema.SchemaField("value", "INTEGER", "REQUIRED"),) + + returned_schema = module_under_test.dataframe_to_bigquery_fields( + dataframe, override_bigquery_fields=expected_schema, index=index + ) + + assert returned_schema == expected_schema + pandas.testing.assert_frame_equal(dataframe, original) + + def test_dataframe_to_bigquery_fields_w_named_index(module_under_test): df_data = collections.OrderedDict( [