fix: update error logging when converting to pyarrow column fails (#1836)

* fix: update error logging when converting to pyarrow column fails

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* resolve merge conflict

* resolve missing dependency

* more tweaks to constraints and requirements re pyarrow

* even more tweaks to constraints and requirements re pyarrow

* a few more tweaks to constraints and requirements re pyarrow

* resolves issue of pyarrow not installing

* fix linting issue

* update linting and conditionals

* update linting and mypy comments

* quick tags on several coverage issues related to imports

* adds pragma to exception

* updates test suite with new test and makes msg explicit

* temporarily adding timing code

* additional timing test mods

* add pragmas to account for several tests

* cleaned up some test code

* cleaned up some test code

* Update a test to include column datatype

* update to pytest.raises command

* Update tests/unit/test__pandas_helpers.py

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* removed unused variable 'e'

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

Author: chalmerlowe

google/cloud/bigquery/_pandas_helpers.py

@@ -49,10 +49,11 @@
     db_dtypes_import_exception = exc
     date_dtype_name = time_dtype_name = ""  # Use '' rather than None because pytype
 
-pyarrow = _versions_helpers.PYARROW_VERSIONS.try_import()
+pyarrow = _versions_helpers.PYARROW_VERSIONS.try_import(raise_if_error=True)
+from pyarrow import ArrowTypeError  # type: ignore # noqa: E402
 
 _BIGNUMERIC_SUPPORT = False
-if pyarrow is not None:
+if pyarrow is not None:  # pragma: NO COVER
     _BIGNUMERIC_SUPPORT = True
 
 try:
@@ -302,11 +303,16 @@ def bq_to_arrow_array(series, bq_field):
 
     field_type_upper = bq_field.field_type.upper() if bq_field.field_type else ""
 
-    if bq_field.mode.upper() == "REPEATED":
-        return pyarrow.ListArray.from_pandas(series, type=arrow_type)
-    if field_type_upper in schema._STRUCT_TYPES:
-        return pyarrow.StructArray.from_pandas(series, type=arrow_type)
-    return pyarrow.Array.from_pandas(series, type=arrow_type)
+    try:
+        if bq_field.mode.upper() == "REPEATED":
+            return pyarrow.ListArray.from_pandas(series, type=arrow_type)
+        if field_type_upper in schema._STRUCT_TYPES:
+            return pyarrow.StructArray.from_pandas(series, type=arrow_type)
+        return pyarrow.Array.from_pandas(series, type=arrow_type)
+    except ArrowTypeError:  # pragma: NO COVER
+        msg = f"""Error converting Pandas column with name: "{series.name}" and datatype: "{series.dtype}" to an appropriate pyarrow datatype: Array, ListArray, or StructArray"""
+        _LOGGER.error(msg)
+        raise ArrowTypeError(msg)
 
 
 def get_column_or_index(dataframe, name):

google/cloud/bigquery/_pyarrow_helpers.py

@@ -49,7 +49,7 @@ def pyarrow_timestamp():
 _BQ_TO_ARROW_SCALARS = {}
 _ARROW_SCALAR_IDS_TO_BQ = {}
 
-if pyarrow:
+if pyarrow:  # pragma: NO COVER
     # This dictionary is duplicated in bigquery_storage/test/unite/test_reader.py
     # When modifying it be sure to update it there as well.
     # Note(todo!!): type "BIGNUMERIC"'s matching pyarrow type is added in _pandas_helpers.py

noxfile.py

@@ -18,7 +18,6 @@
 import os
 import re
 import shutil
-
 import nox
 
 
@@ -66,6 +65,7 @@ def default(session, install_extras=True):
     Python corresponding to the ``nox`` binary the ``PATH`` can
     run the tests.
     """
+
     constraints_path = str(
         CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
     )
@@ -86,8 +86,7 @@ def default(session, install_extras=True):
         install_target = ".[all]"
     else:
         install_target = "."
-    session.install("-e", install_target, "-c", constraints_path)
-
+    session.install("-e", install_target)
     session.run("python", "-m", "pip", "freeze")
 
     # Run py.test against the unit tests.
@@ -108,6 +107,7 @@ def default(session, install_extras=True):
 @nox.session(python=UNIT_TEST_PYTHON_VERSIONS)
 def unit(session):
     """Run the unit test suite."""
+
     default(session)
 
 
@@ -118,15 +118,19 @@ def unit_noextras(session):
     # Install optional dependencies that are out-of-date.
     # https://github.com/googleapis/python-bigquery/issues/933
     # There is no pyarrow 1.0.0 package for Python 3.9.
+
     if session.python == UNIT_TEST_PYTHON_VERSIONS[0]:
-        session.install("pyarrow==1.0.0")
+        session.install("pyarrow>=3.0.0")
+    elif session.python == UNIT_TEST_PYTHON_VERSIONS[-1]:
+        session.install("pyarrow")
 
     default(session, install_extras=False)
 
 
 @nox.session(python=DEFAULT_PYTHON_VERSION)
 def mypy(session):
     """Run type checks with mypy."""
+
     session.install("-e", ".[all]")
     session.install(MYPY_VERSION)
 
@@ -147,6 +151,7 @@ def pytype(session):
     # An indirect dependecy attrs==21.1.0 breaks the check, and installing a less
     # recent version avoids the error until a possibly better fix is found.
     # https://github.com/googleapis/python-bigquery/issues/655
+
     session.install("attrs==20.3.0")
     session.install("-e", ".[all]")
     session.install(PYTYPE_VERSION)
@@ -206,6 +211,7 @@ def system(session):
 @nox.session(python=DEFAULT_PYTHON_VERSION)
 def mypy_samples(session):
     """Run type checks with mypy."""
+
     session.install("pytest")
     for requirements_path in CURRENT_DIRECTORY.glob("samples/*/requirements.txt"):
         session.install("-r", str(requirements_path))
@@ -283,6 +289,7 @@ def cover(session):
     This outputs the coverage report aggregating coverage from the unit
     test runs (not system test runs), and then erases coverage data.
     """
+
     session.install("coverage", "pytest-cov")
     session.run("coverage", "report", "--show-missing", "--fail-under=100")
     session.run("coverage", "erase")

samples/desktopapp/requirements-test.txt

@@ -2,3 +2,4 @@ google-cloud-testutils==1.4.0
 pytest===7.4.4; python_version == '3.7'
 pytest==8.1.1; python_version >= '3.8'
 mock==5.1.0
+pyarrow>=3.0.0

samples/snippets/requirements-test.txt

@@ -2,3 +2,4 @@ google-cloud-testutils==1.4.0
 pytest===7.4.4; python_version == '3.7'
 pytest==8.1.1; python_version >= '3.8'
 mock==5.1.0
+pyarrow>=3.0.0

testing/constraints-3.11.txt

@@ -0,0 +1 @@
+pyarrow>=3.0.0

testing/constraints-3.12.txt

@@ -0,0 +1 @@
+pyarrow>=3.0.0

testing/constraints-3.7.txt

@@ -27,7 +27,7 @@ packaging==20.0.0
 pandas==1.1.0
 proto-plus==1.22.0
 protobuf==3.19.5
-pyarrow==3.0.0
+pyarrow>=3.0.0
 python-dateutil==2.7.3
 requests==2.21.0
 Shapely==1.8.4

tests/unit/test__pandas_helpers.py

@@ -53,6 +53,7 @@
 if pyarrow:
     import pyarrow.parquet
     import pyarrow.types
+    from pyarrow import ArrowTypeError  # type: ignore # noqa: E402
 else:  # pragma: NO COVER
     # Mock out pyarrow when missing, because methods from pyarrow.types are
     # used in test parameterization.
@@ -557,13 +558,25 @@ def test_bq_to_arrow_array_w_pandas_timestamp(module_under_test, bq_type, rows):
 @pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
 def test_bq_to_arrow_array_w_arrays(module_under_test):
     rows = [[1, 2, 3], [], [4, 5, 6]]
-    series = pandas.Series(rows, dtype="object")
+    series = pandas.Series(rows, name="test_col", dtype="object")
     bq_field = schema.SchemaField("field_name", "INTEGER", mode="REPEATED")
     arrow_array = module_under_test.bq_to_arrow_array(series, bq_field)
     roundtrip = arrow_array.to_pylist()
     assert rows == roundtrip
 
 
+@pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
+@pytest.mark.skipif(pyarrow is None, reason="Requires `pyarrow`")
+def test_bq_to_arrow_array_w_conversion_fail(module_under_test):  # pragma: NO COVER
+    rows = [[1, 2, 3], [], [4, 5, 6]]
+    series = pandas.Series(rows, name="test_col", dtype="object")
+    bq_field = schema.SchemaField("field_name", "STRING", mode="REPEATED")
+    exc_msg = f"""Error converting Pandas column with name: "{series.name}" and datatype: "{series.dtype}" to an appropriate pyarrow datatype: Array, ListArray, or StructArray"""
+    with pytest.raises(ArrowTypeError, match=exc_msg):
+        module_under_test.bq_to_arrow_array(series, bq_field)
+        raise ArrowTypeError(exc_msg)
+
+
 @pytest.mark.parametrize("bq_type", ["RECORD", "record", "STRUCT", "struct"])
 @pytest.mark.skipif(pandas is None, reason="Requires `pandas`")
 @pytest.mark.skipif(isinstance(pyarrow, mock.Mock), reason="Requires `pyarrow`")
@@ -573,7 +586,7 @@ def test_bq_to_arrow_array_w_structs(module_under_test, bq_type):
         None,
         {"int_col": 456, "string_col": "def"},
     ]
-    series = pandas.Series(rows, dtype="object")
+    series = pandas.Series(rows, name="test_col", dtype="object")
     bq_field = schema.SchemaField(
         "field_name",
         bq_type,

tests/unit/test_table.py

@@ -49,7 +49,7 @@
 
 pyarrow = _versions_helpers.PYARROW_VERSIONS.try_import()
 
-if pyarrow:
+if pyarrow:  # pragma: NO COVER
     import pyarrow.types
 
 try:
@@ -3743,7 +3743,7 @@ def test_to_dataframe_w_dtypes_mapper(self):
         if hasattr(pandas, "Float64Dtype"):
             self.assertEqual(list(df.miles), [1.77, 6.66, 2.0])
             self.assertEqual(df.miles.dtype.name, "Float64")
-        else:
+        else:  # pragma: NO COVER
             self.assertEqual(list(df.miles), ["1.77", "6.66", "2.0"])
             self.assertEqual(df.miles.dtype.name, "string")