fix(duckdb): use fetch_arrow_table() to be able to handle big timestamps

Author: cpcloud

ibis/backends/duckdb/__init__.py

@@ -153,7 +153,8 @@ def fetch_from_cursor(
         cursor: duckdb.DuckDBPyConnection,
         schema: sch.Schema,
     ):
-        df = cursor.cursor.fetch_df()
+        table = cursor.cursor.fetch_arrow_table()
+        df = table.to_pandas(timestamp_as_object=True)
         return schema.apply_to(df)
 
     def _metadata(self, query: str) -> Iterator[_ColumnMetadata]:

ibis/backends/pandas/client.py

@@ -289,6 +289,11 @@ def convert_element(values, names=out_dtype.names):
     return column.map(convert_element)
 
 
+@sch.convert.register(np.dtype, dt.Array, pd.Series)
+def convert_array_to_series(in_dtype, out_dtype, column):
+    return column.map(lambda x: x if x is None else list(x))
+
+
 dt.DataType.to_pandas = ibis_dtype_to_pandas  # type: ignore
 sch.Schema.to_pandas = ibis_schema_to_pandas  # type: ignore
 

ibis/backends/tests/test_temporal.py

@@ -857,8 +857,8 @@ def test_integer_cast_to_timestamp(backend, alltypes, df):
     ),
 )
 @pytest.mark.notimpl(
-    ["datafusion", "duckdb"],
-    reason="DataFusion and DuckDB backends assume ns resolution timestamps",
+    ["datafusion"],
+    reason="DataFusion backend assumes ns resolution timestamps",
 )
 @pytest.mark.notyet(
     ["pyspark"],

ibis/expr/schema.py

@@ -389,7 +389,7 @@ def apply_to(self, df: pd.DataFrame) -> pd.DataFrame:
                 # assume not equal
                 not_equal = True
 
-            if not_equal or isinstance(dtype, (dt.String, dt.Struct)):
+            if not_equal or not isinstance(dtype, dt.Primitive):
                 new_col = convert(col_dtype, dtype, col)
             else:
                 new_col = col