[ibis 🦩] check backend: use positional join for duckdb and polars, fix ibis DataFrameModel.validate types (#2071)

* update docs banner with ibis announcement

Signed-off-by: Niels Bantilan <[email protected]>

* override ibis DataFrameModel.validate method to pass correct type

Signed-off-by: Niels Bantilan <[email protected]>

* [ibis 🦩] check backend: use positional join for duckdb and polars

Signed-off-by: Niels Bantilan <[email protected]>

* use typing_extensions for self in py<3.11

Signed-off-by: Niels Bantilan <[email protected]>

* Update pandera/backends/ibis/checks.py

Co-authored-by: Deepyaman Datta <[email protected]>

* Update docs/source/conf.py

Co-authored-by: Deepyaman Datta <[email protected]>

* Update conf.py

* Update checks.py

---------

Signed-off-by: Niels Bantilan <[email protected]>
Co-authored-by: Deepyaman Datta <[email protected]>

Author: cosmicBboy

docs/source/conf.py

@@ -134,9 +134,9 @@
 # documentation.
 
 announcement = """
-📢 Pandera 0.24.0 introduces the <i>pandera.pandas</i>
-module, which is the recommended way of defining schemas for <i>pandas objects</i>.
-Learn more details <a href='https://github.com/unionai-oss/pandera/releases/tag/v0.24.0'>here</a>
+📢 Pandera 0.25.0 introduces the <i>🦩 pandera-ibis integration </i>!
+Validate all supported Ibis backends, including Snowflake, BigQuery, and more.
+Learn more details <a href='./ibis.html'>here</a>
 """
 
 html_logo = "_static/pandera-banner.png"

pandera/api/ibis/model.py

@@ -1,11 +1,20 @@
 """Class-based API for Ibis models."""
 
+import sys
 import inspect
-from typing import Dict, List, Tuple
+from typing import (
+    Dict,
+    List,
+    Tuple,
+    Type,
+    Optional,
+    cast,
+)
 
 import ibis
 import ibis.expr.datatypes as dt
 
+from pandera.api.base.schema import BaseSchema
 from pandera.api.checks import Check
 from pandera.api.dataframe.model import DataFrameModel as _DataFrameModel
 from pandera.api.dataframe.model import get_dtype_kwargs
@@ -15,6 +24,15 @@
 from pandera.engines import ibis_engine
 from pandera.errors import SchemaInitError
 from pandera.typing import AnnotationInfo
+from pandera.typing.ibis import Table
+from pandera.utils import docstring_substitution
+
+
+# if python version is < 3.11, import Self from typing_extensions
+if sys.version_info < (3, 11):
+    from typing_extensions import Self
+else:
+    from typing import Self
 
 
 class DataFrameModel(_DataFrameModel[ibis.Table, DataFrameSchema]):
@@ -102,3 +120,21 @@ def _build_columns(  # pylint:disable=too-many-locals
                 )
 
         return columns
+
+    @classmethod
+    @docstring_substitution(validate_doc=BaseSchema.validate.__doc__)
+    def validate(
+        cls: Type[Self],
+        check_obj: ibis.Table,
+        head: Optional[int] = None,
+        tail: Optional[int] = None,
+        sample: Optional[int] = None,
+        random_state: Optional[int] = None,
+        lazy: bool = False,
+        inplace: bool = False,
+    ) -> Table[Self]:
+        """%(validate_doc)s"""
+        result = cls.to_schema().validate(
+            check_obj, head, tail, sample, random_state, lazy, inplace
+        )
+        return cast(Table[Self], result)

pandera/backends/ibis/checks.py

@@ -18,6 +18,11 @@
     from ibis.expr.types.groupby import GroupedTable
 
 
+# Manually maintained list derived from
+# https://github.com/ibis-project/ibis/blob/10.6.0/ibis/backends/tests/test_join.py#L370-L399
+POSITIONAL_JOIN_BACKENDS = {"duckdb", "polars"}
+
+
 class IbisCheckBackend(BaseCheckBackend):
     """Check backend for Ibis."""
 
@@ -75,9 +80,12 @@ def apply(self, check_obj: IbisData):
                 )
             elif isinstance(out, ibis.Table):
                 out = out.rename(f"{{name}}{CHECK_OUTPUT_SUFFIX}")
-                try:
+                if (
+                    check_obj.table.get_backend().name
+                    in POSITIONAL_JOIN_BACKENDS
+                ):
                     out = check_obj.table.join(out, how="positional")
-                except Exception:  # pylint: disable=broad-exception-caught
+                else:
                     # For backends that do not support positional joins:
                     # https://github.com/ibis-project/ibis/issues/9486
                     index_col = "__idx__"

pandera/backends/ibis/components.py

@@ -95,12 +95,12 @@ def validate_column(check_obj, column_name):
                                 check_output=result.check_output,
                                 reason_code=result.reason_code,
                             )
-                            error_handler.collect_error(  # Why indent (unlike in container.py)?
-                                validation_type(result.reason_code),
-                                result.reason_code,
-                                error,
-                                original_exc=result.original_exc,
-                            )
+                        error_handler.collect_error(
+                            validation_type(result.reason_code),
+                            result.reason_code,
+                            error,
+                            original_exc=result.original_exc,
+                        )
 
             finally:
                 # revert the schema component mutations

pandera/typing/ibis.py

@@ -219,7 +219,7 @@ def _get_schema_model(cls, field):
             if not field.sub_fields:
                 raise TypeError(
                     "Expected a typed pandera.typing.ibis.Table,"
-                    " e.g. DataFrame[Schema]"
+                    " e.g. Table[Schema]"
                 )
             schema_model = field.sub_fields[0].type_
             return schema_model

pyproject.toml

@@ -129,7 +129,7 @@ testing = [
     "pytest-cov",
     "pytest-xdist",
     "pytest-asyncio",
-    "ibis-framework[duckdb] >= 9.0.0",
+    "ibis-framework[duckdb,sqlite] >= 9.0.0",
 ]
 docs = [
     "setuptools",

tests/ibis/test_ibis_backends.py

@@ -0,0 +1,33 @@
+"""Unit tests for ibis backends."""
+
+import ibis
+import pytest
+
+import pandera.ibis as pa
+
+
+def test_ibis_sqlite_backend():
+    con = ibis.sqlite.connect()
+    schema = ibis.schema(dict(x="int64", y="float64", z="string"))
+
+    valid_t = con.create_table("valid_table", schema=schema)
+    invalid_t = con.create_table("invalid_table", schema=schema)
+
+    con.insert(
+        "valid_table", obj=[(1, 1.0, "a"), (2, 2.0, "b"), (3, 3.0, "c")]
+    )
+    con.insert(
+        "invalid_table", obj=[(-1, 1.0, "a"), (2, 2.0, "b"), (3, 3.0, "d")]
+    )
+
+    # pylint: disable=missing-class-docstring
+    class TableSchema(pa.DataFrameModel):
+        x: int = pa.Field(ge=0)
+        y: float
+        z: str
+
+    validated_t = TableSchema.validate(valid_t)
+    assert validated_t.execute() is not None
+
+    with pytest.raises(pa.errors.SchemaErrors):
+        TableSchema.validate(invalid_t, lazy=True)