Enh/future annotations py3.13 (#1980)

* fix typing unit test for py3.13

Signed-off-by: cosmicBboy <[email protected]>

* support __future__.annotations, including forwardref types

Signed-off-by: cosmicBboy <[email protected]>

* fix weird issue with DataFrameBase.__setattr__: use dict.update

Signed-off-by: cosmicBboy <[email protected]>

* add python 3.13 to ci

Signed-off-by: cosmicBboy <[email protected]>

* fix up tests

Signed-off-by: cosmicBboy <[email protected]>

* update ci matrix

Signed-off-by: cosmicBboy <[email protected]>

---------

Signed-off-by: cosmicBboy <[email protected]>

Author: cosmicBboy

.github/workflows/ci-tests.yml

@@ -30,7 +30,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     defaults:
       run:
         shell: bash -l {0}
@@ -102,7 +102,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
         pydantic-version: ["2.10.6"]
     steps:
       - uses: actions/checkout@v4
@@ -135,9 +135,12 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
         pandas-version: ["2.1.1", "2.2.3"]
         pydantic-version: ["1.10.11", "2.10.6"]
+        exclude:
+          - pandas-version: "2.1.1"
+            python-version: "3.13"
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
@@ -169,7 +172,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
         pandas-version: ["2.2.3"]
         pydantic-version: ["2.10.6"]
         extra:
@@ -215,7 +218,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
         pandas-version: ["2.2.3"]
         pydantic-version: ["2.10.6"]
         extra:
@@ -235,7 +238,13 @@ jobs:
         - extra: pyspark
           os: windows-latest
         - extra: pyspark
-          python-version: 3.12
+          python-version: "3.12"
+        - extra: pyspark
+          python-version: "3.13"
+        - extra: modin-dask
+          python-version: "3.13"
+        - extra: modin-ray
+          python-version: "3.13"
 
     steps:
       - uses: actions/checkout@v4

noxfile.py

@@ -21,7 +21,7 @@
     "docs",
 )
 
-PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12"]
+PYTHON_VERSIONS = ["3.9", "3.10", "3.11", "3.12", "3.13"]
 PANDAS_VERSIONS = ["2.1.1", "2.2.3"]
 PYDANTIC_VERSIONS = ["1.10.11", "2.10.6"]
 PACKAGE = "pandera"
@@ -174,7 +174,7 @@ def _testing_requirements(
 
     return [
         *_updated_requirements,
-        *nox.project.dependency_groups(PYPROJECT, *["dev", "testing", "docs"]),
+        *nox.project.dependency_groups(PYPROJECT, *["dev", "testing"]),
     ]
 
 

pandera/api/dataframe/model.py

@@ -20,7 +20,7 @@
     Union,
     cast,
 )
-
+from typing_extensions import get_type_hints
 from pandera.api.base.model import BaseModel
 from pandera.api.base.schema import BaseSchema
 from pandera.api.checks import Check
@@ -49,11 +49,6 @@
     from pydantic import GetCoreSchemaHandler, GetJsonSchemaHandler
     from pydantic_core import core_schema
 
-try:
-    from typing_extensions import get_type_hints
-except ImportError:  # pragma: no cover
-    from typing import get_type_hints  # type: ignore
-
 
 TDataFrame = TypeVar("TDataFrame")
 TDataFrameModel = TypeVar("TDataFrameModel", bound="DataFrameModel")
@@ -337,10 +332,7 @@ def _get_model_attrs(cls) -> Dict[str, Any]:
     def _collect_fields(cls) -> Dict[str, Tuple[AnnotationInfo, FieldInfo]]:
         """Centralize publicly named fields and their corresponding annotations."""
         # pylint: disable=unexpected-keyword-arg
-        annotations = get_type_hints(  # type: ignore[call-arg]
-            cls,
-            include_extras=True,
-        )
+        annotations = get_type_hints(cls, include_extras=True)
         # pylint: enable=unexpected-keyword-arg
         attrs = cls._get_model_attrs()
 
@@ -358,6 +350,8 @@ def _collect_fields(cls) -> Dict[str, Tuple[AnnotationInfo, FieldInfo]]:
 
         fields = {}
         for field_name, annotation in annotations.items():
+            if not _is_field(field_name):
+                continue
             field = attrs[field_name]  # __init_subclass__ guarantees existence
             if not isinstance(field, FieldInfo):
                 raise SchemaInitError(

pandera/api/pandas/model.py

@@ -83,6 +83,7 @@ def _build_columns_index(  # pylint:disable=too-many-locals,too-many-branches
 
         columns: Dict[str, Column] = {}
         indices: List[Index] = []
+
         for field_name, (annotation, field) in fields.items():
             field_checks = checks.get(field_name, [])
             field_parsers = parsers.get(field_name, [])

pandera/api/pyspark/model.py

@@ -24,6 +24,7 @@
 
 import pyspark.sql as ps
 from pyspark.sql.types import StructType
+from typing_extensions import get_type_hints
 
 from pandera.api.base.model import BaseModel
 from pandera.api.checks import Check
@@ -43,11 +44,6 @@
 from pandera.typing.common import DataFrameBase
 from pandera.typing.pyspark import DataFrame
 
-try:
-    from typing_extensions import get_type_hints
-except ImportError:  # pragma: no cover
-    from typing import get_type_hints  # type: ignore
-
 
 _CONFIG_KEY = "Config"
 

pandera/decorators.py

@@ -4,7 +4,6 @@
 import inspect
 import sys
 import types
-import typing
 from typing import (
     Any,
     Callable,
@@ -21,6 +20,7 @@
 )
 
 from pydantic import validate_arguments
+from typing_extensions import get_type_hints
 
 from pandera import errors
 from pandera.api.base.error_handler import ErrorHandler
@@ -600,42 +600,55 @@ def check_types(
         )
 
     # Front-load annotation parsing
-    annotated_schema_models: Dict[
+    # @functools.lru_cache
+    def _get_annotated_schema_models(
+        wrapped: Callable,
+    ) -> Dict[
         str,
         Iterable[
             Tuple[Union[DataFrameModel, None], Union[AnnotationInfo, None]]
         ],
-    ] = {}
-    for arg_name_, annotation in typing.get_type_hints(wrapped).items():
-        annotation_info = AnnotationInfo(annotation)
-        if not annotation_info.is_generic_df:
-            # pylint: disable=comparison-with-callable
-            if annotation_info.origin == Union:
-                annotation_model_pairs = []
-                for annot in annotation_info.args:  # type: ignore[union-attr]
-                    sub_annotation_info = AnnotationInfo(annot)
-                    if not sub_annotation_info.is_generic_df:
-                        continue
-
-                    schema_model = cast(
-                        DataFrameModel, sub_annotation_info.arg
-                    )
-                    annotation_model_pairs.append(
-                        (schema_model, sub_annotation_info)
-                    )
+    ]:
+        annotated_schema_models: Dict[
+            str,
+            Iterable[
+                Tuple[Union[DataFrameModel, None], Union[AnnotationInfo, None]]
+            ],
+        ] = {}
+        for arg_name_, annotation in get_type_hints(
+            wrapped, include_extras=True
+        ).items():
+            annotation_info = AnnotationInfo(annotation)
+            if not annotation_info.is_generic_df:
+                # pylint: disable=comparison-with-callable
+                if annotation_info.origin == Union:
+                    annotation_model_pairs = []
+                    for annot in annotation_info.args:  # type: ignore[union-attr]
+                        sub_annotation_info = AnnotationInfo(annot)
+                        if not sub_annotation_info.is_generic_df:
+                            continue
+
+                        schema_model = cast(
+                            DataFrameModel, sub_annotation_info.arg
+                        )
+                        annotation_model_pairs.append(
+                            (schema_model, sub_annotation_info)
+                        )
+                else:
+                    continue
             else:
-                continue
-        else:
-            schema_model = cast(DataFrameModel, annotation_info.arg)
-            annotation_model_pairs = [(schema_model, annotation_info)]
+                schema_model = cast(DataFrameModel, annotation_info.arg)
+                annotation_model_pairs = [(schema_model, annotation_info)]
 
-        annotated_schema_models[arg_name_] = annotation_model_pairs
+            annotated_schema_models[arg_name_] = annotation_model_pairs
+        return annotated_schema_models
 
     def _check_arg(arg_name: str, arg_value: Any) -> Any:
         """
         Validate function's argument if annotated with a schema, else
         pass-through.
         """
+        annotated_schema_models = _get_annotated_schema_models(wrapped)
         annotation_model_pairs = annotated_schema_models.get(
             arg_name, [(None, None)]
         )

pandera/engines/engine.py

@@ -19,10 +19,10 @@
     Tuple,
     Type,
     TypeVar,
-    get_type_hints,
 )
 
 import typing_inspect
+from typing_extensions import get_type_hints
 
 from pandera.dtypes import DataType
 

pandera/typing/common.py

@@ -130,13 +130,14 @@ def __setattr__(self, name: str, value: Any) -> None:
         # pylint: disable=no-member
         object.__setattr__(self, name, value)
         if name == "__orig_class__":
-            orig_class = getattr(self, "__orig_class__")
+            orig_class = value
             class_args = getattr(orig_class, "__args__", None)
             if class_args is not None and any(
                 x.__name__ == "DataFrameModel"
                 for x in inspect.getmro(class_args[0])
             ):
                 schema_model = value.__args__[0]
+                schema = schema_model.to_schema()
             else:
                 raise TypeError("Could not find DataFrameModel in class args")
 
@@ -147,12 +148,12 @@ def __setattr__(self, name: str, value: Any) -> None:
             if (
                 pandera_accessor is None
                 or pandera_accessor.schema is None
-                or pandera_accessor.schema != schema_model.to_schema()
+                or pandera_accessor.schema != schema
             ):
-                self.__dict__ = schema_model.validate(self).__dict__
+                self.__dict__.update(schema.validate(self).__dict__)
                 if pandera_accessor is None:
                     pandera_accessor = getattr(self, "pandera")
-                pandera_accessor.add_schema(schema_model.to_schema())
+                pandera_accessor.add_schema(schema)
 
 
 # pylint:disable=too-few-public-methods
@@ -233,8 +234,14 @@ def _parse_annotation(self, raw_annotation: Type) -> None:
         self.arg = args[0] if args else args
 
         metadata = getattr(raw_annotation, "__metadata__", None)
+
         if metadata:
             self.is_annotated_type = True
+            try:
+                inspect.signature(self.arg)
+            except ValueError:
+                metadata = None
+
         elif metadata := getattr(self.arg, "__metadata__", None):
             self.arg = typing_inspect.get_args(self.arg)[0]
 

pyproject.toml

@@ -27,6 +27,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Topic :: Scientific/Engineering",
 ]
 dependencies = [
@@ -129,6 +130,7 @@ testing = [
     "pytest-cov",
     "pytest-xdist",
     "pytest-asyncio",
+    "sphinx",
     "ibis-framework[duckdb,sqlite] >= 9.0.0",
 ]
 docs = [

tests/pandas/test_model.py

@@ -11,6 +11,7 @@
 import numpy as np
 import pandas as pd
 import pytest
+from pandas._testing import assert_frame_equal
 
 import pandera.pandas as pa
 import pandera.api.extensions as pax
@@ -1118,7 +1119,7 @@ class Config:
     }
     pandera_validated_df = DataFrame[Schema](raw_data)
     pandas_df = pd.DataFrame(raw_data)
-    assert pandera_validated_df.equals(Schema.validate(pandas_df))
+    assert_frame_equal(pandera_validated_df, Schema.validate(pandas_df))
     assert isinstance(pandera_validated_df, DataFrame)
     assert isinstance(pandas_df, pd.DataFrame)