From 89c33c1adf4527e8b831c81f897efeb8258e95b0 Mon Sep 17 00:00:00 2001 From: nqi Date: Fri, 6 Jun 2025 14:15:46 -0700 Subject: [PATCH 1/5] SNOW-2012666: Implemented lazy import for pandas in the connector --- src/snowflake/connector/options.py | 40 +++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/src/snowflake/connector/options.py b/src/snowflake/connector/options.py index 8454ab169..7cfd42622 100644 --- a/src/snowflake/connector/options.py +++ b/src/snowflake/connector/options.py @@ -61,6 +61,21 @@ def warn_incompatible_dep( ) +def pandas(): + try: + return importlib.import_module("pandas") + except ImportError: + return MissingPandas() + + +def pyarrow(): + try: + pyarrow = importlib.import_module("pyarrow") + return pyarrow + except ImportError: + raise errors.MissingDependencyError("pyarrow") + + def _import_or_missing_pandas_option() -> ( tuple[ModuleLikeObject, ModuleLikeObject, bool] ): @@ -114,6 +129,12 @@ def _import_or_missing_pandas_option() -> ( return MissingPandas(), MissingPandas(), False +def installed_pandas() -> bool: + """This function checks if pandas is available and compatible.""" + _, _, installed = _import_or_missing_pandas_option() + return installed + + def _import_or_missing_keyring_option() -> tuple[ModuleLikeObject, bool]: """This function tries importing the following packages: keyring. @@ -127,5 +148,22 @@ def _import_or_missing_keyring_option() -> tuple[ModuleLikeObject, bool]: # Create actual constants to be imported from this file -pandas, pyarrow, installed_pandas = _import_or_missing_pandas_option() +try: + pyarrow = importlib.import_module("pyarrow") +except ImportError: + raise errors.MissingDependencyError("pyarrow") + keyring, installed_keyring = _import_or_missing_keyring_option() + + +def __getattr__(name): + if name == "pandas": + try: + return importlib.import_module("pandas") + except ImportError: + return MissingPandas() + + elif name == "installed_pandas": + return installed_pandas() + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") From 033e90622f3bcff3101d6e6f113e11d88a7272e8 Mon Sep 17 00:00:00 2001 From: nqi Date: Fri, 6 Jun 2025 14:31:52 -0700 Subject: [PATCH 2/5] Fix precommit failures --- DESCRIPTION.md | 1 + src/snowflake/connector/options.py | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/DESCRIPTION.md b/DESCRIPTION.md index fd5f9713d..40c788520 100644 --- a/DESCRIPTION.md +++ b/DESCRIPTION.md @@ -11,6 +11,7 @@ Source code is also available at: https://github.com/snowflakedb/snowflake-conne - Bumped numpy dependency from <2.1.0 to <=2.2.4 - Added Windows support for Python 3.13. - Add `bulk_upload_chunks` parameter to `write_pandas` function. Setting this parameter to True changes the behaviour of write_pandas function to first write all the data chunks to the local disk and then perform the wildcard upload of the chunks folder to the stage. In default behaviour the chunks are being saved, uploaded and deleted one by one. + - Implemented lazy import for pandas to improve module loading performance. - v3.15.1(May 20, 2025) diff --git a/src/snowflake/connector/options.py b/src/snowflake/connector/options.py index 7cfd42622..27033848f 100644 --- a/src/snowflake/connector/options.py +++ b/src/snowflake/connector/options.py @@ -68,14 +68,6 @@ def pandas(): return MissingPandas() -def pyarrow(): - try: - pyarrow = importlib.import_module("pyarrow") - return pyarrow - except ImportError: - raise errors.MissingDependencyError("pyarrow") - - def _import_or_missing_pandas_option() -> ( tuple[ModuleLikeObject, ModuleLikeObject, bool] ): From 85e8f98f013a57e4a41579c3a51f6b9b535ecb00 Mon Sep 17 00:00:00 2001 From: nqi Date: Fri, 6 Jun 2025 15:11:24 -0700 Subject: [PATCH 3/5] Defer errors import to avoid circular dependency --- src/snowflake/connector/options.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/snowflake/connector/options.py b/src/snowflake/connector/options.py index 27033848f..10c45ba56 100644 --- a/src/snowflake/connector/options.py +++ b/src/snowflake/connector/options.py @@ -10,8 +10,6 @@ from packaging.requirements import Requirement -from . import errors - logger = getLogger(__name__) """This module helps to manage optional dependencies. @@ -33,6 +31,7 @@ class MissingOptionalDependency: _dep_name = "not set" def __getattr__(self, item): + from . import errors raise errors.MissingDependencyError(self._dep_name) @@ -143,6 +142,8 @@ def _import_or_missing_keyring_option() -> tuple[ModuleLikeObject, bool]: try: pyarrow = importlib.import_module("pyarrow") except ImportError: + # Defer errors import to avoid circular dependency + from . import errors raise errors.MissingDependencyError("pyarrow") keyring, installed_keyring = _import_or_missing_keyring_option() From 8081d2c8ac3d5fc70685793927e4868604369135 Mon Sep 17 00:00:00 2001 From: nqi Date: Fri, 6 Jun 2025 15:12:21 -0700 Subject: [PATCH 4/5] Fix errors import to avoid circular dependency --- DESCRIPTION.md | 2 +- src/snowflake/connector/options.py | 23 +++++++++-------------- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/DESCRIPTION.md b/DESCRIPTION.md index 40c788520..b08755105 100644 --- a/DESCRIPTION.md +++ b/DESCRIPTION.md @@ -11,7 +11,7 @@ Source code is also available at: https://github.com/snowflakedb/snowflake-conne - Bumped numpy dependency from <2.1.0 to <=2.2.4 - Added Windows support for Python 3.13. - Add `bulk_upload_chunks` parameter to `write_pandas` function. Setting this parameter to True changes the behaviour of write_pandas function to first write all the data chunks to the local disk and then perform the wildcard upload of the chunks folder to the stage. In default behaviour the chunks are being saved, uploaded and deleted one by one. - - Implemented lazy import for pandas to improve module loading performance. + - Implemented lazy import for pandas to improve loading performance. - v3.15.1(May 20, 2025) diff --git a/src/snowflake/connector/options.py b/src/snowflake/connector/options.py index 10c45ba56..0994ada7d 100644 --- a/src/snowflake/connector/options.py +++ b/src/snowflake/connector/options.py @@ -32,6 +32,7 @@ class MissingOptionalDependency: def __getattr__(self, item): from . import errors + raise errors.MissingDependencyError(self._dep_name) @@ -60,13 +61,6 @@ def warn_incompatible_dep( ) -def pandas(): - try: - return importlib.import_module("pandas") - except ImportError: - return MissingPandas() - - def _import_or_missing_pandas_option() -> ( tuple[ModuleLikeObject, ModuleLikeObject, bool] ): @@ -139,13 +133,6 @@ def _import_or_missing_keyring_option() -> tuple[ModuleLikeObject, bool]: # Create actual constants to be imported from this file -try: - pyarrow = importlib.import_module("pyarrow") -except ImportError: - # Defer errors import to avoid circular dependency - from . import errors - raise errors.MissingDependencyError("pyarrow") - keyring, installed_keyring = _import_or_missing_keyring_option() @@ -156,6 +143,14 @@ def __getattr__(name): except ImportError: return MissingPandas() + elif name == "pyarrow": + try: + return importlib.import_module("pyarrow") + except ImportError: + from . import errors + + raise errors.MissingDependencyError("pyarrow") + elif name == "installed_pandas": return installed_pandas() From 83f73a7cbd15b11a95525c810ec30da9262a231a Mon Sep 17 00:00:00 2001 From: nqi Date: Mon, 16 Jun 2025 21:30:47 -0700 Subject: [PATCH 5/5] Update result_set --- src/snowflake/connector/options.py | 7 +++++-- src/snowflake/connector/result_set.py | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/snowflake/connector/options.py b/src/snowflake/connector/options.py index 0994ada7d..2701d636d 100644 --- a/src/snowflake/connector/options.py +++ b/src/snowflake/connector/options.py @@ -116,8 +116,11 @@ def _import_or_missing_pandas_option() -> ( def installed_pandas() -> bool: """This function checks if pandas is available and compatible.""" - _, _, installed = _import_or_missing_pandas_option() - return installed + try: + importlib.import_module("pandas") + return True + except ImportError: + return False def _import_or_missing_keyring_option() -> tuple[ModuleLikeObject, bool]: diff --git a/src/snowflake/connector/result_set.py b/src/snowflake/connector/result_set.py index b633b41a0..81d6fcb13 100644 --- a/src/snowflake/connector/result_set.py +++ b/src/snowflake/connector/result_set.py @@ -18,7 +18,6 @@ from .constants import IterUnit from .errors import NotSupportedError -from .options import pandas from .options import pyarrow as pa from .result_batch import ( ArrowResultBatch, @@ -215,6 +214,8 @@ def _fetch_pandas_batches(self, **kwargs) -> Iterator[DataFrame]: def _fetch_pandas_all(self, **kwargs) -> DataFrame: """Fetches a single Pandas dataframe.""" + from .options import pandas + concat_args = list(inspect.signature(pandas.concat).parameters) concat_kwargs = {k: kwargs.pop(k) for k in dict(kwargs) if k in concat_args} dataframes = list(self._fetch_pandas_batches(is_fetch_all=True, **kwargs))