Move some non-core dependencies to the apache-airflow meta package

potiuk · potiuk · commit 65053f72c47a · 2025-04-27T16:56:04.000+02:00
Some of the extras in `airflow-core` do not really belong there, they
should be moved to the `apache-airflow` meta-package.
diff --git a/airflow-core/pyproject.toml b/airflow-core/pyproject.toml
@@ -145,35 +145,11 @@ dependencies = [
 
 
 [project.optional-dependencies]
-# Aiobotocore required for AWS deferrable operators.
-# There is conflict between boto3 and aiobotocore dependency botocore.
-# TODO: We can remove it once boto3 and aiobotocore both have compatible botocore version or
-# boto3 have native aync support and we move away from aio aiobotocore
-"aiobotocore" =  [
-    "apache-airflow-providers-amazon[aiobotocore]>=9.6.0",
-]
 "async" = [
     "eventlet>=0.37.0",
     "gevent>=24.2.1",
     "greenlet>=0.4.9",
 ]
-"apache-atlas" = [
-    "atlasclient>=0.1.2",
-]
-"apache-webhdfs" = [
-    "apache-airflow-providers-apache-hdfs",
-]
-"cloudpickle" = [
-    "cloudpickle>=2.2.1",
-]
-"github-enterprise" = [
-    "apache-airflow-providers-fab",
-    "authlib>=1.0.0",
-]
-"google-auth" = [
-    "apache-airflow-providers-fab",
-    "authlib>=1.0.0",
-]
 "graphviz" = [
     # The graphviz package creates friction when installing on MacOS as it needs graphviz system package to
     # be installed, and it's really only used for very obscure features of Airflow, so we can skip it on MacOS
@@ -185,27 +161,9 @@ dependencies = [
     "requests-kerberos>=0.14.0",
     "thrift-sasl>=0.4.2",
 ]
-"ldap" = [
-    "python-ldap>=3.4.4",
-]
 "otel" = [
     "opentelemetry-exporter-prometheus>=0.47b0",
 ]
-
-"pandas" = [
-    # In pandas 2.2 minimal version of the sqlalchemy is 2.0
-    # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
-    # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
-    "pandas>=2.1.2,<2.3",
-]
-"rabbitmq" = [
-    "amqp>=5.2.0",
-]
-"s3fs" = [
-    # This is required for support of S3 file system which uses aiobotocore
-    # which can have a conflict with boto3 as mentioned in aiobotocore extra
-    "apache-airflow-providers-amazon[s3fs]",
-]
 "sentry" = [
     "blinker>=1.1",
     # Sentry SDK 1.33 is broken when greenlets are installed and fails to import
@@ -215,11 +173,8 @@ dependencies = [
 "statsd" = [
     "statsd>=3.3.0",
 ]
-"uv" = [
-    "uv>=0.6.17",
-]
 "all" = [
-    "apache-airflow-core[aiobotocore,apache-atlas,apache-webhdfs,cloudpickle,github-enterprise,google-auth,graphviz,kerberos,ldap,otel,pandas,rabbitmq,s3fs,sentry,statsd,uv]"
+    "apache-airflow-core[graphviz,kerberos,otel,sentry,statsd]"
 ]
 
 [project.scripts]
@@ -280,11 +235,14 @@ dev = [
     "apache-airflow-ctl",
     "apache-airflow-devel-common",
     "apache-airflow-task-sdk",
+    # TODO(potiuk): eventually we do not want any providers nor apache-airflow extras to be needed for
+    # airflow-core tests
+    "apache-airflow[pandas]",
     "apache-airflow-providers-amazon",
     "apache-airflow-providers-celery",
     "apache-airflow-providers-cncf-kubernetes",
+    "apache-airflow-providers-fab",
     "apache-airflow-providers-git",
-    # TODO(potiuk): check if this is really needed
     "apache-airflow-providers-ftp",
 ]
 
diff --git a/pyproject.toml b/pyproject.toml
@@ -70,60 +70,27 @@ packages = []
 
 [project.optional-dependencies]
 # Automatically generated airflow optional dependencies
-"aiobotocore" = [
-    "apache-airflow-core[aiobotocore]"
-]
 "all-core" = [
     "apache-airflow-core[all]"
 ]
-"apache-atlas" = [
-    "apache-airflow-core[apache-atlas]"
-]
-"apache-webhdfs" = [
-    "apache-airflow-core[apache-webhdfs]"
-]
 "async" = [
     "apache-airflow-core[async]"
 ]
-"cloudpickle" = [
-    "apache-airflow-core[cloudpickle]"
-]
-"github-enterprise" = [
-    "apache-airflow-core[github-enterprise]"
-]
-"google-auth" = [
-    "apache-airflow-core[google-auth]"
-]
 "graphviz" = [
     "apache-airflow-core[graphviz]"
 ]
 "kerberos" = [
     "apache-airflow-core[kerberos]"
 ]
-"ldap" = [
-    "apache-airflow-core[ldap]"
-]
 "otel" = [
     "apache-airflow-core[otel]"
 ]
-"pandas" = [
-    "apache-airflow-core[pandas]"
-]
-"rabbitmq" = [
-    "apache-airflow-core[rabbitmq]"
-]
-"s3fs" = [
-    "apache-airflow-core[s3fs]"
-]
 "sentry" = [
     "apache-airflow-core[sentry]"
 ]
 "statsd" = [
     "apache-airflow-core[statsd]"
 ]
-"uv" = [
-    "apache-airflow-core[uv]"
-]
 "airbyte" = [
     "apache-airflow-providers-airbyte>=5.0.0"
 ]
@@ -413,6 +380,7 @@ packages = []
     "apache-airflow-providers-zendesk>=4.9.0"
 ]
 "all" = [
+    "apache-airflow[aiobotocore,apache-atlas,apache-webhdfs,async,cloudpickle,github-enterprise,google-auth,graphviz,kerberos,ldap,otel,pandas,rabbitmq,s3fs,sentry,statsd,uv]",
     "apache-airflow-core[all]",
     "apache-airflow-providers-airbyte>=5.0.0",
     "apache-airflow-providers-alibaba>=3.0.0",
@@ -512,6 +480,51 @@ packages = []
     "apache-airflow-providers-zendesk>=4.9.0",
 ]
 # End of automatically generated airflow optional dependencies
+# Aiobotocore required for AWS deferrable operators.
+# There is conflict between boto3 and aiobotocore dependency botocore.
+# TODO: We can remove it once boto3 and aiobotocore both have compatible botocore version or
+# boto3 have native aync support and we move away from aio aiobotocore
+"aiobotocore" =  [
+    "apache-airflow-providers-amazon[aiobotocore]>=9.6.0",
+]
+"apache-atlas" = [
+    "atlasclient>=0.1.2",
+]
+"apache-webhdfs" = [
+    "apache-airflow-providers-apache-hdfs",
+]
+"cloudpickle" = [
+    "cloudpickle>=2.2.1",
+]
+"github-enterprise" = [
+    "apache-airflow-providers-fab",
+    "authlib>=1.0.0",
+]
+"google-auth" = [
+    "apache-airflow-providers-fab",
+    "authlib>=1.0.0",
+]
+"ldap" = [
+    "python-ldap>=3.4.4",
+]
+"pandas" = [
+    # In pandas 2.2 minimal version of the sqlalchemy is 2.0
+    # https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
+    # However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
+    "pandas>=2.1.2,<2.3",
+]
+"rabbitmq" = [
+    "amqp>=5.2.0",
+]
+"s3fs" = [
+    # This is required for support of S3 file system which uses aiobotocore
+    # which can have a conflict with boto3 as mentioned in aiobotocore extra
+    "apache-airflow-providers-amazon[s3fs]",
+]
+"uv" = [
+    "uv>=0.6.17",
+]
+
 
 [project.urls]
 "Bug Tracker" = "https://github.com/apache/airflow/issues"
diff --git a/scripts/ci/pre_commit/update_airflow_pyproject_toml.py b/scripts/ci/pre_commit/update_airflow_pyproject_toml.py
@@ -63,12 +63,12 @@
 }
 
 
-def get_optional_dependencies_from_airflow_core() -> list[str]:
+def get_optional_dependencies(pyproject_toml_path: Path) -> list[str]:
     try:
         import tomllib
     except ImportError:
         import tomli as tomllib
-    airflow_core_toml_dict = tomllib.loads(AIRFLOW_CORE_PYPROJECT_TOML_FILE.read_text())
+    airflow_core_toml_dict = tomllib.loads(pyproject_toml_path.read_text())
     return airflow_core_toml_dict["project"]["optional-dependencies"].keys()
 
 
@@ -131,7 +131,7 @@ def find_min_provider_version(provider_id: str) -> Version | None:
 
 if __name__ == "__main__":
     all_optional_dependencies = []
-    optional_airflow_core_dependencies = get_optional_dependencies_from_airflow_core()
+    optional_airflow_core_dependencies = get_optional_dependencies(AIRFLOW_CORE_PYPROJECT_TOML_FILE)
     for optional in sorted(optional_airflow_core_dependencies):
         if optional == "all":
             all_optional_dependencies.append('"all-core" = [\n    "apache-airflow-core[all]"\n]\n')
@@ -150,7 +150,15 @@ def find_min_provider_version(provider_id: str) -> Version | None:
         else:
             all_optional_dependencies.append(f'"{provider_id}" = [\n    "{distribution_name}"\n]\n')
             all_provider_lines.append(f'    "{distribution_name}",\n')
-    all_optional_dependencies.append('"all" = [\n    "apache-airflow-core[all]",\n')
+    all_optional_dependencies.append('"all" = [\n')
+    optional_apache_airflow_dependencies = get_optional_dependencies(AIRFLOW_PYPROJECT_TOML_FILE)
+    all_local_extras = [
+        extra
+        for extra in sorted(optional_apache_airflow_dependencies)
+        if extra not in all_providers and not extra.startswith("all")
+    ]
+    all_optional_dependencies.append(f'    "apache-airflow[{",".join(all_local_extras)}]",\n')
+    all_optional_dependencies.append('    "apache-airflow-core[all]",\n')
     all_optional_dependencies.extend(all_provider_lines)
     all_optional_dependencies.append("]\n")
     insert_documentation(
