Skip to content

Move some non-core dependencies to the apache-airflow meta package #49846

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 27, 2025
Merged

Move some non-core dependencies to the apache-airflow meta package #49846

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 5 additions & 47 deletions airflow-core/pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,35 +145,11 @@ dependencies = [


[project.optional-dependencies]
# Aiobotocore required for AWS deferrable operators.
# There is conflict between boto3 and aiobotocore dependency botocore.
# TODO: We can remove it once boto3 and aiobotocore both have compatible botocore version or
# boto3 have native aync support and we move away from aio aiobotocore
"aiobotocore" = [
"apache-airflow-providers-amazon[aiobotocore]>=9.6.0",
]
"async" = [
"eventlet>=0.37.0",
"gevent>=24.2.1",
"greenlet>=0.4.9",
]
"apache-atlas" = [
"atlasclient>=0.1.2",
]
"apache-webhdfs" = [
"apache-airflow-providers-apache-hdfs",
]
"cloudpickle" = [
"cloudpickle>=2.2.1",
]
"github-enterprise" = [
"apache-airflow-providers-fab",
"authlib>=1.0.0",
]
"google-auth" = [
"apache-airflow-providers-fab",
"authlib>=1.0.0",
]
"graphviz" = [
# The graphviz package creates friction when installing on MacOS as it needs graphviz system package to
# be installed, and it's really only used for very obscure features of Airflow, so we can skip it on MacOS
Expand All @@ -185,27 +161,9 @@ dependencies = [
"requests-kerberos>=0.14.0",
"thrift-sasl>=0.4.2",
]
"ldap" = [
"python-ldap>=3.4.4",
]
"otel" = [
"opentelemetry-exporter-prometheus>=0.47b0",
]

"pandas" = [
# In pandas 2.2 minimal version of the sqlalchemy is 2.0
# https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
# However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
"pandas>=2.1.2,<2.3",
]
"rabbitmq" = [
"amqp>=5.2.0",
]
"s3fs" = [
# This is required for support of S3 file system which uses aiobotocore
# which can have a conflict with boto3 as mentioned in aiobotocore extra
"apache-airflow-providers-amazon[s3fs]",
]
"sentry" = [
"blinker>=1.1",
# Sentry SDK 1.33 is broken when greenlets are installed and fails to import
Expand All @@ -215,11 +173,8 @@ dependencies = [
"statsd" = [
"statsd>=3.3.0",
]
"uv" = [
"uv>=0.6.17",
]
"all" = [
"apache-airflow-core[aiobotocore,apache-atlas,apache-webhdfs,cloudpickle,github-enterprise,google-auth,graphviz,kerberos,ldap,otel,pandas,rabbitmq,s3fs,sentry,statsd,uv]"
"apache-airflow-core[graphviz,kerberos,otel,sentry,statsd]"
]

[project.scripts]
Expand Down Expand Up @@ -280,11 +235,14 @@ dev = [
"apache-airflow-ctl",
"apache-airflow-devel-common",
"apache-airflow-task-sdk",
# TODO(potiuk): eventually we do not want any providers nor apache-airflow extras to be needed for
# airflow-core tests
"apache-airflow[pandas]",
"apache-airflow-providers-amazon",
"apache-airflow-providers-celery",
"apache-airflow-providers-cncf-kubernetes",
"apache-airflow-providers-fab",
"apache-airflow-providers-git",
# TODO(potiuk): check if this is really needed
"apache-airflow-providers-ftp",
]

Expand Down
79 changes: 46 additions & 33 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,60 +70,27 @@ packages = []

[project.optional-dependencies]
# Automatically generated airflow optional dependencies
"aiobotocore" = [
"apache-airflow-core[aiobotocore]"
]
"all-core" = [
"apache-airflow-core[all]"
]
"apache-atlas" = [
"apache-airflow-core[apache-atlas]"
]
"apache-webhdfs" = [
"apache-airflow-core[apache-webhdfs]"
]
"async" = [
"apache-airflow-core[async]"
]
"cloudpickle" = [
"apache-airflow-core[cloudpickle]"
]
"github-enterprise" = [
"apache-airflow-core[github-enterprise]"
]
"google-auth" = [
"apache-airflow-core[google-auth]"
]
"graphviz" = [
"apache-airflow-core[graphviz]"
]
"kerberos" = [
"apache-airflow-core[kerberos]"
]
"ldap" = [
"apache-airflow-core[ldap]"
]
"otel" = [
"apache-airflow-core[otel]"
]
"pandas" = [
"apache-airflow-core[pandas]"
]
"rabbitmq" = [
"apache-airflow-core[rabbitmq]"
]
"s3fs" = [
"apache-airflow-core[s3fs]"
]
"sentry" = [
"apache-airflow-core[sentry]"
]
"statsd" = [
"apache-airflow-core[statsd]"
]
"uv" = [
"apache-airflow-core[uv]"
]
"airbyte" = [
"apache-airflow-providers-airbyte>=5.0.0"
]
Expand Down Expand Up @@ -413,6 +380,7 @@ packages = []
"apache-airflow-providers-zendesk>=4.9.0"
]
"all" = [
"apache-airflow[aiobotocore,apache-atlas,apache-webhdfs,async,cloudpickle,github-enterprise,google-auth,graphviz,kerberos,ldap,otel,pandas,rabbitmq,s3fs,sentry,statsd,uv]",
"apache-airflow-core[all]",
"apache-airflow-providers-airbyte>=5.0.0",
"apache-airflow-providers-alibaba>=3.0.0",
Expand Down Expand Up @@ -512,6 +480,51 @@ packages = []
"apache-airflow-providers-zendesk>=4.9.0",
]
# End of automatically generated airflow optional dependencies
# Aiobotocore required for AWS deferrable operators.
# There is conflict between boto3 and aiobotocore dependency botocore.
# TODO: We can remove it once boto3 and aiobotocore both have compatible botocore version or
# boto3 have native aync support and we move away from aio aiobotocore
"aiobotocore" = [
"apache-airflow-providers-amazon[aiobotocore]>=9.6.0",
]
"apache-atlas" = [
"atlasclient>=0.1.2",
]
"apache-webhdfs" = [
"apache-airflow-providers-apache-hdfs",
]
"cloudpickle" = [
"cloudpickle>=2.2.1",
]
"github-enterprise" = [
"apache-airflow-providers-fab",
"authlib>=1.0.0",
]
"google-auth" = [
"apache-airflow-providers-fab",
"authlib>=1.0.0",
]
"ldap" = [
"python-ldap>=3.4.4",
]
"pandas" = [
# In pandas 2.2 minimal version of the sqlalchemy is 2.0
# https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
# However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
"pandas>=2.1.2,<2.3",
]
"rabbitmq" = [
"amqp>=5.2.0",
]
"s3fs" = [
# This is required for support of S3 file system which uses aiobotocore
# which can have a conflict with boto3 as mentioned in aiobotocore extra
"apache-airflow-providers-amazon[s3fs]",
]
"uv" = [
"uv>=0.6.17",
]


[project.urls]
"Bug Tracker" = "https://github.com/apache/airflow/issues"
Expand Down
16 changes: 12 additions & 4 deletions scripts/ci/pre_commit/update_airflow_pyproject_toml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,12 @@
}


def get_optional_dependencies_from_airflow_core() -> list[str]:
def get_optional_dependencies(pyproject_toml_path: Path) -> list[str]:
try:
import tomllib
except ImportError:
import tomli as tomllib
airflow_core_toml_dict = tomllib.loads(AIRFLOW_CORE_PYPROJECT_TOML_FILE.read_text())
airflow_core_toml_dict = tomllib.loads(pyproject_toml_path.read_text())
return airflow_core_toml_dict["project"]["optional-dependencies"].keys()


Expand Down Expand Up @@ -131,7 +131,7 @@ def find_min_provider_version(provider_id: str) -> Version | None:

if __name__ == "__main__":
all_optional_dependencies = []
optional_airflow_core_dependencies = get_optional_dependencies_from_airflow_core()
optional_airflow_core_dependencies = get_optional_dependencies(AIRFLOW_CORE_PYPROJECT_TOML_FILE)
for optional in sorted(optional_airflow_core_dependencies):
if optional == "all":
all_optional_dependencies.append('"all-core" = [\n "apache-airflow-core[all]"\n]\n')
Expand All @@ -150,7 +150,15 @@ def find_min_provider_version(provider_id: str) -> Version | None:
else:
all_optional_dependencies.append(f'"{provider_id}" = [\n "{distribution_name}"\n]\n')
all_provider_lines.append(f' "{distribution_name}",\n')
all_optional_dependencies.append('"all" = [\n "apache-airflow-core[all]",\n')
all_optional_dependencies.append('"all" = [\n')
optional_apache_airflow_dependencies = get_optional_dependencies(AIRFLOW_PYPROJECT_TOML_FILE)
all_local_extras = [
extra
for extra in sorted(optional_apache_airflow_dependencies)
if extra not in all_providers and not extra.startswith("all")
]
all_optional_dependencies.append(f' "apache-airflow[{",".join(all_local_extras)}]",\n')
all_optional_dependencies.append(' "apache-airflow-core[all]",\n')
all_optional_dependencies.extend(all_provider_lines)
all_optional_dependencies.append("]\n")
insert_documentation(
Expand Down