Skip to content

Commit 6daa258

Browse files
potiukpotiuk
committed
[v3-0-test] Move some non-core dependencies to the apache-airflow meta package (apache#49846)
Some of the extras in `airflow-core` do not really belong there, they should be moved to the `apache-airflow` meta-package. (cherry picked from commit 3730cba) Co-authored-by: Jarek Potiuk <[email protected]>
1 parent 4a18c45 commit 6daa258

File tree

3 files changed

+64
-85
lines changed

3 files changed

+64
-85
lines changed

‎airflow-core/pyproject.toml

Lines changed: 6 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ classifiers = [
5959
]
6060

6161
# Version is defined in src/airflow/__init__.py and it is automatically synchronized by pre-commit
62-
version = "3.0.1.dev0"
62+
version = "3.0.1"
6363

6464
dependencies = [
6565
"a2wsgi>=1.10.8",
@@ -146,35 +146,11 @@ dependencies = [
146146

147147

148148
[project.optional-dependencies]
149-
# Aiobotocore required for AWS deferrable operators.
150-
# There is conflict between boto3 and aiobotocore dependency botocore.
151-
# TODO: We can remove it once boto3 and aiobotocore both have compatible botocore version or
152-
# boto3 have native aync support and we move away from aio aiobotocore
153-
"aiobotocore" = [
154-
"apache-airflow-providers-amazon[aiobotocore]>=9.6.0",
155-
]
156149
"async" = [
157150
"eventlet>=0.37.0",
158151
"gevent>=24.2.1",
159152
"greenlet>=0.4.9",
160153
]
161-
"apache-atlas" = [
162-
"atlasclient>=0.1.2",
163-
]
164-
"apache-webhdfs" = [
165-
"apache-airflow-providers-apache-hdfs",
166-
]
167-
"cloudpickle" = [
168-
"cloudpickle>=2.2.1",
169-
]
170-
"github-enterprise" = [
171-
"apache-airflow-providers-fab",
172-
"authlib>=1.0.0",
173-
]
174-
"google-auth" = [
175-
"apache-airflow-providers-fab",
176-
"authlib>=1.0.0",
177-
]
178154
"graphviz" = [
179155
# The graphviz package creates friction when installing on MacOS as it needs graphviz system package to
180156
# be installed, and it's really only used for very obscure features of Airflow, so we can skip it on MacOS
@@ -186,27 +162,9 @@ dependencies = [
186162
"requests-kerberos>=0.14.0",
187163
"thrift-sasl>=0.4.2",
188164
]
189-
"ldap" = [
190-
"python-ldap>=3.4.4",
191-
]
192165
"otel" = [
193166
"opentelemetry-exporter-prometheus>=0.47b0",
194167
]
195-
196-
"pandas" = [
197-
# In pandas 2.2 minimal version of the sqlalchemy is 2.0
198-
# https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
199-
# However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
200-
"pandas>=2.1.2,<2.3",
201-
]
202-
"rabbitmq" = [
203-
"amqp>=5.2.0",
204-
]
205-
"s3fs" = [
206-
# This is required for support of S3 file system which uses aiobotocore
207-
# which can have a conflict with boto3 as mentioned in aiobotocore extra
208-
"apache-airflow-providers-amazon[s3fs]",
209-
]
210168
"sentry" = [
211169
"blinker>=1.1",
212170
# Sentry SDK 1.33 is broken when greenlets are installed and fails to import
@@ -216,11 +174,8 @@ dependencies = [
216174
"statsd" = [
217175
"statsd>=3.3.0",
218176
]
219-
"uv" = [
220-
"uv>=0.6.13",
221-
]
222177
"all" = [
223-
"apache-airflow-core[aiobotocore,apache-atlas,apache-webhdfs,cloudpickle,github-enterprise,google-auth,graphviz,kerberos,ldap,otel,pandas,rabbitmq,s3fs,sentry,statsd,uv]"
178+
"apache-airflow-core[graphviz,kerberos,otel,sentry,statsd]"
224179
]
225180

226181
[project.scripts]
@@ -281,11 +236,14 @@ dev = [
281236
"apache-airflow-ctl",
282237
"apache-airflow-devel-common",
283238
"apache-airflow-task-sdk",
239+
# TODO(potiuk): eventually we do not want any providers nor apache-airflow extras to be needed for
240+
# airflow-core tests
241+
"apache-airflow[pandas]",
284242
"apache-airflow-providers-amazon",
285243
"apache-airflow-providers-celery",
286244
"apache-airflow-providers-cncf-kubernetes",
245+
"apache-airflow-providers-fab",
287246
"apache-airflow-providers-git",
288-
# TODO(potiuk): check if this is really needed
289247
"apache-airflow-providers-ftp",
290248
]
291249

‎pyproject.toml

Lines changed: 46 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -70,60 +70,27 @@ packages = []
7070

7171
[project.optional-dependencies]
7272
# Automatically generated airflow optional dependencies
73-
"aiobotocore" = [
74-
"apache-airflow-core[aiobotocore]"
75-
]
7673
"all-core" = [
7774
"apache-airflow-core[all]"
7875
]
79-
"apache-atlas" = [
80-
"apache-airflow-core[apache-atlas]"
81-
]
82-
"apache-webhdfs" = [
83-
"apache-airflow-core[apache-webhdfs]"
84-
]
8576
"async" = [
8677
"apache-airflow-core[async]"
8778
]
88-
"cloudpickle" = [
89-
"apache-airflow-core[cloudpickle]"
90-
]
91-
"github-enterprise" = [
92-
"apache-airflow-core[github-enterprise]"
93-
]
94-
"google-auth" = [
95-
"apache-airflow-core[google-auth]"
96-
]
9779
"graphviz" = [
9880
"apache-airflow-core[graphviz]"
9981
]
10082
"kerberos" = [
10183
"apache-airflow-core[kerberos]"
10284
]
103-
"ldap" = [
104-
"apache-airflow-core[ldap]"
105-
]
10685
"otel" = [
10786
"apache-airflow-core[otel]"
10887
]
109-
"pandas" = [
110-
"apache-airflow-core[pandas]"
111-
]
112-
"rabbitmq" = [
113-
"apache-airflow-core[rabbitmq]"
114-
]
115-
"s3fs" = [
116-
"apache-airflow-core[s3fs]"
117-
]
11888
"sentry" = [
11989
"apache-airflow-core[sentry]"
12090
]
12191
"statsd" = [
12292
"apache-airflow-core[statsd]"
12393
]
124-
"uv" = [
125-
"apache-airflow-core[uv]"
126-
]
12794
"airbyte" = [
12895
"apache-airflow-providers-airbyte>=5.0.0"
12996
]
@@ -410,6 +377,7 @@ packages = []
410377
"apache-airflow-providers-zendesk>=4.9.0"
411378
]
412379
"all" = [
380+
"apache-airflow[aiobotocore,apache-atlas,apache-webhdfs,async,cloudpickle,github-enterprise,google-auth,graphviz,kerberos,ldap,otel,pandas,rabbitmq,s3fs,sentry,statsd,uv]",
413381
"apache-airflow-core[all]",
414382
"apache-airflow-providers-airbyte>=5.0.0",
415383
"apache-airflow-providers-alibaba>=3.0.0",
@@ -508,6 +476,51 @@ packages = []
508476
"apache-airflow-providers-zendesk>=4.9.0",
509477
]
510478
# End of automatically generated airflow optional dependencies
479+
# Aiobotocore required for AWS deferrable operators.
480+
# There is conflict between boto3 and aiobotocore dependency botocore.
481+
# TODO: We can remove it once boto3 and aiobotocore both have compatible botocore version or
482+
# boto3 have native aync support and we move away from aio aiobotocore
483+
"aiobotocore" = [
484+
"apache-airflow-providers-amazon[aiobotocore]>=9.6.0",
485+
]
486+
"apache-atlas" = [
487+
"atlasclient>=0.1.2",
488+
]
489+
"apache-webhdfs" = [
490+
"apache-airflow-providers-apache-hdfs",
491+
]
492+
"cloudpickle" = [
493+
"cloudpickle>=2.2.1",
494+
]
495+
"github-enterprise" = [
496+
"apache-airflow-providers-fab",
497+
"authlib>=1.0.0",
498+
]
499+
"google-auth" = [
500+
"apache-airflow-providers-fab",
501+
"authlib>=1.0.0",
502+
]
503+
"ldap" = [
504+
"python-ldap>=3.4.4",
505+
]
506+
"pandas" = [
507+
# In pandas 2.2 minimal version of the sqlalchemy is 2.0
508+
# https://pandas.pydata.org/docs/whatsnew/v2.2.0.html#increased-minimum-versions-for-dependencies
509+
# However Airflow not fully supports it yet: https://github.com/apache/airflow/issues/28723
510+
"pandas>=2.1.2,<2.3",
511+
]
512+
"rabbitmq" = [
513+
"amqp>=5.2.0",
514+
]
515+
"s3fs" = [
516+
# This is required for support of S3 file system which uses aiobotocore
517+
# which can have a conflict with boto3 as mentioned in aiobotocore extra
518+
"apache-airflow-providers-amazon[s3fs]",
519+
]
520+
"uv" = [
521+
"uv>=0.6.17",
522+
]
523+
511524

512525
[project.urls]
513526
"Bug Tracker" = "https://github.com/apache/airflow/issues"

‎scripts/ci/pre_commit/update_airflow_pyproject_toml.py

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,12 +64,12 @@
6464
}
6565

6666

67-
def get_optional_dependencies_from_airflow_core() -> list[str]:
67+
def get_optional_dependencies(pyproject_toml_path: Path) -> list[str]:
6868
try:
6969
import tomllib
7070
except ImportError:
7171
import tomli as tomllib
72-
airflow_core_toml_dict = tomllib.loads(AIRFLOW_CORE_PYPROJECT_TOML_FILE.read_text())
72+
airflow_core_toml_dict = tomllib.loads(pyproject_toml_path.read_text())
7373
return airflow_core_toml_dict["project"]["optional-dependencies"].keys()
7474

7575

@@ -132,7 +132,7 @@ def find_min_provider_version(provider_id: str) -> Version | None:
132132

133133
if __name__ == "__main__":
134134
all_optional_dependencies = []
135-
optional_airflow_core_dependencies = get_optional_dependencies_from_airflow_core()
135+
optional_airflow_core_dependencies = get_optional_dependencies(AIRFLOW_CORE_PYPROJECT_TOML_FILE)
136136
for optional in sorted(optional_airflow_core_dependencies):
137137
if optional == "all":
138138
all_optional_dependencies.append('"all-core" = [\n "apache-airflow-core[all]"\n]\n')
@@ -151,7 +151,15 @@ def find_min_provider_version(provider_id: str) -> Version | None:
151151
else:
152152
all_optional_dependencies.append(f'"{provider_id}" = [\n "{distribution_name}"\n]\n')
153153
all_provider_lines.append(f' "{distribution_name}",\n')
154-
all_optional_dependencies.append('"all" = [\n "apache-airflow-core[all]",\n')
154+
all_optional_dependencies.append('"all" = [\n')
155+
optional_apache_airflow_dependencies = get_optional_dependencies(AIRFLOW_PYPROJECT_TOML_FILE)
156+
all_local_extras = [
157+
extra
158+
for extra in sorted(optional_apache_airflow_dependencies)
159+
if extra not in all_providers and not extra.startswith("all")
160+
]
161+
all_optional_dependencies.append(f' "apache-airflow[{",".join(all_local_extras)}]",\n')
162+
all_optional_dependencies.append(' "apache-airflow-core[all]",\n')
155163
all_optional_dependencies.extend(all_provider_lines)
156164
all_optional_dependencies.append("]\n")
157165
insert_documentation(

0 commit comments

Comments
 (0)