Upgrade botocore/aiobotocore minimum requirements (#33649)

Botocore has a very peculiar process of releasing new version
every day, which means that it gives `pip` hard time to figure
what will be the non-conflicting set of packages when we have
too low of a minium version set as requirement.

Since we had > 1.24 that means that `pip` had to consider
more than 340 versions for botocore, but also for related
mypy packages and also a number of aiobotocore packages when
resolving eager-upgrade.

We limit all the relevant packages to 1.28 as minimum version
now, and we should continue doing that regularly in the future.

Author: potiuk

Dockerfile.ci

@@ -1393,11 +1393,10 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\
 RUN echo "Airflow version: ${AIRFLOW_VERSION}"
 
 # Those are additional constraints that are needed for some extras but we do not want to
-# force them on the main Airflow package.
-# Recent aiobotocore (2.6.0+) requires this specific botocore version, while
-# boto3 1.28.17 is the last version supporting that, so we pin them to help pip
-# avoid extra work. Revisit this when a new aiobotocore version is released.
-ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="botocore==1.31.17 boto3<=1.28.17"
+# force them on the main Airflow package. Currently we need no extra limits as PIP 23.1+ has much better
+# dependency resolution and we do not need to limit the versions of the dependencies
+# aiobotocore is limited temporarily until it stops backtracking pip
+ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=""
 ARG UPGRADE_TO_NEWER_DEPENDENCIES="false"
 ARG VERSION_SUFFIX_FOR_PYPI=""
 

airflow/providers/amazon/provider.yaml

@@ -67,7 +67,19 @@ dependencies:
   - apache-airflow>=2.4.0
   - apache-airflow-providers-common-sql>=1.3.1
   - apache-airflow-providers-http
-  - boto3>=1.24.0
+  # We should update minimum version of boto3 and here regularly to avoid `pip` backtracking with the number
+  # of candidates to consider. We should also make sure that all the below related packages have also the
+  # same minimum version specified. Boto3 1.28.0 has been released on July 6 2023. We should also make sure we
+  # set it to the version that `aiobotocore` supports (see `aiobotocore` optional dependency at the end
+  # of this file). Currently we set aiobotocore as minimum 2.5.3 - as this is was the first version
+  # that supported boto3 1.28. NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS
+  - boto3>=1.28.0
+  - mypy-boto3-rds>=1.28.0
+  - mypy-boto3-redshift-data>=1.28.0
+  - mypy-boto3-s3>=1.28.0
+  - mypy-boto3-appflow>=1.28.0
+  # NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS
+  - botocore>=1.31.0
   - asgiref
   # watchtower 3 has been released end Jan and introduced breaking change across the board that might
   # change logging behaviour:
@@ -77,11 +89,7 @@ dependencies:
   - jsonpath_ng>=1.5.3
   - redshift_connector>=2.0.888
   - sqlalchemy_redshift>=0.8.6
-  - mypy-boto3-rds>=1.24.0
-  - mypy-boto3-redshift-data>=1.24.0
-  - mypy-boto3-appflow>=1.28.16
   - asgiref
-  - mypy-boto3-s3>=1.24.0
 
 integrations:
   - integration-name: Amazon Athena
@@ -699,7 +707,7 @@ additional-extras:
   # boto3 have native async support and we move away from aio aiobotocore
   - name: aiobotocore
     dependencies:
-      - aiobotocore[boto3]>=2.2.0
+      - aiobotocore[boto3]>=2.5.3
   - name: cncf.kubernetes
     dependencies:
       - apache-airflow-providers-cncf-kubernetes>=7.2.0

generated/provider_dependencies.json

@@ -26,12 +26,13 @@
       "apache-airflow>=2.4.0",
       "asgiref",
       "asgiref",
-      "boto3>=1.24.0",
+      "boto3>=1.28.0",
+      "botocore>=1.31.0",
       "jsonpath_ng>=1.5.3",
-      "mypy-boto3-appflow>=1.28.16",
-      "mypy-boto3-rds>=1.24.0",
-      "mypy-boto3-redshift-data>=1.24.0",
-      "mypy-boto3-s3>=1.24.0",
+      "mypy-boto3-appflow>=1.28.0",
+      "mypy-boto3-rds>=1.28.0",
+      "mypy-boto3-redshift-data>=1.28.0",
+      "mypy-boto3-s3>=1.28.0",
       "redshift_connector>=2.0.888",
       "sqlalchemy_redshift>=0.8.6",
       "watchtower~=2.0.1"

scripts/in_container/_in_container_utils.sh

@@ -297,6 +297,12 @@ function install_all_providers_from_pypi_with_eager_upgrade() {
     local res
     for provider_package in ${ALL_PROVIDERS_PACKAGES}
     do
+        # Until we release "yandex" provider with protobuf support we need to remove it from the list of providers
+        # to install, because it is impossible to find common requirements for already released yandex provider
+        # and current airflow
+        if [[ ${provider_package} == "apache-airflow-providers-yandex" ]]; then
+            continue
+        fi
         echo -n "Checking if ${provider_package} is available in PyPI: "
         res=$(curl --head -s -o /dev/null -w "%{http_code}" "https://pypi.org/project/${provider_package}/")
         if [[ ${res} == "200" ]]; then
@@ -306,7 +312,11 @@ function install_all_providers_from_pypi_with_eager_upgrade() {
             echo "${COLOR_YELLOW}Skipped${COLOR_RESET}"
         fi
     done
+
+
     echo "Installing provider packages: ${packages_to_install[*]}"
+
+
     # we add eager requirements to make sure to take into account limitations that will allow us to
     # install all providers. We install only those packages that are available in PyPI - we might
     # Have some new providers in the works and they might not yet be simply available in PyPI