airbyte-ci: avoid transient publish to PyPi failures (#42034)

Author: alafanechere

airbyte-ci/connectors/pipelines/README.md

@@ -790,6 +790,7 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only
 
 | Version | PR                                                         | Description                                                                                                                  |
 | ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| 4.32.1  | [#41642](https://github.com/airbytehq/airbyte/pull/41642)  | Avoid transient publish failures by increasing `POETRY_REQUESTS_TIMEOUT` and setting retries on `PublishToPythonRegistry`.   |
 | 4.32.0  | [#43969](https://github.com/airbytehq/airbyte/pull/43969)  | Add an `--ignore-connector` option to `up-to-date`                                                                           |
 | 4.31.5  | [#43934](https://github.com/airbytehq/airbyte/pull/43934)  | Track deleted files when generating pull-request                                                                             |
 | 4.31.4  | [#43724](https://github.com/airbytehq/airbyte/pull/43724)  | Do not send slack message on connector pre-release.                                                                          |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py

@@ -26,6 +26,7 @@ class PackageType(Enum):
 class PublishToPythonRegistry(Step):
     context: PythonRegistryPublishContext
     title = "Publish package to python registry"
+    max_retries = 3
 
     def _get_base_container(self) -> Container:
         return with_poetry(self.context)
@@ -120,6 +121,10 @@ async def _poetry_publish(self, package_dir_to_publish: Directory) -> StepResult
             .with_env_variable("CACHEBUSTER", str(uuid.uuid4()))
             .with_exec(["poetry", "config", "repositories.mypypi", self.context.registry])
             .with_exec(sh_dash_c(["poetry config pypi-token.mypypi $PYTHON_REGISTRY_TOKEN"]))
+            # Default timeout is set to 15 seconds
+            # We sometime face 443 HTTP read timeout responses from PyPi
+            # Setting it to 60 seconds to avoid transient publish failures
+            .with_env_variable("POETRY_REQUESTS_TIMEOUT", "60")
             .with_exec(sh_dash_c(["poetry publish --build --repository mypypi -vvv --no-interaction"]))
         )
 

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.32.0"
+version = "4.32.1"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]
 

airbyte-ci/connectors/pipelines/tests/test_commands/test_groups/test_connectors.py

@@ -154,8 +154,11 @@ def test_get_selected_connectors_with_modified_and_support_level():
         metadata_query=None,
         modified_files=modified_files,
     )
-
-    assert len(selected_connectors) == 1
+    has_strict_encrypt_variant = any("-strict-encrypt" in c.technical_name for c in selected_connectors)
+    if has_strict_encrypt_variant:
+        assert len(selected_connectors) == 2
+    else:
+        assert len(selected_connectors) == 1
     assert selected_connectors[0].technical_name == second_modified_connector.technical_name