🎉 Python CDK: handle requests.exceptions.ChunkedEncodingError for broken connections

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.1.60
+- Add `requests.exceptions.ChunkedEncodingError` to transient errors so it could be retried
+
 ## 0.1.59
 - Add `Stream.get_error_display_message()` to retrieve user-friendly messages from exceptions encountered while reading streams.
 - Add default error error message retrieval logic for `HTTPStream`s following common API patterns.

airbyte-cdk/python/airbyte_cdk/sources/streams/http/rate_limiting.py

@@ -12,7 +12,13 @@
 
 from .exceptions import DefaultBackoffException, UserDefinedBackoffException
 
-TRANSIENT_EXCEPTIONS = (DefaultBackoffException, exceptions.ConnectTimeout, exceptions.ReadTimeout, exceptions.ConnectionError)
+TRANSIENT_EXCEPTIONS = (
+    DefaultBackoffException,
+    exceptions.ConnectTimeout,
+    exceptions.ReadTimeout,
+    exceptions.ConnectionError,
+    exceptions.ChunkedEncodingError,
+)
 
 logger = logging.getLogger("airbyte")
 

airbyte-cdk/python/setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.1.59",
+    version="0.1.60",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py

@@ -248,20 +248,22 @@ def test_raise_on_http_errors_off_non_retryable_4xx(mocker, status_code):
     assert response.status_code == status_code
 
 
-def test_raise_on_http_errors_off_timeout(requests_mock):
-    stream = AutoFailFalseHttpStream()
-    requests_mock.register_uri("GET", stream.url_base, exc=requests.exceptions.ConnectTimeout)
-
-    with pytest.raises(requests.exceptions.ConnectTimeout):
-        list(stream.read_records(SyncMode.full_refresh))
-
-
-def test_raise_on_http_errors_off_connection_error(requests_mock):
+@pytest.mark.parametrize(
+    "error",
+    (
+        requests.exceptions.ConnectTimeout,
+        requests.exceptions.ConnectionError,
+        requests.exceptions.ChunkedEncodingError,
+        requests.exceptions.ReadTimeout
+    )
+)
+def test_raise_on_http_errors(mocker, error):
     stream = AutoFailFalseHttpStream()
-    requests_mock.register_uri("GET", stream.url_base, exc=requests.exceptions.ConnectionError)
+    send_mock = mocker.patch.object(requests.Session, "send", side_effect=error())
 
-    with pytest.raises(requests.exceptions.ConnectionError):
+    with pytest.raises(error):
         list(stream.read_records(SyncMode.full_refresh))
+    assert send_mock.call_count == stream.max_retries + 1
 
 
 class PostHttpStream(StubBasicReadHttpStream):