Source Facebook Marketing: add retry for transient error (#37320)

Signed-off-by: Artem Inzhyyants <[email protected]>

Author: artem1205

airbyte-integrations/connectors/source-facebook-marketing/metadata.yaml

@@ -10,7 +10,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
-  dockerImageTag: 2.1.2
+  dockerImageTag: 2.1.3
   dockerRepository: airbyte/source-facebook-marketing
   documentationUrl: https://docs.airbyte.com/integrations/sources/facebook-marketing
   githubIssueLabel: source-facebook-marketing

airbyte-integrations/connectors/source-facebook-marketing/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.1.2"
+version = "2.1.3"
 name = "source-facebook-marketing"
 description = "Source implementation for Facebook Marketing."
 authors = [ "Airbyte <[email protected]>",]

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/streams/common.py

@@ -4,6 +4,7 @@
 
 import http.client
 import logging
+import re
 import sys
 from typing import Any
 
@@ -84,6 +85,12 @@ def revert_request_record_limit(details):
         # set the flag to the api class that the `limit` param is restored
         details.get("args")[0].request_record_limit_is_reduced = False
 
+    def is_transient_cannot_include_error(exc: FacebookRequestError) -> bool:
+        """After migration to API v19.0, some customers randomly face a BAD_REQUEST error (OAuthException) with the pattern:"Cannot include ..."
+        According to the last comment in https://developers.facebook.com/community/threads/286697364476462/, this might be a transient issue that can be solved with a retry."""
+        pattern = r"Cannot include .* in summary param because they weren't there while creating the report run."
+        return bool(exc.http_status() == http.client.BAD_REQUEST and re.search(pattern, exc.api_error_message()))
+
     def should_retry_api_error(exc):
         if isinstance(exc, FacebookRequestError):
             call_rate_limit_error = exc.api_error_code() in FACEBOOK_RATE_LIMIT_ERROR_CODES
@@ -98,6 +105,7 @@ def should_retry_api_error(exc):
                     unknown_error,
                     call_rate_limit_error,
                     batch_timeout_error,
+                    is_transient_cannot_include_error(exc),
                     connection_reset_error,
                     temporary_oauth_error,
                     server_error,

airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_errors.py

@@ -346,7 +346,6 @@ def test_config_error_during_account_info_read(self, requests_mock, name, friend
             assert error.failure_type == FailureType.config_error
             assert friendly_msg in error.message
 
-    # @pytest.mark.parametrize("name, friendly_msg, config_error_response", [CONFIG_ERRORS[-1]])
     @pytest.mark.parametrize("name, friendly_msg, config_error_response", CONFIG_ERRORS)
     def test_config_error_during_actual_nodes_read(self, requests_mock, name, friendly_msg, config_error_response):
         """Error raised during actual nodes read"""
@@ -422,6 +421,37 @@ def test_config_error_insights_during_actual_nodes_read(self, requests_mock, nam
             assert error.failure_type == FailureType.config_error
             assert friendly_msg in error.message
 
+    def test_retry_for_cannot_include_error(self, requests_mock):
+        """Error raised randomly for insights stream. Oncall: https://github.com/airbytehq/oncall/issues/4868 """
+
+        api = API(access_token=some_config["access_token"], page_size=100)
+        stream = AdsInsights(
+            api=api,
+            account_ids=some_config["account_ids"],
+            start_date=datetime(2010, 1, 1),
+            end_date=datetime(2011, 1, 1),
+            fields=["account_id", "account_currency"],
+            insights_lookback_window=28,
+        )
+        requests_mock.register_uri("GET", f"{act_url}", [ad_account_response])
+        response = {
+            "status_code": 400,
+            "json": {
+                "error": {
+                    "message": "(#100) Cannot include video_avg_time_watched_actions, video_continuous_2_sec_watched_actions in summary param because they weren't there while creating the report run.",
+                    "type": "OAuthException",
+                    "code": 100
+                }
+            },
+        }
+        call_insights = requests_mock.register_uri("GET", f"{act_url}insights", [response])
+
+        try:
+            slice = list(stream.stream_slices(sync_mode=SyncMode.full_refresh, stream_state={}))[0]
+            list(stream.read_records(sync_mode=SyncMode.full_refresh, stream_slice=slice, stream_state={}))
+        except Exception:
+            assert call_insights.call_count == 5
+
     @pytest.mark.parametrize(
         "failure_response",
         (

docs/integrations/sources/facebook-marketing.md

@@ -200,6 +200,7 @@ The Facebook Marketing connector uses the `lookback_window` parameter to repeate
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                                                                                                                                           |
 |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 2.1.3   | 2024-04-16 | [37320](https://github.com/airbytehq/airbyte/pull/37320) | Add retry for transient error                                                                                                                                                                                                                                                                     |
 | 2.1.2   | 2024-03-29 | [36689](https://github.com/airbytehq/airbyte/pull/36689) | Fix key error `account_id` for custom reports.                                                                                                                                                                                                                                                    |
 | 2.1.1   | 2024-03-18 | [36025](https://github.com/airbytehq/airbyte/pull/36025) | Fix start_date selection behaviour                                                                                                                                                                                                                                                                |
 | 2.1.0   | 2024-03-12 | [35978](https://github.com/airbytehq/airbyte/pull/35978) | Upgrade CDK to start emitting record counts with state and full refresh state                                                                                                                                                                                                                     |