🐛 Source google-analytics-data-api: fix max time (#43929)

Author: aldogonzalez8

airbyte-integrations/connectors/source-google-analytics-data-api/metadata.yaml

@@ -12,7 +12,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 3cc2eafd-84aa-4dca-93af-322d9dfeec1a
-  dockerImageTag: 2.5.2
+  dockerImageTag: 2.5.3
   dockerRepository: airbyte/source-google-analytics-data-api
   documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-data-api
   githubIssueLabel: source-google-analytics-data-api

airbyte-integrations/connectors/source-google-analytics-data-api/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "2.5.2"
+version = "2.5.3"
 name = "source-google-analytics-data-api"
 description = "Source implementation for Google Analytics Data Api."
 authors = [ "Airbyte <[email protected]>",]

airbyte-integrations/connectors/source-google-analytics-data-api/source_google_analytics_data_api/source.py

@@ -9,6 +9,7 @@
 import re
 import uuid
 from abc import ABC
+from datetime import timedelta
 from http import HTTPStatus
 from typing import Any, Dict, Iterable, List, Mapping, MutableMapping, Optional, Set, Tuple, Type, Union
 
@@ -101,6 +102,20 @@ def backoff_time(
 
 
 class GoogleAnalyticsDatApiErrorHandler(HttpStatusErrorHandler):
+    QUOTA_RECOVERY_TIME = 3600
+
+    def __init__(
+        self,
+        logger: logging.Logger,
+        error_mapping: Optional[Mapping[Union[int, str, type[Exception]], ErrorResolution]] = None,
+    ) -> None:
+        super().__init__(
+            logger=logger,
+            error_mapping=error_mapping,
+            max_retries=5,
+            max_time=timedelta(seconds=GoogleAnalyticsDatApiErrorHandler.QUOTA_RECOVERY_TIME),
+        )
+
     @GoogleAnalyticsQuotaHandler.handle_quota()
     def interpret_response(self, response_or_exception: Optional[Union[requests.Response, Exception]] = None) -> ErrorResolution:
         if not isinstance(response_or_exception, Exception) and response_or_exception.status_code == requests.codes.too_many_requests:

airbyte-integrations/connectors/source-google-analytics-data-api/unit_tests/test_source.py

@@ -10,7 +10,8 @@
 from airbyte_cdk.utils import AirbyteTracedException
 from airbyte_protocol.models import Status
 from source_google_analytics_data_api import SourceGoogleAnalyticsDataApi
-from source_google_analytics_data_api.source import MetadataDescriptor
+from source_google_analytics_data_api.api_quota import GoogleAnalyticsApiQuotaBase
+from source_google_analytics_data_api.source import GoogleAnalyticsDatApiErrorHandler, MetadataDescriptor
 from source_google_analytics_data_api.utils import NO_DIMENSIONS, NO_METRICS, NO_NAME, WRONG_CUSTOM_REPORT_CONFIG, WRONG_JSON_SYNTAX
 
 
@@ -108,7 +109,53 @@ def test_check_failure_throws_exception(requests_mock, config_gen, error_code):
     assert e.value.failure_type == FailureType.config_error
     assert "Access was denied to the property ID entered." in e.value.message
 
-@pytest.mark.parametrize("error_code", (402,404, 405))
+
+def test_exhausted_quota_recovers_after_two_retries(requests_mock, config_gen):
+    """
+        If the account runs out of quota the api will return a message asking us to back off for one hour.
+        We have set backoff time for this scenario to 30 minutes to check if quota is already recovered, if not
+        it will backoff again  30 minutes and quote should be reestablished by then.
+        Now, we don't want wait one hour to test out this retry behavior so we will fix time dividing by 600 the quota
+        recovery time and also the backoff time.
+    """
+    requests_mock.register_uri(
+        "POST", "https://oauth2.googleapis.com/token", json={"access_token": "access_token", "expires_in": 3600, "token_type": "Bearer"}
+    )
+    error_response = {"error": {"message":"Exhausted potentially thresholded requests quota. This quota will refresh in under an hour. To learn more, see"}}
+    requests_mock.register_uri(
+        "GET",
+        "https://analyticsdata.googleapis.com/v1beta/properties/UA-11111111/metadata",
+        # first try we get 429 t=~0
+        [{"json": error_response, "status_code": 429},
+         # first retry we get 429 t=~1800
+         {"json": error_response, "status_code": 429},
+         # second retry quota is recovered, t=~3600
+         {"json": {
+            "dimensions": [{"apiName": "date"}, {"apiName": "country"}, {"apiName": "language"}, {"apiName": "browser"}],
+            "metrics": [{"apiName": "totalUsers"}, {"apiName": "screenPageViews"}, {"apiName": "sessions"}],
+        }, "status_code": 200}
+         ]
+    )
+    def fix_time(time):
+        return int(time / 600 )
+    source = SourceGoogleAnalyticsDataApi()
+    logger = MagicMock()
+    max_time_fixed = fix_time(GoogleAnalyticsDatApiErrorHandler.QUOTA_RECOVERY_TIME)
+    potentially_thresholded_requests_per_hour_mapping = GoogleAnalyticsApiQuotaBase.quota_mapping["potentiallyThresholdedRequestsPerHour"]
+    threshold_backoff_time = potentially_thresholded_requests_per_hour_mapping["backoff"]
+    fixed_threshold_backoff_time = fix_time(threshold_backoff_time)
+    potentially_thresholded_requests_per_hour_mapping_fixed = {
+        **potentially_thresholded_requests_per_hour_mapping,
+        "backoff": fixed_threshold_backoff_time,
+        }
+    with (
+          patch.object(GoogleAnalyticsDatApiErrorHandler, 'QUOTA_RECOVERY_TIME', new=max_time_fixed),
+          patch.object(GoogleAnalyticsApiQuotaBase, 'quota_mapping', new={**GoogleAnalyticsApiQuotaBase.quota_mapping,"potentiallyThresholdedRequestsPerHour": potentially_thresholded_requests_per_hour_mapping_fixed})):
+        output = source.check(logger, config_gen(property_ids=["UA-11111111"]))
+        assert output == AirbyteConnectionStatus(status=Status.SUCCEEDED, message=None)
+
+
+@pytest.mark.parametrize("error_code", (402, 404, 405))
 def test_check_failure(requests_mock, config_gen, error_code):
     requests_mock.register_uri(
         "POST", "https://oauth2.googleapis.com/token", json={"access_token": "access_token", "expires_in": 3600, "token_type": "Bearer"}

docs/integrations/sources/google-analytics-data-api.md

@@ -268,6 +268,7 @@ The Google Analytics connector is subject to Google Analytics Data API quotas. P
 
 | Version | Date       | Pull Request                                             | Subject                                                                                |
 |:--------|:-----------| :------------------------------------------------------- |:---------------------------------------------------------------------------------------|
+| 2.5.3 | 2024-08-13 | [43929](https://github.com/airbytehq/airbyte/pull/43929) | Increase streams max_time to backoff |
 | 2.5.2 | 2024-08-12 | [43909](https://github.com/airbytehq/airbyte/pull/43909) | Update dependencies |
 | 2.5.1 | 2024-08-10 | [43289](https://github.com/airbytehq/airbyte/pull/43289) | Update dependencies |
 | 2.5.0 | 2024-08-07 | [42841](https://github.com/airbytehq/airbyte/pull/42841) | Upgrade to CDK 3 |