Source Amazon Ads: remove "report_wait_timeout", "report_generation_max_retries" from config (#18985)

Signed-off-by: Sergey Chvalyuk <[email protected]>

Author: grubberr

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -47,7 +47,7 @@
 - name: Amazon Ads
   sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
   dockerRepository: airbyte/source-amazon-ads
-  dockerImageTag: 0.1.24
+  dockerImageTag: 0.1.25
   documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads
   icon: amazonads.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -725,7 +725,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-amazon-ads:0.1.24"
+- dockerImage: "airbyte/source-amazon-ads:0.1.25"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads"
     connectionSpecification:
@@ -770,40 +770,20 @@
           type: "string"
           default: "NA"
           order: 4
-        report_wait_timeout:
-          title: "Report Wait Timeout"
-          description: "Timeout duration in minutes for Reports. Default is 60 minutes."
-          default: 60
-          examples:
-          - 60
-          - 120
-          order: 5
-          type: "integer"
-        report_generation_max_retries:
-          title: "Report Generation Maximum Retries"
-          description: "Maximum retries Airbyte will attempt for fetching report data.\
-            \ Default is 5."
-          default: 5
-          examples:
-          - 5
-          - 10
-          - 15
-          order: 6
-          type: "integer"
         start_date:
           title: "Start Date"
           description: "The Start date for collecting reports, should not be more\
             \ than 60 days in the past. In YYYY-MM-DD format"
           examples:
           - "2022-10-10"
           - "2022-10-22"
-          order: 7
+          order: 5
           type: "string"
         profiles:
           title: "Profile IDs"
           description: "Profile IDs you want to fetch data for. See <a href=\"https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles\"\
             >docs</a> for more details."
-          order: 8
+          order: 6
           type: "array"
           items:
             type: "integer"
@@ -820,7 +800,7 @@
             - "archived"
           type: "array"
           uniqueItems: true
-          order: 9
+          order: 7
       required:
       - "client_id"
       - "client_secret"

airbyte-integrations/connectors/source-amazon-ads/Dockerfile

@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=0.1.24
+LABEL io.airbyte.version=0.1.25
 LABEL io.airbyte.name=airbyte/source-amazon-ads

airbyte-integrations/connectors/source-amazon-ads/integration_tests/spec.json

@@ -38,33 +38,17 @@
         "default": "NA",
         "order": 4
       },
-      "report_wait_timeout": {
-        "title": "Report Wait Timeout",
-        "description": "Timeout duration in minutes for Reports. Default is 60 minutes.",
-        "default": 60,
-        "examples": [60, 120],
-        "order": 5,
-        "type": "integer"
-      },
-      "report_generation_max_retries": {
-        "title": "Report Generation Maximum Retries",
-        "description": "Maximum retries Airbyte will attempt for fetching report data. Default is 5.",
-        "default": 5,
-        "examples": [5, 10, 15],
-        "order": 6,
-        "type": "integer"
-      },
       "start_date": {
         "title": "Start Date",
         "description": "The Start date for collecting reports, should not be more than 60 days in the past. In YYYY-MM-DD format",
         "examples": ["2022-10-10", "2022-10-22"],
-        "order": 7,
+        "order": 5,
         "type": "string"
       },
       "profiles": {
         "title": "Profile IDs",
         "description": "Profile IDs you want to fetch data for. See <a href=\"https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles\">docs</a> for more details.",
-        "order": 8,
+        "order": 6,
         "type": "array",
         "items": {
           "type": "integer"
@@ -79,7 +63,7 @@
         },
         "type": "array",
         "uniqueItems": true,
-        "order": 9
+        "order": 7
       }
     },
     "required": ["client_id", "client_secret", "refresh_token"],

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/spec.yaml

@@ -46,27 +46,6 @@ connectionSpecification:
       type: string
       default: NA
       order: 4
-    report_wait_timeout:
-      title: Report Wait Timeout
-      description: Timeout duration in minutes for Reports. Default is 60 minutes.
-      default: 60
-      examples:
-        - 60
-        - 120
-      order: 5
-      type: integer
-    report_generation_max_retries:
-      title: Report Generation Maximum Retries
-      description:
-        Maximum retries Airbyte will attempt for fetching report data.
-        Default is 5.
-      default: 5
-      examples:
-        - 5
-        - 10
-        - 15
-      order: 6
-      type: integer
     start_date:
       title: Start Date
       description:
@@ -75,14 +54,14 @@ connectionSpecification:
       examples:
         - "2022-10-10"
         - "2022-10-22"
-      order: 7
+      order: 5
       type: string
     profiles:
       title: Profile IDs
       description:
         Profile IDs you want to fetch data for. See <a href="https://advertising.amazon.com/API/docs/en-us/concepts/authorization/profiles">docs</a>
         for more details.
-      order: 8
+      order: 6
       type: array
       items:
         type: integer
@@ -97,7 +76,7 @@ connectionSpecification:
           - archived
       type: array
       uniqueItems: true
-      order: 9
+      order: 7
   required:
     - client_id
     - client_secret

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py

@@ -22,7 +22,7 @@
 from pydantic import BaseModel
 from source_amazon_ads.schemas import CatalogModel, MetricsReport, Profile
 from source_amazon_ads.streams.common import BasicAmazonAdsStream
-from source_amazon_ads.utils import iterate_one_by_one
+from source_amazon_ads.utils import get_typed_env, iterate_one_by_one
 
 
 class RecordType(str, Enum):
@@ -112,14 +112,16 @@ class ReportStream(BasicAmazonAdsStream, ABC):
     ]
 
     def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authenticator: Oauth2Authenticator):
+        super().__init__(config, profiles)
         self._state = {}
         self._authenticator = authenticator
         self._session = requests.Session()
         self._model = self._generate_model()
-        self.report_wait_timeout = config.get("report_wait_timeout", 60)
-        self.report_generation_maximum_retries = config.get("report_generation_max_retries", 5)
         self._start_date: Optional[Date] = config.get("start_date")
-        super().__init__(config, profiles)
+        # Timeout duration in minutes for Reports. Default is 180 minutes.
+        self.report_wait_timeout: int = get_typed_env("REPORT_WAIT_TIMEOUT", 180)
+        # Maximum retries Airbyte will attempt for fetching report data. Default is 5.
+        self.report_generation_maximum_retries: int = get_typed_env("REPORT_GENERATION_MAX_RETRIES", 5)
 
     @property
     def model(self) -> CatalogModel:
@@ -407,7 +409,7 @@ def _download_report(self, report_info: ReportInfo, url: str) -> List[dict]:
 
     def get_error_display_message(self, exception: BaseException) -> Optional[str]:
         if isinstance(exception, ReportGenerationInProgress):
-            return f'Report(s) generation time took more than {self.report_wait_timeout} minutes, please increase the "report_wait_timeout" parameter in configuration.'
+            return f"Report(s) generation time took more than {self.report_wait_timeout} minutes and failed because of Amazon API issues. Please wait some time and run synchronization again."
         return super().get_error_display_message(exception)
 
     def _get_response_error_details(self, response) -> Optional[str]:

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/utils.py

@@ -2,6 +2,12 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+import logging
+import os
+from typing import Union
+
+logger = logging.getLogger("airbyte")
+
 
 def iterate_one_by_one(*iterables):
     iterables = list(iterables)
@@ -13,3 +19,14 @@ def iterate_one_by_one(*iterables):
             pass
         else:
             iterables.append(iterable)
+
+
+def get_typed_env(name: str, default: Union[str, int]) -> Union[str, int]:
+    convert = type(default)
+    assert convert in [str, int]
+    value = os.environ.get(name, default)
+    try:
+        return convert(value)
+    except ValueError:
+        logger.warning(f"Cannot convert environment variable {name}={value!r} to type {convert}")
+        return default

airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py

@@ -244,7 +244,7 @@ def test_display_report_stream_init_too_many_requests(mocker, config):
         ),
         (
             [
-                (lambda x: x > 5, None, "2021-01-02 04:04:05"),
+                (lambda x: x > 5, None, "2021-01-02 06:04:05"),
             ],
             ReportGenerationInProgress,
         ),
@@ -259,11 +259,11 @@ def test_display_report_stream_init_too_many_requests(mocker, config):
         (
             [
                 (lambda x: True, "FAILURE", None),
-                (lambda x: x >= 10, None, "2021-01-02 04:04:05"),
-                (lambda x: x >= 15, None, "2021-01-02 05:04:05"),
-                (lambda x: x >= 20, None, "2021-01-02 06:04:05"),
-                (lambda x: x >= 25, None, "2021-01-02 07:04:05"),
-                (lambda x: x >= 30, None, "2021-01-02 08:04:05"),
+                (lambda x: x >= 10, None, "2021-01-02 06:04:05"),
+                (lambda x: x >= 15, None, "2021-01-02 09:04:05"),
+                (lambda x: x >= 20, None, "2021-01-02 12:04:05"),
+                (lambda x: x >= 25, None, "2021-01-02 15:04:05"),
+                (lambda x: x >= 30, None, "2021-01-02 18:04:05"),
             ],
             ReportGenerationFailure,
         ),

airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_utils.py

@@ -0,0 +1,17 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+from source_amazon_ads.utils import get_typed_env
+
+
+def test_get_typed_env(monkeypatch):
+    assert get_typed_env("REPORT_WAIT_TIMEOUT", 180) == 180
+    assert get_typed_env("BOOLEAN_PARAM", "1") == "1"
+    assert get_typed_env("STRING_PARAM", "string") == "string"
+    monkeypatch.setenv("REPORT_WAIT_TIMEOUT", "60")
+    assert get_typed_env("REPORT_WAIT_TIMEOUT", 180) == 60
+    monkeypatch.setenv("REPORT_WAIT_TIMEOUT", "60")
+    assert get_typed_env("REPORT_WAIT_TIMEOUT", "180") == "60"
+    monkeypatch.setenv("REPORT_WAIT_TIMEOUT", "string")
+    assert get_typed_env("REPORT_WAIT_TIMEOUT", 180) == 180

docs/integrations/sources/amazon-ads.md

@@ -91,7 +91,8 @@ Information about expected report generation waiting time you may find [here](ht
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                         |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
-| 0.1.24  | 2022-10-19 | [17475](https://github.com/airbytehq/airbyte/pull/17475) | Add filters for state on brand, product and display campaigns |
+| 0.1.25  | 2022-11-08 | [18985](https://github.com/airbytehq/airbyte/pull/18985) | Remove "report_wait_timeout", "report_generation_max_retries" from config                                       |
+| 0.1.24  | 2022-10-19 | [17475](https://github.com/airbytehq/airbyte/pull/17475) | Add filters for state on brand, product and display campaigns                                                   |
 | 0.1.23  | 2022-09-06 | [16342](https://github.com/airbytehq/airbyte/pull/16342) | Add attribution reports                                                                                         |
 | 0.1.22  | 2022-09-28 | [17304](https://github.com/airbytehq/airbyte/pull/17304) | Migrate to per-stream state.                                                                                    |
 | 0.1.21  | 2022-09-27 | [17202](https://github.com/airbytehq/airbyte/pull/17202) | Improved handling if known reporting errors                                                                     |