Source Amazon Ads: fix bug with handling: "Report date is too far in the past." (#20662)

Signed-off-by: Sergey Chvalyuk <[email protected]>

Author: grubberr

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -54,7 +54,7 @@
 - name: Amazon Ads
   sourceDefinitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
   dockerRepository: airbyte/source-amazon-ads
-  dockerImageTag: 0.1.25
+  dockerImageTag: 0.1.26
   documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads
   icon: amazonads.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -719,7 +719,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-amazon-ads:0.1.25"
+- dockerImage: "airbyte/source-amazon-ads:0.1.26"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/amazon-ads"
     connectionSpecification:

airbyte-integrations/connectors/source-amazon-ads/Dockerfile

@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=0.1.25
+LABEL io.airbyte.version=0.1.26
 LABEL io.airbyte.name=airbyte/source-amazon-ads

airbyte-integrations/connectors/source-amazon-ads/acceptance-test-config.yml

@@ -1,50 +1,55 @@
 # See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference)
 # for more information about how to configure these tests
 connector_image: airbyte/source-amazon-ads:dev
-tests:
+acceptance_tests:
   spec:
-    - spec_path: "integration_tests/spec.json"
+    tests:
+      - spec_path: "integration_tests/spec.json"
   connection:
-    - config_path: "secrets/config.json"
-      status: "succeed"
-    - config_path: "integration_tests/invalid_config.json"
-      status: "failed"
+    tests:
+      - config_path: "secrets/config.json"
+        status: "succeed"
+      - config_path: "integration_tests/invalid_config.json"
+        status: "failed"
   discovery:
-    - config_path: "secrets/config.json"
+    tests:
+      - config_path: "secrets/config.json"
   basic_read:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-      expect_records:
-        path: "integration_tests/expected_records.jsonl"
-        extra_fields: no
-        exact_order: no
-        extra_records: no
-      empty_streams:
-        [
-          "profiles",
-          "sponsored_brands_ad_groups",
-          "sponsored_brands_campaigns",
-          "sponsored_brands_keywords",
-          "attribution_report_performance_creative",
-          "attribution_report_performance_adgroup",
-          "attribution_report_products",
-          "attribution_report_performance_campaign",
-        ]
-    - config_path: "secrets/config_report.json"
-      configured_catalog_path: "integration_tests/configured_catalog_report.json"
-      timeout_seconds: 2400
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+        expect_records:
+          path: "integration_tests/expected_records.jsonl"
+          extra_fields: no
+          exact_order: no
+          extra_records: no
+        empty_streams:
+          - name: "profiles"
+          - name: "sponsored_brands_ad_groups"
+          - name: "sponsored_brands_campaigns"
+          - name: "sponsored_brands_keywords"
+          - name: "attribution_report_performance_creative"
+          - name: "attribution_report_performance_adgroup"
+          - name: "attribution_report_products"
+          - name: "attribution_report_performance_campaign"
+      - config_path: "secrets/config_report.json"
+        configured_catalog_path: "integration_tests/configured_catalog_report.json"
+        timeout_seconds: 2400
   incremental:
-    - config_path: "secrets/config_report.json"
-      configured_catalog_path: "integration_tests/configured_catalog_report.json"
-      future_state_path: "integration_tests/abnormal_state.json"
-      cursor_paths:
-        sponsored_products_report_stream: ["1861552880916640", "reportDate"]
-      timeout_seconds: 2400
+    tests:
+      - config_path: "secrets/config_report.json"
+        configured_catalog_path: "integration_tests/configured_catalog_report.json"
+        future_state:
+          future_state_path: "integration_tests/abnormal_state.json"
+        cursor_paths:
+          sponsored_products_report_stream: ["1861552880916640", "reportDate"]
+        timeout_seconds: 2400
   full_refresh:
-    - config_path: "secrets/config.json"
-      configured_catalog_path: "integration_tests/configured_catalog.json"
-    - config_path: "secrets/config_report.json"
-      configured_catalog_path: "integration_tests/configured_catalog_report.json"
-      ignored_fields:
-        "sponsored_products_report_stream": ["updatedAt"]
-      timeout_seconds: 3600
+    tests:
+      - config_path: "secrets/config.json"
+        configured_catalog_path: "integration_tests/configured_catalog.json"
+      - config_path: "secrets/config_report.json"
+        configured_catalog_path: "integration_tests/configured_catalog_report.json"
+        ignored_fields:
+          "sponsored_products_report_stream": ["updatedAt"]
+        timeout_seconds: 3600

airbyte-integrations/connectors/source-amazon-ads/setup.py

@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = ["airbyte-cdk~=0.2", "requests_oauthlib~=1.3.1", "pendulum~=2.1.2"]
+MAIN_REQUIREMENTS = ["airbyte-cdk~=0.16", "requests_oauthlib~=1.3.1", "pendulum~=2.1.2"]
 
 TEST_REQUIREMENTS = [
     "pytest~=6.1",

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/source.py

@@ -36,17 +36,26 @@
     SponsoredProductsReportStream,
     SponsoredProductTargetings,
 )
+from .streams.report_streams.report_streams import ReportStream
 
 # Oauth 2.0 authentication URL for amazon
 TOKEN_URL = "https://api.amazon.com/auth/o2/token"
 CONFIG_DATE_FORMAT = "YYYY-MM-DD"
 
 
 class SourceAmazonAds(AbstractSource):
-    def _validate_and_transform(self, config: Mapping[str, Any]):
+    def _validate_and_transform(self, config: Mapping[str, Any], check=False):
         start_date = config.get("start_date")
         if start_date:
             config["start_date"] = pendulum.from_format(start_date, CONFIG_DATE_FORMAT).date()
+            if check:
+                # This validation is used only for improving the user UX experience.
+                # The connector can work without this low boundary checking.
+                # We use REPORTING_PERIOD plus ONE additional day because
+                # amazon account can have multiple profiles in different time zones.
+                min_date = pendulum.today().date().subtract(days=ReportStream.REPORTING_PERIOD + 1)
+                if config["start_date"] < min_date:
+                    raise Exception(f"Start Date: minimum allowed value is {min_date}")
         else:
             config["start_date"] = None
         if not config.get("region"):
@@ -61,7 +70,7 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) ->
         :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise.
         """
         try:
-            config = self._validate_and_transform(config)
+            config = self._validate_and_transform(config, check=True)
         except Exception as e:
             return False, str(e)
         # Check connection by sending list of profiles request. Its most simple

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py

@@ -108,7 +108,7 @@ class ReportStream(BasicAmazonAdsStream, ABC):
         # Check if the connector received an error: 'Report date is too far in the past. Reports are only available for 60 days.'
         # In theory, it does not have to get such an error because the connector correctly calculates the start date,
         # but from practice, we can still catch such errors from time to time.
-        (406, "Report date is too far in the past."),
+        (406, re.compile(r"^Report date is too far in the past\.")),
     ]
 
     def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authenticator: Oauth2Authenticator):

airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_source.py

@@ -2,8 +2,10 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+import pendulum
 import responses
 from airbyte_cdk.models import AirbyteConnectionStatus, AirbyteMessage, ConnectorSpecification, Status, Type
+from freezegun import freeze_time
 from jsonschema import Draft4Validator
 from source_amazon_ads import SourceAmazonAds
 
@@ -41,6 +43,7 @@ def test_spec():
 
 
 @responses.activate
+@freeze_time("2022-12-20 10:00:00")
 def test_check(config_gen):
     setup_responses()
     source = SourceAmazonAds()
@@ -51,7 +54,13 @@ def test_check(config_gen):
     assert command_check(source, config_gen(start_date="")) == AirbyteConnectionStatus(status=Status.SUCCEEDED)
     assert len(responses.calls) == 4
 
-    assert source.check(None, config_gen(start_date="2022-02-20")) == AirbyteConnectionStatus(status=Status.SUCCEEDED)
+    assert source.check(None, config_gen(start_date="1900-01-01")) == AirbyteConnectionStatus(
+        status=Status.FAILED, message="'Start Date: minimum allowed value is 2022-10-20'"
+    )
+    assert len(responses.calls) == 4
+
+    start_date = pendulum.today().format("YYYY-MM-DD")
+    assert source.check(None, config_gen(start_date=start_date)) == AirbyteConnectionStatus(status=Status.SUCCEEDED)
     assert len(responses.calls) == 6
 
     assert command_check(source, config_gen(start_date="2022-20-02")) == AirbyteConnectionStatus(

docs/integrations/sources/amazon-ads.md

@@ -94,6 +94,7 @@ Information about expected report generation waiting time you may find [here](ht
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                         |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
+| 0.1.26  | 2022-12-19 | [20662](https://github.com/airbytehq/airbyte/pull/20662) | Fix bug with handling: "Report date is too far in the past."                                                    |
 | 0.1.25  | 2022-11-08 | [18985](https://github.com/airbytehq/airbyte/pull/18985) | Remove "report_wait_timeout", "report_generation_max_retries" from config                                       |
 | 0.1.24  | 2022-10-19 | [17475](https://github.com/airbytehq/airbyte/pull/17475) | Add filters for state on brand, product and display campaigns                                                   |
 | 0.1.23  | 2022-09-06 | [16342](https://github.com/airbytehq/airbyte/pull/16342) | Add attribution reports                                                                                         |