fix(source-amazon-ads): fix download request for sponsored_products_report_stream (#43354)

Author: darynaishchenko

airbyte-integrations/connectors/source-amazon-ads/metadata.yaml

@@ -13,7 +13,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: c6b0a29e-1da9-4512-9002-7bfd0cba2246
-  dockerImageTag: 5.0.10
+  dockerImageTag: 5.0.11
   dockerRepository: airbyte/source-amazon-ads
   documentationUrl: https://docs.airbyte.com/integrations/sources/amazon-ads
   githubIssueLabel: source-amazon-ads

airbyte-integrations/connectors/source-amazon-ads/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "5.0.10"
+version = "5.0.11"
 name = "source-amazon-ads"
 description = "Source implementation for Amazon Ads."
 authors = [ "Airbyte <[email protected]>",]

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/products_report.py

@@ -4,9 +4,14 @@
 
 
 from http import HTTPStatus
-from typing import List
+from typing import Any, List, Mapping
 
-from .report_streams import ReportInfo, ReportStream
+import backoff
+import requests
+from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator
+from source_amazon_ads.schemas import Profile
+
+from .report_streams import ReportInfo, ReportStream, TooManyRequests
 
 METRICS_MAP = {
     "campaigns": [
@@ -266,6 +271,12 @@ class SponsoredProductsReportStream(ReportStream):
     metrics_map = METRICS_MAP
     metrics_type_to_id_map = METRICS_TYPE_TO_ID_MAP
 
+    def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authenticator: Oauth2Authenticator):
+        super().__init__(config, profiles, authenticator)
+        # using session without auth as API returns 400 bad request if Authorization header presents in request
+        # X-Amz-Algorithm and X-Amz-Signature query params already present in the url, that is enough to make proper request
+        self._report_download_session = requests.Session()
+
     def report_init_endpoint(self, record_type: str) -> str:
         return f"/{self.API_VERSION}/reports"
 

airbyte-integrations/connectors/source-amazon-ads/source_amazon_ads/streams/report_streams/report_streams.py

@@ -121,6 +121,7 @@ def __init__(self, config: Mapping[str, Any], profiles: List[Profile], authentic
         self._state = {}
         self._session = requests.Session()
         self._session.auth = authenticator
+        self._report_download_session = self._session
         self._model = self._generate_model()
         self._start_date: Optional[Date] = config.get("start_date")
         self._look_back_window: int = config["look_back_window"]
@@ -294,12 +295,13 @@ def _check_status(self, report_info: ReportInfo) -> Tuple[Status, str]:
         ),
         max_tries=10,
     )
-    def _send_http_request(self, url: str, profile_id: int, json: dict = None):
+    def _send_http_request(self, url: str, profile_id: int, json: dict = None, is_download_report: bool = False):
         headers = self._get_auth_headers(profile_id)
         if json:
             response = self._session.post(url, headers=headers, json=json)
         else:
-            response = self._session.get(url, headers=headers)
+            session = self._report_download_session if is_download_report else self._session
+            response = session.get(url, headers=headers)
         if response.status_code == HTTPStatus.TOO_MANY_REQUESTS:
             raise TooManyRequests()
         return response
@@ -433,7 +435,8 @@ def _download_report(self, report_info: ReportInfo, url: str) -> List[dict]:
         """
         Download and parse report result
         """
-        response = self._send_http_request(url, report_info.profile_id) if report_info else self._send_http_request(url, None)
+        profile_id = report_info.profile_id if report_info else None
+        response = self._send_http_request(url=url, profile_id=profile_id, is_download_report=True)
         response.raise_for_status()
         raw_string = decompress(response.content).decode("utf")
         return json.loads(raw_string)

airbyte-integrations/connectors/source-amazon-ads/unit_tests/test_report_streams.py

@@ -11,6 +11,7 @@
 
 import pendulum
 import pytest
+import requests_mock
 import responses
 from airbyte_cdk.models import SyncMode
 from freezegun import freeze_time
@@ -876,3 +877,17 @@ def test_get_record_id_by_report_type(config, metric_object, record_type):
     profiles = make_profiles(profile_type="vendor")
     stream = SponsoredProductsReportStream(config, profiles, authenticator=mock.MagicMock())
     assert stream.get_record_id(metric_object, record_type), "recordId must be non-empty value"
+
+
+def test_sponsored_products_report_stream_send_http_request_on_download_not_use_headers(config):
+    profiles = make_profiles(profile_type="vendor")
+    stream = SponsoredProductsReportStream(config,profiles, authenticator=mock.MagicMock())
+    download_url = "https://download/url"
+
+    with requests_mock.Mocker() as m:
+        request_mock = m.get(download_url, status_code=200)
+        stream._send_http_request(download_url, None, None, True)
+
+    assert request_mock.called is True
+    assert "Authorization" not in request_mock.request_history[0].matcher.last_request._request.headers.keys()
+

docs/integrations/sources/amazon-ads.md

@@ -131,26 +131,27 @@ Information about expected report generation waiting time can be found [here](ht
   <summary>Expand to review</summary>
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                         |
-| :------ | :--------- | :------------------------------------------------------- | :-------------------------------------------------------------------------------------------------------------- |
-| 5.0.10 | 2024-08-10 | [42162](https://github.com/airbytehq/airbyte/pull/42162) | Update dependencies |
-| 5.0.9 | 2024-07-13 | [41876](https://github.com/airbytehq/airbyte/pull/41876) | Update dependencies |
-| 5.0.8 | 2024-07-10 | [41487](https://github.com/airbytehq/airbyte/pull/41487) | Update dependencies |
-| 5.0.7 | 2024-07-09 | [41143](https://github.com/airbytehq/airbyte/pull/41143) | Update dependencies |
-| 5.0.6 | 2024-07-06 | [40798](https://github.com/airbytehq/airbyte/pull/40798) | Update dependencies |
-| 5.0.5 | 2024-06-25 | [40403](https://github.com/airbytehq/airbyte/pull/40403) | Update dependencies |
-| 5.0.4 | 2024-06-21 | [39926](https://github.com/airbytehq/airbyte/pull/39926) | Update dependencies |
-| 5.0.3 | 2024-06-04 | [38962](https://github.com/airbytehq/airbyte/pull/38962) | [autopull] Upgrade base image to v1.2.1 |
-| 5.0.2 | 2024-05-29 | [38737](https://github.com/airbytehq/airbyte/pull/38737) | Update authenticator to `requests_native_auth` package |
-| 5.0.1 | 2024-04-29 | [37655](https://github.com/airbytehq/airbyte/pull/37655) | Update error messages and spec with info about `agency` profile type. |
-| 5.0.0 | 2024-03-22 | [36169](https://github.com/airbytehq/airbyte/pull/36169) | Update `SponsoredBrand` and `SponsoredProduct` streams due to API endpoint deprecation |
-| 4.1.0 | 2024-03-19 | [36267](https://github.com/airbytehq/airbyte/pull/36267) | Pin airbyte-cdk version to `^0` |
-| 4.0.4 | 2024-02-23 | [35481](https://github.com/airbytehq/airbyte/pull/35481) | Migrate source to `YamlDeclarativeSource` with custom `check_connection` |
-| 4.0.3 | 2024-02-12 | [35180](https://github.com/airbytehq/airbyte/pull/35180) | Manage dependencies with Poetry |
-| 4.0.2 | 2024-02-08 | [35013](https://github.com/airbytehq/airbyte/pull/35013) | Add missing field to `sponsored_display_budget_rules` stream |
-| 4.0.1 | 2023-12-28 | [33833](https://github.com/airbytehq/airbyte/pull/33833) | Updated oauth spec to put region, so we can choose oauth consent url based on it |
-| 4.0.0 | 2023-12-28 | [33817](https://github.com/airbytehq/airbyte/pull/33817) | Fix schema for streams: `SponsoredBrandsAdGroups` and `SponsoredBrandsKeywords` |
-| 3.4.2 | 2023-12-12 | [33361](https://github.com/airbytehq/airbyte/pull/33361) | Fix unexpected crash when handling error messages which don't have `requestId` field |
-| 3.4.1 | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image |
+|:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------|
+| 5.0.11  | 2024-08-12 | [43354](https://github.com/airbytehq/airbyte/pull/43354) | Fix download request for `sponsored_products_report_stream`                                                     |
+| 5.0.10  | 2024-08-10 | [42162](https://github.com/airbytehq/airbyte/pull/42162) | Update dependencies                                                                                             |
+| 5.0.9   | 2024-07-13 | [41876](https://github.com/airbytehq/airbyte/pull/41876) | Update dependencies                                                                                             |
+| 5.0.8   | 2024-07-10 | [41487](https://github.com/airbytehq/airbyte/pull/41487) | Update dependencies                                                                                             |
+| 5.0.7   | 2024-07-09 | [41143](https://github.com/airbytehq/airbyte/pull/41143) | Update dependencies                                                                                             |
+| 5.0.6   | 2024-07-06 | [40798](https://github.com/airbytehq/airbyte/pull/40798) | Update dependencies                                                                                             |
+| 5.0.5   | 2024-06-25 | [40403](https://github.com/airbytehq/airbyte/pull/40403) | Update dependencies                                                                                             |
+| 5.0.4   | 2024-06-21 | [39926](https://github.com/airbytehq/airbyte/pull/39926) | Update dependencies                                                                                             |
+| 5.0.3   | 2024-06-04 | [38962](https://github.com/airbytehq/airbyte/pull/38962) | [autopull] Upgrade base image to v1.2.1                                                                         |
+| 5.0.2   | 2024-05-29 | [38737](https://github.com/airbytehq/airbyte/pull/38737) | Update authenticator to `requests_native_auth` package                                                          |
+| 5.0.1   | 2024-04-29 | [37655](https://github.com/airbytehq/airbyte/pull/37655) | Update error messages and spec with info about `agency` profile type.                                           |
+| 5.0.0   | 2024-03-22 | [36169](https://github.com/airbytehq/airbyte/pull/36169) | Update `SponsoredBrand` and `SponsoredProduct` streams due to API endpoint deprecation                          |
+| 4.1.0   | 2024-03-19 | [36267](https://github.com/airbytehq/airbyte/pull/36267) | Pin airbyte-cdk version to `^0`                                                                                 |
+| 4.0.4   | 2024-02-23 | [35481](https://github.com/airbytehq/airbyte/pull/35481) | Migrate source to `YamlDeclarativeSource` with custom `check_connection`                                        |
+| 4.0.3   | 2024-02-12 | [35180](https://github.com/airbytehq/airbyte/pull/35180) | Manage dependencies with Poetry                                                                                 |
+| 4.0.2   | 2024-02-08 | [35013](https://github.com/airbytehq/airbyte/pull/35013) | Add missing field to `sponsored_display_budget_rules` stream                                                    |
+| 4.0.1   | 2023-12-28 | [33833](https://github.com/airbytehq/airbyte/pull/33833) | Updated oauth spec to put region, so we can choose oauth consent url based on it                                |
+| 4.0.0   | 2023-12-28 | [33817](https://github.com/airbytehq/airbyte/pull/33817) | Fix schema for streams: `SponsoredBrandsAdGroups` and `SponsoredBrandsKeywords`                                 |
+| 3.4.2   | 2023-12-12 | [33361](https://github.com/airbytehq/airbyte/pull/33361) | Fix unexpected crash when handling error messages which don't have `requestId` field                            |
+| 3.4.1   | 2023-10-19 | [31599](https://github.com/airbytehq/airbyte/pull/31599) | Base image migration: remove Dockerfile and use the python-connector-base image                                 |
 | 3.4.0   | 2023-06-09 | [25913](https://github.com/airbytehq/airbyte/pull/26203) | Add Stream `DisplayCreatives`                                                                                   |
 | 3.3.0   | 2023-09-22 | [30679](https://github.com/airbytehq/airbyte/pull/30679) | Fix unexpected column for `SponsoredProductCampaigns` and `SponsoredBrandsKeywords`                             |
 | 3.2.0   | 2023-09-18 | [30517](https://github.com/airbytehq/airbyte/pull/30517) | Add suggested streams; fix unexpected column issue                                                              |