Source Google Ads: rollback patch 500 Internal Server Error (airbytehq#35493)

Author: roman-yermilov-gl

airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records.jsonl

@@ -46,9 +46,9 @@
 {"stream": "custom_audience", "data": {"custom_audience.description": "", "custom_audience.name": "Airbyte", "custom_audience.id": 523469909, "custom_audience.members": ["member_type: KEYWORD\nkeyword: \"etl elt\"\n", "member_type: KEYWORD\nkeyword: \"cloud data management and analytics\"\n", "member_type: KEYWORD\nkeyword: \"data integration\"\n", "member_type: KEYWORD\nkeyword: \"big data analytics database\"\n", "member_type: KEYWORD\nkeyword: \"data\"\n", "member_type: KEYWORD\nkeyword: \"data sherid nada\"\n", "member_type: KEYWORD\nkeyword: \"airbyteforeveryone\"\n", "member_type: KEYWORD\nkeyword: \"Airbyte\"\n"], "custom_audience.resource_name": "customers/4651612872/customAudiences/523469909", "custom_audience.status": "ENABLED", "custom_audience.type": "AUTO"}, "emitted_at": 1704407775427}
 {"stream": "audience", "data": {"customer.id": 4651612872, "audience.description": "", "audience.dimensions": ["audience_segments {\n  segments {\n    custom_audience {\n      custom_audience: \"customers/4651612872/customAudiences/523469909\"\n    }\n  }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 47792633, "audience.name": "Audience name 1", "audience.resource_name": "customers/4651612872/audiences/47792633", "audience.status": "ENABLED"}, "emitted_at": 1704407775721}
 {"stream": "audience", "data": {"customer.id": 4651612872, "audience.description": "", "audience.dimensions": ["audience_segments {\n  segments {\n    user_interest {\n      user_interest_category: \"customers/4651612872/userInterests/80276\"\n    }\n  }\n  segments {\n    user_interest {\n      user_interest_category: \"customers/4651612872/userInterests/80279\"\n    }\n  }\n  segments {\n    user_interest {\n      user_interest_category: \"customers/4651612872/userInterests/80520\"\n    }\n  }\n  segments {\n    user_interest {\n      user_interest_category: \"customers/4651612872/userInterests/80530\"\n    }\n  }\n  segments {\n    user_interest {\n      user_interest_category: \"customers/4651612872/userInterests/92931\"\n    }\n  }\n}\n"], "audience.exclusion_dimension": "", "audience.id": 97300129, "audience.name": "Upgraded Audience 1", "audience.resource_name": "customers/4651612872/audiences/97300129", "audience.status": "ENABLED"}, "emitted_at": 1704407775723}
-{"stream": "user_interest", "data": {"user_interest.availabilities": null, "user_interest.launched_to_all": null, "user_interest.name": "Arts & Entertainment", "user_interest.resource_name": "customers/4651612872/userInterests/3", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 3, "user_interest.user_interest_parent": ""}, "emitted_at": 1704407777549}
-{"stream": "user_interest", "data": {"user_interest.availabilities": null, "user_interest.launched_to_all": null, "user_interest.name": "Computers & Electronics", "user_interest.resource_name": "customers/4651612872/userInterests/5", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 5, "user_interest.user_interest_parent": ""}, "emitted_at": 1704407777550}
-{"stream": "user_interest", "data": {"user_interest.availabilities": null, "user_interest.launched_to_all": null, "user_interest.name": "Finance", "user_interest.resource_name": "customers/4651612872/userInterests/7", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 7, "user_interest.user_interest_parent": ""}, "emitted_at": 1704407777551}
+{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Arts & Entertainment", "user_interest.resource_name": "customers/4651612872/userInterests/3", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 3, "user_interest.user_interest_parent": ""}, "emitted_at": 1704407777549}
+{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Computers & Electronics", "user_interest.resource_name": "customers/4651612872/userInterests/5", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 5, "user_interest.user_interest_parent": ""}, "emitted_at": 1704407777550}
+{"stream": "user_interest", "data": {"user_interest.availabilities": [], "user_interest.launched_to_all": true, "user_interest.name": "Finance", "user_interest.resource_name": "customers/4651612872/userInterests/7", "user_interest.taxonomy_type": "VERTICAL_GEO", "user_interest.user_interest_id": 7, "user_interest.user_interest_parent": ""}, "emitted_at": 1704407777551}
 {"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21585034471, "label.name": "edgao-example-label", "label.resource_name": "customers/4651612872/labels/21585034471", "label.status": "ENABLED", "label.text_label.background_color": "#E993EB", "label.text_label.description": "example label for edgao"}, "emitted_at": 1704407779851}
 {"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21902092838, "label.name": "Test Label", "label.resource_name": "customers/4651612872/labels/21902092838", "label.status": "ENABLED", "label.text_label.background_color": "#8BCBD2", "label.text_label.description": "Description to test label"}, "emitted_at": 1704407779852}
 {"stream": "label", "data": {"customer.id": 4651612872, "label.id": 21906377810, "label.name": "Test Delete label customer", "label.resource_name": "customers/4651612872/labels/21906377810", "label.status": "ENABLED", "label.text_label.background_color": "#8266C9", "label.text_label.description": ""}, "emitted_at": 1704407779852}

airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records_click.jsonl

@@ -11,7 +11,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 253487c0-2246-43ba-a21f-5116b20a2c50
-  dockerImageTag: 3.3.3
+  dockerImageTag: 3.3.4
   dockerRepository: airbyte/source-google-ads
   documentationUrl: https://docs.airbyte.com/integrations/sources/google-ads
   githubIssueLabel: source-google-ads

airbyte-integrations/connectors/source-google-ads/metadata.yaml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "3.3.2"
+version = "3.3.4"
 name = "source-google-ads"
 description = "Source implementation for Google Ads."
 authors = [ "Airbyte <[email protected]>",]

airbyte-integrations/connectors/source-google-ads/pyproject.toml

@@ -214,18 +214,8 @@ def get_field_value(field_value: GoogleAdsRow, field: str, schema_type: Mapping[
         return field_value
 
     @staticmethod
-    def parse_single_result(schema: Mapping[str, Any], result: GoogleAdsRow, nullable: Optional[List[str]] = None):
-        if nullable is None:
-            nullable = []
+    def parse_single_result(schema: Mapping[str, Any], result: GoogleAdsRow):
         props = schema.get("properties")
         fields = GoogleAds.get_fields_from_schema(schema)
-        # Making fields nullable is a temporary try to avoid `500 Internal server error` happen when
-        # `user_interest.availabilities` and `user_interest.launched_to_all` fields are queried while syncing `user_interest` stream.
-        # It seems like the values for the fields have changed following the 500 errors and it is unexpected,
-        # so until we understand the issue with the 500 errors, we will nullify those two fields.
-        # This should affect only the `user_interest` stream.
-        # TODO: we need to get rid of the nullable condition once the issue https://github.com/airbytehq/oncall/issues/4306 to be resolved.
-        single_record = {
-            field: GoogleAds.get_field_value(result, field, props.get(field)) if field not in nullable else None for field in fields
-        }
+        single_record = {field: GoogleAds.get_field_value(result, field, props.get(field)) for field in fields}
         return single_record

airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py

@@ -31,24 +31,14 @@ def __init__(self, api: GoogleAds, customers: List[CustomerModel]):
         self.customers = customers
 
     def get_query(self, stream_slice: Mapping[str, Any]) -> str:
-        # The following construction excludes "user_interest.availabilities" and "user_interest.launched_to_all" fields
-        # from the query because they are a cause of "500 Internal server error" on the google side.
-        # The oncall issue: https://github.com/airbytehq/oncall/issues/4306
-        # TODO: replace with fields = GoogleAds.get_fields_from_schema(self.get_json_schema())
-        fields = [
-            field
-            for field in GoogleAds.get_fields_from_schema(self.get_json_schema())
-            if not (field == "user_interest.availabilities" or field == "user_interest.launched_to_all")
-        ]
+        fields = GoogleAds.get_fields_from_schema(self.get_json_schema())
         table_name = get_resource_name(self.name)
         query = GoogleAds.convert_schema_into_query(fields=fields, table_name=table_name)
         return query
 
     def parse_response(self, response: SearchPager, stream_slice: Optional[Mapping[str, Any]] = None) -> Iterable[Mapping]:
         for result in response:
-            yield self.google_ads_client.parse_single_result(
-                self.get_json_schema(), result, nullable=["user_interest.availabilities", "user_interest.launched_to_all"]
-            )
+            yield self.google_ads_client.parse_single_result(self.get_json_schema(), result)
 
     def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]:
         for customer in self.customers:

airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py

@@ -51,7 +51,7 @@ def mock_response_child():
 
 
 class MockGoogleAds(GoogleAds):
-    def parse_single_result(self, schema, result, nullable = None):
+    def parse_single_result(self, schema, result):
         return result
 
     def send_request(self, query: str, customer_id: str, login_customer_id: str = "default"):
@@ -219,7 +219,7 @@ def mock_response_4():
 class MockGoogleAdsLimit(GoogleAds):
     count = 0
 
-    def parse_single_result(self, schema, result, nullable = None):
+    def parse_single_result(self, schema, result):
         return result
 
     def send_request(self, query: str, customer_id: str, login_customer_id: str = "default"):

airbyte-integrations/connectors/source-google-ads/unit_tests/test_incremental_events_streams.py

@@ -507,7 +507,7 @@ def test_get_customers(mocker, customer_status_filter, expected_ids, send_reques
 
     mock_google_api.get_accessible_accounts.return_value = ["123", "789"]
     mock_google_api.send_request.side_effect = mock_send_request
-    mock_google_api.parse_single_result.side_effect = lambda schema, result, nullable: result
+    mock_google_api.parse_single_result.side_effect = lambda schema, result: result
 
     mock_config = {"customer_status_filter": customer_status_filter, "customer_ids": ["123", "456", "789"]}
 

airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py

@@ -48,7 +48,7 @@ def mock_response_2():
 class MockGoogleAds(GoogleAds):
     count = 0
 
-    def parse_single_result(self, schema, result, nullable = None):
+    def parse_single_result(self, schema, result):
         return result
 
     def send_request(self, query: str, customer_id: str, login_customer_id: str = "none"):

airbyte-integrations/connectors/source-google-ads/unit_tests/test_streams.py

@@ -280,6 +280,7 @@ Due to a limitation in the Google Ads API which does not allow getting performan
 
 | Version  | Date       | Pull Request                                             | Subject                                                                                                                              |
 |:---------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------|
+| `3.3.4`  | 2024-02-21 | [35493](https://github.com/airbytehq/airbyte/pull/35493) | Rolling back the patch 3.3.3 made for `user_interest` steam                                                                          |
 | `3.3.3`  | 2024-02-14 | [35280](https://github.com/airbytehq/airbyte/pull/35280) | Temporary patch that disables some fields to avoid 500 error when syncing `user_interest` steam                                      |
 | `3.3.2`  | 2024-02-12 | [35158](https://github.com/airbytehq/airbyte/pull/35158) | Manage dependencies with Poetry.                                                                                                     |
 | `3.3.1`  | 2024-01-16 | [34007](https://github.com/airbytehq/airbyte/pull/34007) | prepare for airbyte-lib                                                                                                              |