fix(source-google-search-console): fix request body for report streams by keyword (#45196)

Author: darynaishchenko

airbyte-integrations/connectors/source-google-search-console/acceptance-test-config.yml

@@ -14,8 +14,9 @@ acceptance_tests:
     tests:
       - config_path: "secrets/config.json"
         status: "succeed"
-      - config_path: "secrets/service_account_config.json"
-        status: "succeed"
+      #        TODO: uncomment testing on service_account_config.json when we update creds
+      #      - config_path: "secrets/service_account_config.json"
+      #        status: "succeed"
       - config_path: "integration_tests/invalid_config.json"
         status: "failed"
   discovery:

airbyte-integrations/connectors/source-google-search-console/integration_tests/expected_records.jsonl

@@ -15,14 +15,14 @@
 {"stream": "search_analytics_all_fields", "data": {"clicks": 2, "impressions": 2, "ctr": 1, "position": 2, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-01-02", "country": "usa", "device": "DESKTOP", "page": "https://discuss.airbyte.io/t/integrating-keycloak-iam-with-airbyte/2826", "query": "airbyte keycloak"}, "emitted_at": 1709913953146}
 {"stream": "custom_dimensions", "data": {"clicks": 97, "impressions": 2392, "ctr": 0.040551839464882944, "position": 24.149247491638796, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-01-03", "country": "usa", "device": "DESKTOP"}, "emitted_at": 1709913970183}
 {"stream": "custom_dimensions", "data": {"clicks": 81, "impressions": 2220, "ctr": 0.03648648648648649, "position": 27.025675675675675, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-01-02", "country": "usa", "device": "DESKTOP"}, "emitted_at": 1709913970184}
-{"stream": "search_analytics_keyword_page_report", "data": {"clicks": 0, "impressions": 1, "ctr": 0, "position": 6, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-01-08", "country": "usa", "device": "DESKTOP", "query": "fatal: not a dbt project (or any of the parent directories). missing dbt_project.yml file", "page": "https://discuss.airbyte.io/t/how-to-set-workspace-folder-job-id-in-entrypoint-arguments-for-custom-dbt-transformation/2805"}, "emitted_at": 1709913956708}
-{"stream": "search_analytics_keyword_page_report", "data": {"clicks": 0, "impressions": 1, "ctr": 0, "position": 4, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-01-09", "country": "usa", "device": "DESKTOP", "query": "could not find a version that satisfies the requirement comm>=0.1.3", "page": "https://discuss.airbyte.io/t/error-could-not-find-a-version-that-satisfies-the-requirement-airbyte-cdk-0-1-56/1397"}, "emitted_at": 1709913956709}
+{"stream": "search_analytics_keyword_page_report", "data": {"clicks": 1, "impressions": 1, "ctr": 1, "position": 4, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-09-01", "country": "ind", "device": "DESKTOP", "query": "airbyte octavia", "page": "https://discuss.airbyte.io/t/airbyte-connectors-configuration-data-storage-using-octavia/3145"}, "emitted_at": 1725547397864}
+{"stream": "search_analytics_keyword_page_report", "data": {"clicks": 0, "impressions": 1, "ctr": 0, "position": 32, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-08-31", "country": "are", "device": "DESKTOP", "query": "database in k8s", "page": "https://discuss.airbyte.io/t/kubernetes-temporal-deployment-failling-to-connect-with-external-azure-postgres-db/2928"}, "emitted_at": 1725547398182}
 {"stream": "search_analytics_page_report", "data": {"clicks": 2, "impressions": 4, "ctr": 0.5, "position": 2.5, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-01-02", "country": "gbr", "device": "DESKTOP", "page": "https://discuss.airbyte.io/t/using-a-private-git-repo-for-transformations-the-selection-criterion-does-not-match-any-nodes/4170"}, "emitted_at": 1709913968085}
 {"stream": "search_analytics_page_report", "data": {"clicks": 2, "impressions": 3, "ctr": 0.6666666666666666, "position": 1, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-01-03", "country": "deu", "device": "DESKTOP", "page": "https://discuss.airbyte.io/t/mixpanel-connector-issue-follow-up-on-previous-case/2814"}, "emitted_at": 1709913968086}
-{"stream": "search_analytics_keyword_site_report_by_page", "data": {"clicks": 0, "impressions": 1, "ctr": 0, "position": 6, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-01-08", "country": "usa", "device": "DESKTOP", "query": "fatal: not a dbt project (or any of the parent directories). missing dbt_project.yml file"}, "emitted_at": 1709913961303}
-{"stream": "search_analytics_keyword_site_report_by_page", "data": {"clicks": 0, "impressions": 1, "ctr": 0, "position": 4, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-01-09", "country": "usa", "device": "DESKTOP", "query": "could not find a version that satisfies the requirement comm>=0.1.3"}, "emitted_at": 1709913961305}
-{"stream": "search_analytics_keyword_site_report_by_site", "data": {"clicks": 0, "impressions": 1, "ctr": 0, "position": 5, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-01-10", "country": "usa", "device": "DESKTOP", "query": "fatal: not a dbt project (or any of the parent directories). missing dbt_project.yml file"}, "emitted_at": 1709913966177}
-{"stream": "search_analytics_keyword_site_report_by_site", "data": {"clicks": 0, "impressions": 1, "ctr": 0, "position": 5, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-01-11", "country": "usa", "device": "DESKTOP", "query": "dbt_project.yml not found"}, "emitted_at": 1709913966179}
+{"stream": "search_analytics_keyword_site_report_by_page", "data": {"clicks": 1, "impressions": 1, "ctr": 1, "position": 6, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-09-12", "country": "aus", "device": "DESKTOP", "query": "nothing to do. try checking your model configs and model specification args"}, "emitted_at": 1726485253585}
+{"stream": "search_analytics_keyword_site_report_by_page", "data": {"clicks": 0, "impressions": 1, "ctr": 0, "position": 3, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-09-12", "country": "can", "device": "DESKTOP", "query": "did not find matching node for patch with name"}, "emitted_at": 1726485254092}
+{"stream": "search_analytics_keyword_site_report_by_site", "data": {"clicks": 1, "impressions": 1, "ctr": 1, "position": 4, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-09-06", "country": "can", "device": "DESKTOP", "query": "airbyte datadog"}, "emitted_at": 1725874488288}
+{"stream": "search_analytics_keyword_site_report_by_site", "data": {"clicks": 0, "impressions": 1, "ctr": 0, "position": 12, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-09-06", "country": "can", "device": "DESKTOP", "query": "error: could not find a version that satisfies the requirement pyyaml (from versions: none)"}, "emitted_at": 1725874488655}
 {"stream": "search_analytics_site_report_by_page", "data": {"clicks": 105, "impressions": 2905, "ctr": 0.03614457831325301, "position": 21.6447504302926, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-01-03", "country": "usa", "device": "DESKTOP"}, "emitted_at": 1709913968684}
 {"stream": "search_analytics_site_report_by_page", "data": {"clicks": 87, "impressions": 2598, "ctr": 0.03348729792147806, "position": 24.50269438029253, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-01-02", "country": "usa", "device": "DESKTOP"}, "emitted_at": 1709913968685}
 {"stream": "search_analytics_site_report_by_site", "data": {"clicks": 97, "impressions": 2392, "ctr": 0.040551839464882944, "position": 24.149247491638796, "site_url": "sc-domain:airbyte.io", "search_type": "web", "date": "2024-01-03", "country": "usa", "device": "DESKTOP"}, "emitted_at": 1709913969703}

airbyte-integrations/connectors/source-google-search-console/metadata.yaml

@@ -10,7 +10,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: eb4c9e00-db83-4d63-a386-39cfa91012a8
-  dockerImageTag: 1.5.3
+  dockerImageTag: 1.5.4
   dockerRepository: airbyte/source-google-search-console
   documentationUrl: https://docs.airbyte.com/integrations/sources/google-search-console
   erdUrl: https://dbdocs.io/airbyteio/source-google-search-console?view=relationships

airbyte-integrations/connectors/source-google-search-console/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "1.5.3"
+version = "1.5.4"
 name = "source-google-search-console"
 description = "Source implementation for Google Search Console."
 authors = [ "Airbyte <[email protected]>",]

airbyte-integrations/connectors/source-google-search-console/source_google_search_console/streams.py

@@ -374,25 +374,36 @@ class SearchByKeyword(SearchAnalytics):
     """
     Adds searchAppearance value to dimensionFilterGroups in json body
     https://developers.google.com/webmaster-tools/v1/how-tos/all-your-data#search-appearance-data
+
+    groupType: "and" - Whether all filters in this group must return true ("and"), or one or more must return true (not yet supported).
+    filters: {"dimension": "searchAppearance", "operator": "equals", "expression": keyword}
     """
 
+    def stream_slices(
+        self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None
+    ) -> Iterable[Optional[Mapping[str, Any]]]:
+        search_appearance_stream = SearchAppearance(self._session.auth, self._site_urls, self._start_date, self._end_date)
+
+        for stream_slice in super().stream_slices(sync_mode, cursor_field, stream_state):
+            keywords_records = search_appearance_stream.read_records(
+                sync_mode=SyncMode.full_refresh, stream_state=stream_state, stream_slice=stream_slice
+            )
+            keywords = {record["searchAppearance"] for record in keywords_records}
+
+            for keyword in keywords:
+                filters = {"dimension": "searchAppearance", "operator": "equals", "expression": keyword}
+                stream_slice["dimensionFilterGroups"] = [{"groupType": "and", "filters": filters}]
+
+                yield stream_slice
+
     def request_body_json(
         self,
         stream_state: Mapping[str, Any] = None,
         stream_slice: Mapping[str, Any] = None,
         next_page_token: Mapping[str, Any] = None,
     ) -> Optional[Union[Dict[str, Any], str]]:
         data = super().request_body_json(stream_state, stream_slice, next_page_token)
-
-        stream = SearchAppearance(self._session.auth, self._site_urls, self._start_date, self._end_date)
-        keywords_records = stream.read_records(sync_mode=SyncMode.full_refresh, stream_state=stream_state, stream_slice=stream_slice)
-        keywords = {record["searchAppearance"] for record in keywords_records}
-        filters = []
-        for keyword in keywords:
-            filters.append({"dimension": "searchAppearance", "operator": "equals", "expression": keyword})
-
-        data["dimensionFilterGroups"] = [{"filters": filters}]
-
+        data["dimensionFilterGroups"] = stream_slice["dimensionFilterGroups"]
         return data
 
 

airbyte-integrations/connectors/source-google-search-console/unit_tests/unit_test.py

@@ -133,6 +133,7 @@ def test_forbidden_should_retry(requests_mock, forbidden_error_message_json):
 
 def test_bad_aggregation_type_should_retry(requests_mock, bad_aggregation_type):
     stream = SearchAnalyticsKeywordSiteReportBySite(None, ["https://example.com"], "2021-01-01", "2021-01-02")
+    requests_mock.post(f"{stream.url_base}sites/{stream._site_urls[0]}/searchAnalytics/query", status_code=200, json={"rows": [{"keys": ["TPF_QA"]}]})
     slice = list(stream.stream_slices(None))[0]
     url = stream.url_base + stream.path(None, slice)
     requests_mock.get(url, status_code=400, json=bad_aggregation_type)