Source GoogleAds: add end_date to config (#8669)

* GoogleAds add end_date to config

* Update script following review comments

* Add unit test

* Solve conflicts

* Solve conflicts in MR

* Update test_google_ads.py

Instanciate IncrementalGoogleAdsStream in tests + add missing line between functions

* Update test_source.py

remove extra hashtag

* Update tests with missing params

* Add missing time_zone param

* merge user code

* run format

* revert unit test stream count

* remove error log file

* bump connector version

* run seed file

Co-authored-by: Marcos Marx <[email protected]>

Author: iberchid

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/253487c0-2246-43ba-a21f-5116b20a2c50.json

@@ -2,7 +2,7 @@
   "sourceDefinitionId": "253487c0-2246-43ba-a21f-5116b20a2c50",
   "name": "Google Ads",
   "dockerRepository": "airbyte/source-google-ads",
-  "dockerImageTag": "0.1.21",
+  "dockerImageTag": "0.1.23",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/google-ads",
   "icon": "google-adwords.svg"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -245,7 +245,7 @@
 - name: Google Ads
   sourceDefinitionId: 253487c0-2246-43ba-a21f-5116b20a2c50
   dockerRepository: airbyte/source-google-ads
-  dockerImageTag: 0.1.22
+  dockerImageTag: 0.1.23
   documentationUrl: https://docs.airbyte.io/integrations/sources/google-ads
   icon: google-adwords.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -2308,7 +2308,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-google-ads:0.1.22"
+- dockerImage: "airbyte/source-google-ads:0.1.23"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/google-ads"
     connectionSpecification:
@@ -2383,6 +2383,15 @@
           examples:
           - "2017-01-25"
           order: 2
+        end_date:
+          type: "string"
+          title: "End Date"
+          description: "UTC date and time in the format 2017-01-25. Any data after\
+            \ this date will not be replicated."
+          pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
+          examples:
+          - "2017-01-30"
+          order: 6
         custom_queries:
           type: "array"
           title: "Custom GAQL Queries (Optional)"

airbyte-integrations/connectors/source-google-ads/Dockerfile

@@ -13,5 +13,5 @@ RUN pip install .
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.22
+LABEL io.airbyte.version=0.1.23
 LABEL io.airbyte.name=airbyte/source-google-ads

airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py

@@ -90,8 +90,13 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
         google_api = GoogleAds(credentials=self.get_credentials(config), customer_id=config["customer_id"])
         account_info = self.get_account_info(google_api)
         time_zone = self.get_time_zone(account_info)
+        end_date = config.get("end_date")
         incremental_stream_config = dict(
-            api=google_api, conversion_window_days=config["conversion_window_days"], start_date=config["start_date"], time_zone=time_zone
+            api=google_api,
+            conversion_window_days=config["conversion_window_days"],
+            start_date=config["start_date"],
+            time_zone=time_zone,
+            end_date=end_date,
         )
 
         streams = [

airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json

@@ -63,6 +63,14 @@
         "examples": ["2017-01-25"],
         "order": 2
       },
+      "end_date": {
+        "type": "string",
+        "title": "End Date",
+        "description": "UTC date and time in the format 2017-01-25. Any data after this date will not be replicated.",
+        "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$",
+        "examples": ["2017-01-30"],
+        "order": 6
+      },
       "custom_queries": {
         "type": "array",
         "title": "Custom GAQL Queries (Optional)",

airbyte-integrations/connectors/source-google-ads/source_google_ads/streams.py

@@ -71,18 +71,21 @@ class IncrementalGoogleAdsStream(GoogleAdsStream, ABC):
     primary_key = None
     range_days = 15  # date range is set to 15 days, because for conversion_window_days default value is 14. Range less than 15 days will break the integration tests.
 
-    def __init__(self, start_date: str, conversion_window_days: int, time_zone: [pendulum.timezone, str], **kwargs):
+    def __init__(self, start_date: str, conversion_window_days: int, time_zone: [pendulum.timezone, str], end_date: str = None, **kwargs):
         self.conversion_window_days = conversion_window_days
         self._start_date = start_date
         self.time_zone = time_zone
+        self._end_date = end_date
         super().__init__(**kwargs)
 
     def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]:
         stream_state = stream_state or {}
         start_date = stream_state.get(self.cursor_field) or self._start_date
+        end_date = self._end_date
 
         return chunk_date_range(
             start_date=start_date,
+            end_date=end_date,
             conversion_window=self.conversion_window_days,
             field=self.cursor_field,
             days_of_data_storage=self.days_of_data_storage,

airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py

@@ -177,3 +177,65 @@ def test_parse_single_result():
     date = "2001-01-01"
     response = GoogleAds.parse_single_result(SAMPLE_SCHEMA, MockedDateSegment(date))
     assert response == response
+
+
+# Add a sample config with date parameters
+SAMPLE_CONFIG_WITH_DATE = {
+    "credentials": {
+        "developer_token": "developer_token",
+        "client_id": "client_id",
+        "client_secret": "client_secret",
+        "refresh_token": "refresh_token",
+    },
+    "customer_id": "customer_id",
+    "start_date": "2021-11-01",
+    "end_date": "2021-11-15",
+}
+
+
+def test_get_date_params_with_date():
+    # Please note that this is equal to inputted stream_slice start date + 1 day
+    mock_start_date = SAMPLE_CONFIG_WITH_DATE["start_date"]
+    mock_end_date = SAMPLE_CONFIG_WITH_DATE["end_date"]
+    incremental_stream_config = dict(
+        start_date=mock_start_date,
+        end_date=mock_end_date,
+        conversion_window_days=0,
+        time_zone="local",
+        api=MockGoogleAdsClient(SAMPLE_CONFIG_WITH_DATE),
+    )
+    start_date, end_date = IncrementalGoogleAdsStream(**incremental_stream_config).get_date_params(
+        stream_slice={"segments.date": "2021-10-31"}, cursor_field="segments.date", end_date=pendulum.parse("2021-11-15")
+    )
+    assert mock_start_date == start_date and mock_end_date == end_date
+
+
+SAMPLE_CONFIG_WITHOUT_END_DATE = {
+    "credentials": {
+        "developer_token": "developer_token",
+        "client_id": "client_id",
+        "client_secret": "client_secret",
+        "refresh_token": "refresh_token",
+    },
+    "customer_id": "customer_id",
+    "start_date": "2021-11-01",
+}
+
+
+def test_get_date_params_without_end_date():
+    # Please note that this is equal to inputted stream_slice start date + 1 day
+    mock_start_date = SAMPLE_CONFIG_WITHOUT_END_DATE["start_date"]
+    mock_end_date = "2021-11-30"
+    incremental_stream_config = dict(
+        start_date=mock_start_date,
+        end_date=mock_end_date,
+        conversion_window_days=0,
+        time_zone="local",
+        api=MockGoogleAdsClient(SAMPLE_CONFIG_WITHOUT_END_DATE),
+    )
+    start_date, end_date = IncrementalGoogleAdsStream(**incremental_stream_config).get_date_params(
+        stream_slice={"segments.date": "2021-10-31"}, cursor_field="segments.date"
+    )
+    assert mock_start_date == start_date
+    # There is a Google limitation where we capture only a 15-day date range
+    assert end_date == "2021-11-15"

airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py

@@ -2,13 +2,30 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
+import pendulum
 import pytest
 from source_google_ads.custom_query_stream import CustomQuery
 from source_google_ads.google_ads import GoogleAds
 from source_google_ads.source import SourceGoogleAds
 from source_google_ads.streams import AdGroupAdReport, chunk_date_range
 
 
+# Test chunck date range without end date
+def test_chunk_date_range_without_end_date():
+    start_date_str = pendulum.now().subtract(days=5).to_date_string()
+    conversion_window = 0
+    field = "date"
+    response = chunk_date_range(
+        start_date=start_date_str, conversion_window=conversion_window, field=field, end_date=None, days_of_data_storage=None, range_days=1
+    )
+    start_date = pendulum.parse(start_date_str)
+    expected_response = []
+    while start_date < pendulum.now():
+        expected_response.append({field: start_date.to_date_string()})
+        start_date = start_date.add(days=1)
+    assert expected_response == response
+
+
 def test_chunk_date_range():
     start_date = "2021-03-04"
     end_date = "2021-05-04"
@@ -102,6 +119,24 @@ def get_instance_from_config(config, query):
     return instance
 
 
+# get he instance with a config
+def get_instance_from_config_with_end_date(config, query):
+    start_date = "2021-03-04"
+    end_date = "2021-04-04"
+    conversion_window_days = 14
+    google_api = GoogleAds(credentials=config["credentials"], customer_id=config["customer_id"])
+
+    instance = CustomQuery(
+        api=google_api,
+        conversion_window_days=conversion_window_days,
+        start_date=start_date,
+        end_date=end_date,
+        time_zone="local",
+        custom_query_config={"query": query, "table_name": "whatever_table"},
+    )
+    return instance
+
+
 @pytest.mark.parametrize(
     "query, fields",
     [
@@ -244,6 +279,30 @@ def test_get_json_schema_parse_query(config):
     assert set(schema_keys) == set(final_fields)  # test 1
 
 
+# Test get json schema when start and end date are provided in the config file
+def test_get_json_schema_parse_query_with_end_date(config):
+    query = """
+        SELECT
+            campaign.accessible_bidding_strategy,
+            segments.ad_destination_type,
+            campaign.start_date,
+            campaign.end_date
+        FROM campaign
+        """
+    final_fields = [
+        "campaign.accessible_bidding_strategy",
+        "segments.ad_destination_type",
+        "campaign.start_date",
+        "campaign.end_date",
+        "segments.date",
+    ]
+
+    instance = get_instance_from_config_with_end_date(config=config, query=query)
+    final_schema = instance.get_json_schema()
+    schema_keys = final_schema["properties"]
+    assert set(schema_keys) == set(final_fields)  # test 1
+
+
 def test_google_type_conversion(config):
     """
     query may be invalid (fields incompatibility did not checked).

docs/integrations/sources/google-ads.md

@@ -102,6 +102,7 @@ This source is constrained by whatever API limits are set for the Google Ads tha
 
 | Version | Date | Pull Request | Subject |
 | :--- | :--- | :--- | :--- |
+| `0.1.23` | 2022-01-25 | [8669](https://github.com/airbytehq/airbyte/pull/8669) | Add end date parameter in spec. |
 | `0.1.22` | 2022-01-24 | [9608](https://github.com/airbytehq/airbyte/pull/9608) | Reduce stream slice date range. |
 | `0.1.21` | 2021-12-28 | [9149](https://github.com/airbytehq/airbyte/pull/9149) | Update title and description |
 | `0.1.20` | 2021-12-22 | [9071](https://github.com/airbytehq/airbyte/pull/9071) | Fix: Keyword schema enum |