End date configuration for tiktok (#12838)

* added end date configuration.

* restore and added stream_test in unit test

* updated spec.json

* updated with master

* spec.json updated

* updated streams_test.py

* updated spec.json with new end date

* updated in unit test file

* stream test updated

* fix: formatting and spec

* chore: updated the source-pipedrive version

* fix: formatting and spec

* fix: formatting and spec

* fix: formatting and spec

Co-authored-by: Harshith Mullapudi <[email protected]>

Author: Zawar92

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -914,7 +914,7 @@
 - name: TikTok Marketing
   sourceDefinitionId: 4bfac00d-ce15-44ff-95b9-9e3c3e8fbd35
   dockerRepository: airbyte/source-tiktok-marketing
-  dockerImageTag: 0.1.10
+  dockerImageTag: 0.1.11
   documentationUrl: https://docs.airbyte.io/integrations/sources/tiktok-marketing
   icon: tiktok.svg
   sourceType: api

airbyte-integrations/connectors/source-tiktok-marketing/Dockerfile

@@ -32,5 +32,5 @@ COPY source_tiktok_marketing ./source_tiktok_marketing
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.10
+LABEL io.airbyte.version=0.1.11
 LABEL io.airbyte.name=airbyte/source-tiktok-marketing

airbyte-integrations/connectors/source-tiktok-marketing/integration_tests/spec.json

@@ -84,16 +84,28 @@
         "enum": ["LIFETIME", "DAY", "HOUR"],
         "order": 2,
         "type": "string"
+      },
+      "end_date": {
+        "title": "End Date",
+        "description": "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DD. All data generated between start_date and this date will be replicated. Not setting this option will result in always syncing the data till the current date.",
+        "default": "2022-05-23",
+        "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$",
+        "order": 3,
+        "type": "string"
       }
     }
   },
   "supportsIncremental": true,
+  "supportsNormalization": false,
+  "supportsDBT": false,
   "supported_destination_sync_modes": ["overwrite", "append", "append_dedup"],
+  "authSpecification": null,
   "advanced_auth": {
     "auth_flow_type": "oauth2.0",
     "predicate_key": ["credentials", "auth_type"],
     "predicate_value": "oauth2.0",
     "oauth_config_specification": {
+      "oauth_user_input_from_connector_config_specification": null,
       "complete_oauth_output_specification": {
         "title": "CompleteOauthOutputSpecification",
         "type": "object",
@@ -110,14 +122,8 @@
         "title": "CompleteOauthServerInputSpecification",
         "type": "object",
         "properties": {
-          "app_id": {
-            "title": "App Id",
-            "type": "string"
-          },
-          "secret": {
-            "title": "Secret",
-            "type": "string"
-          }
+          "app_id": { "title": "App Id", "type": "string" },
+          "secret": { "title": "Secret", "type": "string" }
         },
         "required": ["app_id", "secret"]
       },

airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/source.py

@@ -18,6 +18,7 @@
     SourceTiktokMarketingSpec,
 )
 from .streams import (
+    DEFAULT_END_DATE,
     DEFAULT_START_DATE,
     AdGroupAudienceReports,
     AdGroups,
@@ -93,6 +94,7 @@ def _prepare_stream_args(config: Mapping[str, Any]) -> Mapping[str, Any]:
         return {
             "authenticator": TiktokTokenAuthenticator(access_token),
             "start_date": config.get("start_date") or DEFAULT_START_DATE,
+            "end_date": config.get("end_date") or DEFAULT_END_DATE,
             "advertiser_id": advertiser_id,
             "app_id": app_id,
             "secret": secret,

airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/spec.py

@@ -10,7 +10,7 @@
 from jsonschema import RefResolver
 from pydantic import BaseModel, Field
 
-from .streams import DEFAULT_START_DATE, ReportGranularity
+from .streams import DEFAULT_END_DATE, DEFAULT_START_DATE, ReportGranularity
 
 
 class OauthCredSpec(BaseModel):
@@ -62,6 +62,18 @@ class Config:
         order=2,
     )
 
+    end_date: str = Field(
+        title="End Date",
+        default=DEFAULT_END_DATE,
+        pattern="^[0-9]{4}-[0-9]{2}-[0-9]{2}$",
+        description=(
+            "The date until which you'd like to replicate data for all incremental streams, in the format YYYY-MM-DD. "
+            "All data generated between start_date and this date will be replicated. "
+            "Not setting this option will result in always syncing the data till the current date."
+        ),
+        order=3,
+    )
+
     @classmethod
     def change_format_to_oneOf(cls, schema: dict) -> dict:
         new_schema = {}

airbyte-integrations/connectors/source-tiktok-marketing/source_tiktok_marketing/streams.py

@@ -22,6 +22,7 @@
 
 # TikTok Initial release date is September 2016
 DEFAULT_START_DATE = "2016-09-01"
+DEFAULT_END_DATE = str(datetime.now().date())
 NOT_AUDIENCE_METRICS = [
     "reach",
     "cost_per_1000_reached",
@@ -226,12 +227,15 @@ def transform_function(original_value: Any, field_schema: Dict[str, Any]) -> Any
             return Decimal(original_value)
         return original_value
 
-    def __init__(self, start_date: str, **kwargs):
+    def __init__(self, start_date: str, end_date: str, **kwargs):
         super().__init__(**kwargs)
         self.kwargs = kwargs
         # convert a start date to TikTok format
         # example:  "2021-08-24" => "2021-08-24 00:00:00"
         self._start_time = pendulum.parse(start_date or DEFAULT_START_DATE).strftime("%Y-%m-%d 00:00:00")
+        # convert end date to TikTok format
+        # example:  "2021-08-24" => "2021-08-24 00:00:00"
+        self._end_time = pendulum.parse(end_date or DEFAULT_END_DATE).strftime("%Y-%m-%d 00:00:00")
         self.max_cursor_date = None
         self._advertiser_ids = []
 
@@ -422,7 +426,9 @@ def cursor_field(self):
         return []
 
     @staticmethod
-    def _get_time_interval(start_date: Union[datetime, str], granularity: ReportGranularity) -> Iterable[Tuple[datetime, datetime]]:
+    def _get_time_interval(
+        start_date: Union[datetime, str], ending_date: Union[datetime, str], granularity: ReportGranularity
+    ) -> Iterable[Tuple[datetime, datetime]]:
         """Due to time range restrictions based on the level of granularity of reports, we have to chunk API calls in order
         to get the desired time range.
         Docs: https://ads.tiktok.com/marketing_api/docs?id=1714590313280513
@@ -432,7 +438,7 @@ def _get_time_interval(start_date: Union[datetime, str], granularity: ReportGran
         """
         if isinstance(start_date, str):
             start_date = pendulum.parse(start_date)
-        end_date = pendulum.now()
+        end_date = pendulum.parse(ending_date) if ending_date else pendulum.now()
 
         # Snapchat API only allows certain amount of days of data based on the reporting granularity
         if granularity == ReportGranularity.DAY:
@@ -530,9 +536,10 @@ def _get_metrics(self):
 
     def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]:
         stream_start = self.select_cursor_field_value(stream_state) or self._start_time
+        stream_end = self._end_time
 
         for slice_adv_id in super().stream_slices(**kwargs):
-            for start_date, end_date in self._get_time_interval(stream_start, self.report_granularity):
+            for start_date, end_date in self._get_time_interval(stream_start, stream_end, self.report_granularity):
                 slice = {
                     "advertiser_id": slice_adv_id["advertiser_id"],
                     "start_date": start_date.strftime("%Y-%m-%d"),

airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/streams_test.py

@@ -26,6 +26,7 @@
     "secret": "secret",
     "authenticator": None,
     "start_date": START_DATE,
+    "end_date": END_DATE,
     "app_id": 1234,
     "advertiser_id": 0,
 }
@@ -34,6 +35,7 @@
     "secret": "secret",
     "authenticator": None,
     "start_date": START_DATE,
+    "end_date": END_DATE,
     "app_id": 1234,
     "advertiser_id": 2000,
 }
@@ -62,13 +64,13 @@ def advertiser_ids_fixture():
     ],
 )
 def test_get_time_interval(pendulum_now_mock, granularity, intervals_len):
-    intervals = BasicReports._get_time_interval(start_date="2020-01-01", granularity=granularity)
+    intervals = BasicReports._get_time_interval(start_date="2020-01-01", ending_date="2020-03-01", granularity=granularity)
     assert len(list(intervals)) == intervals_len
 
 
 @patch.object(pendulum, "now", return_value=pendulum.parse("2018-12-25"))
 def test_get_time_interval_past(pendulum_now_mock_past):
-    intervals = BasicReports._get_time_interval(start_date="2020-01-01", granularity=ReportGranularity.DAY)
+    intervals = BasicReports._get_time_interval(start_date="2020-01-01", ending_date="2020-01-01", granularity=ReportGranularity.DAY)
     assert len(list(intervals)) == 1
 
 

airbyte-integrations/connectors/source-tiktok-marketing/unit_tests/unit_test.py

@@ -112,6 +112,7 @@ def test_random_items(prepared_prod_args):
                 )
                 if not max_updated_value or max_updated_value < ad_items[-1][stream.cursor_field]:
                     max_updated_value = ad_items[-1][stream.cursor_field]
+
             # mock for ads
             for page, page_response in generate_pages(items=ad_items, page_size=page_size, last_empty=True):
                 uri = f"/open_api/v1.2/ad/get/?page_size={page_size}&advertiser_id={advertiser_id}"

docs/integrations/sources/tiktok-marketing.md

@@ -524,6 +524,7 @@ The connector is restricted by [requests limitation](https://ads.tiktok.com/mark
 
 | Version | Date       | Pull Request                                             | Subject                                                                                       |
 |:--------|:-----------|:---------------------------------------------------------|:----------------------------------------------------------------------------------------------|
+| 0.1.11  | 2022-04-27 | [12838](https://github.com/airbytehq/airbyte/pull/12838) | Added end date configuration for tiktok                                                      |
 | 0.1.10  | 2022-05-07 | [12545](https://github.com/airbytehq/airbyte/pull/12545) | Removed odd production authenication method                                                   |
 | 0.1.9   | 2022-04-30 | [12500](https://github.com/airbytehq/airbyte/pull/12500) | Improve input configuration copy                                                              |
 | 0.1.8   | 2022-04-28 | [12435](https://github.com/airbytehq/airbyte/pull/12435) | updated spec descriptions                                                                     |