🎉Source Facebook Marketing: Updated date validation process (#15327)

* Updated date validation process

* Bumped docker version

* auto-bump connector version [ci skip]

Co-authored-by: Octavia Squidington III <[email protected]>

Author: lazebnyi

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -248,7 +248,7 @@
 - name: Facebook Marketing
   sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
   dockerRepository: airbyte/source-facebook-marketing
-  dockerImageTag: 0.2.58
+  dockerImageTag: 0.2.59
   documentationUrl: https://docs.airbyte.io/integrations/sources/facebook-marketing
   icon: facebook.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -1821,7 +1821,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-facebook-marketing:0.2.58"
+- dockerImage: "airbyte/source-facebook-marketing:0.2.59"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"
     changelogUrl: "https://docs.airbyte.io/integrations/sources/facebook-marketing"

airbyte-integrations/connectors/source-facebook-marketing/Dockerfile

@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=0.2.58
+LABEL io.airbyte.version=0.2.59
 LABEL io.airbyte.name=airbyte/source-facebook-marketing

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/source.py

@@ -30,6 +30,8 @@
     Videos,
 )
 
+from .utils import validate_end_date, validate_start_date
+
 logger = logging.getLogger("airbyte")
 
 
@@ -58,6 +60,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Type[Stream]]:
         :return: list of the stream instances
         """
         config: ConnectorConfig = ConnectorConfig.parse_obj(config)
+
+        config.start_date = validate_start_date(config.start_date)
+        config.end_date = validate_end_date(config.start_date, config.end_date)
+
         api = API(account_id=config.account_id, access_token=config.access_token)
 
         insights_args = dict(

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/spec.py

@@ -10,9 +10,7 @@
 import pendulum
 from airbyte_cdk.sources.config import BaseConfig
 from facebook_business.adobjects.adsinsights import AdsInsights
-from pydantic import BaseModel, Field, PositiveInt, validator
-
-from .utils import validate_date_field
+from pydantic import BaseModel, Field, PositiveInt
 
 logger = logging.getLogger("airbyte")
 
@@ -87,14 +85,6 @@ class Config:
         default=28,
     )
 
-    @validator("start_date")
-    def set_start_date(cls, start_date):
-        return validate_date_field("Start date", start_date)
-
-    @validator("end_date")
-    def set_end_date(cls, end_date):
-        return validate_date_field("End date", end_date)
-
 
 class ConnectorConfig(BaseConfig):
     """Connector config"""
@@ -189,11 +179,3 @@ class Config:
         description="Maximum batch size used when sending batch requests to Facebook API. Most users do not need to set this field unless they specifically need to tune the connector to address specific issues or use cases.",
         default=50,
     )
-
-    @validator("start_date")
-    def set_start_date(cls, start_date):
-        return validate_date_field("Start date", start_date)
-
-    @validator("end_date")
-    def set_end_date(cls, end_date):
-        return validate_date_field("End date", end_date)

airbyte-integrations/connectors/source-facebook-marketing/source_facebook_marketing/utils.py

@@ -2,35 +2,42 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
+import logging
 from datetime import datetime
 
 import pendulum
 
+logger = logging.getLogger("airbyte")
+
 # Facebook store metrics maximum of 37 months old. Any time range that
 # older that 37 months from current date would result in 400 Bad request
 # HTTP response.
 # https://developers.facebook.com/docs/marketing-api/reference/ad-account/insights/#overview
-DATA_RETENTION_PERIOD = pendulum.duration(months=37)
-
-
-class ValidationDateException(Exception):
-    def __init__(self, message, *args, **kwargs):
-        self.message = message
-        super().__init__(self.message, *args, **kwargs)
-
-    def __str__(self):
-        return self.message
-
-    def __repr__(self):
-        return self.__str__()
+DATA_RETENTION_PERIOD = 37
 
 
-def validate_date_field(field_name: str, date: datetime) -> datetime:
-    pendulum_date = pendulum.instance(date)
+def validate_start_date(start_date: datetime) -> datetime:
+    pendulum_date = pendulum.instance(start_date)
+    time_zone = start_date.tzinfo
+    current_date = pendulum.today(time_zone)
     if pendulum_date.timestamp() > pendulum.now().timestamp():
-        message = f"{field_name} cannot be in the future. Please set today's date or later."
-        raise ValidationDateException(message)
-    elif pendulum_date.timestamp() < (pendulum.now() - DATA_RETENTION_PERIOD).timestamp():
-        message = f"{field_name} cannot be beyond {DATA_RETENTION_PERIOD.months} months from the current date."
-        raise ValidationDateException(message)
-    return date
+        message = f"The start date cannot be in the future. Set start date to today's date - {current_date}."
+        logger.warning(message)
+        return current_date
+    elif pendulum_date.timestamp() < current_date.subtract(months=DATA_RETENTION_PERIOD).timestamp():
+        current_date = pendulum.today(time_zone)
+        message = (
+            f"The start date cannot be beyond {DATA_RETENTION_PERIOD} months from the current date. "
+            f"Set start date to {current_date.subtract(months=DATA_RETENTION_PERIOD)}."
+        )
+        logger.warning(message)
+        return current_date.subtract(months=DATA_RETENTION_PERIOD)
+    return start_date
+
+
+def validate_end_date(start_date: datetime, end_date: datetime) -> datetime:
+    if start_date > end_date:
+        message = f"The end date must be after start date. Set end date to {start_date}."
+        logger.warning(message)
+        return start_date
+    return end_date

airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_utils.py

@@ -4,28 +4,44 @@
 
 import pendulum
 import pytest
-from source_facebook_marketing.utils import DATA_RETENTION_PERIOD, ValidationDateException, validate_date_field
+from source_facebook_marketing.utils import DATA_RETENTION_PERIOD, validate_end_date, validate_start_date
 
 
 @pytest.mark.parametrize(
-    "date, expected_message, raise_error",
+    "field_name, date, expected_date, expected_messages",
     [
-        (pendulum.now(), "", False),
         (
-            pendulum.now() - pendulum.duration(months=DATA_RETENTION_PERIOD.months + 1),
-            f" cannot be beyond {DATA_RETENTION_PERIOD.months} months from the current date.",
-            True,
+            "start_date",
+            pendulum.today().subtract(months=DATA_RETENTION_PERIOD - 1),
+            pendulum.today().subtract(months=DATA_RETENTION_PERIOD - 1),
+            [],
+        ),
+        (
+            "start_date",
+            pendulum.today().subtract(months=DATA_RETENTION_PERIOD + 1),
+            pendulum.today().subtract(months=DATA_RETENTION_PERIOD),
+            [
+                f"The start date cannot be beyond 37 months from the current date. "
+                f"Set start date to {pendulum.today().subtract(months=DATA_RETENTION_PERIOD)}."
+            ],
+        ),
+        (
+            "start_date",
+            pendulum.today() + pendulum.duration(months=1),
+            pendulum.today(),
+            [f"The start date cannot be in the future. Set start date to today's date - {pendulum.today()}."],
+        ),
+        (
+            "end_date",
+            pendulum.today().subtract(months=DATA_RETENTION_PERIOD),
+            pendulum.today(),
+            [f"The end date must be after start date. Set end date to {pendulum.today()}."],
         ),
-        (pendulum.now() + pendulum.duration(months=1), " cannot be in the future. Please set today's date or later.", True),
     ],
-    ids=["valid_date", f"date in the past by {DATA_RETENTION_PERIOD.months} months", "date in future"],
 )
-def test_validate_date_field(date, expected_message, raise_error):
-    field_name = "test_field_name"
-
-    if raise_error:
-        with pytest.raises(ValidationDateException) as error:
-            assert validate_date_field(field_name, date)
-        assert str(error.value) == field_name + expected_message
-    else:
-        assert validate_date_field(field_name, date)
+def test_date_validators(caplog, field_name, date, expected_date, expected_messages):
+    if field_name == "start_date":
+        assert validate_start_date(date) == expected_date
+    elif field_name == "end_date":
+        assert validate_end_date(expected_date, date) == expected_date
+    assert caplog.messages == expected_messages

docs/integrations/sources/facebook-marketing.md

@@ -120,6 +120,7 @@ Please be informed that the connector uses the `lookback_window` parameter to pe
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                                                                                                                                                                           |
 |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.2.59  | 2022-08-04 | [15327](https://github.com/airbytehq/airbyte/pull/15327) | Shift date validation from config validation to stream method                                                                                                                                                                                                                                   |     
 | 0.2.58  | 2022-07-25 | [15012](https://github.com/airbytehq/airbyte/pull/15012) | Add `DATA_RETENTION_PERIOD`validation and fix `failed_delivery_checks` field schema type issue                                                                                                                                                                                                    |     
 | 0.2.57  | 2022-07-25 | [14831](https://github.com/airbytehq/airbyte/pull/14831) | Update Facebook SDK to version 14.0.0                                                                                                                                                                                                                                                             | 
 | 0.2.56  | 2022-07-19 | [14831](https://github.com/airbytehq/airbyte/pull/14831) | Add future `start_date` and `end_date` validation                                                                                                                                                                                                                                                 |