Airbyte CDK: use pytz.utc instead of datetime.utc (#38026)

Signed-off-by: Artem Inzhyyants <[email protected]>

Author: artem1205

airbyte-cdk/python/airbyte_cdk/sources/declarative/interpolation/macros.py

@@ -4,9 +4,11 @@
 
 import builtins
 import datetime
-import numbers
+import typing
 from typing import Union
 
+import isodate
+import pytz
 from dateutil import parser
 from isodate import parse_duration
 
@@ -15,7 +17,7 @@
 """
 
 
-def now_utc():
+def now_utc() -> datetime.datetime:
     """
     Current local date and time in UTC timezone
 
@@ -25,7 +27,7 @@ def now_utc():
     return datetime.datetime.now(datetime.timezone.utc)
 
 
-def today_utc():
+def today_utc() -> datetime.date:
     """
     Current date in UTC timezone
 
@@ -35,7 +37,7 @@ def today_utc():
     return datetime.datetime.now(datetime.timezone.utc).date()
 
 
-def timestamp(dt: Union[numbers.Number, str]):
+def timestamp(dt: Union[float, str]) -> Union[int, float]:
     """
     Converts a number or a string to a timestamp
 
@@ -48,21 +50,21 @@ def timestamp(dt: Union[numbers.Number, str]):
     :param dt: datetime to convert to timestamp
     :return: unix timestamp
     """
-    if isinstance(dt, numbers.Number):
+    if isinstance(dt, (int, float)):
         return int(dt)
     else:
-        return _str_to_datetime(dt).astimezone(datetime.timezone.utc).timestamp()
+        return _str_to_datetime(dt).astimezone(pytz.utc).timestamp()
 
 
 def _str_to_datetime(s: str) -> datetime.datetime:
     parsed_date = parser.isoparse(s)
     if not parsed_date.tzinfo:
         # Assume UTC if the input does not contain a timezone
-        parsed_date = parsed_date.replace(tzinfo=datetime.timezone.utc)
-    return parsed_date.astimezone(datetime.timezone.utc)
+        parsed_date = parsed_date.replace(tzinfo=pytz.utc)
+    return parsed_date.astimezone(pytz.utc)
 
 
-def max(*args):
+def max(*args: typing.Any) -> typing.Any:
     """
     Returns biggest object of an iterable, or two or more arguments.
 
@@ -95,7 +97,7 @@ def day_delta(num_days: int, format: str = "%Y-%m-%dT%H:%M:%S.%f%z") -> str:
     return (datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=num_days)).strftime(format)
 
 
-def duration(datestring: str) -> datetime.timedelta:
+def duration(datestring: str) -> Union[datetime.timedelta, isodate.Duration]:
     """
     Converts ISO8601 duration to datetime.timedelta
 
@@ -111,10 +113,17 @@ def format_datetime(dt: Union[str, datetime.datetime], format: str) -> str:
 
     Usage:
     `"{{ format_datetime(config.start_date, '%Y-%m-%d') }}"`
+
+    CPython Datetime package has known bug with `stfrtime` method: '%s' formatting uses locale timezone
+    https://github.com/python/cpython/issues/77169
+    https://github.com/python/cpython/issues/56959
     """
     if isinstance(dt, datetime.datetime):
         return dt.strftime(format)
-    return _str_to_datetime(dt).strftime(format)
+    dt_datetime = _str_to_datetime(dt)
+    if format == "%s":
+        return str(int(dt_datetime.timestamp()))
+    return dt_datetime.strftime(format)
 
 
 _macros_list = [now_utc, today_utc, timestamp, max, day_delta, duration, format_datetime]

airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_macros.py

@@ -71,3 +71,10 @@ def test_timestamp(test_name, input_value, expected_output):
     timestamp_function = macros["timestamp"]
     actual_output = timestamp_function(input_value)
     assert actual_output == expected_output
+
+
+def test_utc_datetime_to_local_timestamp_conversion():
+    """
+    This test ensures correct timezone handling independent of the timezone of the system on which the sync is running.
+    """
+    assert macros["format_datetime"](dt="2020-10-01T00:00:00Z", format="%s") == "1601510400"