source-twilio: implement lookback windows (#13896)

Author: alafanechere

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -953,7 +953,7 @@
 - name: Twilio
   sourceDefinitionId: b9dc6155-672e-42ea-b10d-9f1f1fb95ab1
   dockerRepository: airbyte/source-twilio
-  dockerImageTag: 0.1.4
+  dockerImageTag: 0.1.5
   documentationUrl: https://docs.airbyte.io/integrations/sources/twilio
   icon: twilio.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -9248,7 +9248,7 @@
         oauthFlowOutputParameters:
         - - "token"
         - - "key"
-- dockerImage: "airbyte/source-twilio:0.1.4"
+- dockerImage: "airbyte/source-twilio:0.1.5"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/twilio"
     connectionSpecification:
@@ -9266,11 +9266,13 @@
           description: "Twilio account SID"
           airbyte_secret: true
           type: "string"
+          order: 1
         auth_token:
           title: "Auth Token"
           description: "Twilio Auth Token."
           airbyte_secret: true
           type: "string"
+          order: 2
         start_date:
           title: "Replication Start Date"
           description: "UTC date and time in the format 2020-10-01T00:00:00Z. Any\
@@ -9279,6 +9281,15 @@
           examples:
           - "2020-10-01T00:00:00Z"
           type: "string"
+          order: 3
+        lookback_window:
+          title: "Lookback window"
+          description: "How far into the past to look for records. (in minutes)"
+          examples:
+          - 60
+          default: 0
+          type: "integer"
+          order: 4
     supportsIncremental: true
     supportsNormalization: false
     supportsDBT: false

airbyte-integrations/connectors/source-twilio/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.4
+LABEL io.airbyte.version=0.1.5
 LABEL io.airbyte.name=airbyte/source-twilio

airbyte-integrations/connectors/source-twilio/acceptance-test-config.yml

@@ -18,13 +18,11 @@ tests:
       # usage records stream produces and error if cursor date gte than current date
       configured_catalog_path: "integration_tests/no_empty_streams_no_usage_records_catalog.json"
       future_state_path: "integration_tests/abnormal_state.json"
-      cursor_paths:
-        calls: ["end_time"]
-        conferences: ["date_updated"]
-        recordings: ["date_created"]
-        messages: ["date_sent"]
-        message_media: ["date_created"]
-        alerts: ["date_updated"]
+    - config_path: "secrets/config_with_lookback.json"
+      # usage records stream produces and error if cursor date gte than current date
+      configured_catalog_path: "integration_tests/no_empty_streams_no_usage_records_catalog.json"
+      future_state_path: "integration_tests/abnormal_state.json"
+      threshold_days: 30
   full_refresh:
     - config_path: "secrets/config.json"
       # `constant_records_catalog.json` does not contain the available phone numbers streams,

airbyte-integrations/connectors/source-twilio/acceptance-test-docker.sh

@@ -1,20 +1,20 @@
 {
   "calls": {
-    "end_time": "2220-10-01T00:00:00Z"
+    "end_time": "2030-10-01T00:00:00Z"
   },
   "conferences": {
-    "date_updated": "2220-10-01T00:00:00Z"
+    "date_created": "2030-10-01T00:00:00Z"
   },
   "recordings": {
-    "date_created": "2220-10-01T00:00:00Z"
+    "date_created": "2030-10-01T00:00:00Z"
   },
   "messages": {
-    "date_sent": "2220-10-01T00:00:00Z"
+    "date_sent": "2030-10-01T00:00:00Z"
   },
   "message_media": {
-    "date_created": "2220-10-01T00:00:00Z"
+    "date_created": "2030-10-01T00:00:00Z"
   },
   "alerts": {
-    "date_updated": "2220-10-01T00:00:00Z"
+    "date_generated": "2030-10-01T00:00:00Z"
   }
 }

airbyte-integrations/connectors/source-twilio/integration_tests/abnormal_state.json

@@ -108,9 +108,9 @@
         "json_schema": {},
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["date_updated"]
+        "default_cursor_field": ["date_created"]
       },
-      "cursor_field": ["date_updated"],
+      "cursor_field": ["date_created"],
       "sync_mode": "incremental",
       "destination_sync_mode": "append"
     },
@@ -213,9 +213,9 @@
         "json_schema": {},
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["date_updated"]
+        "default_cursor_field": ["date_created"]
       },
-      "cursor_field": ["date_updated"],
+      "cursor_field": ["date_created"],
       "sync_mode": "incremental",
       "destination_sync_mode": "append"
     }

airbyte-integrations/connectors/source-twilio/integration_tests/configured_catalog.json

@@ -63,9 +63,9 @@
         "json_schema": {},
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["date_updated"]
+        "default_cursor_field": ["date_created"]
       },
-      "cursor_field": ["date_updated"],
+      "cursor_field": ["date_created"],
       "sync_mode": "incremental",
       "destination_sync_mode": "append"
     },
@@ -159,9 +159,9 @@
         "json_schema": {},
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["date_updated"]
+        "default_cursor_field": ["date_created"]
       },
-      "cursor_field": ["date_updated"],
+      "cursor_field": ["date_created"],
       "sync_mode": "incremental",
       "destination_sync_mode": "append"
     }

airbyte-integrations/connectors/source-twilio/integration_tests/constant_records_catalog.json

@@ -99,9 +99,9 @@
         "json_schema": {},
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["date_updated"]
+        "default_cursor_field": ["date_created"]
       },
-      "cursor_field": ["date_updated"],
+      "cursor_field": ["date_created"],
       "sync_mode": "incremental",
       "destination_sync_mode": "append"
     },

airbyte-integrations/connectors/source-twilio/integration_tests/no_empty_streams_catalog.json

@@ -99,9 +99,9 @@
         "json_schema": {},
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["date_updated"]
+        "default_cursor_field": ["date_created"]
       },
-      "cursor_field": ["date_updated"],
+      "cursor_field": ["date_created"],
       "sync_mode": "incremental",
       "destination_sync_mode": "append"
     },
@@ -183,9 +183,9 @@
         "json_schema": {},
         "supported_sync_modes": ["incremental", "full_refresh"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["date_updated"]
+        "default_cursor_field": ["date_created"]
       },
-      "cursor_field": ["date_updated"],
+      "cursor_field": ["date_created"],
       "sync_mode": "incremental",
       "destination_sync_mode": "append"
     }

airbyte-integrations/connectors/source-twilio/integration_tests/no_empty_streams_no_usage_records_catalog.json

@@ -1,6 +1,6 @@
 {
   "calls": {
-    "end_time": "2020-01-01T00:00:00Z"
+    "end_time": "2022-06-11T00:00:00Z"
   },
   "conferences": {
     "date_updated": "2020-01-01T00:00:00Z"

airbyte-integrations/connectors/source-twilio/integration_tests/sample_state.json

@@ -8,7 +8,7 @@
       "type": ["null", "string"]
     },
     "date_updated": {
-      "format": "date",
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "api_version": {

airbyte-integrations/connectors/source-twilio/source_twilio/schemas/conferences.json

@@ -48,6 +48,9 @@
     "error_code": {
       "type": ["null", "integer"]
     },
+    "media_url": {
+      "type": ["null", "string"]
+    },
     "uri": {
       "type": ["null", "string"]
     },

airbyte-integrations/connectors/source-twilio/source_twilio/schemas/recordings.json

@@ -62,7 +62,7 @@
       "type": ["null", "number"]
     },
     "start_date": {
-      "format": "date",
+      "format": "date-time",
       "type": ["null", "string"]
     },
     "count_unit": {

airbyte-integrations/connectors/source-twilio/source_twilio/schemas/usage_records.json

@@ -61,7 +61,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             ),
         )
         full_refresh_stream_kwargs = {"authenticator": auth}
-        incremental_stream_kwargs = {"authenticator": auth, "start_date": config["start_date"]}
+        incremental_stream_kwargs = {
+            "authenticator": auth,
+            "start_date": config["start_date"],
+            "lookback_window": config["lookback_window"],
+        }
 
         streams = [
             Accounts(**full_refresh_stream_kwargs),

airbyte-integrations/connectors/source-twilio/source_twilio/source.py

@@ -4,30 +4,51 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "title": "Twilio Spec",
     "type": "object",
-    "required": ["account_sid", "auth_token", "start_date"],
+    "required": [
+      "account_sid",
+      "auth_token",
+      "start_date"
+    ],
     "additionalProperties": false,
     "properties": {
       "account_sid": {
         "title": "Account ID",
         "description": "Twilio account SID",
         "airbyte_secret": true,
-        "type": "string"
+        "type": "string",
+        "order": 1
       },
       "auth_token": {
         "title": "Auth Token",
         "description": "Twilio Auth Token.",
         "airbyte_secret": true,
-        "type": "string"
+        "type": "string",
+        "order": 2
       },
       "start_date": {
         "title": "Replication Start Date",
         "description": "UTC date and time in the format 2020-10-01T00:00:00Z. Any data before this date will not be replicated.",
         "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
-        "examples": ["2020-10-01T00:00:00Z"],
-        "type": "string"
+        "examples": [
+          "2020-10-01T00:00:00Z"
+        ],
+        "type": "string",
+        "order": 3
+      },
+      "lookback_window": {
+        "title": "Lookback window",
+        "description": "How far into the past to look for records. (in minutes)",
+        "examples": [
+          60
+        ],
+        "default": 0,
+        "type": "integer",
+        "order": 4
       }
     }
   },
   "supportsIncremental": true,
-  "supported_destination_sync_modes": ["append"]
+  "supported_destination_sync_modes": [
+    "append"
+  ]
 }