✨ Source Zendesk Support: add tests and clean-up

airbyte-integrations/connectors/source-zendesk-support/acceptance-test-config.yml

@@ -32,6 +32,8 @@ acceptance_tests:
             bypass_reason: "not available in current subscription plan"
           - name: "post_comment_votes"
             bypass_reason: "not available in current subscription plan"
+          - name: "ticket_activities"
+            bypass_reason: "There is a retention period which requires too much maintenance"
           - name: "tags"
             bypass_reason: "API issue" # TODO: remove this after all changes being merged
   incremental:

airbyte-integrations/connectors/source-zendesk-support/integration_tests/expected_records.jsonl

@@ -32,7 +32,6 @@
 {"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/7235633102607.json", "id": 7235633102607, "assignee_id": null, "group_id": null, "requester_id": 361089721035, "ticket_id": 146, "score": "offered", "created_at": "2023-06-19T18:01:40Z", "updated_at": "2023-06-19T18:01:40Z", "comment": null}, "emitted_at": 1720179592962}
 {"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/8178725484175.json", "id": 8178725484175, "assignee_id": null, "group_id": null, "requester_id": 8178212241935, "ticket_id": 158, "score": "offered", "created_at": "2023-10-20T12:01:58Z", "updated_at": "2023-10-20T12:01:58Z", "comment": null}, "emitted_at": 1720179592971}
 {"stream": "satisfaction_ratings", "data": {"url": "https://d3v-airbyte.zendesk.com/api/v2/satisfaction_ratings/9862120719631.json", "id": 9862120719631, "assignee_id": null, "group_id": null, "requester_id": 9861847678735, "ticket_id": 161, "score": "offered", "created_at": "2024-05-28T21:01:33Z", "updated_at": "2024-05-28T21:01:33Z", "comment": null}, "emitted_at": 1720179592979}
-{"stream":"ticket_activities","data":{"url":"https://d3v-airbyte.zendesk.com/api/v2/activities/11836244663055.json","id":11836244663055,"title":"Danylo commented on ticket #160: I hope so!.","verb":"tickets.comment","user_id":360786799676,"actor_id":9515132940047,"updated_at":"2025-01-29T18:21:22Z","created_at":"2025-01-29T18:21:22Z","object":{"comment":{"value":"I hope so!","public":true}},"target":{"ticket":{"id":160,"subject":"Stream filling request"}},"user":{"id":360786799676,"url":"https://d3v-airbyte.zendesk.com/api/v2/users/360786799676.json","name":"Team Airbyte","email":"[email protected]","created_at":"2020-11-17T23:55:24Z","updated_at":"2025-01-31T21:22:08Z","time_zone":"Pacific/Noumea","iana_time_zone":"Pacific/Noumea","phone":null,"shared_phone_number":null,"photo":{"url":"https://d3v-airbyte.zendesk.com/api/v2/attachments/7282857066895.json","id":7282857066895,"file_name":"Airbyte_logo_220x220.png","content_url":"https://d3v-airbyte.zendesk.com/system/photos/7282857066895/Airbyte_logo_220x220.png","mapped_content_url":"https://d3v-airbyte.zendesk.com/system/photos/7282857066895/Airbyte_logo_220x220.png","content_type":"image/png","size":5442,"width":80,"height":80,"inline":false,"deleted":false,"thumbnails":[{"url":"https://d3v-airbyte.zendesk.com/api/v2/attachments/7282824912911.json","id":7282824912911,"file_name":"Airbyte_logo_220x220_thumb.png","content_url":"https://d3v-airbyte.zendesk.com/system/photos/7282857066895/Airbyte_logo_220x220_thumb.png","mapped_content_url":"https://d3v-airbyte.zendesk.com/system/photos/7282857066895/Airbyte_logo_220x220_thumb.png","content_type":"image/png","size":1422,"width":32,"height":32,"inline":false,"deleted":false}]},"locale_id":1,"locale":"en-US","organization_id":360033549136,"role":"admin","verified":true,"external_id":null,"tags":[],"alias":"Team Airbyte","active":true,"shared":false,"shared_agent":false,"last_login_at":"2025-01-31T21:22:08Z","two_factor_auth_enabled":null,"signature":null,"details":null,"notes":null,"role_type":4,"custom_role_id":360006308896,"moderator":true,"ticket_restriction":null,"only_private_comments":false,"restricted_agent":false,"suspended":false,"default_group_id":360003074836,"report_csv":true,"user_fields":{"test_display_name_checkbox_field":false,"test_display_name_decimal_field":null,"test_display_name_text_field":null}},"actor":{"id":9515132940047,"url":"https://d3v-airbyte.zendesk.com/api/v2/users/9515132940047.json","name":"Danylo","email":"[email protected]","created_at":"2024-04-12T13:38:07Z","updated_at":"2024-04-12T13:38:07Z","time_zone":"Pacific/Noumea","iana_time_zone":"Pacific/Noumea","phone":null,"shared_phone_number":null,"photo":null,"locale_id":1,"locale":"en-US","organization_id":null,"role":"end-user","verified":false,"external_id":null,"tags":[],"alias":"","active":true,"shared":false,"shared_agent":false,"last_login_at":null,"two_factor_auth_enabled":null,"signature":null,"details":"","notes":"","role_type":null,"custom_role_id":null,"moderator":false,"ticket_restriction":"requested","only_private_comments":false,"restricted_agent":true,"suspended":false,"default_group_id":null,"report_csv":false,"user_fields":{"test_display_name_checkbox_field":false,"test_display_name_decimal_field":null,"test_display_name_text_field":null}}},"emitted_at":1738602991286}
 {"stream": "ticket_audits", "data": {"id": 10021116193295, "ticket_id": 160, "created_at": "2024-06-19T09:49:54Z", "author_id": 9515132940047, "metadata": {"system": {"message_id": "<[email protected]>", "client": "Microsoft Outlook 16.0", "email_id": "01J0QYE6SGCX3Z936BFETHRR8P", "ip_address": "024.06.19.02", "raw_email_identifier": "10414779/4fe5f3c6-857a-4560-a065-aae562a36b53.eml", "json_email_identifier": "10414779/4fe5f3c6-857a-4560-a065-aae562a36b53.json", "eml_redacted": false, "location": "Mountain View, CA, United States", "latitude": 37.3859, "longitude": -122.0882}, "custom": {}, "flags": [15], "flags_options": {"15": {"trusted": true}}, "trusted": true, "suspension_type_id": null}, "events": [{"id": 10021099820047, "type": "Comment", "author_id": 9515132940047, "body": "\n\n\n\nI hope so!", "html_body": "<div class=\"zd-comment zd-comment-pre-styled\" dir=\"auto\"><div style=\"page: WordSection1;\"><p style=\"font-size: 11.0pt; margin: 0in 0in .0001pt;\" dir=\"auto\">&nbsp;</p><p style=\"font-size: 11.0pt; margin: 0in 0in .0001pt;\" dir=\"auto\">&nbsp;</p><p style=\"font-size: 11.0pt; margin: 0in 0in .0001pt;\" dir=\"auto\">I hope so!</p><p style=\"font-size: 11.0pt; margin: 0in 0in .0001pt;\" dir=\"auto\">&nbsp;</p></div></div>", "plain_body": "&nbsp;  &nbsp;  I hope so!  &nbsp;", "public": true, "attachments": [], "audit_id": 10021116193295}, {"id": 10021116193423, "type": "Notification", "via": {"channel": "rule", "source": {"from": {"deleted": false, "title": "Notify assignee of comment update", "id": 360011363236, "revision_id": 1}, "rel": "trigger"}}, "subject": "[{{ticket.account}}] Re: {{ticket.title}}", "body": "This ticket (#{{ticket.id}}) has been updated.\n\n{{ticket.comments_formatted}}", "recipients": [360786799676]}], "via": {"channel": "email", "source": {"from": {"address": "[email protected]", "name": "Danylo", "original_recipients": ["[email protected]", "[email protected]"]}, "to": {"name": "Airbyte", "address": "[email protected]"}, "rel": null}}}, "emitted_at": 1720179595459}
 {"stream": "ticket_audits", "data": {"id": 10020996855439, "ticket_id": 160, "created_at": "2024-06-19T09:28:21Z", "author_id": 360786799676, "metadata": {"system": {"client": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", "ip_address": "45.89.90.157", "location": "Lviv, 46, Ukraine", "latitude": 49.839, "longitude": 24.0191}, "custom": {}}, "events": [{"id": 10020996855567, "type": "Change", "value": "high", "field_name": "priority", "previous_value": "normal"}], "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}}, "emitted_at": 1720179595472}
 {"stream": "ticket_audits", "data": {"id": 10020982311311, "ticket_id": 160, "created_at": "2024-06-19T09:27:57Z", "author_id": 360786799676, "metadata": {"system": {"client": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", "ip_address": "45.89.90.157", "location": "Lviv, 46, Ukraine", "latitude": 49.839, "longitude": 24.0191}, "custom": {}}, "events": [{"id": 10020982311439, "type": "Change", "value": "normal", "field_name": "priority", "previous_value": "high"}], "via": {"channel": "web", "source": {"from": {}, "to": {}, "rel": null}}}, "emitted_at": 1720179595483}

airbyte-integrations/connectors/source-zendesk-support/metadata.yaml

@@ -11,7 +11,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 79c1aa37-dae3-42ae-b333-d1c105477715
-  dockerImageTag: 4.7.1
+  dockerImageTag: 4.7.2
   dockerRepository: airbyte/source-zendesk-support
   documentationUrl: https://docs.airbyte.com/integrations/sources/zendesk-support
   githubIssueLabel: source-zendesk-support

airbyte-integrations/connectors/source-zendesk-support/pyproject.toml

@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
-version = "4.7.1"
+version = "4.7.2"
 name = "source-zendesk-support"
 description = "Source implementation for Zendesk Support."
 authors = [ "Airbyte <[email protected]>",]

airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/source.py

@@ -117,8 +117,6 @@ def get_nested_streams(self, config: Mapping[str, Any]) -> List[Stream]:
         """
         args = self.convert_config2stream_args(config)
 
-        tickets = Tickets(**args)
-
         streams = [
             Articles(**args),
             ArticleComments(**args),
@@ -128,7 +126,7 @@ def get_nested_streams(self, config: Mapping[str, Any]) -> List[Stream]:
             PostComments(**args),
             PostCommentVotes(**args),
             PostVotes(**args),
-            tickets,
+            Tickets(**args),
             TicketMetrics(**args),
             UserIdentities(**args),
         ]

airbyte-integrations/connectors/source-zendesk-support/source_zendesk_support/streams.py

@@ -261,7 +261,7 @@ def get_stream_state_value(self, stream_state: Mapping[str, Any] = None) -> int:
         Returns the state value, if exists. Otherwise, returns user defined `Start Date`.
         """
         state = stream_state.get(self.cursor_field) or self._start_date if stream_state else self._start_date
-        return calendar.timegm(pendulum.parse(state).utctimetuple())
+        return int(state) if isinstance(state, int) or state.isdigit() else calendar.timegm(pendulum.parse(state).utctimetuple())
 
 
 class CursorPaginationZendeskSupportStream(IncrementalZendeskSupportStream):

airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/config.py

@@ -1,16 +1,17 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 
 import base64
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 from pendulum.datetime import DateTime
 
 
 class ConfigBuilder:
     def __init__(self) -> None:
-        self._subdomain: str = None
-        self._start_date: str = None
+        self._subdomain: Optional[str] = None
+        self._start_date: Optional[str] = None
         self._credentials: Dict[str, str] = {}
+        self._ignore_pagination: Optional[bool] = None
 
     def with_subdomain(self, subdomain: str) -> "ConfigBuilder":
         self._subdomain = subdomain
@@ -31,6 +32,10 @@ def with_start_date(self, start_date: DateTime) -> "ConfigBuilder":
         self._start_date = start_date.format("YYYY-MM-DDTHH:mm:ss[Z]")
         return self
 
+    def with_ignore_pagination(self) -> "ConfigBuilder":
+        self._ignore_pagination = True
+        return self
+
     def build(self) -> Dict[str, Any]:
         config = {}
         if self._subdomain:
@@ -39,4 +44,6 @@ def build(self) -> Dict[str, Any]:
             config["start_date"] = self._start_date
         if self._credentials:
             config["credentials"] = self._credentials
+        if self._ignore_pagination:
+            config["ignore_pagination"] = self._ignore_pagination
         return config

airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/test_posts.py

@@ -0,0 +1,157 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+from datetime import datetime, timezone
+from multiprocessing.context import AuthenticationError
+from unittest import TestCase
+
+import freezegun
+import pendulum
+
+from airbyte_cdk.models import AirbyteStateBlob, SyncMode
+from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest
+from airbyte_cdk.test.state_builder import StateBuilder
+
+from .config import ConfigBuilder
+from .utils import datetime_to_string, read_stream
+from .zs_requests import PostsRequestBuilder
+from .zs_requests.request_authenticators import ApiTokenAuthenticator
+from .zs_requests.request_authenticators.authenticator import Authenticator
+from .zs_responses import PostsResponseBuilder
+from .zs_responses.records import PostsRecordBuilder
+
+
+_NOW = datetime.now(timezone.utc)
+_START_DATE = pendulum.now(tz="UTC").subtract(years=2)
+
+
+@freezegun.freeze_time(_NOW.isoformat())
+class TestPostsStream(TestCase):
+    def _config(self) -> ConfigBuilder:
+        return (
+            ConfigBuilder()
+            .with_basic_auth_credentials("[email protected]", "password")
+            .with_subdomain("d3v-airbyte")
+            .with_start_date(pendulum.now(tz="UTC").subtract(years=2))
+        )
+
+    def _get_authenticator(self, config):
+        return ApiTokenAuthenticator(email=config["credentials"]["email"], password=config["credentials"]["api_token"])
+
+    def _base_posts_request(self, authenticator: Authenticator) -> PostsRequestBuilder:
+        return PostsRequestBuilder.posts_endpoint(authenticator).with_page_size(100)
+
+    @HttpMocker()
+    def test_given_one_page_when_read_then_return_records(self, http_mocker):
+        config = self._config().with_start_date(_START_DATE).build()
+        api_token_authenticator = self._get_authenticator(config)
+        http_mocker.get(
+            self._base_posts_request(api_token_authenticator).with_start_time(datetime_to_string(_START_DATE)).build(),
+            PostsResponseBuilder.posts_response()
+            .with_record(PostsRecordBuilder.posts_record())
+            .with_record(PostsRecordBuilder.posts_record())
+            .build(),
+        )
+
+        output = read_stream("posts", SyncMode.full_refresh, config)
+
+        assert len(output.records) == 2
+
+    @HttpMocker()
+    def test_given_has_more_when_read_then_paginate(self, http_mocker):
+        config = self._config().with_start_date(_START_DATE).build()
+        api_token_authenticator = self._get_authenticator(config)
+        http_mocker.get(
+            self._base_posts_request(api_token_authenticator).with_start_time(datetime_to_string(_START_DATE)).build(),
+            PostsResponseBuilder.posts_response(self._base_posts_request(api_token_authenticator).build())
+            .with_record(PostsRecordBuilder.posts_record())
+            .with_record(PostsRecordBuilder.posts_record())
+            .with_pagination()
+            .build(),
+        )
+        http_mocker.get(
+            self._base_posts_request(api_token_authenticator).with_after_cursor("after-cursor").build(),
+            PostsResponseBuilder.posts_response().with_record(PostsRecordBuilder.posts_record()).build(),
+        )
+
+        output = read_stream("posts", SyncMode.full_refresh, config)
+
+        assert len(output.records) == 3
+
+    @HttpMocker()
+    def test_given_ignore_pagination_when_read_then_do_not_paginate(self, http_mocker):
+        """
+        Given the db query `select * from actor where actor_definition_id = '79c1aa37-dae3-42ae-b333-d1c105477715' and configuration->>'ignore_pagination' = 'true' and tombstone = false;`, we can see that some connections are using this config. I don't exactly understand the use for that but here is the test since this is used.
+        """
+        config = self._config().with_start_date(_START_DATE).with_ignore_pagination().build()
+        api_token_authenticator = self._get_authenticator(config)
+        http_mocker.get(
+            self._base_posts_request(api_token_authenticator).with_start_time(datetime_to_string(_START_DATE)).build(),
+            PostsResponseBuilder.posts_response(self._base_posts_request(api_token_authenticator).build())
+            .with_record(PostsRecordBuilder.posts_record())
+            .with_record(PostsRecordBuilder.posts_record())
+            .with_pagination()
+            .build(),
+        )
+
+        output = read_stream("posts", SyncMode.full_refresh, config)
+
+        assert len(output.records) == 2
+        assert len(output.errors) == 0
+
+    @HttpMocker()
+    def test_when_read_then_set_state_value_to_most_recent_cursor_value(self, http_mocker):
+        config = self._config().with_start_date(_START_DATE).build()
+        api_token_authenticator = self._get_authenticator(config)
+        most_recent_cursor_value = datetime_to_string(_START_DATE.add(days=2))
+        http_mocker.get(
+            PostsRequestBuilder.posts_endpoint(api_token_authenticator)
+            .with_start_time(datetime_to_string(_START_DATE))
+            .with_page_size(100)
+            .build(),
+            PostsResponseBuilder.posts_response()
+            .with_record(PostsRecordBuilder.posts_record().with_cursor(most_recent_cursor_value))
+            .with_record(PostsRecordBuilder.posts_record().with_cursor(datetime_to_string(_START_DATE.add(days=1))))
+            .build(),
+        )
+
+        output = read_stream("posts", SyncMode.full_refresh, config)
+
+        assert output.most_recent_state.stream_state == AirbyteStateBlob({"updated_at": most_recent_cursor_value})
+
+    @HttpMocker()
+    def test_given_input_state_when_read_then_set_state_value_to_most_recent_cursor_value(self, http_mocker):
+        config = self._config().with_start_date(_START_DATE).build()
+        api_token_authenticator = self._get_authenticator(config)
+        state_cursor_value = datetime_to_string(_START_DATE.add(days=2))
+        http_mocker.get(
+            PostsRequestBuilder.posts_endpoint(api_token_authenticator).with_start_time(state_cursor_value).with_page_size(100).build(),
+            PostsResponseBuilder.posts_response().with_record(PostsRecordBuilder.posts_record()).build(),
+        )
+
+        output = read_stream(
+            "posts", SyncMode.full_refresh, config, StateBuilder().with_stream_state("posts", {"updated_at": state_cursor_value}).build()
+        )
+
+        assert len(output.records) == 1
+
+    @HttpMocker()
+    def test_given_input_state_with_low_code_format_when_read_then_set_state_value_to_most_recent_cursor_value(self, http_mocker):
+        config = self._config().with_start_date(_START_DATE).build()
+        api_token_authenticator = self._get_authenticator(config)
+        state_cursor_value = _START_DATE.add(days=2)
+        http_mocker.get(
+            PostsRequestBuilder.posts_endpoint(api_token_authenticator)
+            .with_start_time(datetime_to_string(state_cursor_value))
+            .with_page_size(100)
+            .build(),
+            PostsResponseBuilder.posts_response().with_record(PostsRecordBuilder.posts_record()).build(),
+        )
+
+        output = read_stream(
+            "posts",
+            SyncMode.full_refresh,
+            config,
+            StateBuilder().with_stream_state("posts", {"updated_at": str(state_cursor_value.int_timestamp)}).build(),
+        )
+
+        assert len(output.records) == 1

airbyte-integrations/connectors/source-zendesk-support/unit_tests/integrations/utils.py

@@ -7,6 +7,7 @@
 from pendulum.datetime import DateTime
 from source_zendesk_support import SourceZendeskSupport
 
+from airbyte_cdk.connector_builder.models import HttpRequest
 from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, SyncMode
 from airbyte_cdk.models import Level as LogLevel
 from airbyte_cdk.test.catalog_builder import CatalogBuilder
@@ -34,3 +35,9 @@ def datetime_to_string(dt: DateTime) -> str:
 
 def string_to_datetime(dt_string: str) -> DateTime:
     return pendulum.parse(dt_string)
+
+
+def http_request_to_str(http_request: Optional[HttpRequest]) -> Optional[str]:
+    if http_request is None:
+        return None
+    return http_request._parsed_url._replace(fragment="").geturl()