Source Freshdesk: add availability strategy (airbytehq#22145)

* Source FreshDesk: start using availability strategy

* Source FreshDesk: fix flake issues

* Source Freshdesk: update version

* auto-bump connector version

---------

Co-authored-by: Octavia Squidington III <[email protected]>

Author: 2 people

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -568,11 +568,14 @@
 - name: Freshdesk
   sourceDefinitionId: ec4b9503-13cb-48ab-a4ab-6ade4be46567
   dockerRepository: airbyte/source-freshdesk
-  dockerImageTag: 3.0.0
+  dockerImageTag: 3.0.2
   documentationUrl: https://docs.airbyte.com/integrations/sources/freshdesk
   icon: freshdesk.svg
   sourceType: api
   releaseStage: generally_available
+  allowedHosts:
+    hosts:
+      - "*.freshdesk.com"
 - name: Freshsales
   sourceDefinitionId: eca08d79-7b92-4065-b7f3-79c14836ebe7
   dockerRepository: airbyte/source-freshsales

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -4514,7 +4514,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-freshdesk:3.0.0"
+- dockerImage: "airbyte/source-freshdesk:3.0.2"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/freshdesk"
     connectionSpecification:

airbyte-integrations/connectors/source-freshdesk/Dockerfile

@@ -34,5 +34,5 @@ COPY source_freshdesk ./source_freshdesk
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=3.0.0
+LABEL io.airbyte.version=3.0.2
 LABEL io.airbyte.name=airbyte/source-freshdesk

airbyte-integrations/connectors/source-freshdesk/source_freshdesk/availability_strategy.py

@@ -0,0 +1,18 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+import requests
+from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy
+
+
+class FreshdeskAvailabilityStrategy(HttpAvailabilityStrategy):
+    def reasons_for_unavailable_status_codes(self, stream, logger, source, error):
+        unauthorized_error_message = f"The endpoint to access stream '{stream.name}' returned 401: Unauthorized. "
+        unauthorized_error_message += "This is most likely due to wrong credentials. "
+        unauthorized_error_message += self._visit_docs_message(logger, source)
+
+        reasons = super(FreshdeskAvailabilityStrategy, self).reasons_for_unavailable_status_codes(stream, logger, source, error)
+        reasons[requests.codes.UNAUTHORIZED] = unauthorized_error_message
+
+        return reasons

airbyte-integrations/connectors/source-freshdesk/source_freshdesk/source.py

@@ -3,11 +3,10 @@
 #
 
 import logging
-from typing import Any, List, Mapping, Optional, Tuple
-from urllib.parse import urljoin
+from typing import Any, List, Mapping
 
-import requests
 from airbyte_cdk.sources import AbstractSource
+from airbyte_cdk.sources.declarative.checks import CheckStream
 from airbyte_cdk.sources.streams import Stream
 from requests.auth import HTTPBasicAuth
 from source_freshdesk.streams import (
@@ -52,53 +51,45 @@ def __init__(self, api_key: str) -> None:
 
 
 class SourceFreshdesk(AbstractSource):
-    def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]:
-        alive = True
-        error_msg = None
+    @staticmethod
+    def _get_stream_kwargs(config: Mapping[str, Any]) -> dict:
+        return {"authenticator": FreshdeskAuth(config["api_key"]), "config": config}
+
+    def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]):
         try:
-            url = urljoin(f"https://{config['domain'].rstrip('/')}", "/api/v2/settings/helpdesk")
-            response = requests.get(url=url, auth=FreshdeskAuth(config["api_key"]))
-            response.raise_for_status()
-        except requests.HTTPError as error:
-            alive = False
-            body = error.response.json()
-            error_msg = f"{body.get('code')}: {body.get('message')}"
+            check_stream = CheckStream(stream_names=["settings"], options={})
+            return check_stream.check_connection(self, logger, config)
         except Exception as error:
-            alive = False
-            error_msg = repr(error)
-
-        return alive, error_msg
+            return False, repr(error)
 
     def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        authenticator = FreshdeskAuth(config["api_key"])
-        stream_kwargs = {"authenticator": authenticator, "config": config}
         return [
-            Agents(**stream_kwargs),
-            BusinessHours(**stream_kwargs),
-            CannedResponseFolders(**stream_kwargs),
-            CannedResponses(**stream_kwargs),
-            Companies(**stream_kwargs),
-            Contacts(**stream_kwargs),
-            Conversations(**stream_kwargs),
-            DiscussionCategories(**stream_kwargs),
-            DiscussionComments(**stream_kwargs),
-            DiscussionForums(**stream_kwargs),
-            DiscussionTopics(**stream_kwargs),
-            EmailConfigs(**stream_kwargs),
-            EmailMailboxes(**stream_kwargs),
-            Groups(**stream_kwargs),
-            Products(**stream_kwargs),
-            Roles(**stream_kwargs),
-            ScenarioAutomations(**stream_kwargs),
-            Settings(**stream_kwargs),
-            Skills(**stream_kwargs),
-            SlaPolicies(**stream_kwargs),
-            SolutionArticles(**stream_kwargs),
-            SolutionCategories(**stream_kwargs),
-            SolutionFolders(**stream_kwargs),
-            TimeEntries(**stream_kwargs),
-            TicketFields(**stream_kwargs),
-            Tickets(**stream_kwargs),
-            SatisfactionRatings(**stream_kwargs),
-            Surveys(**stream_kwargs),
+            Agents(**self._get_stream_kwargs(config)),
+            BusinessHours(**self._get_stream_kwargs(config)),
+            CannedResponseFolders(**self._get_stream_kwargs(config)),
+            CannedResponses(**self._get_stream_kwargs(config)),
+            Companies(**self._get_stream_kwargs(config)),
+            Contacts(**self._get_stream_kwargs(config)),
+            Conversations(**self._get_stream_kwargs(config)),
+            DiscussionCategories(**self._get_stream_kwargs(config)),
+            DiscussionComments(**self._get_stream_kwargs(config)),
+            DiscussionForums(**self._get_stream_kwargs(config)),
+            DiscussionTopics(**self._get_stream_kwargs(config)),
+            EmailConfigs(**self._get_stream_kwargs(config)),
+            EmailMailboxes(**self._get_stream_kwargs(config)),
+            Groups(**self._get_stream_kwargs(config)),
+            Products(**self._get_stream_kwargs(config)),
+            Roles(**self._get_stream_kwargs(config)),
+            ScenarioAutomations(**self._get_stream_kwargs(config)),
+            Settings(**self._get_stream_kwargs(config)),
+            Skills(**self._get_stream_kwargs(config)),
+            SlaPolicies(**self._get_stream_kwargs(config)),
+            SolutionArticles(**self._get_stream_kwargs(config)),
+            SolutionCategories(**self._get_stream_kwargs(config)),
+            SolutionFolders(**self._get_stream_kwargs(config)),
+            TimeEntries(**self._get_stream_kwargs(config)),
+            TicketFields(**self._get_stream_kwargs(config)),
+            Tickets(**self._get_stream_kwargs(config)),
+            SatisfactionRatings(**self._get_stream_kwargs(config)),
+            Surveys(**self._get_stream_kwargs(config)),
         ]

airbyte-integrations/connectors/source-freshdesk/source_freshdesk/streams.py

@@ -16,6 +16,7 @@
 from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream
 from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer
 from requests.auth import AuthBase
+from source_freshdesk.availability_strategy import FreshdeskAvailabilityStrategy
 from source_freshdesk.utils import CallCredit
 
 
@@ -48,20 +49,13 @@ def url_base(self) -> str:
         return parse.urljoin(f"https://{self.domain.rstrip('/')}", "/api/v2/")
 
     @property
-    def availability_strategy(self) -> Optional["AvailabilityStrategy"]:
-        return None
+    def availability_strategy(self) -> Optional[AvailabilityStrategy]:
+        return FreshdeskAvailabilityStrategy()
 
     def backoff_time(self, response: requests.Response) -> Optional[float]:
         if response.status_code == requests.codes.too_many_requests:
             return float(response.headers.get("Retry-After", 0))
 
-    def should_retry(self, response: requests.Response) -> bool:
-        if response.status_code == requests.codes.FORBIDDEN:
-            self.forbidden_stream = True
-            setattr(self, "raise_on_http_errors", False)
-            self.logger.warn(f"Stream `{self.name}` is not available. {response.text}")
-        return super().should_retry(response)
-
     def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
         link_header = response.headers.get("Link")
         if not link_header:

airbyte-integrations/connectors/source-freshdesk/unit_tests/test_source.py

@@ -26,7 +26,10 @@ def test_check_connection_invalid_api_key(requests_mock, config):
     requests_mock.register_uri("GET", "/api/v2/settings/helpdesk", responses)
     ok, error_msg = SourceFreshdesk().check_connection(logger, config=config)
 
-    assert not ok and error_msg == "invalid_credentials: You have to be logged in to perform this action."
+    assert not ok and error_msg == "The endpoint to access stream \'settings\' returned 401: Unauthorized. " \
+                                   "This is most likely due to wrong credentials. " \
+                                   "Please visit https://docs.airbyte.com/integrations/sources/freshdesk to learn more. " \
+                                   "You have to be logged in to perform this action."
 
 
 def test_check_connection_empty_config(config):
@@ -45,7 +48,7 @@ def test_check_connection_invalid_config(config):
     assert not ok and error_msg
 
 
-def test_check_connection_exception(config):
+def test_check_connection_exception(requests_mock, config):
     ok, error_msg = SourceFreshdesk().check_connection(logger, config=config)
 
     assert not ok and error_msg

airbyte-integrations/connectors/source-freshdesk/unit_tests/test_streams.py

@@ -209,13 +209,3 @@ def test_full_refresh_discussion_comments(requests_mock, authenticator, config):
     records = _read_full_refresh(stream)
 
     assert len(records) == 120
-
-
-def test_403_skipped(requests_mock, authenticator, config):
-    # this case should neither raise an error nor retry
-    requests_mock.register_uri("GET", "/api/v2/tickets", json=[{"id": 1705, "updated_at": "2022-05-05T00:00:00Z"}])
-    requests_mock.register_uri("GET", "/api/v2/tickets/1705/conversations", status_code=403)
-    stream = Conversations(authenticator=authenticator, config=config)
-    records = _read_full_refresh(stream)
-    assert records == []
-    assert len(requests_mock.request_history) == 2

docs/integrations/sources/freshdesk.md

@@ -67,6 +67,7 @@ The Freshdesk connector should not run into Freshdesk API limitations under norm
 
 | Version | Date       | Pull Request                                             | Subject                                                                               |
 |:--------|:-----------|:---------------------------------------------------------|:--------------------------------------------------------------------------------------|
+| 3.0.2   | 2023-02-06 | [21970](https://github.com/airbytehq/airbyte/pull/21970) | Enable availability strategy for all streams                                                       |
 | 3.0.0   | 2023-01-31 | [22164](https://github.com/airbytehq/airbyte/pull/22164) | Rename nested `business_hours` table to `working_hours`                               |
 | 2.0.1   | 2023-01-27 | [21888](https://github.com/airbytehq/airbyte/pull/21888) | Set `AvailabilityStrategy` for streams explicitly to `None`                           |
 | 2.0.0   | 2022-12-20 | [20416](https://github.com/airbytehq/airbyte/pull/20416) | Fix `SlaPolicies` stream schema                                                       |