🐛 Source Github: Fix error handling for 404 streams (#10878)

* Fix, no error if "Projects" feature is disabled
* improve requests.codes.NOT_FOUND

Signed-off-by: Sergey Chvalyuk <[email protected]>

Author: grubberr

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -245,7 +245,7 @@
 - name: GitHub
   sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
   dockerRepository: airbyte/source-github
-  dockerImageTag: 0.2.21
+  dockerImageTag: 0.2.22
   documentationUrl: https://docs.airbyte.io/integrations/sources/github
   icon: github.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -2357,7 +2357,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-github:0.2.21"
+- dockerImage: "airbyte/source-github:0.2.22"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/github"
     connectionSpecification:

airbyte-integrations/connectors/source-github/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.21
+LABEL io.airbyte.version=0.2.22
 LABEL io.airbyte.name=airbyte/source-github

airbyte-integrations/connectors/source-github/source_github/streams.py

@@ -100,7 +100,13 @@ def read_records(self, stream_slice: Mapping[str, any] = None, **kwargs) -> Iter
             # This whole try/except situation in `read_records()` isn't good but right now in `self._send_request()`
             # function we have `response.raise_for_status()` so we don't have much choice on how to handle errors.
             # Bocked on https://github.com/airbytehq/airbyte/issues/3514.
-            if e.response.status_code == requests.codes.FORBIDDEN:
+            if e.response.status_code == requests.codes.NOT_FOUND:
+                # A lot of streams are not available for repositories owned by a user instead of an organization.
+                if isinstance(self, Organizations):
+                    error_msg = f"Syncing `{self.__class__.__name__}` stream isn't available for organization `{stream_slice['organization']}`."
+                else:
+                    error_msg = f"Syncing `{self.__class__.__name__}` stream isn't available for repository `{stream_slice['repository']}`."
+            elif e.response.status_code == requests.codes.FORBIDDEN:
                 # When using the `check_connection` method, we should raise an error if we do not have access to the repository.
                 if isinstance(self, Repositories):
                     raise e
@@ -116,30 +122,10 @@ def read_records(self, stream_slice: Mapping[str, any] = None, **kwargs) -> Iter
                     error_msg = (
                         f"Syncing `{self.name}` stream isn't available for repository `{repository}`. Full error message: {error_msg}"
                     )
-            elif e.response.status_code == requests.codes.NOT_FOUND and "/teams?" in error_msg:
-                # For private repositories `Teams` stream is not available and we get "404 Client Error: Not Found for
-                # url: https://api.github.com/orgs/<org_name>/teams?per_page=100" error.
-                error_msg = f"Syncing `Team` stream isn't available for organization `{stream_slice['organization']}`."
-            elif e.response.status_code == requests.codes.NOT_FOUND and "/repos?" in error_msg:
-                # `Repositories` stream is not available for repositories not in an organization.
-                # Handle "404 Client Error: Not Found for url: https://api.github.com/orgs/<org_name>/repos?per_page=100" error.
-                error_msg = f"Syncing `Repositories` stream isn't available for organization `{stream_slice['organization']}`."
-            elif e.response.status_code == requests.codes.GONE and "/projects?" in error_msg:
+            elif e.response.status_code == requests.codes.GONE and isinstance(self, Projects):
                 # Some repos don't have projects enabled and we we get "410 Client Error: Gone for
                 # url: https://api.github.com/repos/xyz/projects?per_page=100" error.
                 error_msg = f"Syncing `Projects` stream isn't available for repository `{stream_slice['repository']}`."
-            elif e.response.status_code == requests.codes.NOT_FOUND and "/orgs/" in error_msg:
-                # Some streams are not available for repositories owned by a user instead of an organization.
-                # Handle "404 Client Error: Not Found" errors
-                if isinstance(self, Repositories):
-                    error_msg = f"Syncing `Repositories` stream isn't available for organization `{stream_slice['organization']}`."
-                elif isinstance(self, Users):
-                    error_msg = f"Syncing `Users` stream isn't available for organization `{stream_slice['organization']}`."
-                elif isinstance(self, Organizations):
-                    error_msg = f"Syncing `Organizations` stream isn't available for organization `{stream_slice['organization']}`."
-                else:
-                    self.logger.error(f"Undefined error while reading records: {error_msg}")
-                    raise e
             elif e.response.status_code == requests.codes.CONFLICT:
                 error_msg = (
                     f"Syncing `{self.name}` stream isn't available for repository "

airbyte-integrations/connectors/source-github/unit_tests/test_stream.py

@@ -6,9 +6,12 @@
 from unittest.mock import MagicMock, patch
 
 import pytest
+import requests
 import responses
 from airbyte_cdk.sources.streams.http.exceptions import BaseBackoffException
-from source_github.streams import PullRequestCommentReactions
+from source_github.streams import Projects, PullRequestCommentReactions, Repositories, Teams
+
+from .utils import read_full_refresh
 
 DEFAULT_BACKOFF_DELAYS = [5, 10, 20, 40, 80]
 
@@ -47,3 +50,54 @@ def test_backoff_time(http_status, response_text, expected_backoff_time):
     args = {"authenticator": None, "repositories": ["test_repo"], "start_date": "start_date", "page_size_for_large_streams": 30}
     stream = PullRequestCommentReactions(**args)
     assert stream.backoff_time(response_mock) == expected_backoff_time
+
+
+@responses.activate
+def test_stream_teams_404():
+    kwargs = {"organizations": ["org_name"]}
+    stream = Teams(**kwargs)
+
+    responses.add(
+        "GET",
+        "https://api.github.com/orgs/org_name/teams",
+        status=requests.codes.NOT_FOUND,
+        json={"message": "Not Found", "documentation_url": "https://docs.github.com/rest/reference/teams#list-teams"},
+    )
+
+    assert read_full_refresh(stream) == []
+    assert len(responses.calls) == 1
+    assert responses.calls[0].request.url == "https://api.github.com/orgs/org_name/teams?per_page=100"
+
+
+@responses.activate
+def test_stream_repositories_404():
+    kwargs = {"organizations": ["org_name"]}
+    stream = Repositories(**kwargs)
+
+    responses.add(
+        "GET",
+        "https://api.github.com/orgs/org_name/repos",
+        status=requests.codes.NOT_FOUND,
+        json={"message": "Not Found", "documentation_url": "https://docs.github.com/rest/reference/repos#list-organization-repositories"},
+    )
+
+    assert read_full_refresh(stream) == []
+    assert len(responses.calls) == 1
+    assert responses.calls[0].request.url == "https://api.github.com/orgs/org_name/repos?per_page=100"
+
+
+@responses.activate
+def test_stream_projects_disabled():
+    kwargs = {"start_date": "start_date", "page_size_for_large_streams": 30, "repositories": ["test_repo"]}
+    stream = Projects(**kwargs)
+
+    responses.add(
+        "GET",
+        "https://api.github.com/repos/test_repo/projects",
+        status=requests.codes.GONE,
+        json={"message": "Projects are disabled for this repository", "documentation_url": "https://docs.github.com/v3/projects"},
+    )
+
+    assert read_full_refresh(stream) == []
+    assert len(responses.calls) == 1
+    assert responses.calls[0].request.url == "https://api.github.com/repos/test_repo/projects?per_page=100&state=all"

airbyte-integrations/connectors/source-github/unit_tests/utils.py

@@ -0,0 +1,15 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+
+from airbyte_cdk.models import SyncMode
+from airbyte_cdk.sources.streams import Stream
+
+
+def read_full_refresh(stream_instance: Stream):
+    records = []
+    slices = stream_instance.stream_slices(sync_mode=SyncMode.full_refresh)
+    for slice in slices:
+        records.extend(list(stream_instance.read_records(stream_slice=slice, sync_mode=SyncMode.full_refresh)))
+    return records

docs/integrations/sources/github.md

@@ -95,6 +95,7 @@ Your token should have at least the `repo` scope. Depending on which streams you
 
 | Version | Date       | Pull Request | Subject                                                                                                      |
 |:--------|:-----------| :--- |:-------------------------------------------------------------------------------------------------------------|
+| 0.2.22  | 2022-03-10 | [10878](https://github.com/airbytehq/airbyte/pull/10878) | Fix error handling for unavailable streams with 404 status code                                            |
 | 0.2.21  | 2022-03-04 | [10749](https://github.com/airbytehq/airbyte/pull/10749) | Add new stream `ProjectCards`                                                                              |
 | 0.2.20  | 2022-02-16 | [10385](https://github.com/airbytehq/airbyte/pull/10385) | Add new stream `Deployments`, `ProjectColumns`, `PullRequestCommits`                                       |
 | 0.2.19  | 2022-02-07 | [10211](https://github.com/airbytehq/airbyte/pull/10211) | Add human-readable error in case of incorrect organization or repo name                                    |