Source Github: add integration tests

airbyte-integrations/connectors/source-github/acceptance-test-config.yml

@@ -32,7 +32,7 @@ acceptance_tests:
           extra_records: yes
         empty_streams:
           - name: "events"
-            bypass_reason: "Only events created within the past 90 days can be showed"
+            bypass_reason: "Only events created within the past 90 days can be showed. Stream is tested with integration tests."
         ignored_fields:
           contributor_activity:
             - name: weeks

airbyte-integrations/connectors/source-github/unit_tests/conftest.py

@@ -12,18 +12,8 @@
 def rate_limit_mock_response():
     rate_limit_response = {
         "resources": {
-            "core": {
-                "limit": 5000,
-                "used": 0,
-                "remaining": 5000,
-                "reset": 4070908800
-            },
-            "graphql": {
-                "limit": 5000,
-                "used": 0,
-                "remaining": 5000,
-                "reset": 4070908800
-            }
+            "core": {"limit": 5000, "used": 0, "remaining": 5000, "reset": 4070908800},
+            "graphql": {"limit": 5000, "used": 0, "remaining": 5000, "reset": 4070908800},
         }
     }
     responses.add(responses.GET, "https://api.github.com/rate_limit", json=rate_limit_response)

airbyte-integrations/connectors/source-github/unit_tests/integration/config.py

@@ -0,0 +1,35 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+from datetime import datetime
+from typing import Any, Dict, List
+
+
+class ConfigBuilder:
+    def __init__(self) -> None:
+        self._config: Dict[str, Any] = {
+            "credentials": {"option_title": "PAT Credentials", "personal_access_token": "GITHUB_TEST_TOKEN"},
+            "start_date": "2020-05-01T00:00:00Z",
+        }
+
+    def with_repositories(self, repositories: List[str]) -> "ConfigBuilder":
+        self._config["repositories"] = repositories
+        return self
+
+    def with_client_secret(self, client_secret: str) -> "ConfigBuilder":
+        self._config["client_secret"] = client_secret
+        return self
+
+    def with_start_date(self, start_datetime: datetime) -> "ConfigBuilder":
+        self._config["start_date"] = start_datetime.isoformat()[:-13] + "Z"
+        return self
+
+    def with_branches(self, branches: List[str]) -> "ConfigBuilder":
+        self._config["branches"] = branches
+        return self
+
+    def with_api_url(self, api_url: str) -> "ConfigBuilder":
+        self._config["api_url"] = api_url
+        return self
+
+    def build(self) -> Dict[str, Any]:
+        return self._config

airbyte-integrations/connectors/source-github/unit_tests/integration/test_events.py

@@ -0,0 +1,143 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+from unittest import TestCase
+
+import responses
+from airbyte_cdk.models import SyncMode
+from airbyte_cdk.test.catalog_builder import CatalogBuilder
+from airbyte_cdk.test.entrypoint_wrapper import read
+from airbyte_cdk.test.mock_http.response_builder import find_template
+from airbyte_cdk.test.state_builder import StateBuilder
+from airbyte_protocol.models import Level
+from responses import matchers
+from responses.registries import OrderedRegistry
+from source_github import SourceGithub
+
+from .config import ConfigBuilder
+
+_CONFIG = ConfigBuilder().with_repositories(["airbytehq/integration-test"]).build()
+
+
+def _create_catalog(sync_mode: SyncMode = SyncMode.full_refresh):
+    return CatalogBuilder().with_stream(name="events", sync_mode=sync_mode).build()
+
+
+class EventsTest(TestCase):
+    def setUp(self) -> None:
+        """Base setup for all tests. Add responses for:
+        1. rate limit checker
+        2. repositories
+        3. branches
+        """
+        self.r_mock = responses.RequestsMock(registry=OrderedRegistry)
+        self.r_mock.start()
+        self.r_mock.get(
+            "https://api.github.com/rate_limit",
+            match=[
+                matchers.header_matcher(
+                    {
+                        "Accept": "application/vnd.github+json",
+                        "X-GitHub-Api-Version": "2022-11-28",
+                        "Authorization": "token GITHUB_TEST_TOKEN",
+                    }
+                )
+            ],
+            json={
+                "resources": {
+                    "core": {"limit": 5000, "used": 0, "remaining": 5000, "reset": 5070908800},
+                    "graphql": {"limit": 5000, "used": 0, "remaining": 5000, "reset": 5070908800},
+                }
+            },
+        )
+        self.r_mock.get(
+            f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}",
+            match=[matchers.query_param_matcher({"per_page": 100})],
+            json={"full_name": "airbytehq/integration-test", "default_branch": "master"},
+        )
+        self.r_mock.get(
+            f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}",
+            match=[matchers.query_param_matcher({"per_page": 100})],
+            json={"full_name": "airbytehq/integration-test", "default_branch": "master"},
+        )
+        self.r_mock.get(
+            f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}/branches",
+            match=[matchers.query_param_matcher({"per_page": 100})],
+            json=[{"repository": "airbytehq/integration-test", "name": "master"}],
+        )
+
+    def teardown(self):
+        """Stops and resets RequestsMock instance.
+
+        If ``assert_all_requests_are_fired`` is set to ``True``, will raise an error
+        if some requests were not processed.
+        """
+        self.r_mock.stop()
+        self.r_mock.reset()
+
+    def test_full_refresh_no_pagination(self):
+        """Ensure http integration, record extraction and transformation"""
+        self.r_mock.get(
+            f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}/events",
+            match=[matchers.query_param_matcher({"per_page": 100})],
+            json=find_template("events", __file__),
+        )
+
+        source = SourceGithub()
+        actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
+
+        assert len(actual_messages.records) == 2
+        assert all(("repository", "airbytehq/integration-test") in x.record.data.items() for x in actual_messages.records)
+
+    def test_full_refresh_with_pagination(self):
+        """Ensure pagination"""
+        self.r_mock.get(
+            f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}/events",
+            headers={"Link": '<https://api.github.com/repos/{}/events?page=2>; rel="next"'.format(_CONFIG.get("repositories")[0])},
+            match=[matchers.query_param_matcher({"per_page": 100})],
+            json=find_template("events", __file__),
+        )
+        self.r_mock.get(
+            f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}/events",
+            match=[matchers.query_param_matcher({"per_page": 100, "page": 2})],
+            json=find_template("events", __file__),
+        )
+        source = SourceGithub()
+        actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
+
+        assert len(actual_messages.records) == 4
+
+    def test_incremental_read(self):
+        """Ensure incremental sync.
+        Stream `Events` is semi-incremental, so all request  will be performed and only new records will be extracted"""
+
+        self.r_mock.get(
+            f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}/events",
+            match=[matchers.query_param_matcher({"per_page": 100})],
+            json=find_template("events", __file__),
+        )
+
+        source = SourceGithub()
+        actual_messages = read(
+            source,
+            config=_CONFIG,
+            catalog=_create_catalog(sync_mode=SyncMode.incremental),
+            state=StateBuilder()
+            .with_stream_state("events", {"airbytehq/integration-test": {"created_at": "2022-06-09T10:00:00Z"}})
+            .build(),
+        )
+        assert len(actual_messages.records) == 1
+
+    def test_read_with_error(self):
+        """Ensure read() method does not raise an Exception and log message with error is in output"""
+
+        self.r_mock.get(
+            f"https://api.github.com/repos/{_CONFIG.get('repositories')[0]}/events",
+            match=[matchers.query_param_matcher({"per_page": 100})],
+            body='{"message":"some_error_message"}',
+            status=403,
+        )
+        source = SourceGithub()
+        actual_messages = read(source, config=_CONFIG, catalog=_create_catalog())
+
+        assert len(actual_messages.records) == 0
+        assert Level.ERROR in [x.log.level for x in actual_messages.logs]

airbyte-integrations/connectors/source-github/unit_tests/resource/http/response/events.json

@@ -0,0 +1,63 @@
+[
+  {
+    "id": "22249084964",
+    "type": "PushEvent",
+    "actor": {
+      "id": 583231,
+      "login": "octocat",
+      "display_login": "octocat",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/octocat",
+      "avatar_url": "https://avatars.githubusercontent.com/u/583231?v=4"
+    },
+    "repo": {
+      "id": 1296269,
+      "name": "octocat/Hello-World",
+      "url": "https://api.github.com/repos/octocat/Hello-World"
+    },
+    "payload": {
+      "push_id": 10115855396,
+      "size": 1,
+      "distinct_size": 1,
+      "ref": "refs/heads/master",
+      "head": "7a8f3ac80e2ad2f6842cb86f576d4bfe2c03e300",
+      "before": "883efe034920928c47fe18598c01249d1a9fdabd",
+      "commits": [
+        {
+          "sha": "7a8f3ac80e2ad2f6842cb86f576d4bfe2c03e300",
+          "author": {
+            "email": "[email protected]",
+            "name": "Monalisa Octocat"
+          },
+          "message": "commit",
+          "distinct": true,
+          "url": "https://api.github.com/repos/octocat/Hello-World/commits/7a8f3ac80e2ad2f6842cb86f576d4bfe2c03e300"
+        }
+      ]
+    },
+    "public": true,
+    "created_at": "2022-06-09T12:47:28Z"
+  },
+  {
+    "id": "22237752260",
+    "type": "WatchEvent",
+    "actor": {
+      "id": 583231,
+      "login": "octocat",
+      "display_login": "octocat",
+      "gravatar_id": "",
+      "url": "https://api.github.com/users/octocat",
+      "avatar_url": "https://avatars.githubusercontent.com/u/583231?v=4"
+    },
+    "repo": {
+      "id": 1296269,
+      "name": "octocat/Hello-World",
+      "url": "https://api.github.com/repos/octocat/Hello-World"
+    },
+    "payload": {
+      "action": "started"
+    },
+    "public": true,
+    "created_at": "2022-06-08T23:29:25Z"
+  }
+]

airbyte-integrations/connectors/source-github/unit_tests/test_multiple_token_authenticator.py

@@ -58,18 +58,8 @@ def request_callback_rate_limits(request):
             counter_rate_limits += 1
             resp_body = {
                 "resources": {
-                    "core": {
-                        "limit": 500,
-                        "used": 0,
-                        "remaining": 500,
-                        "reset": 4070908800
-                    },
-                    "graphql": {
-                        "limit": 500,
-                        "used": 0,
-                        "remaining": 500,
-                        "reset": 4070908800
-                    }
+                    "core": {"limit": 500, "used": 0, "remaining": 500, "reset": 4070908800},
+                    "graphql": {"limit": 500, "used": 0, "remaining": 500, "reset": 4070908800},
                 }
             }
             return (200, {}, json.dumps(resp_body))
@@ -96,7 +86,9 @@ def request_callback_orgs(request):
     with pytest.raises(AirbyteTracedException) as e:
         list(read_full_refresh(stream))
     assert [(x.count_rest, x.count_graphql) for x in authenticator._tokens.values()] == [(0, 500), (0, 500), (0, 500)]
-    message = "Stream: `organizations`, slice: `{'organization': 'org1'}`. Limits for all provided tokens are reached, please try again later"
+    message = (
+        "Stream: `organizations`, slice: `{'organization': 'org1'}`. Limits for all provided tokens are reached, please try again later"
+    )
     assert e.value.internal_message == message
 
 
@@ -121,18 +113,8 @@ def request_callback_rate_limits(request):
             counter_rate_limits += 1
             resp_body = {
                 "resources": {
-                    "core": {
-                        "limit": 500,
-                        "used": 0,
-                        "remaining": 500,
-                        "reset": reset_time
-                    },
-                    "graphql": {
-                        "limit": 500,
-                        "used": 0,
-                        "remaining": 500,
-                        "reset": reset_time
-                    }
+                    "core": {"limit": 500, "used": 0, "remaining": 500, "reset": reset_time},
+                    "graphql": {"limit": 500, "used": 0, "remaining": 500, "reset": reset_time},
                 }
             }
             return (200, {}, json.dumps(resp_body))

airbyte-integrations/connectors/source-github/unit_tests/test_stream.py

@@ -1379,17 +1379,17 @@ def test_stream_contributor_activity_accepted_response(caplog, rate_limit_mock_r
         status=200,
     )
     responses.add(
-            responses.GET,
-            "https://api.github.com/repos/airbytehq/test_airbyte?per_page=100",
-            json={"full_name": "airbytehq/test_airbyte", "default_branch": "default_branch"},
-            status=200,
-        )
+        responses.GET,
+        "https://api.github.com/repos/airbytehq/test_airbyte?per_page=100",
+        json={"full_name": "airbytehq/test_airbyte", "default_branch": "default_branch"},
+        status=200,
+    )
     responses.add(
-                responses.GET,
-                "https://api.github.com/repos/airbytehq/test_airbyte/branches?per_page=100",
-                json={},
-                status=200,
-            )
+        responses.GET,
+        "https://api.github.com/repos/airbytehq/test_airbyte/branches?per_page=100",
+        json={},
+        status=200,
+    )
     resp = responses.add(
         responses.GET,
         "https://api.github.com/repos/airbytehq/test_airbyte/stats/contributors?per_page=100",
@@ -1401,9 +1401,14 @@ def test_stream_contributor_activity_accepted_response(caplog, rate_limit_mock_r
     configured_catalog = {
         "streams": [
             {
-                "stream": {"name": "contributor_activity", "json_schema": {}, "supported_sync_modes": ["full_refresh"],"source_defined_primary_key": [["id"]]},
+                "stream": {
+                    "name": "contributor_activity",
+                    "json_schema": {},
+                    "supported_sync_modes": ["full_refresh"],
+                    "source_defined_primary_key": [["id"]],
+                },
                 "sync_mode": "full_refresh",
-                "destination_sync_mode": "overwrite"
+                "destination_sync_mode": "overwrite",
             }
         ]
     }