🎉 Source Github: IssueReactions - re-implemented using GraphQL (#21457)

Signed-off-by: Sergey Chvalyuk <[email protected]>

Author: grubberr

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -597,7 +597,7 @@
 - name: GitHub
   sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
   dockerRepository: airbyte/source-github
-  dockerImageTag: 0.3.11
+  dockerImageTag: 0.4.0
   documentationUrl: https://docs.airbyte.com/integrations/sources/github
   icon: github.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -4705,7 +4705,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-github:0.3.11"
+- dockerImage: "airbyte/source-github:0.4.0"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/github"
     connectionSpecification:

airbyte-integrations/connectors/source-github/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.3.12
+LABEL io.airbyte.version=0.4.0
 LABEL io.airbyte.name=airbyte/source-github

airbyte-integrations/connectors/source-github/acceptance-test-config.yml

@@ -47,7 +47,7 @@ acceptance_tests:
             ["airbytehq/integration-test", "907296275", "created_at"]
           issue_events: ["airbytehq/integration-test", "created_at"]
           issue_milestones: ["airbytehq/integration-test", "updated_at"]
-          issue_reactions: ["airbytehq/integration-test", "11", "created_at"]
+          issue_reactions: ["airbytehq/integration-test", "created_at"]
           issues: ["airbytehq/integration-test", "updated_at"]
           project_cards:
             ["airbytehq/integration-test", "13167124", "17807006", "updated_at"]

airbyte-integrations/connectors/source-github/integration_tests/abnormal_state.json

@@ -89,9 +89,7 @@
     "type": "STREAM",
     "stream": {
       "stream_state": {
-        "airbytehq/integration-test": {
-          "11": { "created_at": "2121-12-31T23:59:59Z" }
-        }
+        "airbytehq/integration-test": { "created_at": "2121-12-31T23:59:59Z" }
       },
       "stream_descriptor": { "name": "issue_reactions" }
     }

airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl

@@ -46,8 +46,8 @@
 {"stream":"issue_labels","data":{"id":3300346197,"node_id":"MDU6TGFiZWwzMzAwMzQ2MTk3","url":"https://api.github.com/repos/airbytehq/integration-test/labels/critical","name":"critical","color":"ededed","default":false,"description":null,"repository":"airbytehq/integration-test"},"emitted_at":1673559178152}
 {"stream":"issue_milestones","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/milestones/4","html_url":"https://github.com/airbytehq/integration-test/milestone/4","labels_url":"https://api.github.com/repos/airbytehq/integration-test/milestones/4/labels","id":7786502,"node_id":"MI_kwDOF9hP9c4AdtAG","number":4,"title":"m2","description":"","creator":{"login":"airbyteio","id":92915184,"node_id":"U_kgDOBYnF8A","avatar_url":"https://avatars.githubusercontent.com/u/92915184?v=4","gravatar_id":"","url":"https://api.github.com/users/airbyteio","html_url":"https://github.com/airbyteio","followers_url":"https://api.github.com/users/airbyteio/followers","following_url":"https://api.github.com/users/airbyteio/following{/other_user}","gists_url":"https://api.github.com/users/airbyteio/gists{/gist_id}","starred_url":"https://api.github.com/users/airbyteio/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/airbyteio/subscriptions","organizations_url":"https://api.github.com/users/airbyteio/orgs","repos_url":"https://api.github.com/users/airbyteio/repos","events_url":"https://api.github.com/users/airbyteio/events{/privacy}","received_events_url":"https://api.github.com/users/airbyteio/received_events","type":"User","site_admin":false},"open_issues":0,"closed_issues":0,"state":"open","created_at":"2022-03-21T08:37:46Z","updated_at":"2022-03-21T08:37:46Z","due_on":null,"closed_at":null,"repository":"airbytehq/integration-test"},"emitted_at":1673559253517}
 {"stream":"issue_milestones","data":{"url":"https://api.github.com/repos/airbytehq/integration-test/milestones/1","html_url":"https://github.com/airbytehq/integration-test/milestone/1","labels_url":"https://api.github.com/repos/airbytehq/integration-test/milestones/1/labels","id":7097357,"node_id":"MI_kwDOF9hP9c4AbEwN","number":1,"title":"main","description":null,"creator":{"login":"gaart","id":743901,"node_id":"MDQ6VXNlcjc0MzkwMQ==","avatar_url":"https://avatars.githubusercontent.com/u/743901?v=4","gravatar_id":"","url":"https://api.github.com/users/gaart","html_url":"https://github.com/gaart","followers_url":"https://api.github.com/users/gaart/followers","following_url":"https://api.github.com/users/gaart/following{/other_user}","gists_url":"https://api.github.com/users/gaart/gists{/gist_id}","starred_url":"https://api.github.com/users/gaart/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/gaart/subscriptions","organizations_url":"https://api.github.com/users/gaart/orgs","repos_url":"https://api.github.com/users/gaart/repos","events_url":"https://api.github.com/users/gaart/events{/privacy}","received_events_url":"https://api.github.com/users/gaart/received_events","type":"User","site_admin":false},"open_issues":3,"closed_issues":1,"state":"open","created_at":"2021-08-27T15:43:44Z","updated_at":"2021-08-27T16:02:49Z","due_on":null,"closed_at":null,"repository":"airbytehq/integration-test"},"emitted_at":1673559253517}
-{"stream":"issue_reactions","data":{"id":127048454,"node_id":"MDEzOklzc3VlUmVhY3Rpb24xMjcwNDg0NTQ=","user":{"login":"yevhenii-ldv","id":34103125,"node_id":"MDQ6VXNlcjM0MTAzMTI1","avatar_url":"https://avatars.githubusercontent.com/u/34103125?u=3e49bb73177a9f70896e3d49b34656ab659c70a5&v=4","gravatar_id":"","url":"https://api.github.com/users/yevhenii-ldv","html_url":"https://github.com/yevhenii-ldv","followers_url":"https://api.github.com/users/yevhenii-ldv/followers","following_url":"https://api.github.com/users/yevhenii-ldv/following{/other_user}","gists_url":"https://api.github.com/users/yevhenii-ldv/gists{/gist_id}","starred_url":"https://api.github.com/users/yevhenii-ldv/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yevhenii-ldv/subscriptions","organizations_url":"https://api.github.com/users/yevhenii-ldv/orgs","repos_url":"https://api.github.com/users/yevhenii-ldv/repos","events_url":"https://api.github.com/users/yevhenii-ldv/events{/privacy}","received_events_url":"https://api.github.com/users/yevhenii-ldv/received_events","type":"User","site_admin":false},"content":"eyes","created_at":"2021-09-06T11:13:31Z","repository":"airbytehq/integration-test","issue_number":11},"emitted_at":1673559327732}
-{"stream":"issue_reactions","data":{"id":127048456,"node_id":"MDEzOklzc3VlUmVhY3Rpb24xMjcwNDg0NTY=","user":{"login":"yevhenii-ldv","id":34103125,"node_id":"MDQ6VXNlcjM0MTAzMTI1","avatar_url":"https://avatars.githubusercontent.com/u/34103125?u=3e49bb73177a9f70896e3d49b34656ab659c70a5&v=4","gravatar_id":"","url":"https://api.github.com/users/yevhenii-ldv","html_url":"https://github.com/yevhenii-ldv","followers_url":"https://api.github.com/users/yevhenii-ldv/followers","following_url":"https://api.github.com/users/yevhenii-ldv/following{/other_user}","gists_url":"https://api.github.com/users/yevhenii-ldv/gists{/gist_id}","starred_url":"https://api.github.com/users/yevhenii-ldv/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yevhenii-ldv/subscriptions","organizations_url":"https://api.github.com/users/yevhenii-ldv/orgs","repos_url":"https://api.github.com/users/yevhenii-ldv/repos","events_url":"https://api.github.com/users/yevhenii-ldv/events{/privacy}","received_events_url":"https://api.github.com/users/yevhenii-ldv/received_events","type":"User","site_admin":false},"content":"rocket","created_at":"2021-09-06T11:13:32Z","repository":"airbytehq/integration-test","issue_number":11},"emitted_at":1673559327732}
+{"stream":"issue_reactions","data":{"node_id":"MDEzOklzc3VlUmVhY3Rpb24xMjcwNDg0NTQ=","id":127048454,"content":"EYES","created_at":"2021-09-06T11:13:31Z","user":{"node_id":"MDQ6VXNlcjM0MTAzMTI1","id":34103125,"login":"yevhenii-ldv","avatar_url":"https://avatars.githubusercontent.com/u/34103125?u=3e49bb73177a9f70896e3d49b34656ab659c70a5&v=4","html_url":"https://github.com/yevhenii-ldv","site_admin":false,"type":"User"},"repository":"airbytehq/integration-test","issue_number":11},"emitted_at":1674174073303}
+{"stream":"issue_reactions","data":{"node_id":"MDEzOklzc3VlUmVhY3Rpb24xMjcwNDg0NTY=","id":127048456,"content":"ROCKET","created_at":"2021-09-06T11:13:32Z","user":{"node_id":"MDQ6VXNlcjM0MTAzMTI1","id":34103125,"login":"yevhenii-ldv","avatar_url":"https://avatars.githubusercontent.com/u/34103125?u=3e49bb73177a9f70896e3d49b34656ab659c70a5&v=4","html_url":"https://github.com/yevhenii-ldv","site_admin":false,"type":"User"},"repository":"airbytehq/integration-test","issue_number":11},"emitted_at":1674174073303}
 {"stream":"issue_comment_reactions","data":{"id":127042034,"node_id":"MDIwOklzc3VlQ29tbWVudFJlYWN0aW9uMTI3MDQyMDM0","user":{"login":"yevhenii-ldv","id":34103125,"node_id":"MDQ6VXNlcjM0MTAzMTI1","avatar_url":"https://avatars.githubusercontent.com/u/34103125?v=4","gravatar_id":"","url":"https://api.github.com/users/yevhenii-ldv","html_url":"https://github.com/yevhenii-ldv","followers_url":"https://api.github.com/users/yevhenii-ldv/followers","following_url":"https://api.github.com/users/yevhenii-ldv/following{/other_user}","gists_url":"https://api.github.com/users/yevhenii-ldv/gists{/gist_id}","starred_url":"https://api.github.com/users/yevhenii-ldv/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yevhenii-ldv/subscriptions","organizations_url":"https://api.github.com/users/yevhenii-ldv/orgs","repos_url":"https://api.github.com/users/yevhenii-ldv/repos","events_url":"https://api.github.com/users/yevhenii-ldv/events{/privacy}","received_events_url":"https://api.github.com/users/yevhenii-ldv/received_events","type":"User","site_admin":false},"content":"hooray","created_at":"2021-09-06T10:18:19Z","repository":"airbytehq/integration-test","comment_id":907296275},"emitted_at":1673559403000}
 {"stream":"issue_comment_reactions","data":{"id":127042037,"node_id":"MDIwOklzc3VlQ29tbWVudFJlYWN0aW9uMTI3MDQyMDM3","user":{"login":"yevhenii-ldv","id":34103125,"node_id":"MDQ6VXNlcjM0MTAzMTI1","avatar_url":"https://avatars.githubusercontent.com/u/34103125?v=4","gravatar_id":"","url":"https://api.github.com/users/yevhenii-ldv","html_url":"https://github.com/yevhenii-ldv","followers_url":"https://api.github.com/users/yevhenii-ldv/followers","following_url":"https://api.github.com/users/yevhenii-ldv/following{/other_user}","gists_url":"https://api.github.com/users/yevhenii-ldv/gists{/gist_id}","starred_url":"https://api.github.com/users/yevhenii-ldv/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/yevhenii-ldv/subscriptions","organizations_url":"https://api.github.com/users/yevhenii-ldv/orgs","repos_url":"https://api.github.com/users/yevhenii-ldv/repos","events_url":"https://api.github.com/users/yevhenii-ldv/events{/privacy}","received_events_url":"https://api.github.com/users/yevhenii-ldv/received_events","type":"User","site_admin":false},"content":"laugh","created_at":"2021-09-06T10:18:20Z","repository":"airbytehq/integration-test","comment_id":907296275},"emitted_at":1673559403000}
 {"stream":"commit_comment_reactions","data":{"id":154935456,"node_id":"REA_lADOF9hP9c4DT3SJzgk8IKA","user":{"login":"grubberr","id":195743,"node_id":"MDQ6VXNlcjE5NTc0Mw==","avatar_url":"https://avatars.githubusercontent.com/u/195743?v=4","gravatar_id":"","url":"https://api.github.com/users/grubberr","html_url":"https://github.com/grubberr","followers_url":"https://api.github.com/users/grubberr/followers","following_url":"https://api.github.com/users/grubberr/following{/other_user}","gists_url":"https://api.github.com/users/grubberr/gists{/gist_id}","starred_url":"https://api.github.com/users/grubberr/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/grubberr/subscriptions","organizations_url":"https://api.github.com/users/grubberr/orgs","repos_url":"https://api.github.com/users/grubberr/repos","events_url":"https://api.github.com/users/grubberr/events{/privacy}","received_events_url":"https://api.github.com/users/grubberr/received_events","type":"User","site_admin":false},"content":"heart","created_at":"2022-03-20T11:30:23Z","repository":"airbytehq/integration-test","comment_id":55538825},"emitted_at":1673560627726}

airbyte-integrations/connectors/source-github/source_github/graphql.py

@@ -99,6 +99,37 @@ def get_query_reviews(owner, name, first, after, number=None):
     return str(op)
 
 
+def get_query_issue_reactions(owner, name, first, after, number=None):
+    op = sgqlc.operation.Operation(_schema_root.query_type)
+    repository = op.repository(owner=owner, name=name)
+    repository.name()
+    repository.owner.login()
+    if number:
+        issue = repository.issue(number=number)
+    else:
+        kwargs = {"first": first}
+        if after:
+            kwargs["after"] = after
+        issues = repository.issues(**kwargs)
+        issues.page_info.__fields__(has_next_page=True, end_cursor=True)
+        issue = issues.nodes
+
+    issue.__fields__(number=True)
+    kwargs = {"first": first}
+    if number and after:
+        kwargs["after"] = after
+    reactions = issue.reactions(**kwargs)
+    reactions.page_info.__fields__(has_next_page=True, end_cursor=True)
+    reactions.nodes.__fields__(
+        id="node_id",
+        database_id="id",
+        content=True,
+        created_at="created_at",
+    )
+    select_user_fields(reactions.nodes.user())
+    return str(op)
+
+
 class QueryReactions:
 
     # AVERAGE_REVIEWS - optimal number of reviews to fetch inside every pull request.

airbyte-integrations/connectors/source-github/source_github/schemas/issue_reactions.json

@@ -16,7 +16,7 @@
       "format": "date-time"
     },
     "user": {
-      "$ref": "user.json"
+      "$ref": "user_graphql.json"
     },
     "repository": {
       "type": "string"

airbyte-integrations/connectors/source-github/source_github/streams.py

@@ -14,7 +14,7 @@
 from airbyte_cdk.sources.streams.http.exceptions import DefaultBackoffException
 from requests.exceptions import HTTPError
 
-from .graphql import CursorStorage, QueryReactions, get_query_pull_requests, get_query_reviews
+from .graphql import CursorStorage, QueryReactions, get_query_issue_reactions, get_query_pull_requests, get_query_reviews
 from .utils import getter
 
 DEFAULT_PAGE_SIZE = 100
@@ -1002,14 +1002,83 @@ class IssueCommentReactions(ReactionStream):
     parent_entity = Comments
 
 
-class IssueReactions(ReactionStream):
+class IssueReactions(SemiIncrementalMixin, GithubStream):
     """
-    API docs: https://docs.github.com/en/rest/reference/reactions#list-reactions-for-an-issue
+    https://docs.github.com/en/graphql/reference/objects#issue
+    https://docs.github.com/en/graphql/reference/objects#reaction
     """
 
-    parent_entity = Issues
-    parent_key = "number"
-    copy_parent_key = "issue_number"
+    http_method = "POST"
+    cursor_field = "created_at"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.issues_cursor = {}
+        self.reactions_cursors = {}
+
+    def path(
+        self, *, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
+    ) -> str:
+        return "graphql"
+
+    def raise_error_from_response(self, response_json):
+        if "errors" in response_json:
+            raise Exception(str(response_json["errors"]))
+
+    def _get_name(self, repository):
+        return repository["owner"]["login"] + "/" + repository["name"]
+
+    def _get_reactions_from_issue(self, issue, repository_name):
+        for reaction in issue["reactions"]["nodes"]:
+            reaction["repository"] = repository_name
+            reaction["issue_number"] = issue["number"]
+            reaction["user"]["type"] = "User"
+            yield reaction
+
+    def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]:
+        self.raise_error_from_response(response_json=response.json())
+        repository = response.json()["data"]["repository"]
+        if repository:
+            repository_name = self._get_name(repository)
+            if "issues" in repository:
+                for issue in repository["issues"]["nodes"]:
+                    yield from self._get_reactions_from_issue(issue, repository_name)
+            elif "issue" in repository:
+                yield from self._get_reactions_from_issue(repository["issue"], repository_name)
+
+    def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]:
+        repository = response.json()["data"]["repository"]
+        if repository:
+            repository_name = self._get_name(repository)
+            reactions_cursors = self.reactions_cursors.setdefault(repository_name, {})
+            if "issues" in repository:
+                if repository["issues"]["pageInfo"]["hasNextPage"]:
+                    self.issues_cursor[repository_name] = repository["issues"]["pageInfo"]["endCursor"]
+                for issue in repository["issues"]["nodes"]:
+                    if issue["reactions"]["pageInfo"]["hasNextPage"]:
+                        issue_number = issue["number"]
+                        reactions_cursors[issue_number] = issue["reactions"]["pageInfo"]["endCursor"]
+            elif "issue" in repository:
+                if repository["issue"]["reactions"]["pageInfo"]["hasNextPage"]:
+                    issue_number = repository["issue"]["number"]
+                    reactions_cursors[issue_number] = repository["issue"]["reactions"]["pageInfo"]["endCursor"]
+            if reactions_cursors:
+                number, after = reactions_cursors.popitem()
+                return {"after": after, "number": number}
+            if repository_name in self.issues_cursor:
+                return {"after": self.issues_cursor.pop(repository_name)}
+
+    def request_body_json(
+        self,
+        stream_state: Mapping[str, Any],
+        stream_slice: Mapping[str, Any] = None,
+        next_page_token: Mapping[str, Any] = None,
+    ) -> Optional[Mapping]:
+        organization, name = stream_slice["repository"].split("/")
+        if not next_page_token:
+            next_page_token = {"after": None}
+        query = get_query_issue_reactions(owner=organization, name=name, first=self.page_size, **next_page_token)
+        return {"query": query}
 
 
 class PullRequestCommentReactions(SemiIncrementalMixin, GithubStream):