✨Source GitHub: add new stream Contributor Activity (#30615)

Co-authored-by: artem1205 <[email protected]>

Author: artem1205

airbyte-integrations/connectors/source-github/Dockerfile

@@ -12,5 +12,5 @@ RUN pip install .
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=1.0.4
+LABEL io.airbyte.version=1.1.0
 LABEL io.airbyte.name=airbyte/source-github

airbyte-integrations/connectors/source-github/acceptance-test-config.yml

@@ -34,6 +34,11 @@ acceptance_tests:
       - name: "events"
         bypass_reason: "Only events created within the past 90 days can be showed"
       ignored_fields:
+        contributor_activity:
+        - name: weeks
+          bypass_reason: "depend on changing data"
+        - name: total
+          bypass_reason: "depend on changing data"
         workflows:
         - name: created_at
           bypass_reason: value may be returned in different time zones
@@ -84,6 +89,7 @@ acceptance_tests:
       configured_catalog_path: "integration_tests/configured_catalog.json"
       future_state:
         future_state_path: "integration_tests/abnormal_state.json"
+      skip_comprehensive_incremental_tests: true
   full_refresh:
     tests:
     - config_path: "secrets/config.json"

airbyte-integrations/connectors/source-github/integration_tests/configured_catalog.json

@@ -79,6 +79,16 @@
       "destination_sync_mode": "append",
       "cursor_field": ["created_at"]
     },
+    {
+      "stream": {
+        "name": "contributor_activity",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite"
+    },
     {
       "stream": {
         "name": "deployments",

airbyte-integrations/connectors/source-github/integration_tests/expected_records.jsonl

@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
-  dockerImageTag: 1.0.4
+  dockerImageTag: 1.1.0
   maxSecondsBetweenMessages: 5400
   dockerRepository: airbyte/source-github
   githubIssueLabel: source-github

airbyte-integrations/connectors/source-github/metadata.yaml

@@ -0,0 +1,108 @@
+{
+  "$schema": "https://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "title": "Contributor Activity",
+  "properties": {
+    "name": {
+      "type": ["null", "string"]
+    },
+    "email": {
+      "type": ["string", "null"]
+    },
+    "login": {
+      "type": ["null", "string"]
+    },
+    "id": {
+      "type": ["null", "integer"]
+    },
+    "node_id": {
+      "type": ["null", "string"]
+    },
+    "avatar_url": {
+      "type": ["null", "string"],
+      "format": "uri"
+    },
+    "gravatar_id": {
+      "type": ["string", "null"]
+    },
+    "url": {
+      "type": ["null", "string"],
+      "format": "uri"
+    },
+    "html_url": {
+      "type": ["null", "string"],
+      "format": "uri"
+    },
+    "followers_url": {
+      "type": ["null", "string"],
+      "format": "uri"
+    },
+    "following_url": {
+      "type": ["null", "string"]
+    },
+    "gists_url": {
+      "type": ["null", "string"]
+    },
+    "starred_url": {
+      "type": ["null", "string"]
+    },
+    "subscriptions_url": {
+      "type": ["null", "string"],
+      "format": "uri"
+    },
+    "organizations_url": {
+      "type": ["null", "string"],
+      "format": "uri"
+    },
+    "repos_url": {
+      "type": ["null", "string"],
+      "format": "uri"
+    },
+    "events_url": {
+      "type": ["null", "string"]
+    },
+    "repository": {
+      "type": ["null", "string"]
+    },
+    "received_events_url": {
+      "type": ["null", "string"],
+      "format": "uri"
+    },
+    "type": {
+      "type": ["null", "string"]
+    },
+    "site_admin": {
+      "type": ["null", "boolean"]
+    },
+    "starred_at": {
+      "type": ["null", "string"]
+    },
+    "total": {
+      "type": ["null", "integer"]
+    },
+    "weeks": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "w": {
+            "type": ["null", "integer"],
+            "description": "Start of the week, given as a Unix timestamp."
+          },
+          "a": {
+            "type": ["null", "integer"],
+            "description": "Number of additions"
+          },
+          "d": {
+            "type": ["null", "integer"],
+            "description": "Number of deletions"
+          },
+          "c": {
+            "type": ["null", "integer"],
+            "description": "Number of commits"
+          }
+        }
+      }
+    }
+  }
+}

airbyte-integrations/connectors/source-github/source_github/schemas/contributor_activity.json

@@ -21,6 +21,7 @@
     CommitCommentReactions,
     CommitComments,
     Commits,
+    ContributorActivity,
     Deployments,
     Events,
     IssueCommentReactions,
@@ -259,6 +260,7 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]:
             CommitCommentReactions(**repository_args_with_start_date),
             CommitComments(**repository_args_with_start_date),
             Commits(**repository_args_with_start_date, branches_to_pull=branches_to_pull, default_branches=default_branches),
+            ContributorActivity(**repository_args),
             Deployments(**repository_args_with_start_date),
             Events(**repository_args_with_start_date),
             IssueCommentReactions(**repository_args_with_start_date),

airbyte-integrations/connectors/source-github/source_github/source.py

@@ -205,9 +205,7 @@ def request_params(
 
     def request_headers(self, **kwargs) -> Mapping[str, Any]:
         # Without sending `User-Agent` header we will be getting `403 Client Error: Forbidden for url` error.
-        return {
-            "User-Agent": "PostmanRuntime/7.28.0",
-        }
+        return {"User-Agent": "PostmanRuntime/7.28.0"}
 
     def parse_response(
         self,
@@ -1543,3 +1541,33 @@ def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str,
         record["team_slug"] = stream_slice["team_slug"]
         record["username"] = stream_slice["username"]
         return record
+
+
+class ContributorActivity(GithubStream):
+    """
+    API docs: https://docs.github.com/en/rest/metrics/statistics?apiVersion=2022-11-28#get-all-contributor-commit-activity
+    """
+
+    def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str:
+        return f"repos/{stream_slice['repository']}/stats/contributors"
+
+    def request_headers(self, **kwargs) -> Mapping[str, Any]:
+        params = super().request_headers(**kwargs)
+        params.update({"Accept": "application/vnd.github+json", "X-GitHub-Api-Version": "2022-11-28"})
+        return params
+
+    def transform(self, record: MutableMapping[str, Any], stream_slice: Mapping[str, Any]) -> MutableMapping[str, Any]:
+        record["repository"] = stream_slice["repository"]
+        record.update(record.pop("author"))
+        return record
+
+    def should_retry(self, response: requests.Response) -> bool:
+        """
+        If the data hasn't been cached when you query a repository's statistics, you'll receive a 202 response, need to retry to get results
+        see for more info https://docs.github.com/en/rest/metrics/statistics?apiVersion=2022-11-28#a-word-about-caching
+        """
+        if super().should_retry(response) or response.status_code == requests.codes.ACCEPTED:
+            return True
+
+    def backoff_time(self, response: requests.Response) -> Optional[float]:
+        return 10 if response.status_code == requests.codes.ACCEPTED else super().backoff_time(response)

airbyte-integrations/connectors/source-github/source_github/streams.py

@@ -74,6 +74,7 @@ This connector outputs the following full refresh streams:
 
 - [Assignees](https://docs.github.com/en/rest/reference/issues#list-assignees)
 - [Branches](https://docs.github.com/en/rest/reference/repos#list-branches)
+- [Contributor Activity](https://docs.github.com/en/rest/metrics/statistics?apiVersion=2022-11-28#get-all-contributor-commit-activity)
 - [Collaborators](https://docs.github.com/en/rest/reference/repos#list-repository-collaborators)
 - [Issue labels](https://docs.github.com/en/rest/issues/labels#list-labels-for-a-repository)
 - [Organizations](https://docs.github.com/en/rest/reference/orgs#get-an-organization)
@@ -163,9 +164,10 @@ The GitHub connector should not run into GitHub API limitations under normal usa
 
 | Version | Date       | Pull Request                                                                                                      | Subject                                                                                                                                                             |
 |:--------|:-----------|:------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| 1.0.4   | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031) | Reverted `advancedAuth` spec changes  |
-| 1.0.3   | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910) | Updated `advancedAuth` broken references  |
-| 1.0.2   | 2023-07-11 | [28144](https://github.com/airbytehq/airbyte/pull/28144)                                                          | Add `archived_at` property to `Organizations` schema parameter                                                                                                                               |
+| 1.1.0   | 2023-08-03 | [30615](https://github.com/airbytehq/airbyte/pull/30615)                                                          | Add new stream `Contributor Activity`                                                                                                                               |
+| 1.0.4   | 2023-08-03 | [29031](https://github.com/airbytehq/airbyte/pull/29031)                                                          | Reverted `advancedAuth` spec changes                                                                                                                                |
+| 1.0.3   | 2023-08-01 | [28910](https://github.com/airbytehq/airbyte/pull/28910)                                                          | Updated `advancedAuth` broken references                                                                                                                            |
+| 1.0.2   | 2023-07-11 | [28144](https://github.com/airbytehq/airbyte/pull/28144)                                                          | Add `archived_at` property to `Organizations` schema parameter                                                                                                      |
 | 1.0.1   | 2023-05-22 | [25838](https://github.com/airbytehq/airbyte/pull/25838)                                                          | Deprecate "page size" input parameter                                                                                                                               |
 | 1.0.0   | 2023-05-19 | [25778](https://github.com/airbytehq/airbyte/pull/25778)                                                          | Improve repo(s) name validation on UI                                                                                                                               |
 | 0.5.0   | 2023-05-16 | [25793](https://github.com/airbytehq/airbyte/pull/25793)                                                          | Implement client-side throttling of requests                                                                                                                        |