🎉 Source Gitlab: add OAuth2.0 authentication support

airbyte-integrations/connectors/source-gitlab/Dockerfile

@@ -13,5 +13,5 @@ COPY main.py ./
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.9
+LABEL io.airbyte.version=1.0.0
 LABEL io.airbyte.name=airbyte/source-gitlab

airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml

@@ -2,19 +2,32 @@ connector_image: airbyte/source-gitlab:dev
 tests:
   spec:
     - spec_path: "source_gitlab/spec.json"
+      backward_compatibility_tests_config:
+        disable_for_version: "0.1.9"
   connection:
     - config_path: "secrets/config.json"
       status: "succeed"
+    - config_path: "secrets/config_oauth.json"
+      status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
       status: "failed"
   discovery:
     - config_path: "secrets/config.json"
+      backward_compatibility_tests_config:
+        disable_for_version: "0.1.9"
+    - config_path: "secrets/config_oauth.json"
+      backward_compatibility_tests_config:
+        disable_for_version: "0.1.9"
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
       empty_streams: ["group_issue_boards"]
       expect_records:
         path: "integration_tests/expected_records.txt"
+#     test read with OAuth2.0
+    - config_path: "secrets/config_oauth.json"
+      configured_catalog_path: "integration_tests/configured_catalog.json"
+      empty_streams: ["group_issue_boards"]
 # We cannot use these tests for testing Incremental, since for Gitlab the State is saved for each Project separately,
 # and the Acceptance Tests at this stage do not support this functionality.
 # Therefore, we hardcode the cursor_paths for our config.

airbyte-integrations/connectors/source-gitlab/integration_tests/invalid_config.json

@@ -1,7 +1,11 @@
 {
   "api_url": "gitlab.com",
   "private_token": "private_token-fake",
-  "groups": "new-group",
-  "projects": "new-ci-test",
-  "start_date": "2021-01-01T00:00:00Z"
-}
+  "start_date": "2021-01-01T00:00:00Z",
+  "groups": "new-group-airbute",
+  "projects": "new-group-airbute/new-ci-test-project",
+  "credentials": {
+      "auth_type": "access_token",
+      "access_token": "migrated_from_old_config"
+  }
+}

airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py

@@ -3,13 +3,16 @@
 #
 
 
-from typing import Any, List, Mapping, Tuple
+from typing import Any, List, MutableMapping, Tuple
 
+import pendulum
 import requests
 from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources import AbstractSource
 from airbyte_cdk.sources.streams import Stream
-from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator
+from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import SingleUseRefreshTokenOauth2Authenticator
+from airbyte_cdk.sources.streams.http.requests_native_auth.token import TokenAuthenticator
+from requests.auth import AuthBase
 
 from .streams import (
     Branches,
@@ -39,11 +42,47 @@
 )
 
 
-class SourceGitlab(AbstractSource):
-    def _generate_main_streams(self, config: Mapping[str, Any]) -> Tuple[GitlabStream, GitlabStream]:
-        auth = TokenAuthenticator(token=config["private_token"])
-        auth_params = dict(authenticator=auth, api_url=config["api_url"])
+class GitlabOauth2Authenticator(SingleUseRefreshTokenOauth2Authenticator):
+    def __init__(self, connector_config, *args, **kwargs):
+        super().__init__(connector_config, *args, **kwargs)
+        access_token = connector_config["credentials"].get("access_token")
+        if not access_token:
+            return
+        api_url = self._connector_config["api_url"]
+        token_valid, access_token_info = self.get_access_token_info(api_url, access_token)
+        if not token_valid:
+            return
+        now = pendulum.now()
+        self.access_token = access_token
+        self.set_token_expiry_date(now.add(seconds=access_token_info["expires_in"]))
+
+    @staticmethod
+    def get_access_token_info(api_url: str, access_token: str) -> Tuple[bool, MutableMapping]:
+        response = requests.get(f"https://{api_url}/oauth/token/info?access_token={access_token}")
+        if response.status_code == 200:
+            return True, response.json()
+        return False, {}
+
+
+def get_authenticator(config: MutableMapping) -> AuthBase:
+    if config["credentials"]["auth_type"] == "access_token":
+        return TokenAuthenticator(token=config["credentials"]["access_token"])
+    return GitlabOauth2Authenticator(config, token_refresh_endpoint=f"https://{config['api_url']}/oauth/token")
 
+
+class SourceGitlab(AbstractSource):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._auth_params = {}
+
+    def auth_params(self, config: MutableMapping[str, Any]) -> MutableMapping[str, Any]:
+        if not self._auth_params:
+            auth = get_authenticator(config)
+            self._auth_params = dict(authenticator=auth, api_url=config["api_url"])
+        return self._auth_params
+
+    def _generate_main_streams(self, config: MutableMapping[str, Any]) -> Tuple[GitlabStream, GitlabStream]:
+        auth_params = self.auth_params(config)
         pids = list(filter(None, config.get("projects", "").split(" ")))
         gids = config.get("groups")
 
@@ -90,10 +129,8 @@ def check_connection(self, logger, config) -> Tuple[bool, any]:
         except Exception as error:
             return False, f"Unable to connect to Gitlab API with the provided credentials - {repr(error)}"
 
-    def streams(self, config: Mapping[str, Any]) -> List[Stream]:
-        auth = TokenAuthenticator(token=config["private_token"])
-        auth_params = dict(authenticator=auth, api_url=config["api_url"])
-
+    def streams(self, config: MutableMapping[str, Any]) -> List[Stream]:
+        auth_params = self.auth_params(config)
         groups, projects = self._generate_main_streams(config)
         pipelines = Pipelines(parent_stream=projects, start_date=config["start_date"], **auth_params)
         merge_requests = MergeRequests(parent_stream=projects, start_date=config["start_date"], **auth_params)

airbyte-integrations/connectors/source-gitlab/source_gitlab/spec.json

@@ -2,23 +2,18 @@
   "documentationUrl": "https://docs.airbyte.com/integrations/sources/gitlab",
   "connectionSpecification": {
     "$schema": "http://json-schema.org/draft-07/schema#",
-    "title": "Source GitLab Singer Spec",
+    "title": "Source Gitlab Spec",
     "type": "object",
-    "required": ["api_url", "private_token", "start_date"],
+    "required": ["api_url", "start_date", "credentials"],
     "additionalProperties": true,
     "properties": {
       "api_url": {
         "type": "string",
         "examples": ["gitlab.com"],
         "title": "API URL",
+        "default": "gitlab.com",
         "description": "Please enter your basic URL from GitLab instance."
       },
-      "private_token": {
-        "type": "string",
-        "title": "Private Token",
-        "description": "Log into your GitLab account and then generate a personal Access Token.",
-        "airbyte_secret": true
-      },
       "groups": {
         "type": "string",
         "examples": ["airbyte.io"],
@@ -37,6 +32,113 @@
         "description": "The date from which you'd like to replicate data for GitLab API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.",
         "examples": ["2021-03-01T00:00:00Z"],
         "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"
+      },
+      "credentials": {
+        "title": "Authorization Method",
+        "type": "object",
+        "oneOf": [
+          {
+            "type": "object",
+            "title": "OAuth2.0",
+            "required": ["client_id", "client_secret", "refresh_token"],
+            "properties": {
+              "auth_type": {
+                "type": "string",
+                "const": "oauth2.0"
+              },
+              "client_id": {
+                "type": "string",
+                "description": "The API ID of the Gitlab developer application.",
+                "airbyte_secret": true
+              },
+              "client_secret": {
+                "type": "string",
+                "description": "The API Secret the Gitlab developer application.",
+                "airbyte_secret": true
+              },
+              "access_token": {
+                "type": "string",
+                "description": "Access Token for making authenticated requests.",
+                "airbyte_secret": true
+              },
+              "refresh_token": {
+                "type": "string",
+                "description": "The key to refresh the expired access_token.",
+                "airbyte_secret": true
+              }
+            }
+          },
+          {
+            "title": "Private Token",
+            "type": "object",
+            "required": ["access_token"],
+            "properties": {
+              "auth_type": {
+                "type": "string",
+                "const": "access_token"
+              },
+              "access_token": {
+                "type": "string",
+                "title": "Private Token",
+                "description": "Log into your Gitlab account and then generate a personal Access Token.",
+                "airbyte_secret": true
+              }
+            }
+          }
+        ]
+      }
+    }
+  },
+  "advanced_auth": {
+    "auth_flow_type": "oauth2.0",
+    "predicate_key": ["credentials", "auth_type"],
+    "predicate_value": "oauth2.0",
+    "oauth_config_specification": {
+      "oauth_user_input_from_connector_config_specification": {
+        "type": "object",
+        "properties": {
+          "domain": {
+            "type": "string",
+            "path_in_connector_config": ["api_url"]
+          }
+        }
+      },
+      "complete_oauth_output_specification": {
+        "type": "object",
+        "properties": {
+          "access_token": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "access_token"]
+          },
+          "refresh_token": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "refresh_token"]
+          }
+        }
+      },
+      "complete_oauth_server_input_specification": {
+        "type": "object",
+        "properties": {
+          "client_id": {
+            "type": "string"
+          },
+          "client_secret": {
+            "type": "string"
+          }
+        }
+      },
+      "complete_oauth_server_output_specification": {
+        "type": "object",
+        "properties": {
+          "client_id": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "client_id"]
+          },
+          "client_secret": {
+            "type": "string",
+            "path_in_connector_config": ["credentials", "client_secret"]
+          }
+        }
       }
     }
   }

airbyte-oauth/src/main/java/io/airbyte/oauth/OAuthImplementationFactory.java

@@ -52,6 +52,7 @@ public OAuthImplementationFactory(final ConfigRepository configRepository, final
         .put("airbyte/source-strava", new StravaOAuthFlow(configRepository, httpClient))
         .put("airbyte/source-surveymonkey", new SurveymonkeyOAuthFlow(configRepository, httpClient))
         .put("airbyte/source-trello", new TrelloOAuthFlow(configRepository))
+        .put("airbyte/source-gitlab", new GitlabOAuthFlow(configRepository, httpClient))
         .put("airbyte/source-youtube-analytics", new YouTubeAnalyticsOAuthFlow(configRepository, httpClient))
         // revert me
         .put("airbyte/source-youtube-analytics-business", new YouTubeAnalyticsBusinessOAuthFlow(configRepository, httpClient))