🏥 Source Gitlab: increase test coverage, update Groups, Commits and Projects schemas. (airbytehq#33676)

Author: darynaishchenko

airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl

@@ -10,7 +10,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80
-  dockerImageTag: 2.0.0
+  dockerImageTag: 2.1.0
   dockerRepository: airbyte/source-gitlab
   documentationUrl: https://docs.airbyte.com/integrations/sources/gitlab
   githubIssueLabel: source-gitlab

airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl

@@ -37,6 +37,17 @@
       "type": ["null", "string"],
       "format": "date-time"
     },
+    "extended_trailers": {
+      "type": ["null", "object"],
+      "properties": {
+        "Cc": {
+          "type": ["null", "array"],
+          "items": {
+            "type": ["null", "string"]
+          }
+        }
+      }
+    },
     "committer_name": {
       "type": ["null", "string"]
     },

airbyte-integrations/connectors/source-gitlab/metadata.yaml

@@ -19,6 +19,9 @@
     "id": {
       "type": ["null", "integer"]
     },
+    "organization_id": {
+      "type": ["null", "integer"]
+    },
     "default_branch_protection_defaults": {
       "type": ["null", "object"],
       "properties": {
@@ -74,6 +77,9 @@
     "emails_disabled": {
       "type": ["null", "boolean"]
     },
+    "emails_enabled": {
+      "type": ["null", "boolean"]
+    },
     "mentions_disabled": {
       "type": ["null", "boolean"]
     },
@@ -144,6 +150,9 @@
     },
     "shared_runners_setting": {
       "type": ["null", "string"]
+    },
+    "service_access_tokens_expiration_enforced": {
+      "type": ["null", "boolean"]
     }
   }
 }

airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/commits.json

@@ -235,20 +235,23 @@
       }
     },
     "epic": {
-      "id": {
-        "type": ["null", "integer"]
-      },
-      "iid": {
-        "type": ["null", "integer"]
-      },
-      "title": {
-        "type": ["null", "string"]
-      },
-      "url": {
-        "type": ["null", "string"]
-      },
-      "group_id": {
-        "type": ["null", "integer"]
+      "type": ["null", "object"],
+      "properties": {
+        "id": {
+          "type": ["null", "integer"]
+        },
+        "iid": {
+          "type": ["null", "integer"]
+        },
+        "title": {
+          "type": ["null", "string"]
+        },
+        "url": {
+          "type": ["null", "string"]
+        },
+        "group_id": {
+          "type": ["null", "integer"]
+        }
       }
     },
     "epic_iid": {

airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/groups.json

@@ -11,6 +11,9 @@
     "stage": {
       "type": ["null", "string"]
     },
+    "archived": {
+      "type": ["null", "boolean"]
+    },
     "name": {
       "type": ["null", "string"]
     },

airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/issues.json

@@ -483,6 +483,15 @@
     },
     "merge_trains_skip_train_allowed": {
       "type": ["null", "boolean"]
+    },
+    "code_suggestions": {
+      "type": ["null", "boolean"]
+    },
+    "model_registry_access_level": {
+      "type": ["null", "string"]
+    },
+    "ci_restrict_pipeline_cancellation_role": {
+      "type": ["null", "string"]
     }
   }
 }

airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/jobs.json

@@ -92,7 +92,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
         elif isinstance(response_data, dict):
             yield self.transform(response_data, **kwargs)
         else:
-            Exception(f"Unsupported type of response data for stream {self.name}")
+            self.logger.info(f"Unsupported type of response data for stream {self.name}")
 
     def transform(self, record: Dict[str, Any], stream_slice: Mapping[str, Any] = None, **kwargs):
         for key in self.flatten_id_keys:
@@ -166,7 +166,7 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late
             current_state = current_state.get(self.cursor_field)
         current_state_value = current_state or latest_cursor_value
         max_value = max(pendulum.parse(current_state_value), pendulum.parse(latest_cursor_value))
-        current_stream_state[str(project_id)] = {self.cursor_field: str(max_value)}
+        current_stream_state[str(project_id)] = {self.cursor_field: max_value.to_iso8601_string()}
         return current_stream_state
 
     @staticmethod

airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json

@@ -0,0 +1 @@
+{ "groups": "a b c", "groups_list": ["a", "c", "b"] }

airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py

@@ -0,0 +1,21 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+import os
+
+from source_gitlab.config_migrations import MigrateGroups
+from source_gitlab.source import SourceGitlab
+
+TEST_CONFIG_PATH = f"{os.path.dirname(__file__)}/test_config.json"
+
+
+def test_should_migrate():
+    assert MigrateGroups._should_migrate({"groups": "group group2 group3"}) is True
+    assert MigrateGroups._should_migrate({"groups_list": ["test", "group2", "group3"]}) is False
+
+
+def test__modify_and_save():
+    source = SourceGitlab()
+    expected = {"groups": "a b c", "groups_list": ["b", "c", "a"]}
+    modified_config = MigrateGroups._modify_and_save(config_path=TEST_CONFIG_PATH, source=source, config={"groups": "a b c"})
+    assert modified_config["groups_list"].sort() == expected["groups_list"].sort()
+    assert modified_config.get("groups")

airbyte-integrations/connectors/source-gitlab/unit_tests/test_config.json

@@ -64,12 +64,39 @@ def test_connection_fail_due_to_api_error(errror_code, expected_status, config,
     assert msg.startswith("Unable to connect to Gitlab API with the provided Private Access Token")
 
 
+def test_connection_fail_due_to_api_error_oauth(oauth_config, mocker, requests_mock):
+    mocker.patch("time.sleep")
+    test_response = {
+        "access_token": "new_access_token",
+        "expires_in": 7200,
+        "created_at": 1735689600,
+        # (7200 + 1735689600).timestamp().to_rfc3339_string() = "2025-01-01T02:00:00+00:00"
+        "refresh_token": "new_refresh_token",
+    }
+    requests_mock.post("https://gitlab.com/oauth/token", status_code=200, json=test_response)
+    requests_mock.get("/api/v4/groups", status_code=500)
+    source = SourceGitlab()
+    status, msg = source.check_connection(logging.getLogger(), oauth_config)
+    assert status is False
+    assert msg.startswith("Unable to connect to Gitlab API with the provided credentials")
+
+
 def test_connection_fail_due_to_expired_access_token_error(oauth_config, requests_mock):
-    expected = "Unable to refresh the `access_token`, please re-auth in Source > Settings."
+    expected = "Unable to refresh the `access_token`, please re-authenticate in Sources > Settings."
     requests_mock.post("https://gitlab.com/oauth/token", status_code=401)
     source = SourceGitlab()
     status, msg = source.check_connection(logging.getLogger("airbyte"), oauth_config)
-    assert status is False, expected in msg
+    assert status is False
+    assert expected in msg
+
+
+def test_connection_refresh_access_token(oauth_config, requests_mock):
+    expected = "Unknown error occurred while checking the connection"
+    requests_mock.post("https://gitlab.com/oauth/token", status_code=200, json={"access_token": "new access token"})
+    source = SourceGitlab()
+    status, msg = source.check_connection(logging.getLogger("airbyte"), oauth_config)
+    assert status is False
+    assert expected in msg
 
 
 def test_refresh_expired_access_token_on_error(oauth_config, requests_mock):
@@ -108,3 +135,27 @@ def test_connection_fail_due_to_config_error(mocker, api_url, deployment_env, ex
     }
     status, msg = source.check_connection(logging.getLogger(), config)
     assert (status, msg) == (False, expected_message)
+
+
+def test_try_refresh_access_token(oauth_config, requests_mock):
+    test_response = {
+        "access_token": "new_access_token",
+        "expires_in": 7200,
+        "created_at": 1735689600,
+        # (7200 + 1735689600).timestamp().to_rfc3339_string() = "2025-01-01T02:00:00+00:00"
+        "refresh_token": "new_refresh_token",
+    }
+    requests_mock.post("https://gitlab.com/oauth/token", status_code=200, json=test_response)
+
+    expected = {"api_url": "gitlab.com",
+                "credentials": {"access_token": "new_access_token",
+                                "auth_type": "oauth2.0",
+                                "client_id": "client_id",
+                                "client_secret": "client_secret",
+                                "refresh_token": "new_refresh_token",
+                                "token_expiry_date": "2025-01-01T02:00:00+00:00"},
+                "start_date": "2021-01-01T00:00:00Z"}
+
+    source = SourceGitlab()
+    source._auth_params(oauth_config)
+    assert source._try_refresh_access_token(logger=logging.getLogger(), config=oauth_config) == expected

airbyte-integrations/connectors/source-gitlab/unit_tests/test_config_migrations.py

@@ -3,8 +3,10 @@
 #
 
 import datetime
+from unittest.mock import MagicMock
 
 import pytest
+from airbyte_cdk.models import SyncMode
 from airbyte_cdk.sources.streams.http.auth import NoAuth
 from source_gitlab.streams import (
     Branches,
@@ -276,3 +278,53 @@ def test_transform(requests_mock, stream, response_mocks, expected_records, requ
 def test_updated_state(stream, current_state, latest_record, new_state, request):
     stream = request.getfixturevalue(stream)
     assert stream.get_updated_state(current_state, latest_record) == new_state
+
+
+def test_parse_response_unsuported_response_type(request, caplog):
+    stream = request.getfixturevalue("pipelines")
+    from unittest.mock import MagicMock
+    response = MagicMock()
+    response.status_code = 200
+    response.json = MagicMock(return_value="")
+    list(stream.parse_response(response=response))
+    assert "Unsupported type of response data for stream pipelines" in caplog.text
+
+
+def test_stream_slices_child_stream(request, requests_mock):
+    commits = request.getfixturevalue("commits")
+    requests_mock.get("https://gitlab.com/api/v4/projects/p_1?per_page=50&statistics=1",
+                      json=[{"id": 13082000, "description": "", "name": "New CI Test Project"}])
+
+    slices = list(commits.stream_slices(sync_mode=SyncMode.full_refresh, stream_state={"13082000": {""'created_at': "2021-03-10T23:58:1213"}}))
+    assert slices
+
+
+def test_next_page_token(request):
+    response = MagicMock()
+    response.status_code = 200
+    response.json = MagicMock(return_value=["some data"])
+    commits = request.getfixturevalue("commits")
+    assert not commits.next_page_token(response)
+    data = ["some data" for x in range(0, 50)]
+    response.json = MagicMock(return_value=data)
+    assert commits.next_page_token(response) == {'page': 2}
+    response.json = MagicMock(return_value={"data": "some data"})
+    assert not commits.next_page_token(response)
+
+
+def test_availability_strategy(request):
+    commits = request.getfixturevalue("commits")
+    assert not commits.availability_strategy
+
+
+def test_request_params(request):
+    commits = request.getfixturevalue("commits")
+    expected = {'per_page': 50, 'page': 2, 'with_stats': True}
+    assert commits.request_params(stream_slice={"updated_after": "2021-03-10T23:58:1213"}, next_page_token={'page': 2}) == expected
+
+
+def test_chunk_date_range(request):
+    commits = request.getfixturevalue("commits")
+    # start point in future
+    start_point = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=1)
+    assert not list(commits._chunk_date_range(start_point))

airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+
+import pytest
+from source_gitlab.utils import parse_url
+
+
+@pytest.mark.parametrize(
+    "url, expected",
+    (
+            ("http://example.com", (True, "http", "example.com")),
+            ("http://example", (True, "http", "example")),
+            ("test://example.com", (False, "", "")),
+            ("https://example.com/test/test2", (False, "", "")),
+    )
+)
+def test_parse_url(url, expected):
+    assert parse_url(url) == expected