🚨🚨🐛 Source Gitlab fix merge_request_commits stream

airbyte-integrations/connectors/source-gitlab/acceptance-test-config.yml

@@ -18,7 +18,7 @@ acceptance_tests:
     tests:
       - config_path: "secrets/config.json"
         backward_compatibility_tests_config:
-          disable_for_version: 1.8.4
+          disable_for_version: 3.0.0
   basic_read:
     tests:
       - config_path: "secrets/config.json"
@@ -29,6 +29,9 @@ acceptance_tests:
           jobs:
             - name: "user"
               bypass_reason: "User object contains local_time which will be different each time test is run"
+          projects:
+            - name: "updated_at"
+              bypass_reason: "value can be changed"
       - config_path: "secrets/config_with_ids.json"
         timeout_seconds: 3600
         empty_streams:
@@ -42,6 +45,9 @@ acceptance_tests:
           jobs:
             - name: "user"
               bypass_reason: "User object contains local_time which will be different each time test is run"
+          projects:
+            - name: "updated_at"
+              bypass_reason: "value can be changed"
       - config_path: "secrets/config_oauth.json"
         timeout_seconds: 3600
         expect_records:
@@ -50,6 +56,9 @@ acceptance_tests:
           jobs:
             - name: "user"
               bypass_reason: "User object contains local_time which will be different each time test is run"
+          projects:
+            - name: "updated_at"
+              bypass_reason: "value can be changed"
   incremental:
     tests:
       - config_path: "secrets/config_with_ids.json"

airbyte-integrations/connectors/source-gitlab/metadata.yaml

@@ -10,7 +10,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80
-  dockerImageTag: 2.1.1
+  dockerImageTag: 2.2.0
   dockerRepository: airbyte/source-gitlab
   documentationUrl: https://docs.airbyte.com/integrations/sources/gitlab
   githubIssueLabel: source-gitlab
@@ -31,6 +31,15 @@ data:
           These changes impact pipeline.created_at and pipeline.updated_at fields for stream Deployments and expires_at field for stream Group Members and stream Project Members.
           Users will need to refresh the source schema and reset affected streams after upgrading.
         upgradeDeadline: "2023-11-09"
+      3.0.0:
+        message:
+          In this release, merge_request_commits stream was fixed so now it returns commits for each found merge_request.
+          These changes leads to complete change of stream schema.
+          Users will need to refresh the source schema and reset merge_request_commits stream after upgrading.
+        upgradeDeadline: "2024-02-09"
+        scopedImpact:
+          - scopeType: stream
+            impactedScopes: ["merge_request_commits"]
   suggestedStreams:
     streams:
       - merge_requests

airbyte-integrations/connectors/source-gitlab/source_gitlab/config_migrations.py

@@ -95,12 +95,10 @@ def migrate(cls, args: List[str], source: SourceGitlab) -> None:
 
 
 class MigrateGroups(MigrateStringToArray):
-
     migrate_from_key: str = "groups"
     migrate_to_key: str = "groups_list"
 
 
 class MigrateProjects(MigrateStringToArray):
-
     migrate_from_key: str = "projects"
     migrate_to_key: str = "projects_list"

airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/groups.json

@@ -74,6 +74,9 @@
     "subgroup_creation_level": {
       "type": ["null", "string"]
     },
+    "enabled_git_access_protocol": {
+      "type": ["null", "string"]
+    },
     "emails_disabled": {
       "type": ["null", "boolean"]
     },

airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/merge_request_commits.json

@@ -2,233 +2,84 @@
   "$schema": "https://json-schema.org/draft-07/schema#",
   "type": "object",
   "properties": {
-    "id": {
-      "type": ["null", "integer"]
-    },
-    "iid": {
-      "type": ["null", "integer"]
-    },
     "project_id": {
       "type": ["null", "integer"]
     },
-    "title": {
-      "type": ["null", "string"]
-    },
-    "description": {
+    "id": {
       "type": ["null", "string"]
     },
-    "state": {
+    "short_id": {
       "type": ["null", "string"]
     },
     "created_at": {
       "type": ["null", "string"],
       "format": "date-time"
     },
-    "updated_at": {
-      "type": ["null", "string"],
-      "format": "date-time"
-    },
-    "merged_by": {
-      "type": ["null", "object"]
-    },
-    "merged_at": {
-      "type": ["null", "string"],
-      "format": "date-time"
-    },
-    "prepared_at": {
-      "type": ["null", "string"],
-      "format": "date-time"
-    },
-    "closed_by": {
-      "type": ["null", "object"]
-    },
-    "closed_at": {
-      "type": ["null", "string"],
-      "format": "date-time"
-    },
-    "target_branch": {
-      "type": ["null", "string"]
-    },
-    "source_branch": {
-      "type": ["null", "string"]
-    },
-    "user_notes_count": {
-      "type": ["null", "integer"]
-    },
-    "upvotes": {
-      "type": ["null", "integer"]
-    },
-    "downvotes": {
-      "type": ["null", "integer"]
-    },
-    "author": {
-      "type": ["null", "object"]
-    },
-    "assignees": {
-      "type": ["null", "array"],
-      "items": {
-        "type": "object"
-      }
-    },
-    "assignee": {
-      "type": ["null", "object"]
-    },
-    "reviewers": {
+    "parent_ids": {
       "type": ["null", "array"],
       "items": {
-        "type": "object"
+        "type": ["null", "string"]
       }
     },
-    "source_project_id": {
-      "type": ["null", "integer"]
-    },
-    "target_project_id": {
-      "type": ["null", "integer"]
-    },
-    "labels": {
-      "type": ["null", "array"],
-      "items": {
-        "type": "string"
-      }
-    },
-    "work_in_progress": {
-      "type": ["null", "boolean"]
-    },
-    "milestone": {
-      "type": ["null", "object"]
-    },
-    "merge_when_pipeline_succeeds": {
-      "type": ["null", "boolean"]
-    },
-    "merge_status": {
-      "type": ["null", "string"]
-    },
-    "sha": {
-      "type": ["null", "string"]
-    },
-    "merge_commit_sha": {
+    "title": {
       "type": ["null", "string"]
     },
-    "squash_commit_sha": {
+    "message": {
       "type": ["null", "string"]
     },
-    "discussion_locked": {
-      "type": ["null", "boolean"]
-    },
-    "should_remove_source_branch": {
-      "type": ["null", "boolean"]
-    },
-    "force_remove_source_branch": {
-      "type": ["null", "boolean"]
-    },
-    "reference": {
+    "author_name": {
       "type": ["null", "string"]
     },
-    "references": {
-      "type": ["null", "object"]
-    },
-    "web_url": {
+    "author_email": {
       "type": ["null", "string"]
     },
-    "time_stats": {
-      "type": ["null", "object"]
-    },
-    "squash": {
-      "type": ["null", "boolean"]
-    },
-    "task_completion_status": {
-      "type": ["null", "object"]
-    },
-    "has_conflicts": {
-      "type": ["null", "boolean"]
-    },
-    "blocking_discussions_resolved": {
-      "type": ["null", "boolean"]
-    },
-    "approvals_before_merge": {
-      "type": ["null", "boolean", "string", "object"]
-    },
-    "subscribed": {
-      "type": ["null", "boolean"]
-    },
-    "changes_count": {
-      "type": ["null", "integer", "string"]
-    },
-    "latest_build_started_at": {
-      "type": ["null", "string"],
-      "format": "date-time"
-    },
-    "latest_build_finished_at": {
+    "authored_date": {
       "type": ["null", "string"],
       "format": "date-time"
     },
-    "first_deployed_to_production_at": {
-      "type": ["null", "string"],
-      "format": "date-time"
-    },
-    "pipeline": {
-      "type": ["null", "object"]
-    },
-    "head_pipeline": {
-      "type": ["null", "object", "string", "boolean", "integer"]
-    },
-    "diff_refs": {
+    "extended_trailers": {
       "type": ["null", "object"],
       "properties": {
-        "base_sha": {
-          "type": ["null", "string"]
-        },
-        "head_sha": {
-          "type": ["null", "string"]
-        },
-        "start_sha": {
-          "type": ["null", "string"]
+        "Cc": {
+          "type": ["null", "array"],
+          "items": {
+            "type": ["null", "string"]
+          }
         }
       }
     },
-    "merge_error": {
-      "type": ["null", "boolean", "string"]
-    },
-    "first_contribution": {
-      "type": ["null", "boolean"]
+    "committer_name": {
+      "type": ["null", "string"]
     },
-    "user": {
-      "type": ["null", "object"]
+    "committer_email": {
+      "type": ["null", "string"]
     },
-    "merge_request_iid": {
-      "type": ["null", "integer"]
+    "committed_date": {
+      "type": ["null", "string"],
+      "format": "date-time"
     },
-    "draft": {
-      "type": ["null", "boolean"]
+    "trailers": {
+      "type": ["null", "object"]
     },
-    "detailed_merge_status": {
+    "web_url": {
       "type": ["null", "string"]
     },
-    "squash_on_merge": {
-      "type": ["null", "boolean"]
-    },
-    "merge_user": {
+    "stats": {
       "type": ["null", "object"],
       "properties": {
-        "id": {
+        "additions": {
           "type": ["null", "integer"]
         },
-        "name": {
-          "type": ["null", "string"]
-        },
-        "username": {
-          "type": ["null", "string"]
-        },
-        "state": {
-          "type": ["null", "string"]
-        },
-        "avatar_url": {
-          "type": ["null", "string"]
+        "deletions": {
+          "type": ["null", "integer"]
         },
-        "web_url": {
-          "type": ["null", "string"]
+        "total": {
+          "type": ["null", "integer"]
         }
       }
+    },
+    "merge_request_iid": {
+      "type": ["null", "integer"]
     }
   }
 }

airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/projects.json

@@ -329,6 +329,9 @@
         },
         "packages_size": {
           "type": ["null", "integer"]
+        },
+        "container_registry_size": {
+          "type": ["null", "integer"]
         }
       }
     },

airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py

@@ -339,8 +339,10 @@ class MergeRequests(IncrementalGitlabChildStream):
 
 
 class MergeRequestCommits(GitlabChildStream):
+    """Docs: https://docs.gitlab.com/ee/api/merge_requests.html#get-single-merge-request-commits"""
+
     path_list = ["project_id", "iid"]
-    path_template = "projects/{project_id}/merge_requests/{iid}"
+    path_template = "projects/{project_id}/merge_requests/{iid}/commits"
 
     def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs):
         super().transform(record, stream_slice, **kwargs)

airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py

@@ -207,7 +207,7 @@ def test_should_retry(mocker, requests_mock, stream, extra_mocks, expected_call_
                 [{"id": "mr_1", "iid": "mr_1", "project_id": "p_1"}],
             ),
             (
-                "/api/v4/projects/p_1/merge_requests/mr_1",
+                "/api/v4/projects/p_1/merge_requests/mr_1/commits",
                 [
                     {
                         "id": "mrc_1",