airbyte-ci: improve git diff comparison (#37616)

Author: alafanechere

airbyte-ci/connectors/pipelines/README.md

@@ -183,7 +183,7 @@ options to the `airbyte-ci` command group.**
 | `--is-local/--is-ci`                           | `--is-local`                    |                               | Determines the environment in which the CLI runs: local environment or CI environment.      |
 | `--git-branch`                                 | The checked out git branch name | `CI_GIT_BRANCH`               | The git branch on which the pipelines will run.                                             |
 | `--git-revision`                               | The current branch head         | `CI_GIT_REVISION`             | The commit hash on which the pipelines will run.                                            |
-| `--diffed-branch`                              | `origin/master`                 |                               | Branch to which the git diff will happen to detect new or modified files.                   |
+| `--diffed-branch`                              | `master`                 |                               | Branch to which the git diff will happen to detect new or modified files.                   |
 | `--gha-workflow-run-id`                        |                                 |                               | GHA CI only - The run id of the GitHub action workflow                                      |
 | `--ci-context`                                 | `manual`                        |                               | The current CI context: `manual` for manual run, `pull_request`, `nightly_builds`, `master` |
 | `--pipeline-start-timestamp`                   | Current epoch time              | `CI_PIPELINE_START_TIMESTAMP` | Start time of the pipeline as epoch time. Used for pipeline run duration computation.       |
@@ -649,6 +649,7 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 
 | Version | PR                                                         | Description                                                                                                                |
 | ------- | ---------------------------------------------------------- |----------------------------------------------------------------------------------------------------------------------------|
+| 4.10.0   | [#37616](https://github.com/airbytehq/airbyte/pull/37616)  | Improve modified files comparison when the target branch is from a fork.                                                                       |
 | 4.9.0   | [#37440](https://github.com/airbytehq/airbyte/pull/37440)  | Run regression tests with `airbyte-ci connectors test`                                                                       |
 | 4.8.0   | [#37404](https://github.com/airbytehq/airbyte/pull/37404)  | Accept a `git-repo-url` option on the `airbyte-ci` root command to checkout forked repo.                                                                                                     |
 | 4.7.4   | [#37485](https://github.com/airbytehq/airbyte/pull/37485)  | Allow java connectors to be written in kotlin.                                                                             |

airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py

@@ -153,7 +153,7 @@ def is_current_process_wrapped_by_dagger_run() -> bool:
 @click.option(
     "--diffed-branch",
     help="Branch to which the git diff will happen to detect new or modified connectors",
-    default="origin/master",
+    default="master",
     type=str,
 )
 @click.option("--gha-workflow-run-id", help="[CI Only] The run id of the GitHub action workflow", default=None, type=str)

airbyte-ci/connectors/pipelines/pipelines/dagger/containers/git.py

@@ -13,7 +13,12 @@ async def checked_out_git_container(
     diffed_branch: Optional[str] = None,
     repo_url: str = AIRBYTE_REPO_URL,
 ) -> Container:
-    """Builds git-based container with the current branch checked out."""
+    """
+    Create a container with git in it.
+    We add the airbyte repo as the origin remote and the target repo as the target remote.
+    We fetch the diffed branch from the origin remote and the current branch from the target remote.
+    We then checkout the current branch.
+    """
     current_git_branch = current_git_branch.removeprefix("origin/")
     diffed_branch = current_git_branch if diffed_branch is None else diffed_branch.removeprefix("origin/")
     return await (
@@ -26,14 +31,19 @@ async def checked_out_git_container(
             [
                 "remote",
                 "add",
-                "--fetch",
-                "--track",
-                current_git_branch,
-                "--track",
-                diffed_branch if diffed_branch is not None else current_git_branch,
                 "origin",
+                AIRBYTE_REPO_URL,
+            ]
+        )
+        .with_exec(
+            [
+                "remote",
+                "add",
+                "target",
                 repo_url,
             ]
         )
-        .with_exec(["checkout", "-t", f"origin/{current_git_branch}"])
+        .with_exec(["fetch", "origin", diffed_branch])
+        .with_exec(["fetch", "target", current_git_branch])
+        .with_exec(["checkout", current_git_branch])
     )

airbyte-ci/connectors/pipelines/pipelines/helpers/git.py

@@ -21,7 +21,7 @@ def get_current_git_branch() -> str:  # noqa D103
 
 
 async def get_modified_files_in_branch_remote(
-    current_git_repo_url: str, current_git_branch: str, current_git_revision: str, diffed_branch: str = "origin/master", retries: int = 3
+    current_git_repo_url: str, current_git_branch: str, current_git_revision: str, diffed_branch: str = "master", retries: int = 3
 ) -> Set[str]:
     """Use git diff to spot the modified files on the remote branch."""
     try:
@@ -30,7 +30,7 @@ async def get_modified_files_in_branch_remote(
                 dagger_client, current_git_branch, current_git_revision, diffed_branch, repo_url=current_git_repo_url
             )
             modified_files = await container.with_exec(
-                ["diff", f"--diff-filter={DIFF_FILTER}", "--name-only", f"{diffed_branch}...{current_git_branch}"]
+                ["diff", f"--diff-filter={DIFF_FILTER}", "--name-only", f"origin/{diffed_branch}...target/{current_git_branch}"]
             ).stdout()
     except SessionError:
         if retries > 0:

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.9.0"
+version = "4.10.0"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]