airbyte-ci makes pull requests (#37649)

Co-authored-by: alafanechere <[email protected]>
Co-authored-by: Augustin <[email protected]>

Author: bleonard

airbyte-ci/connectors/connector_ops/connector_ops/utils.py

@@ -57,6 +57,9 @@ def download_catalog(catalog_url):
 
 OSS_CATALOG = download_catalog(OSS_CATALOG_URL)
 METADATA_FILE_NAME = "metadata.yaml"
+MANIFEST_FILE_NAME = "manifest.yaml"
+DOCKERFILE_FILE_NAME = "Dockerfile"
+PYPROJECT_FILE_NAME = "pyproject.toml"
 ICON_FILE_NAME = "icon.svg"
 
 STRATEGIC_CONNECTOR_THRESHOLDS = {
@@ -347,9 +350,25 @@ def icon_path(self) -> Path:
     def code_directory(self) -> Path:
         return Path(f"./{CONNECTOR_PATH_PREFIX}/{self.relative_connector_path}")
 
+    @property
+    def python_source_dir_path(self) -> Path:
+        return self.code_directory / self.technical_name.replace("-", "_")
+
+    @property
+    def manifest_path(self) -> Path:
+        return self.python_source_dir_path / MANIFEST_FILE_NAME
+
     @property
     def has_dockerfile(self) -> bool:
-        return (self.code_directory / "Dockerfile").is_file()
+        return self.dockerfile_file_path.is_file()
+
+    @property
+    def dockerfile_file_path(self) -> Path:
+        return self.code_directory / DOCKERFILE_FILE_NAME
+
+    @property
+    def pyproject_file_path(self) -> Path:
+        return self.code_directory / PYPROJECT_FILE_NAME
 
     @property
     def metadata_file_path(self) -> Path:

airbyte-ci/connectors/pipelines/README.md

@@ -148,6 +148,8 @@ At this point you can run `airbyte-ci` commands.
 - [`connectors upgrade_base_image` command](#connectors-upgrade_base_image)
 - [`connectors migrate_to_base_image` command](#connectors-migrate_to_base_image)
 - [`connectors migrate-to-poetry` command](#connectors-migrate-to-poetry)
+- [`connectors migrate_to_inline_schemas` command](#migrate_to_inline_schemas)
+- [`connectors pull_request` command](#pull_request)
 - [`format` command subgroup](#format-subgroup)
   - [`format check` command](#format-check-command)
   - [`format fix` command](#format-fix-command)
@@ -464,15 +466,28 @@ Meant to be run on a cron script.
 Actions:
 
 * Upgrades dependecies to the current versions
+* Can make a pull request and bump version, changelog
+
+```
+Usage: airbyte-ci connectors up_to_date [OPTIONS]
+
+Options:
+  --dev       Force update when there are only dev changes.
+  --dep TEXT  Give a specific set of `poetry add` dependencies to update. For
+              example: --dep airbyte-cdk==0.80.0 --dep pytest@^6.2
+  --report    Auto open report browser.
+  --pull      Create a pull request.
+  --help      Show this message and exit.
+```
 
 ### Examples
 
-Bump source-openweather:
+Get source-openweather up to date. If there are changes, bump the version and add to changelog:
 
 * `airbyte-ci connectors --name=source-openweather up_to_date`: upgrades main dependecies
 * `airbyte-ci connectors --name=source-openweather up_to_date --dev`: forces update if there are only dev changes
 * `airbyte-ci connectors --name=source-openweather up_to_date --dep pytest@^8.10 --dep [email protected]`: allows update to toml files as well
-
+* `airbyte-ci connectors --name=source-openweather up_to_date --pull`: make a pull request for it
 
  ### Other things it could do
 
@@ -555,6 +570,60 @@ Migrate connectors the poetry package manager.
 Migrate source-openweather to use the base image:
 `airbyte-ci connectors --name=source-openweather migrate-to-poetry`
 
+### <a id="connectors-migrate_to_inline_schemas"></a>`connectors migrate_to_inline_schemas` command
+
+Migrate `.json` schemas into `manifest.yaml` files, when present.
+
+```
+Usage: airbyte-ci connectors migrate_to_inline_schemas [OPTIONS]
+
+Options:
+  --report  Auto open report browser.
+  --help    Show this message and exit.
+```
+
+#### Examples
+
+Migrate source-quickbooks to use inline schemas:
+`airbyte-ci connectors --name=source-quickbooks migrate_to_inline_schemas`
+
+### <a id="connectors-pull_request"></a>`connectors pull_request` command
+
+Makes a pull request for all changed connectors. If the branch already exists, it will update the existing one.
+
+```
+Usage: airbyte-ci connectors pull_request [OPTIONS]
+
+Options:
+  -m, --message TEXT          Commit message and pull request title and
+                              changelog (if enabled).  [required]
+  -b, --branch_id TEXT        update a branch named <branch_id>/<connector-
+                              name> instead generating one from the message.
+                              [required]
+  --report                    Auto open report browser.
+  --title TEXT                Title of the PR to be created or edited
+                              (optional - defaults to message or no change).
+  --body TEXT                 Body of the PR to be created or edited (optional
+                              - defaults to empty or not change).
+  --changelog                 Add message to the changelog for this version.
+  --bump [patch|minor|major]  Bump the metadata.yaml version. Can be `major`,
+                              `minor`, or `patch`.
+  --dry-run                   Don't actually make the pull requests. Just
+                              print the files that would be changed.
+  --help                      Show this message and exit.
+```
+
+#### Examples
+
+Make a PR for all changes, bump the version and make a changelog in those PRs. They will be on the branch ci_update/round2/<connector-name>:
+`airbyte-ci connectors pull_request -m "upgrading connectors" -b ci_update/round2 --bump patch --changelog`
+
+Do it just for a few connectors:
+`airbyte-ci connectors --name source-aha --name source-quickbooks pull_request -m "upgrading connectors" -b ci_update/round2 --bump patch --changelog`
+
+You can also set or set/change the title or body of the PR:
+`airbyte-ci connectors --name source-aha --name source-quickbooks pull_request -m "upgrading connectors" -b ci_update/round2 --title "New title" --body "full body\n\ngoes here"`
+
 ### <a id="format-subgroup"></a>`format` command subgroup
 
 Available commands:

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/commands.py

@@ -50,6 +50,8 @@ async def build(ctx: click.Context, use_host_gradle_dist_tar: bool, build_archit
             is_local=ctx.obj["is_local"],
             git_branch=ctx.obj["git_branch"],
             git_revision=ctx.obj["git_revision"],
+            diffed_branch=ctx.obj["diffed_branch"],
+            git_repo_url=ctx.obj["git_repo_url"],
             ci_report_bucket=ctx.obj["ci_report_bucket_name"],
             report_output_prefix=ctx.obj["report_output_prefix"],
             use_remote_secrets=ctx.obj["use_remote_secrets"],

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/__init__.py

@@ -6,15 +6,14 @@
 from __future__ import annotations
 
 import anyio
-from connector_ops.utils import ConnectorLanguage # type: ignore
+from connector_ops.utils import ConnectorLanguage  # type: ignore
 from pipelines.airbyte_ci.connectors.build_image.steps import java_connectors, python_connectors
 from pipelines.airbyte_ci.connectors.build_image.steps.common import LoadContainerToLocalDockerHost, StepStatus
 from pipelines.airbyte_ci.connectors.context import ConnectorContext
 from pipelines.airbyte_ci.connectors.reports import ConnectorReport, Report
 from pipelines.models.steps import StepResult
 
 
-
 class NoBuildStepForLanguageError(Exception):
     pass
 

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/build_customization.py

@@ -22,9 +22,11 @@ def get_build_customization_module(connector: Connector) -> Optional[ModuleType]
     """
     build_customization_spec_path = connector.code_directory / BUILD_CUSTOMIZATION_SPEC_NAME
 
-    if not build_customization_spec_path.exists() or not (build_customization_spec := importlib.util.spec_from_file_location(
-        f"{connector.code_directory.name}_{BUILD_CUSTOMIZATION_MODULE_NAME}", build_customization_spec_path
-    )):
+    if not build_customization_spec_path.exists() or not (
+        build_customization_spec := importlib.util.spec_from_file_location(
+            f"{connector.code_directory.name}_{BUILD_CUSTOMIZATION_MODULE_NAME}", build_customization_spec_path
+        )
+    ):
         return None
 
     if build_customization_spec.loader is None:

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/common.py

@@ -15,6 +15,7 @@
 if TYPE_CHECKING:
     from typing import Any
 
+
 class BuildConnectorImagesBase(Step, ABC):
     """
     A step to build connector images for a set of platforms.
@@ -39,13 +40,17 @@ async def _run(self, *args: Any) -> StepResult:
                     await connector.with_exec(["spec"])
                 except ExecError as e:
                     return StepResult(
-                        step=self, status=StepStatus.FAILURE, stderr=str(e),
-                        stdout=f"Failed to run the spec command on the connector container for platform {platform}."
+                        step=self,
+                        status=StepStatus.FAILURE,
+                        stderr=str(e),
+                        stdout=f"Failed to run the spec command on the connector container for platform {platform}.",
+                        exc_info=e,
                     )
                 build_results_per_platform[platform] = connector
             except QueryError as e:
-                return StepResult(step=self, status=StepStatus.FAILURE,
-                                  stderr=f"Failed to build connector image for platform {platform}: {e}")
+                return StepResult(
+                    step=self, status=StepStatus.FAILURE, stderr=f"Failed to build connector image for platform {platform}: {e}"
+                )
         success_message = (
             f"The {self.context.connector.technical_name} docker image "
             f"was successfully built for platform(s) {', '.join(self.build_platforms)}"
@@ -86,6 +91,7 @@ def image_name(self) -> str:
 
     async def _run(self) -> StepResult:
         loaded_images = []
+        image_sha = None
         multi_platforms = len(self.containers) > 1
         for platform, container in self.containers.items():
             _, exported_tar_path = await export_container_to_tarball(self.context, container, platform)
@@ -109,4 +115,6 @@ async def _run(self) -> StepResult:
                     step=self, status=StepStatus.FAILURE, stderr=f"Something went wrong while interacting with the local docker client: {e}"
                 )
 
-        return StepResult(step=self, status=StepStatus.SUCCESS, stdout=f"Loaded image {','.join(loaded_images)} to your Docker host ({image_sha}).")
+        return StepResult(
+            step=self, status=StepStatus.SUCCESS, stdout=f"Loaded image {','.join(loaded_images)} to your Docker host ({image_sha})."
+        )

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/normalization.py

@@ -14,7 +14,7 @@ class BuildOrPullNormalization(Step):
     """A step to build or pull the normalization image for a connector according to the image name."""
 
     context: ConnectorContext
-    
+
     def __init__(self, context: ConnectorContext, normalization_image: str, build_platform: Platform) -> None:
         """Initialize the step to build or pull the normalization image.
 

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/build_image/steps/python_connectors.py

@@ -46,13 +46,9 @@ async def _create_builder_container(self, base_container: Container) -> Containe
             Container: The builder container, with installed dependencies.
         """
         ONLY_BUILD_FILES = ["pyproject.toml", "poetry.lock", "poetry.toml", "setup.py", "requirements.txt", "README.md"]
-        
+
         builder = await with_python_connector_installed(
-            self.context,
-            base_container,
-            str(self.context.connector.code_directory),
-            install_root_package=False,
-            include=ONLY_BUILD_FILES
+            self.context, base_container, str(self.context.connector.code_directory), install_root_package=False, include=ONLY_BUILD_FILES
         )
         return builder
 

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/bump_version/commands.py

@@ -29,6 +29,8 @@ async def bump_version(
             is_local=ctx.obj["is_local"],
             git_branch=ctx.obj["git_branch"],
             git_revision=ctx.obj["git_revision"],
+            diffed_branch=ctx.obj["diffed_branch"],
+            git_repo_url=ctx.obj["git_repo_url"],
             ci_report_bucket=ctx.obj["ci_report_bucket_name"],
             report_output_prefix=ctx.obj["report_output_prefix"],
             use_remote_secrets=ctx.obj["use_remote_secrets"],