airbyte_ci: up_to_date command (#37487)

Co-authored-by: Catherine Noll <[email protected]>

Author: bleonard

airbyte-ci/connectors/pipelines/README.md

@@ -142,6 +142,7 @@ At this point you can run `airbyte-ci` commands.
 - [`connectors publish` command](#connectors-publish-command)
 - [Examples](#examples)
 - [Options](#options-2)
+- [`connectors up_to_date` command](#up_to_date)
 - [`connectors bump_version` command](#connectors-bump_version)
 - [`connectors upgrade_cdk` command](#connectors-upgrade_cdk)
 - [`connectors upgrade_base_image` command](#connectors-upgrade_base_image)
@@ -456,6 +457,32 @@ remoteRegistries:
     packageName: airbyte-source-pokeapi
 ```
 
+### <a id="connectors-up_to_date"></a>`connectors up_to_date` command
+
+Meant to be run on a cron script. 
+
+Actions:
+
+* Upgrades dependecies to the current versions
+
+### Examples
+
+Bump source-openweather:
+
+* `airbyte-ci connectors --name=source-openweather up_to_date`: upgrades main dependecies
+* `airbyte-ci connectors --name=source-openweather up_to_date --dev`: forces update if there are only dev changes
+* `airbyte-ci connectors --name=source-openweather up_to_date --dep pytest@^8.10 --dep [email protected]`: allows update to toml files as well
+
+
+ ### Other things it could do
+
+* upgrade it the latest base image
+* make sure it's the newest version of pytest
+* do a `poetry update` to update everything else
+* make the pull requests on a well known branch, replacing the last one if still open
+* bump the toml and metadata and changelog
+* also bump the manifest version of the CDK
+
 ### <a id="connectors-bump_version"></a>`connectors bump_version` command
 
 Bump the version of the selected connectors.

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/commands.py

@@ -153,7 +153,8 @@ def should_use_remote_secrets(use_remote_secrets: Optional[bool]) -> bool:
         "migrate_to_base_image": "pipelines.airbyte_ci.connectors.migrate_to_base_image.commands.migrate_to_base_image",
         "migrate-to-poetry": "pipelines.airbyte_ci.connectors.migrate_to_poetry.commands.migrate_to_poetry",
         "upgrade_base_image": "pipelines.airbyte_ci.connectors.upgrade_base_image.commands.upgrade_base_image",
-        "upgrade_cdk": "pipelines.airbyte_ci.connectors.upgrade_cdk.commands.bump_version",
+        "upgrade_cdk": "pipelines.airbyte_ci.connectors.upgrade_cdk.commands.upgrade_cdk",
+        "up_to_date": "pipelines.airbyte_ci.connectors.up_to_date.commands.up_to_date",
     },
 )
 @click.option(

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/consts.py

@@ -30,6 +30,8 @@ class CONNECTOR_TEST_STEP_ID(str, Enum):
     CHECK_MIGRATION_CANDIDATE = "check_migration_candidate"
     POETRY_INIT = "poetry_init"
     DELETE_SETUP_PY = "delete_setup_py"
+    CHECK_UPDATE_CANDIDATE = "check_update_candidate"
+    UPDATE_POETRY = "upgrade_poetry"
     CONNECTOR_REGRESSION_TESTS = "connector_regression_tests"
 
     def __str__(self) -> str:

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/up_to_date/commands.py

@@ -0,0 +1,76 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import List
+
+import asyncclick as click
+from pipelines.airbyte_ci.connectors.context import ConnectorContext
+from pipelines.airbyte_ci.connectors.pipeline import run_connectors_pipelines
+from pipelines.airbyte_ci.connectors.up_to_date.pipeline import run_connector_up_to_date_pipeline
+from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand
+
+
+@click.command(
+    cls=DaggerPipelineCommand,
+    short_help="Get the selected Python connectors up to date.",
+)
+@click.option(
+    "--dev",
+    type=bool,
+    default=False,
+    is_flag=True,
+    help="Force update when there are only dev changes.",
+)
+@click.option(
+    "--dep",
+    type=str,
+    multiple=True,
+    default=[],
+    help="Give a specific set of `poetry add` dependencies to update. For example: --dep airbyte-cdk==0.80.0 --dep pytest@^6.2",
+)
+
+# TODO: flag to skip regression tests
+# TODO: flag to make PR
+# TODO: also update the manifest.yaml with the cdk version?
+@click.pass_context
+async def up_to_date(ctx: click.Context, dev: bool, dep: List[str]) -> bool:
+
+    connectors_contexts = [
+        ConnectorContext(
+            pipeline_name=f"Update {connector.technical_name} to latest",
+            connector=connector,
+            is_local=ctx.obj["is_local"],
+            git_branch=ctx.obj["git_branch"],
+            git_revision=ctx.obj["git_revision"],
+            ci_report_bucket=ctx.obj["ci_report_bucket_name"],
+            report_output_prefix=ctx.obj["report_output_prefix"],
+            use_remote_secrets=ctx.obj["use_remote_secrets"],
+            gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"),
+            dagger_logs_url=ctx.obj.get("dagger_logs_url"),
+            pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"),
+            ci_context=ctx.obj.get("ci_context"),
+            ci_gcs_credentials=ctx.obj["ci_gcs_credentials"],
+            ci_git_user=ctx.obj["ci_git_user"],
+            ci_github_access_token=ctx.obj["ci_github_access_token"],
+            enable_report_auto_open=True,
+            docker_hub_username=ctx.obj.get("docker_hub_username"),
+            docker_hub_password=ctx.obj.get("docker_hub_password"),
+            s3_build_cache_access_key_id=ctx.obj.get("s3_build_cache_access_key_id"),
+            s3_build_cache_secret_key=ctx.obj.get("s3_build_cache_secret_key"),
+        )
+        for connector in ctx.obj["selected_connectors_with_modified_files"]
+    ]
+
+    await run_connectors_pipelines(
+        connectors_contexts,
+        run_connector_up_to_date_pipeline,
+        "Get Python connector up to date",
+        ctx.obj["concurrency"],
+        ctx.obj["dagger_logs_path"],
+        ctx.obj["execute_timeout"],
+        dev,
+        dep,
+    )
+
+    return True