From 1f2e7d5fbcc1dbf8a5ad614621a0d054d4f11a69 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 10 Jan 2024 17:32:29 +0100 Subject: [PATCH 01/53] WIP --- .../pipelines/airbyte_ci/poetry/__init__.py | 3 + .../pipelines/airbyte_ci/poetry/commands.py | 92 +++++++++++++ .../pipelines/airbyte_ci/poetry/pipeline.py | 127 ++++++++++++++++++ .../pipelines/pipelines/cli/airbyte_ci.py | 1 + airbyte-ci/connectors/pipelines/poetry.lock | 24 +--- .../connectors/pipelines/pyproject.toml | 1 + 6 files changed, 226 insertions(+), 22 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/__init__.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/__init__.py new file mode 100644 index 0000000000000..c941b30457953 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py new file mode 100644 index 0000000000000..94b647b52cea7 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py @@ -0,0 +1,92 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +""" +Module exposing the format commands. +""" +from __future__ import annotations + +import logging +import sys +from typing import Any, Dict, List + +import asyncclick as click +from pipelines.airbyte_ci.format.configuration import FORMATTERS_CONFIGURATIONS, Formatter +from pipelines.airbyte_ci.format.format_command import FormatCommand +from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.helpers.cli import LogOptions, invoke_commands_concurrently, invoke_commands_sequentially, log_command_results +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context +from pipelines.models.steps import StepStatus +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from .pipeline import PyPIPublishContext, PublishToPyPI + + +@click.group( + name="poetry", + help="Commands related to running poetry commands.", +) +@click_merge_args_into_context_obj +@pass_pipeline_context +@click_ignore_unused_kwargs +async def poetry(pipeline_context: ClickPipelineContext) -> None: + pass + + +@poetry.command(cls=DaggerPipelineCommand, name="publish", help="Publish a Python package to PyPI.") +@click.option( + "--pypi-username", + help="Your username to connect to PyPI.", + type=click.STRING, + required=True, + envvar="PYPI_USERNAME", +) +@click.option( + "--pypi-password", + help="Your password to connect to PyPI.", + type=click.STRING, + required=True, + envvar="PYPI_PASSWORD", +) +@click.option( + "--pypi-repository", + help="The PyPI repository to publish to (pypi, test-pypi).", + type=click.Choice(["pypi", "testpypi"]), + default="pypi", +) +@click.option( + "--package-path", + help="The path to publish", + type=click.STRING, + required=True, +) +@pass_pipeline_context +@click.pass_context +async def publish(ctx: click.Context, + package_path: str, + pypi_username: str, + pypi_password: str, + pypi_repository: str, + click_pipeline_context: ClickPipelineContext + ) -> None: + context = PyPIPublishContext( + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + pypi_username=pypi_username, + pypi_password=pypi_password, + pypi_repository=pypi_repository, + package_path=package_path + ) + dagger_client = await click_pipeline_context.get_dagger_client(pipeline_name=f"Publish {package_path} to PyPI") + context.dagger_client = dagger_client + + await PublishToPyPI(context).run() + diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py new file mode 100644 index 0000000000000..dee547bb28e56 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py @@ -0,0 +1,127 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import tomli +import uuid +from typing import Optional + +import dagger +from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID +from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext +from pipelines.airbyte_ci.steps.docker import SimpleDockerStep +from pipelines.airbyte_ci.steps.poetry import PoetryRunStep +from pipelines.consts import DOCS_DIRECTORY_ROOT_PATH, INTERNAL_TOOL_PATHS +from pipelines.dagger.actions.python.common import with_pip_packages +from pipelines.dagger.containers.python import with_python_base +from pipelines.helpers.run_steps import STEP_TREE, StepToRun, run_steps +from pipelines.helpers.utils import DAGGER_CONFIG, get_secret_host_variable +from pipelines.models.reports import Report +from pipelines.models.steps import MountPath, Step, StepResult +from pipelines.models.contexts.pipeline_context import PipelineContext + + + +from textwrap import dedent +from typing import List + +import anyio + + +class PyPIPublishContext(PipelineContext): + def __init__( + self, + pypi_username: str, + pypi_password: str, + pypi_repository: str, + package_path: str, + ci_report_bucket: str, + report_output_prefix: str, + is_local: bool, + git_branch: bool, + git_revision: bool, + gha_workflow_run_url: Optional[str] = None, + dagger_logs_url: Optional[str] = None, + pipeline_start_timestamp: Optional[int] = None, + ci_context: Optional[str] = None, + ci_gcs_credentials: str = None, + package_name: Optional[str] = None, + version: Optional[str] = None, + ): + self.pypi_username = pypi_username + self.pypi_password = pypi_password + self.pypi_repository = pypi_repository + self.package_path = package_path + self.package_name = package_name + self.version = version + + pipeline_name = f"Publish PyPI {package_path}" + + super().__init__( + pipeline_name=pipeline_name, + report_output_prefix=report_output_prefix, + ci_report_bucket=ci_report_bucket, + is_local=is_local, + git_branch=git_branch, + git_revision=git_revision, + gha_workflow_run_url=gha_workflow_run_url, + dagger_logs_url=dagger_logs_url, + pipeline_start_timestamp=pipeline_start_timestamp, + ci_context=ci_context, + ci_gcs_credentials=ci_gcs_credentials, + ) + + +class PublishToPyPI(Step): + title = "Publish package to PyPI" + + async def _run(self) -> StepResult: + context: PyPIPublishContext = self.context + dir_to_publish = await context.get_repo_dir(context.package_path) + + if not context.package_name or not context.version: + # check whether it has a pyproject.toml file + pyproject_toml = dir_to_publish.file("pyproject.toml") + if not await pyproject_toml.exists(): + return self.skip("Connector does not have a pyproject.toml file and version and package name is not set otherwise, skipping.") + + # get package name and version from pyproject.toml + pyproject_toml_content = await pyproject_toml.contents() + contents = tomli.loads(pyproject_toml_content) + if "tool" not in contents or "poetry" not in contents["tool"] or "name" not in contents["tool"]["poetry"] or "version" not in contents["tool"]["poetry"]: + return self.skip("Connector does not have a pyproject.toml file with a poetry section, skipping.") + + context.package_name = contents["tool"]["poetry"]["name"] + context.version = contents["tool"]["poetry"]["version"] + + setup_cfg = dedent( + f""" + [metadata] + name = {context.package_name} + version = {context.version} + author = Airbyte + author_email = contact@airbyte.io + """ + ) + + twine_username = self.context.dagger_client.set_secret("twine_username", context.pypi_username) + twine_password = self.context.dagger_client.set_secret("twine_password", context.pypi_password) + + twine_upload = ( + self.context.dagger_client.container() + .from_("python:3.10-slim") + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "install", "-y", "twine"]) + .with_directory("package", dir_to_publish) + .with_workdir("package") + .with_exec(["sed", "-i", "/name=/d; /author=/d; /author_email=/d; /version=/d", "setup.py"]) + .with_new_file("setup.cfg", setup_cfg) + .with_exec(["pip", "install", "--upgrade", "setuptools", "wheel"]) + .with_exec(["python", "setup.py", "sdist", "bdist_wheel"]) + .with_secret_variable("TWINE_USERNAME", twine_username) + .with_secret_variable("TWINE_PASSWORD", twine_password) + .with_exec(["twine", "upload", "--verbose", "--repository", self.context.pypi_repository, "dist/*"]) + ) + + return await self.get_step_result(twine_upload) + diff --git a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py index 028bb54df7c74..b1b3ba787e0c1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py +++ b/airbyte-ci/connectors/pipelines/pipelines/cli/airbyte_ci.py @@ -169,6 +169,7 @@ def is_current_process_wrapped_by_dagger_run() -> bool: help="Airbyte CI top-level command group.", lazy_subcommands={ "connectors": "pipelines.airbyte_ci.connectors.commands.connectors", + "poetry": "pipelines.airbyte_ci.poetry.commands.poetry", "format": "pipelines.airbyte_ci.format.commands.format_code", "metadata": "pipelines.airbyte_ci.metadata.commands.metadata", "test": "pipelines.airbyte_ci.test.commands.test", diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index a1e1a4468abe6..0aaed21095c41 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "airbyte-connectors-base-images" @@ -1134,16 +1134,6 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1983,7 +1973,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1991,15 +1980,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2016,7 +1998,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2024,7 +2005,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2531,4 +2511,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "c24e2d2a6c1288b7fe7efa1c0ca598dfef8942bb437986512bbe17bcf1408799" +content-hash = "8e1d9bcd933ad7030fa06fd305f20317391111f2b364874e98e594de762fcf57" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index d370f0a8e71bc..19483e5ea5e86 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -27,6 +27,7 @@ segment-analytics-python = "^2.2.3" pygit2 = "^1.13.1" asyncclick = "^8.1.3.4" certifi = "^2023.11.17" +tomli = "^2.0.1" [tool.poetry.group.test.dependencies] pytest = "^6.2.5" From dcaef6d4da80d07c4da7aa67270c520e48780d4d Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 11 Jan 2024 18:06:44 +0100 Subject: [PATCH 02/53] wip --- .../pipelines/airbyte_ci/poetry/commands.py | 22 +++++++++---------- .../pipelines/airbyte_ci/poetry/pipeline.py | 10 ++++----- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py index 94b647b52cea7..5930832b5766d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py @@ -26,6 +26,12 @@ name="poetry", help="Commands related to running poetry commands.", ) +@click.option( + "--package-path", + help="The path to publish", + type=click.STRING, + required=True, +) @click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs @@ -54,21 +60,13 @@ async def poetry(pipeline_context: ClickPipelineContext) -> None: type=click.Choice(["pypi", "testpypi"]), default="pypi", ) -@click.option( - "--package-path", - help="The path to publish", - type=click.STRING, - required=True, -) @pass_pipeline_context @click.pass_context async def publish(ctx: click.Context, - package_path: str, + click_pipeline_context: ClickPipelineContext, pypi_username: str, pypi_password: str, - pypi_repository: str, - click_pipeline_context: ClickPipelineContext - ) -> None: + pypi_repository: str) -> None: context = PyPIPublishContext( is_local=ctx.obj["is_local"], git_branch=ctx.obj["git_branch"], @@ -83,9 +81,9 @@ async def publish(ctx: click.Context, pypi_username=pypi_username, pypi_password=pypi_password, pypi_repository=pypi_repository, - package_path=package_path + package_path=ctx.obj["package_path"], ) - dagger_client = await click_pipeline_context.get_dagger_client(pipeline_name=f"Publish {package_path} to PyPI") + dagger_client = await click_pipeline_context.get_dagger_client(pipeline_name=f"Publish {ctx.obj['package_path']} to PyPI") context.dagger_client = dagger_client await PublishToPyPI(context).run() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py index dee547bb28e56..24d1ffafd5047 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py @@ -15,7 +15,7 @@ from pipelines.dagger.actions.python.common import with_pip_packages from pipelines.dagger.containers.python import with_python_base from pipelines.helpers.run_steps import STEP_TREE, StepToRun, run_steps -from pipelines.helpers.utils import DAGGER_CONFIG, get_secret_host_variable +from pipelines.helpers.utils import get_file_contents from pipelines.models.reports import Report from pipelines.models.steps import MountPath, Step, StepResult from pipelines.models.contexts.pipeline_context import PipelineContext @@ -76,16 +76,16 @@ class PublishToPyPI(Step): title = "Publish package to PyPI" async def _run(self) -> StepResult: - context: PyPIPublishContext = self.context + context: PyPIPublishContext = self.context # TODO: Add logic to create a PyPIPublishContext out of a ConnectorContext (check the instance type to decide whether it's necessary) dir_to_publish = await context.get_repo_dir(context.package_path) if not context.package_name or not context.version: # check whether it has a pyproject.toml file - pyproject_toml = dir_to_publish.file("pyproject.toml") - if not await pyproject_toml.exists(): + if "pyproject.toml" not in (await dir_to_publish.entries()): return self.skip("Connector does not have a pyproject.toml file and version and package name is not set otherwise, skipping.") # get package name and version from pyproject.toml + pyproject_toml = dir_to_publish.file("pyproject.toml") pyproject_toml_content = await pyproject_toml.contents() contents = tomli.loads(pyproject_toml_content) if "tool" not in contents or "poetry" not in contents["tool"] or "name" not in contents["tool"]["poetry"] or "version" not in contents["tool"]["poetry"]: @@ -115,7 +115,7 @@ async def _run(self) -> StepResult: .with_directory("package", dir_to_publish) .with_workdir("package") .with_exec(["sed", "-i", "/name=/d; /author=/d; /author_email=/d; /version=/d", "setup.py"]) - .with_new_file("setup.cfg", setup_cfg) + .with_new_file("setup.cfg", contents=setup_cfg) .with_exec(["pip", "install", "--upgrade", "setuptools", "wheel"]) .with_exec(["python", "setup.py", "sdist", "bdist_wheel"]) .with_secret_variable("TWINE_USERNAME", twine_username) From 3cd4f0341af93ae7d0ff53f0faeec50d06fcccc4 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 12 Jan 2024 16:24:30 +0100 Subject: [PATCH 03/53] make work for setup.py and pyproject.toml --- .../pipelines/airbyte_ci/poetry/commands.py | 51 ++++--- .../pipelines/airbyte_ci/poetry/pipeline.py | 132 +++++++++++------- airbyte-ci/connectors/pipelines/poetry.lock | 13 +- .../connectors/pipelines/pyproject.toml | 1 + 4 files changed, 126 insertions(+), 71 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py index 5930832b5766d..6adecc0a75a35 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py @@ -9,7 +9,7 @@ import logging import sys -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional import asyncclick as click from pipelines.airbyte_ci.format.configuration import FORMATTERS_CONFIGURATIONS, Formatter @@ -32,6 +32,12 @@ type=click.STRING, required=True, ) +@click.option( + "--docker-image", + help="The docker image to run the command in.", + type=click.STRING, + default="mwalbeck/python-poetry" +) @click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs @@ -41,32 +47,37 @@ async def poetry(pipeline_context: ClickPipelineContext) -> None: @poetry.command(cls=DaggerPipelineCommand, name="publish", help="Publish a Python package to PyPI.") @click.option( - "--pypi-username", - help="Your username to connect to PyPI.", + "--pypi-token", + help="Access token", type=click.STRING, required=True, - envvar="PYPI_USERNAME", + envvar="PYPI_TOKEN", +) +@click.option( + "--test-pypi", + help="Whether to publish to test.pypi.org instead of pypi.org.", + type=click.BOOL, + is_flag=True, + default=False, ) @click.option( - "--pypi-password", - help="Your password to connect to PyPI.", + "--publish-name", + help="The name of the package to publish. If not set, the name will be inferred from the pyproject.toml file of the package.", type=click.STRING, - required=True, - envvar="PYPI_PASSWORD", ) @click.option( - "--pypi-repository", - help="The PyPI repository to publish to (pypi, test-pypi).", - type=click.Choice(["pypi", "testpypi"]), - default="pypi", + "--publish-version", + help="The version of the package to publish. If not set, the version will be inferred from the pyproject.toml file of the package.", + type=click.STRING, ) @pass_pipeline_context @click.pass_context async def publish(ctx: click.Context, click_pipeline_context: ClickPipelineContext, - pypi_username: str, - pypi_password: str, - pypi_repository: str) -> None: + pypi_token: str, + test_pypi: bool, + publish_name: Optional[str], + publish_version: Optional[str]) -> None: context = PyPIPublishContext( is_local=ctx.obj["is_local"], git_branch=ctx.obj["git_branch"], @@ -78,13 +89,17 @@ async def publish(ctx: click.Context, pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), ci_context=ctx.obj.get("ci_context"), ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], - pypi_username=pypi_username, - pypi_password=pypi_password, - pypi_repository=pypi_repository, + pypi_token=pypi_token, + test_pypi=test_pypi, package_path=ctx.obj["package_path"], + build_docker_image=ctx.obj["docker_image"], + package_name=publish_name, + version=publish_version, ) dagger_client = await click_pipeline_context.get_dagger_client(pipeline_name=f"Publish {ctx.obj['package_path']} to PyPI") context.dagger_client = dagger_client await PublishToPyPI(context).run() + return True + diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py index 24d1ffafd5047..bb64cc9ee6ef3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py @@ -3,38 +3,27 @@ # import tomli -import uuid +import tomli_w from typing import Optional +import configparser +import io -import dagger -from pipelines.airbyte_ci.connectors.consts import CONNECTOR_TEST_STEP_ID -from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext -from pipelines.airbyte_ci.steps.docker import SimpleDockerStep -from pipelines.airbyte_ci.steps.poetry import PoetryRunStep -from pipelines.consts import DOCS_DIRECTORY_ROOT_PATH, INTERNAL_TOOL_PATHS -from pipelines.dagger.actions.python.common import with_pip_packages -from pipelines.dagger.containers.python import with_python_base -from pipelines.helpers.run_steps import STEP_TREE, StepToRun, run_steps -from pipelines.helpers.utils import get_file_contents -from pipelines.models.reports import Report -from pipelines.models.steps import MountPath, Step, StepResult +from pipelines.airbyte_ci.connectors.context import PipelineContext +from pipelines.models.steps import Step, StepResult from pipelines.models.contexts.pipeline_context import PipelineContext from textwrap import dedent -from typing import List - -import anyio class PyPIPublishContext(PipelineContext): def __init__( self, - pypi_username: str, - pypi_password: str, - pypi_repository: str, + pypi_token: str, + test_pypi: bool, package_path: str, + build_docker_image: str, ci_report_bucket: str, report_output_prefix: str, is_local: bool, @@ -48,12 +37,12 @@ def __init__( package_name: Optional[str] = None, version: Optional[str] = None, ): - self.pypi_username = pypi_username - self.pypi_password = pypi_password - self.pypi_repository = pypi_repository + self.pypi_token = pypi_token + self.test_pypi = test_pypi self.package_path = package_path self.package_name = package_name self.version = version + self.build_docker_image = build_docker_image pipeline_name = f"Publish PyPI {package_path}" @@ -78,10 +67,14 @@ class PublishToPyPI(Step): async def _run(self) -> StepResult: context: PyPIPublishContext = self.context # TODO: Add logic to create a PyPIPublishContext out of a ConnectorContext (check the instance type to decide whether it's necessary) dir_to_publish = await context.get_repo_dir(context.package_path) + + files = await dir_to_publish.entries() + is_poetry_package = "pyproject.toml" in files + is_pip_package = "setup.py" in files if not context.package_name or not context.version: # check whether it has a pyproject.toml file - if "pyproject.toml" not in (await dir_to_publish.entries()): + if not is_poetry_package: return self.skip("Connector does not have a pyproject.toml file and version and package name is not set otherwise, skipping.") # get package name and version from pyproject.toml @@ -89,39 +82,74 @@ async def _run(self) -> StepResult: pyproject_toml_content = await pyproject_toml.contents() contents = tomli.loads(pyproject_toml_content) if "tool" not in contents or "poetry" not in contents["tool"] or "name" not in contents["tool"]["poetry"] or "version" not in contents["tool"]["poetry"]: - return self.skip("Connector does not have a pyproject.toml file with a poetry section, skipping.") + return self.skip("Connector does not have a pyproject.toml file which specifies package name and version and they are not set otherwise, skipping.") context.package_name = contents["tool"]["poetry"]["name"] context.version = contents["tool"]["poetry"]["version"] - setup_cfg = dedent( - f""" - [metadata] - name = {context.package_name} - version = {context.version} - author = Airbyte - author_email = contact@airbyte.io - """ - ) - twine_username = self.context.dagger_client.set_secret("twine_username", context.pypi_username) - twine_password = self.context.dagger_client.set_secret("twine_password", context.pypi_password) - - twine_upload = ( - self.context.dagger_client.container() - .from_("python:3.10-slim") - .with_exec(["apt-get", "update"]) - .with_exec(["apt-get", "install", "-y", "twine"]) - .with_directory("package", dir_to_publish) - .with_workdir("package") - .with_exec(["sed", "-i", "/name=/d; /author=/d; /author_email=/d; /version=/d", "setup.py"]) - .with_new_file("setup.cfg", contents=setup_cfg) - .with_exec(["pip", "install", "--upgrade", "setuptools", "wheel"]) - .with_exec(["python", "setup.py", "sdist", "bdist_wheel"]) - .with_secret_variable("TWINE_USERNAME", twine_username) - .with_secret_variable("TWINE_PASSWORD", twine_password) - .with_exec(["twine", "upload", "--verbose", "--repository", self.context.pypi_repository, "dist/*"]) - ) + print(f"Uploading package {context.package_name} version {context.version} to {'testpypi' if context.test_pypi else 'pypi'}...") + + if is_pip_package: + pypi_username = self.context.dagger_client.set_secret("pypi_username", "__token__") + pypi_password = self.context.dagger_client.set_secret("pypi_password", f"pypi-{context.pypi_token}") + metadata = { + "name": context.package_name, + "version": context.version, + "author": "Airbyte", + "author_email": "contact@airbyte.io", + } + if "README.md" in files: + metadata["long_description"] = await dir_to_publish.file("README.md").contents() + metadata["long_description_content_type"] = "text/markdown" + + # legacy publish logic + config = configparser.ConfigParser() + config["metadata"] = metadata + + setup_cfg_io = io.StringIO() + config.write(setup_cfg_io) + setup_cfg = setup_cfg_io.getvalue() + + twine_upload = ( + self.context.dagger_client.container() + .from_(context.build_docker_image) + .with_exec(["apt-get", "update"]) + .with_exec(["apt-get", "install", "-y", "twine"]) + .with_directory("package", dir_to_publish) + .with_workdir("package") + # clear out setup.py metadata so setup.cfg is used + .with_exec(["sed", "-i", "/name=/d; /author=/d; /author_email=/d; /version=/d", "setup.py"]) + .with_new_file("setup.cfg", contents=setup_cfg) + .with_exec(["pip", "install", "--upgrade", "setuptools", "wheel"]) + .with_exec(["python", "setup.py", "sdist", "bdist_wheel"]) + .with_secret_variable("TWINE_USERNAME", pypi_username) + .with_secret_variable("TWINE_PASSWORD", pypi_password) + .with_exec(["twine", "upload", "--verbose", "--repository", "testpypi" if context.test_pypi else "pypi", "dist/*"]) + ) + + return await self.get_step_result(twine_upload) + else: + pypi_token = self.context.dagger_client.set_secret("pypi_token", f"pypi-{context.pypi_token}") + pyproject_toml = dir_to_publish.file("pyproject.toml") + pyproject_toml_content = await pyproject_toml.contents() + contents = tomli.loads(pyproject_toml_content) + # set package name and version + contents["tool"]["poetry"]["name"] = context.package_name + contents["tool"]["poetry"]["version"] = context.version + # poetry publish logic + poetry_publish = ( + self.context.dagger_client.container() + .from_(context.build_docker_image) + .with_secret_variable("PYPI_TOKEN", pypi_token) + .with_directory("package", dir_to_publish) + .with_workdir("package") + .with_new_file("pyproject.toml", contents=tomli_w.dumps(contents)) + .with_exec(["poetry", "config", "repositories.testpypi", "https://test.pypi.org/legacy/"]) + .with_exec(["sh", "-c", f"poetry config {'pypi-token.testpypi' if context.test_pypi else 'pypi-token.pypi'} $PYPI_TOKEN"]) + .with_exec(["poetry", "publish", "--build", "--repository", "testpypi" if context.test_pypi else "pypi"]) + ) + + return await self.get_step_result(poetry_publish) - return await self.get_step_result(twine_upload) diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index 0aaed21095c41..fb6f1df90cab6 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -2261,6 +2261,17 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tomli-w" +version = "1.0.0" +description = "A lil' TOML writer" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomli_w-1.0.0-py3-none-any.whl", hash = "sha256:9f2a07e8be30a0729e533ec968016807069991ae2fd921a78d42f429ae5f4463"}, + {file = "tomli_w-1.0.0.tar.gz", hash = "sha256:f463434305e0336248cac9c2dc8076b707d8a12d019dd349f5c1e382dd1ae1b9"}, +] + [[package]] name = "typing-extensions" version = "4.9.0" @@ -2511,4 +2522,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "8e1d9bcd933ad7030fa06fd305f20317391111f2b364874e98e594de762fcf57" +content-hash = "860276a3ba4a5809e9775e2df33b02eb1867420e3fdcca7ee4b3471ac66f16df" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 7a8d8266f7b84..94ed81992e1db 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -28,6 +28,7 @@ pygit2 = "^1.13.1" asyncclick = "^8.1.3.4" certifi = "^2023.11.17" tomli = "^2.0.1" +tomli-w = "^1.0.0" [tool.poetry.group.test.dependencies] pytest = "^6.2.5" From 475b1c5b559468e9f95a69227b421ade6b3d86de Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 12 Jan 2024 17:40:30 +0100 Subject: [PATCH 04/53] make publish work --- .../airbyte_ci/connectors/publish/pipeline.py | 37 +++++++++ .../pipelines/airbyte_ci/poetry/commands.py | 19 ++--- .../pipelines/airbyte_ci/poetry/pipeline.py | 76 ++++++++++++++----- .../pipelines/airbyte_ci/poetry/utils.py | 19 +++++ 4 files changed, 120 insertions(+), 31 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index ba61a521ec66c..f0bb43e29b0b9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -14,6 +14,8 @@ from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.airbyte_ci.metadata.pipeline import MetadataUpload, MetadataValidation +from pipelines.airbyte_ci.poetry.pipeline import PublishToPyPI, PyPIPublishContext +from pipelines.airbyte_ci.poetry.utils import is_package_published from pipelines.dagger.actions.remote_storage import upload_to_gcs from pipelines.dagger.actions.system import docker from pipelines.models.steps import Step, StepResult, StepStatus @@ -52,6 +54,22 @@ async def _run(self) -> StepResult: return StepResult(self, status=StepStatus.SUCCESS, stdout=f"No manifest found for {self.context.docker_image}.") +class CheckPypiPackageExists(Step): + context: PyPIPublishContext + title = "Check if the connector is published on pypi" + + async def _run(self) -> StepResult: + is_published = is_package_published(self.context.package_name, self.context.version, self.context.test_pypi) + if is_published: + return StepResult( + self, status=StepStatus.SKIPPED, stderr=f"{self.context.package_name} already exists in version {self.context.version}." + ) + else: + return StepResult( + self, status=StepStatus.SUCCESS, stdout=f"{self.context.package_name} does not exist in version {self.context.version}." + ) + + class PushConnectorImageToRegistry(Step): context: PublishConnectorContext title = "Push connector image to registry" @@ -273,6 +291,25 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: metadata_upload_results = await metadata_upload_step.run() results.append(metadata_upload_results) + if ( + "remoteRegistries" in context.connector.metadata["metadata"] + and "pypi" in context.connector.metadata["metadata"]["remoteRegistries"] + and context.connector.metadata["metadata"]["remoteRegistries"]["pypi"]["enabled"] + ): + pypi_context = PyPIPublishContext.from_connector_context(context) + check_pypi_package_exists_results = await CheckPypiPackageExists(pypi_context).run() + results.append(check_pypi_package_exists_results) + if check_pypi_package_exists_results.status is StepStatus.SKIPPED: + context.logger.info("The connector version is already published on pypi.") + elif check_pypi_package_exists_results.status is StepStatus.SUCCESS: + context.logger.info("The connector version is not published on pypi. Let's build and publish it.") + publish_to_pypi_results = await PublishToPyPI(pypi_context).run() + results.append(publish_to_pypi_results) + if publish_to_pypi_results.status is StepStatus.FAILURE: + return create_connector_report(results) + elif check_pypi_package_exists_results.status is StepStatus.FAILURE: + return create_connector_report(results) + # Exit early if the connector image already exists or has failed to build if check_connector_image_results.status is not StepStatus.SUCCESS: return create_connector_report(results) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py index 6adecc0a75a35..796ead980110f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py @@ -15,11 +15,12 @@ from pipelines.airbyte_ci.format.configuration import FORMATTERS_CONFIGURATIONS, Formatter from pipelines.airbyte_ci.format.format_command import FormatCommand from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.helpers.cli import LogOptions, invoke_commands_concurrently, invoke_commands_sequentially, log_command_results from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context from pipelines.models.steps import StepStatus -from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand -from .pipeline import PyPIPublishContext, PublishToPyPI + +from .pipeline import PublishToPyPI, PyPIPublishContext @click.group( @@ -32,12 +33,7 @@ type=click.STRING, required=True, ) -@click.option( - "--docker-image", - help="The docker image to run the command in.", - type=click.STRING, - default="mwalbeck/python-poetry" -) +@click.option("--docker-image", help="The docker image to run the command in.", type=click.STRING, default="mwalbeck/python-poetry") @click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs @@ -72,12 +68,14 @@ async def poetry(pipeline_context: ClickPipelineContext) -> None: ) @pass_pipeline_context @click.pass_context -async def publish(ctx: click.Context, +async def publish( + ctx: click.Context, click_pipeline_context: ClickPipelineContext, pypi_token: str, test_pypi: bool, publish_name: Optional[str], - publish_version: Optional[str]) -> None: + publish_version: Optional[str], +) -> None: context = PyPIPublishContext( is_local=ctx.obj["is_local"], git_branch=ctx.obj["git_branch"], @@ -102,4 +100,3 @@ async def publish(ctx: click.Context, await PublishToPyPI(context).run() return True - diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py index bb64cc9ee6ef3..1d0b73fba990c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py @@ -2,19 +2,17 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import tomli -import tomli_w -from typing import Optional import configparser import io +import os +from textwrap import dedent +from typing import Optional -from pipelines.airbyte_ci.connectors.context import PipelineContext -from pipelines.models.steps import Step, StepResult +import tomli +import tomli_w +from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.models.contexts.pipeline_context import PipelineContext - - - -from textwrap import dedent +from pipelines.models.steps import Step, StepResult class PyPIPublishContext(PipelineContext): @@ -60,14 +58,46 @@ def __init__( ci_gcs_credentials=ci_gcs_credentials, ) + @staticmethod + def from_connector_context(connector_context: ConnectorContext) -> "PyPIPublishContext": + if ( + connector_context.connector.metadata["connectorBuildOptions"] + and "baseImage" in connector_context.connector.metadata["connectorBuildOptions"] + ): + build_docker_image = connector_context.connector.metadata["connectorBuildOptions"]["baseImage"] + else: + build_docker_image = "mwalbeck/python-poetry" + # copy everything except the connector + # the package_path is the path to the connector, the package name is airbyte-{connector_name} and the version is the docker image tag of the connector + return PyPIPublishContext( + pypi_token=os.environ["PYPI_TOKEN"], + test_pypi=True, + package_path=connector_context.connector.code_directory, + package_name=connector_context.connector.metadata["name"], + version=connector_context.connector.metadata["dockerImageTag"], + build_docker_image=build_docker_image, + ci_report_bucket=connector_context.ci_report_bucket, + report_output_prefix=connector_context.report_output_prefix, + is_local=connector_context.is_local, + git_branch=connector_context.git_branch, + git_revision=connector_context.git_revision, + gha_workflow_run_url=connector_context.gha_workflow_run_url, + dagger_logs_url=connector_context.dagger_logs_url, + pipeline_start_timestamp=connector_context.pipeline_start_timestamp, + ci_context=connector_context.ci_context, + ci_gcs_credentials=connector_context.ci_gcs_credentials, + ) + class PublishToPyPI(Step): title = "Publish package to PyPI" async def _run(self) -> StepResult: - context: PyPIPublishContext = self.context # TODO: Add logic to create a PyPIPublishContext out of a ConnectorContext (check the instance type to decide whether it's necessary) + context: PyPIPublishContext = ( + self.context + ) # TODO: Add logic to create a PyPIPublishContext out of a ConnectorContext (check the instance type to decide whether it's necessary) dir_to_publish = await context.get_repo_dir(context.package_path) - + files = await dir_to_publish.entries() is_poetry_package = "pyproject.toml" in files is_pip_package = "setup.py" in files @@ -75,21 +105,29 @@ async def _run(self) -> StepResult: if not context.package_name or not context.version: # check whether it has a pyproject.toml file if not is_poetry_package: - return self.skip("Connector does not have a pyproject.toml file and version and package name is not set otherwise, skipping.") - + return self.skip( + "Connector does not have a pyproject.toml file and version and package name is not set otherwise, skipping." + ) + # get package name and version from pyproject.toml pyproject_toml = dir_to_publish.file("pyproject.toml") pyproject_toml_content = await pyproject_toml.contents() contents = tomli.loads(pyproject_toml_content) - if "tool" not in contents or "poetry" not in contents["tool"] or "name" not in contents["tool"]["poetry"] or "version" not in contents["tool"]["poetry"]: - return self.skip("Connector does not have a pyproject.toml file which specifies package name and version and they are not set otherwise, skipping.") - + if ( + "tool" not in contents + or "poetry" not in contents["tool"] + or "name" not in contents["tool"]["poetry"] + or "version" not in contents["tool"]["poetry"] + ): + return self.skip( + "Connector does not have a pyproject.toml file which specifies package name and version and they are not set otherwise, skipping." + ) + context.package_name = contents["tool"]["poetry"]["name"] context.version = contents["tool"]["poetry"]["version"] - print(f"Uploading package {context.package_name} version {context.version} to {'testpypi' if context.test_pypi else 'pypi'}...") - + if is_pip_package: pypi_username = self.context.dagger_client.set_secret("pypi_username", "__token__") pypi_password = self.context.dagger_client.set_secret("pypi_password", f"pypi-{context.pypi_token}") @@ -151,5 +189,3 @@ async def _run(self) -> StepResult: ) return await self.get_step_result(poetry_publish) - - diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py new file mode 100644 index 0000000000000..b0d6d2f076502 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py @@ -0,0 +1,19 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +import requests + + +def is_package_published(package_name, version, test_pypi=False): + """ + Check if a package with a specific version is published on PyPI or Test PyPI. + + :param package_name: The name of the package to check. + :param version: The version of the package. + :param test_pypi: Set to True to check on Test PyPI, False for regular PyPI. + :return: True if the package is found with the specified version, False otherwise. + """ + base_url = "https://test.pypi.org/pypi" if test_pypi else "https://pypi.org/pypi" + url = f"{base_url}/{package_name}/{version}/json" + + response = requests.get(url) + return response.status_code == 200 From 0360603151196288362d474939a30436bbd0a4e6 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 12 Jan 2024 17:47:38 +0100 Subject: [PATCH 05/53] fix --- .../pipelines/airbyte_ci/poetry/pipeline.py | 54 +++++++++---------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py index 1d0b73fba990c..3c1e735e607c9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py @@ -67,11 +67,9 @@ def from_connector_context(connector_context: ConnectorContext) -> "PyPIPublishC build_docker_image = connector_context.connector.metadata["connectorBuildOptions"]["baseImage"] else: build_docker_image = "mwalbeck/python-poetry" - # copy everything except the connector - # the package_path is the path to the connector, the package name is airbyte-{connector_name} and the version is the docker image tag of the connector return PyPIPublishContext( pypi_token=os.environ["PYPI_TOKEN"], - test_pypi=True, + test_pypi=True, # TODO: Go live package_path=connector_context.connector.code_directory, package_name=connector_context.connector.metadata["name"], version=connector_context.connector.metadata["dockerImageTag"], @@ -90,26 +88,24 @@ def from_connector_context(connector_context: ConnectorContext) -> "PyPIPublishC class PublishToPyPI(Step): + context: PyPIPublishContext title = "Publish package to PyPI" async def _run(self) -> StepResult: - context: PyPIPublishContext = ( - self.context - ) # TODO: Add logic to create a PyPIPublishContext out of a ConnectorContext (check the instance type to decide whether it's necessary) - dir_to_publish = await context.get_repo_dir(context.package_path) + dir_to_publish = await self.context.get_repo_dir(self.context.package_path) files = await dir_to_publish.entries() is_poetry_package = "pyproject.toml" in files is_pip_package = "setup.py" in files - if not context.package_name or not context.version: - # check whether it has a pyproject.toml file + # Try to infer package name and version from the pyproject.toml file. If it is not present, we need to have the package name and version set + # Setup.py packages need to set package name and version as parameter + if not self.context.package_name or not self.context.version: if not is_poetry_package: return self.skip( "Connector does not have a pyproject.toml file and version and package name is not set otherwise, skipping." ) - # get package name and version from pyproject.toml pyproject_toml = dir_to_publish.file("pyproject.toml") pyproject_toml_content = await pyproject_toml.contents() contents = tomli.loads(pyproject_toml_content) @@ -123,17 +119,20 @@ async def _run(self) -> StepResult: "Connector does not have a pyproject.toml file which specifies package name and version and they are not set otherwise, skipping." ) - context.package_name = contents["tool"]["poetry"]["name"] - context.version = contents["tool"]["poetry"]["version"] + self.context.package_name = contents["tool"]["poetry"]["name"] + self.context.version = contents["tool"]["poetry"]["version"] - print(f"Uploading package {context.package_name} version {context.version} to {'testpypi' if context.test_pypi else 'pypi'}...") + print( + f"Uploading package {self.context.package_name} version {self.context.version} to {'testpypi' if self.context.test_pypi else 'pypi'}..." + ) if is_pip_package: + # legacy publish logic pypi_username = self.context.dagger_client.set_secret("pypi_username", "__token__") - pypi_password = self.context.dagger_client.set_secret("pypi_password", f"pypi-{context.pypi_token}") + pypi_password = self.context.dagger_client.set_secret("pypi_password", f"pypi-{self.context.pypi_token}") metadata = { - "name": context.package_name, - "version": context.version, + "name": self.context.package_name, + "version": self.context.version, "author": "Airbyte", "author_email": "contact@airbyte.io", } @@ -141,7 +140,6 @@ async def _run(self) -> StepResult: metadata["long_description"] = await dir_to_publish.file("README.md").contents() metadata["long_description_content_type"] = "text/markdown" - # legacy publish logic config = configparser.ConfigParser() config["metadata"] = metadata @@ -151,7 +149,7 @@ async def _run(self) -> StepResult: twine_upload = ( self.context.dagger_client.container() - .from_(context.build_docker_image) + .from_(self.context.build_docker_image) .with_exec(["apt-get", "update"]) .with_exec(["apt-get", "install", "-y", "twine"]) .with_directory("package", dir_to_publish) @@ -163,29 +161,31 @@ async def _run(self) -> StepResult: .with_exec(["python", "setup.py", "sdist", "bdist_wheel"]) .with_secret_variable("TWINE_USERNAME", pypi_username) .with_secret_variable("TWINE_PASSWORD", pypi_password) - .with_exec(["twine", "upload", "--verbose", "--repository", "testpypi" if context.test_pypi else "pypi", "dist/*"]) + .with_exec(["twine", "upload", "--verbose", "--repository", "testpypi" if self.context.test_pypi else "pypi", "dist/*"]) ) return await self.get_step_result(twine_upload) else: - pypi_token = self.context.dagger_client.set_secret("pypi_token", f"pypi-{context.pypi_token}") + # poetry publish logic + pypi_token = self.context.dagger_client.set_secret("pypi_token", f"pypi-{self.context.pypi_token}") pyproject_toml = dir_to_publish.file("pyproject.toml") pyproject_toml_content = await pyproject_toml.contents() contents = tomli.loads(pyproject_toml_content) - # set package name and version - contents["tool"]["poetry"]["name"] = context.package_name - contents["tool"]["poetry"]["version"] = context.version - # poetry publish logic + # make sure package name and version are set to the configured one + contents["tool"]["poetry"]["name"] = self.context.package_name + contents["tool"]["poetry"]["version"] = self.context.version poetry_publish = ( self.context.dagger_client.container() - .from_(context.build_docker_image) + .from_(self.context.build_docker_image) .with_secret_variable("PYPI_TOKEN", pypi_token) .with_directory("package", dir_to_publish) .with_workdir("package") .with_new_file("pyproject.toml", contents=tomli_w.dumps(contents)) .with_exec(["poetry", "config", "repositories.testpypi", "https://test.pypi.org/legacy/"]) - .with_exec(["sh", "-c", f"poetry config {'pypi-token.testpypi' if context.test_pypi else 'pypi-token.pypi'} $PYPI_TOKEN"]) - .with_exec(["poetry", "publish", "--build", "--repository", "testpypi" if context.test_pypi else "pypi"]) + .with_exec( + ["sh", "-c", f"poetry config {'pypi-token.testpypi' if self.context.test_pypi else 'pypi-token.pypi'} $PYPI_TOKEN"] + ) + .with_exec(["poetry", "publish", "--build", "--repository", "testpypi" if self.context.test_pypi else "pypi"]) ) return await self.get_step_result(poetry_publish) From 5f6be6cf1620a37e45aba1a7be7d95ff218f5189 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Fri, 12 Jan 2024 17:59:48 +0100 Subject: [PATCH 06/53] pass pypi token through --- .github/workflows/publish_connectors.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index 228faae2abd94..4db06a255e8f2 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -57,6 +57,7 @@ jobs: with: context: "manual" dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} + pypi_token: ${{ secrets.PYPI_TOKEN }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} From cf698f22360a879f4cf29dcb14b28fd0ef7c2064 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 14:46:59 +0100 Subject: [PATCH 07/53] prepare test --- .../pipelines/pipelines/airbyte_ci/poetry/pipeline.py | 2 +- .../connectors/source-apify-dataset/metadata.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py index 3c1e735e607c9..db54b2264a6fc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py @@ -71,7 +71,7 @@ def from_connector_context(connector_context: ConnectorContext) -> "PyPIPublishC pypi_token=os.environ["PYPI_TOKEN"], test_pypi=True, # TODO: Go live package_path=connector_context.connector.code_directory, - package_name=connector_context.connector.metadata["name"], + package_name=connector_context.connector.metadata["remoteRegistries"]["pypi"]["packageName"], version=connector_context.connector.metadata["dockerImageTag"], build_docker_image=build_docker_image, ci_report_bucket=connector_context.ci_report_bucket, diff --git a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml index f3421f116b826..6dbc2f6b80318 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml +++ b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml @@ -16,6 +16,10 @@ data: icon: apify.svg license: MIT name: Apify Dataset + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-apify-dataset releaseDate: 2023-08-25 releaseStage: alpha releases: From 44f845482afc15890722dc402575256f7a6a6338 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 14:51:39 +0100 Subject: [PATCH 08/53] prepare test --- .../connectors/source-apify-dataset/metadata.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml index 6dbc2f6b80318..b85a146d84663 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml +++ b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml @@ -34,4 +34,5 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/apify-dataset tags: - language:lowcode + - language:python metadataSpecVersion: "1.0" From 6874dc8022c14b7b44ba15444f7a1750ea23464b Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 14:58:42 +0100 Subject: [PATCH 09/53] fix bug --- .../pipelines/airbyte_ci/connectors/publish/pipeline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index f0bb43e29b0b9..e5540904b4508 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -292,9 +292,9 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: results.append(metadata_upload_results) if ( - "remoteRegistries" in context.connector.metadata["metadata"] - and "pypi" in context.connector.metadata["metadata"]["remoteRegistries"] - and context.connector.metadata["metadata"]["remoteRegistries"]["pypi"]["enabled"] + "remoteRegistries" in context.connector.metadata + and "pypi" in context.connector.metadata["remoteRegistries"] + and context.connector.metadata["remoteRegistries"]["pypi"]["enabled"] ): pypi_context = PyPIPublishContext.from_connector_context(context) check_pypi_package_exists_results = await CheckPypiPackageExists(pypi_context).run() From 7c3bb25159a49d3825a523953f7e088780278f70 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 15:03:11 +0100 Subject: [PATCH 10/53] fix bug --- .../pipelines/pipelines/airbyte_ci/poetry/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py index db54b2264a6fc..dea6d08769dd1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py @@ -61,7 +61,7 @@ def __init__( @staticmethod def from_connector_context(connector_context: ConnectorContext) -> "PyPIPublishContext": if ( - connector_context.connector.metadata["connectorBuildOptions"] + "connectorBuildOptions" in connector_context.connector.metadata and "baseImage" in connector_context.connector.metadata["connectorBuildOptions"] ): build_docker_image = connector_context.connector.metadata["connectorBuildOptions"]["baseImage"] From 43cf70a571e11f7dfa486f3ba637a8290b1361fe Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 15:07:08 +0100 Subject: [PATCH 11/53] pass in secret properly --- .github/workflows/publish_connectors.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index 4db06a255e8f2..c70cdba62f8ec 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -57,7 +57,6 @@ jobs: with: context: "manual" dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }} - pypi_token: ${{ secrets.PYPI_TOKEN }} docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} @@ -72,6 +71,8 @@ jobs: tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}" airbyte_ci_binary_url: ${{ github.event.inputs.airbyte-ci-binary-url }} + env: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} set-instatus-incident-on-failure: name: Create Instatus Incident on Failure From bd0d63615bb3dad3b806e731e7a38816d209bcff Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 15:15:56 +0100 Subject: [PATCH 12/53] add debug --- .../pipelines/airbyte_ci/connectors/publish/pipeline.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index e5540904b4508..1684cdf848c80 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -291,12 +291,14 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: metadata_upload_results = await metadata_upload_step.run() results.append(metadata_upload_results) + print(str(context.connector.metadata["remoteRegistries"])) if ( "remoteRegistries" in context.connector.metadata and "pypi" in context.connector.metadata["remoteRegistries"] and context.connector.metadata["remoteRegistries"]["pypi"]["enabled"] ): pypi_context = PyPIPublishContext.from_connector_context(context) + print(str(pypi_context)) check_pypi_package_exists_results = await CheckPypiPackageExists(pypi_context).run() results.append(check_pypi_package_exists_results) if check_pypi_package_exists_results.status is StepStatus.SKIPPED: From 7cc0274b0740887204a1ee1ccda3c17d797fab68 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 15:18:34 +0100 Subject: [PATCH 13/53] remove debug --- .../pipelines/airbyte_ci/connectors/publish/pipeline.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index 1684cdf848c80..e5540904b4508 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -291,14 +291,12 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: metadata_upload_results = await metadata_upload_step.run() results.append(metadata_upload_results) - print(str(context.connector.metadata["remoteRegistries"])) if ( "remoteRegistries" in context.connector.metadata and "pypi" in context.connector.metadata["remoteRegistries"] and context.connector.metadata["remoteRegistries"]["pypi"]["enabled"] ): pypi_context = PyPIPublishContext.from_connector_context(context) - print(str(pypi_context)) check_pypi_package_exists_results = await CheckPypiPackageExists(pypi_context).run() results.append(check_pypi_package_exists_results) if check_pypi_package_exists_results.status is StepStatus.SKIPPED: From e252b4f435f14a9a7161cc3925849ab51383484f Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 15:19:53 +0100 Subject: [PATCH 14/53] prepare google drive --- .../connectors/source-google-drive/metadata.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte-integrations/connectors/source-google-drive/metadata.yaml b/airbyte-integrations/connectors/source-google-drive/metadata.yaml index bf40629baae06..fab6905514c00 100644 --- a/airbyte-integrations/connectors/source-google-drive/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-drive/metadata.yaml @@ -18,6 +18,10 @@ data: enabled: true oss: enabled: true + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-google-drive releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/google-drive tags: From 439f3925c06294c80e7ab02f38952755be4a0dd5 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 15:29:37 +0100 Subject: [PATCH 15/53] add debug --- .../pipelines/airbyte_ci/connectors/publish/pipeline.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index e5540904b4508..1684cdf848c80 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -291,12 +291,14 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: metadata_upload_results = await metadata_upload_step.run() results.append(metadata_upload_results) + print(str(context.connector.metadata["remoteRegistries"])) if ( "remoteRegistries" in context.connector.metadata and "pypi" in context.connector.metadata["remoteRegistries"] and context.connector.metadata["remoteRegistries"]["pypi"]["enabled"] ): pypi_context = PyPIPublishContext.from_connector_context(context) + print(str(pypi_context)) check_pypi_package_exists_results = await CheckPypiPackageExists(pypi_context).run() results.append(check_pypi_package_exists_results) if check_pypi_package_exists_results.status is StepStatus.SKIPPED: From ab4bd79ec91867bd6e886acd893efebc5a8911ae Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 15:34:05 +0100 Subject: [PATCH 16/53] fix bug --- .../pipelines/pipelines/airbyte_ci/poetry/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py index dea6d08769dd1..63c067135b2f9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py @@ -70,7 +70,7 @@ def from_connector_context(connector_context: ConnectorContext) -> "PyPIPublishC return PyPIPublishContext( pypi_token=os.environ["PYPI_TOKEN"], test_pypi=True, # TODO: Go live - package_path=connector_context.connector.code_directory, + package_path=str(connector_context.connector.code_directory), package_name=connector_context.connector.metadata["remoteRegistries"]["pypi"]["packageName"], version=connector_context.connector.metadata["dockerImageTag"], build_docker_image=build_docker_image, From 6de583af6ed580aad7d90dd3e1c639abd93c6fcf Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 15:37:44 +0100 Subject: [PATCH 17/53] fix bug --- .../pipelines/pipelines/airbyte_ci/poetry/pipeline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py index 63c067135b2f9..4cd2271db1321 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py @@ -67,7 +67,7 @@ def from_connector_context(connector_context: ConnectorContext) -> "PyPIPublishC build_docker_image = connector_context.connector.metadata["connectorBuildOptions"]["baseImage"] else: build_docker_image = "mwalbeck/python-poetry" - return PyPIPublishContext( + pypi_context = PyPIPublishContext( pypi_token=os.environ["PYPI_TOKEN"], test_pypi=True, # TODO: Go live package_path=str(connector_context.connector.code_directory), @@ -85,6 +85,7 @@ def from_connector_context(connector_context: ConnectorContext) -> "PyPIPublishC ci_context=connector_context.ci_context, ci_gcs_credentials=connector_context.ci_gcs_credentials, ) + pypi_context.dagger_client = connector_context.dagger_client class PublishToPyPI(Step): From 08103fc538619ae95f084d8b2d01daa14cee1a78 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 15:37:59 +0100 Subject: [PATCH 18/53] fix bug --- .../connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py index 4cd2271db1321..9c4cf54d7050a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py @@ -86,6 +86,7 @@ def from_connector_context(connector_context: ConnectorContext) -> "PyPIPublishC ci_gcs_credentials=connector_context.ci_gcs_credentials, ) pypi_context.dagger_client = connector_context.dagger_client + return pypi_context class PublishToPyPI(Step): From 415dc2315bb584a2926fab6fbbef45fac4357be8 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 15:59:20 +0100 Subject: [PATCH 19/53] use most up to date metadata --- .../airbyte_ci/connectors/publish/pipeline.py | 11 ++------ .../pipelines/airbyte_ci/poetry/pipeline.py | 28 +++++++++++++++---- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index 1684cdf848c80..2772eff210fa1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -291,14 +291,9 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: metadata_upload_results = await metadata_upload_step.run() results.append(metadata_upload_results) - print(str(context.connector.metadata["remoteRegistries"])) - if ( - "remoteRegistries" in context.connector.metadata - and "pypi" in context.connector.metadata["remoteRegistries"] - and context.connector.metadata["remoteRegistries"]["pypi"]["enabled"] - ): - pypi_context = PyPIPublishContext.from_connector_context(context) - print(str(pypi_context)) + # Try to convert the context to a PyPIPublishContext. If it returns None, it means we don't need to publish to pypi. + pypi_context = await PyPIPublishContext.from_connector_context(context) + if pypi_context: check_pypi_package_exists_results = await CheckPypiPackageExists(pypi_context).run() results.append(check_pypi_package_exists_results) if check_pypi_package_exists_results.status is StepStatus.SKIPPED: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py index 9c4cf54d7050a..3d38c0b96275b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py @@ -7,6 +7,7 @@ import os from textwrap import dedent from typing import Optional +import yaml import tomli import tomli_w @@ -59,20 +60,35 @@ def __init__( ) @staticmethod - def from_connector_context(connector_context: ConnectorContext) -> "PyPIPublishContext": + async def from_connector_context(connector_context: ConnectorContext) -> Optional["PyPIPublishContext"]: + """ + Create a PyPIPublishContext from a ConnectorContext. + + The metadata of the connector is read from the current workdir to capture changes that are not yet published. + If pypi is not enabled, this will return None. + """ + + current_metadata = yaml.safe_load(await connector_context.get_repo_file(connector_context.connector.metadata_file_path).contents())["data"] + if( + not "remoteRegistries" in current_metadata + or not "pypi" in current_metadata["remoteRegistries"] + or not current_metadata["remoteRegistries"]["pypi"]["enabled"] + ): + return None + if ( - "connectorBuildOptions" in connector_context.connector.metadata - and "baseImage" in connector_context.connector.metadata["connectorBuildOptions"] + "connectorBuildOptions" in current_metadata + and "baseImage" in current_metadata["connectorBuildOptions"] ): - build_docker_image = connector_context.connector.metadata["connectorBuildOptions"]["baseImage"] + build_docker_image = current_metadata["connectorBuildOptions"]["baseImage"] else: build_docker_image = "mwalbeck/python-poetry" pypi_context = PyPIPublishContext( pypi_token=os.environ["PYPI_TOKEN"], test_pypi=True, # TODO: Go live package_path=str(connector_context.connector.code_directory), - package_name=connector_context.connector.metadata["remoteRegistries"]["pypi"]["packageName"], - version=connector_context.connector.metadata["dockerImageTag"], + package_name=current_metadata["remoteRegistries"]["pypi"]["packageName"], + version=current_metadata["dockerImageTag"], build_docker_image=build_docker_image, ci_report_bucket=connector_context.ci_report_bucket, report_output_prefix=connector_context.report_output_prefix, From 9e71444e42053eca434685041fdf7b5f3ce61069 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 16:03:14 +0100 Subject: [PATCH 20/53] prepare pokeapi --- .../connectors/source-pokeapi/Dockerfile | 2 +- .../connectors/source-pokeapi/main.py | 8 ++------ .../connectors/source-pokeapi/metadata.yaml | 6 +++++- .../connectors/source-pokeapi/setup.py | 5 +++++ .../connectors/source-pokeapi/source_pokeapi/run.py | 12 ++++++++++++ 5 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py diff --git a/airbyte-integrations/connectors/source-pokeapi/Dockerfile b/airbyte-integrations/connectors/source-pokeapi/Dockerfile index 0d27d3737d6fa..9bf098140d9af 100644 --- a/airbyte-integrations/connectors/source-pokeapi/Dockerfile +++ b/airbyte-integrations/connectors/source-pokeapi/Dockerfile @@ -34,5 +34,5 @@ COPY source_pokeapi ./source_pokeapi ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.0 +LABEL io.airbyte.version=0.2.1 LABEL io.airbyte.name=airbyte/source-pokeapi diff --git a/airbyte-integrations/connectors/source-pokeapi/main.py b/airbyte-integrations/connectors/source-pokeapi/main.py index 38a510a3f2d77..f32ce6b381058 100644 --- a/airbyte-integrations/connectors/source-pokeapi/main.py +++ b/airbyte-integrations/connectors/source-pokeapi/main.py @@ -2,11 +2,7 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import sys - -from airbyte_cdk.entrypoint import launch -from source_pokeapi import SourcePokeapi +from source_pokeapi.run import run if __name__ == "__main__": - source = SourcePokeapi() - launch(source, sys.argv[1:]) + run() diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index 85047f016528f..aca63bfa05b91 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6371b14b-bc68-4236-bfbd-468e8df8e968 - dockerImageTag: 0.2.0 + dockerImageTag: 0.2.1 dockerRepository: airbyte/source-pokeapi githubIssueLabel: source-pokeapi icon: pokeapi.svg @@ -19,6 +19,10 @@ data: releaseDate: "2020-05-14" releaseStage: alpha supportLevel: community + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pokeapi documentationUrl: https://docs.airbyte.com/integrations/sources/pokeapi tags: - language:lowcode diff --git a/airbyte-integrations/connectors/source-pokeapi/setup.py b/airbyte-integrations/connectors/source-pokeapi/setup.py index 2fa7839b58fca..ece0a748d58e8 100644 --- a/airbyte-integrations/connectors/source-pokeapi/setup.py +++ b/airbyte-integrations/connectors/source-pokeapi/setup.py @@ -26,4 +26,9 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, + entry_points={ + "console_scripts": [ + "source-pokeapi=source_pokeapi.run:run", + ], + }, ) diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py new file mode 100644 index 0000000000000..9f7014e9dd53a --- /dev/null +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py @@ -0,0 +1,12 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pokeapi import SourcePokeapi + +def run(): + source = SourcePokeapi() + launch(source, sys.argv[1:]) From 96a0715cc4a9aae0685e0deeea667dfe097354f0 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 16:05:50 +0100 Subject: [PATCH 21/53] prepare pokeapi --- airbyte-integrations/connectors/source-pokeapi/metadata.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index aca63bfa05b91..d1b40f63190ea 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -26,4 +26,5 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/pokeapi tags: - language:lowcode + - language:python metadataSpecVersion: "1.0" From d6b70d0ab9060a53f96f6a3546d55d6e05bbd2d2 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 16:15:20 +0100 Subject: [PATCH 22/53] debug --- .../connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py index 3d38c0b96275b..4439284eb9dfe 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py @@ -69,6 +69,7 @@ async def from_connector_context(connector_context: ConnectorContext) -> Optiona """ current_metadata = yaml.safe_load(await connector_context.get_repo_file(connector_context.connector.metadata_file_path).contents())["data"] + print(current_metadata) if( not "remoteRegistries" in current_metadata or not "pypi" in current_metadata["remoteRegistries"] From dd7443a3d1aed61bd7937b6782a8f9b45dba7bd1 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 16:20:25 +0100 Subject: [PATCH 23/53] fix --- .../pipelines/pipelines/airbyte_ci/poetry/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py index 4439284eb9dfe..a867e85e3a32c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py @@ -68,7 +68,7 @@ async def from_connector_context(connector_context: ConnectorContext) -> Optiona If pypi is not enabled, this will return None. """ - current_metadata = yaml.safe_load(await connector_context.get_repo_file(connector_context.connector.metadata_file_path).contents())["data"] + current_metadata = yaml.safe_load(await connector_context.get_repo_file(str(connector_context.connector.metadata_file_path)).contents())["data"] print(current_metadata) if( not "remoteRegistries" in current_metadata From 6063ec797d57537c8722d2c077e06e38c6ff3667 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Mon, 15 Jan 2024 16:40:25 +0100 Subject: [PATCH 24/53] format --- .../pipelines/airbyte_ci/poetry/pipeline.py | 19 +++++++++---------- .../source-pokeapi/source_pokeapi/run.py | 1 + 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py index a867e85e3a32c..40e6c6a6d29bb 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py @@ -7,10 +7,10 @@ import os from textwrap import dedent from typing import Optional -import yaml import tomli import tomli_w +import yaml from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import Step, StepResult @@ -68,19 +68,18 @@ async def from_connector_context(connector_context: ConnectorContext) -> Optiona If pypi is not enabled, this will return None. """ - current_metadata = yaml.safe_load(await connector_context.get_repo_file(str(connector_context.connector.metadata_file_path)).contents())["data"] + current_metadata = yaml.safe_load( + await connector_context.get_repo_file(str(connector_context.connector.metadata_file_path)).contents() + )["data"] print(current_metadata) - if( - not "remoteRegistries" in current_metadata - or not "pypi" in current_metadata["remoteRegistries"] - or not current_metadata["remoteRegistries"]["pypi"]["enabled"] + if ( + not "remoteRegistries" in current_metadata + or not "pypi" in current_metadata["remoteRegistries"] + or not current_metadata["remoteRegistries"]["pypi"]["enabled"] ): return None - if ( - "connectorBuildOptions" in current_metadata - and "baseImage" in current_metadata["connectorBuildOptions"] - ): + if "connectorBuildOptions" in current_metadata and "baseImage" in current_metadata["connectorBuildOptions"]: build_docker_image = current_metadata["connectorBuildOptions"]["baseImage"] else: build_docker_image = "mwalbeck/python-poetry" diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py index 9f7014e9dd53a..2b573e6939542 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py @@ -7,6 +7,7 @@ from airbyte_cdk.entrypoint import launch from source_pokeapi import SourcePokeapi + def run(): source = SourcePokeapi() launch(source, sys.argv[1:]) From 2069bbd356b614e8a5bf5be4e7a9cb1be38733b6 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 18 Jan 2024 12:15:09 +0100 Subject: [PATCH 25/53] review comments --- .../actions/run-dagger-pipeline/action.yml | 4 + .github/workflows/publish_connectors.yml | 3 +- .../airbyte_ci/connectors/publish/pipeline.py | 52 +++-- .../pipelines/airbyte_ci/poetry/commands.py | 69 +----- .../pipelines/airbyte_ci/poetry/pipeline.py | 209 ------------------ .../airbyte_ci/poetry/publish/__init__.py | 3 + .../airbyte_ci/poetry/publish/commands.py | 81 +++++++ .../airbyte_ci/poetry/publish/context.py | 96 ++++++++ .../airbyte_ci/poetry/publish/pipeline.py | 153 +++++++++++++ .../pipelines/airbyte_ci/poetry/utils.py | 3 +- .../connectors/pipelines/pipelines/consts.py | 2 + .../tests/test_poetry/test_poetry_publish.py | 92 ++++++++ .../pipelines/tests/test_publish.py | 68 ++++++ 13 files changed, 540 insertions(+), 295 deletions(-) delete mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/__init__.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py create mode 100644 airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml index 8d28ea9e788cd..0fcebdfe833b9 100644 --- a/.github/actions/run-dagger-pipeline/action.yml +++ b/.github/actions/run-dagger-pipeline/action.yml @@ -83,6 +83,9 @@ inputs: description: "URL to airbyte-ci binary" required: false default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci + pypi_token: + description: "PyPI API token to publish to PyPI" + required: false runs: using: "composite" @@ -182,3 +185,4 @@ runs: CI: "True" TAILSCALE_AUTH_KEY: ${{ inputs.tailscale_auth_key }} DOCKER_REGISTRY_MIRROR_URL: ${{ inputs.docker_registry_mirror_url }} + PYPI_TOKEN: ${{ inputs.pypi_token }} diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index 21bce68f1e401..db44bfd767acf 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -84,8 +84,7 @@ jobs: s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}" - env: - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + pypi_token: ${{ secrets.PYPI_TOKEN }} set-instatus-incident-on-failure: name: Create Instatus Incident on Failure diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index 2772eff210fa1..492ac01882bf1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -14,7 +14,7 @@ from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.airbyte_ci.metadata.pipeline import MetadataUpload, MetadataValidation -from pipelines.airbyte_ci.poetry.pipeline import PublishToPyPI, PyPIPublishContext +from pipelines.airbyte_ci.poetry.publish.pipeline import PublishToPyPI, PyPIPublishContext from pipelines.airbyte_ci.poetry.utils import is_package_published from pipelines.dagger.actions.remote_storage import upload_to_gcs from pipelines.dagger.actions.system import docker @@ -54,12 +54,12 @@ async def _run(self) -> StepResult: return StepResult(self, status=StepStatus.SUCCESS, stdout=f"No manifest found for {self.context.docker_image}.") -class CheckPypiPackageExists(Step): +class CheckPypiPackageDoesNotExist(Step): context: PyPIPublishContext title = "Check if the connector is published on pypi" async def _run(self) -> StepResult: - is_published = is_package_published(self.context.package_name, self.context.version, self.context.test_pypi) + is_published = is_package_published(self.context.package_name, self.context.version, self.context.registry) if is_published: return StepResult( self, status=StepStatus.SKIPPED, stderr=f"{self.context.package_name} already exists in version {self.context.version}." @@ -291,21 +291,10 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: metadata_upload_results = await metadata_upload_step.run() results.append(metadata_upload_results) - # Try to convert the context to a PyPIPublishContext. If it returns None, it means we don't need to publish to pypi. - pypi_context = await PyPIPublishContext.from_connector_context(context) - if pypi_context: - check_pypi_package_exists_results = await CheckPypiPackageExists(pypi_context).run() - results.append(check_pypi_package_exists_results) - if check_pypi_package_exists_results.status is StepStatus.SKIPPED: - context.logger.info("The connector version is already published on pypi.") - elif check_pypi_package_exists_results.status is StepStatus.SUCCESS: - context.logger.info("The connector version is not published on pypi. Let's build and publish it.") - publish_to_pypi_results = await PublishToPyPI(pypi_context).run() - results.append(publish_to_pypi_results) - if publish_to_pypi_results.status is StepStatus.FAILURE: - return create_connector_report(results) - elif check_pypi_package_exists_results.status is StepStatus.FAILURE: - return create_connector_report(results) + pypi_steps, terminate_early = await _run_pypi_publish_pipeline(context) + results.extend(pypi_steps) + if terminate_early: + return create_connector_report(results) # Exit early if the connector image already exists or has failed to build if check_connector_image_results.status is not StepStatus.SUCCESS: @@ -346,6 +335,33 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: return connector_report +async def _run_pypi_publish_pipeline(context: PublishConnectorContext) -> Tuple[List[StepResult], bool]: + """ + Run the pypi publish pipeline for a single connector. + Return the results of the steps and a boolean indicating whether there was an error and the pipeline should be stopped. + """ + results = [] + # Try to convert the context to a PyPIPublishContext. If it returns None, it means we don't need to publish to pypi. + pypi_context = await PyPIPublishContext.from_publish_connector_context(context) + if not pypi_context: + return results, False + + check_pypi_package_exists_results = await CheckPypiPackageDoesNotExist(pypi_context).run() + results.append(check_pypi_package_exists_results) + if check_pypi_package_exists_results.status is StepStatus.SKIPPED: + context.logger.info("The connector version is already published on pypi.") + elif check_pypi_package_exists_results.status is StepStatus.SUCCESS: + context.logger.info("The connector version is not published on pypi. Let's build and publish it.") + publish_to_pypi_results = await PublishToPyPI(pypi_context).run() + results.append(publish_to_pypi_results) + if publish_to_pypi_results.status is StepStatus.FAILURE: + return results, True + elif check_pypi_package_exists_results.status is StepStatus.FAILURE: + return results, True + + return results, False + + def reorder_contexts(contexts: List[PublishConnectorContext]) -> List[PublishConnectorContext]: """Reorder contexts so that the ones that are for strict-encrypt/secure connectors come first. The metadata upload on publish checks if the the connectors referenced in the metadata file are already published to DockerHub. diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py index 796ead980110f..92e70d6651e0d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py @@ -16,16 +16,19 @@ from pipelines.airbyte_ci.format.format_command import FormatCommand from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from pipelines.cli.lazy_group import LazyGroup from pipelines.helpers.cli import LogOptions, invoke_commands_concurrently, invoke_commands_sequentially, log_command_results from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context from pipelines.models.steps import StepStatus -from .pipeline import PublishToPyPI, PyPIPublishContext - @click.group( name="poetry", help="Commands related to running poetry commands.", + cls=LazyGroup, + lazy_subcommands={ + "publish": "pipelines.airbyte_ci.poetry.publish.commands.publish", + }, ) @click.option( "--package-path", @@ -33,70 +36,8 @@ type=click.STRING, required=True, ) -@click.option("--docker-image", help="The docker image to run the command in.", type=click.STRING, default="mwalbeck/python-poetry") @click_merge_args_into_context_obj @pass_pipeline_context @click_ignore_unused_kwargs async def poetry(pipeline_context: ClickPipelineContext) -> None: pass - - -@poetry.command(cls=DaggerPipelineCommand, name="publish", help="Publish a Python package to PyPI.") -@click.option( - "--pypi-token", - help="Access token", - type=click.STRING, - required=True, - envvar="PYPI_TOKEN", -) -@click.option( - "--test-pypi", - help="Whether to publish to test.pypi.org instead of pypi.org.", - type=click.BOOL, - is_flag=True, - default=False, -) -@click.option( - "--publish-name", - help="The name of the package to publish. If not set, the name will be inferred from the pyproject.toml file of the package.", - type=click.STRING, -) -@click.option( - "--publish-version", - help="The version of the package to publish. If not set, the version will be inferred from the pyproject.toml file of the package.", - type=click.STRING, -) -@pass_pipeline_context -@click.pass_context -async def publish( - ctx: click.Context, - click_pipeline_context: ClickPipelineContext, - pypi_token: str, - test_pypi: bool, - publish_name: Optional[str], - publish_version: Optional[str], -) -> None: - context = PyPIPublishContext( - is_local=ctx.obj["is_local"], - git_branch=ctx.obj["git_branch"], - git_revision=ctx.obj["git_revision"], - ci_report_bucket=ctx.obj["ci_report_bucket_name"], - report_output_prefix=ctx.obj["report_output_prefix"], - gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), - dagger_logs_url=ctx.obj.get("dagger_logs_url"), - pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), - ci_context=ctx.obj.get("ci_context"), - ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], - pypi_token=pypi_token, - test_pypi=test_pypi, - package_path=ctx.obj["package_path"], - build_docker_image=ctx.obj["docker_image"], - package_name=publish_name, - version=publish_version, - ) - dagger_client = await click_pipeline_context.get_dagger_client(pipeline_name=f"Publish {ctx.obj['package_path']} to PyPI") - context.dagger_client = dagger_client - - await PublishToPyPI(context).run() - - return True diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py deleted file mode 100644 index 40e6c6a6d29bb..0000000000000 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/pipeline.py +++ /dev/null @@ -1,209 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import configparser -import io -import os -from textwrap import dedent -from typing import Optional - -import tomli -import tomli_w -import yaml -from pipelines.airbyte_ci.connectors.context import ConnectorContext, PipelineContext -from pipelines.models.contexts.pipeline_context import PipelineContext -from pipelines.models.steps import Step, StepResult - - -class PyPIPublishContext(PipelineContext): - def __init__( - self, - pypi_token: str, - test_pypi: bool, - package_path: str, - build_docker_image: str, - ci_report_bucket: str, - report_output_prefix: str, - is_local: bool, - git_branch: bool, - git_revision: bool, - gha_workflow_run_url: Optional[str] = None, - dagger_logs_url: Optional[str] = None, - pipeline_start_timestamp: Optional[int] = None, - ci_context: Optional[str] = None, - ci_gcs_credentials: str = None, - package_name: Optional[str] = None, - version: Optional[str] = None, - ): - self.pypi_token = pypi_token - self.test_pypi = test_pypi - self.package_path = package_path - self.package_name = package_name - self.version = version - self.build_docker_image = build_docker_image - - pipeline_name = f"Publish PyPI {package_path}" - - super().__init__( - pipeline_name=pipeline_name, - report_output_prefix=report_output_prefix, - ci_report_bucket=ci_report_bucket, - is_local=is_local, - git_branch=git_branch, - git_revision=git_revision, - gha_workflow_run_url=gha_workflow_run_url, - dagger_logs_url=dagger_logs_url, - pipeline_start_timestamp=pipeline_start_timestamp, - ci_context=ci_context, - ci_gcs_credentials=ci_gcs_credentials, - ) - - @staticmethod - async def from_connector_context(connector_context: ConnectorContext) -> Optional["PyPIPublishContext"]: - """ - Create a PyPIPublishContext from a ConnectorContext. - - The metadata of the connector is read from the current workdir to capture changes that are not yet published. - If pypi is not enabled, this will return None. - """ - - current_metadata = yaml.safe_load( - await connector_context.get_repo_file(str(connector_context.connector.metadata_file_path)).contents() - )["data"] - print(current_metadata) - if ( - not "remoteRegistries" in current_metadata - or not "pypi" in current_metadata["remoteRegistries"] - or not current_metadata["remoteRegistries"]["pypi"]["enabled"] - ): - return None - - if "connectorBuildOptions" in current_metadata and "baseImage" in current_metadata["connectorBuildOptions"]: - build_docker_image = current_metadata["connectorBuildOptions"]["baseImage"] - else: - build_docker_image = "mwalbeck/python-poetry" - pypi_context = PyPIPublishContext( - pypi_token=os.environ["PYPI_TOKEN"], - test_pypi=True, # TODO: Go live - package_path=str(connector_context.connector.code_directory), - package_name=current_metadata["remoteRegistries"]["pypi"]["packageName"], - version=current_metadata["dockerImageTag"], - build_docker_image=build_docker_image, - ci_report_bucket=connector_context.ci_report_bucket, - report_output_prefix=connector_context.report_output_prefix, - is_local=connector_context.is_local, - git_branch=connector_context.git_branch, - git_revision=connector_context.git_revision, - gha_workflow_run_url=connector_context.gha_workflow_run_url, - dagger_logs_url=connector_context.dagger_logs_url, - pipeline_start_timestamp=connector_context.pipeline_start_timestamp, - ci_context=connector_context.ci_context, - ci_gcs_credentials=connector_context.ci_gcs_credentials, - ) - pypi_context.dagger_client = connector_context.dagger_client - return pypi_context - - -class PublishToPyPI(Step): - context: PyPIPublishContext - title = "Publish package to PyPI" - - async def _run(self) -> StepResult: - dir_to_publish = await self.context.get_repo_dir(self.context.package_path) - - files = await dir_to_publish.entries() - is_poetry_package = "pyproject.toml" in files - is_pip_package = "setup.py" in files - - # Try to infer package name and version from the pyproject.toml file. If it is not present, we need to have the package name and version set - # Setup.py packages need to set package name and version as parameter - if not self.context.package_name or not self.context.version: - if not is_poetry_package: - return self.skip( - "Connector does not have a pyproject.toml file and version and package name is not set otherwise, skipping." - ) - - pyproject_toml = dir_to_publish.file("pyproject.toml") - pyproject_toml_content = await pyproject_toml.contents() - contents = tomli.loads(pyproject_toml_content) - if ( - "tool" not in contents - or "poetry" not in contents["tool"] - or "name" not in contents["tool"]["poetry"] - or "version" not in contents["tool"]["poetry"] - ): - return self.skip( - "Connector does not have a pyproject.toml file which specifies package name and version and they are not set otherwise, skipping." - ) - - self.context.package_name = contents["tool"]["poetry"]["name"] - self.context.version = contents["tool"]["poetry"]["version"] - - print( - f"Uploading package {self.context.package_name} version {self.context.version} to {'testpypi' if self.context.test_pypi else 'pypi'}..." - ) - - if is_pip_package: - # legacy publish logic - pypi_username = self.context.dagger_client.set_secret("pypi_username", "__token__") - pypi_password = self.context.dagger_client.set_secret("pypi_password", f"pypi-{self.context.pypi_token}") - metadata = { - "name": self.context.package_name, - "version": self.context.version, - "author": "Airbyte", - "author_email": "contact@airbyte.io", - } - if "README.md" in files: - metadata["long_description"] = await dir_to_publish.file("README.md").contents() - metadata["long_description_content_type"] = "text/markdown" - - config = configparser.ConfigParser() - config["metadata"] = metadata - - setup_cfg_io = io.StringIO() - config.write(setup_cfg_io) - setup_cfg = setup_cfg_io.getvalue() - - twine_upload = ( - self.context.dagger_client.container() - .from_(self.context.build_docker_image) - .with_exec(["apt-get", "update"]) - .with_exec(["apt-get", "install", "-y", "twine"]) - .with_directory("package", dir_to_publish) - .with_workdir("package") - # clear out setup.py metadata so setup.cfg is used - .with_exec(["sed", "-i", "/name=/d; /author=/d; /author_email=/d; /version=/d", "setup.py"]) - .with_new_file("setup.cfg", contents=setup_cfg) - .with_exec(["pip", "install", "--upgrade", "setuptools", "wheel"]) - .with_exec(["python", "setup.py", "sdist", "bdist_wheel"]) - .with_secret_variable("TWINE_USERNAME", pypi_username) - .with_secret_variable("TWINE_PASSWORD", pypi_password) - .with_exec(["twine", "upload", "--verbose", "--repository", "testpypi" if self.context.test_pypi else "pypi", "dist/*"]) - ) - - return await self.get_step_result(twine_upload) - else: - # poetry publish logic - pypi_token = self.context.dagger_client.set_secret("pypi_token", f"pypi-{self.context.pypi_token}") - pyproject_toml = dir_to_publish.file("pyproject.toml") - pyproject_toml_content = await pyproject_toml.contents() - contents = tomli.loads(pyproject_toml_content) - # make sure package name and version are set to the configured one - contents["tool"]["poetry"]["name"] = self.context.package_name - contents["tool"]["poetry"]["version"] = self.context.version - poetry_publish = ( - self.context.dagger_client.container() - .from_(self.context.build_docker_image) - .with_secret_variable("PYPI_TOKEN", pypi_token) - .with_directory("package", dir_to_publish) - .with_workdir("package") - .with_new_file("pyproject.toml", contents=tomli_w.dumps(contents)) - .with_exec(["poetry", "config", "repositories.testpypi", "https://test.pypi.org/legacy/"]) - .with_exec( - ["sh", "-c", f"poetry config {'pypi-token.testpypi' if self.context.test_pypi else 'pypi-token.pypi'} $PYPI_TOKEN"] - ) - .with_exec(["poetry", "publish", "--build", "--repository", "testpypi" if self.context.test_pypi else "pypi"]) - ) - - return await self.get_step_result(poetry_publish) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/__init__.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/__init__.py new file mode 100644 index 0000000000000..c941b30457953 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py new file mode 100644 index 0000000000000..5953830192df8 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py @@ -0,0 +1,81 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +""" +Module exposing the format commands. +""" +from __future__ import annotations + +from typing import Optional + +import asyncclick as click +from connector_ops.utils import CONNECTOR_PATH_PREFIX +from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context + +from .context import PyPIPublishContext +from .pipeline import PublishToPyPI + + +@click.command(cls=DaggerPipelineCommand, name="publish", help="Publish a Python package to PyPI.") +@click.option( + "--pypi-token", + help="Access token", + type=click.STRING, + required=True, + envvar="PYPI_TOKEN", +) +@click.option( + "--registry-url", + help="Which registry to publish to. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/legacy/", + type=click.STRING, + default="https://pypi.org/simple", +) +@click.option( + "--publish-name", + help="The name of the package to publish. If not set, the name will be inferred from the pyproject.toml file of the package.", + type=click.STRING, +) +@click.option( + "--publish-version", + help="The version of the package to publish. If not set, the version will be inferred from the pyproject.toml file of the package.", + type=click.STRING, +) +@pass_pipeline_context +@click.pass_context +async def publish( + ctx: click.Context, + click_pipeline_context: ClickPipelineContext, + pypi_token: str, + registry_url: bool, + publish_name: Optional[str], + publish_version: Optional[str], +) -> None: + context = PyPIPublishContext( + is_local=ctx.obj["is_local"], + git_branch=ctx.obj["git_branch"], + git_revision=ctx.obj["git_revision"], + ci_report_bucket=ctx.obj["ci_report_bucket_name"], + report_output_prefix=ctx.obj["report_output_prefix"], + gha_workflow_run_url=ctx.obj.get("gha_workflow_run_url"), + dagger_logs_url=ctx.obj.get("dagger_logs_url"), + pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), + ci_context=ctx.obj.get("ci_context"), + ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], + pypi_token=pypi_token, + registry=registry_url, + package_path=ctx.obj["package_path"], + package_name=publish_name, + version=publish_version, + ) + + if context.package_path.startswith(CONNECTOR_PATH_PREFIX): + context.logger.warning("It looks like you are trying to publish a connector. Please use the `connectors` command group instead.") + + dagger_client = await click_pipeline_context.get_dagger_client(pipeline_name=f"Publish {ctx.obj['package_path']} to PyPI") + context.dagger_client = dagger_client + + await PublishToPyPI(context).run() + + return True diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py new file mode 100644 index 0000000000000..676ca760daf93 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py @@ -0,0 +1,96 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import os +from datetime import date +from typing import Optional + +from pipelines.airbyte_ci.connectors.context import PipelineContext +from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext +from pipelines.models.contexts.pipeline_context import PipelineContext + + +class PyPIPublishContext(PipelineContext): + def __init__( + self, + pypi_token: str, + package_path: str, + ci_report_bucket: str, + report_output_prefix: str, + is_local: bool, + git_branch: bool, + git_revision: bool, + registry: Optional[str] = None, + gha_workflow_run_url: Optional[str] = None, + dagger_logs_url: Optional[str] = None, + pipeline_start_timestamp: Optional[int] = None, + ci_context: Optional[str] = None, + ci_gcs_credentials: str = None, + package_name: Optional[str] = None, + version: Optional[str] = None, + ): + self.pypi_token = pypi_token + self.registry = registry or "https://pypi.org/simple" + self.package_path = package_path + self.package_name = package_name + self.version = version + + pipeline_name = f"Publish PyPI {package_path}" + + super().__init__( + pipeline_name=pipeline_name, + report_output_prefix=report_output_prefix, + ci_report_bucket=ci_report_bucket, + is_local=is_local, + git_branch=git_branch, + git_revision=git_revision, + gha_workflow_run_url=gha_workflow_run_url, + dagger_logs_url=dagger_logs_url, + pipeline_start_timestamp=pipeline_start_timestamp, + ci_context=ci_context, + ci_gcs_credentials=ci_gcs_credentials, + ) + + @staticmethod + async def from_publish_connector_context(connector_context: PublishConnectorContext) -> Optional["PyPIPublishContext"]: + """ + Create a PyPIPublishContext from a ConnectorContext. + + The metadata of the connector is read from the current workdir to capture changes that are not yet published. + If pypi is not enabled, this will return None. + """ + + current_metadata = connector_context.connector.metadata + if ( + not "remoteRegistries" in current_metadata + or not "pypi" in current_metadata["remoteRegistries"] + or not current_metadata["remoteRegistries"]["pypi"]["enabled"] + ): + return None + + version = current_metadata["dockerImageTag"] + if connector_context.pre_release: + # use current date as pre-release version + rc_tag = date.today().strftime("%Y-%m-%d-%H-%M") + version = f"{version}rc{rc_tag}" + + pypi_context = PyPIPublishContext( + pypi_token=os.environ["PYPI_TOKEN"], + registry="https://test.pypi.org/", # TODO: go live + package_path=str(connector_context.connector.code_directory), + package_name=current_metadata["remoteRegistries"]["pypi"]["packageName"], + version=current_metadata["dockerImageTag"], + ci_report_bucket=connector_context.ci_report_bucket, + report_output_prefix=connector_context.report_output_prefix, + is_local=connector_context.is_local, + git_branch=connector_context.git_branch, + git_revision=connector_context.git_revision, + gha_workflow_run_url=connector_context.gha_workflow_run_url, + dagger_logs_url=connector_context.dagger_logs_url, + pipeline_start_timestamp=connector_context.pipeline_start_timestamp, + ci_context=connector_context.ci_context, + ci_gcs_credentials=connector_context.ci_gcs_credentials, + ) + pypi_context.dagger_client = connector_context.dagger_client + return pypi_context diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py new file mode 100644 index 0000000000000..ed7651d7cf026 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py @@ -0,0 +1,153 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import configparser +import io +import uuid +from enum import Enum, auto +from typing import Optional, Tuple + +import tomli +import tomli_w +from dagger import Directory +from pipelines.airbyte_ci.poetry.publish.context import PyPIPublishContext +from pipelines.consts import PYPROJECT_TOML_FILE_PATH, SETUP_PY_FILE_PATH +from pipelines.dagger.actions.python.poetry import with_poetry +from pipelines.helpers.utils import sh_dash_c +from pipelines.models.steps import Step, StepResult + + +class PackageType(Enum): + POETRY = auto() + PIP = auto() + NONE = auto() + + +class PublishToPyPI(Step): + context: PyPIPublishContext + title = "Publish package to PyPI" + + def _get_base_container(self): + return with_poetry(self.context) + + async def _get_package_metadata_from_pyproject_toml(self, dir_to_publish: Directory) -> Optional[Tuple[str, str]]: + pyproject_toml = dir_to_publish.file(PYPROJECT_TOML_FILE_PATH) + pyproject_toml_content = await pyproject_toml.contents() + contents = tomli.loads(pyproject_toml_content) + if ( + "tool" not in contents + or "poetry" not in contents["tool"] + or "name" not in contents["tool"]["poetry"] + or "version" not in contents["tool"]["poetry"] + ): + return None + + return (contents["tool"]["poetry"]["name"], contents["tool"]["poetry"]["version"]) + + async def _get_package_type(self, dir_to_publish: Directory) -> PackageType: + files = await dir_to_publish.entries() + has_pyproject_toml = PYPROJECT_TOML_FILE_PATH in files + has_setup_py = SETUP_PY_FILE_PATH in files + if has_pyproject_toml: + return PackageType.POETRY + elif has_setup_py: + return PackageType.PIP + else: + return PackageType.NONE + + async def _run(self) -> StepResult: + dir_to_publish = await self.context.get_repo_dir(self.context.package_path) + package_type = await self._get_package_type(dir_to_publish) + + if package_type == PackageType.NONE: + return self.skip("Connector does not have a pyproject.toml file or setup.py file, skipping.") + + # Try to infer package name and version from the pyproject.toml file. If it is not present, we need to have the package name and version set + # Setup.py packages need to set package name and version as parameter + if not self.context.package_name or not self.context.version: + if not package_type == PackageType.POETRY: + return self.skip( + "Connector does not have a pyproject.toml file and version and package name is not set otherwise, skipping." + ) + + package_metadata = await self._get_package_metadata_from_pyproject_toml(dir_to_publish) + + if not package_metadata: + return self.skip( + "Connector does not have a pyproject.toml file which specifies package name and version and they are not set otherwise, skipping." + ) + + self.context.package_name = package_metadata[0] + self.context.version = package_metadata[1] + + self.logger.info(f"Uploading package {self.context.package_name} version {self.context.version} to {self.context.registry}...") + + if package_type == PackageType.PIP: + return await self._pip_publish(dir_to_publish) + else: + return await self._poetry_publish(dir_to_publish) + + async def _poetry_publish(self, dir_to_publish: Directory) -> StepResult: + pypi_token = self.context.dagger_client.set_secret("pypi_token", f"pypi-{self.context.pypi_token}") + pyproject_toml = dir_to_publish.file(PYPROJECT_TOML_FILE_PATH) + pyproject_toml_content = await pyproject_toml.contents() + contents = tomli.loads(pyproject_toml_content) + # make sure package name and version are set to the configured one + contents["tool"]["poetry"]["name"] = self.context.package_name + contents["tool"]["poetry"]["version"] = self.context.version + # enforce consistent author + contents["tool"]["poetry"]["authors"] = ["Airbyte "] + poetry_publish = ( + self._get_base_container() + .with_secret_variable("PYPI_TOKEN", pypi_token) + .with_directory("package", dir_to_publish) + .with_workdir("package") + .with_new_file(PYPROJECT_TOML_FILE_PATH, contents=tomli_w.dumps(contents)) + .with_exec(["poetry", "config", "repositories.mypypi", self.context.registry]) + .with_exec(sh_dash_c([f"poetry config pypi-token.mypypi $PYPI_TOKEN"])) + .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) + .with_exec(sh_dash_c(["poetry publish --build --repository mypypi -vvv --no-interaction"])) + ) + + return await self.get_step_result(poetry_publish) + + async def _pip_publish(self, dir_to_publish: Directory) -> StepResult: + files = await dir_to_publish.entries() + pypi_username = self.context.dagger_client.set_secret("pypi_username", "__token__") + pypi_password = self.context.dagger_client.set_secret("pypi_password", f"pypi-{self.context.pypi_token}") + metadata = { + "name": self.context.package_name, + "version": self.context.version, + # Enforce consistent author + "author": "Airbyte", + "author_email": "contact@airbyte.io", + } + if "README.md" in files: + metadata["long_description"] = await dir_to_publish.file("README.md").contents() + metadata["long_description_content_type"] = "text/markdown" + + config = configparser.ConfigParser() + config["metadata"] = metadata + + setup_cfg_io = io.StringIO() + config.write(setup_cfg_io) + setup_cfg = setup_cfg_io.getvalue() + + twine_upload = ( + self._get_base_container() + .with_exec(sh_dash_c(["apt-get update", "apt-get install -y twine"])) + .with_directory("package", dir_to_publish) + .with_workdir("package") + # clear out setup.py metadata so setup.cfg is used + .with_exec(["sed", "-i", "/name=/d; /author=/d; /author_email=/d; /version=/d", SETUP_PY_FILE_PATH]) + .with_new_file("setup.cfg", contents=setup_cfg) + .with_exec(["pip", "install", "--upgrade", "setuptools", "wheel"]) + .with_exec(["python", SETUP_PY_FILE_PATH, "sdist", "bdist_wheel"]) + .with_secret_variable("TWINE_USERNAME", pypi_username) + .with_secret_variable("TWINE_PASSWORD", pypi_password) + .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) + .with_exec(["twine", "upload", "--verbose", "--repository-url", self.context.registry, "dist/*"]) + ) + + return await self.get_step_result(twine_upload) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py index b0d6d2f076502..966c596eb428d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py @@ -3,7 +3,7 @@ import requests -def is_package_published(package_name, version, test_pypi=False): +def is_package_published(package_name: str, version: str, base_url: str): """ Check if a package with a specific version is published on PyPI or Test PyPI. @@ -12,7 +12,6 @@ def is_package_published(package_name, version, test_pypi=False): :param test_pypi: Set to True to check on Test PyPI, False for regular PyPI. :return: True if the package is found with the specified version, False otherwise. """ - base_url = "https://test.pypi.org/pypi" if test_pypi else "https://pypi.org/pypi" url = f"{base_url}/{package_name}/{version}/json" response = requests.get(url) diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py index 851578cc7a0ca..57a413a5d18b3 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -60,6 +60,8 @@ POETRY_CACHE_PATH = "/root/.cache/pypoetry" STORAGE_DRIVER = "fuse-overlayfs" TAILSCALE_AUTH_KEY = os.getenv("TAILSCALE_AUTH_KEY") +PYPROJECT_TOML_FILE_PATH = "pyproject.toml" +SETUP_PY_FILE_PATH = "setup.py" class CIContext(str, Enum): diff --git a/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py new file mode 100644 index 0000000000000..e802e71326243 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py @@ -0,0 +1,92 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +import json +import random +from pathlib import Path +from typing import List +from unittest.mock import AsyncMock, MagicMock, patch + +import anyio +import pytest +import requests +from connector_ops.utils import Connector, ConnectorLanguage +from dagger import Client, Directory, Platform +from pipelines.airbyte_ci.connectors.context import ConnectorContext +from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline +from pipelines.airbyte_ci.connectors.upgrade_cdk import pipeline as upgrade_cdk_pipeline +from pipelines.airbyte_ci.poetry.publish.context import PyPIPublishContext +from pipelines.dagger.actions.python.poetry import with_poetry +from pipelines.models.contexts.pipeline_context import PipelineContext +from pipelines.models.steps import StepStatus + +pytestmark = [ + pytest.mark.anyio, +] + + +@pytest.fixture +def context(dagger_client: Client): + context = PyPIPublishContext( + package_path="test", + version="0.2.0", + pypi_token="test", + package_name="test", + registry="http://local_registry:8080/", + is_local=True, + git_branch="test", + git_revision="test", + report_output_prefix="test", + ci_report_bucket="test", + ) + context.dagger_client = dagger_client + return context + + +@pytest.mark.parametrize( + "package_path, package_name, expected_asset", + [ + pytest.param( + "airbyte-integrations/connectors/source-apify-dataset", + "airbyte-source-apify-dataset", + "airbyte_source_apify_dataset-0.2.0-py3-none-any.whl", + id="setup.py project", + ), + pytest.param( + "airbyte-lib", + "airbyte-lib", + "airbyte_lib-0.2.0-py3-none-any.whl", + id="poetry project", + ), + ], +) +async def test_run_poetry_publish(context: PyPIPublishContext, package_path: str, package_name: str, expected_asset: str): + context.package_name = package_name + context.package_path = package_path + pypi_registry = ( + # need to use linux/amd64 because the pypiserver image is only available for that platform + context.dagger_client.container(platform=Platform("linux/amd64")) + .from_("pypiserver/pypiserver:v2.0.1") + .with_exec(["run", "-P", ".", "-a", "."]) + .with_exposed_port(8080) + .as_service() + ) + + base_container = with_poetry(context).with_service_binding("local_registry", pypi_registry) + step = publish_pipeline.PublishToPyPI(context) + step._get_base_container = MagicMock(return_value=base_container) + step_result = await step.run() + assert step_result.status == StepStatus.SUCCESS + + # Query the registry to check that the package was published + tunnel = await context.dagger_client.host().tunnel(pypi_registry).start() + endpoint = await tunnel.endpoint(scheme="http") + list_url = f"{endpoint}/simple/" + list_response = requests.get(list_url) + assert list_response.status_code == 200 + assert package_name in list_response.text + url = f"{endpoint}/simple/{package_name}" + response = requests.get(url) + assert response.status_code == 200 + assert expected_asset in response.text diff --git a/airbyte-ci/connectors/pipelines/tests/test_publish.py b/airbyte-ci/connectors/pipelines/tests/test_publish.py index b7fe7d764d5fb..a2fbc404911cb 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_publish.py @@ -2,8 +2,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # import json +import os import random from typing import List +from unittest.mock import patch import anyio import pytest @@ -153,6 +155,7 @@ def test_parse_spec_output_no_spec(self, publish_context): (publish_pipeline, "PushConnectorImageToRegistry"), (publish_pipeline, "PullConnectorImageFromRegistry"), (publish_pipeline.steps, "run_connector_build"), + (publish_pipeline, "CheckPypiPackageDoesNotExist"), ] @@ -333,3 +336,68 @@ async def test_run_connector_publish_pipeline_when_image_does_not_exist( publish_pipeline.PullConnectorImageFromRegistry.return_value.run.assert_not_called() publish_pipeline.UploadSpecToCache.return_value.run.assert_not_called() publish_pipeline.MetadataUpload.return_value.run.assert_not_called() + + +@pytest.mark.parametrize( + "pypi_enabled, pypi_package_does_not_exist_status, publish_step_status, expect_publish_to_pypi_called, expect_build_connector_called", + [ + pytest.param(True, StepStatus.SUCCESS, StepStatus.SUCCESS, True, True, id="happy_path"), + pytest.param(False, StepStatus.SUCCESS, StepStatus.SUCCESS, False, True, id="pypi_disabled, skip all pypi steps"), + pytest.param(True, StepStatus.SKIPPED, StepStatus.SUCCESS, False, True, id="pypi_package_exists, skip publish_to_pypi"), + pytest.param(True, StepStatus.SUCCESS, StepStatus.FAILURE, True, False, id="publish_step_fails, abort"), + pytest.param(True, StepStatus.FAILURE, StepStatus.FAILURE, False, False, id="pypi_package_does_not_exist_fails, abort"), + ], +) +async def test_run_connector_pypi_publish_pipeline( + mocker, + pypi_enabled, + pypi_package_does_not_exist_status, + publish_step_status, + expect_publish_to_pypi_called, + expect_build_connector_called, +): + + for module, to_mock in STEPS_TO_PATCH: + mocker.patch.object(module, to_mock, return_value=mocker.AsyncMock()) + + mocked_publish_to_pypi = mocker.patch("pipelines.airbyte_ci.connectors.publish.pipeline.PublishToPyPI", return_value=mocker.AsyncMock()) + + for step in [ + publish_pipeline.MetadataValidation, + publish_pipeline.CheckConnectorImageDoesNotExist, + publish_pipeline.UploadSpecToCache, + publish_pipeline.MetadataUpload, + publish_pipeline.PushConnectorImageToRegistry, + publish_pipeline.PullConnectorImageFromRegistry, + ]: + step.return_value.run.return_value = mocker.Mock(name=f"{step.title}_result", status=StepStatus.SUCCESS) + + mocked_publish_to_pypi.return_value.run.return_value = mocker.Mock(name="publish_to_pypi_result", status=publish_step_status) + + publish_pipeline.CheckPypiPackageDoesNotExist.return_value.run.return_value = mocker.Mock( + name="pypi_package_does_not_exist_result", status=pypi_package_does_not_exist_status + ) + + context = mocker.MagicMock( + ci_gcs_credentials="", + connector=mocker.MagicMock( + code_directory="path/to/connector", + metadata={"dockerImageTag": "1.2.3", "remoteRegistries": {"pypi": {"enabled": pypi_enabled, "packageName": "test"}}}, + ), + ) + semaphore = anyio.Semaphore(1) + with patch.dict(os.environ, {"PYPI_TOKEN": "test"}): + await publish_pipeline.run_connector_publish_pipeline(context, semaphore) + if expect_publish_to_pypi_called: + mocked_publish_to_pypi.return_value.run.assert_called_once() + # assert that the first argument passed to mocked_publish_to_pypi contains the things from the context + assert mocked_publish_to_pypi.call_args.args[0].pypi_token == "test" + assert mocked_publish_to_pypi.call_args.args[0].package_name == "test" + assert mocked_publish_to_pypi.call_args.args[0].version == "1.2.3" + assert mocked_publish_to_pypi.call_args.args[0].registry == "https://test.pypi.org/" + assert mocked_publish_to_pypi.call_args.args[0].package_path == "path/to/connector" + else: + mocked_publish_to_pypi.return_value.run.assert_not_called() + + if expect_build_connector_called: + publish_pipeline.steps.run_connector_build.assert_called_once() From 3c408dfec7a512a0f73ba4f9ecc8d488158ffb2e Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 18 Jan 2024 12:25:27 +0100 Subject: [PATCH 26/53] adjust for test pypi publishing --- .../pipelines/pipelines/airbyte_ci/poetry/publish/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py index 676ca760daf93..c09bc771dcf7d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py @@ -77,7 +77,7 @@ async def from_publish_connector_context(connector_context: PublishConnectorCont pypi_context = PyPIPublishContext( pypi_token=os.environ["PYPI_TOKEN"], - registry="https://test.pypi.org/", # TODO: go live + registry="https://test.pypi.org/legacy/", # TODO: go live package_path=str(connector_context.connector.code_directory), package_name=current_metadata["remoteRegistries"]["pypi"]["packageName"], version=current_metadata["dockerImageTag"], From e06d37d9b2e2faa3594bd5ec29ee74697aaa1546 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 18 Jan 2024 12:31:55 +0100 Subject: [PATCH 27/53] fix things --- .../pipelines/pipelines/airbyte_ci/poetry/commands.py | 9 --------- .../pipelines/airbyte_ci/poetry/publish/context.py | 6 +++--- .../pipelines/airbyte_ci/poetry/publish/pipeline.py | 6 +++--- .../pipelines/pipelines/airbyte_ci/poetry/utils.py | 4 ++-- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py index 92e70d6651e0d..72dbe53b170f8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/commands.py @@ -7,19 +7,10 @@ """ from __future__ import annotations -import logging -import sys -from typing import Any, Dict, List, Optional - import asyncclick as click -from pipelines.airbyte_ci.format.configuration import FORMATTERS_CONFIGURATIONS, Formatter -from pipelines.airbyte_ci.format.format_command import FormatCommand from pipelines.cli.click_decorators import click_ignore_unused_kwargs, click_merge_args_into_context_obj -from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.cli.lazy_group import LazyGroup -from pipelines.helpers.cli import LogOptions, invoke_commands_concurrently, invoke_commands_sequentially, log_command_results from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context -from pipelines.models.steps import StepStatus @click.group( diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py index c09bc771dcf7d..faca22fb2392b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py @@ -63,8 +63,8 @@ async def from_publish_connector_context(connector_context: PublishConnectorCont current_metadata = connector_context.connector.metadata if ( - not "remoteRegistries" in current_metadata - or not "pypi" in current_metadata["remoteRegistries"] + "remoteRegistries" not in current_metadata + or "pypi" not in current_metadata["remoteRegistries"] or not current_metadata["remoteRegistries"]["pypi"]["enabled"] ): return None @@ -80,7 +80,7 @@ async def from_publish_connector_context(connector_context: PublishConnectorCont registry="https://test.pypi.org/legacy/", # TODO: go live package_path=str(connector_context.connector.code_directory), package_name=current_metadata["remoteRegistries"]["pypi"]["packageName"], - version=current_metadata["dockerImageTag"], + version=version, ci_report_bucket=connector_context.ci_report_bucket, report_output_prefix=connector_context.report_output_prefix, is_local=connector_context.is_local, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py index ed7651d7cf026..1ad438fe0f65d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py @@ -10,7 +10,7 @@ import tomli import tomli_w -from dagger import Directory +from dagger import Container, Directory from pipelines.airbyte_ci.poetry.publish.context import PyPIPublishContext from pipelines.consts import PYPROJECT_TOML_FILE_PATH, SETUP_PY_FILE_PATH from pipelines.dagger.actions.python.poetry import with_poetry @@ -28,7 +28,7 @@ class PublishToPyPI(Step): context: PyPIPublishContext title = "Publish package to PyPI" - def _get_base_container(self): + def _get_base_container(self) -> Container: return with_poetry(self.context) async def _get_package_metadata_from_pyproject_toml(self, dir_to_publish: Directory) -> Optional[Tuple[str, str]]: @@ -105,7 +105,7 @@ async def _poetry_publish(self, dir_to_publish: Directory) -> StepResult: .with_workdir("package") .with_new_file(PYPROJECT_TOML_FILE_PATH, contents=tomli_w.dumps(contents)) .with_exec(["poetry", "config", "repositories.mypypi", self.context.registry]) - .with_exec(sh_dash_c([f"poetry config pypi-token.mypypi $PYPI_TOKEN"])) + .with_exec(sh_dash_c(["poetry config pypi-token.mypypi $PYPI_TOKEN"])) .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) .with_exec(sh_dash_c(["poetry publish --build --repository mypypi -vvv --no-interaction"])) ) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py index 966c596eb428d..c8aae74f2cba7 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py @@ -3,7 +3,7 @@ import requests -def is_package_published(package_name: str, version: str, base_url: str): +def is_package_published(package_name: str, version: str, base_url: str) -> bool: """ Check if a package with a specific version is published on PyPI or Test PyPI. @@ -12,7 +12,7 @@ def is_package_published(package_name: str, version: str, base_url: str): :param test_pypi: Set to True to check on Test PyPI, False for regular PyPI. :return: True if the package is found with the specified version, False otherwise. """ - url = f"{base_url}/{package_name}/{version}/json" + url = f"{base_url}{package_name}/{version}/json" response = requests.get(url) return response.status_code == 200 From df9502c60d808c4e24210ea4dd603c92661851b7 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 18 Jan 2024 12:39:28 +0100 Subject: [PATCH 28/53] fix --- .../pipelines/airbyte_ci/poetry/publish/context.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py index faca22fb2392b..4d6426139aecc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py @@ -3,7 +3,7 @@ # import os -from datetime import date +from datetime import datetime from typing import Optional from pipelines.airbyte_ci.connectors.context import PipelineContext @@ -72,8 +72,8 @@ async def from_publish_connector_context(connector_context: PublishConnectorCont version = current_metadata["dockerImageTag"] if connector_context.pre_release: # use current date as pre-release version - rc_tag = date.today().strftime("%Y-%m-%d-%H-%M") - version = f"{version}rc{rc_tag}" + rc_tag = datetime.now().strftime("%Y%m%d%H%M") + version = f"{version}.dev{rc_tag}" pypi_context = PyPIPublishContext( pypi_token=os.environ["PYPI_TOKEN"], From 0b6b9de3265a454f4b1ef611e2723f0a41801d7d Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 18 Jan 2024 12:45:14 +0100 Subject: [PATCH 29/53] fix --- .../pipelines/pipelines/airbyte_ci/poetry/publish/context.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py index 4d6426139aecc..ee029491a1cab 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py @@ -8,7 +8,6 @@ from pipelines.airbyte_ci.connectors.context import PipelineContext from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext -from pipelines.models.contexts.pipeline_context import PipelineContext class PyPIPublishContext(PipelineContext): @@ -29,7 +28,7 @@ def __init__( ci_gcs_credentials: str = None, package_name: Optional[str] = None, version: Optional[str] = None, - ): + ) -> None: self.pypi_token = pypi_token self.registry = registry or "https://pypi.org/simple" self.package_path = package_path From a0313acb39fb92877bee1c8058e3e24a43370e8c Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 18 Jan 2024 12:49:10 +0100 Subject: [PATCH 30/53] revert testing stuff --- airbyte-integrations/connectors/convert.sh | 122 ++++++++++++++++++ .../source-apify-dataset/metadata.yaml | 4 - .../source-google-drive/metadata.yaml | 4 - .../connectors/source-pokeapi/Dockerfile | 2 +- .../connectors/source-pokeapi/main.py | 9 +- .../connectors/source-pokeapi/metadata.yaml | 6 +- .../connectors/source-pokeapi/setup.py | 5 - .../source-pokeapi/source_pokeapi/run.py | 13 -- 8 files changed, 131 insertions(+), 34 deletions(-) create mode 100755 airbyte-integrations/connectors/convert.sh delete mode 100644 airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py diff --git a/airbyte-integrations/connectors/convert.sh b/airbyte-integrations/connectors/convert.sh new file mode 100755 index 0000000000000..6ffe1e56760b2 --- /dev/null +++ b/airbyte-integrations/connectors/convert.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +# Install github cli: brew install gh + +# Check if an argument was provided +if [ -z "$1" ]; then + echo "Please provide a folder name." + exit 1 +fi + +FOLDER_NAME="$1" +METADATA_FILE="$FOLDER_NAME/metadata.yaml" +SNAKE_CASED_FOLDER_NAME=$(echo "$FOLDER_NAME" | sed 's/-/_/g') +DOC_FOLDER_NAME=$(echo "$FOLDER_NAME" | sed 's/source-//') +DOC_FILE="../../docs/integrations/sources/${DOC_FOLDER_NAME}.md" +BRANCH_NAME="flash1293/airbyte-lib-convert-${FOLDER_NAME}" + +git checkout origin/master +git checkout -b "$BRANCH_NAME" + +# Step 1: Add entry_points to setup.py +SETUP_FILE="$FOLDER_NAME/setup.py" +if [ -f "$SETUP_FILE" ]; then + sed -i '' "/setup(/a \\ + entry_points={\\ + \"console_scripts\": [\\ + \"${FOLDER_NAME}=${SNAKE_CASED_FOLDER_NAME}.run:run\",\\ + ],\\ + }," "$SETUP_FILE" +else + echo "setup.py not found in $FOLDER_NAME" + exit 1 +fi + +# Step 2: Create run.py and copy contents from main.py +mkdir -p "$FOLDER_NAME/$SNAKE_CASED_FOLDER_NAME" +MAIN_PY="$FOLDER_NAME/main.py" +RUN_PY="$FOLDER_NAME/$SNAKE_CASED_FOLDER_NAME/run.py" +if [ -f "$MAIN_PY" ]; then + cp "$MAIN_PY" "$RUN_PY" + sed -i '' 's/if __name__ == "__main__":/def run():/' "$RUN_PY" +else + echo "main.py not found in $FOLDER_NAME" + exit 1 +fi + +# Step 3: Modify main.py +echo -e "#\n# Copyright (c) 2023 Airbyte, Inc., all rights reserved.\n#\n\nfrom ${SNAKE_CASED_FOLDER_NAME}.run import run\n\nif __name__ == \"__main__\":\n run()" > "$MAIN_PY" + +# Function to update dockerImageTag +update_docker_image_tag() { + if [ -f "$METADATA_FILE" ]; then + # Extract the current dockerImageTag version + CURRENT_VERSION=$(awk -F ': ' '/dockerImageTag/ {print $2}' "$METADATA_FILE" | tr -d '"') + + # Break the version into array (major, minor, patch) + IFS='.' read -r -a VERSION_PARTS <<< "$CURRENT_VERSION" + + # Increment the patch version + PATCH_VERSION=$((VERSION_PARTS[2] + 1)) + + # Construct the new version + NEW_VERSION="${VERSION_PARTS[0]}.${VERSION_PARTS[1]}.$PATCH_VERSION" + + # Use sed to replace the old version with the new version in metadata.yaml + sed -i '' "s/dockerImageTag: [0-9]*\.[0-9]*\.[0-9]*/dockerImageTag: $NEW_VERSION/" "$METADATA_FILE" + + # If there is a Dockerfile in the directory, also udpate the version for the LABEL io.airbyte.version="x.y.z" + DOCKER_FILE="$FOLDER_NAME/Dockerfile" + if [ -f "$DOCKER_FILE" ]; then + sed -i '' "s/LABEL io.airbyte.version=\"[0-9]*\.[0-9]*\.[0-9]*\"/LABEL io.airbyte.version=\"$NEW_VERSION\"/" "$DOCKER_FILE" + fi + + # Return the new version + echo "$NEW_VERSION" + else + echo "metadata.yaml not found in $FOLDER_NAME" + exit 1 + fi +} + +# Function to update changelog +update_changelog() { + local version=$1 + local changelog_entry="| $version | $(date +%Y-%m-%d) | [1234](https://github.com/airbytehq/airbyte/pull/1234) | prepare for airbyte-lib |" + + if [ -f "$DOC_FILE" ]; then + sed -i '' -e '/|.*---.*|.*---.*|.*---.*|.*---.*/a\'$'\n'"$changelog_entry" "$DOC_FILE" + + else + echo "Documentation file not found: $DOC_FILE" + exit 1 + fi +} + +# Main script execution +NEW_VERSION=$(update_docker_image_tag) +update_changelog "$NEW_VERSION" + + +echo "Modifications completed." + +airbyte-ci format fix all + +handle_git_operations() { + local folder_name="$1" + local docs_file="$2" + +t # Add changes + git add "$folder_name" + git add "$docs_file" + + # Commit changes + git commit -m "convert" + + git push --set-upstream origin "$BRANCH_NAME" +} + +handle_git_operations "$FOLDER_NAME" "$DOC_FILE" + +# create github pr using gh tool: +gh pr create --title "$FOLDER_NAME: Convert to airbyte-lib" --body "Make the connector ready to be consumed by airbyte-lib" --base master --head "$BRANCH_NAME" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml index ec478505c9551..0e4e0668f3d49 100644 --- a/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml +++ b/airbyte-integrations/connectors/source-apify-dataset/metadata.yaml @@ -16,10 +16,6 @@ data: icon: apify.svg license: MIT name: Apify Dataset - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-apify-dataset releaseDate: 2023-08-25 releaseStage: alpha releases: diff --git a/airbyte-integrations/connectors/source-google-drive/metadata.yaml b/airbyte-integrations/connectors/source-google-drive/metadata.yaml index fab6905514c00..bf40629baae06 100644 --- a/airbyte-integrations/connectors/source-google-drive/metadata.yaml +++ b/airbyte-integrations/connectors/source-google-drive/metadata.yaml @@ -18,10 +18,6 @@ data: enabled: true oss: enabled: true - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-google-drive releaseStage: alpha documentationUrl: https://docs.airbyte.com/integrations/sources/google-drive tags: diff --git a/airbyte-integrations/connectors/source-pokeapi/Dockerfile b/airbyte-integrations/connectors/source-pokeapi/Dockerfile index 9bf098140d9af..0d27d3737d6fa 100644 --- a/airbyte-integrations/connectors/source-pokeapi/Dockerfile +++ b/airbyte-integrations/connectors/source-pokeapi/Dockerfile @@ -34,5 +34,5 @@ COPY source_pokeapi ./source_pokeapi ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.2.1 +LABEL io.airbyte.version=0.2.0 LABEL io.airbyte.name=airbyte/source-pokeapi diff --git a/airbyte-integrations/connectors/source-pokeapi/main.py b/airbyte-integrations/connectors/source-pokeapi/main.py index f32ce6b381058..b80ba24cafd64 100644 --- a/airbyte-integrations/connectors/source-pokeapi/main.py +++ b/airbyte-integrations/connectors/source-pokeapi/main.py @@ -2,7 +2,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from source_pokeapi.run import run +import sys + +from airbyte_cdk.entrypoint import launch +from source_pokeapi import SourcePokeapi + if __name__ == "__main__": - run() + source = SourcePokeapi() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index 09553b556e054..076a75a780a48 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6371b14b-bc68-4236-bfbd-468e8df8e968 - dockerImageTag: 0.2.1 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-pokeapi githubIssueLabel: source-pokeapi icon: pokeapi.svg @@ -19,10 +19,6 @@ data: releaseDate: "2020-05-14" releaseStage: alpha supportLevel: community - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-pokeapi documentationUrl: https://docs.airbyte.com/integrations/sources/pokeapi tags: - language:low-code diff --git a/airbyte-integrations/connectors/source-pokeapi/setup.py b/airbyte-integrations/connectors/source-pokeapi/setup.py index ece0a748d58e8..2fa7839b58fca 100644 --- a/airbyte-integrations/connectors/source-pokeapi/setup.py +++ b/airbyte-integrations/connectors/source-pokeapi/setup.py @@ -26,9 +26,4 @@ extras_require={ "tests": TEST_REQUIREMENTS, }, - entry_points={ - "console_scripts": [ - "source-pokeapi=source_pokeapi.run:run", - ], - }, ) diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py deleted file mode 100644 index 2b573e6939542..0000000000000 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/run.py +++ /dev/null @@ -1,13 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -import sys - -from airbyte_cdk.entrypoint import launch -from source_pokeapi import SourcePokeapi - - -def run(): - source = SourcePokeapi() - launch(source, sys.argv[1:]) From 638b30c6dc78be5304d0d651168bf517d85c4245 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 18 Jan 2024 12:49:59 +0100 Subject: [PATCH 31/53] revert --- airbyte-integrations/connectors/convert.sh | 122 ------------------ .../connectors/source-pokeapi/main.py | 1 - 2 files changed, 123 deletions(-) delete mode 100755 airbyte-integrations/connectors/convert.sh diff --git a/airbyte-integrations/connectors/convert.sh b/airbyte-integrations/connectors/convert.sh deleted file mode 100755 index 6ffe1e56760b2..0000000000000 --- a/airbyte-integrations/connectors/convert.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/bash - -# Install github cli: brew install gh - -# Check if an argument was provided -if [ -z "$1" ]; then - echo "Please provide a folder name." - exit 1 -fi - -FOLDER_NAME="$1" -METADATA_FILE="$FOLDER_NAME/metadata.yaml" -SNAKE_CASED_FOLDER_NAME=$(echo "$FOLDER_NAME" | sed 's/-/_/g') -DOC_FOLDER_NAME=$(echo "$FOLDER_NAME" | sed 's/source-//') -DOC_FILE="../../docs/integrations/sources/${DOC_FOLDER_NAME}.md" -BRANCH_NAME="flash1293/airbyte-lib-convert-${FOLDER_NAME}" - -git checkout origin/master -git checkout -b "$BRANCH_NAME" - -# Step 1: Add entry_points to setup.py -SETUP_FILE="$FOLDER_NAME/setup.py" -if [ -f "$SETUP_FILE" ]; then - sed -i '' "/setup(/a \\ - entry_points={\\ - \"console_scripts\": [\\ - \"${FOLDER_NAME}=${SNAKE_CASED_FOLDER_NAME}.run:run\",\\ - ],\\ - }," "$SETUP_FILE" -else - echo "setup.py not found in $FOLDER_NAME" - exit 1 -fi - -# Step 2: Create run.py and copy contents from main.py -mkdir -p "$FOLDER_NAME/$SNAKE_CASED_FOLDER_NAME" -MAIN_PY="$FOLDER_NAME/main.py" -RUN_PY="$FOLDER_NAME/$SNAKE_CASED_FOLDER_NAME/run.py" -if [ -f "$MAIN_PY" ]; then - cp "$MAIN_PY" "$RUN_PY" - sed -i '' 's/if __name__ == "__main__":/def run():/' "$RUN_PY" -else - echo "main.py not found in $FOLDER_NAME" - exit 1 -fi - -# Step 3: Modify main.py -echo -e "#\n# Copyright (c) 2023 Airbyte, Inc., all rights reserved.\n#\n\nfrom ${SNAKE_CASED_FOLDER_NAME}.run import run\n\nif __name__ == \"__main__\":\n run()" > "$MAIN_PY" - -# Function to update dockerImageTag -update_docker_image_tag() { - if [ -f "$METADATA_FILE" ]; then - # Extract the current dockerImageTag version - CURRENT_VERSION=$(awk -F ': ' '/dockerImageTag/ {print $2}' "$METADATA_FILE" | tr -d '"') - - # Break the version into array (major, minor, patch) - IFS='.' read -r -a VERSION_PARTS <<< "$CURRENT_VERSION" - - # Increment the patch version - PATCH_VERSION=$((VERSION_PARTS[2] + 1)) - - # Construct the new version - NEW_VERSION="${VERSION_PARTS[0]}.${VERSION_PARTS[1]}.$PATCH_VERSION" - - # Use sed to replace the old version with the new version in metadata.yaml - sed -i '' "s/dockerImageTag: [0-9]*\.[0-9]*\.[0-9]*/dockerImageTag: $NEW_VERSION/" "$METADATA_FILE" - - # If there is a Dockerfile in the directory, also udpate the version for the LABEL io.airbyte.version="x.y.z" - DOCKER_FILE="$FOLDER_NAME/Dockerfile" - if [ -f "$DOCKER_FILE" ]; then - sed -i '' "s/LABEL io.airbyte.version=\"[0-9]*\.[0-9]*\.[0-9]*\"/LABEL io.airbyte.version=\"$NEW_VERSION\"/" "$DOCKER_FILE" - fi - - # Return the new version - echo "$NEW_VERSION" - else - echo "metadata.yaml not found in $FOLDER_NAME" - exit 1 - fi -} - -# Function to update changelog -update_changelog() { - local version=$1 - local changelog_entry="| $version | $(date +%Y-%m-%d) | [1234](https://github.com/airbytehq/airbyte/pull/1234) | prepare for airbyte-lib |" - - if [ -f "$DOC_FILE" ]; then - sed -i '' -e '/|.*---.*|.*---.*|.*---.*|.*---.*/a\'$'\n'"$changelog_entry" "$DOC_FILE" - - else - echo "Documentation file not found: $DOC_FILE" - exit 1 - fi -} - -# Main script execution -NEW_VERSION=$(update_docker_image_tag) -update_changelog "$NEW_VERSION" - - -echo "Modifications completed." - -airbyte-ci format fix all - -handle_git_operations() { - local folder_name="$1" - local docs_file="$2" - -t # Add changes - git add "$folder_name" - git add "$docs_file" - - # Commit changes - git commit -m "convert" - - git push --set-upstream origin "$BRANCH_NAME" -} - -handle_git_operations "$FOLDER_NAME" "$DOC_FILE" - -# create github pr using gh tool: -gh pr create --title "$FOLDER_NAME: Convert to airbyte-lib" --body "Make the connector ready to be consumed by airbyte-lib" --base master --head "$BRANCH_NAME" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-pokeapi/main.py b/airbyte-integrations/connectors/source-pokeapi/main.py index b80ba24cafd64..38a510a3f2d77 100644 --- a/airbyte-integrations/connectors/source-pokeapi/main.py +++ b/airbyte-integrations/connectors/source-pokeapi/main.py @@ -7,7 +7,6 @@ from airbyte_cdk.entrypoint import launch from source_pokeapi import SourcePokeapi - if __name__ == "__main__": source = SourcePokeapi() launch(source, sys.argv[1:]) From ee02e87be6e21a19f5061df1c981baaafc461c75 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 18 Jan 2024 13:49:49 +0100 Subject: [PATCH 32/53] fix --- .../airbyte_ci/connectors/publish/pipeline.py | 8 ++++-- .../airbyte_ci/poetry/publish/commands.py | 7 ++--- .../airbyte_ci/poetry/publish/context.py | 8 +++--- .../airbyte_ci/poetry/publish/pipeline.py | 4 +-- airbyte-ci/connectors/pipelines/poetry.lock | 27 ++++++++++++++++++- .../connectors/pipelines/pyproject.toml | 1 + 6 files changed, 43 insertions(+), 12 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index 492ac01882bf1..24d977f973ec0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -59,7 +59,11 @@ class CheckPypiPackageDoesNotExist(Step): title = "Check if the connector is published on pypi" async def _run(self) -> StepResult: - is_published = is_package_published(self.context.package_name, self.context.version, self.context.registry) + is_published = ( + self.context.package_name + and self.context.version + and is_package_published(self.context.package_name, self.context.version, self.context.registry) + ) if is_published: return StepResult( self, status=StepStatus.SKIPPED, stderr=f"{self.context.package_name} already exists in version {self.context.version}." @@ -340,7 +344,7 @@ async def _run_pypi_publish_pipeline(context: PublishConnectorContext) -> Tuple[ Run the pypi publish pipeline for a single connector. Return the results of the steps and a boolean indicating whether there was an error and the pipeline should be stopped. """ - results = [] + results: List[StepResult] = [] # Try to convert the context to a PyPIPublishContext. If it returns None, it means we don't need to publish to pypi. pypi_context = await PyPIPublishContext.from_publish_connector_context(context) if not pypi_context: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py index 5953830192df8..5a55b0a501f82 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py @@ -10,13 +10,14 @@ from typing import Optional import asyncclick as click -from connector_ops.utils import CONNECTOR_PATH_PREFIX from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context from .context import PyPIPublishContext from .pipeline import PublishToPyPI +CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors" + @click.command(cls=DaggerPipelineCommand, name="publish", help="Publish a Python package to PyPI.") @click.option( @@ -48,10 +49,10 @@ async def publish( ctx: click.Context, click_pipeline_context: ClickPipelineContext, pypi_token: str, - registry_url: bool, + registry_url: str, publish_name: Optional[str], publish_version: Optional[str], -) -> None: +) -> bool: context = PyPIPublishContext( is_local=ctx.obj["is_local"], git_branch=ctx.obj["git_branch"], diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py index ee029491a1cab..294454c617769 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py @@ -15,17 +15,17 @@ def __init__( self, pypi_token: str, package_path: str, - ci_report_bucket: str, report_output_prefix: str, is_local: bool, - git_branch: bool, - git_revision: bool, + git_branch: str, + git_revision: str, + ci_report_bucket: Optional[str] = None, registry: Optional[str] = None, gha_workflow_run_url: Optional[str] = None, dagger_logs_url: Optional[str] = None, pipeline_start_timestamp: Optional[int] = None, ci_context: Optional[str] = None, - ci_gcs_credentials: str = None, + ci_gcs_credentials: Optional[str] = None, package_name: Optional[str] = None, version: Optional[str] = None, ) -> None: diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py index 1ad438fe0f65d..3cc33bbe9ad0c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py @@ -6,7 +6,7 @@ import io import uuid from enum import Enum, auto -from typing import Optional, Tuple +from typing import Mapping, Optional, Tuple import tomli import tomli_w @@ -116,7 +116,7 @@ async def _pip_publish(self, dir_to_publish: Directory) -> StepResult: files = await dir_to_publish.entries() pypi_username = self.context.dagger_client.set_secret("pypi_username", "__token__") pypi_password = self.context.dagger_client.set_secret("pypi_password", f"pypi-{self.context.pypi_token}") - metadata = { + metadata: Mapping[str, str] = { "name": self.context.package_name, "version": self.context.version, # Enforce consistent author diff --git a/airbyte-ci/connectors/pipelines/poetry.lock b/airbyte-ci/connectors/pipelines/poetry.lock index fb6f1df90cab6..8fe5e25cc39b6 100644 --- a/airbyte-ci/connectors/pipelines/poetry.lock +++ b/airbyte-ci/connectors/pipelines/poetry.lock @@ -2272,6 +2272,31 @@ files = [ {file = "tomli_w-1.0.0.tar.gz", hash = "sha256:f463434305e0336248cac9c2dc8076b707d8a12d019dd349f5c1e382dd1ae1b9"}, ] +[[package]] +name = "types-requests" +version = "2.28.2" +description = "Typing stubs for requests" +optional = false +python-versions = "*" +files = [ + {file = "types-requests-2.28.2.tar.gz", hash = "sha256:398f88cd9302c796cb63d1021af2a1fb7ae507741a3d508edf8e0746d8c16a04"}, + {file = "types_requests-2.28.2-py3-none-any.whl", hash = "sha256:c164696bfdce0123901165c5f097a6cc4f6326268c65815d4b6a57eacfec5e81"}, +] + +[package.dependencies] +types-urllib3 = "<1.27" + +[[package]] +name = "types-urllib3" +version = "1.26.25.14" +description = "Typing stubs for urllib3" +optional = false +python-versions = "*" +files = [ + {file = "types-urllib3-1.26.25.14.tar.gz", hash = "sha256:229b7f577c951b8c1b92c1bc2b2fdb0b49847bd2af6d1cc2a2e3dd340f3bda8f"}, + {file = "types_urllib3-1.26.25.14-py3-none-any.whl", hash = "sha256:9683bbb7fb72e32bfe9d2be6e04875fbe1b3eeec3cbb4ea231435aa7fd6b4f0e"}, +] + [[package]] name = "typing-extensions" version = "4.9.0" @@ -2522,4 +2547,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = "~3.10" -content-hash = "860276a3ba4a5809e9775e2df33b02eb1867420e3fdcca7ee4b3471ac66f16df" +content-hash = "4ff412a744be269c2fd8048eaa759b4c02580ec027c41a197505b82f86df22ce" diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 412f0067d2da6..da192d8ade919 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -29,6 +29,7 @@ asyncclick = "^8.1.3.4" certifi = "^2023.11.17" tomli = "^2.0.1" tomli-w = "^1.0.0" +types-requests = "2.28.2" [tool.poetry.group.test.dependencies] pytest = "^6.2.5" From 2dd2124045e5b3b72de3e614b39abaa19ef0bcca Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 18 Jan 2024 14:05:28 +0100 Subject: [PATCH 33/53] fix --- .../pipelines/airbyte_ci/poetry/publish/pipeline.py | 8 ++++---- .../connectors/pipelines/pipelines/helpers/slack.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py index 3cc33bbe9ad0c..ede5c6c550824 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py @@ -6,7 +6,7 @@ import io import uuid from enum import Enum, auto -from typing import Mapping, Optional, Tuple +from typing import Dict, Mapping, Optional, Tuple import tomli import tomli_w @@ -116,9 +116,9 @@ async def _pip_publish(self, dir_to_publish: Directory) -> StepResult: files = await dir_to_publish.entries() pypi_username = self.context.dagger_client.set_secret("pypi_username", "__token__") pypi_password = self.context.dagger_client.set_secret("pypi_password", f"pypi-{self.context.pypi_token}") - metadata: Mapping[str, str] = { - "name": self.context.package_name, - "version": self.context.version, + metadata: Dict[str, str] = { + "name": str(self.context.package_name), + "version": str(self.context.version), # Enforce consistent author "author": "Airbyte", "author_email": "contact@airbyte.io", diff --git a/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py index affc981a60de0..619be4278b575 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/slack.py @@ -4,11 +4,11 @@ import json -import requests # type: ignore +import requests from pipelines import main_logger -def send_message_to_webhook(message: str, channel: str, webhook: str) -> dict: +def send_message_to_webhook(message: str, channel: str, webhook: str) -> requests.Response: payload = {"channel": f"#{channel}", "username": "Connectors CI/CD Bot", "text": message} response = requests.post(webhook, data={"payload": json.dumps(payload)}) From 13fa2a1efc1f0d9134b4e55b9d22ec64da1bb5b3 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Thu, 18 Jan 2024 15:48:41 +0100 Subject: [PATCH 34/53] fix --- .../pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py index ede5c6c550824..921993a56e22f 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py @@ -6,7 +6,7 @@ import io import uuid from enum import Enum, auto -from typing import Dict, Mapping, Optional, Tuple +from typing import Dict, Optional, Tuple import tomli import tomli_w From f80878fff07f95a2cafb3dd42d8bad620de39bc5 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Sat, 20 Jan 2024 12:38:44 +0100 Subject: [PATCH 35/53] review comments --- .../airbyte_ci/connectors/publish/context.py | 6 +- .../airbyte_ci/connectors/publish/pipeline.py | 24 ++-- .../airbyte_ci/poetry/publish/commands.py | 45 +++++-- .../pipelines/airbyte_ci/poetry/utils.py | 10 +- .../python_registry}/context.py | 35 +++--- .../python_registry}/pipeline.py | 112 ++++++++++-------- .../connectors/pipelines/pipelines/consts.py | 2 +- .../tests/test_poetry/test_poetry_publish.py | 23 ++-- .../pipelines/tests/test_publish.py | 11 +- 9 files changed, 157 insertions(+), 111 deletions(-) rename airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/{poetry/publish => steps/python_registry}/context.py (77%) rename airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/{poetry/publish => steps/python_registry}/pipeline.py (50%) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py index a4471bac7ecaf..829ab07b4e0a0 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/context.py @@ -89,12 +89,16 @@ def metadata_service_gcs_credentials_secret(self) -> Secret: def spec_cache_gcs_credentials_secret(self) -> Secret: return self.dagger_client.set_secret("spec_cache_gcs_credentials", self.spec_cache_gcs_credentials) + @property + def pre_release_suffix(self) -> str: + return self.git_revision[:10] + @property def docker_image_tag(self) -> str: # get the docker image tag from the parent class metadata_tag = super().docker_image_tag if self.pre_release: - return f"{metadata_tag}-dev.{self.git_revision[:10]}" + return f"{metadata_tag}-dev.{self.pre_release_suffix}" else: return metadata_tag diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index 24d977f973ec0..801220e4ef241 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -14,7 +14,7 @@ from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.airbyte_ci.metadata.pipeline import MetadataUpload, MetadataValidation -from pipelines.airbyte_ci.poetry.publish.pipeline import PublishToPyPI, PyPIPublishContext +from pipelines.airbyte_ci.poetry.publish.pipeline import PublishToPythonRegistry, PythonRegistryPublishContext from pipelines.airbyte_ci.poetry.utils import is_package_published from pipelines.dagger.actions.remote_storage import upload_to_gcs from pipelines.dagger.actions.system import docker @@ -55,22 +55,22 @@ async def _run(self) -> StepResult: class CheckPypiPackageDoesNotExist(Step): - context: PyPIPublishContext + context: PythonRegistryPublishContext title = "Check if the connector is published on pypi" async def _run(self) -> StepResult: - is_published = ( - self.context.package_name - and self.context.version - and is_package_published(self.context.package_name, self.context.version, self.context.registry) - ) + is_published = is_package_published(self.context.package_metadata, self.context.registry) if is_published: return StepResult( - self, status=StepStatus.SKIPPED, stderr=f"{self.context.package_name} already exists in version {self.context.version}." + self, + status=StepStatus.SKIPPED, + stderr=f"{self.context.package_metadata.name} already exists in version {self.context.package_metadata.version}.", ) else: return StepResult( - self, status=StepStatus.SUCCESS, stdout=f"{self.context.package_name} does not exist in version {self.context.version}." + self, + status=StepStatus.SUCCESS, + stdout=f"{self.context.package_metadata.name} does not exist in version {self.context.package_metadata.version}.", ) @@ -345,8 +345,8 @@ async def _run_pypi_publish_pipeline(context: PublishConnectorContext) -> Tuple[ Return the results of the steps and a boolean indicating whether there was an error and the pipeline should be stopped. """ results: List[StepResult] = [] - # Try to convert the context to a PyPIPublishContext. If it returns None, it means we don't need to publish to pypi. - pypi_context = await PyPIPublishContext.from_publish_connector_context(context) + # Try to convert the context to a PythonRegistryPublishContext. If it returns None, it means we don't need to publish to pypi. + pypi_context = await PythonRegistryPublishContext.from_publish_connector_context(context) if not pypi_context: return results, False @@ -356,7 +356,7 @@ async def _run_pypi_publish_pipeline(context: PublishConnectorContext) -> Tuple[ context.logger.info("The connector version is already published on pypi.") elif check_pypi_package_exists_results.status is StepStatus.SUCCESS: context.logger.info("The connector version is not published on pypi. Let's build and publish it.") - publish_to_pypi_results = await PublishToPyPI(pypi_context).run() + publish_to_pypi_results = await PublishToPythonRegistry(pypi_context).run() results.append(publish_to_pypi_results) if publish_to_pypi_results.status is StepStatus.FAILURE: return results, True diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py index 5a55b0a501f82..01f214d328a8b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py @@ -10,16 +10,35 @@ from typing import Optional import asyncclick as click +from packaging import version +from pipelines.airbyte_ci.steps.python_registry.context import PythonRegistryPublishContext +from pipelines.airbyte_ci.steps.python_registry.pipeline import PublishToPythonRegistry +from pipelines.cli.confirm_prompt import confirm from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand +from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_URL from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context +from pipelines.models.steps import StepStatus -from .context import PyPIPublishContext -from .pipeline import PublishToPyPI -CONNECTOR_PATH_PREFIX = "airbyte-integrations/connectors" +async def _has_metadata_yaml(context: PythonRegistryPublishContext) -> bool: + dir_to_publish = context.get_repo_dir(context.package_path) + return "metadata.yaml" in await dir_to_publish.entries() -@click.command(cls=DaggerPipelineCommand, name="publish", help="Publish a Python package to PyPI.") +def _validate_python_version(_ctx: dict, _param: dict, value: Optional[str]) -> Optional[str]: + """ + Check if an given version is valid. + """ + if value is None: + return value + try: + version.Version(value) + return value + except version.InvalidVersion: + raise click.BadParameter(f"Version {value} is not a valid version.") + + +@click.command(cls=DaggerPipelineCommand, name="publish", help="Publish a Python package to a registry.") @click.option( "--pypi-token", help="Access token", @@ -31,7 +50,7 @@ "--registry-url", help="Which registry to publish to. If not set, the default pypi is used. For test pypi, use https://test.pypi.org/legacy/", type=click.STRING, - default="https://pypi.org/simple", + default=DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, ) @click.option( "--publish-name", @@ -42,6 +61,7 @@ "--publish-version", help="The version of the package to publish. If not set, the version will be inferred from the pyproject.toml file of the package.", type=click.STRING, + callback=_validate_python_version, ) @pass_pipeline_context @click.pass_context @@ -53,7 +73,7 @@ async def publish( publish_name: Optional[str], publish_version: Optional[str], ) -> bool: - context = PyPIPublishContext( + context = PythonRegistryPublishContext( is_local=ctx.obj["is_local"], git_branch=ctx.obj["git_branch"], git_revision=ctx.obj["git_revision"], @@ -71,12 +91,15 @@ async def publish( version=publish_version, ) - if context.package_path.startswith(CONNECTOR_PATH_PREFIX): - context.logger.warning("It looks like you are trying to publish a connector. Please use the `connectors` command group instead.") - dagger_client = await click_pipeline_context.get_dagger_client(pipeline_name=f"Publish {ctx.obj['package_path']} to PyPI") context.dagger_client = dagger_client - await PublishToPyPI(context).run() + if await _has_metadata_yaml(context): + confirm( + "It looks like you are trying to publish a connector. In most cases, the `connectors` command group should be used instead. Do you want to continue?", + abort=True, + ) + + publish_result = await PublishToPythonRegistry(context).run() - return True + return publish_result.status is StepStatus.SUCCESS diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py index c8aae74f2cba7..56c571f5294c1 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py @@ -1,9 +1,10 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. import requests +from pipelines.airbyte_ci.steps.python_registry.context import PythonPackageMetadata -def is_package_published(package_name: str, version: str, base_url: str) -> bool: +def is_package_published(package_metadata: PythonPackageMetadata, base_url: str) -> bool: """ Check if a package with a specific version is published on PyPI or Test PyPI. @@ -12,7 +13,12 @@ def is_package_published(package_name: str, version: str, base_url: str) -> bool :param test_pypi: Set to True to check on Test PyPI, False for regular PyPI. :return: True if the package is found with the specified version, False otherwise. """ - url = f"{base_url}{package_name}/{version}/json" + package_name = package_metadata.name + version = package_metadata.version + if not package_name or not version: + return False + + url = f"{base_url}/{package_name}/{version}/json" response = requests.get(url) return response.status_code == 200 diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py similarity index 77% rename from airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py index 294454c617769..7c7530f46dc87 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py @@ -3,14 +3,21 @@ # import os -from datetime import datetime -from typing import Optional +from dataclasses import dataclass +from typing import Optional, Type from pipelines.airbyte_ci.connectors.context import PipelineContext from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext +from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_URL -class PyPIPublishContext(PipelineContext): +@dataclass +class PythonPackageMetadata: + name: Optional[str] + version: Optional[str] + + +class PythonRegistryPublishContext(PipelineContext): def __init__( self, pypi_token: str, @@ -20,7 +27,7 @@ def __init__( git_branch: str, git_revision: str, ci_report_bucket: Optional[str] = None, - registry: Optional[str] = None, + registry: str = DEFAULT_PYTHON_PACKAGE_REGISTRY_URL, gha_workflow_run_url: Optional[str] = None, dagger_logs_url: Optional[str] = None, pipeline_start_timestamp: Optional[int] = None, @@ -30,10 +37,9 @@ def __init__( version: Optional[str] = None, ) -> None: self.pypi_token = pypi_token - self.registry = registry or "https://pypi.org/simple" + self.registry = registry self.package_path = package_path - self.package_name = package_name - self.version = version + self.package_metadata = PythonPackageMetadata(package_name, version) pipeline_name = f"Publish PyPI {package_path}" @@ -51,10 +57,12 @@ def __init__( ci_gcs_credentials=ci_gcs_credentials, ) - @staticmethod - async def from_publish_connector_context(connector_context: PublishConnectorContext) -> Optional["PyPIPublishContext"]: + @classmethod + async def from_publish_connector_context( + cls: Type["PythonRegistryPublishContext"], connector_context: PublishConnectorContext + ) -> Optional["PythonRegistryPublishContext"]: """ - Create a PyPIPublishContext from a ConnectorContext. + Create a PythonRegistryPublishContext from a ConnectorContext. The metadata of the connector is read from the current workdir to capture changes that are not yet published. If pypi is not enabled, this will return None. @@ -71,12 +79,11 @@ async def from_publish_connector_context(connector_context: PublishConnectorCont version = current_metadata["dockerImageTag"] if connector_context.pre_release: # use current date as pre-release version - rc_tag = datetime.now().strftime("%Y%m%d%H%M") - version = f"{version}.dev{rc_tag}" + version = f"{version}+{connector_context.pre_release_suffix}" - pypi_context = PyPIPublishContext( + pypi_context = cls( pypi_token=os.environ["PYPI_TOKEN"], - registry="https://test.pypi.org/legacy/", # TODO: go live + registry="https://test.pypi.org/legacy", # TODO: go live package_path=str(connector_context.connector.code_directory), package_name=current_metadata["remoteRegistries"]["pypi"]["packageName"], version=version, diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/pipeline.py similarity index 50% rename from airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/pipeline.py index 921993a56e22f..fd9901f548a9a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/pipeline.py @@ -6,12 +6,12 @@ import io import uuid from enum import Enum, auto -from typing import Dict, Optional, Tuple +from typing import Dict, Optional import tomli import tomli_w from dagger import Container, Directory -from pipelines.airbyte_ci.poetry.publish.context import PyPIPublishContext +from pipelines.airbyte_ci.steps.python_registry.context import PythonPackageMetadata, PythonRegistryPublishContext from pipelines.consts import PYPROJECT_TOML_FILE_PATH, SETUP_PY_FILE_PATH from pipelines.dagger.actions.python.poetry import with_poetry from pipelines.helpers.utils import sh_dash_c @@ -21,32 +21,26 @@ class PackageType(Enum): POETRY = auto() PIP = auto() - NONE = auto() -class PublishToPyPI(Step): - context: PyPIPublishContext +class PublishToPythonRegistry(Step): + context: PythonRegistryPublishContext title = "Publish package to PyPI" def _get_base_container(self) -> Container: return with_poetry(self.context) - async def _get_package_metadata_from_pyproject_toml(self, dir_to_publish: Directory) -> Optional[Tuple[str, str]]: - pyproject_toml = dir_to_publish.file(PYPROJECT_TOML_FILE_PATH) + async def _get_package_metadata_from_pyproject_toml(self, package_dir_to_publish: Directory) -> Optional[PythonPackageMetadata]: + pyproject_toml = package_dir_to_publish.file(PYPROJECT_TOML_FILE_PATH) pyproject_toml_content = await pyproject_toml.contents() contents = tomli.loads(pyproject_toml_content) - if ( - "tool" not in contents - or "poetry" not in contents["tool"] - or "name" not in contents["tool"]["poetry"] - or "version" not in contents["tool"]["poetry"] - ): + try: + return PythonPackageMetadata(contents["tool"]["poetry"]["name"], contents["tool"]["poetry"]["version"]) + except KeyError: return None - return (contents["tool"]["poetry"]["name"], contents["tool"]["poetry"]["version"]) - - async def _get_package_type(self, dir_to_publish: Directory) -> PackageType: - files = await dir_to_publish.entries() + async def _get_package_type(self, package_dir_to_publish: Directory) -> Optional[PackageType]: + files = await package_dir_to_publish.entries() has_pyproject_toml = PYPROJECT_TOML_FILE_PATH in files has_setup_py = SETUP_PY_FILE_PATH in files if has_pyproject_toml: @@ -54,54 +48,72 @@ async def _get_package_type(self, dir_to_publish: Directory) -> PackageType: elif has_setup_py: return PackageType.PIP else: - return PackageType.NONE + return None async def _run(self) -> StepResult: - dir_to_publish = await self.context.get_repo_dir(self.context.package_path) - package_type = await self._get_package_type(dir_to_publish) + package_dir_to_publish = await self.context.get_repo_dir(self.context.package_path) + package_type = await self._get_package_type(package_dir_to_publish) - if package_type == PackageType.NONE: + if not package_type: return self.skip("Connector does not have a pyproject.toml file or setup.py file, skipping.") - # Try to infer package name and version from the pyproject.toml file. If it is not present, we need to have the package name and version set - # Setup.py packages need to set package name and version as parameter - if not self.context.package_name or not self.context.version: - if not package_type == PackageType.POETRY: - return self.skip( - "Connector does not have a pyproject.toml file and version and package name is not set otherwise, skipping." - ) + result = await self._ensure_package_name_and_version(package_dir_to_publish, package_type) + if result: + return result + + self.logger.info( + f"Uploading package {self.context.package_metadata.name} version {self.context.package_metadata.version} to {self.context.registry}..." + ) + + return await self._publish(package_dir_to_publish, package_type) + + async def _ensure_package_name_and_version(self, package_dir_to_publish: Directory, package_type: PackageType) -> Optional[StepResult]: + """ + Try to infer package name and version from the pyproject.toml file. If it is not present, we need to have the package name and version set. + Setup.py packages need to set package name and version as parameter. + + Returns None if package name and version are set, otherwise a StepResult with a skip message. + """ + if self.context.package_metadata.name and self.context.package_metadata.version: + return None + + if package_type is not PackageType.POETRY: + return self.skip("Connector does not have a pyproject.toml file and version and package name is not set otherwise, skipping.") - package_metadata = await self._get_package_metadata_from_pyproject_toml(dir_to_publish) + inferred_package_metadata = await self._get_package_metadata_from_pyproject_toml(package_dir_to_publish) - if not package_metadata: - return self.skip( - "Connector does not have a pyproject.toml file which specifies package name and version and they are not set otherwise, skipping." - ) + if not inferred_package_metadata: + return self.skip( + "Connector does not have a pyproject.toml file which specifies package name and version and they are not set otherwise, skipping." + ) - self.context.package_name = package_metadata[0] - self.context.version = package_metadata[1] + if not self.context.package_metadata.name: + self.context.package_metadata.name = inferred_package_metadata.name + if not self.context.package_metadata.version: + self.context.package_metadata.version = inferred_package_metadata.version - self.logger.info(f"Uploading package {self.context.package_name} version {self.context.version} to {self.context.registry}...") + return None - if package_type == PackageType.PIP: - return await self._pip_publish(dir_to_publish) + async def _publish(self, package_dir_to_publish: Directory, package_type: PackageType) -> StepResult: + if package_type is PackageType.PIP: + return await self._pip_publish(package_dir_to_publish) else: - return await self._poetry_publish(dir_to_publish) + return await self._poetry_publish(package_dir_to_publish) - async def _poetry_publish(self, dir_to_publish: Directory) -> StepResult: + async def _poetry_publish(self, package_dir_to_publish: Directory) -> StepResult: pypi_token = self.context.dagger_client.set_secret("pypi_token", f"pypi-{self.context.pypi_token}") - pyproject_toml = dir_to_publish.file(PYPROJECT_TOML_FILE_PATH) + pyproject_toml = package_dir_to_publish.file(PYPROJECT_TOML_FILE_PATH) pyproject_toml_content = await pyproject_toml.contents() contents = tomli.loads(pyproject_toml_content) # make sure package name and version are set to the configured one - contents["tool"]["poetry"]["name"] = self.context.package_name - contents["tool"]["poetry"]["version"] = self.context.version + contents["tool"]["poetry"]["name"] = self.context.package_metadata.name + contents["tool"]["poetry"]["version"] = self.context.package_metadata.version # enforce consistent author contents["tool"]["poetry"]["authors"] = ["Airbyte "] poetry_publish = ( self._get_base_container() .with_secret_variable("PYPI_TOKEN", pypi_token) - .with_directory("package", dir_to_publish) + .with_directory("package", package_dir_to_publish) .with_workdir("package") .with_new_file(PYPROJECT_TOML_FILE_PATH, contents=tomli_w.dumps(contents)) .with_exec(["poetry", "config", "repositories.mypypi", self.context.registry]) @@ -112,19 +124,19 @@ async def _poetry_publish(self, dir_to_publish: Directory) -> StepResult: return await self.get_step_result(poetry_publish) - async def _pip_publish(self, dir_to_publish: Directory) -> StepResult: - files = await dir_to_publish.entries() + async def _pip_publish(self, package_dir_to_publish: Directory) -> StepResult: + files = await package_dir_to_publish.entries() pypi_username = self.context.dagger_client.set_secret("pypi_username", "__token__") pypi_password = self.context.dagger_client.set_secret("pypi_password", f"pypi-{self.context.pypi_token}") metadata: Dict[str, str] = { - "name": str(self.context.package_name), - "version": str(self.context.version), + "name": str(self.context.package_metadata.name), + "version": str(self.context.package_metadata.version), # Enforce consistent author "author": "Airbyte", "author_email": "contact@airbyte.io", } if "README.md" in files: - metadata["long_description"] = await dir_to_publish.file("README.md").contents() + metadata["long_description"] = await package_dir_to_publish.file("README.md").contents() metadata["long_description_content_type"] = "text/markdown" config = configparser.ConfigParser() @@ -137,7 +149,7 @@ async def _pip_publish(self, dir_to_publish: Directory) -> StepResult: twine_upload = ( self._get_base_container() .with_exec(sh_dash_c(["apt-get update", "apt-get install -y twine"])) - .with_directory("package", dir_to_publish) + .with_directory("package", package_dir_to_publish) .with_workdir("package") # clear out setup.py metadata so setup.cfg is used .with_exec(["sed", "-i", "/name=/d; /author=/d; /author_email=/d; /version=/d", SETUP_PY_FILE_PATH]) diff --git a/airbyte-ci/connectors/pipelines/pipelines/consts.py b/airbyte-ci/connectors/pipelines/pipelines/consts.py index 57a413a5d18b3..20bb7bd55211d 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/consts.py +++ b/airbyte-ci/connectors/pipelines/pipelines/consts.py @@ -60,8 +60,8 @@ POETRY_CACHE_PATH = "/root/.cache/pypoetry" STORAGE_DRIVER = "fuse-overlayfs" TAILSCALE_AUTH_KEY = os.getenv("TAILSCALE_AUTH_KEY") -PYPROJECT_TOML_FILE_PATH = "pyproject.toml" SETUP_PY_FILE_PATH = "setup.py" +DEFAULT_PYTHON_PACKAGE_REGISTRY_URL = "https://pypi.org/simple" class CIContext(str, Enum): diff --git a/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py index 4a59ea920bd8d..5ca048ea7dbe8 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py @@ -2,23 +2,14 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import json -import random -from pathlib import Path -from typing import List -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock -import anyio import pytest import requests -from connector_ops.utils import Connector, ConnectorLanguage -from dagger import Client, Directory, Platform -from pipelines.airbyte_ci.connectors.context import ConnectorContext +from dagger import Client, Platform from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline -from pipelines.airbyte_ci.connectors.upgrade_cdk import pipeline as upgrade_cdk_pipeline -from pipelines.airbyte_ci.poetry.publish.context import PyPIPublishContext +from pipelines.airbyte_ci.steps.python_registry.context import PythonPackageMetadata, PythonRegistryPublishContext from pipelines.dagger.actions.python.poetry import with_poetry -from pipelines.models.contexts.pipeline_context import PipelineContext from pipelines.models.steps import StepStatus pytestmark = [ @@ -28,7 +19,7 @@ @pytest.fixture def context(dagger_client: Client): - context = PyPIPublishContext( + context = PythonRegistryPublishContext( package_path="test", version="0.2.0", pypi_token="test", @@ -61,8 +52,8 @@ def context(dagger_client: Client): ), ], ) -async def test_run_poetry_publish(context: PyPIPublishContext, package_path: str, package_name: str, expected_asset: str): - context.package_name = package_name +async def test_run_poetry_publish(context: PythonRegistryPublishContext, package_path: str, package_name: str, expected_asset: str): + context.package_metadata = PythonPackageMetadata(package_name, "0.2.0") context.package_path = package_path pypi_registry = ( # need to use linux/amd64 because the pypiserver image is only available for that platform @@ -74,7 +65,7 @@ async def test_run_poetry_publish(context: PyPIPublishContext, package_path: str ) base_container = with_poetry(context).with_service_binding("local_registry", pypi_registry) - step = publish_pipeline.PublishToPyPI(context) + step = publish_pipeline.PublishToPythonRegistry(context) step._get_base_container = MagicMock(return_value=base_container) step_result = await step.run() assert step_result.status == StepStatus.SUCCESS diff --git a/airbyte-ci/connectors/pipelines/tests/test_publish.py b/airbyte-ci/connectors/pipelines/tests/test_publish.py index a2fbc404911cb..8e06d28de1596 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_publish.py @@ -360,7 +360,9 @@ async def test_run_connector_pypi_publish_pipeline( for module, to_mock in STEPS_TO_PATCH: mocker.patch.object(module, to_mock, return_value=mocker.AsyncMock()) - mocked_publish_to_pypi = mocker.patch("pipelines.airbyte_ci.connectors.publish.pipeline.PublishToPyPI", return_value=mocker.AsyncMock()) + mocked_publish_to_pypi = mocker.patch( + "pipelines.airbyte_ci.connectors.publish.pipeline.PublishToPythonRegistry", return_value=mocker.AsyncMock() + ) for step in [ publish_pipeline.MetadataValidation, @@ -380,6 +382,7 @@ async def test_run_connector_pypi_publish_pipeline( context = mocker.MagicMock( ci_gcs_credentials="", + pre_release=False, connector=mocker.MagicMock( code_directory="path/to/connector", metadata={"dockerImageTag": "1.2.3", "remoteRegistries": {"pypi": {"enabled": pypi_enabled, "packageName": "test"}}}, @@ -392,9 +395,9 @@ async def test_run_connector_pypi_publish_pipeline( mocked_publish_to_pypi.return_value.run.assert_called_once() # assert that the first argument passed to mocked_publish_to_pypi contains the things from the context assert mocked_publish_to_pypi.call_args.args[0].pypi_token == "test" - assert mocked_publish_to_pypi.call_args.args[0].package_name == "test" - assert mocked_publish_to_pypi.call_args.args[0].version == "1.2.3" - assert mocked_publish_to_pypi.call_args.args[0].registry == "https://test.pypi.org/" + assert mocked_publish_to_pypi.call_args.args[0].package_metadata.name == "test" + assert mocked_publish_to_pypi.call_args.args[0].package_metadata.version == "1.2.3" + assert mocked_publish_to_pypi.call_args.args[0].registry == "https://test.pypi.org/legacy" assert mocked_publish_to_pypi.call_args.args[0].package_path == "path/to/connector" else: mocked_publish_to_pypi.return_value.run.assert_not_called() From 707f769e0ecac2891008bba5a63deb7cf795c0c5 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Sat, 20 Jan 2024 12:44:37 +0100 Subject: [PATCH 36/53] review comments --- .../pipelines/airbyte_ci/connectors/publish/pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index 801220e4ef241..d46776c3c344e 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -14,8 +14,8 @@ from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.airbyte_ci.metadata.pipeline import MetadataUpload, MetadataValidation -from pipelines.airbyte_ci.poetry.publish.pipeline import PublishToPythonRegistry, PythonRegistryPublishContext from pipelines.airbyte_ci.poetry.utils import is_package_published +from pipelines.airbyte_ci.steps.python_registry.pipeline import PublishToPythonRegistry, PythonRegistryPublishContext from pipelines.dagger.actions.remote_storage import upload_to_gcs from pipelines.dagger.actions.system import docker from pipelines.models.steps import Step, StepResult, StepStatus From a0fbf5fa001a43ae3f083ba1b60a51f851c037d5 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Sat, 20 Jan 2024 12:49:33 +0100 Subject: [PATCH 37/53] enable pypi --- airbyte-integrations/connectors/source-pokeapi/metadata.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index 076a75a780a48..aaa7f63a8b336 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -20,6 +20,10 @@ data: releaseStage: alpha supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/pokeapi + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-pokeapi tags: - language:low-code metadataSpecVersion: "1.0" From e082d6535c60c4a86c2da9d8f523d7f11e2ebb1d Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Sat, 20 Jan 2024 12:54:44 +0100 Subject: [PATCH 38/53] enable pypi second try --- airbyte-integrations/connectors/source-pokeapi/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index aaa7f63a8b336..fec36e8f0184a 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6371b14b-bc68-4236-bfbd-468e8df8e968 - dockerImageTag: 0.2.0 + dockerImageTag: 0.2.1 dockerRepository: airbyte/source-pokeapi githubIssueLabel: source-pokeapi icon: pokeapi.svg From 26f680a214c5d75816fc7472e5eb0b04b8a0d657 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Sat, 20 Jan 2024 12:59:13 +0100 Subject: [PATCH 39/53] enable pypi debug --- .../pipelines/airbyte_ci/steps/python_registry/context.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py index 7c7530f46dc87..ff6ab06b106bc 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py @@ -69,6 +69,7 @@ async def from_publish_connector_context( """ current_metadata = connector_context.connector.metadata + connector_context.logger.info(f"Current metadata: {str(current_metadata)}") if ( "remoteRegistries" not in current_metadata or "pypi" not in current_metadata["remoteRegistries"] From 0b78a12fecfacf2059d2d85fbcac22c4ebdf8e9e Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Sat, 20 Jan 2024 13:05:38 +0100 Subject: [PATCH 40/53] fix test --- .../pipelines/airbyte_ci/steps/python_registry/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py index ff6ab06b106bc..e3ce9121f2dd6 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py @@ -84,7 +84,7 @@ async def from_publish_connector_context( pypi_context = cls( pypi_token=os.environ["PYPI_TOKEN"], - registry="https://test.pypi.org/legacy", # TODO: go live + registry="https://test.pypi.org/legacy/", # TODO: go live package_path=str(connector_context.connector.code_directory), package_name=current_metadata["remoteRegistries"]["pypi"]["packageName"], version=version, From 391b68cd5239b2d87ee6c2548b3152d074f73ad5 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Sat, 20 Jan 2024 13:05:46 +0100 Subject: [PATCH 41/53] fix test --- airbyte-ci/connectors/pipelines/tests/test_publish.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/tests/test_publish.py b/airbyte-ci/connectors/pipelines/tests/test_publish.py index 8e06d28de1596..929f49d8e3c18 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_publish.py @@ -397,7 +397,7 @@ async def test_run_connector_pypi_publish_pipeline( assert mocked_publish_to_pypi.call_args.args[0].pypi_token == "test" assert mocked_publish_to_pypi.call_args.args[0].package_metadata.name == "test" assert mocked_publish_to_pypi.call_args.args[0].package_metadata.version == "1.2.3" - assert mocked_publish_to_pypi.call_args.args[0].registry == "https://test.pypi.org/legacy" + assert mocked_publish_to_pypi.call_args.args[0].registry == "https://test.pypi.org/legacy/" assert mocked_publish_to_pypi.call_args.args[0].package_path == "path/to/connector" else: mocked_publish_to_pypi.return_value.run.assert_not_called() From bd7c918163c13ffe20f98fbef4c3dfdfa360c4bb Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Sat, 20 Jan 2024 13:24:42 +0100 Subject: [PATCH 42/53] fix test --- .../pipelines/airbyte_ci/connectors/publish/pipeline.py | 2 +- .../pipelines/airbyte_ci/steps/python_registry/context.py | 4 +++- .../airbyte_ci/{poetry => steps/python_registry}/utils.py | 7 ++++++- 3 files changed, 10 insertions(+), 3 deletions(-) rename airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/{poetry => steps/python_registry}/utils.py (81%) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index d46776c3c344e..7be64e00e4e30 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -14,8 +14,8 @@ from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.airbyte_ci.metadata.pipeline import MetadataUpload, MetadataValidation -from pipelines.airbyte_ci.poetry.utils import is_package_published from pipelines.airbyte_ci.steps.python_registry.pipeline import PublishToPythonRegistry, PythonRegistryPublishContext +from pipelines.airbyte_ci.steps.python_registry.utils import is_package_published from pipelines.dagger.actions.remote_storage import upload_to_gcs from pipelines.dagger.actions.system import docker from pipelines.models.steps import Step, StepResult, StepStatus diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py index e3ce9121f2dd6..ecbeb7180c806 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py @@ -4,6 +4,7 @@ import os from dataclasses import dataclass +from datetime import datetime from typing import Optional, Type from pipelines.airbyte_ci.connectors.context import PipelineContext @@ -80,7 +81,8 @@ async def from_publish_connector_context( version = current_metadata["dockerImageTag"] if connector_context.pre_release: # use current date as pre-release version - version = f"{version}+{connector_context.pre_release_suffix}" + release_candidate_tag = datetime.now().strftime("%Y%m%d%H%M") + version = f"{version}.dev{release_candidate_tag}" pypi_context = cls( pypi_token=os.environ["PYPI_TOKEN"], diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/utils.py similarity index 81% rename from airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/utils.py index 56c571f5294c1..6f610f628a132 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/utils.py @@ -1,10 +1,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from urllib.parse import urlparse + import requests from pipelines.airbyte_ci.steps.python_registry.context import PythonPackageMetadata -def is_package_published(package_metadata: PythonPackageMetadata, base_url: str) -> bool: +def is_package_published(package_metadata: PythonPackageMetadata, registry_url: str) -> bool: """ Check if a package with a specific version is published on PyPI or Test PyPI. @@ -18,6 +20,9 @@ def is_package_published(package_metadata: PythonPackageMetadata, base_url: str) if not package_name or not version: return False + parsed_registry_url = urlparse(registry_url) + base_url = f"{parsed_registry_url.scheme}://{parsed_registry_url.netloc}" + url = f"{base_url}/{package_name}/{version}/json" response = requests.get(url) From 1a6e0c3aa2ca5d1ccae732bd650e27c37f79f3db Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Sat, 20 Jan 2024 13:29:02 +0100 Subject: [PATCH 43/53] revert pokeapi changes --- .../connectors/source-pokeapi/metadata.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index fec36e8f0184a..076a75a780a48 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 6371b14b-bc68-4236-bfbd-468e8df8e968 - dockerImageTag: 0.2.1 + dockerImageTag: 0.2.0 dockerRepository: airbyte/source-pokeapi githubIssueLabel: source-pokeapi icon: pokeapi.svg @@ -20,10 +20,6 @@ data: releaseStage: alpha supportLevel: community documentationUrl: https://docs.airbyte.com/integrations/sources/pokeapi - remoteRegistries: - pypi: - enabled: true - packageName: airbyte-source-pokeapi tags: - language:low-code metadataSpecVersion: "1.0" From d5c037f20a85ea357c89b95d963364df1ef6d84b Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 24 Jan 2024 11:57:31 +0100 Subject: [PATCH 44/53] review comments --- .../actions/run-dagger-pipeline/action.yml | 2 +- .github/workflows/publish_connectors.yml | 2 +- .../airbyte_ci/connectors/publish/pipeline.py | 46 ++++++++++--------- .../airbyte_ci/poetry/publish/commands.py | 14 +++--- .../pipeline.py => python_registry.py} | 12 ++--- .../utils.py => helpers/pip.py} | 6 +-- .../contexts/python_registry_publish.py} | 7 +-- .../tests/test_poetry/test_poetry_publish.py | 4 +- .../pipelines/tests/test_publish.py | 30 ++++++------ 9 files changed, 63 insertions(+), 60 deletions(-) rename airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/{python_registry/pipeline.py => python_registry.py} (94%) rename airbyte-ci/connectors/pipelines/pipelines/{airbyte_ci/steps/python_registry/utils.py => helpers/pip.py} (75%) rename airbyte-ci/connectors/pipelines/pipelines/{airbyte_ci/steps/python_registry/context.py => models/contexts/python_registry_publish.py} (92%) diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml index 0fcebdfe833b9..41c1ac2a609af 100644 --- a/.github/actions/run-dagger-pipeline/action.yml +++ b/.github/actions/run-dagger-pipeline/action.yml @@ -185,4 +185,4 @@ runs: CI: "True" TAILSCALE_AUTH_KEY: ${{ inputs.tailscale_auth_key }} DOCKER_REGISTRY_MIRROR_URL: ${{ inputs.docker_registry_mirror_url }} - PYPI_TOKEN: ${{ inputs.pypi_token }} + PYTHON_REGISTRY_TOKEN: ${{ inputs.python_registry_token }} diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index db44bfd767acf..88b8c01475b25 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -84,7 +84,7 @@ jobs: s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}" - pypi_token: ${{ secrets.PYPI_TOKEN }} + python_registry_token: ${{ secrets.PYTHON_REGISTRY_TOKEN }} set-instatus-incident-on-failure: name: Create Instatus Incident on Failure diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index 7be64e00e4e30..f6daa7bb2b985 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -14,10 +14,10 @@ from pipelines.airbyte_ci.connectors.publish.context import PublishConnectorContext from pipelines.airbyte_ci.connectors.reports import ConnectorReport from pipelines.airbyte_ci.metadata.pipeline import MetadataUpload, MetadataValidation -from pipelines.airbyte_ci.steps.python_registry.pipeline import PublishToPythonRegistry, PythonRegistryPublishContext -from pipelines.airbyte_ci.steps.python_registry.utils import is_package_published +from pipelines.airbyte_ci.steps.python_registry import PublishToPythonRegistry, PythonRegistryPublishContext from pipelines.dagger.actions.remote_storage import upload_to_gcs from pipelines.dagger.actions.system import docker +from pipelines.helpers.pip import is_package_published from pipelines.models.steps import Step, StepResult, StepStatus from pydantic import ValidationError @@ -54,12 +54,14 @@ async def _run(self) -> StepResult: return StepResult(self, status=StepStatus.SUCCESS, stdout=f"No manifest found for {self.context.docker_image}.") -class CheckPypiPackageDoesNotExist(Step): +class CheckPythonRegistryPackageDoesNotExist(Step): context: PythonRegistryPublishContext - title = "Check if the connector is published on pypi" + title = "Check if the connector is published on python registry" async def _run(self) -> StepResult: - is_published = is_package_published(self.context.package_metadata, self.context.registry) + is_published = is_package_published( + self.context.package_metadata.name, self.context.package_metadata.version, self.context.registry + ) if is_published: return StepResult( self, @@ -295,8 +297,8 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: metadata_upload_results = await metadata_upload_step.run() results.append(metadata_upload_results) - pypi_steps, terminate_early = await _run_pypi_publish_pipeline(context) - results.extend(pypi_steps) + python_registry_steps, terminate_early = await _run_python_registry_publish_pipeline(context) + results.extend(python_registry_steps) if terminate_early: return create_connector_report(results) @@ -339,28 +341,28 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: return connector_report -async def _run_pypi_publish_pipeline(context: PublishConnectorContext) -> Tuple[List[StepResult], bool]: +async def _run_python_registry_publish_pipeline(context: PublishConnectorContext) -> Tuple[List[StepResult], bool]: """ - Run the pypi publish pipeline for a single connector. + Run the python registry publish pipeline for a single connector. Return the results of the steps and a boolean indicating whether there was an error and the pipeline should be stopped. """ results: List[StepResult] = [] - # Try to convert the context to a PythonRegistryPublishContext. If it returns None, it means we don't need to publish to pypi. - pypi_context = await PythonRegistryPublishContext.from_publish_connector_context(context) - if not pypi_context: + # Try to convert the context to a PythonRegistryPublishContext. If it returns None, it means we don't need to publish to a python registry. + python_registry_context = await PythonRegistryPublishContext.from_publish_connector_context(context) + if not python_registry_context: return results, False - check_pypi_package_exists_results = await CheckPypiPackageDoesNotExist(pypi_context).run() - results.append(check_pypi_package_exists_results) - if check_pypi_package_exists_results.status is StepStatus.SKIPPED: - context.logger.info("The connector version is already published on pypi.") - elif check_pypi_package_exists_results.status is StepStatus.SUCCESS: - context.logger.info("The connector version is not published on pypi. Let's build and publish it.") - publish_to_pypi_results = await PublishToPythonRegistry(pypi_context).run() - results.append(publish_to_pypi_results) - if publish_to_pypi_results.status is StepStatus.FAILURE: + check_python_registry_package_exists_results = await CheckPythonRegistryPackageDoesNotExist(python_registry_context).run() + results.append(check_python_registry_package_exists_results) + if check_python_registry_package_exists_results.status is StepStatus.SKIPPED: + context.logger.info("The connector version is already published on python registry.") + elif check_python_registry_package_exists_results.status is StepStatus.SUCCESS: + context.logger.info("The connector version is not published on python registry. Let's build and publish it.") + publish_to_python_registry_results = await PublishToPythonRegistry(python_registry_context).run() + results.append(publish_to_python_registry_results) + if publish_to_python_registry_results.status is StepStatus.FAILURE: return results, True - elif check_pypi_package_exists_results.status is StepStatus.FAILURE: + elif check_python_registry_package_exists_results.status is StepStatus.FAILURE: return results, True return results, False diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py index 01f214d328a8b..29785f8312661 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/poetry/publish/commands.py @@ -11,12 +11,12 @@ import asyncclick as click from packaging import version -from pipelines.airbyte_ci.steps.python_registry.context import PythonRegistryPublishContext -from pipelines.airbyte_ci.steps.python_registry.pipeline import PublishToPythonRegistry +from pipelines.airbyte_ci.steps.python_registry import PublishToPythonRegistry from pipelines.cli.confirm_prompt import confirm from pipelines.cli.dagger_pipeline_command import DaggerPipelineCommand from pipelines.consts import DEFAULT_PYTHON_PACKAGE_REGISTRY_URL from pipelines.models.contexts.click_pipeline_context import ClickPipelineContext, pass_pipeline_context +from pipelines.models.contexts.python_registry_publish import PythonRegistryPublishContext from pipelines.models.steps import StepStatus @@ -40,11 +40,11 @@ def _validate_python_version(_ctx: dict, _param: dict, value: Optional[str]) -> @click.command(cls=DaggerPipelineCommand, name="publish", help="Publish a Python package to a registry.") @click.option( - "--pypi-token", + "--python-registry-token", help="Access token", type=click.STRING, required=True, - envvar="PYPI_TOKEN", + envvar="PYTHON_REGISTRY_TOKEN", ) @click.option( "--registry-url", @@ -68,7 +68,7 @@ def _validate_python_version(_ctx: dict, _param: dict, value: Optional[str]) -> async def publish( ctx: click.Context, click_pipeline_context: ClickPipelineContext, - pypi_token: str, + python_registry_token: str, registry_url: str, publish_name: Optional[str], publish_version: Optional[str], @@ -84,14 +84,14 @@ async def publish( pipeline_start_timestamp=ctx.obj.get("pipeline_start_timestamp"), ci_context=ctx.obj.get("ci_context"), ci_gcs_credentials=ctx.obj["ci_gcs_credentials"], - pypi_token=pypi_token, + python_registry_token=python_registry_token, registry=registry_url, package_path=ctx.obj["package_path"], package_name=publish_name, version=publish_version, ) - dagger_client = await click_pipeline_context.get_dagger_client(pipeline_name=f"Publish {ctx.obj['package_path']} to PyPI") + dagger_client = await click_pipeline_context.get_dagger_client(pipeline_name=f"Publish {ctx.obj['package_path']} to python registry") context.dagger_client = dagger_client if await _has_metadata_yaml(context): diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py similarity index 94% rename from airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/pipeline.py rename to airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py index fd9901f548a9a..aec2e30bb3da2 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry.py @@ -11,10 +11,10 @@ import tomli import tomli_w from dagger import Container, Directory -from pipelines.airbyte_ci.steps.python_registry.context import PythonPackageMetadata, PythonRegistryPublishContext from pipelines.consts import PYPROJECT_TOML_FILE_PATH, SETUP_PY_FILE_PATH from pipelines.dagger.actions.python.poetry import with_poetry from pipelines.helpers.utils import sh_dash_c +from pipelines.models.contexts.python_registry_publish import PythonPackageMetadata, PythonRegistryPublishContext from pipelines.models.steps import Step, StepResult @@ -25,7 +25,7 @@ class PackageType(Enum): class PublishToPythonRegistry(Step): context: PythonRegistryPublishContext - title = "Publish package to PyPI" + title = "Publish package to python registry" def _get_base_container(self) -> Container: return with_poetry(self.context) @@ -101,7 +101,7 @@ async def _publish(self, package_dir_to_publish: Directory, package_type: Packag return await self._poetry_publish(package_dir_to_publish) async def _poetry_publish(self, package_dir_to_publish: Directory) -> StepResult: - pypi_token = self.context.dagger_client.set_secret("pypi_token", f"pypi-{self.context.pypi_token}") + python_registry_token = self.context.dagger_client.set_secret("python_registry_token", self.context.python_registry_token) pyproject_toml = package_dir_to_publish.file(PYPROJECT_TOML_FILE_PATH) pyproject_toml_content = await pyproject_toml.contents() contents = tomli.loads(pyproject_toml_content) @@ -112,12 +112,12 @@ async def _poetry_publish(self, package_dir_to_publish: Directory) -> StepResult contents["tool"]["poetry"]["authors"] = ["Airbyte "] poetry_publish = ( self._get_base_container() - .with_secret_variable("PYPI_TOKEN", pypi_token) + .with_secret_variable("PYTHON_REGISTRY_TOKEN", python_registry_token) .with_directory("package", package_dir_to_publish) .with_workdir("package") .with_new_file(PYPROJECT_TOML_FILE_PATH, contents=tomli_w.dumps(contents)) .with_exec(["poetry", "config", "repositories.mypypi", self.context.registry]) - .with_exec(sh_dash_c(["poetry config pypi-token.mypypi $PYPI_TOKEN"])) + .with_exec(sh_dash_c(["poetry config pypi-token.mypypi $PYTHON_REGISTRY_TOKEN"])) .with_env_variable("CACHEBUSTER", str(uuid.uuid4())) .with_exec(sh_dash_c(["poetry publish --build --repository mypypi -vvv --no-interaction"])) ) @@ -127,7 +127,7 @@ async def _poetry_publish(self, package_dir_to_publish: Directory) -> StepResult async def _pip_publish(self, package_dir_to_publish: Directory) -> StepResult: files = await package_dir_to_publish.entries() pypi_username = self.context.dagger_client.set_secret("pypi_username", "__token__") - pypi_password = self.context.dagger_client.set_secret("pypi_password", f"pypi-{self.context.pypi_token}") + pypi_password = self.context.dagger_client.set_secret("pypi_password", self.context.python_registry_token) metadata: Dict[str, str] = { "name": str(self.context.package_metadata.name), "version": str(self.context.package_metadata.version), diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/utils.py b/airbyte-ci/connectors/pipelines/pipelines/helpers/pip.py similarity index 75% rename from airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/utils.py rename to airbyte-ci/connectors/pipelines/pipelines/helpers/pip.py index 6f610f628a132..3fa6fa396c097 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/utils.py +++ b/airbyte-ci/connectors/pipelines/pipelines/helpers/pip.py @@ -1,12 +1,12 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. +from typing import Optional from urllib.parse import urlparse import requests -from pipelines.airbyte_ci.steps.python_registry.context import PythonPackageMetadata -def is_package_published(package_metadata: PythonPackageMetadata, registry_url: str) -> bool: +def is_package_published(package_name: Optional[str], version: Optional[str], registry_url: str) -> bool: """ Check if a package with a specific version is published on PyPI or Test PyPI. @@ -15,8 +15,6 @@ def is_package_published(package_metadata: PythonPackageMetadata, registry_url: :param test_pypi: Set to True to check on Test PyPI, False for regular PyPI. :return: True if the package is found with the specified version, False otherwise. """ - package_name = package_metadata.name - version = package_metadata.version if not package_name or not version: return False diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py similarity index 92% rename from airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py rename to airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py index ecbeb7180c806..ee45760d2f9d9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/python_registry/context.py +++ b/airbyte-ci/connectors/pipelines/pipelines/models/contexts/python_registry_publish.py @@ -21,7 +21,7 @@ class PythonPackageMetadata: class PythonRegistryPublishContext(PipelineContext): def __init__( self, - pypi_token: str, + python_registry_token: str, package_path: str, report_output_prefix: str, is_local: bool, @@ -37,7 +37,7 @@ def __init__( package_name: Optional[str] = None, version: Optional[str] = None, ) -> None: - self.pypi_token = pypi_token + self.python_registry_token = python_registry_token self.registry = registry self.package_path = package_path self.package_metadata = PythonPackageMetadata(package_name, version) @@ -81,11 +81,12 @@ async def from_publish_connector_context( version = current_metadata["dockerImageTag"] if connector_context.pre_release: # use current date as pre-release version + # we can't use the git revision because not all python registries allow local version identifiers. Public version identifiers must conform to PEP 440 and only allow digits. release_candidate_tag = datetime.now().strftime("%Y%m%d%H%M") version = f"{version}.dev{release_candidate_tag}" pypi_context = cls( - pypi_token=os.environ["PYPI_TOKEN"], + python_registry_token=os.environ["PYTHON_REGISTRY_TOKEN"], registry="https://test.pypi.org/legacy/", # TODO: go live package_path=str(connector_context.connector.code_directory), package_name=current_metadata["remoteRegistries"]["pypi"]["packageName"], diff --git a/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py index 5ca048ea7dbe8..ee345b4183809 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_poetry/test_poetry_publish.py @@ -8,8 +8,8 @@ import requests from dagger import Client, Platform from pipelines.airbyte_ci.connectors.publish import pipeline as publish_pipeline -from pipelines.airbyte_ci.steps.python_registry.context import PythonPackageMetadata, PythonRegistryPublishContext from pipelines.dagger.actions.python.poetry import with_poetry +from pipelines.models.contexts.python_registry_publish import PythonPackageMetadata, PythonRegistryPublishContext from pipelines.models.steps import StepStatus pytestmark = [ @@ -22,7 +22,7 @@ def context(dagger_client: Client): context = PythonRegistryPublishContext( package_path="test", version="0.2.0", - pypi_token="test", + python_registry_token="test", package_name="test", registry="http://local_registry:8080/", is_local=True, diff --git a/airbyte-ci/connectors/pipelines/tests/test_publish.py b/airbyte-ci/connectors/pipelines/tests/test_publish.py index 929f49d8e3c18..ce8913e648abd 100644 --- a/airbyte-ci/connectors/pipelines/tests/test_publish.py +++ b/airbyte-ci/connectors/pipelines/tests/test_publish.py @@ -155,7 +155,7 @@ def test_parse_spec_output_no_spec(self, publish_context): (publish_pipeline, "PushConnectorImageToRegistry"), (publish_pipeline, "PullConnectorImageFromRegistry"), (publish_pipeline.steps, "run_connector_build"), - (publish_pipeline, "CheckPypiPackageDoesNotExist"), + (publish_pipeline, "CheckPythonRegistryPackageDoesNotExist"), ] @@ -348,7 +348,7 @@ async def test_run_connector_publish_pipeline_when_image_does_not_exist( pytest.param(True, StepStatus.FAILURE, StepStatus.FAILURE, False, False, id="pypi_package_does_not_exist_fails, abort"), ], ) -async def test_run_connector_pypi_publish_pipeline( +async def test_run_connector_python_registry_publish_pipeline( mocker, pypi_enabled, pypi_package_does_not_exist_status, @@ -360,7 +360,7 @@ async def test_run_connector_pypi_publish_pipeline( for module, to_mock in STEPS_TO_PATCH: mocker.patch.object(module, to_mock, return_value=mocker.AsyncMock()) - mocked_publish_to_pypi = mocker.patch( + mocked_publish_to_python_registry = mocker.patch( "pipelines.airbyte_ci.connectors.publish.pipeline.PublishToPythonRegistry", return_value=mocker.AsyncMock() ) @@ -374,10 +374,12 @@ async def test_run_connector_pypi_publish_pipeline( ]: step.return_value.run.return_value = mocker.Mock(name=f"{step.title}_result", status=StepStatus.SUCCESS) - mocked_publish_to_pypi.return_value.run.return_value = mocker.Mock(name="publish_to_pypi_result", status=publish_step_status) + mocked_publish_to_python_registry.return_value.run.return_value = mocker.Mock( + name="publish_to_python_registry_result", status=publish_step_status + ) - publish_pipeline.CheckPypiPackageDoesNotExist.return_value.run.return_value = mocker.Mock( - name="pypi_package_does_not_exist_result", status=pypi_package_does_not_exist_status + publish_pipeline.CheckPythonRegistryPackageDoesNotExist.return_value.run.return_value = mocker.Mock( + name="python_registry_package_does_not_exist_result", status=pypi_package_does_not_exist_status ) context = mocker.MagicMock( @@ -389,18 +391,18 @@ async def test_run_connector_pypi_publish_pipeline( ), ) semaphore = anyio.Semaphore(1) - with patch.dict(os.environ, {"PYPI_TOKEN": "test"}): + with patch.dict(os.environ, {"PYTHON_REGISTRY_TOKEN": "test"}): await publish_pipeline.run_connector_publish_pipeline(context, semaphore) if expect_publish_to_pypi_called: - mocked_publish_to_pypi.return_value.run.assert_called_once() + mocked_publish_to_python_registry.return_value.run.assert_called_once() # assert that the first argument passed to mocked_publish_to_pypi contains the things from the context - assert mocked_publish_to_pypi.call_args.args[0].pypi_token == "test" - assert mocked_publish_to_pypi.call_args.args[0].package_metadata.name == "test" - assert mocked_publish_to_pypi.call_args.args[0].package_metadata.version == "1.2.3" - assert mocked_publish_to_pypi.call_args.args[0].registry == "https://test.pypi.org/legacy/" - assert mocked_publish_to_pypi.call_args.args[0].package_path == "path/to/connector" + assert mocked_publish_to_python_registry.call_args.args[0].python_registry_token == "test" + assert mocked_publish_to_python_registry.call_args.args[0].package_metadata.name == "test" + assert mocked_publish_to_python_registry.call_args.args[0].package_metadata.version == "1.2.3" + assert mocked_publish_to_python_registry.call_args.args[0].registry == "https://test.pypi.org/legacy/" + assert mocked_publish_to_python_registry.call_args.args[0].package_path == "path/to/connector" else: - mocked_publish_to_pypi.return_value.run.assert_not_called() + mocked_publish_to_python_registry.return_value.run.assert_not_called() if expect_build_connector_called: publish_pipeline.steps.run_connector_build.assert_called_once() From a44fdc4070c9559734ecb9684dc007436cf7341e Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 24 Jan 2024 12:12:43 +0100 Subject: [PATCH 45/53] review comments --- airbyte-ci/connectors/pipelines/README.md | 34 ++++++++++++++++++- .../connectors/pipelines/pyproject.toml | 2 +- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index e762b5705d248..fb5d36815d70f 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -491,6 +491,37 @@ This command runs formatting checks and reformats any code that would be reforma Running `airbyte-ci format fix all` will format all of the different types of code. Run `airbyte-ci format fix --help` for subcommands to format only certain types of files. +### `poetry` command subgroup + +Available commands: + +- `airbyte-ci poetry publish` + +### Options + +| Option | Required | Default | Mapped environment variable | Description | +| ------------------- | -------- | ------- | --------------------------- | ------------------------------------------------------------------------------------------- | +| `--package-path` | True | | | The path to the python package to execute a poetry command on. | + +### Examples + +- Publish a python package: `airbyte-ci poetry --package-path=path/to/package publish --publish-name=my-package --publish-version="1.2.3" --python-registry-token="..." --registry-url="http://host.docker.internal:8012/"` + +### `publish` command + +This command publishes poetry packages (using `pyproject.toml`) or python packages (using `setup.py`) to a python registry. + +For poetry packages, the package name and version can be taken from the `pyproject.toml` file or be specified as options. + +#### Options + +| Option | Required | Default | Mapped environment variable | Description | +| ------------------------- | -------- | ----------------------- | --------------------------- | --------------------------------------------------------------------------------------------------------------- | +| `--publish-name` | False | | | The name of the package. Not required for poetry packages that define it in the `pyproject.toml` file | +| `--publish-version` | False | | | The version of the package. Not required for poetry packages that define it in the `pyproject.toml` file | +| `--python-registry-token` | True | | PYTHON_REGISTRY_TOKEN | The API token to authenticate with the registry. For pypi, the `pypi-` prefix needs to be specified | +| `--registry-url` | False | https://pypi.org/simple | | The python registry to publish to. Defaults to main pypi | + ### `metadata` command subgroup Available commands: @@ -547,7 +578,8 @@ E.G.: running `pytest` on a specific test folder: | Version | PR | Description | | ------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- | -| 3.5.3 | [#34339](https://github.com/airbytehq/airbyte/pull/34339) | only do minimal changes on a connector version_bump | +| 3.6.0 | [#34111](https://github.com/airbytehq/airbyte/pull/34111) | Add python registry publishing | +| 3.5.3 | [#34339](https://github.com/airbytehq/airbyte/pull/34339) | only do minimal changes on a connector version_bump | | 3.5.2 | [#34381](https://github.com/airbytehq/airbyte/pull/34381) | Bind a sidecar docker host for `airbyte-ci test` | | 3.5.1 | [#34321](https://github.com/airbytehq/airbyte/pull/34321) | Upgrade to Dagger 0.9.6 . | | 3.5.0 | [#33313](https://github.com/airbytehq/airbyte/pull/33313) | Pass extra params after Gradle tasks. | diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index 669dc43a41fb4..7acd9b13b6f5f 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "3.5.3" +version = "3.6.0" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] From faaf6db804c4ec1047ec37bbf599c41c9beb0ef8 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 24 Jan 2024 12:13:45 +0100 Subject: [PATCH 46/53] test with pokeapi --- airbyte-integrations/connectors/source-pokeapi/metadata.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index 076a75a780a48..07e1efc5f8ce9 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -22,4 +22,8 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/pokeapi tags: - language:low-code + remoteRegistries: + pypi: + enabled: true + packageName: "airbyte-source-pokeapi" metadataSpecVersion: "1.0" From 6923181c6b2da4fe5d442900e35b643e178a38d8 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 24 Jan 2024 13:19:44 +0100 Subject: [PATCH 47/53] switch order --- .../airbyte_ci/connectors/publish/pipeline.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index f6daa7bb2b985..fa6892a9bef10 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -283,6 +283,12 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: check_connector_image_results = await CheckConnectorImageDoesNotExist(context).run() results.append(check_connector_image_results) + python_registry_steps, terminate_early = await _run_python_registry_publish_pipeline(context) + results.extend(python_registry_steps) + if terminate_early: + return create_connector_report(results) + + # If the connector image already exists, we don't need to build it, but we still need to upload the metadata file. # We also need to upload the spec to the spec cache bucket. if check_connector_image_results.status is StepStatus.SKIPPED: @@ -297,11 +303,6 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: metadata_upload_results = await metadata_upload_step.run() results.append(metadata_upload_results) - python_registry_steps, terminate_early = await _run_python_registry_publish_pipeline(context) - results.extend(python_registry_steps) - if terminate_early: - return create_connector_report(results) - # Exit early if the connector image already exists or has failed to build if check_connector_image_results.status is not StepStatus.SUCCESS: return create_connector_report(results) From c30cb89760ca1d34e8907360c932fbccb310e1f0 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 24 Jan 2024 14:14:09 +0100 Subject: [PATCH 48/53] format --- .../pipelines/airbyte_ci/connectors/publish/pipeline.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index fa6892a9bef10..c6e4002829c33 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -288,7 +288,6 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: if terminate_early: return create_connector_report(results) - # If the connector image already exists, we don't need to build it, but we still need to upload the metadata file. # We also need to upload the spec to the spec cache bucket. if check_connector_image_results.status is StepStatus.SKIPPED: From 84da11f6633f22390e9d4f2a1141f31d41330cac Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 24 Jan 2024 14:17:05 +0100 Subject: [PATCH 49/53] fix github actions --- .github/actions/run-dagger-pipeline/action.yml | 4 ++-- .github/workflows/publish_connectors.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/run-dagger-pipeline/action.yml b/.github/actions/run-dagger-pipeline/action.yml index 41c1ac2a609af..5ba6faa69a479 100644 --- a/.github/actions/run-dagger-pipeline/action.yml +++ b/.github/actions/run-dagger-pipeline/action.yml @@ -83,8 +83,8 @@ inputs: description: "URL to airbyte-ci binary" required: false default: https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci - pypi_token: - description: "PyPI API token to publish to PyPI" + python_registry_token: + description: "Python registry API token to publish python package" required: false runs: diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index 88b8c01475b25..9543efdfd1e55 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -63,6 +63,7 @@ jobs: s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release" + python_registry_token: ${{ secrets.PYTHON_REGISTRY_TOKEN }} - name: Publish connectors [manual] id: publish-connectors From 004af84c8531417ae69bcd00a4506df59bbf1583 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 24 Jan 2024 15:14:52 +0100 Subject: [PATCH 50/53] dummy-change --- .../connectors/source-pokeapi/source_pokeapi/source.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py index 4e1813d78e7e4..67efbfecedf50 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py @@ -15,4 +15,5 @@ # Declarative Source class SourcePokeapi(YamlDeclarativeSource): def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) From e8b86eff837ce8967fd3ba4c255723da74538b88 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 24 Jan 2024 15:51:02 +0100 Subject: [PATCH 51/53] dummy-change 2 --- .../pipelines/airbyte_ci/connectors/publish/pipeline.py | 1 + .../connectors/source-pokeapi/source_pokeapi/source.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index c6e4002829c33..08a1c5f8dc807 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -281,6 +281,7 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: return create_connector_report(results) check_connector_image_results = await CheckConnectorImageDoesNotExist(context).run() + results.append(check_connector_image_results) python_registry_steps, terminate_early = await _run_python_registry_publish_pipeline(context) diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py index 67efbfecedf50..802fb9be3aced 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py @@ -15,5 +15,5 @@ # Declarative Source class SourcePokeapi(YamlDeclarativeSource): def __init__(self): - + super().__init__(**{"path_to_yaml": "manifest.yaml"}) From 4691763d06dfebf0377c93d65fd9b277f836988f Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 24 Jan 2024 16:05:52 +0100 Subject: [PATCH 52/53] fix variable in github action flow --- .github/workflows/publish_connectors.yml | 4 ++-- .../pipelines/airbyte_ci/connectors/publish/pipeline.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish_connectors.yml b/.github/workflows/publish_connectors.yml index 9543efdfd1e55..ae431454eda8b 100644 --- a/.github/workflows/publish_connectors.yml +++ b/.github/workflows/publish_connectors.yml @@ -63,7 +63,7 @@ jobs: s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release" - python_registry_token: ${{ secrets.PYTHON_REGISTRY_TOKEN }} + python_registry_token: ${{ secrets.PYPI_TOKEN }} - name: Publish connectors [manual] id: publish-connectors @@ -85,7 +85,7 @@ jobs: s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }} subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}" - python_registry_token: ${{ secrets.PYTHON_REGISTRY_TOKEN }} + python_registry_token: ${{ secrets.PYPI_TOKEN }} set-instatus-incident-on-failure: name: Create Instatus Incident on Failure diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py index 08a1c5f8dc807..c6e4002829c33 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/publish/pipeline.py @@ -281,7 +281,6 @@ def create_connector_report(results: List[StepResult]) -> ConnectorReport: return create_connector_report(results) check_connector_image_results = await CheckConnectorImageDoesNotExist(context).run() - results.append(check_connector_image_results) python_registry_steps, terminate_early = await _run_python_registry_publish_pipeline(context) From f99aaef3e49e911e8667d514c49cf507a51d8c87 Mon Sep 17 00:00:00 2001 From: Joe Reuter Date: Wed, 24 Jan 2024 16:11:50 +0100 Subject: [PATCH 53/53] revert test changes --- airbyte-integrations/connectors/source-pokeapi/metadata.yaml | 4 ---- .../connectors/source-pokeapi/source_pokeapi/source.py | 1 - 2 files changed, 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml index 07e1efc5f8ce9..076a75a780a48 100644 --- a/airbyte-integrations/connectors/source-pokeapi/metadata.yaml +++ b/airbyte-integrations/connectors/source-pokeapi/metadata.yaml @@ -22,8 +22,4 @@ data: documentationUrl: https://docs.airbyte.com/integrations/sources/pokeapi tags: - language:low-code - remoteRegistries: - pypi: - enabled: true - packageName: "airbyte-source-pokeapi" metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py index 802fb9be3aced..4e1813d78e7e4 100644 --- a/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py +++ b/airbyte-integrations/connectors/source-pokeapi/source_pokeapi/source.py @@ -15,5 +15,4 @@ # Declarative Source class SourcePokeapi(YamlDeclarativeSource): def __init__(self): - super().__init__(**{"path_to_yaml": "manifest.yaml"})