airbyte-lib: Add publish action (#34486)

Co-authored-by: alafanechere <[email protected]>

Author: Joe Reuter

.github/workflows/publish-airbyte-lib-command-manually.yml

@@ -0,0 +1,57 @@
+name: Publish AirbyteLib Manually
+on: workflow_dispatch
+
+concurrency:
+  group: publish-airbyte-lib
+  cancel-in-progress: false
+
+jobs:
+  get_ci_runner:
+    runs-on: ubuntu-latest
+    name: Get CI runner
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+          token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          fetch-depth: 1
+      - name: Get CI runner
+        id: get_ci_runner
+        uses: ./.github/actions/airbyte-ci-requirements
+        with:
+          runner_type: "publish"
+          runner_size: "large"
+          # Getting ci requirements for connectors publish command as there is no special one for poetry publish
+          airbyte_ci_command: "connectors publish"
+          github_token: ${{ secrets.GH_PAT_APPROVINGTON_OCTAVIA }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+    outputs:
+      runner_name: ${{ steps.get_ci_runner.outputs.runner_name }}
+  publish_connectors:
+    name: Publish airbyte-lib
+    needs: get_ci_runner
+    runs-on: ${{ needs.get_ci_runner.outputs.runner_name }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v3
+      - name: Publish
+        id: publish-airbyte-lib
+        uses: ./.github/actions/run-dagger-pipeline
+        with:
+          context: "manual"
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN }}
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }}
+          spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }}
+          s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          tailscale_auth_key: ${{ secrets.TAILSCALE_AUTH_KEY }}
+          subcommand: 'poetry --package-path=airbyte-lib publish --registry-url="https://test.pypi.org/legacy/"'
+          python_registry_token: ${{ secrets.PYPI_TOKEN }}

airbyte-lib/README.md

@@ -9,6 +9,17 @@ airbyte-lib is a library that allows to run Airbyte syncs embedded into any Pyth
 * For examples, check out the `examples` folder. They can be run via `poetry run python examples/<example file>`
 * Unit tests and type checks can be run via `poetry run pytest`
 
+## Release
+
+* In your PR:
+   * Bump the version in `pyproject.toml`
+   * Add a changelog entry to the table below
+* Once the PR is merged, go to Github and trigger the `Publish AirbyteLib Manually` workflow. This will publish the new version to PyPI.
+
+### Versioning
+
+Versioning follows [Semantic Versioning](https://semver.org/). For new features, bump the minor version. For bug fixes, bump the patch version. For pre-releases, append `dev.N` to the version. For example, `0.1.0dev.1` is the first pre-release of the `0.1.0` version.
+
 ## Documentation
 
 Regular documentation lives in the `/docs` folder. Based on the doc strings of public methods, we generate API documentation using [pdoc](https://pdoc.dev). To generate the documentation, run `poetry run generate-docs`. The documentation will be generated in the `docs/generate` folder. This needs to be done manually when changing the public interface of the library.
@@ -25,4 +36,10 @@ airbyte-lib-validate-source —connector-dir . -—sample-config secrets/config.
 
 The script will install the python package in the provided directory, and run the connector against the provided config. The config should be a valid JSON file, with the same structure as the one that would be provided to the connector in Airbyte. The script will exit with a non-zero exit code if the connector fails to run.
 
-For a more lightweight check, the `--validate-install-only` flag can be used. This will only check that the connector can be installed and returns a spec, no sample config required.
+For a more lightweight check, the `--validate-install-only` flag can be used. This will only check that the connector can be installed and returns a spec, no sample config required.
+
+## Changelog
+
+| Version     | PR                                                         | Description                                                                                                       |
+| ----------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
+| 0.1.0dev.2   | [#34111](https://github.com/airbytehq/airbyte/pull/34111)  | Initial publish - add publish workflow                                                                            |

airbyte-lib/airbyte_lib/validate.py

@@ -143,5 +143,7 @@ def validate(connector_dir: str, sample_config: str, *, validate_install_only: b
             install_only_test(connector_name)
         else:
             if not sample_config:
-                raise Exception("sample_config is required when -validate-install-only is not set")
+                raise exc.AirbyteLibInputError(
+                    input_value="--sample-config is required without --validate-install-only set"
+                )
             full_tests(connector_name, sample_config)

airbyte-lib/pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "airbyte-lib"
 description = "AirbyteLib"
-version = "0.1.0"
+version = "0.1.0dev.2"
 authors = ["Airbyte <[email protected]>"]
 readme = "README.md"
 packages = [{include = "airbyte_lib"}]