feat(registry): add cdk version (#37809)

## What
Adds the cdk version for python connectors to the regisry



![Screenshot 2024-05-03 at 3.00.59 PM.png](https://graphite-user-uploaded-assets-prod.s3.amazonaws.com/PTsI7qAmiIMkhFQg04QF/4e9b5b92-90f4-476d-b19b-0ee404e49ef4.png)

closes airbytehq/airbyte-internal-issues#7462

Author: bnchrch

airbyte-ci/connectors/metadata_service/lib/README.md

@@ -50,7 +50,7 @@ _💡 Note: A prerequisite is you have [gsutil](https://cloud.google.com/storage
 _⚠️ Warning: Its important to know that this will remove ANY files you have in your destination buckets as it calls `gsutil rsync` with `-d` enabled._
 
 ```bash
-TARGET_BUCKET=<YOUR-DEV_BUCKET> poetry poe replicate-prod
+TARGET_BUCKET=<YOUR-DEV_BUCKET> poetry run poe replicate-prod
 ```
 
 ### Copy specific connector version to your Development Bucket
@@ -60,7 +60,7 @@ This will copy the specified connector version to your development bucket. This
 _💡 Note: A prerequisite is you have [gsutil](https://cloud.google.com/storage/docs/gsutil) installed and have run `gsutil auth login`_
 
 ```bash
-TARGET_BUCKET=<YOUR-DEV_BUCKET> CONNECTOR="airbyte/source-stripe" VERSION="3.17.0-dev.ea013c8741" poetry poe copy-connector-from-prod
+TARGET_BUCKET=<YOUR-DEV_BUCKET> CONNECTOR="airbyte/source-stripe" VERSION="3.17.0-dev.ea013c8741" poetry run poe copy-connector-from-prod
 ```
 
 ### Promote Connector Version to Latest
@@ -72,5 +72,5 @@ _💡 Note: A prerequisite is you have [gsutil](https://cloud.google.com/storage
 _⚠️ Warning: Its important to know that this will remove ANY existing files in the latest folder that are not in the versioned folder as it calls `gsutil rsync` with `-d` enabled._
 
 ```bash
-TARGET_BUCKET=<YOUR-DEV_BUCKET> CONNECTOR="airbyte/source-stripe" VERSION="3.17.0-dev.ea013c8741" poetry poe promote-connector-to-latest
+TARGET_BUCKET=<YOUR-DEV_BUCKET> CONNECTOR="airbyte/source-stripe" VERSION="3.17.0-dev.ea013c8741" poetry run poe promote-connector-to-latest
 ```

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorPackageInfo.py

@@ -0,0 +1,12 @@
+# generated by datamodel-codegen:
+#   filename:  ConnectorPackageInfo.yaml
+
+from __future__ import annotations
+
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class ConnectorPackageInfo(BaseModel):
+    cdk_version: Optional[str] = None

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryDestinationDefinition.py

@@ -97,6 +97,10 @@ class SourceFileInfo(BaseModel):
     registry_entry_generated_at: Optional[str] = None
 
 
+class ConnectorPackageInfo(BaseModel):
+    cdk_version: Optional[str] = None
+
+
 class JobTypeResourceLimit(BaseModel):
     class Config:
         extra = Extra.forbid
@@ -196,3 +200,4 @@ class Config:
     ab_internal: Optional[AirbyteInternal] = None
     supportsRefreshes: Optional[bool] = False
     generated: Optional[GeneratedFields] = None
+    packageInfo: Optional[ConnectorPackageInfo] = None

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistrySourceDefinition.py

@@ -93,6 +93,10 @@ class SourceFileInfo(BaseModel):
     registry_entry_generated_at: Optional[str] = None
 
 
+class ConnectorPackageInfo(BaseModel):
+    cdk_version: Optional[str] = None
+
+
 class JobTypeResourceLimit(BaseModel):
     class Config:
         extra = Extra.forbid
@@ -188,3 +192,4 @@ class Config:
     releases: Optional[ConnectorReleases] = None
     ab_internal: Optional[AirbyteInternal] = None
     generated: Optional[GeneratedFields] = None
+    packageInfo: Optional[ConnectorPackageInfo] = None

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/ConnectorRegistryV0.py

@@ -97,6 +97,10 @@ class SourceFileInfo(BaseModel):
     registry_entry_generated_at: Optional[str] = None
 
 
+class ConnectorPackageInfo(BaseModel):
+    cdk_version: Optional[str] = None
+
+
 class SuggestedStreams(BaseModel):
     class Config:
         extra = Extra.allow
@@ -202,6 +206,7 @@ class Config:
     releases: Optional[ConnectorReleases] = None
     ab_internal: Optional[AirbyteInternal] = None
     generated: Optional[GeneratedFields] = None
+    packageInfo: Optional[ConnectorPackageInfo] = None
 
 
 class ConnectorRegistryDestinationDefinition(BaseModel):
@@ -239,6 +244,7 @@ class Config:
     ab_internal: Optional[AirbyteInternal] = None
     supportsRefreshes: Optional[bool] = False
     generated: Optional[GeneratedFields] = None
+    packageInfo: Optional[ConnectorPackageInfo] = None
 
 
 class ConnectorRegistryV0(BaseModel):

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/generated/__init__.py

@@ -4,6 +4,7 @@
 from .AllowedHosts import *
 from .ConnectorBuildOptions import *
 from .ConnectorMetadataDefinitionV0 import *
+from .ConnectorPackageInfo import *
 from .ConnectorRegistryDestinationDefinition import *
 from .ConnectorRegistrySourceDefinition import *
 from .ConnectorRegistryV0 import *

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorPackageInfo.yaml

@@ -0,0 +1,9 @@
+---
+"$schema": http://json-schema.org/draft-07/schema#
+"$id": https://github.com/airbytehq/airbyte/airbyte-ci/connectors_ci/metadata_service/lib/models/src/ConnectorPackageInfo.yaml
+title: ConnectorPackageInfo
+description: Information about the contents of the connector image
+type: object
+properties:
+  cdk_version:
+    type: string

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistryDestinationDefinition.yaml

@@ -77,3 +77,5 @@ properties:
     default: false
   generated:
     "$ref": GeneratedFields.yaml
+  packageInfo:
+    "$ref": ConnectorPackageInfo.yaml

airbyte-ci/connectors/metadata_service/lib/metadata_service/models/src/ConnectorRegistrySourceDefinition.yaml

@@ -76,3 +76,5 @@ properties:
     "$ref": AirbyteInternal.yaml
   generated:
     "$ref": GeneratedFields.yaml
+  packageInfo:
+    "$ref": ConnectorPackageInfo.yaml

airbyte-ci/connectors/metadata_service/lib/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metadata-service"
-version = "0.5.0"
+version = "0.6.0"
 description = ""
 authors = ["Ben Church <[email protected]>"]
 readme = "README.md"

airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/assets/registry_entry.py

@@ -20,6 +20,7 @@
 from metadata_service.models.transform import to_json_sanitized_dict
 from metadata_service.spec_cache import SpecCache
 from orchestrator.config import MAX_METADATA_PARTITION_RUN_REQUEST, VALID_REGISTRIES, get_public_url_for_gcs_file
+from orchestrator.fetcher.connector_cdk_version import get_cdk_version
 from orchestrator.logging import sentry
 from orchestrator.logging.publish_connector_lifecycle import PublishConnectorLifecycle, PublishConnectorLifecycleStage, StageStatus
 from orchestrator.models.metadata import LatestMetadataEntry, MetadataDefinition
@@ -171,6 +172,22 @@ def apply_generated_fields(metadata_data: dict, metadata_entry: LatestMetadataEn
     return generated_fields
 
 
+@deep_copy_params
+def apply_package_info_fields(metadata_data: dict, metadata_entry: LatestMetadataEntry) -> dict:
+    """Apply package info fields to the metadata data field.
+
+    Args:
+        metadata_data (dict): The metadata data field.
+
+    Returns:
+        dict: The metadata data field with the package info fields applied.
+    """
+    package_info_fields = metadata_data.get("packageInfo") or {}
+    package_info_fields = set_with(package_info_fields, "cdk_version", get_cdk_version(metadata_entry), default_none_to_dict)
+
+    return package_info_fields
+
+
 @deep_copy_params
 @sentry_sdk.trace
 def metadata_to_registry_entry(metadata_entry: LatestMetadataEntry, override_registry_key: str) -> dict:
@@ -214,6 +231,9 @@ def metadata_to_registry_entry(metadata_entry: LatestMetadataEntry, override_reg
     # Add generated fields for source file metadata and git
     overridden_metadata_data["generated"] = apply_generated_fields(overridden_metadata_data, metadata_entry)
 
+    # Add Dependency information
+    overridden_metadata_data["packageInfo"] = apply_package_info_fields(overridden_metadata_data, metadata_entry)
+
     # if there is no supportLevel, set it to "community"
     if not overridden_metadata_data.get("supportLevel"):
         overridden_metadata_data["supportLevel"] = "community"

airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/config.py

@@ -5,6 +5,8 @@
 import os
 from typing import Optional
 
+DEFAULT_ASSET_URL = "https://storage.googleapis.com"
+
 VALID_REGISTRIES = ["oss", "cloud"]
 REGISTRIES_FOLDER = "registries/v0"
 REPORT_FOLDER = "generated_reports"
@@ -20,6 +22,9 @@
 CONNECTORS_PATH = "airbyte-integrations/connectors"
 CONNECTOR_TEST_SUMMARY_FOLDER = "test_summary"
 
+CONNECTOR_DEPENDENCY_FOLDER = "connector_dependencies"
+CONNECTOR_DEPENDENCY_FILE_NAME = "dependencies.json"
+
 MAX_METADATA_PARTITION_RUN_REQUEST = 50
 
 HIGH_QUEUE_PRIORITY = "3"
@@ -39,7 +44,7 @@ def get_public_url_for_gcs_file(bucket_name: str, file_path: str, cdn_url: Optio
     Returns:
         The public URL to the file.
     """
-    return f"{cdn_url}/{file_path}" if cdn_url else f"https://storage.googleapis.com/{bucket_name}/{file_path}"
+    return f"{cdn_url}/{file_path}" if cdn_url else f"{DEFAULT_ASSET_URL}/{bucket_name}/{file_path}"
 
 
 def get_public_metadata_service_url(file_path: str) -> str:

airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/fetcher/connector_cdk_version.py

@@ -0,0 +1,54 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Optional
+
+import requests
+from orchestrator.models.metadata import LatestMetadataEntry
+
+GROUP_NAME = "connector_cdk_versions"
+
+BASE_URL = "https://storage.googleapis.com/dev-airbyte-cloud-connector-metadata-service/"
+DEPENDENCY_FOLDER = "connector_dependencies"
+DEPENDENCY_FILE = "dependencies.json"
+PACKAGE_NAME = "airbyte-cdk"
+PYTHON_CDK_SLUG = "python"
+
+# HELPERS
+
+
+def safe_get_json_from_url(url: str) -> Optional[dict]:
+    try:
+        response = requests.get(url)
+        if response.ok:
+            return response.json()
+        else:
+            return None
+    except requests.exceptions.RequestException:
+        return None
+
+
+def find_package_version(dependencies_body: dict, package_name: str) -> Optional[str]:
+    for package in dependencies_body.get("dependencies", []):
+        if package.get("package_name") == package_name:
+            return package.get("version")
+    return None
+
+
+def get_cdk_version(
+    metadata_entry: LatestMetadataEntry,
+) -> Optional[str]:
+    url = metadata_entry.dependency_file_url
+    if not url:
+        return None
+
+    response = safe_get_json_from_url(url)
+    if not response:
+        return None
+
+    version = find_package_version(response, PACKAGE_NAME)
+
+    # Note: Prefix the version with the python slug as the python cdk is the only one we have
+    # versions available for.
+    return f"{PYTHON_CDK_SLUG}:{version}" if version else None

airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/hacks.py

@@ -84,3 +84,28 @@ def construct_registry_entry_write_path(
     overrode_registry_entry_version_write_path = _get_version_specific_registry_entry_file_path(registry_entry, registry_name)
     _check_for_invalid_write_path(overrode_registry_entry_version_write_path)
     return overrode_registry_entry_version_write_path
+
+
+def sanitize_docker_repo_name_for_dependency_file(docker_repo_name: str) -> str:
+    """
+    Remove the "airbyte/" prefix from the docker repository name.
+
+    e.g. airbyte/source-postgres -> source-postgres
+
+    Problem:
+        The dependency file paths are based on the docker repository name without the "airbyte/" prefix where as all other
+        paths are based on the full docker repository name.
+
+        e.g. https://storage.googleapis.com/prod-airbyte-cloud-connector-metadata-service/connector_dependencies/source-pokeapi/0.2.0/dependencies.json
+
+    Long term solution:
+        Move the dependency file paths to be based on the full docker repository name.
+
+    Args:
+        docker_repo_name (str): The docker repository name
+
+    Returns:
+        str: The docker repository name without the "airbyte/" prefix
+    """
+
+    return docker_repo_name.replace("airbyte/", "")

airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/models/metadata.py

@@ -4,9 +4,12 @@
 
 from typing import Any, Optional, Tuple
 
+import orchestrator.hacks as HACKS
 from metadata_service.constants import METADATA_FILE_NAME
 from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0
+from orchestrator.config import CONNECTOR_DEPENDENCY_FILE_NAME, CONNECTOR_DEPENDENCY_FOLDER, get_public_url_for_gcs_file
 from pydantic import BaseModel, ValidationError
+from pydash import get
 
 
 class PydanticDelayValidationMixin:
@@ -61,3 +64,17 @@ def is_latest_version_path(self) -> bool:
         """
         ending_path = f"latest/{METADATA_FILE_NAME}"
         return self.file_path.endswith(ending_path)
+
+    @property
+    def dependency_file_url(self) -> Optional[str]:
+        if not self.bucket_name or not self.metadata_definition:
+            return None
+
+        connector_technical_name = get(self.metadata_definition, "data.dockerRepository")
+        connector_version = get(self.metadata_definition, "data.dockerImageTag")
+        sanitized_connector_technical_name = HACKS.sanitize_docker_repo_name_for_dependency_file(connector_technical_name)
+
+        file_path = (
+            f"{CONNECTOR_DEPENDENCY_FOLDER}/{sanitized_connector_technical_name}/{connector_version}/{CONNECTOR_DEPENDENCY_FILE_NAME}"
+        )
+        return get_public_url_for_gcs_file(self.bucket_name, file_path)

airbyte-ci/connectors/metadata_service/orchestrator/orchestrator/utils/object_helpers.py

@@ -50,5 +50,8 @@ def default_none_to_dict(value, key, obj):
         key: The key to set in the dictionary.
         obj: The dictionary to set the key in.
     """
+    if obj is None:
+        return
+
     if value is None:
         obj[key] = {}