airbyte-ci/metadata-lib: make dockerhub credentials optional (#37787)

Author: alafanechere

airbyte-ci/connectors/connectors_qa/README.md

@@ -107,6 +107,10 @@ poe lint
 ```
 ## Changelog
 
+### 1.3.1
+
+Remove requirements on DockerHub credentials to run metadata validation.
+
 ### 1.3.0
 
 Added `CheckConnectorMaxSecondsBetweenMessagesValue` check that verifies presence of `maxSecondsBetweenMessages` value in `metadata.yaml` file for all source certified connectors.

airbyte-ci/connectors/connectors_qa/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "connectors-qa"
-version = "1.3.0"
+version = "1.3.1"
 description = "A package to run QA checks on Airbyte connectors, generate reports and documentation."
 authors = ["Airbyte <[email protected]>"]
 readme = "README.md"
@@ -41,5 +41,4 @@ lint = "ruff check src"
 
 [tool.airbyte_ci]
 optional_poetry_groups = ["dev"]
-poe_tasks = ["type_check", "lint", "test"]
-required_environment_variables = ["DOCKER_HUB_USERNAME", "DOCKER_HUB_PASSWORD"]
+poe_tasks = ["type_check", "test"]

airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/__init__.py

@@ -1,9 +1,9 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 from .assets import ENABLED_CHECKS as ASSETS_CHECKS
+from .documentation import ENABLED_CHECKS as DOCUMENTATION_CHECKS
 from .metadata import ENABLED_CHECKS as METADATA_CORRECTNESS_CHECKS
-from .security import ENABLED_CHECKS as SECURITY_CHECKS
 from .packaging import ENABLED_CHECKS as PACKAGING_CHECKS
-from .documentation import ENABLED_CHECKS as DOCUMENTATION_CHECKS
+from .security import ENABLED_CHECKS as SECURITY_CHECKS
 
 ENABLED_CHECKS = (
     DOCUMENTATION_CHECKS

airbyte-ci/connectors/connectors_qa/src/connectors_qa/checks/metadata.py

@@ -18,18 +18,6 @@ class MetadataCheck(Check):
 class ValidateMetadata(MetadataCheck):
     name = f"Connectors must have valid {consts.METADATA_FILE_NAME} file"
     description = f"Connectors must have a `{consts.METADATA_FILE_NAME}` file at the root of their directory. This file is used to build our connector registry. Its structure must follow our metadata schema. Field values are also validated. This is to ensure that all connectors have the required metadata fields and that the metadata is valid. More details in this [documentation]({consts.METADATA_DOCUMENTATION_URL})."
-    # Metadata lib required the following env var to be set
-    # to check if the base image is on DockerHub
-    required_env_vars = {
-        consts.DOCKER_HUB_USERNAME_ENV_VAR_NAME,
-        consts.DOCKER_HUB_PASSWORD_ENV_VAR_NAME,
-    }
-
-    def __init__(self) -> None:
-        for env_var in self.required_env_vars:
-            if env_var not in os.environ:
-                raise ValueError(f"Environment variable {env_var} is required for this check")
-        super().__init__()
 
     def _run(self, connector: Connector) -> CheckResult:
         if not connector.documentation_file_path or not connector.documentation_file_path.exists():

airbyte-ci/connectors/connectors_qa/tests/unit_tests/test_checks/test_metadata.py

@@ -9,23 +9,6 @@
 
 
 class TestValidateMetadata:
-    def test_fail_init_when_required_env_vars_are_not_set(self, random_string, mocker):
-        # Arrange
-        mocker.patch.object(metadata.ValidateMetadata, "required_env_vars", new={random_string})
-
-        # Act
-        with pytest.raises(ValueError):
-            metadata.ValidateMetadata()
-
-    def test_init_when_required_env_vars_are_set(self, random_string, mocker):
-        # Arrange
-        os.environ[random_string] = "test"
-        mocker.patch.object(metadata.ValidateMetadata, "required_env_vars", new={random_string})
-
-        # Act
-        metadata.ValidateMetadata()
-
-        os.environ.pop(random_string)
 
     def test_fail_when_documentation_file_path_is_none(self, mocker):
         # Arrange

airbyte-ci/connectors/metadata_service/lib/metadata_service/docker_hub.py

@@ -40,8 +40,14 @@ def is_image_on_docker_hub(image_name: str, version: str, digest: Optional[str]
         bool: True if the image and version exists on Docker Hub, False otherwise.
     """
 
-    token = get_docker_hub_auth_token()
-    headers = {"Authorization": f"JWT {token}"}
+    if "DOCKER_HUB_USERNAME" not in os.environ or "DOCKER_HUB_PASSWORD" not in os.environ:
+        # If the Docker Hub credentials are not provided, we can only anonymously call the Docker Hub API.
+        # This will only work for public images and lead to a lower rate limit.
+        headers = {}
+    else:
+        token = get_docker_hub_auth_token()
+        headers = {"Authorization": f"JWT {token}"} if token else {}
+
     tag_url = f"https://registry.hub.docker.com/v2/repositories/{image_name}/tags/{version}"
 
     # Allow for retries as the DockerHub API is not always reliable with returning the latest publish.

airbyte-ci/connectors/metadata_service/lib/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metadata-service"
-version = "0.3.4"
+version = "0.3.5"
 description = ""
 authors = ["Ben Church <[email protected]>"]
 readme = "README.md"

airbyte-ci/connectors/pipelines/README.md

@@ -186,7 +186,7 @@ options to the `airbyte-ci` command group.**
 | `--is-local/--is-ci`                           | `--is-local`                    |                               | Determines the environment in which the CLI runs: local environment or CI environment.      |
 | `--git-branch`                                 | The checked out git branch name | `CI_GIT_BRANCH`               | The git branch on which the pipelines will run.                                             |
 | `--git-revision`                               | The current branch head         | `CI_GIT_REVISION`             | The commit hash on which the pipelines will run.                                            |
-| `--diffed-branch`                              | `master`                 |                               | Branch to which the git diff will happen to detect new or modified files.                   |
+| `--diffed-branch`                              | `master`                        |                               | Branch to which the git diff will happen to detect new or modified files.                   |
 | `--gha-workflow-run-id`                        |                                 |                               | GHA CI only - The run id of the GitHub action workflow                                      |
 | `--ci-context`                                 | `manual`                        |                               | The current CI context: `manual` for manual run, `pull_request`, `nightly_builds`, `master` |
 | `--pipeline-start-timestamp`                   | Current epoch time              | `CI_PIPELINE_START_TIMESTAMP` | Start time of the pipeline as epoch time. Used for pipeline run duration computation.       |
@@ -744,7 +744,8 @@ E.G.: running Poe tasks on the modified internal packages of the current branch:
 ## Changelog
 
 | Version | PR                                                         | Description                                                                                                                  |
-|---------| ---------------------------------------------------------- |------------------------------------------------------------------------------------------------------------------------------|
+| ------- | ---------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| 4.12.7  | [#37787](https://github.com/airbytehq/airbyte/pull/37787)  | Remove requirements on dockerhub credentials to run QA checks.                                                               |
 | 4.12.6  | [#36497](https://github.com/airbytehq/airbyte/pull/36497)  | Add airbyte-cdk to list of poetry packages for testing                                                                       |
 | 4.12.5  | [#37785](https://github.com/airbytehq/airbyte/pull/37785)  | Set the `--yes-auto-update` flag to `True` by default.                                                                       |
 | 4.12.4  | [#37786](https://github.com/airbytehq/airbyte/pull/37786)  | (fixed 4.12.2): Do not upload dagger log to GCP when no credentials are available.                                           |

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/steps/docker.py

@@ -18,7 +18,7 @@ def __init__(
         context: PipelineContext,
         paths_to_mount: List[MountPath] = [],
         internal_tools: List[MountPath] = [],
-        secrets: dict[str, dagger.Secret] = {},
+        secrets: dict[str, dagger.Secret | None] = {},
         env_variables: dict[str, str] = {},
         working_directory: str = "/",
         command: Optional[List[str]] = None,
@@ -78,7 +78,8 @@ def _set_env_variables(self, container: dagger.Container) -> dagger.Container:
 
     def _set_secrets(self, container: dagger.Container) -> dagger.Container:
         for key, value in self.secrets.items():
-            container = container.with_secret_variable(key, value)
+            if value is not None:
+                container = container.with_secret_variable(key, value)
         return container
 
     async def init_container(self) -> dagger.Container:

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "pipelines"
-version = "4.12.6"
+version = "4.12.7"
 description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines"
 authors = ["Airbyte <[email protected]>"]