airbyte ci test to support --extras

Author: erohmensing

.github/workflows/airbyte-ci-tests.yml

@@ -30,6 +30,7 @@ jobs:
         with:
           # Note: expressions within a filter are OR'ed
           filters: |
+            # This list is duplicated in `pipelines/airbyte_ci/test/__init__.py`
             internal_poetry_packages:
             - airbyte-lib/**
             - airbyte-ci/connectors/pipelines/**

airbyte-ci/connectors/base_images/pyproject.toml

@@ -36,6 +36,6 @@ publish =  "base_images.commands:publish_existing_version"
 test = "pytest tests"
 
 [tool.airbyte_ci]
-extra_poetry_groups = ["dev"]
+optional_poetry_groups = ["dev"]
 poe_tasks = ["test"]
 mount_docker_socket = true

airbyte-ci/connectors/ci_credentials/pyproject.toml

@@ -32,5 +32,5 @@ ci_credentials = "ci_credentials.main:ci_credentials"
 test = "pytest tests"
 
 [tool.airbyte_ci]
-extra_poetry_groups = ["dev"]
+optional_poetry_groups = ["dev"]
 poe_tasks = ["test"]

airbyte-ci/connectors/common_utils/pyproject.toml

@@ -27,6 +27,6 @@ build-backend = "poetry.core.masonry.api"
 test = "pytest tests"
 
 [tool.airbyte_ci]
-extra_poetry_groups = ["dev"]
+optional_poetry_groups = ["dev"]
 # Disable poe tasks as tests are not passing ATM
 poe_tasks = []

airbyte-ci/connectors/connector_ops/pyproject.toml

@@ -38,5 +38,5 @@ allowed-hosts-checks = "connector_ops.allowed_hosts_checks:check_allowed_hosts"
 test = "pytest tests"
 
 [tool.airbyte_ci]
-extra_poetry_groups = ["dev"]
+optional_poetry_groups = ["dev"]
 poe_tasks = ["test"]

airbyte-ci/connectors/connectors_qa/pyproject.toml

@@ -40,6 +40,6 @@ type_check = "mypy src --disallow-untyped-defs"
 lint = "ruff check src"
 
 [tool.airbyte_ci]
-extra_poetry_groups = ["dev"]
+optional_poetry_groups = ["dev"]
 poe_tasks = ["type_check", "lint", "test"]
 required_environment_variables = ["DOCKER_HUB_USERNAME", "DOCKER_HUB_PASSWORD",]

airbyte-ci/connectors/metadata_service/lib/pyproject.toml

@@ -36,7 +36,7 @@ promote-connector-to-latest = "gsutil  -m rsync -r -d gs://$TARGET_BUCKET/metada
 test = "pytest tests"
 
 [tool.airbyte_ci]
-extra_poetry_groups = ["dev"]
+optional_poetry_groups = ["dev"]
 poe_tasks = ["test"]
 
 [build-system]

airbyte-ci/connectors/metadata_service/orchestrator/pyproject.toml

@@ -49,5 +49,5 @@ module_name = "orchestrator"
 test = "pytest tests"
 
 [tool.airbyte_ci]
-extra_poetry_groups = ["dev"]
+optional_poetry_groups = ["dev"]
 poe_tasks = ["test"]

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/models.py

@@ -11,7 +11,8 @@ class AirbyteCiPackageConfiguration(BaseModel):
     required_environment_variables: Set[str] = Field(
         set(), description="List of unique required environment variables to pass to the container running the poe task"
     )
-    extra_poetry_groups: Set[str] = Field(set(), description="List of unique extra poetry groups to install")
+    poetry_extras: Set[str] = Field(set(), description="List of unique poetry extras to install")
+    optional_poetry_groups: Set[str] = Field(set(), description="List of unique poetry groups to install")
     side_car_docker_engine: bool = Field(
         False, description="Flag indicating the use of a sidecar Docker engine during the poe task executions"
     )

airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/test/pipeline.py

@@ -191,7 +191,11 @@ def prepare_container_for_poe_tasks(
     container = container.with_workdir(f"/airbyte/{poetry_package_path}")
 
     # Install the poetry package
-    container = container.with_exec(["poetry", "install"] + [f"--with={group}" for group in airbyte_ci_package_config.extra_poetry_groups])
+    container = container.with_exec(
+        ["poetry", "install"]
+        + [f"--with={group}" for group in airbyte_ci_package_config.optional_poetry_groups]
+        + [f"--extras={extra}" for extra in airbyte_ci_package_config.poetry_extras]
+    )
     return container
 
 

airbyte-ci/connectors/pipelines/pyproject.toml

@@ -62,6 +62,6 @@ type_check = "mypy pipelines --disallow-untyped-defs"
 lint = "ruff check pipelines"
 
 [tool.airbyte_ci]
-extra_poetry_groups = ["dev"]
+optional_poetry_groups = ["dev"]
 poe_tasks = ["type_check", "lint", "test"]
 mount_docker_socket = true

airbyte-lib/pyproject.toml

@@ -255,7 +255,7 @@ airbyte-lib-validate-source = "airbyte_lib.validate:run"
 test = "pytest tests"
 
 [tool.airbyte_ci]
-extra_poetry_groups = ["dev"]
+optional_poetry_groups = ["dev"]
 poe_tasks = ["test"]
 required_environment_variables = ["GCP_GSM_CREDENTIALS"]
 side_car_docker_engine = true