Move ValidatorOpts higher in call

Author: bnchrch

airbyte-ci/connectors/metadata_service/lib/metadata_service/commands.py

@@ -5,7 +5,7 @@
 
 import click
 from metadata_service.gcs_upload import upload_metadata_to_gcs, MetadataUploadInfo
-from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, validate_and_load
+from metadata_service.validators.metadata_validator import PRE_UPLOAD_VALIDATORS, validate_and_load, ValidatorOptions
 from metadata_service.constants import METADATA_FILE_NAME
 from pydantic import ValidationError
 
@@ -54,9 +54,9 @@ def validate(file_path: pathlib.Path):
 @click.option("--prerelease", type=click.STRING, required=False, default=None, help="The prerelease tag of the connector.")
 def upload(metadata_file_path: pathlib.Path, bucket_name: str, prerelease: str):
     metadata_file_path = metadata_file_path if not metadata_file_path.is_dir() else metadata_file_path / METADATA_FILE_NAME
-
+    validator_opts = ValidatorOptions(prerelease_tag=prerelease)
     try:
-        upload_info = upload_metadata_to_gcs(bucket_name, metadata_file_path, prerelease)
+        upload_info = upload_metadata_to_gcs(bucket_name, metadata_file_path, validator_opts)
         log_metadata_upload_info(upload_info)
     except (ValidationError, FileNotFoundError) as e:
         click.secho(f"The metadata file could not be uploaded: {str(e)}", color="red")

airbyte-ci/connectors/metadata_service/lib/metadata_service/gcs_upload.py

@@ -15,7 +15,7 @@
 from google.oauth2 import service_account
 
 from metadata_service.constants import METADATA_FILE_NAME, METADATA_FOLDER, ICON_FILE_NAME
-from metadata_service.validators.metadata_validator import POST_UPLOAD_VALIDATORS, validate_and_load, ValidatorContext
+from metadata_service.validators.metadata_validator import POST_UPLOAD_VALIDATORS, validate_and_load, ValidatorOptions
 from metadata_service.utils import to_json_sanitized_dict
 from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0
 
@@ -122,30 +122,30 @@ def _icon_upload(metadata: ConnectorMetadataDefinitionV0, bucket: storage.bucket
     return upload_file_if_changed(local_icon_path, bucket, latest_icon_path)
 
 
-def create_prerelease_metadata_file(metadata_file_path: Path, prerelease: str) -> Path:
-    metadata, error = validate_and_load(metadata_file_path, [], prerelease=prerelease)
+def create_prerelease_metadata_file(metadata_file_path: Path, validator_opts: ValidatorOptions) -> Path:
+    metadata, error = validate_and_load(metadata_file_path, [], validator_opts)
     if metadata is None:
         raise ValueError(f"Metadata file {metadata_file_path} is invalid for uploading: {error}")
 
     # replace any dockerImageTag references with the actual tag
     # this includes metadata.data.dockerImageTag, metadata.data.registries[].dockerImageTag
     # where registries is a dictionary of registry name to registry object
     metadata_dict = to_json_sanitized_dict(metadata, exclude_none=True)
-    metadata_dict["data"]["dockerImageTag"] = prerelease
+    metadata_dict["data"]["dockerImageTag"] = validator_opts.prerelease_tag
     for registry in get(metadata_dict, "data.registries", {}).values():
         if "dockerImageTag" in registry:
-            registry["dockerImageTag"] = prerelease
+            registry["dockerImageTag"] = validator_opts.prerelease_tag
 
     # write metadata to yaml file in system tmp folder
-    tmp_metadata_file_path = Path("/tmp") / metadata.data.dockerRepository / prerelease / METADATA_FILE_NAME
+    tmp_metadata_file_path = Path("/tmp") / metadata.data.dockerRepository / validator_opts.prerelease_tag / METADATA_FILE_NAME
     tmp_metadata_file_path.parent.mkdir(parents=True, exist_ok=True)
     with open(tmp_metadata_file_path, "w") as f:
         yaml.dump(metadata_dict, f)
 
     return tmp_metadata_file_path
 
 
-def upload_metadata_to_gcs(bucket_name: str, metadata_file_path: Path, prerelease: Optional[str] = None) -> MetadataUploadInfo:
+def upload_metadata_to_gcs(bucket_name: str, metadata_file_path: Path, validator_opts: ValidatorOptions = ValidatorOptions()) -> MetadataUploadInfo:
     """Upload a metadata file to a GCS bucket.
 
     If the per 'version' key already exists it won't be overwritten.
@@ -155,14 +155,14 @@ def upload_metadata_to_gcs(bucket_name: str, metadata_file_path: Path, prereleas
         bucket_name (str): Name of the GCS bucket to which the metadata file will be uploade.
         metadata_file_path (Path): Path to the metadata file.
         service_account_file_path (Path): Path to the JSON file with the service account allowed to read and write on the bucket.
-        prerelease (Optional[str]): Whether the connector is a prerelease or not.
+        prerelease_tag (Optional[str]): Whether the connector is a prerelease_tag or not.
     Returns:
         Tuple[bool, str]: Whether the metadata file was uploaded and its blob id.
     """
-    if prerelease:
-        metadata_file_path = create_prerelease_metadata_file(metadata_file_path, prerelease)
+    if validator_opts.prerelease_tag:
+        metadata_file_path = create_prerelease_metadata_file(metadata_file_path, validator_opts)
 
-    metadata, error = validate_and_load(metadata_file_path, POST_UPLOAD_VALIDATORS, prerelease=prerelease)
+    metadata, error = validate_and_load(metadata_file_path, POST_UPLOAD_VALIDATORS, validator_opts)
 
     if metadata is None:
         raise ValueError(f"Metadata file {metadata_file_path} is invalid for uploading: {error}")
@@ -175,7 +175,7 @@ def upload_metadata_to_gcs(bucket_name: str, metadata_file_path: Path, prereleas
     icon_uploaded, icon_blob_id = _icon_upload(metadata, bucket, metadata_file_path)
 
     version_uploaded, version_blob_id = _version_upload(metadata, bucket, metadata_file_path)
-    if not prerelease:
+    if not validator_opts.prerelease_tag:
         latest_uploaded, latest_blob_id = _latest_upload(metadata, bucket, metadata_file_path)
     else:
         latest_uploaded, latest_blob_id = False, None

airbyte-ci/connectors/metadata_service/lib/metadata_service/validators/metadata_validator.py

@@ -11,12 +11,12 @@
 
 
 @dataclass(frozen=True)
-class ValidatorContext:
-    prerelease: Optional[str] = None
+class ValidatorOptions:
+    prerelease_tag: Optional[str] = None
 
 
 ValidationResult = Tuple[bool, Optional[Union[ValidationError, str]]]
-Validator = Callable[[ConnectorMetadataDefinitionV0, ValidatorContext], ValidationResult]
+Validator = Callable[[ConnectorMetadataDefinitionV0, ValidatorOptions], ValidationResult]
 
 # TODO: Remove these when each of these connectors ship any new version
 ALREADY_ON_MAJOR_VERSION_EXCEPTIONS = [
@@ -34,7 +34,7 @@ class ValidatorContext:
 
 
 def validate_metadata_images_in_dockerhub(
-    metadata_definition: ConnectorMetadataDefinitionV0, validator_context: ValidatorContext
+    metadata_definition: ConnectorMetadataDefinitionV0, validator_opts: ValidatorOptions
 ) -> ValidationResult:
     metadata_definition_dict = metadata_definition.dict()
     base_docker_image = get(metadata_definition_dict, "data.dockerRepository")
@@ -58,7 +58,7 @@ def validate_metadata_images_in_dockerhub(
         (normalization_docker_image, normalization_docker_version),
     ]
 
-    if not validator_context.prerelease:
+    if not validator_opts.prerelease_tag:
         possible_docker_images.extend([(base_docker_image, version) for version in breaking_change_versions])
 
     # Filter out tuples with None and remove duplicates
@@ -73,7 +73,7 @@ def validate_metadata_images_in_dockerhub(
 
 
 def validate_at_least_one_language_tag(
-    metadata_definition: ConnectorMetadataDefinitionV0, _validator_context: ValidatorContext
+    metadata_definition: ConnectorMetadataDefinitionV0, _validator_opts: ValidatorOptions
 ) -> ValidationResult:
     """Ensure that there is at least one tag in the data.tags field that matches language:<LANG>."""
     tags = get(metadata_definition, "data.tags", [])
@@ -84,7 +84,7 @@ def validate_at_least_one_language_tag(
 
 
 def validate_all_tags_are_keyvalue_pairs(
-    metadata_definition: ConnectorMetadataDefinitionV0, _validator_context: ValidatorContext
+    metadata_definition: ConnectorMetadataDefinitionV0, _validator_opts: ValidatorOptions
 ) -> ValidationResult:
     """Ensure that all tags are of the form <KEY>:<VALUE>."""
     tags = get(metadata_definition, "data.tags", [])
@@ -102,7 +102,7 @@ def is_major_version(version: str) -> bool:
 
 
 def validate_major_version_bump_has_breaking_change_entry(
-    metadata_definition: ConnectorMetadataDefinitionV0, _validator_context: ValidatorContext
+    metadata_definition: ConnectorMetadataDefinitionV0, _validator_opts: ValidatorOptions
 ) -> ValidationResult:
     """Ensure that if the major version is incremented, there is a breaking change entry for that version."""
     metadata_definition_dict = metadata_definition.dict()
@@ -147,18 +147,14 @@ def validate_major_version_bump_has_breaking_change_entry(
 def validate_and_load(
     file_path: pathlib.Path,
     validators_to_run: List[Validator],
-    **kwargs,
+    validator_opts: ValidatorOptions = ValidatorOptions(),
 ) -> Tuple[Optional[ConnectorMetadataDefinitionV0], Optional[ValidationError]]:
     """Load a metadata file from a path (runs jsonschema validation) and run optional extra validators.
 
     Returns a tuple of (metadata_model, error_message).
     If the metadata file is valid, metadata_model will be populated.
     Otherwise, error_message will be populated with a string describing the error.
     """
-
-    # Initialize the validator context with the kwargs only if it is not empty
-    validator_context = ValidatorContext(**kwargs) if kwargs else ValidatorContext()
-
     try:
         # Load the metadata file - this implicitly runs jsonschema validation
         metadata = yaml.safe_load(file_path.read_text())
@@ -167,7 +163,7 @@ def validate_and_load(
         return None, f"Validation error: {e}"
 
     for validator in validators_to_run:
-        is_valid, error = validator(metadata_model, validator_context)
+        is_valid, error = validator(metadata_model, validator_opts)
         if not is_valid:
             return None, f"Validation error: {error}"
 

airbyte-ci/connectors/metadata_service/lib/tests/test_gcs_upload.py

@@ -9,6 +9,7 @@
 from pydash.objects import get
 
 from metadata_service import gcs_upload
+from metadata_service.validators.metadata_validator import ValidatorOptions
 from metadata_service.models.generated.ConnectorMetadataDefinitionV0 import ConnectorMetadataDefinitionV0
 from metadata_service.constants import METADATA_FILE_NAME
 from metadata_service.utils import to_json_sanitized_dict
@@ -185,7 +186,7 @@ def test_upload_metadata_to_gcs_with_prerelease(mocker, valid_metadata_upload_fi
         gcs_upload.upload_metadata_to_gcs(
             "my_bucket",
             metadata_file_path,
-            prerelease_image_tag,
+            ValidatorOptions(prerelease_tag=prerelease_image_tag),
         )
 
         gcs_upload._latest_upload.assert_not_called()