diff --git a/octavia-cli/README.md b/octavia-cli/README.md index afe8dda551f52..26b2242a834b8 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -38,7 +38,8 @@ We welcome community contributions! | Date | Milestone | |------------|-------------------------------------| -| 2022-02-06 | Implement `octavia generate`| +| 2022-03-02 | Implement `octavia apply` (sources and destination only)| +| 2022-02-06 | Implement `octavia generate` (sources and destination only)| | 2022-01-25 | Implement `octavia init` + some context checks| | 2022-01-19 | Implement `octavia list workspace sources`, `octavia list workspace destinations`, `octavia list workspace connections`| | 2022-01-17 | Implement `octavia list connectors source` and `octavia list connectors destinations`| @@ -49,9 +50,9 @@ We welcome community contributions! 1. Install Python 3.8.12. We suggest doing it through `pyenv` 2. Create a virtualenv: `python -m venv .venv` 3. Activate the virtualenv: `source .venv/bin/activate` -4. Install dev dependencies: `pip install -e .\[dev\]` +4. Install dev dependencies: `pip install -e .\[tests\]` 5. Install `pre-commit` hooks: `pre-commit install` -6. Run the test suite: `pytest --cov=octavia_cli unit_tests` +6. Run the unittest suite: `pytest --cov=octavia_cli` 7. Iterate: please check the [Contributing](#contributing) for instructions on contributing. # Contributing diff --git a/octavia-cli/tests/integration/test_generate/__init__.py b/octavia-cli/integration_tests/test_generate/__init__.py similarity index 100% rename from octavia-cli/tests/integration/test_generate/__init__.py rename to octavia-cli/integration_tests/test_generate/__init__.py diff --git a/octavia-cli/tests/conftest.py b/octavia-cli/integration_tests/test_generate/conftest.py similarity index 81% rename from octavia-cli/tests/conftest.py rename to octavia-cli/integration_tests/test_generate/conftest.py index a1323c227ed40..edaf005f66059 100644 --- a/octavia-cli/tests/conftest.py +++ b/octavia-cli/integration_tests/test_generate/conftest.py @@ -11,8 +11,3 @@ def octavia_project_directory(tmpdir): for directory in OCTAVIA_PROJECT_DIRECTORIES: tmpdir.mkdir(directory) return tmpdir - - -@pytest.fixture -def mock_api_client(mocker): - return mocker.Mock() diff --git a/octavia-cli/tests/integration/test_generate/expected_rendered_yaml/destination_postgres/expected.yaml b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_postgres/expected.yaml similarity index 100% rename from octavia-cli/tests/integration/test_generate/expected_rendered_yaml/destination_postgres/expected.yaml rename to octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_postgres/expected.yaml diff --git a/octavia-cli/tests/integration/test_generate/expected_rendered_yaml/destination_postgres/input_spec.yaml b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_postgres/input_spec.yaml similarity index 100% rename from octavia-cli/tests/integration/test_generate/expected_rendered_yaml/destination_postgres/input_spec.yaml rename to octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_postgres/input_spec.yaml diff --git a/octavia-cli/tests/integration/test_generate/expected_rendered_yaml/destination_s3/expected.yaml b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_s3/expected.yaml similarity index 100% rename from octavia-cli/tests/integration/test_generate/expected_rendered_yaml/destination_s3/expected.yaml rename to octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_s3/expected.yaml diff --git a/octavia-cli/tests/integration/test_generate/expected_rendered_yaml/destination_s3/input_spec.yaml b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_s3/input_spec.yaml similarity index 100% rename from octavia-cli/tests/integration/test_generate/expected_rendered_yaml/destination_s3/input_spec.yaml rename to octavia-cli/integration_tests/test_generate/expected_rendered_yaml/destination_s3/input_spec.yaml diff --git a/octavia-cli/tests/integration/test_generate/expected_rendered_yaml/source_postgres/expected.yaml b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/source_postgres/expected.yaml similarity index 100% rename from octavia-cli/tests/integration/test_generate/expected_rendered_yaml/source_postgres/expected.yaml rename to octavia-cli/integration_tests/test_generate/expected_rendered_yaml/source_postgres/expected.yaml diff --git a/octavia-cli/tests/integration/test_generate/expected_rendered_yaml/source_postgres/input_spec.yaml b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/source_postgres/input_spec.yaml similarity index 100% rename from octavia-cli/tests/integration/test_generate/expected_rendered_yaml/source_postgres/input_spec.yaml rename to octavia-cli/integration_tests/test_generate/expected_rendered_yaml/source_postgres/input_spec.yaml diff --git a/octavia-cli/tests/integration/test_generate/test_renderer.py b/octavia-cli/integration_tests/test_generate/test_renderer.py similarity index 52% rename from octavia-cli/tests/integration/test_generate/test_renderer.py rename to octavia-cli/integration_tests/test_generate/test_renderer.py index 917273bb6ef86..e1e2e3c3842fc 100644 --- a/octavia-cli/tests/integration/test_generate/test_renderer.py +++ b/octavia-cli/integration_tests/test_generate/test_renderer.py @@ -14,45 +14,48 @@ DESTINATION_SPECS = "../airbyte-config/init/src/main/resources/seed/destination_specs.yaml" -@pytest.mark.parametrize("spec_type, spec_file_path", [("source", SOURCE_SPECS), ("destination", DESTINATION_SPECS)]) -def test_rendering_all_specs(spec_type, spec_file_path, octavia_project_directory, mocker): - with open(spec_file_path, "r") as f: - specs = yaml.load(f, yaml.FullLoader) - rendered_specs = [] - for i, spec in enumerate(specs): - renderer = ConnectionSpecificationRenderer( - resource_name=f"resource-{i}", - definition=mocker.Mock( - type=spec_type, - id=i, - docker_repository=spec["dockerImage"].split(":")[0], - docker_image_tag=spec["dockerImage"].split(":")[-1], - documentation_url=spec["spec"]["documentationUrl"], - specification=mocker.Mock(connection_specification=spec["spec"]["connectionSpecification"]), - ), - ) - output_path = renderer.write_yaml(octavia_project_directory) - rendered_specs.append(output_path) - assert len(rendered_specs) == len(specs) - for rendered_spec in rendered_specs: - with open(rendered_spec, "r") as f: - parsed_yaml = yaml.load(f, yaml.FullLoader) - assert all( - [ - expected_field in parsed_yaml - for expected_field in [ - "resource_name", - "definition_type", - "definition_id", - "definition_image", - "definition_version", - "configuration", - ] +def get_all_specs_params(): + with open(SOURCE_SPECS, "r") as f: + source_specs = yaml.load(f, yaml.FullLoader) + with open(DESTINATION_SPECS, "r") as f: + destination_specs = yaml.load(f, yaml.FullLoader) + return [pytest.param("source", spec, id=spec["dockerImage"]) for spec in source_specs] + [ + pytest.param("destination", spec, id=spec["dockerImage"]) for spec in destination_specs + ] + + +@pytest.mark.parametrize("spec_type, spec", get_all_specs_params()) +def test_render_spec(spec_type, spec, octavia_project_directory, mocker): + renderer = ConnectionSpecificationRenderer( + resource_name=f"resource-{spec['dockerImage']}", + definition=mocker.Mock( + type=spec_type, + id="foo", + docker_repository=spec["dockerImage"].split(":")[0], + docker_image_tag=spec["dockerImage"].split(":")[-1], + documentation_url=spec["spec"]["documentationUrl"], + specification=mocker.Mock(connection_specification=spec["spec"]["connectionSpecification"]), + ), + ) + output_path = renderer.write_yaml(octavia_project_directory) + with open(output_path, "r") as f: + parsed_yaml = yaml.load(f, yaml.FullLoader) + assert all( + [ + expected_field in parsed_yaml + for expected_field in [ + "resource_name", + "definition_type", + "definition_id", + "definition_image", + "definition_version", + "configuration", ] - ) + ] + ) -EXPECTED_RENDERED_YAML_PATH = "tests/integration/test_generate/expected_rendered_yaml" +EXPECTED_RENDERED_YAML_PATH = f"{os.path.dirname(__file__)}/expected_rendered_yaml" @pytest.mark.parametrize( diff --git a/octavia-cli/tests/unit/test_init/__init__.py b/octavia-cli/octavia_cli/apply/__init__.py similarity index 100% rename from octavia-cli/tests/unit/test_init/__init__.py rename to octavia-cli/octavia_cli/apply/__init__.py diff --git a/octavia-cli/octavia_cli/apply/commands.py b/octavia-cli/octavia_cli/apply/commands.py new file mode 100644 index 0000000000000..49a175a8ac554 --- /dev/null +++ b/octavia-cli/octavia_cli/apply/commands.py @@ -0,0 +1,163 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from glob import glob +from typing import List, Tuple + +import airbyte_api_client +import click +from octavia_cli.check_context import REQUIRED_PROJECT_DIRECTORIES, requires_init + +from .diff_helpers import display_diff_line +from .resources import BaseResource +from .resources import factory as resource_factory + + +@click.command(name="apply", help="Create or update Airbyte remote resources according local YAML configurations.") +@click.option("--file", "-f", "configurations_files", type=click.Path(), multiple=True) +@click.option("--force", is_flag=True, default=False, help="Does not display the diff and updates without user prompt.") +@click.pass_context +@requires_init +def apply(ctx: click.Context, configurations_files: List[click.Path], force: bool): + if not configurations_files: + configurations_files = find_local_configuration_files() + + resources = get_resources_to_apply(configurations_files, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"]) + for resource in resources: + apply_single_resource(resource, force) + + +def get_resources_to_apply( + configuration_files: List[str], api_client: airbyte_api_client.ApiClient, workspace_id: str +) -> List[BaseResource]: + """Create resource objects with factory and sort according to apply priority. + + Args: + configuration_files (List[str]): List of YAML configuration files. + api_client (airbyte_api_client.ApiClient): the Airbyte API client. + workspace_id (str): current Airbyte workspace id. + + Returns: + List[BaseResource]: Resources sorted according to their apply priority. + """ + all_resources = [resource_factory(api_client, workspace_id, path) for path in configuration_files] + return sorted(all_resources, key=lambda resource: resource.apply_priority) + + +def apply_single_resource(resource: BaseResource, force: bool) -> None: + """Runs resource creation if it was not created, update it otherwise. + + Args: + resource (BaseResource): The resource to apply. + force (bool): Whether force mode is on. + """ + if resource.was_created: + click.echo( + click.style(f"🐙 - {resource.resource_name} exists on your Airbyte instance, let's check if we need to update it!", fg="yellow") + ) + messages = update_resource(resource, force) + else: + click.echo(click.style(f"🐙 - {resource.resource_name} does not exists on your Airbyte instance, let's create it!", fg="green")) + messages = create_resource(resource) + click.echo("\n".join(messages)) + + +def should_update_resource(force: bool, diff: str, local_file_changed: bool) -> Tuple[bool, str]: + """Function to decide if the resource needs an update or not. + + Args: + force (bool): Whether force mode is on. + diff (str): The computed diff between local and remote for this resource. + local_file_changed (bool): Whether the local file describing the resource was modified. + + Returns: + Tuple[bool, str]: Boolean to know if resource should be updated and string describing the update reason. + """ + if force: + should_update, update_reason = True, "🚨 - Running update because force mode is on." + elif diff: + should_update, update_reason = True, "✍️ - Running update because a diff was detected between local and remote resource." + elif local_file_changed: + should_update, update_reason = ( + True, + "✍️ - Running update because a local file change was detected and a secret field might have been edited.", + ) + else: + should_update, update_reason = False, "😴 - Did not update because no change detected." + return should_update, click.style(update_reason, fg="green") + + +def prompt_for_diff_validation(resource_name: str, diff: str) -> bool: + """Display the diff to user and prompt them from validation. + + Args: + resource_name (str): Name of the resource the diff was computed for. + diff (str): The diff. + + Returns: + bool: Whether user validated the diff. + """ + if diff: + click.echo( + click.style("👀 - Here's the computed diff (🚨 remind that diff on secret fields are not displayed):", fg="magenta", bold=True) + ) + for line in diff.split("\n"): + display_diff_line(line) + return click.confirm(click.style(f"❓ - Do you want to update {resource_name}?", bold=True)) + else: + return False + + +def create_resource(resource: BaseResource) -> List[str]: + """Run a resource creation. + + Args: + resource (BaseResource): The resource to create. + + Returns: + List[str]: Post create messages to display to standard output. + """ + created_resource, state = resource.create() + return [ + click.style(f"🎉 - Successfully created {created_resource.name} on your Airbyte instance!", fg="green", bold=True), + click.style(f"💾 - New state for {created_resource.name} saved at {state.path}", fg="yellow"), + ] + + +def update_resource(resource: BaseResource, force: bool) -> List[str]: + """Run a resource update. Check if update is required and prompt for user diff validation if needed. + + Args: + resource (BaseResource): Resource to update + force (bool): Whether force mode is on. + + Returns: + List[str]: Post update messages to display to standard output. + """ + diff = resource.get_diff_with_remote_resource() + should_update, update_reason = should_update_resource(force, diff, resource.local_file_changed) + output_messages = [update_reason] + if not force and diff: + should_update = prompt_for_diff_validation(resource.resource_name, diff) + if should_update: + updated_resource, state = resource.update() + output_messages.append( + click.style(f"🎉 - Successfully updated {updated_resource.name} on your Airbyte instance!", fg="green", bold=True) + ) + output_messages.append(click.style(f"💾 - New state for {updated_resource.name} stored at {state.path}.", fg="yellow")) + return output_messages + + +def find_local_configuration_files() -> List[str]: + """Discover local configuration files. + + Returns: + List[str]: Paths to YAML configuration files. + """ + configuration_files = [] + for resource_directory in REQUIRED_PROJECT_DIRECTORIES: + configuration_files += glob(f"./{resource_directory}/**/configuration.yaml") + if not configuration_files: + click.echo(click.style("😒 - No YAML file found to run apply.", fg="red")) + return configuration_files diff --git a/octavia-cli/octavia_cli/apply/diff_helpers.py b/octavia-cli/octavia_cli/apply/diff_helpers.py new file mode 100644 index 0000000000000..a5d5e2e3b8e74 --- /dev/null +++ b/octavia-cli/octavia_cli/apply/diff_helpers.py @@ -0,0 +1,80 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import hashlib +from typing import Any + +import click +from deepdiff import DeepDiff + +SECRET_MASK = "**********" + + +def compute_checksum(file_path: str) -> str: + """Compute SHA256 checksum from a file + + Args: + file_path (str): Path for the file for which you want to compute a checksum. + + Returns: + str: The computed hash digest + """ + BLOCK_SIZE = 65536 + file_hash = hashlib.sha256() + with open(file_path, "rb") as f: + fb = f.read(BLOCK_SIZE) + while len(fb) > 0: + file_hash.update(fb) + fb = f.read(BLOCK_SIZE) + return file_hash.hexdigest() + + +def exclude_secrets_from_diff(obj: Any, path: str) -> bool: + """Callback function used with DeepDiff to ignore secret values from the diff. + + Args: + obj (Any): Object for which a diff will be computed. + path (str): unused. + + Returns: + bool: Whether to ignore the object from the diff. + """ + if isinstance(obj, str): + return True if SECRET_MASK in obj else False + else: + return False + + +def compute_diff(a: Any, b: Any) -> DeepDiff: + """Wrapper around the DeepDiff computation. + + Args: + a (Any): Object to compare with b. + b (Any): Object to compare with a. + + Returns: + DeepDiff: the computed diff object. + """ + return DeepDiff(a, b, view="tree", exclude_obj_callback=exclude_secrets_from_diff) + + +def display_diff_line(diff_line: str) -> None: + """Prettify a diff line and print it to standard output. + + Args: + diff_line (str): The diff line to display. + """ + if "changed from" in diff_line: + color = "yellow" + prefix = "E" + elif "added" in diff_line: + color = "green" + prefix = "+" + elif "removed" in diff_line: + color = "red" + prefix = "-" + else: + prefix = "" + color = None + click.echo(click.style(f"\t{prefix} - {diff_line}", fg=color)) diff --git a/octavia-cli/octavia_cli/apply/resources.py b/octavia-cli/octavia_cli/apply/resources.py new file mode 100644 index 0000000000000..4efe338b535ef --- /dev/null +++ b/octavia-cli/octavia_cli/apply/resources.py @@ -0,0 +1,426 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import abc +import os +import time +from pathlib import Path +from typing import Any, Callable, Optional, Union + +import airbyte_api_client +import yaml +from airbyte_api_client.api import destination_api, source_api +from airbyte_api_client.model.destination_create import DestinationCreate +from airbyte_api_client.model.destination_read import DestinationRead +from airbyte_api_client.model.destination_read_list import DestinationReadList +from airbyte_api_client.model.destination_search import DestinationSearch +from airbyte_api_client.model.destination_update import DestinationUpdate +from airbyte_api_client.model.source_create import SourceCreate +from airbyte_api_client.model.source_read import SourceRead +from airbyte_api_client.model.source_read_list import SourceReadList +from airbyte_api_client.model.source_search import SourceSearch +from airbyte_api_client.model.source_update import SourceUpdate +from click import ClickException + +from .diff_helpers import compute_checksum, compute_diff + + +class DuplicateResourceError(ClickException): + pass + + +class NonExistingResourceError(ClickException): + pass + + +class InvalidConfigurationError(ClickException): + pass + + +class ResourceState: + def __init__(self, configuration_path: str, resource_id: str, generation_timestamp: int, configuration_checksum: str): + """This constructor is meant to be private. Construction shall be made with create or from_file class methods. + + Args: + configuration_path (str): Path to the configuration path the state relates to. + resource_id (str): Id of the resource the state relates to. + generation_timestamp (int): State generation timestamp. + configuration_checksum (str): Checksum of the configuration file. + """ + self.configuration_path = configuration_path + self.resource_id = resource_id + self.generation_timestamp = generation_timestamp + self.configuration_checksum = configuration_checksum + self.path = os.path.join(os.path.dirname(self.configuration_path), "state.yaml") + + def as_dict(self): + return { + "configuration_path": self.configuration_path, + "resource_id": self.resource_id, + "generation_timestamp": self.generation_timestamp, + "configuration_checksum": self.configuration_checksum, + } + + def _save(self) -> None: + """Save the state as a YAML file.""" + with open(self.path, "w") as state_file: + yaml.dump(self.as_dict(), state_file) + + @classmethod + def create(cls, configuration_path: str, resource_id: str) -> "ResourceState": + """Create a state for a resource configuration. + + Args: + configuration_path (str): Path to the YAML file defining the resource. + resource_id (str): UUID of the resource. + + Returns: + ResourceState: state representing the resource. + """ + generation_timestamp = int(time.time()) + configuration_checksum = compute_checksum(configuration_path) + state = ResourceState(configuration_path, resource_id, generation_timestamp, configuration_checksum) + state._save() + return state + + @classmethod + def from_file(cls, file_path: str) -> "ResourceState": + """Deserialize a state from a YAML path. + + Args: + file_path (str): Path to the YAML state. + + Returns: + ResourceState: state deserialized from YAML. + """ + with open(file_path, "r") as f: + raw_state = yaml.load(f, yaml.FullLoader) + return ResourceState( + raw_state["configuration_path"], + raw_state["resource_id"], + raw_state["generation_timestamp"], + raw_state["configuration_checksum"], + ) + + +class BaseResource(abc.ABC): + @property + @abc.abstractmethod + def apply_priority( + self, + ): # pragma: no cover + pass + + @property + @abc.abstractmethod + def api( + self, + ): # pragma: no cover + pass + + @property + @abc.abstractmethod + def create_function_name( + self, + ): # pragma: no cover + pass + + @property + @abc.abstractmethod + def create_payload( + self, + ): # pragma: no cover + pass + + @property + @abc.abstractmethod + def update_payload( + self, + ): # pragma: no cover + pass + + @property + @abc.abstractmethod + def update_function_name( + self, + ): # pragma: no cover + pass + + @property + @abc.abstractmethod + def search_function_name( + self, + ): # pragma: no cover + pass + + @property + @abc.abstractmethod + def search_payload( + self, + ): # pragma: no cover + pass + + @property + @abc.abstractmethod + def resource_id_field( + self, + ): # pragma: no cover + pass + + @property + @abc.abstractmethod + def resource_type( + self, + ): # pragma: no cover + pass + + def __init__( + self, api_client: airbyte_api_client.ApiClient, workspace_id: str, local_configuration: dict, configuration_path: str + ) -> None: + """Create a BaseResource object. + + Args: + api_client (airbyte_api_client.ApiClient): the Airbyte API client. + workspace_id (str): the workspace id. + local_configuration (dict): The local configuration describing the resource. + configuration_path (str): The path to the local configuration describing the resource with YAML. + """ + self._create_fn = getattr(self.api, self.create_function_name) + self._update_fn = getattr(self.api, self.update_function_name) + self._search_fn = getattr(self.api, self.search_function_name) + self.workspace_id = workspace_id + self.local_configuration = local_configuration + self.configuration_path = configuration_path + self.api_instance = self.api(api_client) + self.state = self._get_state_from_file() + self.local_file_changed = ( + True if self.state is None else compute_checksum(self.configuration_path) != self.state.configuration_checksum + ) + + @property + def remote_resource(self): + return self._get_remote_resource() + + @property + def was_created(self): + return True if self.remote_resource else False + + def __getattr__(self, name: str) -> Any: + """Map attribute of the YAML config to the Resource object. + + Args: + name (str): Attribute name + + Raises: + AttributeError: Raised if the attributed was not found in the local configuration. + + Returns: + [Any]: Attribute value + """ + if name in self.local_configuration: + return self.local_configuration.get(name) + raise AttributeError(f"{self.__class__.__name__}.{name} is invalid.") + + def _search(self) -> Union[SourceReadList, DestinationReadList]: + """Run search of a resources on the remote Airbyte instance. + + Returns: + Union[SourceReadList, DestinationReadList]: Search results + """ + return self._search_fn(self.api_instance, self.search_payload) + + def _get_state_from_file(self) -> Optional[ResourceState]: + """Retrieve a state object from a local YAML file if it exists. + + Returns: + Optional[ResourceState]: the deserialized resource state if YAML file found. + """ + expected_state_path = Path(os.path.join(os.path.dirname(self.configuration_path), "state.yaml")) + if expected_state_path.is_file(): + return ResourceState.from_file(expected_state_path) + + def _get_remote_resource(self) -> Optional[Union[SourceRead, DestinationRead]]: + """Find the remote resource on the Airbyte instance associated with the current resource. + + Raises: + DuplicateResourceError: raised if the search results return multiple resources. + + Returns: + Optional[Union[SourceRead, DestinationRead]]: The remote resource found. + """ + search_results = self._search().get(f"{self.resource_type}s", []) + if len(search_results) > 1: + raise DuplicateResourceError("Two or more ressources exist with the same name.") + if len(search_results) == 1: + return search_results[0] + else: + return None + + def get_diff_with_remote_resource(self) -> str: + """Compute the diff between current resource and the remote resource. + + Raises: + NonExistingResourceError: Raised if the remote resource does not exist. + + Returns: + str: The prettyfied diff. + """ + if not self.was_created: + raise NonExistingResourceError("Cannot compute diff with a non existing remote resource.") + current_config = self.configuration + remote_config = self.remote_resource.connection_configuration + diff = compute_diff(remote_config, current_config) + return diff.pretty() + + def _create_or_update( + self, operation_fn: Callable, payload: Union[SourceCreate, SourceUpdate, DestinationCreate, DestinationUpdate] + ) -> Union[SourceRead, DestinationRead]: + """Wrapper to trigger create or update of remote resource. + + Args: + operation_fn (Callable): The API function to run. + payload (Union[SourceCreate, SourceUpdate, DestinationCreate, DestinationUpdate]): The payload to send to create or update the resource. + + Raises: + InvalidConfigurationError: Raised if the create or update payload is invalid. + ApiException: Raised in case of other API errors. + + Returns: + Union[SourceRead, DestinationRead]: The created or updated resource. + """ + try: + result = operation_fn(self.api_instance, payload) + return result, ResourceState.create(self.configuration_path, result[self.resource_id_field]) + except airbyte_api_client.ApiException as api_error: + if api_error.status == 422: + # This API response error is really verbose, but it embodies all the details about why the config is not valid. + # TODO alafanechere: try to parse it and display it in a more readable way. + raise InvalidConfigurationError(api_error.body) + else: + raise api_error + + def create(self) -> Union[SourceRead, DestinationRead]: + """Public function to create the resource on the remote Airbyte instance. + + Returns: + Union[SourceRead, DestinationRead]: The created resource. + """ + return self._create_or_update(self._create_fn, self.create_payload) + + def update(self) -> Union[SourceRead, DestinationRead]: + """Public function to update the resource on the remote Airbyte instance. + + Returns: + Union[SourceRead, DestinationRead]: The updated resource. + """ + return self._create_or_update(self._update_fn, self.update_payload) + + @property + def resource_id(self) -> Optional[str]: + """Exposes the resource UUID of the remote resource + + Returns: + str: Remote resource's UUID + """ + return self.remote_resource.get(self.resource_id_field) if self.was_created else None + + +class Source(BaseResource): + + apply_priority = 0 + api = source_api.SourceApi + create_function_name = "create_source" + resource_id_field = "source_id" + search_function_name = "search_sources" + update_function_name = "update_source" + resource_type = "source" + + @property + def create_payload(self): + return SourceCreate(self.definition_id, self.configuration, self.workspace_id, self.resource_name) + + @property + def search_payload(self): + if self.state is None: + return SourceSearch(source_definition_id=self.definition_id, workspace_id=self.workspace_id, name=self.resource_name) + else: + return SourceSearch(source_definition_id=self.definition_id, workspace_id=self.workspace_id, source_id=self.state.resource_id) + + @property + def update_payload(self): + return SourceUpdate( + source_id=self.resource_id, + connection_configuration=self.configuration, + name=self.resource_name, + ) + + +class Destination(BaseResource): + + apply_priority = 0 + api = destination_api.DestinationApi + create_function_name = "create_destination" + resource_id_field = "destination_id" + search_function_name = "search_destinations" + update_function_name = "update_destination" + resource_type = "destination" + + @property + def create_payload(self) -> DestinationCreate: + """Defines the payload to create the remote resource. + + Returns: + DestinationCreate: The DestinationCreate model instance + """ + return DestinationCreate(self.workspace_id, self.resource_name, self.definition_id, self.configuration) + + @property + def search_payload(self) -> DestinationSearch: + """Defines the payload to search the remote resource. Search by resource name if no state found, otherwise search by resource id found in the state. + Returns: + DestinationSearch: The DestinationSearch model instance + """ + if self.state is None: + return DestinationSearch(destination_definition_id=self.definition_id, workspace_id=self.workspace_id, name=self.resource_name) + else: + return DestinationSearch( + destination_definition_id=self.definition_id, workspace_id=self.workspace_id, destination_id=self.state.resource_id + ) + + @property + def update_payload(self) -> DestinationUpdate: + """Defines the payload to update a remote resource. + + Returns: + DestinationUpdate: The DestinationUpdate model instance. + """ + return DestinationUpdate( + destination_id=self.resource_id, + connection_configuration=self.configuration, + name=self.resource_name, + ) + + +def factory(api_client: airbyte_api_client.ApiClient, workspace_id: str, configuration_path: str) -> Union[Source, Destination]: + """Create resource object according to the definition type field in their YAML configuration. + + Args: + api_client (airbyte_api_client.ApiClient): The Airbyte API client. + workspace_id (str): The current workspace id. + configuration_path (str): Path to the YAML file with the configuration. + + Raises: + NotImplementedError: Raised if the definition type found in the YAML is not a supported resource. + + Returns: + Union[Source, Destination]: The resource object created from the YAML config. + """ + with open(configuration_path, "r") as f: + local_configuration = yaml.load(f, yaml.FullLoader) + if local_configuration["definition_type"] == "source": + return Source(api_client, workspace_id, local_configuration, configuration_path) + if local_configuration["definition_type"] == "destination": + return Destination(api_client, workspace_id, local_configuration, configuration_path) + else: + raise NotImplementedError(f"Resource {local_configuration['definition_type']} was not yet implemented") diff --git a/octavia-cli/octavia_cli/check_context.py b/octavia-cli/octavia_cli/check_context.py index 320d48a745a45..76d839f46caa0 100644 --- a/octavia-cli/octavia_cli/check_context.py +++ b/octavia-cli/octavia_cli/check_context.py @@ -80,3 +80,14 @@ def check_is_initialized(project_directory: str = ".") -> bool: """ sub_directories = [f.name for f in os.scandir(project_directory) if f.is_dir()] return set(REQUIRED_PROJECT_DIRECTORIES).issubset(sub_directories) + + +def requires_init(f): + def wrapper(ctx, **kwargs): + if not ctx.obj["PROJECT_IS_INITIALIZED"]: + raise ProjectNotInitializedError( + "Your octavia project is not initialized, please run 'octavia init' before running this command." + ) + f(ctx, **kwargs) + + return wrapper diff --git a/octavia-cli/octavia_cli/entrypoint.py b/octavia-cli/octavia_cli/entrypoint.py index ebbd73b8254cb..c2ca8c12c5cb9 100644 --- a/octavia-cli/octavia_cli/entrypoint.py +++ b/octavia-cli/octavia_cli/entrypoint.py @@ -8,12 +8,13 @@ import click from airbyte_api_client.api import workspace_api +from .apply import commands as apply_commands from .check_context import check_api_health, check_is_initialized, check_workspace_exists from .generate import commands as generate_commands from .init import commands as init_commands from .list import commands as list_commands -AVAILABLE_COMMANDS: List[click.Command] = [list_commands._list, init_commands.init, generate_commands.generate] +AVAILABLE_COMMANDS: List[click.Command] = [list_commands._list, init_commands.init, generate_commands.generate, apply_commands.apply] @click.group() @@ -66,11 +67,6 @@ def _import() -> None: raise click.ClickException("The import command is not yet implemented.") -@octavia.command(help="Create or update resources according to YAML configurations.") -def apply() -> None: - raise click.ClickException("The apply command is not yet implemented.") - - @octavia.command(help="Delete resources") def delete() -> None: raise click.ClickException("The delete command is not yet implemented.") diff --git a/octavia-cli/octavia_cli/generate/commands.py b/octavia-cli/octavia_cli/generate/commands.py index b25ca6ac461fb..d34c39c82f8c8 100644 --- a/octavia-cli/octavia_cli/generate/commands.py +++ b/octavia-cli/octavia_cli/generate/commands.py @@ -4,7 +4,7 @@ import click import octavia_cli.generate.definitions as definitions -from octavia_cli.check_context import ProjectNotInitializedError +from octavia_cli.check_context import requires_init from .renderer import ConnectionSpecificationRenderer @@ -14,12 +14,8 @@ @click.argument("definition_id", type=click.STRING) @click.argument("resource_name", type=click.STRING) @click.pass_context +@requires_init def generate(ctx: click.Context, definition_type: str, definition_id: str, resource_name: str): - if not ctx.obj["PROJECT_IS_INITIALIZED"]: - raise ProjectNotInitializedError( - "Your octavia project is not initialized, please run 'octavia init' before running 'octavia generate'." - ) - definition = definitions.factory(definition_type, ctx.obj["API_CLIENT"], definition_id) renderer = ConnectionSpecificationRenderer(resource_name, definition) output_path = renderer.write_yaml(project_path=".") diff --git a/octavia-cli/octavia_cli/generate/renderer.py b/octavia-cli/octavia_cli/generate/renderer.py index 9fdcf2e21b1ef..324e92075eb4a 100644 --- a/octavia-cli/octavia_cli/generate/renderer.py +++ b/octavia-cli/octavia_cli/generate/renderer.py @@ -53,7 +53,7 @@ def __getattr__(self, name: str) -> Any: @property def is_array_of_objects(self) -> bool: if self.type == "array" and self.items: - if self.items["type"] == "object": + if self.items.get("type") == "object": return True return False @@ -146,7 +146,10 @@ def _parse_connection_specification(self, schema: dict) -> List[List["FieldToRen return [parse_fields(required_fields, schema["properties"])] def _get_output_path(self, project_path: str) -> str: - return os.path.join(project_path, f"{self.definition.type}s", f"{self.resource_name}.yaml") + directory = os.path.join(project_path, f"{self.definition.type}s", self.resource_name) + if not os.path.exists(directory): + os.makedirs(directory) + return os.path.join(directory, "configuration.yaml") def write_yaml(self, project_path: str) -> str: output_path = self._get_output_path(project_path) diff --git a/octavia-cli/setup.py b/octavia-cli/setup.py index dccb4f9ba5bcb..3337976377b06 100644 --- a/octavia-cli/setup.py +++ b/octavia-cli/setup.py @@ -40,15 +40,16 @@ "Source": "https://github.com/airbytehq/airbyte", "Tracker": "https://github.com/airbytehq/airbyte/issues", }, - packages=find_packages(exclude=("unit_tests", "docs")), + packages=find_packages(exclude=("unit_tests", "integration_tests", "docs")), install_requires=[ "click~=8.0.3", f"airbyte_api_client @ file://{os.getcwd()}/build/airbyte_api_client", "jinja2~=3.0.3", + "deepdiff~=5.7.0", ], python_requires=">=3.8.12", extras_require={ - "dev": ["MyPy~=0.812", "pytest~=6.2.5", "pytest-cov", "pytest-mock", "requests-mock", "pre-commit"], + "tests": ["MyPy~=0.812", "pytest~=6.2.5", "pytest-cov", "pytest-mock", "requests-mock", "pre-commit"], "sphinx-docs": [ "Sphinx~=4.2", "sphinx-rtd-theme~=1.0", diff --git a/octavia-cli/tests/unit/test_list/__init__.py b/octavia-cli/unit_tests/__init__.py similarity index 100% rename from octavia-cli/tests/unit/test_list/__init__.py rename to octavia-cli/unit_tests/__init__.py diff --git a/octavia-cli/unit_tests/conftest.py b/octavia-cli/unit_tests/conftest.py new file mode 100644 index 0000000000000..0e05d6b1c34a0 --- /dev/null +++ b/octavia-cli/unit_tests/conftest.py @@ -0,0 +1,10 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import pytest + + +@pytest.fixture +def mock_api_client(mocker): + return mocker.Mock() diff --git a/octavia-cli/unit_tests/test_apply/__init__.py b/octavia-cli/unit_tests/test_apply/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/octavia-cli/unit_tests/test_apply/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/octavia-cli/unit_tests/test_apply/test_commands.py b/octavia-cli/unit_tests/test_apply/test_commands.py new file mode 100644 index 0000000000000..434be4b7cad7a --- /dev/null +++ b/octavia-cli/unit_tests/test_apply/test_commands.py @@ -0,0 +1,294 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import pytest +from click.testing import CliRunner +from octavia_cli.apply import commands + + +@pytest.fixture +def patch_click(mocker): + mocker.patch.object(commands, "click") + + +@pytest.fixture +def context_object(mock_api_client): + return {"PROJECT_IS_INITIALIZED": True, "API_CLIENT": mock_api_client, "WORKSPACE_ID": "workspace_id"} + + +def test_apply_not_initialized(): + runner = CliRunner() + result = runner.invoke(commands.apply, obj={"PROJECT_IS_INITIALIZED": False}) + assert result.exit_code == 1 + + +def test_apply_without_custom_configuration_file(mocker, context_object): + runner = CliRunner() + local_files = ["foo", "bar"] + mocker.patch.object(commands, "find_local_configuration_files", mocker.Mock(return_value=local_files)) + mock_resources_to_apply = [mocker.Mock(), mocker.Mock()] + mocker.patch.object(commands, "get_resources_to_apply", mocker.Mock(return_value=mock_resources_to_apply)) + mocker.patch.object(commands, "apply_single_resource") + result = runner.invoke(commands.apply, obj=context_object) + assert result.exit_code == 0 + commands.find_local_configuration_files.assert_called_once() + commands.get_resources_to_apply.assert_called_once_with(local_files, context_object["API_CLIENT"], context_object["WORKSPACE_ID"]) + commands.apply_single_resource([mocker.call(r, False) for r in commands.get_resources_to_apply.return_value]) + + +def test_apply_with_custom_configuration_file(mocker, context_object): + runner = CliRunner() + mocker.patch.object(commands, "find_local_configuration_files") + mocker.patch.object(commands, "get_resources_to_apply") + mocker.patch.object(commands, "apply_single_resource") + result = runner.invoke(commands.apply, ["--file", "foo", "--file", "bar"], obj=context_object) + assert result.exit_code == 0 + commands.find_local_configuration_files.assert_not_called() + commands.get_resources_to_apply.assert_called_with(("foo", "bar"), context_object["API_CLIENT"], context_object["WORKSPACE_ID"]) + + +def test_apply_with_custom_configuration_file_force(mocker, context_object): + runner = CliRunner() + mocker.patch.object(commands, "find_local_configuration_files") + mocker.patch.object(commands, "get_resources_to_apply", mocker.Mock(return_value=[mocker.Mock()])) + mocker.patch.object(commands, "apply_single_resource") + result = runner.invoke(commands.apply, ["--file", "foo", "--file", "bar", "--force"], obj=context_object) + assert result.exit_code == 0 + commands.apply_single_resource.assert_called_with(commands.get_resources_to_apply.return_value[0], True) + + +def test_get_resource_to_apply(mocker, mock_api_client): + local_files_priorities = [("foo", 2), ("bar", 1)] + mock_resource_factory = mocker.Mock() + mock_resource_factory.side_effect = [mocker.Mock(apply_priority=priority) for _, priority in local_files_priorities] + mocker.patch.object(commands, "resource_factory", mock_resource_factory) + + resources_to_apply = commands.get_resources_to_apply([f[0] for f in local_files_priorities], mock_api_client, "workspace_id") + assert resources_to_apply == sorted(resources_to_apply, key=lambda r: r.apply_priority) + assert commands.resource_factory.call_count == len(local_files_priorities) + commands.resource_factory.assert_has_calls([mocker.call(mock_api_client, "workspace_id", path) for path, _ in local_files_priorities]) + + +@pytest.mark.parametrize("resource_was_created", [True, False]) +def test_apply_single_resource(patch_click, mocker, resource_was_created): + mocker.patch.object(commands, "update_resource", mocker.Mock(return_value=["updated"])) + mocker.patch.object(commands, "create_resource", mocker.Mock(return_value=["created"])) + resource = mocker.Mock(was_created=resource_was_created, resource_name="my_resource_name") + force = mocker.Mock() + commands.apply_single_resource(resource, force) + if resource_was_created: + commands.update_resource.assert_called_once_with(resource, force) + commands.create_resource.assert_not_called() + expected_message = "🐙 - my_resource_name exists on your Airbyte instance, let's check if we need to update it!" + expected_message_color = "yellow" + expected_echo_calls = [mocker.call(commands.click.style.return_value), mocker.call("\n".join(["updated"]))] + else: + commands.update_resource.assert_not_called() + commands.create_resource.assert_called_once_with(resource) + expected_message = "🐙 - my_resource_name does not exists on your Airbyte instance, let's create it!" + expected_message_color = "green" + expected_echo_calls = [mocker.call(commands.click.style.return_value), mocker.call("\n".join(["created"]))] + commands.click.style.assert_called_with(expected_message, fg=expected_message_color) + commands.click.echo.assert_has_calls(expected_echo_calls) + + +@pytest.mark.parametrize( + "diff,local_file_changed,force,expected_should_update,expected_update_reason", + [ + (True, True, True, True, "🚨 - Running update because force mode is on."), # check if force has the top priority + (True, False, True, True, "🚨 - Running update because force mode is on."), # check if force has the top priority + (True, False, False, True, "🚨 - Running update because force mode is on."), # check if force has the top priority + (True, True, False, True, "🚨 - Running update because force mode is on."), # check if force has the top priority + ( + False, + True, + True, + True, + "✍️ - Running update because a diff was detected between local and remote resource.", + ), # check if diff has priority of file changed + ( + False, + True, + False, + True, + "✍️ - Running update because a diff was detected between local and remote resource.", + ), # check if diff has priority of file changed + ( + False, + False, + True, + True, + "✍️ - Running update because a local file change was detected and a secret field might have been edited.", + ), # check if local_file_changed runs even if no diff found + ( + False, + False, + False, + False, + "😴 - Did not update because no change detected.", + ), # check if local_file_changed runs even if no diff found + ], +) +def test_should_update_resource(patch_click, mocker, diff, local_file_changed, force, expected_should_update, expected_update_reason): + should_update, update_reason = commands.should_update_resource(diff, local_file_changed, force) + assert should_update == expected_should_update + assert update_reason == commands.click.style.return_value + commands.click.style.assert_called_with(expected_update_reason, fg="green") + + +@pytest.mark.parametrize( + "diff,expected_number_calls_to_display_diff_line", + [("", 0), ("First diff line", 1), ("First diff line\nSecond diff line", 2), ("First diff line\nSecond diff line\nThird diff line", 3)], +) +def test_prompt_for_diff_validation(patch_click, mocker, diff, expected_number_calls_to_display_diff_line): + mocker.patch.object(commands, "display_diff_line") + output = commands.prompt_for_diff_validation("my_resource", diff) + assert commands.display_diff_line.call_count == expected_number_calls_to_display_diff_line + if diff and expected_number_calls_to_display_diff_line > 0: + commands.display_diff_line.assert_has_calls([mocker.call(line) for line in diff.split("\n")]) + commands.click.style.assert_has_calls( + [ + mocker.call( + "👀 - Here's the computed diff (🚨 remind that diff on secret fields are not displayed):", fg="magenta", bold=True + ), + mocker.call("❓ - Do you want to update my_resource?", bold=True), + ] + ) + commands.click.echo.assert_called_with(commands.click.style.return_value) + assert output == commands.click.confirm.return_value + else: + assert output is False + + +def test_create_resource(patch_click, mocker): + mock_created_resource = mocker.Mock() + mock_state = mocker.Mock() + mock_resource = mocker.Mock(create=mocker.Mock(return_value=(mock_created_resource, mock_state))) + output_messages = commands.create_resource(mock_resource) + mock_resource.create.assert_called_once() + assert output_messages == [commands.click.style.return_value, commands.click.style.return_value] + commands.click.style.assert_has_calls( + [ + mocker.call(f"🎉 - Successfully created {mock_created_resource.name} on your Airbyte instance!", fg="green", bold=True), + mocker.call(f"💾 - New state for {mock_created_resource.name} saved at {mock_state.path}", fg="yellow"), + ] + ) + + +@pytest.mark.parametrize( + "force,diff,should_update_resource,expect_prompt,user_validate_diff,expect_update,expected_number_of_messages", + [ + (True, True, True, False, False, True, 3), # Force is on, we have a diff, prompt should not be displayed: we expect update. + ( + True, + False, + True, + False, + False, + True, + 3, + ), # Force is on, no diff, should_update_resource == true, prompt should not be displayed, we expect update. + ( + True, + False, + False, + False, + False, + False, + 1, + ), # Force is on, no diff, should_update_resource == false, prompt should not be displayed, we don't expect update. This scenario should not exists in current implementation as force always trigger update. + ( + False, + True, + True, + True, + True, + True, + 3, + ), # Force is off, we have diff, prompt should be displayed, user validate diff: we expected update. + ( + False, + False, + True, + False, + False, + True, + 3, + ), # Force is off, no diff, should_update_resource == true (in case of file change), prompt should not be displayed, we expect update. + ( + False, + True, + True, + True, + False, + False, + 1, + ), # Force is off, we have a diff but the user does not validate it: we don't expect update. + ( + False, + False, + False, + False, + False, + False, + 1, + ), # Force is off, we have a no diff, should_update_resource == false: we don't expect update. + ], +) +def test_update_resource( + patch_click, mocker, force, diff, should_update_resource, user_validate_diff, expect_prompt, expect_update, expected_number_of_messages +): + mock_updated_resource = mocker.Mock() + mock_state = mocker.Mock() + mock_resource = mocker.Mock( + get_diff_with_remote_resource=mocker.Mock(return_value=diff), + resource_name="my_resource", + update=mocker.Mock(return_value=(mock_updated_resource, mock_state)), + ) + update_reason = "foo" + mocker.patch.object(commands, "should_update_resource", mocker.Mock(return_value=(should_update_resource, update_reason))) + mocker.patch.object(commands, "prompt_for_diff_validation", mocker.Mock(return_value=user_validate_diff)) + + output_messages = commands.update_resource(mock_resource, force) + if expect_prompt: + commands.prompt_for_diff_validation.assert_called_once_with("my_resource", diff) + else: + commands.prompt_for_diff_validation.assert_not_called() + if expect_update: + mock_resource.update.assert_called_once() + else: + mock_resource.update.assert_not_called() + assert output_messages[0] == commands.should_update_resource.return_value[1] == update_reason + assert len(output_messages) == expected_number_of_messages + if expected_number_of_messages == 3: + assert output_messages == [ + commands.should_update_resource.return_value[1], + commands.click.style.return_value, + commands.click.style.return_value, + ] + commands.click.style.assert_has_calls( + [ + mocker.call(f"🎉 - Successfully updated {mock_updated_resource.name} on your Airbyte instance!", fg="green", bold=True), + mocker.call(f"💾 - New state for {mock_updated_resource.name} stored at {mock_state.path}.", fg="yellow"), + ] + ) + + +def test_find_local_configuration_files(mocker): + project_directories = ["sources", "connections", "destinations"] + mocker.patch.object(commands, "REQUIRED_PROJECT_DIRECTORIES", project_directories) + mocker.patch.object(commands, "glob", mocker.Mock(return_value=["foo.yaml"])) + configuration_files = commands.find_local_configuration_files() + assert isinstance(configuration_files, list) + commands.glob.assert_has_calls([mocker.call(f"./{directory}/**/configuration.yaml") for directory in project_directories]) + assert configuration_files == ["foo.yaml" for _ in range(len(project_directories))] + + +def test_find_local_configuration_files_no_file_found(patch_click, mocker): + project_directories = ["sources", "connections", "destinations"] + mocker.patch.object(commands, "REQUIRED_PROJECT_DIRECTORIES", project_directories) + mocker.patch.object(commands, "glob", mocker.Mock(return_value=[])) + configuration_files = commands.find_local_configuration_files() + assert not configuration_files + commands.click.style.assert_called_once_with("😒 - No YAML file found to run apply.", fg="red") diff --git a/octavia-cli/unit_tests/test_apply/test_diff_helpers.py b/octavia-cli/unit_tests/test_apply/test_diff_helpers.py new file mode 100644 index 0000000000000..02e3b0c44483c --- /dev/null +++ b/octavia-cli/unit_tests/test_apply/test_diff_helpers.py @@ -0,0 +1,50 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import mock_open, patch + +import pytest +from octavia_cli.apply import diff_helpers + + +def test_compute_checksum(mocker): + with patch("builtins.open", mock_open(read_data=b"data")) as mock_file: + digest = diff_helpers.compute_checksum("test_file_path") + assert digest == "3a6eb0790f39ac87c94f3856b2dd2c5d110e6811602261a9a923d3bb23adc8b7" + mock_file.assert_called_with("test_file_path", "rb") + + +@pytest.mark.parametrize( + "obj, expected_output", + [ + (diff_helpers.SECRET_MASK, True), + ("not secret", False), + ({}, False), + ], +) +def test_exclude_secrets_from_diff(obj, expected_output): + assert diff_helpers.exclude_secrets_from_diff(obj, "foo") == expected_output + + +def test_compute_diff(mocker): + mocker.patch.object(diff_helpers, "DeepDiff") + diff = diff_helpers.compute_diff("foo", "bar") + assert diff == diff_helpers.DeepDiff.return_value + diff_helpers.DeepDiff.assert_called_with("foo", "bar", view="tree", exclude_obj_callback=diff_helpers.exclude_secrets_from_diff) + + +@pytest.mark.parametrize( + "diff_line,expected_message,expected_color", + [ + ("resource changed from", "E - resource changed from", "yellow"), + ("resource added", "+ - resource added", "green"), + ("resource removed", "- - resource removed", "red"), + ("whatever", " - whatever", None), + ], +) +def test_display_diff_line(mocker, diff_line, expected_message, expected_color): + mocker.patch.object(diff_helpers, "click") + diff_helpers.display_diff_line(diff_line) + diff_helpers.click.style.assert_called_with(f"\t{expected_message}", fg=expected_color) + diff_helpers.click.echo.assert_called_with(diff_helpers.click.style.return_value) diff --git a/octavia-cli/unit_tests/test_apply/test_resources.py b/octavia-cli/unit_tests/test_apply/test_resources.py new file mode 100644 index 0000000000000..f4702715c9976 --- /dev/null +++ b/octavia-cli/unit_tests/test_apply/test_resources.py @@ -0,0 +1,323 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import mock_open, patch + +import pytest +from airbyte_api_client import ApiException +from octavia_cli.apply import resources + + +class TestResourceState: + def test_init(self, mocker): + mocker.patch.object(resources, "os") + state = resources.ResourceState("config_path", "resource_id", 123, "config_checksum") + assert state.configuration_path == "config_path" + assert state.resource_id == "resource_id" + assert state.generation_timestamp == 123 + assert state.configuration_checksum == "config_checksum" + assert state.path == resources.os.path.join.return_value + resources.os.path.dirname.assert_called_with("config_path") + resources.os.path.join.assert_called_with(resources.os.path.dirname.return_value, "state.yaml") + + @pytest.fixture + def state(self): + return resources.ResourceState("config_path", "resource_id", 123, "config_checksum") + + def test_as_dict(self, state): + assert state.as_dict() == { + "configuration_path": state.configuration_path, + "resource_id": state.resource_id, + "generation_timestamp": state.generation_timestamp, + "configuration_checksum": state.configuration_checksum, + } + + def test_save(self, mocker, state): + mocker.patch.object(resources, "yaml") + mocker.patch.object(state, "as_dict") + + expected_content = state.as_dict.return_value + with patch("builtins.open", mock_open()) as mock_file: + state._save() + mock_file.assert_called_with(state.path, "w") + resources.yaml.dump.assert_called_with(expected_content, mock_file.return_value) + + def test_create(self, mocker): + mocker.patch.object(resources.time, "time", mocker.Mock(return_value=0)) + mocker.patch.object(resources, "compute_checksum", mocker.Mock(return_value="my_checksum")) + mocker.patch.object(resources.ResourceState, "_save") + state = resources.ResourceState.create("config_path", "resource_id") + assert isinstance(state, resources.ResourceState) + resources.ResourceState._save.assert_called_once() + assert state.configuration_path == "config_path" + assert state.resource_id == "resource_id" + assert state.generation_timestamp == 0 + assert state.configuration_checksum == "my_checksum" + + def test_from_file(self, mocker): + mocker.patch.object(resources, "yaml") + resources.yaml.load.return_value = { + "configuration_path": "config_path", + "resource_id": "resource_id", + "generation_timestamp": 0, + "configuration_checksum": "my_checksum", + } + with patch("builtins.open", mock_open(read_data="data")) as mock_file: + state = resources.ResourceState.from_file("state.yaml") + resources.yaml.load.assert_called_with(mock_file.return_value, resources.yaml.FullLoader) + assert isinstance(state, resources.ResourceState) + assert state.configuration_path == "config_path" + assert state.resource_id == "resource_id" + assert state.generation_timestamp == 0 + assert state.configuration_checksum == "my_checksum" + + +@pytest.fixture +def local_configuration(): + return {"exotic_attribute": "foo", "configuration": {"foo": "bar"}, "resource_name": "bar", "definition_id": "bar"} + + +class TestBaseResource: + @pytest.fixture + def patch_base_class(self, mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(resources.BaseResource, "__abstractmethods__", set()) + mocker.patch.object(resources.BaseResource, "create_function_name", "create_resource") + mocker.patch.object(resources.BaseResource, "resource_id_field", "resource_id") + mocker.patch.object(resources.BaseResource, "search_function_name", "search_resource") + mocker.patch.object(resources.BaseResource, "update_function_name", "update_resource") + mocker.patch.object(resources.BaseResource, "resource_type", "universal_resource") + mocker.patch.object(resources.BaseResource, "api") + + def test_init_no_remote_resource(self, mocker, patch_base_class, mock_api_client, local_configuration): + mocker.patch.object(resources.BaseResource, "_get_state_from_file", mocker.Mock(return_value=None)) + mocker.patch.object(resources.BaseResource, "_get_remote_resource", mocker.Mock(return_value=False)) + mocker.patch.object(resources, "compute_checksum") + resource = resources.BaseResource(mock_api_client, "workspace_id", local_configuration, "bar.yaml") + assert resource.workspace_id == "workspace_id" + assert resource.local_configuration == local_configuration + assert resource.configuration_path == "bar.yaml" + assert resource.api_instance == resource.api.return_value + resource.api.assert_called_with(mock_api_client) + assert resource.state == resource._get_state_from_file.return_value + assert resource.remote_resource == resource._get_remote_resource.return_value + assert resource.was_created == resource._get_remote_resource.return_value + assert resource.local_file_changed is True + assert resource.resource_id is None + + def test_init_with_remote_resource_not_changed(self, mocker, patch_base_class, mock_api_client, local_configuration): + mocker.patch.object( + resources.BaseResource, "_get_state_from_file", mocker.Mock(return_value=mocker.Mock(configuration_checksum="my_checksum")) + ) + mocker.patch.object(resources.BaseResource, "_get_remote_resource", mocker.Mock(return_value={"resource_id": "my_resource_id"})) + + mocker.patch.object(resources, "compute_checksum", mocker.Mock(return_value="my_checksum")) + resource = resources.BaseResource(mock_api_client, "workspace_id", local_configuration, "bar.yaml") + assert resource.was_created is True + assert resource.local_file_changed is False + assert resource.resource_id == "my_resource_id" + + def test_init_with_remote_resource_changed(self, mocker, patch_base_class, mock_api_client, local_configuration): + mocker.patch.object( + resources.BaseResource, + "_get_state_from_file", + mocker.Mock(return_value=mocker.Mock(configuration_checksum="my_state_checksum")), + ) + mocker.patch.object(resources.BaseResource, "_get_remote_resource", mocker.Mock(return_value={"resource_id": "my_resource_id"})) + mocker.patch.object(resources, "compute_checksum", mocker.Mock(return_value="my_new_checksum")) + resource = resources.BaseResource(mock_api_client, "workspace_id", local_configuration, "bar.yaml") + assert resource.was_created is True + assert resource.local_file_changed is True + assert resource.resource_id == "my_resource_id" + + @pytest.fixture + def resource(self, patch_base_class, mock_api_client, local_configuration): + return resources.BaseResource(mock_api_client, "workspace_id", local_configuration, "bar.yaml") + + def test_get_attr(self, resource, local_configuration): + assert resource.exotic_attribute == local_configuration["exotic_attribute"] + with pytest.raises(AttributeError): + resource.wrong_attribute + + def test_search(self, resource): + search_results = resource._search() + assert search_results == resource._search_fn.return_value + resource._search_fn.assert_called_with(resource.api_instance, resource.search_payload) + + @pytest.mark.parametrize( + "search_results,expected_error,expected_output", + [ + ([], None, None), + (["foo"], None, "foo"), + (["foo", "bar"], resources.DuplicateResourceError, None), + ], + ) + def test_get_remote_resource(self, resource, mocker, search_results, expected_error, expected_output): + mock_search_results = mocker.Mock(return_value=search_results) + mocker.patch.object(resource, "_search", mocker.Mock(return_value=mocker.Mock(get=mock_search_results))) + if expected_error is None: + remote_resource = resource._get_remote_resource() + assert remote_resource == expected_output + else: + with pytest.raises(expected_error): + remote_resource = resource._get_remote_resource() + resource._search.return_value.get.assert_called_with("universal_resources", []) + + @pytest.mark.parametrize( + "state_path_is_file", + [True, False], + ) + def test_get_state_from_file(self, mocker, resource, state_path_is_file): + mocker.patch.object(resources, "os") + mock_expected_state_path = mocker.Mock(is_file=mocker.Mock(return_value=state_path_is_file)) + mocker.patch.object(resources, "Path", mocker.Mock(return_value=mock_expected_state_path)) + mocker.patch.object(resources, "ResourceState") + state = resource._get_state_from_file() + resources.os.path.dirname.assert_called_with(resource.configuration_path) + resources.os.path.join.assert_called_with(resources.os.path.dirname.return_value, "state.yaml") + resources.Path.assert_called_with(resources.os.path.join.return_value) + if state_path_is_file: + resources.ResourceState.from_file.assert_called_with(mock_expected_state_path) + assert state == resources.ResourceState.from_file.return_value + else: + assert state is None + + @pytest.mark.parametrize( + "was_created", + [True, False], + ) + def test_get_diff_with_remote_resource(self, mocker, resource, was_created): + + mock_remote_resource = mocker.Mock(return_value=None if not was_created else mocker.Mock()) + mocker.patch.object(resource, "_get_remote_resource", mock_remote_resource) + mocker.patch.object(resources, "compute_diff") + if was_created: + diff = resource.get_diff_with_remote_resource() + resources.compute_diff.assert_called_with(resource.remote_resource.connection_configuration, resource.configuration) + assert diff == resources.compute_diff.return_value.pretty.return_value + else: + with pytest.raises(resources.NonExistingResourceError): + resource.get_diff_with_remote_resource() + + def test_create_or_update(self, mocker, resource): + expected_results = {resource.resource_id_field: "resource_id"} + operation_fn = mocker.Mock(return_value=expected_results) + mocker.patch.object(resources, "ResourceState") + payload = "foo" + result, state = resource._create_or_update(operation_fn, payload) + assert result == expected_results + assert state == resources.ResourceState.create.return_value + resources.ResourceState.create.assert_called_with(resource.configuration_path, "resource_id") + + @pytest.mark.parametrize( + "response_status,expected_error", + [(404, ApiException), (422, resources.InvalidConfigurationError)], + ) + def test_create_or_update_error(self, mocker, resource, response_status, expected_error): + operation_fn = mocker.Mock(side_effect=ApiException(status=response_status)) + mocker.patch.object(resources, "ResourceState") + with pytest.raises(expected_error): + resource._create_or_update(operation_fn, "foo") + + def test_create(self, mocker, resource): + mocker.patch.object(resource, "_create_or_update") + assert resource.create() == resource._create_or_update.return_value + resource._create_or_update.assert_called_with(resource._create_fn, resource.create_payload) + + def test_update(self, mocker, resource): + mocker.patch.object(resource, "_create_or_update") + assert resource.update() == resource._create_or_update.return_value + resource._create_or_update.assert_called_with(resource._update_fn, resource.update_payload) + + +class TestSource: + @pytest.mark.parametrize( + "state", + [None, resources.ResourceState("config_path", "resource_id", 123, "abc")], + ) + def test_init(self, mocker, mock_api_client, local_configuration, state): + assert resources.Source.__base__ == resources.BaseResource + mocker.patch.object(resources.Source, "resource_id", "foo") + source = resources.Source(mock_api_client, "workspace_id", local_configuration, "bar.yaml") + mocker.patch.object(source, "state", state) + assert source.api == resources.source_api.SourceApi + assert source.create_function_name == "create_source" + assert source.resource_id_field == "source_id" + assert source.search_function_name == "search_sources" + assert source.update_function_name == "update_source" + assert source.resource_type == "source" + assert source.apply_priority == 0 + assert source.create_payload == resources.SourceCreate( + source.definition_id, source.configuration, source.workspace_id, source.resource_name + ) + assert source.update_payload == resources.SourceUpdate( + source_id=source.resource_id, connection_configuration=source.configuration, name=source.resource_name + ) + if state is None: + assert source.search_payload == resources.SourceSearch( + source_definition_id=source.definition_id, workspace_id=source.workspace_id, name=source.resource_name + ) + else: + assert source.search_payload == resources.SourceSearch( + source_definition_id=source.definition_id, workspace_id=source.workspace_id, source_id=source.state.resource_id + ) + + +class TestDestination: + @pytest.mark.parametrize( + "state", + [None, resources.ResourceState("config_path", "resource_id", 123, "abc")], + ) + def test_init(self, mocker, mock_api_client, local_configuration, state): + assert resources.Destination.__base__ == resources.BaseResource + mocker.patch.object(resources.Destination, "resource_id", "foo") + destination = resources.Destination(mock_api_client, "workspace_id", local_configuration, "bar.yaml") + mocker.patch.object(destination, "state", state) + assert destination.api == resources.destination_api.DestinationApi + assert destination.create_function_name == "create_destination" + assert destination.resource_id_field == "destination_id" + assert destination.search_function_name == "search_destinations" + assert destination.update_function_name == "update_destination" + assert destination.resource_type == "destination" + assert destination.apply_priority == 0 + assert destination.create_payload == resources.DestinationCreate( + destination.workspace_id, destination.resource_name, destination.definition_id, destination.configuration + ) + assert destination.update_payload == resources.DestinationUpdate( + destination_id=destination.resource_id, connection_configuration=destination.configuration, name=destination.resource_name + ) + if state is None: + assert destination.search_payload == resources.DestinationSearch( + destination_definition_id=destination.definition_id, workspace_id=destination.workspace_id, name=destination.resource_name + ) + else: + assert destination.search_payload == resources.DestinationSearch( + destination_definition_id=destination.definition_id, + workspace_id=destination.workspace_id, + destination_id=destination.state.resource_id, + ) + + +@pytest.mark.parametrize( + "local_configuration,resource_to_mock,expected_error", + [ + ({"definition_type": "source"}, "Source", None), + ({"definition_type": "destination"}, "Destination", None), + ({"definition_type": "connection"}, None, NotImplementedError), + ], +) +def test_factory(mocker, mock_api_client, local_configuration, resource_to_mock, expected_error): + mocker.patch.object(resources, "yaml") + if resource_to_mock is not None: + mocker.patch.object(resources, resource_to_mock) + resources.yaml.load.return_value = local_configuration + with patch("builtins.open", mock_open(read_data="data")) as mock_file: + if not expected_error: + resource = resources.factory(mock_api_client, "workspace_id", "my_config.yaml") + resource == getattr(resources, resource_to_mock).return_value + mock_file.assert_called_with("my_config.yaml", "r") + else: + with pytest.raises(expected_error): + resources.factory(mock_api_client, "workspace_id", "my_config.yaml") + mock_file.assert_called_with("my_config.yaml", "r") diff --git a/octavia-cli/tests/unit/test_check_context.py b/octavia-cli/unit_tests/test_check_context.py similarity index 100% rename from octavia-cli/tests/unit/test_check_context.py rename to octavia-cli/unit_tests/test_check_context.py diff --git a/octavia-cli/tests/unit/test_entrypoint.py b/octavia-cli/unit_tests/test_entrypoint.py similarity index 98% rename from octavia-cli/tests/unit/test_entrypoint.py rename to octavia-cli/unit_tests/test_entrypoint.py index 5b6ce3bb5f276..f17c6700d092a 100644 --- a/octavia-cli/tests/unit/test_entrypoint.py +++ b/octavia-cli/unit_tests/test_entrypoint.py @@ -87,7 +87,7 @@ def test_commands_in_octavia_group(): @pytest.mark.parametrize( "command", - [entrypoint.apply, entrypoint.delete, entrypoint._import], + [entrypoint.delete, entrypoint._import], ) def test_not_implemented_commands(command): runner = CliRunner() @@ -101,4 +101,5 @@ def test_available_commands(): entrypoint.list_commands._list, entrypoint.init_commands.init, entrypoint.generate_commands.generate, + entrypoint.apply_commands.apply, ] diff --git a/octavia-cli/unit_tests/test_generate/__init__.py b/octavia-cli/unit_tests/test_generate/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/octavia-cli/unit_tests/test_generate/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/octavia-cli/tests/unit/test_generate/test_commands.py b/octavia-cli/unit_tests/test_generate/test_commands.py similarity index 95% rename from octavia-cli/tests/unit/test_generate/test_commands.py rename to octavia-cli/unit_tests/test_generate/test_commands.py index 6d1b9a178b828..7985a505768bc 100644 --- a/octavia-cli/tests/unit/test_generate/test_commands.py +++ b/octavia-cli/unit_tests/test_generate/test_commands.py @@ -26,7 +26,7 @@ def test_generate_not_initialized(): context_object = {"PROJECT_IS_INITIALIZED": False} result = runner.invoke(commands.generate, ["source", "uuid", "my_source"], obj=context_object) assert result.exit_code == 1 - assert result.output == "Error: Your octavia project is not initialized, please run 'octavia init' before running 'octavia generate'.\n" + assert result.output == "Error: Your octavia project is not initialized, please run 'octavia init' before running this command.\n" def test_invalid_definition_type(): diff --git a/octavia-cli/tests/unit/test_generate/test_definitions.py b/octavia-cli/unit_tests/test_generate/test_definitions.py similarity index 100% rename from octavia-cli/tests/unit/test_generate/test_definitions.py rename to octavia-cli/unit_tests/test_generate/test_definitions.py diff --git a/octavia-cli/tests/unit/test_generate/test_renderer.py b/octavia-cli/unit_tests/test_generate/test_renderer.py similarity index 91% rename from octavia-cli/tests/unit/test_generate/test_renderer.py rename to octavia-cli/unit_tests/test_generate/test_renderer.py index 728e4c9a1282f..902561cc6d56a 100644 --- a/octavia-cli/tests/unit/test_generate/test_renderer.py +++ b/octavia-cli/unit_tests/test_generate/test_renderer.py @@ -2,6 +2,8 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +from unittest.mock import mock_open, patch + import pytest from octavia_cli.generate import renderer @@ -182,10 +184,22 @@ def test__parse_connection_specification_one_of(self, mocker): renderer.parse_fields.assert_called_with(["free"], {"free": "beer"}) def test__get_output_path(self, mocker): + mocker.patch.object(renderer, "os") + renderer.os.path.exists.return_value = False spec_renderer = renderer.ConnectionSpecificationRenderer("my_resource_name", mocker.Mock(type="source")) - assert spec_renderer._get_output_path(".") == "./sources/my_resource_name.yaml" + renderer.os.path.join.side_effect = ["./source/my_resource_name", "./source/my_resource_name/configuration.yaml"] + output_path = spec_renderer._get_output_path(".") + renderer.os.makedirs.assert_called_once() + renderer.os.path.join.assert_has_calls( + [ + mocker.call(".", "sources", "my_resource_name"), + mocker.call("./source/my_resource_name", "configuration.yaml"), + ] + ) + assert output_path == "./source/my_resource_name/configuration.yaml" def test_write_yaml(self, mocker): + mocker.patch.object(renderer.ConnectionSpecificationRenderer, "_get_output_path") mocker.patch.object(renderer.ConnectionSpecificationRenderer, "_parse_connection_specification") mocker.patch.object( @@ -193,7 +207,8 @@ def test_write_yaml(self, mocker): ) spec_renderer = renderer.ConnectionSpecificationRenderer("my_resource_name", mocker.Mock(type="source")) - output_path = spec_renderer.write_yaml(".") + with patch("builtins.open", mock_open()) as mock_file: + output_path = spec_renderer.write_yaml(".") assert output_path == spec_renderer._get_output_path.return_value spec_renderer.TEMPLATE.render.assert_called_with( { @@ -202,3 +217,4 @@ def test_write_yaml(self, mocker): "configuration_fields": spec_renderer._parse_connection_specification.return_value, } ) + mock_file.assert_called_with(output_path, "w") diff --git a/octavia-cli/unit_tests/test_init/__init__.py b/octavia-cli/unit_tests/test_init/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/octavia-cli/unit_tests/test_init/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/octavia-cli/tests/unit/test_init/test_commands.py b/octavia-cli/unit_tests/test_init/test_commands.py similarity index 100% rename from octavia-cli/tests/unit/test_init/test_commands.py rename to octavia-cli/unit_tests/test_init/test_commands.py diff --git a/octavia-cli/unit_tests/test_list/__init__.py b/octavia-cli/unit_tests/test_list/__init__.py new file mode 100644 index 0000000000000..46b7376756ec6 --- /dev/null +++ b/octavia-cli/unit_tests/test_list/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/octavia-cli/tests/unit/test_list/test_commands.py b/octavia-cli/unit_tests/test_list/test_commands.py similarity index 100% rename from octavia-cli/tests/unit/test_list/test_commands.py rename to octavia-cli/unit_tests/test_list/test_commands.py diff --git a/octavia-cli/tests/unit/test_list/test_formatting.py b/octavia-cli/unit_tests/test_list/test_formatting.py similarity index 100% rename from octavia-cli/tests/unit/test_list/test_formatting.py rename to octavia-cli/unit_tests/test_list/test_formatting.py diff --git a/octavia-cli/tests/unit/test_list/test_listings.py b/octavia-cli/unit_tests/test_list/test_listings.py similarity index 100% rename from octavia-cli/tests/unit/test_list/test_listings.py rename to octavia-cli/unit_tests/test_list/test_listings.py