octavia-cli: implement apply command

Author: alafanechere

octavia-cli/octavia_cli/apply/commands.py

@@ -0,0 +1,183 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+from glob import glob
+from typing import List, Tuple
+
+import airbyte_api_client
+import click
+from octavia_cli.check_context import REQUIRED_PROJECT_DIRECTORIES, requires_init
+
+from .resources import BaseResource
+from .resources import factory as resource_factory
+
+
+@click.command(name="apply", help="Create an Airbyte resources from a YAML definition")
+@click.option("--file", "-f", "configurations_files", type=click.Path(), multiple=True)
+@click.option("--force", is_flag=True, default=False, help="Does not display the diff and updates without user prompt.")
+@click.pass_context
+@requires_init
+def apply(ctx: click.Context, configurations_files: List[click.Path], force: bool):
+    if not configurations_files:
+        configurations_files = find_local_configuration_files()
+
+    resources = get_resources_to_apply(configurations_files, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"])
+    for resource in resources:
+        apply_single_resource(resource, force)
+
+
+def get_resources_to_apply(
+    configuration_files: List[str], api_client: airbyte_api_client.ApiClient, workspace_id: str
+) -> List[BaseResource]:
+    """Create resource objects with factory and sort according to apply priority.
+
+    Args:
+        configuration_files (List[str]): List of YAML configuration files.
+        api_client (airbyte_api_client.ApiClient): the Airbyte API client.
+        workspace_id (str): current Airbyte workspace id.
+
+    Returns:
+        List[BaseResource]: Resources sorted according to their apply priority.
+    """
+    all_resources = [resource_factory(api_client, workspace_id, path) for path in configuration_files]
+    return sorted(all_resources, key=lambda resource: resource.apply_priority)
+
+
+def apply_single_resource(resource: BaseResource, force: bool) -> None:
+    """Runs resource creation if it was not created, update it otherwise.
+
+    Args:
+        resource (BaseResource): The resource to apply.
+        force (bool): Wetheir force mode is on.
+    """
+    if resource.was_created:
+        click.echo(
+            click.style(f"🐙 - {resource.resource_name} exists on your Airbyte instance, let's check if we need to update it!", fg="yellow")
+        )
+        messages = update_resource(resource, force)
+    else:
+        click.echo(click.style(f"🐙 - {resource.resource_name} does not exists on your Airbyte instance, let's create it!", fg="green"))
+        messages = create_resource(resource)
+    click.echo("\n".join(messages))
+
+
+def should_update_resource(force: bool, diff: str, local_file_changed: bool) -> Tuple[bool, str]:
+    """Function to decide if the resource needs an update or not.
+
+    Args:
+        force (bool): Wetheir force mode is on.
+        diff (str): The computed diff between local and remote for this resource.
+        local_file_changed (bool): Wetheir the local file describing the resource was modified.
+
+    Returns:
+        Tuple[bool, str]: Boolean to know if resource should be updated and string describing the update reason.
+    """
+    if force:
+        should_update, update_reason = True, "🚨 - Running update because force mode is on."
+    elif diff:
+        should_update, update_reason = True, "✍️ - Running update because a diff was detected between local and remote resource."
+    elif local_file_changed:
+        should_update, update_reason = (
+            True,
+            "✍️ - Running update because a local file change was detected and a secret field might have been edited.",
+        )
+    else:
+        should_update, update_reason = False, "😴 - Did not update because no change detected."
+    return should_update, click.style(update_reason, fg="green")
+
+
+def display_diff_line(diff_line: str) -> None:
+    """Prettify a diff line and print it to standard output.
+
+    Args:
+        diff_line (str): The diff line to display.
+    """
+    if "changed from" in diff_line:
+        color = "yellow"
+        prefix = "E"
+    elif "added" in diff_line:
+        color = "green"
+        prefix = "+"
+    elif "removed" in diff_line:
+        color = "red"
+        prefix = "-"
+    else:
+        prefix = ""
+        color = None
+    click.echo(click.style(f"\t{prefix} - {diff_line}", fg=color))
+
+
+def prompt_for_diff_validation(resource_name: str, diff: str) -> bool:
+    """Display the diff to user and prompt them from validation.
+
+    Args:
+        resource_name (str): Name of the resource the diff was computed for.
+        diff (str): The diff.
+
+    Returns:
+        bool: Wetheir user validated the diff.
+    """
+    if diff:
+        click.echo(
+            click.style("👀 - Here's the computed diff (🚨 remind that diff on secret fields are not displayed):", fg="magenta", bold=True)
+        )
+        for line in diff.split("\n"):
+            display_diff_line(line)
+        return click.confirm(click.style(f"❓ - Do you want to update {resource_name}?", bold=True))
+    else:
+        return False
+
+
+def create_resource(resource: BaseResource) -> List[str]:
+    """Run a resource creation.
+
+    Args:
+        resource (BaseResource): The resource to create.
+
+    Returns:
+        List[str]: Post create messages to display to standard output.
+    """
+    created_resource, state = resource.create()
+    return [
+        click.style(f"🎉 - Successfully created {created_resource.name} on your Airbyte instance!", fg="green", bold=True),
+        click.style(f"💾 - New state for {created_resource.name} saved at {state.path}", fg="yellow"),
+    ]
+
+
+def update_resource(resource: BaseResource, force: bool) -> List[str]:
+    """Run a resource update. Check if update is required and prompt for user diff validation if needed.
+
+    Args:
+        resource (BaseResource): Resource to update
+        force (bool): Wetheir force mode is on.
+
+    Returns:
+        List[str]: Post update messages to display to standard output.
+    """
+    diff = resource.get_diff_with_remote_resource()
+    should_update, update_reason = should_update_resource(force, diff, resource.local_file_changed)
+    output_messages = [update_reason]
+    if not force and diff:
+        should_update = prompt_for_diff_validation(resource.resource_name, diff)
+    if should_update:
+        updated_resource, state = resource.update()
+        output_messages.append(
+            click.style(f"🎉 - Successfully updated {updated_resource.name} on your Airbyte instance!", fg="green", bold=True)
+        )
+        output_messages.append(click.style(f"💾 - New state for {updated_resource.name} stored at {state.path}.", fg="yellow"))
+    return output_messages
+
+
+def find_local_configuration_files() -> List[str]:
+    """Discover local configuration files.
+
+    Returns:
+        List[str]: Paths to YAML configuration files.
+    """
+    configuration_files = []
+    for resource_directory in REQUIRED_PROJECT_DIRECTORIES:
+        configuration_files += glob(f"./{resource_directory}/**/configuration.yaml")
+    if not configuration_files:
+        click.echo(click.style("😒 - No YAML file found to run apply.", fg="red"))
+    return configuration_files

octavia-cli/octavia_cli/apply/diff_helpers.py

@@ -0,0 +1,58 @@
+#
+# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
+#
+
+import hashlib
+from typing import Any
+
+from deepdiff import DeepDiff
+
+SECRET_MASK = "**********"
+
+
+def compute_checksum(file_path: str) -> str:
+    """Compute SHA256 checksum from a file
+
+    Args:
+        file_path (str): Path for the file for which you want to compute a checksum.
+
+    Returns:
+        str: The computed hash digest
+    """
+    BLOCK_SIZE = 65536
+    file_hash = hashlib.sha256()
+    with open(file_path, "rb") as f:
+        fb = f.read(BLOCK_SIZE)
+        while len(fb) > 0:
+            file_hash.update(fb)
+            fb = f.read(BLOCK_SIZE)
+    return file_hash.hexdigest()
+
+
+def exclude_secrets_from_diff(obj: Any, path: str) -> bool:
+    """Callback function used with DeepDiff to ignore secret values from the diff.
+
+    Args:
+        obj (Any): Object for which a diff will be computed.
+        path (str): unused.
+
+    Returns:
+        bool: Wetheir to ignore the object from the diff.
+    """
+    if isinstance(obj, str):
+        return True if SECRET_MASK in obj else False
+    else:
+        return False
+
+
+def compute_diff(a: Any, b: Any) -> DeepDiff:
+    """Wrapper around the DeepDiff computation.
+
+    Args:
+        a (Any): Object to compare with b.
+        b (Any): Object to compare with a.
+
+    Returns:
+        DeepDiff: the computed diff object.
+    """
+    return DeepDiff(a, b, view="tree", exclude_obj_callback=exclude_secrets_from_diff)