Skip to content

🐙 octavia-cli: implement apply #10703

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Mar 5, 2022
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion octavia-cli/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ We welcome community contributions!

| Date | Milestone |
|------------|-------------------------------------|
| 2022-02-06 | Implement `octavia generate`|
| 2022-03-02 | Implement `octavia apply` (sources and destination only)|
| 2022-02-06 | Implement `octavia generate` (sources and destination only)|
| 2022-01-25 | Implement `octavia init` + some context checks|
| 2022-01-19 | Implement `octavia list workspace sources`, `octavia list workspace destinations`, `octavia list workspace connections`|
| 2022-01-17 | Implement `octavia list connectors source` and `octavia list connectors destinations`|
Expand Down
183 changes: 183 additions & 0 deletions octavia-cli/octavia_cli/apply/commands.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,183 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
#

from glob import glob
from typing import List, Tuple

import airbyte_api_client
import click
from octavia_cli.check_context import REQUIRED_PROJECT_DIRECTORIES, requires_init

from .resources import BaseResource
from .resources import factory as resource_factory


@click.command(name="apply", help="Create or update Airbyte remote resources according local YAML configurations.")
@click.option("--file", "-f", "configurations_files", type=click.Path(), multiple=True)
@click.option("--force", is_flag=True, default=False, help="Does not display the diff and updates without user prompt.")
@click.pass_context
@requires_init
def apply(ctx: click.Context, configurations_files: List[click.Path], force: bool):
if not configurations_files:
configurations_files = find_local_configuration_files()

resources = get_resources_to_apply(configurations_files, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"])
for resource in resources:
apply_single_resource(resource, force)


def get_resources_to_apply(
configuration_files: List[str], api_client: airbyte_api_client.ApiClient, workspace_id: str
) -> List[BaseResource]:
"""Create resource objects with factory and sort according to apply priority.

Args:
configuration_files (List[str]): List of YAML configuration files.
api_client (airbyte_api_client.ApiClient): the Airbyte API client.
workspace_id (str): current Airbyte workspace id.

Returns:
List[BaseResource]: Resources sorted according to their apply priority.
"""
all_resources = [resource_factory(api_client, workspace_id, path) for path in configuration_files]
return sorted(all_resources, key=lambda resource: resource.apply_priority)


def apply_single_resource(resource: BaseResource, force: bool) -> None:
"""Runs resource creation if it was not created, update it otherwise.

Args:
resource (BaseResource): The resource to apply.
force (bool): Wetheir force mode is on.
"""
if resource.was_created:
click.echo(
click.style(f"🐙 - {resource.resource_name} exists on your Airbyte instance, let's check if we need to update it!", fg="yellow")
)
messages = update_resource(resource, force)
else:
click.echo(click.style(f"🐙 - {resource.resource_name} does not exists on your Airbyte instance, let's create it!", fg="green"))
messages = create_resource(resource)
click.echo("\n".join(messages))


def should_update_resource(force: bool, diff: str, local_file_changed: bool) -> Tuple[bool, str]:
"""Function to decide if the resource needs an update or not.

Args:
force (bool): Wetheir force mode is on.
diff (str): The computed diff between local and remote for this resource.
local_file_changed (bool): Wetheir the local file describing the resource was modified.

Returns:
Tuple[bool, str]: Boolean to know if resource should be updated and string describing the update reason.
"""
if force:
should_update, update_reason = True, "🚨 - Running update because force mode is on."
elif diff:
should_update, update_reason = True, "✍️ - Running update because a diff was detected between local and remote resource."
elif local_file_changed:
should_update, update_reason = (
True,
"✍️ - Running update because a local file change was detected and a secret field might have been edited.",
)
else:
should_update, update_reason = False, "😴 - Did not update because no change detected."
return should_update, click.style(update_reason, fg="green")


def display_diff_line(diff_line: str) -> None:
"""Prettify a diff line and print it to standard output.

Args:
diff_line (str): The diff line to display.
"""
if "changed from" in diff_line:
color = "yellow"
prefix = "E"
elif "added" in diff_line:
color = "green"
prefix = "+"
elif "removed" in diff_line:
color = "red"
prefix = "-"
else:
prefix = ""
color = None
click.echo(click.style(f"\t{prefix} - {diff_line}", fg=color))


def prompt_for_diff_validation(resource_name: str, diff: str) -> bool:
"""Display the diff to user and prompt them from validation.

Args:
resource_name (str): Name of the resource the diff was computed for.
diff (str): The diff.

Returns:
bool: Wetheir user validated the diff.
"""
if diff:
click.echo(
click.style("👀 - Here's the computed diff (🚨 remind that diff on secret fields are not displayed):", fg="magenta", bold=True)
)
for line in diff.split("\n"):
display_diff_line(line)
return click.confirm(click.style(f"❓ - Do you want to update {resource_name}?", bold=True))
else:
return False


def create_resource(resource: BaseResource) -> List[str]:
"""Run a resource creation.

Args:
resource (BaseResource): The resource to create.

Returns:
List[str]: Post create messages to display to standard output.
"""
created_resource, state = resource.create()
return [
click.style(f"🎉 - Successfully created {created_resource.name} on your Airbyte instance!", fg="green", bold=True),
click.style(f"💾 - New state for {created_resource.name} saved at {state.path}", fg="yellow"),
]


def update_resource(resource: BaseResource, force: bool) -> List[str]:
"""Run a resource update. Check if update is required and prompt for user diff validation if needed.

Args:
resource (BaseResource): Resource to update
force (bool): Wetheir force mode is on.

Returns:
List[str]: Post update messages to display to standard output.
"""
diff = resource.get_diff_with_remote_resource()
should_update, update_reason = should_update_resource(force, diff, resource.local_file_changed)
output_messages = [update_reason]
if not force and diff:
should_update = prompt_for_diff_validation(resource.resource_name, diff)
if should_update:
updated_resource, state = resource.update()
output_messages.append(
click.style(f"🎉 - Successfully updated {updated_resource.name} on your Airbyte instance!", fg="green", bold=True)
)
output_messages.append(click.style(f"💾 - New state for {updated_resource.name} stored at {state.path}.", fg="yellow"))
return output_messages


def find_local_configuration_files() -> List[str]:
"""Discover local configuration files.

Returns:
List[str]: Paths to YAML configuration files.
"""
configuration_files = []
for resource_directory in REQUIRED_PROJECT_DIRECTORIES:
configuration_files += glob(f"./{resource_directory}/**/configuration.yaml")
if not configuration_files:
click.echo(click.style("😒 - No YAML file found to run apply.", fg="red"))
return configuration_files
58 changes: 58 additions & 0 deletions octavia-cli/octavia_cli/apply/diff_helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
#

import hashlib
from typing import Any

from deepdiff import DeepDiff

SECRET_MASK = "**********"


def compute_checksum(file_path: str) -> str:
"""Compute SHA256 checksum from a file

Args:
file_path (str): Path for the file for which you want to compute a checksum.

Returns:
str: The computed hash digest
"""
BLOCK_SIZE = 65536
file_hash = hashlib.sha256()
with open(file_path, "rb") as f:
fb = f.read(BLOCK_SIZE)
while len(fb) > 0:
file_hash.update(fb)
fb = f.read(BLOCK_SIZE)
return file_hash.hexdigest()


def exclude_secrets_from_diff(obj: Any, path: str) -> bool:
"""Callback function used with DeepDiff to ignore secret values from the diff.

Args:
obj (Any): Object for which a diff will be computed.
path (str): unused.

Returns:
bool: Wetheir to ignore the object from the diff.
"""
if isinstance(obj, str):
return True if SECRET_MASK in obj else False
else:
return False


def compute_diff(a: Any, b: Any) -> DeepDiff:
"""Wrapper around the DeepDiff computation.

Args:
a (Any): Object to compare with b.
b (Any): Object to compare with a.

Returns:
DeepDiff: the computed diff object.
"""
return DeepDiff(a, b, view="tree", exclude_obj_callback=exclude_secrets_from_diff)
Loading