-
Notifications
You must be signed in to change notification settings - Fork 4.6k
🐙 octavia-cli: implement apply
#10703
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from 4 commits
Commits
Show all changes
6 commits
Select commit
Hold shift + click to select a range
2b3e149
octavia-cli: implement apply command
alafanechere 531bef8
update README with achievement
alafanechere d529821
change help message
alafanechere 143dc02
fix typo
alafanechere 4d87355
implement review suggestions + fix builds
alafanechere a6115a4
fix extras in setup.py
alafanechere File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,183 @@ | ||
# | ||
# Copyright (c) 2021 Airbyte, Inc., all rights reserved. | ||
# | ||
|
||
from glob import glob | ||
from typing import List, Tuple | ||
|
||
import airbyte_api_client | ||
import click | ||
from octavia_cli.check_context import REQUIRED_PROJECT_DIRECTORIES, requires_init | ||
|
||
from .resources import BaseResource | ||
from .resources import factory as resource_factory | ||
|
||
|
||
@click.command(name="apply", help="Create or update Airbyte remote resources according local YAML configurations.") | ||
@click.option("--file", "-f", "configurations_files", type=click.Path(), multiple=True) | ||
@click.option("--force", is_flag=True, default=False, help="Does not display the diff and updates without user prompt.") | ||
@click.pass_context | ||
@requires_init | ||
def apply(ctx: click.Context, configurations_files: List[click.Path], force: bool): | ||
if not configurations_files: | ||
configurations_files = find_local_configuration_files() | ||
|
||
resources = get_resources_to_apply(configurations_files, ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"]) | ||
for resource in resources: | ||
apply_single_resource(resource, force) | ||
|
||
|
||
def get_resources_to_apply( | ||
configuration_files: List[str], api_client: airbyte_api_client.ApiClient, workspace_id: str | ||
) -> List[BaseResource]: | ||
"""Create resource objects with factory and sort according to apply priority. | ||
|
||
Args: | ||
configuration_files (List[str]): List of YAML configuration files. | ||
api_client (airbyte_api_client.ApiClient): the Airbyte API client. | ||
workspace_id (str): current Airbyte workspace id. | ||
|
||
Returns: | ||
List[BaseResource]: Resources sorted according to their apply priority. | ||
""" | ||
all_resources = [resource_factory(api_client, workspace_id, path) for path in configuration_files] | ||
return sorted(all_resources, key=lambda resource: resource.apply_priority) | ||
|
||
|
||
def apply_single_resource(resource: BaseResource, force: bool) -> None: | ||
"""Runs resource creation if it was not created, update it otherwise. | ||
|
||
Args: | ||
resource (BaseResource): The resource to apply. | ||
force (bool): Wetheir force mode is on. | ||
""" | ||
if resource.was_created: | ||
click.echo( | ||
click.style(f"🐙 - {resource.resource_name} exists on your Airbyte instance, let's check if we need to update it!", fg="yellow") | ||
) | ||
messages = update_resource(resource, force) | ||
else: | ||
click.echo(click.style(f"🐙 - {resource.resource_name} does not exists on your Airbyte instance, let's create it!", fg="green")) | ||
messages = create_resource(resource) | ||
click.echo("\n".join(messages)) | ||
|
||
|
||
def should_update_resource(force: bool, diff: str, local_file_changed: bool) -> Tuple[bool, str]: | ||
"""Function to decide if the resource needs an update or not. | ||
|
||
Args: | ||
force (bool): Wetheir force mode is on. | ||
diff (str): The computed diff between local and remote for this resource. | ||
local_file_changed (bool): Wetheir the local file describing the resource was modified. | ||
|
||
Returns: | ||
Tuple[bool, str]: Boolean to know if resource should be updated and string describing the update reason. | ||
""" | ||
if force: | ||
should_update, update_reason = True, "🚨 - Running update because force mode is on." | ||
elif diff: | ||
should_update, update_reason = True, "✍️ - Running update because a diff was detected between local and remote resource." | ||
elif local_file_changed: | ||
should_update, update_reason = ( | ||
True, | ||
"✍️ - Running update because a local file change was detected and a secret field might have been edited.", | ||
) | ||
else: | ||
should_update, update_reason = False, "😴 - Did not update because no change detected." | ||
return should_update, click.style(update_reason, fg="green") | ||
|
||
|
||
def display_diff_line(diff_line: str) -> None: | ||
alafanechere marked this conversation as resolved.
Show resolved
Hide resolved
|
||
"""Prettify a diff line and print it to standard output. | ||
|
||
Args: | ||
diff_line (str): The diff line to display. | ||
""" | ||
if "changed from" in diff_line: | ||
color = "yellow" | ||
prefix = "E" | ||
elif "added" in diff_line: | ||
color = "green" | ||
prefix = "+" | ||
elif "removed" in diff_line: | ||
color = "red" | ||
prefix = "-" | ||
else: | ||
prefix = "" | ||
color = None | ||
click.echo(click.style(f"\t{prefix} - {diff_line}", fg=color)) | ||
|
||
|
||
def prompt_for_diff_validation(resource_name: str, diff: str) -> bool: | ||
"""Display the diff to user and prompt them from validation. | ||
|
||
Args: | ||
resource_name (str): Name of the resource the diff was computed for. | ||
diff (str): The diff. | ||
|
||
Returns: | ||
bool: Wetheir user validated the diff. | ||
""" | ||
if diff: | ||
click.echo( | ||
click.style("👀 - Here's the computed diff (🚨 remind that diff on secret fields are not displayed):", fg="magenta", bold=True) | ||
) | ||
for line in diff.split("\n"): | ||
display_diff_line(line) | ||
return click.confirm(click.style(f"❓ - Do you want to update {resource_name}?", bold=True)) | ||
else: | ||
return False | ||
|
||
|
||
def create_resource(resource: BaseResource) -> List[str]: | ||
"""Run a resource creation. | ||
|
||
Args: | ||
resource (BaseResource): The resource to create. | ||
|
||
Returns: | ||
List[str]: Post create messages to display to standard output. | ||
""" | ||
created_resource, state = resource.create() | ||
return [ | ||
click.style(f"🎉 - Successfully created {created_resource.name} on your Airbyte instance!", fg="green", bold=True), | ||
click.style(f"💾 - New state for {created_resource.name} saved at {state.path}", fg="yellow"), | ||
] | ||
|
||
|
||
def update_resource(resource: BaseResource, force: bool) -> List[str]: | ||
"""Run a resource update. Check if update is required and prompt for user diff validation if needed. | ||
|
||
Args: | ||
resource (BaseResource): Resource to update | ||
force (bool): Wetheir force mode is on. | ||
|
||
Returns: | ||
List[str]: Post update messages to display to standard output. | ||
""" | ||
diff = resource.get_diff_with_remote_resource() | ||
should_update, update_reason = should_update_resource(force, diff, resource.local_file_changed) | ||
output_messages = [update_reason] | ||
if not force and diff: | ||
should_update = prompt_for_diff_validation(resource.resource_name, diff) | ||
if should_update: | ||
updated_resource, state = resource.update() | ||
output_messages.append( | ||
click.style(f"🎉 - Successfully updated {updated_resource.name} on your Airbyte instance!", fg="green", bold=True) | ||
) | ||
output_messages.append(click.style(f"💾 - New state for {updated_resource.name} stored at {state.path}.", fg="yellow")) | ||
return output_messages | ||
|
||
|
||
def find_local_configuration_files() -> List[str]: | ||
"""Discover local configuration files. | ||
|
||
Returns: | ||
List[str]: Paths to YAML configuration files. | ||
""" | ||
configuration_files = [] | ||
for resource_directory in REQUIRED_PROJECT_DIRECTORIES: | ||
configuration_files += glob(f"./{resource_directory}/**/configuration.yaml") | ||
if not configuration_files: | ||
click.echo(click.style("😒 - No YAML file found to run apply.", fg="red")) | ||
return configuration_files |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
# | ||
# Copyright (c) 2021 Airbyte, Inc., all rights reserved. | ||
# | ||
|
||
import hashlib | ||
from typing import Any | ||
|
||
from deepdiff import DeepDiff | ||
|
||
SECRET_MASK = "**********" | ||
|
||
|
||
def compute_checksum(file_path: str) -> str: | ||
"""Compute SHA256 checksum from a file | ||
|
||
Args: | ||
file_path (str): Path for the file for which you want to compute a checksum. | ||
|
||
Returns: | ||
str: The computed hash digest | ||
""" | ||
BLOCK_SIZE = 65536 | ||
file_hash = hashlib.sha256() | ||
with open(file_path, "rb") as f: | ||
fb = f.read(BLOCK_SIZE) | ||
while len(fb) > 0: | ||
file_hash.update(fb) | ||
fb = f.read(BLOCK_SIZE) | ||
return file_hash.hexdigest() | ||
|
||
|
||
def exclude_secrets_from_diff(obj: Any, path: str) -> bool: | ||
"""Callback function used with DeepDiff to ignore secret values from the diff. | ||
|
||
Args: | ||
obj (Any): Object for which a diff will be computed. | ||
path (str): unused. | ||
|
||
Returns: | ||
bool: Wetheir to ignore the object from the diff. | ||
""" | ||
if isinstance(obj, str): | ||
return True if SECRET_MASK in obj else False | ||
else: | ||
return False | ||
|
||
|
||
def compute_diff(a: Any, b: Any) -> DeepDiff: | ||
"""Wrapper around the DeepDiff computation. | ||
|
||
Args: | ||
a (Any): Object to compare with b. | ||
b (Any): Object to compare with a. | ||
|
||
Returns: | ||
DeepDiff: the computed diff object. | ||
""" | ||
return DeepDiff(a, b, view="tree", exclude_obj_callback=exclude_secrets_from_diff) |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.