diff --git a/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/connection/expected.yaml b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/connection/expected.yaml new file mode 100644 index 0000000000000..9ba155cc10ded --- /dev/null +++ b/octavia-cli/integration_tests/test_generate/expected_rendered_yaml/connection/expected.yaml @@ -0,0 +1,67 @@ +# Configuration for connection my_new_connection +definition_type: connection +resource_name: my_new_connection +source_id: my_source_id +destination_id: my_destination_id + +# EDIT THE CONFIGURATION BELOW! +configuration: + sourceId: my_source_id # REQUIRED | string + destinationId: my_destination_id # REQUIRED | string + status: active # REQUIRED | string | Allowed values: active, inactive, deprecated + name: my_new_connection # OPTIONAL | string | Optional name of the connection + namespaceDefinition: source # OPTIONAL | string | Allowed values: source, destination, customformat + namespaceFormat: "${SOURCE_NAMESPACE}" # OPTIONAL | string | Used when namespaceDefinition is 'customformat'. If blank then behaves like namespaceDefinition = 'destination'. If "${SOURCE_NAMESPACE}" then behaves like namespaceDefinition = 'source'. + prefix: "" # REQUIRED | Prefix that will be prepended to the name of each stream when it is written to the destination + resourceRequirements: # OPTIONAL | object | Resource requirements to run workers (blank for unbounded allocations) + cpu_limit: "" # OPTIONAL + cpu_request: "" # OPTIONAL + memory_limit: "" # OPTIONAL + memory_request: "" # OPTIONAL + schedule: # OPTIONAL | object + timeUnit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months + units: 1 # REQUIRED | integer + syncCatalog: # OPTIONAL | object | 🚨 ONLY edit streams.config, streams.stream should not be edited as schema cannot be changed. + streams: + - config: + aliasName: aliasMock + cursorField: [] + destinationSyncMode: append + primaryKey: [] + selected: true + syncMode: full_refresh + stream: + defaultCursorField: + - foo + jsonSchema: + $schema: http://json-schema.org/draft-07/schema# + properties: + foo: + type: number + name: stream_1 + namespace: null + sourceDefinedCursor: null + sourceDefinedPrimaryKey: [] + supportedSyncModes: + - full_refresh + - config: + aliasName: aliasMock + cursorField: [] + destinationSyncMode: append + primaryKey: [] + selected: true + syncMode: full_refresh + stream: + defaultCursorField: [] + jsonSchema: + $schema: http://json-schema.org/draft-07/schema# + properties: + bar: + type: number + name: stream_2 + namespace: null + sourceDefinedCursor: null + sourceDefinedPrimaryKey: [] + supportedSyncModes: + - full_refresh + - incremental diff --git a/octavia-cli/integration_tests/test_generate/test_renderer.py b/octavia-cli/integration_tests/test_generate/test_renderers.py similarity index 50% rename from octavia-cli/integration_tests/test_generate/test_renderer.py rename to octavia-cli/integration_tests/test_generate/test_renderers.py index d300bb90c06d0..6a4b823d7e20f 100644 --- a/octavia-cli/integration_tests/test_generate/test_renderer.py +++ b/octavia-cli/integration_tests/test_generate/test_renderers.py @@ -7,7 +7,7 @@ import pytest import yaml -from octavia_cli.generate.renderer import ConnectionSpecificationRenderer +from octavia_cli.generate.renderers import ConnectionRenderer, ConnectorSpecificationRenderer pytestmark = pytest.mark.integration SOURCE_SPECS = "../airbyte-config/init/src/main/resources/seed/source_specs.yaml" @@ -26,7 +26,7 @@ def get_all_specs_params(): @pytest.mark.parametrize("spec_type, spec", get_all_specs_params()) def test_render_spec(spec_type, spec, octavia_project_directory, mocker): - renderer = ConnectionSpecificationRenderer( + renderer = ConnectorSpecificationRenderer( resource_name=f"resource-{spec['dockerImage']}", definition=mocker.Mock( type=spec_type, @@ -66,10 +66,12 @@ def test_render_spec(spec_type, spec, octavia_project_directory, mocker): ("my_s3_destination", "destination", "destination_s3/input_spec.yaml", "destination_s3/expected.yaml"), ], ) -def test_expected_output(resource_name, spec_type, input_spec_path, expected_yaml_path, octavia_project_directory, mocker): +def test_expected_output_connector_specification_renderer( + resource_name, spec_type, input_spec_path, expected_yaml_path, octavia_project_directory, mocker +): with open(os.path.join(EXPECTED_RENDERED_YAML_PATH, input_spec_path), "r") as f: input_spec = yaml.safe_load(f) - renderer = ConnectionSpecificationRenderer( + renderer = ConnectorSpecificationRenderer( resource_name=resource_name, definition=mocker.Mock( type=spec_type, @@ -83,3 +85,71 @@ def test_expected_output(resource_name, spec_type, input_spec_path, expected_yam output_path = renderer.write_yaml(octavia_project_directory) expect_output_path = os.path.join(EXPECTED_RENDERED_YAML_PATH, expected_yaml_path) assert filecmp.cmp(output_path, expect_output_path) + + +def test_expected_output_connection_renderer(octavia_project_directory, mocker): + mock_source = mocker.Mock( + resource_id="my_source_id", + catalog={ + "streams": [ + { + "stream": { + "name": "stream_1", + "jsonSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "foo": { + "type": "number", + } + }, + }, + "supportedSyncModes": ["full_refresh"], + "sourceDefinedCursor": None, + "defaultCursorField": ["foo"], + "sourceDefinedPrimaryKey": [], + "namespace": None, + }, + "config": { + "syncMode": "full_refresh", + "cursorField": [], + "destinationSyncMode": "append", + "primaryKey": [], + "aliasName": "aliasMock", + "selected": True, + }, + }, + { + "stream": { + "name": "stream_2", + "jsonSchema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "properties": { + "bar": { + "type": "number", + } + }, + }, + "supportedSyncModes": ["full_refresh", "incremental"], + "sourceDefinedCursor": None, + "defaultCursorField": [], + "sourceDefinedPrimaryKey": [], + "namespace": None, + }, + "config": { + "syncMode": "full_refresh", + "cursorField": [], + "destinationSyncMode": "append", + "primaryKey": [], + "aliasName": "aliasMock", + "selected": True, + }, + }, + ] + }, + ) + mock_destination = mocker.Mock(resource_id="my_destination_id") + + renderer = ConnectionRenderer("my_new_connection", mock_source, mock_destination) + output_path = renderer.write_yaml(octavia_project_directory) + expect_output_path = os.path.join(EXPECTED_RENDERED_YAML_PATH, "connection/expected.yaml") + assert filecmp.cmp(output_path, expect_output_path) diff --git a/octavia-cli/octavia_cli/apply/resources.py b/octavia-cli/octavia_cli/apply/resources.py index cc9a9022f7c90..fff5009c2aca7 100644 --- a/octavia-cli/octavia_cli/apply/resources.py +++ b/octavia-cli/octavia_cli/apply/resources.py @@ -11,12 +11,14 @@ import airbyte_api_client import yaml from airbyte_api_client.api import destination_api, source_api +from airbyte_api_client.model.airbyte_catalog import AirbyteCatalog from airbyte_api_client.model.destination_create import DestinationCreate from airbyte_api_client.model.destination_read import DestinationRead from airbyte_api_client.model.destination_read_list import DestinationReadList from airbyte_api_client.model.destination_search import DestinationSearch from airbyte_api_client.model.destination_update import DestinationUpdate from airbyte_api_client.model.source_create import SourceCreate +from airbyte_api_client.model.source_id_request_body import SourceIdRequestBody from airbyte_api_client.model.source_read import SourceRead from airbyte_api_client.model.source_read_list import SourceReadList from airbyte_api_client.model.source_search import SourceSearch @@ -355,6 +357,30 @@ def update_payload(self): name=self.resource_name, ) + @property + def resource_id_request_body(self) -> SourceIdRequestBody: + """Creates SourceIdRequestBody from resource id. + + Raises: + NonExistingResourceError: raised if the resource id is None. + + Returns: + SourceIdRequestBody: The SourceIdRequestBody model instance. + """ + if self.resource_id is None: + raise NonExistingResourceError("The resource id could not be retrieved, the remote resource is not existing.") + return SourceIdRequestBody(source_id=self.resource_id) + + @property + def catalog(self) -> AirbyteCatalog: + """Retrieves the source's Airbyte catalog. + + Returns: + AirbyteCatalog: The catalog issued by schema discovery. + """ + schema = self.api_instance.discover_schema_for_source(self.resource_id_request_body, _check_return_type=False) + return schema.catalog + class Destination(BaseResource): diff --git a/octavia-cli/octavia_cli/generate/commands.py b/octavia-cli/octavia_cli/generate/commands.py index d34c39c82f8c8..4a68a57c18bf4 100644 --- a/octavia-cli/octavia_cli/generate/commands.py +++ b/octavia-cli/octavia_cli/generate/commands.py @@ -4,20 +4,74 @@ import click import octavia_cli.generate.definitions as definitions +from octavia_cli.apply import resources from octavia_cli.check_context import requires_init -from .renderer import ConnectionSpecificationRenderer +from .renderers import ConnectionRenderer, ConnectorSpecificationRenderer -@click.command(name="generate", help="Generate a YAML template for a source or a destination.") -@click.argument("definition_type", type=click.Choice(["source", "destination"])) -@click.argument("definition_id", type=click.STRING) -@click.argument("resource_name", type=click.STRING) +@click.group("generate", help="Generate a YAML template for a source, destination or a connection.") @click.pass_context @requires_init -def generate(ctx: click.Context, definition_type: str, definition_id: str, resource_name: str): - definition = definitions.factory(definition_type, ctx.obj["API_CLIENT"], definition_id) - renderer = ConnectionSpecificationRenderer(resource_name, definition) +def generate(ctx: click.Context): + pass + + +def generate_source_or_destination(definition_type, api_client, definition_id, resource_name): + definition = definitions.factory(definition_type, api_client, definition_id) + renderer = ConnectorSpecificationRenderer(resource_name, definition) output_path = renderer.write_yaml(project_path=".") - message = f"✅ - Created the specification template for {resource_name} in {output_path}." + message = f"✅ - Created the {definition_type} template for {resource_name} in {output_path}." + click.echo(click.style(message, fg="green")) + + +@generate.command(name="source", help="Create YAML for a source") +@click.argument("definition_id", type=click.STRING) +@click.argument("resource_name", type=click.STRING) +@click.pass_context +def source(ctx: click.Context, definition_id: str, resource_name: str): + generate_source_or_destination("source", ctx.obj["API_CLIENT"], definition_id, resource_name) + + +@generate.command(name="destination", help="Create YAML for a destination") +@click.argument("definition_id", type=click.STRING) +@click.argument("resource_name", type=click.STRING) +@click.pass_context +def destination(ctx: click.Context, definition_id: str, resource_name: str): + generate_source_or_destination("destination", ctx.obj["API_CLIENT"], definition_id, resource_name) + + +@generate.command(name="connection", help="Generate a YAML template for a connection.") +@click.argument("connection_name", type=click.STRING) +@click.option( + "--source", + "source_path", + type=click.Path(exists=True, readable=True), + required=True, + help="Path to the YAML fine defining your source configuration.", +) +@click.option( + "--destination", + "destination_path", + type=click.Path(exists=True, readable=True), + required=True, + help="Path to the YAML fine defining your destination configuration.", +) +@click.pass_context +def connection(ctx: click.Context, connection_name: str, source_path: str, destination_path: str): + source = resources.factory(ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], source_path) + if not source.was_created: + raise resources.NonExistingResourceError( + f"The source defined at {source_path} does not exists. Please run octavia apply before creating this connection." + ) + + destination = resources.factory(ctx.obj["API_CLIENT"], ctx.obj["WORKSPACE_ID"], destination_path) + if not destination.was_created: + raise resources.NonExistingResourceError( + f"The destination defined at {destination_path} does not exists. Please run octavia apply before creating this connection." + ) + + connection_renderer = ConnectionRenderer(connection_name, source, destination) + output_path = connection_renderer.write_yaml(project_path=".") + message = f"✅ - Created the connection template for {connection_name} in {output_path}." click.echo(click.style(message, fg="green")) diff --git a/octavia-cli/octavia_cli/generate/definitions.py b/octavia-cli/octavia_cli/generate/definitions.py index 5e775a776b814..daa4d22964bd3 100644 --- a/octavia-cli/octavia_cli/generate/definitions.py +++ b/octavia-cli/octavia_cli/generate/definitions.py @@ -86,6 +86,10 @@ def __getattr__(self, name: str) -> Any: raise AttributeError(f"{self.__class__.__name__}.{name} is invalid.") +class ConnectionDefinition(BaseDefinition): + type = "connection" + + class SourceDefinition(BaseDefinition): api = source_definition_api.SourceDefinitionApi type = "source" diff --git a/octavia-cli/octavia_cli/generate/renderer.py b/octavia-cli/octavia_cli/generate/renderers.py similarity index 62% rename from octavia-cli/octavia_cli/generate/renderer.py rename to octavia-cli/octavia_cli/generate/renderers.py index 324e92075eb4a..be50654e0cacb 100644 --- a/octavia-cli/octavia_cli/generate/renderer.py +++ b/octavia-cli/octavia_cli/generate/renderers.py @@ -2,12 +2,16 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +import abc import os from typing import Any, Callable, List -from jinja2 import Environment, PackageLoader, select_autoescape +import yaml +from jinja2 import Environment, PackageLoader, Template, select_autoescape +from octavia_cli.apply import resources -from .definitions import BaseDefinition +from .definitions import BaseDefinition, ConnectionDefinition +from .yaml_dumpers import CatalogDumper JINJA_ENV = Environment(loader=PackageLoader("octavia_cli"), autoescape=select_autoescape(), trim_blocks=False, lstrip_blocks=True) @@ -127,14 +131,77 @@ def get_object_fields(field_metadata: dict) -> List["FieldToRender"]: return [] -class ConnectionSpecificationRenderer: +class BaseRenderer(abc.ABC): + @property + @abc.abstractmethod + def TEMPLATE( + self, + ) -> Template: # pragma: no cover + pass + + def __init__(self, resource_name: str) -> None: + self.resource_name = resource_name + + def _get_output_path(self, project_path: str, definition_type: str) -> str: + """Get rendered file output path + + Args: + project_path (str): Current project path. + definition_type (str): Current definition_type. + + Returns: + str: Full path to the output path. + """ + directory = os.path.join(project_path, f"{definition_type}s", self.resource_name) + if not os.path.exists(directory): + os.makedirs(directory) + return os.path.join(directory, "configuration.yaml") + + @abc.abstractmethod + def _render(self): # pragma: no cover + """Runs the template rendering. + + Raises: + NotImplementedError: Must be implemented on subclasses. + """ + raise NotImplementedError + + def write_yaml(self, project_path: str) -> str: + """Write rendered specification to a YAML file in local project path. + + Args: + project_path (str): Path to directory hosting the octavia project. + + Returns: + str: Path to the rendered specification. + """ + output_path = self._get_output_path(project_path, self.definition.type) + rendered = self._render() + + with open(output_path, "w") as f: + f.write(rendered) + return output_path + + +class ConnectorSpecificationRenderer(BaseRenderer): TEMPLATE = JINJA_ENV.get_template("source_or_destination.yaml.j2") def __init__(self, resource_name: str, definition: BaseDefinition) -> None: - self.resource_name = resource_name + """Connector specification renderer constructor. + + Args: + resource_name (str): Name of the source or destination. + definition (BaseDefinition): The definition related to a source or a destination. + """ + super().__init__(resource_name) self.definition = definition def _parse_connection_specification(self, schema: dict) -> List[List["FieldToRender"]]: + """Create a renderable structure from the specification schema + + Returns: + List[List["FieldToRender"]]: List of list of fields to render. + """ if schema.get("oneOf"): roots = [] for one_of_value in schema.get("oneOf"): @@ -145,19 +212,50 @@ def _parse_connection_specification(self, schema: dict) -> List[List["FieldToRen required_fields = schema.get("required", []) return [parse_fields(required_fields, schema["properties"])] - def _get_output_path(self, project_path: str) -> str: - directory = os.path.join(project_path, f"{self.definition.type}s", self.resource_name) - if not os.path.exists(directory): - os.makedirs(directory) - return os.path.join(directory, "configuration.yaml") - - def write_yaml(self, project_path: str) -> str: - output_path = self._get_output_path(project_path) + def _render(self) -> str: parsed_schema = self._parse_connection_specification(self.definition.specification.connection_specification) - rendered = self.TEMPLATE.render( + return self.TEMPLATE.render( {"resource_name": self.resource_name, "definition": self.definition, "configuration_fields": parsed_schema} ) - with open(output_path, "w") as f: - f.write(rendered) - return output_path + +class ConnectionRenderer(BaseRenderer): + + TEMPLATE = JINJA_ENV.get_template("connection.yaml.j2") + definition = ConnectionDefinition + + def __init__(self, connection_name: str, source: resources.Source, destination: resources.Destination) -> None: + """Connection renderer constructor. + + Args: + connection_name (str): Name of the connection to render. + source (resources.Source): Connection's source. + destination (resources.Destination): Connections's destination. + """ + super().__init__(connection_name) + self.source = source + self.destination = destination + + @staticmethod + def catalog_to_yaml(catalog: dict) -> str: + """Convert the source catalog to a YAML string. + Convert camel case to snake case. + + Args: + catalog (dict): Source's catalog. + + Returns: + str: Catalog rendered as yaml. + """ + return yaml.dump(catalog, Dumper=CatalogDumper, default_flow_style=False) + + def _render(self) -> str: + yaml_catalog = self.catalog_to_yaml(self.source.catalog) + return self.TEMPLATE.render( + { + "connection_name": self.resource_name, + "source_id": self.source.resource_id, + "destination_id": self.destination.resource_id, + "catalog": yaml_catalog, + } + ) diff --git a/octavia-cli/octavia_cli/generate/yaml_dumpers.py b/octavia-cli/octavia_cli/generate/yaml_dumpers.py new file mode 100644 index 0000000000000..bee9fdef2f62d --- /dev/null +++ b/octavia-cli/octavia_cli/generate/yaml_dumpers.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# +import yaml + + +# This custom Dumper allows the list indentation expected by our prettier formatter: +# Normal dumper behavior +# my_list: +# - bar: test2 +# foo: test +# - bar: test4 +# foo: test3 +# Custom behavior to match prettier's rules: +# my_list: +# - bar: test2 +# foo: test +# - bar: test4 +# foo: test3 +class CatalogDumper(yaml.Dumper): + def increase_indent(self, flow=False, indentless=False): + return super(CatalogDumper, self).increase_indent(flow, False) diff --git a/octavia-cli/octavia_cli/templates/connection.yaml.j2 b/octavia-cli/octavia_cli/templates/connection.yaml.j2 new file mode 100644 index 0000000000000..8ddb33caca27e --- /dev/null +++ b/octavia-cli/octavia_cli/templates/connection.yaml.j2 @@ -0,0 +1,25 @@ +# Configuration for connection {{ connection_name }} +definition_type: connection +resource_name: {{ connection_name }} +source_id: {{ source_id }} +destination_id: {{ destination_id }} + +# EDIT THE CONFIGURATION BELOW! +configuration: + sourceId: {{ source_id }} # REQUIRED | string + destinationId: {{ destination_id }} # REQUIRED | string + status: active # REQUIRED | string | Allowed values: active, inactive, deprecated + name: {{ connection_name }} # OPTIONAL | string | Optional name of the connection + namespaceDefinition: source # OPTIONAL | string | Allowed values: source, destination, customformat + namespaceFormat: "${SOURCE_NAMESPACE}" # OPTIONAL | string | Used when namespaceDefinition is 'customformat'. If blank then behaves like namespaceDefinition = 'destination'. If "${SOURCE_NAMESPACE}" then behaves like namespaceDefinition = 'source'. + prefix: "" # REQUIRED | Prefix that will be prepended to the name of each stream when it is written to the destination + resourceRequirements: # OPTIONAL | object | Resource requirements to run workers (blank for unbounded allocations) + cpu_limit: "" # OPTIONAL + cpu_request: "" # OPTIONAL + memory_limit: "" # OPTIONAL + memory_request: "" # OPTIONAL + schedule: # OPTIONAL | object + timeUnit: hours # REQUIRED | string | Allowed values: minutes, hours, days, weeks, months + units: 1 # REQUIRED | integer + syncCatalog: # OPTIONAL | object | 🚨 ONLY edit streams.config, streams.stream should not be edited as schema cannot be changed. + {{ catalog | indent(4)}} diff --git a/octavia-cli/setup.py b/octavia-cli/setup.py index 3337976377b06..f094e7beec9ff 100644 --- a/octavia-cli/setup.py +++ b/octavia-cli/setup.py @@ -46,6 +46,8 @@ f"airbyte_api_client @ file://{os.getcwd()}/build/airbyte_api_client", "jinja2~=3.0.3", "deepdiff~=5.7.0", + "PyYAML~=6.0", + "pyhumps~=3.5.3", ], python_requires=">=3.8.12", extras_require={ diff --git a/octavia-cli/unit_tests/test_apply/test_resources.py b/octavia-cli/unit_tests/test_apply/test_resources.py index ac62c5c65a55a..2358a9b7b0c79 100644 --- a/octavia-cli/unit_tests/test_apply/test_resources.py +++ b/octavia-cli/unit_tests/test_apply/test_resources.py @@ -238,6 +238,7 @@ class TestSource: ) def test_init(self, mocker, mock_api_client, local_configuration, state): assert resources.Source.__base__ == resources.BaseResource + mocker.patch.object(resources, "SourceIdRequestBody") mocker.patch.object(resources.Source, "resource_id", "foo") source = resources.Source(mock_api_client, "workspace_id", local_configuration, "bar.yaml") mocker.patch.object(source, "state", state) @@ -263,6 +264,33 @@ def test_init(self, mocker, mock_api_client, local_configuration, state): source_definition_id=source.definition_id, workspace_id=source.workspace_id, source_id=source.state.resource_id ) + assert source.resource_id_request_body == resources.SourceIdRequestBody.return_value + resources.SourceIdRequestBody.assert_called_with(source_id=source.resource_id) + + @pytest.mark.parametrize( + "resource_id", + [None, "foo"], + ) + def test_resource_id_request_body(self, mocker, mock_api_client, resource_id, local_configuration): + mocker.patch.object(resources, "SourceIdRequestBody") + mocker.patch.object(resources.Source, "resource_id", resource_id) + source = resources.Source(mock_api_client, "workspace_id", local_configuration, "bar.yaml") + if resource_id is None: + with pytest.raises(resources.NonExistingResourceError): + source.resource_id_request_body + resources.SourceIdRequestBody.assert_not_called() + else: + assert source.resource_id_request_body == resources.SourceIdRequestBody.return_value + resources.SourceIdRequestBody.assert_called_with(source_id=source.resource_id) + + def test_catalog(self, mocker, mock_api_client, local_configuration): + mocker.patch.object(resources.Source, "resource_id_request_body") + source = resources.Source(mock_api_client, "workspace_id", local_configuration, "bar.yaml") + source.api_instance = mocker.Mock() + catalog = source.catalog + assert catalog == source.api_instance.discover_schema_for_source.return_value.catalog + source.api_instance.discover_schema_for_source.assert_called_with(source.resource_id_request_body, _check_return_type=False) + class TestDestination: @pytest.mark.parametrize( diff --git a/octavia-cli/unit_tests/test_generate/test_commands.py b/octavia-cli/unit_tests/test_generate/test_commands.py index 7985a505768bc..7617c6a2ed4be 100644 --- a/octavia-cli/unit_tests/test_generate/test_commands.py +++ b/octavia-cli/unit_tests/test_generate/test_commands.py @@ -2,34 +2,116 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +import pytest from click.testing import CliRunner +from octavia_cli.apply.resources import NonExistingResourceError from octavia_cli.generate import commands -def test_generate_initialized(mocker): +@pytest.fixture +def context_object(mocker): + return {"PROJECT_IS_INITIALIZED": True, "API_CLIENT": mocker.Mock(), "WORKSPACE_ID": "foo"} + + +def test_generate_initialized(mocker, context_object): runner = CliRunner() - mocker.patch.object(commands, "definitions", mocker.Mock(return_value=(["dir_a", "dir_b"], []))) - mocker.patch.object(commands, "ConnectionSpecificationRenderer", mocker.Mock()) - mock_renderer = commands.ConnectionSpecificationRenderer.return_value + mocker.patch.object(commands, "definitions") + mocker.patch.object(commands, "ConnectorSpecificationRenderer", mocker.Mock()) + mock_renderer = commands.ConnectorSpecificationRenderer.return_value mock_renderer.write_yaml.return_value = "expected_output_path" - context_object = {"PROJECT_IS_INITIALIZED": True, "API_CLIENT": mocker.Mock()} result = runner.invoke(commands.generate, ["source", "uuid", "my_source"], obj=context_object) assert result.exit_code == 0 - assert result.output == "✅ - Created the specification template for my_source in expected_output_path.\n" - commands.definitions.factory.assert_called_with("source", context_object["API_CLIENT"], "uuid") - commands.ConnectionSpecificationRenderer.assert_called_with("my_source", commands.definitions.factory.return_value) - mock_renderer.write_yaml.assert_called_with(project_path=".") -def test_generate_not_initialized(): +def test_generate_not_initialized(context_object): runner = CliRunner() - context_object = {"PROJECT_IS_INITIALIZED": False} + context_object["PROJECT_IS_INITIALIZED"] = False result = runner.invoke(commands.generate, ["source", "uuid", "my_source"], obj=context_object) assert result.exit_code == 1 + assert result.output == "Error: Your octavia project is not initialized, please run 'octavia init' before running this command.\n" -def test_invalid_definition_type(): +def test_invalid_definition_type(context_object): runner = CliRunner() - result = runner.invoke(commands.generate, ["random_definition", "uuid", "my_source"]) + result = runner.invoke(commands.generate, ["random_definition", "uuid", "my_source"], obj=context_object) assert result.exit_code == 2 + + +@pytest.mark.parametrize( + "command,resource_name,definition_type", + [ + (commands.source, "my_source", "source"), + (commands.destination, "my_destination", "destination"), + ], +) +def test_generate_source_or_destination(mocker, context_object, command, resource_name, definition_type): + runner = CliRunner() + mocker.patch.object(commands, "definitions") + mocker.patch.object(commands, "ConnectorSpecificationRenderer", mocker.Mock()) + mock_renderer = commands.ConnectorSpecificationRenderer.return_value + mock_renderer.write_yaml.return_value = "expected_output_path" + result = runner.invoke(command, ["uuid", resource_name], obj=context_object) + assert result.exit_code == 0 + assert result.output == f"✅ - Created the {definition_type} template for {resource_name} in expected_output_path.\n" + commands.definitions.factory.assert_called_with(definition_type, context_object["API_CLIENT"], "uuid") + commands.ConnectorSpecificationRenderer.assert_called_with(resource_name, commands.definitions.factory.return_value) + mock_renderer.write_yaml.assert_called_with(project_path=".") + + +@pytest.fixture +def tmp_source_path(tmp_path): + source_path = tmp_path / "my_source.yaml" + source_path.write_text("foo") + return source_path + + +@pytest.fixture +def tmp_destination_path(tmp_path): + destination_path = tmp_path / "my_destination.yaml" + destination_path.write_text("foo") + return destination_path + + +@pytest.mark.parametrize( + "source_created,destination_created", + [(True, True), (False, True), (True, False), (False, False)], +) +def test_generate_connection(mocker, context_object, tmp_source_path, tmp_destination_path, source_created, destination_created): + runner = CliRunner() + mock_source = mocker.Mock(was_created=source_created) + mock_destination = mocker.Mock(was_created=destination_created) + + mock_resource_factory = mocker.Mock(side_effect=[mock_source, mock_destination]) + mocker.patch.object( + commands, "resources", mocker.Mock(factory=mock_resource_factory, NonExistingResourceError=NonExistingResourceError) + ) + mocker.patch.object(commands, "ConnectionRenderer", mocker.Mock()) + mock_renderer = commands.ConnectionRenderer.return_value + mock_renderer.write_yaml.return_value = "expected_output_path" + result = runner.invoke( + commands.connection, ["my_new_connection", "--source", tmp_source_path, "--destination", tmp_destination_path], obj=context_object + ) + if source_created and destination_created: + assert result.exit_code == 0 + assert result.output == "✅ - Created the connection template for my_new_connection in expected_output_path.\n" + commands.resources.factory.assert_has_calls( + [ + mocker.call(context_object["API_CLIENT"], context_object["WORKSPACE_ID"], tmp_source_path), + mocker.call(context_object["API_CLIENT"], context_object["WORKSPACE_ID"], tmp_destination_path), + ] + ) + commands.ConnectionRenderer.assert_called_with("my_new_connection", mock_source, mock_destination) + mock_renderer.write_yaml.assert_called_with(project_path=".") + elif not source_created: + assert ( + result.output + == f"Error: The source defined at {tmp_source_path} does not exists. Please run octavia apply before creating this connection.\n" + ) + assert result.exit_code == 1 + elif not destination_created: + assert ( + result.output + == f"Error: The destination defined at {tmp_destination_path} does not exists. Please run octavia apply before creating this connection.\n" + ) + assert result.exit_code == 1 diff --git a/octavia-cli/unit_tests/test_generate/test_connection.py b/octavia-cli/unit_tests/test_generate/test_connection.py new file mode 100644 index 0000000000000..0f599765708a6 --- /dev/null +++ b/octavia-cli/unit_tests/test_generate/test_connection.py @@ -0,0 +1,5 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +# diff --git a/octavia-cli/unit_tests/test_generate/test_renderer.py b/octavia-cli/unit_tests/test_generate/test_renderer.py deleted file mode 100644 index 902561cc6d56a..0000000000000 --- a/octavia-cli/unit_tests/test_generate/test_renderer.py +++ /dev/null @@ -1,220 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# - -from unittest.mock import mock_open, patch - -import pytest -from octavia_cli.generate import renderer - - -class TestFieldToRender: - def test_init(self, mocker): - mocker.patch.object(renderer.FieldToRender, "_get_one_of_values") - mocker.patch.object(renderer, "get_object_fields") - mocker.patch.object(renderer.FieldToRender, "_get_array_items") - mocker.patch.object(renderer.FieldToRender, "_build_comment") - mocker.patch.object(renderer.FieldToRender, "_get_default") - - field_metadata = mocker.Mock() - field_to_render = renderer.FieldToRender("field_name", True, field_metadata) - assert field_to_render.name == "field_name" - assert field_to_render.required - assert field_to_render.field_metadata == field_metadata - assert field_to_render.one_of_values == field_to_render._get_one_of_values.return_value - assert field_to_render.object_properties == renderer.get_object_fields.return_value - assert field_to_render.array_items == field_to_render._get_array_items.return_value - assert field_to_render.comment == field_to_render._build_comment.return_value - assert field_to_render.default == field_to_render._get_default.return_value - field_to_render._build_comment.assert_called_with( - [ - field_to_render._get_secret_comment, - field_to_render._get_required_comment, - field_to_render._get_type_comment, - field_to_render._get_description_comment, - field_to_render._get_example_comment, - ] - ) - - def test_get_attr(self): - field_to_render = renderer.FieldToRender("field_name", True, {"foo": "bar"}) - assert field_to_render.foo == "bar" - assert field_to_render.not_existing is None - - def test_is_array_of_objects(self): - field_to_render = renderer.FieldToRender("field_name", True, {"foo": "bar"}) - field_to_render.type = "array" - field_to_render.items = {"type": "object"} - assert field_to_render.is_array_of_objects - field_to_render.type = "array" - field_to_render.items = {"type": "int"} - assert not field_to_render.is_array_of_objects - - def test__get_one_of_values(self, mocker): - field_to_render = renderer.FieldToRender("field_name", True, {"foo": "bar"}) - field_to_render.oneOf = False - assert field_to_render._get_one_of_values() == [] - - mocker.patch.object(renderer, "get_object_fields") - one_of_value = mocker.Mock() - field_to_render.oneOf = [one_of_value] - one_of_values = field_to_render._get_one_of_values() - renderer.get_object_fields.assert_called_once_with(one_of_value) - assert one_of_values == [renderer.get_object_fields.return_value] - - def test__get_array_items(self, mocker): - mocker.patch.object(renderer, "parse_fields") - mocker.patch.object(renderer.FieldToRender, "is_array_of_objects", False) - - field_to_render = renderer.FieldToRender("field_name", True, {"foo": "bar"}) - assert field_to_render._get_array_items() == [] - field_to_render.items = {"required": [], "properties": []} - mocker.patch.object(renderer.FieldToRender, "is_array_of_objects", True) - assert field_to_render._get_array_items() == renderer.parse_fields.return_value - renderer.parse_fields.assert_called_with([], []) - - def test__get_required_comment(self): - field_to_render = renderer.FieldToRender("field_name", True, {"foo": "bar"}) - field_to_render.required = True - assert field_to_render._get_required_comment() == "REQUIRED" - field_to_render.required = False - assert field_to_render._get_required_comment() == "OPTIONAL" - - def test__get_type_comment(self): - field_to_render = renderer.FieldToRender("field_name", True, {"foo": "bar"}) - field_to_render.type = "mytype" - assert field_to_render._get_type_comment() == "mytype" - field_to_render.type = None - assert field_to_render._get_type_comment() is None - - def test__get_secret_comment(self): - field_to_render = renderer.FieldToRender("field_name", True, {"foo": "bar"}) - field_to_render.airbyte_secret = True - assert field_to_render._get_secret_comment() == "SECRET" - field_to_render.airbyte_secret = False - assert field_to_render._get_secret_comment() is None - - def test__get_description_comment(self): - field_to_render = renderer.FieldToRender("field_name", True, {"foo": "bar"}) - field_to_render.description = "foo" - assert field_to_render._get_description_comment() == "foo" - field_to_render.description = None - assert field_to_render._get_description_comment() is None - - @pytest.mark.parametrize( - "examples_value,expected_output", - [ - (["foo", "bar"], "Examples: foo, bar"), - (["foo"], "Example: foo"), - ("foo", "Example: foo"), - ([5432], "Example: 5432"), - (None, None), - ], - ) - def test__get_example_comment(self, examples_value, expected_output): - field_to_render = renderer.FieldToRender("field_name", True, {"foo": "bar"}) - field_to_render.examples = examples_value - assert field_to_render._get_example_comment() == expected_output - - @pytest.mark.parametrize( - "field_metadata,expected_default", - [ - ({"const": "foo", "default": "bar"}, "foo"), - ({"default": "bar"}, "bar"), - ({}, None), - ], - ) - def test__get_default(self, field_metadata, expected_default): - field_to_render = renderer.FieldToRender("field_name", True, field_metadata) - assert field_to_render.default == expected_default - - def test__build_comment(self, mocker): - comment_functions = [mocker.Mock(return_value="foo"), mocker.Mock(return_value=None), mocker.Mock(return_value="bar")] - comment = renderer.FieldToRender._build_comment(comment_functions) - assert comment == "foo | bar" - - -def test_parse_fields(): - required_fields = ["foo"] - properties = {"foo": {}, "bar": {}} - fields_to_render = renderer.parse_fields(required_fields, properties) - assert fields_to_render[0].name == "foo" - assert fields_to_render[0].required - assert fields_to_render[1].name == "bar" - assert not fields_to_render[1].required - - -def test_get_object_fields(mocker): - mocker.patch.object(renderer, "parse_fields") - field_metadata = {"properties": {"foo": {}, "bar": {}}, "required": ["foo"]} - object_properties = renderer.get_object_fields(field_metadata) - assert object_properties == renderer.parse_fields.return_value - renderer.parse_fields.assert_called_with(["foo"], field_metadata["properties"]) - field_metadata = {} - assert renderer.get_object_fields(field_metadata) == [] - - -class TestConnectionSpecificationRenderer: - def test_init(self, mocker): - assert renderer.ConnectionSpecificationRenderer.TEMPLATE == renderer.JINJA_ENV.get_template("source_or_destination.yaml.j2") - definition = mocker.Mock() - spec_renderer = renderer.ConnectionSpecificationRenderer("my_resource_name", definition) - assert spec_renderer.resource_name == "my_resource_name" - assert spec_renderer.definition == definition - - def test__parse_connection_specification(self, mocker): - mocker.patch.object(renderer, "parse_fields") - schema = {"required": ["foo"], "properties": {"foo": "bar"}} - definition = mocker.Mock() - spec_renderer = renderer.ConnectionSpecificationRenderer("my_resource_name", definition) - parsed_schema = spec_renderer._parse_connection_specification(schema) - assert renderer.parse_fields.call_count == 1 - assert parsed_schema[0], renderer.parse_fields.return_value - renderer.parse_fields.assert_called_with(["foo"], {"foo": "bar"}) - - def test__parse_connection_specification_one_of(self, mocker): - mocker.patch.object(renderer, "parse_fields") - schema = {"oneOf": [{"required": ["foo"], "properties": {"foo": "bar"}}, {"required": ["free"], "properties": {"free": "beer"}}]} - spec_renderer = renderer.ConnectionSpecificationRenderer("my_resource_name", mocker.Mock()) - parsed_schema = spec_renderer._parse_connection_specification(schema) - assert renderer.parse_fields.call_count == 2 - assert parsed_schema[0], renderer.parse_fields.return_value - assert parsed_schema[1], renderer.parse_fields.return_value - assert len(parsed_schema) == len(schema["oneOf"]) - renderer.parse_fields.assert_called_with(["free"], {"free": "beer"}) - - def test__get_output_path(self, mocker): - mocker.patch.object(renderer, "os") - renderer.os.path.exists.return_value = False - spec_renderer = renderer.ConnectionSpecificationRenderer("my_resource_name", mocker.Mock(type="source")) - renderer.os.path.join.side_effect = ["./source/my_resource_name", "./source/my_resource_name/configuration.yaml"] - output_path = spec_renderer._get_output_path(".") - renderer.os.makedirs.assert_called_once() - renderer.os.path.join.assert_has_calls( - [ - mocker.call(".", "sources", "my_resource_name"), - mocker.call("./source/my_resource_name", "configuration.yaml"), - ] - ) - assert output_path == "./source/my_resource_name/configuration.yaml" - - def test_write_yaml(self, mocker): - - mocker.patch.object(renderer.ConnectionSpecificationRenderer, "_get_output_path") - mocker.patch.object(renderer.ConnectionSpecificationRenderer, "_parse_connection_specification") - mocker.patch.object( - renderer.ConnectionSpecificationRenderer, "TEMPLATE", mocker.Mock(render=mocker.Mock(return_value="rendered_string")) - ) - - spec_renderer = renderer.ConnectionSpecificationRenderer("my_resource_name", mocker.Mock(type="source")) - with patch("builtins.open", mock_open()) as mock_file: - output_path = spec_renderer.write_yaml(".") - assert output_path == spec_renderer._get_output_path.return_value - spec_renderer.TEMPLATE.render.assert_called_with( - { - "resource_name": "my_resource_name", - "definition": spec_renderer.definition, - "configuration_fields": spec_renderer._parse_connection_specification.return_value, - } - ) - mock_file.assert_called_with(output_path, "w") diff --git a/octavia-cli/unit_tests/test_generate/test_renderers.py b/octavia-cli/unit_tests/test_generate/test_renderers.py new file mode 100644 index 0000000000000..bb434791b943a --- /dev/null +++ b/octavia-cli/unit_tests/test_generate/test_renderers.py @@ -0,0 +1,308 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import mock_open, patch + +import pytest +from octavia_cli.generate import renderers + + +class TestFieldToRender: + def test_init(self, mocker): + mocker.patch.object(renderers.FieldToRender, "_get_one_of_values") + mocker.patch.object(renderers, "get_object_fields") + mocker.patch.object(renderers.FieldToRender, "_get_array_items") + mocker.patch.object(renderers.FieldToRender, "_build_comment") + mocker.patch.object(renderers.FieldToRender, "_get_default") + + field_metadata = mocker.Mock() + field_to_render = renderers.FieldToRender("field_name", True, field_metadata) + assert field_to_render.name == "field_name" + assert field_to_render.required + assert field_to_render.field_metadata == field_metadata + assert field_to_render.one_of_values == field_to_render._get_one_of_values.return_value + assert field_to_render.object_properties == renderers.get_object_fields.return_value + assert field_to_render.array_items == field_to_render._get_array_items.return_value + assert field_to_render.comment == field_to_render._build_comment.return_value + assert field_to_render.default == field_to_render._get_default.return_value + field_to_render._build_comment.assert_called_with( + [ + field_to_render._get_secret_comment, + field_to_render._get_required_comment, + field_to_render._get_type_comment, + field_to_render._get_description_comment, + field_to_render._get_example_comment, + ] + ) + + def test_get_attr(self): + field_to_render = renderers.FieldToRender("field_name", True, {"foo": "bar"}) + assert field_to_render.foo == "bar" + assert field_to_render.not_existing is None + + def test_is_array_of_objects(self): + field_to_render = renderers.FieldToRender("field_name", True, {"foo": "bar"}) + field_to_render.type = "array" + field_to_render.items = {"type": "object"} + assert field_to_render.is_array_of_objects + field_to_render.type = "array" + field_to_render.items = {"type": "int"} + assert not field_to_render.is_array_of_objects + + def test__get_one_of_values(self, mocker): + field_to_render = renderers.FieldToRender("field_name", True, {"foo": "bar"}) + field_to_render.oneOf = False + assert field_to_render._get_one_of_values() == [] + + mocker.patch.object(renderers, "get_object_fields") + one_of_value = mocker.Mock() + field_to_render.oneOf = [one_of_value] + one_of_values = field_to_render._get_one_of_values() + renderers.get_object_fields.assert_called_once_with(one_of_value) + assert one_of_values == [renderers.get_object_fields.return_value] + + def test__get_array_items(self, mocker): + mocker.patch.object(renderers, "parse_fields") + mocker.patch.object(renderers.FieldToRender, "is_array_of_objects", False) + + field_to_render = renderers.FieldToRender("field_name", True, {"foo": "bar"}) + assert field_to_render._get_array_items() == [] + field_to_render.items = {"required": [], "properties": []} + mocker.patch.object(renderers.FieldToRender, "is_array_of_objects", True) + assert field_to_render._get_array_items() == renderers.parse_fields.return_value + renderers.parse_fields.assert_called_with([], []) + + def test__get_required_comment(self): + field_to_render = renderers.FieldToRender("field_name", True, {"foo": "bar"}) + field_to_render.required = True + assert field_to_render._get_required_comment() == "REQUIRED" + field_to_render.required = False + assert field_to_render._get_required_comment() == "OPTIONAL" + + def test__get_type_comment(self): + field_to_render = renderers.FieldToRender("field_name", True, {"foo": "bar"}) + field_to_render.type = "mytype" + assert field_to_render._get_type_comment() == "mytype" + field_to_render.type = None + assert field_to_render._get_type_comment() is None + + def test__get_secret_comment(self): + field_to_render = renderers.FieldToRender("field_name", True, {"foo": "bar"}) + field_to_render.airbyte_secret = True + assert field_to_render._get_secret_comment() == "SECRET" + field_to_render.airbyte_secret = False + assert field_to_render._get_secret_comment() is None + + def test__get_description_comment(self): + field_to_render = renderers.FieldToRender("field_name", True, {"foo": "bar"}) + field_to_render.description = "foo" + assert field_to_render._get_description_comment() == "foo" + field_to_render.description = None + assert field_to_render._get_description_comment() is None + + @pytest.mark.parametrize( + "examples_value,expected_output", + [ + (["foo", "bar"], "Examples: foo, bar"), + (["foo"], "Example: foo"), + ("foo", "Example: foo"), + ([5432], "Example: 5432"), + (None, None), + ], + ) + def test__get_example_comment(self, examples_value, expected_output): + field_to_render = renderers.FieldToRender("field_name", True, {"foo": "bar"}) + field_to_render.examples = examples_value + assert field_to_render._get_example_comment() == expected_output + + @pytest.mark.parametrize( + "field_metadata,expected_default", + [ + ({"const": "foo", "default": "bar"}, "foo"), + ({"default": "bar"}, "bar"), + ({}, None), + ], + ) + def test__get_default(self, field_metadata, expected_default): + field_to_render = renderers.FieldToRender("field_name", True, field_metadata) + assert field_to_render.default == expected_default + + def test__build_comment(self, mocker): + comment_functions = [mocker.Mock(return_value="foo"), mocker.Mock(return_value=None), mocker.Mock(return_value="bar")] + comment = renderers.FieldToRender._build_comment(comment_functions) + assert comment == "foo | bar" + + +def test_parse_fields(): + required_fields = ["foo"] + properties = {"foo": {}, "bar": {}} + fields_to_render = renderers.parse_fields(required_fields, properties) + assert fields_to_render[0].name == "foo" + assert fields_to_render[0].required + assert fields_to_render[1].name == "bar" + assert not fields_to_render[1].required + + +def test_get_object_fields(mocker): + mocker.patch.object(renderers, "parse_fields") + field_metadata = {"properties": {"foo": {}, "bar": {}}, "required": ["foo"]} + object_properties = renderers.get_object_fields(field_metadata) + assert object_properties == renderers.parse_fields.return_value + renderers.parse_fields.assert_called_with(["foo"], field_metadata["properties"]) + field_metadata = {} + assert renderers.get_object_fields(field_metadata) == [] + + +class TestBaseRenderer: + @pytest.fixture + def patch_base_class(self, mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(renderers.BaseRenderer, "__abstractmethods__", set()) + + def test_init(self, patch_base_class): + base = renderers.BaseRenderer("resource_name") + assert base.resource_name == "resource_name" + + def test_get_output_path(self, patch_base_class, mocker): + mocker.patch.object(renderers, "os") + renderers.os.path.exists.return_value = False + spec_renderer = renderers.BaseRenderer("my_resource_name") + renderers.os.path.join.side_effect = [ + "./my_definition_types/my_resource_name", + "./my_definition_types/my_resource_name/configuration.yaml", + ] + output_path = spec_renderer._get_output_path(".", "my_definition_type") + renderers.os.makedirs.assert_called_once() + renderers.os.path.join.assert_has_calls( + [ + mocker.call(".", "my_definition_types", "my_resource_name"), + mocker.call("./my_definition_types/my_resource_name", "configuration.yaml"), + ] + ) + assert output_path == "./my_definition_types/my_resource_name/configuration.yaml" + + +class TestConnectorSpecificationRenderer: + def test_init(self, mocker): + assert renderers.ConnectorSpecificationRenderer.TEMPLATE == renderers.JINJA_ENV.get_template("source_or_destination.yaml.j2") + definition = mocker.Mock() + spec_renderer = renderers.ConnectorSpecificationRenderer("my_resource_name", definition) + assert spec_renderer.resource_name == "my_resource_name" + assert spec_renderer.definition == definition + + def test__parse_connection_specification(self, mocker): + mocker.patch.object(renderers, "parse_fields") + schema = {"required": ["foo"], "properties": {"foo": "bar"}} + definition = mocker.Mock() + spec_renderer = renderers.ConnectorSpecificationRenderer("my_resource_name", definition) + parsed_schema = spec_renderer._parse_connection_specification(schema) + assert renderers.parse_fields.call_count == 1 + assert parsed_schema[0], renderers.parse_fields.return_value + renderers.parse_fields.assert_called_with(["foo"], {"foo": "bar"}) + + def test__parse_connection_specification_one_of(self, mocker): + mocker.patch.object(renderers, "parse_fields") + schema = {"oneOf": [{"required": ["foo"], "properties": {"foo": "bar"}}, {"required": ["free"], "properties": {"free": "beer"}}]} + spec_renderer = renderers.ConnectorSpecificationRenderer("my_resource_name", mocker.Mock()) + parsed_schema = spec_renderer._parse_connection_specification(schema) + assert renderers.parse_fields.call_count == 2 + assert parsed_schema[0], renderers.parse_fields.return_value + assert parsed_schema[1], renderers.parse_fields.return_value + assert len(parsed_schema) == len(schema["oneOf"]) + renderers.parse_fields.assert_called_with(["free"], {"free": "beer"}) + + def test_write_yaml(self, mocker): + + mocker.patch.object(renderers.ConnectorSpecificationRenderer, "_get_output_path") + mocker.patch.object(renderers.ConnectorSpecificationRenderer, "_parse_connection_specification") + mocker.patch.object( + renderers.ConnectorSpecificationRenderer, "TEMPLATE", mocker.Mock(render=mocker.Mock(return_value="rendered_string")) + ) + + spec_renderer = renderers.ConnectorSpecificationRenderer("my_resource_name", mocker.Mock(type="source")) + with patch("builtins.open", mock_open()) as mock_file: + output_path = spec_renderer.write_yaml(".") + assert output_path == spec_renderer._get_output_path.return_value + spec_renderer.TEMPLATE.render.assert_called_with( + { + "resource_name": "my_resource_name", + "definition": spec_renderer.definition, + "configuration_fields": spec_renderer._parse_connection_specification.return_value, + } + ) + mock_file.assert_called_with(output_path, "w") + + def test__render(self, mocker): + mocker.patch.object(renderers.ConnectorSpecificationRenderer, "_parse_connection_specification") + mocker.patch.object(renderers.ConnectorSpecificationRenderer, "TEMPLATE") + spec_renderer = renderers.ConnectorSpecificationRenderer("my_resource_name", mocker.Mock()) + rendered = spec_renderer._render() + spec_renderer._parse_connection_specification.assert_called_with(spec_renderer.definition.specification.connection_specification) + spec_renderer.TEMPLATE.render.assert_called_with( + { + "resource_name": spec_renderer.resource_name, + "definition": spec_renderer.definition, + "configuration_fields": spec_renderer._parse_connection_specification.return_value, + } + ) + assert rendered == spec_renderer.TEMPLATE.render.return_value + + +class TestConnectionRenderer: + @pytest.fixture + def mock_source(self, mocker): + return mocker.Mock() + + @pytest.fixture + def mock_destination(self, mocker): + return mocker.Mock() + + def test_init(self, mock_source, mock_destination): + assert renderers.ConnectionRenderer.TEMPLATE == renderers.JINJA_ENV.get_template("connection.yaml.j2") + connection_renderer = renderers.ConnectionRenderer("my_resource_name", mock_source, mock_destination) + assert connection_renderer.resource_name == "my_resource_name" + assert connection_renderer.source == mock_source + assert connection_renderer.destination == mock_destination + + def test_catalog_to_yaml(self, mocker): + catalog = {"camelCase": "camelCase", "snake_case": "camelCase", "myArray": ["a", "b"]} + yaml_catalog = renderers.ConnectionRenderer.catalog_to_yaml(catalog) + assert yaml_catalog == "camelCase: camelCase\nmyArray:\n - a\n - b\nsnake_case: camelCase\n" + + def test_write_yaml(self, mocker, mock_source, mock_destination): + mocker.patch.object(renderers.ConnectionRenderer, "_get_output_path") + mocker.patch.object(renderers.ConnectionRenderer, "catalog_to_yaml") + mocker.patch.object(renderers.ConnectionRenderer, "TEMPLATE") + + connection_renderer = renderers.ConnectionRenderer("my_resource_name", mock_source, mock_destination) + with patch("builtins.open", mock_open()) as mock_file: + output_path = connection_renderer.write_yaml(".") + connection_renderer._get_output_path.assert_called_with(".", renderers.ConnectionDefinition.type) + connection_renderer.catalog_to_yaml.assert_called_with(mock_source.catalog) + mock_file.assert_called_with(output_path, "w") + mock_file.return_value.write.assert_called_with(connection_renderer.TEMPLATE.render.return_value) + connection_renderer.TEMPLATE.render.assert_called_with( + { + "connection_name": connection_renderer.resource_name, + "source_id": mock_source.resource_id, + "destination_id": mock_destination.resource_id, + "catalog": connection_renderer.catalog_to_yaml.return_value, + } + ) + + def test__render(self, mocker): + mocker.patch.object(renderers.ConnectionRenderer, "catalog_to_yaml") + mocker.patch.object(renderers.ConnectionRenderer, "TEMPLATE") + connection_renderer = renderers.ConnectionRenderer("my_connection_name", mocker.Mock(), mocker.Mock()) + rendered = connection_renderer._render() + connection_renderer.catalog_to_yaml.assert_called_with(connection_renderer.source.catalog) + connection_renderer.TEMPLATE.render.assert_called_with( + { + "connection_name": connection_renderer.resource_name, + "source_id": connection_renderer.source.resource_id, + "destination_id": connection_renderer.destination.resource_id, + "catalog": connection_renderer.catalog_to_yaml.return_value, + } + ) + assert rendered == connection_renderer.TEMPLATE.render.return_value