From 74e2e2baccb48e1ac9e6f8a179ce264e72a56021 Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Wed, 21 Aug 2024 15:35:45 -0400 Subject: [PATCH 1/7] fixed schemas and migration script --- .../migrate_to_inline_schemas/pipeline.py | 2 +- migrate.sh | 102 ++++++++++++++++++ 2 files changed, 103 insertions(+), 1 deletion(-) create mode 100755 migrate.sh diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/pipeline.py index 1eff14b52fadf..295eae8f9f1d6 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_inline_schemas/pipeline.py @@ -198,7 +198,7 @@ async def _run(self) -> StepResult: _update_inline_schema(schema_loader, json_streams, stream_name) write_yaml(data, manifest_path) - await format_prettier([manifest_path], logger=logger) + # await format_prettier([manifest_path], logger=logger) for json_stream in json_streams.values(): logger.info(f" !! JSON schema not found: {json_stream.name}") diff --git a/migrate.sh b/migrate.sh new file mode 100755 index 0000000000000..44a1685085836 --- /dev/null +++ b/migrate.sh @@ -0,0 +1,102 @@ +#!/bin/zsh + +# Define an array of connector names +connector_names=("datadog") # Replace these with your actual connector names + +for name in $connector_names; do + echo "Checking out a new branch for $name..." + git checkout -b christo/$name-manifest-only + + # # Step to migrate to inline schemas + # echo "Migrating to inline schemas for $name..." + # airbyte-ci connectors --name=source-$name migrate-to-inline_schemas + + # # Ask user if the inline schema migration was successful + # echo "Was the inline schema migration successful for $name? (y/n):" + # read inline_migration_success + + # if [[ "$inline_migration_success" != "y" ]] && [[ "$inline_migration_success" != "Y" ]]; then + # echo "Inline schema migration not successful for $name. Cleaning up and skipping to next." + # # Clean up the directory to prepare for next task + # git restore airbyte-integrations/connectors/source-$name + # git restore docs/integrations/sources/$name.md + # git clean -fd airbyte-integrations/connectors/source-$name + # echo "Workspace cleaned up. Checking out master for $name" + # git checkout master + # git branch -D christo/$name-manifest-only + # continue + # fi + + echo "Starting manifest-only migration for connector: $name" + + # Migrate connector to manifest-only format + echo "Migrating to manifest-only..." + airbyte-ci connectors --name=source-$name migrate-to-manifest-only + + # Ask user if the manifest-only migration was successful + echo "Was the manifest-only migration successful for $name? (y/n):" + read migration_success + + if [[ "$migration_success" == "y" ]] || [[ "$migration_success" == "Y" ]]; then + echo "Proceeding with the next steps for $name..." + + # Bump connector version + echo "Bumping version..." + airbyte-ci connectors --name=source-$name bump-version minor "Refactor connector to manifest-only format" + + # Format fix + echo "Applying format fixes..." + airbyte-ci format fix js + + # Define the PR body content using mixed quotes for safety + pr_body='## What +Migrates source-'$name' to manifest-only format + +## How +Used a script to run the airbyte-ci commands [migrate-to-manifest-only, bump-version, format fix, pull-request] + +## Review Guide +1. manifest.yaml: should be at the root level of the folder, list version 4.3.0, contain an inlined spec, and all $parameters should be resolved. +2. metadata.py: should list the correct manifest-only language tag, have pypi disabled, and point to a recent version of source-declarative-manifest +3. acceptance-test-config.yml: should correctly point to manifest.yaml for the spec test +4. readme.md: Should be updated with correct references to the connector +5. Pretty much everything else should be nuked + +## User Impact +None' + + # Create pull request + branch_name="christo/$name-manifest-only" + pr_title="Source $name: Migrate to manifest-only" + echo "Creating pull request..." + airbyte-ci connectors --name=source-$name pull-request -m "migrate connector to manifest-only" -b $branch_name --title "$pr_title" --body "$pr_body" + + echo "Migration process completed for $name." + + # Clean up the directory to prepare for next task + echo "Cleaning up workspace for $name..." + git restore airbyte-integrations/connectors/source-$name + git restore docs/integrations/sources/$name.md + git clean -fd airbyte-integrations/connectors/source-$name + echo "Workspace cleaned up. Checking out master for $name" + git checkout master + git branch -D christo/$name-manifest-only + + elif [[ "$migration_success" == "n" ]] || [[ "$migration_success" == "N" ]]; then + echo "Migration not successful for $name. Exiting script." + # Clean up the directory to prepare for next task + echo "Cleaning up workspace for $name..." + git restore airbyte-integrations/connectors/source-$name + git restore docs/integrations/sources/$name.md + git clean -fd airbyte-integrations/connectors/source-$name + echo "Workspace cleaned up. Checking out master for $name" + git checkout master + git branch -D christo/$name-manifest-only + # Continue to the next iteration rather than exiting + continue + else + echo "Invalid input for $name. Exiting script." + # Continue to the next iteration rather than exiting + continue + fi +done From 9fb73b255bf2a12170f2a1743d957763b5805b03 Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Wed, 21 Aug 2024 20:35:14 -0400 Subject: [PATCH 2/7] add type to manifest --- .../connectors/migrate_to_manifest_only/pipeline.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py index ea3024741c36e..31b7584a136a9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py @@ -159,12 +159,16 @@ async def _run(self) -> StepResult: ## 2. Update the version in manifest.yaml try: manifest = read_yaml(root_manifest_path) - manifest["version"] = "4.3.0" + manifest["version"] = "4.5.1" + manifest["type"] = "DeclarativeConnector" # Resolve $parameters and types with CDK magic resolved_manifest = ManifestReferenceResolver().preprocess_manifest(manifest) propagated_manifest = ManifestComponentTransformer().propagate_types_and_parameters("", resolved_manifest, {}) cleaned_manifest = remove_parameters_from_manifest(propagated_manifest) + + manifest.pop("type") + write_yaml(cleaned_manifest, root_manifest_path) except Exception as e: return StepResult(step=self, status=StepStatus.FAILURE, stdout=f"Failed to update version in manifest.yaml: {e}") From 48e27331b643ff1aa6bffe0f09dc3ebadfe7a236 Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Thu, 22 Aug 2024 12:19:01 -0400 Subject: [PATCH 3/7] chore: propagate only for valid components --- .../declarative_component_schema.py | 1627 +++++++++++++++++ .../manifest_component_transformer.py | 69 +- .../migrate_to_manifest_only/pipeline.py | 6 +- 3 files changed, 1696 insertions(+), 6 deletions(-) create mode 100644 airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/declarative_component_schema.py diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/declarative_component_schema.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/declarative_component_schema.py new file mode 100644 index 0000000000000..0b2989a187881 --- /dev/null +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/declarative_component_schema.py @@ -0,0 +1,1627 @@ +# generated by datamodel-codegen: +# filename: declarative_component_schema.yaml + +from __future__ import annotations + +from enum import Enum +from typing import Any, Dict, List, Optional, Union + +from pydantic import BaseModel, Extra, Field +from typing_extensions import Literal + +## This is a stupid hack. I've copied the current version of the models from the generated code and pasted them here since we can't access the CDK from the current directory. +## These models won't update alongside the CDK, so this is a temporary solution to unblock manifest-only migrations. + +class AuthFlowType(Enum): + oauth2_0 = 'oauth2.0' + oauth1_0 = 'oauth1.0' + + +class BasicHttpAuthenticator(BaseModel): + type: Literal['BasicHttpAuthenticator'] + username: str = Field( + ..., + description='The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.', + examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], + title='Username', + ) + password: Optional[str] = Field( + '', + description='The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.', + examples=["{{ config['password'] }}", ''], + title='Password', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class BearerAuthenticator(BaseModel): + type: Literal['BearerAuthenticator'] + api_token: str = Field( + ..., + description='Token to inject as request header for authenticating with the API.', + examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], + title='Bearer Token', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class CheckStream(BaseModel): + type: Literal['CheckStream'] + stream_names: List[str] = Field( + ..., + description='Names of the streams to try reading from when running a check operation.', + examples=[['users'], ['users', 'contacts']], + title='Stream Names', + ) + + +class ConstantBackoffStrategy(BaseModel): + type: Literal['ConstantBackoffStrategy'] + backoff_time_in_seconds: Union[float, str] = Field( + ..., + description='Backoff time in seconds.', + examples=[30, 30.5, "{{ config['backoff_time'] }}"], + title='Backoff Time', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class CustomAuthenticator(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['CustomAuthenticator'] + class_name: str = Field( + ..., + description='Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_..`.', + examples=['source_railz.components.ShortLivedTokenAuthenticator'], + title='Class Name', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class CustomBackoffStrategy(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['CustomBackoffStrategy'] + class_name: str = Field( + ..., + description='Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_..`.', + examples=['source_railz.components.MyCustomBackoffStrategy'], + title='Class Name', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class CustomErrorHandler(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['CustomErrorHandler'] + class_name: str = Field( + ..., + description='Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_..`.', + examples=['source_railz.components.MyCustomErrorHandler'], + title='Class Name', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class CustomIncrementalSync(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['CustomIncrementalSync'] + class_name: str = Field( + ..., + description='Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_..`.', + examples=['source_railz.components.MyCustomIncrementalSync'], + title='Class Name', + ) + cursor_field: str = Field( + ..., + description='The location of the value on a record that will be used as a bookmark during sync.', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class CustomPaginationStrategy(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['CustomPaginationStrategy'] + class_name: str = Field( + ..., + description='Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_..`.', + examples=['source_railz.components.MyCustomPaginationStrategy'], + title='Class Name', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class CustomRecordExtractor(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['CustomRecordExtractor'] + class_name: str = Field( + ..., + description='Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_..`.', + examples=['source_railz.components.MyCustomRecordExtractor'], + title='Class Name', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class CustomRecordFilter(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['CustomRecordFilter'] + class_name: str = Field( + ..., + description='Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_..`.', + examples=['source_railz.components.MyCustomCustomRecordFilter'], + title='Class Name', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class CustomRequester(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['CustomRequester'] + class_name: str = Field( + ..., + description='Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_..`.', + examples=['source_railz.components.MyCustomRecordExtractor'], + title='Class Name', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class CustomRetriever(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['CustomRetriever'] + class_name: str = Field( + ..., + description='Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_..`.', + examples=['source_railz.components.MyCustomRetriever'], + title='Class Name', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class CustomPartitionRouter(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['CustomPartitionRouter'] + class_name: str = Field( + ..., + description='Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_..`.', + examples=['source_railz.components.MyCustomPartitionRouter'], + title='Class Name', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class CustomSchemaLoader(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['CustomSchemaLoader'] + class_name: str = Field( + ..., + description='Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_..`.', + examples=['source_railz.components.MyCustomSchemaLoader'], + title='Class Name', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class CustomStateMigration(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['CustomStateMigration'] + class_name: str = Field( + ..., + description='Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_..`.', + examples=['source_railz.components.MyCustomStateMigration'], + title='Class Name', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class CustomTransformation(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['CustomTransformation'] + class_name: str = Field( + ..., + description='Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_..`.', + examples=['source_railz.components.MyCustomTransformation'], + title='Class Name', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class LegacyToPerPartitionStateMigration(BaseModel): + class Config: + extra = Extra.allow + + type: Optional[Literal['LegacyToPerPartitionStateMigration']] = None + + +class Algorithm(Enum): + HS256 = 'HS256' + HS384 = 'HS384' + HS512 = 'HS512' + ES256 = 'ES256' + ES256K = 'ES256K' + ES384 = 'ES384' + ES512 = 'ES512' + RS256 = 'RS256' + RS384 = 'RS384' + RS512 = 'RS512' + PS256 = 'PS256' + PS384 = 'PS384' + PS512 = 'PS512' + EdDSA = 'EdDSA' + + +class JwtHeaders(BaseModel): + class Config: + extra = Extra.forbid + + kid: Optional[str] = Field( + None, + description='Private key ID for user account.', + examples=["{{ config['kid'] }}"], + title='Key Identifier', + ) + typ: Optional[str] = Field( + 'JWT', + description='The media type of the complete JWT.', + examples=['JWT'], + title='Type', + ) + cty: Optional[str] = Field( + None, + description='Content type of JWT header.', + examples=['JWT'], + title='Content Type', + ) + + +class JwtPayload(BaseModel): + class Config: + extra = Extra.forbid + + iss: Optional[str] = Field( + None, + description='The user/principal that issued the JWT. Commonly a value unique to the user.', + examples=["{{ config['iss'] }}"], + title='Issuer', + ) + sub: Optional[str] = Field( + None, + description='The subject of the JWT. Commonly defined by the API.', + title='Subject', + ) + aud: Optional[str] = Field( + None, + description='The recipient that the JWT is intended for. Commonly defined by the API.', + examples=['appstoreconnect-v1'], + title='Audience', + ) + + +class JwtAuthenticator(BaseModel): + type: Literal['JwtAuthenticator'] + secret_key: str = Field( + ..., + description='Secret used to sign the JSON web token.', + examples=["{{ config['secret_key'] }}"], + ) + base64_encode_secret_key: Optional[bool] = Field( + False, + description='When set to true, the secret key will be base64 encoded prior to being encoded as part of the JWT. Only set to "true" when required by the API.', + ) + algorithm: Algorithm = Field( + ..., + description='Algorithm used to sign the JSON web token.', + examples=['ES256', 'HS256', 'RS256', "{{ config['algorithm'] }}"], + ) + token_duration: Optional[int] = Field( + 1200, + description='The amount of time in seconds a JWT token can be valid after being issued.', + examples=[1200, 3600], + title='Token Duration', + ) + header_prefix: Optional[str] = Field( + None, + description='The prefix to be used within the Authentication header.', + examples=['Bearer', 'Basic'], + title='Header Prefix', + ) + jwt_headers: Optional[JwtHeaders] = Field( + None, + description='JWT headers used when signing JSON web token.', + title='JWT Headers', + ) + additional_jwt_headers: Optional[Dict[str, Any]] = Field( + None, + description='Additional headers to be included with the JWT headers object.', + title='Additional JWT Headers', + ) + jwt_payload: Optional[JwtPayload] = Field( + None, + description='JWT Payload used when signing JSON web token.', + title='JWT Payload', + ) + additional_jwt_payload: Optional[Dict[str, Any]] = Field( + None, + description='Additional properties to be added to the JWT payload.', + title='Additional JWT Payload Properties', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class RefreshTokenUpdater(BaseModel): + refresh_token_name: Optional[str] = Field( + 'refresh_token', + description='The name of the property which contains the updated refresh token in the response from the token refresh endpoint.', + examples=['refresh_token'], + title='Refresh Token Property Name', + ) + access_token_config_path: Optional[List[str]] = Field( + ['credentials', 'access_token'], + description='Config path to the access token. Make sure the field actually exists in the config.', + examples=[['credentials', 'access_token'], ['access_token']], + title='Config Path To Access Token', + ) + refresh_token_config_path: Optional[List[str]] = Field( + ['credentials', 'refresh_token'], + description='Config path to the access token. Make sure the field actually exists in the config.', + examples=[['credentials', 'refresh_token'], ['refresh_token']], + title='Config Path To Refresh Token', + ) + token_expiry_date_config_path: Optional[List[str]] = Field( + ['credentials', 'token_expiry_date'], + description='Config path to the expiry date. Make sure actually exists in the config.', + examples=[['credentials', 'token_expiry_date']], + title='Config Path To Expiry Date', + ) + refresh_token_error_status_codes: Optional[List[int]] = Field( + [], + description='Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error', + examples=[[400, 500]], + title='Refresh Token Error Status Codes', + ) + refresh_token_error_key: Optional[str] = Field( + '', + description='Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).', + examples=['error'], + title='Refresh Token Error Key', + ) + refresh_token_error_values: Optional[List[str]] = Field( + [], + description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', + examples=[['invalid_grant', 'invalid_permissions']], + title='Refresh Token Error Values', + ) + + +class OAuthAuthenticator(BaseModel): + type: Literal['OAuthAuthenticator'] + client_id: str = Field( + ..., + description='The OAuth client ID. Fill it in the user inputs.', + examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], + title='Client ID', + ) + client_secret: str = Field( + ..., + description='The OAuth client secret. Fill it in the user inputs.', + examples=[ + "{{ config['client_secret }}", + "{{ config['credentials']['client_secret }}", + ], + title='Client Secret', + ) + refresh_token: Optional[str] = Field( + None, + description='Credential artifact used to get a new access token.', + examples=[ + "{{ config['refresh_token'] }}", + "{{ config['credentials]['refresh_token'] }}", + ], + title='Refresh Token', + ) + token_refresh_endpoint: str = Field( + ..., + description='The full URL to call to obtain a new access token.', + examples=['https://connect.squareup.com/oauth2/token'], + title='Token Refresh Endpoint', + ) + access_token_name: Optional[str] = Field( + 'access_token', + description='The name of the property which contains the access token in the response from the token refresh endpoint.', + examples=['access_token'], + title='Access Token Property Name', + ) + expires_in_name: Optional[str] = Field( + 'expires_in', + description='The name of the property which contains the expiry date in the response from the token refresh endpoint.', + examples=['expires_in'], + title='Token Expiry Property Name', + ) + grant_type: Optional[str] = Field( + 'refresh_token', + description='Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.', + examples=['refresh_token', 'client_credentials'], + title='Grant Type', + ) + refresh_request_body: Optional[Dict[str, Any]] = Field( + None, + description='Body of the request sent to get a new access token.', + examples=[ + { + 'applicationId': "{{ config['application_id'] }}", + 'applicationSecret': "{{ config['application_secret'] }}", + 'token': "{{ config['token'] }}", + } + ], + title='Refresh Request Body', + ) + scopes: Optional[List[str]] = Field( + None, + description='List of scopes that should be granted to the access token.', + examples=[ + ['crm.list.read', 'crm.objects.contacts.read', 'crm.schema.contacts.read'] + ], + title='Scopes', + ) + token_expiry_date: Optional[str] = Field( + None, + description='The access token expiry date.', + examples=['2023-04-06T07:12:10.421833+00:00', 1680842386], + title='Token Expiry Date', + ) + token_expiry_date_format: Optional[str] = Field( + None, + description='The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.', + examples=['%Y-%m-%d %H:%M:%S.%f+00:00'], + title='Token Expiry Date Format', + ) + refresh_token_updater: Optional[RefreshTokenUpdater] = Field( + None, + description='When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.', + title='Token Updater', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class ExponentialBackoffStrategy(BaseModel): + type: Literal['ExponentialBackoffStrategy'] + factor: Optional[Union[float, str]] = Field( + 5, + description='Multiplicative constant applied on each retry.', + examples=[5, 5.5, '10'], + title='Factor', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class SessionTokenRequestBearerAuthenticator(BaseModel): + type: Literal['Bearer'] + + +class HttpMethod(Enum): + GET = 'GET' + POST = 'POST' + + +class Action(Enum): + SUCCESS = 'SUCCESS' + FAIL = 'FAIL' + RETRY = 'RETRY' + IGNORE = 'IGNORE' + RATE_LIMITED = 'RATE_LIMITED' + + +class FailureType(Enum): + system_error = 'system_error' + config_error = 'config_error' + transient_error = 'transient_error' + + +class HttpResponseFilter(BaseModel): + type: Literal['HttpResponseFilter'] + action: Optional[Action] = Field( + None, + description='Action to execute if a response matches the filter.', + examples=['SUCCESS', 'FAIL', 'RETRY', 'IGNORE', 'RATE_LIMITED'], + title='Action', + ) + failure_type: Optional[FailureType] = Field( + None, + description='Failure type of traced exception if a response matches the filter.', + examples=['system_error', 'config_error', 'transient_error'], + title='Failure Type', + ) + error_message: Optional[str] = Field( + None, + description='Error Message to display if the response matches the filter.', + title='Error Message', + ) + error_message_contains: Optional[str] = Field( + None, + description='Match the response if its error message contains the substring.', + example=['This API operation is not enabled for this site'], + title='Error Message Substring', + ) + http_codes: Optional[List[int]] = Field( + None, + description='Match the response if its HTTP code is included in this list.', + examples=[[420, 429], [500]], + title='HTTP Codes', + ) + predicate: Optional[str] = Field( + None, + description='Match the response if the predicate evaluates to true.', + examples=[ + "{{ 'Too much requests' in response }}", + "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", + ], + title='Predicate', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class InlineSchemaLoader(BaseModel): + type: Literal['InlineSchemaLoader'] + schema_: Optional[Dict[str, Any]] = Field( + None, + alias='schema', + description='Describes a streams\' schema. Refer to the Data Types documentation for more details on which types are valid.', + title='Schema', + ) + + +class JsonFileSchemaLoader(BaseModel): + type: Literal['JsonFileSchemaLoader'] + file_path: Optional[str] = Field( + None, + description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", + example=['./schemas/users.json'], + title='File Path', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class JsonDecoder(BaseModel): + type: Literal['JsonDecoder'] + + +class JsonlDecoder(BaseModel): + type: Literal['JsonlDecoder'] + + +class IterableDecoder(BaseModel): + type: Literal['IterableDecoder'] + + +class MinMaxDatetime(BaseModel): + type: Literal['MinMaxDatetime'] + datetime: str = Field( + ..., + description='Datetime value.', + examples=['2021-01-01', '2021-01-01T00:00:00Z', "{{ config['start_time'] }}"], + title='Datetime', + ) + datetime_format: Optional[str] = Field( + '', + description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', + examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s'], + title='Datetime Format', + ) + max_datetime: Optional[str] = Field( + None, + description='Ceiling applied on the datetime value. Must be formatted with the datetime_format field.', + examples=['2021-01-01T00:00:00Z', '2021-01-01'], + title='Max Datetime', + ) + min_datetime: Optional[str] = Field( + None, + description='Floor applied on the datetime value. Must be formatted with the datetime_format field.', + examples=['2010-01-01T00:00:00Z', '2010-01-01'], + title='Min Datetime', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class NoAuth(BaseModel): + type: Literal['NoAuth'] + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class NoPagination(BaseModel): + type: Literal['NoPagination'] + + +class OAuthConfigSpecification(BaseModel): + class Config: + extra = Extra.allow + + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = ( + Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", + examples=[ + {'app_id': {'type': 'string', 'path_in_connector_config': ['app_id']}}, + { + 'app_id': { + 'type': 'string', + 'path_in_connector_config': ['info', 'app_id'], + } + }, + ], + title='OAuth user input', + ) + ) + complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", + examples=[ + { + 'refresh_token': { + 'type': 'string,', + 'path_in_connector_config': ['credentials', 'refresh_token'], + } + } + ], + title='OAuth output specification', + ) + complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( + None, + description='OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }', + examples=[ + {'client_id': {'type': 'string'}, 'client_secret': {'type': 'string'}} + ], + title='OAuth input specification', + ) + complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", + examples=[ + { + 'client_id': { + 'type': 'string,', + 'path_in_connector_config': ['credentials', 'client_id'], + }, + 'client_secret': { + 'type': 'string,', + 'path_in_connector_config': ['credentials', 'client_secret'], + }, + } + ], + title='OAuth server output specification', + ) + + +class OffsetIncrement(BaseModel): + type: Literal['OffsetIncrement'] + page_size: Optional[Union[int, str]] = Field( + None, + description='The number of records to include in each pages.', + examples=[100, "{{ config['page_size'] }}"], + title='Limit', + ) + inject_on_first_request: Optional[bool] = Field( + False, + description='Using the `offset` with value `0` during the first request', + title='Inject Offset', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class PageIncrement(BaseModel): + type: Literal['PageIncrement'] + page_size: Optional[Union[int, str]] = Field( + None, + description='The number of records to include in each pages.', + examples=[100, '100', "{{ config['page_size'] }}"], + title='Page Size', + ) + start_from_page: Optional[int] = Field( + 0, + description='Index of the first page to request.', + examples=[0, 1], + title='Start From Page', + ) + inject_on_first_request: Optional[bool] = Field( + False, + description='Using the `page number` with value defined by `start_from_page` during the first request', + title='Inject Page Number', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class PrimaryKey(BaseModel): + __root__: Union[str, List[str], List[List[str]]] = Field( + ..., + description='The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.', + examples=['id', ['code', 'type']], + title='Primary Key', + ) + + +class RecordFilter(BaseModel): + type: Literal['RecordFilter'] + condition: Optional[str] = Field( + '', + description='The predicate to filter a record. Records will be removed if evaluated to False.', + examples=[ + "{{ record['created_at'] >= stream_interval['start_time'] }}", + "{{ record.status in ['active', 'expired'] }}", + ], + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class SchemaNormalization(Enum): + None_ = 'None' + Default = 'Default' + + +class RemoveFields(BaseModel): + type: Literal['RemoveFields'] + condition: Optional[str] = Field( + '', + description='The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,', + examples=[ + "{{ property|string == '' }}", + '{{ property is integer }}', + '{{ property|length > 5 }}', + "{{ property == 'some_string_to_match' }}", + ], + ) + field_pointers: List[List[str]] = Field( + ..., + description='Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.', + examples=[['tags'], [['content', 'html'], ['content', 'plain_text']]], + title='Field Paths', + ) + + +class RequestPath(BaseModel): + type: Literal['RequestPath'] + + +class InjectInto(Enum): + request_parameter = 'request_parameter' + header = 'header' + body_data = 'body_data' + body_json = 'body_json' + + +class RequestOption(BaseModel): + type: Literal['RequestOption'] + field_name: str = Field( + ..., + description='Configures which key should be used in the location that the descriptor is being injected into', + examples=['segment_id'], + title='Request Option', + ) + inject_into: InjectInto = Field( + ..., + description='Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.', + examples=['request_parameter', 'header', 'body_data', 'body_json'], + title='Inject Into', + ) + + +class Schemas(BaseModel): + pass + + class Config: + extra = Extra.allow + + +class LegacySessionTokenAuthenticator(BaseModel): + type: Literal['LegacySessionTokenAuthenticator'] + header: str = Field( + ..., + description='The name of the session token header that will be injected in the request', + examples=['X-Session'], + title='Session Request Header', + ) + login_url: str = Field( + ..., + description='Path of the login URL (do not include the base URL)', + examples=['session'], + title='Login Path', + ) + session_token: Optional[str] = Field( + None, + description='Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair', + example=["{{ config['session_token'] }}"], + title='Session Token', + ) + session_token_response_key: str = Field( + ..., + description='Name of the key of the session token to be extracted from the response', + examples=['id'], + title='Response Token Response Key', + ) + username: Optional[str] = Field( + None, + description='Username used to authenticate and obtain a session token', + examples=[" {{ config['username'] }}"], + title='Username', + ) + password: Optional[str] = Field( + '', + description='Password used to authenticate and obtain a session token', + examples=["{{ config['password'] }}", ''], + title='Password', + ) + validate_session_url: str = Field( + ..., + description='Path of the URL to use to validate that the session token is valid (do not include the base URL)', + examples=['user/current'], + title='Validate Session Path', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class ValueType(Enum): + string = 'string' + number = 'number' + integer = 'integer' + boolean = 'boolean' + + +class WaitTimeFromHeader(BaseModel): + type: Literal['WaitTimeFromHeader'] + header: str = Field( + ..., + description='The name of the response header defining how long to wait before retrying.', + examples=['Retry-After'], + title='Response Header Name', + ) + regex: Optional[str] = Field( + None, + description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.', + examples=['([-+]?\\d+)'], + title='Extraction Regex', + ) + max_waiting_time_in_seconds: Optional[float] = Field( + None, + description='Given the value extracted from the header is greater than this value, stop the stream.', + examples=[3600], + title='Max Waiting Time in Seconds', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class WaitUntilTimeFromHeader(BaseModel): + type: Literal['WaitUntilTimeFromHeader'] + header: str = Field( + ..., + description='The name of the response header defining how long to wait before retrying.', + examples=['wait_time'], + title='Response Header', + ) + min_wait: Optional[Union[float, str]] = Field( + None, + description='Minimum time to wait before retrying.', + examples=[10, '60'], + title='Minimum Wait Time', + ) + regex: Optional[str] = Field( + None, + description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.', + examples=['([-+]?\\d+)'], + title='Extraction Regex', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class AddedFieldDefinition(BaseModel): + type: Literal['AddedFieldDefinition'] + path: List[str] = Field( + ..., + description='List of strings defining the path where to add the value on the record.', + examples=[['segment_id'], ['metadata', 'segment_id']], + title='Path', + ) + value: str = Field( + ..., + description="Value of the new field. Use {{ record['existing_field'] }} syntax to refer to other fields in the record.", + examples=[ + "{{ record['updates'] }}", + "{{ record['MetaData']['LastUpdatedTime'] }}", + "{{ stream_partition['segment_id'] }}", + ], + title='Value', + ) + value_type: Optional[ValueType] = Field( + None, + description='Type of the value. If not specified, the type will be inferred from the value.', + title='Value Type', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class AddFields(BaseModel): + type: Literal['AddFields'] + fields: List[AddedFieldDefinition] = Field( + ..., + description='List of transformations (path and corresponding value) that will be added to the record.', + title='Fields', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class ApiKeyAuthenticator(BaseModel): + type: Literal['ApiKeyAuthenticator'] + api_token: Optional[str] = Field( + None, + description='The API key to inject in the request. Fill it in the user inputs.', + examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], + title='API Key', + ) + header: Optional[str] = Field( + None, + description='The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.', + examples=['Authorization', 'Api-Token', 'X-Auth-Token'], + title='Header Name', + ) + inject_into: Optional[RequestOption] = Field( + None, + description='Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.', + examples=[ + {'inject_into': 'header', 'field_name': 'Authorization'}, + {'inject_into': 'request_parameter', 'field_name': 'authKey'}, + ], + title='Inject API Key Into Outgoing HTTP Request', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class AuthFlow(BaseModel): + auth_flow_type: Optional[AuthFlowType] = Field( + None, description='The type of auth to use', title='Auth flow type' + ) + predicate_key: Optional[List[str]] = Field( + None, + description='JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.', + examples=[['credentials', 'auth_type']], + title='Predicate key', + ) + predicate_value: Optional[str] = Field( + None, + description='Value of the predicate_key fields for the advanced auth to be applicable.', + examples=['Oauth'], + title='Predicate value', + ) + oauth_config_specification: Optional[OAuthConfigSpecification] = None + + +class CursorPagination(BaseModel): + type: Literal['CursorPagination'] + cursor_value: str = Field( + ..., + description='Value of the cursor defining the next page to fetch.', + examples=[ + '{{ headers.link.next.cursor }}', + "{{ last_record['key'] }}", + "{{ response['nextPage'] }}", + ], + title='Cursor Value', + ) + page_size: Optional[int] = Field( + None, + description='The number of records to include in each pages.', + examples=[100], + title='Page Size', + ) + stop_condition: Optional[str] = Field( + None, + description='Template string evaluating when to stop paginating.', + examples=[ + '{{ response.data.has_more is false }}', + "{{ 'next' not in headers['link'] }}", + ], + title='Stop Condition', + ) + decoder: Optional[JsonDecoder] = Field( + None, + description='Component decoding the response so records can be extracted.', + title='Decoder', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class DatetimeBasedCursor(BaseModel): + type: Literal['DatetimeBasedCursor'] + cursor_field: str = Field( + ..., + description='The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.', + examples=['created_at', "{{ config['record_cursor'] }}"], + title='Cursor Field', + ) + datetime_format: str = Field( + ..., + description='The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', + examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s', '%ms', '%s_as_float'], + title='Outgoing Datetime Format', + ) + start_datetime: Union[str, MinMaxDatetime] = Field( + ..., + description='The datetime that determines the earliest record that should be synced.', + examples=['2020-01-1T00:00:00Z', "{{ config['start_time'] }}"], + title='Start Datetime', + ) + cursor_datetime_formats: Optional[List[str]] = Field( + None, + description='The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.', + title='Cursor Datetime Formats', + ) + cursor_granularity: Optional[str] = Field( + None, + description='Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.', + examples=['PT1S'], + title='Cursor Granularity', + ) + end_datetime: Optional[Union[str, MinMaxDatetime]] = Field( + None, + description='The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.', + examples=['2021-01-1T00:00:00Z', '{{ now_utc() }}', '{{ day_delta(-1) }}'], + title='End Datetime', + ) + end_time_option: Optional[RequestOption] = Field( + None, + description='Optionally configures how the end datetime will be sent in requests to the source API.', + title='Inject End Time Into Outgoing HTTP Request', + ) + is_data_feed: Optional[bool] = Field( + None, + description='A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.', + title='Whether the target API is formatted as a data feed', + ) + is_client_side_incremental: Optional[bool] = Field( + None, + description='If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.', + title='Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)', + ) + is_compare_strictly: Optional[bool] = Field( + False, + description='Set to True if the target API does not accept queries where the start time equal the end time.', + title='Whether to skip requests if the start time equals the end time', + ) + lookback_window: Optional[str] = Field( + None, + description='Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.', + examples=['P1D', "P{{ config['lookback_days'] }}D"], + title='Lookback Window', + ) + partition_field_end: Optional[str] = Field( + None, + description='Name of the partition start time field.', + examples=['ending_time'], + title='Partition Field End', + ) + partition_field_start: Optional[str] = Field( + None, + description='Name of the partition end time field.', + examples=['starting_time'], + title='Partition Field Start', + ) + start_time_option: Optional[RequestOption] = Field( + None, + description='Optionally configures how the start datetime will be sent in requests to the source API.', + title='Inject Start Time Into Outgoing HTTP Request', + ) + step: Optional[str] = Field( + None, + description='The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.', + examples=['P1W', "{{ config['step_increment'] }}"], + title='Step', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class DefaultErrorHandler(BaseModel): + type: Literal['DefaultErrorHandler'] + backoff_strategies: Optional[ + List[ + Union[ + ConstantBackoffStrategy, + CustomBackoffStrategy, + ExponentialBackoffStrategy, + WaitTimeFromHeader, + WaitUntilTimeFromHeader, + ] + ] + ] = Field( + None, + description='List of backoff strategies to use to determine how long to wait before retrying a retryable request.', + title='Backoff Strategies', + ) + max_retries: Optional[int] = Field( + 5, + description='The maximum number of time to retry a retryable request before giving up and failing.', + examples=[5, 0, 10], + title='Max Retry Count', + ) + response_filters: Optional[List[HttpResponseFilter]] = Field( + None, + description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", + title='Response Filters', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class DefaultPaginator(BaseModel): + type: Literal['DefaultPaginator'] + pagination_strategy: Union[ + CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement + ] = Field( + ..., + description='Strategy defining how records are paginated.', + title='Pagination Strategy', + ) + decoder: Optional[JsonDecoder] = Field( + None, + description='Component decoding the response so records can be extracted.', + title='Decoder', + ) + page_size_option: Optional[RequestOption] = None + page_token_option: Optional[Union[RequestOption, RequestPath]] = None + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class DpathExtractor(BaseModel): + type: Literal['DpathExtractor'] + field_path: List[str] = Field( + ..., + description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', + examples=[ + ['data'], + ['data', 'records'], + ['data', '{{ parameters.name }}'], + ['data', '*', 'record'], + ], + title='Field Path', + ) + decoder: Optional[Union[JsonDecoder, JsonlDecoder, IterableDecoder]] = Field( + None, title='Decoder' + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class SessionTokenRequestApiKeyAuthenticator(BaseModel): + type: Literal['ApiKey'] + inject_into: RequestOption = Field( + ..., + description='Configure how the API Key will be sent in requests to the source API.', + examples=[ + {'inject_into': 'header', 'field_name': 'Authorization'}, + {'inject_into': 'request_parameter', 'field_name': 'authKey'}, + ], + title='Inject API Key Into Outgoing HTTP Request', + ) + + +class ListPartitionRouter(BaseModel): + type: Literal['ListPartitionRouter'] + cursor_field: str = Field( + ..., + description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', + examples=['section', "{{ config['section_key'] }}"], + title='Current Partition Value Identifier', + ) + values: Union[str, List[str]] = Field( + ..., + description='The list of attributes being iterated over and used as input for the requests made to the source API.', + examples=[['section_a', 'section_b', 'section_c'], "{{ config['sections'] }}"], + title='Partition Values', + ) + request_option: Optional[RequestOption] = Field( + None, + description='A request option describing where the list value should be injected into and under what field name if applicable.', + title='Inject Partition Value Into Outgoing HTTP Request', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class RecordSelector(BaseModel): + type: Literal['RecordSelector'] + extractor: Union[CustomRecordExtractor, DpathExtractor] + record_filter: Optional[Union[CustomRecordFilter, RecordFilter]] = Field( + None, + description='Responsible for filtering records to be emitted by the Source.', + title='Record Filter', + ) + schema_normalization: Optional[SchemaNormalization] = SchemaNormalization.None_ + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class Spec(BaseModel): + type: Literal['Spec'] + connection_specification: Dict[str, Any] = Field( + ..., + description='A connection specification describing how a the connector can be configured.', + title='Connection Specification', + ) + documentation_url: Optional[str] = Field( + None, + description="URL of the connector's documentation page.", + examples=['https://docs.airbyte.com/integrations/sources/dremio'], + title='Documentation URL', + ) + advanced_auth: Optional[AuthFlow] = Field( + None, + description='Advanced specification for configuring the authentication flow.', + title='Advanced Auth', + ) + + +class CompositeErrorHandler(BaseModel): + type: Literal['CompositeErrorHandler'] + error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( + ..., + description='List of error handlers to iterate on to determine how to handle a failed response.', + title='Error Handlers', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class DeclarativeSource(BaseModel): + class Config: + extra = Extra.forbid + + type: Literal['DeclarativeSource'] + check: CheckStream + streams: List[DeclarativeStream] + version: str = Field( + ..., + description='The version of the Airbyte CDK used to build and test the source.', + ) + schemas: Optional[Schemas] = None + definitions: Optional[Dict[str, Any]] = None + spec: Optional[Spec] = None + metadata: Optional[Dict[str, Any]] = Field( + None, + description='For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.', + ) + description: Optional[str] = Field( + None, + description='A description of the connector. It will be presented on the Source documentation page.', + ) + + +class SelectiveAuthenticator(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['SelectiveAuthenticator'] + authenticator_selection_path: List[str] = Field( + ..., + description='Path of the field in config with selected authenticator name', + examples=[['auth'], ['auth', 'type']], + title='Authenticator Selection Path', + ) + authenticators: Dict[ + str, + Union[ + ApiKeyAuthenticator, + BasicHttpAuthenticator, + BearerAuthenticator, + CustomAuthenticator, + OAuthAuthenticator, + JwtAuthenticator, + NoAuth, + SessionTokenAuthenticator, + LegacySessionTokenAuthenticator, + ], + ] = Field( + ..., + description='Authenticators to select from.', + examples=[ + { + 'authenticators': { + 'token': '#/definitions/ApiKeyAuthenticator', + 'oauth': '#/definitions/OAuthAuthenticator', + 'jwt': '#/definitions/JwtAuthenticator', + } + } + ], + title='Authenticators', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class DeclarativeStream(BaseModel): + class Config: + extra = Extra.allow + + type: Literal['DeclarativeStream'] + retriever: Union[CustomRetriever, SimpleRetriever] = Field( + ..., + description='Component used to coordinate how records are extracted across stream slices and request pages.', + title='Retriever', + ) + incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = ( + Field( + None, + description='Component used to fetch data incrementally based on a time field in the data.', + title='Incremental Sync', + ) + ) + name: Optional[str] = Field( + '', description='The stream name.', example=['Users'], title='Name' + ) + primary_key: Optional[PrimaryKey] = Field( + '', description='The primary key of the stream.', title='Primary Key' + ) + schema_loader: Optional[ + Union[InlineSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader] + ] = Field( + None, + description='Component used to retrieve the schema for the current stream.', + title='Schema Loader', + ) + transformations: Optional[ + List[Union[AddFields, CustomTransformation, RemoveFields]] + ] = Field( + None, + description='A list of transformations to be applied to each output record.', + title='Transformations', + ) + state_migrations: Optional[ + List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] + ] = Field( + [], + description='Array of state migrations to be applied on the input state', + title='State Migrations', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class SessionTokenAuthenticator(BaseModel): + type: Literal['SessionTokenAuthenticator'] + login_requester: HttpRequester = Field( + ..., + description='Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.', + examples=[ + { + 'type': 'HttpRequester', + 'url_base': 'https://my_api.com', + 'path': '/login', + 'authenticator': { + 'type': 'BasicHttpAuthenticator', + 'username': '{{ config.username }}', + 'password': '{{ config.password }}', + }, + } + ], + title='Login Requester', + ) + session_token_path: List[str] = Field( + ..., + description='The path in the response body returned from the login requester to the session token.', + examples=[['access_token'], ['result', 'token']], + title='Session Token Path', + ) + expiration_duration: Optional[str] = Field( + None, + description='The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.', + examples=['PT1H', 'P1D'], + title='Expiration Duration', + ) + request_authentication: Union[ + SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator + ] = Field( + ..., + description='Authentication method to use for requests sent to the API, specifying how to inject the session token.', + title='Data Request Authentication', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class HttpRequester(BaseModel): + type: Literal['HttpRequester'] + url_base: str = Field( + ..., + description='Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.', + examples=[ + 'https://connect.squareup.com/v2', + "{{ config['base_url'] or 'https://app.posthog.com'}}/api/", + ], + title='API Base URL', + ) + path: str = Field( + ..., + description='Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.', + examples=[ + '/products', + "/quotes/{{ stream_partition['id'] }}/quote_line_groups", + "/trades/{{ config['symbol_id'] }}/history", + ], + title='URL Path', + ) + authenticator: Optional[ + Union[ + ApiKeyAuthenticator, + BasicHttpAuthenticator, + BearerAuthenticator, + CustomAuthenticator, + OAuthAuthenticator, + JwtAuthenticator, + NoAuth, + SessionTokenAuthenticator, + LegacySessionTokenAuthenticator, + SelectiveAuthenticator, + ] + ] = Field( + None, + description='Authentication method to use for requests sent to the API.', + title='Authenticator', + ) + error_handler: Optional[ + Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler] + ] = Field( + None, + description='Error handler component that defines how to handle errors.', + title='Error Handler', + ) + http_method: Optional[HttpMethod] = Field( + HttpMethod.GET, + description='The HTTP method used to fetch data from the source (can be GET or POST).', + examples=['GET', 'POST'], + title='HTTP Method', + ) + request_body_data: Optional[Union[str, Dict[str, str]]] = Field( + None, + description='Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.', + examples=[ + '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' + ], + title='Request Body Payload (Non-JSON)', + ) + request_body_json: Optional[Union[str, Dict[str, Any]]] = Field( + None, + description='Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.', + examples=[ + {'sort_order': 'ASC', 'sort_field': 'CREATED_AT'}, + {'key': "{{ config['value'] }}"}, + {'sort': {'field': 'updated_at', 'order': 'ascending'}}, + ], + title='Request Body JSON Payload', + ) + request_headers: Optional[Union[str, Dict[str, str]]] = Field( + None, + description='Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.', + examples=[{'Output-Format': 'JSON'}, {'Version': "{{ config['version'] }}"}], + title='Request Headers', + ) + request_parameters: Optional[Union[str, Dict[str, str]]] = Field( + None, + description='Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.', + examples=[ + {'unit': 'day'}, + { + 'query': 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + }, + {'searchIn': "{{ ','.join(config.get('search_in', [])) }}"}, + {'sort_by[asc]': 'updated_at'}, + ], + title='Query Parameters', + ) + use_cache: Optional[bool] = Field( + False, + description='Enables stream requests caching. This field is automatically set by the CDK.', + title='Use Cache', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class ParentStreamConfig(BaseModel): + type: Literal['ParentStreamConfig'] + parent_key: str = Field( + ..., + description='The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.', + examples=['id', "{{ config['parent_record_id'] }}"], + title='Parent Key', + ) + stream: DeclarativeStream = Field( + ..., description='Reference to the parent stream.', title='Parent Stream' + ) + partition_field: str = Field( + ..., + description='While iterating over parent records during a sync, the parent_key value can be referenced by using this field.', + examples=['parent_id', "{{ config['parent_partition_field'] }}"], + title='Current Parent Key Value Identifier', + ) + request_option: Optional[RequestOption] = Field( + None, + description='A request option describing where the parent key value should be injected into and under what field name if applicable.', + title='Request Option', + ) + incremental_dependency: Optional[bool] = Field( + False, + description='Indicates whether the parent stream should be read incrementally based on updates in the child stream.', + title='Incremental Dependency', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class SimpleRetriever(BaseModel): + type: Literal['SimpleRetriever'] + record_selector: RecordSelector = Field( + ..., + description='Component that describes how to extract records from a HTTP response.', + ) + requester: Union[CustomRequester, HttpRequester] = Field( + ..., + description='Requester component that describes how to prepare HTTP requests to send to the source API.', + ) + paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( + None, + description="Paginator component that describes how to navigate through the API's pages.", + ) + ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( + False, + description='If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.', + ) + partition_router: Optional[ + Union[ + CustomPartitionRouter, + ListPartitionRouter, + SubstreamPartitionRouter, + List[ + Union[ + CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter + ] + ], + ] + ] = Field( + [], + description='PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.', + title='Partition Router', + ) + decoder: Optional[Union[JsonDecoder, JsonlDecoder, IterableDecoder]] = Field( + None, + description='Component decoding the response so records can be extracted.', + title='Decoder', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +class SubstreamPartitionRouter(BaseModel): + type: Literal['SubstreamPartitionRouter'] + parent_stream_configs: List[ParentStreamConfig] = Field( + ..., + description='Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.', + title='Parent Stream Configs', + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + + +CompositeErrorHandler.update_forward_refs() +DeclarativeSource.update_forward_refs() +SelectiveAuthenticator.update_forward_refs() +DeclarativeStream.update_forward_refs() +SessionTokenAuthenticator.update_forward_refs() +SimpleRetriever.update_forward_refs() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/manifest_component_transformer.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/manifest_component_transformer.py index b3dfef8eb43ce..8efa4d6185804 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/manifest_component_transformer.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/manifest_component_transformer.py @@ -4,10 +4,61 @@ import copy import typing -from typing import Any, Mapping +import logging +from typing import Any, Mapping, Set, Type +from .declarative_component_schema import * +from pydantic import BaseModel PARAMETERS_STR = "$parameters" +# Mapping of component type to the class that implements it. This is used to fetch the Pydantic model class for a component type +COMPONENT_TYPE_REGISTY: Dict[str, type] = { + 'BasicHttpAuthenticator': BasicHttpAuthenticator, + 'BearerAuthenticator': BearerAuthenticator, + 'ConstantBackoffStrategy': ConstantBackoffStrategy, + 'CustomAuthenticator': CustomAuthenticator, + 'CustomBackoffStrategy': CustomBackoffStrategy, + 'CustomErrorHandler': CustomErrorHandler, + 'CustomIncrementalSync': CustomIncrementalSync, + 'CustomPaginationStrategy': CustomPaginationStrategy, + 'CustomRecordExtractor': CustomRecordExtractor, + 'CustomRecordFilter': CustomRecordFilter, + 'CustomRequester': CustomRequester, + 'CustomRetriever': CustomRetriever, + 'CustomPartitionRouter': CustomPartitionRouter, + 'CustomSchemaLoader': CustomSchemaLoader, + 'CustomStateMigration': CustomStateMigration, + 'CustomTransformation': CustomTransformation, + 'JwtAuthenticator': JwtAuthenticator, + '0AuthAuthenticator': OAuthAuthenticator, + 'ExponentialBackoffStrategy': ExponentialBackoffStrategy, + 'HttpResponseFilter': HttpResponseFilter, + 'JsonFileSchemaLoader': JsonFileSchemaLoader, + 'MinMaxDatetime': MinMaxDatetime, + 'OffsetIncrement': OffsetIncrement, + 'PageIncrement': PageIncrement, + 'RecordFilter': RecordFilter, + 'LegacySessionTokenAuthenticator': LegacySessionTokenAuthenticator, + 'WaitTimeFromHeader': WaitTimeFromHeader, + 'WaitUntilTimeFromHeader': WaitUntilTimeFromHeader, + 'ApiKeyAuthenticator': ApiKeyAuthenticator, + 'CursorPagination': CursorPagination, + 'DatetimeBasedCursor': DatetimeBasedCursor, + 'DefaultErrorHandler': DefaultErrorHandler, + 'DefaultPaginator': DefaultPaginator, + 'DpathExtractor': DpathExtractor, + 'ListPartitionRouter': ListPartitionRouter, + 'RecordSelector': RecordSelector, + 'CompositeErrorHandler': CompositeErrorHandler, + 'SelectiveAuthenticator': SelectiveAuthenticator, + 'DeclarativeStream': DeclarativeStream, + 'SessionTokenAuthenticator': SessionTokenAuthenticator, + 'HttpRequester': HttpRequester, + 'ParentStreamConfig': ParentStreamConfig, + 'SimpleRetriever': SimpleRetriever, + 'SubstreamPartitionRouter': SubstreamPartitionRouter, + 'DeclarativeSource': DeclarativeSource +} DEFAULT_MODEL_TYPES: Mapping[str, str] = { # CompositeErrorHandler @@ -74,6 +125,13 @@ "SimpleRetriever.partition_router": "CustomPartitionRouter", } +logger = logging.getLogger(__name__) + +def get_model_fields(model_class: Optional[Type[BaseModel]]) -> Set[str]: + """ Fetches field names from a Pydantic model class if available. """ + if model_class is not None: + return set(model_class.__fields__.keys()) + return set() class ManifestComponentTransformer: def propagate_types_and_parameters( @@ -111,6 +169,11 @@ def propagate_types_and_parameters( # * connection_specification is a Mapping if "type" not in propagated_component or self._is_json_schema_object(propagated_component): return propagated_component + + component_type = propagated_component.get("type", "") + model_class = COMPONENT_TYPE_REGISTY.get(component_type) + # Grab the list of expected fields for the component type + valid_fields = get_model_fields(model_class) # Combines parameters defined at the current level with parameters from parent components. Parameters at the current # level take precedence @@ -121,7 +184,9 @@ def propagate_types_and_parameters( # Parameters should be applied to the current component fields with the existing field taking precedence over parameters if # both exist for parameter_key, parameter_value in current_parameters.items(): - propagated_component[parameter_key] = propagated_component.get(parameter_key) or parameter_value + if parameter_key in valid_fields: + logger.info(f"Adding parameter {parameter_key} to {component_type}") + propagated_component[parameter_key] = propagated_component.get(parameter_key) or parameter_value for field_name, field_value in propagated_component.items(): if isinstance(field_value, dict): diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py index 31b7584a136a9..2483ce61bdb57 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py @@ -159,15 +159,13 @@ async def _run(self) -> StepResult: ## 2. Update the version in manifest.yaml try: manifest = read_yaml(root_manifest_path) - manifest["version"] = "4.5.1" - manifest["type"] = "DeclarativeConnector" + manifest["version"] = "4.3.2" + manifest["type"] = "DeclarativeSource" # Resolve $parameters and types with CDK magic resolved_manifest = ManifestReferenceResolver().preprocess_manifest(manifest) propagated_manifest = ManifestComponentTransformer().propagate_types_and_parameters("", resolved_manifest, {}) cleaned_manifest = remove_parameters_from_manifest(propagated_manifest) - - manifest.pop("type") write_yaml(cleaned_manifest, root_manifest_path) except Exception as e: From 39b1e5db8be32f79ecdcec3f88d8060a9f57b6da Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Thu, 22 Aug 2024 12:20:54 -0400 Subject: [PATCH 4/7] chore: remove migration script --- migrate.sh | 102 ----------------------------------------------------- 1 file changed, 102 deletions(-) delete mode 100755 migrate.sh diff --git a/migrate.sh b/migrate.sh deleted file mode 100755 index 44a1685085836..0000000000000 --- a/migrate.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/zsh - -# Define an array of connector names -connector_names=("datadog") # Replace these with your actual connector names - -for name in $connector_names; do - echo "Checking out a new branch for $name..." - git checkout -b christo/$name-manifest-only - - # # Step to migrate to inline schemas - # echo "Migrating to inline schemas for $name..." - # airbyte-ci connectors --name=source-$name migrate-to-inline_schemas - - # # Ask user if the inline schema migration was successful - # echo "Was the inline schema migration successful for $name? (y/n):" - # read inline_migration_success - - # if [[ "$inline_migration_success" != "y" ]] && [[ "$inline_migration_success" != "Y" ]]; then - # echo "Inline schema migration not successful for $name. Cleaning up and skipping to next." - # # Clean up the directory to prepare for next task - # git restore airbyte-integrations/connectors/source-$name - # git restore docs/integrations/sources/$name.md - # git clean -fd airbyte-integrations/connectors/source-$name - # echo "Workspace cleaned up. Checking out master for $name" - # git checkout master - # git branch -D christo/$name-manifest-only - # continue - # fi - - echo "Starting manifest-only migration for connector: $name" - - # Migrate connector to manifest-only format - echo "Migrating to manifest-only..." - airbyte-ci connectors --name=source-$name migrate-to-manifest-only - - # Ask user if the manifest-only migration was successful - echo "Was the manifest-only migration successful for $name? (y/n):" - read migration_success - - if [[ "$migration_success" == "y" ]] || [[ "$migration_success" == "Y" ]]; then - echo "Proceeding with the next steps for $name..." - - # Bump connector version - echo "Bumping version..." - airbyte-ci connectors --name=source-$name bump-version minor "Refactor connector to manifest-only format" - - # Format fix - echo "Applying format fixes..." - airbyte-ci format fix js - - # Define the PR body content using mixed quotes for safety - pr_body='## What -Migrates source-'$name' to manifest-only format - -## How -Used a script to run the airbyte-ci commands [migrate-to-manifest-only, bump-version, format fix, pull-request] - -## Review Guide -1. manifest.yaml: should be at the root level of the folder, list version 4.3.0, contain an inlined spec, and all $parameters should be resolved. -2. metadata.py: should list the correct manifest-only language tag, have pypi disabled, and point to a recent version of source-declarative-manifest -3. acceptance-test-config.yml: should correctly point to manifest.yaml for the spec test -4. readme.md: Should be updated with correct references to the connector -5. Pretty much everything else should be nuked - -## User Impact -None' - - # Create pull request - branch_name="christo/$name-manifest-only" - pr_title="Source $name: Migrate to manifest-only" - echo "Creating pull request..." - airbyte-ci connectors --name=source-$name pull-request -m "migrate connector to manifest-only" -b $branch_name --title "$pr_title" --body "$pr_body" - - echo "Migration process completed for $name." - - # Clean up the directory to prepare for next task - echo "Cleaning up workspace for $name..." - git restore airbyte-integrations/connectors/source-$name - git restore docs/integrations/sources/$name.md - git clean -fd airbyte-integrations/connectors/source-$name - echo "Workspace cleaned up. Checking out master for $name" - git checkout master - git branch -D christo/$name-manifest-only - - elif [[ "$migration_success" == "n" ]] || [[ "$migration_success" == "N" ]]; then - echo "Migration not successful for $name. Exiting script." - # Clean up the directory to prepare for next task - echo "Cleaning up workspace for $name..." - git restore airbyte-integrations/connectors/source-$name - git restore docs/integrations/sources/$name.md - git clean -fd airbyte-integrations/connectors/source-$name - echo "Workspace cleaned up. Checking out master for $name" - git checkout master - git branch -D christo/$name-manifest-only - # Continue to the next iteration rather than exiting - continue - else - echo "Invalid input for $name. Exiting script." - # Continue to the next iteration rather than exiting - continue - fi -done From b067aaf8ef23ec5d545c78d6df4993ca317000d9 Mon Sep 17 00:00:00 2001 From: ChristoGrab Date: Thu, 22 Aug 2024 13:03:11 -0400 Subject: [PATCH 5/7] format and changelog --- airbyte-ci/connectors/pipelines/README.md | 1 + .../declarative_component_schema.py | 1291 ++++++++--------- .../manifest_component_transformer.py | 102 +- .../migrate_to_manifest_only/pipeline.py | 2 +- .../connectors/pipelines/pyproject.toml | 2 +- 5 files changed, 686 insertions(+), 712 deletions(-) diff --git a/airbyte-ci/connectors/pipelines/README.md b/airbyte-ci/connectors/pipelines/README.md index b94a76beb8825..258382d04c64d 100644 --- a/airbyte-ci/connectors/pipelines/README.md +++ b/airbyte-ci/connectors/pipelines/README.md @@ -843,6 +843,7 @@ airbyte-ci connectors --language=low-code migrate-to-manifest-only | Version | PR | Description | | ------- | ---------------------------------------------------------- |------------------------------------------------------------------------------------------------------------------------------| +| 4.33.2 | [#44557](https://github.com/airbytehq/airbyte/pull/44557) | Conditionally propagate parameters in manifest-only migration | | 4.33.1 | [#44465](https://github.com/airbytehq/airbyte/pull/44465) | Ignore version check if only erd folder is changed | | 4.33.0 | [#44377](https://github.com/airbytehq/airbyte/pull/44377) | Upload connector SBOM to metadata service bucket on publish. | | 4.32.5 | [#44173](https://github.com/airbytehq/airbyte/pull/44173) | Bug fix for live tests' --should-read-with-state handling. | diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/declarative_component_schema.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/declarative_component_schema.py index 0b2989a187881..810d52efac27b 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/declarative_component_schema.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/declarative_component_schema.py @@ -1,3 +1,5 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + # generated by datamodel-codegen: # filename: declarative_component_schema.yaml @@ -12,268 +14,269 @@ ## This is a stupid hack. I've copied the current version of the models from the generated code and pasted them here since we can't access the CDK from the current directory. ## These models won't update alongside the CDK, so this is a temporary solution to unblock manifest-only migrations. + class AuthFlowType(Enum): - oauth2_0 = 'oauth2.0' - oauth1_0 = 'oauth1.0' + oauth2_0 = "oauth2.0" + oauth1_0 = "oauth1.0" class BasicHttpAuthenticator(BaseModel): - type: Literal['BasicHttpAuthenticator'] + type: Literal["BasicHttpAuthenticator"] username: str = Field( ..., - description='The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.', + description="The username that will be combined with the password, base64 encoded and used to make requests. Fill it in the user inputs.", examples=["{{ config['username'] }}", "{{ config['api_key'] }}"], - title='Username', + title="Username", ) password: Optional[str] = Field( - '', - description='The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.', - examples=["{{ config['password'] }}", ''], - title='Password', + "", + description="The password that will be combined with the username, base64 encoded and used to make requests. Fill it in the user inputs.", + examples=["{{ config['password'] }}", ""], + title="Password", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class BearerAuthenticator(BaseModel): - type: Literal['BearerAuthenticator'] + type: Literal["BearerAuthenticator"] api_token: str = Field( ..., - description='Token to inject as request header for authenticating with the API.', + description="Token to inject as request header for authenticating with the API.", examples=["{{ config['api_key'] }}", "{{ config['token'] }}"], - title='Bearer Token', + title="Bearer Token", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CheckStream(BaseModel): - type: Literal['CheckStream'] + type: Literal["CheckStream"] stream_names: List[str] = Field( ..., - description='Names of the streams to try reading from when running a check operation.', - examples=[['users'], ['users', 'contacts']], - title='Stream Names', + description="Names of the streams to try reading from when running a check operation.", + examples=[["users"], ["users", "contacts"]], + title="Stream Names", ) class ConstantBackoffStrategy(BaseModel): - type: Literal['ConstantBackoffStrategy'] + type: Literal["ConstantBackoffStrategy"] backoff_time_in_seconds: Union[float, str] = Field( ..., - description='Backoff time in seconds.', + description="Backoff time in seconds.", examples=[30, 30.5, "{{ config['backoff_time'] }}"], - title='Backoff Time', + title="Backoff Time", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomAuthenticator(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomAuthenticator'] + type: Literal["CustomAuthenticator"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_..`.', - examples=['source_railz.components.ShortLivedTokenAuthenticator'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom authentication strategy. Has to be a sub class of DeclarativeAuthenticator. The format is `source_..`.", + examples=["source_railz.components.ShortLivedTokenAuthenticator"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomBackoffStrategy(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomBackoffStrategy'] + type: Literal["CustomBackoffStrategy"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomBackoffStrategy'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom backoff strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomBackoffStrategy"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomErrorHandler(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomErrorHandler'] + type: Literal["CustomErrorHandler"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_..`.', - examples=['source_railz.components.MyCustomErrorHandler'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom error handler. The format is `source_..`.", + examples=["source_railz.components.MyCustomErrorHandler"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomIncrementalSync(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomIncrementalSync'] + type: Literal["CustomIncrementalSync"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_..`.', - examples=['source_railz.components.MyCustomIncrementalSync'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom incremental sync. The format is `source_..`.", + examples=["source_railz.components.MyCustomIncrementalSync"], + title="Class Name", ) cursor_field: str = Field( ..., - description='The location of the value on a record that will be used as a bookmark during sync.', + description="The location of the value on a record that will be used as a bookmark during sync.", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomPaginationStrategy(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomPaginationStrategy'] + type: Literal["CustomPaginationStrategy"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomPaginationStrategy'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom pagination strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomPaginationStrategy"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomRecordExtractor(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomRecordExtractor'] + type: Literal["CustomRecordExtractor"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomRecordExtractor'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom record extraction strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomRecordExtractor"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomRecordFilter(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomRecordFilter'] + type: Literal["CustomRecordFilter"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomCustomRecordFilter'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom record filter strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomCustomRecordFilter"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomRequester(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomRequester'] + type: Literal["CustomRequester"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomRecordExtractor'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom requester strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomRecordExtractor"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomRetriever(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomRetriever'] + type: Literal["CustomRetriever"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_..`.', - examples=['source_railz.components.MyCustomRetriever'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom retriever strategy. The format is `source_..`.", + examples=["source_railz.components.MyCustomRetriever"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomPartitionRouter(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomPartitionRouter'] + type: Literal["CustomPartitionRouter"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_..`.', - examples=['source_railz.components.MyCustomPartitionRouter'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom partition router. The format is `source_..`.", + examples=["source_railz.components.MyCustomPartitionRouter"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomSchemaLoader(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomSchemaLoader'] + type: Literal["CustomSchemaLoader"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_..`.', - examples=['source_railz.components.MyCustomSchemaLoader'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom schema loader. The format is `source_..`.", + examples=["source_railz.components.MyCustomSchemaLoader"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomStateMigration(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomStateMigration'] + type: Literal["CustomStateMigration"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_..`.', - examples=['source_railz.components.MyCustomStateMigration'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom state migration. The format is `source_..`.", + examples=["source_railz.components.MyCustomStateMigration"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class CustomTransformation(BaseModel): class Config: extra = Extra.allow - type: Literal['CustomTransformation'] + type: Literal["CustomTransformation"] class_name: str = Field( ..., - description='Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_..`.', - examples=['source_railz.components.MyCustomTransformation'], - title='Class Name', + description="Fully-qualified name of the class that will be implementing the custom transformation. The format is `source_..`.", + examples=["source_railz.components.MyCustomTransformation"], + title="Class Name", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class LegacyToPerPartitionStateMigration(BaseModel): class Config: extra = Extra.allow - type: Optional[Literal['LegacyToPerPartitionStateMigration']] = None + type: Optional[Literal["LegacyToPerPartitionStateMigration"]] = None class Algorithm(Enum): - HS256 = 'HS256' - HS384 = 'HS384' - HS512 = 'HS512' - ES256 = 'ES256' - ES256K = 'ES256K' - ES384 = 'ES384' - ES512 = 'ES512' - RS256 = 'RS256' - RS384 = 'RS384' - RS512 = 'RS512' - PS256 = 'PS256' - PS384 = 'PS384' - PS512 = 'PS512' - EdDSA = 'EdDSA' + HS256 = "HS256" + HS384 = "HS384" + HS512 = "HS512" + ES256 = "ES256" + ES256K = "ES256K" + ES384 = "ES384" + ES512 = "ES512" + RS256 = "RS256" + RS384 = "RS384" + RS512 = "RS512" + PS256 = "PS256" + PS384 = "PS384" + PS512 = "PS512" + EdDSA = "EdDSA" class JwtHeaders(BaseModel): @@ -282,21 +285,21 @@ class Config: kid: Optional[str] = Field( None, - description='Private key ID for user account.', + description="Private key ID for user account.", examples=["{{ config['kid'] }}"], - title='Key Identifier', + title="Key Identifier", ) typ: Optional[str] = Field( - 'JWT', - description='The media type of the complete JWT.', - examples=['JWT'], - title='Type', + "JWT", + description="The media type of the complete JWT.", + examples=["JWT"], + title="Type", ) cty: Optional[str] = Field( None, - description='Content type of JWT header.', - examples=['JWT'], - title='Content Type', + description="Content type of JWT header.", + examples=["JWT"], + title="Content Type", ) @@ -306,28 +309,28 @@ class Config: iss: Optional[str] = Field( None, - description='The user/principal that issued the JWT. Commonly a value unique to the user.', + description="The user/principal that issued the JWT. Commonly a value unique to the user.", examples=["{{ config['iss'] }}"], - title='Issuer', + title="Issuer", ) sub: Optional[str] = Field( None, - description='The subject of the JWT. Commonly defined by the API.', - title='Subject', + description="The subject of the JWT. Commonly defined by the API.", + title="Subject", ) aud: Optional[str] = Field( None, - description='The recipient that the JWT is intended for. Commonly defined by the API.', - examples=['appstoreconnect-v1'], - title='Audience', + description="The recipient that the JWT is intended for. Commonly defined by the API.", + examples=["appstoreconnect-v1"], + title="Audience", ) class JwtAuthenticator(BaseModel): - type: Literal['JwtAuthenticator'] + type: Literal["JwtAuthenticator"] secret_key: str = Field( ..., - description='Secret used to sign the JSON web token.', + description="Secret used to sign the JSON web token.", examples=["{{ config['secret_key'] }}"], ) base64_encode_secret_key: Optional[bool] = Field( @@ -336,496 +339,490 @@ class JwtAuthenticator(BaseModel): ) algorithm: Algorithm = Field( ..., - description='Algorithm used to sign the JSON web token.', - examples=['ES256', 'HS256', 'RS256', "{{ config['algorithm'] }}"], + description="Algorithm used to sign the JSON web token.", + examples=["ES256", "HS256", "RS256", "{{ config['algorithm'] }}"], ) token_duration: Optional[int] = Field( 1200, - description='The amount of time in seconds a JWT token can be valid after being issued.', + description="The amount of time in seconds a JWT token can be valid after being issued.", examples=[1200, 3600], - title='Token Duration', + title="Token Duration", ) header_prefix: Optional[str] = Field( None, - description='The prefix to be used within the Authentication header.', - examples=['Bearer', 'Basic'], - title='Header Prefix', + description="The prefix to be used within the Authentication header.", + examples=["Bearer", "Basic"], + title="Header Prefix", ) jwt_headers: Optional[JwtHeaders] = Field( None, - description='JWT headers used when signing JSON web token.', - title='JWT Headers', + description="JWT headers used when signing JSON web token.", + title="JWT Headers", ) additional_jwt_headers: Optional[Dict[str, Any]] = Field( None, - description='Additional headers to be included with the JWT headers object.', - title='Additional JWT Headers', + description="Additional headers to be included with the JWT headers object.", + title="Additional JWT Headers", ) jwt_payload: Optional[JwtPayload] = Field( None, - description='JWT Payload used when signing JSON web token.', - title='JWT Payload', + description="JWT Payload used when signing JSON web token.", + title="JWT Payload", ) additional_jwt_payload: Optional[Dict[str, Any]] = Field( None, - description='Additional properties to be added to the JWT payload.', - title='Additional JWT Payload Properties', + description="Additional properties to be added to the JWT payload.", + title="Additional JWT Payload Properties", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class RefreshTokenUpdater(BaseModel): refresh_token_name: Optional[str] = Field( - 'refresh_token', - description='The name of the property which contains the updated refresh token in the response from the token refresh endpoint.', - examples=['refresh_token'], - title='Refresh Token Property Name', + "refresh_token", + description="The name of the property which contains the updated refresh token in the response from the token refresh endpoint.", + examples=["refresh_token"], + title="Refresh Token Property Name", ) access_token_config_path: Optional[List[str]] = Field( - ['credentials', 'access_token'], - description='Config path to the access token. Make sure the field actually exists in the config.', - examples=[['credentials', 'access_token'], ['access_token']], - title='Config Path To Access Token', + ["credentials", "access_token"], + description="Config path to the access token. Make sure the field actually exists in the config.", + examples=[["credentials", "access_token"], ["access_token"]], + title="Config Path To Access Token", ) refresh_token_config_path: Optional[List[str]] = Field( - ['credentials', 'refresh_token'], - description='Config path to the access token. Make sure the field actually exists in the config.', - examples=[['credentials', 'refresh_token'], ['refresh_token']], - title='Config Path To Refresh Token', + ["credentials", "refresh_token"], + description="Config path to the access token. Make sure the field actually exists in the config.", + examples=[["credentials", "refresh_token"], ["refresh_token"]], + title="Config Path To Refresh Token", ) token_expiry_date_config_path: Optional[List[str]] = Field( - ['credentials', 'token_expiry_date'], - description='Config path to the expiry date. Make sure actually exists in the config.', - examples=[['credentials', 'token_expiry_date']], - title='Config Path To Expiry Date', + ["credentials", "token_expiry_date"], + description="Config path to the expiry date. Make sure actually exists in the config.", + examples=[["credentials", "token_expiry_date"]], + title="Config Path To Expiry Date", ) refresh_token_error_status_codes: Optional[List[int]] = Field( [], - description='Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error', + description="Status Codes to Identify refresh token error in response (Refresh Token Error Key and Refresh Token Error Values should be also specified). Responses with one of the error status code and containing an error value will be flagged as a config error", examples=[[400, 500]], - title='Refresh Token Error Status Codes', + title="Refresh Token Error Status Codes", ) refresh_token_error_key: Optional[str] = Field( - '', - description='Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).', - examples=['error'], - title='Refresh Token Error Key', + "", + description="Key to Identify refresh token error in response (Refresh Token Error Status Codes and Refresh Token Error Values should be also specified).", + examples=["error"], + title="Refresh Token Error Key", ) refresh_token_error_values: Optional[List[str]] = Field( [], description='List of values to check for exception during token refresh process. Used to check if the error found in the response matches the key from the Refresh Token Error Key field (e.g. response={"error": "invalid_grant"}). Only responses with one of the error status code and containing an error value will be flagged as a config error', - examples=[['invalid_grant', 'invalid_permissions']], - title='Refresh Token Error Values', + examples=[["invalid_grant", "invalid_permissions"]], + title="Refresh Token Error Values", ) class OAuthAuthenticator(BaseModel): - type: Literal['OAuthAuthenticator'] + type: Literal["OAuthAuthenticator"] client_id: str = Field( ..., - description='The OAuth client ID. Fill it in the user inputs.', + description="The OAuth client ID. Fill it in the user inputs.", examples=["{{ config['client_id }}", "{{ config['credentials']['client_id }}"], - title='Client ID', + title="Client ID", ) client_secret: str = Field( ..., - description='The OAuth client secret. Fill it in the user inputs.', + description="The OAuth client secret. Fill it in the user inputs.", examples=[ "{{ config['client_secret }}", "{{ config['credentials']['client_secret }}", ], - title='Client Secret', + title="Client Secret", ) refresh_token: Optional[str] = Field( None, - description='Credential artifact used to get a new access token.', + description="Credential artifact used to get a new access token.", examples=[ "{{ config['refresh_token'] }}", "{{ config['credentials]['refresh_token'] }}", ], - title='Refresh Token', + title="Refresh Token", ) token_refresh_endpoint: str = Field( ..., - description='The full URL to call to obtain a new access token.', - examples=['https://connect.squareup.com/oauth2/token'], - title='Token Refresh Endpoint', + description="The full URL to call to obtain a new access token.", + examples=["https://connect.squareup.com/oauth2/token"], + title="Token Refresh Endpoint", ) access_token_name: Optional[str] = Field( - 'access_token', - description='The name of the property which contains the access token in the response from the token refresh endpoint.', - examples=['access_token'], - title='Access Token Property Name', + "access_token", + description="The name of the property which contains the access token in the response from the token refresh endpoint.", + examples=["access_token"], + title="Access Token Property Name", ) expires_in_name: Optional[str] = Field( - 'expires_in', - description='The name of the property which contains the expiry date in the response from the token refresh endpoint.', - examples=['expires_in'], - title='Token Expiry Property Name', + "expires_in", + description="The name of the property which contains the expiry date in the response from the token refresh endpoint.", + examples=["expires_in"], + title="Token Expiry Property Name", ) grant_type: Optional[str] = Field( - 'refresh_token', - description='Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.', - examples=['refresh_token', 'client_credentials'], - title='Grant Type', + "refresh_token", + description="Specifies the OAuth2 grant type. If set to refresh_token, the refresh_token needs to be provided as well. For client_credentials, only client id and secret are required. Other grant types are not officially supported.", + examples=["refresh_token", "client_credentials"], + title="Grant Type", ) refresh_request_body: Optional[Dict[str, Any]] = Field( None, - description='Body of the request sent to get a new access token.', + description="Body of the request sent to get a new access token.", examples=[ { - 'applicationId': "{{ config['application_id'] }}", - 'applicationSecret': "{{ config['application_secret'] }}", - 'token': "{{ config['token'] }}", + "applicationId": "{{ config['application_id'] }}", + "applicationSecret": "{{ config['application_secret'] }}", + "token": "{{ config['token'] }}", } ], - title='Refresh Request Body', + title="Refresh Request Body", ) scopes: Optional[List[str]] = Field( None, - description='List of scopes that should be granted to the access token.', - examples=[ - ['crm.list.read', 'crm.objects.contacts.read', 'crm.schema.contacts.read'] - ], - title='Scopes', + description="List of scopes that should be granted to the access token.", + examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], + title="Scopes", ) token_expiry_date: Optional[str] = Field( None, - description='The access token expiry date.', - examples=['2023-04-06T07:12:10.421833+00:00', 1680842386], - title='Token Expiry Date', + description="The access token expiry date.", + examples=["2023-04-06T07:12:10.421833+00:00", 1680842386], + title="Token Expiry Date", ) token_expiry_date_format: Optional[str] = Field( None, - description='The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.', - examples=['%Y-%m-%d %H:%M:%S.%f+00:00'], - title='Token Expiry Date Format', + description="The format of the time to expiration datetime. Provide it if the time is returned as a date-time string instead of seconds.", + examples=["%Y-%m-%d %H:%M:%S.%f+00:00"], + title="Token Expiry Date Format", ) refresh_token_updater: Optional[RefreshTokenUpdater] = Field( None, - description='When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.', - title='Token Updater', + description="When the token updater is defined, new refresh tokens, access tokens and the access token expiry date are written back from the authentication response to the config object. This is important if the refresh token can only used once.", + title="Token Updater", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class ExponentialBackoffStrategy(BaseModel): - type: Literal['ExponentialBackoffStrategy'] + type: Literal["ExponentialBackoffStrategy"] factor: Optional[Union[float, str]] = Field( 5, - description='Multiplicative constant applied on each retry.', - examples=[5, 5.5, '10'], - title='Factor', + description="Multiplicative constant applied on each retry.", + examples=[5, 5.5, "10"], + title="Factor", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SessionTokenRequestBearerAuthenticator(BaseModel): - type: Literal['Bearer'] + type: Literal["Bearer"] class HttpMethod(Enum): - GET = 'GET' - POST = 'POST' + GET = "GET" + POST = "POST" class Action(Enum): - SUCCESS = 'SUCCESS' - FAIL = 'FAIL' - RETRY = 'RETRY' - IGNORE = 'IGNORE' - RATE_LIMITED = 'RATE_LIMITED' + SUCCESS = "SUCCESS" + FAIL = "FAIL" + RETRY = "RETRY" + IGNORE = "IGNORE" + RATE_LIMITED = "RATE_LIMITED" class FailureType(Enum): - system_error = 'system_error' - config_error = 'config_error' - transient_error = 'transient_error' + system_error = "system_error" + config_error = "config_error" + transient_error = "transient_error" class HttpResponseFilter(BaseModel): - type: Literal['HttpResponseFilter'] + type: Literal["HttpResponseFilter"] action: Optional[Action] = Field( None, - description='Action to execute if a response matches the filter.', - examples=['SUCCESS', 'FAIL', 'RETRY', 'IGNORE', 'RATE_LIMITED'], - title='Action', + description="Action to execute if a response matches the filter.", + examples=["SUCCESS", "FAIL", "RETRY", "IGNORE", "RATE_LIMITED"], + title="Action", ) failure_type: Optional[FailureType] = Field( None, - description='Failure type of traced exception if a response matches the filter.', - examples=['system_error', 'config_error', 'transient_error'], - title='Failure Type', + description="Failure type of traced exception if a response matches the filter.", + examples=["system_error", "config_error", "transient_error"], + title="Failure Type", ) error_message: Optional[str] = Field( None, - description='Error Message to display if the response matches the filter.', - title='Error Message', + description="Error Message to display if the response matches the filter.", + title="Error Message", ) error_message_contains: Optional[str] = Field( None, - description='Match the response if its error message contains the substring.', - example=['This API operation is not enabled for this site'], - title='Error Message Substring', + description="Match the response if its error message contains the substring.", + example=["This API operation is not enabled for this site"], + title="Error Message Substring", ) http_codes: Optional[List[int]] = Field( None, - description='Match the response if its HTTP code is included in this list.', + description="Match the response if its HTTP code is included in this list.", examples=[[420, 429], [500]], - title='HTTP Codes', + title="HTTP Codes", ) predicate: Optional[str] = Field( None, - description='Match the response if the predicate evaluates to true.', + description="Match the response if the predicate evaluates to true.", examples=[ "{{ 'Too much requests' in response }}", "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", ], - title='Predicate', + title="Predicate", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class InlineSchemaLoader(BaseModel): - type: Literal['InlineSchemaLoader'] + type: Literal["InlineSchemaLoader"] schema_: Optional[Dict[str, Any]] = Field( None, - alias='schema', + alias="schema", description='Describes a streams\' schema. Refer to the Data Types documentation for more details on which types are valid.', - title='Schema', + title="Schema", ) class JsonFileSchemaLoader(BaseModel): - type: Literal['JsonFileSchemaLoader'] + type: Literal["JsonFileSchemaLoader"] file_path: Optional[str] = Field( None, description="Path to the JSON file defining the schema. The path is relative to the connector module's root.", - example=['./schemas/users.json'], - title='File Path', + example=["./schemas/users.json"], + title="File Path", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class JsonDecoder(BaseModel): - type: Literal['JsonDecoder'] + type: Literal["JsonDecoder"] class JsonlDecoder(BaseModel): - type: Literal['JsonlDecoder'] + type: Literal["JsonlDecoder"] class IterableDecoder(BaseModel): - type: Literal['IterableDecoder'] + type: Literal["IterableDecoder"] class MinMaxDatetime(BaseModel): - type: Literal['MinMaxDatetime'] + type: Literal["MinMaxDatetime"] datetime: str = Field( ..., - description='Datetime value.', - examples=['2021-01-01', '2021-01-01T00:00:00Z', "{{ config['start_time'] }}"], - title='Datetime', + description="Datetime value.", + examples=["2021-01-01", "2021-01-01T00:00:00Z", "{{ config['start_time'] }}"], + title="Datetime", ) datetime_format: Optional[str] = Field( - '', + "", description='Format of the datetime value. Defaults to "%Y-%m-%dT%H:%M:%S.%f%z" if left empty. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`, `000001`, ..., `999999`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (Sunday as first day) - `00`, `01`, ..., `53`\n * **%W**: Week number of the year (Monday as first day) - `00`, `01`, ..., `53`\n * **%c**: Date and time representation - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date representation - `08/16/1988`\n * **%X**: Time representation - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', - examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s'], - title='Datetime Format', + examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s"], + title="Datetime Format", ) max_datetime: Optional[str] = Field( None, - description='Ceiling applied on the datetime value. Must be formatted with the datetime_format field.', - examples=['2021-01-01T00:00:00Z', '2021-01-01'], - title='Max Datetime', + description="Ceiling applied on the datetime value. Must be formatted with the datetime_format field.", + examples=["2021-01-01T00:00:00Z", "2021-01-01"], + title="Max Datetime", ) min_datetime: Optional[str] = Field( None, - description='Floor applied on the datetime value. Must be formatted with the datetime_format field.', - examples=['2010-01-01T00:00:00Z', '2010-01-01'], - title='Min Datetime', + description="Floor applied on the datetime value. Must be formatted with the datetime_format field.", + examples=["2010-01-01T00:00:00Z", "2010-01-01"], + title="Min Datetime", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class NoAuth(BaseModel): - type: Literal['NoAuth'] - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + type: Literal["NoAuth"] + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class NoPagination(BaseModel): - type: Literal['NoPagination'] + type: Literal["NoPagination"] class OAuthConfigSpecification(BaseModel): class Config: extra = Extra.allow - oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = ( - Field( - None, - description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", - examples=[ - {'app_id': {'type': 'string', 'path_in_connector_config': ['app_id']}}, - { - 'app_id': { - 'type': 'string', - 'path_in_connector_config': ['info', 'app_id'], - } - }, - ], - title='OAuth user input', - ) + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", + examples=[ + {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, + { + "app_id": { + "type": "string", + "path_in_connector_config": ["info", "app_id"], + } + }, + ], + title="OAuth user input", ) complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations produced by the OAuth flows as they are\nreturned by the distant OAuth APIs.\nMust be a valid JSON describing the fields to merge back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_output_specification={\n refresh_token: {\n type: string,\n path_in_connector_config: ['credentials', 'refresh_token']\n }\n }", examples=[ { - 'refresh_token': { - 'type': 'string,', - 'path_in_connector_config': ['credentials', 'refresh_token'], + "refresh_token": { + "type": "string,", + "path_in_connector_config": ["credentials", "refresh_token"], } } ], - title='OAuth output specification', + title="OAuth output specification", ) complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( None, - description='OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }', - examples=[ - {'client_id': {'type': 'string'}, 'client_secret': {'type': 'string'}} - ], - title='OAuth input specification', + description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", + examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], + title="OAuth input specification", ) complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations that\nalso need to be merged back into the connector configuration at runtime.\nThis is a subset configuration of `complete_oauth_server_input_specification` that filters fields out to retain only the ones that\nare necessary for the connector to function with OAuth. (some fields could be used during oauth flows but not needed afterwards, therefore\nthey would be listed in the `complete_oauth_server_input_specification` but not `complete_oauth_server_output_specification`)\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nconnector when using OAuth flow APIs.\nThese fields are to be merged back to `ConnectorSpecification.connectionSpecification`.\nFor each field, a special annotation `path_in_connector_config` can be specified to determine where to merge it,\nExamples:\n complete_oauth_server_output_specification={\n client_id: {\n type: string,\n path_in_connector_config: ['credentials', 'client_id']\n },\n client_secret: {\n type: string,\n path_in_connector_config: ['credentials', 'client_secret']\n }\n }", examples=[ { - 'client_id': { - 'type': 'string,', - 'path_in_connector_config': ['credentials', 'client_id'], + "client_id": { + "type": "string,", + "path_in_connector_config": ["credentials", "client_id"], }, - 'client_secret': { - 'type': 'string,', - 'path_in_connector_config': ['credentials', 'client_secret'], + "client_secret": { + "type": "string,", + "path_in_connector_config": ["credentials", "client_secret"], }, } ], - title='OAuth server output specification', + title="OAuth server output specification", ) class OffsetIncrement(BaseModel): - type: Literal['OffsetIncrement'] + type: Literal["OffsetIncrement"] page_size: Optional[Union[int, str]] = Field( None, - description='The number of records to include in each pages.', + description="The number of records to include in each pages.", examples=[100, "{{ config['page_size'] }}"], - title='Limit', + title="Limit", ) inject_on_first_request: Optional[bool] = Field( False, - description='Using the `offset` with value `0` during the first request', - title='Inject Offset', + description="Using the `offset` with value `0` during the first request", + title="Inject Offset", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class PageIncrement(BaseModel): - type: Literal['PageIncrement'] + type: Literal["PageIncrement"] page_size: Optional[Union[int, str]] = Field( None, - description='The number of records to include in each pages.', - examples=[100, '100', "{{ config['page_size'] }}"], - title='Page Size', + description="The number of records to include in each pages.", + examples=[100, "100", "{{ config['page_size'] }}"], + title="Page Size", ) start_from_page: Optional[int] = Field( 0, - description='Index of the first page to request.', + description="Index of the first page to request.", examples=[0, 1], - title='Start From Page', + title="Start From Page", ) inject_on_first_request: Optional[bool] = Field( False, - description='Using the `page number` with value defined by `start_from_page` during the first request', - title='Inject Page Number', + description="Using the `page number` with value defined by `start_from_page` during the first request", + title="Inject Page Number", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class PrimaryKey(BaseModel): __root__: Union[str, List[str], List[List[str]]] = Field( ..., - description='The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.', - examples=['id', ['code', 'type']], - title='Primary Key', + description="The stream field to be used to distinguish unique records. Can either be a single field, an array of fields representing a composite key, or an array of arrays representing a composite key where the fields are nested fields.", + examples=["id", ["code", "type"]], + title="Primary Key", ) class RecordFilter(BaseModel): - type: Literal['RecordFilter'] + type: Literal["RecordFilter"] condition: Optional[str] = Field( - '', - description='The predicate to filter a record. Records will be removed if evaluated to False.', + "", + description="The predicate to filter a record. Records will be removed if evaluated to False.", examples=[ "{{ record['created_at'] >= stream_interval['start_time'] }}", "{{ record.status in ['active', 'expired'] }}", ], ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SchemaNormalization(Enum): - None_ = 'None' - Default = 'Default' + None_ = "None" + Default = "Default" class RemoveFields(BaseModel): - type: Literal['RemoveFields'] + type: Literal["RemoveFields"] condition: Optional[str] = Field( - '', - description='The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,', + "", + description="The predicate to filter a property by a property value. Property will be removed if it is empty OR expression is evaluated to True.,", examples=[ "{{ property|string == '' }}", - '{{ property is integer }}', - '{{ property|length > 5 }}', + "{{ property is integer }}", + "{{ property|length > 5 }}", "{{ property == 'some_string_to_match' }}", ], ) field_pointers: List[List[str]] = Field( ..., - description='Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.', - examples=[['tags'], [['content', 'html'], ['content', 'plain_text']]], - title='Field Paths', + description="Array of paths defining the field to remove. Each item is an array whose field describe the path of a field to remove.", + examples=[["tags"], [["content", "html"], ["content", "plain_text"]]], + title="Field Paths", ) class RequestPath(BaseModel): - type: Literal['RequestPath'] + type: Literal["RequestPath"] class InjectInto(Enum): - request_parameter = 'request_parameter' - header = 'header' - body_data = 'body_data' - body_json = 'body_json' + request_parameter = "request_parameter" + header = "header" + body_data = "body_data" + body_json = "body_json" class RequestOption(BaseModel): - type: Literal['RequestOption'] + type: Literal["RequestOption"] field_name: str = Field( ..., - description='Configures which key should be used in the location that the descriptor is being injected into', - examples=['segment_id'], - title='Request Option', + description="Configures which key should be used in the location that the descriptor is being injected into", + examples=["segment_id"], + title="Request Option", ) inject_into: InjectInto = Field( ..., - description='Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.', - examples=['request_parameter', 'header', 'body_data', 'body_json'], - title='Inject Into', + description="Configures where the descriptor should be set on the HTTP requests. Note that request parameters that are already encoded in the URL path will not be duplicated.", + examples=["request_parameter", "header", "body_data", "body_json"], + title="Inject Into", ) @@ -837,112 +834,112 @@ class Config: class LegacySessionTokenAuthenticator(BaseModel): - type: Literal['LegacySessionTokenAuthenticator'] + type: Literal["LegacySessionTokenAuthenticator"] header: str = Field( ..., - description='The name of the session token header that will be injected in the request', - examples=['X-Session'], - title='Session Request Header', + description="The name of the session token header that will be injected in the request", + examples=["X-Session"], + title="Session Request Header", ) login_url: str = Field( ..., - description='Path of the login URL (do not include the base URL)', - examples=['session'], - title='Login Path', + description="Path of the login URL (do not include the base URL)", + examples=["session"], + title="Login Path", ) session_token: Optional[str] = Field( None, - description='Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair', + description="Session token to use if using a pre-defined token. Not needed if authenticating with username + password pair", example=["{{ config['session_token'] }}"], - title='Session Token', + title="Session Token", ) session_token_response_key: str = Field( ..., - description='Name of the key of the session token to be extracted from the response', - examples=['id'], - title='Response Token Response Key', + description="Name of the key of the session token to be extracted from the response", + examples=["id"], + title="Response Token Response Key", ) username: Optional[str] = Field( None, - description='Username used to authenticate and obtain a session token', + description="Username used to authenticate and obtain a session token", examples=[" {{ config['username'] }}"], - title='Username', + title="Username", ) password: Optional[str] = Field( - '', - description='Password used to authenticate and obtain a session token', - examples=["{{ config['password'] }}", ''], - title='Password', + "", + description="Password used to authenticate and obtain a session token", + examples=["{{ config['password'] }}", ""], + title="Password", ) validate_session_url: str = Field( ..., - description='Path of the URL to use to validate that the session token is valid (do not include the base URL)', - examples=['user/current'], - title='Validate Session Path', + description="Path of the URL to use to validate that the session token is valid (do not include the base URL)", + examples=["user/current"], + title="Validate Session Path", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class ValueType(Enum): - string = 'string' - number = 'number' - integer = 'integer' - boolean = 'boolean' + string = "string" + number = "number" + integer = "integer" + boolean = "boolean" class WaitTimeFromHeader(BaseModel): - type: Literal['WaitTimeFromHeader'] + type: Literal["WaitTimeFromHeader"] header: str = Field( ..., - description='The name of the response header defining how long to wait before retrying.', - examples=['Retry-After'], - title='Response Header Name', + description="The name of the response header defining how long to wait before retrying.", + examples=["Retry-After"], + title="Response Header Name", ) regex: Optional[str] = Field( None, - description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.', - examples=['([-+]?\\d+)'], - title='Extraction Regex', + description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", + examples=["([-+]?\\d+)"], + title="Extraction Regex", ) max_waiting_time_in_seconds: Optional[float] = Field( None, - description='Given the value extracted from the header is greater than this value, stop the stream.', + description="Given the value extracted from the header is greater than this value, stop the stream.", examples=[3600], - title='Max Waiting Time in Seconds', + title="Max Waiting Time in Seconds", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class WaitUntilTimeFromHeader(BaseModel): - type: Literal['WaitUntilTimeFromHeader'] + type: Literal["WaitUntilTimeFromHeader"] header: str = Field( ..., - description='The name of the response header defining how long to wait before retrying.', - examples=['wait_time'], - title='Response Header', + description="The name of the response header defining how long to wait before retrying.", + examples=["wait_time"], + title="Response Header", ) min_wait: Optional[Union[float, str]] = Field( None, - description='Minimum time to wait before retrying.', - examples=[10, '60'], - title='Minimum Wait Time', + description="Minimum time to wait before retrying.", + examples=[10, "60"], + title="Minimum Wait Time", ) regex: Optional[str] = Field( None, - description='Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.', - examples=['([-+]?\\d+)'], - title='Extraction Regex', + description="Optional regex to apply on the header to extract its value. The regex should define a capture group defining the wait time.", + examples=["([-+]?\\d+)"], + title="Extraction Regex", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class AddedFieldDefinition(BaseModel): - type: Literal['AddedFieldDefinition'] + type: Literal["AddedFieldDefinition"] path: List[str] = Field( ..., - description='List of strings defining the path where to add the value on the record.', - examples=[['segment_id'], ['metadata', 'segment_id']], - title='Path', + description="List of strings defining the path where to add the value on the record.", + examples=[["segment_id"], ["metadata", "segment_id"]], + title="Path", ) value: str = Field( ..., @@ -952,197 +949,195 @@ class AddedFieldDefinition(BaseModel): "{{ record['MetaData']['LastUpdatedTime'] }}", "{{ stream_partition['segment_id'] }}", ], - title='Value', + title="Value", ) value_type: Optional[ValueType] = Field( None, - description='Type of the value. If not specified, the type will be inferred from the value.', - title='Value Type', + description="Type of the value. If not specified, the type will be inferred from the value.", + title="Value Type", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class AddFields(BaseModel): - type: Literal['AddFields'] + type: Literal["AddFields"] fields: List[AddedFieldDefinition] = Field( ..., - description='List of transformations (path and corresponding value) that will be added to the record.', - title='Fields', + description="List of transformations (path and corresponding value) that will be added to the record.", + title="Fields", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class ApiKeyAuthenticator(BaseModel): - type: Literal['ApiKeyAuthenticator'] + type: Literal["ApiKeyAuthenticator"] api_token: Optional[str] = Field( None, - description='The API key to inject in the request. Fill it in the user inputs.', + description="The API key to inject in the request. Fill it in the user inputs.", examples=["{{ config['api_key'] }}", "Token token={{ config['api_key'] }}"], - title='API Key', + title="API Key", ) header: Optional[str] = Field( None, - description='The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.', - examples=['Authorization', 'Api-Token', 'X-Auth-Token'], - title='Header Name', + description="The name of the HTTP header that will be set to the API key. This setting is deprecated, use inject_into instead. Header and inject_into can not be defined at the same time.", + examples=["Authorization", "Api-Token", "X-Auth-Token"], + title="Header Name", ) inject_into: Optional[RequestOption] = Field( None, - description='Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.', + description="Configure how the API Key will be sent in requests to the source API. Either inject_into or header has to be defined.", examples=[ - {'inject_into': 'header', 'field_name': 'Authorization'}, - {'inject_into': 'request_parameter', 'field_name': 'authKey'}, + {"inject_into": "header", "field_name": "Authorization"}, + {"inject_into": "request_parameter", "field_name": "authKey"}, ], - title='Inject API Key Into Outgoing HTTP Request', + title="Inject API Key Into Outgoing HTTP Request", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class AuthFlow(BaseModel): - auth_flow_type: Optional[AuthFlowType] = Field( - None, description='The type of auth to use', title='Auth flow type' - ) + auth_flow_type: Optional[AuthFlowType] = Field(None, description="The type of auth to use", title="Auth flow type") predicate_key: Optional[List[str]] = Field( None, - description='JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.', - examples=[['credentials', 'auth_type']], - title='Predicate key', + description="JSON path to a field in the connectorSpecification that should exist for the advanced auth to be applicable.", + examples=[["credentials", "auth_type"]], + title="Predicate key", ) predicate_value: Optional[str] = Field( None, - description='Value of the predicate_key fields for the advanced auth to be applicable.', - examples=['Oauth'], - title='Predicate value', + description="Value of the predicate_key fields for the advanced auth to be applicable.", + examples=["Oauth"], + title="Predicate value", ) oauth_config_specification: Optional[OAuthConfigSpecification] = None class CursorPagination(BaseModel): - type: Literal['CursorPagination'] + type: Literal["CursorPagination"] cursor_value: str = Field( ..., - description='Value of the cursor defining the next page to fetch.', + description="Value of the cursor defining the next page to fetch.", examples=[ - '{{ headers.link.next.cursor }}', + "{{ headers.link.next.cursor }}", "{{ last_record['key'] }}", "{{ response['nextPage'] }}", ], - title='Cursor Value', + title="Cursor Value", ) page_size: Optional[int] = Field( None, - description='The number of records to include in each pages.', + description="The number of records to include in each pages.", examples=[100], - title='Page Size', + title="Page Size", ) stop_condition: Optional[str] = Field( None, - description='Template string evaluating when to stop paginating.', + description="Template string evaluating when to stop paginating.", examples=[ - '{{ response.data.has_more is false }}', + "{{ response.data.has_more is false }}", "{{ 'next' not in headers['link'] }}", ], - title='Stop Condition', + title="Stop Condition", ) decoder: Optional[JsonDecoder] = Field( None, - description='Component decoding the response so records can be extracted.', - title='Decoder', + description="Component decoding the response so records can be extracted.", + title="Decoder", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DatetimeBasedCursor(BaseModel): - type: Literal['DatetimeBasedCursor'] + type: Literal["DatetimeBasedCursor"] cursor_field: str = Field( ..., - description='The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.', - examples=['created_at', "{{ config['record_cursor'] }}"], - title='Cursor Field', + description="The location of the value on a record that will be used as a bookmark during sync. To ensure no data loss, the API must return records in ascending order based on the cursor field. Nested fields are not supported, so the field must be at the top level of the record. You can use a combination of Add Field and Remove Field transformations to move the nested field to the top.", + examples=["created_at", "{{ config['record_cursor'] }}"], + title="Cursor Field", ) datetime_format: str = Field( ..., - description='The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with "%" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal \'%\' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n', - examples=['%Y-%m-%dT%H:%M:%S.%f%z', '%Y-%m-%d', '%s', '%ms', '%s_as_float'], - title='Outgoing Datetime Format', + description="The datetime format used to format the datetime values that are sent in outgoing requests to the API. Use placeholders starting with \"%\" to describe the format the API is using. The following placeholders are available:\n * **%s**: Epoch unix timestamp - `1686218963`\n * **%s_as_float**: Epoch unix timestamp in seconds as float with microsecond precision - `1686218963.123456`\n * **%ms**: Epoch unix timestamp (milliseconds) - `1686218963123`\n * **%a**: Weekday (abbreviated) - `Sun`\n * **%A**: Weekday (full) - `Sunday`\n * **%w**: Weekday (decimal) - `0` (Sunday), `6` (Saturday)\n * **%d**: Day of the month (zero-padded) - `01`, `02`, ..., `31`\n * **%b**: Month (abbreviated) - `Jan`\n * **%B**: Month (full) - `January`\n * **%m**: Month (zero-padded) - `01`, `02`, ..., `12`\n * **%y**: Year (without century, zero-padded) - `00`, `01`, ..., `99`\n * **%Y**: Year (with century) - `0001`, `0002`, ..., `9999`\n * **%H**: Hour (24-hour, zero-padded) - `00`, `01`, ..., `23`\n * **%I**: Hour (12-hour, zero-padded) - `01`, `02`, ..., `12`\n * **%p**: AM/PM indicator\n * **%M**: Minute (zero-padded) - `00`, `01`, ..., `59`\n * **%S**: Second (zero-padded) - `00`, `01`, ..., `59`\n * **%f**: Microsecond (zero-padded to 6 digits) - `000000`\n * **%z**: UTC offset - `(empty)`, `+0000`, `-04:00`\n * **%Z**: Time zone name - `(empty)`, `UTC`, `GMT`\n * **%j**: Day of the year (zero-padded) - `001`, `002`, ..., `366`\n * **%U**: Week number of the year (starting Sunday) - `00`, ..., `53`\n * **%W**: Week number of the year (starting Monday) - `00`, ..., `53`\n * **%c**: Date and time - `Tue Aug 16 21:30:00 1988`\n * **%x**: Date standard format - `08/16/1988`\n * **%X**: Time standard format - `21:30:00`\n * **%%**: Literal '%' character\n\n Some placeholders depend on the locale of the underlying system - in most cases this locale is configured as en/US. For more information see the [Python documentation](https://docs.python.org/3/library/datetime.html#strftime-and-strptime-format-codes).\n", + examples=["%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%d", "%s", "%ms", "%s_as_float"], + title="Outgoing Datetime Format", ) start_datetime: Union[str, MinMaxDatetime] = Field( ..., - description='The datetime that determines the earliest record that should be synced.', - examples=['2020-01-1T00:00:00Z', "{{ config['start_time'] }}"], - title='Start Datetime', + description="The datetime that determines the earliest record that should be synced.", + examples=["2020-01-1T00:00:00Z", "{{ config['start_time'] }}"], + title="Start Datetime", ) cursor_datetime_formats: Optional[List[str]] = Field( None, - description='The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.', - title='Cursor Datetime Formats', + description="The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used.", + title="Cursor Datetime Formats", ) cursor_granularity: Optional[str] = Field( None, - description='Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.', - examples=['PT1S'], - title='Cursor Granularity', + description="Smallest increment the datetime_format has (ISO 8601 duration) that is used to ensure the start of a slice does not overlap with the end of the previous one, e.g. for %Y-%m-%d the granularity should be P1D, for %Y-%m-%dT%H:%M:%SZ the granularity should be PT1S. Given this field is provided, `step` needs to be provided as well.", + examples=["PT1S"], + title="Cursor Granularity", ) end_datetime: Optional[Union[str, MinMaxDatetime]] = Field( None, - description='The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.', - examples=['2021-01-1T00:00:00Z', '{{ now_utc() }}', '{{ day_delta(-1) }}'], - title='End Datetime', + description="The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used.", + examples=["2021-01-1T00:00:00Z", "{{ now_utc() }}", "{{ day_delta(-1) }}"], + title="End Datetime", ) end_time_option: Optional[RequestOption] = Field( None, - description='Optionally configures how the end datetime will be sent in requests to the source API.', - title='Inject End Time Into Outgoing HTTP Request', + description="Optionally configures how the end datetime will be sent in requests to the source API.", + title="Inject End Time Into Outgoing HTTP Request", ) is_data_feed: Optional[bool] = Field( None, - description='A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.', - title='Whether the target API is formatted as a data feed', + description="A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination.", + title="Whether the target API is formatted as a data feed", ) is_client_side_incremental: Optional[bool] = Field( None, - description='If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.', - title='Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)', + description="If the target API endpoint does not take cursor values to filter records and returns all records anyway, the connector with this cursor will filter out records locally, and only emit new records from the last sync, hence incremental. This means that all records would be read from the API, but only new records will be emitted to the destination.", + title="Whether the target API does not support filtering and returns all data (the cursor filters records in the client instead of the API side)", ) is_compare_strictly: Optional[bool] = Field( False, - description='Set to True if the target API does not accept queries where the start time equal the end time.', - title='Whether to skip requests if the start time equals the end time', + description="Set to True if the target API does not accept queries where the start time equal the end time.", + title="Whether to skip requests if the start time equals the end time", ) lookback_window: Optional[str] = Field( None, - description='Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.', - examples=['P1D', "P{{ config['lookback_days'] }}D"], - title='Lookback Window', + description="Time interval before the start_datetime to read data for, e.g. P1M for looking back one month.", + examples=["P1D", "P{{ config['lookback_days'] }}D"], + title="Lookback Window", ) partition_field_end: Optional[str] = Field( None, - description='Name of the partition start time field.', - examples=['ending_time'], - title='Partition Field End', + description="Name of the partition start time field.", + examples=["ending_time"], + title="Partition Field End", ) partition_field_start: Optional[str] = Field( None, - description='Name of the partition end time field.', - examples=['starting_time'], - title='Partition Field Start', + description="Name of the partition end time field.", + examples=["starting_time"], + title="Partition Field Start", ) start_time_option: Optional[RequestOption] = Field( None, - description='Optionally configures how the start datetime will be sent in requests to the source API.', - title='Inject Start Time Into Outgoing HTTP Request', + description="Optionally configures how the start datetime will be sent in requests to the source API.", + title="Inject Start Time Into Outgoing HTTP Request", ) step: Optional[str] = Field( None, - description='The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.', - examples=['P1W', "{{ config['step_increment'] }}"], - title='Step', + description="The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well.", + examples=["P1W", "{{ config['step_increment'] }}"], + title="Step", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DefaultErrorHandler(BaseModel): - type: Literal['DefaultErrorHandler'] + type: Literal["DefaultErrorHandler"] backoff_strategies: Optional[ List[ Union[ @@ -1155,159 +1150,155 @@ class DefaultErrorHandler(BaseModel): ] ] = Field( None, - description='List of backoff strategies to use to determine how long to wait before retrying a retryable request.', - title='Backoff Strategies', + description="List of backoff strategies to use to determine how long to wait before retrying a retryable request.", + title="Backoff Strategies", ) max_retries: Optional[int] = Field( 5, - description='The maximum number of time to retry a retryable request before giving up and failing.', + description="The maximum number of time to retry a retryable request before giving up and failing.", examples=[5, 0, 10], - title='Max Retry Count', + title="Max Retry Count", ) response_filters: Optional[List[HttpResponseFilter]] = Field( None, description="List of response filters to iterate on when deciding how to handle an error. When using an array of multiple filters, the filters will be applied sequentially and the response will be selected if it matches any of the filter's predicate.", - title='Response Filters', + title="Response Filters", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DefaultPaginator(BaseModel): - type: Literal['DefaultPaginator'] - pagination_strategy: Union[ - CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement - ] = Field( + type: Literal["DefaultPaginator"] + pagination_strategy: Union[CursorPagination, CustomPaginationStrategy, OffsetIncrement, PageIncrement] = Field( ..., - description='Strategy defining how records are paginated.', - title='Pagination Strategy', + description="Strategy defining how records are paginated.", + title="Pagination Strategy", ) decoder: Optional[JsonDecoder] = Field( None, - description='Component decoding the response so records can be extracted.', - title='Decoder', + description="Component decoding the response so records can be extracted.", + title="Decoder", ) page_size_option: Optional[RequestOption] = None page_token_option: Optional[Union[RequestOption, RequestPath]] = None - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DpathExtractor(BaseModel): - type: Literal['DpathExtractor'] + type: Literal["DpathExtractor"] field_path: List[str] = Field( ..., description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', examples=[ - ['data'], - ['data', 'records'], - ['data', '{{ parameters.name }}'], - ['data', '*', 'record'], + ["data"], + ["data", "records"], + ["data", "{{ parameters.name }}"], + ["data", "*", "record"], ], - title='Field Path', - ) - decoder: Optional[Union[JsonDecoder, JsonlDecoder, IterableDecoder]] = Field( - None, title='Decoder' + title="Field Path", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + decoder: Optional[Union[JsonDecoder, JsonlDecoder, IterableDecoder]] = Field(None, title="Decoder") + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SessionTokenRequestApiKeyAuthenticator(BaseModel): - type: Literal['ApiKey'] + type: Literal["ApiKey"] inject_into: RequestOption = Field( ..., - description='Configure how the API Key will be sent in requests to the source API.', + description="Configure how the API Key will be sent in requests to the source API.", examples=[ - {'inject_into': 'header', 'field_name': 'Authorization'}, - {'inject_into': 'request_parameter', 'field_name': 'authKey'}, + {"inject_into": "header", "field_name": "Authorization"}, + {"inject_into": "request_parameter", "field_name": "authKey"}, ], - title='Inject API Key Into Outgoing HTTP Request', + title="Inject API Key Into Outgoing HTTP Request", ) class ListPartitionRouter(BaseModel): - type: Literal['ListPartitionRouter'] + type: Literal["ListPartitionRouter"] cursor_field: str = Field( ..., description='While iterating over list values, the name of field used to reference a list value. The partition value can be accessed with string interpolation. e.g. "{{ stream_partition[\'my_key\'] }}" where "my_key" is the value of the cursor_field.', - examples=['section', "{{ config['section_key'] }}"], - title='Current Partition Value Identifier', + examples=["section", "{{ config['section_key'] }}"], + title="Current Partition Value Identifier", ) values: Union[str, List[str]] = Field( ..., - description='The list of attributes being iterated over and used as input for the requests made to the source API.', - examples=[['section_a', 'section_b', 'section_c'], "{{ config['sections'] }}"], - title='Partition Values', + description="The list of attributes being iterated over and used as input for the requests made to the source API.", + examples=[["section_a", "section_b", "section_c"], "{{ config['sections'] }}"], + title="Partition Values", ) request_option: Optional[RequestOption] = Field( None, - description='A request option describing where the list value should be injected into and under what field name if applicable.', - title='Inject Partition Value Into Outgoing HTTP Request', + description="A request option describing where the list value should be injected into and under what field name if applicable.", + title="Inject Partition Value Into Outgoing HTTP Request", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class RecordSelector(BaseModel): - type: Literal['RecordSelector'] + type: Literal["RecordSelector"] extractor: Union[CustomRecordExtractor, DpathExtractor] record_filter: Optional[Union[CustomRecordFilter, RecordFilter]] = Field( None, - description='Responsible for filtering records to be emitted by the Source.', - title='Record Filter', + description="Responsible for filtering records to be emitted by the Source.", + title="Record Filter", ) schema_normalization: Optional[SchemaNormalization] = SchemaNormalization.None_ - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class Spec(BaseModel): - type: Literal['Spec'] + type: Literal["Spec"] connection_specification: Dict[str, Any] = Field( ..., - description='A connection specification describing how a the connector can be configured.', - title='Connection Specification', + description="A connection specification describing how a the connector can be configured.", + title="Connection Specification", ) documentation_url: Optional[str] = Field( None, description="URL of the connector's documentation page.", - examples=['https://docs.airbyte.com/integrations/sources/dremio'], - title='Documentation URL', + examples=["https://docs.airbyte.com/integrations/sources/dremio"], + title="Documentation URL", ) advanced_auth: Optional[AuthFlow] = Field( None, - description='Advanced specification for configuring the authentication flow.', - title='Advanced Auth', + description="Advanced specification for configuring the authentication flow.", + title="Advanced Auth", ) class CompositeErrorHandler(BaseModel): - type: Literal['CompositeErrorHandler'] + type: Literal["CompositeErrorHandler"] error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler]] = Field( ..., - description='List of error handlers to iterate on to determine how to handle a failed response.', - title='Error Handlers', + description="List of error handlers to iterate on to determine how to handle a failed response.", + title="Error Handlers", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DeclarativeSource(BaseModel): class Config: extra = Extra.forbid - type: Literal['DeclarativeSource'] + type: Literal["DeclarativeSource"] check: CheckStream streams: List[DeclarativeStream] version: str = Field( ..., - description='The version of the Airbyte CDK used to build and test the source.', + description="The version of the Airbyte CDK used to build and test the source.", ) schemas: Optional[Schemas] = None definitions: Optional[Dict[str, Any]] = None spec: Optional[Spec] = None metadata: Optional[Dict[str, Any]] = Field( None, - description='For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.', + description="For internal Airbyte use only - DO NOT modify manually. Used by consumers of declarative manifests for storing related metadata.", ) description: Optional[str] = Field( None, - description='A description of the connector. It will be presented on the Source documentation page.', + description="A description of the connector. It will be presented on the Source documentation page.", ) @@ -1315,12 +1306,12 @@ class SelectiveAuthenticator(BaseModel): class Config: extra = Extra.allow - type: Literal['SelectiveAuthenticator'] + type: Literal["SelectiveAuthenticator"] authenticator_selection_path: List[str] = Field( ..., - description='Path of the field in config with selected authenticator name', - examples=[['auth'], ['auth', 'type']], - title='Authenticator Selection Path', + description="Path of the field in config with selected authenticator name", + examples=[["auth"], ["auth", "type"]], + title="Authenticator Selection Path", ) authenticators: Dict[ str, @@ -1337,129 +1328,115 @@ class Config: ], ] = Field( ..., - description='Authenticators to select from.', + description="Authenticators to select from.", examples=[ { - 'authenticators': { - 'token': '#/definitions/ApiKeyAuthenticator', - 'oauth': '#/definitions/OAuthAuthenticator', - 'jwt': '#/definitions/JwtAuthenticator', + "authenticators": { + "token": "#/definitions/ApiKeyAuthenticator", + "oauth": "#/definitions/OAuthAuthenticator", + "jwt": "#/definitions/JwtAuthenticator", } } ], - title='Authenticators', + title="Authenticators", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class DeclarativeStream(BaseModel): class Config: extra = Extra.allow - type: Literal['DeclarativeStream'] + type: Literal["DeclarativeStream"] retriever: Union[CustomRetriever, SimpleRetriever] = Field( ..., - description='Component used to coordinate how records are extracted across stream slices and request pages.', - title='Retriever', + description="Component used to coordinate how records are extracted across stream slices and request pages.", + title="Retriever", ) - incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = ( - Field( - None, - description='Component used to fetch data incrementally based on a time field in the data.', - title='Incremental Sync', - ) - ) - name: Optional[str] = Field( - '', description='The stream name.', example=['Users'], title='Name' - ) - primary_key: Optional[PrimaryKey] = Field( - '', description='The primary key of the stream.', title='Primary Key' + incremental_sync: Optional[Union[CustomIncrementalSync, DatetimeBasedCursor]] = Field( + None, + description="Component used to fetch data incrementally based on a time field in the data.", + title="Incremental Sync", ) - schema_loader: Optional[ - Union[InlineSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader] - ] = Field( + name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") + primary_key: Optional[PrimaryKey] = Field("", description="The primary key of the stream.", title="Primary Key") + schema_loader: Optional[Union[InlineSchemaLoader, JsonFileSchemaLoader, CustomSchemaLoader]] = Field( None, - description='Component used to retrieve the schema for the current stream.', - title='Schema Loader', + description="Component used to retrieve the schema for the current stream.", + title="Schema Loader", ) - transformations: Optional[ - List[Union[AddFields, CustomTransformation, RemoveFields]] - ] = Field( + transformations: Optional[List[Union[AddFields, CustomTransformation, RemoveFields]]] = Field( None, - description='A list of transformations to be applied to each output record.', - title='Transformations', + description="A list of transformations to be applied to each output record.", + title="Transformations", ) - state_migrations: Optional[ - List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]] - ] = Field( + state_migrations: Optional[List[Union[LegacyToPerPartitionStateMigration, CustomStateMigration]]] = Field( [], - description='Array of state migrations to be applied on the input state', - title='State Migrations', + description="Array of state migrations to be applied on the input state", + title="State Migrations", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SessionTokenAuthenticator(BaseModel): - type: Literal['SessionTokenAuthenticator'] + type: Literal["SessionTokenAuthenticator"] login_requester: HttpRequester = Field( ..., - description='Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.', + description="Description of the request to perform to obtain a session token to perform data requests. The response body is expected to be a JSON object with a session token property.", examples=[ { - 'type': 'HttpRequester', - 'url_base': 'https://my_api.com', - 'path': '/login', - 'authenticator': { - 'type': 'BasicHttpAuthenticator', - 'username': '{{ config.username }}', - 'password': '{{ config.password }}', + "type": "HttpRequester", + "url_base": "https://my_api.com", + "path": "/login", + "authenticator": { + "type": "BasicHttpAuthenticator", + "username": "{{ config.username }}", + "password": "{{ config.password }}", }, } ], - title='Login Requester', + title="Login Requester", ) session_token_path: List[str] = Field( ..., - description='The path in the response body returned from the login requester to the session token.', - examples=[['access_token'], ['result', 'token']], - title='Session Token Path', + description="The path in the response body returned from the login requester to the session token.", + examples=[["access_token"], ["result", "token"]], + title="Session Token Path", ) expiration_duration: Optional[str] = Field( None, - description='The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.', - examples=['PT1H', 'P1D'], - title='Expiration Duration', + description="The duration in ISO 8601 duration notation after which the session token expires, starting from the time it was obtained. Omitting it will result in the session token being refreshed for every request.", + examples=["PT1H", "P1D"], + title="Expiration Duration", ) - request_authentication: Union[ - SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator - ] = Field( + request_authentication: Union[SessionTokenRequestApiKeyAuthenticator, SessionTokenRequestBearerAuthenticator] = Field( ..., - description='Authentication method to use for requests sent to the API, specifying how to inject the session token.', - title='Data Request Authentication', + description="Authentication method to use for requests sent to the API, specifying how to inject the session token.", + title="Data Request Authentication", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class HttpRequester(BaseModel): - type: Literal['HttpRequester'] + type: Literal["HttpRequester"] url_base: str = Field( ..., - description='Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.', + description="Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", examples=[ - 'https://connect.squareup.com/v2', + "https://connect.squareup.com/v2", "{{ config['base_url'] or 'https://app.posthog.com'}}/api/", ], - title='API Base URL', + title="API Base URL", ) path: str = Field( ..., - description='Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.', + description="Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", examples=[ - '/products', + "/products", "/quotes/{{ stream_partition['id'] }}/quote_line_groups", "/trades/{{ config['symbol_id'] }}/history", ], - title='URL Path', + title="URL Path", ) authenticator: Optional[ Union[ @@ -1476,106 +1453,102 @@ class HttpRequester(BaseModel): ] ] = Field( None, - description='Authentication method to use for requests sent to the API.', - title='Authenticator', + description="Authentication method to use for requests sent to the API.", + title="Authenticator", ) - error_handler: Optional[ - Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler] - ] = Field( + error_handler: Optional[Union[DefaultErrorHandler, CustomErrorHandler, CompositeErrorHandler]] = Field( None, - description='Error handler component that defines how to handle errors.', - title='Error Handler', + description="Error handler component that defines how to handle errors.", + title="Error Handler", ) http_method: Optional[HttpMethod] = Field( HttpMethod.GET, - description='The HTTP method used to fetch data from the source (can be GET or POST).', - examples=['GET', 'POST'], - title='HTTP Method', + description="The HTTP method used to fetch data from the source (can be GET or POST).", + examples=["GET", "POST"], + title="HTTP Method", ) request_body_data: Optional[Union[str, Dict[str, str]]] = Field( None, - description='Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.', + description="Specifies how to populate the body of the request with a non-JSON payload. Plain text will be sent as is, whereas objects will be converted to a urlencoded form.", examples=[ '[{"clause": {"type": "timestamp", "operator": 10, "parameters":\n [{"value": {{ stream_interval[\'start_time\'] | int * 1000 }} }]\n }, "orderBy": 1, "columnName": "Timestamp"}]/\n' ], - title='Request Body Payload (Non-JSON)', + title="Request Body Payload (Non-JSON)", ) request_body_json: Optional[Union[str, Dict[str, Any]]] = Field( None, - description='Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.', + description="Specifies how to populate the body of the request with a JSON payload. Can contain nested objects.", examples=[ - {'sort_order': 'ASC', 'sort_field': 'CREATED_AT'}, - {'key': "{{ config['value'] }}"}, - {'sort': {'field': 'updated_at', 'order': 'ascending'}}, + {"sort_order": "ASC", "sort_field": "CREATED_AT"}, + {"key": "{{ config['value'] }}"}, + {"sort": {"field": "updated_at", "order": "ascending"}}, ], - title='Request Body JSON Payload', + title="Request Body JSON Payload", ) request_headers: Optional[Union[str, Dict[str, str]]] = Field( None, - description='Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.', - examples=[{'Output-Format': 'JSON'}, {'Version': "{{ config['version'] }}"}], - title='Request Headers', + description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", + examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], + title="Request Headers", ) request_parameters: Optional[Union[str, Dict[str, str]]] = Field( None, - description='Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.', + description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", examples=[ - {'unit': 'day'}, + {"unit": "day"}, { - 'query': 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' }, - {'searchIn': "{{ ','.join(config.get('search_in', [])) }}"}, - {'sort_by[asc]': 'updated_at'}, + {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, + {"sort_by[asc]": "updated_at"}, ], - title='Query Parameters', + title="Query Parameters", ) use_cache: Optional[bool] = Field( False, - description='Enables stream requests caching. This field is automatically set by the CDK.', - title='Use Cache', + description="Enables stream requests caching. This field is automatically set by the CDK.", + title="Use Cache", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class ParentStreamConfig(BaseModel): - type: Literal['ParentStreamConfig'] + type: Literal["ParentStreamConfig"] parent_key: str = Field( ..., - description='The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.', - examples=['id', "{{ config['parent_record_id'] }}"], - title='Parent Key', - ) - stream: DeclarativeStream = Field( - ..., description='Reference to the parent stream.', title='Parent Stream' + description="The primary key of records from the parent stream that will be used during the retrieval of records for the current substream. This parent identifier field is typically a characteristic of the child records being extracted from the source API.", + examples=["id", "{{ config['parent_record_id'] }}"], + title="Parent Key", ) + stream: DeclarativeStream = Field(..., description="Reference to the parent stream.", title="Parent Stream") partition_field: str = Field( ..., - description='While iterating over parent records during a sync, the parent_key value can be referenced by using this field.', - examples=['parent_id', "{{ config['parent_partition_field'] }}"], - title='Current Parent Key Value Identifier', + description="While iterating over parent records during a sync, the parent_key value can be referenced by using this field.", + examples=["parent_id", "{{ config['parent_partition_field'] }}"], + title="Current Parent Key Value Identifier", ) request_option: Optional[RequestOption] = Field( None, - description='A request option describing where the parent key value should be injected into and under what field name if applicable.', - title='Request Option', + description="A request option describing where the parent key value should be injected into and under what field name if applicable.", + title="Request Option", ) incremental_dependency: Optional[bool] = Field( False, - description='Indicates whether the parent stream should be read incrementally based on updates in the child stream.', - title='Incremental Dependency', + description="Indicates whether the parent stream should be read incrementally based on updates in the child stream.", + title="Incremental Dependency", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SimpleRetriever(BaseModel): - type: Literal['SimpleRetriever'] + type: Literal["SimpleRetriever"] record_selector: RecordSelector = Field( ..., - description='Component that describes how to extract records from a HTTP response.', + description="Component that describes how to extract records from a HTTP response.", ) requester: Union[CustomRequester, HttpRequester] = Field( ..., - description='Requester component that describes how to prepare HTTP requests to send to the source API.', + description="Requester component that describes how to prepare HTTP requests to send to the source API.", ) paginator: Optional[Union[DefaultPaginator, NoPagination]] = Field( None, @@ -1583,40 +1556,36 @@ class SimpleRetriever(BaseModel): ) ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( False, - description='If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.', + description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", ) partition_router: Optional[ Union[ CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter, - List[ - Union[ - CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter - ] - ], + List[Union[CustomPartitionRouter, ListPartitionRouter, SubstreamPartitionRouter]], ] ] = Field( [], - description='PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.', - title='Partition Router', + description="PartitionRouter component that describes how to partition the stream, enabling incremental syncs and checkpointing.", + title="Partition Router", ) decoder: Optional[Union[JsonDecoder, JsonlDecoder, IterableDecoder]] = Field( None, - description='Component decoding the response so records can be extracted.', - title='Decoder', + description="Component decoding the response so records can be extracted.", + title="Decoder", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") class SubstreamPartitionRouter(BaseModel): - type: Literal['SubstreamPartitionRouter'] + type: Literal["SubstreamPartitionRouter"] parent_stream_configs: List[ParentStreamConfig] = Field( ..., - description='Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.', - title='Parent Stream Configs', + description="Specifies which parent streams are being iterated over and how parent records should be used to partition the child stream data set.", + title="Parent Stream Configs", ) - parameters: Optional[Dict[str, Any]] = Field(None, alias='$parameters') + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") CompositeErrorHandler.update_forward_refs() diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/manifest_component_transformer.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/manifest_component_transformer.py index 8efa4d6185804..a7f78c4f2b0d8 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/manifest_component_transformer.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/manifest_component_transformer.py @@ -3,61 +3,63 @@ # import copy -import typing import logging +import typing from typing import Any, Mapping, Set, Type -from .declarative_component_schema import * + from pydantic import BaseModel +from .declarative_component_schema import * + PARAMETERS_STR = "$parameters" # Mapping of component type to the class that implements it. This is used to fetch the Pydantic model class for a component type COMPONENT_TYPE_REGISTY: Dict[str, type] = { - 'BasicHttpAuthenticator': BasicHttpAuthenticator, - 'BearerAuthenticator': BearerAuthenticator, - 'ConstantBackoffStrategy': ConstantBackoffStrategy, - 'CustomAuthenticator': CustomAuthenticator, - 'CustomBackoffStrategy': CustomBackoffStrategy, - 'CustomErrorHandler': CustomErrorHandler, - 'CustomIncrementalSync': CustomIncrementalSync, - 'CustomPaginationStrategy': CustomPaginationStrategy, - 'CustomRecordExtractor': CustomRecordExtractor, - 'CustomRecordFilter': CustomRecordFilter, - 'CustomRequester': CustomRequester, - 'CustomRetriever': CustomRetriever, - 'CustomPartitionRouter': CustomPartitionRouter, - 'CustomSchemaLoader': CustomSchemaLoader, - 'CustomStateMigration': CustomStateMigration, - 'CustomTransformation': CustomTransformation, - 'JwtAuthenticator': JwtAuthenticator, - '0AuthAuthenticator': OAuthAuthenticator, - 'ExponentialBackoffStrategy': ExponentialBackoffStrategy, - 'HttpResponseFilter': HttpResponseFilter, - 'JsonFileSchemaLoader': JsonFileSchemaLoader, - 'MinMaxDatetime': MinMaxDatetime, - 'OffsetIncrement': OffsetIncrement, - 'PageIncrement': PageIncrement, - 'RecordFilter': RecordFilter, - 'LegacySessionTokenAuthenticator': LegacySessionTokenAuthenticator, - 'WaitTimeFromHeader': WaitTimeFromHeader, - 'WaitUntilTimeFromHeader': WaitUntilTimeFromHeader, - 'ApiKeyAuthenticator': ApiKeyAuthenticator, - 'CursorPagination': CursorPagination, - 'DatetimeBasedCursor': DatetimeBasedCursor, - 'DefaultErrorHandler': DefaultErrorHandler, - 'DefaultPaginator': DefaultPaginator, - 'DpathExtractor': DpathExtractor, - 'ListPartitionRouter': ListPartitionRouter, - 'RecordSelector': RecordSelector, - 'CompositeErrorHandler': CompositeErrorHandler, - 'SelectiveAuthenticator': SelectiveAuthenticator, - 'DeclarativeStream': DeclarativeStream, - 'SessionTokenAuthenticator': SessionTokenAuthenticator, - 'HttpRequester': HttpRequester, - 'ParentStreamConfig': ParentStreamConfig, - 'SimpleRetriever': SimpleRetriever, - 'SubstreamPartitionRouter': SubstreamPartitionRouter, - 'DeclarativeSource': DeclarativeSource + "BasicHttpAuthenticator": BasicHttpAuthenticator, + "BearerAuthenticator": BearerAuthenticator, + "ConstantBackoffStrategy": ConstantBackoffStrategy, + "CustomAuthenticator": CustomAuthenticator, + "CustomBackoffStrategy": CustomBackoffStrategy, + "CustomErrorHandler": CustomErrorHandler, + "CustomIncrementalSync": CustomIncrementalSync, + "CustomPaginationStrategy": CustomPaginationStrategy, + "CustomRecordExtractor": CustomRecordExtractor, + "CustomRecordFilter": CustomRecordFilter, + "CustomRequester": CustomRequester, + "CustomRetriever": CustomRetriever, + "CustomPartitionRouter": CustomPartitionRouter, + "CustomSchemaLoader": CustomSchemaLoader, + "CustomStateMigration": CustomStateMigration, + "CustomTransformation": CustomTransformation, + "JwtAuthenticator": JwtAuthenticator, + "0AuthAuthenticator": OAuthAuthenticator, + "ExponentialBackoffStrategy": ExponentialBackoffStrategy, + "HttpResponseFilter": HttpResponseFilter, + "JsonFileSchemaLoader": JsonFileSchemaLoader, + "MinMaxDatetime": MinMaxDatetime, + "OffsetIncrement": OffsetIncrement, + "PageIncrement": PageIncrement, + "RecordFilter": RecordFilter, + "LegacySessionTokenAuthenticator": LegacySessionTokenAuthenticator, + "WaitTimeFromHeader": WaitTimeFromHeader, + "WaitUntilTimeFromHeader": WaitUntilTimeFromHeader, + "ApiKeyAuthenticator": ApiKeyAuthenticator, + "CursorPagination": CursorPagination, + "DatetimeBasedCursor": DatetimeBasedCursor, + "DefaultErrorHandler": DefaultErrorHandler, + "DefaultPaginator": DefaultPaginator, + "DpathExtractor": DpathExtractor, + "ListPartitionRouter": ListPartitionRouter, + "RecordSelector": RecordSelector, + "CompositeErrorHandler": CompositeErrorHandler, + "SelectiveAuthenticator": SelectiveAuthenticator, + "DeclarativeStream": DeclarativeStream, + "SessionTokenAuthenticator": SessionTokenAuthenticator, + "HttpRequester": HttpRequester, + "ParentStreamConfig": ParentStreamConfig, + "SimpleRetriever": SimpleRetriever, + "SubstreamPartitionRouter": SubstreamPartitionRouter, + "DeclarativeSource": DeclarativeSource, } DEFAULT_MODEL_TYPES: Mapping[str, str] = { @@ -127,12 +129,14 @@ logger = logging.getLogger(__name__) + def get_model_fields(model_class: Optional[Type[BaseModel]]) -> Set[str]: - """ Fetches field names from a Pydantic model class if available. """ + """Fetches field names from a Pydantic model class if available.""" if model_class is not None: return set(model_class.__fields__.keys()) return set() + class ManifestComponentTransformer: def propagate_types_and_parameters( self, @@ -169,7 +173,7 @@ def propagate_types_and_parameters( # * connection_specification is a Mapping if "type" not in propagated_component or self._is_json_schema_object(propagated_component): return propagated_component - + component_type = propagated_component.get("type", "") model_class = COMPONENT_TYPE_REGISTY.get(component_type) # Grab the list of expected fields for the component type diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py index 2483ce61bdb57..f85ed87c4eff9 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/pipeline.py @@ -166,7 +166,7 @@ async def _run(self) -> StepResult: resolved_manifest = ManifestReferenceResolver().preprocess_manifest(manifest) propagated_manifest = ManifestComponentTransformer().propagate_types_and_parameters("", resolved_manifest, {}) cleaned_manifest = remove_parameters_from_manifest(propagated_manifest) - + write_yaml(cleaned_manifest, root_manifest_path) except Exception as e: return StepResult(step=self, status=StepStatus.FAILURE, stdout=f"Failed to update version in manifest.yaml: {e}") diff --git a/airbyte-ci/connectors/pipelines/pyproject.toml b/airbyte-ci/connectors/pipelines/pyproject.toml index c7f076475ba71..8a6c4fe762fb2 100644 --- a/airbyte-ci/connectors/pipelines/pyproject.toml +++ b/airbyte-ci/connectors/pipelines/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "pipelines" -version = "4.33.1" +version = "4.33.2" description = "Packaged maintained by the connector operations team to perform CI for connectors' pipelines" authors = ["Airbyte "] From b4a0ba9a95c27e39ee7de9f880cee8db39842f07 Mon Sep 17 00:00:00 2001 From: Natik Gadzhi Date: Thu, 22 Aug 2024 12:11:17 -0700 Subject: [PATCH 6/7] Update airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/declarative_component_schema.py --- .../migrate_to_manifest_only/declarative_component_schema.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/declarative_component_schema.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/declarative_component_schema.py index 810d52efac27b..c58c68dc0968a 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/declarative_component_schema.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/declarative_component_schema.py @@ -11,7 +11,9 @@ from pydantic import BaseModel, Extra, Field from typing_extensions import Literal -## This is a stupid hack. I've copied the current version of the models from the generated code and pasted them here since we can't access the CDK from the current directory. +## WARNING +## This is a hack. I've copied the current version of the models from the generated code and pasted them here since we can't access the CDK from the current directory. +## These are from CDK 4.5.3. ## These models won't update alongside the CDK, so this is a temporary solution to unblock manifest-only migrations. From 453b847d68426ca8e842f3c0fd893ed4c28ddd77 Mon Sep 17 00:00:00 2001 From: Christo Grabowski <108154848+ChristoGrab@users.noreply.github.com> Date: Thu, 22 Aug 2024 15:17:18 -0400 Subject: [PATCH 7/7] Update airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/manifest_component_transformer.py --- .../migrate_to_manifest_only/manifest_component_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/manifest_component_transformer.py b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/manifest_component_transformer.py index a7f78c4f2b0d8..0c35f8ced463c 100644 --- a/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/manifest_component_transformer.py +++ b/airbyte-ci/connectors/pipelines/pipelines/airbyte_ci/connectors/migrate_to_manifest_only/manifest_component_transformer.py @@ -32,7 +32,7 @@ "CustomStateMigration": CustomStateMigration, "CustomTransformation": CustomTransformation, "JwtAuthenticator": JwtAuthenticator, - "0AuthAuthenticator": OAuthAuthenticator, + "OAuthAuthenticator": OAuthAuthenticator, "ExponentialBackoffStrategy": ExponentialBackoffStrategy, "HttpResponseFilter": HttpResponseFilter, "JsonFileSchemaLoader": JsonFileSchemaLoader,