[low code connectors] perform schema validation of the input config against the declarative language schema (#15543)

* draft: first pass at complete schema language generation and factory validator

* actually a working validator and fixes to the schema that went uncaught

* remove extra spike file

* fix formatting file

* pr feedback and a little bit of refactoring

* fix some types that were erroneously marked as invalid schema

* some comments

* add jsonschemamixin to interfaces

* update changelog

* bump version

Author: brianjlai

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.1.77
+- Add schema validation for declarative YAML connector configs
+
 ## 0.1.76
 - Bugfix: Correctly set parent slice stream for sub-resource streams
 

airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/declarative_authenticator.py

@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+from dataclasses import dataclass
+
+from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
+from dataclasses_jsonschema import JsonSchemaMixin
+
+
+@dataclass
+class DeclarativeAuthenticator(JsonSchemaMixin):
+    """
+    Interface used to associate which authenticators can be used as part of the declarative framework
+    """
+
+
+@dataclass
+class NoAuth(AbstractHeaderAuthenticator, DeclarativeAuthenticator, JsonSchemaMixin):
+    @property
+    def auth_header(self) -> str:
+        return ""
+
+    @property
+    def token(self) -> str:
+        return ""

airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/oauth.py

@@ -6,14 +6,15 @@
 from typing import Any, List, Mapping, Optional, Union
 
 import pendulum
+from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
 from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_oauth import AbstractOauth2Authenticator
 from dataclasses_jsonschema import JsonSchemaMixin
 
 
 @dataclass
-class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, JsonSchemaMixin):
+class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, DeclarativeAuthenticator, JsonSchemaMixin):
     """
     Generates OAuth2.0 access tokens from an OAuth2.0 refresh token and client credentials based on
     a declarative connector configuration file. Credentials can be defined explicitly or via interpolation
@@ -40,7 +41,7 @@ class DeclarativeOauth2Authenticator(AbstractOauth2Authenticator, JsonSchemaMixi
     options: InitVar[Mapping[str, Any]]
     scopes: Optional[List[str]] = None
     token_expiry_date: Optional[Union[InterpolatedString, str]] = None
-    _token_expiry_date: pendulum.DateTime = field(init=False, repr=False)
+    _token_expiry_date: pendulum.DateTime = field(init=False, repr=False, default=None)
     access_token_name: Union[InterpolatedString, str] = "access_token"
     expires_in_name: Union[InterpolatedString, str] = "expires_in"
     refresh_request_body: Optional[Mapping[str, Any]] = None

airbyte-cdk/python/airbyte_cdk/sources/declarative/auth/token.py

@@ -6,14 +6,15 @@
 from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Union
 
+from airbyte_cdk.sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.types import Config
 from airbyte_cdk.sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
 from dataclasses_jsonschema import JsonSchemaMixin
 
 
 @dataclass
-class ApiKeyAuthenticator(AbstractHeaderAuthenticator, JsonSchemaMixin):
+class ApiKeyAuthenticator(AbstractHeaderAuthenticator, DeclarativeAuthenticator, JsonSchemaMixin):
     """
     ApiKeyAuth sets a request header on the HTTP requests sent.
 
@@ -51,7 +52,7 @@ def token(self) -> str:
 
 
 @dataclass
-class BearerAuthenticator(AbstractHeaderAuthenticator, JsonSchemaMixin):
+class BearerAuthenticator(AbstractHeaderAuthenticator, DeclarativeAuthenticator, JsonSchemaMixin):
     """
     Authenticator that sets the Authorization header on the HTTP requests sent.
 
@@ -81,7 +82,7 @@ def token(self) -> str:
 
 
 @dataclass
-class BasicHttpAuthenticator(AbstractHeaderAuthenticator):
+class BasicHttpAuthenticator(AbstractHeaderAuthenticator, DeclarativeAuthenticator, JsonSchemaMixin):
     """
     Builds auth based off the basic authentication scheme as defined by RFC 7617, which transmits credentials as USER ID/password pairs, encoded using base64
     https://developer.mozilla.org/en-US/docs/Web/HTTP/Authentication#basic_authentication_scheme

airbyte-cdk/python/airbyte_cdk/sources/declarative/declarative_stream.py

@@ -36,10 +36,10 @@ class DeclarativeStream(Stream, JsonSchemaMixin):
     config: Config
     options: InitVar[Mapping[str, Any]]
     name: str
-    _name: str = field(init=False, repr=False)
+    _name: str = field(init=False, repr=False, default="")
     primary_key: Optional[Union[str, List[str], List[List[str]]]]
-    _primary_key: str = field(init=False, repr=False)
-    stream_cursor_field: Optional[List[str]] = None
+    _primary_key: str = field(init=False, repr=False, default="")
+    stream_cursor_field: Optional[Union[List[str], str]] = None
     transformations: List[RecordTransformation] = None
     checkpoint_interval: Optional[int] = None
 

airbyte-cdk/python/airbyte_cdk/sources/declarative/decoders/decoder.py

@@ -2,15 +2,16 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
-from abc import ABC, abstractmethod
+from abc import abstractmethod
 from dataclasses import dataclass
 from typing import Any, List, Mapping, Union
 
 import requests
+from dataclasses_jsonschema import JsonSchemaMixin
 
 
 @dataclass
-class Decoder(ABC):
+class Decoder(JsonSchemaMixin):
     """
     Decoder strategy to transform a requests.Response into a Mapping[str, Any]
     """

airbyte-cdk/python/airbyte_cdk/sources/declarative/decoders/json_decoder.py

@@ -7,10 +7,11 @@
 
 import requests
 from airbyte_cdk.sources.declarative.decoders.decoder import Decoder
+from dataclasses_jsonschema import JsonSchemaMixin
 
 
 @dataclass
-class JsonDecoder(Decoder):
+class JsonDecoder(Decoder, JsonSchemaMixin):
     """
     Decoder strategy that returns the json-encoded content of a response, if any.
     """

airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/http_selector.py

@@ -2,16 +2,17 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
-from abc import ABC, abstractmethod
+from abc import abstractmethod
 from dataclasses import dataclass
 from typing import Any, List, Mapping, Optional
 
 import requests
 from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState
+from dataclasses_jsonschema import JsonSchemaMixin
 
 
 @dataclass
-class HttpSelector(ABC):
+class HttpSelector(JsonSchemaMixin):
     """
     Responsible for translating an HTTP response into a list of records by extracting records from the response and optionally filtering
     records based on a heuristic.

airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_extractor.py

@@ -2,16 +2,17 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
-from abc import ABC, abstractmethod
+from abc import abstractmethod
 from dataclasses import dataclass
 from typing import List
 
 import requests
 from airbyte_cdk.sources.declarative.types import Record
+from dataclasses_jsonschema import JsonSchemaMixin
 
 
 @dataclass
-class RecordExtractor(ABC):
+class RecordExtractor(JsonSchemaMixin):
     """
     Responsible for translating an HTTP response into a list of records by extracting records from the response.
     """

airbyte-cdk/python/airbyte_cdk/sources/declarative/extractors/record_filter.py

@@ -2,7 +2,7 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
-from dataclasses import InitVar, dataclass, field
+from dataclasses import InitVar, dataclass
 from typing import Any, List, Mapping, Optional
 
 from airbyte_cdk.sources.declarative.interpolation.interpolated_boolean import InterpolatedBoolean
@@ -20,7 +20,7 @@ class RecordFilter(JsonSchemaMixin):
     """
 
     options: InitVar[Mapping[str, Any]]
-    config: Config = field(default=dict)
+    config: Config
     condition: str = ""
 
     def __post_init__(self, options: Mapping[str, Any]):