diff --git a/airbyte-cdk/python/airbyte_cdk/config_observation.py b/airbyte-cdk/python/airbyte_cdk/config_observation.py index 55e03f335c869..94a3d64a511b3 100644 --- a/airbyte-cdk/python/airbyte_cdk/config_observation.py +++ b/airbyte-cdk/python/airbyte_cdk/config_observation.py @@ -10,7 +10,15 @@ from copy import copy from typing import Any, List, MutableMapping -from airbyte_cdk.models import AirbyteControlConnectorConfigMessage, AirbyteControlMessage, AirbyteMessage, OrchestratorType, Type +from airbyte_cdk.models import ( + AirbyteControlConnectorConfigMessage, + AirbyteControlMessage, + AirbyteMessage, + AirbyteMessageSerializer, + OrchestratorType, + Type, +) +from orjson import orjson class ObservedDict(dict): # type: ignore # disallow_any_generics is set to True, and dict is equivalent to dict[Any] @@ -76,7 +84,7 @@ def emit_configuration_as_airbyte_control_message(config: MutableMapping[str, An See the airbyte_cdk.sources.message package """ airbyte_message = create_connector_config_control_message(config) - print(airbyte_message.model_dump_json(exclude_unset=True)) + print(orjson.dumps(AirbyteMessageSerializer.dump(airbyte_message)).decode()) def create_connector_config_control_message(config: MutableMapping[str, Any]) -> AirbyteMessage: diff --git a/airbyte-cdk/python/airbyte_cdk/connector.py b/airbyte-cdk/python/airbyte_cdk/connector.py index e40ace288e2b2..658a0b167077f 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector.py +++ b/airbyte-cdk/python/airbyte_cdk/connector.py @@ -11,7 +11,7 @@ from typing import Any, Generic, Mapping, Optional, Protocol, TypeVar import yaml -from airbyte_cdk.models import AirbyteConnectionStatus, ConnectorSpecification +from airbyte_cdk.models import AirbyteConnectionStatus, ConnectorSpecification, ConnectorSpecificationSerializer def load_optional_package_file(package: str, filename: str) -> Optional[bytes]: @@ -84,7 +84,7 @@ def spec(self, logger: logging.Logger) -> ConnectorSpecification: else: raise FileNotFoundError("Unable to find spec.yaml or spec.json in the package.") - return ConnectorSpecification.parse_obj(spec_obj) + return ConnectorSpecificationSerializer.load(spec_obj) @abstractmethod def check(self, logger: logging.Logger, config: TConfig) -> AirbyteConnectionStatus: diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py b/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py index 9f12b8aaeb114..1691b41b090d9 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/main.py @@ -9,10 +9,17 @@ from airbyte_cdk.connector import BaseConnector from airbyte_cdk.connector_builder.connector_builder_handler import TestReadLimits, create_source, get_limits, read_stream, resolve_manifest from airbyte_cdk.entrypoint import AirbyteEntrypoint -from airbyte_cdk.models import AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteMessageSerializer, + AirbyteStateMessage, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, +) from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource from airbyte_cdk.sources.source import Source from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from orjson import orjson def get_config_and_catalog_from_args(args: List[str]) -> Tuple[str, Mapping[str, Any], Optional[ConfiguredAirbyteCatalog], Any]: @@ -32,7 +39,7 @@ def get_config_and_catalog_from_args(args: List[str]) -> Tuple[str, Mapping[str, command = config["__command"] if command == "test_read": - catalog = ConfiguredAirbyteCatalog.parse_obj(BaseConnector.read_config(catalog_path)) + catalog = ConfiguredAirbyteCatalogSerializer.load(BaseConnector.read_config(catalog_path)) state = Source.read_state(state_path) else: catalog = None @@ -67,7 +74,7 @@ def handle_request(args: List[str]) -> AirbyteMessage: command, config, catalog, state = get_config_and_catalog_from_args(args) limits = get_limits(config) source = create_source(config, limits) - return handle_connector_builder_request(source, command, config, catalog, state, limits).json(exclude_unset=True) + return AirbyteMessageSerializer.dump(handle_connector_builder_request(source, command, config, catalog, state, limits)) # type: ignore[no-any-return] # Serializer.dump() always returns AirbyteMessage if __name__ == "__main__": @@ -76,4 +83,4 @@ def handle_request(args: List[str]) -> AirbyteMessage: except Exception as exc: error = AirbyteTracedException.from_exception(exc, message=f"Error handling request: {str(exc)}") m = error.as_airbyte_message() - print(error.as_airbyte_message().model_dump_json(exclude_unset=True)) + print(orjson.dumps(AirbyteMessageSerializer.dump(m)).decode()) diff --git a/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py b/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py index 80cb8c36178ea..4b00fc874cf53 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py +++ b/airbyte-cdk/python/airbyte_cdk/connector_builder/message_grouper.py @@ -18,13 +18,7 @@ StreamReadSlices, ) from airbyte_cdk.entrypoint import AirbyteEntrypoint -from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource -from airbyte_cdk.sources.utils.slice_logger import SliceLogger -from airbyte_cdk.sources.utils.types import JsonType -from airbyte_cdk.utils import AirbyteTracedException -from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer -from airbyte_cdk.utils.schema_inferrer import SchemaInferrer, SchemaValidationException -from airbyte_protocol.models.airbyte_protocol import ( +from airbyte_cdk.models import ( AirbyteControlMessage, AirbyteLogMessage, AirbyteMessage, @@ -34,7 +28,13 @@ OrchestratorType, TraceType, ) -from airbyte_protocol.models.airbyte_protocol import Type as MessageType +from airbyte_cdk.models import Type as MessageType +from airbyte_cdk.sources.declarative.declarative_source import DeclarativeSource +from airbyte_cdk.sources.utils.slice_logger import SliceLogger +from airbyte_cdk.sources.utils.types import JsonType +from airbyte_cdk.utils import AirbyteTracedException +from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer +from airbyte_cdk.utils.schema_inferrer import SchemaInferrer, SchemaValidationException class MessageGrouper: @@ -182,19 +182,19 @@ def _get_message_groups( if ( at_least_one_page_in_group and message.type == MessageType.LOG - and message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX) + and message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX) # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message ): yield StreamReadSlices( pages=current_slice_pages, slice_descriptor=current_slice_descriptor, state=[latest_state_message] if latest_state_message else [], ) - current_slice_descriptor = self._parse_slice_description(message.log.message) + current_slice_descriptor = self._parse_slice_description(message.log.message) # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message current_slice_pages = [] at_least_one_page_in_group = False - elif message.type == MessageType.LOG and message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX): + elif message.type == MessageType.LOG and message.log.message.startswith(SliceLogger.SLICE_LOG_PREFIX): # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message # parsing the first slice - current_slice_descriptor = self._parse_slice_description(message.log.message) + current_slice_descriptor = self._parse_slice_description(message.log.message) # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message elif message.type == MessageType.LOG: if json_message is not None and self._is_http_log(json_message): if self._is_auxiliary_http_request(json_message): @@ -221,17 +221,17 @@ def _get_message_groups( else: yield message.log elif message.type == MessageType.TRACE: - if message.trace.type == TraceType.ERROR: + if message.trace.type == TraceType.ERROR: # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has trace.type yield message.trace elif message.type == MessageType.RECORD: - current_page_records.append(message.record.data) + current_page_records.append(message.record.data) # type: ignore[union-attr] # AirbyteMessage with MessageType.RECORD has record.data records_count += 1 schema_inferrer.accumulate(message.record) datetime_format_inferrer.accumulate(message.record) - elif message.type == MessageType.CONTROL and message.control.type == OrchestratorType.CONNECTOR_CONFIG: + elif message.type == MessageType.CONTROL and message.control.type == OrchestratorType.CONNECTOR_CONFIG: # type: ignore[union-attr] # AirbyteMessage with MessageType.CONTROL has control.type yield message.control elif message.type == MessageType.STATE: - latest_state_message = message.state + latest_state_message = message.state # type: ignore[assignment] else: if current_page_request or current_page_response or current_page_records: self._close_page(current_page_request, current_page_response, current_slice_pages, current_page_records) @@ -246,7 +246,7 @@ def _need_to_close_page(at_least_one_page_in_group: bool, message: AirbyteMessag return ( at_least_one_page_in_group and message.type == MessageType.LOG - and (MessageGrouper._is_page_http_request(json_message) or message.log.message.startswith("slice:")) + and (MessageGrouper._is_page_http_request(json_message) or message.log.message.startswith("slice:")) # type: ignore[union-attr] # AirbyteMessage with MessageType.LOG has log.message ) @staticmethod diff --git a/airbyte-cdk/python/airbyte_cdk/destinations/destination.py b/airbyte-cdk/python/airbyte_cdk/destinations/destination.py index f95e185aabfe4..336a54a94e8f1 100644 --- a/airbyte-cdk/python/airbyte_cdk/destinations/destination.py +++ b/airbyte-cdk/python/airbyte_cdk/destinations/destination.py @@ -11,10 +11,10 @@ from airbyte_cdk.connector import Connector from airbyte_cdk.exception_handler import init_uncaught_exception_handler -from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Type +from airbyte_cdk.models import AirbyteMessage, AirbyteMessageSerializer, ConfiguredAirbyteCatalog, ConfiguredAirbyteCatalogSerializer, Type from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from pydantic import ValidationError as V2ValidationError +from orjson import orjson logger = logging.getLogger("airbyte") @@ -36,14 +36,14 @@ def _parse_input_stream(self, input_stream: io.TextIOWrapper) -> Iterable[Airbyt """Reads from stdin, converting to Airbyte messages""" for line in input_stream: try: - yield AirbyteMessage.parse_raw(line) - except V2ValidationError: + yield AirbyteMessageSerializer.load(orjson.loads(line)) + except orjson.JSONDecodeError: logger.info(f"ignoring input which can't be deserialized as Airbyte Message: {line}") def _run_write( self, config: Mapping[str, Any], configured_catalog_path: str, input_stream: io.TextIOWrapper ) -> Iterable[AirbyteMessage]: - catalog = ConfiguredAirbyteCatalog.parse_file(configured_catalog_path) + catalog = ConfiguredAirbyteCatalogSerializer.load(orjson.loads(open(configured_catalog_path).read())) input_messages = self._parse_input_stream(input_stream) logger.info("Begin writing to the destination...") yield from self.write(config=config, configured_catalog=catalog, input_messages=input_messages) @@ -117,4 +117,4 @@ def run(self, args: List[str]) -> None: parsed_args = self.parse_args(args) output_messages = self.run_cmd(parsed_args) for message in output_messages: - print(message.model_dump_json(exclude_unset=True)) + print(orjson.dumps(AirbyteMessageSerializer.dump(message)).decode()) diff --git a/airbyte-cdk/python/airbyte_cdk/entrypoint.py b/airbyte-cdk/python/airbyte_cdk/entrypoint.py index cc9c3662ff671..57b604691bc17 100644 --- a/airbyte-cdk/python/airbyte_cdk/entrypoint.py +++ b/airbyte-cdk/python/airbyte_cdk/entrypoint.py @@ -19,8 +19,15 @@ from airbyte_cdk.connector import TConfig from airbyte_cdk.exception_handler import init_uncaught_exception_handler from airbyte_cdk.logger import init_logger -from airbyte_cdk.models import AirbyteMessage, FailureType, Status, Type -from airbyte_cdk.models.airbyte_protocol import AirbyteStateStats, ConnectorSpecification # type: ignore [attr-defined] +from airbyte_cdk.models import ( # type: ignore [attr-defined] + AirbyteMessage, + AirbyteMessageSerializer, + AirbyteStateStats, + ConnectorSpecification, + FailureType, + Status, + Type, +) from airbyte_cdk.sources import Source from airbyte_cdk.sources.connector_state_manager import HashableStreamDescriptor from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit, split_config @@ -28,6 +35,7 @@ from airbyte_cdk.utils.airbyte_secrets_utils import get_secrets, update_secrets from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from orjson import orjson from requests import PreparedRequest, Response, Session logger = init_logger("airbyte") @@ -170,13 +178,13 @@ def read(self, source_spec: ConnectorSpecification, config: TConfig, catalog: An def handle_record_counts(message: AirbyteMessage, stream_message_count: DefaultDict[HashableStreamDescriptor, float]) -> AirbyteMessage: match message.type: case Type.RECORD: - stream_message_count[HashableStreamDescriptor(name=message.record.stream, namespace=message.record.namespace)] += 1.0 + stream_message_count[HashableStreamDescriptor(name=message.record.stream, namespace=message.record.namespace)] += 1.0 # type: ignore[union-attr] # record has `stream` and `namespace` case Type.STATE: stream_descriptor = message_utils.get_stream_descriptor(message) # Set record count from the counter onto the state message - message.state.sourceStats = message.state.sourceStats or AirbyteStateStats() - message.state.sourceStats.recordCount = stream_message_count.get(stream_descriptor, 0.0) + message.state.sourceStats = message.state.sourceStats or AirbyteStateStats() # type: ignore[union-attr] # state has `sourceStats` + message.state.sourceStats.recordCount = stream_message_count.get(stream_descriptor, 0.0) # type: ignore[union-attr] # state has `sourceStats` # Reset the counter stream_message_count[stream_descriptor] = 0.0 @@ -197,8 +205,8 @@ def set_up_secret_filter(config: TConfig, connection_specification: Mapping[str, update_secrets(config_secrets) @staticmethod - def airbyte_message_to_string(airbyte_message: AirbyteMessage) -> Any: - return airbyte_message.model_dump_json(exclude_unset=True) + def airbyte_message_to_string(airbyte_message: AirbyteMessage) -> str: + return orjson.dumps(AirbyteMessageSerializer.dump(airbyte_message)).decode() # type: ignore[no-any-return] # orjson.dumps(message).decode() always returns string @classmethod def extract_state(cls, args: List[str]) -> Optional[Any]: diff --git a/airbyte-cdk/python/airbyte_cdk/logger.py b/airbyte-cdk/python/airbyte_cdk/logger.py index 6f40e581df94b..c74be7f4b09dc 100644 --- a/airbyte-cdk/python/airbyte_cdk/logger.py +++ b/airbyte-cdk/python/airbyte_cdk/logger.py @@ -7,8 +7,10 @@ import logging.config from typing import Any, Mapping, Optional, Tuple -from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage -from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteMessageSerializer, Level, Type +from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets as base_filter_secrets, filter_secrets +from orjson import dumps as orjson_dumps, orjson +from logging import LogRecord LOGGING_CONFIG = { "version": 1, @@ -42,11 +44,11 @@ class AirbyteLogFormatter(logging.Formatter): # Transforming Python log levels to Airbyte protocol log levels level_mapping = { - logging.FATAL: "FATAL", - logging.ERROR: "ERROR", - logging.WARNING: "WARN", - logging.INFO: "INFO", - logging.DEBUG: "DEBUG", + logging.FATAL: Level.FATAL, + logging.ERROR: Level.ERROR, + logging.WARNING: Level.WARN, + logging.INFO: Level.INFO, + logging.DEBUG: Level.DEBUG, } def format(self, record: logging.LogRecord) -> str: @@ -55,23 +57,25 @@ def format(self, record: logging.LogRecord) -> str: if airbyte_level == "DEBUG": extras = self.extract_extra_args_from_record(record) debug_dict = {"type": "DEBUG", "message": record.getMessage(), "data": extras} - return filter_secrets(json.dumps(debug_dict)) + return base_filter_secrets(json.dumps(debug_dict)) else: message = super().format(record) - message = filter_secrets(message) - log_message = AirbyteMessage(type="LOG", log=AirbyteLogMessage(level=airbyte_level, message=message)) - return log_message.model_dump_json(exclude_unset=True) # type: ignore + message = base_filter_secrets(message) + log_message = AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=airbyte_level, message=message)) + return orjson_dumps(AirbyteMessageSerializer.dump(log_message)).decode() # type: ignore[no-any-return] - @staticmethod def extract_extra_args_from_record(record: logging.LogRecord) -> Mapping[str, Any]: """ The python logger conflates default args with extra args. We use an empty log record and set operations to isolate fields passed to the log record via extra by the developer. """ - default_attrs = logging.LogRecord("", 0, "", 0, None, None, None).__dict__.keys() - extra_keys = set(record.__dict__.keys()) - default_attrs + extra_keys = set(record.__dict__.keys()) - self.default_attrs return {k: str(getattr(record, k)) for k in extra_keys if hasattr(record, k)} + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.default_attrs = set(LogRecord("", 0, "", 0, None, None, None).__dict__.keys()) + def log_by_prefix(msg: str, default_level: str) -> Tuple[int, str]: """Custom method, which takes log level from first word of message""" diff --git a/airbyte-cdk/python/airbyte_cdk/models/__init__.py b/airbyte-cdk/python/airbyte_cdk/models/__init__.py index b062a4468c22f..c56df9adc43a4 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/__init__.py +++ b/airbyte-cdk/python/airbyte_cdk/models/__init__.py @@ -7,6 +7,7 @@ # of airbyte-cdk rather than a standalone package. from .airbyte_protocol import ( AdvancedAuth, + AirbyteStateStats, AirbyteAnalyticsTraceMessage, AirbyteCatalog, AirbyteConnectionStatus, @@ -58,3 +59,12 @@ TimeWithoutTimezone, TimeWithTimezone, ) + +from .airbyte_protocol_serializers import ( +AirbyteStreamStateSerializer, +AirbyteStateMessageSerializer, +AirbyteMessageSerializer, +ConfiguredAirbyteCatalogSerializer, +ConfiguredAirbyteStreamSerializer, +ConnectorSpecificationSerializer, +) \ No newline at end of file diff --git a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py index 74639c8bf3c1f..24c80f91f1c03 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py +++ b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol.py @@ -2,4 +2,81 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_protocol.models import * +from dataclasses import InitVar, dataclass +from typing import Annotated, Any, Dict, List, Mapping, Optional + +from airbyte_protocol_dataclasses.models import * +from serpyco_rs.metadata import Alias + + +@dataclass +class AirbyteStateBlob: + """ + A dataclass that dynamically sets attributes based on provided keyword arguments and positional arguments. + Used to "mimic" pydantic Basemodel with ConfigDict(extra='allow') option. + + The `AirbyteStateBlob` class allows for flexible instantiation by accepting any number of keyword arguments + and positional arguments. These are used to dynamically update the instance's attributes. This class is useful + in scenarios where the attributes of an object are not known until runtime and need to be set dynamically. + + Attributes: + kwargs (InitVar[Mapping[str, Any]]): A dictionary of keyword arguments used to set attributes dynamically. + + Methods: + __init__(*args: Any, **kwargs: Any) -> None: + Initializes the `AirbyteStateBlob` by setting attributes from the provided arguments. + + __eq__(other: object) -> bool: + Checks equality between two `AirbyteStateBlob` instances based on their internal dictionaries. + Returns `False` if the other object is not an instance of `AirbyteStateBlob`. + """ + + kwargs: InitVar[Mapping[str, Any]] + + def __init__(self, *args: Any, **kwargs: Any) -> None: + # Set any attribute passed in through kwargs + for arg in args: + self.__dict__.update(arg) + for key, value in kwargs.items(): + setattr(self, key, value) + + def __eq__(self, other: object) -> bool: + return False if not isinstance(other, AirbyteStateBlob) else bool(self.__dict__ == other.__dict__) + + +# The following dataclasses have been redeclared to include the new version of AirbyteStateBlob +@dataclass +class AirbyteStreamState: + stream_descriptor: StreamDescriptor # type: ignore [name-defined] + stream_state: Optional[AirbyteStateBlob] = None + + +@dataclass +class AirbyteGlobalState: + stream_states: List[AirbyteStreamState] + shared_state: Optional[AirbyteStateBlob] = None + + +@dataclass +class AirbyteStateMessage: + type: Optional[AirbyteStateType] = None # type: ignore [name-defined] + stream: Optional[AirbyteStreamState] = None + global_: Annotated[AirbyteGlobalState | None, Alias("global")] = ( + None # "global" is a reserved keyword in python ⇒ Alias is used for (de-)serialization + ) + data: Optional[Dict[str, Any]] = None + sourceStats: Optional[AirbyteStateStats] = None # type: ignore [name-defined] + destinationStats: Optional[AirbyteStateStats] = None # type: ignore [name-defined] + + +@dataclass +class AirbyteMessage: + type: Type # type: ignore [name-defined] + log: Optional[AirbyteLogMessage] = None # type: ignore [name-defined] + spec: Optional[ConnectorSpecification] = None # type: ignore [name-defined] + connectionStatus: Optional[AirbyteConnectionStatus] = None # type: ignore [name-defined] + catalog: Optional[AirbyteCatalog] = None # type: ignore [name-defined] + record: Optional[AirbyteRecordMessage] = None # type: ignore [name-defined] + state: Optional[AirbyteStateMessage] = None + trace: Optional[AirbyteTraceMessage] = None # type: ignore [name-defined] + control: Optional[AirbyteControlMessage] = None # type: ignore [name-defined] diff --git a/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol_serializers.py b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol_serializers.py new file mode 100644 index 0000000000000..aeac43f794ced --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/models/airbyte_protocol_serializers.py @@ -0,0 +1,38 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +from typing import Any, Dict + +from serpyco_rs import CustomType, Serializer + +from .airbyte_protocol import ( # type: ignore[attr-defined] # all classes are imported to airbyte_protocol via * + AirbyteMessage, + AirbyteStateBlob, + AirbyteStateMessage, + AirbyteStreamState, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + ConnectorSpecification, +) + + +class AirbyteStateBlobType(CustomType[AirbyteStateBlob, Dict[str, Any]]): + def serialize(self, value: AirbyteStateBlob) -> Dict[str, Any]: + # cant use orjson.dumps() directly because private attributes are excluded, e.g. "__ab_full_refresh_sync_complete" + return {k: v for k, v in value.__dict__.items()} + + def deserialize(self, value: Dict[str, Any]) -> AirbyteStateBlob: + return AirbyteStateBlob(value) + + def get_json_schema(self) -> Dict[str, Any]: + return {"type": "object"} + + +def custom_type_resolver(t: type) -> CustomType[AirbyteStateBlob, Dict[str, Any]] | None: + return AirbyteStateBlobType() if t is AirbyteStateBlob else None + + +AirbyteStreamStateSerializer = Serializer(AirbyteStreamState, omit_none=True, custom_type_resolver=custom_type_resolver) +AirbyteStateMessageSerializer = Serializer(AirbyteStateMessage, omit_none=True, custom_type_resolver=custom_type_resolver) +AirbyteMessageSerializer = Serializer(AirbyteMessage, omit_none=True, custom_type_resolver=custom_type_resolver) +ConfiguredAirbyteCatalogSerializer = Serializer(ConfiguredAirbyteCatalog, omit_none=True) +ConfiguredAirbyteStreamSerializer = Serializer(ConfiguredAirbyteStream, omit_none=True) +ConnectorSpecificationSerializer = Serializer(ConnectorSpecification, omit_none=True) diff --git a/airbyte-cdk/python/airbyte_cdk/models/well_known_types.py b/airbyte-cdk/python/airbyte_cdk/models/well_known_types.py index 0cc409c7e0709..a063ad7db03a8 100644 --- a/airbyte-cdk/python/airbyte_cdk/models/well_known_types.py +++ b/airbyte-cdk/python/airbyte_cdk/models/well_known_types.py @@ -2,4 +2,4 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_protocol.models.well_known_types import * +from airbyte_protocol_dataclasses.models.well_known_types import * diff --git a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py index f345c6b4bd754..40cc771ab7800 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/concurrent_source/concurrent_read_processor.py @@ -5,7 +5,7 @@ from typing import Dict, Iterable, List, Optional, Set from airbyte_cdk.exception_handler import generate_failed_streams_error_message -from airbyte_cdk.models import AirbyteMessage, AirbyteStreamStatus +from airbyte_cdk.models import AirbyteMessage, AirbyteStreamStatus, FailureType, StreamDescriptor from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.concurrent_source.partition_generation_completed_sentinel import PartitionGenerationCompletedSentinel from airbyte_cdk.sources.concurrent_source.stream_thread_exception import StreamThreadException @@ -21,7 +21,6 @@ from airbyte_cdk.sources.utils.slice_logger import SliceLogger from airbyte_cdk.utils import AirbyteTracedException from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message -from airbyte_protocol.models import FailureType, StreamDescriptor class ConcurrentReadProcessor: @@ -76,7 +75,7 @@ def on_partition_generation_completed(self, sentinel: PartitionGenerationComplet if self._is_stream_done(stream_name) or len(self._streams_to_running_partitions[stream_name]) == 0: yield from self._on_stream_is_done(stream_name) if self._stream_instances_to_start_partition_generation: - yield self.start_next_partition_generator() + yield self.start_next_partition_generator() # type:ignore # None may be yielded def on_partition(self, partition: Partition) -> None: """ diff --git a/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py b/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py index b550d81b7a276..547f4bb23dca0 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/connector_state_manager.py @@ -3,20 +3,22 @@ # import copy +from dataclasses import dataclass from typing import Any, List, Mapping, MutableMapping, Optional, Tuple, Union from airbyte_cdk.models import AirbyteMessage, AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor from airbyte_cdk.models import Type as MessageType -from pydantic import ConfigDict as V2ConfigDict -class HashableStreamDescriptor(StreamDescriptor): +@dataclass(frozen=True) +class HashableStreamDescriptor: """ Helper class that overrides the existing StreamDescriptor class that is auto generated from the Airbyte Protocol and freezes its fields so that it be used as a hash key. This is only marked public because we use it outside for unit tests. """ - model_config = V2ConfigDict(extra="allow", frozen=True) + name: str + namespace: Optional[str] = None class ConnectorStateManager: @@ -47,9 +49,9 @@ def get_stream_state(self, stream_name: str, namespace: Optional[str]) -> Mutabl :param namespace: Namespace of the stream being fetched :return: The per-stream state for a stream """ - stream_state = self.per_stream_states.get(HashableStreamDescriptor(name=stream_name, namespace=namespace)) + stream_state: AirbyteStateBlob | None = self.per_stream_states.get(HashableStreamDescriptor(name=stream_name, namespace=namespace)) if stream_state: - return stream_state.dict() # type: ignore # mypy thinks dict() returns any, but it returns a dict + return copy.deepcopy({k: v for k, v in stream_state.__dict__.items()}) return {} def update_state_for_stream(self, stream_name: str, namespace: Optional[str], value: Mapping[str, Any]) -> None: @@ -60,7 +62,7 @@ def update_state_for_stream(self, stream_name: str, namespace: Optional[str], va :param value: A stream state mapping that is being updated for a stream """ stream_descriptor = HashableStreamDescriptor(name=stream_name, namespace=namespace) - self.per_stream_states[stream_descriptor] = AirbyteStateBlob.parse_obj(value) + self.per_stream_states[stream_descriptor] = AirbyteStateBlob(value) def create_state_message(self, stream_name: str, namespace: Optional[str]) -> AirbyteMessage: """ @@ -100,19 +102,19 @@ def _extract_from_state_message( if is_global: global_state = state[0].global_ # type: ignore # We verified state is a list in _is_global_state - shared_state = copy.deepcopy(global_state.shared_state, {}) + shared_state = copy.deepcopy(global_state.shared_state, {}) # type: ignore[union-attr] # global_state has shared_state streams = { HashableStreamDescriptor( name=per_stream_state.stream_descriptor.name, namespace=per_stream_state.stream_descriptor.namespace ): per_stream_state.stream_state - for per_stream_state in global_state.stream_states + for per_stream_state in global_state.stream_states # type: ignore[union-attr] # global_state has shared_state } return shared_state, streams else: streams = { HashableStreamDescriptor( - name=per_stream_state.stream.stream_descriptor.name, namespace=per_stream_state.stream.stream_descriptor.namespace - ): per_stream_state.stream.stream_state + name=per_stream_state.stream.stream_descriptor.name, namespace=per_stream_state.stream.stream_descriptor.namespace # type: ignore[union-attr] # stream has stream_descriptor + ): per_stream_state.stream.stream_state # type: ignore[union-attr] # stream has stream_state for per_stream_state in state if per_stream_state.type == AirbyteStateType.STREAM and hasattr(per_stream_state, "stream") # type: ignore # state is always a list of AirbyteStateMessage if is_per_stream is True } diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py index f0869b72fa292..28c2f0eb6b8b8 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/incremental/per_partition_cursor.py @@ -4,12 +4,12 @@ from typing import Any, Callable, Iterable, Mapping, MutableMapping, Optional, Union +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter from airbyte_cdk.sources.streams.checkpoint.per_partition_key_serializer import PerPartitionKeySerializer from airbyte_cdk.sources.types import Record, StreamSlice, StreamState from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import FailureType class CursorFactory: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py index d437a5c12ae5d..79eb8a7fe23d6 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/error_handlers/backoff_strategies/wait_time_from_header_backoff_strategy.py @@ -7,12 +7,12 @@ from typing import Any, Mapping, Optional, Union import requests +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.header_helper import get_numeric_value_from_header from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategy import BackoffStrategy from airbyte_cdk.sources.types import Config from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import FailureType @dataclass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py index a0d499f5d13da..87c8911d6aa6e 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/spec/spec.py @@ -5,7 +5,7 @@ from dataclasses import InitVar, dataclass from typing import Any, Mapping, Optional -from airbyte_cdk.models.airbyte_protocol import AdvancedAuth, ConnectorSpecification # type: ignore [attr-defined] +from airbyte_cdk.models import AdvancedAuth, ConnectorSpecification, ConnectorSpecificationSerializer # type: ignore [attr-defined] from airbyte_cdk.sources.declarative.models.declarative_component_schema import AuthFlow @@ -36,7 +36,7 @@ def generate_spec(self) -> ConnectorSpecification: if self.advanced_auth: self.advanced_auth.auth_flow_type = self.advanced_auth.auth_flow_type.value # type: ignore # We know this is always assigned to an AuthFlow which has the auth_flow_type field # Map CDK AuthFlow model to protocol AdvancedAuth model - obj["advanced_auth"] = AdvancedAuth.parse_obj(self.advanced_auth.dict()) + obj["advanced_auth"] = self.advanced_auth.dict() # We remap these keys to camel case because that's the existing format expected by the rest of the platform - return ConnectorSpecification.parse_obj(obj) + return ConnectorSpecificationSerializer.load(obj) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/embedded/base_integration.py b/airbyte-cdk/python/airbyte_cdk/sources/embedded/base_integration.py index 158dea4d135a4..79c9bd850a3a5 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/embedded/base_integration.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/embedded/base_integration.py @@ -6,11 +6,11 @@ from typing import Generic, Iterable, Optional, TypeVar from airbyte_cdk.connector import TConfig +from airbyte_cdk.models import AirbyteRecordMessage, AirbyteStateMessage, SyncMode, Type from airbyte_cdk.sources.embedded.catalog import create_configured_catalog, get_stream, get_stream_names from airbyte_cdk.sources.embedded.runner import SourceRunner from airbyte_cdk.sources.embedded.tools import get_defined_id from airbyte_cdk.sources.utils.schema_helpers import check_config_against_spec_or_exit -from airbyte_protocol.models import AirbyteRecordMessage, AirbyteStateMessage, SyncMode, Type TOutput = TypeVar("TOutput") @@ -43,7 +43,7 @@ def _load_data(self, stream_name: str, state: Optional[AirbyteStateMessage] = No for message in self.source.read(self.config, configured_catalog, state): if message.type == Type.RECORD: - output = self._handle_record(message.record, get_defined_id(stream, message.record.data)) + output = self._handle_record(message.record, get_defined_id(stream, message.record.data)) # type: ignore[union-attr] # record has `data` if output: yield output elif message.type is Type.STATE and message.state: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py index bbae84b287bfa..b033afa57fb37 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/avro_parser.py @@ -166,7 +166,9 @@ def file_read_mode(self) -> FileReadMode: @staticmethod def _to_output_value(avro_format: AvroFormat, record_type: Mapping[str, Any], record_value: Any) -> Any: - if not isinstance(record_type, Mapping): + if isinstance(record_value, bytes): + return record_value.decode() + elif not isinstance(record_type, Mapping): if record_type == "double" and avro_format.double_as_string: return str(record_value) return record_value diff --git a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/excel_parser.py b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/excel_parser.py index 579a85390c663..93add4108deab 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/excel_parser.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/file_based/file_types/excel_parser.py @@ -17,6 +17,7 @@ from numpy import datetime64 from numpy import dtype as dtype_ from numpy import issubdtype +from orjson import orjson from pydantic.v1 import BaseModel @@ -97,7 +98,10 @@ def parse_records( with stream_reader.open_file(file, self.file_read_mode, self.ENCODING, logger) as fp: df = self.open_and_parse_file(fp) # Yield records as dictionaries - yield from df.to_dict(orient="records") + # DataFrame.to_dict() method returns datetime values in pandas.Timestamp values, which are not serializable by orjson + # DataFrame.to_json() returns string with datetime values serialized to iso8601 with microseconds to align with pydantic behavior + # see PR description: https://github.com/airbytehq/airbyte/pull/44444/ + yield from orjson.loads(df.to_json(orient="records", date_format="iso", date_unit="us")) except Exception as exc: # Raise a RecordParseError if any exception occurs during parsing diff --git a/airbyte-cdk/python/airbyte_cdk/sources/source.py b/airbyte-cdk/python/airbyte_cdk/sources/source.py index 77de81fbe7f1d..975770c889499 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/source.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/source.py @@ -8,7 +8,14 @@ from typing import Any, Generic, Iterable, List, Mapping, Optional, TypeVar from airbyte_cdk.connector import BaseConnector, DefaultConnectorMixin, TConfig -from airbyte_cdk.models import AirbyteCatalog, AirbyteMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog +from airbyte_cdk.models import ( + AirbyteCatalog, + AirbyteMessage, + AirbyteStateMessage, + AirbyteStateMessageSerializer, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, +) TState = TypeVar("TState") TCatalog = TypeVar("TCatalog") @@ -61,7 +68,7 @@ def read_state(cls, state_path: str) -> List[AirbyteStateMessage]: state_obj = BaseConnector._read_json_file(state_path) if state_obj: for state in state_obj: # type: ignore # `isinstance(state_obj, List)` ensures that this is a list - parsed_message = AirbyteStateMessage.parse_obj(state) + parsed_message = AirbyteStateMessageSerializer.load(state) if not parsed_message.stream and not parsed_message.data and not parsed_message.global_: raise ValueError("AirbyteStateMessage should contain either a stream, global, or state field") parsed_state_messages.append(parsed_message) @@ -70,7 +77,7 @@ def read_state(cls, state_path: str) -> List[AirbyteStateMessage]: # can be overridden to change an input catalog @classmethod def read_catalog(cls, catalog_path: str) -> ConfiguredAirbyteCatalog: - return ConfiguredAirbyteCatalog.parse_obj(cls._read_json_file(catalog_path)) + return ConfiguredAirbyteCatalogSerializer.load(cls._read_json_file(catalog_path)) @property def name(self) -> str: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py index 761a37e1f1801..0afc2974fa9ae 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/checkpoint/substream_resumable_full_refresh_cursor.py @@ -3,11 +3,11 @@ from dataclasses import dataclass from typing import Any, Mapping, MutableMapping, Optional +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.checkpoint import Cursor from airbyte_cdk.sources.streams.checkpoint.per_partition_key_serializer import PerPartitionKeySerializer from airbyte_cdk.sources.types import Record, StreamSlice, StreamState from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import FailureType FULL_REFRESH_COMPLETE_STATE: Mapping[str, Any] = {"__ab_full_refresh_sync_complete": True} diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http_client.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http_client.py index d52b926275770..b1f23aeb4e254 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http_client.py @@ -10,7 +10,14 @@ import requests import requests_cache -from airbyte_cdk.models import AirbyteStreamStatus, AirbyteStreamStatusReason, AirbyteStreamStatusReasonType, Level, StreamDescriptor +from airbyte_cdk.models import ( + AirbyteMessageSerializer, + AirbyteStreamStatus, + AirbyteStreamStatusReason, + AirbyteStreamStatusReasonType, + Level, + StreamDescriptor, +) from airbyte_cdk.sources.http_config import MAX_CONNECTION_POOL_SIZE from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.streams.call_rate import APIBudget, CachedLimiterSession, LimiterSession @@ -38,6 +45,7 @@ from airbyte_cdk.utils.constants import ENV_REQUEST_CACHE_PATH from airbyte_cdk.utils.stream_status_utils import as_airbyte_message as stream_status_as_airbyte_message from airbyte_cdk.utils.traced_exception import AirbyteTracedException +from orjson import orjson from requests.auth import AuthBase BODY_REQUEST_METHODS = ("GET", "POST", "PUT", "PATCH") @@ -281,9 +289,11 @@ def _send( if error_resolution.response_action == ResponseAction.RATE_LIMITED: # TODO: Update to handle with message repository when concurrent message repository is ready reasons = [AirbyteStreamStatusReason(type=AirbyteStreamStatusReasonType.RATE_LIMITED)] - message = stream_status_as_airbyte_message( - StreamDescriptor(name=self._name), AirbyteStreamStatus.RUNNING, reasons - ).model_dump_json(exclude_unset=True) + message = orjson.dumps( + AirbyteMessageSerializer.dump( + stream_status_as_airbyte_message(StreamDescriptor(name=self._name), AirbyteStreamStatus.RUNNING, reasons) + ) + ).decode() # Simply printing the stream status is a temporary solution and can cause future issues. Currently, the _send method is # wrapped with backoff decorators, and we can only emit messages by iterating record_iterator in the abstract source at the diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/catalog_helpers.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/catalog_helpers.py deleted file mode 100644 index 415374a44bc1e..0000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/catalog_helpers.py +++ /dev/null @@ -1,22 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.models import AirbyteCatalog, SyncMode - - -class CatalogHelper: - @staticmethod - def coerce_catalog_as_full_refresh(catalog: AirbyteCatalog) -> AirbyteCatalog: - """ - Updates the sync mode on all streams in this catalog to be full refresh - """ - coerced_catalog = catalog.copy() - for stream in catalog.streams: - stream.source_defined_cursor = False - stream.supported_sync_modes = [SyncMode.full_refresh] - stream.default_cursor_field = None - - # remove nulls - return AirbyteCatalog.parse_raw(coerced_catalog.model_dump_json(exclude_unset=True, exclude_none=True)) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_models.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_models.py deleted file mode 100644 index de011bfb896b9..0000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/schema_models.py +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import Any, Dict, Optional, Type - -from airbyte_cdk.sources.utils.schema_helpers import expand_refs -from pydantic.v1 import BaseModel, Extra -from pydantic.v1.main import ModelMetaclass -from pydantic.v1.typing import resolve_annotations - - -class AllOptional(ModelMetaclass): - """ - Metaclass for marking all Pydantic model fields as Optional - Here is example of declaring model using this metaclass like: - ''' - class MyModel(BaseModel, metaclass=AllOptional): - a: str - b: str - ''' - it is an equivalent of: - ''' - class MyModel(BaseModel): - a: Optional[str] - b: Optional[str] - ''' - It would make code more clear and eliminate a lot of manual work. - """ - - def __new__(mcs, name, bases, namespaces, **kwargs): # type: ignore[no-untyped-def] # super().__new__ is also untyped - """ - Iterate through fields and wrap then with typing.Optional type. - """ - annotations = resolve_annotations(namespaces.get("__annotations__", {}), namespaces.get("__module__", None)) - for base in bases: - annotations = {**annotations, **getattr(base, "__annotations__", {})} - for field in annotations: - if not field.startswith("__"): - annotations[field] = Optional[annotations[field]] # type: ignore[assignment] - namespaces["__annotations__"] = annotations - return super().__new__(mcs, name, bases, namespaces, **kwargs) - - -class BaseSchemaModel(BaseModel): - """ - Base class for all schema models. It has some extra schema postprocessing. - Can be used in combination with AllOptional metaclass - """ - - class Config: - extra = Extra.allow - - @classmethod - def schema_extra(cls, schema: Dict[str, Any], model: Type[BaseModel]) -> None: - """Modify generated jsonschema, remove "title", "description" and "required" fields. - - Pydantic doesn't treat Union[None, Any] type correctly when generate jsonschema, - so we can't set field as nullable (i.e. field that can have either null and non-null values), - We generate this jsonschema value manually. - - :param schema: generated jsonschema - :param model: - """ - schema.pop("title", None) - schema.pop("description", None) - schema.pop("required", None) - for name, prop in schema.get("properties", {}).items(): - prop.pop("title", None) - prop.pop("description", None) - allow_none = model.__fields__[name].allow_none - if allow_none: - if "type" in prop: - prop["type"] = ["null", prop["type"]] - elif "$ref" in prop: - ref = prop.pop("$ref") - prop["oneOf"] = [{"type": "null"}, {"$ref": ref}] - - @classmethod - def schema(cls, *args: Any, **kwargs: Any) -> Dict[str, Any]: - """We're overriding the schema classmethod to enable some post-processing""" - schema = super().schema(*args, **kwargs) - expand_refs(schema) - return schema # type: ignore[no-any-return] diff --git a/airbyte-cdk/python/airbyte_cdk/test/catalog_builder.py b/airbyte-cdk/python/airbyte_cdk/test/catalog_builder.py index c3e3578f34941..235be7c579b6e 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/catalog_builder.py +++ b/airbyte-cdk/python/airbyte_cdk/test/catalog_builder.py @@ -2,7 +2,7 @@ from typing import Any, Dict, List, Union, overload -from airbyte_protocol.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, SyncMode +from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, ConfiguredAirbyteStreamSerializer, SyncMode class ConfiguredAirbyteStreamBuilder: @@ -37,7 +37,7 @@ def with_json_schema(self, json_schema: Dict[str, Any]) -> "ConfiguredAirbyteStr return self def build(self) -> ConfiguredAirbyteStream: - return ConfiguredAirbyteStream.parse_obj(self._stream) + return ConfiguredAirbyteStreamSerializer.load(self._stream) class CatalogBuilder: diff --git a/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py b/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py index ef300bd864f0d..9cc74ec2669b8 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py +++ b/airbyte-cdk/python/airbyte_cdk/test/entrypoint_wrapper.py @@ -26,18 +26,23 @@ from airbyte_cdk.entrypoint import AirbyteEntrypoint from airbyte_cdk.exception_handler import assemble_uncaught_exception from airbyte_cdk.logger import AirbyteLogFormatter -from airbyte_cdk.sources import Source -from airbyte_protocol.models import ( +from airbyte_cdk.models import ( AirbyteLogMessage, AirbyteMessage, + AirbyteMessageSerializer, AirbyteStateMessage, + AirbyteStateMessageSerializer, AirbyteStreamStatus, ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, Level, TraceType, Type, ) +from airbyte_cdk.sources import Source +from orjson import orjson from pydantic import ValidationError as V2ValidationError +from serpyco_rs import SchemaValidationError class EntrypointOutput: @@ -53,8 +58,8 @@ def __init__(self, messages: List[str], uncaught_exception: Optional[BaseExcepti @staticmethod def _parse_message(message: str) -> AirbyteMessage: try: - return AirbyteMessage.parse_obj(json.loads(message)) - except (json.JSONDecodeError, V2ValidationError): + return AirbyteMessageSerializer.load(orjson.loads(message)) # type: ignore[no-any-return] # Serializer.load() always returns AirbyteMessage + except (orjson.JSONDecodeError, SchemaValidationError): # The platform assumes that logs that are not of AirbyteMessage format are log messages return AirbyteMessage(type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message=message)) @@ -75,7 +80,7 @@ def most_recent_state(self) -> Any: state_messages = self._get_message_by_types([Type.STATE]) if not state_messages: raise ValueError("Can't provide most recent state as there are no state messages") - return state_messages[-1].state.stream + return state_messages[-1].state.stream # type: ignore[union-attr] # state has `stream` @property def logs(self) -> List[AirbyteMessage]: @@ -102,9 +107,9 @@ def catalog(self) -> AirbyteMessage: def get_stream_statuses(self, stream_name: str) -> List[AirbyteStreamStatus]: status_messages = map( - lambda message: message.trace.stream_status.status, + lambda message: message.trace.stream_status.status, # type: ignore filter( - lambda message: message.trace.stream_status.stream_descriptor.name == stream_name, + lambda message: message.trace.stream_status.stream_descriptor.name == stream_name, # type: ignore # callable; trace has `stream_status` self._get_trace_message_by_trace_type(TraceType.STREAM_STATUS), ), ) @@ -114,11 +119,11 @@ def _get_message_by_types(self, message_types: List[Type]) -> List[AirbyteMessag return [message for message in self._messages if message.type in message_types] def _get_trace_message_by_trace_type(self, trace_type: TraceType) -> List[AirbyteMessage]: - return [message for message in self._get_message_by_types([Type.TRACE]) if message.trace.type == trace_type] + return [message for message in self._get_message_by_types([Type.TRACE]) if message.trace.type == trace_type] # type: ignore[union-attr] # trace has `type` def is_in_logs(self, pattern: str) -> bool: """Check if any log message case-insensitive matches the pattern.""" - return any(re.search(pattern, entry.log.message, flags=re.IGNORECASE) for entry in self.logs) + return any(re.search(pattern, entry.log.message, flags=re.IGNORECASE) for entry in self.logs) # type: ignore[union-attr] # log has `message` def is_not_in_logs(self, pattern: str) -> bool: """Check if no log message matches the case-insensitive pattern.""" @@ -188,7 +193,9 @@ def read( with tempfile.TemporaryDirectory() as tmp_directory: tmp_directory_path = Path(tmp_directory) config_file = make_file(tmp_directory_path / "config.json", config) - catalog_file = make_file(tmp_directory_path / "catalog.json", catalog.model_dump_json()) + catalog_file = make_file( + tmp_directory_path / "catalog.json", orjson.dumps(ConfiguredAirbyteCatalogSerializer.dump(catalog)).decode() + ) args = [ "read", "--config", @@ -201,7 +208,8 @@ def read( [ "--state", make_file( - tmp_directory_path / "state.json", f"[{','.join([stream_state.model_dump_json() for stream_state in state])}]" + tmp_directory_path / "state.json", + f"[{','.join([orjson.dumps(AirbyteStateMessageSerializer.dump(stream_state)).decode() for stream_state in state])}]", ), ] ) diff --git a/airbyte-cdk/python/airbyte_cdk/test/state_builder.py b/airbyte-cdk/python/airbyte_cdk/test/state_builder.py index 0c43d43204287..50b5dbe5f793b 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/state_builder.py +++ b/airbyte-cdk/python/airbyte_cdk/test/state_builder.py @@ -2,7 +2,7 @@ from typing import Any, List -from airbyte_protocol.models import AirbyteStateMessage +from airbyte_cdk.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor class StateBuilder: @@ -11,7 +11,13 @@ def __init__(self) -> None: def with_stream_state(self, stream_name: str, state: Any) -> "StateBuilder": self._state.append( - AirbyteStateMessage.parse_obj({"type": "STREAM", "stream": {"stream_state": state, "stream_descriptor": {"name": stream_name}}}) + AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_state=state if isinstance(state, AirbyteStateBlob) else AirbyteStateBlob(state), + stream_descriptor=StreamDescriptor(**{"name": stream_name}), + ), + ) ) return self diff --git a/airbyte-cdk/python/airbyte_cdk/test/utils/reading.py b/airbyte-cdk/python/airbyte_cdk/test/utils/reading.py index f8100187d4fb5..2d89cb8709842 100644 --- a/airbyte-cdk/python/airbyte_cdk/test/utils/reading.py +++ b/airbyte-cdk/python/airbyte_cdk/test/utils/reading.py @@ -3,9 +3,9 @@ from typing import Any, List, Mapping, Optional from airbyte_cdk import AbstractSource +from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, read -from airbyte_protocol.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, SyncMode def catalog(stream_name: str, sync_mode: SyncMode) -> ConfiguredAirbyteCatalog: diff --git a/airbyte-cdk/python/airbyte_cdk/utils/message_utils.py b/airbyte-cdk/python/airbyte_cdk/utils/message_utils.py index 37d9d1351afcb..a862d46964950 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/message_utils.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/message_utils.py @@ -1,18 +1,18 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. +from airbyte_cdk.models import AirbyteMessage, Type from airbyte_cdk.sources.connector_state_manager import HashableStreamDescriptor -from airbyte_protocol.models import AirbyteMessage, Type def get_stream_descriptor(message: AirbyteMessage) -> HashableStreamDescriptor: match message.type: case Type.RECORD: - return HashableStreamDescriptor(name=message.record.stream, namespace=message.record.namespace) + return HashableStreamDescriptor(name=message.record.stream, namespace=message.record.namespace) # type: ignore[union-attr] # record has `stream` and `namespace` case Type.STATE: - if not message.state.stream or not message.state.stream.stream_descriptor: + if not message.state.stream or not message.state.stream.stream_descriptor: # type: ignore[union-attr] # state has `stream` raise ValueError("State message was not in per-stream state format, which is required for record counts.") return HashableStreamDescriptor( - name=message.state.stream.stream_descriptor.name, namespace=message.state.stream.stream_descriptor.namespace + name=message.state.stream.stream_descriptor.name, namespace=message.state.stream.stream_descriptor.namespace # type: ignore[union-attr] # state has `stream` ) case _: raise NotImplementedError(f"get_stream_descriptor is not implemented for message type '{message.type}'.") diff --git a/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py b/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py index 9bec5ac095c08..bd96ea398146e 100644 --- a/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py +++ b/airbyte-cdk/python/airbyte_cdk/utils/traced_exception.py @@ -1,15 +1,15 @@ # # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # - +import time import traceback -from datetime import datetime from typing import Optional from airbyte_cdk.models import ( AirbyteConnectionStatus, AirbyteErrorTraceMessage, AirbyteMessage, + AirbyteMessageSerializer, AirbyteTraceMessage, FailureType, Status, @@ -18,6 +18,7 @@ ) from airbyte_cdk.models import Type as MessageType from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets +from orjson import orjson class AirbyteTracedException(Exception): @@ -54,7 +55,7 @@ def as_airbyte_message(self, stream_descriptor: Optional[StreamDescriptor] = Non :param stream_descriptor is deprecated, please use the stream_description in `__init__ or `from_exception`. If many stream_descriptors are defined, the one from `as_airbyte_message` will be discarded. """ - now_millis = datetime.now().timestamp() * 1000.0 + now_millis = time.time_ns() // 1_000_000 trace_exc = self._exception or self stack_trace_str = "".join(traceback.TracebackException.from_exception(trace_exc).format()) @@ -85,7 +86,7 @@ def emit_message(self) -> None: Prints the exception as an AirbyteTraceMessage. Note that this will be called automatically on uncaught exceptions when using the airbyte_cdk entrypoint. """ - message = self.as_airbyte_message().model_dump_json(exclude_unset=True) + message = orjson.dumps(AirbyteMessageSerializer.dump(self.as_airbyte_message())).decode() filtered_message = filter_secrets(message) print(filtered_message) @@ -106,10 +107,10 @@ def as_sanitized_airbyte_message(self, stream_descriptor: Optional[StreamDescrip stream_descriptors are defined, the one from `as_sanitized_airbyte_message` will be discarded. """ error_message = self.as_airbyte_message(stream_descriptor=stream_descriptor) - if error_message.trace.error.message: - error_message.trace.error.message = filter_secrets(error_message.trace.error.message) - if error_message.trace.error.internal_message: - error_message.trace.error.internal_message = filter_secrets(error_message.trace.error.internal_message) - if error_message.trace.error.stack_trace: - error_message.trace.error.stack_trace = filter_secrets(error_message.trace.error.stack_trace) + if error_message.trace.error.message: # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage + error_message.trace.error.message = filter_secrets(error_message.trace.error.message) # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage + if error_message.trace.error.internal_message: # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage + error_message.trace.error.internal_message = filter_secrets(error_message.trace.error.internal_message) # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage + if error_message.trace.error.stack_trace: # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage + error_message.trace.error.stack_trace = filter_secrets(error_message.trace.error.stack_trace) # type: ignore[union-attr] # AirbyteMessage with MessageType.TRACE has AirbyteTraceMessage return error_message diff --git a/airbyte-cdk/python/cdk-migrations.md b/airbyte-cdk/python/cdk-migrations.md index 2f38fb8452fef..02ebf2e751c47 100644 --- a/airbyte-cdk/python/cdk-migrations.md +++ b/airbyte-cdk/python/cdk-migrations.md @@ -1,5 +1,36 @@ # CDK Migration Guide +## Upgrading to 5.0.0 + +Version 5.0.0 of the CDK updates the `airbyte_cdk.models` dependency to replace Pydantic v2 models with Python `dataclasses`. It also +updates the `airbyte-protocol-models` dependency to a version that uses dataclasses models. + +The changes to Airbyte CDK itself are backwards-compatible, but some changes are required if the connector: +- uses the `airbyte_protocol` models directly, or `airbyte_cdk.models`, which points to `airbyte_protocol` models +- uses third-party libraries, such as `pandas`, to read data from sources, which output non-native Python objects that cannot be serialized by the [orjson](https://github.com/ijl/orjson) library. + +### Updating direct usage of Pydantic based Airbyte Protocol Models + +If the connector uses Pydantic based Airbyte Protocol Models, the code will need to be updated to reflect the changes `pydantic`. +It is recommended to import protocol classes not directly by `import airbyte_protocol` statement, but from `airbyte_cdk.models` package. +It is also recommended to use `Serializers` from `airbyte_cdk.models` to manipulate the data or convert to/from JSON. + +### Updating third-party libraries + +For example, if `pandas` outputs data from the source, which has date-time `pandas.Timestamp` object in +it, [Orjson supported Types](https://github.com/ijl/orjson?tab=readme-ov-file#types), these fields should be transformed to native JSON +objects. + +```python3 +# Before +yield from df.to_dict(orient="records") + +# After - Option 1 +yield orjson.loads(df.to_json(orient="records", date_format="iso", date_unit="us")) + +``` + + ## Upgrading to 4.5.0 In this release, we are no longer supporting the legacy state format in favor of the current per-stream state diff --git a/airbyte-cdk/python/poetry.lock b/airbyte-cdk/python/poetry.lock index f327a8d2bf002..4564f78918122 100644 --- a/airbyte-cdk/python/poetry.lock +++ b/airbyte-cdk/python/poetry.lock @@ -2,98 +2,113 @@ [[package]] name = "aiohappyeyeballs" -version = "2.3.5" +version = "2.4.0" description = "Happy Eyeballs for asyncio" optional = true python-versions = ">=3.8" files = [ - {file = "aiohappyeyeballs-2.3.5-py3-none-any.whl", hash = "sha256:4d6dea59215537dbc746e93e779caea8178c866856a721c9c660d7a5a7b8be03"}, - {file = "aiohappyeyeballs-2.3.5.tar.gz", hash = "sha256:6fa48b9f1317254f122a07a131a86b71ca6946ca989ce6326fff54a99a920105"}, + {file = "aiohappyeyeballs-2.4.0-py3-none-any.whl", hash = "sha256:7ce92076e249169a13c2f49320d1967425eaf1f407522d707d59cac7628d62bd"}, + {file = "aiohappyeyeballs-2.4.0.tar.gz", hash = "sha256:55a1714f084e63d49639800f95716da97a1f173d46a16dfcfda0016abb93b6b2"}, ] [[package]] name = "aiohttp" -version = "3.10.3" +version = "3.10.5" description = "Async http client/server framework (asyncio)" optional = true python-versions = ">=3.8" files = [ - {file = "aiohttp-3.10.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cc36cbdedf6f259371dbbbcaae5bb0e95b879bc501668ab6306af867577eb5db"}, - {file = "aiohttp-3.10.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:85466b5a695c2a7db13eb2c200af552d13e6a9313d7fa92e4ffe04a2c0ea74c1"}, - {file = "aiohttp-3.10.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:71bb1d97bfe7e6726267cea169fdf5df7658831bb68ec02c9c6b9f3511e108bb"}, - {file = "aiohttp-3.10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:baec1eb274f78b2de54471fc4c69ecbea4275965eab4b556ef7a7698dee18bf2"}, - {file = "aiohttp-3.10.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:13031e7ec1188274bad243255c328cc3019e36a5a907978501256000d57a7201"}, - {file = "aiohttp-3.10.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2bbc55a964b8eecb341e492ae91c3bd0848324d313e1e71a27e3d96e6ee7e8e8"}, - {file = "aiohttp-3.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8cc0564b286b625e673a2615ede60a1704d0cbbf1b24604e28c31ed37dc62aa"}, - {file = "aiohttp-3.10.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f817a54059a4cfbc385a7f51696359c642088710e731e8df80d0607193ed2b73"}, - {file = "aiohttp-3.10.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:8542c9e5bcb2bd3115acdf5adc41cda394e7360916197805e7e32b93d821ef93"}, - {file = "aiohttp-3.10.3-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:671efce3a4a0281060edf9a07a2f7e6230dca3a1cbc61d110eee7753d28405f7"}, - {file = "aiohttp-3.10.3-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:0974f3b5b0132edcec92c3306f858ad4356a63d26b18021d859c9927616ebf27"}, - {file = "aiohttp-3.10.3-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:44bb159b55926b57812dca1b21c34528e800963ffe130d08b049b2d6b994ada7"}, - {file = "aiohttp-3.10.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:6ae9ae382d1c9617a91647575255ad55a48bfdde34cc2185dd558ce476bf16e9"}, - {file = "aiohttp-3.10.3-cp310-cp310-win32.whl", hash = "sha256:aed12a54d4e1ee647376fa541e1b7621505001f9f939debf51397b9329fd88b9"}, - {file = "aiohttp-3.10.3-cp310-cp310-win_amd64.whl", hash = "sha256:b51aef59370baf7444de1572f7830f59ddbabd04e5292fa4218d02f085f8d299"}, - {file = "aiohttp-3.10.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e021c4c778644e8cdc09487d65564265e6b149896a17d7c0f52e9a088cc44e1b"}, - {file = "aiohttp-3.10.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:24fade6dae446b183e2410a8628b80df9b7a42205c6bfc2eff783cbeedc224a2"}, - {file = "aiohttp-3.10.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bc8e9f15939dacb0e1f2d15f9c41b786051c10472c7a926f5771e99b49a5957f"}, - {file = "aiohttp-3.10.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5a9ec959b5381271c8ec9310aae1713b2aec29efa32e232e5ef7dcca0df0279"}, - {file = "aiohttp-3.10.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2a5d0ea8a6467b15d53b00c4e8ea8811e47c3cc1bdbc62b1aceb3076403d551f"}, - {file = "aiohttp-3.10.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c9ed607dbbdd0d4d39b597e5bf6b0d40d844dfb0ac6a123ed79042ef08c1f87e"}, - {file = "aiohttp-3.10.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3e66d5b506832e56add66af88c288c1d5ba0c38b535a1a59e436b300b57b23e"}, - {file = "aiohttp-3.10.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fda91ad797e4914cca0afa8b6cccd5d2b3569ccc88731be202f6adce39503189"}, - {file = "aiohttp-3.10.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:61ccb867b2f2f53df6598eb2a93329b5eee0b00646ee79ea67d68844747a418e"}, - {file = "aiohttp-3.10.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6d881353264e6156f215b3cb778c9ac3184f5465c2ece5e6fce82e68946868ef"}, - {file = "aiohttp-3.10.3-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b031ce229114825f49cec4434fa844ccb5225e266c3e146cb4bdd025a6da52f1"}, - {file = "aiohttp-3.10.3-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:5337cc742a03f9e3213b097abff8781f79de7190bbfaa987bd2b7ceb5bb0bdec"}, - {file = "aiohttp-3.10.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ab3361159fd3dcd0e48bbe804006d5cfb074b382666e6c064112056eb234f1a9"}, - {file = "aiohttp-3.10.3-cp311-cp311-win32.whl", hash = "sha256:05d66203a530209cbe40f102ebaac0b2214aba2a33c075d0bf825987c36f1f0b"}, - {file = "aiohttp-3.10.3-cp311-cp311-win_amd64.whl", hash = "sha256:70b4a4984a70a2322b70e088d654528129783ac1ebbf7dd76627b3bd22db2f17"}, - {file = "aiohttp-3.10.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:166de65e2e4e63357cfa8417cf952a519ac42f1654cb2d43ed76899e2319b1ee"}, - {file = "aiohttp-3.10.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:7084876352ba3833d5d214e02b32d794e3fd9cf21fdba99cff5acabeb90d9806"}, - {file = "aiohttp-3.10.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d98c604c93403288591d7d6d7d6cc8a63459168f8846aeffd5b3a7f3b3e5e09"}, - {file = "aiohttp-3.10.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d73b073a25a0bb8bf014345374fe2d0f63681ab5da4c22f9d2025ca3e3ea54fc"}, - {file = "aiohttp-3.10.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8da6b48c20ce78f5721068f383e0e113dde034e868f1b2f5ee7cb1e95f91db57"}, - {file = "aiohttp-3.10.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a9dcdccf50284b1b0dc72bc57e5bbd3cc9bf019060dfa0668f63241ccc16aa7"}, - {file = "aiohttp-3.10.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56fb94bae2be58f68d000d046172d8b8e6b1b571eb02ceee5535e9633dcd559c"}, - {file = "aiohttp-3.10.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bf75716377aad2c718cdf66451c5cf02042085d84522aec1f9246d3e4b8641a6"}, - {file = "aiohttp-3.10.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6c51ed03e19c885c8e91f574e4bbe7381793f56f93229731597e4a499ffef2a5"}, - {file = "aiohttp-3.10.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b84857b66fa6510a163bb083c1199d1ee091a40163cfcbbd0642495fed096204"}, - {file = "aiohttp-3.10.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c124b9206b1befe0491f48185fd30a0dd51b0f4e0e7e43ac1236066215aff272"}, - {file = "aiohttp-3.10.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3461d9294941937f07bbbaa6227ba799bc71cc3b22c40222568dc1cca5118f68"}, - {file = "aiohttp-3.10.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:08bd0754d257b2db27d6bab208c74601df6f21bfe4cb2ec7b258ba691aac64b3"}, - {file = "aiohttp-3.10.3-cp312-cp312-win32.whl", hash = "sha256:7f9159ae530297f61a00116771e57516f89a3de6ba33f314402e41560872b50a"}, - {file = "aiohttp-3.10.3-cp312-cp312-win_amd64.whl", hash = "sha256:e1128c5d3a466279cb23c4aa32a0f6cb0e7d2961e74e9e421f90e74f75ec1edf"}, - {file = "aiohttp-3.10.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:d1100e68e70eb72eadba2b932b185ebf0f28fd2f0dbfe576cfa9d9894ef49752"}, - {file = "aiohttp-3.10.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a541414578ff47c0a9b0b8b77381ea86b0c8531ab37fc587572cb662ccd80b88"}, - {file = "aiohttp-3.10.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d5548444ef60bf4c7b19ace21f032fa42d822e516a6940d36579f7bfa8513f9c"}, - {file = "aiohttp-3.10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5ba2e838b5e6a8755ac8297275c9460e729dc1522b6454aee1766c6de6d56e5e"}, - {file = "aiohttp-3.10.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:48665433bb59144aaf502c324694bec25867eb6630fcd831f7a893ca473fcde4"}, - {file = "aiohttp-3.10.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bac352fceed158620ce2d701ad39d4c1c76d114255a7c530e057e2b9f55bdf9f"}, - {file = "aiohttp-3.10.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2b0f670502100cdc567188c49415bebba947eb3edaa2028e1a50dd81bd13363f"}, - {file = "aiohttp-3.10.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:43b09f38a67679e32d380fe512189ccb0b25e15afc79b23fbd5b5e48e4fc8fd9"}, - {file = "aiohttp-3.10.3-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:cd788602e239ace64f257d1c9d39898ca65525583f0fbf0988bcba19418fe93f"}, - {file = "aiohttp-3.10.3-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:214277dcb07ab3875f17ee1c777d446dcce75bea85846849cc9d139ab8f5081f"}, - {file = "aiohttp-3.10.3-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:32007fdcaab789689c2ecaaf4b71f8e37bf012a15cd02c0a9db8c4d0e7989fa8"}, - {file = "aiohttp-3.10.3-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:123e5819bfe1b87204575515cf448ab3bf1489cdeb3b61012bde716cda5853e7"}, - {file = "aiohttp-3.10.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:812121a201f0c02491a5db335a737b4113151926a79ae9ed1a9f41ea225c0e3f"}, - {file = "aiohttp-3.10.3-cp38-cp38-win32.whl", hash = "sha256:b97dc9a17a59f350c0caa453a3cb35671a2ffa3a29a6ef3568b523b9113d84e5"}, - {file = "aiohttp-3.10.3-cp38-cp38-win_amd64.whl", hash = "sha256:3731a73ddc26969d65f90471c635abd4e1546a25299b687e654ea6d2fc052394"}, - {file = "aiohttp-3.10.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38d91b98b4320ffe66efa56cb0f614a05af53b675ce1b8607cdb2ac826a8d58e"}, - {file = "aiohttp-3.10.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9743fa34a10a36ddd448bba8a3adc2a66a1c575c3c2940301bacd6cc896c6bf1"}, - {file = "aiohttp-3.10.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:7c126f532caf238031c19d169cfae3c6a59129452c990a6e84d6e7b198a001dc"}, - {file = "aiohttp-3.10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:926e68438f05703e500b06fe7148ef3013dd6f276de65c68558fa9974eeb59ad"}, - {file = "aiohttp-3.10.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:434b3ab75833accd0b931d11874e206e816f6e6626fd69f643d6a8269cd9166a"}, - {file = "aiohttp-3.10.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d35235a44ec38109b811c3600d15d8383297a8fab8e3dec6147477ec8636712a"}, - {file = "aiohttp-3.10.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59c489661edbd863edb30a8bd69ecb044bd381d1818022bc698ba1b6f80e5dd1"}, - {file = "aiohttp-3.10.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:50544fe498c81cb98912afabfc4e4d9d85e89f86238348e3712f7ca6a2f01dab"}, - {file = "aiohttp-3.10.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:09bc79275737d4dc066e0ae2951866bb36d9c6b460cb7564f111cc0427f14844"}, - {file = "aiohttp-3.10.3-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:af4dbec58e37f5afff4f91cdf235e8e4b0bd0127a2a4fd1040e2cad3369d2f06"}, - {file = "aiohttp-3.10.3-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:b22cae3c9dd55a6b4c48c63081d31c00fc11fa9db1a20c8a50ee38c1a29539d2"}, - {file = "aiohttp-3.10.3-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:ba562736d3fbfe9241dad46c1a8994478d4a0e50796d80e29d50cabe8fbfcc3f"}, - {file = "aiohttp-3.10.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:f25d6c4e82d7489be84f2b1c8212fafc021b3731abdb61a563c90e37cced3a21"}, - {file = "aiohttp-3.10.3-cp39-cp39-win32.whl", hash = "sha256:b69d832e5f5fa15b1b6b2c8eb6a9fd2c0ec1fd7729cb4322ed27771afc9fc2ac"}, - {file = "aiohttp-3.10.3-cp39-cp39-win_amd64.whl", hash = "sha256:673bb6e3249dc8825df1105f6ef74e2eab779b7ff78e96c15cadb78b04a83752"}, - {file = "aiohttp-3.10.3.tar.gz", hash = "sha256:21650e7032cc2d31fc23d353d7123e771354f2a3d5b05a5647fc30fea214e696"}, + {file = "aiohttp-3.10.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:18a01eba2574fb9edd5f6e5fb25f66e6ce061da5dab5db75e13fe1558142e0a3"}, + {file = "aiohttp-3.10.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:94fac7c6e77ccb1ca91e9eb4cb0ac0270b9fb9b289738654120ba8cebb1189c6"}, + {file = "aiohttp-3.10.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2f1f1c75c395991ce9c94d3e4aa96e5c59c8356a15b1c9231e783865e2772699"}, + {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4f7acae3cf1a2a2361ec4c8e787eaaa86a94171d2417aae53c0cca6ca3118ff6"}, + {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:94c4381ffba9cc508b37d2e536b418d5ea9cfdc2848b9a7fea6aebad4ec6aac1"}, + {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c31ad0c0c507894e3eaa843415841995bf8de4d6b2d24c6e33099f4bc9fc0d4f"}, + {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0912b8a8fadeb32ff67a3ed44249448c20148397c1ed905d5dac185b4ca547bb"}, + {file = "aiohttp-3.10.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0d93400c18596b7dc4794d48a63fb361b01a0d8eb39f28800dc900c8fbdaca91"}, + {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d00f3c5e0d764a5c9aa5a62d99728c56d455310bcc288a79cab10157b3af426f"}, + {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:d742c36ed44f2798c8d3f4bc511f479b9ceef2b93f348671184139e7d708042c"}, + {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:814375093edae5f1cb31e3407997cf3eacefb9010f96df10d64829362ae2df69"}, + {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:8224f98be68a84b19f48e0bdc14224b5a71339aff3a27df69989fa47d01296f3"}, + {file = "aiohttp-3.10.5-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d9a487ef090aea982d748b1b0d74fe7c3950b109df967630a20584f9a99c0683"}, + {file = "aiohttp-3.10.5-cp310-cp310-win32.whl", hash = "sha256:d9ef084e3dc690ad50137cc05831c52b6ca428096e6deb3c43e95827f531d5ef"}, + {file = "aiohttp-3.10.5-cp310-cp310-win_amd64.whl", hash = "sha256:66bf9234e08fe561dccd62083bf67400bdbf1c67ba9efdc3dac03650e97c6088"}, + {file = "aiohttp-3.10.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:8c6a4e5e40156d72a40241a25cc226051c0a8d816610097a8e8f517aeacd59a2"}, + {file = "aiohttp-3.10.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2c634a3207a5445be65536d38c13791904fda0748b9eabf908d3fe86a52941cf"}, + {file = "aiohttp-3.10.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4aff049b5e629ef9b3e9e617fa6e2dfeda1bf87e01bcfecaf3949af9e210105e"}, + {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1942244f00baaacaa8155eca94dbd9e8cc7017deb69b75ef67c78e89fdad3c77"}, + {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e04a1f2a65ad2f93aa20f9ff9f1b672bf912413e5547f60749fa2ef8a644e061"}, + {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7f2bfc0032a00405d4af2ba27f3c429e851d04fad1e5ceee4080a1c570476697"}, + {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:424ae21498790e12eb759040bbb504e5e280cab64693d14775c54269fd1d2bb7"}, + {file = "aiohttp-3.10.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:975218eee0e6d24eb336d0328c768ebc5d617609affaca5dbbd6dd1984f16ed0"}, + {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4120d7fefa1e2d8fb6f650b11489710091788de554e2b6f8347c7a20ceb003f5"}, + {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:b90078989ef3fc45cf9221d3859acd1108af7560c52397ff4ace8ad7052a132e"}, + {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:ba5a8b74c2a8af7d862399cdedce1533642fa727def0b8c3e3e02fcb52dca1b1"}, + {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:02594361128f780eecc2a29939d9dfc870e17b45178a867bf61a11b2a4367277"}, + {file = "aiohttp-3.10.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8fb4fc029e135859f533025bc82047334e24b0d489e75513144f25408ecaf058"}, + {file = "aiohttp-3.10.5-cp311-cp311-win32.whl", hash = "sha256:e1ca1ef5ba129718a8fc827b0867f6aa4e893c56eb00003b7367f8a733a9b072"}, + {file = "aiohttp-3.10.5-cp311-cp311-win_amd64.whl", hash = "sha256:349ef8a73a7c5665cca65c88ab24abe75447e28aa3bc4c93ea5093474dfdf0ff"}, + {file = "aiohttp-3.10.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:305be5ff2081fa1d283a76113b8df7a14c10d75602a38d9f012935df20731487"}, + {file = "aiohttp-3.10.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3a1c32a19ee6bbde02f1cb189e13a71b321256cc1d431196a9f824050b160d5a"}, + {file = "aiohttp-3.10.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:61645818edd40cc6f455b851277a21bf420ce347baa0b86eaa41d51ef58ba23d"}, + {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c225286f2b13bab5987425558baa5cbdb2bc925b2998038fa028245ef421e75"}, + {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8ba01ebc6175e1e6b7275c907a3a36be48a2d487549b656aa90c8a910d9f3178"}, + {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8eaf44ccbc4e35762683078b72bf293f476561d8b68ec8a64f98cf32811c323e"}, + {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b1c43eb1ab7cbf411b8e387dc169acb31f0ca0d8c09ba63f9eac67829585b44f"}, + {file = "aiohttp-3.10.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de7a5299827253023c55ea549444e058c0eb496931fa05d693b95140a947cb73"}, + {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4790f0e15f00058f7599dab2b206d3049d7ac464dc2e5eae0e93fa18aee9e7bf"}, + {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:44b324a6b8376a23e6ba25d368726ee3bc281e6ab306db80b5819999c737d820"}, + {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0d277cfb304118079e7044aad0b76685d30ecb86f83a0711fc5fb257ffe832ca"}, + {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:54d9ddea424cd19d3ff6128601a4a4d23d54a421f9b4c0fff740505813739a91"}, + {file = "aiohttp-3.10.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4f1c9866ccf48a6df2b06823e6ae80573529f2af3a0992ec4fe75b1a510df8a6"}, + {file = "aiohttp-3.10.5-cp312-cp312-win32.whl", hash = "sha256:dc4826823121783dccc0871e3f405417ac116055bf184ac04c36f98b75aacd12"}, + {file = "aiohttp-3.10.5-cp312-cp312-win_amd64.whl", hash = "sha256:22c0a23a3b3138a6bf76fc553789cb1a703836da86b0f306b6f0dc1617398abc"}, + {file = "aiohttp-3.10.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:7f6b639c36734eaa80a6c152a238242bedcee9b953f23bb887e9102976343092"}, + {file = "aiohttp-3.10.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f29930bc2921cef955ba39a3ff87d2c4398a0394ae217f41cb02d5c26c8b1b77"}, + {file = "aiohttp-3.10.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f489a2c9e6455d87eabf907ac0b7d230a9786be43fbe884ad184ddf9e9c1e385"}, + {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:123dd5b16b75b2962d0fff566effb7a065e33cd4538c1692fb31c3bda2bfb972"}, + {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b98e698dc34966e5976e10bbca6d26d6724e6bdea853c7c10162a3235aba6e16"}, + {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3b9162bab7e42f21243effc822652dc5bb5e8ff42a4eb62fe7782bcbcdfacf6"}, + {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1923a5c44061bffd5eebeef58cecf68096e35003907d8201a4d0d6f6e387ccaa"}, + {file = "aiohttp-3.10.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d55f011da0a843c3d3df2c2cf4e537b8070a419f891c930245f05d329c4b0689"}, + {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:afe16a84498441d05e9189a15900640a2d2b5e76cf4efe8cbb088ab4f112ee57"}, + {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f8112fb501b1e0567a1251a2fd0747baae60a4ab325a871e975b7bb67e59221f"}, + {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:1e72589da4c90337837fdfe2026ae1952c0f4a6e793adbbfbdd40efed7c63599"}, + {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:4d46c7b4173415d8e583045fbc4daa48b40e31b19ce595b8d92cf639396c15d5"}, + {file = "aiohttp-3.10.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:33e6bc4bab477c772a541f76cd91e11ccb6d2efa2b8d7d7883591dfb523e5987"}, + {file = "aiohttp-3.10.5-cp313-cp313-win32.whl", hash = "sha256:c58c6837a2c2a7cf3133983e64173aec11f9c2cd8e87ec2fdc16ce727bcf1a04"}, + {file = "aiohttp-3.10.5-cp313-cp313-win_amd64.whl", hash = "sha256:38172a70005252b6893088c0f5e8a47d173df7cc2b2bd88650957eb84fcf5022"}, + {file = "aiohttp-3.10.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f6f18898ace4bcd2d41a122916475344a87f1dfdec626ecde9ee802a711bc569"}, + {file = "aiohttp-3.10.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:5ede29d91a40ba22ac1b922ef510aab871652f6c88ef60b9dcdf773c6d32ad7a"}, + {file = "aiohttp-3.10.5-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:673f988370f5954df96cc31fd99c7312a3af0a97f09e407399f61583f30da9bc"}, + {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58718e181c56a3c02d25b09d4115eb02aafe1a732ce5714ab70326d9776457c3"}, + {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4b38b1570242fbab8d86a84128fb5b5234a2f70c2e32f3070143a6d94bc854cf"}, + {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:074d1bff0163e107e97bd48cad9f928fa5a3eb4b9d33366137ffce08a63e37fe"}, + {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd31f176429cecbc1ba499d4aba31aaccfea488f418d60376b911269d3b883c5"}, + {file = "aiohttp-3.10.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7384d0b87d4635ec38db9263e6a3f1eb609e2e06087f0aa7f63b76833737b471"}, + {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:8989f46f3d7ef79585e98fa991e6ded55d2f48ae56d2c9fa5e491a6e4effb589"}, + {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:c83f7a107abb89a227d6c454c613e7606c12a42b9a4ca9c5d7dad25d47c776ae"}, + {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:cde98f323d6bf161041e7627a5fd763f9fd829bcfcd089804a5fdce7bb6e1b7d"}, + {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:676f94c5480d8eefd97c0c7e3953315e4d8c2b71f3b49539beb2aa676c58272f"}, + {file = "aiohttp-3.10.5-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2d21ac12dc943c68135ff858c3a989f2194a709e6e10b4c8977d7fcd67dfd511"}, + {file = "aiohttp-3.10.5-cp38-cp38-win32.whl", hash = "sha256:17e997105bd1a260850272bfb50e2a328e029c941c2708170d9d978d5a30ad9a"}, + {file = "aiohttp-3.10.5-cp38-cp38-win_amd64.whl", hash = "sha256:1c19de68896747a2aa6257ae4cf6ef59d73917a36a35ee9d0a6f48cff0f94db8"}, + {file = "aiohttp-3.10.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7e2fe37ac654032db1f3499fe56e77190282534810e2a8e833141a021faaab0e"}, + {file = "aiohttp-3.10.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f5bf3ead3cb66ab990ee2561373b009db5bc0e857549b6c9ba84b20bc462e172"}, + {file = "aiohttp-3.10.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1b2c16a919d936ca87a3c5f0e43af12a89a3ce7ccbce59a2d6784caba945b68b"}, + {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad146dae5977c4dd435eb31373b3fe9b0b1bf26858c6fc452bf6af394067e10b"}, + {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8c5c6fa16412b35999320f5c9690c0f554392dc222c04e559217e0f9ae244b92"}, + {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:95c4dc6f61d610bc0ee1edc6f29d993f10febfe5b76bb470b486d90bbece6b22"}, + {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da452c2c322e9ce0cfef392e469a26d63d42860f829026a63374fde6b5c5876f"}, + {file = "aiohttp-3.10.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:898715cf566ec2869d5cb4d5fb4be408964704c46c96b4be267442d265390f32"}, + {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:391cc3a9c1527e424c6865e087897e766a917f15dddb360174a70467572ac6ce"}, + {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:380f926b51b92d02a34119d072f178d80bbda334d1a7e10fa22d467a66e494db"}, + {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:ce91db90dbf37bb6fa0997f26574107e1b9d5ff939315247b7e615baa8ec313b"}, + {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:9093a81e18c45227eebe4c16124ebf3e0d893830c6aca7cc310bfca8fe59d857"}, + {file = "aiohttp-3.10.5-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:ee40b40aa753d844162dcc80d0fe256b87cba48ca0054f64e68000453caead11"}, + {file = "aiohttp-3.10.5-cp39-cp39-win32.whl", hash = "sha256:03f2645adbe17f274444953bdea69f8327e9d278d961d85657cb0d06864814c1"}, + {file = "aiohttp-3.10.5-cp39-cp39-win_amd64.whl", hash = "sha256:d17920f18e6ee090bdd3d0bfffd769d9f2cb4c8ffde3eb203777a3895c128862"}, + {file = "aiohttp-3.10.5.tar.gz", hash = "sha256:f071854b47d39591ce9a17981c46790acb30518e2f83dfca8db2dfa091178691"}, ] [package.dependencies] @@ -123,19 +138,16 @@ files = [ frozenlist = ">=1.1.0" [[package]] -name = "airbyte-protocol-models-pdv2" -version = "0.12.2" -description = "Declares the Airbyte Protocol." +name = "airbyte-protocol-models-dataclasses" +version = "0.13.0" +description = "Declares the Airbyte Protocol using Python Dataclasses. Dataclasses in Python have less performance overhead compared to Pydantic models, making them a more efficient choice for scenarios where speed and memory usage are critical" optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models_pdv2-0.12.2-py3-none-any.whl", hash = "sha256:8b3f9d0388928547cdf2e9134c0d589e4bcaa6f63bf71a21299f6824bfb7ad0e"}, - {file = "airbyte_protocol_models_pdv2-0.12.2.tar.gz", hash = "sha256:130c9ab289f3f53749ce63ff1abbfb67a44b7e5bd2794865315a2976138b672b"}, + {file = "airbyte_protocol_models_dataclasses-0.13.0-py3-none-any.whl", hash = "sha256:0aedb99ffc4f9aab0ce91bba2c292fa17cd8fd4b42eeba196d6a16c20bbbd7a5"}, + {file = "airbyte_protocol_models_dataclasses-0.13.0.tar.gz", hash = "sha256:72e67850d661e2808406aec5839b3158ebb94d3553b798dbdae1b4a278548d2f"}, ] -[package.dependencies] -pydantic = ">=2.7.2,<3.0.0" - [[package]] name = "alabaster" version = "0.7.16" @@ -169,6 +181,28 @@ files = [ {file = "ansicon-1.89.0.tar.gz", hash = "sha256:e4d039def5768a47e4afec8e89e83ec3ae5a26bf00ad851f914d1240b444d2b1"}, ] +[[package]] +name = "anyio" +version = "4.4.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = true +python-versions = ">=3.8" +files = [ + {file = "anyio-4.4.0-py3-none-any.whl", hash = "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7"}, + {file = "anyio-4.4.0.tar.gz", hash = "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" +typing-extensions = {version = ">=4.1", markers = "python_version < \"3.11\""} + +[package.extras] +doc = ["Sphinx (>=7)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (>=0.17)"] +trio = ["trio (>=0.23)"] + [[package]] name = "async-timeout" version = "4.0.3" @@ -193,6 +227,17 @@ files = [ {file = "asyncio-3.4.3.tar.gz", hash = "sha256:83360ff8bc97980e4ff25c964c7bd3923d333d177aa4f7fb736b019f26c7cb41"}, ] +[[package]] +name = "attributes-doc" +version = "0.4.0" +description = "PEP 224 implementation" +optional = false +python-versions = ">=3.8" +files = [ + {file = "attributes-doc-0.4.0.tar.gz", hash = "sha256:b1576c94a714e9fc2c65c47cf10d0c8e1a5f7c4f5ae7f69006be108d95cbfbfb"}, + {file = "attributes_doc-0.4.0-py2.py3-none-any.whl", hash = "sha256:4c3007d9e58f3a6cb4b9c614c4d4ce2d92161581f28e594ddd8241cc3a113bdd"}, +] + [[package]] name = "attrs" version = "24.2.0" @@ -347,24 +392,24 @@ files = [ [[package]] name = "cachetools" -version = "5.4.0" +version = "5.5.0" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" files = [ - {file = "cachetools-5.4.0-py3-none-any.whl", hash = "sha256:3ae3b49a3d5e28a77a0be2b37dbcb89005058959cb2323858c2657c4a8cab474"}, - {file = "cachetools-5.4.0.tar.gz", hash = "sha256:b8adc2e7c07f105ced7bc56dbb6dfbe7c4a00acce20e2227b3f355be89bc6827"}, + {file = "cachetools-5.5.0-py3-none-any.whl", hash = "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292"}, + {file = "cachetools-5.5.0.tar.gz", hash = "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a"}, ] [[package]] name = "cattrs" -version = "23.2.3" +version = "24.1.0" description = "Composable complex class support for attrs and dataclasses." optional = false python-versions = ">=3.8" files = [ - {file = "cattrs-23.2.3-py3-none-any.whl", hash = "sha256:0341994d94971052e9ee70662542699a3162ea1e0c62f7ce1b4a57f563685108"}, - {file = "cattrs-23.2.3.tar.gz", hash = "sha256:a934090d95abaa9e911dac357e3a8699e0b4b14f8529bcc7d2b1ad9d51672b9f"}, + {file = "cattrs-24.1.0-py3-none-any.whl", hash = "sha256:043bb8af72596432a7df63abcff0055ac0f198a4d2e95af8db5a936a7074a761"}, + {file = "cattrs-24.1.0.tar.gz", hash = "sha256:8274f18b253bf7674a43da851e3096370d67088165d23138b04a1c04c8eaf48e"}, ] [package.dependencies] @@ -376,6 +421,7 @@ typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_ver bson = ["pymongo (>=4.4.0)"] cbor2 = ["cbor2 (>=5.4.6)"] msgpack = ["msgpack (>=1.0.5)"] +msgspec = ["msgspec (>=0.18.5)"] orjson = ["orjson (>=3.9.2)"] pyyaml = ["pyyaml (>=6.0)"] tomlkit = ["tomlkit (>=0.11.8)"] @@ -383,13 +429,13 @@ ujson = ["ujson (>=5.7.0)"] [[package]] name = "certifi" -version = "2024.7.4" +version = "2024.8.30" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" files = [ - {file = "certifi-2024.7.4-py3-none-any.whl", hash = "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"}, - {file = "certifi-2024.7.4.tar.gz", hash = "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b"}, + {file = "certifi-2024.8.30-py3-none-any.whl", hash = "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8"}, + {file = "certifi-2024.8.30.tar.gz", hash = "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9"}, ] [[package]] @@ -597,13 +643,13 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "codeflash" -version = "0.6.17" +version = "0.6.19" description = "Client for codeflash.ai - automatic code performance optimization, powered by AI" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "codeflash-0.6.17-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2d767d1bf035fbfd95d4f8d62a8800ceee833cfff919ea65e6ec406e618228b3"}, - {file = "codeflash-0.6.17.tar.gz", hash = "sha256:96afca1263230c0dd0c6fc3a4601e2680fb25ffa43657310bde4cbaeb83b9000"}, + {file = "codeflash-0.6.19-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:f4e19b78ef4dfbbbaeab6d953422027a6e9120545de0657e70adfe23a3fdb51a"}, + {file = "codeflash-0.6.19.tar.gz", hash = "sha256:0cb61aec565b286b1d5fcaa64e55f3e91e03bd6154e03c718c086186c30040da"}, ] [package.dependencies] @@ -662,66 +708,87 @@ files = [ [[package]] name = "contourpy" -version = "1.2.1" +version = "1.3.0" description = "Python library for calculating contours of 2D quadrilateral grids" optional = true python-versions = ">=3.9" files = [ - {file = "contourpy-1.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bd7c23df857d488f418439686d3b10ae2fbf9bc256cd045b37a8c16575ea1040"}, - {file = "contourpy-1.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5b9eb0ca724a241683c9685a484da9d35c872fd42756574a7cfbf58af26677fd"}, - {file = "contourpy-1.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c75507d0a55378240f781599c30e7776674dbaf883a46d1c90f37e563453480"}, - {file = "contourpy-1.2.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:11959f0ce4a6f7b76ec578576a0b61a28bdc0696194b6347ba3f1c53827178b9"}, - {file = "contourpy-1.2.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eb3315a8a236ee19b6df481fc5f997436e8ade24a9f03dfdc6bd490fea20c6da"}, - {file = "contourpy-1.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39f3ecaf76cd98e802f094e0d4fbc6dc9c45a8d0c4d185f0f6c2234e14e5f75b"}, - {file = "contourpy-1.2.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:94b34f32646ca0414237168d68a9157cb3889f06b096612afdd296003fdd32fd"}, - {file = "contourpy-1.2.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:457499c79fa84593f22454bbd27670227874cd2ff5d6c84e60575c8b50a69619"}, - {file = "contourpy-1.2.1-cp310-cp310-win32.whl", hash = "sha256:ac58bdee53cbeba2ecad824fa8159493f0bf3b8ea4e93feb06c9a465d6c87da8"}, - {file = "contourpy-1.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:9cffe0f850e89d7c0012a1fb8730f75edd4320a0a731ed0c183904fe6ecfc3a9"}, - {file = "contourpy-1.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6022cecf8f44e36af10bd9118ca71f371078b4c168b6e0fab43d4a889985dbb5"}, - {file = "contourpy-1.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ef5adb9a3b1d0c645ff694f9bca7702ec2c70f4d734f9922ea34de02294fdf72"}, - {file = "contourpy-1.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6150ffa5c767bc6332df27157d95442c379b7dce3a38dff89c0f39b63275696f"}, - {file = "contourpy-1.2.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4c863140fafc615c14a4bf4efd0f4425c02230eb8ef02784c9a156461e62c965"}, - {file = "contourpy-1.2.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:00e5388f71c1a0610e6fe56b5c44ab7ba14165cdd6d695429c5cd94021e390b2"}, - {file = "contourpy-1.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4492d82b3bc7fbb7e3610747b159869468079fe149ec5c4d771fa1f614a14df"}, - {file = "contourpy-1.2.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:49e70d111fee47284d9dd867c9bb9a7058a3c617274900780c43e38d90fe1205"}, - {file = "contourpy-1.2.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b59c0ffceff8d4d3996a45f2bb6f4c207f94684a96bf3d9728dbb77428dd8cb8"}, - {file = "contourpy-1.2.1-cp311-cp311-win32.whl", hash = "sha256:7b4182299f251060996af5249c286bae9361fa8c6a9cda5efc29fe8bfd6062ec"}, - {file = "contourpy-1.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2855c8b0b55958265e8b5888d6a615ba02883b225f2227461aa9127c578a4922"}, - {file = "contourpy-1.2.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:62828cada4a2b850dbef89c81f5a33741898b305db244904de418cc957ff05dc"}, - {file = "contourpy-1.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:309be79c0a354afff9ff7da4aaed7c3257e77edf6c1b448a779329431ee79d7e"}, - {file = "contourpy-1.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e785e0f2ef0d567099b9ff92cbfb958d71c2d5b9259981cd9bee81bd194c9a4"}, - {file = "contourpy-1.2.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1cac0a8f71a041aa587410424ad46dfa6a11f6149ceb219ce7dd48f6b02b87a7"}, - {file = "contourpy-1.2.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:af3f4485884750dddd9c25cb7e3915d83c2db92488b38ccb77dd594eac84c4a0"}, - {file = "contourpy-1.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9ce6889abac9a42afd07a562c2d6d4b2b7134f83f18571d859b25624a331c90b"}, - {file = "contourpy-1.2.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:a1eea9aecf761c661d096d39ed9026574de8adb2ae1c5bd7b33558af884fb2ce"}, - {file = "contourpy-1.2.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:187fa1d4c6acc06adb0fae5544c59898ad781409e61a926ac7e84b8f276dcef4"}, - {file = "contourpy-1.2.1-cp312-cp312-win32.whl", hash = "sha256:c2528d60e398c7c4c799d56f907664673a807635b857df18f7ae64d3e6ce2d9f"}, - {file = "contourpy-1.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:1a07fc092a4088ee952ddae19a2b2a85757b923217b7eed584fdf25f53a6e7ce"}, - {file = "contourpy-1.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bb6834cbd983b19f06908b45bfc2dad6ac9479ae04abe923a275b5f48f1a186b"}, - {file = "contourpy-1.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1d59e739ab0e3520e62a26c60707cc3ab0365d2f8fecea74bfe4de72dc56388f"}, - {file = "contourpy-1.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd3db01f59fdcbce5b22afad19e390260d6d0222f35a1023d9adc5690a889364"}, - {file = "contourpy-1.2.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a12a813949e5066148712a0626895c26b2578874e4cc63160bb007e6df3436fe"}, - {file = "contourpy-1.2.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe0ccca550bb8e5abc22f530ec0466136379c01321fd94f30a22231e8a48d985"}, - {file = "contourpy-1.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e1d59258c3c67c865435d8fbeb35f8c59b8bef3d6f46c1f29f6123556af28445"}, - {file = "contourpy-1.2.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f32c38afb74bd98ce26de7cc74a67b40afb7b05aae7b42924ea990d51e4dac02"}, - {file = "contourpy-1.2.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d31a63bc6e6d87f77d71e1abbd7387ab817a66733734883d1fc0021ed9bfa083"}, - {file = "contourpy-1.2.1-cp39-cp39-win32.whl", hash = "sha256:ddcb8581510311e13421b1f544403c16e901c4e8f09083c881fab2be80ee31ba"}, - {file = "contourpy-1.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:10a37ae557aabf2509c79715cd20b62e4c7c28b8cd62dd7d99e5ed3ce28c3fd9"}, - {file = "contourpy-1.2.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:a31f94983fecbac95e58388210427d68cd30fe8a36927980fab9c20062645609"}, - {file = "contourpy-1.2.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef2b055471c0eb466033760a521efb9d8a32b99ab907fc8358481a1dd29e3bd3"}, - {file = "contourpy-1.2.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b33d2bc4f69caedcd0a275329eb2198f560b325605810895627be5d4b876bf7f"}, - {file = "contourpy-1.2.1.tar.gz", hash = "sha256:4d8908b3bee1c889e547867ca4cdc54e5ab6be6d3e078556814a22457f49423c"}, + {file = "contourpy-1.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:880ea32e5c774634f9fcd46504bf9f080a41ad855f4fef54f5380f5133d343c7"}, + {file = "contourpy-1.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:76c905ef940a4474a6289c71d53122a4f77766eef23c03cd57016ce19d0f7b42"}, + {file = "contourpy-1.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92f8557cbb07415a4d6fa191f20fd9d2d9eb9c0b61d1b2f52a8926e43c6e9af7"}, + {file = "contourpy-1.3.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:36f965570cff02b874773c49bfe85562b47030805d7d8360748f3eca570f4cab"}, + {file = "contourpy-1.3.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cacd81e2d4b6f89c9f8a5b69b86490152ff39afc58a95af002a398273e5ce589"}, + {file = "contourpy-1.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:69375194457ad0fad3a839b9e29aa0b0ed53bb54db1bfb6c3ae43d111c31ce41"}, + {file = "contourpy-1.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7a52040312b1a858b5e31ef28c2e865376a386c60c0e248370bbea2d3f3b760d"}, + {file = "contourpy-1.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3faeb2998e4fcb256542e8a926d08da08977f7f5e62cf733f3c211c2a5586223"}, + {file = "contourpy-1.3.0-cp310-cp310-win32.whl", hash = "sha256:36e0cff201bcb17a0a8ecc7f454fe078437fa6bda730e695a92f2d9932bd507f"}, + {file = "contourpy-1.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:87ddffef1dbe5e669b5c2440b643d3fdd8622a348fe1983fad7a0f0ccb1cd67b"}, + {file = "contourpy-1.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0fa4c02abe6c446ba70d96ece336e621efa4aecae43eaa9b030ae5fb92b309ad"}, + {file = "contourpy-1.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:834e0cfe17ba12f79963861e0f908556b2cedd52e1f75e6578801febcc6a9f49"}, + {file = "contourpy-1.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dbc4c3217eee163fa3984fd1567632b48d6dfd29216da3ded3d7b844a8014a66"}, + {file = "contourpy-1.3.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4865cd1d419e0c7a7bf6de1777b185eebdc51470800a9f42b9e9decf17762081"}, + {file = "contourpy-1.3.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:303c252947ab4b14c08afeb52375b26781ccd6a5ccd81abcdfc1fafd14cf93c1"}, + {file = "contourpy-1.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:637f674226be46f6ba372fd29d9523dd977a291f66ab2a74fbeb5530bb3f445d"}, + {file = "contourpy-1.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:76a896b2f195b57db25d6b44e7e03f221d32fe318d03ede41f8b4d9ba1bff53c"}, + {file = "contourpy-1.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e1fd23e9d01591bab45546c089ae89d926917a66dceb3abcf01f6105d927e2cb"}, + {file = "contourpy-1.3.0-cp311-cp311-win32.whl", hash = "sha256:d402880b84df3bec6eab53cd0cf802cae6a2ef9537e70cf75e91618a3801c20c"}, + {file = "contourpy-1.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:6cb6cc968059db9c62cb35fbf70248f40994dfcd7aa10444bbf8b3faeb7c2d67"}, + {file = "contourpy-1.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:570ef7cf892f0afbe5b2ee410c507ce12e15a5fa91017a0009f79f7d93a1268f"}, + {file = "contourpy-1.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:da84c537cb8b97d153e9fb208c221c45605f73147bd4cadd23bdae915042aad6"}, + {file = "contourpy-1.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0be4d8425bfa755e0fd76ee1e019636ccc7c29f77a7c86b4328a9eb6a26d0639"}, + {file = "contourpy-1.3.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9c0da700bf58f6e0b65312d0a5e695179a71d0163957fa381bb3c1f72972537c"}, + {file = "contourpy-1.3.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eb8b141bb00fa977d9122636b16aa67d37fd40a3d8b52dd837e536d64b9a4d06"}, + {file = "contourpy-1.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3634b5385c6716c258d0419c46d05c8aa7dc8cb70326c9a4fb66b69ad2b52e09"}, + {file = "contourpy-1.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0dce35502151b6bd35027ac39ba6e5a44be13a68f55735c3612c568cac3805fd"}, + {file = "contourpy-1.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aea348f053c645100612b333adc5983d87be69acdc6d77d3169c090d3b01dc35"}, + {file = "contourpy-1.3.0-cp312-cp312-win32.whl", hash = "sha256:90f73a5116ad1ba7174341ef3ea5c3150ddf20b024b98fb0c3b29034752c8aeb"}, + {file = "contourpy-1.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:b11b39aea6be6764f84360fce6c82211a9db32a7c7de8fa6dd5397cf1d079c3b"}, + {file = "contourpy-1.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3e1c7fa44aaae40a2247e2e8e0627f4bea3dd257014764aa644f319a5f8600e3"}, + {file = "contourpy-1.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:364174c2a76057feef647c802652f00953b575723062560498dc7930fc9b1cb7"}, + {file = "contourpy-1.3.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32b238b3b3b649e09ce9aaf51f0c261d38644bdfa35cbaf7b263457850957a84"}, + {file = "contourpy-1.3.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d51fca85f9f7ad0b65b4b9fe800406d0d77017d7270d31ec3fb1cc07358fdea0"}, + {file = "contourpy-1.3.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:732896af21716b29ab3e988d4ce14bc5133733b85956316fb0c56355f398099b"}, + {file = "contourpy-1.3.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d73f659398a0904e125280836ae6f88ba9b178b2fed6884f3b1f95b989d2c8da"}, + {file = "contourpy-1.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c6c7c2408b7048082932cf4e641fa3b8ca848259212f51c8c59c45aa7ac18f14"}, + {file = "contourpy-1.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f317576606de89da6b7e0861cf6061f6146ead3528acabff9236458a6ba467f8"}, + {file = "contourpy-1.3.0-cp313-cp313-win32.whl", hash = "sha256:31cd3a85dbdf1fc002280c65caa7e2b5f65e4a973fcdf70dd2fdcb9868069294"}, + {file = "contourpy-1.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:4553c421929ec95fb07b3aaca0fae668b2eb5a5203d1217ca7c34c063c53d087"}, + {file = "contourpy-1.3.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:345af746d7766821d05d72cb8f3845dfd08dd137101a2cb9b24de277d716def8"}, + {file = "contourpy-1.3.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3bb3808858a9dc68f6f03d319acd5f1b8a337e6cdda197f02f4b8ff67ad2057b"}, + {file = "contourpy-1.3.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:420d39daa61aab1221567b42eecb01112908b2cab7f1b4106a52caaec8d36973"}, + {file = "contourpy-1.3.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4d63ee447261e963af02642ffcb864e5a2ee4cbfd78080657a9880b8b1868e18"}, + {file = "contourpy-1.3.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:167d6c890815e1dac9536dca00828b445d5d0df4d6a8c6adb4a7ec3166812fa8"}, + {file = "contourpy-1.3.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:710a26b3dc80c0e4febf04555de66f5fd17e9cf7170a7b08000601a10570bda6"}, + {file = "contourpy-1.3.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:75ee7cb1a14c617f34a51d11fa7524173e56551646828353c4af859c56b766e2"}, + {file = "contourpy-1.3.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:33c92cdae89ec5135d036e7218e69b0bb2851206077251f04a6c4e0e21f03927"}, + {file = "contourpy-1.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a11077e395f67ffc2c44ec2418cfebed032cd6da3022a94fc227b6faf8e2acb8"}, + {file = "contourpy-1.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e8134301d7e204c88ed7ab50028ba06c683000040ede1d617298611f9dc6240c"}, + {file = "contourpy-1.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e12968fdfd5bb45ffdf6192a590bd8ddd3ba9e58360b29683c6bb71a7b41edca"}, + {file = "contourpy-1.3.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fd2a0fc506eccaaa7595b7e1418951f213cf8255be2600f1ea1b61e46a60c55f"}, + {file = "contourpy-1.3.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4cfb5c62ce023dfc410d6059c936dcf96442ba40814aefbfa575425a3a7f19dc"}, + {file = "contourpy-1.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68a32389b06b82c2fdd68276148d7b9275b5f5cf13e5417e4252f6d1a34f72a2"}, + {file = "contourpy-1.3.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:94e848a6b83da10898cbf1311a815f770acc9b6a3f2d646f330d57eb4e87592e"}, + {file = "contourpy-1.3.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:d78ab28a03c854a873787a0a42254a0ccb3cb133c672f645c9f9c8f3ae9d0800"}, + {file = "contourpy-1.3.0-cp39-cp39-win32.whl", hash = "sha256:81cb5ed4952aae6014bc9d0421dec7c5835c9c8c31cdf51910b708f548cf58e5"}, + {file = "contourpy-1.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:14e262f67bd7e6eb6880bc564dcda30b15e351a594657e55b7eec94b6ef72843"}, + {file = "contourpy-1.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:fe41b41505a5a33aeaed2a613dccaeaa74e0e3ead6dd6fd3a118fb471644fd6c"}, + {file = "contourpy-1.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eca7e17a65f72a5133bdbec9ecf22401c62bcf4821361ef7811faee695799779"}, + {file = "contourpy-1.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:1ec4dc6bf570f5b22ed0d7efba0dfa9c5b9e0431aeea7581aa217542d9e809a4"}, + {file = "contourpy-1.3.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:00ccd0dbaad6d804ab259820fa7cb0b8036bda0686ef844d24125d8287178ce0"}, + {file = "contourpy-1.3.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ca947601224119117f7c19c9cdf6b3ab54c5726ef1d906aa4a69dfb6dd58102"}, + {file = "contourpy-1.3.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c6ec93afeb848a0845a18989da3beca3eec2c0f852322efe21af1931147d12cb"}, + {file = "contourpy-1.3.0.tar.gz", hash = "sha256:7ffa0db17717a8ffb127efd0c95a4362d996b892c2904db72428d5b52e1938a4"}, ] [package.dependencies] -numpy = ">=1.20" +numpy = ">=1.23" [package.extras] bokeh = ["bokeh", "selenium"] docs = ["furo", "sphinx (>=7.2)", "sphinx-copybutton"] -mypy = ["contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.8.0)", "types-Pillow"] +mypy = ["contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.11.1)", "types-Pillow"] test = ["Pillow", "contourpy[test-no-images]", "matplotlib"] -test-no-images = ["pytest", "pytest-cov", "pytest-xdist", "wurlitzer"] +test-no-images = ["pytest", "pytest-cov", "pytest-rerunfailures", "pytest-xdist", "wurlitzer"] [[package]] name = "coverage" @@ -1350,6 +1417,63 @@ files = [ docs = ["Sphinx", "furo"] test = ["objgraph", "psutil"] +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = true +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + +[[package]] +name = "httpcore" +version = "1.0.5" +description = "A minimal low-level HTTP client." +optional = true +python-versions = ">=3.8" +files = [ + {file = "httpcore-1.0.5-py3-none-any.whl", hash = "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"}, + {file = "httpcore-1.0.5.tar.gz", hash = "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.13,<0.15" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<0.26.0)"] + +[[package]] +name = "httpx" +version = "0.27.2" +description = "The next generation HTTP client." +optional = true +python-versions = ">=3.8" +files = [ + {file = "httpx-0.27.2-py3-none-any.whl", hash = "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0"}, + {file = "httpx-0.27.2.tar.gz", hash = "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + [[package]] name = "humanize" version = "4.10.0" @@ -1366,13 +1490,13 @@ tests = ["freezegun", "pytest", "pytest-cov"] [[package]] name = "idna" -version = "3.7" +version = "3.8" description = "Internationalized Domain Names in Applications (IDNA)" optional = false -python-versions = ">=3.5" +python-versions = ">=3.6" files = [ - {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, - {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, + {file = "idna-3.8-py3-none-any.whl", hash = "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac"}, + {file = "idna-3.8.tar.gz", hash = "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603"}, ] [[package]] @@ -1824,16 +1948,17 @@ six = "*" [[package]] name = "langsmith" -version = "0.1.99" +version = "0.1.107" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." optional = true python-versions = "<4.0,>=3.8.1" files = [ - {file = "langsmith-0.1.99-py3-none-any.whl", hash = "sha256:ef8d1d74a2674c514aa429b0171a9fbb661207dc3835142cca0e8f1bf97b26b0"}, - {file = "langsmith-0.1.99.tar.gz", hash = "sha256:b5c6a1f158abda61600a4a445081ee848b4a28b758d91f2793dc02aeffafcaf1"}, + {file = "langsmith-0.1.107-py3-none-any.whl", hash = "sha256:ddd0c846980474e271a553e9c220122e32d1f2ce877cc87d39ecd86726b9e78c"}, + {file = "langsmith-0.1.107.tar.gz", hash = "sha256:f44de0a5f199381d0b518ecbe295d541c44ff33d13f18098ecc54a4547eccb3f"}, ] [package.dependencies] +httpx = ">=0.23.0,<1" orjson = ">=3.9.14,<4.0.0" pydantic = [ {version = ">=1,<3", markers = "python_full_version < \"3.12.4\""}, @@ -2057,13 +2182,13 @@ source = ["Cython (>=3.0.11)"] [[package]] name = "markdown" -version = "3.6" +version = "3.7" description = "Python implementation of John Gruber's Markdown." optional = true python-versions = ">=3.8" files = [ - {file = "Markdown-3.6-py3-none-any.whl", hash = "sha256:48f276f4d8cfb8ce6527c8f79e2ee29708508bf4d40aa410fbc3b4ee832c850f"}, - {file = "Markdown-3.6.tar.gz", hash = "sha256:ed4f41f6daecbeeb96e576ce414c41d2d876daa9a16cb35fa8ed8c2ddfad0224"}, + {file = "Markdown-3.7-py3-none-any.whl", hash = "sha256:7eb6df5690b81a1d7942992c97fad2938e956e79df20cbc6186e9c3a77b1c803"}, + {file = "markdown-3.7.tar.gz", hash = "sha256:2ae2471477cfd02dbbf038d5d9bc226d40def84b4fe2986e49b59b6b472bbed2"}, ] [package.extras] @@ -2167,13 +2292,13 @@ files = [ [[package]] name = "marshmallow" -version = "3.21.3" +version = "3.22.0" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = true python-versions = ">=3.8" files = [ - {file = "marshmallow-3.21.3-py3-none-any.whl", hash = "sha256:86ce7fb914aa865001a4b2092c4c2872d13bc347f3d42673272cabfdbad386f1"}, - {file = "marshmallow-3.21.3.tar.gz", hash = "sha256:4f57c5e050a54d66361e826f94fba213eb10b67b2fdb02c3e0343ce207ba1662"}, + {file = "marshmallow-3.22.0-py3-none-any.whl", hash = "sha256:71a2dce49ef901c3f97ed296ae5051135fd3febd2bf43afe0ae9a82143a494d9"}, + {file = "marshmallow-3.22.0.tar.gz", hash = "sha256:4972f529104a220bb8637d595aa4c9762afbe7f7a77d82dc58c1615d70c5823e"}, ] [package.dependencies] @@ -2181,45 +2306,56 @@ packaging = ">=17.0" [package.extras] dev = ["marshmallow[tests]", "pre-commit (>=3.5,<4.0)", "tox"] -docs = ["alabaster (==0.7.16)", "autodocsumm (==0.2.12)", "sphinx (==7.3.7)", "sphinx-issues (==4.1.0)", "sphinx-version-warning (==1.1.2)"] +docs = ["alabaster (==1.0.0)", "autodocsumm (==0.2.13)", "sphinx (==8.0.2)", "sphinx-issues (==4.1.0)", "sphinx-version-warning (==1.1.2)"] tests = ["pytest", "pytz", "simplejson"] [[package]] name = "matplotlib" -version = "3.9.1.post1" +version = "3.9.2" description = "Python plotting package" optional = true python-versions = ">=3.9" files = [ - {file = "matplotlib-3.9.1.post1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3779ad3e8b72df22b8a622c5796bbcfabfa0069b835412e3c1dec8ee3de92d0c"}, - {file = "matplotlib-3.9.1.post1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ec400340f8628e8e2260d679078d4e9b478699f386e5cc8094e80a1cb0039c7c"}, - {file = "matplotlib-3.9.1.post1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82c18791b8862ea095081f745b81f896b011c5a5091678fb33204fef641476af"}, - {file = "matplotlib-3.9.1.post1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:621a628389c09a6b9f609a238af8e66acecece1cfa12febc5fe4195114ba7446"}, - {file = "matplotlib-3.9.1.post1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9a54734ca761ebb27cd4f0b6c2ede696ab6861052d7d7e7b8f7a6782665115f5"}, - {file = "matplotlib-3.9.1.post1-cp310-cp310-win_amd64.whl", hash = "sha256:0721f93db92311bb514e446842e2b21c004541dcca0281afa495053e017c5458"}, - {file = "matplotlib-3.9.1.post1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:b08b46058fe2a31ecb81ef6aa3611f41d871f6a8280e9057cb4016cb3d8e894a"}, - {file = "matplotlib-3.9.1.post1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:22b344e84fcc574f561b5731f89a7625db8ef80cdbb0026a8ea855a33e3429d1"}, - {file = "matplotlib-3.9.1.post1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b49fee26d64aefa9f061b575f0f7b5fc4663e51f87375c7239efa3d30d908fa"}, - {file = "matplotlib-3.9.1.post1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89eb7e89e2b57856533c5c98f018aa3254fa3789fcd86d5f80077b9034a54c9a"}, - {file = "matplotlib-3.9.1.post1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c06e742bade41fda6176d4c9c78c9ea016e176cd338e62a1686384cb1eb8de41"}, - {file = "matplotlib-3.9.1.post1-cp311-cp311-win_amd64.whl", hash = "sha256:c44edab5b849e0fc1f1c9d6e13eaa35ef65925f7be45be891d9784709ad95561"}, - {file = "matplotlib-3.9.1.post1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:bf28b09986aee06393e808e661c3466be9c21eff443c9bc881bce04bfbb0c500"}, - {file = "matplotlib-3.9.1.post1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:92aeb8c439d4831510d8b9d5e39f31c16c7f37873879767c26b147cef61e54cd"}, - {file = "matplotlib-3.9.1.post1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f15798b0691b45c80d3320358a88ce5a9d6f518b28575b3ea3ed31b4bd95d009"}, - {file = "matplotlib-3.9.1.post1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d59fc6096da7b9c1df275f9afc3fef5cbf634c21df9e5f844cba3dd8deb1847d"}, - {file = "matplotlib-3.9.1.post1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ab986817a32a70ce22302438691e7df4c6ee4a844d47289db9d583d873491e0b"}, - {file = "matplotlib-3.9.1.post1-cp312-cp312-win_amd64.whl", hash = "sha256:0d78e7d2d86c4472da105d39aba9b754ed3dfeaeaa4ac7206b82706e0a5362fa"}, - {file = "matplotlib-3.9.1.post1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:bd07eba6431b4dc9253cce6374a28c415e1d3a7dc9f8aba028ea7592f06fe172"}, - {file = "matplotlib-3.9.1.post1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ca230cc4482010d646827bd2c6d140c98c361e769ae7d954ebf6fff2a226f5b1"}, - {file = "matplotlib-3.9.1.post1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ace27c0fdeded399cbc43f22ffa76e0f0752358f5b33106ec7197534df08725a"}, - {file = "matplotlib-3.9.1.post1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a4f3aeb7ba14c497dc6f021a076c48c2e5fbdf3da1e7264a5d649683e284a2f"}, - {file = "matplotlib-3.9.1.post1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:23f96fbd4ff4cfa9b8a6b685a65e7eb3c2ced724a8d965995ec5c9c2b1f7daf5"}, - {file = "matplotlib-3.9.1.post1-cp39-cp39-win_amd64.whl", hash = "sha256:2808b95452b4ffa14bfb7c7edffc5350743c31bda495f0d63d10fdd9bc69e895"}, - {file = "matplotlib-3.9.1.post1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ffc91239f73b4179dec256b01299d46d0ffa9d27d98494bc1476a651b7821cbe"}, - {file = "matplotlib-3.9.1.post1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:f965ebca9fd4feaaca45937c4849d92b70653057497181100fcd1e18161e5f29"}, - {file = "matplotlib-3.9.1.post1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:801ee9323fd7b2da0d405aebbf98d1da77ea430bbbbbec6834c0b3af15e5db44"}, - {file = "matplotlib-3.9.1.post1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:50113e9b43ceb285739f35d43db36aa752fb8154325b35d134ff6e177452f9ec"}, - {file = "matplotlib-3.9.1.post1.tar.gz", hash = "sha256:c91e585c65092c975a44dc9d4239ba8c594ba3c193d7c478b6d178c4ef61f406"}, + {file = "matplotlib-3.9.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:9d78bbc0cbc891ad55b4f39a48c22182e9bdaea7fc0e5dbd364f49f729ca1bbb"}, + {file = "matplotlib-3.9.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c375cc72229614632c87355366bdf2570c2dac01ac66b8ad048d2dabadf2d0d4"}, + {file = "matplotlib-3.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d94ff717eb2bd0b58fe66380bd8b14ac35f48a98e7c6765117fe67fb7684e64"}, + {file = "matplotlib-3.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab68d50c06938ef28681073327795c5db99bb4666214d2d5f880ed11aeaded66"}, + {file = "matplotlib-3.9.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:65aacf95b62272d568044531e41de26285d54aec8cb859031f511f84bd8b495a"}, + {file = "matplotlib-3.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:3fd595f34aa8a55b7fc8bf9ebea8aa665a84c82d275190a61118d33fbc82ccae"}, + {file = "matplotlib-3.9.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d8dd059447824eec055e829258ab092b56bb0579fc3164fa09c64f3acd478772"}, + {file = "matplotlib-3.9.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c797dac8bb9c7a3fd3382b16fe8f215b4cf0f22adccea36f1545a6d7be310b41"}, + {file = "matplotlib-3.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d719465db13267bcef19ea8954a971db03b9f48b4647e3860e4bc8e6ed86610f"}, + {file = "matplotlib-3.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8912ef7c2362f7193b5819d17dae8629b34a95c58603d781329712ada83f9447"}, + {file = "matplotlib-3.9.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7741f26a58a240f43bee74965c4882b6c93df3e7eb3de160126d8c8f53a6ae6e"}, + {file = "matplotlib-3.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:ae82a14dab96fbfad7965403c643cafe6515e386de723e498cf3eeb1e0b70cc7"}, + {file = "matplotlib-3.9.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:ac43031375a65c3196bee99f6001e7fa5bdfb00ddf43379d3c0609bdca042df9"}, + {file = "matplotlib-3.9.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:be0fc24a5e4531ae4d8e858a1a548c1fe33b176bb13eff7f9d0d38ce5112a27d"}, + {file = "matplotlib-3.9.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf81de2926c2db243c9b2cbc3917619a0fc85796c6ba4e58f541df814bbf83c7"}, + {file = "matplotlib-3.9.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f6ee45bc4245533111ced13f1f2cace1e7f89d1c793390392a80c139d6cf0e6c"}, + {file = "matplotlib-3.9.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:306c8dfc73239f0e72ac50e5a9cf19cc4e8e331dd0c54f5e69ca8758550f1e1e"}, + {file = "matplotlib-3.9.2-cp312-cp312-win_amd64.whl", hash = "sha256:5413401594cfaff0052f9d8b1aafc6d305b4bd7c4331dccd18f561ff7e1d3bd3"}, + {file = "matplotlib-3.9.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:18128cc08f0d3cfff10b76baa2f296fc28c4607368a8402de61bb3f2eb33c7d9"}, + {file = "matplotlib-3.9.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4876d7d40219e8ae8bb70f9263bcbe5714415acfdf781086601211335e24f8aa"}, + {file = "matplotlib-3.9.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d9f07a80deab4bb0b82858a9e9ad53d1382fd122be8cde11080f4e7dfedb38b"}, + {file = "matplotlib-3.9.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7c0410f181a531ec4e93bbc27692f2c71a15c2da16766f5ba9761e7ae518413"}, + {file = "matplotlib-3.9.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:909645cce2dc28b735674ce0931a4ac94e12f5b13f6bb0b5a5e65e7cea2c192b"}, + {file = "matplotlib-3.9.2-cp313-cp313-win_amd64.whl", hash = "sha256:f32c7410c7f246838a77d6d1eff0c0f87f3cb0e7c4247aebea71a6d5a68cab49"}, + {file = "matplotlib-3.9.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:37e51dd1c2db16ede9cfd7b5cabdfc818b2c6397c83f8b10e0e797501c963a03"}, + {file = "matplotlib-3.9.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b82c5045cebcecd8496a4d694d43f9cc84aeeb49fe2133e036b207abe73f4d30"}, + {file = "matplotlib-3.9.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f053c40f94bc51bc03832a41b4f153d83f2062d88c72b5e79997072594e97e51"}, + {file = "matplotlib-3.9.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dbe196377a8248972f5cede786d4c5508ed5f5ca4a1e09b44bda889958b33f8c"}, + {file = "matplotlib-3.9.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:5816b1e1fe8c192cbc013f8f3e3368ac56fbecf02fb41b8f8559303f24c5015e"}, + {file = "matplotlib-3.9.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:cef2a73d06601437be399908cf13aee74e86932a5ccc6ccdf173408ebc5f6bb2"}, + {file = "matplotlib-3.9.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e0830e188029c14e891fadd99702fd90d317df294c3298aad682739c5533721a"}, + {file = "matplotlib-3.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:03ba9c1299c920964e8d3857ba27173b4dbb51ca4bab47ffc2c2ba0eb5e2cbc5"}, + {file = "matplotlib-3.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1cd93b91ab47a3616b4d3c42b52f8363b88ca021e340804c6ab2536344fad9ca"}, + {file = "matplotlib-3.9.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:6d1ce5ed2aefcdce11904fc5bbea7d9c21fff3d5f543841edf3dea84451a09ea"}, + {file = "matplotlib-3.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:b2696efdc08648536efd4e1601b5fd491fd47f4db97a5fbfd175549a7365c1b2"}, + {file = "matplotlib-3.9.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d52a3b618cb1cbb769ce2ee1dcdb333c3ab6e823944e9a2d36e37253815f9556"}, + {file = "matplotlib-3.9.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:039082812cacd6c6bec8e17a9c1e6baca230d4116d522e81e1f63a74d01d2e21"}, + {file = "matplotlib-3.9.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6758baae2ed64f2331d4fd19be38b7b4eae3ecec210049a26b6a4f3ae1c85dcc"}, + {file = "matplotlib-3.9.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:050598c2b29e0b9832cde72bcf97627bf00262adbc4a54e2b856426bb2ef0697"}, + {file = "matplotlib-3.9.2.tar.gz", hash = "sha256:96ab43906269ca64a6366934106fa01534454a69e471b7bf3d79083981aaab92"}, ] [package.dependencies] @@ -2447,38 +2583,38 @@ files = [ [[package]] name = "mypy" -version = "1.11.1" +version = "1.11.2" description = "Optional static typing for Python" optional = false python-versions = ">=3.8" files = [ - {file = "mypy-1.11.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:a32fc80b63de4b5b3e65f4be82b4cfa362a46702672aa6a0f443b4689af7008c"}, - {file = "mypy-1.11.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c1952f5ea8a5a959b05ed5f16452fddadbaae48b5d39235ab4c3fc444d5fd411"}, - {file = "mypy-1.11.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1e30dc3bfa4e157e53c1d17a0dad20f89dc433393e7702b813c10e200843b03"}, - {file = "mypy-1.11.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2c63350af88f43a66d3dfeeeb8d77af34a4f07d760b9eb3a8697f0386c7590b4"}, - {file = "mypy-1.11.1-cp310-cp310-win_amd64.whl", hash = "sha256:a831671bad47186603872a3abc19634f3011d7f83b083762c942442d51c58d58"}, - {file = "mypy-1.11.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7b6343d338390bb946d449677726edf60102a1c96079b4f002dedff375953fc5"}, - {file = "mypy-1.11.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e4fe9f4e5e521b458d8feb52547f4bade7ef8c93238dfb5bbc790d9ff2d770ca"}, - {file = "mypy-1.11.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:886c9dbecc87b9516eff294541bf7f3655722bf22bb898ee06985cd7269898de"}, - {file = "mypy-1.11.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fca4a60e1dd9fd0193ae0067eaeeb962f2d79e0d9f0f66223a0682f26ffcc809"}, - {file = "mypy-1.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:0bd53faf56de9643336aeea1c925012837432b5faf1701ccca7fde70166ccf72"}, - {file = "mypy-1.11.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f39918a50f74dc5969807dcfaecafa804fa7f90c9d60506835036cc1bc891dc8"}, - {file = "mypy-1.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0bc71d1fb27a428139dd78621953effe0d208aed9857cb08d002280b0422003a"}, - {file = "mypy-1.11.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b868d3bcff720dd7217c383474008ddabaf048fad8d78ed948bb4b624870a417"}, - {file = "mypy-1.11.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a707ec1527ffcdd1c784d0924bf5cb15cd7f22683b919668a04d2b9c34549d2e"}, - {file = "mypy-1.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:64f4a90e3ea07f590c5bcf9029035cf0efeae5ba8be511a8caada1a4893f5525"}, - {file = "mypy-1.11.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:749fd3213916f1751fff995fccf20c6195cae941dc968f3aaadf9bb4e430e5a2"}, - {file = "mypy-1.11.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:b639dce63a0b19085213ec5fdd8cffd1d81988f47a2dec7100e93564f3e8fb3b"}, - {file = "mypy-1.11.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4c956b49c5d865394d62941b109728c5c596a415e9c5b2be663dd26a1ff07bc0"}, - {file = "mypy-1.11.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:45df906e8b6804ef4b666af29a87ad9f5921aad091c79cc38e12198e220beabd"}, - {file = "mypy-1.11.1-cp38-cp38-win_amd64.whl", hash = "sha256:d44be7551689d9d47b7abc27c71257adfdb53f03880841a5db15ddb22dc63edb"}, - {file = "mypy-1.11.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2684d3f693073ab89d76da8e3921883019ea8a3ec20fa5d8ecca6a2db4c54bbe"}, - {file = "mypy-1.11.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:79c07eb282cb457473add5052b63925e5cc97dfab9812ee65a7c7ab5e3cb551c"}, - {file = "mypy-1.11.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:11965c2f571ded6239977b14deebd3f4c3abd9a92398712d6da3a772974fad69"}, - {file = "mypy-1.11.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a2b43895a0f8154df6519706d9bca8280cda52d3d9d1514b2d9c3e26792a0b74"}, - {file = "mypy-1.11.1-cp39-cp39-win_amd64.whl", hash = "sha256:1a81cf05975fd61aec5ae16501a091cfb9f605dc3e3c878c0da32f250b74760b"}, - {file = "mypy-1.11.1-py3-none-any.whl", hash = "sha256:0624bdb940255d2dd24e829d99a13cfeb72e4e9031f9492148f410ed30bcab54"}, - {file = "mypy-1.11.1.tar.gz", hash = "sha256:f404a0b069709f18bbdb702eb3dcfe51910602995de00bd39cea3050b5772d08"}, + {file = "mypy-1.11.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d42a6dd818ffce7be66cce644f1dff482f1d97c53ca70908dff0b9ddc120b77a"}, + {file = "mypy-1.11.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:801780c56d1cdb896eacd5619a83e427ce436d86a3bdf9112527f24a66618fef"}, + {file = "mypy-1.11.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41ea707d036a5307ac674ea172875f40c9d55c5394f888b168033177fce47383"}, + {file = "mypy-1.11.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:6e658bd2d20565ea86da7d91331b0eed6d2eee22dc031579e6297f3e12c758c8"}, + {file = "mypy-1.11.2-cp310-cp310-win_amd64.whl", hash = "sha256:478db5f5036817fe45adb7332d927daa62417159d49783041338921dcf646fc7"}, + {file = "mypy-1.11.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:75746e06d5fa1e91bfd5432448d00d34593b52e7e91a187d981d08d1f33d4385"}, + {file = "mypy-1.11.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a976775ab2256aadc6add633d44f100a2517d2388906ec4f13231fafbb0eccca"}, + {file = "mypy-1.11.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd953f221ac1379050a8a646585a29574488974f79d8082cedef62744f0a0104"}, + {file = "mypy-1.11.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:57555a7715c0a34421013144a33d280e73c08df70f3a18a552938587ce9274f4"}, + {file = "mypy-1.11.2-cp311-cp311-win_amd64.whl", hash = "sha256:36383a4fcbad95f2657642a07ba22ff797de26277158f1cc7bd234821468b1b6"}, + {file = "mypy-1.11.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e8960dbbbf36906c5c0b7f4fbf2f0c7ffb20f4898e6a879fcf56a41a08b0d318"}, + {file = "mypy-1.11.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06d26c277962f3fb50e13044674aa10553981ae514288cb7d0a738f495550b36"}, + {file = "mypy-1.11.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6e7184632d89d677973a14d00ae4d03214c8bc301ceefcdaf5c474866814c987"}, + {file = "mypy-1.11.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:3a66169b92452f72117e2da3a576087025449018afc2d8e9bfe5ffab865709ca"}, + {file = "mypy-1.11.2-cp312-cp312-win_amd64.whl", hash = "sha256:969ea3ef09617aff826885a22ece0ddef69d95852cdad2f60c8bb06bf1f71f70"}, + {file = "mypy-1.11.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:37c7fa6121c1cdfcaac97ce3d3b5588e847aa79b580c1e922bb5d5d2902df19b"}, + {file = "mypy-1.11.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4a8a53bc3ffbd161b5b2a4fff2f0f1e23a33b0168f1c0778ec70e1a3d66deb86"}, + {file = "mypy-1.11.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ff93107f01968ed834f4256bc1fc4475e2fecf6c661260066a985b52741ddce"}, + {file = "mypy-1.11.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:edb91dded4df17eae4537668b23f0ff6baf3707683734b6a818d5b9d0c0c31a1"}, + {file = "mypy-1.11.2-cp38-cp38-win_amd64.whl", hash = "sha256:ee23de8530d99b6db0573c4ef4bd8f39a2a6f9b60655bf7a1357e585a3486f2b"}, + {file = "mypy-1.11.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:801ca29f43d5acce85f8e999b1e431fb479cb02d0e11deb7d2abb56bdaf24fd6"}, + {file = "mypy-1.11.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:af8d155170fcf87a2afb55b35dc1a0ac21df4431e7d96717621962e4b9192e70"}, + {file = "mypy-1.11.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7821776e5c4286b6a13138cc935e2e9b6fde05e081bdebf5cdb2bb97c9df81d"}, + {file = "mypy-1.11.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:539c570477a96a4e6fb718b8d5c3e0c0eba1f485df13f86d2970c91f0673148d"}, + {file = "mypy-1.11.2-cp39-cp39-win_amd64.whl", hash = "sha256:3f14cd3d386ac4d05c5a39a51b84387403dadbd936e17cb35882134d4f8f0d24"}, + {file = "mypy-1.11.2-py3-none-any.whl", hash = "sha256:b499bc07dbdcd3de92b0a8b29fdf592c111276f6a12fe29c30f6c417dd546d12"}, + {file = "mypy-1.11.2.tar.gz", hash = "sha256:7f9993ad3e0ffdc95c2a14b66dee63729f021968bff8ad911867579c65d13a79"}, ] [package.dependencies] @@ -3016,13 +3152,13 @@ type = ["mypy (>=1.8)"] [[package]] name = "plotly" -version = "5.23.0" +version = "5.24.0" description = "An open-source, interactive data visualization library for Python" optional = true python-versions = ">=3.8" files = [ - {file = "plotly-5.23.0-py3-none-any.whl", hash = "sha256:76cbe78f75eddc10c56f5a4ee3e7ccaade7c0a57465546f02098c0caed6c2d1a"}, - {file = "plotly-5.23.0.tar.gz", hash = "sha256:89e57d003a116303a34de6700862391367dd564222ab71f8531df70279fc0193"}, + {file = "plotly-5.24.0-py3-none-any.whl", hash = "sha256:0e54efe52c8cef899f7daa41be9ed97dfb6be622613a2a8f56a86a0634b2b67e"}, + {file = "plotly-5.24.0.tar.gz", hash = "sha256:eae9f4f54448682442c92c1e97148e3ad0c52f0cf86306e1b76daba24add554a"}, ] [package.dependencies] @@ -3064,13 +3200,13 @@ poetry-plugin = ["poetry (>=1.0,<2.0)"] [[package]] name = "posthog" -version = "3.5.0" +version = "3.6.0" description = "Integrate PostHog into any python application." optional = false python-versions = "*" files = [ - {file = "posthog-3.5.0-py2.py3-none-any.whl", hash = "sha256:3c672be7ba6f95d555ea207d4486c171d06657eb34b3ce25eb043bfe7b6b5b76"}, - {file = "posthog-3.5.0.tar.gz", hash = "sha256:8f7e3b2c6e8714d0c0c542a2109b83a7549f63b7113a133ab2763a89245ef2ef"}, + {file = "posthog-3.6.0-py2.py3-none-any.whl", hash = "sha256:6f8dacc6d14d80734b1d15bd4ab08b049629c5f0fc420cafcf1ce0667c76c83c"}, + {file = "posthog-3.6.0.tar.gz", hash = "sha256:27dbf537241a69fb5f6a3e9561caa2d555d5891d95fa65c27ffa6b52d1fb63b6"}, ] [package.dependencies] @@ -3322,13 +3458,13 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] [[package]] name = "pyparsing" -version = "3.1.2" +version = "3.1.4" description = "pyparsing module - Classes and methods to define and execute parsing grammars" optional = true python-versions = ">=3.6.8" files = [ - {file = "pyparsing-3.1.2-py3-none-any.whl", hash = "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742"}, - {file = "pyparsing-3.1.2.tar.gz", hash = "sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad"}, + {file = "pyparsing-3.1.4-py3-none-any.whl", hash = "sha256:a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c"}, + {file = "pyparsing-3.1.4.tar.gz", hash = "sha256:f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032"}, ] [package.extras] @@ -4094,13 +4230,13 @@ compatible-mypy = ["mypy (>=1.10,<1.11)"] [[package]] name = "rich" -version = "13.7.1" +version = "13.8.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"}, - {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, + {file = "rich-13.8.0-py3-none-any.whl", hash = "sha256:2e85306a063b9492dffc86278197a60cbece75bcb766022f3436f567cae11bdc"}, + {file = "rich-13.8.0.tar.gz", hash = "sha256:a5ac1f1cd448ade0d59cc3356f7db7a7ccda2c8cbae9c7a90c28ff463d3e91f4"}, ] [package.dependencies] @@ -4171,36 +4307,44 @@ tests = ["black (>=24.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.9)", "numpydoc ( [[package]] name = "scipy" -version = "1.14.0" +version = "1.14.1" description = "Fundamental algorithms for scientific computing in Python" optional = true python-versions = ">=3.10" files = [ - {file = "scipy-1.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7e911933d54ead4d557c02402710c2396529540b81dd554fc1ba270eb7308484"}, - {file = "scipy-1.14.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:687af0a35462402dd851726295c1a5ae5f987bd6e9026f52e9505994e2f84ef6"}, - {file = "scipy-1.14.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:07e179dc0205a50721022344fb85074f772eadbda1e1b3eecdc483f8033709b7"}, - {file = "scipy-1.14.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:6a9c9a9b226d9a21e0a208bdb024c3982932e43811b62d202aaf1bb59af264b1"}, - {file = "scipy-1.14.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:076c27284c768b84a45dcf2e914d4000aac537da74236a0d45d82c6fa4b7b3c0"}, - {file = "scipy-1.14.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42470ea0195336df319741e230626b6225a740fd9dce9642ca13e98f667047c0"}, - {file = "scipy-1.14.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:176c6f0d0470a32f1b2efaf40c3d37a24876cebf447498a4cefb947a79c21e9d"}, - {file = "scipy-1.14.0-cp310-cp310-win_amd64.whl", hash = "sha256:ad36af9626d27a4326c8e884917b7ec321d8a1841cd6dacc67d2a9e90c2f0359"}, - {file = "scipy-1.14.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6d056a8709ccda6cf36cdd2eac597d13bc03dba38360f418560a93050c76a16e"}, - {file = "scipy-1.14.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:f0a50da861a7ec4573b7c716b2ebdcdf142b66b756a0d392c236ae568b3a93fb"}, - {file = "scipy-1.14.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:94c164a9e2498e68308e6e148646e486d979f7fcdb8b4cf34b5441894bdb9caf"}, - {file = "scipy-1.14.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:a7d46c3e0aea5c064e734c3eac5cf9eb1f8c4ceee756262f2c7327c4c2691c86"}, - {file = "scipy-1.14.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9eee2989868e274aae26125345584254d97c56194c072ed96cb433f32f692ed8"}, - {file = "scipy-1.14.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e3154691b9f7ed73778d746da2df67a19d046a6c8087c8b385bc4cdb2cfca74"}, - {file = "scipy-1.14.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:c40003d880f39c11c1edbae8144e3813904b10514cd3d3d00c277ae996488cdb"}, - {file = "scipy-1.14.0-cp311-cp311-win_amd64.whl", hash = "sha256:5b083c8940028bb7e0b4172acafda6df762da1927b9091f9611b0bcd8676f2bc"}, - {file = "scipy-1.14.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:bff2438ea1330e06e53c424893ec0072640dac00f29c6a43a575cbae4c99b2b9"}, - {file = "scipy-1.14.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:bbc0471b5f22c11c389075d091d3885693fd3f5e9a54ce051b46308bc787e5d4"}, - {file = "scipy-1.14.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:64b2ff514a98cf2bb734a9f90d32dc89dc6ad4a4a36a312cd0d6327170339eb0"}, - {file = "scipy-1.14.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:7d3da42fbbbb860211a811782504f38ae7aaec9de8764a9bef6b262de7a2b50f"}, - {file = "scipy-1.14.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d91db2c41dd6c20646af280355d41dfa1ec7eead235642178bd57635a3f82209"}, - {file = "scipy-1.14.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a01cc03bcdc777c9da3cfdcc74b5a75caffb48a6c39c8450a9a05f82c4250a14"}, - {file = "scipy-1.14.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:65df4da3c12a2bb9ad52b86b4dcf46813e869afb006e58be0f516bc370165159"}, - {file = "scipy-1.14.0-cp312-cp312-win_amd64.whl", hash = "sha256:4c4161597c75043f7154238ef419c29a64ac4a7c889d588ea77690ac4d0d9b20"}, - {file = "scipy-1.14.0.tar.gz", hash = "sha256:b5923f48cb840380f9854339176ef21763118a7300a88203ccd0bdd26e58527b"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:b28d2ca4add7ac16ae8bb6632a3c86e4b9e4d52d3e34267f6e1b0c1f8d87e389"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d0d2821003174de06b69e58cef2316a6622b60ee613121199cb2852a873f8cf3"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8bddf15838ba768bb5f5083c1ea012d64c9a444e16192762bd858f1e126196d0"}, + {file = "scipy-1.14.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:97c5dddd5932bd2a1a31c927ba5e1463a53b87ca96b5c9bdf5dfd6096e27efc3"}, + {file = "scipy-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ff0a7e01e422c15739ecd64432743cf7aae2b03f3084288f399affcefe5222d"}, + {file = "scipy-1.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e32dced201274bf96899e6491d9ba3e9a5f6b336708656466ad0522d8528f69"}, + {file = "scipy-1.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8426251ad1e4ad903a4514712d2fa8fdd5382c978010d1c6f5f37ef286a713ad"}, + {file = "scipy-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:a49f6ed96f83966f576b33a44257d869756df6cf1ef4934f59dd58b25e0327e5"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:2da0469a4ef0ecd3693761acbdc20f2fdeafb69e6819cc081308cc978153c675"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c0ee987efa6737242745f347835da2cc5bb9f1b42996a4d97d5c7ff7928cb6f2"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3a1b111fac6baec1c1d92f27e76511c9e7218f1695d61b59e05e0fe04dc59617"}, + {file = "scipy-1.14.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8475230e55549ab3f207bff11ebfc91c805dc3463ef62eda3ccf593254524ce8"}, + {file = "scipy-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:278266012eb69f4a720827bdd2dc54b2271c97d84255b2faaa8f161a158c3b37"}, + {file = "scipy-1.14.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fef8c87f8abfb884dac04e97824b61299880c43f4ce675dd2cbeadd3c9b466d2"}, + {file = "scipy-1.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b05d43735bb2f07d689f56f7b474788a13ed8adc484a85aa65c0fd931cf9ccd2"}, + {file = "scipy-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:716e389b694c4bb564b4fc0c51bc84d381735e0d39d3f26ec1af2556ec6aad94"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:631f07b3734d34aced009aaf6fedfd0eb3498a97e581c3b1e5f14a04164a456d"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:af29a935803cc707ab2ed7791c44288a682f9c8107bc00f0eccc4f92c08d6e07"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2843f2d527d9eebec9a43e6b406fb7266f3af25a751aa91d62ff416f54170bc5"}, + {file = "scipy-1.14.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:eb58ca0abd96911932f688528977858681a59d61a7ce908ffd355957f7025cfc"}, + {file = "scipy-1.14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30ac8812c1d2aab7131a79ba62933a2a76f582d5dbbc695192453dae67ad6310"}, + {file = "scipy-1.14.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f9ea80f2e65bdaa0b7627fb00cbeb2daf163caa015e59b7516395fe3bd1e066"}, + {file = "scipy-1.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:edaf02b82cd7639db00dbff629995ef185c8df4c3ffa71a5562a595765a06ce1"}, + {file = "scipy-1.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2ff38e22128e6c03ff73b6bb0f85f897d2362f8c052e3b8ad00532198fbdae3f"}, + {file = "scipy-1.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1729560c906963fc8389f6aac023739ff3983e727b1a4d87696b7bf108316a79"}, + {file = "scipy-1.14.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:4079b90df244709e675cdc8b93bfd8a395d59af40b72e339c2287c91860deb8e"}, + {file = "scipy-1.14.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e0cf28db0f24a38b2a0ca33a85a54852586e43cf6fd876365c86e0657cfe7d73"}, + {file = "scipy-1.14.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:0c2f95de3b04e26f5f3ad5bb05e74ba7f68b837133a4492414b3afd79dfe540e"}, + {file = "scipy-1.14.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b99722ea48b7ea25e8e015e8341ae74624f72e5f21fc2abd45f3a93266de4c5d"}, + {file = "scipy-1.14.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5149e3fd2d686e42144a093b206aef01932a0059c2a33ddfa67f5f035bdfe13e"}, + {file = "scipy-1.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e4f5a7c49323533f9103d4dacf4e4f07078f360743dec7f7596949149efeec06"}, + {file = "scipy-1.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:baff393942b550823bfce952bb62270ee17504d02a1801d7fd0719534dfb9c84"}, + {file = "scipy-1.14.1.tar.gz", hash = "sha256:5a275584e726026a5699459aa72f828a610821006228e841b94275c4a7c08417"}, ] [package.dependencies] @@ -4208,8 +4352,8 @@ numpy = ">=1.23.5,<2.3" [package.extras] dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodestyle", "pydevtool", "rich-click", "ruff (>=0.0.292)", "types-psutil", "typing_extensions"] -doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.13.1)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0)", "sphinx-design (>=0.4.0)"] -test = ["Cython", "array-api-strict", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.13.1)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<=7.3.7)", "sphinx-design (>=0.4.0)"] +test = ["Cython", "array-api-strict (>=2.0)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] [[package]] name = "sentry-sdk" @@ -4262,21 +4406,71 @@ starlette = ["starlette (>=0.19.1)"] starlite = ["starlite (>=1.48)"] tornado = ["tornado (>=6)"] +[[package]] +name = "serpyco-rs" +version = "1.10.2" +description = "" +optional = false +python-versions = ">=3.9" +files = [ + {file = "serpyco_rs-1.10.2-cp310-cp310-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:e01d824fdebb9bded57ec40b9ac0ca3b312ad617fd5deba61113a3b23bcb915d"}, + {file = "serpyco_rs-1.10.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ef9a31f8d62c17b1ccfffb3e91c5aed2d6fd2187c7611ee3ca1b572046150cd"}, + {file = "serpyco_rs-1.10.2-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:aab2241b2d87bca5f15d5d34a3948b1c9ad1724cc55d1332e0c5325aff02635f"}, + {file = "serpyco_rs-1.10.2-cp310-cp310-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:87d8118e9ba6e37aee1b0f7c14b19fe494f1589dc81ae0cc5168812779e1bfab"}, + {file = "serpyco_rs-1.10.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3d18a77d23aeb49904b2462410e57b4027511158845291bf6251e5857a881d60"}, + {file = "serpyco_rs-1.10.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8da7ff487ada75f6b724d6ef9e40cde5cf703a2b89e6a3f466a8db0049e153a"}, + {file = "serpyco_rs-1.10.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5212fa00ff8874ecabca0cf5f11eb7c1291b55ec9ee6aa7ee3ae2ec344abcf7f"}, + {file = "serpyco_rs-1.10.2-cp310-none-win_amd64.whl", hash = "sha256:ff83f5296f0ab08e77d09a4888020e636d4385a642fec52eacd2ab480d0ec22c"}, + {file = "serpyco_rs-1.10.2-cp311-cp311-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:8d0e6d6546145ba30d6032381b27261e338f7c1b96b9fb0773a481970a809827"}, + {file = "serpyco_rs-1.10.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf4d5c69d1fcd7007b7792cb5ea62a0702822f6f8982349f44b795677ab7414c"}, + {file = "serpyco_rs-1.10.2-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9fc4c1241c0707bfdd93991c0a2cea3f51a17acad343d9b5c296fc0a9f044d78"}, + {file = "serpyco_rs-1.10.2-cp311-cp311-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:413fe29db4cab826269371a89ff9ccbd897ee7ff0eaaf1090362fdb86d5b8beb"}, + {file = "serpyco_rs-1.10.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:54ce4d5ac0ac4d62911998bfba1ac149a61c43f5dbfa23f831f0d87290c1861a"}, + {file = "serpyco_rs-1.10.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bd9c8a31440a3158c601fdcd523e77cd5fefa2ae5be061a4151c38a7a6060624"}, + {file = "serpyco_rs-1.10.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e8e323f5420c3e6f99627291a2d47d7fcd7f5c4433aaa6cc35e15d5b22ba19d6"}, + {file = "serpyco_rs-1.10.2-cp311-none-win_amd64.whl", hash = "sha256:743c1e1943f51883cb498c2c16c5f49bab2adb991c842077fcd0fa5a1658da25"}, + {file = "serpyco_rs-1.10.2-cp312-cp312-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:6379d789daff44e5f535d7e1c0131b30cee86988e9561cc9d98e87021188220d"}, + {file = "serpyco_rs-1.10.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:805162d7b67fd08b04b1e2ef1deeaedc37c7ee24a200f24778fb98b9fe7f5cdd"}, + {file = "serpyco_rs-1.10.2-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1366df15ae2548a8a063eca84b9a8c2af92ac55df73ce60a7c4f2dfe71e2526b"}, + {file = "serpyco_rs-1.10.2-cp312-cp312-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:35d0a1a1a69ae074b123f6ad1487dc67717727d9dce4f95a393298743d60aafb"}, + {file = "serpyco_rs-1.10.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4a79517070e0b021803cafdf11d326e1149eac4a226443040e9fa1492c74337b"}, + {file = "serpyco_rs-1.10.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bdd2b8d3b9160ddcab0400ca5e258c16e870ae49c6586ed5405c18e8910c957b"}, + {file = "serpyco_rs-1.10.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:045965a32c651797a73c7b7165165ed0d78efc233af4bf24c47acd41d222fae8"}, + {file = "serpyco_rs-1.10.2-cp312-none-win_amd64.whl", hash = "sha256:c6c95f6c9e04af94c33e4e514291df7380c3960a155e9fe264ccaaa46d4d0de8"}, + {file = "serpyco_rs-1.10.2-cp39-cp39-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:f19a82836699d102b288b17ae370dd4d37af60ccd2254f5bfdbd053d168cecee"}, + {file = "serpyco_rs-1.10.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c3830bb3f6a342825e27592e86baa46774bfb1f08c82dbf561b5f1380a18b48"}, + {file = "serpyco_rs-1.10.2-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f726392e6380b1e7d642d7633ac27929c8616a59db0a54632f5a9ab80987e071"}, + {file = "serpyco_rs-1.10.2-cp39-cp39-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:9ce029f8f29f4f335d0f3c9e005b71d7e8a934735d9654e3f03ccc54d50c107a"}, + {file = "serpyco_rs-1.10.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e1f011370259602b55141ce866bf31dcdc9d8b68105c32f18ee442bc651ee880"}, + {file = "serpyco_rs-1.10.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:14d9e22016e2860c1f524aa123cfadd4a4eea25af10d1be76cc3d97d9c85c2e2"}, + {file = "serpyco_rs-1.10.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:441b8045f91f30120c00a1f617a0ad6f22c1753c6b98899e8476d6e7775a3667"}, + {file = "serpyco_rs-1.10.2-cp39-none-win_amd64.whl", hash = "sha256:a124608cc998e3854fc743dea5dd7d948edbeaa70c1c1777b6dbb4b64ce465b0"}, + {file = "serpyco_rs-1.10.2.tar.gz", hash = "sha256:9cf06956eb14b326e522c9665aa5136f8fd7ece2df8a393c2e84bee8204362d0"}, +] + +[package.dependencies] +attributes-doc = "*" +typing-extensions = "*" + [[package]] name = "setuptools" -version = "72.1.0" +version = "74.0.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-72.1.0-py3-none-any.whl", hash = "sha256:5a03e1860cf56bb6ef48ce186b0e557fdba433237481a9a625176c2831be15d1"}, - {file = "setuptools-72.1.0.tar.gz", hash = "sha256:8d243eff56d095e5817f796ede6ae32941278f542e0f941867cc05ae52b162ec"}, + {file = "setuptools-74.0.0-py3-none-any.whl", hash = "sha256:0274581a0037b638b9fc1c6883cc71c0210865aaa76073f7882376b641b84e8f"}, + {file = "setuptools-74.0.0.tar.gz", hash = "sha256:a85e96b8be2b906f3e3e789adec6a9323abf79758ecfa3065bd740d81158b11e"}, ] [package.extras] -core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "ordered-set (>=3.1.1)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "mypy (==1.11.*)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (<0.4)", "pytest-ruff (>=0.2.1)", "pytest-ruff (>=0.3.2)", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.5.2)"] +core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.text (>=3.7)", "more-itertools (>=8.8)", "packaging (>=24)", "platformdirs (>=2.6.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.11.*)", "pytest-mypy"] [[package]] name = "six" @@ -4300,6 +4494,17 @@ files = [ {file = "smmap-5.0.1.tar.gz", hash = "sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62"}, ] +[[package]] +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" +optional = true +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, +] + [[package]] name = "snowballstemmer" version = "2.2.0" @@ -4313,13 +4518,13 @@ files = [ [[package]] name = "soupsieve" -version = "2.5" +version = "2.6" description = "A modern CSS selector implementation for Beautiful Soup." optional = true python-versions = ">=3.8" files = [ - {file = "soupsieve-2.5-py3-none-any.whl", hash = "sha256:eaa337ff55a1579b6549dc679565eac1e3d000563bcb1c8ab0d0fefbc0c2cdc7"}, - {file = "soupsieve-2.5.tar.gz", hash = "sha256:5663d5a7b3bfaeee0bc4372e7fc48f9cff4940b3eec54a6451cc5299f1097690"}, + {file = "soupsieve-2.6-py3-none-any.whl", hash = "sha256:e72c4ff06e4fb6e4b5a9f0f55fe6e81514581fca1515028625d0f299c602ccc9"}, + {file = "soupsieve-2.6.tar.gz", hash = "sha256:e2e68417777af359ec65daac1057404a3c8a5455bb8abc36f1a9866ab1a51abb"}, ] [[package]] @@ -4587,17 +4792,18 @@ test = ["pytest", "tornado (>=4.5)", "typeguard"] [[package]] name = "textual" -version = "0.76.0" +version = "0.79.0" description = "Modern Text User Interface framework" optional = false python-versions = "<4.0.0,>=3.8.1" files = [ - {file = "textual-0.76.0-py3-none-any.whl", hash = "sha256:e2035609c889dba507d34a5d7b333f1c8c53a29fb170962cb92101507663517a"}, - {file = "textual-0.76.0.tar.gz", hash = "sha256:b12e8879d591090c0901b5cb8121d086e28e677353b368292d3865ec99b83b70"}, + {file = "textual-0.79.0-py3-none-any.whl", hash = "sha256:59785f20e13b0e530e3d21c0fca5eb09bd1ff329f47abce29a8e50a59646228d"}, + {file = "textual-0.79.0.tar.gz", hash = "sha256:b5ae63ae11227c158da90e486e99a6db7ef198470219edaf8c200a999d27577a"}, ] [package.dependencies] markdown-it-py = {version = ">=2.1.0", extras = ["linkify", "plugins"]} +platformdirs = ">=4.2.2,<5.0.0" rich = ">=13.3.3" typing-extensions = ">=4.4.0,<5.0.0" @@ -4881,13 +5087,13 @@ xlsx = ["networkx", "openpyxl", "pandas", "xlrd"] [[package]] name = "unstructured-pytesseract" -version = "0.3.12" +version = "0.3.13" description = "Python-tesseract is a python wrapper for Google's Tesseract-OCR" optional = true python-versions = ">=3.8" files = [ - {file = "unstructured.pytesseract-0.3.12-py3-none-any.whl", hash = "sha256:6ed42530fc697bb08d1ae4884cc517ee808620c1c1414efe8d5d90334da068d3"}, - {file = "unstructured.pytesseract-0.3.12.tar.gz", hash = "sha256:751a21d67b1f109036bf4daf796d3e04631697a355efd650f3373412b249de2e"}, + {file = "unstructured.pytesseract-0.3.13-py3-none-any.whl", hash = "sha256:8001bc860470d56185176eb3ceb4623e888eba058ca3b30af79003784bc40e19"}, + {file = "unstructured.pytesseract-0.3.13.tar.gz", hash = "sha256:ff2e6391496e457dbf4b4e327f4a4577cce18921ea6570dc74bd64381b10e963"}, ] [package.dependencies] @@ -4952,13 +5158,13 @@ files = [ [[package]] name = "werkzeug" -version = "3.0.3" +version = "3.0.4" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.8" files = [ - {file = "werkzeug-3.0.3-py3-none-any.whl", hash = "sha256:fc9645dc43e03e4d630d23143a04a7f947a9a3b5727cd535fdfe155a17cc48c8"}, - {file = "werkzeug-3.0.3.tar.gz", hash = "sha256:097e5bfda9f0aba8da6b8545146def481d06aa7d3266e7448e2cccf67dd8bd18"}, + {file = "werkzeug-3.0.4-py3-none-any.whl", hash = "sha256:02c9eb92b7d6c06f31a782811505d2157837cea66aaede3e217c7c27c039476c"}, + {file = "werkzeug-3.0.4.tar.gz", hash = "sha256:34f2371506b250df4d4f84bfe7b0921e4762525762bbd936614909fe25cd7306"}, ] [package.dependencies] @@ -5173,18 +5379,22 @@ multidict = ">=4.0" [[package]] name = "zipp" -version = "3.20.0" +version = "3.20.1" description = "Backport of pathlib-compatible object wrapper for zip files" optional = true python-versions = ">=3.8" files = [ - {file = "zipp-3.20.0-py3-none-any.whl", hash = "sha256:58da6168be89f0be59beb194da1250516fdaa062ccebd30127ac65d30045e10d"}, - {file = "zipp-3.20.0.tar.gz", hash = "sha256:0145e43d89664cfe1a2e533adc75adafed82fe2da404b4bbb6b026c0157bdb31"}, + {file = "zipp-3.20.1-py3-none-any.whl", hash = "sha256:9960cd8967c8f85a56f920d5d507274e74f9ff813a0ab8889a5b5be2daf44064"}, + {file = "zipp-3.20.1.tar.gz", hash = "sha256:c22b14cc4763c5a5b04134207736c107db42e9d3ef2d9779d465f5f1bcba572b"}, ] [package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy"] [extras] file-based = ["avro", "fastavro", "markdown", "pandas", "pdf2image", "pdfminer.six", "pyarrow", "pytesseract", "python-calamine", "unstructured", "unstructured.pytesseract"] @@ -5194,4 +5404,4 @@ vector-db-based = ["cohere", "langchain", "openai", "tiktoken"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "a61b0b329edd46e9efd6ff722f9518d63210de55c4f770a29785e630c372bb0e" +content-hash = "1759d8574c392cf39fccff997263873168087159c5f741314ceff6db4e5a32af" diff --git a/airbyte-cdk/python/pyproject.toml b/airbyte-cdk/python/pyproject.toml index 0eda309b44a80..4b07bed0626f9 100644 --- a/airbyte-cdk/python/pyproject.toml +++ b/airbyte-cdk/python/pyproject.toml @@ -22,9 +22,10 @@ classifiers = [ ] keywords = ["airbyte", "connector-development-kit", "cdk"] + [tool.poetry.dependencies] python = "^3.10" -airbyte-protocol-models-pdv2 = "^0.12.2" +airbyte-protocol-models-dataclasses = "^0.13" backoff = "*" cachetools = "*" Deprecated = "~1.2" @@ -66,6 +67,7 @@ pyjwt = "^2.8.0" cryptography = "^42.0.5" pytz = "2024.1" orjson = "^3.10.7" +serpyco-rs = "^1.10.2" [tool.poetry.group.dev.dependencies] freezegun = "*" diff --git a/airbyte-cdk/python/unit_tests/conftest.py b/airbyte-cdk/python/unit_tests/conftest.py index a5883fe095a56..5d1e1f03f3424 100644 --- a/airbyte-cdk/python/unit_tests/conftest.py +++ b/airbyte-cdk/python/unit_tests/conftest.py @@ -10,6 +10,6 @@ @pytest.fixture() def mock_sleep(monkeypatch): - with freezegun.freeze_time(datetime.datetime.now(), ignore=['_pytest.runner', '_pytest.terminal']) as frozen_datetime: - monkeypatch.setattr('time.sleep', lambda x: frozen_datetime.tick(x)) + with freezegun.freeze_time(datetime.datetime.now(), ignore=["_pytest.runner", "_pytest.terminal"]) as frozen_datetime: + monkeypatch.setattr("time.sleep", lambda x: frozen_datetime.tick(x)) yield diff --git a/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py b/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py index a967087f0d0e7..ca6b8e47ea689 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/test_connector_builder_handler.py @@ -27,11 +27,13 @@ from airbyte_cdk.models import ( AirbyteLogMessage, AirbyteMessage, + AirbyteMessageSerializer, AirbyteRecordMessage, AirbyteStateMessage, AirbyteStream, AirbyteStreamState, ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, ConfiguredAirbyteStream, ConnectorSpecification, DestinationSyncMode, @@ -46,6 +48,7 @@ from airbyte_cdk.sources.declarative.retrievers import SimpleRetrieverTestReadDecorator from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets, update_secrets +from orjson import orjson from unit_tests.connector_builder.utils import create_configured_catalog _stream_name = "stream_with_custom_requester" @@ -73,8 +76,8 @@ }, ], "parent_state": {}, - } - ) + }, + ), ) ] @@ -277,13 +280,13 @@ def _mocked_send(self, request, **kwargs) -> requests.Response: def test_handle_resolve_manifest(valid_resolve_manifest_config_file, dummy_catalog): - with mock.patch.object(connector_builder.main, "handle_connector_builder_request") as patched_handle: + with mock.patch.object(connector_builder.main, "handle_connector_builder_request", return_value=AirbyteMessage(type=MessageType.RECORD)) as patched_handle: handle_request(["read", "--config", str(valid_resolve_manifest_config_file), "--catalog", str(dummy_catalog)]) assert patched_handle.call_count == 1 def test_handle_test_read(valid_read_config_file, configured_catalog): - with mock.patch.object(connector_builder.main, "handle_connector_builder_request") as patch: + with mock.patch.object(connector_builder.main, "handle_connector_builder_request", return_value=AirbyteMessage(type=MessageType.RECORD)) as patch: handle_request(["read", "--config", str(valid_read_config_file), "--catalog", str(configured_catalog)]) assert patch.call_count == 1 @@ -487,11 +490,14 @@ def test_read(): limits = TestReadLimits() with patch("airbyte_cdk.connector_builder.message_grouper.MessageGrouper.get_message_groups", return_value=stream_read) as mock: output_record = handle_connector_builder_request( - source, "test_read", config, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), _A_STATE, limits + source, "test_read", config, ConfiguredAirbyteCatalogSerializer.load(CONFIGURED_CATALOG), _A_STATE, limits ) - mock.assert_called_with(source, config, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), _A_STATE, limits.max_records) + mock.assert_called_with(source, config, ConfiguredAirbyteCatalogSerializer.load(CONFIGURED_CATALOG), _A_STATE, limits.max_records) output_record.record.emitted_at = 1 - assert output_record == expected_airbyte_message + assert ( + orjson.dumps(AirbyteMessageSerializer.dump(output_record)).decode() + == orjson.dumps(AirbyteMessageSerializer.dump(expected_airbyte_message)).decode() + ) def test_config_update(): @@ -523,7 +529,12 @@ def test_config_update(): return_value=refresh_request_response, ): output = handle_connector_builder_request( - source, "test_read", config, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), _A_PER_PARTITION_STATE, TestReadLimits() + source, + "test_read", + config, + ConfiguredAirbyteCatalogSerializer.load(CONFIGURED_CATALOG), + _A_PER_PARTITION_STATE, + TestReadLimits(), ) assert output.record.data["latest_config_update"] @@ -560,7 +571,7 @@ def check_config_against_spec(self): source = MockManifestDeclarativeSource() limits = TestReadLimits() - response = read_stream(source, TEST_READ_CONFIG, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), _A_STATE, limits) + response = read_stream(source, TEST_READ_CONFIG, ConfiguredAirbyteCatalogSerializer.load(CONFIGURED_CATALOG), _A_STATE, limits) expected_stream_read = StreamRead( logs=[LogMessage("error_message - a stack trace", "ERROR")], @@ -584,13 +595,8 @@ def test_handle_429_response(): response = _create_429_page_response({"result": [{"error": "too many requests"}], "_metadata": {"next": "next"}}) # Add backoff strategy to avoid default endless backoff loop - TEST_READ_CONFIG["__injected_declarative_manifest"]['definitions']['retriever']['requester']['error_handler'] = { - "backoff_strategies": [ - { - "type": "ConstantBackoffStrategy", - "backoff_time_in_seconds": 5 - } - ] + TEST_READ_CONFIG["__injected_declarative_manifest"]["definitions"]["retriever"]["requester"]["error_handler"] = { + "backoff_strategies": [{"type": "ConstantBackoffStrategy", "backoff_time_in_seconds": 5}] } config = TEST_READ_CONFIG @@ -599,7 +605,7 @@ def test_handle_429_response(): with patch("requests.Session.send", return_value=response) as mock_send: response = handle_connector_builder_request( - source, "test_read", config, ConfiguredAirbyteCatalog.parse_obj(CONFIGURED_CATALOG), _A_PER_PARTITION_STATE, limits + source, "test_read", config, ConfiguredAirbyteCatalogSerializer.load(CONFIGURED_CATALOG), _A_PER_PARTITION_STATE, limits ) mock_send.assert_called_once() diff --git a/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py b/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py index b865c719b2111..41ce94513560d 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/test_message_grouper.py @@ -23,6 +23,7 @@ StreamDescriptor, ) from airbyte_cdk.models import Type as MessageType +from orjson import orjson from unit_tests.connector_builder.utils import create_configured_catalog _NO_PK = [[]] @@ -147,7 +148,10 @@ def test_get_grouped_messages(mock_entrypoint_read: Mock) -> None: connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) actual_response: StreamRead = connector_builder_handler.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) assert actual_response.inferred_schema == expected_schema @@ -212,7 +216,10 @@ def test_get_grouped_messages_with_logs(mock_entrypoint_read: Mock) -> None: connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) actual_response: StreamRead = connector_builder_handler.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) single_slice = actual_response.slices[0] for i, actual_page in enumerate(single_slice.pages): @@ -230,7 +237,9 @@ def test_get_grouped_messages_with_logs(mock_entrypoint_read: Mock) -> None: ], ) @patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") -def test_get_grouped_messages_record_limit(mock_entrypoint_read: Mock, request_record_limit: int, max_record_limit: int, should_fail: bool) -> None: +def test_get_grouped_messages_record_limit( + mock_entrypoint_read: Mock, request_record_limit: int, max_record_limit: int, should_fail: bool +) -> None: url = "https://demonslayers.com/api/v1/hashiras?era=taisho" request = { "headers": {"Content-Type": "application/json"}, @@ -258,11 +267,19 @@ def test_get_grouped_messages_record_limit(mock_entrypoint_read: Mock, request_r if should_fail: with pytest.raises(ValueError): api.get_message_groups( - mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, record_limit=request_record_limit + mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, + record_limit=request_record_limit, ) else: actual_response: StreamRead = api.get_message_groups( - mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, record_limit=request_record_limit + mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, + record_limit=request_record_limit, ) single_slice = actual_response.slices[0] total_records = 0 @@ -338,7 +355,9 @@ def test_get_grouped_messages_limit_0(mock_entrypoint_read: Mock) -> None: api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) with pytest.raises(ValueError): - api.get_message_groups(source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, record_limit=0) + api.get_message_groups( + source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, record_limit=0 + ) @patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") @@ -386,7 +405,10 @@ def test_get_grouped_messages_no_records(mock_entrypoint_read: Mock) -> None: message_grouper = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) actual_response: StreamRead = message_grouper.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) single_slice = actual_response.slices[0] @@ -484,7 +506,10 @@ def test_get_grouped_messages_with_many_slices(mock_entrypoint_read: Mock) -> No connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) assert not stream_read.test_read_limit_reached @@ -501,7 +526,10 @@ def test_get_grouped_messages_with_many_slices(mock_entrypoint_read: Mock) -> No assert len(stream_read.slices[1].pages[1].records) == 1 assert len(stream_read.slices[1].pages[2].records) == 0 - assert stream_read.slices[1].state[0].stream.stream_state == AirbyteStateBlob(a_timestamp=123) + assert ( + orjson.dumps(stream_read.slices[1].state[0].stream.stream_state).decode() + == orjson.dumps(AirbyteStateBlob(a_timestamp=123)).decode() + ) @patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") @@ -516,7 +544,10 @@ def test_get_grouped_messages_given_maximum_number_of_slices_then_test_read_limi api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = api.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) assert stream_read.test_read_limit_reached @@ -535,7 +566,10 @@ def test_get_grouped_messages_given_maximum_number_of_pages_then_test_read_limit api = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = api.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) assert stream_read.test_read_limit_reached @@ -550,7 +584,10 @@ def test_read_stream_returns_error_if_stream_does_not_exist() -> None: message_grouper = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) actual_response = message_grouper.get_message_groups( - source=mock_source, config=full_config, configured_catalog=create_configured_catalog("not_in_manifest"), state=_NO_STATE, + source=mock_source, + config=full_config, + configured_catalog=create_configured_catalog("not_in_manifest"), + state=_NO_STATE, ) assert len(actual_response.logs) == 1 @@ -566,7 +603,10 @@ def test_given_control_message_then_stream_read_has_config_update(mock_entrypoin ) connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) assert stream_read.latest_config_update == updated_config @@ -591,7 +631,10 @@ def test_given_multiple_control_messages_then_stream_read_has_latest_based_on_em ) connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) assert stream_read.latest_config_update == latest_config @@ -616,7 +659,10 @@ def test_given_multiple_control_messages_with_same_timestamp_then_stream_read_ha ) connector_builder_handler = MessageGrouper(MAX_PAGES_PER_SLICE, MAX_SLICES) stream_read: StreamRead = connector_builder_handler.get_message_groups( - source=mock_source, config=CONFIG, configured_catalog=create_configured_catalog("hashiras"), state=_NO_STATE, + source=mock_source, + config=CONFIG, + configured_catalog=create_configured_catalog("hashiras"), + state=_NO_STATE, ) assert stream_read.latest_config_update == latest_config @@ -646,11 +692,16 @@ def test_given_no_slices_then_return_empty_slices(mock_entrypoint_read: Mock) -> @patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_given_pk_then_ensure_pk_is_pass_to_schema_inferrence(mock_entrypoint_read: Mock) -> None: - mock_source = make_mock_source(mock_entrypoint_read, iter([ - request_response_log_message({"request": 1}, {"response": 2}, "http://any_url.com"), - record_message("hashiras", {"id": "Shinobu Kocho", "date": "2023-03-03"}), - record_message("hashiras", {"id": "Muichiro Tokito", "date": "2023-03-04"}), - ])) + mock_source = make_mock_source( + mock_entrypoint_read, + iter( + [ + request_response_log_message({"request": 1}, {"response": 2}, "http://any_url.com"), + record_message("hashiras", {"id": "Shinobu Kocho", "date": "2023-03-03"}), + record_message("hashiras", {"id": "Muichiro Tokito", "date": "2023-03-04"}), + ] + ), + ) mock_source.streams.return_value = [Mock()] mock_source.streams.return_value[0].primary_key = [["id"]] mock_source.streams.return_value[0].cursor_field = _NO_CURSOR_FIELD @@ -665,11 +716,16 @@ def test_given_pk_then_ensure_pk_is_pass_to_schema_inferrence(mock_entrypoint_re @patch("airbyte_cdk.connector_builder.message_grouper.AirbyteEntrypoint.read") def test_given_cursor_field_then_ensure_cursor_field_is_pass_to_schema_inferrence(mock_entrypoint_read: Mock) -> None: - mock_source = make_mock_source(mock_entrypoint_read, iter([ - request_response_log_message({"request": 1}, {"response": 2}, "http://any_url.com"), - record_message("hashiras", {"id": "Shinobu Kocho", "date": "2023-03-03"}), - record_message("hashiras", {"id": "Muichiro Tokito", "date": "2023-03-04"}), - ])) + mock_source = make_mock_source( + mock_entrypoint_read, + iter( + [ + request_response_log_message({"request": 1}, {"response": 2}, "http://any_url.com"), + record_message("hashiras", {"id": "Shinobu Kocho", "date": "2023-03-03"}), + record_message("hashiras", {"id": "Muichiro Tokito", "date": "2023-03-04"}), + ] + ), + ) mock_source.streams.return_value = [Mock()] mock_source.streams.return_value[0].primary_key = _NO_PK mock_source.streams.return_value[0].cursor_field = ["date"] @@ -709,10 +765,10 @@ def record_message(stream: str, data: Mapping[str, Any]) -> AirbyteMessage: def state_message(stream: str, data: Mapping[str, Any]) -> AirbyteMessage: - return AirbyteMessage(type=MessageType.STATE, state=AirbyteStateMessage(stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name=stream), - stream_state=data - ))) + return AirbyteMessage( + type=MessageType.STATE, + state=AirbyteStateMessage(stream=AirbyteStreamState(stream_descriptor=StreamDescriptor(name=stream), stream_state=data)), + ) def slice_message(slice_descriptor: str = '{"key": "value"}') -> AirbyteMessage: diff --git a/airbyte-cdk/python/unit_tests/connector_builder/utils.py b/airbyte-cdk/python/unit_tests/connector_builder/utils.py index 15abdd30b9d92..a94a0416437c7 100644 --- a/airbyte-cdk/python/unit_tests/connector_builder/utils.py +++ b/airbyte-cdk/python/unit_tests/connector_builder/utils.py @@ -4,7 +4,7 @@ from typing import Any, Mapping -from airbyte_cdk.models.airbyte_protocol import ConfiguredAirbyteCatalog +from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteCatalogSerializer def create_configured_catalog_dict(stream_name: str) -> Mapping[str, Any]: @@ -24,4 +24,4 @@ def create_configured_catalog_dict(stream_name: str) -> Mapping[str, Any]: def create_configured_catalog(stream_name: str) -> ConfiguredAirbyteCatalog: - return ConfiguredAirbyteCatalog.parse_obj(create_configured_catalog_dict(stream_name)) + return ConfiguredAirbyteCatalogSerializer.load(create_configured_catalog_dict(stream_name)) diff --git a/airbyte-cdk/python/unit_tests/destinations/test_destination.py b/airbyte-cdk/python/unit_tests/destinations/test_destination.py index 89d16453d5300..a03d7ffcc6b0f 100644 --- a/airbyte-cdk/python/unit_tests/destinations/test_destination.py +++ b/airbyte-cdk/python/unit_tests/destinations/test_destination.py @@ -16,10 +16,12 @@ AirbyteCatalog, AirbyteConnectionStatus, AirbyteMessage, + AirbyteMessageSerializer, AirbyteRecordMessage, AirbyteStateMessage, AirbyteStream, ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, ConfiguredAirbyteStream, ConnectorSpecification, DestinationSyncMode, @@ -27,6 +29,7 @@ SyncMode, Type, ) +from orjson import orjson @pytest.fixture(name="destination") @@ -194,7 +197,7 @@ def test_run_check_with_invalid_config(self, mocker, destination: Destination, t parsed_args = argparse.Namespace(**args) destination.run_cmd(parsed_args) - spec = {'type': 'integer'} + spec = {"type": "integer"} spec_msg = ConnectorSpecification(connectionSpecification=spec) mocker.patch.object(destination, "spec", return_value=spec_msg) @@ -214,7 +217,7 @@ def test_run_check_with_invalid_config(self, mocker, destination: Destination, t assert returned_check_result.type == Type.CONNECTION_STATUS assert returned_check_result.connectionStatus.status == Status.FAILED # the specific phrasing is not relevant, so only check for the keywords - assert 'validation error' in returned_check_result.connectionStatus.message + assert "validation error" in returned_check_result.connectionStatus.message def test_run_write(self, mocker, destination: Destination, tmp_path, monkeypatch): config_path, dummy_config = tmp_path / "config.json", {"user": "sherif"} @@ -230,7 +233,7 @@ def test_run_write(self, mocker, destination: Destination, tmp_path, monkeypatch ] ) catalog_path = tmp_path / "catalog.json" - write_file(catalog_path, dummy_catalog.json(exclude_unset=True)) + write_file(catalog_path, ConfiguredAirbyteCatalogSerializer.dump(dummy_catalog)) args = {"command": "write", "config": config_path, "catalog": catalog_path} parsed_args = argparse.Namespace(**args) @@ -244,7 +247,7 @@ def test_run_write(self, mocker, destination: Destination, tmp_path, monkeypatch validate_mock = mocker.patch("airbyte_cdk.destinations.destination.check_config_against_spec_or_exit") # mock input is a record followed by some state messages mocked_input: List[AirbyteMessage] = [_wrapped(_record("s1", {"k1": "v1"})), *expected_write_result] - mocked_stdin_string = "\n".join([record.json(exclude_unset=True) for record in mocked_input]) + mocked_stdin_string = "\n".join([orjson.dumps(AirbyteMessageSerializer.dump(record)).decode() for record in mocked_input]) mocked_stdin_string += "\n add this non-serializable string to verify the destination does not break on malformed input" mocked_stdin = io.TextIOWrapper(io.BytesIO(bytes(mocked_stdin_string, "utf-8"))) diff --git a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py index 41da649163682..db3ce730c89e9 100644 --- a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py +++ b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/document_processor_test.py @@ -14,8 +14,14 @@ SeparatorSplitterConfigModel, ) from airbyte_cdk.destinations.vector_db_based.document_processor import DocumentProcessor -from airbyte_cdk.models import AirbyteStream, ConfiguredAirbyteCatalog, ConfiguredAirbyteStream -from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage, DestinationSyncMode, SyncMode +from airbyte_cdk.models import ( + AirbyteRecordMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + SyncMode, +) from airbyte_cdk.utils.traced_exception import AirbyteTracedException diff --git a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/embedder_test.py b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/embedder_test.py index a5f22b752ed26..600a4c0890d3b 100644 --- a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/embedder_test.py +++ b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/embedder_test.py @@ -24,7 +24,7 @@ OpenAICompatibleEmbedder, OpenAIEmbedder, ) -from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage +from airbyte_cdk.models import AirbyteRecordMessage from airbyte_cdk.utils.traced_exception import AirbyteTracedException diff --git a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/writer_test.py b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/writer_test.py index c906d0f3e9b57..ac831694c7261 100644 --- a/airbyte-cdk/python/unit_tests/destinations/vector_db_based/writer_test.py +++ b/airbyte-cdk/python/unit_tests/destinations/vector_db_based/writer_test.py @@ -7,12 +7,13 @@ import pytest from airbyte_cdk.destinations.vector_db_based import ProcessingConfigModel, Writer -from airbyte_cdk.models.airbyte_protocol import ( +from airbyte_cdk.models import ( AirbyteLogMessage, AirbyteMessage, AirbyteRecordMessage, AirbyteStateMessage, ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, Level, Type, ) @@ -61,7 +62,7 @@ def test_write(omit_raw_text: bool): """ config_model = ProcessingConfigModel(chunk_overlap=0, chunk_size=1000, metadata_fields=None, text_fields=["column_name"]) - configured_catalog: ConfiguredAirbyteCatalog = ConfiguredAirbyteCatalog.parse_obj({"streams": [generate_stream()]}) + configured_catalog: ConfiguredAirbyteCatalog = ConfiguredAirbyteCatalogSerializer.load({"streams": [generate_stream()]}) # messages are flushed after 32 records or after a state message, so this will trigger two batches to be processed input_messages = [_generate_record_message(i) for i in range(BATCH_SIZE + 5)] state_message = AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage()) @@ -126,7 +127,7 @@ def test_write_stream_namespace_split(): """ config_model = ProcessingConfigModel(chunk_overlap=0, chunk_size=1000, metadata_fields=None, text_fields=["column_name"]) - configured_catalog: ConfiguredAirbyteCatalog = ConfiguredAirbyteCatalog.parse_obj( + configured_catalog: ConfiguredAirbyteCatalog = ConfiguredAirbyteCatalogSerializer.load( { "streams": [ generate_stream(), diff --git a/airbyte-cdk/python/unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py b/airbyte-cdk/python/unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py index f6ff8684fa941..1c7315cb6969b 100644 --- a/airbyte-cdk/python/unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py +++ b/airbyte-cdk/python/unit_tests/sources/concurrent_source/test_concurrent_source_adapter.py @@ -43,7 +43,15 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> def streams(self, config: Mapping[str, Any]) -> List[Stream]: return [ - StreamFacade.create_from_stream(s, self, self._logger, None, FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=InMemoryMessageRepository())) if is_concurrent else s + StreamFacade.create_from_stream( + s, + self, + self._logger, + None, + FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=InMemoryMessageRepository()), + ) + if is_concurrent + else s for s, is_concurrent in self._streams_to_is_concurrent.items() ] @@ -96,7 +104,13 @@ def test_concurrent_source_adapter(as_stream_status, remove_stack_trace): assert records == expected_records - unavailable_stream_trace_messages = [m for m in messages if m.type == MessageType.TRACE and m.trace.type == TraceType.STREAM_STATUS and m.trace.stream_status.status == AirbyteStreamStatus.INCOMPLETE] + unavailable_stream_trace_messages = [ + m + for m in messages + if m.type == MessageType.TRACE + and m.trace.type == TraceType.STREAM_STATUS + and m.trace.stream_status.status == AirbyteStreamStatus.INCOMPLETE + ] expected_status = [as_stream_status("s3", AirbyteStreamStatus.INCOMPLETE)] assert len(unavailable_stream_trace_messages) == 1 @@ -133,7 +147,9 @@ def _configured_catalog(streams: List[Stream]): @pytest.mark.parametrize("raise_exception_on_missing_stream", [True, False]) -def test_read_nonexistent_concurrent_stream_emit_incomplete_stream_status(mocker, remove_stack_trace, as_stream_status, raise_exception_on_missing_stream): +def test_read_nonexistent_concurrent_stream_emit_incomplete_stream_status( + mocker, remove_stack_trace, as_stream_status, raise_exception_on_missing_stream +): """ Tests that attempting to sync a stream which the source does not return from the `streams` method emits incomplete stream status. """ diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_jwt.py b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_jwt.py index b625ddd5b3577..51bef48230c95 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_jwt.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/auth/test_jwt.py @@ -19,11 +19,18 @@ class TestJwtAuthenticator: """ @pytest.mark.parametrize( - "algorithm, kid, typ, cty, additional_jwt_headers, expected", - [ - ("ALGORITHM", "test_kid", "test_typ", "test_cty", {"test": "test"}, {"kid": "test_kid", "typ": "test_typ", "cty": "test_cty", "test": "test", "alg": "ALGORITHM"}), - ("ALGORITHM", None, None, None, None, {"alg": "ALGORITHM"}) - ] + "algorithm, kid, typ, cty, additional_jwt_headers, expected", + [ + ( + "ALGORITHM", + "test_kid", + "test_typ", + "test_cty", + {"test": "test"}, + {"kid": "test_kid", "typ": "test_typ", "cty": "test_cty", "test": "test", "alg": "ALGORITHM"}, + ), + ("ALGORITHM", None, None, None, None, {"alg": "ALGORITHM"}), + ], ) def test_get_jwt_headers(self, algorithm, kid, typ, cty, additional_jwt_headers, expected): authenticator = JwtAuthenticator( @@ -61,14 +68,8 @@ def test_given_overriden_reserverd_properties_get_jwt_headers_throws_error(self) {"test": "test"}, {"iss": "test_iss", "sub": "test_sub", "aud": "test_aud", "test": "test"}, ), - ( - None, - None, - None, - None, - {} - ), - ] + (None, None, None, None, {}), + ], ) def test_get_jwt_payload(self, iss, sub, aud, additional_jwt_payload, expected): authenticator = JwtAuthenticator( @@ -105,7 +106,7 @@ def test_given_overriden_reserverd_properties_get_jwt_payload_throws_error(self) [ (True, "test", base64.b64encode("test".encode()).decode()), (False, "test", "test"), - ] + ], ) def test_get_secret_key(self, base64_encode_secret_key, secret_key, expected): authenticator = JwtAuthenticator( @@ -152,13 +153,7 @@ def test_given_invalid_algorithm_get_signed_token_throws_error(self): with pytest.raises(ValueError): authenticator._get_signed_token() - @pytest.mark.parametrize( - "header_prefix, expected", - [ - ("test", "test"), - (None, None) - ] - ) + @pytest.mark.parametrize("header_prefix, expected", [("test", "test"), (None, None)]) def test_get_header_prefix(self, header_prefix, expected): authenticator = JwtAuthenticator( config={}, diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py index 8ccf70b4e7a96..4ebe449dcd691 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/checks/test_check_stream.py @@ -92,9 +92,7 @@ def test_check_stream_with_no_stream_slices_aborts(): "test_stream_unavailable_handled_error", 403, False, - [ - "Forbidden. You don't have permission to access this resource." - ], + ["Forbidden. You don't have permission to access this resource."], ), ("test_stream_available", 200, True, []), ], diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py index 0a25be1129c1c..1a7d45f7a78f7 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_datetime_parser.py @@ -55,7 +55,12 @@ def test_parse_date(test_name, input_date, date_format, expected_output_date): [ ("test_format_timestamp", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%s", "1609459200"), ("test_format_timestamp_ms", datetime.datetime(2021, 1, 1, 0, 0, 0, 1000, tzinfo=datetime.timezone.utc), "%ms", "1609459200001"), - ("test_format_timestamp_as_float", datetime.datetime(2023, 1, 30, 15, 28, 28, 873709, tzinfo=datetime.timezone.utc), "%s_as_float", "1675092508.873709"), + ( + "test_format_timestamp_as_float", + datetime.datetime(2023, 1, 30, 15, 28, 28, 873709, tzinfo=datetime.timezone.utc), + "%s_as_float", + "1675092508.873709", + ), ("test_format_string", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y-%m-%d", "2021-01-01"), ("test_format_to_number", datetime.datetime(2021, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), "%Y%m%d", "20210101"), ], diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py index 84a63969cec64..ff9aedf0752ae 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/datetime/test_min_max_datetime.py @@ -114,11 +114,12 @@ def test_min_max_datetime_lazy_eval(): @pytest.mark.parametrize( - "input_datetime", [ + "input_datetime", + [ pytest.param("2022-01-01T00:00:00", id="test_create_min_max_datetime_from_string"), pytest.param(InterpolatedString.create("2022-01-01T00:00:00", parameters={}), id="test_create_min_max_datetime_from_string"), - pytest.param(MinMaxDatetime("2022-01-01T00:00:00", parameters={}), id="test_create_min_max_datetime_from_minmaxdatetime") - ] + pytest.param(MinMaxDatetime("2022-01-01T00:00:00", parameters={}), id="test_create_min_max_datetime_from_minmaxdatetime"), + ], ) def test_create_min_max_datetime(input_datetime): minMaxDatetime = MinMaxDatetime.create(input_datetime, parameters={}) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/decoders/test_json_decoder.py b/airbyte-cdk/python/unit_tests/sources/declarative/decoders/test_json_decoder.py index 52bc55201bbed..65ed78698ca63 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/decoders/test_json_decoder.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/decoders/test_json_decoder.py @@ -7,19 +7,15 @@ import pytest import requests from airbyte_cdk import YamlDeclarativeSource +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder, JsonlDecoder from airbyte_cdk.sources.declarative.models import DeclarativeStream as DeclarativeStreamModel from airbyte_cdk.sources.declarative.parsers.model_to_component_factory import ModelToComponentFactory -from airbyte_protocol.models import SyncMode @pytest.mark.parametrize( "response_body, first_element", - [ - ("", {}), - ("[]", {}), - ('{"healthcheck": {"status": "ok"}}', {"healthcheck": {"status": "ok"}}) - ], + [("", {}), ("[]", {}), ('{"healthcheck": {"status": "ok"}}', {"healthcheck": {"status": "ok"}})], ) def test_json_decoder(requests_mock, response_body, first_element): requests_mock.register_uri("GET", "https://airbyte.io/", text=response_body) @@ -45,13 +41,13 @@ def test_jsonl_decoder(requests_mock, response_body, expected_json): @pytest.fixture(name="large_events_response") def large_event_response_fixture(): data = {"email": "email1@example.com"} - json_string = json.dumps(data) - lines_in_response = 5_000_000 + jsonl_string = f"{json.dumps(data)}\n" + lines_in_response = 2_000_000 # ≈ 58 MB of response dir_path = os.path.dirname(os.path.realpath(__file__)) file_path = f"{dir_path}/test_response.txt" with open(file_path, "w") as file: for _ in range(lines_in_response): - file.write(json_string + "\n") + file.write(jsonl_string) yield (lines_in_response, file_path) os.remove(file_path) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py index 24fb662d726ac..92b4ffbb4804f 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/extractors/test_dpath_extractor.py @@ -56,12 +56,15 @@ def create_response(body: Union[Dict, bytes]): ["data"], decoder_jsonl, b'{"data": [{"id": 1, "text_field": "This is a text\\n. New paragraph start here."}]}\n{"data": [{"id": 2, "text_field": "This is another text\\n. New paragraph start here."}]}', - [{"id": 1, "text_field": "This is a text\n. New paragraph start here."}, {"id": 2, "text_field": "This is another text\n. New paragraph start here."}], + [ + {"id": 1, "text_field": "This is a text\n. New paragraph start here."}, + {"id": 2, "text_field": "This is another text\n. New paragraph start here."}, + ], ), ( [], decoder_iterable, - b'user1@example.com\nuser2@example.com', + b"user1@example.com\nuser2@example.com", [{"record": "user1@example.com"}, {"record": "user2@example.com"}], ), ], diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py index 8132e4b603490..33bd6786c1525 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_datetime_based_cursor.py @@ -333,7 +333,7 @@ def mock_datetime_now(monkeypatch): [ {"start_time": "2021-01-01T00:00:00.000000+0000", "end_time": "2021-01-31T23:59:59.999999+0000"}, ], - ) + ), ], ) def test_stream_slices( @@ -580,10 +580,11 @@ def test_request_option(test_name, inject_into, field_name, expected_req_params, @pytest.mark.parametrize( - "stream_slice", [ + "stream_slice", + [ pytest.param(None, id="test_none_stream_slice"), pytest.param({}, id="test_none_stream_slice"), - ] + ], ) def test_request_option_with_empty_stream_slice(stream_slice): start_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, parameters={}, field_name="starttime") diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor.py b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor.py index 96eb3c86e52a4..b2c8d5faf46d6 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor.py @@ -6,12 +6,12 @@ from unittest.mock import Mock import pytest +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.incremental.declarative_cursor import DeclarativeCursor from airbyte_cdk.sources.declarative.incremental.per_partition_cursor import PerPartitionCursor, PerPartitionKeySerializer, StreamSlice from airbyte_cdk.sources.declarative.partition_routers.partition_router import PartitionRouter from airbyte_cdk.sources.types import Record from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import FailureType PARTITION = { "partition_key string": "partition value", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py index 1b3550a99861c..4d2141b423730 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_per_partition_cursor_integration.py @@ -37,9 +37,8 @@ def with_substream_partition_router(self, stream_name): "stream": "#/definitions/Rates", "parent_key": "id", "partition_field": "parent_id", - } - ] + ], } return self @@ -100,10 +99,7 @@ def build(self): }, }, }, - "streams": [ - {"$ref": "#/definitions/Rates"}, - {"$ref": "#/definitions/AnotherStream"} - ], + "streams": [{"$ref": "#/definitions/Rates"}, {"$ref": "#/definitions/AnotherStream"}], "spec": { "connection_specification": { "$schema": "http://json-schema.org/draft-07/schema#", @@ -180,11 +176,9 @@ def test_given_record_for_partition_when_read_then_update_state(): stream_instance = source.streams({})[0] list(stream_instance.stream_slices(sync_mode=SYNC_MODE)) - stream_slice = StreamSlice(partition={"partition_field": "1"}, - cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}) + stream_slice = StreamSlice(partition={"partition_field": "1"}, cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}) with patch.object( - SimpleRetriever, "_read_pages", - side_effect=[[Record({"a record key": "a record value", CURSOR_FIELD: "2022-01-15"}, stream_slice)]] + SimpleRetriever, "_read_pages", side_effect=[[Record({"a record key": "a record value", CURSOR_FIELD: "2022-01-15"}, stream_slice)]] ): list( stream_instance.read_records( @@ -236,17 +230,41 @@ def test_substream_without_input_state(): # This mocks the resulting records of the Rates stream which acts as the parent stream of the SubstreamPartitionRouter being tested with patch.object( - SimpleRetriever, "_read_pages", side_effect=[[Record({"id": "1", CURSOR_FIELD: "2022-01-15"}, parent_stream_slice)], - [Record({"id": "2", CURSOR_FIELD: "2022-01-15"}, parent_stream_slice)]] + SimpleRetriever, + "_read_pages", + side_effect=[ + [Record({"id": "1", CURSOR_FIELD: "2022-01-15"}, parent_stream_slice)], + [Record({"id": "2", CURSOR_FIELD: "2022-01-15"}, parent_stream_slice)], + ], ): slices = list(stream_instance.stream_slices(sync_mode=SYNC_MODE)) assert list(slices) == [ - StreamSlice(partition={"parent_id": "1", "parent_slice": {}, }, - cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}), - StreamSlice(partition={"parent_id": "1", "parent_slice": {}, }, - cursor_slice={"start_time": "2022-02-01", "end_time": "2022-02-28"}), - StreamSlice(partition={"parent_id": "2", "parent_slice": {}, }, - cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}), - StreamSlice(partition={"parent_id": "2", "parent_slice": {}, }, - cursor_slice={"start_time": "2022-02-01", "end_time": "2022-02-28"}), + StreamSlice( + partition={ + "parent_id": "1", + "parent_slice": {}, + }, + cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}, + ), + StreamSlice( + partition={ + "parent_id": "1", + "parent_slice": {}, + }, + cursor_slice={"start_time": "2022-02-01", "end_time": "2022-02-28"}, + ), + StreamSlice( + partition={ + "parent_id": "2", + "parent_slice": {}, + }, + cursor_slice={"start_time": "2022-01-01", "end_time": "2022-01-31"}, + ), + StreamSlice( + partition={ + "parent_id": "2", + "parent_slice": {}, + }, + cursor_slice={"start_time": "2022-02-01", "end_time": "2022-02-28"}, + ), ] diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_resumable_full_refresh_cursor.py b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_resumable_full_refresh_cursor.py index bb15e465e8fa4..b45973283aadf 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_resumable_full_refresh_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/incremental/test_resumable_full_refresh_cursor.py @@ -32,7 +32,7 @@ StreamSlice(cursor_slice={}, partition={}), id="test_empty_substream_resumable_full_refresh_stream_state", ), - ] + ], ) def test_stream_slices(stream_state, cursor, expected_slice): cursor = cursor(parameters={}) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/migrations/test_legacy_to_per_partition_migration.py b/airbyte-cdk/python/unit_tests/sources/declarative/migrations/test_legacy_to_per_partition_migration.py index 7fce15031ee19..97e5efd69f977 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/migrations/test_legacy_to_per_partition_migration.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/migrations/test_legacy_to_per_partition_migration.py @@ -23,12 +23,8 @@ def test_migrate_a_valid_legacy_state_to_per_partition(): input_state = { - "13506132": { - "last_changed": "2022-12-27T08:34:39+00:00" - }, - "14351124": { - "last_changed": "2022-12-27T08:35:39+00:00" - }, + "13506132": {"last_changed": "2022-12-27T08:34:39+00:00"}, + "14351124": {"last_changed": "2022-12-27T08:35:39+00:00"}, } migrator = _migrator() @@ -37,14 +33,8 @@ def test_migrate_a_valid_legacy_state_to_per_partition(): expected_state = { "states": [ - { - "partition": {"parent_id": "13506132"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, - { - "partition": {"parent_id": "14351124"}, - "cursor": {"last_changed": "2022-12-27T08:35:39+00:00"} - }, + {"partition": {"parent_id": "13506132"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + {"partition": {"parent_id": "14351124"}, "cursor": {"last_changed": "2022-12-27T08:35:39+00:00"}}, ] } @@ -52,115 +42,88 @@ def test_migrate_a_valid_legacy_state_to_per_partition(): @pytest.mark.parametrize( - "input_state", [ - pytest.param({ - "states": [ - { - "partition": {"id": "13506132"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, - { - "partition": {"id": "14351124"}, - "cursor": {"last_changed": "2022-12-27T08:35:39+00:00"} - }, - ] - }, id="test_should_not_migrate_a_per_partition_state"), - pytest.param({ - "states": [ - { - "partition": {"id": "13506132"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, - { - "partition": {"id": "14351124"}, - }, - ] - }, id="test_should_not_migrate_state_without_a_cursor_component"), - pytest.param({ - "states": [ - { - "partition": {"id": "13506132"}, - "cursor": {"updated_at": "2022-12-27T08:34:39+00:00"} - }, - { - "partition": {"id": "14351124"}, - "cursor": {"updated_at": "2022-12-27T08:35:39+00:00"} - }, - ] - }, id="test_should_not_migrate_a_per_partition_state_with_wrong_cursor_field"), - pytest.param({ - "states": [ - { - "partition": {"id": "13506132"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, - { - "partition": {"id": "14351124"}, - "cursor": {"last_changed": "2022-12-27T08:35:39+00:00", "updated_at": "2021-01-01"} - }, - ] - }, id="test_should_not_migrate_a_per_partition_state_with_multiple_cursor_fields"), + "input_state", + [ pytest.param( { "states": [ - { - "partition": {"id": "13506132"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, - { - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, + {"partition": {"id": "13506132"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + {"partition": {"id": "14351124"}, "cursor": {"last_changed": "2022-12-27T08:35:39+00:00"}}, ] - }, id="test_should_not_migrate_state_without_a_partition_component" + }, + id="test_should_not_migrate_a_per_partition_state", ), pytest.param( { "states": [ + {"partition": {"id": "13506132"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, { - "partition": {"id": "13506132", "another_id": "A"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, - { - "partition": {"id": "13506134"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} + "partition": {"id": "14351124"}, }, ] - }, id="test_should_not_migrate_state_if_multiple_partition_keys" + }, + id="test_should_not_migrate_state_without_a_cursor_component", + ), + pytest.param( + { + "states": [ + {"partition": {"id": "13506132"}, "cursor": {"updated_at": "2022-12-27T08:34:39+00:00"}}, + {"partition": {"id": "14351124"}, "cursor": {"updated_at": "2022-12-27T08:35:39+00:00"}}, + ] + }, + id="test_should_not_migrate_a_per_partition_state_with_wrong_cursor_field", ), pytest.param( { "states": [ - { - "partition": {"identifier": "13506132"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, - { - "partition": {"id": "13506134"}, - "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"} - }, + {"partition": {"id": "13506132"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + {"partition": {"id": "14351124"}, "cursor": {"last_changed": "2022-12-27T08:35:39+00:00", "updated_at": "2021-01-01"}}, + ] + }, + id="test_should_not_migrate_a_per_partition_state_with_multiple_cursor_fields", + ), + pytest.param( + { + "states": [ + {"partition": {"id": "13506132"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + {"cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, ] - }, id="test_should_not_migrate_state_if_invalid_partition_key" + }, + id="test_should_not_migrate_state_without_a_partition_component", ), pytest.param( { - "13506132": { - "last_changed": "2022-12-27T08:34:39+00:00" - }, - "14351124": { - "last_changed": "2022-12-27T08:35:39+00:00", - "another_key": "2022-12-27T08:35:39+00:00" - }, - }, id="test_should_not_migrate_if_the_partitioned_state_has_more_than_one_key" + "states": [ + {"partition": {"id": "13506132", "another_id": "A"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + {"partition": {"id": "13506134"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + ] + }, + id="test_should_not_migrate_state_if_multiple_partition_keys", + ), + pytest.param( + { + "states": [ + {"partition": {"identifier": "13506132"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + {"partition": {"id": "13506134"}, "cursor": {"last_changed": "2022-12-27T08:34:39+00:00"}}, + ] + }, + id="test_should_not_migrate_state_if_invalid_partition_key", ), - pytest.param({ - "13506132": { - "last_changed": "2022-12-27T08:34:39+00:00" + pytest.param( + { + "13506132": {"last_changed": "2022-12-27T08:34:39+00:00"}, + "14351124": {"last_changed": "2022-12-27T08:35:39+00:00", "another_key": "2022-12-27T08:35:39+00:00"}, }, - "14351124": { - "another_key": "2022-12-27T08:35:39+00:00" + id="test_should_not_migrate_if_the_partitioned_state_has_more_than_one_key", + ), + pytest.param( + { + "13506132": {"last_changed": "2022-12-27T08:34:39+00:00"}, + "14351124": {"another_key": "2022-12-27T08:35:39+00:00"}, }, - }, id="test_should_not_migrate_if_the_partitioned_state_key_is_not_the_cursor_field"), - ] + id="test_should_not_migrate_if_the_partitioned_state_key_is_not_the_cursor_field", + ), + ], ) def test_should_not_migrate(input_state): migrator = _migrator() @@ -169,12 +132,8 @@ def test_should_not_migrate(input_state): def test_should_not_migrate_stream_with_multiple_parent_streams(): input_state = { - "13506132": { - "last_changed": "2022-12-27T08:34:39+00:00" - }, - "14351124": { - "last_changed": "2022-12-27T08:35:39+00:00" - }, + "13506132": {"last_changed": "2022-12-27T08:34:39+00:00"}, + "14351124": {"last_changed": "2022-12-27T08:35:39+00:00"}, } migrator = _migrator_with_multiple_parent_streams() @@ -191,14 +150,10 @@ def _migrator(): parent_key="{{ parameters['parent_key_id'] }}", partition_field="parent_id", stream=DeclarativeStream( - type="DeclarativeStream", - retriever=CustomRetriever( - type="CustomRetriever", - class_name="a_class_name" - ) - ) + type="DeclarativeStream", retriever=CustomRetriever(type="CustomRetriever", class_name="a_class_name") + ), ) - ] + ], ) cursor = DatetimeBasedCursor( type="DatetimeBasedCursor", @@ -220,26 +175,18 @@ def _migrator_with_multiple_parent_streams(): parent_key="id", partition_field="parent_id", stream=DeclarativeStream( - type="DeclarativeStream", - retriever=CustomRetriever( - type="CustomRetriever", - class_name="a_class_name" - ) - ) + type="DeclarativeStream", retriever=CustomRetriever(type="CustomRetriever", class_name="a_class_name") + ), ), ParentStreamConfig( type="ParentStreamConfig", parent_key="id", partition_field="parent_id", stream=DeclarativeStream( - type="DeclarativeStream", - retriever=CustomRetriever( - type="CustomRetriever", - class_name="a_class_name" - ) - ) + type="DeclarativeStream", retriever=CustomRetriever(type="CustomRetriever", class_name="a_class_name") + ), ), - ] + ], ) cursor = DatetimeBasedCursor( type="DatetimeBasedCursor", @@ -256,10 +203,28 @@ def _migrator_with_multiple_parent_streams(): "retriever_type, partition_router_class, is_parent_stream_config, expected_exception, expected_error_message", [ (SimpleRetriever, CustomPartitionRouter, True, None, None), - (None, CustomPartitionRouter, True, ValueError, "LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever. Got "), - (SimpleRetriever, None, False, ValueError, "LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a Substream partition router. Got "), - (SimpleRetriever, CustomPartitionRouter, False, ValueError, "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration."), - ] + ( + None, + CustomPartitionRouter, + True, + ValueError, + "LegacyToPerPartitionStateMigrations can only be applied on a DeclarativeStream with a SimpleRetriever. Got ", + ), + ( + SimpleRetriever, + None, + False, + ValueError, + "LegacyToPerPartitionStateMigrations can only be applied on a SimpleRetriever with a Substream partition router. Got ", + ), + ( + SimpleRetriever, + CustomPartitionRouter, + False, + ValueError, + "LegacyToPerPartitionStateMigrations can only be applied with a parent stream configuration.", + ), + ], ) def test_create_legacy_to_per_partition_state_migration( retriever_type, @@ -283,13 +248,30 @@ def test_create_legacy_to_per_partition_state_migration( state_migrations_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["state_migrations"][0], {}) if is_parent_stream_config: - parent_stream_config = ParentStreamConfig(type="ParentStreamConfig", parent_key="id", partition_field="parent_id", stream=DeclarativeStream(type="DeclarativeStream", retriever=CustomRetriever(type="CustomRetriever", class_name="a_class_name"))) + parent_stream_config = ParentStreamConfig( + type="ParentStreamConfig", + parent_key="id", + partition_field="parent_id", + stream=DeclarativeStream( + type="DeclarativeStream", retriever=CustomRetriever(type="CustomRetriever", class_name="a_class_name") + ), + ) partition_router.parent_stream_configs = [parent_stream_config] if expected_exception: with pytest.raises(expected_exception) as excinfo: - factory.create_component(model_type=LegacyToPerPartitionStateMigrationModel, component_definition=state_migrations_manifest, config={}, declarative_stream=stream) + factory.create_component( + model_type=LegacyToPerPartitionStateMigrationModel, + component_definition=state_migrations_manifest, + config={}, + declarative_stream=stream, + ) assert str(excinfo.value) == expected_error_message else: - migration_instance = factory.create_component(model_type=LegacyToPerPartitionStateMigrationModel, component_definition=state_migrations_manifest, config={}, declarative_stream=stream) + migration_instance = factory.create_component( + model_type=LegacyToPerPartitionStateMigrationModel, + component_definition=state_migrations_manifest, + config={}, + declarative_stream=stream, + ) assert migration_instance is not None diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index d73527ad6f0f3..d574ed8724e87 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -9,7 +9,7 @@ import freezegun import pytest from airbyte_cdk import AirbyteTracedException -from airbyte_cdk.models import Level +from airbyte_cdk.models import FailureType, Level from airbyte_cdk.sources.declarative.auth import DeclarativeOauth2Authenticator, JwtAuthenticator from airbyte_cdk.sources.declarative.auth.token import ( ApiKeyAuthenticator, @@ -82,7 +82,6 @@ from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction from airbyte_cdk.sources.streams.http.requests_native_auth.oauth import SingleUseRefreshTokenOauth2Authenticator -from airbyte_protocol.models import FailureType from unit_tests.sources.declarative.parsers.testing_components import TestingCustomSubstreamPartitionRouter, TestingSomeComponent factory = ModelToComponentFactory() @@ -1043,10 +1042,7 @@ def test_create_record_selector(test_name, record_selector, expected_runtime_sel selector_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["selector"], {}) selector = factory.create_component( - model_type=RecordSelectorModel, component_definition=selector_manifest, - decoder=None, - transformations=[], - config=input_config + model_type=RecordSelectorModel, component_definition=selector_manifest, decoder=None, transformations=[], config=input_config ) assert isinstance(selector, RecordSelector) @@ -1127,7 +1123,8 @@ def test_create_requester(test_name, error_handler, expected_backoff_strategy_ty selector = factory.create_component( model_type=HttpRequesterModel, - component_definition=requester_manifest, config=input_config, + component_definition=requester_manifest, + config=input_config, name=name, decoder=None, ) @@ -1179,8 +1176,7 @@ def test_create_request_with_legacy_session_authenticator(): requester_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["requester"], {}) selector = factory.create_component( - model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name=name, - decoder=None + model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name=name, decoder=None ) assert isinstance(selector, HttpRequester) @@ -1265,11 +1261,13 @@ def test_given_composite_error_handler_does_not_match_response_then_fallback_on_ resolved_manifest = resolver.preprocess_manifest(parsed_manifest) requester_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["requester"], {}) http_requester = factory.create_component( - model_type=HttpRequesterModel, component_definition=requester_manifest, config=input_config, name="any name", decoder=JsonDecoder(parameters={}) - ) - requests_mock.get( - "https://api.sendgrid.com/v3/marketing/lists", status_code=401 + model_type=HttpRequesterModel, + component_definition=requester_manifest, + config=input_config, + name="any name", + decoder=JsonDecoder(parameters={}), ) + requests_mock.get("https://api.sendgrid.com/v3/marketing/lists", status_code=401) with pytest.raises(AirbyteTracedException) as exception: http_requester.send_request() @@ -1453,8 +1451,11 @@ def test_create_default_paginator(): paginator_manifest = transformer.propagate_types_and_parameters("", resolved_manifest["paginator"], {}) paginator = factory.create_component( - model_type=DefaultPaginatorModel, component_definition=paginator_manifest, config=input_config, url_base="https://airbyte.io", - decoder=JsonDecoder(parameters={}) + model_type=DefaultPaginatorModel, + component_definition=paginator_manifest, + config=input_config, + url_base="https://airbyte.io", + decoder=JsonDecoder(parameters={}), ) assert isinstance(paginator, DefaultPaginator) @@ -1481,7 +1482,12 @@ def test_create_default_paginator(): "subcomponent_field_with_hint": {"type": "DpathExtractor", "field_path": [], "decoder": {"type": "JsonDecoder"}}, }, "subcomponent_field_with_hint", - DpathExtractor(field_path=[], config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}, decoder=JsonDecoder(parameters={}), parameters={}), + DpathExtractor( + field_path=[], + config={"apikey": "verysecrettoken", "repos": ["airbyte", "airbyte-cloud"]}, + decoder=JsonDecoder(parameters={}), + parameters={}, + ), None, id="test_create_custom_component_with_subcomponent_that_must_be_parsed", ), @@ -2118,10 +2124,7 @@ def test_create_page_increment_with_interpolated_page_size(): start_from_page=1, inject_on_first_request=True, ) - config = { - **input_config, - "page_size": 5 - } + config = {**input_config, "page_size": 5} expected_strategy = PageIncrement(page_size=5, start_from_page=1, inject_on_first_request=True, parameters={}, config=config) strategy = factory.create_page_increment(model, config) @@ -2156,7 +2159,7 @@ def test_create_custom_schema_loader(): definition = { "type": "CustomSchemaLoader", - "class_name": "unit_tests.sources.declarative.parsers.test_model_to_component_factory.MyCustomSchemaLoader" + "class_name": "unit_tests.sources.declarative.parsers.test_model_to_component_factory.MyCustomSchemaLoader", } component = factory.create_component(CustomSchemaLoaderModel, definition, {}) assert isinstance(component, MyCustomSchemaLoader) @@ -2181,12 +2184,9 @@ def test_create_custom_schema_loader(): "algorithm": "HS256", "base64_encode_secret_key": False, "token_duration": 1200, - "jwt_headers": { - "typ": "JWT", - "alg": "HS256" - }, - "jwt_payload": {} - } + "jwt_headers": {"typ": "JWT", "alg": "HS256"}, + "jwt_payload": {}, + }, ), ( { @@ -2228,7 +2228,6 @@ def test_create_custom_schema_loader(): "alg": "RS256", "cty": "JWT", "test": "test custom header", - }, "jwt_payload": { "iss": "test iss", @@ -2236,7 +2235,7 @@ def test_create_custom_schema_loader(): "aud": "test aud", "test": "test custom payload", }, - } + }, ), ( { @@ -2261,12 +2260,11 @@ def test_create_custom_schema_loader(): "typ": "JWT", "alg": "HS256", "custom_header": "custom header value", - }, "jwt_payload": { "custom_payload": "custom payload value", }, - } + }, ), ( { @@ -2280,7 +2278,7 @@ def test_create_custom_schema_loader(): """, { "expect_error": True, - } + }, ), ], ) @@ -2297,9 +2295,7 @@ def test_create_jwt_authenticator(config, manifest, expected): ) return - authenticator = factory.create_component( - model_type=JwtAuthenticatorModel, component_definition=authenticator_manifest, config=config - ) + authenticator = factory.create_component(model_type=JwtAuthenticatorModel, component_definition=authenticator_manifest, config=config) assert isinstance(authenticator, JwtAuthenticator) assert authenticator._secret_key.eval(config) == expected["secret_key"] @@ -2310,9 +2306,11 @@ def test_create_jwt_authenticator(config, manifest, expected): assert authenticator._header_prefix.eval(config) == expected["header_prefix"] assert authenticator._get_jwt_headers() == expected["jwt_headers"] jwt_payload = expected["jwt_payload"] - jwt_payload.update({ - "iat": int(datetime.datetime.now().timestamp()), - "nbf": int(datetime.datetime.now().timestamp()), - "exp": int(datetime.datetime.now().timestamp()) + expected["token_duration"] - }) + jwt_payload.update( + { + "iat": int(datetime.datetime.now().timestamp()), + "nbf": int(datetime.datetime.now().timestamp()), + "exp": int(datetime.datetime.now().timestamp()) + expected["token_duration"], + } + ) assert authenticator._get_jwt_payload() == jwt_payload diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_cartesian_product_partition_router.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_cartesian_product_partition_router.py index 3ec2537e0072f..2b9313b3ebd7e 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_cartesian_product_partition_router.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_cartesian_product_partition_router.py @@ -17,9 +17,11 @@ ( "test_single_stream_slicer", [ListPartitionRouter(values=["customer", "store", "subscription"], cursor_field="owner_resource", config={}, parameters={})], - [StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={})], + [ + StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={}), + ], ), ( "test_two_stream_slicers", @@ -37,24 +39,24 @@ ], ), ( - "test_singledatetime", - [ - DatetimeBasedCursor( - start_datetime=MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d", parameters={}), - end_datetime=MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d", parameters={}), - step="P1D", - cursor_field=InterpolatedString.create("", parameters={}), - datetime_format="%Y-%m-%d", - cursor_granularity="P1D", - config={}, - parameters={}, - ), - ], - [ - StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-01", "end_time": "2021-01-01"}), - StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-02", "end_time": "2021-01-02"}), - StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-03", "end_time": "2021-01-03"}), - ], + "test_singledatetime", + [ + DatetimeBasedCursor( + start_datetime=MinMaxDatetime(datetime="2021-01-01", datetime_format="%Y-%m-%d", parameters={}), + end_datetime=MinMaxDatetime(datetime="2021-01-03", datetime_format="%Y-%m-%d", parameters={}), + step="P1D", + cursor_field=InterpolatedString.create("", parameters={}), + datetime_format="%Y-%m-%d", + cursor_granularity="P1D", + config={}, + parameters={}, + ), + ], + [ + StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-01", "end_time": "2021-01-01"}), + StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-02", "end_time": "2021-01-02"}), + StreamSlice(partition={}, cursor_slice={"start_time": "2021-01-03", "end_time": "2021-01-03"}), + ], ), ( "test_list_and_datetime", @@ -78,9 +80,15 @@ StreamSlice(partition={"owner_resource": "store"}, cursor_slice={"start_time": "2021-01-01", "end_time": "2021-01-01"}), StreamSlice(partition={"owner_resource": "store"}, cursor_slice={"start_time": "2021-01-02", "end_time": "2021-01-02"}), StreamSlice(partition={"owner_resource": "store"}, cursor_slice={"start_time": "2021-01-03", "end_time": "2021-01-03"}), - StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={"start_time": "2021-01-01", "end_time": "2021-01-01"}), - StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={"start_time": "2021-01-02", "end_time": "2021-01-02"}), - StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={"start_time": "2021-01-03", "end_time": "2021-01-03"}), + StreamSlice( + partition={"owner_resource": "subscription"}, cursor_slice={"start_time": "2021-01-01", "end_time": "2021-01-01"} + ), + StreamSlice( + partition={"owner_resource": "subscription"}, cursor_slice={"start_time": "2021-01-02", "end_time": "2021-01-02"} + ), + StreamSlice( + partition={"owner_resource": "subscription"}, cursor_slice={"start_time": "2021-01-03", "end_time": "2021-01-03"} + ), ], ), ], diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py index 387579783e4df..87aa18f5a0b4a 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_list_partition_router.py @@ -18,23 +18,29 @@ ( ["customer", "store", "subscription"], "owner_resource", - [StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={})], + [ + StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={}), + ], ), ( '["customer", "store", "subscription"]', "owner_resource", - [StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={})], + [ + StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={}), + ], ), ( '["customer", "store", "subscription"]', "{{ parameters['cursor_field'] }}", - [StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), - StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={})], + [ + StreamSlice(partition={"owner_resource": "customer"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "store"}, cursor_slice={}), + StreamSlice(partition={"owner_resource": "subscription"}, cursor_slice={}), + ], ), ], ids=[ @@ -106,8 +112,8 @@ def test_request_option(request_option, expected_req_params, expected_headers, e [ pytest.param({}, id="test_request_option_is_empty_if_empty_stream_slice"), pytest.param({"not the cursor": "value"}, id="test_request_option_is_empty_if_the_stream_slice_does_not_have_cursor_field"), - pytest.param(None, id="test_request_option_is_empty_if_no_stream_slice") - ] + pytest.param(None, id="test_request_option_is_empty_if_no_stream_slice"), + ], ) def test_request_option_is_empty_if_no_stream_slice(stream_slice): request_option = RequestOption(inject_into=RequestOptionType.body_data, parameters={}, field_name="owner_resource") diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_parent_state_stream.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_parent_state_stream.py index 773ed96571d05..9ced561742f60 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_parent_state_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_parent_state_stream.py @@ -19,6 +19,7 @@ SyncMode, ) from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource +from orjson import orjson SUBSTREAM_MANIFEST: MutableMapping[str, Any] = { "version": "0.51.42", @@ -349,7 +350,7 @@ def _run_read( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="post_comment_votes", namespace=None), - stream_state=AirbyteStateBlob.parse_obj( + stream_state=AirbyteStateBlob( { "parent_state": { "post_comments": { @@ -424,7 +425,7 @@ def test_incremental_parent_state(test_name, manifest, mock_requests, expected_r output_data = [message.record.data for message in output if message.record] assert output_data == expected_records - final_state = [message.state.stream.stream_state.dict() for message in output if message.state] + final_state = [orjson.loads(orjson.dumps(message.state.stream.stream_state)) for message in output if message.state] assert final_state[-1] == expected_state @@ -467,56 +468,56 @@ def test_incremental_parent_state(test_name, manifest, mock_requests, expected_r ), # Fetch the first page of votes for comment 10 of post 1 ( - "https://api.example.com/community/posts/1/comments/10/votes?per_page=100&start_time=2024-01-02T00:00:00Z", - { - "votes": [{"id": 100, "comment_id": 10, "created_at": "2024-01-15T00:00:00Z"}], - "next_page": "https://api.example.com/community/posts/1/comments/10/votes?per_page=100&page=2&start_time=2024-01-01T00:00:01Z", - }, + "https://api.example.com/community/posts/1/comments/10/votes?per_page=100&start_time=2024-01-02T00:00:00Z", + { + "votes": [{"id": 100, "comment_id": 10, "created_at": "2024-01-15T00:00:00Z"}], + "next_page": "https://api.example.com/community/posts/1/comments/10/votes?per_page=100&page=2&start_time=2024-01-01T00:00:01Z", + }, ), # Fetch the second page of votes for comment 10 of post 1 ( - "https://api.example.com/community/posts/1/comments/10/votes?per_page=100&page=2&start_time=2024-01-01T00:00:01Z", - {"votes": [{"id": 101, "comment_id": 10, "created_at": "2024-01-14T00:00:00Z"}]}, + "https://api.example.com/community/posts/1/comments/10/votes?per_page=100&page=2&start_time=2024-01-01T00:00:01Z", + {"votes": [{"id": 101, "comment_id": 10, "created_at": "2024-01-14T00:00:00Z"}]}, ), # Fetch the first page of votes for comment 11 of post 1 ( - "https://api.example.com/community/posts/1/comments/11/votes?per_page=100&start_time=2024-01-03T00:00:00Z", - {"votes": [{"id": 102, "comment_id": 11, "created_at": "2024-01-13T00:00:00Z"}]}, + "https://api.example.com/community/posts/1/comments/11/votes?per_page=100&start_time=2024-01-03T00:00:00Z", + {"votes": [{"id": 102, "comment_id": 11, "created_at": "2024-01-13T00:00:00Z"}]}, ), # Fetch the first page of votes for comment 12 of post 1 ("https://api.example.com/community/posts/1/comments/12/votes?per_page=100&start_time=2024-01-01T00:00:01Z", {"votes": []}), # Fetch the first page of comments for post 2 ( - "https://api.example.com/community/posts/2/comments?per_page=100", - { - "comments": [{"id": 20, "post_id": 2, "updated_at": "2024-01-22T00:00:00Z"}], - "next_page": "https://api.example.com/community/posts/2/comments?per_page=100&page=2", - }, + "https://api.example.com/community/posts/2/comments?per_page=100", + { + "comments": [{"id": 20, "post_id": 2, "updated_at": "2024-01-22T00:00:00Z"}], + "next_page": "https://api.example.com/community/posts/2/comments?per_page=100&page=2", + }, ), # Fetch the second page of comments for post 2 ( - "https://api.example.com/community/posts/2/comments?per_page=100&page=2", - {"comments": [{"id": 21, "post_id": 2, "updated_at": "2024-01-21T00:00:00Z"}]}, + "https://api.example.com/community/posts/2/comments?per_page=100&page=2", + {"comments": [{"id": 21, "post_id": 2, "updated_at": "2024-01-21T00:00:00Z"}]}, ), # Fetch the first page of votes for comment 20 of post 2 ( - "https://api.example.com/community/posts/2/comments/20/votes?per_page=100&start_time=2024-01-01T00:00:01Z", - {"votes": [{"id": 200, "comment_id": 20, "created_at": "2024-01-12T00:00:00Z"}]}, + "https://api.example.com/community/posts/2/comments/20/votes?per_page=100&start_time=2024-01-01T00:00:01Z", + {"votes": [{"id": 200, "comment_id": 20, "created_at": "2024-01-12T00:00:00Z"}]}, ), # Fetch the first page of votes for comment 21 of post 2 ( - "https://api.example.com/community/posts/2/comments/21/votes?per_page=100&start_time=2024-01-01T00:00:01Z", - {"votes": [{"id": 201, "comment_id": 21, "created_at": "2024-01-12T00:00:15Z"}]}, + "https://api.example.com/community/posts/2/comments/21/votes?per_page=100&start_time=2024-01-01T00:00:01Z", + {"votes": [{"id": 201, "comment_id": 21, "created_at": "2024-01-12T00:00:15Z"}]}, ), # Fetch the first page of comments for post 3 ( - "https://api.example.com/community/posts/3/comments?per_page=100", - {"comments": [{"id": 30, "post_id": 3, "updated_at": "2024-01-09T00:00:00Z"}]}, + "https://api.example.com/community/posts/3/comments?per_page=100", + {"comments": [{"id": 30, "post_id": 3, "updated_at": "2024-01-09T00:00:00Z"}]}, ), # Fetch the first page of votes for comment 30 of post 3 ( - "https://api.example.com/community/posts/3/comments/30/votes?per_page=100", - {"votes": [{"id": 300, "comment_id": 30, "created_at": "2024-01-10T00:00:00Z"}]}, + "https://api.example.com/community/posts/3/comments/30/votes?per_page=100", + {"votes": [{"id": 300, "comment_id": 30, "created_at": "2024-01-10T00:00:00Z"}]}, ), ], # Expected records @@ -534,7 +535,7 @@ def test_incremental_parent_state(test_name, manifest, mock_requests, expected_r type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="post_comment_votes", namespace=None), - stream_state=AirbyteStateBlob.parse_obj( + stream_state=AirbyteStateBlob( { # This should not happen since parent state is disabled, but I've added this to validate that and # incoming parent_state is ignored when the parent stream's incremental_dependency is disabled @@ -590,12 +591,7 @@ def test_incremental_parent_state(test_name, manifest, mock_requests, expected_r ], ) def test_incremental_parent_state_no_incremental_dependency( - test_name, - manifest, - mock_requests, - expected_records, - initial_state, - expected_state + test_name, manifest, mock_requests, expected_records, initial_state, expected_state ): """ This is a pretty complicated test that syncs a low-code connector stream with three levels of substreams @@ -614,8 +610,12 @@ def test_incremental_parent_state_no_incremental_dependency( config = {"start_date": "2024-01-01T00:00:01Z", "credentials": {"email": "email", "api_token": "api_token"}} # Disable incremental_dependency - manifest["definitions"]["post_comments_stream"]["retriever"]["partition_router"]["parent_stream_configs"][0]["incremental_dependency"] = False - manifest["definitions"]["post_comment_votes_stream"]["retriever"]["partition_router"]["parent_stream_configs"][0]["incremental_dependency"] = False + manifest["definitions"]["post_comments_stream"]["retriever"]["partition_router"]["parent_stream_configs"][0][ + "incremental_dependency" + ] = False + manifest["definitions"]["post_comment_votes_stream"]["retriever"]["partition_router"]["parent_stream_configs"][0][ + "incremental_dependency" + ] = False with requests_mock.Mocker() as m: for url, response in mock_requests: @@ -625,5 +625,5 @@ def test_incremental_parent_state_no_incremental_dependency( output_data = [message.record.data for message in output if message.record] assert output_data == expected_records - final_state = [message.state.stream.stream_state.dict() for message in output if message.state] + final_state = [orjson.loads(orjson.dumps(message.state.stream.stream_state)) for message in output if message.state] assert final_state[-1] == expected_state diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py index 5201dbc8f241b..3a80407cea964 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/partition_routers/test_substream_partition_router.py @@ -41,9 +41,7 @@ def __init__(self, slices, records, name, cursor_field="", cursor=None): self._slices = slices self._records = records self._stream_cursor_field = ( - InterpolatedString.create(cursor_field, parameters={}) - if isinstance(cursor_field, str) - else cursor_field + InterpolatedString.create(cursor_field, parameters={}) if isinstance(cursor_field, str) else cursor_field ) self._name = name self._state = {"states": []} @@ -311,15 +309,17 @@ def test_substream_partition_router(parent_stream_configs, expected_slices): def test_substream_partition_router_invalid_parent_record_type(): partition_router = SubstreamPartitionRouter( - parent_stream_configs=[ParentStreamConfig( - stream=MockStream([{}], [list()], "first_stream"), - parent_key="id", - partition_field="first_stream_id", - parameters={}, - config={}, - )], + parent_stream_configs=[ + ParentStreamConfig( + stream=MockStream([{}], [list()], "first_stream"), + parent_key="id", + partition_field="first_stream_id", + parameters={}, + config={}, + ) + ], parameters={}, - config={} + config={}, ) with pytest.raises(AirbyteTracedException): @@ -664,7 +664,7 @@ def test_substream_checkpoints_after_each_parent_partition(): [ pytest.param(False, id="test_resumable_full_refresh_stream_without_parent_checkpoint"), pytest.param(True, id="test_resumable_full_refresh_stream_with_use_incremental_dependency_for_parent_checkpoint"), - ] + ], ) def test_substream_using_resumable_full_refresh_parent_stream(use_incremental_dependency): mock_slices = [ @@ -687,8 +687,8 @@ def test_substream_using_resumable_full_refresh_parent_stream(use_incremental_de {"next_page_token": 2}, {"next_page_token": 3}, {"next_page_token": 3}, - {'__ab_full_refresh_sync_complete': True}, - {'__ab_full_refresh_sync_complete': True}, + {"__ab_full_refresh_sync_complete": True}, + {"__ab_full_refresh_sync_complete": True}, ] partition_router = SubstreamPartitionRouter( @@ -737,7 +737,7 @@ def test_substream_using_resumable_full_refresh_parent_stream(use_incremental_de [ pytest.param(False, id="test_substream_resumable_full_refresh_stream_without_parent_checkpoint"), pytest.param(True, id="test_substream_resumable_full_refresh_stream_with_use_incremental_dependency_for_parent_checkpoint"), - ] + ], ) def test_substream_using_resumable_full_refresh_parent_stream_slices(use_incremental_dependency): mock_parent_slices = [ @@ -760,72 +760,20 @@ def test_substream_using_resumable_full_refresh_parent_stream_slices(use_increme {"next_page_token": 2}, {"next_page_token": 3}, {"next_page_token": 3}, - {'__ab_full_refresh_sync_complete': True}, - {'__ab_full_refresh_sync_complete': True}, + {"__ab_full_refresh_sync_complete": True}, + {"__ab_full_refresh_sync_complete": True}, ] expected_substream_state = { "states": [ - { - "partition": { - "parent_slice": {}, - "partition_field": "makoto_yuki" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "parent_slice": {}, - "partition_field": "yukari_takeba" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "parent_slice": {}, - "partition_field": "mitsuru_kirijo" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "parent_slice": {}, - "partition_field": "akihiko_sanada" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "parent_slice": {}, - "partition_field": "junpei_iori" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "parent_slice": {}, - "partition_field": "fuuka_yamagishi" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - } + {"partition": {"parent_slice": {}, "partition_field": "makoto_yuki"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"parent_slice": {}, "partition_field": "yukari_takeba"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"parent_slice": {}, "partition_field": "mitsuru_kirijo"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"parent_slice": {}, "partition_field": "akihiko_sanada"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"parent_slice": {}, "partition_field": "junpei_iori"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"parent_slice": {}, "partition_field": "fuuka_yamagishi"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, ], - "parent_state": { - "persona_3_characters": { - "__ab_full_refresh_sync_complete": True - } - } + "parent_state": {"persona_3_characters": {"__ab_full_refresh_sync_complete": True}}, } partition_router = SubstreamPartitionRouter( @@ -874,7 +822,9 @@ def test_substream_using_resumable_full_refresh_parent_stream_slices(use_increme assert actual_slice == expected_parent_slices[expected_counter] # check for parent state if use_incremental_dependency: - assert substream_cursor_slicer._partition_router._parent_state["persona_3_characters"] == expected_parent_state[expected_counter] + assert ( + substream_cursor_slicer._partition_router._parent_state["persona_3_characters"] == expected_parent_state[expected_counter] + ) expected_counter += 1 # validate final state for closed substream slices diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py index b57fd714b7359..59dbb6b419a79 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/backoff_strategies/test_wait_time_from_header.py @@ -6,10 +6,10 @@ import pytest from airbyte_cdk import AirbyteTracedException +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.requesters.error_handlers.backoff_strategies.wait_time_from_header_backoff_strategy import ( WaitTimeFromHeaderBackoffStrategy, ) -from airbyte_protocol.models import FailureType from requests import Response SOME_BACKOFF_TIME = 60 @@ -44,7 +44,9 @@ def test_given_retry_after_smaller_than_max_time_then_raise_transient_error(): response_mock = MagicMock(spec=Response) retry_after = _A_MAX_TIME - 1 response_mock.headers = {_A_RETRY_HEADER: str(retry_after)} - backoff_strategy = WaitTimeFromHeaderBackoffStrategy(header=_A_RETRY_HEADER, max_waiting_time_in_seconds=_A_MAX_TIME, parameters={}, config={}) + backoff_strategy = WaitTimeFromHeaderBackoffStrategy( + header=_A_RETRY_HEADER, max_waiting_time_in_seconds=_A_MAX_TIME, parameters={}, config={} + ) assert backoff_strategy.backoff_time(response_mock, 1) == retry_after @@ -52,7 +54,9 @@ def test_given_retry_after_smaller_than_max_time_then_raise_transient_error(): def test_given_retry_after_greater_than_max_time_then_raise_transient_error(): response_mock = MagicMock(spec=Response) response_mock.headers = {_A_RETRY_HEADER: str(_A_MAX_TIME + 1)} - backoff_strategy = WaitTimeFromHeaderBackoffStrategy(header=_A_RETRY_HEADER, max_waiting_time_in_seconds=_A_MAX_TIME, parameters={}, config={}) + backoff_strategy = WaitTimeFromHeaderBackoffStrategy( + header=_A_RETRY_HEADER, max_waiting_time_in_seconds=_A_MAX_TIME, parameters={}, config={} + ) with pytest.raises(AirbyteTracedException) as exception: backoff_strategy.backoff_time(response_mock, 1) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py index 5f34bb28c9692..574f3eec0e753 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py @@ -6,11 +6,11 @@ import pytest import requests +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.requesters.error_handlers import HttpResponseFilter from airbyte_cdk.sources.declarative.requesters.error_handlers.composite_error_handler import CompositeErrorHandler from airbyte_cdk.sources.declarative.requesters.error_handlers.default_error_handler import DefaultErrorHandler from airbyte_cdk.sources.streams.http.error_handlers.response_models import ErrorResolution, ResponseAction -from airbyte_protocol.models import FailureType SOME_BACKOFF_TIME = 60 @@ -34,7 +34,7 @@ response_action=ResponseAction.SUCCESS, failure_type=None, error_message=None, - ) + ), ), ( "test_chain_retrier_ignore_fail", @@ -83,7 +83,7 @@ ErrorResolution( response_action=ResponseAction.IGNORE, ), - ) + ), ], ) def test_composite_error_handler(test_name, first_handler_behavior, second_handler_behavior, expected_behavior): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py index f80aef233ebee..6fc99159afed1 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_error_handler.py @@ -18,42 +18,42 @@ @pytest.mark.parametrize( - "test_name, http_status_code, expected_error_resolution", - [ - ( - "_with_http_response_status_200", - 200, - ErrorResolution( - response_action=ResponseAction.SUCCESS, - failure_type=None, - error_message=None, - ), - ), - ( - "_with_http_response_status_400", - 400, - DEFAULT_ERROR_MAPPING[400], - ), - ( - "_with_http_response_status_404", - 404, - DEFAULT_ERROR_MAPPING[404], + "test_name, http_status_code, expected_error_resolution", + [ + ( + "_with_http_response_status_200", + 200, + ErrorResolution( + response_action=ResponseAction.SUCCESS, + failure_type=None, + error_message=None, ), - ( - "_with_http_response_status_408", - 408, - DEFAULT_ERROR_MAPPING[408], + ), + ( + "_with_http_response_status_400", + 400, + DEFAULT_ERROR_MAPPING[400], + ), + ( + "_with_http_response_status_404", + 404, + DEFAULT_ERROR_MAPPING[404], + ), + ( + "_with_http_response_status_408", + 408, + DEFAULT_ERROR_MAPPING[408], + ), + ( + "_with_unmapped_http_status_418", + 418, + ErrorResolution( + response_action=ResponseAction.RETRY, + failure_type=FailureType.system_error, + error_message="Unexpected response with HTTP status 418", ), - ( - "_with_unmapped_http_status_418", - 418, - ErrorResolution( - response_action=ResponseAction.RETRY, - failure_type=FailureType.system_error, - error_message="Unexpected response with HTTP status 418", - ), - ) - ], + ), + ], ) def test_default_error_handler_with_default_response_filter(test_name, http_status_code: int, expected_error_resolution: ErrorResolution): response_mock = create_response(http_status_code) @@ -65,76 +65,78 @@ def test_default_error_handler_with_default_response_filter(test_name, http_stat @pytest.mark.parametrize( - "test_name, http_status_code, test_response_filter, response_action, failure_type, error_message", - [ - ( - "_with_http_response_status_400_fail_with_default_failure_type", - 400, - HttpResponseFilter( - http_codes=[400], - action=ResponseAction.RETRY, - config={}, - parameters={}, - ), - ResponseAction.RETRY, - FailureType.system_error, - "Bad request. Please check your request parameters.", + "test_name, http_status_code, test_response_filter, response_action, failure_type, error_message", + [ + ( + "_with_http_response_status_400_fail_with_default_failure_type", + 400, + HttpResponseFilter( + http_codes=[400], + action=ResponseAction.RETRY, + config={}, + parameters={}, ), - ( - "_with_http_response_status_402_fail_with_default_failure_type", - 402, - HttpResponseFilter( - http_codes=[402], - action=ResponseAction.FAIL, - config={}, - parameters={}, - ), - ResponseAction.FAIL, - FailureType.system_error, - "", + ResponseAction.RETRY, + FailureType.system_error, + "Bad request. Please check your request parameters.", + ), + ( + "_with_http_response_status_402_fail_with_default_failure_type", + 402, + HttpResponseFilter( + http_codes=[402], + action=ResponseAction.FAIL, + config={}, + parameters={}, ), - ( - "_with_http_response_status_403_fail_with_default_failure_type", - 403, - HttpResponseFilter( - http_codes=[403], - action="FAIL", - config={}, - parameters={}, - ), - ResponseAction.FAIL, - FailureType.config_error, - "Forbidden. You don't have permission to access this resource.", + ResponseAction.FAIL, + FailureType.system_error, + "", + ), + ( + "_with_http_response_status_403_fail_with_default_failure_type", + 403, + HttpResponseFilter( + http_codes=[403], + action="FAIL", + config={}, + parameters={}, ), - ( - "_with_http_response_status_200_fail_with_contained_error_message", - 418, - HttpResponseFilter( - action=ResponseAction.FAIL, - error_message_contains="test", - config={}, - parameters={}, - ), - ResponseAction.FAIL, - FailureType.system_error, - "", + ResponseAction.FAIL, + FailureType.config_error, + "Forbidden. You don't have permission to access this resource.", + ), + ( + "_with_http_response_status_200_fail_with_contained_error_message", + 418, + HttpResponseFilter( + action=ResponseAction.FAIL, + error_message_contains="test", + config={}, + parameters={}, ), - ( - "_fail_with_predicate", - 418, - HttpResponseFilter( - action=ResponseAction.FAIL, - predicate="{{ 'error' in response }}", - config={}, - parameters={}, - ), - ResponseAction.FAIL, - FailureType.system_error, - "", + ResponseAction.FAIL, + FailureType.system_error, + "", + ), + ( + "_fail_with_predicate", + 418, + HttpResponseFilter( + action=ResponseAction.FAIL, + predicate="{{ 'error' in response }}", + config={}, + parameters={}, ), - ], + ResponseAction.FAIL, + FailureType.system_error, + "", + ), + ], ) -def test_default_error_handler_with_custom_response_filter(test_name, http_status_code, test_response_filter, response_action, failure_type, error_message): +def test_default_error_handler_with_custom_response_filter( + test_name, http_status_code, test_response_filter, response_action, failure_type, error_message +): response_mock = create_response(http_status_code) if http_status_code == 418: response_mock.json.return_value = {"error": "test"} @@ -148,11 +150,11 @@ def test_default_error_handler_with_custom_response_filter(test_name, http_statu @pytest.mark.parametrize( - "http_status_code, expected_response_action", - [ - (400, ResponseAction.RETRY), - (402, ResponseAction.FAIL), - ], + "http_status_code, expected_response_action", + [ + (400, ResponseAction.RETRY), + (402, ResponseAction.FAIL), + ], ) def test_default_error_handler_with_multiple_response_filters(http_status_code, expected_response_action): response_filter_one = HttpResponseFilter( @@ -175,15 +177,17 @@ def test_default_error_handler_with_multiple_response_filters(http_status_code, @pytest.mark.parametrize( - "first_response_filter_action, second_response_filter_action, expected_response_action", - [ - (ResponseAction.RETRY, ResponseAction.FAIL, ResponseAction.RETRY), - (ResponseAction.FAIL, ResponseAction.RETRY, ResponseAction.FAIL), - (ResponseAction.IGNORE, ResponseAction.IGNORE, ResponseAction.IGNORE), - (ResponseAction.SUCCESS, ResponseAction.IGNORE, ResponseAction.SUCCESS), - ] + "first_response_filter_action, second_response_filter_action, expected_response_action", + [ + (ResponseAction.RETRY, ResponseAction.FAIL, ResponseAction.RETRY), + (ResponseAction.FAIL, ResponseAction.RETRY, ResponseAction.FAIL), + (ResponseAction.IGNORE, ResponseAction.IGNORE, ResponseAction.IGNORE), + (ResponseAction.SUCCESS, ResponseAction.IGNORE, ResponseAction.SUCCESS), + ], ) -def test_default_error_handler_with_conflicting_response_filters(first_response_filter_action, second_response_filter_action, expected_response_action): +def test_default_error_handler_with_conflicting_response_filters( + first_response_filter_action, second_response_filter_action, expected_response_action +): response_filter_one = HttpResponseFilter( http_codes=[400], action=first_response_filter_action, @@ -205,19 +209,29 @@ def test_default_error_handler_with_conflicting_response_filters(first_response_ def test_default_error_handler_with_constant_backoff_strategy(): response_mock = create_response(429) - error_handler = DefaultErrorHandler(config={}, parameters={}, backoff_strategies=[ConstantBackoffStrategy(SOME_BACKOFF_TIME, config={}, parameters={})]) + error_handler = DefaultErrorHandler( + config={}, parameters={}, backoff_strategies=[ConstantBackoffStrategy(SOME_BACKOFF_TIME, config={}, parameters={})] + ) assert error_handler.backoff_time(response_or_exception=response_mock, attempt_count=0) == SOME_BACKOFF_TIME @pytest.mark.parametrize( "attempt_count", [ - 0, 1, 2, 3, 4, 5, 6, + 0, + 1, + 2, + 3, + 4, + 5, + 6, ], ) def test_default_error_handler_with_exponential_backoff_strategy(attempt_count): response_mock = create_response(429) - error_handler = DefaultErrorHandler(config={}, parameters={}, backoff_strategies=[ExponentialBackoffStrategy(factor=1, config={}, parameters={})]) + error_handler = DefaultErrorHandler( + config={}, parameters={}, backoff_strategies=[ExponentialBackoffStrategy(factor=1, config={}, parameters={})] + ) assert error_handler.backoff_time(response_or_exception=response_mock, attempt_count=attempt_count) == (1 * 2**attempt_count) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_http_response_filter.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_http_response_filter.py index 6da87a183ff2f..b3e4c517da269 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_http_response_filter.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_default_http_response_filter.py @@ -5,31 +5,19 @@ from unittest.mock import MagicMock import pytest +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.declarative.requesters.error_handlers.default_http_response_filter import DefaultHttpResponseFilter from airbyte_cdk.sources.streams.http.error_handlers.default_error_mapping import DEFAULT_ERROR_MAPPING from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction -from airbyte_protocol.models import FailureType from requests import RequestException, Response @pytest.mark.parametrize( "http_code, expected_error_resolution", [ - pytest.param( - 403, - DEFAULT_ERROR_MAPPING[403], - id="403 mapping" - ), - pytest.param( - 404, - DEFAULT_ERROR_MAPPING[404], - id="404 mapping" - ), - pytest.param( - 408, - DEFAULT_ERROR_MAPPING[408], - id="408 mapping" - ), + pytest.param(403, DEFAULT_ERROR_MAPPING[403], id="403 mapping"), + pytest.param(404, DEFAULT_ERROR_MAPPING[404], id="404 mapping"), + pytest.param(408, DEFAULT_ERROR_MAPPING[408], id="408 mapping"), ], ) def test_matches_mapped_http_status_code(http_code, expected_error_resolution): diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py index 5fd5990e898ba..9c6817c268c42 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py @@ -22,7 +22,9 @@ "", "custom error message", {"status_code": 503}, - ErrorResolution(response_action=ResponseAction.FAIL, failure_type=FailureType.transient_error, error_message="custom error message"), + ErrorResolution( + response_action=ResponseAction.FAIL, failure_type=FailureType.transient_error, error_message="custom error message" + ), id="test_http_code_matches", ), pytest.param( @@ -33,7 +35,11 @@ "", "", {"status_code": 403}, - ErrorResolution(response_action=ResponseAction.IGNORE, failure_type=FailureType.config_error, error_message="Forbidden. You don't have permission to access this resource."), + ErrorResolution( + response_action=ResponseAction.IGNORE, + failure_type=FailureType.config_error, + error_message="Forbidden. You don't have permission to access this resource.", + ), id="test_http_code_matches_ignore_action", ), pytest.param( @@ -44,7 +50,9 @@ "", "", {"status_code": 429}, - ErrorResolution(response_action=ResponseAction.RETRY, failure_type=FailureType.transient_error, error_message="Too many requests."), + ErrorResolution( + response_action=ResponseAction.RETRY, failure_type=FailureType.transient_error, error_message="Too many requests." + ), id="test_http_code_matches_retry_action", ), pytest.param( @@ -55,7 +63,9 @@ "", "error message was: {{ response.failure }}", {"status_code": 404, "json": {"the_body": "do_i_match", "failure": "i failed you"}}, - ErrorResolution(response_action=ResponseAction.FAIL, failure_type=FailureType.system_error, error_message="error message was: i failed you"), + ErrorResolution( + response_action=ResponseAction.FAIL, failure_type=FailureType.system_error, error_message="error message was: i failed you" + ), id="test_predicate_matches_json", ), pytest.param( @@ -66,7 +76,9 @@ "", "error from header: {{ headers.warning }}", {"status_code": 404, "headers": {"the_key": "header_match", "warning": "this failed"}}, - ErrorResolution(response_action=ResponseAction.FAIL, failure_type=FailureType.system_error, error_message="error from header: this failed"), + ErrorResolution( + response_action=ResponseAction.FAIL, failure_type=FailureType.system_error, error_message="error from header: this failed" + ), id="test_predicate_matches_headers", ), pytest.param( @@ -80,7 +92,7 @@ ErrorResolution( response_action=ResponseAction.FAIL, failure_type=FailureType.config_error, - error_message="Forbidden. You don't have permission to access this resource." + error_message="Forbidden. You don't have permission to access this resource.", ), id="test_predicate_matches_headers", ), @@ -147,12 +159,16 @@ "", "rate limits", {"status_code": 500}, - ErrorResolution(response_action=ResponseAction.RATE_LIMITED, failure_type=FailureType.transient_error, error_message="rate limits"), + ErrorResolution( + response_action=ResponseAction.RATE_LIMITED, failure_type=FailureType.transient_error, error_message="rate limits" + ), id="test_http_code_matches_response_action_rate_limited", ), ], ) -def test_matches(requests_mock, action, failure_type, http_codes, predicate, error_contains, error_message, response, expected_error_resolution): +def test_matches( + requests_mock, action, failure_type, http_codes, predicate, error_contains, error_message, response, expected_error_resolution +): requests_mock.register_uri( "GET", "https://airbyte.io/", diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py index 6f010323c8f02..31d9ae5e05f52 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_cursor_pagination_strategy.py @@ -50,7 +50,7 @@ "test_static_token_with_string_stop_condition", "test_token_from_header", "test_token_from_response_header_links", - ] + ], ) def test_cursor_pagination_strategy(template_string, stop_condition, expected_token, page_size): decoder = JsonDecoder(parameters={}) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py index 109f153cbcc8d..54fcb2883ab28 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_default_paginator.py @@ -247,7 +247,7 @@ def test_page_size_option_cannot_be_set_if_strategy_has_no_limit(): ids=[ "test_reset_inject_on_first_request", "test_reset_no_inject_on_first_request", - ] + ], ) def test_reset(inject_on_first_request): page_size_request_option = RequestOption(inject_into=RequestOptionType.request_parameter, field_name="limit", parameters={}) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py index 1ca14cc60481a..da2bf6d9450ec 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/paginators/test_page_increment.py @@ -40,13 +40,7 @@ def test_page_increment_paginator_strategy(page_size, start_from, last_page_size assert start_from == paginator_strategy._page -@pytest.mark.parametrize( - "page_size", - [ - pytest.param("{{ config['value'] }}"), - pytest.param("not-an-integer") - ] -) +@pytest.mark.parametrize("page_size", [pytest.param("{{ config['value'] }}"), pytest.param("not-an-integer")]) def test_page_increment_paginator_strategy_malformed_page_size(page_size): with pytest.raises(Exception, match=".* is of type . Expected "): PageIncrement(page_size=page_size, parameters={}, start_from_page=0, config={"value": "not-an-integer"}) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py index 77e94778ea2f4..404bf9f50e15e 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py @@ -415,7 +415,7 @@ def test_send_request_params(provider_params, param_params, authenticator_params {"k": [1, 2]}, "%5B%22a%22%2C+%22b%22%5D=1&%5B%22a%22%2C+%22b%22%5D=2", id="test-key-with-list-to-be-interpolated", - ) + ), ], ) def test_request_param_interpolation(request_parameters, config, expected_query_params): @@ -464,8 +464,7 @@ def test_request_param_interpolation_with_incorrect_values(request_parameters, c requester.send_request() assert ( - error.value.args[0] - == f"Invalid value for `{invalid_value_for_key}` parameter. The values of request params cannot be an object." + error.value.args[0] == f"Invalid value for `{invalid_value_for_key}` parameter. The values of request params cannot be an object." ) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index 9c36c65b85536..fd3db0452f04c 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -147,7 +147,7 @@ def test_simple_retriever_with_request_response_logs(mock_http_stream): [ pytest.param(None, None, 1, id="test_initial_sync_no_state"), pytest.param({"next_page_token": 10}, 10, 11, id="test_reset_with_next_page_token"), - ] + ], ) def test_simple_retriever_resumable_full_refresh_cursor_page_increment(initial_state, expected_reset_value, expected_next_page): expected_records = [ @@ -184,7 +184,7 @@ def test_simple_retriever_resumable_full_refresh_cursor_page_increment(initial_s expected_records[5], expected_records[6], expected_records[7], - ] + ], ] page_increment_strategy = PageIncrement(config={}, page_size=5, parameters={}) @@ -230,11 +230,13 @@ def test_simple_retriever_resumable_full_refresh_cursor_page_increment(initial_s {"next_page_token": "https://for-all-mankind.nasa.com/api/v1/astronauts?next_page=tracy_stevens"}, "https://for-all-mankind.nasa.com/api/v1/astronauts?next_page=tracy_stevens", "https://for-all-mankind.nasa.com/api/v1/astronauts?next_page=gordo_stevens", - id="test_reset_with_next_page_token" + id="test_reset_with_next_page_token", ), - ] + ], ) -def test_simple_retriever_resumable_full_refresh_cursor_reset_cursor_pagination(initial_state, expected_reset_value, expected_next_page, requests_mock): +def test_simple_retriever_resumable_full_refresh_cursor_reset_cursor_pagination( + initial_state, expected_reset_value, expected_next_page, requests_mock +): expected_records = [ Record(data={"name": "ed_baldwin"}, associated_slice=None), Record(data={"name": "danielle_poole"}, associated_slice=None), @@ -288,7 +290,7 @@ def test_simple_retriever_resumable_full_refresh_cursor_reset_cursor_pagination( stream = factory.create_component(model_type=DeclarativeStreamModel, component_definition=stream_manifest, config={}) response_body = { "data": [r.data for r in expected_records[:5]], - "next_page": "https://for-all-mankind.nasa.com/api/v1/astronauts?next_page=gordo_stevens" + "next_page": "https://for-all-mankind.nasa.com/api/v1/astronauts?next_page=gordo_stevens", } requests_mock.get("https://for-all-mankind.nasa.com/api/v1/astronauts", json=response_body) requests_mock.get("https://for-all-mankind.nasa.com/astronauts?next_page=tracy_stevens", json=response_body) @@ -334,7 +336,10 @@ def test_simple_retriever_resumable_full_refresh_cursor_reset_skip_completed_str ] record_selector = MagicMock() - record_selector.select_records.return_value = [expected_records[0],expected_records[1],] + record_selector.select_records.return_value = [ + expected_records[0], + expected_records[1], + ] page_increment_strategy = PageIncrement(config={}, page_size=5, parameters={}) paginator = DefaultPaginator(config={}, pagination_strategy=page_increment_strategy, url_base="https://airbyte.io", parameters={}) @@ -463,13 +468,39 @@ def test_get_request_headers(test_name, paginator_mapping, expected_mapping): @pytest.mark.parametrize( "test_name, paginator_mapping, ignore_stream_slicer_parameters_on_paginated_requests, next_page_token, expected_mapping", [ - ("test_do_not_ignore_stream_slicer_params_if_ignore_is_true_but_no_next_page_token", {"key_from_pagination": "1000"}, True, None, {"key_from_pagination": "1000"}), - ("test_do_not_ignore_stream_slicer_params_if_ignore_is_false_and_no_next_page_token", {"key_from_pagination": "1000"}, False, None, {"key_from_pagination": "1000", "key_from_slicer": "value"}), - ("test_ignore_stream_slicer_params_on_paginated_request", {"key_from_pagination": "1000"}, True, {"page": 2}, {"key_from_pagination": "1000"}), - ("test_do_not_ignore_stream_slicer_params_on_paginated_request", {"key_from_pagination": "1000"}, False, {"page": 2}, {"key_from_pagination": "1000", "key_from_slicer": "value"}), + ( + "test_do_not_ignore_stream_slicer_params_if_ignore_is_true_but_no_next_page_token", + {"key_from_pagination": "1000"}, + True, + None, + {"key_from_pagination": "1000"}, + ), + ( + "test_do_not_ignore_stream_slicer_params_if_ignore_is_false_and_no_next_page_token", + {"key_from_pagination": "1000"}, + False, + None, + {"key_from_pagination": "1000", "key_from_slicer": "value"}, + ), + ( + "test_ignore_stream_slicer_params_on_paginated_request", + {"key_from_pagination": "1000"}, + True, + {"page": 2}, + {"key_from_pagination": "1000"}, + ), + ( + "test_do_not_ignore_stream_slicer_params_on_paginated_request", + {"key_from_pagination": "1000"}, + False, + {"page": 2}, + {"key_from_pagination": "1000", "key_from_slicer": "value"}, + ), ], ) -def test_ignore_stream_slicer_parameters_on_paginated_requests(test_name, paginator_mapping, ignore_stream_slicer_parameters_on_paginated_requests, next_page_token, expected_mapping): +def test_ignore_stream_slicer_parameters_on_paginated_requests( + test_name, paginator_mapping, ignore_stream_slicer_parameters_on_paginated_requests, next_page_token, expected_mapping +): # This test is separate from the other request options because request headers must be strings paginator = MagicMock() paginator.get_request_headers.return_value = paginator_mapping diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/spec/test_spec.py b/airbyte-cdk/python/unit_tests/sources/declarative/spec/test_spec.py index 46b892256a25d..1e1ef498082f6 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/spec/test_spec.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/spec/test_spec.py @@ -3,32 +3,34 @@ # import pytest -from airbyte_cdk.models.airbyte_protocol import AdvancedAuth, ConnectorSpecification +from airbyte_cdk.models import AdvancedAuth, AuthFlowType, ConnectorSpecification from airbyte_cdk.sources.declarative.models.declarative_component_schema import AuthFlow from airbyte_cdk.sources.declarative.spec.spec import Spec @pytest.mark.parametrize( - "test_name, spec, expected_connection_specification", + "spec, expected_connection_specification", [ ( - "test_only_connection_specification", Spec(connection_specification={"client_id": "my_client_id"}, parameters={}), ConnectorSpecification(connectionSpecification={"client_id": "my_client_id"}), ), ( - "test_with_doc_url", Spec(connection_specification={"client_id": "my_client_id"}, parameters={}, documentation_url="https://airbyte.io"), ConnectorSpecification(connectionSpecification={"client_id": "my_client_id"}, documentationUrl="https://airbyte.io"), ), ( - "test_auth_flow", Spec(connection_specification={"client_id": "my_client_id"}, parameters={}, advanced_auth=AuthFlow(auth_flow_type="oauth2.0")), ConnectorSpecification( - connectionSpecification={"client_id": "my_client_id"}, advanced_auth=AdvancedAuth(auth_flow_type="oauth2.0") + connectionSpecification={"client_id": "my_client_id"}, advanced_auth=AdvancedAuth(auth_flow_type=AuthFlowType.oauth2_0) ), ), ], + ids=[ + "test_only_connection_specification", + "test_with_doc_url", + "test_auth_flow", + ], ) -def test_spec(test_name, spec, expected_connection_specification): +def test_spec(spec, expected_connection_specification): assert spec.generate_spec() == expected_connection_specification diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py index bd8281b80b46c..8906b625fb8f0 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py @@ -189,7 +189,7 @@ def test_no_state_migration_is_applied_if_the_state_should_not_be_migrated(): [ pytest.param(True, True, id="test_retriever_has_cursor"), pytest.param(False, False, id="test_retriever_has_cursor"), - ] + ], ) def test_is_resumable(use_cursor, expected_supports_checkpointing): schema_loader = _schema_loader() diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py index d7017eb52dd5f..2d350fa12b4b7 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_manifest_declarative_source.py @@ -29,7 +29,6 @@ from airbyte_cdk.sources.declarative.manifest_declarative_source import ManifestDeclarativeSource from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever from jsonschema.exceptions import ValidationError -from pydantic import AnyUrl logger = logging.getLogger("airbyte") @@ -210,7 +209,7 @@ def test_manifest_with_spec(self): source = ManifestDeclarativeSource(source_config=manifest) connector_specification = source.spec(logger) assert connector_specification is not None - assert connector_specification.documentationUrl == AnyUrl("https://airbyte.com/#yaml-from-manifest") + assert connector_specification.documentationUrl == "https://airbyte.com/#yaml-from-manifest" assert connector_specification.connectionSpecification["title"] == "Test Spec" assert connector_specification.connectionSpecification["required"][0] == "api_key" assert connector_specification.connectionSpecification["additionalProperties"] is False @@ -277,7 +276,7 @@ def test_manifest_with_external_spec(self, use_external_yaml_spec): connector_specification = source.spec(logger) - assert connector_specification.documentationUrl == AnyUrl("https://airbyte.com/#yaml-from-external") + assert connector_specification.documentationUrl == "https://airbyte.com/#yaml-from-external" assert connector_specification.connectionSpecification == EXTERNAL_CONNECTION_SPECIFICATION def test_source_is_not_created_if_toplevel_fields_are_unknown(self): @@ -1045,8 +1044,12 @@ def _create_page(response_body): ), [{"ABC": 0, "partition": 0}, {"AED": 1, "partition": 0}, {"ABC": 2, "partition": 1}], [ - call({'states': []}, {"partition": "0"}, None), - call({'states': [{'partition': {'partition': '0'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}]}, {"partition": "1"}, None), + call({"states": []}, {"partition": "0"}, None), + call( + {"states": [{"partition": {"partition": "0"}, "cursor": {"__ab_full_refresh_sync_complete": True}}]}, + {"partition": "1"}, + None, + ), ], ), ( @@ -1119,9 +1122,13 @@ def _create_page(response_body): ), [{"ABC": 0, "partition": 0}, {"AED": 1, "partition": 0}, {"USD": 3, "partition": 0}, {"ABC": 2, "partition": 1}], [ - call({'states': []}, {"partition": "0"}, None), - call({'states': []}, {"partition": "0"}, {"next_page_token": "next"}), - call({'states': [{'partition': {'partition': '0'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}]}, {'partition': '1'}, None), + call({"states": []}, {"partition": "0"}, None), + call({"states": []}, {"partition": "0"}, {"next_page_token": "next"}), + call( + {"states": [{"partition": {"partition": "0"}, "cursor": {"__ab_full_refresh_sync_complete": True}}]}, + {"partition": "1"}, + None, + ), ], ), ], @@ -1269,14 +1276,14 @@ def _run_read(manifest: Mapping[str, Any], stream_name: str) -> List[AirbyteMess def test_declarative_component_schema_valid_ref_links(): def load_yaml(file_path) -> Mapping[str, Any]: - with open(file_path, 'r') as file: + with open(file_path, "r") as file: return yaml.safe_load(file) - def extract_refs(data, base_path='#') -> List[str]: + def extract_refs(data, base_path="#") -> List[str]: refs = [] if isinstance(data, dict): for key, value in data.items(): - if key == '$ref' and isinstance(value, str) and value.startswith('#'): + if key == "$ref" and isinstance(value, str) and value.startswith("#"): ref_path = value refs.append(ref_path) else: @@ -1287,11 +1294,11 @@ def extract_refs(data, base_path='#') -> List[str]: return refs def resolve_pointer(data: Mapping[str, Any], pointer: str) -> bool: - parts = pointer.split('/')[1:] # Skip the first empty part due to leading '#/' + parts = pointer.split("/")[1:] # Skip the first empty part due to leading '#/' current = data try: for part in parts: - part = part.replace('~1', '/').replace('~0', '~') # Unescape JSON Pointer + part = part.replace("~1", "/").replace("~0", "~") # Unescape JSON Pointer current = current[part] return True except (KeyError, TypeError): @@ -1300,8 +1307,10 @@ def resolve_pointer(data: Mapping[str, Any], pointer: str) -> bool: def validate_refs(yaml_file: str) -> List[str]: data = load_yaml(yaml_file) refs = extract_refs(data) - invalid_refs = [ref for ref in refs if not resolve_pointer(data, ref.replace('#', ''))] + invalid_refs = [ref for ref in refs if not resolve_pointer(data, ref.replace("#", ""))] return invalid_refs - yaml_file_path = Path(__file__).resolve().parent.parent.parent.parent / 'airbyte_cdk/sources/declarative/declarative_component_schema.yaml' + yaml_file_path = ( + Path(__file__).resolve().parent.parent.parent.parent / "airbyte_cdk/sources/declarative/declarative_component_schema.yaml" + ) assert not validate_refs(yaml_file_path) diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_types.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_types.py index dd3f8e5b4ab26..b6eb42f940b6d 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_types.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_types.py @@ -7,13 +7,25 @@ @pytest.mark.parametrize( "stream_slice, expected_partition", [ - pytest.param(StreamSlice(partition={},cursor_slice={}), {}, id="test_partition_with_empty_partition"), - pytest.param(StreamSlice(partition=StreamSlice(partition={}, cursor_slice={}), cursor_slice={}), {}, id="test_partition_nested_empty"), - pytest.param(StreamSlice(partition={"key": "value"}, cursor_slice={}), {"key": "value"}, id="test_partition_with_mapping_partition"), - pytest.param(StreamSlice(partition={},cursor_slice={"cursor": "value"}), {}, id="test_partition_with_only_cursor"), - pytest.param(StreamSlice(partition=StreamSlice(partition={}, cursor_slice={}), cursor_slice={"cursor": "value"}), {}, id="test_partition_nested_empty_and_cursor_value_mapping"), - pytest.param(StreamSlice(partition=StreamSlice(partition={}, cursor_slice={"cursor": "value"}), cursor_slice={}), {}, id="test_partition_nested_empty_and_cursor_value"), - ] + pytest.param(StreamSlice(partition={}, cursor_slice={}), {}, id="test_partition_with_empty_partition"), + pytest.param( + StreamSlice(partition=StreamSlice(partition={}, cursor_slice={}), cursor_slice={}), {}, id="test_partition_nested_empty" + ), + pytest.param( + StreamSlice(partition={"key": "value"}, cursor_slice={}), {"key": "value"}, id="test_partition_with_mapping_partition" + ), + pytest.param(StreamSlice(partition={}, cursor_slice={"cursor": "value"}), {}, id="test_partition_with_only_cursor"), + pytest.param( + StreamSlice(partition=StreamSlice(partition={}, cursor_slice={}), cursor_slice={"cursor": "value"}), + {}, + id="test_partition_nested_empty_and_cursor_value_mapping", + ), + pytest.param( + StreamSlice(partition=StreamSlice(partition={}, cursor_slice={"cursor": "value"}), cursor_slice={}), + {}, + id="test_partition_nested_empty_and_cursor_value", + ), + ], ) def test_partition(stream_slice, expected_partition): partition = stream_slice.partition @@ -24,14 +36,25 @@ def test_partition(stream_slice, expected_partition): @pytest.mark.parametrize( "stream_slice, expected_cursor_slice", [ - pytest.param(StreamSlice(partition={},cursor_slice={}), {}, id="test_cursor_slice_with_empty_cursor"), - pytest.param(StreamSlice(partition={}, cursor_slice=StreamSlice(partition={}, cursor_slice={})), {}, id="test_cursor_slice_nested_empty"), - - pytest.param(StreamSlice(partition={}, cursor_slice={"key": "value"}), {"key": "value"}, id="test_cursor_slice_with_mapping_cursor_slice"), + pytest.param(StreamSlice(partition={}, cursor_slice={}), {}, id="test_cursor_slice_with_empty_cursor"), + pytest.param( + StreamSlice(partition={}, cursor_slice=StreamSlice(partition={}, cursor_slice={})), {}, id="test_cursor_slice_nested_empty" + ), + pytest.param( + StreamSlice(partition={}, cursor_slice={"key": "value"}), {"key": "value"}, id="test_cursor_slice_with_mapping_cursor_slice" + ), pytest.param(StreamSlice(partition={"partition": "value"}, cursor_slice={}), {}, id="test_cursor_slice_with_only_partition"), - pytest.param(StreamSlice(partition={"partition": "value"}, cursor_slice=StreamSlice(partition={}, cursor_slice={})), {}, id="test_cursor_slice_nested_empty_and_partition_mapping"), - pytest.param(StreamSlice(partition=StreamSlice(partition={"partition": "value"}, cursor_slice={}), cursor_slice={}), {}, id="test_cursor_slice_nested_empty_and_partition"), - ] + pytest.param( + StreamSlice(partition={"partition": "value"}, cursor_slice=StreamSlice(partition={}, cursor_slice={})), + {}, + id="test_cursor_slice_nested_empty_and_partition_mapping", + ), + pytest.param( + StreamSlice(partition=StreamSlice(partition={"partition": "value"}, cursor_slice={}), cursor_slice={}), + {}, + id="test_cursor_slice_nested_empty_and_partition", + ), + ], ) def test_cursor_slice(stream_slice, expected_cursor_slice): cursor_slice = stream_slice.cursor_slice diff --git a/airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py b/airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py index d2bad84128e2f..7560dc403ecde 100644 --- a/airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py +++ b/airbyte-cdk/python/unit_tests/sources/embedded/test_embedded_integration.py @@ -6,9 +6,7 @@ from typing import Any, Mapping, Optional from unittest.mock import MagicMock -from airbyte_cdk.sources.embedded.base_integration import BaseEmbeddedIntegration -from airbyte_cdk.utils import AirbyteTracedException -from airbyte_protocol.models import ( +from airbyte_cdk.models import ( AirbyteCatalog, AirbyteLogMessage, AirbyteMessage, @@ -23,6 +21,8 @@ SyncMode, Type, ) +from airbyte_cdk.sources.embedded.base_integration import BaseEmbeddedIntegration +from airbyte_cdk.utils import AirbyteTracedException class TestIntegration(BaseEmbeddedIntegration): diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/config/test_csv_format.py b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_csv_format.py index 5ce69276d974a..c233bd7ac9e91 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/config/test_csv_format.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/config/test_csv_format.py @@ -30,5 +30,5 @@ def test_given_from_csv_then_csv_has_header_row(self) -> None: class CsvDelimiterTest(unittest.TestCase): def test_tab_delimter(self): - assert CsvFormat(delimiter=r"\t").delimiter == '\t' + assert CsvFormat(delimiter=r"\t").delimiter == "\t" assert len(CsvFormat(delimiter=r"\t").delimiter) == 1 diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_avro_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_avro_parser.py index 3882d823e1968..a45d424b7a2bb 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_avro_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_avro_parser.py @@ -222,7 +222,7 @@ def test_convert_primitive_avro_type_to_json(avro_format, avro_type, expected_js pytest.param(_default_avro_format, "float", 123.456, 123.456, id="test_float"), pytest.param(_default_avro_format, "double", 123.456, 123.456, id="test_double_default_config"), pytest.param(_double_as_string_avro_format, "double", 123.456, "123.456", id="test_double_as_string"), - pytest.param(_default_avro_format, "bytes", b"hello world", b"hello world", id="test_bytes"), + pytest.param(_default_avro_format, "bytes", b"hello world", "hello world", id="test_bytes"), pytest.param(_default_avro_format, "string", "hello world", "hello world", id="test_string"), pytest.param(_default_avro_format, {"logicalType": "decimal"}, 3.1415, "3.1415", id="test_decimal"), pytest.param(_default_avro_format, {"logicalType": "uuid"}, _uuid_value, str(_uuid_value), id="test_uuid"), diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_excel_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_excel_parser.py index dbee93fd57a62..bd9d8338f094f 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_excel_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_excel_parser.py @@ -47,12 +47,14 @@ def setup_parser(remote_file): parser = ExcelParser() # Sample data for the mock Excel file - data = pd.DataFrame({ - "column1": [1, 2, 3], - "column2": ["a", "b", "c"], - "column3": [True, False, True], - "column4": pd.to_datetime(["2021-01-01", "2022-01-01", "2023-01-01"]), - }) + data = pd.DataFrame( + { + "column1": [1, 2, 3], + "column2": ["a", "b", "c"], + "column3": [True, False, True], + "column4": pd.to_datetime(["2021-01-01", "2022-01-01", "2023-01-01"]), + } + ) # Convert the DataFrame to an Excel byte stream excel_bytes = BytesIO() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_parquet_parser.py b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_parquet_parser.py index 1fa2dcbf66fce..c4768facc7dd3 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_parquet_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/file_types/test_parquet_parser.py @@ -238,7 +238,8 @@ def test_value_dictionary() -> None: pytest.param(pa.decimal256(2), _decimal_as_float_parquet_format, id="test_decimal256_as_float"), pytest.param(pa.map_(pa.int32(), pa.int32()), _default_parquet_format, id="test_map"), pytest.param(pa.null(), _default_parquet_format, id="test_null"), - ]) + ], +) def test_null_value_does_not_throw(parquet_type, parquet_format) -> None: pyarrow_value = pa.scalar(None, type=parquet_type) assert ParquetParser._to_output_value(pyarrow_value, parquet_format) is None diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py b/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py index a6de1b290c565..0a26819112117 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/in_memory_files_source.py @@ -16,7 +16,7 @@ import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -from airbyte_cdk.models import ConfiguredAirbyteCatalog +from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteCatalogSerializer from airbyte_cdk.sources.file_based.availability_strategy import AbstractFileBasedAvailabilityStrategy, DefaultFileBasedAvailabilityStrategy from airbyte_cdk.sources.file_based.config.abstract_file_based_spec import AbstractFileBasedSpec from airbyte_cdk.sources.file_based.discovery_policy import AbstractDiscoveryPolicy, DefaultDiscoveryPolicy @@ -53,7 +53,7 @@ def __init__( self.files = files self.file_type = file_type self.catalog = catalog - self.configured_catalog = ConfiguredAirbyteCatalog(streams=self.catalog["streams"]) if self.catalog else None + self.configured_catalog = ConfiguredAirbyteCatalogSerializer.load(self.catalog) if self.catalog else None self.config = config self.state = state @@ -224,8 +224,8 @@ def _make_file_contents(self, file_name: str) -> bytes: df = pd.DataFrame(contents) with io.BytesIO() as fp: - writer = pd.ExcelWriter(fp, engine='xlsxwriter') - df.to_excel(writer, index=False, sheet_name='Sheet1') + writer = pd.ExcelWriter(fp, engine="xlsxwriter") + df.to_excel(writer, index=False, sheet_name="Sheet1") writer._save() fp.seek(0) return fp.read() diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py index 0b662519f276b..e5a7ee4194523 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/concurrent_incremental_scenarios.py @@ -2227,12 +2227,12 @@ .set_expected_records( [ { - "history": { - "b.csv": "2023-06-05T03:54:07.000000Z", - "c.csv": "2023-06-05T03:54:07.000000Z", - "d.csv": "2023-06-05T03:54:07.000000Z", - }, - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv", + "history": { + "b.csv": "2023-06-05T03:54:07.000000Z", + "c.csv": "2023-06-05T03:54:07.000000Z", + "d.csv": "2023-06-05T03:54:07.000000Z", + }, + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z_d.csv", } ] ) diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py index 723550cc36d7d..dc0a97bc1cf2d 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/csv_scenarios.py @@ -2,12 +2,11 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -from airbyte_cdk.models import AirbyteAnalyticsTraceMessage +from airbyte_cdk.models import AirbyteAnalyticsTraceMessage, SyncMode from airbyte_cdk.sources.file_based.config.csv_format import CsvFormat from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError from airbyte_cdk.test.catalog_builder import CatalogBuilder from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from airbyte_protocol.models import SyncMode from unit_tests.sources.file_based.helpers import EmptySchemaParser, LowInferenceLimitDiscoveryPolicy from unit_tests.sources.file_based.in_memory_files_source import InMemoryFilesSource from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder @@ -417,8 +416,8 @@ "properties": { "filetype": {"title": "Filetype", "default": "excel", "const": "excel", "type": "string"} }, - "required": ["filetype"] - } + "required": ["filetype"], + }, ], }, "schemaless": { @@ -432,7 +431,7 @@ "description": "The number of resent files which will be used to discover the schema for this stream.", "exclusiveMinimum": 0, "type": "integer", - } + }, }, "required": ["name", "format"], }, @@ -440,6 +439,8 @@ }, "required": ["streams"], }, + "supportsDBT": False, + "supportsNormalization": False, } ) .set_expected_catalog( @@ -505,7 +506,7 @@ "format": {"filetype": "csv"}, "globs": ["b.csv"], "validation_policy": "Emit Record", - } + }, ] } ) @@ -568,50 +569,52 @@ "source_defined_cursor": True, "supported_sync_modes": ["full_refresh", "incremental"], "is_resumable": True, - } + }, ] } ) - .set_expected_records([ - { - "data": { - "col1": "val11a", - "col2": "val12a", - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv", + .set_expected_records( + [ + { + "data": { + "col1": "val11a", + "col2": "val12a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", }, - "stream": "stream1", - }, - { - "data": { - "col1": "val21a", - "col2": "val22a", - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "a.csv", + { + "data": { + "col1": "val21a", + "col2": "val22a", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "a.csv", + }, + "stream": "stream1", }, - "stream": "stream1", - }, - { - "data": { - "col1": "val11b", - "col2": "val12b", - "col3": "val13b", - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv", + { + "data": { + "col1": "val11b", + "col2": "val12b", + "col3": "val13b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream2", }, - "stream": "stream2", - }, - { - "data": { - "col1": "val21b", - "col2": "val22b", - "col3": "val23b", - "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", - "_ab_source_file_url": "b.csv", + { + "data": { + "col1": "val21b", + "col2": "val22b", + "col3": "val23b", + "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", + "_ab_source_file_url": "b.csv", + }, + "stream": "stream2", }, - "stream": "stream2", - }, - ]) + ] + ) .set_expected_analytics( [ AirbyteAnalyticsTraceMessage(type="file-cdk-csv-stream-count", value="2"), @@ -2094,7 +2097,6 @@ { "data": { "col1": "2", - "col2": None, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv", }, @@ -2305,7 +2307,6 @@ { "data": { "col1": "2", - "col2": None, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv", }, @@ -3174,7 +3175,6 @@ [ { "data": { - "col1": None, "col2": "na", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv", @@ -3316,11 +3316,7 @@ "start_date": "2023-06-10T03:54:07.000000Z", } ) - .set_source_builder( - FileBasedSourceBuilder() - .set_files({}) - .set_file_type("csv") - ) + .set_source_builder(FileBasedSourceBuilder().set_files({}).set_file_type("csv")) .set_expected_check_status("FAILED") .set_expected_catalog( { diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/excel_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/excel_scenarios.py index f92c8420099d1..6653296535d58 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/excel_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/excel_scenarios.py @@ -22,7 +22,11 @@ "a.xlsx": { "contents": [ {"col_double": 20.02, "col_string": "Robbers", "col_album": "The 1975"}, - {"col_double": 20.23, "col_string": "Somebody Else", "col_album": "I Like It When You Sleep, for You Are So Beautiful yet So Unaware of It"}, + { + "col_double": 20.23, + "col_string": "Somebody Else", + "col_album": "I Like It When You Sleep, for You Are So Beautiful yet So Unaware of It", + }, ], "last_modified": "2023-06-05T03:54:07.000Z", }, @@ -60,14 +64,22 @@ {"col_title": "White Lies", "col_album": "IN_RETURN", "col_year": 2014, "col_vocals": True}, {"col_title": "Wide Awake", "col_album": "THE_LAST_GOODBYE", "col_year": 2022, "col_vocals": True}, ], - "last_modified": "2023-06-05T03:54:07.000Z" + "last_modified": "2023-06-05T03:54:07.000Z", }, "california_festivals.xlsx": { "contents": [ - {"col_name": "Lightning in a Bottle", "col_location": {"country": "USA", "state": "California", "city": "Buena Vista Lake"}, "col_attendance": 18000}, - {"col_name": "Outside Lands", "col_location": {"country": "USA", "state": "California", "city": "San Francisco"}, "col_attendance": 220000}, + { + "col_name": "Lightning in a Bottle", + "col_location": {"country": "USA", "state": "California", "city": "Buena Vista Lake"}, + "col_attendance": 18000, + }, + { + "col_name": "Outside Lands", + "col_location": {"country": "USA", "state": "California", "city": "San Francisco"}, + "col_attendance": 220000, + }, ], - "last_modified": "2023-06-06T03:54:07.000Z" + "last_modified": "2023-06-06T03:54:07.000Z", }, } @@ -257,7 +269,7 @@ "col_long": 1992, "col_float": 999.09723456, "col_string": "Love It If We Made It", - "col_date": "2022-05-29T00:00:00", + "col_date": "2022-05-29T00:00:00.000000", "col_time_millis": "06:00:00.456000", "col_time_micros": "12:00:00.456789", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", @@ -407,7 +419,7 @@ "type": "object", "properties": { "col_name": {"type": ["null", "string"]}, - "col_location": {"type": ["null", "string"]}, + "col_location": {"type": ["null", "string"]}, "col_attendance": {"type": ["null", "number"]}, "_ab_source_file_last_modified": {"type": "string"}, "_ab_source_file_url": {"type": "string"}, diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/file_based_source_builder.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/file_based_source_builder.py index f3d72ab67e7ab..6675df380c7c5 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/file_based_source_builder.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/file_based_source_builder.py @@ -33,7 +33,9 @@ def __init__(self) -> None: self._config: Optional[Mapping[str, Any]] = None self._state: Optional[TState] = None - def build(self, configured_catalog: Optional[Mapping[str, Any]], config: Optional[Mapping[str, Any]], state: Optional[TState]) -> InMemoryFilesSource: + def build( + self, configured_catalog: Optional[Mapping[str, Any]], config: Optional[Mapping[str, Any]], state: Optional[TState] + ) -> InMemoryFilesSource: if self._file_type is None: raise ValueError("file_type is not set") return InMemoryFilesSource( diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py index 25811a9e60adb..8158225ac8f42 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/scenario_builder.py @@ -6,10 +6,15 @@ from dataclasses import dataclass, field from typing import Any, Generic, List, Mapping, Optional, Set, Tuple, Type, TypeVar -from airbyte_cdk.models import AirbyteAnalyticsTraceMessage, AirbyteStateMessage, SyncMode +from airbyte_cdk.models import ( + AirbyteAnalyticsTraceMessage, + AirbyteStateMessageSerializer, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, + SyncMode, +) from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.source import TState -from airbyte_protocol.models import ConfiguredAirbyteCatalog @dataclass @@ -27,7 +32,9 @@ class SourceBuilder(ABC, Generic[SourceType]): """ @abstractmethod - def build(self, configured_catalog: Optional[Mapping[str, Any]], config: Optional[Mapping[str, Any]], state: Optional[TState]) -> SourceType: + def build( + self, configured_catalog: Optional[Mapping[str, Any]], config: Optional[Mapping[str, Any]], state: Optional[TState] + ) -> SourceType: raise NotImplementedError() @@ -78,7 +85,7 @@ def configured_catalog(self, sync_mode: SyncMode) -> Optional[Mapping[str, Any]] # exception to be raised as part of the actual check/discover/read commands # Note that to avoid a breaking change, we still attempt to automatically generate the catalog based on the streams if self.catalog: - return self.catalog.dict() # type: ignore # dict() is not typed + return ConfiguredAirbyteCatalogSerializer.dump(self.catalog) catalog: Mapping[str, Any] = {"streams": []} for stream in catalog["streams"]: @@ -90,7 +97,7 @@ def configured_catalog(self, sync_mode: SyncMode) -> Optional[Mapping[str, Any]] "supported_sync_modes": [sync_mode.value], }, "sync_mode": sync_mode.value, - "destination_sync_mode": "append" + "destination_sync_mode": "append", } ) @@ -192,7 +199,9 @@ def build(self) -> "TestScenario[SourceType]": if self.source_builder is None: raise ValueError("source_builder is not set") if self._incremental_scenario_config and self._incremental_scenario_config.input_state: - state = [AirbyteStateMessage.parse_obj(s) for s in self._incremental_scenario_config.input_state] + state = [ + AirbyteStateMessageSerializer.load(s) if isinstance(s, dict) else s for s in self._incremental_scenario_config.input_state + ] else: state = None source = self.source_builder.build( diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py index 4257da83e6049..97c0c491510a9 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/unstructured_scenarios.py @@ -90,7 +90,6 @@ "content": "# Title 1\n\n## Title 2\n\n### Title 3\n\n#### Title 4\n\n##### Title 5\n\n###### Title 6\n\n", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.md", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -100,7 +99,6 @@ "content": "Just some text", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b.md", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -110,7 +108,6 @@ "content": "Detected via mime type", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "c", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -173,7 +170,6 @@ "content": "Just some raw text", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.txt", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -183,7 +179,6 @@ "content": "Detected via mime type", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "b", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -290,7 +285,6 @@ { "data": { "document_key": "a.csv", - "content": None, "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.csv", "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=a.csv message=File type FileType.CSV is not supported. Supported file types are FileType.MD, FileType.PDF, FileType.DOCX, FileType.PPTX, FileType.TXT", @@ -358,7 +352,6 @@ "content": "A harmless markdown file", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "a.md", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -439,7 +432,6 @@ "content": "# Hello World", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "sample.pdf", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -449,7 +441,6 @@ "content": "# Content", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", "_ab_source_file_url": "sample.docx", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -459,7 +450,6 @@ "content": "# Title", "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", "_ab_source_file_url": "sample.pptx", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -515,7 +505,6 @@ { "data": { "document_key": "sample.pdf", - "content": None, "_ab_source_file_parse_error": "Error parsing record. This could be due to a mismatch between the config's file type and the actual file type, or because the file or record is not parseable. Contact Support if you need assistance.\nfilename=sample.pdf message=No /Root object! - Is this really a PDF?", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "sample.pdf", @@ -587,7 +576,6 @@ "content": "# Hello World", "_ab_source_file_last_modified": "2023-06-05T03:54:07.000000Z", "_ab_source_file_url": "pdf_without_extension", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -597,7 +585,6 @@ "content": "# Content", "_ab_source_file_last_modified": "2023-06-06T03:54:07.000000Z", "_ab_source_file_url": "docx_without_extension", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, @@ -607,7 +594,6 @@ "content": "# Title", "_ab_source_file_last_modified": "2023-06-07T03:54:07.000000Z", "_ab_source_file_url": "pptx_without_extension", - "_ab_source_file_parse_error": None, }, "stream": "stream1", }, diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py index e83ab53457879..3c10e701c6292 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/scenarios/user_input_schema_scenarios.py @@ -3,9 +3,9 @@ # +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources.file_based.exceptions import ConfigValidationError, FileBasedSourceError from airbyte_cdk.test.catalog_builder import CatalogBuilder -from airbyte_protocol.models import SyncMode from unit_tests.sources.file_based.scenarios.file_based_source_builder import FileBasedSourceBuilder from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenarioBuilder diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py b/airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py index f8122da702bba..96c907901a388 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/stream/concurrent/test_file_based_concurrent_cursor.py @@ -30,7 +30,7 @@ def _make_cursor(input_state: Optional[MutableMapping[str, Any]]) -> FileBasedCo None, input_state, MagicMock(), - ConnectorStateManager(state=[AirbyteStateMessage.parse_obj(input_state)] if input_state is not None else None), + ConnectorStateManager(state=[AirbyteStateMessage(input_state)] if input_state is not None else None), CursorField(FileBasedConcurrentCursor.CURSOR_FIELD), ) return cursor diff --git a/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py b/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py index 119dab4bb6fb6..9563fe0af9c19 100644 --- a/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py +++ b/airbyte-cdk/python/unit_tests/sources/file_based/test_scenarios.py @@ -11,14 +11,13 @@ from _pytest.capture import CaptureFixture from _pytest.reports import ExceptionInfo from airbyte_cdk.entrypoint import launch -from airbyte_cdk.models import AirbyteAnalyticsTraceMessage, SyncMode +from airbyte_cdk.models import AirbyteAnalyticsTraceMessage, AirbyteLogMessage, AirbyteMessage, ConfiguredAirbyteCatalogSerializer, SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.file_based.stream.concurrent.cursor import AbstractConcurrentFileBasedCursor from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput from airbyte_cdk.test.entrypoint_wrapper import read as entrypoint_read from airbyte_cdk.utils import message_utils from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from airbyte_protocol.models import AirbyteLogMessage, AirbyteMessage, ConfiguredAirbyteCatalog from unit_tests.sources.file_based.scenarios.scenario_builder import TestScenario @@ -112,10 +111,10 @@ def _verify_read_output(output: EntrypointOutput, scenario: TestScenario[Abstrac if hasattr(scenario.source, "cursor_cls") and issubclass(scenario.source.cursor_cls, AbstractConcurrentFileBasedCursor): # Only check the last state emitted because we don't know the order the others will be in. # This may be needed for non-file-based concurrent scenarios too. - assert states[-1].state.stream.stream_state.dict() == expected_states[-1] + assert {k: v for k, v in states[-1].state.stream.stream_state.__dict__.items()} == expected_states[-1] else: for actual, expected in zip(states, expected_states): # states should be emitted in sorted order - assert actual.state.stream.stream_state.dict() == expected + assert {k: v for k, v in actual.state.stream.stream_state.__dict__.items()} == expected if scenario.expected_logs: read_logs = scenario.expected_logs.get("read") @@ -138,8 +137,7 @@ def _verify_state_record_counts(records: List[AirbyteMessage], states: List[Airb for state_message in states: stream_descriptor = message_utils.get_stream_descriptor(state_message) state_record_count_sums[stream_descriptor] = ( - state_record_count_sums.get(stream_descriptor, 0) - + state_message.state.sourceStats.recordCount + state_record_count_sums.get(stream_descriptor, 0) + state_message.state.sourceStats.recordCount ) for stream, actual_count in actual_record_counts.items(): @@ -154,8 +152,8 @@ def _verify_state_record_counts(records: List[AirbyteMessage], states: List[Airb def _verify_analytics(analytics: List[AirbyteMessage], expected_analytics: Optional[List[AirbyteAnalyticsTraceMessage]]) -> None: if expected_analytics: assert len(analytics) == len( - expected_analytics), \ - f"Number of actual analytics messages ({len(analytics)}) did not match expected ({len(expected_analytics)})" + expected_analytics + ), f"Number of actual analytics messages ({len(analytics)}) did not match expected ({len(expected_analytics)})" for actual, expected in zip(analytics, expected_analytics): actual_type, actual_value = actual.trace.analytics.type, actual.trace.analytics.value expected_type = expected.type @@ -228,7 +226,7 @@ def read(scenario: TestScenario[AbstractSource]) -> EntrypointOutput: return entrypoint_read( scenario.source, scenario.config, - ConfiguredAirbyteCatalog.parse_obj(scenario.configured_catalog(SyncMode.full_refresh)), + ConfiguredAirbyteCatalogSerializer.load(scenario.configured_catalog(SyncMode.full_refresh)), ) @@ -236,7 +234,7 @@ def read_with_state(scenario: TestScenario[AbstractSource]) -> EntrypointOutput: return entrypoint_read( scenario.source, scenario.config, - ConfiguredAirbyteCatalog.parse_obj(scenario.configured_catalog(SyncMode.incremental)), + ConfiguredAirbyteCatalogSerializer.load(scenario.configured_catalog(SyncMode.incremental)), scenario.input_state(), ) diff --git a/airbyte-cdk/python/unit_tests/sources/message/test_repository.py b/airbyte-cdk/python/unit_tests/sources/message/test_repository.py index 95c8f96a154d8..48778b657cb82 100644 --- a/airbyte-cdk/python/unit_tests/sources/message/test_repository.py +++ b/airbyte-cdk/python/unit_tests/sources/message/test_repository.py @@ -12,7 +12,6 @@ MessageRepository, NoopMessageRepository, ) -from pydantic.error_wrappers import ValidationError A_CONTROL = AirbyteControlMessage( type=OrchestratorType.CONNECTOR_CONFIG, @@ -90,14 +89,6 @@ def test_given_unknown_log_level_as_threshold_when_log_message_then_allow_messag repo.log_message(Level.DEBUG, lambda: {"message": "this is a log message"}) assert list(repo.consume_queue()) - def test_given_unknown_log_level_for_log_when_log_message_then_raise_error(self): - """ - Pydantic will fail if the log level is unknown but on our side, we should try to log at least - """ - repo = InMemoryMessageRepository(Level.ERROR) - with pytest.raises(ValidationError): - repo.log_message(UNKNOWN_LEVEL, lambda: {"message": "this is a log message"}) - class TestNoopMessageRepository: def test_given_message_emitted_when_consume_queue_then_return_empty(self): diff --git a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/mock_source_fixture.py b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/mock_source_fixture.py index ac7aa179b6359..ece5039ba465c 100644 --- a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/mock_source_fixture.py +++ b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/mock_source_fixture.py @@ -9,12 +9,12 @@ import pendulum import requests +from airbyte_cdk.models import ConnectorSpecification, SyncMode from airbyte_cdk.sources import AbstractSource, Source from airbyte_cdk.sources.streams import CheckpointMixin, IncrementalMixin, Stream from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.availability_strategy import HttpAvailabilityStrategy -from airbyte_protocol.models import ConnectorSpecification, SyncMode from requests import HTTPError @@ -23,10 +23,12 @@ class FixtureAvailabilityStrategy(HttpAvailabilityStrategy): Inherit from HttpAvailabilityStrategy with slight modification to 403 error message. """ - def reasons_for_unavailable_status_codes(self, stream: Stream, logger: logging.Logger, source: Source, error: HTTPError) -> Dict[int, str]: + def reasons_for_unavailable_status_codes( + self, stream: Stream, logger: logging.Logger, source: Source, error: HTTPError + ) -> Dict[int, str]: reasons_for_codes: Dict[int, str] = { requests.codes.FORBIDDEN: "This is likely due to insufficient permissions for your Notion integration. " - "Please make sure your integration has read access for the resources you are trying to sync" + "Please make sure your integration has read access for the resources you are trying to sync" } return reasons_for_codes @@ -94,28 +96,16 @@ def get_json_schema(self) -> Mapping[str, Any]: "type": "object", "additionalProperties": True, "properties": { - "type": { - "type": "string" - }, - "id": { - "type": "string" - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "first_name": { - "type": "string" - }, - "last_name": { - "type": "string" - } - } + "type": {"type": "string"}, + "id": {"type": "string"}, + "created_at": {"type": "string", "format": "date-time"}, + "first_name": {"type": "string"}, + "last_name": {"type": "string"}, + }, } class Planets(IncrementalIntegrationStream): - def __init__(self, **kwargs): super().__init__(**kwargs) self._state: MutableMapping[str, Any] = {} @@ -129,20 +119,11 @@ def get_json_schema(self) -> Mapping[str, Any]: "type": "object", "additionalProperties": True, "properties": { - "type": { - "type": "string" - }, - "id": { - "type": "string" - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "name": { - "type": "string" - } - } + "type": {"type": "string"}, + "id": {"type": "string"}, + "created_at": {"type": "string", "format": "date-time"}, + "name": {"type": "string"}, + }, } def request_params( @@ -151,10 +132,7 @@ def request_params( stream_slice: Optional[Mapping[str, Any]] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> MutableMapping[str, Any]: - return { - "start_date": stream_slice.get("start_date"), - "end_date": stream_slice.get("end_date") - } + return {"start_date": stream_slice.get("start_date"), "end_date": stream_slice.get("end_date")} def stream_slices( self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None @@ -170,7 +148,10 @@ def stream_slices( while start_date < end_date: end_date_slice = min(start_date.add(days=7), end_date) - date_slice = {"start_date": start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), "end_date": end_date_slice.strftime("%Y-%m-%dT%H:%M:%SZ")} + date_slice = { + "start_date": start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), + "end_date": end_date_slice.strftime("%Y-%m-%dT%H:%M:%SZ"), + } date_slices.append(date_slice) start_date = end_date_slice @@ -195,20 +176,11 @@ def get_json_schema(self) -> Mapping[str, Any]: "type": "object", "additionalProperties": True, "properties": { - "type": { - "type": "string" - }, - "id": { - "type": "string" - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "quote": { - "type": "string" - } - } + "type": {"type": "string"}, + "id": {"type": "string"}, + "created_at": {"type": "string", "format": "date-time"}, + "quote": {"type": "string"}, + }, } def get_updated_state( @@ -221,11 +193,11 @@ def get_updated_state( return {} def read_records( - self, - sync_mode: SyncMode, - cursor_field: Optional[List[str]] = None, - stream_slice: Optional[Mapping[str, Any]] = None, - stream_state: Optional[Mapping[str, Any]] = None, + self, + sync_mode: SyncMode, + cursor_field: Optional[List[str]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + stream_state: Optional[Mapping[str, Any]] = None, ) -> Iterable[StreamData]: yield from super().read_records(sync_mode, cursor_field, stream_slice, stream_state) @@ -235,10 +207,7 @@ def request_params( stream_slice: Optional[Mapping[str, Any]] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> MutableMapping[str, Any]: - return { - "start_date": stream_slice.get("start_date"), - "end_date": stream_slice.get("end_date") - } + return {"start_date": stream_slice.get("start_date"), "end_date": stream_slice.get("end_date")} def stream_slices( self, *, sync_mode: SyncMode, cursor_field: Optional[List[str]] = None, stream_state: Optional[Mapping[str, Any]] = None @@ -254,7 +223,10 @@ def stream_slices( while start_date < end_date: end_date_slice = min(start_date.add(days=7), end_date) - date_slice = {"start_date": start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), "end_date": end_date_slice.strftime("%Y-%m-%dT%H:%M:%SZ")} + date_slice = { + "start_date": start_date.strftime("%Y-%m-%dT%H:%M:%SZ"), + "end_date": end_date_slice.strftime("%Y-%m-%dT%H:%M:%SZ"), + } date_slices.append(date_slice) start_date = end_date_slice @@ -272,20 +244,11 @@ def get_json_schema(self) -> Mapping[str, Any]: "type": "object", "additionalProperties": True, "properties": { - "type": { - "type": "string" - }, - "id": { - "type": "string" - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "divide_category": { - "type": "string" - } - } + "type": {"type": "string"}, + "id": {"type": "string"}, + "created_at": {"type": "string", "format": "date-time"}, + "divide_category": {"type": "string"}, + }, } def stream_slices( @@ -319,23 +282,12 @@ def get_json_schema(self) -> Mapping[str, Any]: "type": "object", "additionalProperties": True, "properties": { - "type": { - "type": "string" - }, - "id": { - "type": "string" - }, - "created_at": { - "type": "string", - "format": "date-time" - }, - "name": { - "type": "string" - }, - "album": { - "type": "string" - } - } + "type": {"type": "string"}, + "id": {"type": "string"}, + "created_at": {"type": "string", "format": "date-time"}, + "name": {"type": "string"}, + "album": {"type": "string"}, + }, } @property @@ -360,9 +312,7 @@ def request_params( stream_slice: Optional[Mapping[str, Any]] = None, next_page_token: Optional[Mapping[str, Any]] = None, ) -> MutableMapping[str, Any]: - return { - "page": next_page_token.get("page") - } + return {"page": next_page_token.get("page")} def read_records( self, @@ -433,7 +383,7 @@ def spec(self, logger: logging.Logger) -> ConnectorSpecification: "pattern_descriptor": "YYYY-MM-DDTHH:MM:SS.000Z", "examples": ["2020-11-16T00:00:00.000Z"], "type": "string", - "format": "date-time" + "format": "date-time", } } } diff --git a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_helpers/airbyte_message_assertions.py b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_helpers/airbyte_message_assertions.py index 52affbb6d76e7..04b65594cf017 100644 --- a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_helpers/airbyte_message_assertions.py +++ b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_helpers/airbyte_message_assertions.py @@ -5,13 +5,16 @@ from typing import List import pytest -from airbyte_cdk.models import AirbyteMessage, Type -from airbyte_protocol.models import AirbyteStreamStatus +from airbyte_cdk.models import AirbyteMessage, AirbyteStreamStatus, Type def emits_successful_sync_status_messages(status_messages: List[AirbyteStreamStatus]) -> bool: - return (len(status_messages) == 3 and status_messages[0] == AirbyteStreamStatus.STARTED - and status_messages[1] == AirbyteStreamStatus.RUNNING and status_messages[2] == AirbyteStreamStatus.COMPLETE) + return ( + len(status_messages) == 3 + and status_messages[0] == AirbyteStreamStatus.STARTED + and status_messages[1] == AirbyteStreamStatus.RUNNING + and status_messages[2] == AirbyteStreamStatus.COMPLETE + ) def validate_message_order(expected_message_order: List[Type], messages: List[AirbyteMessage]): @@ -20,4 +23,6 @@ def validate_message_order(expected_message_order: List[Type], messages: List[Ai for i, message in enumerate(messages): if message.type != expected_message_order[i]: - pytest.fail(f"At index {i} actual message type {message.type.name} did not match expected message type {expected_message_order[i].name}") + pytest.fail( + f"At index {i} actual message type {message.type.name} did not match expected message type {expected_message_order[i].name}" + ) diff --git a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_mock_server_abstract_source.py b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_mock_server_abstract_source.py index 6e68db646675e..c7fd2cef433ed 100644 --- a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_mock_server_abstract_source.py +++ b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_mock_server_abstract_source.py @@ -114,16 +114,7 @@ def _create_justice_songs_request() -> RequestBuilder: return RequestBuilder.justice_songs_endpoint() -RESPONSE_TEMPLATE = { - "object": "list", - "has_more": False, - "data": [ - { - "id": "123", - "created_at": "2024-01-01T07:04:28.000Z" - } - ] -} +RESPONSE_TEMPLATE = {"object": "list", "has_more": False, "data": [{"id": "123", "created_at": "2024-01-01T07:04:28.000Z"}]} USER_TEMPLATE = { "object": "list", @@ -135,7 +126,7 @@ def _create_justice_songs_request() -> RequestBuilder: "first_name": "Paul", "last_name": "Atreides", } - ] + ], } PLANET_TEMPLATE = { @@ -147,7 +138,7 @@ def _create_justice_songs_request() -> RequestBuilder: "created_at": "2024-01-01T07:04:28.000Z", "name": "Giedi Prime", } - ] + ], } LEGACY_TEMPLATE = { @@ -159,7 +150,7 @@ def _create_justice_songs_request() -> RequestBuilder: "created_at": "2024-02-01T07:04:28.000Z", "quote": "What do you leave behind?", } - ] + ], } DIVIDER_TEMPLATE = { @@ -171,7 +162,7 @@ def _create_justice_songs_request() -> RequestBuilder: "created_at": "2024-02-01T07:04:28.000Z", "divide_category": "dukes", } - ] + ], } @@ -190,8 +181,8 @@ def _create_justice_songs_request() -> RequestBuilder: "created_at": "2024-02-01T07:04:28.000Z", "name": "dukes", "album": "", - } - ] + }, + ], } @@ -208,7 +199,7 @@ def _create_response(pagination_has_more: bool = False) -> HttpResponseBuilder: return create_response_builder( response_template=RESPONSE_TEMPLATE, records_path=FieldPath("data"), - pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more"), pagination_has_more) + pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more"), pagination_has_more), ) @@ -225,9 +216,7 @@ class FullRefreshStreamTest(TestCase): @HttpMocker() def test_full_refresh_sync(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} http_mocker.get( _create_users_request().build(), @@ -248,17 +237,15 @@ def test_full_refresh_sync(self, http_mocker): @HttpMocker() def test_substream_resumable_full_refresh_with_parent_slices(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} expected_first_substream_per_stream_state = [ - {'partition': {'divide_category': 'dukes'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}, + {"partition": {"divide_category": "dukes"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, ] expected_second_substream_per_stream_state = [ - {'partition': {'divide_category': 'dukes'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}, - {'partition': {'divide_category': 'mentats'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}, + {"partition": {"divide_category": "dukes"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"divide_category": "mentats"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, ] http_mocker.get( @@ -277,10 +264,16 @@ def test_substream_resumable_full_refresh_with_parent_slices(self, http_mocker): assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("dividers")) assert len(actual_messages.records) == 4 assert len(actual_messages.state_messages) == 2 - validate_message_order([Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], actual_messages.records_and_state_messages) - assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(states=expected_first_substream_per_stream_state) + validate_message_order( + [Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], actual_messages.records_and_state_messages + ) + assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob( + states=expected_first_substream_per_stream_state + ) assert actual_messages.state_messages[0].state.sourceStats.recordCount == 2.0 - assert actual_messages.state_messages[1].state.stream.stream_state == AirbyteStateBlob(states=expected_second_substream_per_stream_state) + assert actual_messages.state_messages[1].state.stream.stream_state == AirbyteStateBlob( + states=expected_second_substream_per_stream_state + ) assert actual_messages.state_messages[1].state.sourceStats.recordCount == 2.0 @@ -289,20 +282,25 @@ class IncrementalStreamTest(TestCase): @HttpMocker() def test_incremental_sync(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} last_record_date_0 = (start_datetime + timedelta(days=4)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_planets_request().with_start_date(start_datetime).with_end_date(start_datetime + timedelta(days=7)).build(), - _create_response().with_record(record=_create_record("planets").with_cursor(last_record_date_0)).with_record(record=_create_record("planets").with_cursor(last_record_date_0)).with_record(record=_create_record("planets").with_cursor(last_record_date_0)).build(), + _create_response() + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .build(), ) last_record_date_1 = (_NOW - timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_planets_request().with_start_date(start_datetime + timedelta(days=7)).with_end_date(_NOW).build(), - _create_response().with_record(record=_create_record("planets").with_cursor(last_record_date_1)).with_record(record=_create_record("planets").with_cursor(last_record_date_1)).build(), + _create_response() + .with_record(record=_create_record("planets").with_cursor(last_record_date_1)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_1)) + .build(), ) source = SourceFixture() @@ -311,7 +309,10 @@ def test_incremental_sync(self, http_mocker): assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("planets")) assert len(actual_messages.records) == 5 assert len(actual_messages.state_messages) == 2 - validate_message_order([Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], actual_messages.records_and_state_messages) + validate_message_order( + [Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], + actual_messages.records_and_state_messages, + ) assert actual_messages.state_messages[0].state.stream.stream_descriptor.name == "planets" assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(created_at=last_record_date_0) assert actual_messages.state_messages[0].state.sourceStats.recordCount == 3.0 @@ -322,20 +323,25 @@ def test_incremental_sync(self, http_mocker): @HttpMocker() def test_incremental_running_as_full_refresh(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} last_record_date_0 = (start_datetime + timedelta(days=4)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_planets_request().with_start_date(start_datetime).with_end_date(start_datetime + timedelta(days=7)).build(), - _create_response().with_record(record=_create_record("planets").with_cursor(last_record_date_0)).with_record(record=_create_record("planets").with_cursor(last_record_date_0)).with_record(record=_create_record("planets").with_cursor(last_record_date_0)).build(), + _create_response() + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .build(), ) last_record_date_1 = (_NOW - timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_planets_request().with_start_date(start_datetime + timedelta(days=7)).with_end_date(_NOW).build(), - _create_response().with_record(record=_create_record("planets").with_cursor(last_record_date_1)).with_record(record=_create_record("planets").with_cursor(last_record_date_1)).build(), + _create_response() + .with_record(record=_create_record("planets").with_cursor(last_record_date_1)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_1)) + .build(), ) source = SourceFixture() @@ -344,7 +350,10 @@ def test_incremental_running_as_full_refresh(self, http_mocker): assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("planets")) assert len(actual_messages.records) == 5 assert len(actual_messages.state_messages) == 2 - validate_message_order([Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], actual_messages.records_and_state_messages) + validate_message_order( + [Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], + actual_messages.records_and_state_messages, + ) assert actual_messages.state_messages[0].state.stream.stream_descriptor.name == "planets" assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(created_at=last_record_date_0) @@ -356,20 +365,25 @@ def test_incremental_running_as_full_refresh(self, http_mocker): @HttpMocker() def test_legacy_incremental_sync(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} last_record_date_0 = (start_datetime + timedelta(days=4)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_legacies_request().with_start_date(start_datetime).with_end_date(start_datetime + timedelta(days=7)).build(), - _create_response().with_record(record=_create_record("legacies").with_cursor(last_record_date_0)).with_record(record=_create_record("legacies").with_cursor(last_record_date_0)).with_record(record=_create_record("legacies").with_cursor(last_record_date_0)).build(), + _create_response() + .with_record(record=_create_record("legacies").with_cursor(last_record_date_0)) + .with_record(record=_create_record("legacies").with_cursor(last_record_date_0)) + .with_record(record=_create_record("legacies").with_cursor(last_record_date_0)) + .build(), ) last_record_date_1 = (_NOW - timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_legacies_request().with_start_date(start_datetime + timedelta(days=7)).with_end_date(_NOW).build(), - _create_response().with_record(record=_create_record("legacies").with_cursor(last_record_date_1)).with_record(record=_create_record("legacies").with_cursor(last_record_date_1)).build(), + _create_response() + .with_record(record=_create_record("legacies").with_cursor(last_record_date_1)) + .with_record(record=_create_record("legacies").with_cursor(last_record_date_1)) + .build(), ) source = SourceFixture() @@ -378,7 +392,10 @@ def test_legacy_incremental_sync(self, http_mocker): assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("legacies")) assert len(actual_messages.records) == 5 assert len(actual_messages.state_messages) == 2 - validate_message_order([Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], actual_messages.records_and_state_messages) + validate_message_order( + [Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], + actual_messages.records_and_state_messages, + ) assert actual_messages.state_messages[0].state.stream.stream_descriptor.name == "legacies" assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(created_at=last_record_date_0) assert actual_messages.state_messages[0].state.sourceStats.recordCount == 3.0 @@ -389,9 +406,7 @@ def test_legacy_incremental_sync(self, http_mocker): @HttpMocker() def test_legacy_no_records_retains_incoming_state(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} last_record_date_1 = (_NOW - timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( @@ -412,9 +427,7 @@ def test_legacy_no_records_retains_incoming_state(self, http_mocker): @HttpMocker() def test_legacy_no_slices_retains_incoming_state(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} last_record_date_1 = _NOW.strftime("%Y-%m-%dT%H:%M:%SZ") @@ -434,17 +447,15 @@ class MultipleStreamTest(TestCase): @HttpMocker() def test_incremental_and_full_refresh_streams(self, http_mocker): start_datetime = _NOW - timedelta(days=14) - config = { - "start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") - } + config = {"start_date": start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ")} expected_first_substream_per_stream_state = [ - {'partition': {'divide_category': 'dukes'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}, + {"partition": {"divide_category": "dukes"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, ] expected_second_substream_per_stream_state = [ - {'partition': {'divide_category': 'dukes'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}, - {'partition': {'divide_category': 'mentats'}, 'cursor': {'__ab_full_refresh_sync_complete': True}}, + {"partition": {"divide_category": "dukes"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"divide_category": "mentats"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, ] # Mocks for users full refresh stream @@ -457,13 +468,20 @@ def test_incremental_and_full_refresh_streams(self, http_mocker): last_record_date_0 = (start_datetime + timedelta(days=4)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_planets_request().with_start_date(start_datetime).with_end_date(start_datetime + timedelta(days=7)).build(), - _create_response().with_record(record=_create_record("planets").with_cursor(last_record_date_0)).with_record(record=_create_record("planets").with_cursor(last_record_date_0)).with_record(record=_create_record("planets").with_cursor(last_record_date_0)).build(), + _create_response() + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_0)) + .build(), ) last_record_date_1 = (_NOW - timedelta(days=1)).strftime("%Y-%m-%dT%H:%M:%SZ") http_mocker.get( _create_planets_request().with_start_date(start_datetime + timedelta(days=7)).with_end_date(_NOW).build(), - _create_response().with_record(record=_create_record("planets").with_cursor(last_record_date_1)).with_record(record=_create_record("planets").with_cursor(last_record_date_1)).build(), + _create_response() + .with_record(record=_create_record("planets").with_cursor(last_record_date_1)) + .with_record(record=_create_record("planets").with_cursor(last_record_date_1)) + .build(), ) # Mocks for dividers full refresh stream @@ -478,7 +496,13 @@ def test_incremental_and_full_refresh_streams(self, http_mocker): ) source = SourceFixture() - actual_messages = read(source, config=config, catalog=_create_catalog([("users", SyncMode.full_refresh), ("planets", SyncMode.incremental), ("dividers", SyncMode.full_refresh)])) + actual_messages = read( + source, + config=config, + catalog=_create_catalog( + [("users", SyncMode.full_refresh), ("planets", SyncMode.incremental), ("dividers", SyncMode.full_refresh)] + ), + ) assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("users")) assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("planets")) @@ -486,24 +510,27 @@ def test_incremental_and_full_refresh_streams(self, http_mocker): assert len(actual_messages.records) == 11 assert len(actual_messages.state_messages) == 5 - validate_message_order([ - Type.RECORD, - Type.RECORD, - Type.STATE, - Type.RECORD, - Type.RECORD, - Type.RECORD, - Type.STATE, - Type.RECORD, - Type.RECORD, - Type.STATE, - Type.RECORD, - Type.RECORD, - Type.STATE, - Type.RECORD, - Type.RECORD, - Type.STATE - ], actual_messages.records_and_state_messages) + validate_message_order( + [ + Type.RECORD, + Type.RECORD, + Type.STATE, + Type.RECORD, + Type.RECORD, + Type.RECORD, + Type.STATE, + Type.RECORD, + Type.RECORD, + Type.STATE, + Type.RECORD, + Type.RECORD, + Type.STATE, + Type.RECORD, + Type.RECORD, + Type.STATE, + ], + actual_messages.records_and_state_messages, + ) assert actual_messages.state_messages[0].state.stream.stream_descriptor.name == "users" assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(__ab_full_refresh_sync_complete=True) assert actual_messages.state_messages[0].state.sourceStats.recordCount == 2.0 @@ -514,8 +541,12 @@ def test_incremental_and_full_refresh_streams(self, http_mocker): assert actual_messages.state_messages[2].state.stream.stream_state == AirbyteStateBlob(created_at=last_record_date_1) assert actual_messages.state_messages[2].state.sourceStats.recordCount == 2.0 assert actual_messages.state_messages[3].state.stream.stream_descriptor.name == "dividers" - assert actual_messages.state_messages[3].state.stream.stream_state == AirbyteStateBlob(states=expected_first_substream_per_stream_state) + assert actual_messages.state_messages[3].state.stream.stream_state == AirbyteStateBlob( + states=expected_first_substream_per_stream_state + ) assert actual_messages.state_messages[3].state.sourceStats.recordCount == 2.0 assert actual_messages.state_messages[4].state.stream.stream_descriptor.name == "dividers" - assert actual_messages.state_messages[4].state.stream.stream_state == AirbyteStateBlob(states=expected_second_substream_per_stream_state) + assert actual_messages.state_messages[4].state.stream.stream_state == AirbyteStateBlob( + states=expected_second_substream_per_stream_state + ) assert actual_messages.state_messages[4].state.sourceStats.recordCount == 2.0 diff --git a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_resumable_full_refresh.py b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_resumable_full_refresh.py index bc5fe899f343e..f5a9e8578ab9d 100644 --- a/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_resumable_full_refresh.py +++ b/airbyte-cdk/python/unit_tests/sources/mock_server_tests/test_resumable_full_refresh.py @@ -7,7 +7,7 @@ from unittest import TestCase import freezegun -from airbyte_cdk.models import AirbyteStateBlob, ConfiguredAirbyteCatalog, SyncMode, Type +from airbyte_cdk.models import AirbyteStateBlob, AirbyteStreamStatus, ConfiguredAirbyteCatalog, FailureType, SyncMode, Type from airbyte_cdk.test.catalog_builder import ConfiguredAirbyteStreamBuilder from airbyte_cdk.test.entrypoint_wrapper import read from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest @@ -20,7 +20,6 @@ create_response_builder, ) from airbyte_cdk.test.state_builder import StateBuilder -from airbyte_protocol.models import AirbyteStreamStatus, FailureType from unit_tests.sources.mock_server_tests.mock_source_fixture import SourceFixture from unit_tests.sources.mock_server_tests.test_helpers import emits_successful_sync_status_messages, validate_message_order @@ -64,16 +63,7 @@ def _create_justice_songs_request() -> RequestBuilder: return RequestBuilder.justice_songs_endpoint() -RESPONSE_TEMPLATE = { - "object": "list", - "has_more": False, - "data": [ - { - "id": "123", - "created_at": "2024-01-01T07:04:28.000Z" - } - ] -} +RESPONSE_TEMPLATE = {"object": "list", "has_more": False, "data": [{"id": "123", "created_at": "2024-01-01T07:04:28.000Z"}]} JUSTICE_SONGS_TEMPLATE = { @@ -91,8 +81,8 @@ def _create_justice_songs_request() -> RequestBuilder: "created_at": "2024-02-01T07:04:28.000Z", "name": "dukes", "album": "", - } - ] + }, + ], } @@ -105,7 +95,7 @@ def _create_response(pagination_has_more: bool = False) -> HttpResponseBuilder: return create_response_builder( response_template=RESPONSE_TEMPLATE, records_path=FieldPath("data"), - pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more"), pagination_has_more) + pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more"), pagination_has_more), ) @@ -126,12 +116,20 @@ def test_resumable_full_refresh_sync(self, http_mocker): http_mocker.get( _create_justice_songs_request().build(), - _create_response(pagination_has_more=True).with_pagination().with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).build(), + _create_response(pagination_has_more=True) + .with_pagination() + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .build(), ) http_mocker.get( _create_justice_songs_request().with_page(1).build(), - _create_response(pagination_has_more=True).with_pagination().with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).build(), + _create_response(pagination_has_more=True) + .with_pagination() + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .build(), ) http_mocker.get( @@ -145,7 +143,10 @@ def test_resumable_full_refresh_sync(self, http_mocker): assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("justice_songs")) assert len(actual_messages.records) == 5 assert len(actual_messages.state_messages) == 4 - validate_message_order([Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.STATE, Type.STATE], actual_messages.records_and_state_messages) + validate_message_order( + [Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.STATE, Type.STATE], + actual_messages.records_and_state_messages, + ) assert actual_messages.state_messages[0].state.stream.stream_descriptor.name == "justice_songs" assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(page=1) assert actual_messages.state_messages[0].state.sourceStats.recordCount == 2.0 @@ -167,17 +168,31 @@ def test_resumable_full_refresh_second_attempt(self, http_mocker): http_mocker.get( _create_justice_songs_request().with_page(100).build(), - _create_response(pagination_has_more=True).with_pagination().with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).build(), + _create_response(pagination_has_more=True) + .with_pagination() + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .build(), ) http_mocker.get( _create_justice_songs_request().with_page(101).build(), - _create_response(pagination_has_more=True).with_pagination().with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).build(), + _create_response(pagination_has_more=True) + .with_pagination() + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .build(), ) http_mocker.get( _create_justice_songs_request().with_page(102).build(), - _create_response(pagination_has_more=False).with_pagination().with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).build(), + _create_response(pagination_has_more=False) + .with_pagination() + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .build(), ) source = SourceFixture() @@ -186,7 +201,23 @@ def test_resumable_full_refresh_second_attempt(self, http_mocker): assert emits_successful_sync_status_messages(actual_messages.get_stream_statuses("justice_songs")) assert len(actual_messages.records) == 8 assert len(actual_messages.state_messages) == 4 - validate_message_order([Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE, Type.STATE], actual_messages.records_and_state_messages) + validate_message_order( + [ + Type.RECORD, + Type.RECORD, + Type.RECORD, + Type.STATE, + Type.RECORD, + Type.RECORD, + Type.RECORD, + Type.STATE, + Type.RECORD, + Type.RECORD, + Type.STATE, + Type.STATE, + ], + actual_messages.records_and_state_messages, + ) assert actual_messages.state_messages[0].state.stream.stream_descriptor.name == "justice_songs" assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(page=101) assert actual_messages.state_messages[0].state.sourceStats.recordCount == 3.0 @@ -206,25 +237,37 @@ def test_resumable_full_refresh_failure(self, http_mocker): http_mocker.get( _create_justice_songs_request().build(), - _create_response(pagination_has_more=True).with_pagination().with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).build(), + _create_response(pagination_has_more=True) + .with_pagination() + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .build(), ) http_mocker.get( _create_justice_songs_request().with_page(1).build(), - _create_response(pagination_has_more=True).with_pagination().with_record(record=_create_record("justice_songs")).with_record(record=_create_record("justice_songs")).build(), + _create_response(pagination_has_more=True) + .with_pagination() + .with_record(record=_create_record("justice_songs")) + .with_record(record=_create_record("justice_songs")) + .build(), ) http_mocker.get(_create_justice_songs_request().with_page(2).build(), _create_response().with_status_code(status_code=400).build()) source = SourceFixture() - actual_messages = read(source, config=config, catalog=_create_catalog([("justice_songs", SyncMode.full_refresh, {})]), expecting_exception=True) + actual_messages = read( + source, config=config, catalog=_create_catalog([("justice_songs", SyncMode.full_refresh, {})]), expecting_exception=True + ) status_messages = actual_messages.get_stream_statuses("justice_songs") assert status_messages[-1] == AirbyteStreamStatus.INCOMPLETE assert len(actual_messages.records) == 4 assert len(actual_messages.state_messages) == 2 - validate_message_order([Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], actual_messages.records_and_state_messages) + validate_message_order( + [Type.RECORD, Type.RECORD, Type.STATE, Type.RECORD, Type.RECORD, Type.STATE], actual_messages.records_and_state_messages + ) assert actual_messages.state_messages[0].state.stream.stream_descriptor.name == "justice_songs" assert actual_messages.state_messages[0].state.stream.stream_state == AirbyteStateBlob(page=1) assert actual_messages.state_messages[1].state.stream.stream_descriptor.name == "justice_songs" diff --git a/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_checkpoint_reader.py b/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_checkpoint_reader.py index 2ccfaf33e8b62..01ddd363b0d36 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_checkpoint_reader.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_checkpoint_reader.py @@ -310,7 +310,12 @@ def test_legacy_cursor_based_checkpoint_reader_resumable_full_refresh(): {"parent_id": 400, "next_page_token": 2, "partition": {"parent_id": 400}, "cursor_slice": {"next_page_token": 2}}, {"parent_id": 400, "next_page_token": 3, "partition": {"parent_id": 400}, "cursor_slice": {"next_page_token": 3}}, {"parent_id": 400, "next_page_token": 4, "partition": {"parent_id": 400}, "cursor_slice": {"next_page_token": 4}}, - {"parent_id": 400, "__ab_full_refresh_sync_complete": True, "partition": {"parent_id": 400}, "cursor_slice": {"__ab_full_refresh_sync_complete": True}}, + { + "parent_id": 400, + "__ab_full_refresh_sync_complete": True, + "partition": {"parent_id": 400}, + "cursor_slice": {"__ab_full_refresh_sync_complete": True}, + }, ] mocked_state = [ diff --git a/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_substream_resumable_full_refresh_cursor.py b/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_substream_resumable_full_refresh_cursor.py index eb762ee08f330..4944518535f9a 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_substream_resumable_full_refresh_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/checkpoint/test_substream_resumable_full_refresh_cursor.py @@ -14,22 +14,8 @@ def test_substream_resumable_full_refresh_cursor(): expected_ending_state = { "states": [ - { - "partition": { - "musician_id": "kousei_arima" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "musician_id": "kaori_miyazono" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - } + {"partition": {"musician_id": "kousei_arima"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"musician_id": "kaori_miyazono"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, ] } @@ -58,65 +44,18 @@ def test_substream_resumable_full_refresh_cursor_with_state(): """ initial_state = { "states": [ - { - "partition": { - "musician_id": "kousei_arima" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "musician_id": "kaori_miyazono" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "musician_id": "takeshi_aiza" - }, - "cursor": {} - } + {"partition": {"musician_id": "kousei_arima"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"musician_id": "kaori_miyazono"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"musician_id": "takeshi_aiza"}, "cursor": {}}, ] } expected_ending_state = { "states": [ - { - "partition": { - "musician_id": "kousei_arima" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "musician_id": "kaori_miyazono" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "musician_id": "takeshi_aiza" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - }, - { - "partition": { - "musician_id": "emi_igawa" - }, - "cursor": { - "__ab_full_refresh_sync_complete": True - } - } + {"partition": {"musician_id": "kousei_arima"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"musician_id": "kaori_miyazono"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"musician_id": "takeshi_aiza"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, + {"partition": {"musician_id": "emi_igawa"}, "cursor": {"__ab_full_refresh_sync_complete": True}}, ] } @@ -146,9 +85,7 @@ def test_substream_resumable_full_refresh_cursor_with_state(): def test_set_initial_state_invalid_incoming_state(): - bad_state = { - "next_page_token": 2 - } + bad_state = {"next_page_token": 2} cursor = SubstreamResumableFullRefreshCursor() with pytest.raises(AirbyteTracedException): diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py index e6c91686209b0..090950aa14c75 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/stream_facade_builder.py @@ -6,7 +6,14 @@ import logging from typing import Any, List, Mapping, Optional, Tuple, Union -from airbyte_cdk.models import AirbyteStateMessage, ConfiguredAirbyteCatalog, ConnectorSpecification, DestinationSyncMode, SyncMode +from airbyte_cdk.models import ( + AirbyteStateMessage, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + ConnectorSpecification, + DestinationSyncMode, + SyncMode, +) from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter from airbyte_cdk.sources.concurrent_source.thread_pool_manager import ThreadPoolManager @@ -17,7 +24,6 @@ from airbyte_cdk.sources.streams.concurrent.adapters import StreamFacade from airbyte_cdk.sources.streams.concurrent.cursor import ConcurrentCursor, CursorField, FinalStateCursor from airbyte_cdk.sources.streams.concurrent.state_converters.datetime_stream_state_converter import EpochValueConcurrentStreamStateConverter -from airbyte_protocol.models import ConfiguredAirbyteStream from unit_tests.sources.file_based.scenarios.scenario_builder import SourceBuilder from unit_tests.sources.streams.concurrent.scenarios.thread_based_concurrent_stream_source_builder import NeverLogSliceLogger @@ -46,7 +52,7 @@ def __init__( self._threadpool = threadpool_manager self._cursor_field = cursor_field self._cursor_boundaries = cursor_boundaries - self._state = [AirbyteStateMessage.parse_obj(s) for s in input_state] if input_state else None + self._state = [AirbyteStateMessage(s) for s in input_state] if input_state else None def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> Tuple[bool, Optional[Any]]: return True, None @@ -74,10 +80,12 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: self._cursor_field, self._cursor_boundaries, None, - EpochValueConcurrentStreamStateConverter.get_end_provider() + EpochValueConcurrentStreamStateConverter.get_end_provider(), ) if self._cursor_field - else FinalStateCursor(stream_name=stream.name, stream_namespace=stream.namespace, message_repository=self.message_repository), + else FinalStateCursor( + stream_name=stream.name, stream_namespace=stream.namespace, message_repository=self.message_repository + ), ) for stream, state in zip(self._streams, stream_states) ] @@ -129,6 +137,8 @@ def set_input_state(self, state: List[Mapping[str, Any]]) -> "StreamFacadeSource self._input_state = state return self - def build(self, configured_catalog: Optional[Mapping[str, Any]], config: Optional[Mapping[str, Any]], state: Optional[TState]) -> StreamFacadeSource: + def build( + self, configured_catalog: Optional[Mapping[str, Any]], config: Optional[Mapping[str, Any]], state: Optional[TState] + ) -> StreamFacadeSource: threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=self._max_workers, thread_name_prefix="workerpool") return StreamFacadeSource(self._streams, threadpool, self._cursor_field, self._cursor_boundaries, state) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py index 43c198916a67a..51d83084041e9 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/scenarios/thread_based_concurrent_stream_source_builder.py @@ -5,7 +5,7 @@ import logging from typing import Any, Iterable, List, Mapping, Optional, Tuple, Union -from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConnectorSpecification, DestinationSyncMode, SyncMode +from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, ConnectorSpecification, DestinationSyncMode, SyncMode from airbyte_cdk.sources.concurrent_source.concurrent_source import ConcurrentSource from airbyte_cdk.sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter from airbyte_cdk.sources.message import InMemoryMessageRepository, MessageRepository @@ -19,7 +19,6 @@ from airbyte_cdk.sources.streams.concurrent.partitions.record import Record from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.utils.slice_logger import SliceLogger -from airbyte_protocol.models import ConfiguredAirbyteStream from unit_tests.sources.file_based.scenarios.scenario_builder import SourceBuilder @@ -49,7 +48,16 @@ def check_connection(self, logger: logging.Logger, config: Mapping[str, Any]) -> return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: - return [StreamFacade(s, LegacyStream(), FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=self.message_repository), NeverLogSliceLogger(), s._logger) for s in self._streams] + return [ + StreamFacade( + s, + LegacyStream(), + FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=self.message_repository), + NeverLogSliceLogger(), + s._logger, + ) + for s in self._streams + ] def spec(self, *args: Any, **kwargs: Any) -> ConnectorSpecification: return ConnectorSpecification(connectionSpecification={}) @@ -58,7 +66,13 @@ def read_catalog(self, catalog_path: str) -> ConfiguredAirbyteCatalog: return ConfiguredAirbyteCatalog( streams=[ ConfiguredAirbyteStream( - stream=StreamFacade(s, LegacyStream(), FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=InMemoryMessageRepository()), NeverLogSliceLogger(), s._logger).as_airbyte_stream(), + stream=StreamFacade( + s, + LegacyStream(), + FinalStateCursor(stream_name=s.name, stream_namespace=s.namespace, message_repository=InMemoryMessageRepository()), + NeverLogSliceLogger(), + s._logger, + ).as_airbyte_stream(), sync_mode=SyncMode.full_refresh, destination_sync_mode=DestinationSyncMode.overwrite, ) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py index 19a4cdb62627d..31688999372d0 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_concurrent_read_processor.py @@ -566,7 +566,9 @@ def test_on_exception_return_trace_message_and_on_stream_complete_return_stream_ handler.is_done() @freezegun.freeze_time("2020-01-01T00:00:00") - def test_given_underlying_exception_is_traced_exception_on_exception_return_trace_message_and_on_stream_complete_return_stream_status(self): + def test_given_underlying_exception_is_traced_exception_on_exception_return_trace_message_and_on_stream_complete_return_stream_status( + self, + ): stream_instances_to_read_from = [self._stream, self._another_stream] handler = ConcurrentReadProcessor( diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_cursor.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_cursor.py index b8fa8b2f79e0c..3f511c7b51da8 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_cursor.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_cursor.py @@ -102,10 +102,7 @@ def test_given_state_not_sequential_when_close_partition_then_emit_state(self) - self._state_manager.update_state_for_stream.assert_called_once_with( _A_STREAM_NAME, _A_STREAM_NAMESPACE, - { - "slices": [{"end": 0, "start": 0}, {"end": 30, "start": 12}], - "state_type": "date-range" - }, + {"slices": [{"end": 0, "start": 0}, {"end": 30, "start": 12}], "state_type": "date-range"}, ) def test_given_boundary_fields_when_close_partition_then_emit_updated_state(self) -> None: @@ -197,7 +194,7 @@ def test_given_one_slice_when_generate_slices_then_create_slice_from_slice_upper "state_type": ConcurrencyCompatibleStateType.date_range.value, "slices": [ {EpochValueConcurrentStreamStateConverter.START_KEY: 0, EpochValueConcurrentStreamStateConverter.END_KEY: 20}, - ] + ], }, self._message_repository, self._state_manager, @@ -225,7 +222,7 @@ def test_given_start_after_slices_when_generate_slices_then_generate_from_start( "state_type": ConcurrencyCompatibleStateType.date_range.value, "slices": [ {EpochValueConcurrentStreamStateConverter.START_KEY: 0, EpochValueConcurrentStreamStateConverter.END_KEY: 20}, - ] + ], }, self._message_repository, self._state_manager, @@ -254,7 +251,7 @@ def test_given_state_with_gap_and_start_after_slices_when_generate_slices_then_g "slices": [ {EpochValueConcurrentStreamStateConverter.START_KEY: 0, EpochValueConcurrentStreamStateConverter.END_KEY: 10}, {EpochValueConcurrentStreamStateConverter.START_KEY: 15, EpochValueConcurrentStreamStateConverter.END_KEY: 20}, - ] + ], }, self._message_repository, self._state_manager, @@ -283,7 +280,7 @@ def test_given_small_slice_range_when_generate_slices_then_create_many_slices(se "state_type": ConcurrencyCompatibleStateType.date_range.value, "slices": [ {EpochValueConcurrentStreamStateConverter.START_KEY: 0, EpochValueConcurrentStreamStateConverter.END_KEY: 20}, - ] + ], }, self._message_repository, self._state_manager, @@ -316,7 +313,7 @@ def test_given_difference_between_slices_match_slice_range_when_generate_slices_ "slices": [ {EpochValueConcurrentStreamStateConverter.START_KEY: 0, EpochValueConcurrentStreamStateConverter.END_KEY: 30}, {EpochValueConcurrentStreamStateConverter.START_KEY: 40, EpochValueConcurrentStreamStateConverter.END_KEY: 50}, - ] + ], }, self._message_repository, self._state_manager, @@ -346,7 +343,7 @@ def test_given_non_continuous_state_when_generate_slices_then_create_slices_betw {EpochValueConcurrentStreamStateConverter.START_KEY: 0, EpochValueConcurrentStreamStateConverter.END_KEY: 10}, {EpochValueConcurrentStreamStateConverter.START_KEY: 20, EpochValueConcurrentStreamStateConverter.END_KEY: 25}, {EpochValueConcurrentStreamStateConverter.START_KEY: 30, EpochValueConcurrentStreamStateConverter.END_KEY: 40}, - ] + ], }, self._message_repository, self._state_manager, @@ -378,7 +375,7 @@ def test_given_lookback_window_when_generate_slices_then_apply_lookback_on_most_ "slices": [ {EpochValueConcurrentStreamStateConverter.START_KEY: 0, EpochValueConcurrentStreamStateConverter.END_KEY: 20}, {EpochValueConcurrentStreamStateConverter.START_KEY: 30, EpochValueConcurrentStreamStateConverter.END_KEY: 40}, - ] + ], }, self._message_repository, self._state_manager, @@ -407,7 +404,7 @@ def test_given_start_is_before_first_slice_lower_boundary_when_generate_slices_t "state_type": ConcurrencyCompatibleStateType.date_range.value, "slices": [ {EpochValueConcurrentStreamStateConverter.START_KEY: 10, EpochValueConcurrentStreamStateConverter.END_KEY: 20}, - ] + ], }, self._message_repository, self._state_manager, diff --git a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py index d11154e712978..da67ff82588d2 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/concurrent/test_partition_enqueuer.py @@ -68,7 +68,10 @@ def test_given_exception_when_generate_partitions_then_return_exception_and_sent self._partition_generator.generate_partitions(stream) queue_content = self._consume_queue() - assert queue_content == _SOME_PARTITIONS + [StreamThreadException(exception, _A_STREAM_NAME), PartitionGenerationCompletedSentinel(stream)] + assert queue_content == _SOME_PARTITIONS + [ + StreamThreadException(exception, _A_STREAM_NAME), + PartitionGenerationCompletedSentinel(stream), + ] def _partitions_before_raising(self, partitions: List[Partition], exception: Exception) -> Callable[[], Iterable[Partition]]: def inner_function() -> Iterable[Partition]: diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_default_backoff_strategy.py b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_default_backoff_strategy.py index de795a409b1ab..67e7e3503c6c5 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_default_backoff_strategy.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_default_backoff_strategy.py @@ -15,8 +15,9 @@ def test_given_no_arguments_default_backoff_strategy_returns_default_values(): class CustomBackoffStrategy(BackoffStrategy): - - def backoff_time(self, response_or_exception: Optional[Union[requests.Response, requests.RequestException]], attempt_count: int) -> Optional[float]: + def backoff_time( + self, response_or_exception: Optional[Union[requests.Response, requests.RequestException]], attempt_count: int + ) -> Optional[float]: return response_or_exception.headers["Retry-After"] diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_http_status_error_handler.py b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_http_status_error_handler.py index e56d97a4fa182..6da3e15b2a69b 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_http_status_error_handler.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_http_status_error_handler.py @@ -26,10 +26,12 @@ def test_given_ok_response_http_status_error_handler_returns_success_action(mock "error, expected_action, expected_failure_type, expected_error_message", [ (403, ResponseAction.FAIL, FailureType.config_error, "Forbidden. You don't have permission to access this resource."), - (404, ResponseAction.FAIL, FailureType.system_error, "Not found. The requested resource was not found on the server.") - ] + (404, ResponseAction.FAIL, FailureType.system_error, "Not found. The requested resource was not found on the server."), + ], ) -def test_given_error_code_in_response_http_status_error_handler_returns_expected_actions(error, expected_action, expected_failure_type, expected_error_message): +def test_given_error_code_in_response_http_status_error_handler_returns_expected_actions( + error, expected_action, expected_failure_type, expected_error_message +): response = requests.Response() response.status_code = error error_resolution = HttpStatusErrorHandler(logger).interpret_response(response) @@ -98,14 +100,10 @@ def test_given_injected_error_mapping_returns_expected_action(): assert default_error_resolution.error_message == f"Unexpected HTTP Status Code in error handler: {mock_response.status_code}" mapped_error_resolution = ErrorResolution( - response_action=ResponseAction.IGNORE, - failure_type=FailureType.transient_error, - error_message="Injected mapping" - ) - - error_mapping = { - 509: mapped_error_resolution - } + response_action=ResponseAction.IGNORE, failure_type=FailureType.transient_error, error_message="Injected mapping" + ) + + error_mapping = {509: mapped_error_resolution} actual_error_resolution = HttpStatusErrorHandler(logger, error_mapping).interpret_response(mock_response) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_json_error_message_parser.py b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_json_error_message_parser.py index 81f838170341d..2eff4bc3f05e1 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_json_error_message_parser.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_json_error_message_parser.py @@ -8,23 +8,21 @@ @pytest.mark.parametrize( - "response_body,expected_error_message", - [ - (b'{"message": "json error message"}', "json error message"), - (b'[{"message": "list error message"}]', "list error message"), - (b'[{"message": "list error message 1"}, {"message": "list error message 2"}]', "list error message 1, list error message 2"), - (b'{"error": "messages error message"}', "messages error message"), - (b'[{"errors": "list error message 1"}, {"errors": "list error message 2"}]', "list error message 1, list error message 2"), - (b'{"failures": "failures error message"}', "failures error message"), - (b'{"failure": "failure error message"}', "failure error message"), - (b'{"detail": "detail error message"}', "detail error message"), - (b'{"err": "err error message"}', "err error message"), - (b'{"error_message": "error_message error message"}', "error_message error message"), - (b'{"msg": "msg error message"}', "msg error message"), - (b'{"reason": "reason error message"}', "reason error message"), - (b'{"status_message": "status_message error message"}', "status_message error message"), - ] - + "response_body,expected_error_message", + [ + (b'{"message": "json error message"}', "json error message"), + (b'[{"message": "list error message"}]', "list error message"), + (b'[{"message": "list error message 1"}, {"message": "list error message 2"}]', "list error message 1, list error message 2"), + (b'{"error": "messages error message"}', "messages error message"), + (b'[{"errors": "list error message 1"}, {"errors": "list error message 2"}]', "list error message 1, list error message 2"), + (b'{"failures": "failures error message"}', "failures error message"), + (b'{"failure": "failure error message"}', "failure error message"), + (b'{"detail": "detail error message"}', "detail error message"), + (b'{"err": "err error message"}', "err error message"), + (b'{"error_message": "error_message error message"}', "error_message error message"), + (b'{"msg": "msg error message"}', "msg error message"), + (b'{"reason": "reason error message"}', "reason error message"), + (b'{"status_message": "status_message error message"}', "status_message error message"),], ) def test_given_error_message_in_response_body_parse_response_error_message_returns_error_message(response_body, expected_error_message): response = requests.Response() @@ -35,6 +33,6 @@ def test_given_error_message_in_response_body_parse_response_error_message_retur def test_given_invalid_json_body_parse_response_error_message_returns_none(): response = requests.Response() - response._content = b'invalid json body' + response._content = b"invalid json body" error_message = JsonErrorMessageParser().parse_response_error_message(response) assert error_message is None diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_response_models.py b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_response_models.py index 62cde8d866905..a19d3c8d5fe0d 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_response_models.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/error_handlers/test_response_models.py @@ -4,16 +4,15 @@ import requests import requests_mock +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.streams.http.error_handlers.response_models import ResponseAction, create_fallback_error_resolution from airbyte_cdk.utils.airbyte_secrets_utils import update_secrets -from airbyte_protocol.models import FailureType _A_SECRET = "a-secret" _A_URL = "https://a-url.com" class DefaultErrorResolutionTest(TestCase): - def setUp(self) -> None: update_secrets([_A_SECRET]) @@ -26,7 +25,10 @@ def test_given_none_when_create_fallback_error_resolution_then_return_error_reso assert error_resolution.failure_type == FailureType.system_error assert error_resolution.response_action == ResponseAction.RETRY - assert error_resolution.error_message == "Error handler did not receive a valid response or exception. This is unexpected please contact Airbyte Support" + assert ( + error_resolution.error_message + == "Error handler did not receive a valid response or exception. This is unexpected please contact Airbyte Support" + ) def test_given_exception_when_create_fallback_error_resolution_then_return_error_resolution(self) -> None: exception = ValueError("This is an exception") diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py index 1300ad8e94dfd..42975d8ed5a90 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_availability_strategy.py @@ -83,7 +83,7 @@ def read_records(self, *args, **kvargs): http_stream = MockListHttpStream() response = requests.Response() response.status_code = status_code - response.raw = io.BytesIO(json.dumps(json_contents).encode('utf-8')) + response.raw = io.BytesIO(json.dumps(json_contents).encode("utf-8")) mocker.patch.object(requests.Session, "send", return_value=response) actual_is_available, reason = HttpAvailabilityStrategy().check_availability(http_stream, logger) @@ -104,7 +104,9 @@ def test_http_availability_raises_unhandled_error(mocker): req.status_code = 404 mocker.patch.object(requests.Session, "send", return_value=req) - assert (False, 'Not found. The requested resource was not found on the server.') == HttpAvailabilityStrategy().check_availability(http_stream, logger) + assert (False, "Not found. The requested resource was not found on the server.") == HttpAvailabilityStrategy().check_availability( + http_stream, logger + ) def test_send_handles_retries_when_checking_availability(mocker, caplog): @@ -120,7 +122,7 @@ def test_send_handles_retries_when_checking_availability(mocker, caplog): mock_send = mocker.patch.object(requests.Session, "send", side_effect=[req_1, req_2, req_3]) with caplog.at_level(logging.INFO): - stream_is_available, _ = HttpAvailabilityStrategy().check_availability(stream=http_stream,logger=logger) + stream_is_available, _ = HttpAvailabilityStrategy().check_availability(stream=http_stream, logger=logger) assert stream_is_available assert mock_send.call_count == 3 diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py index 27c7c7414e361..8737289a780f9 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py @@ -490,15 +490,13 @@ def should_retry(self, *args, **kwargs): [ (300, True, True, ResponseAction.RETRY), (200, False, True, ResponseAction.SUCCESS), - (503, False,True, ResponseAction.FAIL), - (503,False,False, ResponseAction.IGNORE) - ] + (503, False, True, ResponseAction.FAIL), + (503, False, False, ResponseAction.IGNORE), + ], ) -def test_http_stream_adapter_http_status_error_handler_should_retry_false_raise_on_http_errors(mocker, - response_status_code: int, - should_retry: bool, - raise_on_http_errors: bool, - expected_response_action: ResponseAction): +def test_http_stream_adapter_http_status_error_handler_should_retry_false_raise_on_http_errors( + mocker, response_status_code: int, should_retry: bool, raise_on_http_errors: bool, expected_response_action: ResponseAction +): stream = AutoFailTrueHttpStream() mocker.patch.object(stream, "should_retry", return_value=should_retry) mocker.patch.object(stream, "raise_on_http_errors", raise_on_http_errors) @@ -664,9 +662,19 @@ def test_duplicate_request_params_are_deduped(deduplicate_query_params, path, pa if expected_url is None: with pytest.raises(ValueError): - stream._http_client._create_prepared_request(http_method=stream.http_method, url=stream._join_url(stream.url_base, path), params=params, dedupe_query_params=deduplicate_query_params) + stream._http_client._create_prepared_request( + http_method=stream.http_method, + url=stream._join_url(stream.url_base, path), + params=params, + dedupe_query_params=deduplicate_query_params, + ) else: - prepared_request = stream._http_client._create_prepared_request(http_method=stream.http_method, url=stream._join_url(stream.url_base, path), params=params, dedupe_query_params=deduplicate_query_params) + prepared_request = stream._http_client._create_prepared_request( + http_method=stream.http_method, + url=stream._join_url(stream.url_base, path), + params=params, + dedupe_query_params=deduplicate_query_params, + ) assert prepared_request.url == expected_url @@ -689,8 +697,13 @@ def __init__(self, records: List[Mapping[str, Any]]): def url_base(self) -> str: return "https://airbyte.io/api/v1" - def path(self, *, stream_state: Optional[Mapping[str, Any]] = None, stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None) -> str: + def path( + self, + *, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: return "/stub" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -709,12 +722,12 @@ def _read_single_page( self.state = {"__ab_full_refresh_sync_complete": True} def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: return [] @@ -736,8 +749,13 @@ def __init__(self, record_pages: List[List[Mapping[str, Any]]]): def url_base(self) -> str: return "https://airbyte.io/api/v1" - def path(self, *, stream_state: Optional[Mapping[str, Any]] = None, stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None) -> str: + def path( + self, + *, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: return "/stub" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -759,12 +777,12 @@ def read_records( self.state = {"__ab_full_refresh_sync_complete": True} def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: return [] @@ -779,8 +797,13 @@ class StubHttpSubstream(HttpSubStream): def url_base(self) -> str: return "https://airbyte.io/api/v1" - def path(self, *, stream_state: Optional[Mapping[str, Any]] = None, stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None) -> str: + def path( + self, + *, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: return "/stub" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -800,12 +823,12 @@ def _read_pages( ] def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: return [] @@ -841,7 +864,7 @@ def test_substream_with_resumable_full_refresh_parent(): [ {"id": "page_3_abc"}, {"id": "page_3_def"}, - ] + ], ] expected_slices = [ @@ -987,10 +1010,7 @@ def test_resumable_full_refresh_read_from_state(mocker): mocker.patch.object(stream, method, wraps=getattr(stream, method)) checkpoint_reader = stream._get_checkpoint_reader( - cursor_field=[], - logger=logging.getLogger("airbyte"), - sync_mode=SyncMode.full_refresh, - stream_state={"page": 3} + cursor_field=[], logger=logging.getLogger("airbyte"), sync_mode=SyncMode.full_refresh, stream_state={"page": 3} ) next_stream_slice = checkpoint_reader.next() records = [] @@ -1036,10 +1056,7 @@ def test_resumable_full_refresh_legacy_stream_slice(mocker): mocker.patch.object(stream, method, wraps=getattr(stream, method)) checkpoint_reader = stream._get_checkpoint_reader( - cursor_field=[], - logger=logging.getLogger("airbyte"), - sync_mode=SyncMode.full_refresh, - stream_state={"page": 2} + cursor_field=[], logger=logging.getLogger("airbyte"), sync_mode=SyncMode.full_refresh, stream_state={"page": 2} ) next_stream_slice = checkpoint_reader.next() records = [] @@ -1082,8 +1099,13 @@ def __init__(self, parent: HttpStream, partition_id_to_child_records: Mapping[st def url_base(self) -> str: return "https://airbyte.io/api/v1" - def path(self, *, stream_state: Optional[Mapping[str, Any]] = None, stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None) -> str: + def path( + self, + *, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: return f"/parents/{stream_slice.get('parent_id')}/children" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: @@ -1113,12 +1135,12 @@ def _fetch_next_page( return requests.PreparedRequest(), requests.Response() def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: partition_id = stream_slice.get("parent").get("parent_id") if partition_id in self._partition_id_to_child_records: @@ -1141,14 +1163,21 @@ def test_substream_resumable_full_refresh_read_from_start(mocker): {"parent_id": "100", "name": "christopher_nolan"}, {"parent_id": "101", "name": "celine_song"}, {"parent_id": "102", "name": "david_fincher"}, - ] parent_stream = StubParentHttpStream(records=parent_records) parents_to_children_records = { - "100": [{"id": "a200", "parent_id": "100", "film": "interstellar"}, {"id": "a201", "parent_id": "100", "film": "oppenheimer"}, {"id": "a202", "parent_id": "100", "film": "inception"}], + "100": [ + {"id": "a200", "parent_id": "100", "film": "interstellar"}, + {"id": "a201", "parent_id": "100", "film": "oppenheimer"}, + {"id": "a202", "parent_id": "100", "film": "inception"}, + ], "101": [{"id": "b200", "parent_id": "101", "film": "past_lives"}, {"id": "b201", "parent_id": "101", "film": "materialists"}], - "102": [{"id": "c200", "parent_id": "102", "film": "the_social_network"}, {"id": "c201", "parent_id": "102", "film": "gone_girl"}, {"id": "c202", "parent_id": "102", "film": "the_curious_case_of_benjamin_button"}], + "102": [ + {"id": "c200", "parent_id": "102", "film": "the_social_network"}, + {"id": "c201", "parent_id": "102", "film": "gone_girl"}, + {"id": "c202", "parent_id": "102", "film": "the_curious_case_of_benjamin_button"}, + ], } stream = StubSubstreamResumableFullRefreshStream(parent=parent_stream, partition_id_to_child_records=parents_to_children_records) @@ -1168,61 +1197,31 @@ def test_substream_resumable_full_refresh_read_from_start(mocker): { "states": [ { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "christopher_nolan", "parent_id": "100"} - } + "cursor": {"__ab_full_refresh_sync_complete": True}, + "partition": {"parent": {"name": "christopher_nolan", "parent_id": "100"}}, } ] }, { "states": [ { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "christopher_nolan", "parent_id": "100"} - } + "cursor": {"__ab_full_refresh_sync_complete": True}, + "partition": {"parent": {"name": "christopher_nolan", "parent_id": "100"}}, }, - { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "celine_song", "parent_id": "101"} - } - } + {"cursor": {"__ab_full_refresh_sync_complete": True}, "partition": {"parent": {"name": "celine_song", "parent_id": "101"}}}, ] }, { "states": [ { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "christopher_nolan", "parent_id": "100"} - } + "cursor": {"__ab_full_refresh_sync_complete": True}, + "partition": {"parent": {"name": "christopher_nolan", "parent_id": "100"}}, }, + {"cursor": {"__ab_full_refresh_sync_complete": True}, "partition": {"parent": {"name": "celine_song", "parent_id": "101"}}}, { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "celine_song", "parent_id": "101"} - } + "cursor": {"__ab_full_refresh_sync_complete": True}, + "partition": {"parent": {"name": "david_fincher", "parent_id": "102"}}, }, - { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "david_fincher", "parent_id": "102"} - } - } ] }, ] @@ -1239,46 +1238,14 @@ def test_substream_resumable_full_refresh_read_from_start(mocker): assert getattr(stream, "_read_pages").call_count == 3 expected = [ - { - "film": "interstellar", - "id": "a200", - "parent_id": "100" - }, - { - "film": "oppenheimer", - "id": "a201", - "parent_id": "100" - }, - { - "film": "inception", - "id": "a202", - "parent_id": "100" - }, - { - "film": "past_lives", - "id": "b200", - "parent_id": "101" - }, - { - "film": "materialists", - "id": "b201", - "parent_id": "101" - }, - { - "film": "the_social_network", - "id": "c200", - "parent_id": "102" - }, - { - "film": "gone_girl", - "id": "c201", - "parent_id": "102" - }, - { - "film": "the_curious_case_of_benjamin_button", - "id": "c202", - "parent_id": "102" - } + {"film": "interstellar", "id": "a200", "parent_id": "100"}, + {"film": "oppenheimer", "id": "a201", "parent_id": "100"}, + {"film": "inception", "id": "a202", "parent_id": "100"}, + {"film": "past_lives", "id": "b200", "parent_id": "101"}, + {"film": "materialists", "id": "b201", "parent_id": "101"}, + {"film": "the_social_network", "id": "c200", "parent_id": "102"}, + {"film": "gone_girl", "id": "c201", "parent_id": "102"}, + {"film": "the_curious_case_of_benjamin_button", "id": "c202", "parent_id": "102"}, ] assert records == expected @@ -1294,13 +1261,15 @@ def test_substream_resumable_full_refresh_read_from_state(mocker): parent_records = [ {"parent_id": "100", "name": "christopher_nolan"}, {"parent_id": "101", "name": "celine_song"}, - ] parent_stream = StubParentHttpStream(records=parent_records) parents_to_children_records = { - "100": [{"id": "a200", "parent_id": "100", "film": "interstellar"}, {"id": "a201", "parent_id": "100", "film": "oppenheimer"}, - {"id": "a202", "parent_id": "100", "film": "inception"}], + "100": [ + {"id": "a200", "parent_id": "100", "film": "interstellar"}, + {"id": "a201", "parent_id": "100", "film": "oppenheimer"}, + {"id": "a202", "parent_id": "100", "film": "inception"}, + ], "101": [{"id": "b200", "parent_id": "101", "film": "past_lives"}, {"id": "b201", "parent_id": "101", "film": "materialists"}], } stream = StubSubstreamResumableFullRefreshStream(parent=parent_stream, partition_id_to_child_records=parents_to_children_records) @@ -1318,15 +1287,11 @@ def test_substream_resumable_full_refresh_read_from_state(mocker): stream_state={ "states": [ { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "christopher_nolan", "parent_id": "100"} - } + "cursor": {"__ab_full_refresh_sync_complete": True}, + "partition": {"parent": {"name": "christopher_nolan", "parent_id": "100"}}, }, ] - } + }, ) next_stream_slice = checkpoint_reader.next() records = [] @@ -1335,21 +1300,10 @@ def test_substream_resumable_full_refresh_read_from_state(mocker): { "states": [ { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "christopher_nolan", "parent_id": "100"} - } + "cursor": {"__ab_full_refresh_sync_complete": True}, + "partition": {"parent": {"name": "christopher_nolan", "parent_id": "100"}}, }, - { - "cursor": { - "__ab_full_refresh_sync_complete": True - }, - "partition": { - "parent": {"name": "celine_song", "parent_id": "101"} - } - } + {"cursor": {"__ab_full_refresh_sync_complete": True}, "partition": {"parent": {"name": "celine_song", "parent_id": "101"}}}, ] }, ] @@ -1366,16 +1320,8 @@ def test_substream_resumable_full_refresh_read_from_state(mocker): assert getattr(stream, "_read_pages").call_count == 1 expected = [ - { - "film": "past_lives", - "id": "b200", - "parent_id": "101" - }, - { - "film": "materialists", - "id": "b201", - "parent_id": "101" - }, + {"film": "past_lives", "id": "b200", "parent_id": "101"}, + {"film": "materialists", "id": "b201", "parent_id": "101"}, ] assert records == expected @@ -1398,8 +1344,13 @@ def cursor_field(self) -> Union[str, List[str]]: pytest.param([], False, ResumableFullRefreshCursor(), id="test_stream_supports_resumable_full_refresh_cursor"), pytest.param(["updated_at"], False, None, id="test_incremental_stream_does_not_use_cursor"), pytest.param(["updated_at"], True, None, id="test_incremental_substream_does_not_use_cursor"), - pytest.param([], True, SubstreamResumableFullRefreshCursor(), id="test_full_refresh_substream_automatically_applies_substream_resumable_full_refresh_cursor"), - ] + pytest.param( + [], + True, + SubstreamResumableFullRefreshCursor(), + id="test_full_refresh_substream_automatically_applies_substream_resumable_full_refresh_cursor", + ), + ], ) def test_get_cursor(cursor_field, is_substream, expected_cursor): stream = StubWithCursorFields(set_cursor_field=cursor_field, has_multiple_slices=is_substream) diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http_client.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http_client.py index efb4a9f1f4bd3..f7b0a11f69f4e 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http_client.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http_client.py @@ -48,19 +48,23 @@ def test_request_session_returns_valid_session(use_cache, expected_session): True, "https://test_base_url.com/v1/endpoint?param1=value1", {}, - "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_only_in_path" + "https://test_base_url.com/v1/endpoint?param1=value1", + id="test_params_only_in_path", ), pytest.param( True, "https://test_base_url.com/v1/endpoint", {"param1": "value1"}, - "https://test_base_url.com/v1/endpoint?param1=value1", id="test_params_only_in_path" + "https://test_base_url.com/v1/endpoint?param1=value1", + id="test_params_only_in_path", ), pytest.param( True, "https://test_base_url.com/v1/endpoint", None, - "https://test_base_url.com/v1/endpoint", id="test_params_is_none_and_no_params_in_path"), + "https://test_base_url.com/v1/endpoint", + id="test_params_is_none_and_no_params_in_path", + ), pytest.param( True, "https://test_base_url.com/v1/endpoint?param1=value1", @@ -119,7 +123,9 @@ def test_duplicate_request_params_are_deduped(deduplicate_query_params, url, par with pytest.raises(ValueError): http_client._create_prepared_request(http_method="get", url=url, dedupe_query_params=deduplicate_query_params, params=params) else: - prepared_request = http_client._create_prepared_request(http_method="get", url=url, dedupe_query_params=deduplicate_query_params, params=params) + prepared_request = http_client._create_prepared_request( + http_method="get", url=url, dedupe_query_params=deduplicate_query_params, params=params + ) assert prepared_request.url == expected_url @@ -127,7 +133,9 @@ def test_create_prepared_response_given_given_both_json_and_data_raises_request_ http_client = test_http_client() with pytest.raises(RequestBodyException): - http_client._create_prepared_request(http_method="get", url="https://test_base_url.com/v1/endpoint", json={"test": "json"}, data={"test": "data"}) + http_client._create_prepared_request( + http_method="get", url="https://test_base_url.com/v1/endpoint", json={"test": "json"}, data={"test": "data"} + ) @pytest.mark.parametrize( @@ -139,7 +147,9 @@ def test_create_prepared_response_given_given_both_json_and_data_raises_request_ ) def test_create_prepared_response_given_either_json_or_data_returns_valid_request(json, data): http_client = test_http_client() - prepared_request = http_client._create_prepared_request(http_method="get", url="https://test_base_url.com/v1/endpoint", json=json, data=data) + prepared_request = http_client._create_prepared_request( + http_method="get", url="https://test_base_url.com/v1/endpoint", json=json, data=data + ) assert prepared_request assert isinstance(prepared_request, requests.PreparedRequest) @@ -155,7 +165,9 @@ def test_valid_basic_send_request(mocker): mocked_response.status_code = 200 mocked_response.headers = {} mocker.patch.object(requests.Session, "send", return_value=mocked_response) - returned_request, returned_response = http_client.send_request(http_method="get", url="https://test_base_url.com/v1/endpoint", request_kwargs={}) + returned_request, returned_response = http_client.send_request( + http_method="get", url="https://test_base_url.com/v1/endpoint", request_kwargs={} + ) assert isinstance(returned_request, requests.PreparedRequest) assert returned_response == mocked_response @@ -166,8 +178,10 @@ def test_send_raises_airbyte_traced_exception_with_fail_response_action(): http_client = HttpClient( name="test", logger=MagicMock(), - error_handler=HttpStatusErrorHandler(logger=MagicMock(), error_mapping={400: ErrorResolution(ResponseAction.FAIL, FailureType.system_error, "test error message")}), - session=mocked_session + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), error_mapping={400: ErrorResolution(ResponseAction.FAIL, FailureType.system_error, "test error message")} + ), + session=mocked_session, ) prepared_request = requests.PreparedRequest() mocked_response = requests.Response() @@ -190,8 +204,10 @@ def test_send_ignores_with_ignore_reponse_action_and_returns_response(): http_client = HttpClient( name="test", logger=mocked_logger, - error_handler=HttpStatusErrorHandler(logger=MagicMock(), error_mapping={300: ErrorResolution(ResponseAction.IGNORE, FailureType.system_error, "test ignore message")}), - session=mocked_session + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), error_mapping={300: ErrorResolution(ResponseAction.IGNORE, FailureType.system_error, "test ignore message")} + ), + session=mocked_session, ) prepared_request = http_client._create_prepared_request(http_method="get", url="https://test_base_url.com/v1/endpoint") @@ -204,7 +220,6 @@ def test_send_ignores_with_ignore_reponse_action_and_returns_response(): class CustomBackoffStrategy(BackoffStrategy): - def __init__(self, backoff_time_value: float) -> None: self._backoff_time_value = backoff_time_value @@ -212,19 +227,15 @@ def backoff_time(self, *args, **kwargs) -> float: return self._backoff_time_value -@pytest.mark.parametrize( - "backoff_time_value, exception_type", - [ - (0.1, UserDefinedBackoffException), - (None, DefaultBackoffException) - ] -) +@pytest.mark.parametrize("backoff_time_value, exception_type", [(0.1, UserDefinedBackoffException), (None, DefaultBackoffException)]) def test_raises_backoff_exception_with_retry_response_action(mocker, backoff_time_value, exception_type): http_client = HttpClient( name="test", logger=MagicMock(), - error_handler=HttpStatusErrorHandler(logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.FAIL, FailureType.system_error, "test retry message")}), - backoff_strategy=CustomBackoffStrategy(backoff_time_value=backoff_time_value) + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.FAIL, FailureType.system_error, "test retry message")} + ), + backoff_strategy=CustomBackoffStrategy(backoff_time_value=backoff_time_value), ) prepared_request = http_client._create_prepared_request(http_method="get", url="https://test_base_url.com/v1/endpoint") mocked_response = MagicMock(spec=requests.Response) @@ -233,25 +244,25 @@ def test_raises_backoff_exception_with_retry_response_action(mocker, backoff_tim http_client._logger.info = MagicMock() mocker.patch.object(requests.Session, "send", return_value=mocked_response) - mocker.patch.object(http_client._error_handler, "interpret_response", return_value=ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")) + mocker.patch.object( + http_client._error_handler, + "interpret_response", + return_value=ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message"), + ) with pytest.raises(exception_type): http_client._send(prepared_request, {}) -@pytest.mark.parametrize( - "backoff_time_value, exception_type", - [ - (0.1, UserDefinedBackoffException), - (None, DefaultBackoffException) - ] -) +@pytest.mark.parametrize("backoff_time_value, exception_type", [(0.1, UserDefinedBackoffException), (None, DefaultBackoffException)]) def test_raises_backoff_exception_with_response_with_unmapped_error(mocker, backoff_time_value, exception_type): http_client = HttpClient( name="test", logger=MagicMock(), - error_handler=HttpStatusErrorHandler(logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.FAIL, FailureType.system_error, "test retry message")}), - backoff_strategy=CustomBackoffStrategy(backoff_time_value=backoff_time_value) + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.FAIL, FailureType.system_error, "test retry message")} + ), + backoff_strategy=CustomBackoffStrategy(backoff_time_value=backoff_time_value), ) prepared_request = requests.PreparedRequest() mocked_response = MagicMock(spec=requests.Response) @@ -289,8 +300,10 @@ def update_response(*args, **kwargs): http_client = HttpClient( name="test", logger=MagicMock(), - error_handler=HttpStatusErrorHandler(logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")}), - session=mocked_session + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")} + ), + session=mocked_session, ) prepared_request = requests.PreparedRequest() @@ -302,15 +315,15 @@ def update_response(*args, **kwargs): def test_session_request_exception_raises_backoff_exception(): - error_handler = HttpStatusErrorHandler(logger=MagicMock(), error_mapping={requests.exceptions.RequestException: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")}) - mocked_session = MagicMock(spec=requests.Session) - mocked_session.send.side_effect = requests.RequestException - http_client = HttpClient( - name="test", + error_handler = HttpStatusErrorHandler( logger=MagicMock(), - error_handler=error_handler, - session=mocked_session + error_mapping={ + requests.exceptions.RequestException: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message") + }, ) + mocked_session = MagicMock(spec=requests.Session) + mocked_session.send.side_effect = requests.RequestException + http_client = HttpClient(name="test", logger=MagicMock(), error_handler=error_handler, session=mocked_session) prepared_request = requests.PreparedRequest() with pytest.raises(DefaultBackoffException): @@ -347,12 +360,7 @@ def test_send_handles_response_action_given_session_send_raises_request_exceptio mocked_session = MagicMock(spec=requests.Session) mocked_session.send.side_effect = requests.RequestException - http_client = HttpClient( - name="test", - logger=MagicMock(), - error_handler=custom_error_handler, - session=mocked_session - ) + http_client = HttpClient(name="test", logger=MagicMock(), error_handler=custom_error_handler, session=mocked_session) prepared_request = requests.PreparedRequest() with pytest.raises(AirbyteTracedException) as e: @@ -383,8 +391,10 @@ def update_response(*args, **kwargs): http_client = HttpClient( name="test", logger=MagicMock(), - error_handler=HttpStatusErrorHandler(logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")}), - session=mocked_session + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), error_mapping={408: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")} + ), + session=mocked_session, ) prepared_request = requests.PreparedRequest() @@ -400,7 +410,13 @@ class BackoffStrategy: def backoff_time(self, *args, **kwargs): return 0.001 - http_client = HttpClient(name="test", logger=MagicMock(), error_handler=HttpStatusErrorHandler(logger=MagicMock()), backoff_strategy=BackoffStrategy(), disable_retries=True) + http_client = HttpClient( + name="test", + logger=MagicMock(), + error_handler=HttpStatusErrorHandler(logger=MagicMock()), + backoff_strategy=BackoffStrategy(), + disable_retries=True, + ) mocked_response = MagicMock(spec=requests.Response) mocked_response.status_code = 429 @@ -421,7 +437,9 @@ class BackoffStrategy: def backoff_time(self, *args, **kwargs): return 0.001 - http_client = HttpClient(name="test", logger=MagicMock(), error_handler=HttpStatusErrorHandler(logger=MagicMock()), backoff_strategy=BackoffStrategy()) + http_client = HttpClient( + name="test", logger=MagicMock(), error_handler=HttpStatusErrorHandler(logger=MagicMock()), backoff_strategy=BackoffStrategy() + ) mocked_response = MagicMock(spec=requests.Response) mocked_response.status_code = 429 @@ -444,7 +462,12 @@ def backoff_time(self, *args, **kwargs): retries = 3 - http_client = HttpClient(name="test", logger=MagicMock(), error_handler=HttpStatusErrorHandler(logger=MagicMock(), max_retries=retries), backoff_strategy=BackoffStrategy()) + http_client = HttpClient( + name="test", + logger=MagicMock(), + error_handler=HttpStatusErrorHandler(logger=MagicMock(), max_retries=retries), + backoff_strategy=BackoffStrategy(), + ) mocked_response = MagicMock(spec=requests.Response) mocked_response.status_code = 429 @@ -461,7 +484,12 @@ def backoff_time(self, *args, **kwargs): @pytest.mark.usefixtures("mock_sleep") def test_backoff_strategy_max_time(): - error_handler = HttpStatusErrorHandler(logger=MagicMock(), error_mapping={requests.RequestException: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")}, max_retries=10, max_time=timedelta(seconds=2)) + error_handler = HttpStatusErrorHandler( + logger=MagicMock(), + error_mapping={requests.RequestException: ErrorResolution(ResponseAction.RETRY, FailureType.system_error, "test retry message")}, + max_retries=10, + max_time=timedelta(seconds=2), + ) class BackoffStrategy: def backoff_time(self, *args, **kwargs): @@ -488,7 +516,9 @@ class BackoffStrategy: def backoff_time(self, *args, **kwargs): return 0.001 - http_client = HttpClient(name="test", logger=MagicMock(), error_handler=HttpStatusErrorHandler(logger=MagicMock()), backoff_strategy=BackoffStrategy()) + http_client = HttpClient( + name="test", logger=MagicMock(), error_handler=HttpStatusErrorHandler(logger=MagicMock()), backoff_strategy=BackoffStrategy() + ) mocked_response = MagicMock(spec=requests.Response) mocked_response.status_code = 429 @@ -505,7 +535,9 @@ def backoff_time(self, *args, **kwargs): assert len(trace_messages) == mocked_send.call_count -@pytest.mark.parametrize("exit_on_rate_limit, expected_call_count, expected_error",[[True, 6, DefaultBackoffException] ,[False, 38, OverflowError]]) +@pytest.mark.parametrize( + "exit_on_rate_limit, expected_call_count, expected_error", [[True, 6, DefaultBackoffException], [False, 38, OverflowError]] +) @pytest.mark.usefixtures("mock_sleep") def test_backoff_strategy_endless(exit_on_rate_limit, expected_call_count, expected_error): http_client = HttpClient(name="test", logger=MagicMock(), error_handler=HttpStatusErrorHandler(logger=MagicMock())) @@ -519,5 +551,7 @@ def test_backoff_strategy_endless(exit_on_rate_limit, expected_call_count, expec with patch.object(requests.Session, "send", return_value=mocked_response) as mocked_send: with pytest.raises(expected_error): - http_client.send_request(http_method="get", url="https://test_base_url.com/v1/endpoint", request_kwargs={}, exit_on_rate_limit=exit_on_rate_limit) + http_client.send_request( + http_method="get", url="https://test_base_url.com/v1/endpoint", request_kwargs={}, exit_on_rate_limit=exit_on_rate_limit + ) assert mocked_send.call_count == expected_call_count diff --git a/airbyte-cdk/python/unit_tests/sources/streams/test_stream_read.py b/airbyte-cdk/python/unit_tests/sources/streams/test_stream_read.py index b40f93ed03277..9f6f943e08406 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/test_stream_read.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/test_stream_read.py @@ -470,7 +470,9 @@ def test_configured_json_schema(): }, } - configured_stream, internal_config, logger, slice_logger, message_repository, state_manager = setup_stream_dependencies(current_json_schema) + configured_stream, internal_config, logger, slice_logger, message_repository, state_manager = setup_stream_dependencies( + current_json_schema + ) records = [ {"id": 1, "partition": 1}, {"id": 2, "partition": 1}, @@ -506,7 +508,9 @@ def test_configured_json_schema_with_invalid_properties(): del stream_schema["properties"][old_user_insights] del stream_schema["properties"][old_feature_info] - configured_stream, internal_config, logger, slice_logger, message_repository, state_manager = setup_stream_dependencies(configured_json_schema) + configured_stream, internal_config, logger, slice_logger, message_repository, state_manager = setup_stream_dependencies( + configured_json_schema + ) records = [ {"id": 1, "partition": 1}, {"id": 2, "partition": 1}, @@ -521,7 +525,9 @@ def test_configured_json_schema_with_invalid_properties(): assert old_user_insights not in configured_json_schema_properties assert old_feature_info not in configured_json_schema_properties for stream_schema_property in stream_schema["properties"]: - assert stream_schema_property in configured_json_schema_properties, f"Stream schema property: {stream_schema_property} missing in configured schema" + assert ( + stream_schema_property in configured_json_schema_properties + ), f"Stream schema property: {stream_schema_property} missing in configured schema" assert stream_schema["properties"][stream_schema_property] == configured_json_schema_properties[stream_schema_property] diff --git a/airbyte-cdk/python/unit_tests/sources/streams/test_streams_core.py b/airbyte-cdk/python/unit_tests/sources/streams/test_streams_core.py index 019705d1cd754..9f356b5c80bb8 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/test_streams_core.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/test_streams_core.py @@ -46,6 +46,7 @@ class StreamStubIncremental(Stream, CheckpointMixin): """ Stub full incremental class to assist with testing. """ + _state = {} def read_records( @@ -74,6 +75,7 @@ class StreamStubResumableFullRefresh(Stream, CheckpointMixin): """ Stub full incremental class to assist with testing. """ + _state = {} def read_records( @@ -100,6 +102,7 @@ class StreamStubLegacyStateInterface(Stream): """ Stub full incremental class to assist with testing. """ + _state = {} def read_records( @@ -154,17 +157,22 @@ def url_base(self) -> str: def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: pass - def path(self, *, stream_state: Optional[Mapping[str, Any]] = None, stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None) -> str: + def path( + self, + *, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: return "/stub" def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: return [] @@ -203,17 +211,22 @@ def stream_slices( def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: pass - def path(self, *, stream_state: Optional[Mapping[str, Any]] = None, stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None) -> str: + def path( + self, + *, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: return "/stub" def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: return [] @@ -234,17 +247,22 @@ def read_records( def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None - def path(self, *, stream_state: Optional[Mapping[str, Any]] = None, stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None) -> str: + def path( + self, + *, + stream_state: Optional[Mapping[str, Any]] = None, + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, + ) -> str: return "/parent" def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Optional[Mapping[str, Any]] = None, - next_page_token: Optional[Mapping[str, Any]] = None + self, + response: requests.Response, + *, + stream_state: Mapping[str, Any], + stream_slice: Optional[Mapping[str, Any]] = None, + next_page_token: Optional[Mapping[str, Any]] = None, ) -> Iterable[Mapping[str, Any]]: return [] @@ -374,11 +392,25 @@ def test_get_json_schema_is_cached(mocked_method): [ pytest.param(StreamStubIncremental(), {}, IncrementalCheckpointReader, id="test_incremental_checkpoint_reader"), pytest.param(StreamStubFullRefresh(), {}, FullRefreshCheckpointReader, id="test_full_refresh_checkpoint_reader"), - pytest.param(StreamStubResumableFullRefresh(), {}, ResumableFullRefreshCheckpointReader, id="test_resumable_full_refresh_checkpoint_reader"), - pytest.param(StreamStubLegacyStateInterface(), {}, IncrementalCheckpointReader, id="test_incremental_checkpoint_reader_with_legacy_state"), - pytest.param(CursorBasedStreamStubFullRefresh(), {"next_page_token": 10}, CursorBasedCheckpointReader, id="test_checkpoint_reader_using_rfr_cursor"), - pytest.param(LegacyCursorBasedStreamStubFullRefresh(), {}, LegacyCursorBasedCheckpointReader, id="test_full_refresh_checkpoint_reader_for_legacy_slice_format"), - ] + pytest.param( + StreamStubResumableFullRefresh(), {}, ResumableFullRefreshCheckpointReader, id="test_resumable_full_refresh_checkpoint_reader" + ), + pytest.param( + StreamStubLegacyStateInterface(), {}, IncrementalCheckpointReader, id="test_incremental_checkpoint_reader_with_legacy_state" + ), + pytest.param( + CursorBasedStreamStubFullRefresh(), + {"next_page_token": 10}, + CursorBasedCheckpointReader, + id="test_checkpoint_reader_using_rfr_cursor", + ), + pytest.param( + LegacyCursorBasedStreamStubFullRefresh(), + {}, + LegacyCursorBasedCheckpointReader, + id="test_full_refresh_checkpoint_reader_for_legacy_slice_format", + ), + ], ) def test_get_checkpoint_reader(stream: Stream, stream_state, expected_checkpoint_reader_type): checkpoint_reader = stream._get_checkpoint_reader( diff --git a/airbyte-cdk/python/unit_tests/sources/streams/utils/test_stream_helper.py b/airbyte-cdk/python/unit_tests/sources/streams/utils/test_stream_helper.py index 8cf1996853fd2..da76a78714d70 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/utils/test_stream_helper.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/utils/test_stream_helper.py @@ -11,8 +11,7 @@ def __init__(self, records, exit_on_rate_limit=True): self.records = records self._exit_on_rate_limit = exit_on_rate_limit type(self).exit_on_rate_limit = property( - lambda self: self._get_exit_on_rate_limit(), - lambda self, value: self._set_exit_on_rate_limit(value) + lambda self: self._get_exit_on_rate_limit(), lambda self, value: self._set_exit_on_rate_limit(value) ) def _get_exit_on_rate_limit(self): @@ -31,7 +30,7 @@ def read_records(self, sync_mode, stream_slice): ([{"id": 1}], None, True, {"id": 1}, False), # Single record, with setter ([{"id": 1}, {"id": 2}], None, True, {"id": 1}, False), # Multiple records, with setter ([], None, True, None, True), # No records, with setter - ] + ], ) def test_get_first_record_for_slice(records, stream_slice, exit_on_rate_limit, expected_result, raises_exception): stream = MockStream(records, exit_on_rate_limit) diff --git a/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py b/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py index af6a8b0a5f031..9de46b9e116f3 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_abstract_source.py @@ -198,7 +198,6 @@ def __init__(self, inputs_and_mocked_outputs: List[Tuple[Mapping[str, Any], Iter class MockStreamWithState(MockStreamWithCursor): - def __init__(self, inputs_and_mocked_outputs: List[Tuple[Mapping[str, Any], Iterable[Mapping[str, Any]]]], name: str, state=None): super().__init__(inputs_and_mocked_outputs, name) self._state = state @@ -422,7 +421,7 @@ def _as_state(stream_name: str = "", per_stream_state: Dict[str, Any] = None): state=AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name=stream_name), stream_state=AirbyteStateBlob.parse_obj(per_stream_state) + stream_descriptor=StreamDescriptor(name=stream_name), stream_state=AirbyteStateBlob(per_stream_state) ), ), ) @@ -606,9 +605,7 @@ def test_with_state_attribute(self, mocker): input_state = [ AirbyteStateMessage( type=AirbyteStateType.STREAM, - stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="s1"), stream_state=AirbyteStateBlob.parse_obj(old_state) - ), + stream=AirbyteStreamState(stream_descriptor=StreamDescriptor(name="s1"), stream_state=AirbyteStateBlob(old_state)), ), ] new_state_from_connector = {"cursor": "new_value"} @@ -860,13 +857,7 @@ def test_with_slices(self, mocker): assert messages == expected - @pytest.mark.parametrize( - "slices", - [ - pytest.param([], id="test_slices_as_list"), - pytest.param(iter([]), id="test_slices_as_iterator") - ] - ) + @pytest.mark.parametrize("slices", [pytest.param([], id="test_slices_as_list"), pytest.param(iter([]), id="test_slices_as_iterator")]) def test_no_slices(self, mocker, slices): """ Tests that an incremental read returns at least one state messages even if no records were read: @@ -876,15 +867,11 @@ def test_no_slices(self, mocker, slices): input_state = [ AirbyteStateMessage( type=AirbyteStateType.STREAM, - stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="s1"), stream_state=AirbyteStateBlob.parse_obj(state) - ), + stream=AirbyteStreamState(stream_descriptor=StreamDescriptor(name="s1"), stream_state=AirbyteStateBlob(state)), ), AirbyteStateMessage( type=AirbyteStateType.STREAM, - stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="s2"), stream_state=AirbyteStateBlob.parse_obj(state) - ), + stream=AirbyteStreamState(stream_descriptor=StreamDescriptor(name="s2"), stream_state=AirbyteStateBlob(state)), ), ] @@ -1185,15 +1172,12 @@ def test_without_state_attribute_for_stream_with_desc_records(self, mocker): AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name=stream_name), stream_state=AirbyteStateBlob.parse_obj(initial_state) + stream_descriptor=StreamDescriptor(name=stream_name), stream_state=AirbyteStateBlob(initial_state) ), ), ] stream_with_cursor = MockStreamWithCursor( - [ - ( - {"sync_mode": SyncMode.incremental, "stream_slice": {}, "stream_state": initial_state}, stream_output) - ], + [({"sync_mode": SyncMode.incremental, "stream_slice": {}, "stream_state": initial_state}, stream_output)], name=stream_name, ) @@ -1201,6 +1185,7 @@ def mock_get_updated_state(current_stream, current_stream_state, latest_record): state_cursor_value = current_stream_state.get(current_stream.cursor_field, 0) latest_record_value = latest_record.get(current_stream.cursor_field) return {current_stream.cursor_field: max(latest_record_value, state_cursor_value)} + mocker.patch.object(MockStreamWithCursor, "get_updated_state", mock_get_updated_state) mocker.patch.object(MockStreamWithCursor, "get_json_schema", return_value={}) src = MockSource(streams=[stream_with_cursor]) @@ -1306,7 +1291,7 @@ def test_resumable_full_refresh_with_incoming_state(self, mocker): type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="s1"), - stream_state=AirbyteStateBlob.parse_obj({"page": 10}), + stream_state=AirbyteStateBlob({"page": 10}), ), ) ] @@ -1433,16 +1418,16 @@ def test_resumable_full_refresh_skip_prior_successful_streams(self, mocker): type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="s1"), - stream_state=AirbyteStateBlob.parse_obj({"__ab_full_refresh_sync_complete": True}), + stream_state=AirbyteStateBlob({"__ab_full_refresh_sync_complete": True}), ), ), AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="s2"), - stream_state=AirbyteStateBlob.parse_obj({"page": 10}), + stream_state=AirbyteStateBlob({"page": 10}), ), - ) + ), ] src = MockSource(streams=[s1, s2]) @@ -1713,8 +1698,10 @@ def test_read_nonexistent_stream_emit_incomplete_stream_status(mocker, remove_st expected = _fix_emitted_at([as_stream_status("this_stream_doesnt_exist_in_the_source", AirbyteStreamStatus.INCOMPLETE)]) - expected_error_message = "The stream 'this_stream_doesnt_exist_in_the_source' in your connection configuration was not found in the " \ - "source. Refresh the schema in your replication settings and remove this stream from future sync attempts." + expected_error_message = ( + "The stream 'this_stream_doesnt_exist_in_the_source' in your connection configuration was not found in the " + "source. Refresh the schema in your replication settings and remove this stream from future sync attempts." + ) with pytest.raises(AirbyteTracedException) as exc_info: messages = [remove_stack_trace(message) for message in src.read(logger, {}, catalog)] diff --git a/airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py b/airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py index bcef13b9783ca..1a5526b105d5c 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py +++ b/airbyte-cdk/python/unit_tests/sources/test_connector_state_manager.py @@ -6,7 +6,15 @@ from typing import List import pytest -from airbyte_cdk.models import AirbyteMessage, AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteStateBlob, + AirbyteStateMessage, + AirbyteStateMessageSerializer, + AirbyteStateType, + AirbyteStreamState, + StreamDescriptor, +) from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager, HashableStreamDescriptor @@ -17,24 +25,24 @@ pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "actors", "namespace": "public"}, "stream_state": {"id": "mando_michael"}}, }, { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "actresses", "namespace": "public"}, "stream_state": {"id": "seehorn_rhea"}}, }, ], { - HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob.parse_obj({"id": "mando_michael"}), - HashableStreamDescriptor(name="actresses", namespace="public"): AirbyteStateBlob.parse_obj({"id": "seehorn_rhea"}), + HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob({"id": "mando_michael"}), + HashableStreamDescriptor(name="actresses", namespace="public"): AirbyteStateBlob({"id": "seehorn_rhea"}), }, does_not_raise(), id="test_incoming_per_stream_state", ), pytest.param([], {}, does_not_raise(), id="test_incoming_empty_stream_state"), pytest.param( - [{"type": AirbyteStateType.STREAM, "stream": {"stream_descriptor": {"name": "actresses", "namespace": "public"}}}], + [{"type": "STREAM", "stream": {"stream_descriptor": {"name": "actresses", "namespace": "public"}}}], {HashableStreamDescriptor(name="actresses", namespace="public"): None}, does_not_raise(), id="test_stream_states_that_have_none_state_blob", @@ -42,25 +50,25 @@ pytest.param( [ { - "type": AirbyteStateType.GLOBAL, + "type": "GLOBAL", "global": { "shared_state": {"television": "better_call_saul"}, "stream_states": [ { - "stream_descriptor": StreamDescriptor(name="actors", namespace="public"), - "stream_state": AirbyteStateBlob.parse_obj({"id": "mando_michael"}), + "stream_descriptor": {"name": "actors", "namespace": "public"}, + "stream_state": {"id": "mando_michael"}, }, { - "stream_descriptor": StreamDescriptor(name="actresses", namespace="public"), - "stream_state": AirbyteStateBlob.parse_obj({"id": "seehorn_rhea"}), + "stream_descriptor": {"name": "actresses", "namespace": "public"}, + "stream_state": {"id": "seehorn_rhea"}, }, ], }, }, ], { - HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob.parse_obj({"id": "mando_michael"}), - HashableStreamDescriptor(name="actresses", namespace="public"): AirbyteStateBlob.parse_obj({"id": "seehorn_rhea"}), + HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob({"id": "mando_michael"}), + HashableStreamDescriptor(name="actresses", namespace="public"): AirbyteStateBlob({"id": "seehorn_rhea"}), }, pytest.raises(ValueError), id="test_incoming_global_state_with_shared_state_throws_error", @@ -68,7 +76,7 @@ pytest.param( [ { - "type": AirbyteStateType.GLOBAL, + "type": "GLOBAL", "global": { "stream_states": [ {"stream_descriptor": {"name": "actors", "namespace": "public"}, "stream_state": {"id": "mando_michael"}}, @@ -77,7 +85,7 @@ }, ], { - HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob.parse_obj({"id": "mando_michael"}), + HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob({"id": "mando_michael"}), }, does_not_raise(), id="test_incoming_global_state_without_shared", @@ -85,20 +93,20 @@ pytest.param( [ { - "type": AirbyteStateType.GLOBAL, + "type": "GLOBAL", "global": { "shared_state": None, "stream_states": [ { - "stream_descriptor": StreamDescriptor(name="actors", namespace="public"), - "stream_state": AirbyteStateBlob.parse_obj({"id": "mando_michael"}), + "stream_descriptor": {"name": "actors", "namespace": "public"}, + "stream_state": {"id": "mando_michael"}, }, ], }, }, ], { - HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob.parse_obj({"id": "mando_michael"}), + HashableStreamDescriptor(name="actors", namespace="public"): AirbyteStateBlob({"id": "mando_michael"}), }, does_not_raise(), id="test_incoming_global_state_with_none_shared", @@ -106,7 +114,7 @@ pytest.param( [ { - "type": AirbyteStateType.GLOBAL, + "type": "GLOBAL", "global": { "stream_states": [ {"stream_descriptor": {"name": "actresses", "namespace": "public"}}, @@ -122,7 +130,7 @@ ) def test_initialize_state_manager(input_stream_state, expected_stream_state, expected_error): if isinstance(input_stream_state, List): - input_stream_state = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(input_stream_state)] + input_stream_state = [AirbyteStateMessageSerializer.load(state_obj) for state_obj in list(input_stream_state)] with expected_error: state_manager = ConnectorStateManager(input_stream_state) @@ -136,11 +144,11 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "users", "namespace": "public"}, "stream_state": {"created_at": 12345}}, }, { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "accounts", "namespace": "public"}, "stream_state": {"id": "abc"}}, }, ], @@ -152,10 +160,10 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "users"}, "stream_state": {"created_at": 12345}}, }, - {"type": AirbyteStateType.STREAM, "stream": {"stream_descriptor": {"name": "accounts"}, "stream_state": {"id": "abc"}}}, + {"type": "STREAM", "stream": {"stream_descriptor": {"name": "accounts"}, "stream_state": {"id": "abc"}}}, ], "users", None, @@ -164,8 +172,8 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp ), pytest.param( [ - {"type": AirbyteStateType.STREAM, "stream": {"stream_descriptor": {"name": "users"}}}, - {"type": AirbyteStateType.STREAM, "stream": {"stream_descriptor": {"name": "accounts"}, "stream_state": {"id": "abc"}}}, + {"type": "STREAM", "stream": {"stream_descriptor": {"name": "users"}}}, + {"type": "STREAM", "stream": {"stream_descriptor": {"name": "accounts"}, "stream_state": {"id": "abc"}}}, ], "users", None, @@ -175,11 +183,11 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "users", "namespace": "public"}, "stream_state": {"created_at": 12345}}, }, { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "accounts", "namespace": "public"}, "stream_state": {"id": "abc"}}, }, ], @@ -191,11 +199,11 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "users", "namespace": "public"}, "stream_state": {"created_at": 12345}}, }, { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "accounts", "namespace": "public"}, "stream_state": {"id": "abc"}}, }, ], @@ -208,7 +216,7 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "users", "namespace": "public"}, "stream_state": None}, }, ], @@ -220,7 +228,7 @@ def test_initialize_state_manager(input_stream_state, expected_stream_state, exp ], ) def test_get_stream_state(input_state, stream_name, namespace, expected_state): - state_messages = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(input_state)] + state_messages = [AirbyteStateMessageSerializer.load(state_obj) for state_obj in list(input_state)] state_manager = ConnectorStateManager(state_messages) actual_state = state_manager.get_stream_state(stream_name, namespace) @@ -234,7 +242,7 @@ def test_get_state_returns_deep_copy(): type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="episodes", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"id": [109]}), + stream_state=AirbyteStateBlob({"id": [109]}), ), ) ] @@ -252,11 +260,11 @@ def test_get_state_returns_deep_copy(): pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "actors", "namespace": "public"}, "stream_state": {"id": "mckean_michael"}}, }, { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "actresses", "namespace": "public"}, "stream_state": {"id": "seehorn_rhea"}}, }, ], @@ -275,7 +283,7 @@ def test_get_state_returns_deep_copy(): pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "actresses", "namespace": "public"}, "stream_state": {"id": "seehorn_rhea"}}, } ], @@ -287,7 +295,7 @@ def test_get_state_returns_deep_copy(): pytest.param( [ { - "type": AirbyteStateType.STREAM, + "type": "STREAM", "stream": {"stream_descriptor": {"name": "actresses", "namespace": "public"}, "stream_state": {"id": "seehorn_rhea"}}, } ], @@ -299,14 +307,14 @@ def test_get_state_returns_deep_copy(): ], ) def test_update_state_for_stream(start_state, update_name, update_namespace, update_value): - state_messages = [AirbyteStateMessage.parse_obj(state_obj) for state_obj in list(start_state)] + state_messages = [AirbyteStateMessage(state_obj) for state_obj in list(start_state)] state_manager = ConnectorStateManager(state_messages) state_manager.update_state_for_stream(update_name, update_namespace, update_value) - assert state_manager.per_stream_states[ - HashableStreamDescriptor(name=update_name, namespace=update_namespace) - ] == AirbyteStateBlob.parse_obj(update_value) + assert state_manager.per_stream_states[HashableStreamDescriptor(name=update_name, namespace=update_namespace)] == AirbyteStateBlob( + update_value + ) @pytest.mark.parametrize( @@ -318,14 +326,14 @@ def test_update_state_for_stream(start_state, update_name, update_namespace, upd type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="episodes", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"created_at": "2022_05_22"}), + stream_state=AirbyteStateBlob({"created_at": "2022_05_22"}), ), ), AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="seasons", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"id": 1}), + stream_state=AirbyteStateBlob({"id": 1}), ), ), ], @@ -337,7 +345,7 @@ def test_update_state_for_stream(start_state, update_name, update_namespace, upd type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="episodes", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"created_at": "2022_05_22"}), + stream_state=AirbyteStateBlob({"created_at": "2022_05_22"}), ), ), ), @@ -373,7 +381,7 @@ def test_update_state_for_stream(start_state, update_name, update_namespace, upd type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="episodes", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"id": 507}), + stream_state=AirbyteStateBlob({"id": 507}), ), ) ], @@ -396,7 +404,7 @@ def test_update_state_for_stream(start_state, update_name, update_namespace, upd type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="episodes", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"id": 507}), + stream_state=AirbyteStateBlob({"id": 507}), ), ) ], diff --git a/airbyte-cdk/python/unit_tests/sources/test_source.py b/airbyte-cdk/python/unit_tests/sources/test_source.py index c7b8e884653bc..d548a51b1ebbd 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_source.py +++ b/airbyte-cdk/python/unit_tests/sources/test_source.py @@ -13,9 +13,11 @@ AirbyteGlobalState, AirbyteStateBlob, AirbyteStateMessage, + AirbyteStateMessageSerializer, AirbyteStateType, AirbyteStreamState, ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, StreamDescriptor, SyncMode, Type, @@ -24,7 +26,8 @@ from airbyte_cdk.sources.streams.core import Stream from airbyte_cdk.sources.streams.http.http import HttpStream from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer -from pydantic import ValidationError +from orjson import orjson +from serpyco_rs import SchemaValidationError class MockSource(Source): @@ -74,7 +77,7 @@ def catalog(): }, ] } - return ConfiguredAirbyteCatalog.model_validate(configured_catalog) + return ConfiguredAirbyteCatalogSerializer.load(configured_catalog) @pytest.fixture @@ -154,7 +157,7 @@ def streams(self, config): type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="movies", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"created_at": "2009-07-19"}), + stream_state=AirbyteStateBlob({"created_at": "2009-07-19"}), ), ) ], @@ -190,21 +193,21 @@ def streams(self, config): type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="movies", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"created_at": "2009-07-19"}), + stream_state=AirbyteStateBlob({"created_at": "2009-07-19"}), ), ), AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="directors", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"id": "villeneuve_denis"}), + stream_state=AirbyteStateBlob({"id": "villeneuve_denis"}), ), ), AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( stream_descriptor=StreamDescriptor(name="actors", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"created_at": "1995-12-27"}), + stream_state=AirbyteStateBlob({"created_at": "1995-12-27"}), ), ), ], @@ -224,19 +227,17 @@ def streams(self, config): } ], [ - AirbyteStateMessage.parse_obj( - { - "type": AirbyteStateType.GLOBAL, - "global": AirbyteGlobalState( - shared_state=AirbyteStateBlob.parse_obj({"shared_key": "shared_val"}), - stream_states=[ - AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="movies", namespace="public"), - stream_state=AirbyteStateBlob.parse_obj({"created_at": "2009-07-19"}), - ) - ], - ), - } + AirbyteStateMessage( + type=AirbyteStateType.GLOBAL, + global_=AirbyteGlobalState( + shared_state=AirbyteStateBlob({"shared_key": "shared_val"}), + stream_states=[ + AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="movies", namespace="public"), + stream_state=AirbyteStateBlob({"created_at": "2009-07-19"}), + ) + ], + ), ), ], does_not_raise(), @@ -255,19 +256,19 @@ def streams(self, config): } ], None, - pytest.raises(ValidationError), + pytest.raises(SchemaValidationError), id="test_invalid_stream_state_invalid_type", ), pytest.param( [{"type": "STREAM", "stream": {"stream_state": {"created_at": "2009-07-19"}}}], None, - pytest.raises(ValidationError), + pytest.raises(SchemaValidationError), id="test_invalid_stream_state_missing_descriptor", ), pytest.param( [{"type": "GLOBAL", "global": {"shared_state": {"shared_key": "shared_val"}}}], None, - pytest.raises(ValidationError), + pytest.raises(SchemaValidationError), id="test_invalid_global_state_missing_streams", ), pytest.param( @@ -284,7 +285,7 @@ def streams(self, config): } ], None, - pytest.raises(ValidationError), + pytest.raises(SchemaValidationError), id="test_invalid_global_state_streams_not_list", ), ], @@ -295,7 +296,8 @@ def test_read_state(source, incoming_state, expected_state, expected_error): state_file.flush() with expected_error: actual = source.read_state(state_file.name) - assert actual == expected_state + if expected_state and actual: + assert AirbyteStateMessageSerializer.dump(actual[0]) == AirbyteStateMessageSerializer.dump(expected_state[0]) def test_read_invalid_state(source): @@ -330,9 +332,9 @@ def test_read_catalog(source): } ] } - expected = ConfiguredAirbyteCatalog.parse_obj(configured_catalog) + expected = ConfiguredAirbyteCatalogSerializer.load(configured_catalog) with tempfile.NamedTemporaryFile("w") as catalog_file: - catalog_file.write(expected.json(exclude_unset=True)) + catalog_file.write(orjson.dumps(ConfiguredAirbyteCatalogSerializer.dump(expected)).decode()) catalog_file.flush() actual = source.read_catalog(catalog_file.name) assert actual == expected diff --git a/airbyte-cdk/python/unit_tests/sources/test_source_read.py b/airbyte-cdk/python/unit_tests/sources/test_source_read.py index 00471ae86f825..05c71d1eae399 100644 --- a/airbyte-cdk/python/unit_tests/sources/test_source_read.py +++ b/airbyte-cdk/python/unit_tests/sources/test_source_read.py @@ -309,11 +309,19 @@ def test_concurrent_source_yields_the_same_messages_as_abstract_source_when_an_e def _assert_status_messages(messages_from_abstract_source, messages_from_concurrent_source): - status_from_concurrent_source = [message for message in messages_from_concurrent_source if message.type == MessageType.TRACE and message.trace.type == TraceType.STREAM_STATUS] + status_from_concurrent_source = [ + message + for message in messages_from_concurrent_source + if message.type == MessageType.TRACE and message.trace.type == TraceType.STREAM_STATUS + ] assert status_from_concurrent_source _verify_messages( - [message for message in messages_from_abstract_source if message.type == MessageType.TRACE and message.trace.type == TraceType.STREAM_STATUS], + [ + message + for message in messages_from_abstract_source + if message.type == MessageType.TRACE and message.trace.type == TraceType.STREAM_STATUS + ], status_from_concurrent_source, ) @@ -329,8 +337,14 @@ def _assert_record_messages(messages_from_abstract_source, messages_from_concurr def _assert_errors(messages_from_abstract_source, messages_from_concurrent_source): - errors_from_concurrent_source = [message for message in messages_from_concurrent_source if message.type == MessageType.TRACE and message.trace.type == TraceType.ERROR] - errors_from_abstract_source = [message for message in messages_from_abstract_source if message.type == MessageType.TRACE and message.trace.type == TraceType.ERROR] + errors_from_concurrent_source = [ + message + for message in messages_from_concurrent_source + if message.type == MessageType.TRACE and message.trace.type == TraceType.ERROR + ] + errors_from_abstract_source = [ + message for message in messages_from_abstract_source if message.type == MessageType.TRACE and message.trace.type == TraceType.ERROR + ] assert errors_from_concurrent_source # exceptions might differ from both framework hence we only assert the count @@ -352,7 +366,13 @@ def _init_sources(stream_slice_to_partitions, state, logger): def _init_source(stream_slice_to_partitions, state, logger, source): streams = [ - StreamFacade.create_from_stream(_MockStream(stream_slices, f"stream{i}"), source, logger, state, FinalStateCursor(stream_name=f"stream{i}", stream_namespace=None, message_repository=InMemoryMessageRepository())) + StreamFacade.create_from_stream( + _MockStream(stream_slices, f"stream{i}"), + source, + logger, + state, + FinalStateCursor(stream_name=f"stream{i}", stream_namespace=None, message_repository=InMemoryMessageRepository()), + ) for i, stream_slices in enumerate(stream_slice_to_partitions) ] source.set_streams(streams) diff --git a/airbyte-cdk/python/unit_tests/sources/utils/test_catalog_helpers.py b/airbyte-cdk/python/unit_tests/sources/utils/test_catalog_helpers.py deleted file mode 100644 index 8f4862332ea81..0000000000000 --- a/airbyte-cdk/python/unit_tests/sources/utils/test_catalog_helpers.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.models import AirbyteCatalog, AirbyteStream, SyncMode -from airbyte_cdk.sources.utils.catalog_helpers import CatalogHelper - - -def test_coerce_catalog_as_full_refresh(): - incremental = AirbyteStream( - name="1", - json_schema={"k": "v"}, - supported_sync_modes=[SyncMode.incremental, SyncMode.full_refresh], - source_defined_cursor=True, - default_cursor_field=["cursor"], - ) - full_refresh = AirbyteStream( - name="2", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh], source_defined_cursor=False - ) - input = AirbyteCatalog(streams=[incremental, full_refresh]) - - expected = AirbyteCatalog( - streams=[ - AirbyteStream(name="1", json_schema={"k": "v"}, supported_sync_modes=[SyncMode.full_refresh], source_defined_cursor=False), - full_refresh, - ] - ) - - assert CatalogHelper.coerce_catalog_as_full_refresh(input) == expected diff --git a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py b/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py index 0b76f5eef5c23..76b7a9b1c772f 100644 --- a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py +++ b/airbyte-cdk/python/unit_tests/sources/utils/test_schema_helpers.py @@ -14,7 +14,7 @@ import jsonref import pytest -from airbyte_cdk.models.airbyte_protocol import ConnectorSpecification, FailureType +from airbyte_cdk.models import ConnectorSpecification, ConnectorSpecificationSerializer, FailureType from airbyte_cdk.sources.utils.schema_helpers import InternalConfig, ResourceSchemaLoader, check_config_against_spec_or_exit from airbyte_cdk.utils.traced_exception import AirbyteTracedException from pytest import fixture @@ -42,7 +42,7 @@ def create_schema(name: str, content: Mapping): @fixture -def spec_object(): +def spec_object() -> ConnectorSpecification: spec = { "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", @@ -54,7 +54,7 @@ def spec_object(): }, }, } - yield ConnectorSpecification.parse_obj(spec) + yield ConnectorSpecificationSerializer.load(spec) def test_check_config_against_spec_or_exit_does_not_print_schema(capsys, spec_object): diff --git a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_models.py b/airbyte-cdk/python/unit_tests/sources/utils/test_schema_models.py deleted file mode 100644 index 1ef6b23349e7c..0000000000000 --- a/airbyte-cdk/python/unit_tests/sources/utils/test_schema_models.py +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright (c) 2023 Airbyte, Inc., all rights reserved. -# - -from typing import List, Optional - -from airbyte_cdk.sources.utils.schema_models import AllOptional, BaseSchemaModel - - -class InnerClass(BaseSchemaModel): - field1: Optional[str] - field2: int - - -class SchemaWithFewNullables(BaseSchemaModel): - name: Optional[str] - optional_item: Optional[InnerClass] - items: List[InnerClass] - - -class SchemaWithAllOptional(BaseSchemaModel, metaclass=AllOptional): - object_id: int - item: InnerClass - - -class TestSchemaWithFewNullables: - EXPECTED_SCHEMA = { - "type": "object", - "properties": { - "name": {"type": ["null", "string"]}, - "optional_item": { - "oneOf": [ - {"type": "null"}, - {"type": "object", "properties": {"field1": {"type": ["null", "string"]}, "field2": {"type": "integer"}}}, - ] - }, - "items": { - "type": "array", - "items": {"type": "object", "properties": {"field1": {"type": ["null", "string"]}, "field2": {"type": "integer"}}}, - }, - }, - } - - def test_schema_postprocessing(self): - schema = SchemaWithFewNullables.schema() - assert schema == self.EXPECTED_SCHEMA - - -class TestSchemaWithAllOptional: - EXPECTED_SCHEMA = { - "type": "object", - "properties": { - "object_id": {"type": ["null", "integer"]}, - "item": { - "oneOf": [ - {"type": "null"}, - {"type": "object", "properties": {"field1": {"type": ["null", "string"]}, "field2": {"type": "integer"}}}, - ] - }, - }, - } - - def test_schema_postprocessing(self): - schema = SchemaWithAllOptional.schema() - assert schema == self.EXPECTED_SCHEMA diff --git a/airbyte-cdk/python/unit_tests/test/mock_http/test_response_builder.py b/airbyte-cdk/python/unit_tests/test/mock_http/test_response_builder.py index 328db535ca36b..c8ccdc41b9bf4 100644 --- a/airbyte-cdk/python/unit_tests/test/mock_http/test_response_builder.py +++ b/airbyte-cdk/python/unit_tests/test/mock_http/test_response_builder.py @@ -46,9 +46,7 @@ def _any_record_builder() -> RecordBuilder: def _response_builder( - response_template: Dict[str, Any], - records_path: Union[FieldPath, NestedPath], - pagination_strategy: Optional[PaginationStrategy] = None + response_template: Dict[str, Any], records_path: Union[FieldPath, NestedPath], pagination_strategy: Optional[PaginationStrategy] = None ) -> HttpResponseBuilder: return create_response_builder(deepcopy(response_template), records_path, pagination_strategy=pagination_strategy) @@ -64,7 +62,9 @@ def test_given_with_id_when_build_then_set_id(self) -> None: assert record[_ID_FIELD] == "another id" def test_given_nested_id_when_build_then_set_id(self) -> None: - builder = _record_builder({_RECORDS_FIELD: [{"nested": {_ID_FIELD: "id"}}]}, FieldPath(_RECORDS_FIELD), NestedPath(["nested", _ID_FIELD])) + builder = _record_builder( + {_RECORDS_FIELD: [{"nested": {_ID_FIELD: "id"}}]}, FieldPath(_RECORDS_FIELD), NestedPath(["nested", _ID_FIELD]) + ) record = builder.with_id("another id").build() assert record["nested"][_ID_FIELD] == "another id" @@ -79,9 +79,7 @@ def test_given_no_id_in_template_for_path_when_build_then_raise_error(self) -> N def test_given_with_cursor_when_build_then_set_id(self) -> None: builder = _record_builder( - {_RECORDS_FIELD: [{_CURSOR_FIELD: "a cursor"}]}, - FieldPath(_RECORDS_FIELD), - record_cursor_path=FieldPath(_CURSOR_FIELD) + {_RECORDS_FIELD: [{_CURSOR_FIELD: "a cursor"}]}, FieldPath(_RECORDS_FIELD), record_cursor_path=FieldPath(_CURSOR_FIELD) ) record = builder.with_cursor("another cursor").build() assert record[_CURSOR_FIELD] == "another cursor" @@ -90,7 +88,7 @@ def test_given_nested_cursor_when_build_then_set_cursor(self) -> None: builder = _record_builder( {_RECORDS_FIELD: [{"nested": {_CURSOR_FIELD: "a cursor"}}]}, FieldPath(_RECORDS_FIELD), - record_cursor_path=NestedPath(["nested", _CURSOR_FIELD]) + record_cursor_path=NestedPath(["nested", _CURSOR_FIELD]), ) record = builder.with_cursor("another cursor").build() assert record["nested"][_CURSOR_FIELD] == "another cursor" @@ -115,7 +113,7 @@ def test_given_no_cursor_in_template_for_path_when_build_then_raise_error(self) _record_builder( {_RECORDS_FIELD: [{"record without cursor": "should fail"}]}, FieldPath(_RECORDS_FIELD), - record_cursor_path=FieldPath(_ID_FIELD) + record_cursor_path=FieldPath(_ID_FIELD), ) @@ -150,7 +148,7 @@ def test_given_pagination_with_strategy_when_build_then_apply_strategy(self) -> builder = _response_builder( {"has_more_pages": False} | _SOME_RECORDS, FieldPath(_RECORDS_FIELD), - pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more_pages"), "yes more page") + pagination_strategy=FieldUpdatePaginationStrategy(FieldPath("has_more_pages"), "yes more page"), ) response = builder.with_pagination().build() diff --git a/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py b/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py index 8e0bbe9fc93c0..11dfc58775722 100644 --- a/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py +++ b/airbyte-cdk/python/unit_tests/test/test_entrypoint_wrapper.py @@ -7,32 +7,40 @@ from unittest import TestCase from unittest.mock import Mock, patch -from airbyte_cdk.sources.abstract_source import AbstractSource -from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, discover, read -from airbyte_cdk.test.state_builder import StateBuilder -from airbyte_protocol.models import ( +from airbyte_cdk.models import ( AirbyteAnalyticsTraceMessage, AirbyteCatalog, AirbyteErrorTraceMessage, AirbyteLogMessage, AirbyteMessage, + AirbyteMessageSerializer, AirbyteRecordMessage, AirbyteStateBlob, AirbyteStateMessage, AirbyteStreamState, + AirbyteStreamStateSerializer, AirbyteStreamStatus, AirbyteStreamStatusTraceMessage, AirbyteTraceMessage, - ConfiguredAirbyteCatalog, + ConfiguredAirbyteCatalogSerializer, Level, StreamDescriptor, TraceType, Type, ) +from airbyte_cdk.sources.abstract_source import AbstractSource +from airbyte_cdk.test.entrypoint_wrapper import EntrypointOutput, discover, read +from airbyte_cdk.test.state_builder import StateBuilder +from orjson import orjson def _a_state_message(stream_name: str, stream_state: Mapping[str, Any]) -> AirbyteMessage: - return AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(stream=AirbyteStreamState(stream_descriptor=StreamDescriptor(name=stream_name), stream_state=AirbyteStateBlob(**stream_state)))) + return AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + stream=AirbyteStreamState(stream_descriptor=StreamDescriptor(name=stream_name), stream_state=AirbyteStateBlob(**stream_state)) + ), + ) def _a_status_message(stream_name: str, status: AirbyteStreamStatus) -> AirbyteMessage: @@ -77,7 +85,7 @@ def _a_status_message(stream_name: str, status: AirbyteStreamStatus) -> AirbyteM _A_STREAM_NAME = "a stream name" _A_CONFIG = {"config_key": "config_value"} -_A_CATALOG = ConfiguredAirbyteCatalog.parse_obj( +_A_CATALOG = ConfiguredAirbyteCatalogSerializer.load( { "streams": [ { @@ -97,7 +105,7 @@ def _a_status_message(stream_name: str, status: AirbyteStreamStatus) -> AirbyteM def _to_entrypoint_output(messages: List[AirbyteMessage]) -> Iterator[str]: - return (message.json(exclude_unset=True) for message in messages) + return (orjson.dumps(AirbyteMessageSerializer.dump(message)).decode() for message in messages) def _a_mocked_source() -> AbstractSource: @@ -112,7 +120,11 @@ def _validate_tmp_json_file(expected, file_path) -> None: def _validate_tmp_catalog(expected, file_path) -> None: - assert ConfiguredAirbyteCatalog.parse_file(file_path) == expected + assert ConfiguredAirbyteCatalogSerializer.load( + orjson.loads( + open(file_path).read() + ) + ) == expected def _create_tmp_file_validation(entrypoint, expected_config, expected_catalog: Optional[Any] = None, expected_state: Optional[Any] = None): @@ -176,19 +188,19 @@ def _do_some_logging(self): def test_given_record_when_discover_then_output_has_record(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_CATALOG_MESSAGE]) output = discover(self._a_source, _A_CONFIG) - assert output.catalog == _A_CATALOG_MESSAGE + assert AirbyteMessageSerializer.dump(output.catalog) == AirbyteMessageSerializer.dump(_A_CATALOG_MESSAGE) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_log_when_discover_then_output_has_log(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_LOG]) output = discover(self._a_source, _A_CONFIG) - assert output.logs == [_A_LOG] + assert AirbyteMessageSerializer.dump(output.logs[0]) == AirbyteMessageSerializer.dump(_A_LOG) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_trace_message_when_discover_then_output_has_trace_messages(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_AN_ANALYTIC_MESSAGE]) output = discover(self._a_source, _A_CONFIG) - assert output.analytics_messages == [_AN_ANALYTIC_MESSAGE] + assert AirbyteMessageSerializer.dump(output.analytics_messages[0]) == AirbyteMessageSerializer.dump(_AN_ANALYTIC_MESSAGE) @patch("airbyte_cdk.test.entrypoint_wrapper.print", create=True) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") @@ -254,41 +266,45 @@ def _do_some_logging(self): def test_given_record_when_read_then_output_has_record(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_RECORD]) output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) - assert output.records == [_A_RECORD] + assert AirbyteMessageSerializer.dump(output.records[0]) == AirbyteMessageSerializer.dump(_A_RECORD) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_state_message_when_read_then_output_has_state_message(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_STATE_MESSAGE]) output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) - assert output.state_messages == [_A_STATE_MESSAGE] + assert AirbyteMessageSerializer.dump(output.state_messages[0]) == AirbyteMessageSerializer.dump(_A_STATE_MESSAGE) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_state_message_and_records_when_read_then_output_has_records_and_state_message(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_RECORD, _A_STATE_MESSAGE]) output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) - assert output.records_and_state_messages == [_A_RECORD, _A_STATE_MESSAGE] + assert [AirbyteMessageSerializer.dump(message) for message in output.records_and_state_messages] == [ + AirbyteMessageSerializer.dump(message) for message in (_A_RECORD, _A_STATE_MESSAGE) + ] @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_many_state_messages_and_records_when_read_then_output_has_records_and_state_message(self, entrypoint): state_value = {"state_key": "last state value"} - last_emitted_state = AirbyteStreamState(stream_descriptor=StreamDescriptor(name="stream_name"), stream_state=AirbyteStateBlob(**state_value)) + last_emitted_state = AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="stream_name"), stream_state=AirbyteStateBlob(**state_value) + ) entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_STATE_MESSAGE, _a_state_message("stream_name", state_value)]) output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) - assert output.most_recent_state == last_emitted_state + assert AirbyteStreamStateSerializer.dump(output.most_recent_state) == AirbyteStreamStateSerializer.dump(last_emitted_state) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_log_when_read_then_output_has_log(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_A_LOG]) output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) - assert output.logs == [_A_LOG] + assert AirbyteMessageSerializer.dump(output.logs[0]) == AirbyteMessageSerializer.dump(_A_LOG) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_trace_message_when_read_then_output_has_trace_messages(self, entrypoint): entrypoint.return_value.run.return_value = _to_entrypoint_output([_AN_ANALYTIC_MESSAGE]) output = read(self._a_source, _A_CONFIG, _A_CATALOG, _A_STATE) - assert output.analytics_messages == [_AN_ANALYTIC_MESSAGE] + assert AirbyteMessageSerializer.dump(output.analytics_messages[0]) == AirbyteMessageSerializer.dump(_AN_ANALYTIC_MESSAGE) @patch("airbyte_cdk.test.entrypoint_wrapper.AirbyteEntrypoint") def test_given_stream_statuses_when_read_then_return_statuses(self, entrypoint): diff --git a/airbyte-cdk/python/unit_tests/test_connector.py b/airbyte-cdk/python/unit_tests/test_connector.py index 444397b4b0d62..ea7de2e406958 100644 --- a/airbyte-cdk/python/unit_tests/test_connector.py +++ b/airbyte-cdk/python/unit_tests/test_connector.py @@ -15,7 +15,6 @@ import yaml from airbyte_cdk import Connector from airbyte_cdk.models import AirbyteConnectionStatus -from pydantic import AnyUrl logger = logging.getLogger("airbyte") @@ -113,7 +112,7 @@ def use_yaml_spec(self): def test_spec_from_json_file(self, integration, use_json_spec): connector_spec = integration.spec(logger) - assert connector_spec.documentationUrl == AnyUrl("https://airbyte.com/#json") + assert connector_spec.documentationUrl == "https://airbyte.com/#json" assert connector_spec.connectionSpecification == self.CONNECTION_SPECIFICATION def test_spec_from_improperly_formatted_json_file(self, integration, use_invalid_json_spec): @@ -122,7 +121,7 @@ def test_spec_from_improperly_formatted_json_file(self, integration, use_invalid def test_spec_from_yaml_file(self, integration, use_yaml_spec): connector_spec = integration.spec(logger) - assert connector_spec.documentationUrl == AnyUrl("https://airbyte.com/#yaml") + assert connector_spec.documentationUrl == "https://airbyte.com/#yaml" assert connector_spec.connectionSpecification == self.CONNECTION_SPECIFICATION def test_multiple_spec_files_raises_exception(self, integration, use_yaml_spec, use_json_spec): diff --git a/airbyte-cdk/python/unit_tests/test_entrypoint.py b/airbyte-cdk/python/unit_tests/test_entrypoint.py index 1c5f8427bbb03..571042e202d14 100644 --- a/airbyte-cdk/python/unit_tests/test_entrypoint.py +++ b/airbyte-cdk/python/unit_tests/test_entrypoint.py @@ -20,9 +20,11 @@ AirbyteControlConnectorConfigMessage, AirbyteControlMessage, AirbyteMessage, + AirbyteMessageSerializer, AirbyteRecordMessage, AirbyteStateBlob, AirbyteStateMessage, + AirbyteStateStats, AirbyteStateType, AirbyteStream, AirbyteStreamState, @@ -37,10 +39,10 @@ TraceType, Type, ) -from airbyte_cdk.models.airbyte_protocol import AirbyteStateStats from airbyte_cdk.sources import Source from airbyte_cdk.sources.connector_state_manager import HashableStreamDescriptor from airbyte_cdk.utils import AirbyteTracedException +from orjson import orjson class MockSource(Source): @@ -106,14 +108,14 @@ def test_airbyte_entrypoint_init(mocker): ("check", {"config": "config_path"}, {"command": "check", "config": "config_path", "debug": False}), ("discover", {"config": "config_path", "debug": ""}, {"command": "discover", "config": "config_path", "debug": True}), ( - "read", - {"config": "config_path", "catalog": "catalog_path", "state": "None"}, - {"command": "read", "config": "config_path", "catalog": "catalog_path", "state": "None", "debug": False}, + "read", + {"config": "config_path", "catalog": "catalog_path", "state": "None"}, + {"command": "read", "config": "config_path", "catalog": "catalog_path", "state": "None", "debug": False}, ), ( - "read", - {"config": "config_path", "catalog": "catalog_path", "state": "state_path", "debug": ""}, - {"command": "read", "config": "config_path", "catalog": "catalog_path", "state": "state_path", "debug": True}, + "read", + {"config": "config_path", "catalog": "catalog_path", "state": "state_path", "debug": ""}, + {"command": "read", "config": "config_path", "catalog": "catalog_path", "state": "state_path", "debug": True}, ), ], ) @@ -152,7 +154,7 @@ def _wrap_message(submessage: Union[AirbyteConnectionStatus, ConnectorSpecificat else: raise Exception(f"Unknown message type: {submessage}") - return message.json(exclude_unset=True) + return orjson.dumps(AirbyteMessageSerializer.dump(message)).decode() def test_run_spec(entrypoint: AirbyteEntrypoint, mocker): @@ -162,7 +164,7 @@ def test_run_spec(entrypoint: AirbyteEntrypoint, mocker): messages = list(entrypoint.run(parsed_args)) - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(expected)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode(), _wrap_message(expected)] == messages @pytest.fixture @@ -181,9 +183,9 @@ def config_mock(mocker, request): ({"username": "fake"}, {"type": "object", "properties": {"user": {"type": "string"}}}, True), ({"username": "fake"}, {"type": "object", "properties": {"user": {"type": "string", "airbyte_secret": True}}}, True), ( - {"username": "fake", "_limit": 22}, - {"type": "object", "properties": {"username": {"type": "string"}}, "additionalProperties": False}, - True, + {"username": "fake", "_limit": 22}, + {"type": "object", "properties": {"username": {"type": "string"}}, "additionalProperties": False}, + True, ), ], indirect=["config_mock"], @@ -196,14 +198,14 @@ def test_config_validate(entrypoint: AirbyteEntrypoint, mocker, config_mock, sch messages = list(entrypoint.run(parsed_args)) if config_valid: - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(check_value)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode(), _wrap_message(check_value)] == messages else: assert len(messages) == 2 - assert messages[0] == MESSAGE_FROM_REPOSITORY.json(exclude_unset=True) - connection_status_message = AirbyteMessage.parse_raw(messages[1]) - assert connection_status_message.type == Type.CONNECTION_STATUS - assert connection_status_message.connectionStatus.status == Status.FAILED - assert connection_status_message.connectionStatus.message.startswith("Config validation error:") + assert messages[0] == orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode() + connection_status_message = AirbyteMessage(**orjson.loads(messages[1])) + assert connection_status_message.type == Type.CONNECTION_STATUS.value + assert connection_status_message.connectionStatus.get("status") == Status.FAILED.value + assert connection_status_message.connectionStatus.get("message").startswith("Config validation error:") def test_run_check(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): @@ -213,7 +215,7 @@ def test_run_check(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock messages = list(entrypoint.run(parsed_args)) - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(check_value)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode(), _wrap_message(check_value)] == messages assert spec_mock.called @@ -223,7 +225,7 @@ def test_run_check_with_exception(entrypoint: AirbyteEntrypoint, mocker, spec_mo with pytest.raises(ValueError): messages = list(entrypoint.run(parsed_args)) - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode()] == messages def test_run_discover(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): @@ -233,7 +235,7 @@ def test_run_discover(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_m messages = list(entrypoint.run(parsed_args)) - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(expected)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode(), _wrap_message(expected)] == messages assert spec_mock.called @@ -243,7 +245,7 @@ def test_run_discover_with_exception(entrypoint: AirbyteEntrypoint, mocker, spec with pytest.raises(ValueError): messages = list(entrypoint.run(parsed_args)) - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode()] == messages def test_run_read(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock): @@ -255,18 +257,18 @@ def test_run_read(entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock) messages = list(entrypoint.run(parsed_args)) - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True), _wrap_message(expected)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode(), _wrap_message(expected)] == messages assert spec_mock.called def test_given_message_emitted_during_config_when_read_then_emit_message_before_next_steps( - entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock + entrypoint: AirbyteEntrypoint, mocker, spec_mock, config_mock ): parsed_args = Namespace(command="read", config="config_path", state="statepath", catalog="catalogpath") mocker.patch.object(MockSource, "read_catalog", side_effect=ValueError) messages = entrypoint.run(parsed_args) - assert next(messages) == MESSAGE_FROM_REPOSITORY.json(exclude_unset=True) + assert next(messages) == orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode() with pytest.raises(ValueError): next(messages) @@ -279,7 +281,7 @@ def test_run_read_with_exception(entrypoint: AirbyteEntrypoint, mocker, spec_moc with pytest.raises(ValueError): messages = list(entrypoint.run(parsed_args)) - assert [MESSAGE_FROM_REPOSITORY.json(exclude_unset=True)] == messages + assert [orjson.dumps(AirbyteMessageSerializer.dump(MESSAGE_FROM_REPOSITORY)).decode()] == messages def test_invalid_command(entrypoint: AirbyteEntrypoint, config_mock): @@ -334,12 +336,26 @@ def test_filter_internal_requests(deployment_mode, url, expected_error): id="test_handle_record_message", ), pytest.param( - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="customers"), stream_state=AirbyteStateBlob(updated_at="2024-02-02")))), + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="customers"), stream_state=AirbyteStateBlob(updated_at="2024-02-02") + ), + ), + ), {HashableStreamDescriptor(name="customers"): 100.0}, - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="customers"), stream_state=AirbyteStateBlob(updated_at="2024-02-02")), - sourceStats=AirbyteStateStats(recordCount=100.0))), + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="customers"), stream_state=AirbyteStateBlob(updated_at="2024-02-02") + ), + sourceStats=AirbyteStateStats(recordCount=100.0), + ), + ), {HashableStreamDescriptor(name="customers"): 0.0}, id="test_handle_state_message", ), @@ -351,15 +367,27 @@ def test_filter_internal_requests(deployment_mode, url, expected_error): id="test_handle_first_record_message", ), pytest.param( - AirbyteMessage(type=Type.TRACE, trace=AirbyteTraceMessage(type=TraceType.STREAM_STATUS, - stream_status=AirbyteStreamStatusTraceMessage( - stream_descriptor=StreamDescriptor(name="customers"), - status=AirbyteStreamStatus.COMPLETE), emitted_at=1)), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="customers"), status=AirbyteStreamStatus.COMPLETE + ), + emitted_at=1, + ), + ), {HashableStreamDescriptor(name="customers"): 5.0}, - AirbyteMessage(type=Type.TRACE, trace=AirbyteTraceMessage(type=TraceType.STREAM_STATUS, - stream_status=AirbyteStreamStatusTraceMessage( - stream_descriptor=StreamDescriptor(name="customers"), - status=AirbyteStreamStatus.COMPLETE), emitted_at=1)), + AirbyteMessage( + type=Type.TRACE, + trace=AirbyteTraceMessage( + type=TraceType.STREAM_STATUS, + stream_status=AirbyteStreamStatusTraceMessage( + stream_descriptor=StreamDescriptor(name="customers"), status=AirbyteStreamStatus.COMPLETE + ), + emitted_at=1, + ), + ), {HashableStreamDescriptor(name="customers"): 5.0}, id="test_handle_other_message_type", ), @@ -371,48 +399,96 @@ def test_filter_internal_requests(deployment_mode, url, expected_error): id="test_handle_record_message_for_other_stream", ), pytest.param( - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="others"), stream_state=AirbyteStateBlob(updated_at="2024-02-02")))), + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="others"), stream_state=AirbyteStateBlob(updated_at="2024-02-02") + ), + ), + ), {HashableStreamDescriptor(name="customers"): 100.0, HashableStreamDescriptor(name="others"): 27.0}, - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="others"), stream_state=AirbyteStateBlob(updated_at="2024-02-02")), - sourceStats=AirbyteStateStats(recordCount=27.0))), + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="others"), stream_state=AirbyteStateBlob(updated_at="2024-02-02") + ), + sourceStats=AirbyteStateStats(recordCount=27.0), + ), + ), {HashableStreamDescriptor(name="customers"): 100.0, HashableStreamDescriptor(name="others"): 0.0}, id="test_handle_state_message_for_other_stream", ), pytest.param( - AirbyteMessage(type=Type.RECORD, - record=AirbyteRecordMessage(stream="customers", namespace="public", data={"id": "12345"}, emitted_at=1)), + AirbyteMessage( + type=Type.RECORD, record=AirbyteRecordMessage(stream="customers", namespace="public", data={"id": "12345"}, emitted_at=1) + ), {HashableStreamDescriptor(name="customers", namespace="public"): 100.0}, - AirbyteMessage(type=Type.RECORD, - record=AirbyteRecordMessage(stream="customers", namespace="public", data={"id": "12345"}, emitted_at=1)), + AirbyteMessage( + type=Type.RECORD, record=AirbyteRecordMessage(stream="customers", namespace="public", data={"id": "12345"}, emitted_at=1) + ), {HashableStreamDescriptor(name="customers", namespace="public"): 101.0}, id="test_handle_record_message_with_descriptor", ), pytest.param( - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="customers", namespace="public"), - stream_state=AirbyteStateBlob(updated_at="2024-02-02")))), + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="customers", namespace="public"), + stream_state=AirbyteStateBlob(updated_at="2024-02-02"), + ), + ), + ), {HashableStreamDescriptor(name="customers", namespace="public"): 100.0}, - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="customers", namespace="public"), - stream_state=AirbyteStateBlob(updated_at="2024-02-02")), sourceStats=AirbyteStateStats(recordCount=100.0))), + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="customers", namespace="public"), + stream_state=AirbyteStateBlob(updated_at="2024-02-02"), + ), + sourceStats=AirbyteStateStats(recordCount=100.0), + ), + ), {HashableStreamDescriptor(name="customers", namespace="public"): 0.0}, id="test_handle_state_message_with_descriptor", ), pytest.param( - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="others", namespace="public"), - stream_state=AirbyteStateBlob(updated_at="2024-02-02")))), + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="others", namespace="public"), + stream_state=AirbyteStateBlob(updated_at="2024-02-02"), + ), + ), + ), {HashableStreamDescriptor(name="customers", namespace="public"): 100.0}, - AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor(name="others", namespace="public"), - stream_state=AirbyteStateBlob(updated_at="2024-02-02")), sourceStats=AirbyteStateStats(recordCount=0.0))), - {HashableStreamDescriptor(name="customers", namespace="public"): 100.0, - HashableStreamDescriptor(name="others", namespace="public"): 0.0}, + AirbyteMessage( + type=Type.STATE, + state=AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name="others", namespace="public"), + stream_state=AirbyteStateBlob(updated_at="2024-02-02"), + ), + sourceStats=AirbyteStateStats(recordCount=0.0), + ), + ), + { + HashableStreamDescriptor(name="customers", namespace="public"): 100.0, + HashableStreamDescriptor(name="others", namespace="public"): 0.0, + }, id="test_handle_state_message_no_records", ), - ] + ], ) def test_handle_record_counts(incoming_message, stream_message_count, expected_message, expected_records_by_stream): entrypoint = AirbyteEntrypoint(source=MockSource()) diff --git a/airbyte-cdk/python/unit_tests/test_exception_handler.py b/airbyte-cdk/python/unit_tests/test_exception_handler.py index 42819942ade19..f135c19fd5a96 100644 --- a/airbyte-cdk/python/unit_tests/test_exception_handler.py +++ b/airbyte-cdk/python/unit_tests/test_exception_handler.py @@ -9,7 +9,17 @@ import pytest from airbyte_cdk.exception_handler import assemble_uncaught_exception -from airbyte_cdk.models import AirbyteErrorTraceMessage, AirbyteLogMessage, AirbyteMessage, AirbyteTraceMessage +from airbyte_cdk.models import ( + AirbyteErrorTraceMessage, + AirbyteLogMessage, + AirbyteMessage, + AirbyteMessageSerializer, + AirbyteTraceMessage, + FailureType, + Level, + TraceType, +) +from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.streams.concurrent.exceptions import ExceptionWithDisplayMessage from airbyte_cdk.utils.traced_exception import AirbyteTracedException @@ -43,16 +53,16 @@ def test_uncaught_exception_handler(): ) expected_log_message = AirbyteMessage( - type="LOG", log=AirbyteLogMessage(level="FATAL", message=f"{exception_message}\n{exception_trace}") + type=MessageType.LOG, log=AirbyteLogMessage(level=Level.FATAL, message=f"{exception_message}\n{exception_trace}") ) expected_trace_message = AirbyteMessage( - type="TRACE", + type=MessageType.TRACE, trace=AirbyteTraceMessage( - type="ERROR", + type=TraceType.ERROR, emitted_at=0.0, error=AirbyteErrorTraceMessage( - failure_type="system_error", + failure_type=FailureType.system_error, message="Something went wrong in the connector. See the logs for more details.", internal_message=exception_message, stack_trace=f"{exception_trace}\n", @@ -70,10 +80,10 @@ def test_uncaught_exception_handler(): log_output, trace_output = stdout_lines - out_log_message = AirbyteMessage.parse_obj(json.loads(log_output)) + out_log_message = AirbyteMessageSerializer.load(json.loads(log_output)) assert out_log_message == expected_log_message, "Log message should be emitted in expected form" - out_trace_message = AirbyteMessage.parse_obj(json.loads(trace_output)) + out_trace_message = AirbyteMessageSerializer.load(json.loads(trace_output)) assert out_trace_message.trace.emitted_at > 0 out_trace_message.trace.emitted_at = 0.0 # set a specific emitted_at value for testing assert out_trace_message == expected_trace_message, "Trace message should be emitted in expected form" diff --git a/airbyte-cdk/python/unit_tests/utils/test_datetime_format_inferrer.py b/airbyte-cdk/python/unit_tests/utils/test_datetime_format_inferrer.py index 7660074671843..5e76b9cfa193b 100644 --- a/airbyte-cdk/python/unit_tests/utils/test_datetime_format_inferrer.py +++ b/airbyte-cdk/python/unit_tests/utils/test_datetime_format_inferrer.py @@ -5,7 +5,7 @@ from typing import Dict, List import pytest -from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage +from airbyte_cdk.models import AirbyteRecordMessage from airbyte_cdk.utils.datetime_format_inferrer import DatetimeFormatInferrer NOW = 1234567 diff --git a/airbyte-cdk/python/unit_tests/utils/test_message_utils.py b/airbyte-cdk/python/unit_tests/utils/test_message_utils.py index 496360ea46f37..84fabf1a8fa8c 100644 --- a/airbyte-cdk/python/unit_tests/utils/test_message_utils.py +++ b/airbyte-cdk/python/unit_tests/utils/test_message_utils.py @@ -1,9 +1,7 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. import pytest -from airbyte_cdk.sources.connector_state_manager import HashableStreamDescriptor -from airbyte_cdk.utils.message_utils import get_stream_descriptor -from airbyte_protocol.models import ( +from airbyte_cdk.models import ( AirbyteControlConnectorConfigMessage, AirbyteControlMessage, AirbyteMessage, @@ -17,6 +15,8 @@ StreamDescriptor, Type, ) +from airbyte_cdk.sources.connector_state_manager import HashableStreamDescriptor +from airbyte_cdk.utils.message_utils import get_stream_descriptor def test_get_record_message_stream_descriptor(): @@ -36,9 +36,7 @@ def test_get_record_message_stream_descriptor(): def test_get_record_message_stream_descriptor_no_namespace(): message = AirbyteMessage( type=Type.RECORD, - record=AirbyteRecordMessage( - stream="test_stream", data={"id": "12345"}, emitted_at=1 - ), + record=AirbyteRecordMessage(stream="test_stream", data={"id": "12345"}, emitted_at=1), ) expected_descriptor = HashableStreamDescriptor(name="test_stream", namespace=None) assert get_stream_descriptor(message) == expected_descriptor @@ -50,9 +48,7 @@ def test_get_state_message_stream_descriptor(): state=AirbyteStateMessage( type=AirbyteStateType.STREAM, stream=AirbyteStreamState( - stream_descriptor=StreamDescriptor( - name="test_stream", namespace="test_namespace" - ), + stream_descriptor=StreamDescriptor(name="test_stream", namespace="test_namespace"), stream_state=AirbyteStateBlob(updated_at="2024-02-02"), ), sourceStats=AirbyteStateStats(recordCount=27.0), diff --git a/airbyte-cdk/python/unit_tests/utils/test_schema_inferrer.py b/airbyte-cdk/python/unit_tests/utils/test_schema_inferrer.py index 51a055401a297..98d227c40ec6d 100644 --- a/airbyte-cdk/python/unit_tests/utils/test_schema_inferrer.py +++ b/airbyte-cdk/python/unit_tests/utils/test_schema_inferrer.py @@ -5,7 +5,7 @@ from typing import List, Mapping import pytest -from airbyte_cdk.models.airbyte_protocol import AirbyteRecordMessage +from airbyte_cdk.models import AirbyteRecordMessage from airbyte_cdk.utils.schema_inferrer import SchemaInferrer, SchemaValidationException NOW = 1234567 @@ -133,7 +133,10 @@ { "my_stream": { "field_A": {"type": ["string", "null"]}, - "nested": {"type": ["array", "null"], "items": {"type": ["object", "null"], "properties": {"field_C": {"type": ["string", "null"]}}}}, + "nested": { + "type": ["array", "null"], + "items": {"type": ["object", "null"], "properties": {"field_C": {"type": ["string", "null"]}}}, + }, } }, id="test_array_nested_null", @@ -146,7 +149,10 @@ { "my_stream": { "field_A": {"type": ["string", "null"]}, - "nested": {"type": ["array", "null"], "items": {"type": ["object", "null"], "properties": {"field_C": {"type": ["string", "null"]}}}}, + "nested": { + "type": ["array", "null"], + "items": {"type": ["object", "null"], "properties": {"field_C": {"type": ["string", "null"]}}}, + }, } }, id="test_array_top_level_null", @@ -166,80 +172,42 @@ "data": { "root_property_object": { "property_array": [ - { - "title": "Nested_1", - "type": "multi-value", - "value": ["XL"] - }, + {"title": "Nested_1", "type": "multi-value", "value": ["XL"]}, { "title": "Nested_2", "type": "location", - "value": { - "nested_key_1": "GB", - "nested_key_2": "United Kingdom" - } - } + "value": {"nested_key_1": "GB", "nested_key_2": "United Kingdom"}, + }, ], } - } + }, }, ], { "data_with_nested_arrays": { "root_property_object": { - "type": [ - "object", - "null" - ], + "type": ["object", "null"], "properties": { "property_array": { - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { - "type": [ - "object", - "null" - ], + "type": ["object", "null"], "properties": { - "title": { - "type": [ - "string", - "null" - ] - }, - "type": { - "type": [ - "string", - "null" - ] - }, + "title": {"type": ["string", "null"]}, + "type": {"type": ["string", "null"]}, "value": { "anyOf": [ - { - "type": "array", - "items": { - "type": "string" - } - }, + {"type": "array", "items": {"type": "string"}}, { "type": "object", - "properties": { - "nested_key_1": { - "type": "string" - }, - "nested_key_2": { - "type": "string" - } - } - } + "properties": {"nested_key_1": {"type": "string"}, "nested_key_2": {"type": "string"}}, + }, ] - } - } - } + }, + }, + }, } - } + }, } } }, @@ -277,7 +245,7 @@ def _create_inferrer_with_required_field(is_pk: bool, field: List[List[str]]) -> [ pytest.param(_IS_PK, id="required_field_is_pk"), pytest.param(_IS_CURSOR_FIELD, id="required_field_is_cursor_field"), - ] + ], ) def test_field_is_on_root(is_pk: bool): inferrer = _create_inferrer_with_required_field(is_pk, [["property"]]) @@ -293,7 +261,7 @@ def test_field_is_on_root(is_pk: bool): [ pytest.param(_IS_PK, id="required_field_is_pk"), pytest.param(_IS_CURSOR_FIELD, id="required_field_is_cursor_field"), - ] + ], ) def test_field_is_nested(is_pk: bool): inferrer = _create_inferrer_with_required_field(is_pk, [["property", "nested_property"]]) @@ -310,11 +278,13 @@ def test_field_is_nested(is_pk: bool): [ pytest.param(_IS_PK, id="required_field_is_pk"), pytest.param(_IS_CURSOR_FIELD, id="required_field_is_cursor_field"), - ] + ], ) def test_field_is_composite(is_pk: bool): inferrer = _create_inferrer_with_required_field(is_pk, [["property 1"], ["property 2"]]) - inferrer.accumulate(AirbyteRecordMessage(stream=_STREAM_NAME, data={"property 1": _ANY_VALUE, "property 2": _ANY_VALUE}, emitted_at=NOW)) + inferrer.accumulate( + AirbyteRecordMessage(stream=_STREAM_NAME, data={"property 1": _ANY_VALUE, "property 2": _ANY_VALUE}, emitted_at=NOW) + ) assert inferrer.get_stream_schema(_STREAM_NAME)["required"] == ["property 1", "property 2"] @@ -323,12 +293,14 @@ def test_field_is_composite(is_pk: bool): [ pytest.param(_IS_PK, id="required_field_is_pk"), pytest.param(_IS_CURSOR_FIELD, id="required_field_is_cursor_field"), - ] + ], ) def test_field_is_composite_and_nested(is_pk: bool): inferrer = _create_inferrer_with_required_field(is_pk, [["property 1", "nested"], ["property 2"]]) - inferrer.accumulate(AirbyteRecordMessage(stream=_STREAM_NAME, data={"property 1": {"nested": _ANY_VALUE}, "property 2": _ANY_VALUE}, emitted_at=NOW)) + inferrer.accumulate( + AirbyteRecordMessage(stream=_STREAM_NAME, data={"property 1": {"nested": _ANY_VALUE}, "property 2": _ANY_VALUE}, emitted_at=NOW) + ) assert inferrer.get_stream_schema(_STREAM_NAME)["required"] == ["property 1", "property 2"] assert inferrer.get_stream_schema(_STREAM_NAME)["properties"]["property 1"]["type"] == "object" diff --git a/airbyte-cdk/python/unit_tests/utils/test_traced_exception.py b/airbyte-cdk/python/unit_tests/utils/test_traced_exception.py index e0d3b9a50353a..ea559a319467a 100644 --- a/airbyte-cdk/python/unit_tests/utils/test_traced_exception.py +++ b/airbyte-cdk/python/unit_tests/utils/test_traced_exception.py @@ -2,20 +2,21 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. # -import json import pytest -from airbyte_cdk.models.airbyte_protocol import ( +from airbyte_cdk.models import ( AirbyteErrorTraceMessage, AirbyteMessage, + AirbyteMessageSerializer, AirbyteTraceMessage, FailureType, Status, + StreamDescriptor, TraceType, ) -from airbyte_cdk.models.airbyte_protocol import Type as MessageType +from airbyte_cdk.models import Type as MessageType from airbyte_cdk.utils.traced_exception import AirbyteTracedException -from airbyte_protocol.models import StreamDescriptor +from orjson import orjson _AN_EXCEPTION = ValueError("An exception") _A_STREAM_DESCRIPTOR = StreamDescriptor(name="a_stream") @@ -90,12 +91,12 @@ def test_emit_message(capsys): ) expected_message = AirbyteMessage( - type="TRACE", + type=MessageType.TRACE, trace=AirbyteTraceMessage( - type="ERROR", + type=TraceType.ERROR, emitted_at=0.0, error=AirbyteErrorTraceMessage( - failure_type="system_error", + failure_type=FailureType.system_error, message="user-friendly message", internal_message="internal message", stack_trace="RuntimeError: oh no\n", @@ -106,9 +107,8 @@ def test_emit_message(capsys): traced_exc.emit_message() stdout = capsys.readouterr().out - printed_message = AirbyteMessage.parse_obj(json.loads(stdout)) + printed_message = AirbyteMessageSerializer.load(orjson.loads(stdout)) printed_message.trace.emitted_at = 0.0 - assert printed_message == expected_message