Skip to content

Commit 0ceb769

Browse files
girardaerohmensing
girarda
and
erohmensing
authored
refactor!(airbyte-cdk): Delete deprecated AirbyteLogger, AirbyteSpec, and Authenticators + move public classes to the top level init file (#37805)
Co-authored-by: Ella Rohm-Ensing <[email protected]>
1 parent 7a35021 commit 0ceb769

File tree

15 files changed

+317
-485
lines changed

15 files changed

+317
-485
lines changed

‎airbyte-cdk/python/airbyte_cdk/__init__.py

+237-4
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,243 @@
22
# Copyright (c) 2021 Airbyte, Inc., all rights reserved.
33
#
44

5-
from .connector import AirbyteSpec, Connector
6-
from .entrypoint import AirbyteEntrypoint
7-
from .logger import AirbyteLogger
85
from importlib import metadata
96

10-
__all__ = ["AirbyteEntrypoint", "AirbyteLogger", "AirbyteSpec", "Connector"]
7+
from .destinations import Destination
8+
from .models import AirbyteConnectionStatus, AirbyteMessage, ConfiguredAirbyteCatalog, Status, Type, FailureType, AirbyteStream, AdvancedAuth, DestinationSyncMode, ConnectorSpecification, OAuthConfigSpecification, OrchestratorType, ConfiguredAirbyteStream, SyncMode, AirbyteLogMessage, Level, AirbyteRecordMessage
9+
10+
from .sources import Source
11+
from .config_observation import create_connector_config_control_message, emit_configuration_as_airbyte_control_message
12+
from .connector import BaseConnector, Connector
13+
14+
from .entrypoint import launch, AirbyteEntrypoint
15+
16+
from .logger import AirbyteLogFormatter, init_logger
17+
from .sources import AbstractSource
18+
from .sources.concurrent_source.concurrent_source import ConcurrentSource
19+
from .sources.concurrent_source.concurrent_source_adapter import ConcurrentSourceAdapter
20+
from .sources.config import BaseConfig
21+
from .sources.types import Config, Record, StreamSlice
22+
from .sources.connector_state_manager import ConnectorStateManager
23+
from .sources.declarative.auth import DeclarativeOauth2Authenticator
24+
from .sources.declarative.auth.declarative_authenticator import DeclarativeAuthenticator
25+
from .sources.declarative.auth.declarative_authenticator import NoAuth
26+
from .sources.declarative.auth.oauth import DeclarativeSingleUseRefreshTokenOauth2Authenticator
27+
from .sources.declarative.auth.token import BasicHttpAuthenticator, BearerAuthenticator, ApiKeyAuthenticator
28+
from .sources.declarative.datetime.min_max_datetime import MinMaxDatetime
29+
from .sources.declarative.declarative_stream import DeclarativeStream
30+
from .sources.declarative.decoders import Decoder, JsonDecoder
31+
from .sources.declarative.exceptions import ReadException
32+
from .sources.declarative.extractors import DpathExtractor, RecordSelector
33+
from .sources.declarative.extractors.record_extractor import RecordExtractor
34+
from .sources.declarative.extractors.record_filter import RecordFilter
35+
from .sources.declarative.incremental import DatetimeBasedCursor
36+
from .sources.declarative.interpolation import InterpolatedString, InterpolatedBoolean
37+
from .sources.declarative.manifest_declarative_source import ManifestDeclarativeSource
38+
from .sources.declarative.migrations.legacy_to_per_partition_state_migration import LegacyToPerPartitionStateMigration
39+
40+
from .sources.declarative.partition_routers import SinglePartitionRouter, SubstreamPartitionRouter
41+
from .sources.declarative.partition_routers.substream_partition_router import ParentStreamConfig
42+
from .sources.declarative.requesters import Requester, HttpRequester
43+
44+
from .sources.declarative.requesters.error_handlers import BackoffStrategy
45+
from .sources.declarative.requesters.error_handlers.response_status import ResponseStatus
46+
from .sources.declarative.requesters.paginators import DefaultPaginator, PaginationStrategy
47+
from .sources.declarative.requesters.paginators.strategies import OffsetIncrement, CursorPaginationStrategy, PageIncrement, StopConditionPaginationStrategyDecorator
48+
49+
from .sources.declarative.requesters.request_option import RequestOption, RequestOptionType
50+
51+
from .sources.declarative.requesters.request_options.interpolated_request_input_provider import InterpolatedRequestInputProvider
52+
from .sources.declarative.requesters.requester import HttpMethod
53+
from .sources.declarative.retrievers import SimpleRetriever
54+
from .sources.declarative.schema import JsonFileSchemaLoader
55+
from .sources.declarative.stream_slicers import CartesianProductStreamSlicer
56+
from .sources.declarative.transformations.add_fields import AddFields, AddedFieldDefinition
57+
from .sources.declarative.transformations.transformation import RecordTransformation
58+
from .sources.declarative.types import FieldPointer
59+
from .sources.declarative.yaml_declarative_source import YamlDeclarativeSource
60+
from .sources.message import InMemoryMessageRepository, MessageRepository
61+
from .sources.source import TState
62+
from .sources.streams.availability_strategy import AvailabilityStrategy
63+
from .sources.streams.call_rate import AbstractAPIBudget, HttpAPIBudget, HttpRequestMatcher, MovingWindowCallRatePolicy, Rate, CachedLimiterSession, LimiterSession
64+
from .sources.streams.concurrent.adapters import StreamFacade
65+
from .sources.streams.concurrent.cursor import ConcurrentCursor, CursorField, FinalStateCursor
66+
from .sources.streams.concurrent.cursor import Cursor
67+
from .sources.streams.concurrent.state_converters.datetime_stream_state_converter import EpochValueConcurrentStreamStateConverter, IsoMillisConcurrentStreamStateConverter
68+
from .sources.streams.core import Stream, IncrementalMixin, package_name_from_class
69+
from .sources.streams.http import HttpStream, HttpSubStream
70+
from .sources.streams.http.availability_strategy import HttpAvailabilityStrategy
71+
from .sources.streams.http.exceptions import BaseBackoffException, DefaultBackoffException, UserDefinedBackoffException
72+
from .sources.streams.http.rate_limiting import default_backoff_handler
73+
from .sources.streams.http.requests_native_auth import Oauth2Authenticator, TokenAuthenticator, SingleUseRefreshTokenOauth2Authenticator
74+
from .sources.streams.http.requests_native_auth.abstract_token import AbstractHeaderAuthenticator
75+
from .sources.utils import casing
76+
from .sources.utils.schema_helpers import InternalConfig, ResourceSchemaLoader, check_config_against_spec_or_exit, split_config, expand_refs
77+
from .sources.utils.transform import TransformConfig, TypeTransformer
78+
from .utils import AirbyteTracedException, is_cloud_environment
79+
from .utils.constants import ENV_REQUEST_CACHE_PATH
80+
from .utils.event_timing import create_timer
81+
from .utils.oneof_option_config import OneOfOptionConfig
82+
from .utils.spec_schema_transformations import resolve_refs
83+
from .utils.stream_status_utils import as_airbyte_message
84+
85+
86+
__all__ = [
87+
# Availability strategy
88+
"AvailabilityStrategy",
89+
"HttpAvailabilityStrategy",
90+
91+
# Concurrent
92+
"ConcurrentCursor",
93+
"ConcurrentSource",
94+
"ConcurrentSourceAdapter",
95+
"Cursor",
96+
"CursorField",
97+
"DEFAULT_CONCURRENCY",
98+
"EpochValueConcurrentStreamStateConverter",
99+
"FinalStateCursor",
100+
"IsoMillisConcurrentStreamStateConverter",
101+
"StreamFacade",
102+
103+
# Config observation
104+
"create_connector_config_control_message",
105+
"emit_configuration_as_airbyte_control_message",
106+
107+
# Connector
108+
"AbstractSource",
109+
"BaseConfig",
110+
"BaseConnector",
111+
"Connector",
112+
"Destination",
113+
"Source",
114+
"TState",
115+
116+
# Declarative
117+
"AddFields",
118+
"AddedFieldDefinition",
119+
"ApiKeyAuthenticator",
120+
"BackoffStrategy",
121+
"BasicHttpAuthenticator",
122+
"BearerAuthenticator",
123+
"CartesianProductStreamSlicer",
124+
"CursorPaginationStrategy",
125+
"DatetimeBasedCursor"
126+
"DeclarativeAuthenticator",
127+
"DeclarativeOauth2Authenticator",
128+
"DeclarativeSingleUseRefreshTokenOauth2Authenticator",
129+
"DeclarativeStream",
130+
"Decoder",
131+
"DefaultPaginator",
132+
"DpathExtractor",
133+
"FieldPointer",
134+
"HttpMethod",
135+
"HttpRequester",
136+
"InterpolatedBoolean",
137+
"InterpolatedRequestInputProvider",
138+
"InterpolatedString",
139+
"JsonDecoder",
140+
"JsonFileSchemaLoader",
141+
"LegacyToPerPartitionStateMigration",
142+
"ManifestDeclarativeSource",
143+
"MinMaxDatetime",
144+
"NoAuth",
145+
"OffsetIncrement",
146+
"PageIncrement",
147+
"PaginationStrategy",
148+
"ParentStreamConfig",
149+
"ReadException",
150+
"RecordExtractor",
151+
"RecordFilter",
152+
"RecordSelector",
153+
"RecordTransformation",
154+
"RequestOption",
155+
"RequestOptionType",
156+
"Requester",
157+
"ResponseStatus",
158+
"SimpleRetriever",
159+
"SinglePartitionRouter",
160+
"StopConditionPaginationStrategyDecorator",
161+
"StreamSlice",
162+
"SubstreamPartitionRouter",
163+
"YamlDeclarativeSource",
164+
165+
# Entrypoint
166+
"launch",
167+
"AirbyteEntrypoint",
168+
169+
# HTTP
170+
"AbstractAPIBudget",
171+
"AbstractHeaderAuthenticator",
172+
"BaseBackoffException",
173+
"CachedLimiterSession",
174+
"DefaultBackoffException",
175+
"default_backoff_handler",
176+
"HttpAPIBudget",
177+
"HttpAuthenticator",
178+
"HttpRequestMatcher",
179+
"HttpStream",
180+
"HttpSubStream",
181+
"LimiterSession",
182+
"MovingWindowCallRatePolicy",
183+
"MultipleTokenAuthenticator"
184+
"Oauth2Authenticator",
185+
"Rate",
186+
"SingleUseRefreshTokenOauth2Authenticator",
187+
"TokenAuthenticator",
188+
"UserDefinedBackoffException",
189+
190+
# Logger
191+
"AirbyteLogFormatter",
192+
"init_logger",
193+
194+
# Protocol classes
195+
"AirbyteStream",
196+
"AirbyteConnectionStatus", "AirbyteMessage", "ConfiguredAirbyteCatalog", "Status", "Type",
197+
"OrchestratorType",
198+
"ConfiguredAirbyteStream",
199+
"DestinationSyncMode",
200+
"SyncMode",
201+
"FailureType",
202+
"AdvancedAuth",
203+
"AirbyteLogMessage",
204+
"OAuthConfigSpecification",
205+
"ConnectorSpecification",
206+
"Level",
207+
"AirbyteRecordMessage",
208+
209+
# Repository
210+
"InMemoryMessageRepository",
211+
"MessageRepository",
212+
213+
# State management
214+
"ConnectorStateManager",
215+
216+
# Stream
217+
"IncrementalMixin",
218+
"Stream",
219+
"StreamData",
220+
"package_name_from_class",
221+
222+
# Utils
223+
"AirbyteTracedException",
224+
"is_cloud_environment",
225+
"casing",
226+
"InternalConfig",
227+
"ResourceSchemaLoader",
228+
"check_config_against_spec_or_exit",
229+
"split_config",
230+
"TransformConfig",
231+
"TypeTransformer",
232+
"ENV_REQUEST_CACHE_PATH",
233+
"create_timer",
234+
"OneOfOptionConfig",
235+
"resolve_refs",
236+
"as_airbyte_message",
237+
238+
# Types
239+
"Config",
240+
"Record",
241+
"Source",
242+
"StreamSlice",
243+
]
11244
__version__ = metadata.version("airbyte_cdk")

‎airbyte-cdk/python/airbyte_cdk/connector.py

+4-15
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import os
99
import pkgutil
1010
from abc import ABC, abstractmethod
11-
from typing import Any, Generic, List, Mapping, Optional, Protocol, TypeVar, Union
11+
from typing import Any, Generic, Mapping, Optional, Protocol, TypeVar
1212

1313
import yaml
1414
from airbyte_cdk.models import AirbyteConnectionStatus, ConnectorSpecification
@@ -22,17 +22,6 @@ def load_optional_package_file(package: str, filename: str) -> Optional[bytes]:
2222
return None
2323

2424

25-
class AirbyteSpec(object):
26-
@staticmethod
27-
def from_file(file_name: str):
28-
with open(file_name) as file:
29-
spec_text = file.read()
30-
return AirbyteSpec(spec_text)
31-
32-
def __init__(self, spec_string):
33-
self.spec_string = spec_string
34-
35-
3625
TConfig = TypeVar("TConfig", bound=Mapping[str, Any])
3726

3827

@@ -57,7 +46,7 @@ def read_config(config_path: str) -> Mapping[str, Any]:
5746
)
5847

5948
@staticmethod
60-
def _read_json_file(file_path: str) -> Union[None, bool, float, int, str, List[Any], Mapping[str, Any]]:
49+
def _read_json_file(file_path: str) -> Any:
6150
with open(file_path, "r") as file:
6251
contents = file.read()
6352

@@ -67,7 +56,7 @@ def _read_json_file(file_path: str) -> Union[None, bool, float, int, str, List[A
6756
raise ValueError(f"Could not read json file {file_path}: {error}. Please ensure that it is a valid JSON.")
6857

6958
@staticmethod
70-
def write_config(config: TConfig, config_path: str):
59+
def write_config(config: TConfig, config_path: str) -> None:
7160
with open(config_path, "w") as fh:
7261
fh.write(json.dumps(config))
7362

@@ -107,7 +96,7 @@ def check(self, logger: logging.Logger, config: TConfig) -> AirbyteConnectionSta
10796

10897
class _WriteConfigProtocol(Protocol):
10998
@staticmethod
110-
def write_config(config: Mapping[str, Any], config_path: str):
99+
def write_config(config: Mapping[str, Any], config_path: str) -> None:
111100
...
112101

113102

‎airbyte-cdk/python/airbyte_cdk/logger.py

+4-36
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,10 @@
55
import json
66
import logging
77
import logging.config
8-
import traceback
9-
from typing import Tuple
8+
from typing import Any, Mapping, Optional, Tuple
109

1110
from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage
1211
from airbyte_cdk.utils.airbyte_secrets_utils import filter_secrets
13-
from deprecated import deprecated
1412

1513
LOGGING_CONFIG = {
1614
"version": 1,
@@ -31,7 +29,7 @@
3129
}
3230

3331

34-
def init_logger(name: str = None):
32+
def init_logger(name: Optional[str] = None) -> logging.Logger:
3533
"""Initial set up of logger"""
3634
logger = logging.getLogger(name)
3735
logger.setLevel(logging.INFO)
@@ -62,10 +60,10 @@ def format(self, record: logging.LogRecord) -> str:
6260
message = super().format(record)
6361
message = filter_secrets(message)
6462
log_message = AirbyteMessage(type="LOG", log=AirbyteLogMessage(level=airbyte_level, message=message))
65-
return log_message.json(exclude_unset=True)
63+
return log_message.json(exclude_unset=True) # type: ignore
6664

6765
@staticmethod
68-
def extract_extra_args_from_record(record: logging.LogRecord):
66+
def extract_extra_args_from_record(record: logging.LogRecord) -> Mapping[str, Any]:
6967
"""
7068
The python logger conflates default args with extra args. We use an empty log record and set operations
7169
to isolate fields passed to the log record via extra by the developer.
@@ -88,33 +86,3 @@ def log_by_prefix(msg: str, default_level: str) -> Tuple[int, str]:
8886
rendered_message = msg
8987

9088
return log_level, rendered_message
91-
92-
93-
@deprecated(version="0.1.47", reason="Use logging.getLogger('airbyte') instead")
94-
class AirbyteLogger:
95-
def log(self, level, message):
96-
log_record = AirbyteLogMessage(level=level, message=message)
97-
log_message = AirbyteMessage(type="LOG", log=log_record)
98-
print(log_message.json(exclude_unset=True))
99-
100-
def fatal(self, message):
101-
self.log("FATAL", message)
102-
103-
def exception(self, message):
104-
message = f"{message}\n{traceback.format_exc()}"
105-
self.error(message)
106-
107-
def error(self, message):
108-
self.log("ERROR", message)
109-
110-
def warn(self, message):
111-
self.log("WARN", message)
112-
113-
def info(self, message):
114-
self.log("INFO", message)
115-
116-
def debug(self, message):
117-
self.log("DEBUG", message)
118-
119-
def trace(self, message):
120-
self.log("TRACE", message)

‎airbyte-cdk/python/airbyte_cdk/sources/file_based/__init__.py

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
from .config.abstract_file_based_spec import AbstractFileBasedSpec
2+
from .config.csv_format import CsvFormat
3+
from .config.file_based_stream_config import FileBasedStreamConfig
4+
from .config.jsonl_format import JsonlFormat
5+
from .exceptions import CustomFileBasedException, ErrorListingFiles, FileBasedSourceError
6+
from .file_based_source import DEFAULT_CONCURRENCY, FileBasedSource
7+
from .file_based_stream_reader import AbstractFileBasedStreamReader, FileReadMode
8+
from .remote_file import RemoteFile
9+
from .stream.cursor import DefaultFileBasedCursor
10+
11+
__all__ = [
12+
"AbstractFileBasedSpec",
13+
"AbstractFileBasedStreamReader",
14+
"CsvFormat",
15+
"CustomFileBasedException",
16+
"DefaultFileBasedCursor",
17+
"ErrorListingFiles",
18+
"FileBasedSource",
19+
"FileBasedSourceError",
20+
"FileBasedStreamConfig",
21+
"FileReadMode",
22+
"JsonlFormat",
23+
"RemoteFile",
24+
]

0 commit comments

Comments
 (0)