|
| 1 | +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. |
| 2 | + |
| 3 | +from dataclasses import InitVar, dataclass |
| 4 | +from typing import Any, Iterable, Mapping, Optional |
| 5 | + |
| 6 | +from airbyte_cdk.sources.declarative.incremental import DeclarativeCursor |
| 7 | +from airbyte_cdk.sources.declarative.types import Record, StreamSlice, StreamState |
| 8 | + |
| 9 | + |
| 10 | +@dataclass |
| 11 | +class ResumableFullRefreshCursor(DeclarativeCursor): |
| 12 | + parameters: InitVar[Mapping[str, Any]] |
| 13 | + |
| 14 | + def __post_init__(self, parameters: Mapping[str, Any]) -> None: |
| 15 | + self._cursor: StreamState = {} |
| 16 | + |
| 17 | + def get_stream_state(self) -> StreamState: |
| 18 | + return self._cursor |
| 19 | + |
| 20 | + def set_initial_state(self, stream_state: StreamState) -> None: |
| 21 | + self._cursor = stream_state |
| 22 | + |
| 23 | + def observe(self, stream_slice: StreamSlice, record: Record) -> None: |
| 24 | + """ |
| 25 | + Resumable full refresh manages state using a page number so it does not need to update state by observing incoming records. |
| 26 | + """ |
| 27 | + pass |
| 28 | + |
| 29 | + def close_slice(self, stream_slice: StreamSlice, *args: Any) -> None: |
| 30 | + # The ResumableFullRefreshCursor doesn't support nested streams yet so receiving a partition is unexpected |
| 31 | + if stream_slice.partition: |
| 32 | + raise ValueError(f"Stream slice {stream_slice} should not have a partition. Got {stream_slice.partition}.") |
| 33 | + self._cursor = stream_slice.cursor_slice |
| 34 | + |
| 35 | + def should_be_synced(self, record: Record) -> bool: |
| 36 | + """ |
| 37 | + Unlike date-based cursors which filter out records outside slice boundaries, resumable full refresh records exist within pages |
| 38 | + that don't have filterable bounds. We should always return them. |
| 39 | + """ |
| 40 | + return True |
| 41 | + |
| 42 | + def is_greater_than_or_equal(self, first: Record, second: Record) -> bool: |
| 43 | + """ |
| 44 | + RFR record don't have ordering to be compared between one another. |
| 45 | + """ |
| 46 | + return False |
| 47 | + |
| 48 | + def select_state(self, stream_slice: Optional[StreamSlice] = None) -> Optional[StreamState]: |
| 49 | + # A top-level RFR cursor only manages the state of a single partition |
| 50 | + return self._cursor |
| 51 | + |
| 52 | + def stream_slices(self) -> Iterable[StreamSlice]: |
| 53 | + """ |
| 54 | + Resumable full refresh cursors only return a single slice and can't perform partitioning because iteration is done per-page |
| 55 | + along an unbounded set. |
| 56 | + """ |
| 57 | + yield from [StreamSlice(cursor_slice=self._cursor, partition={})] |
| 58 | + |
| 59 | + # This is an interesting pattern that might not seem obvious at first glance. This cursor itself has no functional need to |
| 60 | + # inject any request values into the outbound response because the up-to-date pagination state is already loaded and |
| 61 | + # maintained by the paginator component |
| 62 | + def get_request_params( |
| 63 | + self, |
| 64 | + *, |
| 65 | + stream_state: Optional[StreamState] = None, |
| 66 | + stream_slice: Optional[StreamSlice] = None, |
| 67 | + next_page_token: Optional[Mapping[str, Any]] = None, |
| 68 | + ) -> Mapping[str, Any]: |
| 69 | + return {} |
| 70 | + |
| 71 | + def get_request_headers( |
| 72 | + self, |
| 73 | + *, |
| 74 | + stream_state: Optional[StreamState] = None, |
| 75 | + stream_slice: Optional[StreamSlice] = None, |
| 76 | + next_page_token: Optional[Mapping[str, Any]] = None, |
| 77 | + ) -> Mapping[str, Any]: |
| 78 | + return {} |
| 79 | + |
| 80 | + def get_request_body_data( |
| 81 | + self, |
| 82 | + *, |
| 83 | + stream_state: Optional[StreamState] = None, |
| 84 | + stream_slice: Optional[StreamSlice] = None, |
| 85 | + next_page_token: Optional[Mapping[str, Any]] = None, |
| 86 | + ) -> Mapping[str, Any]: |
| 87 | + return {} |
| 88 | + |
| 89 | + def get_request_body_json( |
| 90 | + self, |
| 91 | + *, |
| 92 | + stream_state: Optional[StreamState] = None, |
| 93 | + stream_slice: Optional[StreamSlice] = None, |
| 94 | + next_page_token: Optional[Mapping[str, Any]] = None, |
| 95 | + ) -> Mapping[str, Any]: |
| 96 | + return {} |
0 commit comments