1717
1818logger = logging .getLogger ("airbyte" )
1919
20- LINKEDIN_VERSION_API = "202305 "
20+ LINKEDIN_VERSION_API = "202404 "
2121
2222
2323class LinkedinAdsStream (HttpStream , ABC ):
@@ -64,26 +64,13 @@ def path(
6464
6565 def next_page_token (self , response : requests .Response ) -> Optional [Mapping [str , Any ]]:
6666 """
67- To paginate through results, begin with a start value of 0 and a count value of N.
68- To get the next page, set start value to N, while the count value stays the same.
69- We have reached the end of the dataset when the response contains fewer elements than the `count` parameter request.
70- https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/pagination?context=linkedin/marketing/context
67+ Cursor based pagination using the pageSize and pageToken parameters.
7168 """
7269 parsed_response = response .json ()
73- is_elements_less_than_limit = len (parsed_response .get ("elements" )) < self .records_limit
74-
75- # Note: The API might return fewer records than requested within the limits during pagination.
76- # This behavior is documented at: https://github.com/airbytehq/airbyte/issues/34164
77- paging_params = parsed_response .get ("paging" , {})
78- is_end_of_records = (
79- paging_params ["total" ] - paging_params ["start" ] <= self .records_limit
80- if all (param in paging_params for param in ("total" , "start" ))
81- else True
82- )
83-
84- if is_elements_less_than_limit and is_end_of_records :
70+ if parsed_response .get ("metadata" , {}).get ("nextPageToken" ):
71+ return {"pageToken" : parsed_response ["metadata" ]["nextPageToken" ]}
72+ else :
8573 return None
86- return {"start" : paging_params .get ("start" ) + self .records_limit }
8774
8875 def request_headers (
8976 self , stream_state : Mapping [str , Any ], stream_slice : Mapping [str , Any ] = None , next_page_token : Mapping [str , Any ] = None
@@ -96,7 +83,7 @@ def request_params(
9683 stream_slice : Mapping [str , Any ] = None ,
9784 next_page_token : Mapping [str , Any ] = None ,
9885 ) -> MutableMapping [str , Any ]:
99- params = {"count " : self .records_limit , "q" : "search" }
86+ params = {"pageSize " : self .records_limit , "q" : "search" }
10087 if next_page_token :
10188 params .update (** next_page_token )
10289 return params
@@ -130,6 +117,44 @@ def should_retry(self, response: requests.Response) -> bool:
130117 return super ().should_retry (response )
131118
132119
120+ class OffsetPaginationMixin :
121+ """Mixin for offset based pagination for endpoints tha tdoesnt support cursor based pagination"""
122+
123+ def request_params (
124+ self ,
125+ stream_state : Mapping [str , Any ],
126+ stream_slice : Mapping [str , Any ] = None ,
127+ next_page_token : Mapping [str , Any ] = None ,
128+ ) -> MutableMapping [str , Any ]:
129+ params = {"count" : self .records_limit , "q" : "search" }
130+ if next_page_token :
131+ params .update (** next_page_token )
132+ return params
133+
134+ def next_page_token (self , response : requests .Response ) -> Optional [Mapping [str , Any ]]:
135+ """
136+ To paginate through results, begin with a start value of 0 and a count value of N.
137+ To get the next page, set start value to N, while the count value stays the same.
138+ We have reached the end of the dataset when the response contains fewer elements than the `count` parameter request.
139+ https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/pagination?context=linkedin/marketing/context
140+ """
141+ parsed_response = response .json ()
142+ is_elements_less_than_limit = len (parsed_response .get ("elements" )) < self .records_limit
143+
144+ # Note: The API might return fewer records than requested within the limits during pagination.
145+ # This behavior is documented at: https://github.com/airbytehq/airbyte/issues/34164
146+ paging_params = parsed_response .get ("paging" , {})
147+ is_end_of_records = (
148+ paging_params ["total" ] - paging_params ["start" ] <= self .records_limit
149+ if all (param in paging_params for param in ("total" , "start" ))
150+ else True
151+ )
152+
153+ if is_elements_less_than_limit and is_end_of_records :
154+ return None
155+ return {"start" : paging_params .get ("start" ) + self .records_limit }
156+
157+
133158class Accounts (LinkedinAdsStream ):
134159 """
135160 Get Accounts data. More info about LinkedIn Ads / Accounts:
@@ -227,7 +252,7 @@ def read_records(
227252 yield from self .filter_records_newer_than_state (stream_state = stream_state , records_slice = child_stream_slice )
228253
229254
230- class AccountUsers (LinkedInAdsStreamSlicing ):
255+ class AccountUsers (OffsetPaginationMixin , LinkedInAdsStreamSlicing ):
231256 """
232257 Get AccountUsers data using `account_id` slicing. More info about LinkedIn Ads / AccountUsers:
233258 https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads/account-structure/create-and-manage-account-users?tabs=http&view=li-lms-2023-05#find-ad-account-users-by-accounts
@@ -365,7 +390,7 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late
365390 return {self .cursor_field : max (latest_record .get (self .cursor_field ), int (current_stream_state .get (self .cursor_field )))}
366391
367392
368- class Conversions (LinkedInAdsStreamSlicing ):
393+ class Conversions (OffsetPaginationMixin , LinkedInAdsStreamSlicing ):
369394 """
370395 Get Conversions data using `account_id` slicing.
371396 https://learn.microsoft.com/en-us/linkedin/marketing/integrations/ads-reporting/conversion-tracking?view=li-lms-2023-05&tabs=curl#find-conversions-by-ad-account
0 commit comments