1111
1212from .exceptions import TypeSalesforceException
1313from .rate_limiting import default_backoff_handler
14- from .utils import filter_streams
14+ from .utils import filter_streams_by_criteria
1515
1616STRING_TYPES = [
1717 "byte" ,
@@ -191,7 +191,9 @@ def __init__(
191191 self .access_token = None
192192 self .instance_url = None
193193 self .session = requests .Session ()
194- self .is_sandbox = is_sandbox is True or (isinstance (is_sandbox , str ) and is_sandbox .lower () == "true" )
194+ self .is_sandbox = is_sandbox in [True , "true" ]
195+ if self .is_sandbox :
196+ self .logger .info ("using SANDBOX of Salesforce" )
195197 self .start_date = start_date
196198
197199 def _get_standard_headers (self ):
@@ -206,30 +208,37 @@ def filter_streams(self, stream_name: str) -> bool:
206208 return False
207209 return True
208210
209- def get_validated_streams (self , config : Mapping [str , Any ], catalog : ConfiguredAirbyteCatalog = None ):
210- salesforce_objects = self .describe ()["sobjects" ]
211- stream_objects = []
212- for stream_object in salesforce_objects :
211+ def get_validated_streams (self , config : Mapping [str , Any ], catalog : ConfiguredAirbyteCatalog = None ) -> Mapping [str , Any ]:
212+ """Selects all validated streams with additional filtering:
213+ 1) skip all sobjects with negative value of the flag "queryable"
214+ 2) user can set search criterias of necessary streams
215+ 3) selection by catalog settings
216+ """
217+ stream_objects = {}
218+ for stream_object in self .describe ()["sobjects" ]:
213219 if stream_object ["queryable" ]:
214- stream_objects . append ( stream_object )
220+ stream_objects [ stream_object . pop ( "name" )] = stream_object
215221 else :
216222 self .logger .warn (f"Stream { stream_object ['name' ]} )
217223
218- stream_names = [stream_object ["name" ] for stream_object in stream_objects ]
219224 if catalog :
220- return [configured_stream .stream .name for configured_stream in catalog .streams ], stream_objects
225+ return {
226+ configured_stream .stream .name : stream_objects [configured_stream .stream .name ]
227+ for configured_stream in catalog .streams
228+ if configured_stream .stream .name in stream_objects
229+ }
221230
231+ stream_names = list (stream_objects .keys ())
222232 if config .get ("streams_criteria" ):
223233 filtered_stream_list = []
224234 for stream_criteria in config ["streams_criteria" ]:
225- filtered_stream_list += filter_streams (
235+ filtered_stream_list += filter_streams_by_criteria (
226236 streams_list = stream_names , search_word = stream_criteria ["value" ], search_criteria = stream_criteria ["criteria" ]
227237 )
228238 stream_names = list (set (filtered_stream_list ))
229239
230240 validated_streams = [stream_name for stream_name in stream_names if self .filter_streams (stream_name )]
231- validated_stream_objects = [stream_object for stream_object in stream_objects if stream_object ["name" ] in validated_streams ]
232- return validated_streams , validated_stream_objects
241+ return {stream_name : sobject_options for stream_name , sobject_options in stream_objects .items () if stream_name in validated_streams }
233242
234243 @default_backoff_handler (max_tries = 5 , factor = 15 )
235244 def _make_request (
@@ -261,20 +270,20 @@ def login(self):
261270 self .access_token = auth ["access_token" ]
262271 self .instance_url = auth ["instance_url" ]
263272
264- def describe (self , sobject : str = None , stream_objects : List = None ) -> Mapping [str , Any ]:
273+ def describe (self , sobject : str = None , sobject_options : Mapping [ str , Any ] = None ) -> Mapping [str , Any ]:
265274 """Describes all objects or a specific object"""
266275 headers = self ._get_standard_headers ()
267276
268277 endpoint = "sobjects" if not sobject else f"sobjects/{ sobject }
269278
270279 url = f"{ self .instance_url } { self .version } { endpoint }
271280 resp = self ._make_request ("GET" , url , headers = headers )
272- if resp .status_code == 404 :
273- self .logger .error (f"Filtered stream objects : { stream_objects } )
281+ if resp .status_code == 404 and sobject :
282+ self .logger .error (f"not found a description for the sobject ' { sobject } '. Sobject options : { sobject_options } )
274283 return resp .json ()
275284
276- def generate_schema (self , stream_name : str = None , stream_objects : List = None ) -> Mapping [str , Any ]:
277- response = self .describe (stream_name , stream_objects )
285+ def generate_schema (self , stream_name : str = None , stream_options : Mapping [ str , Any ] = None ) -> Mapping [str , Any ]:
286+ response = self .describe (stream_name , stream_options )
278287 schema = {"$schema" : "http://json-schema.org/draft-07/schema#" , "type" : "object" , "additionalProperties" : True , "properties" : {}}
279288 for field in response ["fields" ]:
280289 schema ["properties" ][field ["name" ]] = self .field_to_property_schema (field )
0 commit comments