284
284
},
285
285
"record_selector" : {
286
286
"type" : "RecordSelector" ,
287
- "extractor" : {"type" : "DpathExtractor" , "field_path" : []},
287
+ "extractor" : {
288
+ "type" : "DpathExtractor" ,
289
+ "field_path" : [],
290
+ },
288
291
},
289
292
},
290
293
"schema_loader" : {
291
294
"type" : "InlineSchemaLoader" ,
292
295
"schema" : {
293
296
"$schema" : "http://json-schema.org/schema#" ,
294
- "properties" : {
295
- "id" : {"type" : "integer" }
296
- },
297
+ "properties" : {"id" : {"type" : "integer" }},
297
298
"type" : "object" ,
298
299
},
299
300
},
300
- }
301
+ },
301
302
}
302
- ]
303
- }
303
+ ],
304
+ },
304
305
},
305
306
"components_mapping" : [
306
307
{
322
323
}
323
324
],
324
325
}
326
+
327
+
325
328
@pytest .mark .parametrize (
326
329
"components_mapping, retriever_data, stream_template_config, expected_result" ,
327
330
[
@@ -358,6 +361,7 @@ def test_http_components_resolver(
358
361
result = list (resolver .resolve_components (stream_template_config = stream_template_config ))
359
362
assert result == expected_result
360
363
364
+
361
365
@pytest .mark .parametrize (
362
366
"components_mapping, retriever_data, stream_template_config, expected_result" ,
363
367
[
@@ -394,6 +398,7 @@ def test_http_components_resolver_with_stream_slices(
394
398
result = list (resolver .resolve_components (stream_template_config = stream_template_config ))
395
399
assert result == expected_result
396
400
401
+
397
402
def test_dynamic_streams_read_with_http_components_resolver ():
398
403
expected_stream_names = ["item_1" , "item_2" ]
399
404
with HttpMocker () as http_mocker :
@@ -466,16 +471,18 @@ def test_duplicated_dynamic_streams_read_with_http_components_resolver():
466
471
== "Dynamic streams list contains a duplicate name: item_2. Please contact Airbyte Support."
467
472
)
468
473
474
+
469
475
def test_dynamic_streams_with_http_components_resolver_retriever_with_parent_stream ():
470
476
expected_stream_names = [
471
- "parent_1_item_1" , "parent_1_item_2" , "parent_2_item_1" , "parent_2_item_2"
477
+ "parent_1_item_1" ,
478
+ "parent_1_item_2" ,
479
+ "parent_2_item_1" ,
480
+ "parent_2_item_2" ,
472
481
]
473
482
with HttpMocker () as http_mocker :
474
483
http_mocker .get (
475
484
HttpRequest (url = "https://api.test.com/parents" ),
476
- HttpResponse (
477
- body = json .dumps ([{"id" : 1 }, {"id" : 2 }])
478
- ),
485
+ HttpResponse (body = json .dumps ([{"id" : 1 }, {"id" : 2 }])),
479
486
)
480
487
parent_ids = [1 , 2 ]
481
488
for parent_id in parent_ids :
@@ -494,13 +501,14 @@ def test_dynamic_streams_with_http_components_resolver_retriever_with_parent_str
494
501
for dynamic_stream_path in dynamic_stream_paths :
495
502
http_mocker .get (
496
503
HttpRequest (url = f"https://api.test.com/{ dynamic_stream_path } " ),
497
- HttpResponse (
498
- body = json .dumps ([{"ABC" : 1 , "AED" : 2 }])
499
- ),
504
+ HttpResponse (body = json .dumps ([{"ABC" : 1 , "AED" : 2 }])),
500
505
)
501
506
502
507
source = ConcurrentDeclarativeSource (
503
- source_config = _MANIFEST_WITH_HTTP_COMPONENT_RESOLVER_WITH_RETRIEVER_WITH_PARENT_STREAM , config = _CONFIG , catalog = None , state = None
508
+ source_config = _MANIFEST_WITH_HTTP_COMPONENT_RESOLVER_WITH_RETRIEVER_WITH_PARENT_STREAM ,
509
+ config = _CONFIG ,
510
+ catalog = None ,
511
+ state = None ,
504
512
)
505
513
506
514
actual_catalog = source .discover (logger = source .logger , config = _CONFIG )
@@ -520,4 +528,4 @@ def test_dynamic_streams_with_http_components_resolver_retriever_with_parent_str
520
528
assert len (actual_catalog .streams ) == 4
521
529
assert [stream .name for stream in actual_catalog .streams ] == expected_stream_names
522
530
assert len (records ) == 4
523
- assert [record .stream for record in records ] == expected_stream_names
531
+ assert [record .stream for record in records ] == expected_stream_names
0 commit comments