Skip to content

Commit 63062d9

Browse files
committed
pr feedback and cleaning up a few things, fix a test that was failing
1 parent abbc2a1 commit 63062d9

File tree

5 files changed

+9
-6
lines changed

5 files changed

+9
-6
lines changed

airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2892,6 +2892,9 @@ def create_simple_retriever(
28922892
):
28932893
query_properties_definitions = []
28942894
for key, request_parameter in model.requester.request_parameters.items():
2895+
# When translating JSON schema into Pydantic models, enforcing types for arrays containing both
2896+
# concrete string complex object definitions like QueryProperties would get resolved to Union[str, Any].
2897+
# This adds the extra validation that we couldn't get for free in Pydantic model generation
28952898
if (
28962899
isinstance(request_parameter, Mapping)
28972900
and request_parameter.get("type") == "QueryProperties"

airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313
@dataclass
1414
class PropertiesFromEndpoint:
1515
"""
16-
tbd
16+
Component that defines the behavior around how to dynamically retrieve a set of request properties from an
17+
API endpoint. The set retrieved can then be injected into the requests to extract records from an API source.
1718
"""
1819

1920
property_field_path: List[str]
@@ -36,5 +37,4 @@ def get_properties_from_endpoint(self, stream_slice: Optional[StreamSlice]) -> I
3637
node.eval(self.config) if not isinstance(node, str) else node
3738
for node in self._property_field_path
3839
]
39-
40-
yield dpath.get(property_obj, path) # type: ignore # extracted will be a MutableMapping, given input data structure
40+
yield dpath.get(property_obj, path, default=[]) # type: ignore # extracted will be a MutableMapping, given input data structure

airbyte_cdk/sources/declarative/requesters/query_properties/property_chunking.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313

1414
class PropertyLimitType(Enum):
1515
"""
16-
yeah
16+
The heuristic that determines when the maximum size of the current chunk of properties and when a new
17+
one should be started.
1718
"""
1819

1920
characters = "characters"

airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ def get_request_property_chunks(
3939
property_fields=fields, always_include_properties=self.always_include_properties
4040
)
4141
else:
42-
yield from [list(fields)]
42+
yield list(fields)
4343

4444
def has_multiple_chunks(self, stream_slice: Optional[StreamSlice]) -> bool:
4545
property_chunks = iter(self.get_request_property_chunks(stream_slice=stream_slice))

unit_tests/sources/declarative/test_manifest_declarative_source.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -627,7 +627,6 @@ def test_source_missing_checker_fails_validation(self):
627627
},
628628
}
629629
],
630-
"check": {"type": "CheckStream"},
631630
}
632631
with pytest.raises(ValidationError):
633632
ManifestDeclarativeSource(source_config=manifest)

0 commit comments

Comments
 (0)