Skip to content

File-based CDK: allow to merge schemas with nullable object values #37773

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 2, 2024
Merged

File-based CDK: allow to merge schemas with nullable object values #37773

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions airbyte-cdk/python/airbyte_cdk/sources/file_based/schema_helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,7 @@ def merge_schemas(schema1: SchemaType, schema2: SchemaType) -> SchemaType:
t1 = merged_schema.get(k2)
if t1 is None:
merged_schema[k2] = t2
# do not pass fields with null values to the _choose_wider_type method if their keys are already
# in the merged schema due to issue observed in https://github.com/airbytehq/oncall/issues/4948
elif t1 == t2 or t2["type"] == "null":
elif t1 == t2:
continue
else:
merged_schema[k2] = _choose_wider_type(k2, t1, t2)
Expand All @@ -119,23 +117,27 @@ def _is_valid_type(t: JsonSchemaSupportedType) -> bool:


def _choose_wider_type(key: str, t1: Mapping[str, Any], t2: Mapping[str, Any]) -> Mapping[str, Any]:
if (t1["type"] == "array" or t2["type"] == "array") and t1 != t2:
t1_type = t1["type"]
t2_type = t2["type"]

if (t1_type == "array" or t2_type == "array") and t1 != t2:
raise SchemaInferenceError(
FileBasedSourceError.SCHEMA_INFERENCE_ERROR,
details="Cannot merge schema for unequal array types.",
key=key,
detected_types=f"{t1},{t2}",
)
elif (t1["type"] == "object" or t2["type"] == "object") and t1 != t2:
# Schemas can still be merged if a key contains a null value in either t1 or t2, but it is still an object
elif (t1_type == "object" or t2_type == "object") and t1_type != "null" and t2_type != "null" and t1 != t2:
raise SchemaInferenceError(
FileBasedSourceError.SCHEMA_INFERENCE_ERROR,
details="Cannot merge schema for unequal object types.",
key=key,
detected_types=f"{t1},{t2}",
)
else:
comparable_t1 = get_comparable_type(TYPE_PYTHON_MAPPING[t1["type"]][0]) # accessing the type_mapping value
comparable_t2 = get_comparable_type(TYPE_PYTHON_MAPPING[t2["type"]][0]) # accessing the type_mapping value
comparable_t1 = get_comparable_type(TYPE_PYTHON_MAPPING[t1_type][0]) # accessing the type_mapping value
comparable_t2 = get_comparable_type(TYPE_PYTHON_MAPPING[t2_type][0]) # accessing the type_mapping value
if not comparable_t1 and comparable_t2:
raise SchemaInferenceError(FileBasedSourceError.UNRECOGNIZED_TYPE, key=key, detected_types=f"{t1},{t2}")
return max(
Expand Down
9 changes: 8 additions & 1 deletion airbyte-cdk/python/unit_tests/sources/file_based/test_schema_helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -264,13 +264,20 @@ def test_comparable_types() -> None:
{"a": {"type": "object"}},
{"a": {"type": "null"}},
{"a": {"type": "object"}},
id="single-key-with-null-object-schema2"),
id="single-key-with-null-object-schema2",
),
pytest.param(
{"a": {"type": "object"}},
{"b": {"type": "null"}},
{"a": {"type": "object"}, "b": {"type": "null"}},
id="new-key-with-null-type",
),
pytest.param(
{"a": {"type": "null"}},
{"a": {"type": "object"}},
{"a": {"type": "object"}},
id="single-key-with-null-object-schema1",
),
],
)
def test_merge_schemas(schema1: SchemaType, schema2: SchemaType, expected_result: Optional[SchemaType]) -> None:
Expand Down
Loading