Fix for nullable top-level schemas (#44401)

Author: johnny-schmidt

airbyte-cdk/java/airbyte-cdk/core/src/main/resources/version.properties

@@ -1 +1 @@
-version=0.44.14
+version=0.44.15

airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/problematic_types_coerced_schemaless_messages_out.txt

@@ -1,3 +1,3 @@
-{"schemaless_object":"{\"uuid\":\"38F52396-736D-4B23-B5B4-F504D8894B97\",\"probability\":1.5}","schematized_object":{"id":1,"name":"Joe"},"combined_type":"string1","union_type":10,"schemaless_array":"[10,\"foo\",null,{\"bar\":\"qua\"}]","mixed_array_integer_and_schemaless_object":[15,null,"{\"hello\":\"world\"}"],"array_of_union_integer_and_schemaless_array":[25,null,"[\"goodbye\",\"cruel world\"]"],"union_of_objects_with_properties_identical":{"id":10,"name":"Joe"},"union_of_objects_with_properties_overlapping":{"id":20,"name":"Jane","flagged":true},"union_of_objects_with_properties_nonoverlapping":{"id":30,"name":"Phil","flagged":false,"description":"Very Phil"}, "union_of_objects_with_properties_contradicting": { "id": 1, "name": "Jenny" }, "empty_object": "{}", "object_with_null_properties": "{}", "combined_with_null": "foobar", "union_with_null": "barfoo", "combined_nulls": null}
-{"schemaless_object":"{\"address\":{\"street\":\"113 Hickey Rd\",\"zip\":\"37932\"},\"flags\":[true,false,false]}","schematized_object":{"id":2,"name":"Jane"},"combined_type":20,"union_type":"string2","schemaless_array":"[]","mixed_array_integer_and_schemaless_object":[],"array_of_union_integer_and_schemaless_array":[],"union_of_objects_with_properties_identical":{"id":null,"name":null},"union_of_objects_with_properties_overlapping":{"id":null,"name":null,"flagged":null},"union_of_objects_with_properties_nonoverlapping":{"id":null,"name":null,"flagged":null,"description":null}, "union_of_objects_with_properties_contradicting": { "id": "seal-one-hippity", "name": "James" }, "empty_object": "{\"extra\":\"stuff\"}", "object_with_null_properties": "{\"more\":{\"extra\":\"stuff\"}}", "combined_with_null": "foobar2", "union_with_null": "barfoo2", "combined_nulls": null}
-{ "schemaless_object": null, "schematized_object": null, "combined_type": null, "union_type": null, "schemaless_array": null, "mixed_array_integer_and_schemaless_object": null, "array_of_union_integer_and_schemaless_array": null, "union_of_objects_with_properties_identical": null, "union_of_objects_with_properties_overlapping": null, "union_of_objects_with_properties_nonoverlapping": null, "union_of_objects_with_properties_contradicting":null, "empty_object": null, "object_with_null_properties": null, "combined_with_null": null, "union_with_null": null, "combined_nulls": null }
+{"schemaless_object":"{\"uuid\":\"38F52396-736D-4B23-B5B4-F504D8894B97\",\"probability\":1.5}","schematized_object":{"id":1,"name":"Joe"},"combined_type":"string1","union_type":10,"schemaless_array":"[10,\"foo\",null,{\"bar\":\"qua\"}]","mixed_array_integer_and_schemaless_object":[15,null,"{\"hello\":\"world\"}"],"array_of_union_integer_and_schemaless_array":[25,null,"[\"goodbye\",\"cruel world\"]"],"union_of_objects_with_properties_identical":{"id":10,"name":"Joe"},"union_of_objects_with_properties_overlapping":{"id":20,"name":"Jane","flagged":true},"union_of_objects_with_properties_nonoverlapping":{"id":30,"name":"Phil","flagged":false,"description":"Very Phil"}, "union_of_objects_with_properties_contradicting": { "id": 1, "name": "Jenny" }, "empty_object": "{}", "object_with_null_properties": "{}", "combined_with_null": "foobar", "union_with_null": "barfoo", "combined_nulls": null, "compact_union": { "id": 10, "name": "Tyler" } }
+{"schemaless_object":"{\"address\":{\"street\":\"113 Hickey Rd\",\"zip\":\"37932\"},\"flags\":[true,false,false]}","schematized_object":{"id":2,"name":"Jane"},"combined_type":20,"union_type":"string2","schemaless_array":"[]","mixed_array_integer_and_schemaless_object":[],"array_of_union_integer_and_schemaless_array":[],"union_of_objects_with_properties_identical":{"id":null,"name":null},"union_of_objects_with_properties_overlapping":{"id":null,"name":null,"flagged":null},"union_of_objects_with_properties_nonoverlapping":{"id":null,"name":null,"flagged":null,"description":null}, "union_of_objects_with_properties_contradicting": { "id": "seal-one-hippity", "name": "James" }, "empty_object": "{\"extra\":\"stuff\"}", "object_with_null_properties": "{\"more\":{\"extra\":\"stuff\"}}", "combined_with_null": "foobar2", "union_with_null": "barfoo2", "combined_nulls": null, "compact_union": 4444 }
+{ "schemaless_object": null, "schematized_object": null, "combined_type": null, "union_type": null, "schemaless_array": null, "mixed_array_integer_and_schemaless_object": null, "array_of_union_integer_and_schemaless_array": null, "union_of_objects_with_properties_identical": null, "union_of_objects_with_properties_overlapping": null, "union_of_objects_with_properties_nonoverlapping": null, "union_of_objects_with_properties_contradicting":null, "empty_object": null, "object_with_null_properties": null, "combined_with_null": null, "union_with_null": null, "combined_nulls": null, "compact_union": null }

airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/problematic_types_coerced_schemaless_schema.json

@@ -154,6 +154,24 @@
     },
     "combined_nulls": {
       "type": "null"
+    },
+    "compact_union": {
+      "oneOf": [
+        {
+          "type": "object",
+          "properties": {
+            "id": {
+              "type": "integer"
+            },
+            "name": {
+              "type": "string"
+            }
+          }
+        },
+        {
+          "type": "integer"
+        }
+      ]
     }
   }
 }

airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/problematic_types_configured_catalog.json

@@ -189,6 +189,17 @@
             },
             "combined_nulls": {
               "type": ["null", "null"]
+            },
+            "compact_union": {
+              "type": ["object", "integer"],
+              "properties": {
+                "id": {
+                  "type": "integer"
+                },
+                "name": {
+                  "type": "string"
+                }
+              }
             }
           }
         }

airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/problematic_types_disjoint_union_messages_out.txt

@@ -1,3 +1,3 @@
-{"schemaless_object":"{\"uuid\":\"38F52396-736D-4B23-B5B4-F504D8894B97\",\"probability\":1.5}","schematized_object":{"id":1,"name":"Joe"},"combined_type":{"type":"string","string":"string1","integer":null},"union_type":{"type":"integer","string":null,"integer":10},"schemaless_array":"[10,\"foo\",null,{\"bar\":\"qua\"}]","mixed_array_integer_and_schemaless_object":[15,null,"{\"hello\":\"world\"}"],"array_of_union_integer_and_schemaless_array":[{"type":"integer","integer":25,"string":null},null,{"type":"string","integer":null,"string":"[\"goodbye\",\"cruel world\"]"}],"union_of_objects_with_properties_identical":{"id":10,"name":"Joe"},"union_of_objects_with_properties_overlapping":{"id":20,"name":"Jane","flagged":true},"union_of_objects_with_properties_nonoverlapping":{"id":30,"name":"Phil","flagged":false,"description":"Very Phil"}, "union_of_objects_with_properties_contradicting": { "id": {"type":"integer","integer":1,"string":null}, "name": "Jenny" }, "empty_object": "{}","object_with_null_properties": "{}", "combined_with_null": "foobar", "union_with_null":"barfoo", "combined_nulls": null }}
-{"schemaless_object":"{\"address\":{\"street\":\"113 Hickey Rd\",\"zip\":\"37932\"},\"flags\":[true,false,false]}","schematized_object":{"id":2,"name":"Jane"},"combined_type":{"type":"integer","string":null,"integer":20},"union_type":{"type":"string","string":"string2","integer":null},"schemaless_array":"[]","mixed_array_integer_and_schemaless_object":[],"array_of_union_integer_and_schemaless_array":[],"union_of_objects_with_properties_identical":{"id":null,"name":null},"union_of_objects_with_properties_overlapping":{"id":null,"name":null,"flagged":null},"union_of_objects_with_properties_nonoverlapping":{"id":null,"name":null,"flagged":null,"description":null}, "union_of_objects_with_properties_contradicting": { "id": {"type":"string","integer":null,"string":"seal-one-hippity"}, "name": "James" }, "empty_object": "{\"extra\":\"stuff\"}", "object_with_null_properties": "{\"more\":{\"extra\":\"stuff\"}}", "combined_with_null": "foobar2", "union_with_null": "barfoo2", "combined_nulls": null}
-{ "schemaless_object": null, "schematized_object": null, "combined_type": null, "union_type": null, "schemaless_array": null, "mixed_array_integer_and_schemaless_object": null, "array_of_union_integer_and_schemaless_array": null, "union_of_objects_with_properties_identical": null, "union_of_objects_with_properties_overlapping": null, "union_of_objects_with_properties_nonoverlapping": null, "union_of_objects_with_properties_contradicting": null, "empty_object": null, "object_with_null_properties": null, "combined_with_null": null, "union_with_null": null, "combined_nulls": null }
+{"schemaless_object":"{\"uuid\":\"38F52396-736D-4B23-B5B4-F504D8894B97\",\"probability\":1.5}","schematized_object":{"id":1,"name":"Joe"},"combined_type":{"type":"string","string":"string1","integer":null},"union_type":{"type":"integer","string":null,"integer":10},"schemaless_array":"[10,\"foo\",null,{\"bar\":\"qua\"}]","mixed_array_integer_and_schemaless_object":[15,null,"{\"hello\":\"world\"}"],"array_of_union_integer_and_schemaless_array":[{"type":"integer","integer":25,"string":null},null,{"type":"string","integer":null,"string":"[\"goodbye\",\"cruel world\"]"}],"union_of_objects_with_properties_identical":{"id":10,"name":"Joe"},"union_of_objects_with_properties_overlapping":{"id":20,"name":"Jane","flagged":true},"union_of_objects_with_properties_nonoverlapping":{"id":30,"name":"Phil","flagged":false,"description":"Very Phil"}, "union_of_objects_with_properties_contradicting": { "id": {"type":"integer","integer":1,"string":null}, "name": "Jenny" }, "empty_object": "{}","object_with_null_properties": "{}", "combined_with_null": "foobar", "union_with_null":"barfoo", "combined_nulls": null, "compact_union": {"type": "object", "object": { "id": 10, "name": "Tyler" }, "integer": null } }
+{"schemaless_object":"{\"address\":{\"street\":\"113 Hickey Rd\",\"zip\":\"37932\"},\"flags\":[true,false,false]}","schematized_object":{"id":2,"name":"Jane"},"combined_type":{"type":"integer","string":null,"integer":20},"union_type":{"type":"string","string":"string2","integer":null},"schemaless_array":"[]","mixed_array_integer_and_schemaless_object":[],"array_of_union_integer_and_schemaless_array":[],"union_of_objects_with_properties_identical":{"id":null,"name":null},"union_of_objects_with_properties_overlapping":{"id":null,"name":null,"flagged":null},"union_of_objects_with_properties_nonoverlapping":{"id":null,"name":null,"flagged":null,"description":null}, "union_of_objects_with_properties_contradicting": { "id": {"type":"string","integer":null,"string":"seal-one-hippity"}, "name": "James" }, "empty_object": "{\"extra\":\"stuff\"}", "object_with_null_properties": "{\"more\":{\"extra\":\"stuff\"}}", "combined_with_null": "foobar2", "union_with_null": "barfoo2", "combined_nulls": null, "compact_union": {"type":"integer","integer":4444,"object":null} }
+{ "schemaless_object": null, "schematized_object": null, "combined_type": null, "union_type": null, "schemaless_array": null, "mixed_array_integer_and_schemaless_object": null, "array_of_union_integer_and_schemaless_array": null, "union_of_objects_with_properties_identical": null, "union_of_objects_with_properties_overlapping": null, "union_of_objects_with_properties_nonoverlapping": null, "union_of_objects_with_properties_contradicting": null, "empty_object": null, "object_with_null_properties": null, "combined_with_null": null, "union_with_null": null, "combined_nulls": null, "compact_union": null }

airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/problematic_types_disjoint_union_schema.json

@@ -163,6 +163,24 @@
     },
     "combined_nulls": {
       "type": "null"
+    },
+    "compact_union": {
+      "oneOf": [
+        {
+          "type": "object",
+          "properties": {
+            "id": {
+              "type": "integer"
+            },
+            "name": {
+              "type": "string"
+            }
+          }
+        },
+        {
+          "type": "integer"
+        }
+      ]
     }
   }
 }

airbyte-cdk/java/airbyte-cdk/db-destinations/src/testFixtures/resources/v0/problematic_types_messages_in.txt

@@ -1,5 +1,5 @@
-{"type": "RECORD", "record": {"stream": "problematic_types", "emitted_at": 1602637589100, "data": { "schemaless_object": { "uuid": "38F52396-736D-4B23-B5B4-F504D8894B97", "probability": 1.5 }, "schematized_object": { "id": 1, "name": "Joe" }, "combined_type": "string1", "union_type": 10, "schemaless_array": [ 10, "foo", null, { "bar": "qua" } ], "mixed_array_integer_and_schemaless_object": [ 15, null, { "hello": "world" } ], "array_of_union_integer_and_schemaless_array": [ 25, null, ["goodbye", "cruel world"] ], "union_of_objects_with_properties_identical": { "id": 10, "name": "Joe" }, "union_of_objects_with_properties_overlapping": { "id": 20, "name": "Jane", "flagged": true }, "union_of_objects_with_properties_contradicting": { "id": 1, "name": "Jenny" }, "union_of_objects_with_properties_nonoverlapping": { "id": 30, "name": "Phil", "flagged": false, "description":"Very Phil" }, "empty_object": {},"object_with_null_properties": {}, "combined_with_null": "foobar", "union_with_null": "barfoo", "combined_nulls": null } } }
-{"type": "RECORD", "record": {"stream": "problematic_types", "emitted_at": 1602637589200, "data": { "schemaless_object": { "address": { "street": "113 Hickey Rd", "zip": "37932" }, "flags": [ true, false, false ] }, "schematized_object": { "id": 2, "name": "Jane" }, "combined_type": 20, "union_type": "string2", "schemaless_array": [], "mixed_array_integer_and_schemaless_object": [ ], "array_of_union_integer_and_schemaless_array": [ ], "union_of_objects_with_properties_identical": { }, "union_of_objects_with_properties_overlapping": {}, "union_of_objects_with_properties_nonoverlapping": {}, "union_of_objects_with_properties_contradicting": { "id": "seal-one-hippity", "name": "James" }, "empty_object": {"extra": "stuff"}, "object_with_null_properties": { "more": { "extra": "stuff" } }, "combined_with_null": "foobar2", "union_with_null": "barfoo2", "combined_nulls": null } } }
-{"type": "RECORD", "record": {"stream": "problematic_types", "emitted_at": 1602637589300, "data": { "schemaless_object": null, "schematized_object": null, "combined_type": null, "union_type": null, "schemaless_array": null, "mixed_array_integer_and_schemaless_object": null, "array_of_union_integer_and_schemaless_array": null, "union_of_objects_with_properties_identical": null, "union_of_objects_with_properties_overlapping": null, "union_of_objects_with_properties_nonoverlapping": null, "empty_object": null, "object_with_null_properties": null, "combined_with_null": null, "union_with_null": null, "combined_nulls": null } } }
+{"type": "RECORD", "record": {"stream": "problematic_types", "emitted_at": 1602637589100, "data": { "schemaless_object": { "uuid": "38F52396-736D-4B23-B5B4-F504D8894B97", "probability": 1.5 }, "schematized_object": { "id": 1, "name": "Joe" }, "combined_type": "string1", "union_type": 10, "schemaless_array": [ 10, "foo", null, { "bar": "qua" } ], "mixed_array_integer_and_schemaless_object": [ 15, null, { "hello": "world" } ], "array_of_union_integer_and_schemaless_array": [ 25, null, ["goodbye", "cruel world"] ], "union_of_objects_with_properties_identical": { "id": 10, "name": "Joe" }, "union_of_objects_with_properties_overlapping": { "id": 20, "name": "Jane", "flagged": true }, "union_of_objects_with_properties_contradicting": { "id": 1, "name": "Jenny" }, "union_of_objects_with_properties_nonoverlapping": { "id": 30, "name": "Phil", "flagged": false, "description":"Very Phil" }, "empty_object": {},"object_with_null_properties": {}, "combined_with_null": "foobar", "union_with_null": "barfoo", "combined_nulls": null, "compact_union": { "id": 10, "name": "Tyler" } } } }
+{"type": "RECORD", "record": {"stream": "problematic_types", "emitted_at": 1602637589200, "data": { "schemaless_object": { "address": { "street": "113 Hickey Rd", "zip": "37932" }, "flags": [ true, false, false ] }, "schematized_object": { "id": 2, "name": "Jane" }, "combined_type": 20, "union_type": "string2", "schemaless_array": [], "mixed_array_integer_and_schemaless_object": [ ], "array_of_union_integer_and_schemaless_array": [ ], "union_of_objects_with_properties_identical": { }, "union_of_objects_with_properties_overlapping": {}, "union_of_objects_with_properties_nonoverlapping": {}, "union_of_objects_with_properties_contradicting": { "id": "seal-one-hippity", "name": "James" }, "empty_object": {"extra": "stuff"}, "object_with_null_properties": { "more": { "extra": "stuff" } }, "combined_with_null": "foobar2", "union_with_null": "barfoo2", "combined_nulls": null, "compact_union": 4444 } } }
+{"type": "RECORD", "record": {"stream": "problematic_types", "emitted_at": 1602637589300, "data": { "schemaless_object": null, "schematized_object": null, "combined_type": null, "union_type": null, "schemaless_array": null, "mixed_array_integer_and_schemaless_object": null, "array_of_union_integer_and_schemaless_array": null, "union_of_objects_with_properties_identical": null, "union_of_objects_with_properties_overlapping": null, "union_of_objects_with_properties_nonoverlapping": null, "empty_object": null, "object_with_null_properties": null, "combined_with_null": null, "union_with_null": null, "combined_nulls": null, "compact_union": null } } }
 {"type": "STATE", "state": { "data": {"start_date": "2022-02-14"}}}
 {"type": "TRACE", "trace": { "type": "STREAM_STATUS", "stream_status": {"stream_descriptor": {"name": "problematic_types"}, "status": "COMPLETE"}, "emitted_at": 1721428636000}}

airbyte-cdk/java/airbyte-cdk/s3-destinations/src/main/kotlin/io/airbyte/cdk/integrations/destination/s3/jsonschema/AirbyteJsonSchemaType.kt

@@ -84,15 +84,22 @@ enum class AirbyteJsonSchemaType {
 
             val type = schema["type"]
             if (type != null) {
-                if (type.isArray && type.size() > 1) {
+                val typeArray =
+                    if (type.isArray) {
+                        type.elements().asSequence().filter { it.asText() != "null" }.toList()
+                    } else {
+                        listOf(type)
+                    }
+
+                if (typeArray.size > 1) {
                     return COMBINED
                 }
 
                 val typeStr =
-                    if (type.isArray) {
-                        type[0].asText()
+                    if (typeArray.isEmpty()) {
+                        "null"
                     } else {
-                        type.asText()
+                        typeArray[0].asText()
                     }
 
                 val format = schema["format"]?.asText()
@@ -233,7 +240,9 @@ enum class AirbyteJsonSchemaType {
                     }
                     .toList()
             if (matching.isEmpty()) {
-                throw IllegalArgumentException("Union type does not match any options")
+                throw IllegalArgumentException(
+                    "Union type ${value::class}(value redacted) does not match any options: $optionsAsList"
+                )
             }
             return matching.first()
         }