Fix: Handle array types with json schema parsing errors in subtypes (#…

…113)
airbytehq · Mar 8, 2024 · bfad07b · bfad07b
1 parent 5cbde18
commit bfad07b
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 1 deletion.
diff --git a/airbyte/types.py b/airbyte/types.py
@@ -71,7 +71,12 @@ def _get_airbyte_type(  # noqa: PLR0911  # Too many return statements
     if json_schema_type == "array":
         items_def = json_schema_property_def.get("items", None)
         if isinstance(items_def, dict):
-            subtype, _ = _get_airbyte_type(items_def)
+            try:
+                subtype, _ = _get_airbyte_type(items_def)
+            except SQLTypeConversionError:
+                # We have enough information, so we can ignore parsing errors on subtype.
+                subtype = None
+
             return "array", subtype
 
         return "array", None

diff --git a/tests/unit_tests/test_type_translation.py b/tests/unit_tests/test_type_translation.py
@@ -27,7 +27,12 @@
         ({"type": "number", "airbyte_type": "integer"}, types.BIGINT),
         ({"type": "number"}, types.DECIMAL),
         ({"type": "array", "items": {"type": "object"}}, types.JSON),
+        ({"type": ["null", "array"], "items": {"type": "object"}}, types.JSON),
         ({"type": "object", "properties": {}}, types.JSON),
+        ({"type": ["null", "object"], "properties": {}}, types.JSON),
+        # Malformed JSON schema seen in the wild:
+        ({'type': 'array', 'items': {}}, types.JSON),
+        ({'type': ['null', 'array'], 'items': {'items': {}}}, types.JSON),
     ],
 )
 def test_to_sql_type(json_schema_property_def, expected_sql_type):
@@ -53,8 +58,15 @@ def test_to_sql_type(json_schema_property_def, expected_sql_type):
         ({"type": "integer"}, "integer"),
         ({"type": "number", "airbyte_type": "integer"}, "integer"),
         ({"type": "number"}, "number"),
+        # Array type:
         ({"type": "array"}, "array"),
+        ({"type": "array", "items": {"type": "object"}}, "array"),
+        ({"type": ["null", "array"], "items": {"type": "object"}}, "array"),
+        # Object type:
         ({"type": "object"}, "object"),
+        # Malformed JSON schema seen in the wild:
+        ({'type': 'array', 'items': {'items': {}}}, "array"),
+        ({'type': ['null', 'array'], 'items': {'items': {}}}, "array"),
     ],
 )
 def test_to_airbyte_type(json_schema_property_def, expected_airbyte_type):
@@ -71,6 +83,9 @@ def test_to_airbyte_type(json_schema_property_def, expected_airbyte_type):
         ({"type": "object"}, "object", None),
         ({"type": "array", "items": {"type": ["null", "string"]}}, "array", "string"),
         ({"type": "array", "items": {"type": ["boolean"]}}, "array", "boolean"),
+        # Malformed JSON schema seen in the wild:
+        ({'type': 'array', 'items': {'items': {}}}, "array", None),
+        ({'type': ['null', 'array'], 'items': {'items': {}}}, "array", None),
     ],
 )
 def test_to_airbyte_subtype(