Merge pull request #139 from OpenDataServices/2023-11-23

Descend into nullable objects and arrays

Author: James (ODSC)

CHANGELOG.md

@@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Fixed
 
 - Calculate additional codelist values for schema using `anyOf` or `oneOf`, like OCDS record packages https://github.com/open-contracting/lib-cove-ocds/issues/106
+- Descend into nullable objects and arrays. (For example, OCDS `parties/details` is nullable, and additional codes for `parties/details/scale` were unreported.) https://github.com/OpenDataServices/lib-cove/pull/131
 
 ## [0.31.0] - 2023-07-06
 

libcove/lib/common.py

@@ -430,6 +430,13 @@ def schema_dict_fields_generator(schema_dict):
                     yield field
 
 
+def _get_types(value: dict):
+    types = value.get("type", [])
+    if not isinstance(types, list):
+        return [types]
+    return types
+
+
 def get_schema_codelist_paths(
     schema_obj, obj=None, current_path=(), codelist_paths=None, use_extensions=False
 ):
@@ -465,10 +472,11 @@ def get_schema_codelist_paths(
             descendants = [value]
 
         for value in descendants:
-            if value.get("type") == "object":
+            types = _get_types(value)
+            if "object" in types:
                 get_schema_codelist_paths(None, value, path, codelist_paths)
-            elif value.get("type") == "array" and isinstance(value.get("items"), dict):
-                if value.get("items").get("type") == "string":
+            elif "array" in types and isinstance(value.get("items"), dict):
+                if "string" in _get_types(value["items"]):
                     if "codelist" in value["items"] and path not in codelist_paths:
                         codelist_paths[path] = (
                             value["items"]["codelist"],
@@ -1258,10 +1266,11 @@ def _get_schema_deprecated_paths(
                     )
                 )
 
-        if value.get("type") == "object":
+        types = _get_types(value)
+        if "object" in types:
             _get_schema_deprecated_paths(None, value, path, deprecated_paths)
         elif (
-            value.get("type") == "array"
+            "array" in types
             and isinstance(value.get("items"), dict)
             and value.get("items").get("properties")
         ):
@@ -1303,10 +1312,11 @@ def _get_schema_non_required_ids(
         if prop == "id" and no_required_id and array_parent and not list_merge:
             id_paths.append(path)
 
-        if value.get("type") == "object":
+        types = _get_types(value)
+        if "object" in types:
             _get_schema_non_required_ids(None, value, path, id_paths)
         elif (
-            value.get("type") == "array"
+            "array" in types
             and isinstance(value.get("items"), dict)
             and value.get("items").get("properties")
         ):
@@ -1350,16 +1360,18 @@ def add_is_codelist(obj):
             )
             continue
 
+        types = _get_types(value)
+
         if "codelist" in value:
-            if "array" in value.get("type", ""):
+            if "array" in types:
                 value["items"]["isCodelist"] = True
             else:
                 value["isCodelist"] = True
 
-        if value.get("type") == "object":
+        if "object" in types:
             add_is_codelist(value)
         elif (
-            value.get("type") == "array"
+            "array" in types
             and isinstance(value.get("items"), dict)
             and value.get("items").get("properties")
         ):

tests/lib/fixtures/common/schema_nullable_object_and_array.json

@@ -0,0 +1,39 @@
+{
+    "properties": {
+        "array": {
+            "type": ["array", "null"],
+            "items": {
+                "$ref": "#/definitions/Object"
+            }
+        },
+        "object": {
+            "$ref": "#/definitions/Object"
+        }
+    },
+    "definitions": {
+        "Object": {
+            "type": ["object", "null"],
+            "properties": {
+                "id": {
+                    "type": "string"
+                },
+                "scale": {
+                    "type": ["array", "null"],
+                    "items": {
+                        "type": "string",
+                        "enum": [
+                            "small",
+                            "large"
+                        ]
+                    },
+                    "codelist": "partyScale.csv",
+                    "openCodelist": false,
+                    "deprecated": {
+                        "deprecatedVersion": "1.1",
+                        "description": ""
+                    }
+                }
+            }
+        }
+    }
+}

tests/lib/test_common.py

@@ -13,6 +13,7 @@
     _get_schema_deprecated_paths,
     add_field_coverage,
     add_field_coverage_percentages,
+    common_checks_context,
     fields_present_generator,
     get_additional_codelist_values,
     get_additional_fields_info,
@@ -21,13 +22,26 @@
     get_json_data_deprecated_fields,
     get_json_data_generic_paths,
     get_orgids_prefixes,
+    get_schema_codelist_paths,
     get_schema_validation_errors,
     org_id_file_fresh,
     schema_dict_fields_generator,
     unique_ids,
 )
 
 
+def get_schema_obj(fixture):
+    schema_obj = SchemaJsonMixin()
+    schema_obj.schema_host = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)), "fixtures", "common/"
+    )
+    schema_obj.release_pkg_schema_name = f"{fixture}.json"
+    schema_obj.pkg_schema_url = os.path.join(
+        schema_obj.schema_host, schema_obj.release_pkg_schema_name
+    )
+    return schema_obj
+
+
 def test_unique_ids_False():
     ui = False
     schema = {"uniqueItems": ui}
@@ -187,15 +201,8 @@ def test_get_json_data_deprecated_fields():
     ) as fp:
         json_data_w_deprecations = json.load(fp)
 
-    schema_obj = SchemaJsonMixin()
-    schema_obj.schema_host = os.path.join(
-        os.path.dirname(os.path.realpath(__file__)), "fixtures", "common/"
-    )
-    schema_obj.release_pkg_schema_name = (
-        "release_package_schema_ref_release_schema_deprecated_fields.json"
-    )
-    schema_obj.pkg_schema_url = os.path.join(
-        schema_obj.schema_host, schema_obj.release_pkg_schema_name
+    schema_obj = get_schema_obj(
+        "release_package_schema_ref_release_schema_deprecated_fields"
     )
     json_data_paths = get_json_data_generic_paths(
         json_data_w_deprecations, generic_paths={}
@@ -293,15 +300,8 @@ def test_fields_present_10():
 
 
 def test_get_schema_deprecated_paths():
-    schema_obj = SchemaJsonMixin()
-    schema_obj.schema_host = os.path.join(
-        os.path.dirname(os.path.realpath(__file__)), "fixtures", "common/"
-    )
-    schema_obj.release_pkg_schema_name = (
-        "release_package_schema_ref_release_schema_deprecated_fields.json"
-    )
-    schema_obj.pkg_schema_url = os.path.join(
-        schema_obj.schema_host, schema_obj.release_pkg_schema_name
+    schema_obj = get_schema_obj(
+        "release_package_schema_ref_release_schema_deprecated_fields"
     )
     deprecated_paths = _get_schema_deprecated_paths(schema_obj)
     expected_results = [
@@ -1379,3 +1379,26 @@ def test_get_additional_codelist_values_oneOf():
             "extension_codelist": False,
         }
     }
+
+
+def test_nullable_objects_and_arrays(tmpdir):
+    json_data = {
+        "array": [{"scale": ["a"]}],
+        "object": {"scale": ["b"]},
+    }
+    schema_obj = get_schema_obj("schema_nullable_object_and_array")
+    context = {"file_type": "json"}
+    common_checks_context(tmpdir, json_data, schema_obj, "", context)
+
+    # _get_schema_non_required_ids
+    assert context["structure_warnings"] == {"missing_ids": ["array/0/id"]}
+
+    # _get_schema_deprecated_paths
+    assert context["deprecated_fields"] == {
+        "scale": {"explanation": ("1.1", ""), "paths": ("object",)}
+    }
+
+    assert get_schema_codelist_paths(schema_obj, json_data) == {
+        ("array", "scale"): ("partyScale.csv", False),
+        ("object", "scale"): ("partyScale.csv", False),
+    }