diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 93f34aa..4f0c792 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -25,12 +25,7 @@ jobs: if: matrix.cove == 'ocds' run: | git clone https://github.com/open-contracting/cove-ocds.git - cd cove-ocds - git checkout main - cd .. git clone https://github.com/open-contracting/lib-cove-ocds.git - cd lib-cove-ocds - git checkout 0.11.3 - name: Install run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index bfaf5c4..841a244 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Fixed + +- Descend into nullable objects and arrays. (For example, OCDS `parties/details` is nullable, and additional codes for `parties/details/scale` were unreported.) + ### Removed - Dropped support for Python 3.6 & 3.7, as these are now end of life. diff --git a/libcove/lib/common.py b/libcove/lib/common.py index 4465342..2a12054 100644 --- a/libcove/lib/common.py +++ b/libcove/lib/common.py @@ -466,6 +466,13 @@ def schema_dict_fields_generator(schema_dict): yield field +def _get_types(value: dict): + types = value.get("type", []) + if not isinstance(types, list): + return [types] + return types + + def get_schema_codelist_paths( schema_obj, obj=None, current_path=(), codelist_paths=None, use_extensions=False ): @@ -492,10 +499,11 @@ def get_schema_codelist_paths( if "codelist" in value and path not in codelist_paths: codelist_paths[path] = (value["codelist"], value.get("openCodelist", False)) - if value.get("type") == "object": + types = _get_types(value) + if "object" in types: get_schema_codelist_paths(None, value, path, codelist_paths) - elif value.get("type") == "array" and isinstance(value.get("items"), dict): - if value.get("items").get("type") == "string": + elif "array" in types and isinstance(value.get("items"), dict): + if "string" in _get_types(value["items"]): if "codelist" in value["items"] and path not in codelist_paths: codelist_paths[path] = ( value["items"]["codelist"], @@ -1279,10 +1287,11 @@ def _get_schema_deprecated_paths( ) ) - if value.get("type") == "object": + types = _get_types(value) + if "object" in types: _get_schema_deprecated_paths(None, value, path, deprecated_paths) elif ( - value.get("type") == "array" + "array" in types and isinstance(value.get("items"), dict) and value.get("items").get("properties") ): @@ -1324,10 +1333,11 @@ def _get_schema_non_required_ids( if prop == "id" and no_required_id and array_parent and not list_merge: id_paths.append(path) - if value.get("type") == "object": + types = _get_types(value) + if "object" in types: _get_schema_non_required_ids(None, value, path, id_paths) elif ( - value.get("type") == "array" + "array" in types and isinstance(value.get("items"), dict) and value.get("items").get("properties") ): @@ -1371,16 +1381,18 @@ def add_is_codelist(obj): ) continue + types = _get_types(value) + if "codelist" in value: - if "array" in value.get("type", ""): + if "array" in types: value["items"]["isCodelist"] = True else: value["isCodelist"] = True - if value.get("type") == "object": + if "object" in types: add_is_codelist(value) elif ( - value.get("type") == "array" + "array" in types and isinstance(value.get("items"), dict) and value.get("items").get("properties") ): diff --git a/tests/lib/fixtures/common/schema_nullable_object_and_array.json b/tests/lib/fixtures/common/schema_nullable_object_and_array.json new file mode 100644 index 0000000..2d909b5 --- /dev/null +++ b/tests/lib/fixtures/common/schema_nullable_object_and_array.json @@ -0,0 +1,39 @@ +{ + "properties": { + "array": { + "type": ["array", "null"], + "items": { + "$ref": "#/definitions/Object" + } + }, + "object": { + "$ref": "#/definitions/Object" + } + }, + "definitions": { + "Object": { + "type": ["object", "null"], + "properties": { + "id": { + "type": "string" + }, + "scale": { + "type": ["array", "null"], + "items": { + "type": "string", + "enum": [ + "small", + "large" + ] + }, + "codelist": "partyScale.csv", + "openCodelist": false, + "deprecated": { + "deprecatedVersion": "1.1", + "description": "" + } + } + } + } + } +} diff --git a/tests/lib/test_common.py b/tests/lib/test_common.py index fd4ae36..eec5891 100644 --- a/tests/lib/test_common.py +++ b/tests/lib/test_common.py @@ -13,6 +13,7 @@ _get_schema_deprecated_paths, add_field_coverage, add_field_coverage_percentages, + common_checks_context, fields_present_generator, get_additional_codelist_values, get_additional_fields_info, @@ -21,6 +22,7 @@ get_json_data_deprecated_fields, get_json_data_generic_paths, get_orgids_prefixes, + get_schema_codelist_paths, get_schema_validation_errors, org_id_file_fresh, schema_dict_fields_generator, @@ -28,6 +30,18 @@ ) +def get_schema_obj(fixture): + schema_obj = SchemaJsonMixin() + schema_obj.schema_host = os.path.join( + os.path.dirname(os.path.realpath(__file__)), "fixtures", "common/" + ) + schema_obj.release_pkg_schema_name = f"{fixture}.json" + schema_obj.pkg_schema_url = os.path.join( + schema_obj.schema_host, schema_obj.release_pkg_schema_name + ) + return schema_obj + + def test_unique_ids_False(): ui = False schema = {"uniqueItems": ui} @@ -187,15 +201,8 @@ def test_get_json_data_deprecated_fields(): ) as fp: json_data_w_deprecations = json.load(fp) - schema_obj = SchemaJsonMixin() - schema_obj.schema_host = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "fixtures", "common/" - ) - schema_obj.release_pkg_schema_name = ( - "release_package_schema_ref_release_schema_deprecated_fields.json" - ) - schema_obj.pkg_schema_url = os.path.join( - schema_obj.schema_host, schema_obj.release_pkg_schema_name + schema_obj = get_schema_obj( + "release_package_schema_ref_release_schema_deprecated_fields" ) json_data_paths = get_json_data_generic_paths( json_data_w_deprecations, generic_paths={} @@ -293,15 +300,8 @@ def test_fields_present_10(): def test_get_schema_deprecated_paths(): - schema_obj = SchemaJsonMixin() - schema_obj.schema_host = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "fixtures", "common/" - ) - schema_obj.release_pkg_schema_name = ( - "release_package_schema_ref_release_schema_deprecated_fields.json" - ) - schema_obj.pkg_schema_url = os.path.join( - schema_obj.schema_host, schema_obj.release_pkg_schema_name + schema_obj = get_schema_obj( + "release_package_schema_ref_release_schema_deprecated_fields" ) deprecated_paths = _get_schema_deprecated_paths(schema_obj) expected_results = [ @@ -1347,3 +1347,26 @@ def test_get_additional_codelist_values(): "extension_codelist": False, }, } + + +def test_nullable_objects_and_arrays(tmpdir): + json_data = { + "array": [{"scale": ["a"]}], + "object": {"scale": ["b"]}, + } + schema_obj = get_schema_obj("schema_nullable_object_and_array") + context = {"file_type": "json"} + common_checks_context(tmpdir, json_data, schema_obj, "", context) + + # _get_schema_non_required_ids + assert context["structure_warnings"] == {"missing_ids": ["array/0/id"]} + + # _get_schema_deprecated_paths + assert context["deprecated_fields"] == { + "scale": {"explanation": ("1.1", ""), "paths": ("object",)} + } + + assert get_schema_codelist_paths(schema_obj, json_data) == { + ("array", "scale"): ("partyScale.csv", False), + ("object", "scale"): ("partyScale.csv", False), + }