From 346f530587befecbc88e2c142958c5b8ea904f2d Mon Sep 17 00:00:00 2001 From: Tobias Klockau Date: Fri, 20 Oct 2023 11:48:42 +0200 Subject: [PATCH 01/13] refactor!: rename validate dir to validation and move schema --- raillabel_providerkit/format/understand_ai/metadata.py | 5 ++++- raillabel_providerkit/{validate => validation}/__init__.py | 0 .../validate_schema}/raillabel_schema.json | 0 .../validate_schema}/raillabel_schema.json.license | 0 tests/conftest.py | 2 +- .../{validate => validation}/test_raillabel_v2_schema.py | 0 6 files changed, 5 insertions(+), 2 deletions(-) rename raillabel_providerkit/{validate => validation}/__init__.py (100%) rename raillabel_providerkit/{validate => validation/validate_schema}/raillabel_schema.json (100%) rename raillabel_providerkit/{validate => validation/validate_schema}/raillabel_schema.json.license (100%) rename tests/test_raillabel_providerkit/{validate => validation}/test_raillabel_v2_schema.py (100%) diff --git a/raillabel_providerkit/format/understand_ai/metadata.py b/raillabel_providerkit/format/understand_ai/metadata.py index 7d5fbf0..b049237 100644 --- a/raillabel_providerkit/format/understand_ai/metadata.py +++ b/raillabel_providerkit/format/understand_ai/metadata.py @@ -82,7 +82,10 @@ def to_raillabel(self) -> dict: def _get_subschema_version(self) -> str: RAILLABEL_SCHEMA_PATH = ( - Path(__file__).parent.parent.parent / "validate" / "raillabel_schema.json" + Path(__file__).parent.parent.parent + / "validation" + / "validate_schema" + / "raillabel_schema.json" ) with RAILLABEL_SCHEMA_PATH.open() as schema_file: diff --git a/raillabel_providerkit/validate/__init__.py b/raillabel_providerkit/validation/__init__.py similarity index 100% rename from raillabel_providerkit/validate/__init__.py rename to raillabel_providerkit/validation/__init__.py diff --git a/raillabel_providerkit/validate/raillabel_schema.json b/raillabel_providerkit/validation/validate_schema/raillabel_schema.json similarity index 100% rename from raillabel_providerkit/validate/raillabel_schema.json rename to raillabel_providerkit/validation/validate_schema/raillabel_schema.json diff --git a/raillabel_providerkit/validate/raillabel_schema.json.license b/raillabel_providerkit/validation/validate_schema/raillabel_schema.json.license similarity index 100% rename from raillabel_providerkit/validate/raillabel_schema.json.license rename to raillabel_providerkit/validation/validate_schema/raillabel_schema.json.license diff --git a/tests/conftest.py b/tests/conftest.py index cf0802b..935c63d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ json_data_directories = [ Path(__file__).parent / "__test_assets__", - Path(__file__).parent.parent / "raillabel_providerkit" / "validate" + Path(__file__).parent.parent / "raillabel_providerkit" / "validation" / "validate_schema" ] @pytest.fixture diff --git a/tests/test_raillabel_providerkit/validate/test_raillabel_v2_schema.py b/tests/test_raillabel_providerkit/validation/test_raillabel_v2_schema.py similarity index 100% rename from tests/test_raillabel_providerkit/validate/test_raillabel_v2_schema.py rename to tests/test_raillabel_providerkit/validation/test_raillabel_v2_schema.py From 2133dc0b868bfcf2a8a53fac4cc4c64be608c723 Mon Sep 17 00:00:00 2001 From: Tobias Klockau Date: Fri, 20 Oct 2023 11:50:48 +0200 Subject: [PATCH 02/13] test: move test_raillabel_v2_schema --- .../validation/{ => validate_schema}/test_raillabel_v2_schema.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/test_raillabel_providerkit/validation/{ => validate_schema}/test_raillabel_v2_schema.py (100%) diff --git a/tests/test_raillabel_providerkit/validation/test_raillabel_v2_schema.py b/tests/test_raillabel_providerkit/validation/validate_schema/test_raillabel_v2_schema.py similarity index 100% rename from tests/test_raillabel_providerkit/validation/test_raillabel_v2_schema.py rename to tests/test_raillabel_providerkit/validation/validate_schema/test_raillabel_v2_schema.py From 83b273bd74dd1dbe3dc45460d208ab57c22bf5cf Mon Sep 17 00:00:00 2001 From: Tobias Klockau Date: Fri, 20 Oct 2023 12:06:46 +0200 Subject: [PATCH 03/13] refactor!: move raillabel schema --- .../validate_schema => format}/raillabel_schema.json | 0 .../validate_schema => format}/raillabel_schema.json.license | 0 raillabel_providerkit/format/understand_ai/metadata.py | 5 +---- tests/conftest.py | 2 +- .../validate_schema => format}/test_raillabel_v2_schema.py | 0 5 files changed, 2 insertions(+), 5 deletions(-) rename raillabel_providerkit/{validation/validate_schema => format}/raillabel_schema.json (100%) rename raillabel_providerkit/{validation/validate_schema => format}/raillabel_schema.json.license (100%) rename tests/test_raillabel_providerkit/{validation/validate_schema => format}/test_raillabel_v2_schema.py (100%) diff --git a/raillabel_providerkit/validation/validate_schema/raillabel_schema.json b/raillabel_providerkit/format/raillabel_schema.json similarity index 100% rename from raillabel_providerkit/validation/validate_schema/raillabel_schema.json rename to raillabel_providerkit/format/raillabel_schema.json diff --git a/raillabel_providerkit/validation/validate_schema/raillabel_schema.json.license b/raillabel_providerkit/format/raillabel_schema.json.license similarity index 100% rename from raillabel_providerkit/validation/validate_schema/raillabel_schema.json.license rename to raillabel_providerkit/format/raillabel_schema.json.license diff --git a/raillabel_providerkit/format/understand_ai/metadata.py b/raillabel_providerkit/format/understand_ai/metadata.py index b049237..59b954e 100644 --- a/raillabel_providerkit/format/understand_ai/metadata.py +++ b/raillabel_providerkit/format/understand_ai/metadata.py @@ -82,10 +82,7 @@ def to_raillabel(self) -> dict: def _get_subschema_version(self) -> str: RAILLABEL_SCHEMA_PATH = ( - Path(__file__).parent.parent.parent - / "validation" - / "validate_schema" - / "raillabel_schema.json" + Path(__file__).parent.parent.parent / "format" / "raillabel_schema.json" ) with RAILLABEL_SCHEMA_PATH.open() as schema_file: diff --git a/tests/conftest.py b/tests/conftest.py index 935c63d..f2f58e4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,7 +13,7 @@ json_data_directories = [ Path(__file__).parent / "__test_assets__", - Path(__file__).parent.parent / "raillabel_providerkit" / "validation" / "validate_schema" + Path(__file__).parent.parent / "raillabel_providerkit" / "format" ] @pytest.fixture diff --git a/tests/test_raillabel_providerkit/validation/validate_schema/test_raillabel_v2_schema.py b/tests/test_raillabel_providerkit/format/test_raillabel_v2_schema.py similarity index 100% rename from tests/test_raillabel_providerkit/validation/validate_schema/test_raillabel_v2_schema.py rename to tests/test_raillabel_providerkit/format/test_raillabel_v2_schema.py From dcc0cc2b2d20c9f8748ced1a9a0ac86abe046d35 Mon Sep 17 00:00:00 2001 From: Tobias Klockau Date: Fri, 20 Oct 2023 12:14:01 +0200 Subject: [PATCH 04/13] feat: onthology schema --- .../onthology_schema_v1.yaml | 87 ++++++++ .../test_onthology_schema_v1.py | 196 ++++++++++++++++++ 2 files changed, 283 insertions(+) create mode 100644 raillabel_providerkit/validation/validate_onthology/onthology_schema_v1.yaml create mode 100644 tests/test_raillabel_providerkit/validation/validate_onthology/test_onthology_schema_v1.py diff --git a/raillabel_providerkit/validation/validate_onthology/onthology_schema_v1.yaml b/raillabel_providerkit/validation/validate_onthology/onthology_schema_v1.yaml new file mode 100644 index 0000000..8b3db11 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/onthology_schema_v1.yaml @@ -0,0 +1,87 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +"$schema": http://json-schema.org/draft-07/schema# +version: 1.0.0 + +definitions: + + attribute: + oneOf: + [ + "$ref": "#/definitions/boolean_attribute", + "$ref": "#/definitions/integer_attribute", + "$ref": "#/definitions/multi_select_attribute", + "$ref": "#/definitions/single_select_attribute", + "$ref": "#/definitions/string_attribute", + "$ref": "#/definitions/vector_attribute", + ] + + boolean_attribute: + const: boolean + + class: + additionalProperties: false + properties: + attributes: + additionalProperties: false + patternProperties: + "^": + "$ref": "#/definitions/attribute" + type: object + sensor_types: + additionalProperties: false + patternProperties: + "^(camera|lidar|radar)$": + "$ref": "#/definitions/sensor_type" + type: object + type: object + + integer_attribute: + const: integer + + single_select_attribute: + additionalProperties: false + properties: + type: + const: single-select + options: + type: array + items: + type: string + type: object + + multi_select_attribute: + additionalProperties: false + properties: + type: + const: multi-select + options: + type: array + items: + type: string + type: object + + sensor_type: + additionalProperties: false + properties: + attributes: + additionalProperties: false + patternProperties: + "^": + "$ref": "#/definitions/attribute" + type: object + type: object + + string_attribute: + const: string + + vector_attribute: + const: vector + +additionalProperties: false +patternProperties: + "^": + "$ref": "#/definitions/class" + +type: object diff --git a/tests/test_raillabel_providerkit/validation/validate_onthology/test_onthology_schema_v1.py b/tests/test_raillabel_providerkit/validation/validate_onthology/test_onthology_schema_v1.py new file mode 100644 index 0000000..85685b4 --- /dev/null +++ b/tests/test_raillabel_providerkit/validation/validate_onthology/test_onthology_schema_v1.py @@ -0,0 +1,196 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import os +from pathlib import Path + +import jsonschema +import pytest +import yaml + +# == Fixtures ========================= + +@pytest.fixture +def schema_path() -> Path: + return ( + Path(__file__).parent.parent.parent.parent.parent + / "raillabel_providerkit" + / "validation" + / "validate_onthology" + / "onthology_schema_v1.yaml" + ) + +@pytest.fixture +def schema(schema_path) -> dict: + with schema_path.open() as f: + schema_data = yaml.safe_load(f) + return schema_data + +@pytest.fixture +def validator(schema) -> jsonschema.Draft7Validator: + return jsonschema.Draft7Validator(schema) + +def schema_errors(data: dict, validator: jsonschema.Draft7Validator) -> list[str]: + errors = [] + + for error in validator.iter_errors(data): + errors.append("$" + error.json_path[1:] + ": " + str(error.message)) + + return errors + +# == Tests ========================= + +def test_classes(validator): + data = { + "person": {}, + "train": {}, + } + + assert schema_errors(data, validator) == [] + +def test_class_unsupported_field(validator): + data = { + "person": { + "UNSUPPORTED_FIELD": {} + } + } + + assert schema_errors(data, validator) == [ + "$.person: Additional properties are not allowed ('UNSUPPORTED_FIELD' was unexpected)", + ] + + +def test_attributes_field(validator): + data = { + "person": { + "attributes": {} + } + } + + assert schema_errors(data, validator) == [] + +def test_attribute_string(validator): + data = { + "person": { + "attributes": { + "name": "string" + } + } + } + + assert schema_errors(data, validator) == [] + +def test_attribute_integer(validator): + data = { + "person": { + "attributes": { + "number_of_fingers": "integer" + } + } + } + + assert schema_errors(data, validator) == [] + +def test_attribute_boolean(validator): + data = { + "person": { + "attributes": { + "number_of_fingers": "boolean" + } + } + } + + assert schema_errors(data, validator) == [] + +def test_attribute_single_select(validator): + data = { + "person": { + "attributes": { + "carrying": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the new Slicer-Dicer 3000 (WOW!)" + ] + } + } + } + } + + assert schema_errors(data, validator) == [] + +def test_attribute_multi_select(validator): + data = { + "person": { + "attributes": { + "carrying": { + "type": "multi-select", + "options": [ + "groceries", + "a baby", + "the new Slicer-Dicer 3000 (WOW!)" + ] + } + } + } + } + + assert schema_errors(data, validator) == [] + +def test_attribute_vector(validator): + data = { + "person": { + "attributes": { + "carrying": "vector" + } + } + } + + assert schema_errors(data, validator) == [] + + +def test_sensor_types(validator): + data = { + "person": { + "sensor_types": { + "camera": {}, + "lidar": {}, + "radar": {}, + } + } + } + + assert schema_errors(data, validator) == [] + +def test_sensor_types_unsupported_type(validator): + data = { + "person": { + "sensor_types": { + "UNSUPPORTED_SENSOR_TYPE": {}, + "lidar": {}, + } + } + } + + assert len(schema_errors(data, validator)) == 1 + +def test_sensor_type_attributes(validator): + data = { + "person": { + "sensor_types": { + "lidar": { + "attributes": { + "name": "string" + } + }, + } + } + } + + assert schema_errors(data, validator) == [] + + +if __name__ == "__main__": + os.system("clear") + pytest.main([__file__, "--disable-pytest-warnings", "--cache-clear", "-v"]) From 74527c17af35bd563b8e8513755093324745487a Mon Sep 17 00:00:00 2001 From: Tobias Klockau Date: Fri, 20 Oct 2023 12:48:03 +0200 Subject: [PATCH 05/13] feat: onthology validation --- raillabel_providerkit/exceptions.py | 6 + raillabel_providerkit/validation/__init__.py | 2 + .../validation/validate_onthology/__init__.py | 3 + .../_onthology_classes/__init__.py | 2 + .../_attributes/__init__.py | 2 + .../_attributes/_attribute_abc.py | 52 ++ .../_attributes/_boolean_attribute.py | 29 + .../_attributes/_integer_attribute.py | 29 + .../_attributes/_multi_select_attribute.py | 51 ++ .../_attributes/_single_select_attribute.py | 50 ++ .../_attributes/_string_attribute.py | 29 + .../_attributes/_vector_attribute.py | 29 + .../_onthology_classes/_object_classes.py | 113 +++ .../_onthology_classes/_onthology.py | 46 ++ .../_onthology_classes/_sensor_type.py | 34 + .../validate_onthology/validate_onthology.py | 66 ++ .../test_onthology_schema_v1.py | 3 +- .../test_validate_onthology.py | 705 ++++++++++++++++++ 18 files changed, 1250 insertions(+), 1 deletion(-) create mode 100644 raillabel_providerkit/validation/validate_onthology/__init__.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/__init__.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/__init__.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_boolean_attribute.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_integer_attribute.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_multi_select_attribute.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_single_select_attribute.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_string_attribute.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_vector_attribute.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py create mode 100644 raillabel_providerkit/validation/validate_onthology/_onthology_classes/_sensor_type.py create mode 100644 raillabel_providerkit/validation/validate_onthology/validate_onthology.py create mode 100644 tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py diff --git a/raillabel_providerkit/exceptions.py b/raillabel_providerkit/exceptions.py index 37319d8..47ff8f7 100644 --- a/raillabel_providerkit/exceptions.py +++ b/raillabel_providerkit/exceptions.py @@ -12,3 +12,9 @@ class SchemaError(Exception): """Raised when the data does not validate against a given schema.""" __module__ = "raillabel_providerkit" + + +class OnthologySchemaError(Exception): + """Raised when the .yaml-file provided is not valid against the schema.""" + + __module__ = "raillabel_providerkit" diff --git a/raillabel_providerkit/validation/__init__.py b/raillabel_providerkit/validation/__init__.py index 4c38e62..098b319 100644 --- a/raillabel_providerkit/validation/__init__.py +++ b/raillabel_providerkit/validation/__init__.py @@ -1,3 +1,5 @@ # Copyright DB Netz AG and contributors # SPDX-License-Identifier: Apache-2.0 """Package for validating raillabel data regarding the format requirements.""" + +from .validate_onthology.validate_onthology import validate_onthology diff --git a/raillabel_providerkit/validation/validate_onthology/__init__.py b/raillabel_providerkit/validation/validate_onthology/__init__.py new file mode 100644 index 0000000..56bbc60 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/__init__.py @@ -0,0 +1,3 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 +"""Package for validating a scene via an onthology.""" diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/__init__.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/__init__.py new file mode 100644 index 0000000..aabc64e --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/__init__.py @@ -0,0 +1,2 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/__init__.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/__init__.py new file mode 100644 index 0000000..aabc64e --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/__init__.py @@ -0,0 +1,2 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py new file mode 100644 index 0000000..0599d7b --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py @@ -0,0 +1,52 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import abc +import typing as t +from dataclasses import dataclass +from importlib import import_module +from inspect import isclass +from pathlib import Path +from pkgutil import iter_modules + + +@dataclass +class _Attribute(abc.ABC): + @abc.abstractclassmethod + def supports(cls, data_dict: dict) -> bool: + raise NotImplementedError + + @abc.abstractclassmethod + def fromdict(cls, data_dict: dict) -> t.Type["_Attribute"]: + raise NotImplementedError + + @abc.abstractmethod + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + raise NotImplementedError + + +def attribute_classes() -> list[t.Type[_Attribute]]: + """Return dictionary with Attribute child classes.""" + return ATTRIBUTE_CLASSES + + +def _collect_attribute_classes(): + """Collect attribute child classes and store them.""" + + global ATTRIBUTE_CLASSES + + package_dir = str(Path(__file__).resolve().parent) + for (_, module_name, _) in iter_modules([package_dir]): + + module = import_module( + f"raillabel_providerkit.validation.validate_onthology._onthology_classes._attributes.{module_name}" + ) + for class_name in dir(module): + class_ = getattr(module, class_name) + + if isclass(class_) and issubclass(class_, _Attribute) and class_ != _Attribute: + ATTRIBUTE_CLASSES.append(class_) + + +ATTRIBUTE_CLASSES = [] +_collect_attribute_classes() diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_boolean_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_boolean_attribute.py new file mode 100644 index 0000000..0a9ead5 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_boolean_attribute.py @@ -0,0 +1,29 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _BooleanAttribute(_Attribute): + @classmethod + def supports(cls, data_dict: dict): + return data_dict == "boolean" + + @classmethod + def fromdict(cls, data_dict: dict): + return _BooleanAttribute() + + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + errors = [] + + if type(attribute_value) != bool: + errors.append( + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_value.__class__.__name__}' (should be 'bool')." + ) + + return errors diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_integer_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_integer_attribute.py new file mode 100644 index 0000000..19931f0 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_integer_attribute.py @@ -0,0 +1,29 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _IntegerAttribute(_Attribute): + @classmethod + def supports(cls, data_dict: dict): + return data_dict == "integer" + + @classmethod + def fromdict(cls, data_dict: dict): + return _IntegerAttribute() + + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + errors = [] + + if type(attribute_value) != int: + errors.append( + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_value.__class__.__name__}' (should be 'int')." + ) + + return errors diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_multi_select_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_multi_select_attribute.py new file mode 100644 index 0000000..68faa59 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_multi_select_attribute.py @@ -0,0 +1,51 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _MultiSelectAttribute(_Attribute): + + options: set[str] + + @classmethod + def supports(cls, data_dict: dict): + return ( + type(data_dict) == dict and "type" in data_dict and data_dict["type"] == "multi-select" + ) + + @classmethod + def fromdict(cls, data_dict: dict): + return _MultiSelectAttribute(options=set(data_dict["options"])) + + def check(self, attribute_name: str, attribute_values, annotation_id: str) -> t.List[str]: + + if type(attribute_values) != list: + return [ + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_values.__class__.__name__}' (should be 'list')." + ] + + for attribute_value in attribute_values: + if attribute_value not in self.options: + return [ + f"Attribute '{attribute_name}' of annotation {annotation_id} has an undefined " + + f"value '{attribute_value}' (defined options: {self._stringify_options()})." + ] + + return [] + + def _stringify_options(self) -> str: + options_str = "" + + for option in sorted(list(self.options)): + options_str += f"'{option}', " + + if options_str != "": + options_str = options_str[:-2] + + return options_str diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_single_select_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_single_select_attribute.py new file mode 100644 index 0000000..ecf5c2a --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_single_select_attribute.py @@ -0,0 +1,50 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _SingleSelectAttribute(_Attribute): + + options: set[str] + + @classmethod + def supports(cls, data_dict: dict): + return ( + type(data_dict) == dict and "type" in data_dict and data_dict["type"] == "single-select" + ) + + @classmethod + def fromdict(cls, data_dict: dict): + return _SingleSelectAttribute(options=set(data_dict["options"])) + + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + + if type(attribute_value) != str: + return [ + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_value.__class__.__name__}' (should be 'str')." + ] + + if attribute_value not in self.options: + return [ + f"Attribute '{attribute_name}' of annotation {annotation_id} has an undefined " + + f"value '{attribute_value}' (defined options: {self._stringify_options()})." + ] + + return [] + + def _stringify_options(self) -> str: + options_str = "" + + for option in sorted(list(self.options)): + options_str += f"'{option}', " + + if options_str != "": + options_str = options_str[:-2] + + return options_str diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_string_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_string_attribute.py new file mode 100644 index 0000000..1aef07f --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_string_attribute.py @@ -0,0 +1,29 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _StringAttribute(_Attribute): + @classmethod + def supports(cls, data_dict: dict): + return data_dict == "string" + + @classmethod + def fromdict(cls, data_dict: dict): + return _StringAttribute() + + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + errors = [] + + if type(attribute_value) != str: + errors.append( + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_value.__class__.__name__}' (should be 'str')." + ) + + return errors diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_vector_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_vector_attribute.py new file mode 100644 index 0000000..3d9deaa --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_vector_attribute.py @@ -0,0 +1,29 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attribute_abc import _Attribute + + +@dataclass +class _VectorAttribute(_Attribute): + @classmethod + def supports(cls, data_dict: dict): + return data_dict == "vector" + + @classmethod + def fromdict(cls, data_dict: dict): + return _VectorAttribute() + + def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.List[str]: + errors = [] + + if type(attribute_value) != list: + errors.append( + f"Attribute '{attribute_name}' of annotation {annotation_id} is of type " + + f"'{attribute_value.__class__.__name__}' (should be 'list')." + ) + + return errors diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py new file mode 100644 index 0000000..c335237 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py @@ -0,0 +1,113 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +import raillabel + +from ._attributes._attribute_abc import _Attribute, attribute_classes +from ._sensor_type import _SensorType + + +@dataclass +class _ObjectClass: + attributes: dict[str, t.Type[_Attribute]] + sensor_types: dict[raillabel.format.SensorType, _SensorType] + + @classmethod + def fromdict(cls, data_dict: dict) -> "_ObjectClass": + + if "attributes" not in data_dict: + data_dict["attributes"] = {} + + if "sensor_types" not in data_dict: + data_dict["sensor_types"] = {} + + return _ObjectClass( + attributes={ + attr_name: cls._attribute_fromdict(attr) + for attr_name, attr in data_dict["attributes"].items() + }, + sensor_types=cls._sensor_types_fromdict(data_dict["sensor_types"]), + ) + + def check(self, annotation: t.Type[raillabel.format._ObjectAnnotation]) -> t.List[str]: + errors = [] + + errors.extend(self._check_undefined_attributes(annotation)) + errors.extend(self._check_missing_attributes(annotation)) + errors.extend(self._check_false_attribute_type(annotation)) + + return errors + + @classmethod + def _attribute_fromdict(cls, attribute: dict or str) -> t.Type[_Attribute]: + + for attribute_class in attribute_classes(): + if attribute_class.supports(attribute): + return attribute_class.fromdict(attribute) + + raise ValueError() + + @classmethod + def _sensor_types_fromdict(cls, sensor_types_dict: dict) -> dict[str, _SensorType]: + sensor_types = {} + + for type_id, sensor_type_dict in sensor_types_dict.items(): + sensor_types[raillabel.format.SensorType(type_id)] = _SensorType.fromdict( + sensor_type_dict + ) + + return sensor_types + + def _check_undefined_attributes( + self, annotation: t.Type[raillabel.format._ObjectAnnotation] + ) -> t.List[str]: + errors = [] + + applicable_attributes = self._compile_applicable_attributes(annotation) + + for attr_name in annotation.attributes.keys(): + if attr_name not in applicable_attributes: + errors.append(f"Undefined attribute '{attr_name}' in annotation {annotation.uid}.") + + return errors + + def _check_missing_attributes( + self, annotation: t.Type[raillabel.format._ObjectAnnotation] + ) -> t.List[str]: + errors = [] + + for attr_name in self._compile_applicable_attributes(annotation): + if attr_name not in annotation.attributes: + errors.append(f"Missing attribute '{attr_name}' in annotation {annotation.uid}.") + + return errors + + def _check_false_attribute_type( + self, annotation: t.Type[raillabel.format._ObjectAnnotation] + ) -> t.List[str]: + errors = [] + + applicable_attributes = self._compile_applicable_attributes(annotation) + for attr_name, attr_value in annotation.attributes.items(): + if attr_name not in applicable_attributes: + continue + + errors.extend( + applicable_attributes[attr_name].check(attr_name, attr_value, annotation.uid) + ) + + return errors + + def _compile_applicable_attributes( + self, annotation: t.Type[raillabel.format._ObjectAnnotation] + ) -> dict[str, t.Type[_Attribute]]: + + applicable_attributes = self.attributes + + if annotation.sensor.type in self.sensor_types: + applicable_attributes.update(self.sensor_types[annotation.sensor.type].attributes) + + return applicable_attributes diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py new file mode 100644 index 0000000..9d74fba --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py @@ -0,0 +1,46 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +import raillabel + +from ._object_classes import _ObjectClass + + +@dataclass +class _Onthology: + classes: dict[str, _ObjectClass] + + @classmethod + def fromdict(cls, data_dict: dict) -> "_Onthology": + return _Onthology( + {class_id: _ObjectClass.fromdict(class_) for class_id, class_ in data_dict.items()} + ) + + def check(self, scene: raillabel.Scene) -> t.List[str]: + self.errors = [] + + self._check_class_validity(scene) + annotations = self._compile_annotations(scene) + for annotation in annotations: + self.errors.extend(self.classes[annotation.object.type].check(annotation)) + + return self.errors + + def _check_class_validity(self, scene: raillabel.Scene) -> t.List[str]: + object_classes_in_scene = [obj.type for obj in scene.objects.values()] + + for object_class in object_classes_in_scene: + if object_class not in self.classes: + self.errors.append(f"Object type '{object_class}' is not defined.") + + def _compile_annotations( + self, scene: raillabel.Scene + ) -> list[t.Type[raillabel.format._ObjectAnnotation]]: + annotations = [] + for frame in scene.frames.values(): + annotations.extend(list(frame.annotations.values())) + + return annotations diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_sensor_type.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_sensor_type.py new file mode 100644 index 0000000..7bf7258 --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_sensor_type.py @@ -0,0 +1,34 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from dataclasses import dataclass + +from ._attributes._attribute_abc import _Attribute, attribute_classes + + +@dataclass +class _SensorType: + attributes: dict[str, t.Type[_Attribute]] + + @classmethod + def fromdict(cls, data_dict: dict) -> "_SensorType": + + if "attributes" not in data_dict: + data_dict["attributes"] = {} + + return _SensorType( + attributes={ + attr_name: cls._attribute_fromdict(attr) + for attr_name, attr in data_dict["attributes"].items() + } + ) + + @classmethod + def _attribute_fromdict(cls, attribute: dict or str) -> t.Type[_Attribute]: + + for attribute_class in attribute_classes(): + if attribute_class.supports(attribute): + return attribute_class.fromdict(attribute) + + raise ValueError() diff --git a/raillabel_providerkit/validation/validate_onthology/validate_onthology.py b/raillabel_providerkit/validation/validate_onthology/validate_onthology.py new file mode 100644 index 0000000..95bb40a --- /dev/null +++ b/raillabel_providerkit/validation/validate_onthology/validate_onthology.py @@ -0,0 +1,66 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from pathlib import Path + +import jsonschema +import raillabel +import yaml + +from ...exceptions import OnthologySchemaError +from ._onthology_classes._onthology import _Onthology + + +def validate_onthology(scene: raillabel.Scene, onthology: t.Union[dict, Path]) -> t.List[str]: + """Validate a scene based on the classes and attributes. + + Parameters + ---------- + scene : raillabel.Scene + The scene containing the annotations. + onthology : dict or Path + Onthology YAML-data or file containing a information about all classes and their + attributes. The onthology must adhere to the onthology_schema. If a path is provided, the + file is loaded as a YAML. + + Returns + ------- + list[str] + list of all onthology errors in the scene. If an empty list is returned, then there are no + errors present. + """ + + if isinstance(onthology, Path): + onthology = _load_onthology(Path(onthology)) + + _validate_onthology_schema(onthology) + + onthology = _Onthology.fromdict(onthology) + + return onthology.check(scene) + + +def _load_onthology(path: Path) -> dict: + with path.open() as f: + onthology = yaml.safe_load(f) + return onthology + + +def _validate_onthology_schema(onthology: dict): + SCHEMA_PATH = Path(__file__).parent / "onthology_schema_v1.yaml" + + with SCHEMA_PATH.open() as f: + onthology_schema = yaml.safe_load(f) + + validator = jsonschema.Draft7Validator(schema=onthology_schema) + + schema_errors = "" + for error in validator.iter_errors(onthology): + schema_errors += f"${error.json_path[1:]}: {error.message}\n" + + if schema_errors != "": + raise OnthologySchemaError( + "The provided onthology is not valid. The following errors have been found:\n" + + schema_errors + ) diff --git a/tests/test_raillabel_providerkit/validation/validate_onthology/test_onthology_schema_v1.py b/tests/test_raillabel_providerkit/validation/validate_onthology/test_onthology_schema_v1.py index 85685b4..8d9ad12 100644 --- a/tests/test_raillabel_providerkit/validation/validate_onthology/test_onthology_schema_v1.py +++ b/tests/test_raillabel_providerkit/validation/validate_onthology/test_onthology_schema_v1.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import os +import typing as t from pathlib import Path import jsonschema @@ -30,7 +31,7 @@ def schema(schema_path) -> dict: def validator(schema) -> jsonschema.Draft7Validator: return jsonschema.Draft7Validator(schema) -def schema_errors(data: dict, validator: jsonschema.Draft7Validator) -> list[str]: +def schema_errors(data: dict, validator: jsonschema.Draft7Validator) -> t.List[str]: errors = [] for error in validator.iter_errors(data): diff --git a/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py b/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py new file mode 100644 index 0000000..d207464 --- /dev/null +++ b/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py @@ -0,0 +1,705 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +import typing as t +from pathlib import Path +from uuid import uuid4 + +import pytest +import raillabel + +sys.path.append(str(Path(__file__).parent.parent.parent.parent.parent)) +from raillabel_providerkit import exceptions +from raillabel_providerkit.validation import validate_onthology + +# == Helpers ========================== + +def make_dict_with_uids(objects: list) -> dict: + return {obj.uid: obj for obj in objects} + +def build_scene( + sensors: list[raillabel.format.Sensor], + objects: list[raillabel.format.Object], + annotations: list[t.Type[raillabel.format._ObjectAnnotation]] +) -> raillabel.Scene: + if type(sensors) == list: + sensors = make_dict_with_uids(sensors) + + return raillabel.Scene( + metadata=raillabel.format.Metadata(schema_version="1.0.0"), + sensors=sensors, + objects=make_dict_with_uids(objects), + frames={ + 0: raillabel.format.Frame( + uid=0, + annotations=make_dict_with_uids(annotations) + ) + } + ) + +@pytest.fixture +def metadata(): + return raillabel.format.Metadata(schema_version="1.0.0") + +@pytest.fixture +def sensors() -> list[raillabel.format.Sensor]: + return { + "rgb_middle": raillabel.format.Sensor( + uid="rgb_middle", + type=raillabel.format.SensorType.CAMERA, + ), + "lidar": raillabel.format.Sensor( + uid="lidar", + type=raillabel.format.SensorType.LIDAR, + ), + "radar": raillabel.format.Sensor( + uid="radar", + type=raillabel.format.SensorType.RADAR, + ), + } + +@pytest.fixture +def object_person() -> raillabel.format.Object: + return raillabel.format.Object( + uid="973ecc31-36f3-4b41-a1d8-9b584f265822", + name="person_0000", + type="person", + ) + +def build_object(type: str) -> raillabel.format.Object: + return raillabel.format.Object( + uid=uuid4, + name=type, + type=type, + ) + +def build_annotation( + object: raillabel.format.Object, + uid: str="a3f3abe5-082d-42ce-966c-bae9c6dae9d9", + sensor: raillabel.format.Sensor=raillabel.format.Sensor( + uid="rgb_middle", + type=raillabel.format.SensorType.CAMERA, + ), + attributes: dict={} +) -> raillabel.format.Bbox: + return raillabel.format.Bbox( + uid=uid, + object=object, + sensor=sensor, + attributes=attributes, + pos=[], + size=[], + ) + +# == Tests ============================ + +def test_onthology_schema_invalid(): + onthology = { + "person": { + "INVALID_FIELD": {} + } + } + + with pytest.raises(exceptions.OnthologySchemaError): + validate_onthology(None, onthology) + + +def test_valid_classes(metadata): + onthology = { + "person": {}, + "train": {}, + } + + scene = raillabel.format.Scene( + metadata=metadata, + objects=make_dict_with_uids([ + build_object("person"), + build_object("person"), + build_object("train"), + ]) + ) + + assert validate_onthology(scene, onthology) == [] + +def test_invalid_class(metadata): + onthology = { + "person": {}, + "train": {}, + } + + scene = raillabel.format.Scene( + metadata=metadata, + objects=make_dict_with_uids([ + build_object("person"), + build_object("UNDEFINED_CLASS"), + ]) + ) + + assert validate_onthology(scene, onthology) == [ + "Object type 'UNDEFINED_CLASS' is not defined." + ] + + +def test_undefined_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": {} + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "UNKNOWN_ATTRIBUTE": 10 + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Undefined attribute 'UNKNOWN_ATTRIBUTE' in annotation {annotation.uid}." + ] + +def test_missing_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "number_of_fingers": "integer" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={} + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Missing attribute 'number_of_fingers' in annotation {annotation.uid}." + ] + + +def test_valid_integer_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "number_of_fingers": "integer" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "number_of_fingers": 10 + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_integer_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "number_of_fingers": "integer" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "number_of_fingers": "THIS SHOULD BE AN INTEGER" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'number_of_fingers' of annotation {annotation.uid} is of type 'str' (should be 'int')." + ] + +def test_valid_string_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "first_name": "string" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "first_name": "Gudrun" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_string_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "first_name": "string" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "first_name": 42 + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'first_name' of annotation {annotation.uid} is of type 'int' (should be 'str')." + ] + +def test_valid_boolean_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "has_cool_blue_shirt": "boolean" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "has_cool_blue_shirt": False + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_boolean_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "has_cool_blue_shirt": "boolean" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "has_cool_blue_shirt": "NO THE SHIRT IS ORANGE ... AND THIS SHOULD BE A BOOL" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'has_cool_blue_shirt' of annotation {annotation.uid} is of type 'str' (should be 'bool')." + ] + +def test_valid_vector_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "favorite_pizzas": "vector" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "favorite_pizzas": ["Diavolo", "Neapolitan", "Quattro Formaggi"] + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_vector_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "favorite_pizzas": "vector" + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "favorite_pizzas": "does not like pizza (ikr)... THIS SHOULD BE A VECTOR AS WELL" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'favorite_pizzas' of annotation {annotation.uid} is of type 'str' (should be 'list')." + ] + +def test_valid_single_select_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": "groceries" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_single_select_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": False + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'carries' of annotation {annotation.uid} is of type 'bool' (should be 'str')." + ] + +def test_single_select_attribute_undefined_option(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": "something very unexpected" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'carries' of annotation {annotation.uid} has an undefined value " + + "'something very unexpected' (defined options: 'a baby', 'groceries', 'the SlicerDicer 3000™ (wow!)')." + ] + +def test_valid_multi_select_attribute(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "multi-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": ["groceries", "a baby"] + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_false_multi_select_attribute_type(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "multi-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": "a baby" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'carries' of annotation {annotation.uid} is of type 'str' (should be 'list')." + ] + +def test_multi_select_attribute_undefined_option(sensors, object_person): + onthology = { + "person": { + "attributes": { + "carries": { + "type": "multi-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": ["a baby", "something very unexpected"] + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'carries' of annotation {annotation.uid} has an undefined value " + + "'something very unexpected' (defined options: 'a baby', 'groceries', 'the SlicerDicer 3000™ (wow!)')." + ] + +def test_multiple_attributes_valid(sensors, object_person): + onthology = { + "person": { + "attributes": { + "number_of_fingers": "integer", + "first_name": "string", + "carries": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + } + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": "groceries", + "number_of_fingers": 9, + "first_name": "Brunhilde", + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_multiple_attributes_invalid(sensors, object_person): + onthology = { + "person": { + "attributes": { + "number_of_fingers": "integer", + "first_name": "string", + "carries": { + "type": "single-select", + "options": [ + "groceries", + "a baby", + "the SlicerDicer 3000™ (wow!)", + ] + } + } + } + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "carries": "something very unexpected", + "number_of_fingers": 9, + "first_name": True, + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + c = validate_onthology(scene, onthology) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'carries' of annotation {annotation.uid} has an undefined value " + + "'something very unexpected' (defined options: 'a baby', 'groceries', 'the SlicerDicer 3000™ (wow!)').", + f"Attribute 'first_name' of annotation {annotation.uid} is of type 'bool' (should be 'str').", + ] + + +def test_valid_sensor_type_attribute(sensors, object_person): + onthology = { + "person": { + "sensor_types": { + "lidar": { + "attributes": { + "number_of_fingers": "integer" + } + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "number_of_fingers": 10 + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_invalid_sensor_type_attribute(sensors, object_person): + onthology = { + "person": { + "sensor_types": { + "lidar": { + "attributes": { + "number_of_fingers": "integer" + } + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "number_of_fingers": "None" + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Attribute 'number_of_fingers' of annotation {annotation.uid} is of type 'str' (should be 'int')." + ] + + +def test_valid_sensor_type_attributes_and_attributes(sensors, object_person): + onthology = { + "person": { + "attributes": { + "first_name": "string" + }, + "sensor_types": { + "lidar": { + "attributes": { + "number_of_fingers": "integer" + } + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "number_of_fingers": 10, + "first_name": "Brunhilde", + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [] + +def test_invalid_sensor_type_attributes_and_attributes(sensors, object_person): + onthology = { + "person": { + "attributes": { + "first_name": "string" + }, + "sensor_types": { + "lidar": { + "attributes": { + "number_of_fingers": "integer" + } + } + } + }, + } + + annotation = build_annotation( + object=object_person, + sensor=sensors["lidar"], + attributes={ + "first_name": "Brunhilde", + } + ) + + scene = build_scene(sensors, [object_person], [annotation]) + assert validate_onthology(scene, onthology) == [ + f"Missing attribute 'number_of_fingers' in annotation {annotation.uid}." + ] + + +if __name__ == "__main__": + os.system("clear") + pytest.main([__file__, "--disable-pytest-warnings", "--cache-clear", "-v"]) From 46ccef8961232002d282dbd04aa0c534dc7c42ac Mon Sep 17 00:00:00 2001 From: Tobias Klockau Date: Fri, 20 Oct 2023 15:02:38 +0200 Subject: [PATCH 06/13] fix: dependency --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 475c838..7ca671f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,8 @@ classifiers = [ dependencies = [ "jsonschema>=4.4.0", "fastjsonschema>=2.16.2", - "raillabel>=3.1.0" + "raillabel>=3.1.0", + "pyyaml>=6.0.0" ] [project.urls] From f1efbb80039d0ab7ffd1b8bdf43c2ebd8347ff10 Mon Sep 17 00:00:00 2001 From: Tobias Klockau Date: Fri, 20 Oct 2023 15:09:44 +0200 Subject: [PATCH 07/13] fix: python 3.8 compatibility --- .../_onthology_classes/_attributes/_attribute_abc.py | 2 +- .../validate_onthology/_onthology_classes/_onthology.py | 2 +- .../validate_onthology/test_validate_onthology.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py index 0599d7b..ae6b9d5 100644 --- a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py @@ -25,7 +25,7 @@ def check(self, attribute_name: str, attribute_value, annotation_id: str) -> t.L raise NotImplementedError -def attribute_classes() -> list[t.Type[_Attribute]]: +def attribute_classes() -> t.List[t.Type[_Attribute]]: """Return dictionary with Attribute child classes.""" return ATTRIBUTE_CLASSES diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py index 9d74fba..c78526c 100644 --- a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py @@ -38,7 +38,7 @@ def _check_class_validity(self, scene: raillabel.Scene) -> t.List[str]: def _compile_annotations( self, scene: raillabel.Scene - ) -> list[t.Type[raillabel.format._ObjectAnnotation]]: + ) -> t.List[t.Type[raillabel.format._ObjectAnnotation]]: annotations = [] for frame in scene.frames.values(): annotations.extend(list(frame.annotations.values())) diff --git a/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py b/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py index d207464..0ff0138 100644 --- a/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py +++ b/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py @@ -20,9 +20,9 @@ def make_dict_with_uids(objects: list) -> dict: return {obj.uid: obj for obj in objects} def build_scene( - sensors: list[raillabel.format.Sensor], - objects: list[raillabel.format.Object], - annotations: list[t.Type[raillabel.format._ObjectAnnotation]] + sensors: t.List[raillabel.format.Sensor], + objects: t.List[raillabel.format.Object], + annotations: t.List[t.Type[raillabel.format._ObjectAnnotation]] ) -> raillabel.Scene: if type(sensors) == list: sensors = make_dict_with_uids(sensors) @@ -44,7 +44,7 @@ def metadata(): return raillabel.format.Metadata(schema_version="1.0.0") @pytest.fixture -def sensors() -> list[raillabel.format.Sensor]: +def sensors() -> t.List[raillabel.format.Sensor]: return { "rgb_middle": raillabel.format.Sensor( uid="rgb_middle", From 5ab31cae4ecd3e5943844c11e7837f42f85da441 Mon Sep 17 00:00:00 2001 From: Tobias Klockau Date: Fri, 20 Oct 2023 15:12:57 +0200 Subject: [PATCH 08/13] fix: python 3.8 compatibility - dict typing --- .../_onthology_classes/_object_classes.py | 6 +++--- .../validate_onthology/_onthology_classes/_onthology.py | 2 +- .../validate_onthology/_onthology_classes/_sensor_type.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py index c335237..c4ede47 100644 --- a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py @@ -12,8 +12,8 @@ @dataclass class _ObjectClass: - attributes: dict[str, t.Type[_Attribute]] - sensor_types: dict[raillabel.format.SensorType, _SensorType] + attributes: t.Dict[str, t.Type[_Attribute]] + sensor_types: t.Dict[raillabel.format.SensorType, _SensorType] @classmethod def fromdict(cls, data_dict: dict) -> "_ObjectClass": @@ -103,7 +103,7 @@ def _check_false_attribute_type( def _compile_applicable_attributes( self, annotation: t.Type[raillabel.format._ObjectAnnotation] - ) -> dict[str, t.Type[_Attribute]]: + ) -> t.Dict[str, t.Type[_Attribute]]: applicable_attributes = self.attributes diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py index c78526c..15e319e 100644 --- a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py @@ -11,7 +11,7 @@ @dataclass class _Onthology: - classes: dict[str, _ObjectClass] + classes: t.Dict[str, _ObjectClass] @classmethod def fromdict(cls, data_dict: dict) -> "_Onthology": diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_sensor_type.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_sensor_type.py index 7bf7258..a74960b 100644 --- a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_sensor_type.py +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_sensor_type.py @@ -9,7 +9,7 @@ @dataclass class _SensorType: - attributes: dict[str, t.Type[_Attribute]] + attributes: t.Dict[str, t.Type[_Attribute]] @classmethod def fromdict(cls, data_dict: dict) -> "_SensorType": From ed2ce2d1ca157eb9a54613add63c154eb351b377 Mon Sep 17 00:00:00 2001 From: Tobias Klockau Date: Fri, 20 Oct 2023 15:14:33 +0200 Subject: [PATCH 09/13] fix: python 3.8 compatibility - set typing --- .../_onthology_classes/_attributes/_multi_select_attribute.py | 2 +- .../_onthology_classes/_attributes/_single_select_attribute.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_multi_select_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_multi_select_attribute.py index 68faa59..5805194 100644 --- a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_multi_select_attribute.py +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_multi_select_attribute.py @@ -10,7 +10,7 @@ @dataclass class _MultiSelectAttribute(_Attribute): - options: set[str] + options: t.Set[str] @classmethod def supports(cls, data_dict: dict): diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_single_select_attribute.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_single_select_attribute.py index ecf5c2a..7ce4309 100644 --- a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_single_select_attribute.py +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_single_select_attribute.py @@ -10,7 +10,7 @@ @dataclass class _SingleSelectAttribute(_Attribute): - options: set[str] + options: t.Set[str] @classmethod def supports(cls, data_dict: dict): From f67049abeb57e7ae9075497b261c672c7accce0a Mon Sep 17 00:00:00 2001 From: Tobias Klockau Date: Fri, 20 Oct 2023 15:15:51 +0200 Subject: [PATCH 10/13] fix: python 3.8 compatibility - dict typing 2 --- .../validate_onthology/_onthology_classes/_object_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py index c4ede47..b691f82 100644 --- a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_object_classes.py @@ -51,7 +51,7 @@ def _attribute_fromdict(cls, attribute: dict or str) -> t.Type[_Attribute]: raise ValueError() @classmethod - def _sensor_types_fromdict(cls, sensor_types_dict: dict) -> dict[str, _SensorType]: + def _sensor_types_fromdict(cls, sensor_types_dict: dict) -> t.Dict[str, _SensorType]: sensor_types = {} for type_id, sensor_type_dict in sensor_types_dict.items(): From c625e4f599f966796d58870d0e6e28d69f53e6af Mon Sep 17 00:00:00 2001 From: Tobias Klockau Date: Fri, 20 Oct 2023 15:23:32 +0200 Subject: [PATCH 11/13] lint: fix warnings --- .../_onthology_classes/_attributes/_attribute_abc.py | 6 ++++-- .../validate_onthology/_onthology_classes/_onthology.py | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py index ae6b9d5..6edebc5 100644 --- a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_attributes/_attribute_abc.py @@ -12,11 +12,13 @@ @dataclass class _Attribute(abc.ABC): - @abc.abstractclassmethod + @classmethod + @abc.abstractmethod def supports(cls, data_dict: dict) -> bool: raise NotImplementedError - @abc.abstractclassmethod + @classmethod + @abc.abstractmethod def fromdict(cls, data_dict: dict) -> t.Type["_Attribute"]: raise NotImplementedError diff --git a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py index 15e319e..456f0b2 100644 --- a/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py +++ b/raillabel_providerkit/validation/validate_onthology/_onthology_classes/_onthology.py @@ -12,6 +12,7 @@ @dataclass class _Onthology: classes: t.Dict[str, _ObjectClass] + errors = [] @classmethod def fromdict(cls, data_dict: dict) -> "_Onthology": From 0078322fee39b607db7f887da9aa9a8bf25a6b44 Mon Sep 17 00:00:00 2001 From: Tobias Klockau Date: Fri, 20 Oct 2023 15:50:16 +0200 Subject: [PATCH 12/13] test: add fixtures for onthology and scenes --- .../validation/conftest.py | 9 +++++ .../test_validate_onthology.py | 37 +++++++++++++++++-- 2 files changed, 42 insertions(+), 4 deletions(-) create mode 100644 tests/test_raillabel_providerkit/validation/conftest.py diff --git a/tests/test_raillabel_providerkit/validation/conftest.py b/tests/test_raillabel_providerkit/validation/conftest.py new file mode 100644 index 0000000..cfdf202 --- /dev/null +++ b/tests/test_raillabel_providerkit/validation/conftest.py @@ -0,0 +1,9 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +from validate_onthology.test_validate_onthology import ( + demo_onthology, + invalid_onthology_scene, + metadata, + valid_onthology_scene, +) diff --git a/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py b/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py index 0ff0138..53baa7f 100644 --- a/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py +++ b/tests/test_raillabel_providerkit/validation/validate_onthology/test_validate_onthology.py @@ -39,10 +39,6 @@ def build_scene( } ) -@pytest.fixture -def metadata(): - return raillabel.format.Metadata(schema_version="1.0.0") - @pytest.fixture def sensors() -> t.List[raillabel.format.Sensor]: return { @@ -93,6 +89,39 @@ def build_annotation( size=[], ) +# == Fixtures ========================= + +@pytest.fixture +def metadata(): + return raillabel.format.Metadata(schema_version="1.0.0") + +@pytest.fixture +def demo_onthology() -> dict: + return { + "person": {}, + "train": {}, + } + +@pytest.fixture +def valid_onthology_scene(metadata) -> raillabel.Scene: + return raillabel.format.Scene( + metadata=metadata, + objects=make_dict_with_uids([ + build_object("person"), + build_object("person"), + build_object("train"), + ]) + ) + +@pytest.fixture +def invalid_onthology_scene(metadata) -> raillabel.Scene: + return raillabel.format.Scene( + metadata=metadata, + objects=make_dict_with_uids([ + build_object("INVALID_CLASS"), + ]) + ) + # == Tests ============================ def test_onthology_schema_invalid(): From 578bbfbe011e1e23fafc587ae405b8bc27851596 Mon Sep 17 00:00:00 2001 From: Tobias Klockau Date: Fri, 20 Oct 2023 15:51:21 +0200 Subject: [PATCH 13/13] feat: validate function --- raillabel_providerkit/__init__.py | 1 + raillabel_providerkit/validation/validate.py | 35 +++++++++++++++++++ .../validation/test_validate.py | 27 ++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 raillabel_providerkit/validation/validate.py create mode 100644 tests/test_raillabel_providerkit/validation/test_validate.py diff --git a/raillabel_providerkit/__init__.py b/raillabel_providerkit/__init__.py index a94acf2..733f88e 100644 --- a/raillabel_providerkit/__init__.py +++ b/raillabel_providerkit/__init__.py @@ -6,6 +6,7 @@ from . import format from .convert import loader_classes from .convert.convert import convert +from .validation.validate import validate try: __version__ = metadata.version("raillabel-providerkit") diff --git a/raillabel_providerkit/validation/validate.py b/raillabel_providerkit/validation/validate.py new file mode 100644 index 0000000..a3dddde --- /dev/null +++ b/raillabel_providerkit/validation/validate.py @@ -0,0 +1,35 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import typing as t +from pathlib import Path + +import raillabel + +from . import validate_onthology + + +def validate(scene: raillabel.Scene, onthology: t.Union[dict, Path]) -> t.List[str]: + """Validate a scene based on the Deutsche Bahn Requirements. + + Parameters + ---------- + scene : raillabel.Scene + The scene containing the annotations. + onthology : dict or Path + Onthology YAML-data or file containing a information about all classes and their + attributes. The onthology must adhere to the onthology_schema. If a path is provided, the + file is loaded as a YAML. + + Returns + ------- + list[str] + list of all requirement errors in the scene. If an empty list is returned, then there are + no errors present and the scene is valid. + """ + + errors = [] + + errors += validate_onthology(scene, onthology) + + return errors diff --git a/tests/test_raillabel_providerkit/validation/test_validate.py b/tests/test_raillabel_providerkit/validation/test_validate.py new file mode 100644 index 0000000..e3ce73d --- /dev/null +++ b/tests/test_raillabel_providerkit/validation/test_validate.py @@ -0,0 +1,27 @@ +# Copyright DB Netz AG and contributors +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +import typing as t +from pathlib import Path +from uuid import uuid4 + +import pytest +import raillabel + +sys.path.append(str(Path(__file__).parent.parent.parent.parent.parent)) +from raillabel_providerkit import validate + +# == Tests ============================ + +def test_no_errors(demo_onthology, valid_onthology_scene): + assert validate(valid_onthology_scene, demo_onthology) == [] + +def test_onthology_errors(demo_onthology, invalid_onthology_scene): + assert len(validate(invalid_onthology_scene, demo_onthology)) == 1 + + +if __name__ == "__main__": + os.system("clear") + pytest.main([__file__, "--disable-pytest-warnings", "--cache-clear", "-v"])