From f7276fc32c721e4820746813df8e257233840055 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20G=C5=82owacki?= Date: Wed, 30 Oct 2024 07:29:12 +0000 Subject: [PATCH 1/8] 2x processing speedup --- .../perception_eval_conversions.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py b/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py index 48f1f2fb..e45453f5 100644 --- a/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py +++ b/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py @@ -22,7 +22,7 @@ from geometry_msgs.msg import Pose from geometry_msgs.msg import Quaternion as RosQuaternion from geometry_msgs.msg import Vector3 -import jsonschema +from jsonschema.validators import validator_for import numpy as np from perception_eval.common import ObjectType from perception_eval.common.object import DynamicObject @@ -315,15 +315,19 @@ def load_schema(cls) -> None: ) with schema_file_path.open() as file: cls.schema = json.load(file) + # Compile the schema into a validator instance + validator_cls = validator_for(cls.schema) + validator_cls.check_schema(cls.schema) + cls.validator = validator_cls(cls.schema) @classmethod def is_object_structure_valid(cls, objdata: dict | None) -> bool: cls.load_schema() try: - jsonschema.validate(objdata, cls.schema) + cls.validator.validate(objdata) + return True except jsonschema.exceptions.ValidationError: return False - return True @staticmethod def object_to_description(obj: ObjectType | None) -> dict: From 29e90304d57d377e10b6eaa3977042176cc6b817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20G=C5=82owacki?= Date: Wed, 30 Oct 2024 08:48:37 +0000 Subject: [PATCH 2/8] fixes for ruff --- .../driving_log_replayer/perception_eval_conversions.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py b/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py index e45453f5..2dbe2779 100644 --- a/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py +++ b/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py @@ -22,7 +22,7 @@ from geometry_msgs.msg import Pose from geometry_msgs.msg import Quaternion as RosQuaternion from geometry_msgs.msg import Vector3 -from jsonschema.validators import validator_for +import jsonschema import numpy as np from perception_eval.common import ObjectType from perception_eval.common.object import DynamicObject @@ -316,7 +316,7 @@ def load_schema(cls) -> None: with schema_file_path.open() as file: cls.schema = json.load(file) # Compile the schema into a validator instance - validator_cls = validator_for(cls.schema) + validator_cls = jsonschema.validators.validator_for(cls.schema) validator_cls.check_schema(cls.schema) cls.validator = validator_cls(cls.schema) @@ -325,9 +325,10 @@ def is_object_structure_valid(cls, objdata: dict | None) -> bool: cls.load_schema() try: cls.validator.validate(objdata) - return True except jsonschema.exceptions.ValidationError: return False + else: + return True @staticmethod def object_to_description(obj: ObjectType | None) -> dict: From 5aabcd3225e114602205868dbe51e82e00dd18ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20G=C5=82owacki?= Date: Wed, 30 Oct 2024 11:00:37 +0000 Subject: [PATCH 3/8] fastjson --- .../perception_eval_conversions.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py b/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py index 2dbe2779..91280b25 100644 --- a/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py +++ b/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py @@ -22,7 +22,7 @@ from geometry_msgs.msg import Pose from geometry_msgs.msg import Quaternion as RosQuaternion from geometry_msgs.msg import Vector3 -import jsonschema +import fastjsonschema import numpy as np from perception_eval.common import ObjectType from perception_eval.common.object import DynamicObject @@ -305,27 +305,26 @@ def fill_xyzw_quat(q: Quaternion | None) -> dict: # utils for writing each perception frame result to a file class FrameDescriptionWriter: schema: dict = None + validate_func = None @classmethod def load_schema(cls) -> None: - if cls.schema is None: + if cls.schema is None or cls.validate_func is None: package_share_directory = get_package_share_directory("driving_log_replayer") schema_file_path = ( Path(package_share_directory) / "config" / "object_output_schema.json" ) with schema_file_path.open() as file: cls.schema = json.load(file) - # Compile the schema into a validator instance - validator_cls = jsonschema.validators.validator_for(cls.schema) - validator_cls.check_schema(cls.schema) - cls.validator = validator_cls(cls.schema) + # Compile the schema into a validation function + cls.validate_func = fastjsonschema.compile(cls.schema) @classmethod def is_object_structure_valid(cls, objdata: dict | None) -> bool: cls.load_schema() try: - cls.validator.validate(objdata) - except jsonschema.exceptions.ValidationError: + cls.validate_func(objdata) + except fastjsonschema.exceptions.JsonSchemaException: return False else: return True From dc1ff4c5d63d3ae894ba3b4cf6b042f966a86992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20G=C5=82owacki?= Date: Wed, 30 Oct 2024 11:31:09 +0000 Subject: [PATCH 4/8] update project for dependency --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 122b8b67..aa277b10 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "simplejson>=3.19.2", "pydantic>=2.6.1", "kaleido>=0.2.1", + "fastjsonschema>=2.15.1", ] readme = "README.md" requires-python = ">=3.10,<3.11" From 710b5c190e8de2fb104d84ad040202722df86d54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20G=C5=82owacki?= Date: Wed, 30 Oct 2024 12:16:10 +0000 Subject: [PATCH 5/8] remove useless comment --- .../driving_log_replayer/perception_eval_conversions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py b/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py index 91280b25..f6069994 100644 --- a/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py +++ b/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py @@ -316,7 +316,6 @@ def load_schema(cls) -> None: ) with schema_file_path.open() as file: cls.schema = json.load(file) - # Compile the schema into a validation function cls.validate_func = fastjsonschema.compile(cls.schema) @classmethod From 6241b1ec9d9c03ae1a0449eb2e965cae1438308d Mon Sep 17 00:00:00 2001 From: Hayato Mizushima Date: Fri, 1 Nov 2024 14:31:43 +0900 Subject: [PATCH 6/8] fix: dependency Signed-off-by: Hayato Mizushima --- driving_log_replayer/package.xml | 2 +- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/driving_log_replayer/package.xml b/driving_log_replayer/package.xml index f7cd092a..92816277 100644 --- a/driving_log_replayer/package.xml +++ b/driving_log_replayer/package.xml @@ -38,7 +38,7 @@ driving_log_replayer_analyzer lanelet2_extension_python perception_eval - python3-jsonschema + python3-fastjsonschema python3-pandas python3-simplejson python3-termcolor diff --git a/pyproject.toml b/pyproject.toml index aa277b10..122b8b67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,6 @@ dependencies = [ "simplejson>=3.19.2", "pydantic>=2.6.1", "kaleido>=0.2.1", - "fastjsonschema>=2.15.1", ] readme = "README.md" requires-python = ">=3.10,<3.11" From b7f075c964a81833f2124f2134ed2634fbc3632e Mon Sep 17 00:00:00 2001 From: Hayato Mizushima Date: Fri, 1 Nov 2024 14:32:33 +0900 Subject: [PATCH 7/8] fix: import order Signed-off-by: Hayato Mizushima --- .../driving_log_replayer/perception_eval_conversions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py b/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py index f6069994..ecb12bc2 100644 --- a/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py +++ b/driving_log_replayer/driving_log_replayer/perception_eval_conversions.py @@ -17,12 +17,12 @@ from ament_index_python.packages import get_package_share_directory from builtin_interfaces.msg import Time +import fastjsonschema from geometry_msgs.msg import Point from geometry_msgs.msg import Polygon as RosPolygon from geometry_msgs.msg import Pose from geometry_msgs.msg import Quaternion as RosQuaternion from geometry_msgs.msg import Vector3 -import fastjsonschema import numpy as np from perception_eval.common import ObjectType from perception_eval.common.object import DynamicObject From d99b8643b9254f4a493b24ad3666b6ac0f8b45b7 Mon Sep 17 00:00:00 2001 From: Hayato Mizushima Date: Fri, 1 Nov 2024 14:37:46 +0900 Subject: [PATCH 8/8] chore: add word Signed-off-by: Hayato Mizushima --- .driving_log_replayer.cspell.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.driving_log_replayer.cspell.json b/.driving_log_replayer.cspell.json index b9941db4..a5d49c2c 100644 --- a/.driving_log_replayer.cspell.json +++ b/.driving_log_replayer.cspell.json @@ -38,6 +38,7 @@ "youtalk", "UMiho", "nuscenes", - "devkit" + "devkit", + "fastjsonschema" ] }