JStore cleanup (#1001)

Fix issue #1000 - use JStore to remove duplicated code.

Make get_instance() checking the entity service and add a workaround for a bug in entity service stripping off dimensions.

* Cleaned up iteration over instances in a JSON file.
* Make dlite-validate work for storages that don't implement query()
* Added test
* Added python interface to jstore
* Reduce code duplication.
* Add condition for deprecation warnings

---------

Co-authored-by: Francesca L. Bleken <[email protected]>

bindings/python/dlite-entity-python.i

@@ -168,8 +168,26 @@ def get_instance(
             metaid = str(metaid).rstrip("#/")
         inst = _dlite.get_instance(id, metaid, check_storages)
 
+    if inst is None and id.startswith("http://onto-ns.com/"):
+        try:
+            import requests
+        except ModuleNotFoundError as e:
+            import warnings
+            warnings.warn(f"{e}: skip trying to fetch from entity service")
+        else:
+            import json
+            r = requests.get(id)
+            if r.ok:
+                d = json.loads(r.content.decode())
+                # Workaround for bugs in the entity service
+                if "meta" not in d or d["meta"] == dlite.ENTITY_SCHEMA:
+                    if "dimensions" not in d:
+                        d["dimensions"] = {}
+            inst = Instance.from_dict(d)
+
     if inst is None:
         raise _dlite.DLiteMissingInstanceError(f"no such instance: {id}")
+
     return instance_cast(inst)
 
 %}

bindings/python/dlite-jstore-python.i

@@ -160,6 +160,16 @@ def format_dict(
     return {uri if uri and urikey else uuid: dct}
 
 
+class _JSONEncoder(json.JSONEncoder):
+    """JSON encoder that also handle bytes object."""
+    def default(self, o):
+        if isinstance(o, bytes):
+            return "".join(f"{c:x}" for c in o)
+        elif (isinstance(o, (str, bool, int, float, list, dict)) or o is None):
+            return super().default(o)
+        else:
+            return str(o)
+
 %}
 
 
@@ -203,7 +213,7 @@ def format_dict(
           if "uuid" in d:
               uuid = d["uuid"]
           elif "uri" in d or "identity" in d:
-              uuid = _dlite.get_uuid(d.get("uri", d.get("identity")))
+              uuid = _dlite.get_uuid(str(d.get("uri", d.get("identity"))))
 
           if id and uuid and _dlite.get_uuid(id) != uuid:
               raise _dlite.DLiteInconsistentDataError(
@@ -223,7 +233,7 @@ def format_dict(
           if id and id != uuid:
               d.setdefault("uri", id)
 
-          self.load_json(json.dumps(d))
+          self.load_json(json.dumps(d, cls=_JSONEncoder))
 
       def get_dict(self, id=None, soft7=True, single=None, with_uuid=None,
                    with_meta=False, with_parent=True, urikey=False):

bindings/python/dlite-jstore.i

@@ -5,8 +5,8 @@
 #include "dlite-errors.h"
 #include "dlite-json.h"
 
-  /* If store `js` only has one instance, return its id, otherwise raise a
-     DLiteLookupError. */
+  /* If store `js` only has one instance, return borrowed reference to its id,
+     otherwise raise a DLiteLookupError. */
   static const char *_single_id(JStore *js)
   {
     const char *key = jstore_get_single_key(js);
@@ -97,7 +97,6 @@ struct _JStore {};
   %feature("docstring",
            "If there is one instance in storage, return its id. "
            "Otherwise, raise an DLiteLookupError exception.") get_single_id;
-  %newobject get_single_id;
   const char *get_single_id(void) {
     return _single_id($self);
   }

bindings/python/tests/test_utils.py

@@ -65,6 +65,7 @@
         ],
     },
 }
+
 inst = instance_from_dict(d)
 print(inst)
 

bindings/python/utils.py

@@ -160,157 +160,35 @@ def add(self, d, id=None):
 def instance_from_dict(d, id=None, single=None, check_storages=True):
     """Returns a new DLite instance created from dict.
 
-    Parameters
-    ----------
-    d: dict
-        Dict to parse.  It should be of the same form as returned
-        by the Instance.asdict() method.
-    id: str
-        Identity of the returned instance.
-
-        If `d` is in single-entity form with no explicit 'uuid' or
-        'uri', its identity will be assigned by `id`.  Otherwise
-        `id` must be consistent with the 'uuid' and/or 'uri'
-        fields of `d`.
-
-        If `d` is in multi-entity form, `id` is used to select the
-        instance to return.
-    single: bool | None | "auto"
-        Whether the dict is assumed to be in single-entity form
-        If `single` is None or "auto", the form is inferred.
-    check_storages: bool
-        Whether to check if the instance already exists in storages
-        specified in `dlite.storage_path`.
+    Arguments:
+        d: Dict to parse.  It should be of the same form as returned
+            by the Instance.asdict() method.
+        id: ID (URI or UUID) if instance to return.
+        single: Unsused. Provided for backward compabitility.
+        check_storages: bool
+            Whether to check if the instance already exists in storages
+            specified in `dlite.storage_path`.
     """
-    if single is None or single == "auto":
-        single = True if "properties" in d else False
-
-    if single:
-        if not id and "uuid" not in d and "uri" not in d:
-            if "namespace" in d and "version" in d and "name" in d:
-                id = f"{d['namespace']}/{d['version']}/{d['name']}"
-            else:
-                raise ValueError(
-                    "`id` required for dicts in single-entry "
-                    "form with no explicit uuid or uri."
-                )
-    else:
-        if not id:
-            if len(d) == 1:
-                (id,) = d.keys()
-            else:
-                raise ValueError(
-                    "`id` required for dicts in multi-entry form."
-                )
-        if id in d:
-            return instance_from_dict(
-                d[id], id=id, single=True, check_storages=check_storages
-            )
-        else:
-            uuid = dlite.get_uuid(id)
-            if uuid in d:
-                return instance_from_dict(
-                    d[uuid], id=id, single=True, check_storages=check_storages
-                )
-            else:
-                raise ValueError(f"no such id in dict: {id}")
-
-    if "uri" in d or "uuid" in d:
-        if "uri" in d and "uuid" in d:
-            if dlite.get_uuid(d["uri"]) != d["uuid"]:
-                raise dlite.DLiteError(
-                    "uri and uuid in dict are not consistent"
-                )
-        uuid = dlite.get_uuid(str(d.get("uuid", d.get("uri"))))
-        if id:
-            if dlite.get_uuid(id) != uuid:
-                raise ValueError(
-                    f"`id` is not consistent with uri/uuid in dict"
-                )
-
-    meta = dlite.get_instance(d.get("meta", dlite.ENTITY_SCHEMA))
-
-    if meta.is_metameta:
-        if "uri" in d:
-            uri = d["uri"]
-        elif "identity" in d:
-            uri = d["identity"]
-        elif "name" in d and "version" in d and "namespace" in d:
-            uri = dlite.join_meta_uri(d["name"], d["version"], d["namespace"])
-        elif id and dlite.urlparse(id).scheme:
-            uri = id
-        else:
-            raise TypeError(
-                "`id` required for metadata when the URI is not in the dict"
-            )
-
-        if check_storages:
-            try:
-                with dlite.silent:
-                    return dlite.get_instance(uri)
-            except dlite.DLiteError:
-                pass
-
-        if "dimensions" not in d:
-            dimensions = []
-        elif isinstance(d["dimensions"], Sequence):
-            dimensions = [
-                dlite.Dimension(d["name"], d.get("description"))
-                for d in d["dimensions"]
-            ]
-        elif isinstance(d["dimensions"], Mapping):
-            dimensions = [
-                dlite.Dimension(k, v) for k, v in d["dimensions"].items()
-            ]
-        else:
-            raise TypeError(
-                "`dimensions` must be either a sequence or a mapping"
-            )
-
-        props = []
-        if isinstance(d["properties"], Sequence):
-            for p in d["properties"]:
-                props.append(
-                    dlite.Property(
-                        name=p["name"],
-                        type=p["type"],
-                        ref=p.get("$ref", p.get("ref")),
-                        shape=p.get("shape", p.get("dims")),
-                        unit=p.get("unit"),
-                        description=p.get("description"),
-                    )
-                )
-        elif isinstance(d["properties"], Mapping):
-            for k, v in d["properties"].items():
-                props.append(
-                    dlite.Property(
-                        name=k,
-                        type=v["type"],
-                        ref=v.get("$ref", v.get("ref")),
-                        shape=v.get("shape", v.get("dims")),
-                        unit=v.get("unit"),
-                        description=v.get("description"),
-                    )
-                )
-        else:
-            raise TypeError(
-                "`properties` must be either a sequence or a mapping"
-            )
-
-        inst = dlite.Instance.create_metadata(
-            uri, dimensions, props, d.get("description")
+    if single is not None:
+        dlite.deprecation_warning(
+            "0.7.0",
+            "Argument `single` is deprecated and not used. Single/multi-"
+            "instance format is now inferred.",
         )
-    else:
-        dims = [
-            d["dimensions"][dim.name] for dim in meta.properties["dimensions"]
-        ]
-        inst_id = d.get("uri", d.get("uuid", id))
-        inst = dlite.Instance.from_metaid(meta.uri, dimensions=dims, id=inst_id)
-        for p in meta["properties"]:
-            value = d["properties"][p.name]
-            inst.set_property(p.name, value)
 
-    return inst
+    js = dlite.JStore()
+    js.load_dict(d)
+
+    if check_storages:
+        if not id:
+            id = js.get_single_id()
+        try:
+            with dlite.silent:
+                return dlite.get_instance(id)
+        except dlite.DLiteMissingInstanceError:
+            pass
+
+    return js.get(id=id)
 
 
 def to_metadata(obj):

src/dlite-json.c

@@ -624,7 +624,7 @@ static DLiteInstance *parse_instance(const char *src, jsmntok_t *obj,
   /* Parse dimensions */
   if (dlite_meta_is_metameta(meta)) {
     /* For metadata, dimension sizes are inferred from the size of
-       "dimensions", "propertis" and "relations". */
+       "dimensions", "properties" and "relations". */
     size_t n=0;
     if ((t = jsmn_item(src, obj, "dimensions"))) dims[n++] = t->size;
     if ((t = jsmn_item(src, obj, "properties"))) dims[n++] = t->size;
@@ -722,11 +722,8 @@ static DLiteInstance *parse_instance(const char *src, jsmntok_t *obj,
           if (dlite_property_jscan(src, t, NULL, ptr, p, pdims, 0) < 0)
             goto fail;
         } else if (t->type == JSMN_OBJECT) {
-          if (dlite_instance_is_meta(inst)) {
-            if (dlite_property_jscan(src, t, p->name, ptr, p, pdims, 0) < 0)
-              goto fail;
-          } else
-            FAIL1("missing key \"%s\" in JSON object", p->name);
+          if (dlite_property_jscan(src, t, p->name, ptr, p, pdims, 0) < 0)
+            goto fail;
         } else {
           if (dlite_type_scan(src+t->start, t->end - t->start, ptr, p->type,
                               p->size, 0) < 0)

src/utils/jstore.h

@@ -126,7 +126,8 @@ int jstore_to_file(JStore *js, const char *filename);
 /** Return number of elements in the store. */
 int jstore_count(JStore *js);
 
-/** If there is one item in the store, return its key.  Otherwise return NULL. */
+/** If there is one item in the store, return a borrowed pointer to its key.
+   Otherwise return NULL. */
 const char *jstore_get_single_key(JStore *js);