Optimisations in Instrument lists (#661)

* instance_signature_from_owl

* allow passing ontology instance as ontology_path

* avoid constructing unnecessary instruments

* instrument factory iterator

* revert return signature

* fix _instrument recreation

* separate ontology_path and ontology_object

* fix

* Update cdci_data_analysis/analysis/parameters.py

Co-authored-by: Volodymyr <[email protected]>

* Update cdci_data_analysis/flask_app/app.py

Co-authored-by: Volodymyr <[email protected]>

---------

Co-authored-by: Volodymyr <[email protected]>

cdci_data_analysis/analysis/instrument.py

@@ -793,3 +793,13 @@ def build_catalog(cat_dic,catalog_selected_objects=None):
     user_catalog.select_IDs(IDs)
 
     return user_catalog
+
+class InstrumentFactoryIterator:
+    def __init__(self):
+        self._partlist = []
+
+    def extend(self, lst):
+        self._partlist.append(lst)
+
+    def __iter__(self):
+        return (y for x in self._partlist for y in x)

cdci_data_analysis/analysis/parameters.py

@@ -30,6 +30,7 @@
 import six
 import decorator
 import logging
+import os
 
 from astropy.time import Time as astropyTime
 from astropy.time import TimeDelta as astropyTimeDelta
@@ -481,24 +482,37 @@ def from_owl_uri(cls,
                      owl_uri,
                      extra_ttl = None,
                      ontology_path = None, 
+                     ontology_object = None,
                      **kwargs):
         from oda_api.ontology_helper import Ontology
 
-        if ontology_path:
-            onto = Ontology(ontology_path)
+        if ontology_path is not None and ontology_object is not None:
+            raise RuntimeError("Both ontology_path and ontology_object parameters are set.")
+        elif ontology_path is None and ontology_object is None:
+            logger.warning('Ontology path/object not set in Parameter.from_owl_uri(). '
+                'Trying to find parameter which have %s directly set. '
+                'extra_ttl will be ignored ', owl_uri)
+            parameter_hierarchy = [ owl_uri ]
+            par_format = par_unit = allowed_values = min_value = max_value = None
+        else:
+            if ontology_path is not None:
+                if isinstance(ontology_path, (str, os.PathLike)):
+                    onto = Ontology(ontology_path)
+                else:
+                    raise RuntimeError("Wrong ontology_path")
+            else:
+                if isinstance(ontology_object, Ontology):
+                    onto = ontology_object
+                else:
+                    raise RuntimeError("Wrong ontology_object")
+
             if extra_ttl is not None:
                 onto.parse_extra_triples(extra_ttl)
             parameter_hierarchy = onto.get_parameter_hierarchy(owl_uri)
             par_format = onto.get_parameter_format(owl_uri)
             par_unit = onto.get_parameter_unit(owl_uri)
             min_value, max_value = onto.get_limits(owl_uri)
-            allowed_values = onto.get_allowed_values(owl_uri)            
-        else:
-            logger.warning('Ontology path not set in Parameter.from_owl_uri(). '
-                           'Trying to find parameter which have %s directly set. '
-                           'extra_ttl will be ignored ', owl_uri)
-            parameter_hierarchy = [ owl_uri ]
-            par_format = par_unit = allowed_values = min_value = max_value = None
+            allowed_values = onto.get_allowed_values(owl_uri)
 
         for owl_superclass_uri in parameter_hierarchy:
             for python_subclass in subclasses_recursive(cls):
@@ -546,7 +560,7 @@ def from_owl_uri(cls,
         logger.warning(('Unknown owl type uri %s or failed to construct any parameter. '
                         'Creating basic Parameter object.'), owl_uri)
         return cls(**kwargs)
-
+    
 class String(Parameter):
     owl_uris = ("http://www.w3.org/2001/XMLSchema#str", "http://odahub.io/ontology#String")
 

cdci_data_analysis/flask_app/app.py

@@ -207,8 +207,8 @@ def common_exception_payload():
 
     _l = []
 
-    for instrument_factory in importer.instrument_factory_list:
-        _l.append('%s' % instrument_factory().name)
+    for instrument_factory in importer.instrument_factory_iter:
+        _l.append(str(getattr(instrument_factory, 'instr_name', instrument_factory().name)))
 
     payload['installed_instruments'] = _l
 

cdci_data_analysis/flask_app/dispatcher_query.py

@@ -446,11 +446,18 @@ def get_user_specific_instrument_list(app):
                 email = tokenHelper.get_token_user_email_address(decoded_token)
 
         out_instrument_list = []
-        for instrument_factory in importer.instrument_factory_list:
-            instrument = instrument_factory()
+        for instrument_factory in importer.instrument_factory_iter:
+            if hasattr(instrument_factory, 'instrument_query'):
+                instrument_query = instrument_factory.instrument_query
+                instr_name = getattr(instrument_factory, 'instr_name', instrument_factory().name)
+            else:
+                instrument = instrument_factory()
+                instrument_query = instrument.instrumet_query
+                instr_name = instrument.name
+
 
-            if instrument.instrumet_query.check_instrument_access(roles, email):
-                out_instrument_list.append(instrument.name)
+            if instrument_query.check_instrument_access(roles, email):
+                out_instrument_list.append(instr_name)
 
         return jsonify(out_instrument_list)
 
@@ -1105,8 +1112,8 @@ def get_paramters_dict(self):
 
     def get_instr_list(self, name=None):
         _l = []
-        for instrument_factory in importer.instrument_factory_list:
-            _l.append(instrument_factory().name)
+        for instrument_factory in importer.instrument_factory_iter:
+            _l.append(getattr(instrument_factory, 'instr_name', instrument_factory().name))
 
         return jsonify(_l)
 
@@ -1490,15 +1497,30 @@ def set_instrument(self, instrument_name, roles, email):
         if instrument_name == 'mock':
             new_instrument = 'mock'
         else:
-            for instrument_factory in importer.instrument_factory_list:
-                instrument = instrument_factory()
-                if instrument.name == instrument_name:
-                    if instrument.instrumet_query.check_instrument_access(roles, email):
-                        new_instrument = instrument  # multiple assignment? TODO
+            for instrument_factory in importer.instrument_factory_iter:
+                _instrument = None
+                if hasattr(instrument_factory, 'instr_name'):
+                    instr_name = instrument_factory.instr_name
+                else:
+                    _instrument = instrument_factory()
+                    instr_name = _instrument.name
+
+                if instr_name == instrument_name:
+                    if _instrument is None and hasattr(instrument_factory, 'instrument_query'):
+                        instr_query = instrument_factory.instrument_query
+                    else:
+                        if _instrument is None:
+                            _instrument = instrument_factory()
+                        instr_query = _instrument.instrumet_query
+
+                    if instr_query.check_instrument_access(roles, email):
+                        if _instrument is None:
+                            _instrument = instrument_factory()
+                        new_instrument = _instrument  # multiple assignment? TODO
                     else:
                         no_access = True
 
-                known_instruments.append(instrument.name)
+                known_instruments.append(instr_name)
         if new_instrument is None:
             if no_access:
                 raise RequestNotAuthorized(f"Unfortunately, your priviledges are not sufficient "

cdci_data_analysis/plugins/importer.py

@@ -41,6 +41,7 @@
 logger = logging.getLogger(__name__)
 import sys
 from importlib import reload
+from cdci_data_analysis.analysis.instrument import InstrumentFactoryIterator
 
 
 #plugin_list=['cdci_osa_plugin','cdci_polar_plugin']
@@ -56,30 +57,30 @@
 if os.environ.get('DISPATCHER_DEBUG_MODE', 'no') == 'yes':
     cdci_plugins_dict['dummy_plugin'] = importlib.import_module('.dummy_plugin', 'cdci_data_analysis.plugins')
 
-def build_instrument_factory_list():
+def build_instrument_factory_iter():
     activate_plugins = os.environ.get('DISPATCHER_PLUGINS', 'auto')
-    instr_factory_list = []
+    instr_factory_iter = InstrumentFactoryIterator()
 
     for plugin_name in cdci_plugins_dict:
         if activate_plugins == 'auto' or plugin_name in activate_plugins:   
             logger.info("found plugin: %s", plugin_name)
 
             try:
                 e = importlib.import_module('.exposer', cdci_plugins_dict[plugin_name].__name__)
-                instr_factory_list.extend(e.instr_factory_list)
+                instr_factory_iter.extend(e.instr_factory_list)
                 logger.info(render('{GREEN}imported plugin: %s{/}'), plugin_name)
 
             except Exception as e:
                 logger.error('failed to import %s: %s', plugin_name,e )
                 traceback.print_exc()
-    return instr_factory_list
+    return instr_factory_iter
 
-instrument_factory_list = build_instrument_factory_list()
+instrument_factory_iter = build_instrument_factory_iter()
 
 def reload_plugin(plugin_name):
-    global instrument_factory_list
+    global instrument_factory_iter
     if plugin_name not in cdci_plugins_dict.keys():
         raise ModuleNotFoundError(plugin_name)
     reload(cdci_plugins_dict[plugin_name])
     reload(sys.modules[cdci_plugins_dict[plugin_name].__name__+'.exposer'])
-    instrument_factory_list = build_instrument_factory_list()
+    instrument_factory_iter = build_instrument_factory_iter()