diff --git a/.gitignore b/.gitignore index e42e3b07..3693adf0 100644 --- a/.gitignore +++ b/.gitignore @@ -72,4 +72,5 @@ js9.fits test-dispatcher-conf-with-gallery.yaml tests/oda-ontology.ttl tests/request_files -tests/local_request_files \ No newline at end of file +tests/local_request_files +.lock_* \ No newline at end of file diff --git a/cdci_data_analysis/analysis/instrument.py b/cdci_data_analysis/analysis/instrument.py index a7e8d983..717f9c94 100644 --- a/cdci_data_analysis/analysis/instrument.py +++ b/cdci_data_analysis/analysis/instrument.py @@ -435,7 +435,7 @@ def run_query(self, product_type, job.set_done() if query_out.status_dictionary['status'] == 0: query_out.set_done(message='dry-run',job_status=job.status) - query_out.set_instrument_parameters(self.get_parameters_list_as_json(prod_name=product_type)) + query_out.set_instrument_parameters(self.get_parameters_list_jsonifiable(prod_name=product_type)) else: if query_out.status_dictionary['status'] == 0: query_out = QueryOutput() @@ -593,7 +593,7 @@ def show_parameters_list(self): _query.show_parameters_list() print("-------------") - def get_parameters_list_as_json(self, add_src_query=True, add_instr_query=True, prod_name=None): + def get_parameters_list_jsonifiable(self, add_src_query=True, add_instr_query=True, prod_name=None): l=[{'instrumet':self.name}] l.append({'prod_dict':self.query_dictionary}) @@ -608,7 +608,7 @@ def get_parameters_list_as_json(self, add_src_query=True, add_instr_query=True, if isinstance(_query, ProductQuery) and prod_name is not None and _query.name!=self.query_dictionary[prod_name]: continue - l.append(_query.get_parameters_list_as_json(prod_dict=self.query_dictionary)) + l.append(_query.get_parameters_list_jsonifiable(prod_dict=self.query_dictionary)) return l diff --git a/cdci_data_analysis/analysis/parameters.py b/cdci_data_analysis/analysis/parameters.py index ad84535f..c11d1e6b 100644 --- a/cdci_data_analysis/analysis/parameters.py +++ b/cdci_data_analysis/analysis/parameters.py @@ -191,7 +191,7 @@ class Parameter: TODO see through that this is implemented https://github.com/oda-hub/dispatcher-app/issues/339 """ def __init__(self, - value=None, + value, units=None, name: Union[str, None] = None, # TODO should we remove units/type/format knowledge from the Parameter class? @@ -208,8 +208,9 @@ def __init__(self, check_value=None, allowed_values=None, - min_value = None, - max_value = None, + min_value=None, + max_value=None, + is_optional=False, extra_metadata=None, **kwargs @@ -248,6 +249,7 @@ def __init__(self, # if not (name is None or type(name) in [str]): # raise RuntimeError(f"can not initialize parameter with name {name} and type {type(name)}") + self.is_optional = is_optional self._allowed_units = allowed_units self._allowed_values = allowed_values self._allowed_types = allowed_types @@ -317,6 +319,8 @@ def value(self, v): if v not in self._allowed_values: raise RequestNotUnderstood(f'Parameter {self.name} wrong value {v}: not in allowed {self._allowed_values}') else: + if not self.is_optional: + raise RequestNotUnderstood(f'Non-optional parameter {self.name} is set to None') self._value = None def set_par_internal_value(self, value): @@ -455,7 +459,7 @@ def check_value(self): def reprJSONifiable(self): # produces json-serialisable list reprjson = [dict(name=self.name, units=self.units, value=self.value)] - restrictions = {} + restrictions = {'is_optional': self.is_optional} if self._allowed_values is not None: restrictions['allowed_values'] = self._allowed_values if getattr(self, '_min_value', None) is not None: @@ -578,7 +582,7 @@ def from_owl_uri(cls, class String(Parameter): owl_uris = ("http://www.w3.org/2001/XMLSchema#str", "http://odahub.io/ontology#String") - def __init__(self, value=None, name_format='str', name=None, allowed_values = None, extra_metadata = None): + def __init__(self, value, name_format='str', name=None, allowed_values = None, is_optional=False, extra_metadata = None): _allowed_units = ['str'] super().__init__(value=value, @@ -587,6 +591,7 @@ def __init__(self, value=None, name_format='str', name=None, allowed_values = No name=name, allowed_units=_allowed_units, allowed_values=allowed_values, + is_optional=is_optional, extra_metadata=extra_metadata) @staticmethod @@ -616,7 +621,7 @@ def __init__(self, *args, **kwargs): kwargs['allowed_types'] = [int, float] if kwargs.get('default_type') is None: - val = kwargs['value'] if kwargs.get('value') is not None else args[0] + val = kwargs.get('value', args[0]) if type(val) in kwargs['allowed_types']: kwargs['default_type'] = type(val) else: @@ -647,6 +652,8 @@ def set_par_internal_value(self, value): self._quantity = None def get_value_in_units(self, units): + if self.value is None: + return None if units is None: return self.value if self._quantity is None: @@ -676,7 +683,7 @@ def check_bounds(self): class Float(NumericParameter): owl_uris = ("http://www.w3.org/2001/XMLSchema#float", "http://odahub.io/ontology#Float") def __init__(self, - value=None, + value, units=None, name=None, allowed_units=None, @@ -684,7 +691,8 @@ def __init__(self, check_value=None, min_value= None, max_value = None, - units_name = None, + units_name = None, + is_optional=False, extra_metadata=None): super().__init__(value=value, @@ -699,6 +707,7 @@ def __init__(self, min_value=min_value, max_value=max_value, units_name = units_name, + is_optional=is_optional, extra_metadata=extra_metadata) @@ -707,7 +716,7 @@ class Integer(NumericParameter): owl_uris = ("http://www.w3.org/2001/XMLSchema#int", "http://odahub.io/ontology#Integer") def __init__(self, - value=None, + value, units=None, name=None, check_value=None, @@ -716,6 +725,7 @@ def __init__(self, units_name = None, default_units = None, allowed_units = None, + is_optional=False, extra_metadata=None): super().__init__(value=value, @@ -729,6 +739,7 @@ def __init__(self, min_value = min_value, max_value = max_value, units_name = units_name, + is_optional=is_optional, extra_metadata=extra_metadata) def set_par_internal_value(self, value): @@ -742,19 +753,29 @@ class Time(Parameter): owl_uris = ("http://odahub.io/ontology#TimeInstant",) format_kw = 'T_format' - def __init__(self, value=None, T_format='isot', name=None, Time_format_name='T_format', par_default_format='isot', extra_metadata=None): + def __init__(self, + value, + T_format='isot', + name=None, + Time_format_name='T_format', + par_default_format='isot', + is_optional=False, + extra_metadata=None): super().__init__(value=value, par_format=T_format, par_format_name=Time_format_name, par_default_format=par_default_format, name=name, + is_optional=is_optional, extra_metadata=extra_metadata) def get_default_value(self): return self.get_value_in_default_format() def get_value_in_format(self, par_format): + if self.value is None: + return None return getattr(self._astropy_time, par_format) @property @@ -779,6 +800,9 @@ def value(self, v): super(self.__class__, self.__class__).value.fset(self, v) def set_par_internal_value(self, value): + if value is None: + self._value = None + return try: self._astropy_time = astropyTime(value, format=self.par_format) except ValueError as e: @@ -794,7 +818,13 @@ class TimeDelta(Time): owl_uris = ("http://odahub.io/ontology#TimeDeltaIsDeprecated",) format_kw = 'delta_T_format' - def __init__(self, value=None, delta_T_format='sec', name=None, delta_T_format_name=None, par_default_format='sec', extra_metadata=None): + def __init__(self, + value, + delta_T_format='sec', + name=None, + delta_T_format_name=None, + par_default_format='sec', + extra_metadata=None): logging.warning(('TimeDelta parameter is deprecated. ' 'It derives from Time, which is confusing. ' 'Consider using TimeInterval parameter.')) @@ -818,6 +848,9 @@ def value(self, v): self._set_time(v, format=units) def _set_time(self, value, format): + if value is None: + self._value = None + return try: self._astropy_time_delta = astropyTimeDelta(value, format=format) except ValueError as e: @@ -829,13 +862,14 @@ class TimeInterval(Float): owl_uris = ("http://odahub.io/ontology#TimeInterval",) def __init__(self, - value=None, + value, units='s', name=None, default_units='s', min_value=None, max_value=None, units_name = None, + is_optional=False, extra_metadata=None): _allowed_units = ['s', 'minute', 'hour', 'day', 'year'] @@ -847,6 +881,7 @@ def __init__(self, max_value=max_value, units_name = units_name, allowed_units=_allowed_units, + is_optional=is_optional, extra_metadata=extra_metadata) class InputProdList(Parameter): @@ -864,6 +899,7 @@ def __init__(self, value=None, _format='names_list', name: str = None, extra_met check_value=self.check_list_value, name=name, allowed_units=_allowed_units, + is_optional=True, extra_metadata=extra_metadata) @staticmethod @@ -919,13 +955,14 @@ class Angle(Float): owl_uris = ("http://odahub.io/ontology#Angle") def __init__(self, - value=None, + value, units=None, default_units='deg', name=None, min_value = None, max_value = None, units_name = None, + is_optional=False, extra_metadata=None): super().__init__(value=value, @@ -937,6 +974,7 @@ def __init__(self, min_value = min_value, max_value = max_value, units_name = units_name, + is_optional=is_optional, extra_metadata=extra_metadata) @@ -945,14 +983,15 @@ class Energy(Float): units_kw = 'E_units' def __init__(self, - value=None, + value, E_units='keV', default_units='keV', name=None, check_value=None, - min_value = None, - max_value = None, - units_name = None, + min_value=None, + max_value=None, + units_name=None, + is_optional=False, extra_metadata=None): _allowed_units = ['keV', 'eV', 'MeV', 'GeV', 'TeV', 'Hz', 'MHz', 'GHz'] @@ -963,14 +1002,15 @@ def __init__(self, check_value=check_value, name=name, allowed_units=_allowed_units, - min_value = min_value, - max_value = max_value, - units_name = units_name, + min_value=min_value, + max_value=max_value, + units_name=units_name, + is_optional=is_optional, extra_metadata=extra_metadata) class SpectralBoundary(Energy): def __init__(self, - value=None, + value, E_units='keV', default_units='keV', name=None, @@ -978,6 +1018,7 @@ def __init__(self, min_value=None, max_value=None, units_name=None, + is_optional=False, extra_metadata=None): # retro-compatibility with integral plugin @@ -992,6 +1033,7 @@ def __init__(self, min_value=min_value, max_value=max_value, units_name=units_name, + is_optional=is_optional, extra_metadata=extra_metadata) @staticmethod @@ -1001,7 +1043,14 @@ def check_energy_value(value, units, name): class DetectionThreshold(Float): owl_uris = ("http://odahub.io/ontology#DetectionThreshold",) - def __init__(self, value=None, units='sigma', name=None, min_value = None, max_value = None, extra_metadata=None): + def __init__(self, + value, + units='sigma', + name=None, + min_value=None, + max_value=None, + is_optional=False, + extra_metadata=None): _allowed_units = ['sigma'] super().__init__(value=value, @@ -1009,8 +1058,9 @@ def __init__(self, value=None, units='sigma', name=None, min_value = None, max_v check_value=None, name=name, allowed_units=_allowed_units, - min_value = min_value, - max_value = max_value, + min_value=min_value, + max_value=max_value, + is_optional=is_optional, extra_metadata=extra_metadata) # 'sigma' is not astropy unit, so need to override methods @@ -1031,18 +1081,20 @@ def __init__(self, value=None, name_format='str', name=None, extra_metadata=None check_value=None, name=name, allowed_units=_allowed_units, + is_optional=True, extra_metadata=extra_metadata) class Boolean(Parameter): owl_uris = ('http://www.w3.org/2001/XMLSchema#bool',"http://odahub.io/ontology#Boolean") - def __init__(self, value=None, name=None, extra_metadata=None): + def __init__(self, value, name=None, is_optional=False, extra_metadata=None): self._true_rep = ['True', 'true', 'yes', '1', True] self._false_rep = ['False', 'false', 'no', '0', False] super().__init__(value=value, name=name, allowed_values=self._true_rep+self._false_rep, + is_optional=is_optional, extra_metadata = extra_metadata ) @@ -1062,7 +1114,12 @@ def value(self, v): class StructuredParameter(Parameter): owl_uris = ("http://odahub.io/ontology#StructuredParameter") - def __init__(self, value=None, name=None, schema={"oneOf": [{"type": "object"}, {"type": "array"}]}, extra_metadata=None): + def __init__(self, + value, + name=None, + schema={"oneOf": [{"type": "object"}, {"type": "array"}]}, + is_optional=False, + extra_metadata=None): self.schema = schema @@ -1071,6 +1128,7 @@ def __init__(self, value=None, name=None, schema={"oneOf": [{"type": "object"}, super().__init__(value=value, name=name, + is_optional=is_optional, extra_metadata=extra_metadata) def check_schema(self): @@ -1091,13 +1149,15 @@ def check_value(self): raise RuntimeError(f"Wrong schema for parameter {self.name}: {self.schema}") def get_default_value(self): + if self.value is None: + return None return json.dumps(self.value, sort_keys=True) class PhosphorosFiltersTable(StructuredParameter): owl_uris = ('http://odahub.io/ontology#PhosphorosFiltersTable') - def __init__(self, value=None, name=None, extra_metadata=None): + def __init__(self, value, name=None, extra_metadata=None): # TODO: either list or the whole schema may be loaded from the external file, purely based on URI. # If there is no additional check, this would allow to avoid even having the class. @@ -1134,7 +1194,7 @@ def __init__(self, value=None, name=None, extra_metadata=None): "required": ["filter", "flux", "flux_error"] } - super().__init__(value=value, name=name, schema=schema, extra_metadata=extra_metadata) + super().__init__(value=value, name=name, schema=schema, is_optional=False, extra_metadata=extra_metadata) def additional_check(self): assert len(self._value['filter']) == len(self._value['flux']) == len(self._value['flux_error']) diff --git a/cdci_data_analysis/analysis/queries.py b/cdci_data_analysis/analysis/queries.py index 507596ac..58f27af5 100644 --- a/cdci_data_analysis/analysis/queries.py +++ b/cdci_data_analysis/analysis/queries.py @@ -275,13 +275,13 @@ def print_form_dictionary_list(self,l): else: return l - def get_parameters_list_as_json(self,**kwargs): + def get_parameters_list_jsonifiable(self,**kwargs): l=[ {'query_name':self.name}] for par in self._parameters_list: l.extend(par.reprJSONifiable()) l1 = self._remove_duplicates_from_par_list(l) - return json.dumps(l1) + return l1 # Check if the given query cn be executed given a list of roles extracted from the token def check_query_roles(self, roles, par_dic): @@ -314,7 +314,7 @@ def __init__(self, name): t_range = ParameterRange(t1, t2, 'time') - token = Name(name_format='str', name='token', value=None) + token = Name(name_format='str', name='token', value=None, is_optional=True) #time_group = ParameterGroup([t_range_iso, t_range_mjd], 'time_range', selected='t_range_iso') #time_group_selector = time_group.build_selector('time_group_selector') @@ -385,7 +385,7 @@ def get_data_server_query(self,instrument,config=None,**kwargs): traceback.print_stack() raise RuntimeError(f'{self}: get_data_server_query needs to be implemented in derived class') - def get_parameters_list_as_json(self, prod_dict=None): + def get_parameters_list_jsonifiable(self, prod_dict=None): l=[ {'query_name':self.name}] prod_name=None @@ -402,7 +402,13 @@ def get_parameters_list_as_json(self, prod_dict=None): l.extend(par.reprJSONifiable()) l1 = self._remove_duplicates_from_par_list(l) - return json.dumps(l1) + return l1 + + def get_parameters_list_as_json(self, prod_dict=None): + logger.warning("Method name 'get_parameters_list_as_json' is deptrecated, " + "please use 'get_parameters_list_jsonifiable'") + self.get_parameters_list_jsonifiable(prod_dict=prod_dict) + def get_prod_by_name(self,name): return self.query_prod_list.get_prod_by_name(name) @@ -860,8 +866,8 @@ def __init__(self,name,parameters_list=[],**kwargs): else: parameters_list = [detection_th] - image_scale_min=Float(value=None,name='image_scale_min') - image_scale_max = Float(value=None, name='image_scale_max') + image_scale_min=Float(value=None,name='image_scale_min',is_optional=True) + image_scale_max = Float(value=None, name='image_scale_max',is_optional=True) parameters_list.extend([image_scale_min, image_scale_max]) super(ImageQuery, self).__init__(name, parameters_list, **kwargs) diff --git a/cdci_data_analysis/flask_app/dispatcher_query.py b/cdci_data_analysis/flask_app/dispatcher_query.py index 11baccce..fac51539 100644 --- a/cdci_data_analysis/flask_app/dispatcher_query.py +++ b/cdci_data_analysis/flask_app/dispatcher_query.py @@ -885,6 +885,9 @@ def set_args(self, request, verbose=False, download_products=False, download_fil if k in ['catalog_selected_objects', 'selected_catalog']: self.par_dic[k] = v continue + if v == '\x00': + self.par_dic[k] = None + continue try: decoded = json.loads(v) if isinstance(decoded, (dict, list)): @@ -1198,17 +1201,17 @@ def get_meta_data(self, meta_name=None): else: prod_name = None if hasattr(self, 'instrument'): - l.append(self.instrument.get_parameters_list_as_json(prod_name=prod_name)) + l.append(self.instrument.get_parameters_list_jsonifiable(prod_name=prod_name)) src_query.show_parameters_list() else: l = ['instrument not recognized'] if meta_name == 'src_query': - l = [src_query.get_parameters_list_as_json()] + l = [src_query.get_parameters_list_jsonifiable()] src_query.show_parameters_list() if meta_name == 'instrument': - l = [self.instrument.get_parameters_list_as_json()] + l = [self.instrument.get_parameters_list_jsonifiable()] self.instrument.show_parameters_list() return jsonify(l) diff --git a/cdci_data_analysis/plugins/dummy_plugin/data_server_dispatcher.py b/cdci_data_analysis/plugins/dummy_plugin/data_server_dispatcher.py index cce154b8..3da610bb 100644 --- a/cdci_data_analysis/plugins/dummy_plugin/data_server_dispatcher.py +++ b/cdci_data_analysis/plugins/dummy_plugin/data_server_dispatcher.py @@ -422,7 +422,7 @@ def get_dummy_products(self, instrument, config=None, **kwargs): # create dummy NumpyDataProduct meta_data = {'product': 'mosaic', 'instrument': 'empty', 'src_name': '', - 'query_parameters': self.get_parameters_list_as_json()} + 'query_parameters': self.get_parameters_list_jsonifiable()} ima = NumpyDataUnit(np.zeros((100, 100)), hdu_type='image') data = NumpyDataProduct(data_unit=ima) @@ -554,7 +554,7 @@ def test_has_products(self, instrument, job=None, query_type='Real', logger=None query_out.set_done() return query_out -class StructuredEchoProductQuery(EchoProductQuery): +class DefaultEchoProductQuery(EchoProductQuery): def get_data_server_query(self, instrument, config=None, **kwargs): param_names = instrument.get_parameters_name_list() # and here it's passed as in nb2w plugin diff --git a/cdci_data_analysis/plugins/dummy_plugin/empty_instrument.py b/cdci_data_analysis/plugins/dummy_plugin/empty_instrument.py index fba7e758..cd78d081 100644 --- a/cdci_data_analysis/plugins/dummy_plugin/empty_instrument.py +++ b/cdci_data_analysis/plugins/dummy_plugin/empty_instrument.py @@ -42,7 +42,7 @@ FailingProductQuery, DataServerParametricQuery, EchoProductQuery, - StructuredEchoProductQuery, + DefaultEchoProductQuery, FileParameterQuery) # duplicated with jemx, but this staticmethod makes it complex. @@ -82,7 +82,7 @@ def my_instr_factory(): numerical_query = DataServerNumericQuery('numerical_parameters_dummy_query', parameters_list=[p]) - f = FileURL(name='dummy_file') + f = FileURL(value=None, name='dummy_file', is_optional=True) file_query = FileParameterQuery('file_parameters_dummy_query', parameters_list=[p, f]) @@ -106,7 +106,12 @@ def my_instr_factory(): string_select_param]) struct_par = StructuredParameter({'a': [1, 2]}, name='struct') - structured_echo_query = StructuredEchoProductQuery('structured_param_dummy_query', parameters_list=[struct_par]) + structured_echo_query = DefaultEchoProductQuery('structured_param_dummy_query', parameters_list=[struct_par]) + + optional_par0 = Float(5., name = 'optional_def_float', is_optional=True) + optional_par1 = Float(None, name = 'optional_def_none', is_optional=True) + optional_echo_query = DefaultEchoProductQuery('optional_param_dummy_query', parameters_list=[optional_par0, optional_par1]) + # this dicts binds the product query name to the product name from frontend # eg my_instr_image is the parameter passed by the fronted to access the # the MyInstrMosaicQuery, and the dictionary will bind @@ -122,6 +127,7 @@ def my_instr_factory(): query_dictionary['echo'] = 'echo_parameters_dummy_query' query_dictionary['restricted'] = 'restricted_parameters_dummy_query' query_dictionary['structured'] = 'structured_param_dummy_query' + query_dictionary['optional'] = 'optional_param_dummy_query' return Instrument('empty', src_query=src_query, @@ -133,5 +139,6 @@ def my_instr_factory(): parametrical_query, echo_param_query, restricted_param_query, - structured_echo_query], + structured_echo_query, + optional_echo_query], query_dictionary=query_dictionary) diff --git a/pytest.ini b/pytest.ini index 0ef8c6ff..c846afdc 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,6 @@ # content of pytest.ini [pytest] +testpaths = tests markers = dda: mark a test as dependent on dda, integral backend isgri_plugin: mark a test as one using the isgri instrument diff --git a/setup.py b/setup.py index de2adfba..a28f3183 100644 --- a/setup.py +++ b/setup.py @@ -53,6 +53,8 @@ ] test_req = [ + 'pytest', + 'pytest-sentry', 'psutil', ] diff --git a/tests/test_parameters.py b/tests/test_parameters.py index 779e5beb..93d8e021 100644 --- a/tests/test_parameters.py +++ b/tests/test_parameters.py @@ -276,7 +276,7 @@ def test_parameter_normalization_no_units(): ]: def constructor(): - return parameter_type(value=input_value, name="my-parameter-name") + return parameter_type(value=input_value, name="my-parameter-name", is_optional=input_value is None) if isinstance(outcome, type) and issubclass(outcome, Exception): with pytest.raises(outcome): @@ -360,24 +360,26 @@ def test_parameter_meta_data(): bool_parameter = Boolean(value = True, name = 'bool') assert bounded_parameter.reprJSONifiable() == [{'name': 'bounded', 'units': None, 'value': 1.0, - 'restrictions': {'min_value': 0.1, 'max_value': 2.0}, + 'restrictions': {'is_optional': False, 'min_value': 0.1, 'max_value': 2.0}, 'owl_uri': ["http://www.w3.org/2001/XMLSchema#float", "http://odahub.io/ontology#Float"]}] assert choice_parameter.reprJSONifiable() == [{'name': 'choice', 'units': 'str', 'value': 'spam', - 'restrictions': {'allowed_values': ['spam', 'eggs', 'hams']}, + 'restrictions': {'is_optional': False, 'allowed_values': ['spam', 'eggs', 'hams']}, 'owl_uri': ["http://www.w3.org/2001/XMLSchema#str", "http://odahub.io/ontology#String"]}] assert long_choice_parameter.reprJSONifiable() == [{'name': 'choice', 'units': 'str', 'value': 'long_spam', + 'restrictions': {'is_optional': False}, 'owl_uri': ["http://www.w3.org/2001/XMLSchema#str", "http://odahub.io/ontology#String", "http://odahub.io/ontology#LongString"]}] assert bool_parameter.reprJSONifiable() == [{'name': 'bool', 'units': None, 'value': True, - 'restrictions': {'allowed_values': ['True', 'true', 'yes', '1', True, - 'False', 'false', 'no', '0', False]}, + 'restrictions': {'is_optional': False, + 'allowed_values': ['True', 'true', 'yes', '1', True, + 'False', 'false', 'no', '0', False]}, 'owl_uri': ["http://www.w3.org/2001/XMLSchema#bool","http://odahub.io/ontology#Boolean"]}] @pytest.mark.fast @@ -673,11 +675,23 @@ def test_structured_get_default_value(): (25.0, float, 25.0), ('25.2', float, 25.2), (25.2, float, 25.2)]) -def test_numeric_nonetype(value, expected_type, expected_value): +def test_numeric_fixed_nonetype_exception(value, expected_type, expected_value): np = NumericParameter(value) assert np.value == expected_value assert type(np.value) == expected_type +@pytest.mark.fast def test_numeric_wrong(): with pytest.raises(RequestNotUnderstood): - NumericParameter("ImNotaNumber") \ No newline at end of file + NumericParameter("ImNotaNumber") + +@pytest.mark.fast +@pytest.mark.parametrize('is_optional', [True, False]) +@pytest.mark.parametrize('value', [5.0, None]) +def test_optional_parameters(is_optional, value): + if value is None and not is_optional: + with pytest.raises(RequestNotUnderstood): + Float(value, is_optional=is_optional) + else: + op = Float(value, is_optional=is_optional) + assert op.value == value diff --git a/tests/test_server_basic.py b/tests/test_server_basic.py index 6f78ab6e..27e59a61 100644 --- a/tests/test_server_basic.py +++ b/tests/test_server_basic.py @@ -119,10 +119,10 @@ def test_reload_plugin(safe_dummy_plugin_conf, dispatcher_live_fixture): c = requests.get(server + "/api/instr-list", params={'instrument': 'mock'}) logger.info("content: %s", c.text) + assert c.status_code == 200 jdata = c.json() logger.info(json.dumps(jdata, indent=4, sort_keys=True)) logger.info(jdata) - assert c.status_code == 200 assert 'empty' in jdata assert 'empty-async' in jdata assert 'empty-semi-async' in jdata @@ -4582,13 +4582,17 @@ def test_parameter_bounds_metadata(dispatcher_live_fixture): jdata=c.json() metadata = [json.loads(x) for x in jdata[0] if isinstance(x, str)] - restricted_meta = [x for x in metadata if x[0]['query_name'] == 'restricted_parameters_dummy_query'][0] + if len(metadata) == 0: + # new behaviour, metadata is not string-encoded in the request + metadata = jdata[0] + restricted_meta = [x for x in metadata if isinstance(x, list) and x[0]['query_name'] == 'restricted_parameters_dummy_query'][0] + def meta_for_par(parname): return [x for x in restricted_meta if x.get('name', None) == parname][0] - assert meta_for_par('bounded_int_par')['restrictions'] == {'min_value': 2, 'max_value': 8} - assert meta_for_par('bounded_float_par')['restrictions'] == {'min_value': 2.2, 'max_value': 7.7} - assert meta_for_par('string_select_par')['restrictions'] == {'allowed_values': ['spam', 'eggs', 'ham']} + assert meta_for_par('bounded_int_par')['restrictions'] == {'is_optional': False, 'min_value': 2, 'max_value': 8} + assert meta_for_par('bounded_float_par')['restrictions'] == {'is_optional': False, 'min_value': 2.2, 'max_value': 7.7} + assert meta_for_par('string_select_par')['restrictions'] == {'is_optional': False, 'allowed_values': ['spam', 'eggs', 'ham']} @pytest.mark.fast def test_restricted_parameters_good_request(dispatcher_live_fixture): @@ -4681,4 +4685,73 @@ def test_malformed_structured_parameter(dispatcher_live_fixture): assert c.status_code == 400 print("content:", c.text) jdata=c.json() - assert 'Wrong value of structured parameter struct' in jdata['error_message'] \ No newline at end of file + assert 'Wrong value of structured parameter struct' in jdata['error_message'] + + +@pytest.mark.fast +@pytest.mark.parametrize('par0', [None, 2.0, '\x00']) +@pytest.mark.parametrize('par1', [None, 3.0, '\x00']) +def test_optional_parameters(dispatcher_live_fixture, par0, par1): + # NOTE: when request argument is None, it's just ignored by requests + # to set optional parameter to be empty (internally None), we use '\x00' in request + + server = dispatcher_live_fixture + print("constructed server:", server) + + c = requests.get(server + '/meta-data', + params = {'instrument': 'empty'}) + assert c.status_code == 200 + + optional_query_meta = [q for q in c.json()[0][4:] if q[1]['product_name'] == "optional"][0] + opt_pars = optional_query_meta[2:] + + for par in opt_pars: + assert par['restrictions']['is_optional'] + + defaults = {p['name']: p['value'] for p in opt_pars} + expected = defaults.copy() + if par0 is not None: + expected[opt_pars[0]['name']] = None if par0=='\x00' else par0 + if par1 is not None: + expected[opt_pars[1]['name']] = None if par1=='\x00' else par1 + + req = {'instrument': 'empty', + 'product_type': 'optional', + 'query_status': 'new', + 'query_type': 'Real', + opt_pars[0]['name']: par0, + opt_pars[1]['name']: par1, + } + c = requests.get(server + '/run_analysis', + params = req) + + assert c.status_code == 200 + print("content:", c.text) + jdata=c.json() + + assert jdata['exit_status']['status'] == 0 + assert jdata['exit_status']['job_status'] == 'done' + for k in expected.keys(): + assert jdata['products']['echo'][k] == expected[k] + + +@pytest.mark.fast +def test_nonoptional_parameter_is_not_nullable(dispatcher_live_fixture): + + server = dispatcher_live_fixture + print("constructed server:", server) + + req = {'instrument': 'empty', + 'product_type': 'numerical', + 'query_status': 'new', + 'query_type': 'Real', + 'p': '\x00' + } + c = requests.get(server + '/run_analysis', + params = req) + + assert c.status_code == 400 + print("content:", c.text) + jdata=c.json() + + assert jdata['error_message'] == 'Non-optional parameter p is set to None'