From 027973b5a832f486098622d5bcfca210d0039b79 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 15 Nov 2024 20:03:19 +0100 Subject: [PATCH 01/12] add token only if not present --- nb2workflow/nbadapter.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index f491e03..794fb5b 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -840,11 +840,12 @@ def handle_url_params(self, parameters, tmpdir, context={}): logger.debug(f'adding token to the url: {arg_par_value}') url_parts = urlparse(adapted_parameters[input_par_name]) url_args = parse_qs(url_parts.query) - url_args['token'] = [token] # the values in the dictionary need to be lists - new_url_parts = url_parts._replace(query=urlencode(url_args, doseq=True)) - adapted_parameters[input_par_name] = urlunparse(new_url_parts) - logger.debug(f"updated url: {adapted_parameters[input_par_name]}") - arg_par_value = adapted_parameters[input_par_name] + if token not in url_args: + url_args['token'] = [token] # the values in the dictionary need to be lists + new_url_parts = url_parts._replace(query=urlencode(url_args, doseq=True)) + adapted_parameters[input_par_name] = urlunparse(new_url_parts) + logger.debug(f"updated url: {adapted_parameters[input_par_name]}") + arg_par_value = adapted_parameters[input_par_name] logger.debug(f'download {arg_par_value}') try: From 1cae628f3f74eaa96c798e50a1bb63348a087796 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 19 Nov 2024 13:53:26 +0100 Subject: [PATCH 02/12] adding token also to fileURL and FileReference that are mmoda URL --- nb2workflow/nbadapter.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index 794fb5b..4b48412 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -820,14 +820,22 @@ def handle_url_params(self, parameters, tmpdir, context={}): adapted_parameters = copy.deepcopy(parameters) exceptions = [] posix_path_with_annotations_pattern = re.compile(rf"^{re.escape(oda_prefix)}.*_POSIXPath_") + file_reference_with_annotations_pattern = re.compile(rf"^{re.escape(oda_prefix)}.*_FileReference_") + file_url_with_annotations_pattern = re.compile(rf"^{re.escape(oda_prefix)}.*_FileURL_") for input_par_name, input_par_obj in self.input_parameters.items(): if ontology.is_ontology_available: parameter_hierarchy = ontology.get_parameter_hierarchy(input_par_obj['owl_type']) is_posix_path = f"{oda_prefix}POSIXPath" in parameter_hierarchy + is_file_reference = f"{oda_prefix}FileReference" in parameter_hierarchy + is_file_url = f"{oda_prefix}FileURL" in parameter_hierarchy else: is_posix_path = f"{oda_prefix}POSIXPath" == input_par_obj['owl_type'] or \ posix_path_with_annotations_pattern.match(input_par_obj['owl_type']) is not None - if is_posix_path: + is_file_reference = f"{oda_prefix}FileReference" == input_par_obj['owl_type'] or \ + file_reference_with_annotations_pattern.match(input_par_obj['owl_type']) is not None + is_file_url = f"{oda_prefix}FileURL" == input_par_obj['owl_type'] or \ + file_url_with_annotations_pattern.match(input_par_obj['owl_type']) is not None + if is_posix_path or is_file_reference or is_file_url: arg_par_value = parameters.get(input_par_name, None) if arg_par_value is None: arg_par_value = input_par_obj['default_value'] @@ -849,8 +857,9 @@ def handle_url_params(self, parameters, tmpdir, context={}): logger.debug(f'download {arg_par_value}') try: - file_name = self.download_file(arg_par_value, tmpdir) - adapted_parameters[input_par_name] = file_name + if is_posix_path: + file_name = self.download_file(arg_par_value, tmpdir) + adapted_parameters[input_par_name] = file_name except Exception as e: exceptions.append(e) From 90e2bfdd8fbf1df1ddf9d127129529939f13294e Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 19 Nov 2024 15:47:17 +0100 Subject: [PATCH 03/12] only checking file url and using the default value in case no parameter value was provided --- nb2workflow/nbadapter.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index 4b48412..6f6cf10 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -820,22 +820,18 @@ def handle_url_params(self, parameters, tmpdir, context={}): adapted_parameters = copy.deepcopy(parameters) exceptions = [] posix_path_with_annotations_pattern = re.compile(rf"^{re.escape(oda_prefix)}.*_POSIXPath_") - file_reference_with_annotations_pattern = re.compile(rf"^{re.escape(oda_prefix)}.*_FileReference_") file_url_with_annotations_pattern = re.compile(rf"^{re.escape(oda_prefix)}.*_FileURL_") for input_par_name, input_par_obj in self.input_parameters.items(): if ontology.is_ontology_available: parameter_hierarchy = ontology.get_parameter_hierarchy(input_par_obj['owl_type']) is_posix_path = f"{oda_prefix}POSIXPath" in parameter_hierarchy - is_file_reference = f"{oda_prefix}FileReference" in parameter_hierarchy is_file_url = f"{oda_prefix}FileURL" in parameter_hierarchy else: is_posix_path = f"{oda_prefix}POSIXPath" == input_par_obj['owl_type'] or \ posix_path_with_annotations_pattern.match(input_par_obj['owl_type']) is not None - is_file_reference = f"{oda_prefix}FileReference" == input_par_obj['owl_type'] or \ - file_reference_with_annotations_pattern.match(input_par_obj['owl_type']) is not None is_file_url = f"{oda_prefix}FileURL" == input_par_obj['owl_type'] or \ file_url_with_annotations_pattern.match(input_par_obj['owl_type']) is not None - if is_posix_path or is_file_reference or is_file_url: + if is_posix_path or is_file_url: arg_par_value = parameters.get(input_par_name, None) if arg_par_value is None: arg_par_value = input_par_obj['default_value'] @@ -846,7 +842,8 @@ def handle_url_params(self, parameters, tmpdir, context={}): token = context.get('token', None) if token is not None: logger.debug(f'adding token to the url: {arg_par_value}') - url_parts = urlparse(adapted_parameters[input_par_name]) + url_to_adapt = adapted_parameters.get(input_par_name, arg_par_value) + url_parts = urlparse(url_to_adapt) url_args = parse_qs(url_parts.query) if token not in url_args: url_args['token'] = [token] # the values in the dictionary need to be lists From 62f8fb383551d34a24f934f2970d63c191f5b2e5 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 19 Nov 2024 15:49:42 +0100 Subject: [PATCH 04/12] test --- tests/test_input_types.py | 50 +++++++++++++- .../testfileurl_extra_annotated.ipynb | 67 +++++++++++++++++++ 2 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 tests/testfiles/testfileurl_extra_annotated.ipynb diff --git a/tests/test_input_types.py b/tests/test_input_types.py index c91d149..d457f05 100644 --- a/tests/test_input_types.py +++ b/tests/test_input_types.py @@ -7,7 +7,8 @@ logger = logging.getLogger(__name__) -from urllib.parse import urlencode +from urllib.parse import urlencode, urlparse, parse_qs + @pytest.fixture def app(): @@ -39,6 +40,53 @@ def test_posix_download_file_extra_annotations(client): r = client.get('/api/v1.0/get/testposixpath_extra_annotated', query_string={'fits_file_path': 'https://fits.gsfc.nasa.gov/samples/testkeys.fits'}) assert r.json['output']['output_file_download'] == 'file downloaded successfully' +def test_mmoda_file_url(client): + status_callback_file = "status.json" + callback_url = 'file://' + status_callback_file + token = 'abc123' + query_string = dict( + _async_request='no', + _async_request_callback=callback_url, + _token=token) + r = client.get('/api/v1.0/get/testfileurl_extra_annotated', query_string=query_string) + assert r.status_code == 201 + + from nb2workflow.service import AsyncWorker + + def test_worker_run(): + AsyncWorker('test-worker').run_one() + + test_worker_thread = threading.Thread(target=test_worker_run) + test_worker_thread.start() + + while True: + options = client.get('/api/v1.0/options') + assert options.status_code == 200 + + r = client.get("/api/v1.0/get/testfileurl_extra_annotated", + query_string=query_string) + + logger.info('service returns %s %s', r, r.json) + + if r.json['workflow_status'] == 'done': + logger.info('workflow done!') + break + + time.sleep(0.1) + + test_worker_thread.join() + assert 'data' in r.json + assert 'output' in r.json['data'] + assert 'mmoda_url_modified' in r.json['data']['output'] + url_parts = urlparse(r.json['data']['output']['mmoda_url_modified']) + url_args = parse_qs(url_parts.query) + assert 'token' in url_args + + assert 'fits_file_url_modified' in r.json['data']['output'] + url_parts = urlparse(r.json['data']['output']['fits_file_url_modified']) + url_args = parse_qs(url_parts.query) + assert 'token' not in url_args + def test_posix_download_file_with_arg_low_download_limit(client, app_low_download_limit): r = client.get('/api/v1.0/get/testposixpath', query_string={'fits_file_path': 'https://fits.gsfc.nasa.gov/samples/testkeys.fits'}) assert r.json['output'] == {} diff --git a/tests/testfiles/testfileurl_extra_annotated.ipynb b/tests/testfiles/testfileurl_extra_annotated.ipynb new file mode 100644 index 0000000..f7e75fd --- /dev/null +++ b/tests/testfiles/testfileurl_extra_annotated.ipynb @@ -0,0 +1,67 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "# oda:oda_token_access oda:InOdaContext .\n", + "fits_file_url = \"https://fits.gsfc.nasa.gov/samples/testkeys.fits\" # oda:FileURL ; oda:label \"Test url\" ; oda:description \"Description test url\" .\n", + "mmoda_url = \"https://www.astro.unige.ch/mmoda/dispatch-data/test.fits\" # oda:FileURL ; oda:label \"Test mmoda url\" ; oda:description \"Description mmoda url\" ." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fits_file_url_modified = fits_file_url\n", + "mmoda_url_modified = mmoda_url" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "outputs" + ] + }, + "outputs": [], + "source": [ + "fits_file_url_modified\n", + "mmoda_url_modified" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "767d51c1340bd893661ea55ea3124f6de3c7a262a8b4abca0554b478b1e2ff90" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.15" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From edda64e84bd641fe4ce7b1f523caacc64d9bf9c6 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 19 Nov 2024 17:09:08 +0100 Subject: [PATCH 05/12] adapted another test --- tests/test_nbadapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_nbadapter.py b/tests/test_nbadapter.py index 77ffaaf..cae1601 100644 --- a/tests/test_nbadapter.py +++ b/tests/test_nbadapter.py @@ -84,7 +84,7 @@ def test_find_notebooks(caplog): assert 'Ignoring pattern.' in caplog.text nbas = find_notebooks(nb_dir) - assert len(nbas) == 9 + assert len(nbas) == 10 nbas = find_notebooks(nb_dir, pattern=r'.*bool') assert len(nbas) == 1 From d9460092d913d65f323b1587b0514d73ef6f64ef Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 22 Nov 2024 13:45:13 +0100 Subject: [PATCH 06/12] reintroduced condition with FileReference --- nb2workflow/nbadapter.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index 6f6cf10..ff1fead 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -821,17 +821,21 @@ def handle_url_params(self, parameters, tmpdir, context={}): exceptions = [] posix_path_with_annotations_pattern = re.compile(rf"^{re.escape(oda_prefix)}.*_POSIXPath_") file_url_with_annotations_pattern = re.compile(rf"^{re.escape(oda_prefix)}.*_FileURL_") + file_reference_with_annotations_pattern = re.compile(rf"^{re.escape(oda_prefix)}.*_FileReference_") for input_par_name, input_par_obj in self.input_parameters.items(): if ontology.is_ontology_available: parameter_hierarchy = ontology.get_parameter_hierarchy(input_par_obj['owl_type']) is_posix_path = f"{oda_prefix}POSIXPath" in parameter_hierarchy is_file_url = f"{oda_prefix}FileURL" in parameter_hierarchy + is_file_reference = f"{oda_prefix}FileReference" in parameter_hierarchy else: is_posix_path = f"{oda_prefix}POSIXPath" == input_par_obj['owl_type'] or \ posix_path_with_annotations_pattern.match(input_par_obj['owl_type']) is not None is_file_url = f"{oda_prefix}FileURL" == input_par_obj['owl_type'] or \ file_url_with_annotations_pattern.match(input_par_obj['owl_type']) is not None - if is_posix_path or is_file_url: + is_file_reference = f"{oda_prefix}FileReference" == input_par_obj['owl_type'] or \ + file_reference_with_annotations_pattern.match(input_par_obj['owl_type']) is not None + if is_posix_path or is_file_url or is_file_reference: arg_par_value = parameters.get(input_par_name, None) if arg_par_value is None: arg_par_value = input_par_obj['default_value'] From e03a025ab757f63067ea5807bc0edac6e498f432 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 22 Nov 2024 16:50:21 +0100 Subject: [PATCH 07/12] extended test --- tests/test_input_types.py | 7 +- tests/test_nbadapter.py | 2 +- .../testfilereference_extra_annotated.ipynb | 69 +++++++++++++++++++ 3 files changed, 74 insertions(+), 4 deletions(-) create mode 100644 tests/testfiles/testfilereference_extra_annotated.ipynb diff --git a/tests/test_input_types.py b/tests/test_input_types.py index d457f05..ad38cd4 100644 --- a/tests/test_input_types.py +++ b/tests/test_input_types.py @@ -40,7 +40,8 @@ def test_posix_download_file_extra_annotations(client): r = client.get('/api/v1.0/get/testposixpath_extra_annotated', query_string={'fits_file_path': 'https://fits.gsfc.nasa.gov/samples/testkeys.fits'}) assert r.json['output']['output_file_download'] == 'file downloaded successfully' -def test_mmoda_file_url(client): +@pytest.mark.parametrize("workflow", ["testfileurl_extra_annotated", "testfilereference_extra_annotated"]) +def test_mmoda_file_url(client, workflow): status_callback_file = "status.json" callback_url = 'file://' + status_callback_file token = 'abc123' @@ -48,7 +49,7 @@ def test_mmoda_file_url(client): _async_request='no', _async_request_callback=callback_url, _token=token) - r = client.get('/api/v1.0/get/testfileurl_extra_annotated', query_string=query_string) + r = client.get(f'/api/v1.0/get/{workflow}', query_string=query_string) assert r.status_code == 201 from nb2workflow.service import AsyncWorker @@ -63,7 +64,7 @@ def test_worker_run(): options = client.get('/api/v1.0/options') assert options.status_code == 200 - r = client.get("/api/v1.0/get/testfileurl_extra_annotated", + r = client.get(f'/api/v1.0/get/{workflow}', query_string=query_string) logger.info('service returns %s %s', r, r.json) diff --git a/tests/test_nbadapter.py b/tests/test_nbadapter.py index cae1601..23d352a 100644 --- a/tests/test_nbadapter.py +++ b/tests/test_nbadapter.py @@ -84,7 +84,7 @@ def test_find_notebooks(caplog): assert 'Ignoring pattern.' in caplog.text nbas = find_notebooks(nb_dir) - assert len(nbas) == 10 + assert len(nbas) == 11 nbas = find_notebooks(nb_dir, pattern=r'.*bool') assert len(nbas) == 1 diff --git a/tests/testfiles/testfilereference_extra_annotated.ipynb b/tests/testfiles/testfilereference_extra_annotated.ipynb new file mode 100644 index 0000000..82dbdf1 --- /dev/null +++ b/tests/testfiles/testfilereference_extra_annotated.ipynb @@ -0,0 +1,69 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "fits_file_path = \"/home/local/test.fits\" # oda:FileReference ; oda:label \"Test file path\" ; oda:description \"Description test file path\" .\n", + "fits_file_url = \"https://fits.gsfc.nasa.gov/samples/testkeys.fits\" # oda:FileReference ; oda:label \"Test file path\" ; oda:description \"Description test file path\" .\n", + "mmoda_url = \"https://www.astro.unige.ch/mmoda/dispatch-data/test.fits\" # oda:FileReference ; oda:label \"Test file path\" ; oda:description \"Description test file path\" ." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "fits_file_url_modified = fits_file_url\n", + "mmoda_url_modified = mmoda_url\n", + "fits_file_path_modified = fits_file_path" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "tags": [ + "outputs" + ] + }, + "outputs": [], + "source": [ + "fits_file_url_modified\n", + "mmoda_url_modified\n", + "fits_file_path_modified" + ] + } + ], + "metadata": { + "interpreter": { + "hash": "767d51c1340bd893661ea55ea3124f6de3c7a262a8b4abca0554b478b1e2ff90" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.15" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From e076a4f02d87c510e3670ec3edc75c753c10627b Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 22 Nov 2024 19:30:38 +0100 Subject: [PATCH 08/12] better test --- tests/test_input_types.py | 64 +++++++++++++++++-- .../testfilereference_extra_annotated.ipynb | 8 +-- 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/tests/test_input_types.py b/tests/test_input_types.py index ad38cd4..918a358 100644 --- a/tests/test_input_types.py +++ b/tests/test_input_types.py @@ -40,8 +40,8 @@ def test_posix_download_file_extra_annotations(client): r = client.get('/api/v1.0/get/testposixpath_extra_annotated', query_string={'fits_file_path': 'https://fits.gsfc.nasa.gov/samples/testkeys.fits'}) assert r.json['output']['output_file_download'] == 'file downloaded successfully' -@pytest.mark.parametrize("workflow", ["testfileurl_extra_annotated", "testfilereference_extra_annotated"]) -def test_mmoda_file_url(client, workflow): +@pytest.mark.parametrize("query_string_fits_file_path", ["generic_url", "mmoda_url", None]) +def test_file_reference(client, query_string_fits_file_path): status_callback_file = "status.json" callback_url = 'file://' + status_callback_file token = 'abc123' @@ -49,7 +49,63 @@ def test_mmoda_file_url(client, workflow): _async_request='no', _async_request_callback=callback_url, _token=token) - r = client.get(f'/api/v1.0/get/{workflow}', query_string=query_string) + + if query_string_fits_file_path == "mmoda_url": + query_string['fits_file_path'] = "https://www.astro.unige.ch/mmoda/dispatch-data/test.fits" + elif query_string_fits_file_path == "generic_url": + query_string['fits_file_path'] = "https://fits.gsfc.nasa.gov/samples/testkeys.fits" + + r = client.get(f'/api/v1.0/get/testfilereference_extra_annotated', query_string=query_string) + assert r.status_code == 201 + + from nb2workflow.service import AsyncWorker + + def test_worker_run(): + AsyncWorker('test-worker').run_one() + + test_worker_thread = threading.Thread(target=test_worker_run) + test_worker_thread.start() + + while True: + options = client.get('/api/v1.0/options') + assert options.status_code == 200 + + r = client.get(f'/api/v1.0/get/testfilereference_extra_annotated', + query_string=query_string) + + logger.info('service returns %s %s', r, r.json) + + if r.json['workflow_status'] == 'done': + logger.info('workflow done!') + break + + time.sleep(0.1) + + test_worker_thread.join() + assert 'data' in r.json + assert 'output' in r.json['data'] + assert 'fits_file_path_modified' in r.json['data']['output'] + + if query_string_fits_file_path is None: + assert r.json['data']['output']['fits_file_path_modified'] == "/home/local/test.fits" + elif query_string_fits_file_path == "mmoda_url": + url_parts = urlparse(r.json['data']['output']['fits_file_path_modified']) + url_args = parse_qs(url_parts.query) + assert 'token' in url_args + elif query_string_fits_file_path == "generic_url": + url_parts = urlparse(r.json['data']['output']['fits_file_path_modified']) + url_args = parse_qs(url_parts.query) + assert 'token' not in url_args + +def test_mmoda_file_url(client): + status_callback_file = "status.json" + callback_url = 'file://' + status_callback_file + token = 'abc123' + query_string = dict( + _async_request='no', + _async_request_callback=callback_url, + _token=token) + r = client.get(f'/api/v1.0/get/testfileurl_extra_annotated', query_string=query_string) assert r.status_code == 201 from nb2workflow.service import AsyncWorker @@ -64,7 +120,7 @@ def test_worker_run(): options = client.get('/api/v1.0/options') assert options.status_code == 200 - r = client.get(f'/api/v1.0/get/{workflow}', + r = client.get(f'/api/v1.0/get/testfileurl_extra_annotated', query_string=query_string) logger.info('service returns %s %s', r, r.json) diff --git a/tests/testfiles/testfilereference_extra_annotated.ipynb b/tests/testfiles/testfilereference_extra_annotated.ipynb index 82dbdf1..d7eaf3c 100644 --- a/tests/testfiles/testfilereference_extra_annotated.ipynb +++ b/tests/testfiles/testfilereference_extra_annotated.ipynb @@ -10,9 +10,7 @@ }, "outputs": [], "source": [ - "fits_file_path = \"/home/local/test.fits\" # oda:FileReference ; oda:label \"Test file path\" ; oda:description \"Description test file path\" .\n", - "fits_file_url = \"https://fits.gsfc.nasa.gov/samples/testkeys.fits\" # oda:FileReference ; oda:label \"Test file path\" ; oda:description \"Description test file path\" .\n", - "mmoda_url = \"https://www.astro.unige.ch/mmoda/dispatch-data/test.fits\" # oda:FileReference ; oda:label \"Test file path\" ; oda:description \"Description test file path\" ." + "fits_file_path = \"/home/local/test.fits\" # oda:FileReference ; oda:label \"Test file path\" ; oda:description \"Description test file path\" ." ] }, { @@ -21,8 +19,6 @@ "metadata": {}, "outputs": [], "source": [ - "fits_file_url_modified = fits_file_url\n", - "mmoda_url_modified = mmoda_url\n", "fits_file_path_modified = fits_file_path" ] }, @@ -36,8 +32,6 @@ }, "outputs": [], "source": [ - "fits_file_url_modified\n", - "mmoda_url_modified\n", "fits_file_path_modified" ] } From f78485041f22ef18f59340807400dfa87e1292d2 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 25 Nov 2024 12:23:17 +0100 Subject: [PATCH 09/12] issuing valueerror in case arg is not FileURL --- nb2workflow/nbadapter.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index ff1fead..249bb22 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -122,6 +122,21 @@ def cast_parameter(x,par): return None else: raise ValueError(f'Non-optional parameter is set to None') + # file_url_with_annotations_pattern = re.compile(rf"^{re.escape(oda_prefix)}.*_FileURL_") + # if ontology.is_ontology_available: + # parameter_hierarchy = ontology.get_parameter_hierarchy(par['owl_type']) + # is_file_url = f"{oda_prefix}FileURL" in parameter_hierarchy + # else: + # is_file_url = f"{oda_prefix}FileURL" == par['owl_type'] or \ + # file_url_with_annotations_pattern.match(par['owl_type']) is not None + # + # if is_file_url: + # # arg_par_value = parameters.get(input_par_name, None) + # # if arg_par_value is None: + # # arg_par_value = par['default_value'] + # if not validators.url(x, simple_host=True): + # raise ValueError( + # f'Parameter {par["name"]} value "{x}" can not be interpreted as FileURL.') return par['python_type'](x) @@ -863,7 +878,9 @@ def handle_url_params(self, parameters, tmpdir, context={}): adapted_parameters[input_par_name] = file_name except Exception as e: exceptions.append(e) - + elif is_file_url: + exceptions.append(ValueError( + f'Parameter {input_par_name} value "{arg_par_value}" can not be interpreted as FileURL.')) return dict( adapted_parameters=adapted_parameters, exceptions=exceptions From 9cc2fd02075da37e99015ba5e11d009520c66083 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 25 Nov 2024 12:23:25 +0100 Subject: [PATCH 10/12] extended test --- tests/test_input_types.py | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/tests/test_input_types.py b/tests/test_input_types.py index 918a358..5e690e6 100644 --- a/tests/test_input_types.py +++ b/tests/test_input_types.py @@ -97,7 +97,8 @@ def test_worker_run(): url_args = parse_qs(url_parts.query) assert 'token' not in url_args -def test_mmoda_file_url(client): +@pytest.mark.parametrize("query_string_fits_file_url", ["generic_url", "file_path", None]) +def test_mmoda_file_url(client, query_string_fits_file_url): status_callback_file = "status.json" callback_url = 'file://' + status_callback_file token = 'abc123' @@ -105,7 +106,14 @@ def test_mmoda_file_url(client): _async_request='no', _async_request_callback=callback_url, _token=token) + + if query_string_fits_file_url == "generic_url": + query_string['fits_file_url'] = "https://fits.gsfc.nasa.gov/samples/testkeys.fits" + elif query_string_fits_file_url == "file_path": + query_string['fits_file_url'] = "/home/local/test.fits" + r = client.get(f'/api/v1.0/get/testfileurl_extra_annotated', query_string=query_string) + assert r.status_code == 201 from nb2workflow.service import AsyncWorker @@ -134,15 +142,20 @@ def test_worker_run(): test_worker_thread.join() assert 'data' in r.json assert 'output' in r.json['data'] - assert 'mmoda_url_modified' in r.json['data']['output'] - url_parts = urlparse(r.json['data']['output']['mmoda_url_modified']) - url_args = parse_qs(url_parts.query) - assert 'token' in url_args - - assert 'fits_file_url_modified' in r.json['data']['output'] - url_parts = urlparse(r.json['data']['output']['fits_file_url_modified']) - url_args = parse_qs(url_parts.query) - assert 'token' not in url_args + if query_string_fits_file_url != "file_path": + assert 'mmoda_url_modified' in r.json['data']['output'] + url_parts = urlparse(r.json['data']['output']['mmoda_url_modified']) + url_args = parse_qs(url_parts.query) + assert 'token' in url_args + + assert 'fits_file_url_modified' in r.json['data']['output'] + url_parts = urlparse(r.json['data']['output']['fits_file_url_modified']) + url_args = parse_qs(url_parts.query) + assert 'token' not in url_args + + else: + assert r.json['data']['output'] == 'incomplete' + assert r.json['data']['exceptions'][0]['edump'] == "ValueError('Parameter fits_file_url value \"/home/local/test.fits\" can not be interpreted as FileURL.')" def test_posix_download_file_with_arg_low_download_limit(client, app_low_download_limit): r = client.get('/api/v1.0/get/testposixpath', query_string={'fits_file_path': 'https://fits.gsfc.nasa.gov/samples/testkeys.fits'}) From fd472cede4b920dd6884d294c2d1cc306d5b429c Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 25 Nov 2024 13:32:38 +0100 Subject: [PATCH 11/12] removed commented code --- nb2workflow/nbadapter.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/nb2workflow/nbadapter.py b/nb2workflow/nbadapter.py index 249bb22..7430e5a 100644 --- a/nb2workflow/nbadapter.py +++ b/nb2workflow/nbadapter.py @@ -122,21 +122,6 @@ def cast_parameter(x,par): return None else: raise ValueError(f'Non-optional parameter is set to None') - # file_url_with_annotations_pattern = re.compile(rf"^{re.escape(oda_prefix)}.*_FileURL_") - # if ontology.is_ontology_available: - # parameter_hierarchy = ontology.get_parameter_hierarchy(par['owl_type']) - # is_file_url = f"{oda_prefix}FileURL" in parameter_hierarchy - # else: - # is_file_url = f"{oda_prefix}FileURL" == par['owl_type'] or \ - # file_url_with_annotations_pattern.match(par['owl_type']) is not None - # - # if is_file_url: - # # arg_par_value = parameters.get(input_par_name, None) - # # if arg_par_value is None: - # # arg_par_value = par['default_value'] - # if not validators.url(x, simple_host=True): - # raise ValueError( - # f'Parameter {par["name"]} value "{x}" can not be interpreted as FileURL.') return par['python_type'](x) From 48d36a3e4342aa1aff14ca7263b710f863d08fa5 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 25 Nov 2024 13:32:45 +0100 Subject: [PATCH 12/12] test optimization code --- tests/test_input_types.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tests/test_input_types.py b/tests/test_input_types.py index 5e690e6..c169b55 100644 --- a/tests/test_input_types.py +++ b/tests/test_input_types.py @@ -50,10 +50,13 @@ def test_file_reference(client, query_string_fits_file_path): _async_request_callback=callback_url, _token=token) - if query_string_fits_file_path == "mmoda_url": - query_string['fits_file_path'] = "https://www.astro.unige.ch/mmoda/dispatch-data/test.fits" - elif query_string_fits_file_path == "generic_url": - query_string['fits_file_path'] = "https://fits.gsfc.nasa.gov/samples/testkeys.fits" + fits_file_path_value = None + if query_string_fits_file_path == "generic_url": + fits_file_path_value = "https://fits.gsfc.nasa.gov/samples/testkeys.fits" + elif query_string_fits_file_path == "mmoda_url": + fits_file_path_value = "https://www.astro.unige.ch/mmoda/dispatch-data/test.fits" + + query_string['fits_file_path'] = fits_file_path_value r = client.get(f'/api/v1.0/get/testfilereference_extra_annotated', query_string=query_string) assert r.status_code == 201 @@ -97,7 +100,7 @@ def test_worker_run(): url_args = parse_qs(url_parts.query) assert 'token' not in url_args -@pytest.mark.parametrize("query_string_fits_file_url", ["generic_url", "file_path", None]) +@pytest.mark.parametrize("query_string_fits_file_url", ["generic_url", "file_path", "numeric", None]) def test_mmoda_file_url(client, query_string_fits_file_url): status_callback_file = "status.json" callback_url = 'file://' + status_callback_file @@ -107,10 +110,15 @@ def test_mmoda_file_url(client, query_string_fits_file_url): _async_request_callback=callback_url, _token=token) + fits_file_url_value = None if query_string_fits_file_url == "generic_url": - query_string['fits_file_url'] = "https://fits.gsfc.nasa.gov/samples/testkeys.fits" + fits_file_url_value = "https://fits.gsfc.nasa.gov/samples/testkeys.fits" elif query_string_fits_file_url == "file_path": - query_string['fits_file_url'] = "/home/local/test.fits" + fits_file_url_value = "/home/local/test.fits" + elif query_string_fits_file_url == "numeric": + fits_file_url_value = 123456 + + query_string['fits_file_url'] = fits_file_url_value r = client.get(f'/api/v1.0/get/testfileurl_extra_annotated', query_string=query_string) @@ -142,7 +150,7 @@ def test_worker_run(): test_worker_thread.join() assert 'data' in r.json assert 'output' in r.json['data'] - if query_string_fits_file_url != "file_path": + if query_string_fits_file_url != "file_path" and query_string_fits_file_url != "numeric": assert 'mmoda_url_modified' in r.json['data']['output'] url_parts = urlparse(r.json['data']['output']['mmoda_url_modified']) url_args = parse_qs(url_parts.query) @@ -155,7 +163,7 @@ def test_worker_run(): else: assert r.json['data']['output'] == 'incomplete' - assert r.json['data']['exceptions'][0]['edump'] == "ValueError('Parameter fits_file_url value \"/home/local/test.fits\" can not be interpreted as FileURL.')" + assert r.json['data']['exceptions'][0]['edump'] == f"ValueError('Parameter fits_file_url value \"{fits_file_url_value}\" can not be interpreted as FileURL.')" def test_posix_download_file_with_arg_low_download_limit(client, app_low_download_limit): r = client.get('/api/v1.0/get/testposixpath', query_string={'fits_file_path': 'https://fits.gsfc.nasa.gov/samples/testkeys.fits'})