From 5ad751b05bf0da59295e3b0d092c437882e6f995 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 15 Nov 2024 18:38:34 +0100 Subject: [PATCH 1/3] passing token to the funciotn that builds the uploaded file url --- cdci_data_analysis/analysis/instrument.py | 13 ++++++++----- cdci_data_analysis/flask_app/dispatcher_query.py | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/cdci_data_analysis/analysis/instrument.py b/cdci_data_analysis/analysis/instrument.py index 619c70f4..55c8e1ec 100644 --- a/cdci_data_analysis/analysis/instrument.py +++ b/cdci_data_analysis/analysis/instrument.py @@ -251,6 +251,7 @@ def parse_inputs_files(self, bind_port, request_files_dir, decoded_token, + token=None, sentry_dsn=None): error_message = 'Error while {step} {temp_dir_content_msg}{additional}' # TODO probably exception handling can be further improved and/or optmized @@ -281,7 +282,8 @@ def parse_inputs_files(self, uploaded_files_obj=uploaded_files_obj, products_url=products_url, bind_host=bind_host, - bind_port=bind_port) + bind_port=bind_port, + token=token) step = 'updating ownership files' self.update_ownership_files(uploaded_files_obj, request_files_dir=request_files_dir, @@ -708,16 +710,17 @@ def set_input_products_from_fronted(self, input_file_path, par_dic, verbose=Fals else: raise RuntimeError - def update_par_dic_with_uploaded_files(self, par_dic, uploaded_files_obj, products_url, bind_host, bind_port): + def update_par_dic_with_uploaded_files(self, par_dic, uploaded_files_obj, products_url, bind_host, bind_port, token=None): if validators.url(products_url, simple_host=True): # TODO remove the dispatch-data part, better to have it extracted from the configuration file basepath = os.path.join(products_url, 'dispatch-data/download_file') else: basepath = os.path.join(f"http://{bind_host}:{bind_port}", 'download_file') for f in uploaded_files_obj: - dpars = urlencode(dict(file_list=uploaded_files_obj[f], - _is_mmoda_url=True, - return_archive=False)) + dict_args = dict(file_list=uploaded_files_obj[f], _is_mmoda_url=True, return_archive=False) + if token is not None: + dict_args['token'] = token + dpars = urlencode(dict_args) download_file_url = f"{basepath}?{dpars}" par_dic[f] = download_file_url diff --git a/cdci_data_analysis/flask_app/dispatcher_query.py b/cdci_data_analysis/flask_app/dispatcher_query.py index 11baccce..8fa498fb 100644 --- a/cdci_data_analysis/flask_app/dispatcher_query.py +++ b/cdci_data_analysis/flask_app/dispatcher_query.py @@ -309,6 +309,7 @@ def __init__(self, app, bind_host=bind_host, bind_port=bind_port, request_files_dir=self.request_files_dir, + token=self.token, decoded_token=self.decoded_token, sentry_dsn=self.sentry_dsn ) From a02b6fe15b47b2b6994f34ed61fb8e152ed9f7f8 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 18 Nov 2024 09:06:18 +0100 Subject: [PATCH 2/3] adapted tests --- tests/test_server_basic.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/test_server_basic.py b/tests/test_server_basic.py index 192d7141..e383b368 100644 --- a/tests/test_server_basic.py +++ b/tests/test_server_basic.py @@ -17,13 +17,15 @@ from dateutil import parser, tz from functools import reduce from urllib import parse -from urllib.parse import urlencode +from urllib.parse import urlencode, urlparse, parse_qs, urlunparse import nbformat as nbf import yaml import gzip import random import string +from lxml.html.diff import token + from cdci_data_analysis.analysis.catalog import BasicCatalog from cdci_data_analysis.pytest_fixtures import DispatcherJobState, ask, make_hash, dispatcher_fetch_dummy_products, make_hash_file from cdci_data_analysis.flask_app.dispatcher_query import InstrumentQueryBackEnd @@ -1714,18 +1716,24 @@ def test_arg_file(dispatcher_live_fixture, dispatcher_test_conf, public_download file_hash = make_hash_file(p_file_path) dpars = urlencode(dict(file_list=file_hash, _is_mmoda_url=True, - return_archive=False)) + return_archive=False, + token=encoded_token)) local_download_url = f"{os.path.join(products_host_port, 'download_file')}?{dpars}" assert arg_download_url == local_download_url if public_download_request: + url_parts = urlparse(arg_download_url) + url_args = parse_qs(url_parts.query) + del url_args['token'] + new_url_parts = url_parts._replace(query=urlencode(url_args, doseq=True)) + arg_download_url = urlunparse(new_url_parts) c = requests.get(arg_download_url) assert c.status_code == 403 jdata = c.json() assert jdata['exit_status']['message'] == "User cannot access the file" else: - arg_download_url += f'&token={encoded_token}' + # arg_download_url += f'&token={encoded_token}' c = requests.get(arg_download_url) assert c.status_code == 200 with open(p_file_path) as p_file: @@ -1787,7 +1795,8 @@ def test_arg_file_external_product_url(dispatcher_live_fixture_with_external_pro file_hash = make_hash_file(p_file_path) dpars = urlencode(dict(file_list=file_hash, _is_mmoda_url=True, - return_archive=False)) + return_archive=False, + token=encoded_token)) local_download_url = f"{os.path.join(dispatcher_test_conf_with_external_products_url['products_url'], 'dispatch-data/download_file')}?{dpars}" assert jdata['products']['analysis_parameters']['dummy_file'] == local_download_url @@ -1847,7 +1856,8 @@ def test_arg_file_default_product_url(dispatcher_live_fixture_with_default_route file_hash = make_hash_file(p_file_path) dpars = urlencode(dict(file_list=file_hash, _is_mmoda_url=True, - return_archive=False)) + return_archive=False, + token=encoded_token)) local_download_url = f"{os.path.join(dispatcher_test_conf_with_default_route_products_url['products_url'], 'dispatch-data/download_file')}?{dpars}" assert jdata['products']['analysis_parameters']['dummy_file'] == local_download_url From 5b33fe9ff8654707bd4d19821e004692dd3e9fd4 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 18 Nov 2024 09:16:40 +0100 Subject: [PATCH 3/3] not needed import --- tests/test_server_basic.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_server_basic.py b/tests/test_server_basic.py index e383b368..94f8b3fc 100644 --- a/tests/test_server_basic.py +++ b/tests/test_server_basic.py @@ -24,8 +24,6 @@ import random import string -from lxml.html.diff import token - from cdci_data_analysis.analysis.catalog import BasicCatalog from cdci_data_analysis.pytest_fixtures import DispatcherJobState, ask, make_hash, dispatcher_fetch_dummy_products, make_hash_file from cdci_data_analysis.flask_app.dispatcher_query import InstrumentQueryBackEnd