From 37dca7a4f616af5a1331254f9ccc3dc43f18fe3a Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 15 Sep 2023 10:32:15 +0200 Subject: [PATCH 01/12] get_data_product_list_with_conditions call the drupal rest endpoint --- cdci_data_analysis/analysis/drupal_helper.py | 45 ++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/cdci_data_analysis/analysis/drupal_helper.py b/cdci_data_analysis/analysis/drupal_helper.py index 55685ebcb..60d635c47 100644 --- a/cdci_data_analysis/analysis/drupal_helper.py +++ b/cdci_data_analysis/analysis/drupal_helper.py @@ -840,6 +840,51 @@ def get_all_revolutions(product_gallery_url, gallery_jwt_token, sentry_dsn=None) return entities +def get_data_product_list_with_conditions(product_gallery_url, gallery_jwt_token, + src_name=None, + instrument_name=None, + product_type=None, + e1=None, e2=None, + rev1=None, rev2=None, + sentry_dsn=None) -> Optional[list]: + headers = get_drupal_request_headers(gallery_jwt_token) + product_list = [] + if src_name is None: + source_entity_id = "all" + else: + source_entity_list = get_source_astrophysical_entity_info_by_source_and_alternative_name(product_gallery_url, + gallery_jwt_token, + source_name=src_name, + sentry_dsn=sentry_dsn) + + source_entity_id = None + if len(source_entity_list) >= 1: + source_entity_id = source_entity_list[0]['nid'] + + if instrument_name is None: + instrument_name = "all" + + if product_type is None: + product_type = "all" + + if source_entity_id is not None: + log_res = execute_drupal_request(f"{product_gallery_url}/data_products/source_products_conditions/{instrument_name}/{product_type}/{source_entity_id}", + headers=headers, + sentry_dsn=sentry_dsn) + output_get = analyze_drupal_output(log_res, operation_performed="retrieving the astrophysical entity information") + if isinstance(output_get, list): + for obj in output_get: + refactored_obj = {} + for k, v in obj.items(): + refactored_key = k + if k.startswith('field_'): + refactored_key = k.replace('field_', '') + refactored_obj[refactored_key] = v + product_list.append(refactored_obj) + + return product_list + + def get_data_product_list_by_source_name(product_gallery_url, gallery_jwt_token, src_name=None, sentry_dsn=None) -> Optional[list]: product_list = [] if src_name is None: From c6c01c8112c2b9faa58e2a2cc75bada6d2f4416b Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 15 Sep 2023 10:40:02 +0200 Subject: [PATCH 02/12] flask endpoint --- cdci_data_analysis/analysis/drupal_helper.py | 25 ++++++++-- cdci_data_analysis/flask_app/app.py | 51 ++++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/cdci_data_analysis/analysis/drupal_helper.py b/cdci_data_analysis/analysis/drupal_helper.py index 60d635c47..63a3506a4 100644 --- a/cdci_data_analysis/analysis/drupal_helper.py +++ b/cdci_data_analysis/analysis/drupal_helper.py @@ -844,8 +844,8 @@ def get_data_product_list_with_conditions(product_gallery_url, gallery_jwt_token src_name=None, instrument_name=None, product_type=None, - e1=None, e2=None, - rev1=None, rev2=None, + e1_kev_value=None, e2_kev_value=None, + rev1_value=None, rev2_value=None, sentry_dsn=None) -> Optional[list]: headers = get_drupal_request_headers(gallery_jwt_token) product_list = [] @@ -867,11 +867,28 @@ def get_data_product_list_with_conditions(product_gallery_url, gallery_jwt_token if product_type is None: product_type = "all" + request_url = f"{product_gallery_url}/data_products/source_products_conditions/{instrument_name}/{product_type}/{source_entity_id}" + + params = {"_format": "hal_json"} + + if e1_kev_value is not None: + params["field_e1_kev_value"] = e1_kev_value + + if e2_kev_value is not None: + params["field_e2_kev_value"] = e2_kev_value + + if rev1_value is not None: + params["field_rev1_value"] = rev1_value + + if rev2_value is not None: + params["field_rev2_value"] = rev2_value + if source_entity_id is not None: - log_res = execute_drupal_request(f"{product_gallery_url}/data_products/source_products_conditions/{instrument_name}/{product_type}/{source_entity_id}", + log_res = execute_drupal_request(request_url, + params=params, headers=headers, sentry_dsn=sentry_dsn) - output_get = analyze_drupal_output(log_res, operation_performed="retrieving the astrophysical entity information") + output_get = analyze_drupal_output(log_res, operation_performed="retrieving list of data products with conditions") if isinstance(output_get, list): for obj in output_get: refactored_obj = {} diff --git a/cdci_data_analysis/flask_app/app.py b/cdci_data_analysis/flask_app/app.py index 51ade86a4..b9d75e9e5 100644 --- a/cdci_data_analysis/flask_app/app.py +++ b/cdci_data_analysis/flask_app/app.py @@ -800,6 +800,57 @@ def get_astro_entity_info_by_source_name(): return refactored_astro_entity_info +@app.route('/get_data_product_list_with_conditions', methods=['GET']) +def get_data_product_list_with_conditions(): + logger.info("request.args: %s ", request.args) + logger.info("request.files: %s ", request.files) + + token = request.args.get('token', None) + app_config = app.config.get('conf') + secret_key = app_config.secret_key + + output, output_code = tokenHelper.validate_token_from_request(token=token, secret_key=secret_key, + required_roles=['gallery contributor'], + action="getting all the astro entities from the product gallery") + + if output_code is not None: + return make_response(output, output_code) + decoded_token = output + + par_dic = request.values.to_dict() + par_dic.pop('token') + + sentry_dsn = sentry.sentry_url + + gallery_secret_key = app_config.product_gallery_secret_key + product_gallery_url = app_config.product_gallery_url + user_email = tokenHelper.get_token_user_email_address(decoded_token) + user_id_product_creator = drupal_helper.get_user_id(product_gallery_url=product_gallery_url, + user_email=user_email, + sentry_dsn=sentry_dsn) + # update the token + gallery_jwt_token = drupal_helper.generate_gallery_jwt_token(gallery_secret_key, user_id=user_id_product_creator) + + src_name = request.args.get('src_name', None) + instrument_name = request.args.get('instrument_name', None) + product_type = request.args.get('product_type', None) + e1_kev_value = request.args.get('e1_kev_value', None) + e2_kev_value = request.args.get('e2_kev_value', None) + rev1_value = request.args.get('rev1_value', None) + rev2_value = request.args.get('rev2_value', None) + + output_get = drupal_helper.get_data_product_list_with_conditions(product_gallery_url=product_gallery_url, + gallery_jwt_token=gallery_jwt_token, + src_name=src_name, + instrument_name=instrument_name, + product_type=product_type, + e1_kev_value=e1_kev_value, e2_kev_value=e2_kev_value, + rev1_value=rev1_value, rev2_value=rev2_value, + sentry_dsn=sentry_dsn) + output_list = json.dumps(output_get) + + return output_list + @app.route('/get_data_product_list_by_source_name', methods=['GET']) def get_data_product_list_by_source_name(): From 1293773d2bba079c74348909ef37d28a4d86905e Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 15 Sep 2023 11:11:59 +0200 Subject: [PATCH 03/12] more general approach --- cdci_data_analysis/analysis/drupal_helper.py | 23 ++++++++------------ cdci_data_analysis/flask_app/app.py | 21 ++++++------------ 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/cdci_data_analysis/analysis/drupal_helper.py b/cdci_data_analysis/analysis/drupal_helper.py index 63a3506a4..0620c07f8 100644 --- a/cdci_data_analysis/analysis/drupal_helper.py +++ b/cdci_data_analysis/analysis/drupal_helper.py @@ -844,9 +844,9 @@ def get_data_product_list_with_conditions(product_gallery_url, gallery_jwt_token src_name=None, instrument_name=None, product_type=None, - e1_kev_value=None, e2_kev_value=None, - rev1_value=None, rev2_value=None, - sentry_dsn=None) -> Optional[list]: + sentry_dsn=None, + **kwargs + ) -> Optional[list]: headers = get_drupal_request_headers(gallery_jwt_token) product_list = [] if src_name is None: @@ -871,17 +871,12 @@ def get_data_product_list_with_conditions(product_gallery_url, gallery_jwt_token params = {"_format": "hal_json"} - if e1_kev_value is not None: - params["field_e1_kev_value"] = e1_kev_value - - if e2_kev_value is not None: - params["field_e2_kev_value"] = e2_kev_value - - if rev1_value is not None: - params["field_rev1_value"] = rev1_value - - if rev2_value is not None: - params["field_rev2_value"] = rev2_value + for k, v in kwargs.items(): + # the machine name of the field in drupal starts by default with field_ + field_name = str.lower('field_' + k) + params[field_name] = [{ + "value": v + }] if source_entity_id is not None: log_res = execute_drupal_request(request_url, diff --git a/cdci_data_analysis/flask_app/app.py b/cdci_data_analysis/flask_app/app.py index b9d75e9e5..69b6d94b2 100644 --- a/cdci_data_analysis/flask_app/app.py +++ b/cdci_data_analysis/flask_app/app.py @@ -805,7 +805,8 @@ def get_data_product_list_with_conditions(): logger.info("request.args: %s ", request.args) logger.info("request.files: %s ", request.files) - token = request.args.get('token', None) + par_dic = request.values.to_dict() + token = par_dic.pop('token', None) app_config = app.config.get('conf') secret_key = app_config.secret_key @@ -817,9 +818,6 @@ def get_data_product_list_with_conditions(): return make_response(output, output_code) decoded_token = output - par_dic = request.values.to_dict() - par_dic.pop('token') - sentry_dsn = sentry.sentry_url gallery_secret_key = app_config.product_gallery_secret_key @@ -831,22 +829,17 @@ def get_data_product_list_with_conditions(): # update the token gallery_jwt_token = drupal_helper.generate_gallery_jwt_token(gallery_secret_key, user_id=user_id_product_creator) - src_name = request.args.get('src_name', None) - instrument_name = request.args.get('instrument_name', None) - product_type = request.args.get('product_type', None) - e1_kev_value = request.args.get('e1_kev_value', None) - e2_kev_value = request.args.get('e2_kev_value', None) - rev1_value = request.args.get('rev1_value', None) - rev2_value = request.args.get('rev2_value', None) + src_name = par_dic.pop('src_name', None) + instrument_name = par_dic.pop('instrument_name', None) + product_type = par_dic.pop('product_type', None) output_get = drupal_helper.get_data_product_list_with_conditions(product_gallery_url=product_gallery_url, gallery_jwt_token=gallery_jwt_token, src_name=src_name, instrument_name=instrument_name, product_type=product_type, - e1_kev_value=e1_kev_value, e2_kev_value=e2_kev_value, - rev1_value=rev1_value, rev2_value=rev2_value, - sentry_dsn=sentry_dsn) + sentry_dsn=sentry_dsn, + **par_dic) output_list = json.dumps(output_get) return output_list From ebacc31f95cdd9e70414788e2498dc1c99a7b8d2 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 15 Sep 2023 12:19:24 +0200 Subject: [PATCH 04/12] properly assigning conditions params for request to drupal --- cdci_data_analysis/analysis/drupal_helper.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cdci_data_analysis/analysis/drupal_helper.py b/cdci_data_analysis/analysis/drupal_helper.py index 0620c07f8..a2e26316f 100644 --- a/cdci_data_analysis/analysis/drupal_helper.py +++ b/cdci_data_analysis/analysis/drupal_helper.py @@ -874,9 +874,7 @@ def get_data_product_list_with_conditions(product_gallery_url, gallery_jwt_token for k, v in kwargs.items(): # the machine name of the field in drupal starts by default with field_ field_name = str.lower('field_' + k) - params[field_name] = [{ - "value": v - }] + params[field_name] = v if source_entity_id is not None: log_res = execute_drupal_request(request_url, From cba8cce8cf0c561feb91d92def10e1d8a7c9b1cd Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 15 Sep 2023 12:19:29 +0200 Subject: [PATCH 05/12] extended test --- tests/test_server_basic.py | 144 +++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/tests/test_server_basic.py b/tests/test_server_basic.py index 428d980c6..1c34dcbae 100644 --- a/tests/test_server_basic.py +++ b/tests/test_server_basic.py @@ -2283,6 +2283,150 @@ def test_product_gallery_get_all_astro_entities(dispatcher_live_fixture_with_gal assert any(src['title'] == params['src_name'] for src in drupal_res_obj) +@pytest.mark.test_drupal +@pytest.mark.parametrize("source_name", ["new", "known", "unknown"]) +@pytest.mark.parametrize("include_products_fields_conditions", [True, False]) +def test_product_gallery_get_data_products_list_with_conditions(dispatcher_live_fixture_with_gallery, dispatcher_test_conf_with_gallery, source_name, include_products_fields_conditions): + server = dispatcher_live_fixture_with_gallery + + logger.info("constructed server: %s", server) + + # let's generate a valid token + token_payload = { + **default_token_payload, + "roles": "general, gallery contributor", + } + encoded_token = jwt.encode(token_payload, secret_key, algorithm='HS256') + instrument_name = 'isgri' + product_type = 'isgri_image' + instrument_query = 'isgri' + product_type_query = 'image' + + if source_name == 'new': + source_name = 'test astro entity' + '_' + str(uuid.uuid4()) + # let's create a source + source_params = { + 'token': encoded_token, + 'src_name': source_name + } + + c = requests.post(os.path.join(server, "post_astro_entity_to_gallery"), + params={**source_params}, + ) + + assert c.status_code == 200 + + # let's post a product with the source just created + product_params = { + 'instrument': instrument_name, + 'product_type': product_type, + 'E1_keV': 150, + 'E2_keV': 350, + 'src_name': source_name, + 'content_type': 'data_product', + 'token': encoded_token, + 'insert_new_source': True + } + c = requests.post(os.path.join(server, "post_product_to_gallery"), + params={**product_params} + ) + + assert c.status_code == 200 + + params = { + 'token': encoded_token, + 'src_name': source_name, + 'instrument_name': instrument_query, + 'product_type': product_type_query + } + if include_products_fields_conditions: + for e1_kev, e2_kev in [ + (100, 350), + (50, 400), + (200, 350), + (200, 350), + (50, 300), + ]: + logger.info(f"testing with e1_kev_value {e1_kev}, e2_kev_value {e2_kev}") + params['e1_kev_value'] = e1_kev + params['e2_kev_value'] = e2_kev + + c = requests.get(os.path.join(server, "get_data_product_list_with_conditions"), + params=params + ) + + assert c.status_code == 200 + drupal_res_obj = c.json() + assert isinstance(drupal_res_obj, list) + + if e1_kev > 100 or e2_kev < 350: + assert len(drupal_res_obj) == 0 + else: + assert len(drupal_res_obj) == 1 + else: + c = requests.get(os.path.join(server, "get_data_product_list_with_conditions"), + params=params + ) + + assert c.status_code == 200 + drupal_res_obj = c.json() + assert isinstance(drupal_res_obj, list) + assert len(drupal_res_obj) == 1 + elif source_name == 'unknown': + source_name = "aaaaaaaaaaaaaaaaa" + params = { + 'token': encoded_token, + 'src_name': source_name + } + c = requests.get(os.path.join(server, "get_data_product_list_with_conditions"), + params=params + ) + + assert c.status_code == 200 + drupal_res_obj = c.json() + assert isinstance(drupal_res_obj, list) + assert len(drupal_res_obj) == 0 + else: + source_name = "V404 Cyg" + params = { + 'token': encoded_token, + 'src_name': source_name + } + c = requests.get(os.path.join(server, "get_data_product_list_with_conditions"), + params=params + ) + + assert c.status_code == 200 + drupal_res_obj_source_name = c.json() + assert isinstance(drupal_res_obj_source_name, list) + + source_name = "1RXS J202405.3+335157" + params = { + 'token': encoded_token, + 'src_name': source_name + } + c = requests.get(os.path.join(server, "get_data_product_list_with_conditions"), + params=params + ) + + assert c.status_code == 200 + drupal_res_obj_alternative_name = c.json() + assert isinstance(drupal_res_obj_alternative_name, list) + + assert len(drupal_res_obj_alternative_name) == len(drupal_res_obj_source_name) + + # Create sets of dictionaries + set1 = set(map(lambda d: frozenset(d.items()), drupal_res_obj_source_name)) + set2 = set(map(lambda d: frozenset(d.items()), drupal_res_obj_alternative_name)) + + # Find the differences + diff1 = set1 - set2 + diff2 = set2 - set1 + + assert diff2 == set() + assert diff1 == set() + + @pytest.mark.test_drupal @pytest.mark.parametrize("source_name", ["new", "known"]) def test_product_gallery_get_data_products_list_for_given_source(dispatcher_live_fixture_with_gallery, dispatcher_test_conf_with_gallery, source_name): From d716f0605c06837df905a727482735164900c88f Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 15 Sep 2023 14:38:27 +0200 Subject: [PATCH 06/12] more flexible query towards the gallery, less code --- cdci_data_analysis/analysis/drupal_helper.py | 10 +--------- cdci_data_analysis/flask_app/app.py | 4 ---- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/cdci_data_analysis/analysis/drupal_helper.py b/cdci_data_analysis/analysis/drupal_helper.py index a2e26316f..97ec30bcc 100644 --- a/cdci_data_analysis/analysis/drupal_helper.py +++ b/cdci_data_analysis/analysis/drupal_helper.py @@ -842,8 +842,6 @@ def get_all_revolutions(product_gallery_url, gallery_jwt_token, sentry_dsn=None) def get_data_product_list_with_conditions(product_gallery_url, gallery_jwt_token, src_name=None, - instrument_name=None, - product_type=None, sentry_dsn=None, **kwargs ) -> Optional[list]: @@ -861,13 +859,7 @@ def get_data_product_list_with_conditions(product_gallery_url, gallery_jwt_token if len(source_entity_list) >= 1: source_entity_id = source_entity_list[0]['nid'] - if instrument_name is None: - instrument_name = "all" - - if product_type is None: - product_type = "all" - - request_url = f"{product_gallery_url}/data_products/source_products_conditions/{instrument_name}/{product_type}/{source_entity_id}" + request_url = f"{product_gallery_url}/data_products/source_products_conditions/{source_entity_id}" params = {"_format": "hal_json"} diff --git a/cdci_data_analysis/flask_app/app.py b/cdci_data_analysis/flask_app/app.py index 69b6d94b2..055d9206e 100644 --- a/cdci_data_analysis/flask_app/app.py +++ b/cdci_data_analysis/flask_app/app.py @@ -830,14 +830,10 @@ def get_data_product_list_with_conditions(): gallery_jwt_token = drupal_helper.generate_gallery_jwt_token(gallery_secret_key, user_id=user_id_product_creator) src_name = par_dic.pop('src_name', None) - instrument_name = par_dic.pop('instrument_name', None) - product_type = par_dic.pop('product_type', None) output_get = drupal_helper.get_data_product_list_with_conditions(product_gallery_url=product_gallery_url, gallery_jwt_token=gallery_jwt_token, src_name=src_name, - instrument_name=instrument_name, - product_type=product_type, sentry_dsn=sentry_dsn, **par_dic) output_list = json.dumps(output_get) From 2cba012246887a4719be36682010bdd645c5c266 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 15 Sep 2023 14:54:58 +0200 Subject: [PATCH 07/12] extended test --- tests/test_server_basic.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/test_server_basic.py b/tests/test_server_basic.py index 1c34dcbae..642bceb73 100644 --- a/tests/test_server_basic.py +++ b/tests/test_server_basic.py @@ -2325,7 +2325,9 @@ def test_product_gallery_get_data_products_list_with_conditions(dispatcher_live_ 'src_name': source_name, 'content_type': 'data_product', 'token': encoded_token, - 'insert_new_source': True + 'insert_new_source': True, + 'T1': '2022-07-21T00:29:47', + 'T2': '2022-08-23T05:29:11' } c = requests.post(os.path.join(server, "post_product_to_gallery"), params={**product_params} @@ -2340,17 +2342,24 @@ def test_product_gallery_get_data_products_list_with_conditions(dispatcher_live_ 'product_type': product_type_query } if include_products_fields_conditions: - for e1_kev, e2_kev in [ - (100, 350), - (50, 400), - (200, 350), - (200, 350), - (50, 300), + for e1_kev, e2_kev, rev1, rev2 in [ + (100, 350, 2528, 2540), + (100, 350, 2526, 2541), + (100, 350, 2529, 2539), + (100, 350, 2529, 2541), + (100, 350, 2527, 2539), + (50, 400, 2528, 2540), + (200, 350, 2528, 2540), + (200, 350, 2528, 2540), + (50, 300, 2528, 2540), ]: logger.info(f"testing with e1_kev_value {e1_kev}, e2_kev_value {e2_kev}") params['e1_kev_value'] = e1_kev params['e2_kev_value'] = e2_kev + params['rev1_value'] = rev1 + params['rev2_value'] = rev2 + c = requests.get(os.path.join(server, "get_data_product_list_with_conditions"), params=params ) @@ -2359,7 +2368,7 @@ def test_product_gallery_get_data_products_list_with_conditions(dispatcher_live_ drupal_res_obj = c.json() assert isinstance(drupal_res_obj, list) - if e1_kev > 100 or e2_kev < 350: + if e1_kev > 100 or e2_kev < 350 or rev1 > 2528 or rev2 < 2540: assert len(drupal_res_obj) == 0 else: assert len(drupal_res_obj) == 1 From 8a7e9fefbc485f7d30a212c39b5c2727ed404567 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 15 Sep 2023 14:56:40 +0200 Subject: [PATCH 08/12] better naming --- cdci_data_analysis/analysis/drupal_helper.py | 10 +++++----- cdci_data_analysis/flask_app/app.py | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cdci_data_analysis/analysis/drupal_helper.py b/cdci_data_analysis/analysis/drupal_helper.py index 97ec30bcc..fa5e946e7 100644 --- a/cdci_data_analysis/analysis/drupal_helper.py +++ b/cdci_data_analysis/analysis/drupal_helper.py @@ -840,11 +840,11 @@ def get_all_revolutions(product_gallery_url, gallery_jwt_token, sentry_dsn=None) return entities -def get_data_product_list_with_conditions(product_gallery_url, gallery_jwt_token, - src_name=None, - sentry_dsn=None, - **kwargs - ) -> Optional[list]: +def get_data_product_list_by_source_name_with_conditions(product_gallery_url, gallery_jwt_token, + src_name=None, + sentry_dsn=None, + **kwargs + ) -> Optional[list]: headers = get_drupal_request_headers(gallery_jwt_token) product_list = [] if src_name is None: diff --git a/cdci_data_analysis/flask_app/app.py b/cdci_data_analysis/flask_app/app.py index 055d9206e..40de85430 100644 --- a/cdci_data_analysis/flask_app/app.py +++ b/cdci_data_analysis/flask_app/app.py @@ -800,8 +800,8 @@ def get_astro_entity_info_by_source_name(): return refactored_astro_entity_info -@app.route('/get_data_product_list_with_conditions', methods=['GET']) -def get_data_product_list_with_conditions(): +@app.route('/get_data_product_list_by_source_name_with_conditions', methods=['GET']) +def get_data_product_list_by_source_name_with_conditions(): logger.info("request.args: %s ", request.args) logger.info("request.files: %s ", request.files) @@ -831,11 +831,11 @@ def get_data_product_list_with_conditions(): src_name = par_dic.pop('src_name', None) - output_get = drupal_helper.get_data_product_list_with_conditions(product_gallery_url=product_gallery_url, - gallery_jwt_token=gallery_jwt_token, - src_name=src_name, - sentry_dsn=sentry_dsn, - **par_dic) + output_get = drupal_helper.get_data_product_list_by_source_name_with_conditions(product_gallery_url=product_gallery_url, + gallery_jwt_token=gallery_jwt_token, + src_name=src_name, + sentry_dsn=sentry_dsn, + **par_dic) output_list = json.dumps(output_get) return output_list From 16f5c94766cd8ee361c61c38efc90c4e5459539d Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 18 Sep 2023 12:38:28 +0200 Subject: [PATCH 09/12] src_name no longer as part of the url, included as field_ arg --- cdci_data_analysis/analysis/drupal_helper.py | 53 ++++++++++---------- cdci_data_analysis/flask_app/app.py | 7 ++- 2 files changed, 29 insertions(+), 31 deletions(-) diff --git a/cdci_data_analysis/analysis/drupal_helper.py b/cdci_data_analysis/analysis/drupal_helper.py index fa5e946e7..6ac3a0d7f 100644 --- a/cdci_data_analysis/analysis/drupal_helper.py +++ b/cdci_data_analysis/analysis/drupal_helper.py @@ -841,25 +841,24 @@ def get_all_revolutions(product_gallery_url, gallery_jwt_token, sentry_dsn=None) def get_data_product_list_by_source_name_with_conditions(product_gallery_url, gallery_jwt_token, - src_name=None, sentry_dsn=None, **kwargs ) -> Optional[list]: headers = get_drupal_request_headers(gallery_jwt_token) product_list = [] - if src_name is None: - source_entity_id = "all" - else: - source_entity_list = get_source_astrophysical_entity_info_by_source_and_alternative_name(product_gallery_url, - gallery_jwt_token, - source_name=src_name, - sentry_dsn=sentry_dsn) + # if src_name is None: + # source_entity_id = "all" + # else: + # source_entity_list = get_source_astrophysical_entity_info_by_source_and_alternative_name(product_gallery_url, + # gallery_jwt_token, + # source_name=src_name, + # sentry_dsn=sentry_dsn) - source_entity_id = None - if len(source_entity_list) >= 1: - source_entity_id = source_entity_list[0]['nid'] + # source_entity_id = None + # if len(source_entity_list) >= 1: + # source_entity_id = source_entity_list[0]['nid'] - request_url = f"{product_gallery_url}/data_products/source_products_conditions/{source_entity_id}" + request_url = f"{product_gallery_url}/data_products/source_products_conditions" params = {"_format": "hal_json"} @@ -868,21 +867,21 @@ def get_data_product_list_by_source_name_with_conditions(product_gallery_url, ga field_name = str.lower('field_' + k) params[field_name] = v - if source_entity_id is not None: - log_res = execute_drupal_request(request_url, - params=params, - headers=headers, - sentry_dsn=sentry_dsn) - output_get = analyze_drupal_output(log_res, operation_performed="retrieving list of data products with conditions") - if isinstance(output_get, list): - for obj in output_get: - refactored_obj = {} - for k, v in obj.items(): - refactored_key = k - if k.startswith('field_'): - refactored_key = k.replace('field_', '') - refactored_obj[refactored_key] = v - product_list.append(refactored_obj) + # if source_entity_id is not None: + log_res = execute_drupal_request(request_url, + params=params, + headers=headers, + sentry_dsn=sentry_dsn) + output_get = analyze_drupal_output(log_res, operation_performed="retrieving list of data products with conditions") + if isinstance(output_get, list): + for obj in output_get: + refactored_obj = {} + for k, v in obj.items(): + refactored_key = k + if k.startswith('field_'): + refactored_key = k.replace('field_', '') + refactored_obj[refactored_key] = v + product_list.append(refactored_obj) return product_list diff --git a/cdci_data_analysis/flask_app/app.py b/cdci_data_analysis/flask_app/app.py index 40de85430..aeefdf390 100644 --- a/cdci_data_analysis/flask_app/app.py +++ b/cdci_data_analysis/flask_app/app.py @@ -800,8 +800,8 @@ def get_astro_entity_info_by_source_name(): return refactored_astro_entity_info -@app.route('/get_data_product_list_by_source_name_with_conditions', methods=['GET']) -def get_data_product_list_by_source_name_with_conditions(): +@app.route('/get_data_product_list_with_conditions', methods=['GET']) +def get_data_product_list_with_conditions(): logger.info("request.args: %s ", request.args) logger.info("request.files: %s ", request.files) @@ -829,11 +829,10 @@ def get_data_product_list_by_source_name_with_conditions(): # update the token gallery_jwt_token = drupal_helper.generate_gallery_jwt_token(gallery_secret_key, user_id=user_id_product_creator) - src_name = par_dic.pop('src_name', None) + # src_name = par_dic.pop('src_name', None) output_get = drupal_helper.get_data_product_list_by_source_name_with_conditions(product_gallery_url=product_gallery_url, gallery_jwt_token=gallery_jwt_token, - src_name=src_name, sentry_dsn=sentry_dsn, **par_dic) output_list = json.dumps(output_get) From 8f9abb7fb920cec282c15f8d21736f99d2556a7b Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 18 Sep 2023 12:45:01 +0200 Subject: [PATCH 10/12] TODO --- cdci_data_analysis/flask_app/app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdci_data_analysis/flask_app/app.py b/cdci_data_analysis/flask_app/app.py index aeefdf390..983dcf2e2 100644 --- a/cdci_data_analysis/flask_app/app.py +++ b/cdci_data_analysis/flask_app/app.py @@ -840,6 +840,7 @@ def get_data_product_list_with_conditions(): return output_list +# TODO to refactor using get_data_product_list_with_conditions @app.route('/get_data_product_list_by_source_name', methods=['GET']) def get_data_product_list_by_source_name(): logger.info("request.args: %s ", request.args) @@ -874,7 +875,7 @@ def get_data_product_list_by_source_name(): src_name = request.args.get('src_name', None) output_get = drupal_helper.get_data_product_list_by_source_name(product_gallery_url=product_gallery_url, - gallery_jwt_token=gallery_jwt_token, + gallery_jwt_token=gallery_jwt_token, src_name=src_name, sentry_dsn=sentry_dsn) output_list = json.dumps(output_get) From 441b28fcce39585f836d3b0ba663048c4512fa37 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Tue, 19 Sep 2023 12:21:29 +0200 Subject: [PATCH 11/12] using new dev gallery version --- cdci_data_analysis/pytest_fixtures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdci_data_analysis/pytest_fixtures.py b/cdci_data_analysis/pytest_fixtures.py index 6371da90a..0216cb7cc 100644 --- a/cdci_data_analysis/pytest_fixtures.py +++ b/cdci_data_analysis/pytest_fixtures.py @@ -449,7 +449,7 @@ def dispatcher_test_conf_with_gallery_fn(dispatcher_test_conf_fn): f.write(f_default.read()) f.write('\n product_gallery_options:' - '\n product_gallery_url: "http://cdciweb02.isdc.unige.ch/mmoda/galleryd"' + '\n product_gallery_url: "http://cdciweb02.isdc.unige.ch/mmoda/gallery-dev"' f'\n product_gallery_secret_key: "{os.getenv("DISPATCHER_PRODUCT_GALLERY_SECRET_KEY", "secret_key")}"' '\n product_gallery_timezone: "Europe/Zurich"' '\n name_resolver_url: "https://resolver-prod.obsuks1.unige.ch/api/v1.1/byname/{}"' @@ -468,7 +468,7 @@ def dispatcher_test_conf_with_gallery_no_resolver_fn(dispatcher_test_conf_fn): f.write(f_default.read()) f.write('\n product_gallery_options:' - '\n product_gallery_url: "http://cdciweb02.isdc.unige.ch/mmoda/galleryd"' + '\n product_gallery_url: "http://cdciweb02.isdc.unige.ch/mmoda/gallery-dev"' '\n product_gallery_timezone: "Europe/Zurich"' f'\n product_gallery_secret_key: "{os.getenv("DISPATCHER_PRODUCT_GALLERY_SECRET_KEY", "secret_key")}"') From ad600b6aa24fcba33163ed9e38d60e6eee7abc17 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 20 Sep 2023 09:59:36 +0200 Subject: [PATCH 12/12] back on galleryd --- cdci_data_analysis/pytest_fixtures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdci_data_analysis/pytest_fixtures.py b/cdci_data_analysis/pytest_fixtures.py index 0216cb7cc..6371da90a 100644 --- a/cdci_data_analysis/pytest_fixtures.py +++ b/cdci_data_analysis/pytest_fixtures.py @@ -449,7 +449,7 @@ def dispatcher_test_conf_with_gallery_fn(dispatcher_test_conf_fn): f.write(f_default.read()) f.write('\n product_gallery_options:' - '\n product_gallery_url: "http://cdciweb02.isdc.unige.ch/mmoda/gallery-dev"' + '\n product_gallery_url: "http://cdciweb02.isdc.unige.ch/mmoda/galleryd"' f'\n product_gallery_secret_key: "{os.getenv("DISPATCHER_PRODUCT_GALLERY_SECRET_KEY", "secret_key")}"' '\n product_gallery_timezone: "Europe/Zurich"' '\n name_resolver_url: "https://resolver-prod.obsuks1.unige.ch/api/v1.1/byname/{}"' @@ -468,7 +468,7 @@ def dispatcher_test_conf_with_gallery_no_resolver_fn(dispatcher_test_conf_fn): f.write(f_default.read()) f.write('\n product_gallery_options:' - '\n product_gallery_url: "http://cdciweb02.isdc.unige.ch/mmoda/gallery-dev"' + '\n product_gallery_url: "http://cdciweb02.isdc.unige.ch/mmoda/galleryd"' '\n product_gallery_timezone: "Europe/Zurich"' f'\n product_gallery_secret_key: "{os.getenv("DISPATCHER_PRODUCT_GALLERY_SECRET_KEY", "secret_key")}"')