Skip to content

Commit

Permalink
Merge pull request #588 from oda-hub/get-products-conditions-endpoints
Browse files Browse the repository at this point in the history 
Get products with conditions endpoint
  • Loading branch information
@burnout87
burnout87 authored Sep 22, 2023
2 parents e0a5b04 + ad600b6 commit 96cc9ce
Show file tree
Hide file tree
Showing 3 changed files with 240 additions and 1 deletion.
46 changes: 46 additions & 0 deletions cdci_data_analysis/analysis/drupal_helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -840,6 +840,52 @@ def get_all_revolutions(product_gallery_url, gallery_jwt_token, sentry_dsn=None)
return entities


def get_data_product_list_by_source_name_with_conditions(product_gallery_url, gallery_jwt_token,
sentry_dsn=None,
**kwargs
) -> Optional[list]:
headers = get_drupal_request_headers(gallery_jwt_token)
product_list = []
# if src_name is None:
# source_entity_id = "all"
# else:
# source_entity_list = get_source_astrophysical_entity_info_by_source_and_alternative_name(product_gallery_url,
# gallery_jwt_token,
# source_name=src_name,
# sentry_dsn=sentry_dsn)

# source_entity_id = None
# if len(source_entity_list) >= 1:
# source_entity_id = source_entity_list[0]['nid']

request_url = f"{product_gallery_url}/data_products/source_products_conditions"

params = {"_format": "hal_json"}

for k, v in kwargs.items():
# the machine name of the field in drupal starts by default with field_
field_name = str.lower('field_' + k)
params[field_name] = v

# if source_entity_id is not None:
log_res = execute_drupal_request(request_url,
params=params,
headers=headers,
sentry_dsn=sentry_dsn)
output_get = analyze_drupal_output(log_res, operation_performed="retrieving list of data products with conditions")
if isinstance(output_get, list):
for obj in output_get:
refactored_obj = {}
for k, v in obj.items():
refactored_key = k
if k.startswith('field_'):
refactored_key = k.replace('field_', '')
refactored_obj[refactored_key] = v
product_list.append(refactored_obj)

return product_list


def get_data_product_list_by_source_name(product_gallery_url, gallery_jwt_token, src_name=None, sentry_dsn=None) -> Optional[list]:
product_list = []
if src_name is None:
Expand Down
42 changes: 41 additions & 1 deletion cdci_data_analysis/flask_app/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -800,7 +800,47 @@ def get_astro_entity_info_by_source_name():
return refactored_astro_entity_info


@app.route('/get_data_product_list_with_conditions', methods=['GET'])
def get_data_product_list_with_conditions():
logger.info("request.args: %s ", request.args)
logger.info("request.files: %s ", request.files)

par_dic = request.values.to_dict()
token = par_dic.pop('token', None)
app_config = app.config.get('conf')
secret_key = app_config.secret_key

output, output_code = tokenHelper.validate_token_from_request(token=token, secret_key=secret_key,
required_roles=['gallery contributor'],
action="getting all the astro entities from the product gallery")

if output_code is not None:
return make_response(output, output_code)
decoded_token = output

sentry_dsn = sentry.sentry_url

gallery_secret_key = app_config.product_gallery_secret_key
product_gallery_url = app_config.product_gallery_url
user_email = tokenHelper.get_token_user_email_address(decoded_token)
user_id_product_creator = drupal_helper.get_user_id(product_gallery_url=product_gallery_url,
user_email=user_email,
sentry_dsn=sentry_dsn)
# update the token
gallery_jwt_token = drupal_helper.generate_gallery_jwt_token(gallery_secret_key, user_id=user_id_product_creator)

# src_name = par_dic.pop('src_name', None)

output_get = drupal_helper.get_data_product_list_by_source_name_with_conditions(product_gallery_url=product_gallery_url,
gallery_jwt_token=gallery_jwt_token,
sentry_dsn=sentry_dsn,
**par_dic)
output_list = json.dumps(output_get)

return output_list


# TODO to refactor using get_data_product_list_with_conditions
@app.route('/get_data_product_list_by_source_name', methods=['GET'])
def get_data_product_list_by_source_name():
logger.info("request.args: %s ", request.args)
Expand Down Expand Up @@ -835,7 +875,7 @@ def get_data_product_list_by_source_name():
src_name = request.args.get('src_name', None)

output_get = drupal_helper.get_data_product_list_by_source_name(product_gallery_url=product_gallery_url,
gallery_jwt_token=gallery_jwt_token,
gallery_jwt_token=gallery_jwt_token,
src_name=src_name,
sentry_dsn=sentry_dsn)
output_list = json.dumps(output_get)
Expand Down
153 changes: 153 additions & 0 deletions tests/test_server_basic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2283,6 +2283,159 @@ def test_product_gallery_get_all_astro_entities(dispatcher_live_fixture_with_gal
assert any(src['title'] == params['src_name'] for src in drupal_res_obj)


@pytest.mark.test_drupal
@pytest.mark.parametrize("source_name", ["new", "known", "unknown"])
@pytest.mark.parametrize("include_products_fields_conditions", [True, False])
def test_product_gallery_get_data_products_list_with_conditions(dispatcher_live_fixture_with_gallery, dispatcher_test_conf_with_gallery, source_name, include_products_fields_conditions):
server = dispatcher_live_fixture_with_gallery

logger.info("constructed server: %s", server)

# let's generate a valid token
token_payload = {
**default_token_payload,
"roles": "general, gallery contributor",
}
encoded_token = jwt.encode(token_payload, secret_key, algorithm='HS256')
instrument_name = 'isgri'
product_type = 'isgri_image'
instrument_query = 'isgri'
product_type_query = 'image'

if source_name == 'new':
source_name = 'test astro entity' + '_' + str(uuid.uuid4())
# let's create a source
source_params = {
'token': encoded_token,
'src_name': source_name
}

c = requests.post(os.path.join(server, "post_astro_entity_to_gallery"),
params={**source_params},
)

assert c.status_code == 200

# let's post a product with the source just created
product_params = {
'instrument': instrument_name,
'product_type': product_type,
'E1_keV': 150,
'E2_keV': 350,
'src_name': source_name,
'content_type': 'data_product',
'token': encoded_token,
'insert_new_source': True,
'T1': '2022-07-21T00:29:47',
'T2': '2022-08-23T05:29:11'
}
c = requests.post(os.path.join(server, "post_product_to_gallery"),
params={**product_params}
)

assert c.status_code == 200

params = {
'token': encoded_token,
'src_name': source_name,
'instrument_name': instrument_query,
'product_type': product_type_query
}
if include_products_fields_conditions:
for e1_kev, e2_kev, rev1, rev2 in [
(100, 350, 2528, 2540),
(100, 350, 2526, 2541),
(100, 350, 2529, 2539),
(100, 350, 2529, 2541),
(100, 350, 2527, 2539),
(50, 400, 2528, 2540),
(200, 350, 2528, 2540),
(200, 350, 2528, 2540),
(50, 300, 2528, 2540),
]:
logger.info(f"testing with e1_kev_value {e1_kev}, e2_kev_value {e2_kev}")
params['e1_kev_value'] = e1_kev
params['e2_kev_value'] = e2_kev

params['rev1_value'] = rev1
params['rev2_value'] = rev2

c = requests.get(os.path.join(server, "get_data_product_list_with_conditions"),
params=params
)

assert c.status_code == 200
drupal_res_obj = c.json()
assert isinstance(drupal_res_obj, list)

if e1_kev > 100 or e2_kev < 350 or rev1 > 2528 or rev2 < 2540:
assert len(drupal_res_obj) == 0
else:
assert len(drupal_res_obj) == 1
else:
c = requests.get(os.path.join(server, "get_data_product_list_with_conditions"),
params=params
)

assert c.status_code == 200
drupal_res_obj = c.json()
assert isinstance(drupal_res_obj, list)
assert len(drupal_res_obj) == 1
elif source_name == 'unknown':
source_name = "aaaaaaaaaaaaaaaaa"
params = {
'token': encoded_token,
'src_name': source_name
}
c = requests.get(os.path.join(server, "get_data_product_list_with_conditions"),
params=params
)

assert c.status_code == 200
drupal_res_obj = c.json()
assert isinstance(drupal_res_obj, list)
assert len(drupal_res_obj) == 0
else:
source_name = "V404 Cyg"
params = {
'token': encoded_token,
'src_name': source_name
}
c = requests.get(os.path.join(server, "get_data_product_list_with_conditions"),
params=params
)

assert c.status_code == 200
drupal_res_obj_source_name = c.json()
assert isinstance(drupal_res_obj_source_name, list)

source_name = "1RXS J202405.3+335157"
params = {
'token': encoded_token,
'src_name': source_name
}
c = requests.get(os.path.join(server, "get_data_product_list_with_conditions"),
params=params
)

assert c.status_code == 200
drupal_res_obj_alternative_name = c.json()
assert isinstance(drupal_res_obj_alternative_name, list)

assert len(drupal_res_obj_alternative_name) == len(drupal_res_obj_source_name)

# Create sets of dictionaries
set1 = set(map(lambda d: frozenset(d.items()), drupal_res_obj_source_name))
set2 = set(map(lambda d: frozenset(d.items()), drupal_res_obj_alternative_name))

# Find the differences
diff1 = set1 - set2
diff2 = set2 - set1

assert diff2 == set()
assert diff1 == set()


@pytest.mark.test_drupal
@pytest.mark.parametrize("source_name", ["new", "known"])
def test_product_gallery_get_data_products_list_for_given_source(dispatcher_live_fixture_with_gallery, dispatcher_test_conf_with_gallery, source_name):
Expand Down

0 comments on commit 96cc9ce

Please sign in to comment.