From e8f678e96c329fdf0c5523bcb68fcedd61981d03 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 23 Oct 2023 14:53:45 +0200 Subject: [PATCH 1/4] including list of file inside scratch_dir --- cdci_data_analysis/flask_app/dispatcher_query.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cdci_data_analysis/flask_app/dispatcher_query.py b/cdci_data_analysis/flask_app/dispatcher_query.py index c3f0eee48..2e04a6a68 100644 --- a/cdci_data_analysis/flask_app/dispatcher_query.py +++ b/cdci_data_analysis/flask_app/dispatcher_query.py @@ -496,13 +496,19 @@ def inspect_state(app): def read_scratch_dir(scratch_dir, include_session_log=False): result = {} + file_list = [] + for f in glob.glob(os.path.join(scratch_dir, "*")): + file_list.append(f) + result['file_list'] = file_list + try: fn = os.path.join(scratch_dir, 'analysis_parameters.json') result['analysis_parameters'] = json.load(open(fn)) except Exception as e: # write something logger.warning('unable to read: %s', fn) - return {'error': f'problem reading {fn}: {repr(e)}'} + # return {'error': f'problem reading {fn}: {repr(e)}'} + result['analysis_parameters'] = f'problem reading {fn}: {repr(e)}' if include_session_log: result['analysis_parameters']['session_log'] = '' From cd6785208f92e9a7d1392ec9727ee6331f88aac1 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 23 Oct 2023 14:58:18 +0200 Subject: [PATCH 2/4] extended test --- tests/test_job_management.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_job_management.py b/tests/test_job_management.py index 27f5395e5..03285c8fc 100644 --- a/tests/test_job_management.py +++ b/tests/test_job_management.py @@ -2575,6 +2575,9 @@ def test_inspect_status(dispatcher_live_fixture, request_cred, roles, include_se else: assert 'session_log' not in jdata['records'][0]['analysis_parameters'] + assert 'file_list' in jdata['records'][0] + assert isinstance(jdata['records'][0]['file_list'], list) + @pytest.mark.parametrize("request_cred", ['public', 'valid_token', 'invalid_token']) def test_incident_report(dispatcher_live_fixture, dispatcher_local_mail_server, dispatcher_test_conf, request_cred): From 552f4ffd574e86372f76d421c9f091062ca8de8c Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 23 Oct 2023 20:21:24 +0200 Subject: [PATCH 3/4] include error details in case analysis_parameters.json is not present, rest of folder content is in separate fields --- .../flask_app/dispatcher_query.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cdci_data_analysis/flask_app/dispatcher_query.py b/cdci_data_analysis/flask_app/dispatcher_query.py index 2e04a6a68..45b72dd5b 100644 --- a/cdci_data_analysis/flask_app/dispatcher_query.py +++ b/cdci_data_analysis/flask_app/dispatcher_query.py @@ -511,47 +511,47 @@ def read_scratch_dir(scratch_dir, include_session_log=False): result['analysis_parameters'] = f'problem reading {fn}: {repr(e)}' if include_session_log: - result['analysis_parameters']['session_log'] = '' + result['session_log'] = '' session_log_fn = os.path.join(scratch_dir, 'session.log') if os.path.exists(session_log_fn): with open(session_log_fn) as session_log_fn_f: - result['analysis_parameters']['session_log'] = session_log_fn_f.read() + result['session_log'] = session_log_fn_f.read() if 'token' in result['analysis_parameters']: result['analysis_parameters']['token'] = tokenHelper.get_decoded_token( result['analysis_parameters']['token'], secret_key=None, validate_token=False) - result['analysis_parameters']['email_history'] = [] + result['email_history'] = [] for email in glob.glob(os.path.join(scratch_dir, 'email_history/*')): ctime = os.stat(email).st_ctime, - result['analysis_parameters']['email_history'].append(dict( + result['email_history'].append(dict( ctime=ctime, ctime_isot=time_.strftime("%Y-%m-%dT%H:%M:%S", time_.gmtime(os.stat(email).st_ctime)), fn=email, )) - result['analysis_parameters']['matrix_message_history'] = [] + result['matrix_message_history'] = [] for msg in glob.glob(os.path.join(scratch_dir, 'matrix_message_history/*')): ctime = os.stat(msg).st_ctime, - result['analysis_parameters']['matrix_message_history'].append(dict( + result['matrix_message_history'].append(dict( ctime=ctime, ctime_isot=time_.strftime("%Y-%m-%dT%H:%M:%S", time_.gmtime(os.stat(msg).st_ctime)), fn=msg, )) - result['analysis_parameters']['fits_files'] = [] + result['fits_files'] = [] for fits_fn in glob.glob(os.path.join(scratch_dir, '*fits*')): ctime = os.stat(fits_fn).st_ctime - result['analysis_parameters']['fits_files'].append(dict( + result['fits_files'].append(dict( ctime=ctime, ctime_isot=time_.strftime("%Y-%m-%dT%H:%M:%S", time_.gmtime(ctime)), fn=fits_fn, )) - result['analysis_parameters']['job_monitor'] = [] + result['job_monitor'] = [] for fn in glob.glob(os.path.join(scratch_dir, 'job_monitor*')): ctime = os.stat(fn).st_ctime - result['analysis_parameters']['job_monitor'].append(dict( + result['job_monitor'].append(dict( ctime=ctime, ctime_isot=time_.strftime("%Y-%m-%dT%H:%M:%S", time_.gmtime(ctime)), fn=fn, From f6c7b2123dfc5a773de572873d85aa645858c786 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Mon, 23 Oct 2023 20:21:34 +0200 Subject: [PATCH 4/4] adapted and extended test --- tests/test_job_management.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/test_job_management.py b/tests/test_job_management.py index 03285c8fc..0f5ffe0b2 100644 --- a/tests/test_job_management.py +++ b/tests/test_job_management.py @@ -2479,7 +2479,8 @@ def test_free_up_space(dispatcher_live_fixture, number_folders_to_delete, soft_m @pytest.mark.parametrize("request_cred", ['public', 'private', 'invalid_token']) @pytest.mark.parametrize("roles", ["general, job manager", "administrator", ""]) @pytest.mark.parametrize("include_session_log", [True, False, None]) -def test_inspect_status(dispatcher_live_fixture, request_cred, roles, include_session_log): +@pytest.mark.parametrize("remove_analysis_parameters_json", [True, False]) +def test_inspect_status(dispatcher_live_fixture, request_cred, roles, include_session_log, remove_analysis_parameters_json): required_roles = ['job manager'] DispatcherJobState.remove_scratch_folders() @@ -2521,6 +2522,8 @@ def test_inspect_status(dispatcher_live_fixture, request_cred, roles, include_se session_id = jdata['session_id'] scratch_dir_fn = f'scratch_sid_{session_id}_jid_{job_id}' + if remove_analysis_parameters_json: + os.remove(os.path.join(scratch_dir_fn, "analysis_parameters.json")) scratch_dir_ctime = os.stat(scratch_dir_fn).st_ctime assert os.path.exists(scratch_dir_fn) @@ -2565,15 +2568,18 @@ def test_inspect_status(dispatcher_live_fixture, request_cred, roles, include_se assert jdata['records'][0]['ctime'] == scratch_dir_ctime assert jdata['records'][0]['mtime'] == scratch_dir_mtime - assert 'email_history' in jdata['records'][0]['analysis_parameters'] - assert 'matrix_message_history' in jdata['records'][0]['analysis_parameters'] + assert 'analysis_parameters' in jdata['records'][0] + if remove_analysis_parameters_json: + assert jdata['records'][0]['analysis_parameters'] == f"problem reading {os.path.join(scratch_dir_fn, 'analysis_parameters.json')}: FileNotFoundError(2, 'No such file or directory')" + assert 'email_history' in jdata['records'][0] + assert 'matrix_message_history' in jdata['records'][0] - assert len(jdata['records'][0]['analysis_parameters']['email_history']) == 0 - assert len(jdata['records'][0]['analysis_parameters']['matrix_message_history']) == 0 + assert len(jdata['records'][0]['email_history']) == 0 + assert len(jdata['records'][0]['matrix_message_history']) == 0 if include_session_log: - assert 'session_log' in jdata['records'][0]['analysis_parameters'] + assert 'session_log' in jdata['records'][0] else: - assert 'session_log' not in jdata['records'][0]['analysis_parameters'] + assert 'session_log' not in jdata['records'][0] assert 'file_list' in jdata['records'][0] assert isinstance(jdata['records'][0]['file_list'], list)