diff --git a/cdci_data_analysis/flask_app/dispatcher_query.py b/cdci_data_analysis/flask_app/dispatcher_query.py index c3f0eee48..45b72dd5b 100644 --- a/cdci_data_analysis/flask_app/dispatcher_query.py +++ b/cdci_data_analysis/flask_app/dispatcher_query.py @@ -496,56 +496,62 @@ def inspect_state(app): def read_scratch_dir(scratch_dir, include_session_log=False): result = {} + file_list = [] + for f in glob.glob(os.path.join(scratch_dir, "*")): + file_list.append(f) + result['file_list'] = file_list + try: fn = os.path.join(scratch_dir, 'analysis_parameters.json') result['analysis_parameters'] = json.load(open(fn)) except Exception as e: # write something logger.warning('unable to read: %s', fn) - return {'error': f'problem reading {fn}: {repr(e)}'} + # return {'error': f'problem reading {fn}: {repr(e)}'} + result['analysis_parameters'] = f'problem reading {fn}: {repr(e)}' if include_session_log: - result['analysis_parameters']['session_log'] = '' + result['session_log'] = '' session_log_fn = os.path.join(scratch_dir, 'session.log') if os.path.exists(session_log_fn): with open(session_log_fn) as session_log_fn_f: - result['analysis_parameters']['session_log'] = session_log_fn_f.read() + result['session_log'] = session_log_fn_f.read() if 'token' in result['analysis_parameters']: result['analysis_parameters']['token'] = tokenHelper.get_decoded_token( result['analysis_parameters']['token'], secret_key=None, validate_token=False) - result['analysis_parameters']['email_history'] = [] + result['email_history'] = [] for email in glob.glob(os.path.join(scratch_dir, 'email_history/*')): ctime = os.stat(email).st_ctime, - result['analysis_parameters']['email_history'].append(dict( + result['email_history'].append(dict( ctime=ctime, ctime_isot=time_.strftime("%Y-%m-%dT%H:%M:%S", time_.gmtime(os.stat(email).st_ctime)), fn=email, )) - result['analysis_parameters']['matrix_message_history'] = [] + result['matrix_message_history'] = [] for msg in glob.glob(os.path.join(scratch_dir, 'matrix_message_history/*')): ctime = os.stat(msg).st_ctime, - result['analysis_parameters']['matrix_message_history'].append(dict( + result['matrix_message_history'].append(dict( ctime=ctime, ctime_isot=time_.strftime("%Y-%m-%dT%H:%M:%S", time_.gmtime(os.stat(msg).st_ctime)), fn=msg, )) - result['analysis_parameters']['fits_files'] = [] + result['fits_files'] = [] for fits_fn in glob.glob(os.path.join(scratch_dir, '*fits*')): ctime = os.stat(fits_fn).st_ctime - result['analysis_parameters']['fits_files'].append(dict( + result['fits_files'].append(dict( ctime=ctime, ctime_isot=time_.strftime("%Y-%m-%dT%H:%M:%S", time_.gmtime(ctime)), fn=fits_fn, )) - result['analysis_parameters']['job_monitor'] = [] + result['job_monitor'] = [] for fn in glob.glob(os.path.join(scratch_dir, 'job_monitor*')): ctime = os.stat(fn).st_ctime - result['analysis_parameters']['job_monitor'].append(dict( + result['job_monitor'].append(dict( ctime=ctime, ctime_isot=time_.strftime("%Y-%m-%dT%H:%M:%S", time_.gmtime(ctime)), fn=fn, diff --git a/tests/test_job_management.py b/tests/test_job_management.py index 27f5395e5..0f5ffe0b2 100644 --- a/tests/test_job_management.py +++ b/tests/test_job_management.py @@ -2479,7 +2479,8 @@ def test_free_up_space(dispatcher_live_fixture, number_folders_to_delete, soft_m @pytest.mark.parametrize("request_cred", ['public', 'private', 'invalid_token']) @pytest.mark.parametrize("roles", ["general, job manager", "administrator", ""]) @pytest.mark.parametrize("include_session_log", [True, False, None]) -def test_inspect_status(dispatcher_live_fixture, request_cred, roles, include_session_log): +@pytest.mark.parametrize("remove_analysis_parameters_json", [True, False]) +def test_inspect_status(dispatcher_live_fixture, request_cred, roles, include_session_log, remove_analysis_parameters_json): required_roles = ['job manager'] DispatcherJobState.remove_scratch_folders() @@ -2521,6 +2522,8 @@ def test_inspect_status(dispatcher_live_fixture, request_cred, roles, include_se session_id = jdata['session_id'] scratch_dir_fn = f'scratch_sid_{session_id}_jid_{job_id}' + if remove_analysis_parameters_json: + os.remove(os.path.join(scratch_dir_fn, "analysis_parameters.json")) scratch_dir_ctime = os.stat(scratch_dir_fn).st_ctime assert os.path.exists(scratch_dir_fn) @@ -2565,15 +2568,21 @@ def test_inspect_status(dispatcher_live_fixture, request_cred, roles, include_se assert jdata['records'][0]['ctime'] == scratch_dir_ctime assert jdata['records'][0]['mtime'] == scratch_dir_mtime - assert 'email_history' in jdata['records'][0]['analysis_parameters'] - assert 'matrix_message_history' in jdata['records'][0]['analysis_parameters'] + assert 'analysis_parameters' in jdata['records'][0] + if remove_analysis_parameters_json: + assert jdata['records'][0]['analysis_parameters'] == f"problem reading {os.path.join(scratch_dir_fn, 'analysis_parameters.json')}: FileNotFoundError(2, 'No such file or directory')" + assert 'email_history' in jdata['records'][0] + assert 'matrix_message_history' in jdata['records'][0] - assert len(jdata['records'][0]['analysis_parameters']['email_history']) == 0 - assert len(jdata['records'][0]['analysis_parameters']['matrix_message_history']) == 0 + assert len(jdata['records'][0]['email_history']) == 0 + assert len(jdata['records'][0]['matrix_message_history']) == 0 if include_session_log: - assert 'session_log' in jdata['records'][0]['analysis_parameters'] + assert 'session_log' in jdata['records'][0] else: - assert 'session_log' not in jdata['records'][0]['analysis_parameters'] + assert 'session_log' not in jdata['records'][0] + + assert 'file_list' in jdata['records'][0] + assert isinstance(jdata['records'][0]['file_list'], list) @pytest.mark.parametrize("request_cred", ['public', 'valid_token', 'invalid_token'])