Merge pull request #613 from oda-hub/include-file-list-inspect_state

Include file list inspect state

cdci_data_analysis/flask_app/dispatcher_query.py

@@ -496,56 +496,62 @@ def inspect_state(app):
     def read_scratch_dir(scratch_dir, include_session_log=False):
         result = {}
 
+        file_list = []
+        for f in glob.glob(os.path.join(scratch_dir, "*")):
+            file_list.append(f)
+        result['file_list'] = file_list
+
         try:
             fn = os.path.join(scratch_dir, 'analysis_parameters.json')
             result['analysis_parameters'] = json.load(open(fn))
         except Exception as e:
             # write something
             logger.warning('unable to read: %s', fn)
-            return {'error': f'problem reading {fn}: {repr(e)}'}
+            # return {'error': f'problem reading {fn}: {repr(e)}'}
+            result['analysis_parameters'] = f'problem reading {fn}: {repr(e)}'
 
         if include_session_log:
-            result['analysis_parameters']['session_log'] = ''
+            result['session_log'] = ''
             session_log_fn = os.path.join(scratch_dir, 'session.log')
             if os.path.exists(session_log_fn):
                 with open(session_log_fn) as session_log_fn_f:
-                    result['analysis_parameters']['session_log'] = session_log_fn_f.read()
+                    result['session_log'] = session_log_fn_f.read()
 
         if 'token' in result['analysis_parameters']:
             result['analysis_parameters']['token'] = tokenHelper.get_decoded_token(
                 result['analysis_parameters']['token'], secret_key=None, validate_token=False)
 
-        result['analysis_parameters']['email_history'] = []
+        result['email_history'] = []
         for email in glob.glob(os.path.join(scratch_dir, 'email_history/*')):
             ctime = os.stat(email).st_ctime,
-            result['analysis_parameters']['email_history'].append(dict(
+            result['email_history'].append(dict(
                 ctime=ctime,
                 ctime_isot=time_.strftime("%Y-%m-%dT%H:%M:%S", time_.gmtime(os.stat(email).st_ctime)),
                 fn=email,
             ))
 
-        result['analysis_parameters']['matrix_message_history'] = []
+        result['matrix_message_history'] = []
         for msg in glob.glob(os.path.join(scratch_dir, 'matrix_message_history/*')):
             ctime = os.stat(msg).st_ctime,
-            result['analysis_parameters']['matrix_message_history'].append(dict(
+            result['matrix_message_history'].append(dict(
                 ctime=ctime,
                 ctime_isot=time_.strftime("%Y-%m-%dT%H:%M:%S", time_.gmtime(os.stat(msg).st_ctime)),
                 fn=msg,
             ))
 
-        result['analysis_parameters']['fits_files'] = []
+        result['fits_files'] = []
         for fits_fn in glob.glob(os.path.join(scratch_dir, '*fits*')):
             ctime = os.stat(fits_fn).st_ctime
-            result['analysis_parameters']['fits_files'].append(dict(
+            result['fits_files'].append(dict(
                 ctime=ctime,
                 ctime_isot=time_.strftime("%Y-%m-%dT%H:%M:%S", time_.gmtime(ctime)),
                 fn=fits_fn,
             ))
 
-        result['analysis_parameters']['job_monitor'] = []
+        result['job_monitor'] = []
         for fn in glob.glob(os.path.join(scratch_dir, 'job_monitor*')):
             ctime = os.stat(fn).st_ctime
-            result['analysis_parameters']['job_monitor'].append(dict(
+            result['job_monitor'].append(dict(
                 ctime=ctime,
                 ctime_isot=time_.strftime("%Y-%m-%dT%H:%M:%S", time_.gmtime(ctime)),
                 fn=fn,

tests/test_job_management.py

@@ -2479,7 +2479,8 @@ def test_free_up_space(dispatcher_live_fixture, number_folders_to_delete, soft_m
 @pytest.mark.parametrize("request_cred", ['public', 'private', 'invalid_token'])
 @pytest.mark.parametrize("roles", ["general, job manager", "administrator", ""])
 @pytest.mark.parametrize("include_session_log", [True, False, None])
-def test_inspect_status(dispatcher_live_fixture, request_cred, roles, include_session_log):
+@pytest.mark.parametrize("remove_analysis_parameters_json", [True, False])
+def test_inspect_status(dispatcher_live_fixture, request_cred, roles, include_session_log, remove_analysis_parameters_json):
     required_roles = ['job manager']
     DispatcherJobState.remove_scratch_folders()
 
@@ -2521,6 +2522,8 @@ def test_inspect_status(dispatcher_live_fixture, request_cred, roles, include_se
     session_id = jdata['session_id']
 
     scratch_dir_fn = f'scratch_sid_{session_id}_jid_{job_id}'
+    if remove_analysis_parameters_json:
+        os.remove(os.path.join(scratch_dir_fn, "analysis_parameters.json"))
     scratch_dir_ctime = os.stat(scratch_dir_fn).st_ctime
 
     assert os.path.exists(scratch_dir_fn)
@@ -2565,15 +2568,21 @@ def test_inspect_status(dispatcher_live_fixture, request_cred, roles, include_se
         assert jdata['records'][0]['ctime'] == scratch_dir_ctime
         assert jdata['records'][0]['mtime'] == scratch_dir_mtime
 
-        assert 'email_history' in jdata['records'][0]['analysis_parameters']
-        assert 'matrix_message_history' in jdata['records'][0]['analysis_parameters']
+        assert 'analysis_parameters' in jdata['records'][0]
+        if remove_analysis_parameters_json:
+            assert jdata['records'][0]['analysis_parameters'] == f"problem reading {os.path.join(scratch_dir_fn, 'analysis_parameters.json')}: FileNotFoundError(2, 'No such file or directory')"
+        assert 'email_history' in jdata['records'][0]
+        assert 'matrix_message_history' in jdata['records'][0]
 
-        assert len(jdata['records'][0]['analysis_parameters']['email_history']) == 0
-        assert len(jdata['records'][0]['analysis_parameters']['matrix_message_history']) == 0
+        assert len(jdata['records'][0]['email_history']) == 0
+        assert len(jdata['records'][0]['matrix_message_history']) == 0
         if include_session_log:
-            assert 'session_log' in jdata['records'][0]['analysis_parameters']
+            assert 'session_log' in jdata['records'][0]
         else:
-            assert 'session_log' not in jdata['records'][0]['analysis_parameters']
+            assert 'session_log' not in jdata['records'][0]
+
+        assert 'file_list' in jdata['records'][0]
+        assert isinstance(jdata['records'][0]['file_list'], list)
 
 
 @pytest.mark.parametrize("request_cred", ['public', 'valid_token', 'invalid_token'])