From c03dd75cbd46a8e4e14124fcd516d676413ab0ce Mon Sep 17 00:00:00 2001 From: gadorlhiac Date: Mon, 2 Dec 2024 14:54:51 -0800 Subject: [PATCH 1/4] MNT Add option to specify detector image sum algorithms in smalldata template. --- config/templates/smd_producer_template.py | 17 +++++++++++++++++ config/test.yaml | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/config/templates/smd_producer_template.py b/config/templates/smd_producer_template.py index 19103b51..acc99dd9 100644 --- a/config/templates/smd_producer_template.py +++ b/config/templates/smd_producer_template.py @@ -431,7 +431,24 @@ def define_dets(run): if detname in svd: det.addFunc(svdFit(**svd[detname])) + {%- if detSumAlgos is defined %} + {%- for detector, det_algos in detSumAlgos.items() %} + {%- if detector == "all" %} + {%- for algo in det_algos %} + det.storeSum(sumAlgo='{{ algo }}') + {% endfor %} + {% else %} + if detname.find("{{ detector }}") >= 0: + {% for algo in det_algos %} + det.storeSum(sumAlgo='{{ algo }}') + {% endfor %} + {% endif %} + {% endfor %} + {% else %} det.storeSum(sumAlgo='calib') + det.storeSum(sumAlgo='calib_dropped') + det.storeSum(sumAlgo='calib_dropped_square') + {% endif %} #det.storeSum(sumAlgo='calib_img') dets.append(det) return dets diff --git a/config/test.yaml b/config/test.yaml index 8ec059a7..f9c8a500 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -40,6 +40,23 @@ SubmitSMD: #detnames: [] #epicsPV: [] #ttCalib: [] + # Provide detector image sum algorithms. + # If detSumAlgos is not defined, it will default to calib, calib_dropped and + # calib_dropped_square for every detector. You can add processing algorithms for + # single detectors (e.g. epix10k2M or Rayonix) as below, or add algorithms which + # will apply to every detector defined in detnames by placing them under "all" + #detSumAlgos: + # all: + # - "calib" + # - "calib_dropped" + # - "calib_dropped_square" + # - "calib_thresADU1" + # epix10k2M: + # - "calib_thresADU5" + # - "calib_max" + # Rayonix: + # - "calib_skipFirst_thresADU1" + # - "calib_skipFirst_max" #getROIs: # jungfrau1M: # Change to detector name # ROIs: [[[1, 2], [157, 487], [294, 598]]] From f6324f3c7d8d2d27c74fcefeea5535baca4b516c Mon Sep 17 00:00:00 2001 From: gadorlhiac Date: Tue, 3 Dec 2024 11:48:01 -0800 Subject: [PATCH 2/4] MNT Update hutch-specific config YAMLs for smalldata template change. --- config/mfx.yaml | 17 +++++++++++++++++ config/xcs.yaml | 17 +++++++++++++++++ config/xpp.yaml | 17 +++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/config/mfx.yaml b/config/mfx.yaml index 67e5a7b8..365eb390 100644 --- a/config/mfx.yaml +++ b/config/mfx.yaml @@ -119,6 +119,23 @@ SubmitSMD: #detnames: [] #epicsPV: [] #ttCalib: [] + # Provide detector image sum algorithms. + # If detSumAlgos is not defined, it will default to calib, calib_dropped and + # calib_dropped_square for every detector. You can add processing algorithms for + # single detectors (e.g. epix10k2M or Rayonix) as below, or add algorithms which + # will apply to every detector defined in detnames by placing them under "all" + #detSumAlgos: + # all: + # - "calib" + # - "calib_dropped" + # - "calib_dropped_square" + # - "calib_thresADU1" + # epix10k2M: + # - "calib_thresADU5" + # - "calib_max" + # Rayonix: + # - "calib_skipFirst_thresADU1" + # - "calib_skipFirst_max" #getROIs: # jungfrau1M: # Change to detector name # ROIs: [[[1, 2], [157, 487], [294, 598]]] diff --git a/config/xcs.yaml b/config/xcs.yaml index 890647e1..d85f1895 100644 --- a/config/xcs.yaml +++ b/config/xcs.yaml @@ -106,6 +106,23 @@ SubmitSMD: #detnames: [] #epicsPV: [] #ttCalib: [] + # Provide detector image sum algorithms. + # If detSumAlgos is not defined, it will default to calib, calib_dropped and + # calib_dropped_square for every detector. You can add processing algorithms for + # single detectors (e.g. epix10k2M or Rayonix) as below, or add algorithms which + # will apply to every detector defined in detnames by placing them under "all" + #detSumAlgos: + # all: + # - "calib" + # - "calib_dropped" + # - "calib_dropped_square" + # - "calib_thresADU1" + # epix10k2M: + # - "calib_thresADU5" + # - "calib_max" + # Rayonix: + # - "calib_skipFirst_thresADU1" + # - "calib_skipFirst_max" #getROIs: # jungfrau1M: # Change to detector name # ROIs: [[[1, 2], [157, 487], [294, 598]]] diff --git a/config/xpp.yaml b/config/xpp.yaml index 00a5bdfd..60f0f0ca 100644 --- a/config/xpp.yaml +++ b/config/xpp.yaml @@ -106,6 +106,23 @@ SubmitSMD: #detnames: [] #epicsPV: [] #ttCalib: [] + # Provide detector image sum algorithms. + # If detSumAlgos is not defined, it will default to calib, calib_dropped and + # calib_dropped_square for every detector. You can add processing algorithms for + # single detectors (e.g. epix10k2M or Rayonix) as below, or add algorithms which + # will apply to every detector defined in detnames by placing them under "all" + #detSumAlgos: + # all: + # - "calib" + # - "calib_dropped" + # - "calib_dropped_square" + # - "calib_thresADU1" + # epix10k2M: + # - "calib_thresADU5" + # - "calib_max" + # Rayonix: + # - "calib_skipFirst_thresADU1" + # - "calib_skipFirst_max" #getROIs: # jungfrau1M: # Change to detector name # ROIs: [[[1, 2], [157, 487], [294, 598]]] From 8e246947071bf7f0d893be486c6e4b27482d51d4 Mon Sep 17 00:00:00 2001 From: gadorlhiac Date: Tue, 3 Dec 2024 12:21:09 -0800 Subject: [PATCH 3/4] MNT mypy typing --- utilities/src/dbview/dbview.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utilities/src/dbview/dbview.py b/utilities/src/dbview/dbview.py index 95f7287b..f07b00fa 100644 --- a/utilities/src/dbview/dbview.py +++ b/utilities/src/dbview/dbview.py @@ -83,7 +83,7 @@ def pull_table_data(self, table: DataTable) -> DataTable: return table def action_toggle_dark(self) -> None: - self.dark = not self.dark + self.dark: bool = not self.dark if __name__ == "__main__": From 0c3a87f70d844cdccc5f4fd9c9c3ea29fe59f41a Mon Sep 17 00:00:00 2001 From: gadorlhiac Date: Wed, 4 Dec 2024 08:57:13 -0800 Subject: [PATCH 4/4] MNT Add explicit close to smd template to suppress warning. --- config/templates/smd_producer_template.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/templates/smd_producer_template.py b/config/templates/smd_producer_template.py index acc99dd9..7e8a26d3 100644 --- a/config/templates/smd_producer_template.py +++ b/config/templates/smd_producer_template.py @@ -1002,7 +1002,7 @@ def get_sd_file(write_dir, exp, hutch): dets_time_end = (end_setup_dets-start_job)/60 evt_time_start = (start_evt_loop-start_job)/60 evt_time_end = (end_evt_loop-start_job)/60 -logger.debug(f"##### Timing benchmarks core {ds.rank}: ##### """) +logger.debug(f"##### Timing benchmarks core {ds.rank}: #####") logger.debug(f'Setup dets: \n\tStart: {dets_time_start:.2f} min\n\tEnd: {dets_time_end:.2f} min') logger.debug(f'\tDuration:{dets_time_end-dets_time_start:.2f}') logger.debug(f'Event loop: \n\tStart: {evt_time_start:.2f} min\n\tEnd: {evt_time_end:.2f} min') @@ -1018,6 +1018,7 @@ def get_sd_file(write_dir, exp, hutch): #finishing up here.... logger.debug('rank {0} on {1} is finished'.format(ds.rank, hostname)) small_data.save() +small_data.close() if os.environ.get('ARP_JOB_ID', None) is not None: if ds.size > 1: if ds.rank == 0: @@ -1053,7 +1054,7 @@ def get_sd_file(write_dir, exp, hutch): else: with open('/cds/home/opr/%s/forElogPost.txt'%user,'r') as reader: answer = reader.readline() - r = requests.post(ws_url, params={"run_num": args.run}, json=runtable_data, + r = requests.post(ws_url, params={"run_num": args.run}, json=runtable_data, auth=HTTPBasicAuth(args.experiment[:3]+'opr', answer[:-1])) print(r) if det_presence!={}: