From 36276e21066120cfddaf3d1f1a2173afa821c7ca Mon Sep 17 00:00:00 2001 From: nstelter-slac Date: Tue, 20 Feb 2024 11:07:25 -0800 Subject: [PATCH 1/6] MNT: move Stats and cluster file to correct location, since are library files --- {standalone_scripts => calibrationSuite}/Stats.py | 0 {standalone_scripts => calibrationSuite}/cluster.py | 0 suite_scripts/CalcNoise.py | 2 +- suite_scripts/CalcNoiseAndMean.py | 2 +- 4 files changed, 2 insertions(+), 2 deletions(-) rename {standalone_scripts => calibrationSuite}/Stats.py (100%) rename {standalone_scripts => calibrationSuite}/cluster.py (100%) diff --git a/standalone_scripts/Stats.py b/calibrationSuite/Stats.py similarity index 100% rename from standalone_scripts/Stats.py rename to calibrationSuite/Stats.py diff --git a/standalone_scripts/cluster.py b/calibrationSuite/cluster.py similarity index 100% rename from standalone_scripts/cluster.py rename to calibrationSuite/cluster.py diff --git a/suite_scripts/CalcNoise.py b/suite_scripts/CalcNoise.py index 9bc3643..00d13e3 100755 --- a/suite_scripts/CalcNoise.py +++ b/suite_scripts/CalcNoise.py @@ -1,5 +1,5 @@ from calibrationSuite.basicSuiteScript import * -import Stats +import calibrationSuite.Stats class CalcNoise(BasicSuiteScript): def __init__(self): diff --git a/suite_scripts/CalcNoiseAndMean.py b/suite_scripts/CalcNoiseAndMean.py index ee9b545..3d3f7f4 100755 --- a/suite_scripts/CalcNoiseAndMean.py +++ b/suite_scripts/CalcNoiseAndMean.py @@ -1,5 +1,5 @@ from calibrationSuite.basicSuiteScript import * -import Stats +import calibrationSuite.Stats class CalcNoise(BasicSuiteScript): def __init__(self): From 19e86067fe832e909fe3bb4a59bb833b55f810d4 Mon Sep 17 00:00:00 2001 From: nstelter-slac Date: Tue, 20 Feb 2024 11:31:32 -0800 Subject: [PATCH 2/6] DOC: Add more info to readme, update bad import statement --- README.md | 39 +++++++++++++++++++++++++ suite_scripts/SimpleClustersParallel.py | 2 +- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1e608d6..3892911 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,48 @@ # beamtime-calibration-suite [![Build Status](https://github.com/slaclab/beamtime-calibration-suite/actions/workflows/run-tests.yml/badge.svg?branch=main)](https://github.com/slaclab/beamtime-calibration-suite/actions/workflows/run-tests.yml) +To run to suite_scripts or use the library in scripts outside the project directory, you need to append the project directory to your PYTHONPATH, for example: +export PYTHONPATH="${PYTHONPATH}:~/beamtime_slaclab/beamtime-calibration-suite" + +(can add this to your ~/.bashrc so persists between terminal sessions. Library can later have +a better release method than cloning and adding to path (pip?)) + +File organization: +/calibrationSuite: The library code lives here, and the functions can be imported into other scripts as such: + +from calibrationSuite.basicSuiteScript import * +from calibrationSuite.fitFunctions import * +from calibrationSuite.Stats import * +from calibrationSuite.cluster import * + +(documentation on the library functionality is still to come, but example usage is seen in the /suite_scripts folder) + +/suite_scripts: scripts that use the calibrationSuite library code + +/standalone_scripts: scripts that do not use the calibrationSuite library code + + +Current Status: + +main branch tag v1.0.0 are the scripts used for the 2/17/24 beamtime + -only changes made are to file organization, and to import statements so work with new organization + -large changes will be merged into ontop of this, but original scripts can be accessed by checking out this tag + -future beamtimes can be tagged as well + + To run tests: + ssh into psana source /sdf/group/lcls/ds/ana/sw/conda2/manage/bin/psconda.sh clone this repo cd beamtime-calibration-suite/tests pytest . + + +Developers: + +If you are new to git/github, start here: https://confluence.slac.stanford.edu/pages/viewpage.action?pageId=428802060 + +Then read the following for an overview of the development process: https://confluence.slac.stanford.edu/pages/viewpage.action?pageId=429562464 + + diff --git a/suite_scripts/SimpleClustersParallel.py b/suite_scripts/SimpleClustersParallel.py index c733600..55a2c59 100644 --- a/suite_scripts/SimpleClustersParallel.py +++ b/suite_scripts/SimpleClustersParallel.py @@ -1,5 +1,5 @@ from calibrationSuite.basicSuiteScript import * -from cluster import Cluster, BuildClusters +from calibrationSuite.cluster import Cluster, BuildClusters class SimpleClusters(BasicSuiteScript): def __init__(self): From b8391537f3e1c36022925541966d62b0c56569df Mon Sep 17 00:00:00 2001 From: nstelter-slac Date: Tue, 20 Feb 2024 12:09:43 -0800 Subject: [PATCH 3/6] WIP: Update readme, enable working fitFunction test and disable non-working test-scripts --- .github/workflows/run-tests.yml | 6 +++--- README.md | 4 +--- standalone_scripts/AnalyzeH5.py | 2 +- tests/{ => disabled}/OffXavierV4_2.npy | Bin tests/{ => disabled}/XavierV4_2.npy | Bin .../disabled_test_AnalyzeH5.py} | 0 .../disabled_test_EventScanParallelSlice.py} | 0 .../AnalyzeH5_r666_c0_r1_c14_testLabel_E.png | Bin .../AnalyzeH5_r666_c0_r1_c14_testLabel_fitInfo.npy | Bin .../AnalyzeH5_r666_c0_testLabel_E.png | Bin ...AnalyzeH5_r666_c0_testLabel_gainDistribution.png | Bin .../EventScanParallel_c0_r349_n1.h5 | Bin .../EventScanParallel_r349_c0_timestamps_All1.png | Bin .../EventScanParallel_r349_c0_timestamps_ROI0.png | Bin .../EventScanParallel_r349_c0_timestamps_ROI1.png | Bin .../EventScanParallel_r349_c0_timestamps_pixel0.png | Bin ...tScanParallel_r349_c0_timestamps_pixel0_hist.png | Bin .../EventScanParallel_r349_c0_timestamps_pixel1.png | Bin ...tScanParallel_r349_c0_timestamps_pixel1_hist.png | Bin .../EventScanParallel_r349_c0_timestamps_pixel2.png | Bin ...tScanParallel_r349_c0_timestamps_pixel2_hist.png | Bin .../EventScanParallel_r349_c0_timestamps_pixel3.png | Bin ...tScanParallel_r349_c0_timestamps_pixel3_hist.png | Bin .../EventScanParallel_r349_c0_timestamps_pixel4.png | Bin ...tScanParallel_r349_c0_timestamps_pixel4_hist.png | Bin .../EventScanParallel_r349_c0_timestamps_pixel5.png | Bin ...tScanParallel_r349_c0_timestamps_pixel5_hist.png | Bin .../EventScanParallel_r349_c0_timestamps_pixel6.png | Bin ...tScanParallel_r349_c0_timestamps_pixel6_hist.png | Bin .../EventScanParallel_r349_c0_timestamps_pixel7.png | Bin ...tScanParallel_r349_c0_timestamps_pixel7_hist.png | Bin .../EventScanParallel_r349_c0_timestamps_pixel8.png | Bin ...tScanParallel_r349_c0_timestamps_pixel8_hist.png | Bin .../EventScanParallel_r349_c0_timestamps_pixel9.png | Bin ...tScanParallel_r349_c0_timestamps_pixel9_hist.png | Bin .../bitSlice_c0_r349_rixx1003721.npy | Bin .../eventNumbers_c0_r349_rixx1003721.npy | Bin .../means_c0_r349_rixx1003721.npy | Bin .../pulseIds_c0_r349_rixx1003721.npy | Bin .../testData/lowFlux/SimpleClusters_c0_r334_n100.h5 | Bin .../lowFlux/SimpleClusters_c0_r334_n100_part0.h5 | Bin 41 files changed, 5 insertions(+), 7 deletions(-) rename tests/{ => disabled}/OffXavierV4_2.npy (100%) rename tests/{ => disabled}/XavierV4_2.npy (100%) rename tests/{test_AnalyzeH5.py => disabled/disabled_test_AnalyzeH5.py} (100%) rename tests/{test_EventScanParallelSlice.py => disabled/disabled_test_EventScanParallelSlice.py} (100%) rename tests/{ => disabled}/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_r1_c14_testLabel_E.png (100%) rename tests/{ => disabled}/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_r1_c14_testLabel_fitInfo.npy (100%) rename tests/{ => disabled}/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_testLabel_E.png (100%) rename tests/{ => disabled}/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_testLabel_gainDistribution.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_c0_r349_n1.h5 (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_All1.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_ROI0.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_ROI1.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel0.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel0_hist.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel1.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel1_hist.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel2.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel2_hist.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel3.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel3_hist.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel4.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel4_hist.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel5.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel5_hist.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel6.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel6_hist.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel7.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel7_hist.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel8.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel8_hist.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel9.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel9_hist.png (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/bitSlice_c0_r349_rixx1003721.npy (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/eventNumbers_c0_r349_rixx1003721.npy (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/means_c0_r349_rixx1003721.npy (100%) rename tests/{ => disabled}/testData/expected/eventScanParallelSliceExpected/pulseIds_c0_r349_rixx1003721.npy (100%) rename tests/{ => disabled}/testData/lowFlux/SimpleClusters_c0_r334_n100.h5 (100%) rename tests/{ => disabled}/testData/lowFlux/SimpleClusters_c0_r334_n100_part0.h5 (100%) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 9086112..7843923 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -22,6 +22,6 @@ jobs: pip install scipy pip install statsmodels -# - name: Run tests -# run: | -# pytest tests/test_FitFunctions.py + - name: Run tests + run: | + pytest tests/test_FitFunctions.py diff --git a/README.md b/README.md index 3892911..914ca99 100644 --- a/README.md +++ b/README.md @@ -32,12 +32,10 @@ main branch tag v1.0.0 are the scripts used for the 2/17/24 beamtime To run tests: -ssh into psana -source /sdf/group/lcls/ds/ana/sw/conda2/manage/bin/psconda.sh -clone this repo cd beamtime-calibration-suite/tests pytest . +() Developers: diff --git a/standalone_scripts/AnalyzeH5.py b/standalone_scripts/AnalyzeH5.py index e185d68..f818c1c 100644 --- a/standalone_scripts/AnalyzeH5.py +++ b/standalone_scripts/AnalyzeH5.py @@ -1,6 +1,6 @@ import h5py import numpy as np -import fitFunctions, ancillaryMethods +import calibrationSuite.fitFunctions, calibrationSuite.ancillaryMethods ##import seaborn as sns import matplotlib.pyplot as plt from matplotlib.ticker import AutoMinorLocator diff --git a/tests/OffXavierV4_2.npy b/tests/disabled/OffXavierV4_2.npy similarity index 100% rename from tests/OffXavierV4_2.npy rename to tests/disabled/OffXavierV4_2.npy diff --git a/tests/XavierV4_2.npy b/tests/disabled/XavierV4_2.npy similarity index 100% rename from tests/XavierV4_2.npy rename to tests/disabled/XavierV4_2.npy diff --git a/tests/test_AnalyzeH5.py b/tests/disabled/disabled_test_AnalyzeH5.py similarity index 100% rename from tests/test_AnalyzeH5.py rename to tests/disabled/disabled_test_AnalyzeH5.py diff --git a/tests/test_EventScanParallelSlice.py b/tests/disabled/disabled_test_EventScanParallelSlice.py similarity index 100% rename from tests/test_EventScanParallelSlice.py rename to tests/disabled/disabled_test_EventScanParallelSlice.py diff --git a/tests/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_r1_c14_testLabel_E.png b/tests/disabled/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_r1_c14_testLabel_E.png similarity index 100% rename from tests/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_r1_c14_testLabel_E.png rename to tests/disabled/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_r1_c14_testLabel_E.png diff --git a/tests/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_r1_c14_testLabel_fitInfo.npy b/tests/disabled/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_r1_c14_testLabel_fitInfo.npy similarity index 100% rename from tests/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_r1_c14_testLabel_fitInfo.npy rename to tests/disabled/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_r1_c14_testLabel_fitInfo.npy diff --git a/tests/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_testLabel_E.png b/tests/disabled/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_testLabel_E.png similarity index 100% rename from tests/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_testLabel_E.png rename to tests/disabled/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_testLabel_E.png diff --git a/tests/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_testLabel_gainDistribution.png b/tests/disabled/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_testLabel_gainDistribution.png similarity index 100% rename from tests/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_testLabel_gainDistribution.png rename to tests/disabled/testData/expected/analyzeH5Expected/AnalyzeH5_r666_c0_testLabel_gainDistribution.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_c0_r349_n1.h5 b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_c0_r349_n1.h5 similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_c0_r349_n1.h5 rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_c0_r349_n1.h5 diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_All1.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_All1.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_All1.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_All1.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_ROI0.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_ROI0.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_ROI0.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_ROI0.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_ROI1.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_ROI1.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_ROI1.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_ROI1.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel0.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel0.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel0.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel0.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel0_hist.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel0_hist.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel0_hist.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel0_hist.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel1.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel1.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel1.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel1.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel1_hist.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel1_hist.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel1_hist.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel1_hist.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel2.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel2.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel2.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel2.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel2_hist.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel2_hist.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel2_hist.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel2_hist.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel3.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel3.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel3.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel3.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel3_hist.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel3_hist.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel3_hist.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel3_hist.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel4.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel4.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel4.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel4.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel4_hist.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel4_hist.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel4_hist.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel4_hist.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel5.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel5.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel5.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel5.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel5_hist.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel5_hist.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel5_hist.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel5_hist.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel6.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel6.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel6.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel6.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel6_hist.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel6_hist.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel6_hist.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel6_hist.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel7.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel7.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel7.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel7.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel7_hist.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel7_hist.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel7_hist.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel7_hist.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel8.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel8.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel8.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel8.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel8_hist.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel8_hist.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel8_hist.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel8_hist.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel9.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel9.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel9.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel9.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel9_hist.png b/tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel9_hist.png similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel9_hist.png rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/EventScanParallel_r349_c0_timestamps_pixel9_hist.png diff --git a/tests/testData/expected/eventScanParallelSliceExpected/bitSlice_c0_r349_rixx1003721.npy b/tests/disabled/testData/expected/eventScanParallelSliceExpected/bitSlice_c0_r349_rixx1003721.npy similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/bitSlice_c0_r349_rixx1003721.npy rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/bitSlice_c0_r349_rixx1003721.npy diff --git a/tests/testData/expected/eventScanParallelSliceExpected/eventNumbers_c0_r349_rixx1003721.npy b/tests/disabled/testData/expected/eventScanParallelSliceExpected/eventNumbers_c0_r349_rixx1003721.npy similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/eventNumbers_c0_r349_rixx1003721.npy rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/eventNumbers_c0_r349_rixx1003721.npy diff --git a/tests/testData/expected/eventScanParallelSliceExpected/means_c0_r349_rixx1003721.npy b/tests/disabled/testData/expected/eventScanParallelSliceExpected/means_c0_r349_rixx1003721.npy similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/means_c0_r349_rixx1003721.npy rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/means_c0_r349_rixx1003721.npy diff --git a/tests/testData/expected/eventScanParallelSliceExpected/pulseIds_c0_r349_rixx1003721.npy b/tests/disabled/testData/expected/eventScanParallelSliceExpected/pulseIds_c0_r349_rixx1003721.npy similarity index 100% rename from tests/testData/expected/eventScanParallelSliceExpected/pulseIds_c0_r349_rixx1003721.npy rename to tests/disabled/testData/expected/eventScanParallelSliceExpected/pulseIds_c0_r349_rixx1003721.npy diff --git a/tests/testData/lowFlux/SimpleClusters_c0_r334_n100.h5 b/tests/disabled/testData/lowFlux/SimpleClusters_c0_r334_n100.h5 similarity index 100% rename from tests/testData/lowFlux/SimpleClusters_c0_r334_n100.h5 rename to tests/disabled/testData/lowFlux/SimpleClusters_c0_r334_n100.h5 diff --git a/tests/testData/lowFlux/SimpleClusters_c0_r334_n100_part0.h5 b/tests/disabled/testData/lowFlux/SimpleClusters_c0_r334_n100_part0.h5 similarity index 100% rename from tests/testData/lowFlux/SimpleClusters_c0_r334_n100_part0.h5 rename to tests/disabled/testData/lowFlux/SimpleClusters_c0_r334_n100_part0.h5 From dc8d08a9dd36fe7776a9db8979368d70c9647ddf Mon Sep 17 00:00:00 2001 From: nstelter-slac Date: Tue, 20 Feb 2024 12:11:52 -0800 Subject: [PATCH 4/6] MNT: h5 and npy scripts use library utilities, so move to suite_scripts folder --- standalone_scripts/AnalyzeH5.py | 172 ------------------------------ standalone_scripts/analyze_npy.py | 87 --------------- 2 files changed, 259 deletions(-) delete mode 100644 standalone_scripts/AnalyzeH5.py delete mode 100644 standalone_scripts/analyze_npy.py diff --git a/standalone_scripts/AnalyzeH5.py b/standalone_scripts/AnalyzeH5.py deleted file mode 100644 index f818c1c..0000000 --- a/standalone_scripts/AnalyzeH5.py +++ /dev/null @@ -1,172 +0,0 @@ -import h5py -import numpy as np -import calibrationSuite.fitFunctions, calibrationSuite.ancillaryMethods -##import seaborn as sns -import matplotlib.pyplot as plt -from matplotlib.ticker import AutoMinorLocator -##import sys -import argparse - -class AnalyzeH5(object): - def __init__(self): - print('in init') - ## this parsing may be common - move elsewhere if so - parser = argparse.ArgumentParser( - description='Configures calibration suite, overriding experimentHash', - formatter_class=argparse.ArgumentDefaultsHelpFormatter - ) - parser.add_argument('-e', '--exp', help='experiment') - ##parser.add_argument('-l', '--location', help='hutch location, e.g. MfxEndstation or DetLab') - parser.add_argument('-r', '--run', type=int, help='run') - parser.add_argument('-R', '--runRange', help='run range, format ...') - parser.add_argument('-p', '--path', type=str, default='../lowFlux/', help='the base path to the output directory') - parser.add_argument('-d', '--detType', type=str, default='', help='Epix100, Epix10ka, Epix10kaQuad, Epix10ka2M, ...') - parser.add_argument('-f','--files', type=str, default=None, help='run analysis on file or comma-separated files') - parser.add_argument('-L','--label', type=str, default='foo', help='analysis label') - - args = parser.parse_args() - - self.run = args.run - self.files = args.files.replace(' ', '') - print(self.files) - self.outputDir = args.path - self.label = args.label - self.camera = 0 - - def getFiles(self): - fileNames = self.files.split(',') - self.h5Files = [] - for f in fileNames: - print(f) - self.h5Files.append(h5py.File(f)) - - def identifyAnalysis(self): - try: - self.analysisType = self.h5Files[0]['analysisType'] - self.sliceCoordinates = self.h5Files[0]['sliceCoordinates'][()] - except: - ## do something useful here, maybe - self.analysisType = None - ## but for now - self.analysisType = 'cluster' - self.sliceCoordinates = [[270, 288], [59, 107]] - self.sliceEdges = [288-270, 107-59] - - def sliceToDetector(self, sliceRow, sliceCol): - return sliceRow + self.sliceCoordinates[0][0], sliceCol + self.sliceCoordinates[1][0] - - def analyze(self): - if self.analysisType == 'cluster': - self.clusterAnalysis() - else: - print("unknown analysis type %s" %(self.analysisType)) - - def clusterAnalysis(self): - clusters = None - energyHist = None - clusters = np.concatenate([h5['clusterData'][()] for h5 in self.h5Files]) - try: - energyHist = np.concatenate(energyHist, h5['energyHistogram'][()]) - except: - pass - - self.nBins = 100 - self.lowEnergyCut = 1 ## fix - should be 0.5 photons or something - self.highEnergyCut = 10 ## fix - should be 1.5 photons or something - ##tmp - np.save("%s/r%d_clusters.npy" %(self.outputDir,self.run), clusters) - self.analyzeSimpleClusters(clusters) - - if energyHist is None: - return - - _, bins = np.histogram(energyHist, 250, [-5, 45]) - plt.hist(bins[:-1], bins, weights=energyHist)##, log=True) - plt.grid(which='major',linewidth=0.5) - plt.title = "All pixel energies in run after common mode correction" - plt.xlabel = "energy (keV)" - print("I hate matplotlib so much") - plt.savefig("%s/%s_r%d_c%d_%s_energyHistogram.png" %(self.outputDir,self.__class__.__name__, self.run, self.camera, self.label)) - np.save("%s/%s_r%d_c%d_%s_energyHistogram.npy" %(self.outputDir,self.__class__.__name__, self.run, self.camera, self.label), energyHist) - plt.close() - - - - def analyzeSimpleClusters(self, clusters): - ax = plt.subplot() - energy = clusters[:, :, 0]##.flatten() - ##energy *= 2 ## temporary, due to bit shift - ##print(energy[energy>0][666:777]) - print('mean energy above 0:', energy[energy>0].mean()) - foo = ax.hist(energy[energy>0], 100) - plt.xlabel = "energy (keV)" - plt.title = "All pixels" - plt.savefig("%s/%s_r%d_c%d_%s_E.png" %(self.outputDir, - self.__class__.__name__, - self.run, self.camera, - self.label)) - plt.close() - - rows = self.sliceEdges[0] - cols = self.sliceEdges[1] - fitInfo = np.zeros((rows, cols, 5)) ## mean, std, area, mu, sigma - for i in range(rows): - for j in range(cols): - detRow, detCol = self.sliceToDetector(i,j) - ax = plt.subplot() - goodClusters = ancillaryMethods.goodClusters(clusters, i, j, nPixelCut=4, isSquare=1) - if len(goodClusters) <5: - print("too few clusters in slice pixel %d, %d: %d" %(i,j, len(goodClusters))) - continue - energies = ancillaryMethods.getClusterEnergies(goodClusters) - photonEcut = np.bitwise_and(energies>self.lowEnergyCut, energies0).sum() - print("pixel %d,%d has about %d photons" %(i,j,nPixelClusters)) - photonRegion = energies[photonEcut] - mean = photonRegion.mean() - std = photonRegion.std() - a, mu, sigma = self.histogramAndFitGaussian(ax, energies, self.nBins) - area = fitFunctions.gaussianArea(a, sigma) - ax.set_xlabel('energy (keV)') - ax.set_title('pixel %d,%d, small cluster cuts' %(detRow, detCol)) - plt.figtext(0.7, 0.8, "%d entries (peak)" %(area)) - plt.figtext(0.7, 0.75, "mu %0.2f" %(mu)) - plt.figtext(0.7, 0.7, "sigma %0.2f" %(sigma)) - plt.savefig("%s/%s_r%d_c%d_r%d_c%d_%s_E.png" %(self.outputDir,self.__class__.__name__, self.run, self.camera, detRow, detCol, self.label)) - plt.close() - - np.save("%s/%s_r%d_c%d_r%d_c%d_%s_fitInfo.npy" %(self.outputDir,self.__class__.__name__, self.run, self.camera, detRow, detCol, self.label), fitInfo) - fitInfo[i,j] = mean, std, area, mu, sigma - - gains = fitInfo[:,:,3] - goodGains = gains[np.bitwise_and(gains>0, gains<15)] - ax = plt.subplot() - ax.hist(goodGains, 100) - ax.set_xlabel("energy (keV)") - ax.set_title("pixel single photon fitted energy") - plt.savefig("%s/%s_r%d_c%d_%s_gainDistribution.png" %(self.outputDir,self.__class__.__name__, self.run, self.camera, self.label)) - - - def histogramAndFitGaussian(self, ax, energies, nBins): - y, bin_edges, _ = ax.hist(energies, nBins) - bins = (bin_edges[:-1] + bin_edges[1:])/2 - ##print(y, bins) - a, mean, std = fitFunctions.estimateGaussianParametersFromUnbinnedArray(energies) - try: - popt, pcov = fitFunctions.curve_fit(fitFunctions.gaussian, bins, y, [a, mean, std]) - mu = popt[1] - sigma = popt[2] - fittedFunc = fitFunctions.gaussian(bins, *popt) - ax.plot(bins, fittedFunc,color='b') - return popt - except: - return 0, 0, 0 - - -if __name__ == "__main__": - ah5 = AnalyzeH5() - ah5.getFiles() - ah5.identifyAnalysis() - ah5.analyze() - - diff --git a/standalone_scripts/analyze_npy.py b/standalone_scripts/analyze_npy.py deleted file mode 100644 index 96350a3..0000000 --- a/standalone_scripts/analyze_npy.py +++ /dev/null @@ -1,87 +0,0 @@ -import matplotlib.pyplot as plt -import numpy as np -import fitFunctions, ancillaryMethods -import sys - -class AnalyzeLinearity(object): - def __init__(self, data, label): - self.data = data - self.label = label - self.dataIndices = {"g0slope":0, - "g0intercept":1, - "g0r2":2, - "g1slope":3, - "g1intercept":4, - "g1r2":5, - "g0max":6, - "g1min":7 - } - self.dataRanges = {##"g0slope":[0,1000], - "g0slope":[0,5], - "g0intercept":[0,10000], - "g0r2":[0.9,1.0], - "g1slope":[0,0.1], - "g1intercept":[0,10000], - "g1r2":[0.9,1.], - "g0max":[10000, 16384], - "g1min":[0, 16384] - } - - def analyzeLinearity(self): - self.plotLinearity("g0slope", "g1slope") - self.plotLinearity("g0r2", "g1r2") - self.plotLinearity("g0max", "g1min") - self.plotRatio("g1slope", "g0slope", clipRange=[0, 0.05]) - - def plotLinearity(self, stat0, stat1): - ##return - g0Range = self.dataRanges[stat0] - g1Range = self.dataRanges[stat1] - g0Index = self.dataIndices[stat0] - g1Index = self.dataIndices[stat1] - - fig, ax = plt.subplots(2,2) - d = self.data[:,:,g0Index] - print(stat0, "median:", np.median(d)) - im = ax[0,0].imshow(d.clip(*tuple(g0Range))) - fig.colorbar(im) - ax[0,0].set_title(stat0) - ax[0,1].hist(d.clip(*tuple(g0Range)), 100) - ax[0,1].set_title(stat0) - d = self.data[:,:,g1Index] - print(stat1, "median:", np.median(d)) - im = ax[1,0].imshow(d.clip(*tuple(g1Range))) - fig.colorbar(im) - ax[1,0].set_title(stat1) - ax[1,1].hist(d.clip(*tuple(g1Range)), 100) - ax[1,1].set_title(stat1) - ##plt.show() - plt.savefig("%s_%s_%s_maps_and_histos.png" %(self.label, stat0, stat1)) - plt.close() - - def plotRatio(self, statA, statB, clipRange=None): - ##g0Range = self.dataRanges[stat0] - ##g1Range = self.dataRanges[stat1] - indexA = self.dataIndices[statA] - indexB = self.dataIndices[statB] - fig, ax = plt.subplots(2,1) - d = self.data[:,:,indexA]/self.data[:,:,indexB] - print(statA, statB, "ratio median:", np.median(d)) - d = d.clip(*tuple(clipRange)) - ##im = ax[0,0].imshow(d.clip(*tuple(g0Range))) - im = ax[0].imshow(d) - fig.colorbar(im) - ax[0].set_title("%s/%s" %(statA, statB)) - ax[1].hist(d, 100) - ax[1].set_title("%s/%s" %(statA, statB)) - plt.savefig("%s_%s_%s_ratio_map_and_histo.png" %(self.label, statA, statB)) - plt.close() - - -f = sys.argv[1] -data = np.load(f) -label = f.split('.npy')[0] -if 'Linearity' in f: - print(f) - a = AnalyzeLinearity(data, label) - a.analyzeLinearity() From a2ffcf4f37fea8efd4d972a81e6aaa44dca99116 Mon Sep 17 00:00:00 2001 From: nstelter-slac Date: Tue, 20 Feb 2024 12:50:51 -0800 Subject: [PATCH 5/6] MNT: accidently removed these files, adding back in --- suite_scripts/AnalyzeH5.py | 172 +++++++++++++++++++++++++++++++++++ suite_scripts/analyze_npy.py | 87 ++++++++++++++++++ 2 files changed, 259 insertions(+) create mode 100644 suite_scripts/AnalyzeH5.py create mode 100644 suite_scripts/analyze_npy.py diff --git a/suite_scripts/AnalyzeH5.py b/suite_scripts/AnalyzeH5.py new file mode 100644 index 0000000..f818c1c --- /dev/null +++ b/suite_scripts/AnalyzeH5.py @@ -0,0 +1,172 @@ +import h5py +import numpy as np +import calibrationSuite.fitFunctions, calibrationSuite.ancillaryMethods +##import seaborn as sns +import matplotlib.pyplot as plt +from matplotlib.ticker import AutoMinorLocator +##import sys +import argparse + +class AnalyzeH5(object): + def __init__(self): + print('in init') + ## this parsing may be common - move elsewhere if so + parser = argparse.ArgumentParser( + description='Configures calibration suite, overriding experimentHash', + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument('-e', '--exp', help='experiment') + ##parser.add_argument('-l', '--location', help='hutch location, e.g. MfxEndstation or DetLab') + parser.add_argument('-r', '--run', type=int, help='run') + parser.add_argument('-R', '--runRange', help='run range, format ...') + parser.add_argument('-p', '--path', type=str, default='../lowFlux/', help='the base path to the output directory') + parser.add_argument('-d', '--detType', type=str, default='', help='Epix100, Epix10ka, Epix10kaQuad, Epix10ka2M, ...') + parser.add_argument('-f','--files', type=str, default=None, help='run analysis on file or comma-separated files') + parser.add_argument('-L','--label', type=str, default='foo', help='analysis label') + + args = parser.parse_args() + + self.run = args.run + self.files = args.files.replace(' ', '') + print(self.files) + self.outputDir = args.path + self.label = args.label + self.camera = 0 + + def getFiles(self): + fileNames = self.files.split(',') + self.h5Files = [] + for f in fileNames: + print(f) + self.h5Files.append(h5py.File(f)) + + def identifyAnalysis(self): + try: + self.analysisType = self.h5Files[0]['analysisType'] + self.sliceCoordinates = self.h5Files[0]['sliceCoordinates'][()] + except: + ## do something useful here, maybe + self.analysisType = None + ## but for now + self.analysisType = 'cluster' + self.sliceCoordinates = [[270, 288], [59, 107]] + self.sliceEdges = [288-270, 107-59] + + def sliceToDetector(self, sliceRow, sliceCol): + return sliceRow + self.sliceCoordinates[0][0], sliceCol + self.sliceCoordinates[1][0] + + def analyze(self): + if self.analysisType == 'cluster': + self.clusterAnalysis() + else: + print("unknown analysis type %s" %(self.analysisType)) + + def clusterAnalysis(self): + clusters = None + energyHist = None + clusters = np.concatenate([h5['clusterData'][()] for h5 in self.h5Files]) + try: + energyHist = np.concatenate(energyHist, h5['energyHistogram'][()]) + except: + pass + + self.nBins = 100 + self.lowEnergyCut = 1 ## fix - should be 0.5 photons or something + self.highEnergyCut = 10 ## fix - should be 1.5 photons or something + ##tmp + np.save("%s/r%d_clusters.npy" %(self.outputDir,self.run), clusters) + self.analyzeSimpleClusters(clusters) + + if energyHist is None: + return + + _, bins = np.histogram(energyHist, 250, [-5, 45]) + plt.hist(bins[:-1], bins, weights=energyHist)##, log=True) + plt.grid(which='major',linewidth=0.5) + plt.title = "All pixel energies in run after common mode correction" + plt.xlabel = "energy (keV)" + print("I hate matplotlib so much") + plt.savefig("%s/%s_r%d_c%d_%s_energyHistogram.png" %(self.outputDir,self.__class__.__name__, self.run, self.camera, self.label)) + np.save("%s/%s_r%d_c%d_%s_energyHistogram.npy" %(self.outputDir,self.__class__.__name__, self.run, self.camera, self.label), energyHist) + plt.close() + + + + def analyzeSimpleClusters(self, clusters): + ax = plt.subplot() + energy = clusters[:, :, 0]##.flatten() + ##energy *= 2 ## temporary, due to bit shift + ##print(energy[energy>0][666:777]) + print('mean energy above 0:', energy[energy>0].mean()) + foo = ax.hist(energy[energy>0], 100) + plt.xlabel = "energy (keV)" + plt.title = "All pixels" + plt.savefig("%s/%s_r%d_c%d_%s_E.png" %(self.outputDir, + self.__class__.__name__, + self.run, self.camera, + self.label)) + plt.close() + + rows = self.sliceEdges[0] + cols = self.sliceEdges[1] + fitInfo = np.zeros((rows, cols, 5)) ## mean, std, area, mu, sigma + for i in range(rows): + for j in range(cols): + detRow, detCol = self.sliceToDetector(i,j) + ax = plt.subplot() + goodClusters = ancillaryMethods.goodClusters(clusters, i, j, nPixelCut=4, isSquare=1) + if len(goodClusters) <5: + print("too few clusters in slice pixel %d, %d: %d" %(i,j, len(goodClusters))) + continue + energies = ancillaryMethods.getClusterEnergies(goodClusters) + photonEcut = np.bitwise_and(energies>self.lowEnergyCut, energies0).sum() + print("pixel %d,%d has about %d photons" %(i,j,nPixelClusters)) + photonRegion = energies[photonEcut] + mean = photonRegion.mean() + std = photonRegion.std() + a, mu, sigma = self.histogramAndFitGaussian(ax, energies, self.nBins) + area = fitFunctions.gaussianArea(a, sigma) + ax.set_xlabel('energy (keV)') + ax.set_title('pixel %d,%d, small cluster cuts' %(detRow, detCol)) + plt.figtext(0.7, 0.8, "%d entries (peak)" %(area)) + plt.figtext(0.7, 0.75, "mu %0.2f" %(mu)) + plt.figtext(0.7, 0.7, "sigma %0.2f" %(sigma)) + plt.savefig("%s/%s_r%d_c%d_r%d_c%d_%s_E.png" %(self.outputDir,self.__class__.__name__, self.run, self.camera, detRow, detCol, self.label)) + plt.close() + + np.save("%s/%s_r%d_c%d_r%d_c%d_%s_fitInfo.npy" %(self.outputDir,self.__class__.__name__, self.run, self.camera, detRow, detCol, self.label), fitInfo) + fitInfo[i,j] = mean, std, area, mu, sigma + + gains = fitInfo[:,:,3] + goodGains = gains[np.bitwise_and(gains>0, gains<15)] + ax = plt.subplot() + ax.hist(goodGains, 100) + ax.set_xlabel("energy (keV)") + ax.set_title("pixel single photon fitted energy") + plt.savefig("%s/%s_r%d_c%d_%s_gainDistribution.png" %(self.outputDir,self.__class__.__name__, self.run, self.camera, self.label)) + + + def histogramAndFitGaussian(self, ax, energies, nBins): + y, bin_edges, _ = ax.hist(energies, nBins) + bins = (bin_edges[:-1] + bin_edges[1:])/2 + ##print(y, bins) + a, mean, std = fitFunctions.estimateGaussianParametersFromUnbinnedArray(energies) + try: + popt, pcov = fitFunctions.curve_fit(fitFunctions.gaussian, bins, y, [a, mean, std]) + mu = popt[1] + sigma = popt[2] + fittedFunc = fitFunctions.gaussian(bins, *popt) + ax.plot(bins, fittedFunc,color='b') + return popt + except: + return 0, 0, 0 + + +if __name__ == "__main__": + ah5 = AnalyzeH5() + ah5.getFiles() + ah5.identifyAnalysis() + ah5.analyze() + + diff --git a/suite_scripts/analyze_npy.py b/suite_scripts/analyze_npy.py new file mode 100644 index 0000000..80784da --- /dev/null +++ b/suite_scripts/analyze_npy.py @@ -0,0 +1,87 @@ +import matplotlib.pyplot as plt +import numpy as np +import calibrationSuite.fitFunctions, calibrationSuite.ancillaryMethods +import sys + +class AnalyzeLinearity(object): + def __init__(self, data, label): + self.data = data + self.label = label + self.dataIndices = {"g0slope":0, + "g0intercept":1, + "g0r2":2, + "g1slope":3, + "g1intercept":4, + "g1r2":5, + "g0max":6, + "g1min":7 + } + self.dataRanges = {##"g0slope":[0,1000], + "g0slope":[0,5], + "g0intercept":[0,10000], + "g0r2":[0.9,1.0], + "g1slope":[0,0.1], + "g1intercept":[0,10000], + "g1r2":[0.9,1.], + "g0max":[10000, 16384], + "g1min":[0, 16384] + } + + def analyzeLinearity(self): + self.plotLinearity("g0slope", "g1slope") + self.plotLinearity("g0r2", "g1r2") + self.plotLinearity("g0max", "g1min") + self.plotRatio("g1slope", "g0slope", clipRange=[0, 0.05]) + + def plotLinearity(self, stat0, stat1): + ##return + g0Range = self.dataRanges[stat0] + g1Range = self.dataRanges[stat1] + g0Index = self.dataIndices[stat0] + g1Index = self.dataIndices[stat1] + + fig, ax = plt.subplots(2,2) + d = self.data[:,:,g0Index] + print(stat0, "median:", np.median(d)) + im = ax[0,0].imshow(d.clip(*tuple(g0Range))) + fig.colorbar(im) + ax[0,0].set_title(stat0) + ax[0,1].hist(d.clip(*tuple(g0Range)), 100) + ax[0,1].set_title(stat0) + d = self.data[:,:,g1Index] + print(stat1, "median:", np.median(d)) + im = ax[1,0].imshow(d.clip(*tuple(g1Range))) + fig.colorbar(im) + ax[1,0].set_title(stat1) + ax[1,1].hist(d.clip(*tuple(g1Range)), 100) + ax[1,1].set_title(stat1) + ##plt.show() + plt.savefig("%s_%s_%s_maps_and_histos.png" %(self.label, stat0, stat1)) + plt.close() + + def plotRatio(self, statA, statB, clipRange=None): + ##g0Range = self.dataRanges[stat0] + ##g1Range = self.dataRanges[stat1] + indexA = self.dataIndices[statA] + indexB = self.dataIndices[statB] + fig, ax = plt.subplots(2,1) + d = self.data[:,:,indexA]/self.data[:,:,indexB] + print(statA, statB, "ratio median:", np.median(d)) + d = d.clip(*tuple(clipRange)) + ##im = ax[0,0].imshow(d.clip(*tuple(g0Range))) + im = ax[0].imshow(d) + fig.colorbar(im) + ax[0].set_title("%s/%s" %(statA, statB)) + ax[1].hist(d, 100) + ax[1].set_title("%s/%s" %(statA, statB)) + plt.savefig("%s_%s_%s_ratio_map_and_histo.png" %(self.label, statA, statB)) + plt.close() + + +f = sys.argv[1] +data = np.load(f) +label = f.split('.npy')[0] +if 'Linearity' in f: + print(f) + a = AnalyzeLinearity(data, label) + a.analyzeLinearity() From c26ffb43d1733f85cef95d8d30011ce7b3acc548 Mon Sep 17 00:00:00 2001 From: nstelter-slac Date: Tue, 20 Feb 2024 13:02:43 -0800 Subject: [PATCH 6/6] DOC: minor update to readme --- README.md | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 914ca99..d2bca4a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # beamtime-calibration-suite [![Build Status](https://github.com/slaclab/beamtime-calibration-suite/actions/workflows/run-tests.yml/badge.svg?branch=main)](https://github.com/slaclab/beamtime-calibration-suite/actions/workflows/run-tests.yml) -To run to suite_scripts or use the library in scripts outside the project directory, you need to append the project directory to your PYTHONPATH, for example: +To run any suite_scripts or use the library in scripts outside the project directory, you need to append the project directory to your PYTHONPATH, for example: export PYTHONPATH="${PYTHONPATH}:~/beamtime_slaclab/beamtime-calibration-suite" (can add this to your ~/.bashrc so persists between terminal sessions. Library can later have @@ -21,6 +21,9 @@ from calibrationSuite.cluster import * /standalone_scripts: scripts that do not use the calibrationSuite library code +/tests: tests files, can be ran with 'pytest .' from the root project directory +(Currently only test for the fitFunctions libraryh file is running, more tests are to be added) + Current Status: @@ -29,18 +32,8 @@ main branch tag v1.0.0 are the scripts used for the 2/17/24 beamtime -large changes will be merged into ontop of this, but original scripts can be accessed by checking out this tag -future beamtimes can be tagged as well - -To run tests: - -cd beamtime-calibration-suite/tests -pytest . - -() - Developers: If you are new to git/github, start here: https://confluence.slac.stanford.edu/pages/viewpage.action?pageId=428802060 -Then read the following for an overview of the development process: https://confluence.slac.stanford.edu/pages/viewpage.action?pageId=429562464 - - +Then read the following for an overview of the development process: https://confluence.slac.stanford.edu/pages/viewpage.action?pageId=429562464 \ No newline at end of file