Merge pull request #146 from slaclab/development

Merge development into main (post beamtime nov 8 2024 maintenance)

.gitignore

@@ -1,3 +1,5 @@
+suite_scripts/test_*/
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

calibrationSuite/argumentParser.py

@@ -55,6 +55,9 @@ def __init__(self):
         self.parser.add_argument(
             "-f", "--files", type=str, default=None, help="run analysis on file or comma-separated files"
         )
+        self.parser.add_argument(
+            "-F", "--nonXtcFiles", type=str, default=None, help="run initial analysis on file or comma-separated files"
+        )
         self.parser.add_argument("-cf", "--configFile", type=str, help="config file path, can be relative or full path")
         self.parser.add_argument("-L", "--label", type=str, help="analysis label")
         self.parser.add_argument("-t", "--threshold", help="threshold (ADU or keV or wave8) depending on --detObj")
@@ -64,7 +67,7 @@ def __init__(self):
         self.parser.add_argument("--photonEnergy", type=float, help="photon energy")
         self.parser.add_argument("--aduPerKeV", type=float, help="ADU per keV")
         self.parser.add_argument("--gainMode", type=int, help="gain mode (int)")
-        
+
         self.parser.add_argument(
             "--special",
             type=str,

calibrationSuite/basicSuiteScript.py

@@ -21,7 +21,10 @@
 
 logger = logging.getLogger(__name__)
 
-if os.getenv("foo") == "1":
+if os.getenv("foo") == "0":
+    print("non-psana")
+    from calibrationSuite.nonPsanaBase import PsanaBase
+elif os.getenv("foo") == "1":
     print("psana1")
     from calibrationSuite.psana1Base import PsanaBase
 else:
@@ -96,19 +99,22 @@ def getRawData(self, evt, gainBitsMasked=True, negativeGain=False):
         if frames is None:
             return None
 
-        nZero = frames.size - np.count_nonzero(frames)
+        nZero = 0
+        if not "skipZeroCheck" in dir(self):
+            nZero = frames.size - np.count_nonzero(frames)
         try:
             dz = self.nZero - nZero
             if dz != 0:
-                print("found %d new zero pixels, expected %d, setting frame to None" % (dz, self.nZero))
+            ##if abs(dz) > 10: ## add a flag for just epixM...
+                print("found %d new zero pixels, expected %d, setting frame to None, size was %d" % (dz, self.nZero, frames.size))
                 return None
         except Exception:
             self.nZero = nZero
-            print("Starting with %d zero pixels, will require exactly that many for this run" %(nZero))
-            
+            print("Starting with %d zero pixels, will maybe require exactly that many for this run, size is %d" % (nZero, frames.size))
+
             try:
                 self.dumpEpixMHeaderInfo(evt)
-            except:
+            except Exception:
                 pass
 
         if False and self.special:  ## turned off for a tiny bit of speed
@@ -124,12 +130,14 @@ def getRawData(self, evt, gainBitsMasked=True, negativeGain=False):
             elif "tenBits" in self.special:
                 frames = frames & 0xFFF0
                 ##print("10bits")
+
         if self.negativeGain or negativeGain:
             zeroPixels = frames == 0
             maskedData = frames & self.gainBitsMask
             gainData = frames - maskedData
             frames = gainData + self.gainBitsMask - maskedData
             frames[zeroPixels] = 0
+
         if gainBitsMasked:
             return frames & self.gainBitsMask
         return frames
@@ -171,6 +179,9 @@ def sliceToDetector(self, sliceRow, sliceCol):  ## cp from AnalyzeH5: import?
     def getNswitchedPixels(self, data, region=None):
         return ((data >= self.g0cut) * 1).sum()
 
+    def getSwitchedPixels(self, data, region=None):
+        return data >= self.g0cut
+
     def dumpEventCodeStatistics(self):
         print(
             "have counted %d run triggers, %d DAQ triggers, %d beam events"
@@ -217,9 +228,9 @@ def rowCommonModeCorrection3d(self, frames, arbitraryCut=1000):
             frames[module] = self.rowCommonModeCorrection(frames[module], arbitraryCut)
         return frames
 
-    def colCommonModeCorrection3d(self, frames, arbitraryCut=1000):
+    def colCommonModeCorrection3d(self, frames, cut=1000, switchedPixels=None):
         for module in self.analyzedModules:
-            frames[module] = self.colCommonModeCorrection(frames[module], arbitraryCut)
+            frames[module] = self.colCommonModeCorrection(frames[module], cut, switchedPixels[module])
         return frames
 
     def rowCommonModeCorrection(self, frame, arbitraryCut=1000):
@@ -236,7 +247,7 @@ def rowCommonModeCorrection(self, frame, arbitraryCut=1000):
                             frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank] < arbitraryCut
                         ]
                     )
-                    if not np.isnan(rowCM): ## no pixels found under cut
+                    if not np.isnan(rowCM):  ## no pixels found under cut
                         frame[r, colOffset : colOffset + self.detectorInfo.nColsPerBank] -= rowCM
                 except Exception:
                     print("rowCM problem")
@@ -245,7 +256,7 @@ def rowCommonModeCorrection(self, frame, arbitraryCut=1000):
                 colOffset += self.detectorInfo.nColsPerBank
         return frame
 
-    def colCommonModeCorrection(self, frame, arbitraryCut=1000):
+    def colCommonModeCorrection(self, frame, cut=1000, switchedPixels=None):
         ## this takes a 2d frame
         ## cut keeps photons in common mode - e.g. set to <<1 photon
 
@@ -255,16 +266,19 @@ def colCommonModeCorrection(self, frame, arbitraryCut=1000):
             rowOffset = 0
             for b in range(0, self.detectorInfo.nBanksRow):
                 try:
-                    colCM = np.median(
-                        frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c][
-                            frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c] < arbitraryCut
-                        ]
-                    )
-                    if not np.isnan(colCM): ## if no pixels < cut we get nan
+                    testPixels = np.s_[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c]
+                    relevantPixels = frame[testPixels] < cut
+                    if switchedPixels is not None:
+                        ##print(testPixels, relevantPixels)
+                        relevantPixels = np.bitwise_and(relevantPixels, ~switchedPixels[testPixels])
+                    colCM = np.median(frame[testPixels][relevantPixels])
+                    if not np.isnan(colCM):  ## if no pixels < cut we get nan
                         if False:
-                            if c<100:
+                            if c < 100:
                                 self.commonModeVals.append(colCM)
-                        frame[rowOffset : rowOffset + self.detectorInfo.nRowsPerBank, c] -= colCM
+                        if colCM > cut:
+                            raise Exception("overcorrection: colCM, cut:", colCM, cut)
+                        frame[testPixels] -= colCM
                 except Exception:
                     print("colCM problem")
                     logger.error("colCM problem")

calibrationSuite/detectorInfo.py

@@ -42,6 +42,7 @@ def __init__(self, detType, detSubtype="1d", detVersion=0):
 
         knownTypes = [
             "epixhr",
+            "epixuhr",
             "epixm",
             "epix100",
             "Epix100a",
@@ -60,6 +61,8 @@ def __init__(self, detType, detSubtype="1d", detVersion=0):
     def setupDetector(self):  ## needs nModules to be set
         if self.detectorType == "epixhr":
             self.setup_epixhr()
+        elif self.detectorType == "epixuhr":
+            self.setup_epixUHR()
         elif self.detectorType == "epixm":
             self.setup_epixM()
         elif self.detectorType in ["epix100", "Epix100a"]:
@@ -70,7 +73,9 @@ def setupDetector(self):  ## needs nModules to be set
             self.setup_jungfrau()
         elif "epix10k" in self.detectorType.lower():
             self.setup_epix10k()
-
+        else:
+            raise Exception("detector not matched", self.detectorType)
+
     def setNModules(self, n):
         self.nModules = n
 
@@ -93,6 +98,24 @@ def setup_epixhr(self):
         self.seedCut = 4  ## maybe the minimum sensible
         self.neighborCut = 0.5  ## probably too low given the noise
 
+    def setup_epixUHR(self):
+        self.cameraType = "epixUHR"
+        self.g0cut = 1 << 11
+        self.nModules = 4
+        ## per module (aka asic)
+        self.nRows = 168
+        self.nBanksRow = 28
+        self.nRowsPerBank = int(self.nRows / self.nBanksRow)
+        self.nCols = 192
+        self.nBanksCol = 16
+        self.nColsPerBank = int(self.nCols / self.nBanksCol)
+        self.preferredCommonMode = "clusterCommonMode"  ## not yet defined
+        self.clusterShape = [3, 3]
+        self.gainMode = None  ## high, medium, low, lower
+        self.aduPerKeV = 3 ## high gain; medium is down by 2.8
+        self.seedCut = 2
+        self.neighborCut = 0.25  ## ditto
+
     def setup_epixM(self):
         self.cameraType = "epixM"
         self.g0cut = 1 << 15
@@ -178,11 +201,16 @@ def setup_rixsCCD(self):
         self.nBanks = 16
         self.nCols = 4800 - self.nBanks * self.nTestPixelsPerBank
         self.preferredCommonMode = "rixsCCDTestPixelSubtraction"
+        self.dimension = 2
+        self.nModules = 1
         if self.detectorSubtype == "1d":
             self.nRows = 1
             self.clusterShape = [1, 5]  ## might be [1,3]
-            self.dimension = 2
+            ##self.dimension = 2
         else:
             self.nRows = 1200
             self.clusterShape = [3, 5]  ## maybe
         self.g0cut = 1 << 16
+        self.seedCut = 100
+        self.neighborCut = 5
+        ##self.aduPerKeV = 1000/3.6