Updates for UHR and incidental archon and non-psana stuff

calibrationSuite/basicSuiteScript.py

@@ -104,12 +104,13 @@ def getRawData(self, evt, gainBitsMasked=True, negativeGain=False):
             nZero = frames.size - np.count_nonzero(frames)
         try:
             dz = self.nZero - nZero
-            if dz != 0:
-                print("found %d new zero pixels, expected %d, setting frame to None" % (dz, self.nZero))
+            ##if dz != 0:
+            if abs(dz) > 10: ## add a flag for just epixM...
+                print("found %d new zero pixels, expected %d, setting frame to None, size was %d" % (dz, self.nZero, frames.size))
                 return None
         except Exception:
             self.nZero = nZero
-            print("Starting with %d zero pixels, will require exactly that many for this run" % (nZero))
+            print("Starting with %d zero pixels, will maybe require exactly that many for this run, size is %d" % (nZero, frames.size))
 
             try:
                 self.dumpEpixMHeaderInfo(evt)
@@ -129,12 +130,14 @@ def getRawData(self, evt, gainBitsMasked=True, negativeGain=False):
             elif "tenBits" in self.special:
                 frames = frames & 0xFFF0
                 ##print("10bits")
+
         if self.negativeGain or negativeGain:
             zeroPixels = frames == 0
             maskedData = frames & self.gainBitsMask
             gainData = frames - maskedData
             frames = gainData + self.gainBitsMask - maskedData
             frames[zeroPixels] = 0
+
         if gainBitsMasked:
             return frames & self.gainBitsMask
         return frames

calibrationSuite/detectorInfo.py

@@ -42,6 +42,7 @@ def __init__(self, detType, detSubtype="1d", detVersion=0):
 
         knownTypes = [
             "epixhr",
+            "epixuhr",
             "epixm",
             "epix100",
             "Epix100a",
@@ -60,6 +61,8 @@ def __init__(self, detType, detSubtype="1d", detVersion=0):
     def setupDetector(self):  ## needs nModules to be set
         if self.detectorType == "epixhr":
             self.setup_epixhr()
+        elif self.detectorType == "epixuhr":
+            self.setup_epixUHR()
         elif self.detectorType == "epixm":
             self.setup_epixM()
         elif self.detectorType in ["epix100", "Epix100a"]:
@@ -70,7 +73,9 @@ def setupDetector(self):  ## needs nModules to be set
             self.setup_jungfrau()
         elif "epix10k" in self.detectorType.lower():
             self.setup_epix10k()
-
+        else:
+            raise Exception("detector not matched", self.detectorType)
+
     def setNModules(self, n):
         self.nModules = n
 
@@ -93,6 +98,24 @@ def setup_epixhr(self):
         self.seedCut = 4  ## maybe the minimum sensible
         self.neighborCut = 0.5  ## probably too low given the noise
 
+    def setup_epixUHR(self):
+        self.cameraType = "epixUHR"
+        self.g0cut = 1 << 11
+        self.nModules = 4
+        ## per module (aka asic)
+        self.nRows = 168
+        self.nBanksRow = 28
+        self.nRowsPerBank = int(self.nRows / self.nBanksRow)
+        self.nCols = 192
+        self.nBanksCol = 16
+        self.nColsPerBank = int(self.nCols / self.nBanksCol)
+        self.preferredCommonMode = "clusterCommonMode"  ## not yet defined
+        self.clusterShape = [3, 3]
+        self.gainMode = None  ## high, medium, low, lower
+        self.aduPerKeV = 40 ## high gain; medium is down by 2.8
+        self.seedCut = 2
+        self.neighborCut = 0.25  ## ditto
+
     def setup_epixM(self):
         self.cameraType = "epixM"
         self.g0cut = 1 << 15
@@ -178,11 +201,16 @@ def setup_rixsCCD(self):
         self.nBanks = 16
         self.nCols = 4800 - self.nBanks * self.nTestPixelsPerBank
         self.preferredCommonMode = "rixsCCDTestPixelSubtraction"
+        self.dimension = 2
+        self.nModules = 1
         if self.detectorSubtype == "1d":
             self.nRows = 1
             self.clusterShape = [1, 5]  ## might be [1,3]
-            self.dimension = 2
+            ##self.dimension = 2
         else:
             self.nRows = 1200
             self.clusterShape = [3, 5]  ## maybe
         self.g0cut = 1 << 16
+        self.seedCut = 100
+        self.neighborCut = 5
+        ##self.aduPerKeV = 1000/3.6  

calibrationSuite/psana2Base.py

@@ -77,8 +77,7 @@ def setupPsana(self):
         ## use a dict etc.
         self.det = self.myrun.Detector(self.experimentHash["detectorType"])
         if self.det is None:
-            print("no det object for epixhr, what?  Pretend it's ok.")
-            ##raise Exception
+            raise Exception("no det object found")
         ## could set to None and reset with first frame I guess, or does the det object know?
 
         try:
@@ -220,7 +219,7 @@ def getEventCodes(self, evt):
 
     def getPulseId(self, evt):
         if self.timing is None:
-            return []
+            return 0
         return self.timing.raw.pulseId(evt)
 
     def isKicked(self, evt):

config_files/archonSuiteConfig.py

@@ -13,9 +13,9 @@
 # experimentHash = {'exp':'rixc00121', 'location':'RixEndstation',
 experimentHash = {
     "detectorType": "archon",
-    ##"detectorSubtype": "2d",
-    "detectorSubtype": "1d",
-    "exp": "rixc00122",
+    "detectorSubtype": "2d",
+    ##"detectorSubtype": "1d",
+    "exp": "rixx1017523",
     "location": "RixEndstation",
     "seedCut": 30,  ##from old scripts
     # "fluxSource": "MfxDg1BmMon",
@@ -27,14 +27,22 @@
         [0, 10],
         [0, 100],
         [0, 1000],
-        [0, 2000],
+        [0, 2000]
     ],
     "ignoreEventCodeCheck": True,
+    "analyzedModules": [0],
     # 'ROIs':['module0', 'module2', 'module4', 'module6', 'module10','module12', 'module14']
     # 'ROIs':['roiFromSwitched_e557_rmfxx1005021']
     # 'ROIs':['allHRasicPixels', 'goodboxROI']#'roiAbove7k_raw_r123']
     # "ROIs": ["../data/XavierV4_2", "../data/OffXavierV4_2"],
-    "regionSlice": np.s_[0:1, 0:1, 0:4800],  ##[1d]
+    ##"regionSlice": np.s_[0:1, 0:1, 0:4800],  ##[1d]
+    "regionSlice": np.s_[0:1, 0:600, 0:4800],  ##[2d with ROI....]
 }
 # more complex approach allowing run ranges
 # fluxHash = {1:['MFX-USR-DIO', 11]}
+if experimentHash['detectorSubtype'] == "2d":
+    experimentHash['singlePixels'] += [[100, 100],
+                                       [200, 200],
+                                       [300, 300],
+                                       [400, 400],
+                                       [500, 500]]

config_files/epixUHRSuiteConfig.py

@@ -0,0 +1,47 @@
+##############################################################################
+## This file is part of 'SLAC Beamtime Calibration Suite'.
+## It is subject to the license terms in the LICENSE.txt file found in the
+## top-level directory of this distribution and at:
+##    https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html.
+## No part of 'SLAC Beamtime Calibration Suite', including this file,
+## may be copied, modified, propagated, or distributed except according to
+## the terms contained in the LICENSE.txt file.
+##############################################################################
+import numpy as np
+
+# experimentHash = {'exp':'mfxx1005021', 'location':'MfxEndstation', 'fluxSource':'MFX-USR-DIO', 'fluxChannels':[11], 'fluxSign':-1}
+# experimentHash = {'exp':'rixc00121', 'location':'RixEndstation',
+singlePixelArray = []
+##for i in [0, 2, 3]:##range(0, 3):
+for i in [1, 3]:  ##range(0, 3):
+    singlePixelArray.append([i, 55, 10])
+    singlePixelArray.append([i, 120, 10])
+    singlePixelArray.append([i, 55, 90])
+    singlePixelArray.append([i, 120, 90])
+    singlePixelArray.append([i, 55, 190])
+    singlePixelArray.append([i, 120, 190])
+
+experimentHash = {
+    "detectorType": "epixuhr",
+    ##"detectorVersion": 1,  ## new firmware
+    "exp": "rixx1005922",
+    "location": "RixEndstation",
+    "analyzedModules": [0, 1, 2, 3],
+    "seedCut": 40,  ## pure guess
+    "neighborCut": 10,  ##pure guess
+    "fluxSource": "MfxDg1BmMon",
+    # "fluxSource": "MfxDg2BmMon",
+    "fluxChannels": [15],  ## or 11 if we see saturation
+    # "fluxSign": 1,  ## for dg2
+    "fluxSign": -1,
+    "singlePixels": singlePixelArray,
+    # 'ROIs':['module0', 'module2', 'module4', 'module6', 'module10','module12', 'module14']
+    # 'ROIs':['roiFromSwitched_e557_rmfxx1005021']
+    # 'ROIs':['allHRasicPixels', 'goodboxROI']#'roiAbove7k_raw_r123']
+    # "ROIs": ["../data/XavierV4_2", "../data/OffXavierV4_2"],
+    "ROIs": [
+        "../data/fake_UHR_ROI_0.npy",
+        "../data/fake_UHR_ROI_1.npy",
+    ],
+    "regionSlice": np.s_[0:4, 0:168:, 0:192] ## whole thing
+}

config_files/npy_epixMSuiteConfig.py

@@ -0,0 +1,42 @@
+##############################################################################
+## This file is part of 'SLAC Beamtime Calibration Suite'.
+## It is subject to the license terms in the LICENSE.txt file found in the
+## top-level directory of this distribution and at:
+##    https://confluence.slac.stanford.edu/display/ppareg/LICENSE.html.
+## No part of 'SLAC Beamtime Calibration Suite', including this file,
+## may be copied, modified, propagated, or distributed except according to
+## the terms contained in the LICENSE.txt file.
+##############################################################################
+import numpy as np
+
+singlePixelArray = []
+for i in [0]:  ##range(0, 3):
+    singlePixelArray.append([0, 66, 66])
+    singlePixelArray.append([0, 6, 6])
+    singlePixelArray.append([0, 55, 200])
+    singlePixelArray.append([0, 120, 200])
+    singlePixelArray.append([0, 55, 300])
+    singlePixelArray.append([0, 120, 300])
+
+experimentHash = {
+    "detectorType": "epixm",
+    "detectorVersion": 1,  ## new firmware
+    "exp": "rixx1005922",
+    "location": "RixEndstation",
+    "location": "ascLab",
+    "analyzedModules": [0],
+    "seedCut": 40,  ## pure guess
+    "neighborCut": 10,  ##pure guess
+    ##"fluxSource": "MfxDg1BmMon",
+    # "fluxSource": "MfxDg2BmMon",
+    "fluxChannels": [15],  ## or 11 if we see saturation
+    # "fluxSign": 1,  ## for dg2
+    "fluxSign": -1,
+    "singlePixels": singlePixelArray,
+    # 'ROIs':['module0', 'module2', 'module4', 'module6', 'module10','module12', 'module14']
+    # 'ROIs':['roiFromSwitched_e557_rmfxx1005021']
+    # 'ROIs':['allHRasicPixels', 'goodboxROI']#'roiAbove7k_raw_r123']
+    # "ROIs": ["../data/XavierV4_2", "../data/OffXavierV4_2"],
+    "ROIs": ["../data/square9_0_epixM_roi.npy", "../data/square9_1_epixM_roi.npy"],
+    "regionSlice": np.s_[0:1, 0:192:, 0:384],  ## small region near Kaz rec
+}

setup_archon.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# Setup environment variables
+
+
+# current_dir=$(pwd)
+git_project_root_dir=$(git rev-parse --show-toplevel)
+# so scripts can find the calibrationSuite library code
+export PYTHONPATH="$PYTHONPATH:$git_project_root_dir"
+echo "PYTHONPATH = $PYTHONPATH"
+
+# so output folders are written in a shared location
+export OUTPUT_ROOT="/sdf/data/lcls/ds/rix/rixx1017523/results/philiph/"
+echo "OUTPUT_ROOT = $OUTPUT_ROOT"
+
+# point to which config file to use
+export SUITE_CONFIG="$git_project_root_dir/config_files/archonSuiteConfig.py"
+echo "SUITE_CONFIG = $SUITE_CONFIG"

suite_scripts/AnalyzeH5.py

@@ -41,6 +41,7 @@ def __init__(self):
         self.label = args.label
         self.camera = 0
         self.seedCut = args.seedCut
+        self.photonEnergy = args.photonEnergy
         self.isTestRun = args.special is not None and "testRun" in args.special
 
     def getFiles(self):
@@ -114,7 +115,7 @@ def getRowsColsFromSliceCoordinates(self):
     def analyze(self):
         if self.analysis == "cluster":
             self.clusterAnalysis()
-        if self.analysis == "linearity":
+        elif self.analysis == "linearity":
             self.linearityAnalysis()
         else:
             print("unknown analysis type %s" % (self.analysis))
@@ -140,6 +141,8 @@ def clusterAnalysis(self):
         else:
             self.lowEnergyCut = self.seedCut * 0.8  ## 0.8 is dumb here
         self.highEnergyCut = 15  ## fix - should be 1.5 photons or something
+        if self.photonEnergy is not None:
+            self.highEnergyCut = self.photonEnergy*1.5
         ##tmp
         npyFileName = "%s/r%d_clusters.npy" % (self.outputDir, self.run)
         np.save(npyFileName, clusters)
@@ -150,7 +153,12 @@ def clusterAnalysis(self):
         if energyHist is None:
             return
 
-        _, bins = np.histogram(energyHist, 250, [-5, 45])
+        minE, maxE = -5, 45
+        if self.photonEnergy is not None:
+            minE = -1*self.photonEnergy/2.
+            maxE = self.photonEnergy*3
+
+        _, bins = np.histogram(energyHist, 250, [minE, maxE])
         plt.hist(bins[:-1], bins, weights=energyHist)  ##, log=True)
         plt.grid(which="major", linewidth=0.5)
         plt.title = "All pixel energies in run after common mode correction"
@@ -217,7 +225,10 @@ def analyzeSimpleClusters(self, clusters):
         ##maximumModule = int(clusters[:, 1].max())
         ##fitInfo = np.zeros((maximumModule + 1, rows, cols, 5))  ## mean, std, area, mu, sigma
         fitInfo = np.zeros((self.detModules, self.detRows, self.detCols, 5))  ## mean, std, area, mu, sigma
-        smallSquareClusters = ancillaryMethods.getSmallSquareClusters(clusters, nPixelCut=3)
+        if False: ## turn on if one believes the baseline is right
+            smallSquareClusters = ancillaryMethods.getSmallSquareClusters(clusters, nPixelCut=3)
+        else:
+            smallSquareClusters = clusters
         for m in self.analyzedModules:
             modClusters = ancillaryMethods.getMatchedClusters(smallSquareClusters, "module", m)
             for i in range(self.rowStart, self.rowStop):
@@ -232,8 +243,9 @@ def analyzeSimpleClusters(self, clusters):
                     detRow, detCol = i, j  ## mostly for clarity
                     currGoodClusters = ancillaryMethods.getMatchedClusters(rowModClusters, "column", j)
                     if len(currGoodClusters) < 5:
-                        print("too few clusters in slice pixel %d, %d, %d: %d" % (m, i, j, len(currGoodClusters)))
-                        logger.info("too few clusters in slice pixel %d, %d, %d: %d" % (m, i, j, len(currGoodClusters)))
+                        if i%10==0 and j%10==0:
+                            print("too few clusters in slice pixel %d, %d, %d: %d" % (m, i, j, len(currGoodClusters)))
+                            logger.info("too few clusters in slice pixel %d, %d, %d: %d" % (m, i, j, len(currGoodClusters)))
                         continue
                     energies = ancillaryMethods.getClusterEnergies(currGoodClusters)
                     photonEcut = np.bitwise_and(energies > self.lowEnergyCut, energies < self.highEnergyCut)
@@ -280,7 +292,10 @@ def analyzeSimpleClusters(self, clusters):
         logger.info("Wrote file: " + npyFileName)
 
         gains = fitInfo[:, :, 3]
-        goodGains = gains[np.bitwise_and(gains > 0, gains < 15)]
+        maxE = 15
+        if self.photonEnergy is not None:
+            maxE = self.photonEnergy*2
+        goodGains = gains[np.bitwise_and(gains > 0, gains < maxE)]
         ax = plt.subplot()
         ax.hist(goodGains, 100)
         ax.set_xlabel("energy (keV)")

suite_scripts/EventScanParallelSlice.py

@@ -210,7 +210,7 @@ def analyze_h5(self, dataFile, label):
         ##skip_283_check = "skip283" in esp.special
         skip_283_check = "fakeBeamCode" in esp.special
     except Exception:
-        skip_283_check = False  ## for running at MFX
+        skip_283_check = False  ## for not running at MFX
 
     zeroLowGain = False
     zeroHighGain = False
@@ -242,7 +242,7 @@ def analyze_h5(self, dataFile, label):
             continue
         ec = esp.getEventCodes(evt)
         if not skip_283_check:
-            if not ec[283]:
+            if not esp.isBeamEvent(evt):
                 ##print(ec)
                 continue
         frames = esp.getRawData(evt, gainBitsMasked=True)
@@ -312,6 +312,8 @@ def analyze_h5(self, dataFile, label):
         else:
             timestamp = evt.datetime().timestamp()
             pulseId = esp.getPulseId(evt)
+            if pulseId == 0:
+                pulseId = timestamp ## better than nothing
         smdDict = {
             "timestamps": timestamp,
             "pulseIds": pulseId,