From 1197f8f5e5a18f0710aa3cb747e76e1e6ab28c6c Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sun, 29 May 2022 15:46:32 +0200 Subject: [PATCH 01/78] Remove unused imports. --- ctapipe_io_magic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index c0f9db7..f64d28c 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -17,7 +17,7 @@ from pkg_resources import resource_filename from ctapipe.io import EventSource, DataLevel -from ctapipe.core import Container, Field, Provenance +from ctapipe.core import Provenance from ctapipe.core.traits import Bool from ctapipe.coordinates import CameraFrame From 177bfc413a053091e3b306028ac98766a492d9da Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sun, 29 May 2022 17:29:29 +0200 Subject: [PATCH 02/78] Add class to read superstar files. --- ctapipe_io_magic/__init__.py | 300 +++++++++++++++++++++++++++++++++++ 1 file changed, 300 insertions(+) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index f64d28c..21bc979 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1516,3 +1516,303 @@ def _load_data(self): f'You may have to match events by timestamp at a later stage.') return calib_data + + +class MarsSuperstarRun: + """ + This class implements reading of cosmic events from a MAGIC superstar run file. + """ + + def __init__(self, uproot_file, is_mc, n_cam_pixels=1039): + """ + Constructor of the class. Loads an input uproot file + and store the informaiton to constructor variables. + + Parameters + ---------- + uproot_file: uproot.reading.ReadOnlyDirectory + A calibrated file opened by uproot via uproot.open(file_path) + is_mc: bool + Flag to MC data + n_cam_pixels: int + The number of pixels of the MAGIC camera + """ + + self.uproot_file = uproot_file + self.is_mc = is_mc + self.n_cam_pixels = n_cam_pixels + + # Load the input data: + stereo_data = self._load_data() + + self.cosmic_events = stereo_data["cosmic_events"] + + @property + def n_cosmic_events(self): + return len(self.cosmic_events[1].get('trigger_pattern', [])) + + def _load_data(self): + """ + This method loads cosmic and pedestal events, and monitoring data + from an input calibrated file and returns them as a dictionary. + + Returns + ------- + stereo_data: dict + A dictionary with the event properties, + cosmic and pedestal events, and monitoring data + """ + + common_branches = dict() + pointing_branches = dict() + timing_branches = dict() + hillas_branches = dict() + hillas_src_branches = dict() + imagepar_branches = dict() + new_imagepar_branches = dict() + hillas_timefit_branches = dict() + mc_branches = dict() + + for tel_id in [1, 2]: + + common_branches[tel_id] = [ + f'MRawEvtHeader_{tel_id}.fStereoEvtNumber', + f'MTriggerPattern_{tel_id}.fPrescaled', + ] + + pointing_branches[tel_id] = [ + f'MPointingPos_{tel_id}.fZd', # [deg] + f'MPointingPos_{tel_id}.fAz', # [deg] + f'MPointingPos_{tel_id}.fRa', # [h] + f'MPointingPos_{tel_id}.fDec', # [deg] + f'MPointingPos_{tel_id}.fDevZd', # [deg] + f'MPointingPos_{tel_id}.fDevAz', # [deg] + f'MPointingPos_{tel_id}.fDevHa', # [h] + f'MPointingPos_{tel_id}.fDevDec', # [deg] + ] + + # Branches applicable for real data: + timing_branches[tel_id] = [ + f'MTime_{tel_id}.fMjd', + f'MTime_{tel_id}.fTime.fMilliSec', + f'MTime_{tel_id}.fNanoSec', + ] + + hillas_branches[tel_id] = [ + f'MHillas_{tel_id}.fLength', # [mm] + f'MHillas_{tel_id}.fWidth', # [mm] + f'MHillas_{tel_id}.fDelta', # [rad] + f'MHillas_{tel_id}.fSize', + f'MHillas_{tel_id}.fMeanX', # [mm] + f'MHillas_{tel_id}.fMeanY', # [mm] + ] + + hillas_src_branches[tel_id] = [ + f'MHillasSrc_{tel_id}.fDCADelta', # [deg] Angle of the shower axis with respect to the x-axis + f'MHillasSrc_{tel_id}.fDist', # [mm] distance between src and center of ellipse + ] + + imagepar_branches[tel_id] = [ + f'MImagePar_{tel_id}.fNumIslands', # Number of distinct pixel groupings in image + ] + + new_imagepar_branches[tel_id] = [ + f'MNewImagePar_{tel_id}.fNumUsedPixels', # Number of pixels which survived the image cleaning + f'MNewImagePar_{tel_id}.fLeakage1', # (photons in most outer ring of pixels) over fSize + f'MNewImagePar_{tel_id}.fLeakage2', # (photons in the 2 outer rings of pixels) over fSize + ] + + hillas_timefit_branches[tel_id] = [ + f'MHillaTimeFit_{tel_id}.fP1Const', # [Time_slices] + f'MHillaTimeFit_{tel_id}.fP1Grad', # [Time_slices]/[mm] + ] + + mc_branches[tel_id] = [ + f'MMcEvt_{tel_id}.fEnergy', + f'MMcEvt_{tel_id}.fTheta', + f'MMcEvt_{tel_id}.fPhi', + f'MMcEvt_{tel_id}.fPartId', + f'MMcEvt_{tel_id}.fZFirstInteraction', + f'MMcEvt_{tel_id}.fCoreX', + f'MMcEvt_{tel_id}.fCoreY', + ] + + stereo_branches = [ + 'MStereoPar.fValid', + 'MStereoPar.fDirectionX', + 'MStereoPar.fDirectionY', + 'MStereoPar.fDirectionZd', + 'MStereoPar.fDirectionAz', + 'MStereoPar.fDirectionDec', + 'MStereoPar.fDirectionRA', + 'MStereoPar.fTheta2', + 'MStereoPar.fCoreX', + 'MStereoPar.fCoreY', + 'MStereoPar.fM1Impact', + 'MStereoPar.fM2Impact', + 'MStereoPar.fMaxHeight', + ] + + stereo_data = { + 'cosmic_events': dict(), + } + + event_data = dict() + + for tel_id in [1, 2]: + + # Initialize the data container: + + event_data[tel_id] = dict() + + if self.is_mc: + # Only for cosmic events because MC data do not have pedestal events: + events_cut = f'(MTriggerPattern_{tel_id}.fPrescaled == {MC_STEREO_TRIGGER_PATTERN})' \ + f' & (MRawEvtHeader_{tel_id}.fStereoEvtNumber != 0)' + else: + events_cut = f'(MTriggerPattern_{tel_id}.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})' + + # read common information from RunHeaders + sampling_speed = self.uproot_file['RunHeaders'][f'MRawRunHeader_{tel_id}.fSamplingFrequency'].array(library='np')[0]/1000. # GHz + + # Reading the information common to cosmic and pedestal events: + common_info = self.uproot_file['Events'].arrays( + expressions=common_branches[tel_id], + cut=events_cut, + library='np', + ) + + event_data[tel_id]['trigger_pattern'] = np.array(common_info[f'MTriggerPattern_{tel_id}.fPrescaled'], dtype=int) + event_data[tel_id]['stereo_event_number'] = np.array(common_info[f'MRawEvtHeader_{tel_id}.fStereoEvtNumber'], dtype=int) + + if self.is_mc: + + # Reading the MC information: + mc_info = self.uproot_file['Events'].arrays( + expressions=mc_branches[tel_id], + cut=events_cut, + library='np', + ) + + # Note that the branch "MMcEvt.fPhi" seems to be the angle between the direction + # of the magnetic north and the momentum of a simulated primary particle, defined + # between -pi and pi, negative if the momentum pointing eastward, positive westward. + # The conversion to azimuth should be 180 - fPhi + magnetic_declination: + event_data[tel_id]['mc_energy'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fEnergy'], u.GeV) + event_data[tel_id]['mc_theta'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fTheta'], u.rad) + event_data[tel_id]['mc_phi'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fPhi'], u.rad) + event_data[tel_id]['mc_shower_primary_id'] = np.array(mc_info[f'MMcEvt_{tel_id}.fPartId'], dtype=int) + event_data[tel_id]['mc_h_first_int'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fZFirstInteraction'], u.cm) + event_data[tel_id]['mc_core_x'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fCoreX'], u.cm) + event_data[tel_id]['mc_core_y'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fCoreY'], u.cm) + + else: + # Reading the event timing information: + timing_info = self.uproot_file['Events'].arrays( + expressions=timing_branches[tel_id], + cut=events_cut, + library='np', + ) + + # In order to keep the precision of timestamps, here the Decimal module is used. + # At the later steps, the precise information can be extracted by specifying + # the sub-format of value to 'long', i.e., Time.to_value(format='unix', subfmt='long'): + event_obs_day = Time(timing_info[f'MTime_{tel_id}.fMjd'], format='mjd', scale='utc') + event_obs_day = np.round(event_obs_day.unix) + event_obs_day = np.array([Decimal(str(x)) for x in event_obs_day]) + + event_millisec = np.round(timing_info[f'MTime_{tel_id}.fTime.fMilliSec'] * msec2sec, 3) + event_millisec = np.array([Decimal(str(x)) for x in event_millisec]) + + event_nanosec = np.round(timing_info[f'MTime_{tel_id}.fNanoSec'] * nsec2sec, 7) + event_nanosec = np.array([Decimal(str(x)) for x in event_nanosec]) + + event_time = event_obs_day + event_millisec + event_nanosec + event_data[tel_id]['time'] = Time(event_time, format='unix', scale='utc') + + # Reading the telescope pointing information: + pointing = self.uproot_file['Events'].arrays( + expressions=pointing_branches[tel_id], + cut=events_cut, + library='np', + ) + + pointing_az = pointing[f'MPointingPos_{tel_id}.fAz'] - pointing[f'MPointingPos_{tel_id}.fDevAz'] + pointing_zd = pointing[f'MPointingPos_{tel_id}.fZd'] - pointing[f'MPointingPos_{tel_id}.fDevZd'] + + # N.B. the positive sign here, as HA = local sidereal time - ra: + pointing_ra = (pointing[f'MPointingPos_{tel_id}.fRa'] + pointing[f'MPointingPos_{tel_id}.fDevHa']) * degrees_per_hour + pointing_dec = pointing[f'MPointingPos_{tel_id}.fDec'] - pointing[f'MPointingPos_{tel_id}.fDevDec'] + + event_data[tel_id]['pointing_az'] = u.Quantity(pointing_az, u.deg) + event_data[tel_id]['pointing_alt'] = u.Quantity(90 - pointing_zd, u.deg) + event_data[tel_id]['pointing_ra'] = u.Quantity(pointing_ra, u.deg) + event_data[tel_id]['pointing_dec'] = u.Quantity(pointing_dec, u.deg) + + hillas = self.uproot_file['Events'].arrays( + expressions=hillas_branches[tel_id], + cut=events_cut, + library='np', + ) + + event_data[tel_id]['length'] = u.Quantity(hillas[f'MHillas_{tel_id}.fLength'], u.mm).to(u.m) + event_data[tel_id]['width'] = u.Quantity(hillas[f'MHillas_{tel_id}.fWidth'], u.mm).to(u.m) + event_data[tel_id]['x'] = u.Quantity(hillas[f'MHillas_{tel_id}.fMeanX'], u.mm).to(u.m) + event_data[tel_id]['y'] = u.Quantity(hillas[f'MHillas_{tel_id}.fMeanY'], u.mm).to(u.m) + event_data[tel_id]['intensity'] = u.Quantity(hillas[f'MHillas_{tel_id}.fSize'], u.mm).to(u.m) + event_data[tel_id]['psi'] = u.Quantity(hillas[f'MHillas_{tel_id}.fDelta'], u.rad).to(u.deg) + + hillas_src = self.uproot_file['Events'].arrays( + expressions=hillas_src_branches[tel_id], + cut=events_cut, + library='np', + ) + + event_data[tel_id]['phi'] = u.Quantity(hillas_src[f'MHillasSrc_{tel_id}.fDCADelta'], u.deg) + + imagepar = self.uproot_file['Events'].arrays( + expressions=imagepar_branches[tel_id], + cut=events_cut, + library='np', + ) + + event_data[tel_id]['num_islands'] = imagepar[f'MImagePar_{tel_id}.fNumIslands'] + + new_imagepar = self.uproot_file['Events'].arrays( + expressions=new_imagepar_branches[tel_id], + cut=events_cut, + library='np', + ) + + event_data[tel_id]['num_pixels'] = new_imagepar[f'MNewImagePar_{tel_id}.fNumUsedPixels'] + event_data[tel_id]['intensity_width_1'] = new_imagepar[f'MNewImagePar_{tel_id}.fLeakage1'] * event_data[tel_id]['intensity'] + event_data[tel_id]['intensity_width_2'] = new_imagepar[f'MNewImagePar_{tel_id}.fLeakage2'] * event_data[tel_id]['intensity'] + + hillas_timefit = self.uproot_file['Events'].arrays( + expressions=hillas_timefit_branches[tel_id], + cut=events_cut, + library='np', + ) + + event_data[tel_id]['slope'] = u.Quantity(hillas_timefit[f'MHillaTimeFit_{tel_id}.fP1Grad'], 1/u.mm).to(1/u.m) * (1./sampling_speed) + event_data[tel_id]['intercept'] = hillas_timefit[f'MHillaTimeFit_{tel_id}.fP1Const'] * (1./sampling_speed) + + stereo_data["cosmic_events"] = event_data + + stereo_parameters = self.uproot_file['Events'].arrays( + expressions=stereo_branches[tel_id], + cut=events_cut, + library='np', + ) + + stereo_params = dict() + stereo_params['alt'] = u.Quantity(90 - stereo_parameters['MStereoPar.fDirectionZd'], u.deg) + stereo_params['az'] = u.Quantity(stereo_parameters['MStereoPar.fDirectionAz'], u.deg) + stereo_params['core_x'] = u.Quantity(stereo_parameters['MStereoPar.fCoreX'], u.cm).to(u.m) + stereo_params['core_y'] = u.Quantity(stereo_parameters['MStereoPar.fCoreY'], u.cm).to(u.m) + stereo_params['h_max'] = u.Quantity(stereo_parameters['MStereoPar.fMaxHeight'], u.cm).to(u.m) + stereo_params['is_valid'] = stereo_parameters['MStereoPar.fValid'] + + stereo_data["cosmic_events"]["stereo"] = stereo_params + + return stereo_data From 8139d421d9c028a8ab66c3c1313f54dd97553a66 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sun, 29 May 2022 18:34:27 +0200 Subject: [PATCH 03/78] Use different regex for different datalevels. --- ctapipe_io_magic/__init__.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 21bc979..6ca57df 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -165,9 +165,22 @@ def __init__(self, input_url=None, config=None, parent=None, **kwargs): info_tuple = self.get_run_info_from_name(name) run = info_tuple[0] telescope = info_tuple[2] - - regex = rf"\d{{8}}_M{telescope}_0{run}\.\d{{3}}_Y_.*\.root" - regex_mc = rf"GA_M{telescope}_\w+_{run}_Y_.*\.root" + datalevel = info_tuple[3] + + if datalevel == MARSDataLevel.CALIBRATED: + regex = rf"\d{{8}}_M{telescope}_0{run}\.\d{{3}}_Y_.*\.root" + regex_mc = rf"GA_M{telescope}_\w+_{run}_Y_.*\.root" + elif datalevel == MARSDataLevel.STAR: + regex = rf"\d{{8}}_M{telescope}_0{run}\.\d{{3}}_I_.*\.root" + regex_mc = rf"GA_M{telescope}_\w+_{run}_I_.*\.root" + elif datalevel == MARSDataLevel.SUPERSTAR: + regex = rf"\d{{8}}_0{run}_S_.*\.root" + regex_mc = r"GA_za.*_S_.*\.root" + elif datalevel == MARSDataLevel.MELIBEA: + regex = rf"\d{{8}}_0{run}_Q_.*\.root" + regex_mc = r"GA_za.*_Q_.*\.root" + else: + raise ValueError(f"Datalevel not recognized: {datalevel}") reg_comp = re.compile(regex) reg_comp_mc = re.compile(regex_mc) From 418a7ef515cb3907bd145f654742c6455b721ec5 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sun, 29 May 2022 18:34:59 +0200 Subject: [PATCH 04/78] Prepare drive information only for calibrated files. --- ctapipe_io_magic/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 6ca57df..77b733e 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -226,7 +226,8 @@ def __init__(self, input_url=None, config=None, parent=None, **kwargs): self._subarray_info = self.prepare_subarray_info() if not self.is_simulation: - self.drive_information = self.prepare_drive_information() + if self.mars_datalevel == MARSDataLevel.CALIBRATED: + self.drive_information = self.prepare_drive_information() # Get the arrival time differences self.event_time_diffs = self.get_event_time_difference() From b7032e858d63ff934ddcffde4a1328a69d6a5886 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sun, 29 May 2022 18:36:00 +0200 Subject: [PATCH 05/78] Adapt parse_run_info for different datalevels. --- ctapipe_io_magic/__init__.py | 47 ++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 77b733e..f90dcb4 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -402,6 +402,15 @@ def parse_run_info(self): 'MRawRunHeader.fTelescopeNumber', ] + runinfo_stereo_array_list = [ + 'MRawRunHeader_1.fRunNumber', + 'MRawRunHeader_1.fRunType', + 'MRawRunHeader_1.fTelescopeNumber', + 'MRawRunHeader_2.fRunNumber', + 'MRawRunHeader_2.fRunType', + 'MRawRunHeader_2.fTelescopeNumber', + ] + run_numbers = [] is_simulation = [] telescope_numbers = [] @@ -409,11 +418,39 @@ def parse_run_info(self): for rootf in self.files_: - run_info = rootf['RunHeaders'].arrays( - runinfo_array_list, library="np") - run_number = int(run_info['MRawRunHeader.fRunNumber'][0]) - run_type = int(run_info['MRawRunHeader.fRunType'][0]) - telescope_number = int(run_info['MRawRunHeader.fTelescopeNumber'][0]) + events_tree = rootf['Events'] + + melibea_trees = ['MHadronness', 'MStereoParDisp', 'MEnergyEst'] + superstar_trees = ['MHillas_1', 'MHillas_2', 'MStereoPar'] + star_trees = ['MHillas'] + + datalevel = MARSDataLevel.CALIBRATED + events_keys = events_tree.keys() + trees_in_file = [tree in events_keys for tree in melibea_trees] + if all(trees_in_file): + datalevel = MARSDataLevel.MELIBEA + trees_in_file = [tree in events_keys for tree in superstar_trees] + if all(trees_in_file): + datalevel = MARSDataLevel.SUPERSTAR + trees_in_file = [tree in events_keys for tree in star_trees] + if all(trees_in_file): + datalevel = MARSDataLevel.STAR + + if datalevel <= MARSDataLevel.STAR: + run_info = rootf['RunHeaders'].arrays( + runinfo_array_list, library="np") + + run_number = int(run_info['MRawRunHeader.fRunNumber'][0]) + run_type = int(run_info['MRawRunHeader.fRunType'][0]) + telescope_numbers.append(int(run_info['MRawRunHeader.fTelescopeNumber'][0])) + else: + run_info = rootf['RunHeaders'].arrays( + runinfo_stereo_array_list, library="np") + + run_number = int(run_info['MRawRunHeader_1.fRunNumber'][0]) + run_type = int(run_info['MRawRunHeader_1.fRunType'][0]) + telescope_numbers.append(int(run_info['MRawRunHeader_1.fTelescopeNumber'][0])) + telescope_numbers.append(int(run_info['MRawRunHeader_2.fTelescopeNumber'][0])) # a note about run numbers: # mono data has run numbers starting with 1 or 2 (telescope dependent) From 0f65eb07d3630879534cc1ab4ff4f61b5ea5a8ae Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sun, 29 May 2022 18:36:34 +0200 Subject: [PATCH 06/78] Remainder of previous commit. --- ctapipe_io_magic/__init__.py | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index f90dcb4..d91f726 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -473,27 +473,8 @@ def parse_run_info(self): else: is_mc = False - events_tree = rootf['Events'] - - melibea_trees = ['MHadronness', 'MStereoParDisp', 'MEnergyEst'] - superstar_trees = ['MHillas_1', 'MHillas_2', 'MStereoPar'] - star_trees = ['MHillas'] - - datalevel = MARSDataLevel.CALIBRATED - events_keys = events_tree.keys() - trees_in_file = [tree in events_keys for tree in melibea_trees] - if all(trees_in_file): - datalevel = MARSDataLevel.MELIBEA - trees_in_file = [tree in events_keys for tree in superstar_trees] - if all(trees_in_file): - datalevel = MARSDataLevel.SUPERSTAR - trees_in_file = [tree in events_keys for tree in star_trees] - if all(trees_in_file): - datalevel = MARSDataLevel.STAR - run_numbers.append(run_number) is_simulation.append(is_mc) - telescope_numbers.append(telescope_number) datalevels.append(datalevel) run_numbers = np.unique(run_numbers).tolist() @@ -505,14 +486,14 @@ def parse_run_info(self): raise ValueError( "Loaded files contain both real data and simulation runs. \ Please load only data OR Monte Carlos.") - if len(telescope_numbers) > 1: - raise ValueError( - "Loaded files contain data from different telescopes. \ - Please load data belonging to the same telescope.") if len(datalevels) > 1: raise ValueError( "Loaded files contain data at different datalevels. \ Please load data belonging to the same datalevel.") + if len(telescope_numbers) > 1 and datalevels[0] <= MARSDataLevel.STAR: + raise ValueError( + "Loaded files contain data from different telescopes. \ + Please load data belonging to the same telescope.") return run_numbers, is_simulation, telescope_numbers, datalevels From 6112f6b933bedb3ed33c4ff29f5d65d0bb770bdb Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sun, 29 May 2022 18:37:00 +0200 Subject: [PATCH 07/78] Adapt parse_data_info for different datalevels. --- ctapipe_io_magic/__init__.py | 39 +++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index d91f726..483817e 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -527,26 +527,29 @@ def parse_data_info(self): trigger_tree = rootf["Trigger"] L3T_tree = rootf["L3T"] - # here we take the 2nd element (if possible) because sometimes - # the first trigger report has still the old prescaler values from a previous run - try: - prescaler_array = trigger_tree["MTriggerPrescFact.fPrescFact"].array(library="np") - except AssertionError: - logger.warning("No prescaler info found. Will assume standard stereo data.") - stereo = True - sumt = False - return stereo, sumt + if self.mars_datalevel <= MARSDataLevel.STAR: + # here we take the 2nd element (if possible) because sometimes + # the first trigger report has still the old prescaler values from a previous run + try: + prescaler_array = trigger_tree["MTriggerPrescFact.fPrescFact"].array(library="np") + except AssertionError: + logger.warning("No prescaler info found. Will assume standard stereo data.") + stereo = True + sumt = False + return stereo, sumt - prescaler_size = prescaler_array.size - if prescaler_size > 1: - prescaler = prescaler_array[1] - else: - prescaler = prescaler_array[0] + prescaler_size = prescaler_array.size + if prescaler_size > 1: + prescaler = prescaler_array[1] + else: + prescaler = prescaler_array[0] - if prescaler == prescaler_mono_nosumt or prescaler == prescaler_mono_sumt: - stereo = False - elif prescaler == prescaler_stereo: - stereo = True + if prescaler == prescaler_mono_nosumt or prescaler == prescaler_mono_sumt: + stereo = False + elif prescaler == prescaler_stereo: + stereo = True + else: + stereo = True else: stereo = True From 6f9f663d35aa3f9910b3d897a4264a723d13c53a Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sun, 29 May 2022 18:37:18 +0200 Subject: [PATCH 08/78] Adapt parse_subarray_info for different datalevels. --- ctapipe_io_magic/__init__.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 483817e..41f0441 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -621,10 +621,16 @@ def prepare_subarray_info(self): pulse_shape_lo_gain = np.array([0., 1., 2., 1., 0.]) pulse_shape_hi_gain = np.array([1., 2., 3., 2., 1.]) pulse_shape = np.vstack((pulse_shape_lo_gain, pulse_shape_hi_gain)) - sampling_speed = u.Quantity( - self.files_[0]['RunHeaders']['MRawRunHeader.fSamplingFrequency'].array(library='np')[0]/1000, - u.GHz - ) + if self.mars_datalevel <= MARSDataLevel.STAR: + sampling_speed = u.Quantity( + self.files_[0]['RunHeaders']['MRawRunHeader.fSamplingFrequency'].array(library='np')[0]/1000, + u.GHz + ) + else: + sampling_speed = u.Quantity( + self.files_[0]['RunHeaders']['MRawRunHeader_1.fSamplingFrequency'].array(library='np')[0]/1000, + u.GHz + ) camera_readout = CameraReadout( camera_name='MAGICCam', sampling_rate=sampling_speed, From 6771da021d0e80ec669ccf7477426318deda63a6 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sun, 29 May 2022 18:37:40 +0200 Subject: [PATCH 09/78] Adapt parse_metadata_info for different datalevels. --- ctapipe_io_magic/__init__.py | 77 +++++++++++++++++++++++++----------- 1 file changed, 54 insertions(+), 23 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 41f0441..c0aaff3 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -702,33 +702,54 @@ def parse_metadata_info(self): - ROOT version """ + if self.mars_datalevel <= MARSDataLevel.STAR: + suffix = "" + else: + suffix = "_1" + metadatainfo_array_list_runheaders = [ - 'MRawRunHeader.fSubRunIndex', - 'MRawRunHeader.fSourceRA', - 'MRawRunHeader.fSourceDEC', - 'MRawRunHeader.fSourceName[80]', - 'MRawRunHeader.fObservationMode[60]', - 'MRawRunHeader.fProjectName[100]', + f'MRawRunHeader{suffix}.fSubRunIndex', + f'MRawRunHeader{suffix}.fSourceRA', + f'MRawRunHeader{suffix}.fSourceDEC', + f'MRawRunHeader{suffix}.fSourceName[80]', + f'MRawRunHeader{suffix}.fObservationMode[60]', + f'MRawRunHeader{suffix}.fProjectName[100]', ] - metadatainfo_array_list_runtails = [ - 'MMarsVersion_sorcerer.fMARSVersionCode', - 'MMarsVersion_sorcerer.fROOTVersionCode', - ] + if self.mars_datalevel == MARSDataLevel.CALIBRATED: + metadatainfo_array_list_runtails = [ + 'MMarsVersion_sorcerer.fMARSVersionCode', + 'MMarsVersion_sorcerer.fROOTVersionCode', + ] + elif self.mars_datalevel == MARSDataLevel.STAR: + metadatainfo_array_list_runtails = [ + 'MMarsVersion_star.fMARSVersionCode', + 'MMarsVersion_star.fROOTVersionCode', + ] + elif self.mars_datalevel == MARSDataLevel.SUPERSTAR: + metadatainfo_array_list_runtails = [ + 'MMarsVersion_superstar.fMARSVersionCode', + 'MMarsVersion_superstar.fROOTVersionCode', + ] + elif self.mars_datalevel == MARSDataLevel.MELIBEA: + metadatainfo_array_list_runtails = [ + 'MMarsVersion_melibea.fMARSVersionCode', + 'MMarsVersion_melibea.fROOTVersionCode', + ] metadata = dict() metadata["file_list"] = self.file_list metadata['run_numbers'] = self.run_numbers metadata['is_simulation'] = self.is_simulation - metadata['telescope'] = self.telescope + metadata['telescopes'] = self.telescope metadata['subrun_number'] = [] metadata['source_ra'] = [] metadata['source_dec'] = [] metadata['source_name'] = [] metadata['observation_mode'] = [] metadata['project_name'] = [] - metadata['mars_version_sorcerer'] = [] - metadata['root_version_sorcerer'] = [] + metadata['mars_version'] = [] + metadata['root_version'] = [] for rootf in self.files_: @@ -736,19 +757,19 @@ def parse_metadata_info(self): metadatainfo_array_list_runheaders, library="np" ) - metadata['subrun_number'].append(int(meta_info_runh['MRawRunHeader.fSubRunIndex'][0])) + metadata['subrun_number'].append(int(meta_info_runh[f'MRawRunHeader{suffix}.fSubRunIndex'][0])) metadata['source_ra'].append( - meta_info_runh['MRawRunHeader.fSourceRA'][0] / seconds_per_hour * degrees_per_hour * u.deg + meta_info_runh[f'MRawRunHeader{suffix}.fSourceRA'][0] / seconds_per_hour * degrees_per_hour * u.deg ) metadata['source_dec'].append( - meta_info_runh['MRawRunHeader.fSourceDEC'][0] / seconds_per_hour * u.deg + meta_info_runh[f'MRawRunHeader{suffix}.fSourceDEC'][0] / seconds_per_hour * u.deg ) if not self.is_simulation: - src_name_array = meta_info_runh['MRawRunHeader.fSourceName[80]'][0] + src_name_array = meta_info_runh[f'MRawRunHeader{suffix}.fSourceName[80]'][0] metadata['source_name'].append("".join([chr(item) for item in src_name_array if item != 0])) - obs_mode_array = meta_info_runh['MRawRunHeader.fObservationMode[60]'][0] + obs_mode_array = meta_info_runh[f'MRawRunHeader{suffix}.fObservationMode[60]'][0] metadata['observation_mode'].append("".join([chr(item) for item in obs_mode_array if item != 0])) - project_name_array = meta_info_runh['MRawRunHeader.fProjectName[100]'][0] + project_name_array = meta_info_runh[f'MRawRunHeader{suffix}.fProjectName[100]'][0] metadata['project_name'].append("".join([chr(item) for item in project_name_array if item != 0])) meta_info_runt = rootf['RunTails'].arrays( @@ -756,10 +777,20 @@ def parse_metadata_info(self): library="np" ) - mars_version_encoded = int(meta_info_runt['MMarsVersion_sorcerer.fMARSVersionCode'][0]) - root_version_encoded = int(meta_info_runt['MMarsVersion_sorcerer.fROOTVersionCode'][0]) - metadata['mars_version_sorcerer'].append(self.decode_version_number(mars_version_encoded)) - metadata['root_version_sorcerer'].append(self.decode_version_number(root_version_encoded)) + if self.mars_datalevel == MARSDataLevel.CALIBRATED: + mars_version_encoded = int(meta_info_runt['MMarsVersion_sorcerer.fMARSVersionCode'][0]) + root_version_encoded = int(meta_info_runt['MMarsVersion_sorcerer.fROOTVersionCode'][0]) + elif self.mars_datalevel == MARSDataLevel.STAR: + mars_version_encoded = int(meta_info_runt['MMarsVersion_star.fMARSVersionCode'][0]) + root_version_encoded = int(meta_info_runt['MMarsVersion_star.fROOTVersionCode'][0]) + elif self.mars_datalevel == MARSDataLevel.SUPERSTAR: + mars_version_encoded = int(meta_info_runt['MMarsVersion_superstar.fMARSVersionCode'][0]) + root_version_encoded = int(meta_info_runt['MMarsVersion_superstar.fROOTVersionCode'][0]) + elif self.mars_datalevel == MARSDataLevel.MELIBEA: + mars_version_encoded = int(meta_info_runt['MMarsVersion_melibea.fMARSVersionCode'][0]) + root_version_encoded = int(meta_info_runt['MMarsVersion_melibea.fROOTVersionCode'][0]) + metadata['mars_version'].append(self.decode_version_number(mars_version_encoded)) + metadata['root_version'].append(self.decode_version_number(root_version_encoded)) return metadata From 246c15e2a4726ffe7c22d3fb70d582eac27ff335 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sun, 29 May 2022 18:37:58 +0200 Subject: [PATCH 10/78] Adapt get_event_time_difference for different datalevels. --- ctapipe_io_magic/__init__.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index c0aaff3..5784e2c 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -962,19 +962,33 @@ def get_event_time_difference(self): Trigger time differences of consecutive events """ - time_diffs = np.array([]) + time_diffs = dict() + for tel_id in self.telescope: + time_diffs[tel_id] = np.array([]) for uproot_file in self.files_: - event_info = uproot_file['Events'].arrays( - expressions=['MRawEvtHeader.fTimeDiff'], - cut=f'(MTriggerPattern.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})', - library='np', - ) + if self.mars_datalevel <= MARSDataLevel.STAR: + event_info = uproot_file['Events'].arrays( + expressions=['MRawEvtHeader.fTimeDiff'], + cut=f'(MTriggerPattern.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})', + library='np', + ) + + time_diffs[self.telescope] = np.append(time_diffs, event_info['MRawEvtHeader.fTimeDiff']) + + else: + for tel_id in self.telescope: + event_info = uproot_file['Events'].arrays( + expressions=[f'MRawEvtHeader_{tel_id}.fTimeDiff'], + cut=f'(MTriggerPattern_{tel_id}.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})', + library='np', + ) - time_diffs = np.append(time_diffs, event_info['MRawEvtHeader.fTimeDiff']) + time_diffs[tel_id] = np.append(time_diffs[tel_id], event_info[f'MRawEvtHeader_{tel_id}.fTimeDiff']) - time_diffs *= u.s + for tel_id in self.telescope: + time_diffs[tel_id] *= u.s return time_diffs From 346bf049ad2e53e9d3a2b0ee7f90bd36ea94f9f0 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Mon, 30 May 2022 09:52:53 +0200 Subject: [PATCH 11/78] Make self.telescopes a list. --- ctapipe_io_magic/__init__.py | 17 +++++++++-------- .../tests/test_magic_event_source.py | 4 ++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 5784e2c..8cbe600 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -205,7 +205,7 @@ def __init__(self, input_url=None, config=None, parent=None, **kwargs): self.run_numbers = run_info[0] self.is_mc = run_info[1][0] - self.telescope = run_info[2][0] + self.telescopes = run_info[2] self.mars_datalevel = run_info[3][0] self.metadata = self.parse_metadata_info() @@ -741,7 +741,7 @@ def parse_metadata_info(self): metadata["file_list"] = self.file_list metadata['run_numbers'] = self.run_numbers metadata['is_simulation'] = self.is_simulation - metadata['telescopes'] = self.telescope + metadata['telescopes'] = self.telescopes metadata['subrun_number'] = [] metadata['source_ra'] = [] metadata['source_dec'] = [] @@ -963,7 +963,7 @@ def get_event_time_difference(self): """ time_diffs = dict() - for tel_id in self.telescope: + for tel_id in self.telescopes: time_diffs[tel_id] = np.array([]) for uproot_file in self.files_: @@ -975,10 +975,11 @@ def get_event_time_difference(self): library='np', ) - time_diffs[self.telescope] = np.append(time_diffs, event_info['MRawEvtHeader.fTimeDiff']) + for tel_id in self.telescopes: + time_diffs[tel_id] = np.append(time_diffs[tel_id], event_info['MRawEvtHeader.fTimeDiff']) else: - for tel_id in self.telescope: + for tel_id in self.telescopes: event_info = uproot_file['Events'].arrays( expressions=[f'MRawEvtHeader_{tel_id}.fTimeDiff'], cut=f'(MTriggerPattern_{tel_id}.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})', @@ -987,7 +988,7 @@ def get_event_time_difference(self): time_diffs[tel_id] = np.append(time_diffs[tel_id], event_info[f'MRawEvtHeader_{tel_id}.fTimeDiff']) - for tel_id in self.telescope: + for tel_id in self.telescopes: time_diffs[tel_id] *= u.s return time_diffs @@ -1030,7 +1031,7 @@ def _get_badrmspixel_mask(self, event): pedestal_level = 400 pedestal_level_variance = 4.5 - tel_id = self.telescope + tel_id = self.telescopes[0] event_time = event.trigger.tel[tel_id].time.unix pedestal_times = event.mon.tel[tel_id].pedestal.sample_time.unix @@ -1151,7 +1152,7 @@ def _event_generator(self): event.meta['max_events'] = self.max_events event.index.obs_id = self.obs_ids[0] - tel_id = self.telescope + tel_id = self.telescopes[0] event.trigger.tels_with_trigger = np.array([tel_id]) counter = 0 diff --git a/ctapipe_io_magic/tests/test_magic_event_source.py b/ctapipe_io_magic/tests/test_magic_event_source.py index 2a7e80e..0c4906d 100644 --- a/ctapipe_io_magic/tests/test_magic_event_source.py +++ b/ctapipe_io_magic/tests/test_magic_event_source.py @@ -173,7 +173,7 @@ def test_run_info(dataset): datalevel = [i[3] for i in run_info][0] assert run_numbers == source.run_numbers assert is_mc == source.is_simulation - assert telescope == source.telescope + assert telescope == source.telescopes[0] assert datalevel == source.mars_datalevel @@ -188,7 +188,7 @@ def test_multiple_runs_real(): for i, event in enumerate(source): assert event.trigger.event_type == EventType.SUBARRAY assert event.count == i - assert event.trigger.tels_with_trigger == [source.telescope] + assert event.trigger.tels_with_trigger == source.telescopes assert (i + 1) == n_events From 56f2c8a59c9319e47f9f0ae89a7303c64c0dbdac Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 1 Jun 2022 09:49:45 +0200 Subject: [PATCH 12/78] Adapt MARSCalibratedRun. --- ctapipe_io_magic/__init__.py | 61 +++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 29 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 8cbe600..c04ea9b 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1306,7 +1306,7 @@ class MarsCalibratedRun: and monitoring data from a MAGIC calibrated subrun file. """ - def __init__(self, uproot_file, is_mc, n_cam_pixels=1039): + def __init__(self, uproot_file, is_mc, tel_id, n_cam_pixels=1039): """ Constructor of the class. Loads an input uproot file and store the informaiton to constructor variables. @@ -1317,12 +1317,15 @@ def __init__(self, uproot_file, is_mc, n_cam_pixels=1039): A calibrated file opened by uproot via uproot.open(file_path) is_mc: bool Flag to MC data + tel_id: int + Telescope number (1 for M1, 2 for M2) n_cam_pixels: int The number of pixels of the MAGIC camera """ self.uproot_file = uproot_file self.is_mc = is_mc + self.tel_id = tel_id self.n_cam_pixels = n_cam_pixels # Load the input data: @@ -1334,11 +1337,11 @@ def __init__(self, uproot_file, is_mc, n_cam_pixels=1039): @property def n_cosmic_events(self): - return len(self.cosmic_events.get('trigger_pattern', [])) + return len(self.cosmic_events[self.tel_id].get('trigger_pattern', [])) @property def n_pedestal_events(self): - return len(self.pedestal_events.get('trigger_pattern', [])) + return len(self.pedestal_events[self.tel_id].get('trigger_pattern', [])) def _load_data(self): """ @@ -1403,9 +1406,9 @@ def _load_data(self): # Initialize the data container: calib_data = { - 'cosmic_events': dict(), - 'pedestal_events': dict(), - 'monitoring_data': dict(), + 'cosmic_events': {self.tel_id: dict()}, + 'pedestal_events': {self.tel_id: dict()}, + 'monitoring_data': {self.tel_id: dict()}, } # Set event cuts: @@ -1434,16 +1437,16 @@ def _load_data(self): library='np', ) - calib_data[event_type]['trigger_pattern'] = np.array(common_info['MTriggerPattern.fPrescaled'], dtype=int) - calib_data[event_type]['stereo_event_number'] = np.array(common_info['MRawEvtHeader.fStereoEvtNumber'], dtype=int) + calib_data[event_type][self.tel_id]['trigger_pattern'] = np.array(common_info['MTriggerPattern.fPrescaled'], dtype=int) + calib_data[event_type][self.tel_id]['stereo_event_number'] = np.array(common_info['MRawEvtHeader.fStereoEvtNumber'], dtype=int) # Set pixel-wise charge and peak time information. # The length of the pixel array is 1183, but here only the first part of the pixel # information are extracted (i.e., for the current MAGIC camera geometry, the pixels # between 0 and 1039 are extracted, since the other part of pixels has only zeros): - calib_data[event_type]['image'] = np.array(common_info['MCerPhotEvt.fPixels.fPhot'].tolist())[:, :self.n_cam_pixels] - calib_data[event_type]['peak_time'] = np.array(common_info['MArrivalTime.fData'].tolist())[:, :self.n_cam_pixels] - calib_data[event_type]['peak_time']/=sampling_speed # [ns] + calib_data[event_type][self.tel_id]['image'] = np.array(common_info['MCerPhotEvt.fPixels.fPhot'].tolist())[:, :self.n_cam_pixels] + calib_data[event_type][self.tel_id]['peak_time'] = np.array(common_info['MArrivalTime.fData'].tolist())[:, :self.n_cam_pixels] + calib_data[event_type][self.tel_id]['peak_time']/=sampling_speed # [ns] if self.is_mc: @@ -1458,13 +1461,13 @@ def _load_data(self): # of the magnetic north and the momentum of a simulated primary particle, defined # between -pi and pi, negative if the momentum pointing eastward, positive westward. # The conversion to azimuth should be 180 - fPhi + magnetic_declination: - calib_data[event_type]['mc_energy'] = u.Quantity(mc_info['MMcEvt.fEnergy'], u.GeV) - calib_data[event_type]['mc_theta'] = u.Quantity(mc_info['MMcEvt.fTheta'], u.rad) - calib_data[event_type]['mc_phi'] = u.Quantity(mc_info['MMcEvt.fPhi'], u.rad) - calib_data[event_type]['mc_shower_primary_id'] = np.array(mc_info['MMcEvt.fPartId'], dtype=int) - calib_data[event_type]['mc_h_first_int'] = u.Quantity(mc_info['MMcEvt.fZFirstInteraction'], u.cm) - calib_data[event_type]['mc_core_x'] = u.Quantity(mc_info['MMcEvt.fCoreX'], u.cm) - calib_data[event_type]['mc_core_y'] = u.Quantity(mc_info['MMcEvt.fCoreY'], u.cm) + calib_data[event_type][self.tel_id]['mc_energy'] = u.Quantity(mc_info['MMcEvt.fEnergy'], u.GeV) + calib_data[event_type][self.tel_id]['mc_theta'] = u.Quantity(mc_info['MMcEvt.fTheta'], u.rad) + calib_data[event_type][self.tel_id]['mc_phi'] = u.Quantity(mc_info['MMcEvt.fPhi'], u.rad) + calib_data[event_type][self.tel_id]['mc_shower_primary_id'] = np.array(mc_info['MMcEvt.fPartId'], dtype=int) + calib_data[event_type][self.tel_id]['mc_h_first_int'] = u.Quantity(mc_info['MMcEvt.fZFirstInteraction'], u.cm) + calib_data[event_type][self.tel_id]['mc_core_x'] = u.Quantity(mc_info['MMcEvt.fCoreX'], u.cm) + calib_data[event_type][self.tel_id]['mc_core_y'] = u.Quantity(mc_info['MMcEvt.fCoreY'], u.cm) # Reading the telescope pointing information: pointing = self.uproot_file['Events'].arrays( @@ -1480,10 +1483,10 @@ def _load_data(self): pointing_ra = (pointing['MPointingPos.fRa'] + pointing['MPointingPos.fDevHa']) * degrees_per_hour pointing_dec = pointing['MPointingPos.fDec'] - pointing['MPointingPos.fDevDec'] - calib_data[event_type]['pointing_az'] = u.Quantity(pointing_az, u.deg) - calib_data[event_type]['pointing_alt'] = u.Quantity(90 - pointing_zd, u.deg) - calib_data[event_type]['pointing_ra'] = u.Quantity(pointing_ra, u.deg) - calib_data[event_type]['pointing_dec'] = u.Quantity(pointing_dec, u.deg) + calib_data[event_type][self.tel_id]['pointing_az'] = u.Quantity(pointing_az, u.deg) + calib_data[event_type][self.tel_id]['pointing_alt'] = u.Quantity(90 - pointing_zd, u.deg) + calib_data[event_type][self.tel_id]['pointing_ra'] = u.Quantity(pointing_ra, u.deg) + calib_data[event_type][self.tel_id]['pointing_dec'] = u.Quantity(pointing_dec, u.deg) else: # Reading the event timing information: @@ -1507,7 +1510,7 @@ def _load_data(self): event_nanosec = np.array([Decimal(str(x)) for x in event_nanosec]) event_time = event_obs_day + event_millisec + event_nanosec - calib_data[event_type]['time'] = Time(event_time, format='unix', scale='utc') + calib_data[event_type][self.tel_id]['time'] = Time(event_time, format='unix', scale='utc') # Reading the monitoring data: if not self.is_mc: @@ -1533,7 +1536,7 @@ def _load_data(self): for i_pix in range(self.n_cam_pixels): unsuitable_pixels[i_pix] = int('{:08b}'.format(unsuitable_pixels_bit[i_pix])[-2]) - calib_data['monitoring_data']['bad_pixel'] = unsuitable_pixels + calib_data['monitoring_data'][self.tel_id]['bad_pixel'] = unsuitable_pixels # Try to read the Pedestals tree (soft fail if not present): try: @@ -1555,20 +1558,20 @@ def _load_data(self): pedestal_sample_time = pedestal_obs_day + pedestal_millisec + pedestal_nanosec - calib_data['monitoring_data']['pedestal_sample_time'] = Time( + calib_data['monitoring_data'][self.tel_id]['pedestal_sample_time'] = Time( pedestal_sample_time, format='unix', scale='utc' ) # Set the mean and RMS of pedestal charges: - calib_data['monitoring_data']['pedestal_fundamental'] = { + calib_data['monitoring_data'][self.tel_id]['pedestal_fundamental'] = { 'mean': np.array(pedestal_info[f'MPedPhotFundamental.fArray.fMean'].tolist())[:, :self.n_cam_pixels], 'rms': np.array(pedestal_info[f'MPedPhotFundamental.fArray.fRms'].tolist())[:, :self.n_cam_pixels], } - calib_data['monitoring_data']['pedestal_from_extractor'] = { + calib_data['monitoring_data'][self.tel_id]['pedestal_from_extractor'] = { 'mean': np.array(pedestal_info[f'MPedPhotFromExtractor.fArray.fMean'].tolist())[:, :self.n_cam_pixels], 'rms': np.array(pedestal_info[f'MPedPhotFromExtractor.fArray.fRms'].tolist())[:, :self.n_cam_pixels], } - calib_data['monitoring_data']['pedestal_from_extractor_rndm'] = { + calib_data['monitoring_data'][self.tel_id]['pedestal_from_extractor_rndm'] = { 'mean': np.array(pedestal_info[f'MPedPhotFromExtractorRndm.fArray.fMean'].tolist())[:, :self.n_cam_pixels], 'rms': np.array(pedestal_info[f'MPedPhotFromExtractorRndm.fArray.fRms'].tolist())[:, :self.n_cam_pixels], } @@ -1579,7 +1582,7 @@ def _load_data(self): # Check for bit flips in the stereo event IDs: uplim_total_jumps = 100 - stereo_event_number = calib_data['cosmic_events']['stereo_event_number'].astype(int) + stereo_event_number = calib_data['cosmic_events'][self.tel_id]['stereo_event_number'].astype(int) number_difference = np.diff(stereo_event_number) indices_flip = np.where(number_difference < 0)[0] From 4d2cddafa34e71e5fdb562e800c46e7492e40b99 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 1 Jun 2022 09:50:08 +0200 Subject: [PATCH 13/78] Adapt event generator. --- ctapipe_io_magic/__init__.py | 204 ++++++++++++++++++----------------- 1 file changed, 104 insertions(+), 100 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index c04ea9b..e45f9e5 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1152,8 +1152,7 @@ def _event_generator(self): event.meta['max_events'] = self.max_events event.index.obs_id = self.obs_ids[0] - tel_id = self.telescopes[0] - event.trigger.tels_with_trigger = np.array([tel_id]) + event.trigger.tels_with_trigger = np.array(self.telescopes) counter = 0 @@ -1172,130 +1171,135 @@ def _event_generator(self): event_data = self.current_run['data'].cosmic_events n_events = self.current_run['data'].n_cosmic_events - if not self.is_simulation: - - monitoring_data = self.current_run['data'].monitoring_data - - # Set the pedestal information: - event.mon.tel[tel_id].pedestal.n_events = 500 # hardcoded number of pedestal events averaged over - event.mon.tel[tel_id].pedestal.sample_time = monitoring_data['pedestal_sample_time'] - - event.mon.tel[tel_id].pedestal.charge_mean = [ - monitoring_data['pedestal_fundamental']['mean'], - monitoring_data['pedestal_from_extractor']['mean'], - monitoring_data['pedestal_from_extractor_rndm']['mean'], - ] - - event.mon.tel[tel_id].pedestal.charge_std = [ - monitoring_data['pedestal_fundamental']['rms'], - monitoring_data['pedestal_from_extractor']['rms'], - monitoring_data['pedestal_from_extractor_rndm']['rms'], - ] - - # Set the bad pixel information: - event.mon.tel[tel_id].pixel_status.hardware_failing_pixels = np.reshape( - monitoring_data['bad_pixel'], (1, len(monitoring_data['bad_pixel'])) - ) - - # Interpolate drive information: - drive_data = self.drive_information - n_drive_reports = len(drive_data['mjd']) - - if self.use_pedestals: - logger.warning(f'Interpolating pedestals events information from {n_drive_reports} drive reports.') - else: - logger.warning(f'Interpolating cosmic events information from {n_drive_reports} drive reports.') + for tel_id in self.telescopes: - first_drive_report_time = Time(drive_data['mjd'][0], scale='utc', format='mjd') - last_drive_report_time = Time(drive_data['mjd'][-1], scale='utc', format='mjd') + if not self.is_simulation: - logger.warning(f'Drive reports available from {first_drive_report_time.iso} to {last_drive_report_time.iso}.') + monitoring_data = self.current_run['data'].monitoring_data - # Create azimuth and zenith angles interpolators: - drive_az_pointing_interpolator = scipy.interpolate.interp1d( - drive_data['mjd'], drive_data['az'], fill_value='extrapolate' - ) - drive_zd_pointing_interpolator = scipy.interpolate.interp1d( - drive_data['mjd'], drive_data['zd'], fill_value='extrapolate' - ) + # Set the pedestal information: + event.mon.tel[tel_id].pedestal.n_events = 500 # hardcoded number of pedestal events averaged over + event.mon.tel[tel_id].pedestal.sample_time = monitoring_data[tel_id]['pedestal_sample_time'] - # Create RA and Dec interpolators: - drive_ra_pointing_interpolator = scipy.interpolate.interp1d( - drive_data['mjd'], drive_data['ra'], fill_value='extrapolate' - ) - drive_dec_pointing_interpolator = scipy.interpolate.interp1d( - drive_data['mjd'], drive_data['dec'], fill_value='extrapolate' - ) + event.mon.tel[tel_id].pedestal.charge_mean = [ + monitoring_data[tel_id]['pedestal_fundamental']['mean'], + monitoring_data[tel_id]['pedestal_from_extractor']['mean'], + monitoring_data[tel_id]['pedestal_from_extractor_rndm']['mean'], + ] - # Interpolate the drive pointing to the event timestamps: - event_times = event_data['time'].mjd + event.mon.tel[tel_id].pedestal.charge_std = [ + monitoring_data[tel_id]['pedestal_fundamental']['rms'], + monitoring_data[tel_id]['pedestal_from_extractor']['rms'], + monitoring_data[tel_id]['pedestal_from_extractor_rndm']['rms'], + ] - pointing_az = drive_az_pointing_interpolator(event_times) - pointing_zd = drive_zd_pointing_interpolator(event_times) - pointing_ra = drive_ra_pointing_interpolator(event_times) - pointing_dec = drive_dec_pointing_interpolator(event_times) + # Set the bad pixel information: + event.mon.tel[tel_id].pixel_status.hardware_failing_pixels = np.reshape( + monitoring_data[tel_id]['bad_pixel'], (1, len(monitoring_data[tel_id]['bad_pixel'])) + ) - event_data['pointing_az'] = u.Quantity(pointing_az, u.deg) - event_data['pointing_alt'] = u.Quantity(90 - pointing_zd, u.deg) - event_data['pointing_ra'] = u.Quantity(pointing_ra, u.deg) - event_data['pointing_dec'] = u.Quantity(pointing_dec, u.deg) + if self.mars_datalevel == MARSDataLevel.CALIBRATED: + # Interpolate drive information: + drive_data = self.drive_information + n_drive_reports = len(drive_data['mjd']) + + if self.use_pedestals: + logger.warning(f'Interpolating pedestals events information from {n_drive_reports} drive reports.') + else: + logger.warning(f'Interpolating cosmic events information from {n_drive_reports} drive reports.') + + first_drive_report_time = Time(drive_data['mjd'][0], scale='utc', format='mjd') + last_drive_report_time = Time(drive_data['mjd'][-1], scale='utc', format='mjd') + + logger.warning(f'Drive reports available from {first_drive_report_time.iso} to {last_drive_report_time.iso}.') + + # Create azimuth and zenith angles interpolators: + drive_az_pointing_interpolator = scipy.interpolate.interp1d( + drive_data['mjd'], drive_data['az'], fill_value='extrapolate' + ) + drive_zd_pointing_interpolator = scipy.interpolate.interp1d( + drive_data['mjd'], drive_data['zd'], fill_value='extrapolate' + ) + + # Create RA and Dec interpolators: + drive_ra_pointing_interpolator = scipy.interpolate.interp1d( + drive_data['mjd'], drive_data['ra'], fill_value='extrapolate' + ) + drive_dec_pointing_interpolator = scipy.interpolate.interp1d( + drive_data['mjd'], drive_data['dec'], fill_value='extrapolate' + ) + + # Interpolate the drive pointing to the event timestamps: + event_times = event_data[tel_id]['time'].mjd + + pointing_az = drive_az_pointing_interpolator(event_times) + pointing_zd = drive_zd_pointing_interpolator(event_times) + pointing_ra = drive_ra_pointing_interpolator(event_times) + pointing_dec = drive_dec_pointing_interpolator(event_times) + + event_data[tel_id]['pointing_az'] = u.Quantity(pointing_az, u.deg) + event_data[tel_id]['pointing_alt'] = u.Quantity(90 - pointing_zd, u.deg) + event_data[tel_id]['pointing_ra'] = u.Quantity(pointing_ra, u.deg) + event_data[tel_id]['pointing_dec'] = u.Quantity(pointing_dec, u.deg) # Loop over the events: for i_event in range(n_events): - event.count = counter - event.index.event_id = event_data['stereo_event_number'][i_event] + for tel_id in self.telescopes: - event.trigger.event_type = MAGIC_TO_CTA_EVENT_TYPE.get(event_data['trigger_pattern'][i_event]) + event.count = counter + event.index.event_id = event_data[tel_id]['stereo_event_number'][i_event] - if not self.is_simulation: - event.trigger.time = event_data['time'][i_event] - event.trigger.tel[tel_id].time = event_data['time'][i_event] + event.trigger.event_type = MAGIC_TO_CTA_EVENT_TYPE.get(event_data[tel_id]['trigger_pattern'][i_event]) + + if not self.is_simulation: + event.trigger.time = event_data[tel_id]['time'][i_event] + event.trigger.tel[tel_id].time = event_data[tel_id]['time'][i_event] - if not self.use_pedestals: - badrmspixel_mask = self._get_badrmspixel_mask(event) - event.mon.tel[tel_id].pixel_status.pedestal_failing_pixels = badrmspixel_mask + if not self.use_pedestals: + badrmspixel_mask = self._get_badrmspixel_mask(event) + event.mon.tel[tel_id].pixel_status.pedestal_failing_pixels = badrmspixel_mask - # Set the telescope pointing container: - event.pointing.array_azimuth = event_data['pointing_az'][i_event].to(u.rad) - event.pointing.array_altitude = event_data['pointing_alt'][i_event].to(u.rad) - event.pointing.array_ra = event_data['pointing_ra'][i_event].to(u.rad) - event.pointing.array_dec = event_data['pointing_dec'][i_event].to(u.rad) + # Set the telescope pointing container: + event.pointing.array_azimuth = event_data[tel_id]['pointing_az'][i_event].to(u.rad) + event.pointing.array_altitude = event_data[tel_id]['pointing_alt'][i_event].to(u.rad) + event.pointing.array_ra = event_data[tel_id]['pointing_ra'][i_event].to(u.rad) + event.pointing.array_dec = event_data[tel_id]['pointing_dec'][i_event].to(u.rad) - event.pointing.tel[tel_id].azimuth = event_data['pointing_az'][i_event].to(u.rad) - event.pointing.tel[tel_id].altitude = event_data['pointing_alt'][i_event].to(u.rad) + event.pointing.tel[tel_id].azimuth = event_data[tel_id]['pointing_az'][i_event].to(u.rad) + event.pointing.tel[tel_id].altitude = event_data[tel_id]['pointing_alt'][i_event].to(u.rad) - # Set event charge and peak positions per pixel: - event.dl1.tel[tel_id].image = event_data['image'][i_event] - event.dl1.tel[tel_id].peak_time = event_data['peak_time'][i_event] + # Set event charge and peak positions per pixel: + event.dl1.tel[tel_id].image = event_data[tel_id]['image'][i_event] + event.dl1.tel[tel_id].peak_time = event_data[tel_id]['peak_time'][i_event] - # Set the simulated event container: - if self.is_simulation: + # Set the simulated event container: + if self.is_simulation: - event.simulation = SimulatedEventContainer() + event.simulation = SimulatedEventContainer() - event.simulation.shower.energy = event_data['mc_energy'][i_event].to(u.TeV) - event.simulation.shower.shower_primary_id = 1 - event_data['mc_shower_primary_id'][i_event] - event.simulation.shower.h_first_int = event_data['mc_h_first_int'][i_event].to(u.m) + event.simulation.shower.energy = event_data[tel_id]['mc_energy'][i_event].to(u.TeV) + event.simulation.shower.shower_primary_id = 1 - event_data[tel_id]['mc_shower_primary_id'][i_event] + event.simulation.shower.h_first_int = event_data[tel_id]['mc_h_first_int'][i_event].to(u.m) - # Convert the corsika coordinate (x-axis: magnetic north) to the geographical one. - # Rotate the corsika coordinate by the magnetic declination (= 7 deg): - mfield_dec = self.simulation_config[self.obs_ids[0]]['prod_site_B_declination'] + # Convert the corsika coordinate (x-axis: magnetic north) to the geographical one. + # Rotate the corsika coordinate by the magnetic declination (= 7 deg): + mfield_dec = self.simulation_config[self.obs_ids[0]]['prod_site_B_declination'] - event.simulation.shower.alt = u.Quantity(90, u.deg) - event_data['mc_theta'][i_event].to(u.deg) - event.simulation.shower.az = u.Quantity(180, u.deg) - event_data['mc_phi'][i_event].to(u.deg) + mfield_dec + event.simulation.shower.alt = u.Quantity(90, u.deg) - event_data[tel_id]['mc_theta'][i_event].to(u.deg) + event.simulation.shower.az = u.Quantity(180, u.deg) - event_data[tel_id]['mc_phi'][i_event].to(u.deg) + mfield_dec - if event.simulation.shower.az > u.Quantity(180, u.deg): - event.simulation.shower.az -= u.Quantity(360, u.deg) + if event.simulation.shower.az > u.Quantity(180, u.deg): + event.simulation.shower.az -= u.Quantity(360, u.deg) - event.simulation.shower.core_x = event_data['mc_core_x'][i_event].to(u.m) * np.cos(mfield_dec) \ - + event_data['mc_core_y'][i_event].to(u.m) * np.sin(mfield_dec) + event.simulation.shower.core_x = event_data[tel_id]['mc_core_x'][i_event].to(u.m) * np.cos(mfield_dec) \ + + event_data[tel_id]['mc_core_y'][i_event].to(u.m) * np.sin(mfield_dec) - event.simulation.shower.core_y = event_data['mc_core_y'][i_event].to(u.m) * np.cos(mfield_dec) \ - - event_data['mc_core_x'][i_event].to(u.m) * np.sin(mfield_dec) + event.simulation.shower.core_y = event_data[tel_id]['mc_core_y'][i_event].to(u.m) * np.cos(mfield_dec) \ + - event_data[tel_id]['mc_core_x'][i_event].to(u.m) * np.sin(mfield_dec) - yield event - counter += 1 + yield event + counter += 1 return From d870323ff7428b71596014979ccffe317da1c229 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 1 Jun 2022 09:50:28 +0200 Subject: [PATCH 14/78] Use single value for calibrated data. --- ctapipe_io_magic/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index e45f9e5..d54f84d 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1117,6 +1117,7 @@ def _set_active_run(self, uproot_file): run['data'] = MarsCalibratedRun( uproot_file, self.is_simulation, + self.telescopes[0], ) return run From 1c03ba7c8739bfd062c6ff4c8b2f83deb9e4e936 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 1 Jun 2022 09:50:44 +0200 Subject: [PATCH 15/78] Do not use different event generators for different data levels. --- ctapipe_io_magic/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index d54f84d..478165f 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1132,8 +1132,7 @@ def _generator(self): self._event_generator """ - if self.mars_datalevel == MARSDataLevel.CALIBRATED: - return self._event_generator() + return self._event_generator() def _event_generator(self): """ From 137313a2ed66149bf26a13f25714374129129a20 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sun, 5 Jun 2022 21:44:46 +0200 Subject: [PATCH 16/78] Add tel_id argument to get_badrmspixel_mask. --- ctapipe_io_magic/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 478165f..bc04edd 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1010,7 +1010,7 @@ def obs_ids(self): # ToCheck: will this be compatible in the future, e.g. with merged MC files return self.run_numbers - def _get_badrmspixel_mask(self, event): + def _get_badrmspixel_mask(self, event, tel_id): """ Fetch bad RMS pixel mask for a given event. @@ -1018,6 +1018,8 @@ def _get_badrmspixel_mask(self, event): ---------- event: ctapipe.containers.ArrayEventContainer array event container + tel_id: int + telescope ID Returns ------- @@ -1031,8 +1033,6 @@ def _get_badrmspixel_mask(self, event): pedestal_level = 400 pedestal_level_variance = 4.5 - tel_id = self.telescopes[0] - event_time = event.trigger.tel[tel_id].time.unix pedestal_times = event.mon.tel[tel_id].pedestal.sample_time.unix @@ -1257,7 +1257,7 @@ def _event_generator(self): event.trigger.tel[tel_id].time = event_data[tel_id]['time'][i_event] if not self.use_pedestals: - badrmspixel_mask = self._get_badrmspixel_mask(event) + badrmspixel_mask = self._get_badrmspixel_mask(event, tel_id) event.mon.tel[tel_id].pixel_status.pedestal_failing_pixels = badrmspixel_mask # Set the telescope pointing container: From 0668594867eaec3793cc3d2012d2bf50d26944bd Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Fri, 10 Jun 2022 09:55:06 +0200 Subject: [PATCH 17/78] Start filling containers. --- ctapipe_io_magic/__init__.py | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index bc04edd..e35ed4f 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -26,6 +26,11 @@ ArrayEventContainer, SimulationConfigContainer, SimulatedEventContainer, + ImageParametersContainer, + CameraHillasParametersContainer, + TimingParametersContainer, + LeakageContainer, + MorphologyContainer, ) from ctapipe.instrument import ( @@ -1269,9 +1274,23 @@ def _event_generator(self): event.pointing.tel[tel_id].azimuth = event_data[tel_id]['pointing_az'][i_event].to(u.rad) event.pointing.tel[tel_id].altitude = event_data[tel_id]['pointing_alt'][i_event].to(u.rad) - # Set event charge and peak positions per pixel: - event.dl1.tel[tel_id].image = event_data[tel_id]['image'][i_event] - event.dl1.tel[tel_id].peak_time = event_data[tel_id]['peak_time'][i_event] + if self.mars_datalevel == MARSDataLevel.CALIBRATED: + # Set event charge and peak positions per pixel: + event.dl1.tel[tel_id].image = event_data[tel_id]['image'][i_event] + event.dl1.tel[tel_id].peak_time = event_data[tel_id]['peak_time'][i_event] + + if self.mars_datalevel == MARSDataLevel.SUPERSTAR: + event.dl1.tel[tel_id].parameters = ImageParametersContainer() + event.dl1.tel[tel_id].parameters.hillas = CameraHillasParametersContainer( + x=-event_data[tel_id]["y"], + y=-event_data[tel_id]["x"], + length=event_data[tel_id]["length"], + width=event_data[tel_id]["width"], + intensity=event_data[tel_id]["intensity"], + r=np.sqrt(event_data[tel_id]["x"]*event_data[tel_id]["x"]+event_data[tel_id]["y"]*event_data[tel_id]["y"]), + psi=event_data[tel_id]["psi"], + phi=event_data[tel_id]["phi"], + ) # Set the simulated event container: if self.is_simulation: From 2e8f381219095c7d879cd296cf1f96d28ddb484b Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 20 Jul 2022 11:29:16 +0200 Subject: [PATCH 18/78] Download superstar files for tests. --- download_test_data.sh | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/download_test_data.sh b/download_test_data.sh index 8e0fcd1..e543ee8 100644 --- a/download_test_data.sh +++ b/download_test_data.sh @@ -7,11 +7,16 @@ echo "https://data.magic.pic.es/Users/ctapipe_io_magic/test_data/real/calibrated echo "https://data.magic.pic.es/Users/ctapipe_io_magic/test_data/real/calibrated/20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root" >> test_data_real.txt echo "https://data.magic.pic.es/Users/ctapipe_io_magic/test_data/real/calibrated/20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root" >> test_data_real.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/superstar/20210314_05095172_S_CrabNebula-W0.40+035.root" > test_data_superstar_real.txt + echo "https://data.magic.pic.es/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M1_za35to50_8_824318_Y_w0.root" > test_data_simulated.txt echo "https://data.magic.pic.es/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M1_za35to50_8_824319_Y_w0.root" >> test_data_simulated.txt echo "https://data.magic.pic.es/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M2_za35to50_8_824318_Y_w0.root" >> test_data_simulated.txt echo "https://data.magic.pic.es/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M2_za35to50_8_824319_Y_w0.root" >> test_data_simulated.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824318_S_w0.root" > test_data_superstar_simulated.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824319_S_w0.root" > test_data_superstar_simulated.txt + if [ -z "$TEST_DATA_USER" ]; then echo -n "Username: " read TEST_DATA_USER @@ -35,6 +40,17 @@ if ! wget \ echo "Problem in downloading the test data set for real data." fi +if ! wget \ + -i test_data_superstar_real.txt \ + --user="$TEST_DATA_USER" \ + --password="$TEST_DATA_PASSWORD" \ + --no-check-certificate \ + --no-verbose \ + --timestamping \ + --directory-prefix=test_data/real/superstar; then + echo "Problem in downloading the test data set for real data." +fi + if ! wget \ -i test_data_simulated.txt \ --user="$TEST_DATA_USER" \ @@ -46,4 +62,15 @@ if ! wget \ echo "Problem in downloading the test data set for simulated data." fi -rm -f test_data_real.txt test_data_simulated.txt +if ! wget \ + -i test_data_superstar_simulated.txt \ + --user="$TEST_DATA_USER" \ + --password="$TEST_DATA_PASSWORD" \ + --no-check-certificate \ + --no-verbose \ + --timestamping \ + --directory-prefix=test_data/simulated/superstar; then + echo "Problem in downloading the test data set for simulated data." +fi + +rm -f test_data_real.txt test_data_simulated.txt test_data_superstar_real.txt test_data_superstar_simulated.txt From b70edda43d4d0a60060dfeb3d592a2516b7b6781 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 20 Jul 2022 12:33:13 +0200 Subject: [PATCH 19/78] Fill containers. --- ctapipe_io_magic/__init__.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index e35ed4f..bb617d3 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -31,6 +31,7 @@ TimingParametersContainer, LeakageContainer, MorphologyContainer, + ReconstructedGeometryContainer, ) from ctapipe.instrument import ( @@ -67,6 +68,8 @@ mc_data_type = 256 +m2deg = 180./(17.*np.pi) + MAGIC_TO_CTA_EVENT_TYPE = { MC_STEREO_TRIGGER_PATTERN: EventType.SUBARRAY, DATA_STEREO_TRIGGER_PATTERN: EventType.SUBARRAY, @@ -1279,7 +1282,7 @@ def _event_generator(self): event.dl1.tel[tel_id].image = event_data[tel_id]['image'][i_event] event.dl1.tel[tel_id].peak_time = event_data[tel_id]['peak_time'][i_event] - if self.mars_datalevel == MARSDataLevel.SUPERSTAR: + if self.mars_datalevel >= MARSDataLevel.SUPERSTAR: event.dl1.tel[tel_id].parameters = ImageParametersContainer() event.dl1.tel[tel_id].parameters.hillas = CameraHillasParametersContainer( x=-event_data[tel_id]["y"], @@ -1292,6 +1295,34 @@ def _event_generator(self): phi=event_data[tel_id]["phi"], ) + event.dl1.tel[tel_id].parameters.timing = TimingParametersContainer( + slope=event_data[tel_id]["slope"].value*(1./m2deg)*u.deg, + intercept=event_data[tel_id]["intercept"], + ) + + # not fully filled + event.dl1.tel[tel_id].parameters.leakage = LeakageContainer( + intensity_width_1=event_data[tel_id]["intensity_width_1"], + intensity_width_2=event_data[tel_id]["intensity_width_2"], + ) + + # not fully filled + event.dl1.tel[tel_id].parameters.morphology = MorphologyContainer( + num_pixels=event_data[tel_id]["num_pixels"], + num_islands=event_data[tel_id]["num_islands"], + ) + + event.dl2.stereo.geometry["HillasReconstructor"] = ReconstructedGeometryContainer( + alt=event_data["stereo"]["alt"], + az=event_data["stereo"]["az"], + core_x=event_data["stereo"]["core_x"], + core_y=event_data["stereo"]["core_y"], + h_max=event_data["stereo"]["h_max"], + is_valid=True if event_data["stereo"]["is_valid"] == 1 else False, + average_intensity=(event_data[1]["intensity"]+event_data[2]["intensity"]) / 2., + tel_ids=[1, 2], + ) + # Set the simulated event container: if self.is_simulation: From 245bd7023273a38dfc2475b77edbace4b3d00548 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 20 Jul 2022 12:33:25 +0200 Subject: [PATCH 20/78] Add superstar files to tests. --- .../tests/test_magic_event_source.py | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/ctapipe_io_magic/tests/test_magic_event_source.py b/ctapipe_io_magic/tests/test_magic_event_source.py index 0c4906d..b33e2fa 100644 --- a/ctapipe_io_magic/tests/test_magic_event_source.py +++ b/ctapipe_io_magic/tests/test_magic_event_source.py @@ -21,8 +21,22 @@ test_calibrated_simulated_dir / 'GA_M2_za35to50_8_824319_Y_w0.root', ] +test_superstar_real_dir = test_data / "real/superstar" +test_superstar_simulated_dir = test_data / "simulated/superstar" + +test_superstar_mars_real = [ + test_superstar_real_dir / "20210314_05095172_S_CrabNebula-W0.40+035.root", +] + +test_superstar_mars_simulated = [ + test_superstar_simulated_dir / "GA_za35to50_8_824318_S_w0.root", + test_superstar_simulated_dir / "GA_za35to50_8_824319_S_w0.root", +] + test_calibrated_all = test_calibrated_real+test_calibrated_simulated +test_superstar_all = test_superstar_mars_real + test_superstar_mars_simulated + data_dict = dict() data_dict['20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root'] = dict() @@ -75,7 +89,7 @@ data_dict['GA_M2_za35to50_8_824319_Y_w0.root']['n_events_mc_mono'] = 52 -@pytest.mark.parametrize('dataset', test_calibrated_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) def test_event_source_for_magic_file(dataset): from ctapipe.io import EventSource @@ -88,7 +102,7 @@ def test_event_source_for_magic_file(dataset): assert reader.input_url == dataset -@pytest.mark.parametrize('dataset', test_calibrated_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) def test_compatible(dataset): from ctapipe_io_magic import MAGICEventSource assert MAGICEventSource.is_compatible(dataset) @@ -99,7 +113,7 @@ def test_not_compatible(): assert MAGICEventSource.is_compatible(None) is False -@pytest.mark.parametrize('dataset', test_calibrated_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) def test_stream(dataset): from ctapipe_io_magic import MAGICEventSource with MAGICEventSource(input_url=dataset, process_run=False) as source: From cac95feeefd1af02829d9154b290ef15919f5ce5 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 20 Jul 2022 12:41:48 +0200 Subject: [PATCH 21/78] Fix regex for superstar and melibea files. --- ctapipe_io_magic/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index bb617d3..8e0ffba 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -331,8 +331,8 @@ def get_run_info_from_name(file_name): mask_data_melibea = r"\d{8}_(\d+)_Q_.*" mask_mc_calibrated = r"GA_M(\d)_za\d+to\d+_\d_(\d+)_Y_.*" mask_mc_star = r"GA_M(\d)_za\d+to\d+_\d_(\d+)_I_.*" - mask_mc_superstar = r"GA_za\d+to\d+_\d_S_.*" - mask_mc_melibea = r"GA_za\d+to\d+_\d_Q_.*" + mask_mc_superstar = r"GA_za\d+to\d+_\d_(\d+)_S_.*" + mask_mc_melibea = r"GA_za\d+to\d+_\d_(\d+)_Q_.*" if re.match(mask_data_calibrated, file_name) is not None: parsed_info = re.match(mask_data_calibrated, file_name) telescope = int(parsed_info.group(1)) @@ -384,7 +384,7 @@ def get_run_info_from_name(file_name): else: raise IndexError( 'Can not identify the run number and type (data/MC) of the file' - '{:s}'.format(file_name)) + ' {:s}'.format(file_name)) return run_number, is_mc, telescope, datalevel From 318fc27bec6fa07736a99ec0bba348920785fd13 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 20 Jul 2022 12:58:29 +0200 Subject: [PATCH 22/78] FIx typo. --- ctapipe_io_magic/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 8e0ffba..1253f74 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -348,13 +348,13 @@ def get_run_info_from_name(file_name): elif re.match(mask_data_superstar, file_name) is not None: parsed_info = re.match(mask_data_superstar, file_name) telescope = None - run_number = int(parsed_info.grou(1)) + run_number = int(parsed_info.group(1)) datalevel = MARSDataLevel.SUPERSTAR is_mc = False elif re.match(mask_data_melibea, file_name) is not None: parsed_info = re.match(mask_data_melibea, file_name) telescope = None - run_number = int(parsed_info.grou(1)) + run_number = int(parsed_info.group(1)) datalevel = MARSDataLevel.MELIBEA is_mc = False elif re.match(mask_mc_calibrated, file_name) is not None: From 9d285a28583ed8b6f41c072425b5393ed46760e6 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 20 Jul 2022 12:58:45 +0200 Subject: [PATCH 23/78] Use correct names for trees. --- ctapipe_io_magic/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 1253f74..3d395e8 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -428,9 +428,9 @@ def parse_run_info(self): events_tree = rootf['Events'] - melibea_trees = ['MHadronness', 'MStereoParDisp', 'MEnergyEst'] - superstar_trees = ['MHillas_1', 'MHillas_2', 'MStereoPar'] - star_trees = ['MHillas'] + melibea_trees = ['MHadronness.', 'MStereoParDisp.', 'MEnergyEst.'] + superstar_trees = ['MHillas_1.', 'MHillas_2.', 'MStereoPar.'] + star_trees = ['MHillas.'] datalevel = MARSDataLevel.CALIBRATED events_keys = events_tree.keys() From 4d2a50c27e7c49536bf2137b86b2cab6d85880ba Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 20 Jul 2022 13:10:54 +0200 Subject: [PATCH 24/78] Change parse_data_info for superstar files. --- ctapipe_io_magic/__init__.py | 49 +++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 3d395e8..1221f63 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -532,10 +532,10 @@ def parse_data_info(self): for rootf in self.files_: - trigger_tree = rootf["Trigger"] - L3T_tree = rootf["L3T"] - if self.mars_datalevel <= MARSDataLevel.STAR: + + trigger_tree = rootf["Trigger"] + L3T_tree = rootf["L3T"] # here we take the 2nd element (if possible) because sometimes # the first trigger report has still the old prescaler values from a previous run try: @@ -558,30 +558,33 @@ def parse_data_info(self): stereo = True else: stereo = True - else: - stereo = True - sumt = False - if stereo: - # here we take the 2nd element for the same reason as above - # L3Table is empty for mono data i.e. taken with one telescope only - # if both telescopes take data with no L3, L3Table is filled anyway - L3Table_array = L3T_tree["MReportL3T.fTablename"].array(library="np") - L3Table_size = L3Table_array.size - if L3Table_size > 1: - L3Table = L3Table_array[1] + sumt = False + if stereo: + # here we take the 2nd element for the same reason as above + # L3Table is empty for mono data i.e. taken with one telescope only + # if both telescopes take data with no L3, L3Table is filled anyway + L3Table_array = L3T_tree["MReportL3T.fTablename"].array(library="np") + L3Table_size = L3Table_array.size + if L3Table_size > 1: + L3Table = L3Table_array[1] + else: + L3Table = L3Table_array[0] + + if L3Table == L3_table_sumt: + sumt = True + elif L3Table == L3_table_nosumt: + sumt = False + else: + sumt = False else: - L3Table = L3Table_array[0] + if prescaler == prescaler_mono_sumt: + sumt = True - if L3Table == L3_table_sumt: - sumt = True - elif L3Table == L3_table_nosumt: - sumt = False - else: - sumt = False else: - if prescaler == prescaler_mono_sumt: - sumt = True + # to be changed + stereo = True + sumt = False is_stereo.append(stereo) is_sumt.append(sumt) From 53e99905b04114ed160dc1b1e93dd8fc356ef7f2 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 20 Jul 2022 13:20:31 +0200 Subject: [PATCH 25/78] Update URLs. --- download_test_data.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/download_test_data.sh b/download_test_data.sh index e543ee8..7660913 100644 --- a/download_test_data.sh +++ b/download_test_data.sh @@ -2,17 +2,17 @@ set -e -echo "https://data.magic.pic.es/Users/ctapipe_io_magic/test_data/real/calibrated/20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root" > test_data_real.txt -echo "https://data.magic.pic.es/Users/ctapipe_io_magic/test_data/real/calibrated/20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root" >> test_data_real.txt -echo "https://data.magic.pic.es/Users/ctapipe_io_magic/test_data/real/calibrated/20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root" >> test_data_real.txt -echo "https://data.magic.pic.es/Users/ctapipe_io_magic/test_data/real/calibrated/20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root" >> test_data_real.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/calibrated/20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root" > test_data_real.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/calibrated/20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root" >> test_data_real.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/calibrated/20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root" >> test_data_real.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/calibrated/20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root" >> test_data_real.txt echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/superstar/20210314_05095172_S_CrabNebula-W0.40+035.root" > test_data_superstar_real.txt -echo "https://data.magic.pic.es/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M1_za35to50_8_824318_Y_w0.root" > test_data_simulated.txt -echo "https://data.magic.pic.es/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M1_za35to50_8_824319_Y_w0.root" >> test_data_simulated.txt -echo "https://data.magic.pic.es/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M2_za35to50_8_824318_Y_w0.root" >> test_data_simulated.txt -echo "https://data.magic.pic.es/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M2_za35to50_8_824319_Y_w0.root" >> test_data_simulated.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M1_za35to50_8_824318_Y_w0.root" > test_data_simulated.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M1_za35to50_8_824319_Y_w0.root" >> test_data_simulated.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M2_za35to50_8_824318_Y_w0.root" >> test_data_simulated.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M2_za35to50_8_824319_Y_w0.root" >> test_data_simulated.txt echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824318_S_w0.root" > test_data_superstar_simulated.txt echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824319_S_w0.root" > test_data_superstar_simulated.txt From 44a53b724bdcb6343644020154d745df758d6bb0 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Thu, 21 Jul 2022 17:11:11 +0200 Subject: [PATCH 26/78] Add pedestal number property. --- ctapipe_io_magic/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 1221f63..196a853 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1698,6 +1698,10 @@ def __init__(self, uproot_file, is_mc, n_cam_pixels=1039): def n_cosmic_events(self): return len(self.cosmic_events[1].get('trigger_pattern', [])) + @property + def n_pedestal_events(self): + return 0 + def _load_data(self): """ This method loads cosmic and pedestal events, and monitoring data From af288b35f582dcd0db9f66af1a531a423175d242 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Thu, 21 Jul 2022 17:11:37 +0200 Subject: [PATCH 27/78] Add number of events. --- .../tests/test_magic_event_source.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ctapipe_io_magic/tests/test_magic_event_source.py b/ctapipe_io_magic/tests/test_magic_event_source.py index b33e2fa..62e8f25 100644 --- a/ctapipe_io_magic/tests/test_magic_event_source.py +++ b/ctapipe_io_magic/tests/test_magic_event_source.py @@ -43,10 +43,13 @@ data_dict['20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root'] = dict() data_dict['20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root'] = dict() data_dict['20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root'] = dict() +data_dict['20210314_05095172_S_CrabNebula-W0.40+035.root'] = dict() data_dict['GA_M1_za35to50_8_824318_Y_w0.root'] = dict() data_dict['GA_M1_za35to50_8_824319_Y_w0.root'] = dict() data_dict['GA_M2_za35to50_8_824318_Y_w0.root'] = dict() data_dict['GA_M2_za35to50_8_824319_Y_w0.root'] = dict() +data_dict['GA_za35to50_8_824318_S_w0.root'] = dict() +data_dict['GA_za35to50_8_824319_S_w0.root'] = dict() data_dict['20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root']['n_events_tot'] = 500 data_dict['20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root']['n_events_stereo'] = 458 @@ -88,6 +91,21 @@ data_dict['GA_M2_za35to50_8_824319_Y_w0.root']['n_events_pedestal'] = 0 data_dict['GA_M2_za35to50_8_824319_Y_w0.root']['n_events_mc_mono'] = 52 +data_dict['20210314_05095172_S_CrabNebula-W0.40+035.root']['n_events_tot'] = 356 +data_dict['20210314_05095172_S_CrabNebula-W0.40+035.root']['n_events_stereo'] = 356 +data_dict['20210314_05095172_S_CrabNebula-W0.40+035.root']['n_events_pedestal'] = 0 +data_dict['20210314_05095172_S_CrabNebula-W0.40+035.root']['n_events_mc_mono'] = 0 + +data_dict['GA_za35to50_8_824318_S_w0.root']['n_events_tot'] = 53 +data_dict['GA_za35to50_8_824318_S_w0.root']['n_events_stereo'] = 53 +data_dict['GA_za35to50_8_824318_S_w0.root']['n_events_pedestal'] = 0 +data_dict['GA_za35to50_8_824318_S_w0.root']['n_events_mc_mono'] = 0 + +data_dict['GA_za35to50_8_824319_S_w0.root']['n_events_tot'] = 67 +data_dict['GA_za35to50_8_824319_S_w0.root']['n_events_stereo'] = 67 +data_dict['GA_za35to50_8_824319_S_w0.root']['n_events_pedestal'] = 0 +data_dict['GA_za35to50_8_824319_S_w0.root']['n_events_mc_mono'] = 0 + @pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) def test_event_source_for_magic_file(dataset): From fa17b94d12d0610dca30667acaac0c16d6308e65 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Thu, 21 Jul 2022 17:11:51 +0200 Subject: [PATCH 28/78] Add superstar files to tests. --- .../tests/test_magic_event_source.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/ctapipe_io_magic/tests/test_magic_event_source.py b/ctapipe_io_magic/tests/test_magic_event_source.py index 62e8f25..03b7438 100644 --- a/ctapipe_io_magic/tests/test_magic_event_source.py +++ b/ctapipe_io_magic/tests/test_magic_event_source.py @@ -146,17 +146,20 @@ def test_allowed_tels(): assert np.array_equal(source.subarray.tel_ids, np.array([1])) -@pytest.mark.parametrize('dataset', test_calibrated_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) def test_loop(dataset): - from ctapipe_io_magic import MAGICEventSource + from ctapipe_io_magic import MAGICEventSource, MARSDataLevel n_events = 10 with MAGICEventSource(input_url=dataset, max_events=n_events, process_run=False) as source: for i, event in enumerate(source): assert event.count == i - if "_M1_" in dataset.name: - assert event.trigger.tels_with_trigger == [1] - if "_M2_" in dataset.name: - assert event.trigger.tels_with_trigger == [2] + if source.mars_datalevel <= MARSDataLevel.STAR: + if "_M1_" in dataset.name: + assert event.trigger.tels_with_trigger == [1] + if "_M2_" in dataset.name: + assert event.trigger.tels_with_trigger == [2] + else: + assert event.trigger.tels_with_trigger == [1, 2] assert (i + 1) == n_events @@ -176,7 +179,7 @@ def test_loop_pedestal(dataset): assert event.trigger.event_type == EventType.SKY_PEDESTAL -@pytest.mark.parametrize('dataset', test_calibrated_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) def test_number_of_events(dataset): from ctapipe_io_magic import MAGICEventSource @@ -193,7 +196,7 @@ def test_number_of_events(dataset): # assert run['data'].n_pedestal_events_m2 == data_dict[source.input_url.name]['n_events_pedestal'] -@pytest.mark.parametrize('dataset', test_calibrated_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) def test_run_info(dataset): from ctapipe_io_magic import MAGICEventSource @@ -240,7 +243,7 @@ def test_subarray_multiple_runs(): assert list(sim_config.keys()) == source.obs_ids -@pytest.mark.parametrize('dataset', test_calibrated_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) def test_that_event_is_not_modified_after_loop(dataset): from ctapipe_io_magic import MAGICEventSource n_events = 10 @@ -256,7 +259,7 @@ def test_that_event_is_not_modified_after_loop(dataset): assert event.index.event_id == last_event.index.event_id -@pytest.mark.parametrize('dataset', test_calibrated_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) def test_geom(dataset): from ctapipe_io_magic import MAGICEventSource From 56de5ff3641c79c774097b2b3f7ede4116f72497 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Thu, 21 Jul 2022 17:33:18 +0200 Subject: [PATCH 29/78] Add run object for superstar. --- ctapipe_io_magic/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 196a853..b60ccdd 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1130,6 +1130,11 @@ def _set_active_run(self, uproot_file): self.is_simulation, self.telescopes[0], ) + if self.mars_datalevel == MARSDataLevel.SUPERSTAR: + run['data'] = MarsSuperstarRun( + uproot_file, + self.is_simulation, + ) return run From 355a5fea3722f8f93069356aaba9231dd07c0e39 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Thu, 21 Jul 2022 17:33:29 +0200 Subject: [PATCH 30/78] Fix name of container. --- ctapipe_io_magic/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index b60ccdd..be1801d 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1779,8 +1779,8 @@ def _load_data(self): ] hillas_timefit_branches[tel_id] = [ - f'MHillaTimeFit_{tel_id}.fP1Const', # [Time_slices] - f'MHillaTimeFit_{tel_id}.fP1Grad', # [Time_slices]/[mm] + f'MHillasTimeFit_{tel_id}.fP1Const', # [Time_slices] + f'MHillasTimeFit_{tel_id}.fP1Grad', # [Time_slices]/[mm] ] mc_branches[tel_id] = [ From ec0f55af7636cf2e93e819fac057623bcef701da Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Thu, 21 Jul 2022 17:35:10 +0200 Subject: [PATCH 31/78] Extract correct values. --- ctapipe_io_magic/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index be1801d..8707811 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1950,8 +1950,8 @@ def _load_data(self): library='np', ) - event_data[tel_id]['slope'] = u.Quantity(hillas_timefit[f'MHillaTimeFit_{tel_id}.fP1Grad'], 1/u.mm).to(1/u.m) * (1./sampling_speed) - event_data[tel_id]['intercept'] = hillas_timefit[f'MHillaTimeFit_{tel_id}.fP1Const'] * (1./sampling_speed) + event_data[tel_id]['slope'] = u.Quantity(hillas_timefit[f'MHillasTimeFit_{tel_id}.fP1Grad'], 1/u.mm).to(1/u.m) * (1./sampling_speed) + event_data[tel_id]['intercept'] = hillas_timefit[f'MHillasTimeFit_{tel_id}.fP1Const'] * (1./sampling_speed) stereo_data["cosmic_events"] = event_data From 6c6cf3c2571439065e635f5f20a567fbd01472c2 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Thu, 21 Jul 2022 17:56:16 +0200 Subject: [PATCH 32/78] Extract only for calibrated data. --- ctapipe_io_magic/__init__.py | 49 ++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 8707811..9a946fe 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1191,30 +1191,31 @@ def _event_generator(self): if not self.is_simulation: - monitoring_data = self.current_run['data'].monitoring_data - - # Set the pedestal information: - event.mon.tel[tel_id].pedestal.n_events = 500 # hardcoded number of pedestal events averaged over - event.mon.tel[tel_id].pedestal.sample_time = monitoring_data[tel_id]['pedestal_sample_time'] - - event.mon.tel[tel_id].pedestal.charge_mean = [ - monitoring_data[tel_id]['pedestal_fundamental']['mean'], - monitoring_data[tel_id]['pedestal_from_extractor']['mean'], - monitoring_data[tel_id]['pedestal_from_extractor_rndm']['mean'], - ] - - event.mon.tel[tel_id].pedestal.charge_std = [ - monitoring_data[tel_id]['pedestal_fundamental']['rms'], - monitoring_data[tel_id]['pedestal_from_extractor']['rms'], - monitoring_data[tel_id]['pedestal_from_extractor_rndm']['rms'], - ] - - # Set the bad pixel information: - event.mon.tel[tel_id].pixel_status.hardware_failing_pixels = np.reshape( - monitoring_data[tel_id]['bad_pixel'], (1, len(monitoring_data[tel_id]['bad_pixel'])) - ) - if self.mars_datalevel == MARSDataLevel.CALIBRATED: + + monitoring_data = self.current_run['data'].monitoring_data + + # Set the pedestal information: + event.mon.tel[tel_id].pedestal.n_events = 500 # hardcoded number of pedestal events averaged over + event.mon.tel[tel_id].pedestal.sample_time = monitoring_data[tel_id]['pedestal_sample_time'] + + event.mon.tel[tel_id].pedestal.charge_mean = [ + monitoring_data[tel_id]['pedestal_fundamental']['mean'], + monitoring_data[tel_id]['pedestal_from_extractor']['mean'], + monitoring_data[tel_id]['pedestal_from_extractor_rndm']['mean'], + ] + + event.mon.tel[tel_id].pedestal.charge_std = [ + monitoring_data[tel_id]['pedestal_fundamental']['rms'], + monitoring_data[tel_id]['pedestal_from_extractor']['rms'], + monitoring_data[tel_id]['pedestal_from_extractor_rndm']['rms'], + ] + + # Set the bad pixel information: + event.mon.tel[tel_id].pixel_status.hardware_failing_pixels = np.reshape( + monitoring_data[tel_id]['bad_pixel'], (1, len(monitoring_data[tel_id]['bad_pixel'])) + ) + # Interpolate drive information: drive_data = self.drive_information n_drive_reports = len(drive_data['mjd']) @@ -1272,7 +1273,7 @@ def _event_generator(self): event.trigger.time = event_data[tel_id]['time'][i_event] event.trigger.tel[tel_id].time = event_data[tel_id]['time'][i_event] - if not self.use_pedestals: + if not self.use_pedestals and self.mars_datalevel == MARSDataLevel.CALIBRATED: badrmspixel_mask = self._get_badrmspixel_mask(event, tel_id) event.mon.tel[tel_id].pixel_status.pedestal_failing_pixels = badrmspixel_mask From 5e2530666a8189610157ca424d22e887c37cca6c Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Thu, 21 Jul 2022 17:56:31 +0200 Subject: [PATCH 33/78] Do not use tel_id. --- ctapipe_io_magic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 9a946fe..0a01925 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1957,7 +1957,7 @@ def _load_data(self): stereo_data["cosmic_events"] = event_data stereo_parameters = self.uproot_file['Events'].arrays( - expressions=stereo_branches[tel_id], + expressions=stereo_branches, cut=events_cut, library='np', ) From 0483583a439eccf9d6fdb658c75d1f6581162d8a Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Thu, 21 Jul 2022 17:56:58 +0200 Subject: [PATCH 34/78] Extract event by event. --- ctapipe_io_magic/__init__.py | 42 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 0a01925..788ef9b 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1294,41 +1294,41 @@ def _event_generator(self): if self.mars_datalevel >= MARSDataLevel.SUPERSTAR: event.dl1.tel[tel_id].parameters = ImageParametersContainer() event.dl1.tel[tel_id].parameters.hillas = CameraHillasParametersContainer( - x=-event_data[tel_id]["y"], - y=-event_data[tel_id]["x"], - length=event_data[tel_id]["length"], - width=event_data[tel_id]["width"], - intensity=event_data[tel_id]["intensity"], - r=np.sqrt(event_data[tel_id]["x"]*event_data[tel_id]["x"]+event_data[tel_id]["y"]*event_data[tel_id]["y"]), - psi=event_data[tel_id]["psi"], - phi=event_data[tel_id]["phi"], + x=-event_data[tel_id]["y"][i_event], + y=-event_data[tel_id]["x"][i_event], + length=event_data[tel_id]["length"][i_event], + width=event_data[tel_id]["width"][i_event], + intensity=event_data[tel_id]["intensity"][i_event], + r=np.sqrt(event_data[tel_id]["x"][i_event]*event_data[tel_id]["x"][i_event]+event_data[tel_id]["y"][i_event]*event_data[tel_id]["y"][i_event]), + psi=event_data[tel_id]["psi"][i_event], + phi=event_data[tel_id]["phi"][i_event], ) event.dl1.tel[tel_id].parameters.timing = TimingParametersContainer( - slope=event_data[tel_id]["slope"].value*(1./m2deg)*u.deg, - intercept=event_data[tel_id]["intercept"], + slope=event_data[tel_id]["slope"][i_event].value*(1./m2deg)*u.deg, + intercept=event_data[tel_id]["intercept"][i_event], ) # not fully filled event.dl1.tel[tel_id].parameters.leakage = LeakageContainer( - intensity_width_1=event_data[tel_id]["intensity_width_1"], - intensity_width_2=event_data[tel_id]["intensity_width_2"], + intensity_width_1=event_data[tel_id]["intensity_width_1"][i_event], + intensity_width_2=event_data[tel_id]["intensity_width_2"][i_event], ) # not fully filled event.dl1.tel[tel_id].parameters.morphology = MorphologyContainer( - num_pixels=event_data[tel_id]["num_pixels"], - num_islands=event_data[tel_id]["num_islands"], + num_pixels=event_data[tel_id]["num_pixels"][i_event], + num_islands=event_data[tel_id]["num_islands"][i_event], ) event.dl2.stereo.geometry["HillasReconstructor"] = ReconstructedGeometryContainer( - alt=event_data["stereo"]["alt"], - az=event_data["stereo"]["az"], - core_x=event_data["stereo"]["core_x"], - core_y=event_data["stereo"]["core_y"], - h_max=event_data["stereo"]["h_max"], - is_valid=True if event_data["stereo"]["is_valid"] == 1 else False, - average_intensity=(event_data[1]["intensity"]+event_data[2]["intensity"]) / 2., + alt=event_data["stereo"]["alt"][i_event], + az=event_data["stereo"]["az"][i_event], + core_x=event_data["stereo"]["core_x"][i_event], + core_y=event_data["stereo"]["core_y"][i_event], + h_max=event_data["stereo"]["h_max"][i_event], + is_valid=True if event_data["stereo"]["is_valid"][i_event] == 1 else False, + average_intensity=(event_data[1]["intensity"][i_event]+event_data[2]["intensity"][i_event]) / 2., tel_ids=[1, 2], ) From 892ab70997ae477331e5c549568deacec1d4fc3b Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Thu, 21 Jul 2022 17:57:09 +0200 Subject: [PATCH 35/78] Use array_equal. --- ctapipe_io_magic/tests/test_magic_event_source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ctapipe_io_magic/tests/test_magic_event_source.py b/ctapipe_io_magic/tests/test_magic_event_source.py index 03b7438..3d5a6c9 100644 --- a/ctapipe_io_magic/tests/test_magic_event_source.py +++ b/ctapipe_io_magic/tests/test_magic_event_source.py @@ -149,6 +149,7 @@ def test_allowed_tels(): @pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) def test_loop(dataset): from ctapipe_io_magic import MAGICEventSource, MARSDataLevel + import numpy as np n_events = 10 with MAGICEventSource(input_url=dataset, max_events=n_events, process_run=False) as source: for i, event in enumerate(source): @@ -159,7 +160,7 @@ def test_loop(dataset): if "_M2_" in dataset.name: assert event.trigger.tels_with_trigger == [2] else: - assert event.trigger.tels_with_trigger == [1, 2] + assert np.array_equal(event.trigger.tels_with_trigger, np.array([1, 2])) assert (i + 1) == n_events From 1b9fcc08b1fe9041e9434aa9fbeda323f2a47ee6 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 18:38:47 +0200 Subject: [PATCH 36/78] Use lists. --- ctapipe_io_magic/__init__.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 788ef9b..d151980 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -335,50 +335,50 @@ def get_run_info_from_name(file_name): mask_mc_melibea = r"GA_za\d+to\d+_\d_(\d+)_Q_.*" if re.match(mask_data_calibrated, file_name) is not None: parsed_info = re.match(mask_data_calibrated, file_name) - telescope = int(parsed_info.group(1)) + telescope = [int(parsed_info.group(1))] run_number = int(parsed_info.group(2)) datalevel = MARSDataLevel.CALIBRATED is_mc = False elif re.match(mask_data_star, file_name) is not None: parsed_info = re.match(mask_data_star, file_name) - telescope = int(parsed_info.group(1)) + telescope = [int(parsed_info.group(1))] run_number = int(parsed_info.group(2)) datalevel = MARSDataLevel.STAR is_mc = False elif re.match(mask_data_superstar, file_name) is not None: parsed_info = re.match(mask_data_superstar, file_name) - telescope = None + telescope = [1, 2] run_number = int(parsed_info.group(1)) datalevel = MARSDataLevel.SUPERSTAR is_mc = False elif re.match(mask_data_melibea, file_name) is not None: parsed_info = re.match(mask_data_melibea, file_name) - telescope = None + telescope = [1, 2] run_number = int(parsed_info.group(1)) datalevel = MARSDataLevel.MELIBEA is_mc = False elif re.match(mask_mc_calibrated, file_name) is not None: parsed_info = re.match(mask_mc_calibrated, file_name) - telescope = int(parsed_info.group(1)) + telescope = [int(parsed_info.group(1))] run_number = int(parsed_info.group(2)) datalevel = MARSDataLevel.CALIBRATED is_mc = True elif re.match(mask_mc_star, file_name) is not None: parsed_info = re.match(mask_mc_star, file_name) - telescope = int(parsed_info.group(1)) + telescope = [int(parsed_info.group(1))] run_number = int(parsed_info.group(2)) datalevel = MARSDataLevel.STAR is_mc = True elif re.match(mask_mc_superstar, file_name) is not None: parsed_info = re.match(mask_mc_superstar, file_name) - telescope = None - run_number = None + telescope = [1, 2] + run_number = int(parsed_info.group(1)) datalevel = MARSDataLevel.SUPERSTAR is_mc = True elif re.match(mask_mc_melibea, file_name) is not None: parsed_info = re.match(mask_mc_melibea, file_name) - telescope = None - run_number = None + telescope = [1, 2] + run_number = int(parsed_info.group(1)) datalevel = MARSDataLevel.MELIBEA is_mc = True else: From 09ea52f34b17094f554d2d283b80c1c3d40bf3a1 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 18:38:57 +0200 Subject: [PATCH 37/78] Compare lists. --- ctapipe_io_magic/tests/test_magic_event_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe_io_magic/tests/test_magic_event_source.py b/ctapipe_io_magic/tests/test_magic_event_source.py index 3d5a6c9..e26407e 100644 --- a/ctapipe_io_magic/tests/test_magic_event_source.py +++ b/ctapipe_io_magic/tests/test_magic_event_source.py @@ -209,7 +209,7 @@ def test_run_info(dataset): datalevel = [i[3] for i in run_info][0] assert run_numbers == source.run_numbers assert is_mc == source.is_simulation - assert telescope == source.telescopes[0] + assert telescope == source.telescopes assert datalevel == source.mars_datalevel From 3d6f5cd4e00c3de59d16feb224c8348fd739584d Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 18:45:39 +0200 Subject: [PATCH 38/78] Fix test data download script. --- download_test_data.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/download_test_data.sh b/download_test_data.sh index 7660913..5512679 100644 --- a/download_test_data.sh +++ b/download_test_data.sh @@ -15,7 +15,7 @@ echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulate echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M2_za35to50_8_824319_Y_w0.root" >> test_data_simulated.txt echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824318_S_w0.root" > test_data_superstar_simulated.txt -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824319_S_w0.root" > test_data_superstar_simulated.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824319_S_w0.root" >> test_data_superstar_simulated.txt if [ -z "$TEST_DATA_USER" ]; then echo -n "Username: " From 683f23e4db40cff8dc30c5d28e8e12850807ab19 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 19:37:34 +0200 Subject: [PATCH 39/78] Add melibea files to download files. --- download_test_data.sh | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/download_test_data.sh b/download_test_data.sh index 5512679..4d44426 100644 --- a/download_test_data.sh +++ b/download_test_data.sh @@ -9,6 +9,8 @@ echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/cal echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/superstar/20210314_05095172_S_CrabNebula-W0.40+035.root" > test_data_superstar_real.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/superstar/20210314_05095172_Q_CrabNebula-W0.40+035.root" > test_data_melibea_real.txt + echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M1_za35to50_8_824318_Y_w0.root" > test_data_simulated.txt echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M1_za35to50_8_824319_Y_w0.root" >> test_data_simulated.txt echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M2_za35to50_8_824318_Y_w0.root" >> test_data_simulated.txt @@ -17,6 +19,9 @@ echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulate echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824318_S_w0.root" > test_data_superstar_simulated.txt echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824319_S_w0.root" >> test_data_superstar_simulated.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824318_Q_w0.root" > test_data_melibea_simulated.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824319_Q_w0.root" >> test_data_melibea_simulated.txt + if [ -z "$TEST_DATA_USER" ]; then echo -n "Username: " read TEST_DATA_USER @@ -51,6 +56,17 @@ if ! wget \ echo "Problem in downloading the test data set for real data." fi +if ! wget \ + -i test_data_melibea_real.txt \ + --user="$TEST_DATA_USER" \ + --password="$TEST_DATA_PASSWORD" \ + --no-check-certificate \ + --no-verbose \ + --timestamping \ + --directory-prefix=test_data/real/superstar; then + echo "Problem in downloading the test data set for real data." +fi + if ! wget \ -i test_data_simulated.txt \ --user="$TEST_DATA_USER" \ @@ -73,4 +89,15 @@ if ! wget \ echo "Problem in downloading the test data set for simulated data." fi -rm -f test_data_real.txt test_data_simulated.txt test_data_superstar_real.txt test_data_superstar_simulated.txt +if ! wget \ + -i test_data_melibea_simulated.txt \ + --user="$TEST_DATA_USER" \ + --password="$TEST_DATA_PASSWORD" \ + --no-check-certificate \ + --no-verbose \ + --timestamping \ + --directory-prefix=test_data/simulated/superstar; then + echo "Problem in downloading the test data set for simulated data." +fi + +rm -f test_data_real.txt test_data_simulated.txt test_data_superstar_real.txt test_data_superstar_simulated.txt test_data_melibea_real.txt test_data_melibea_simulated.txt From ce8d2c412872642abb9a50c874baee8ceb212351 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 19:38:34 +0200 Subject: [PATCH 40/78] Start adding melibea files to tests. --- .../tests/test_magic_event_source.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/ctapipe_io_magic/tests/test_magic_event_source.py b/ctapipe_io_magic/tests/test_magic_event_source.py index e26407e..3a61933 100644 --- a/ctapipe_io_magic/tests/test_magic_event_source.py +++ b/ctapipe_io_magic/tests/test_magic_event_source.py @@ -33,6 +33,18 @@ test_superstar_simulated_dir / "GA_za35to50_8_824319_S_w0.root", ] +test_melibea_real_dir = test_data / "real/melibea" +test_melibea_simulated_dir = test_data / "simulated/melibea" + +test_melibea_mars_real = [ + test_melibea_real_dir / "20210314_05095172_Q_CrabNebula-W0.40+035.root", +] + +test_melibea_mars_simulated = [ + test_melibea_simulated_dir / "GA_za35to50_8_824318_Q_w0.root", + test_melibea_simulated_dir / "GA_za35to50_8_824319_Q_w0.root", +] + test_calibrated_all = test_calibrated_real+test_calibrated_simulated test_superstar_all = test_superstar_mars_real + test_superstar_mars_simulated @@ -44,12 +56,15 @@ data_dict['20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root'] = dict() data_dict['20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root'] = dict() data_dict['20210314_05095172_S_CrabNebula-W0.40+035.root'] = dict() +data_dict['20210314_05095172_Q_CrabNebula-W0.40+035.root'] = dict() data_dict['GA_M1_za35to50_8_824318_Y_w0.root'] = dict() data_dict['GA_M1_za35to50_8_824319_Y_w0.root'] = dict() data_dict['GA_M2_za35to50_8_824318_Y_w0.root'] = dict() data_dict['GA_M2_za35to50_8_824319_Y_w0.root'] = dict() data_dict['GA_za35to50_8_824318_S_w0.root'] = dict() data_dict['GA_za35to50_8_824319_S_w0.root'] = dict() +data_dict['GA_za35to50_8_824318_Q_w0.root'] = dict() +data_dict['GA_za35to50_8_824319_Q_w0.root'] = dict() data_dict['20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root']['n_events_tot'] = 500 data_dict['20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root']['n_events_stereo'] = 458 @@ -106,6 +121,21 @@ data_dict['GA_za35to50_8_824319_S_w0.root']['n_events_pedestal'] = 0 data_dict['GA_za35to50_8_824319_S_w0.root']['n_events_mc_mono'] = 0 +data_dict['20210314_05095172_Q_CrabNebula-W0.40+035.root']['n_events_tot'] = 356 +data_dict['20210314_05095172_Q_CrabNebula-W0.40+035.root']['n_events_stereo'] = 356 +data_dict['20210314_05095172_Q_CrabNebula-W0.40+035.root']['n_events_pedestal'] = 0 +data_dict['20210314_05095172_Q_CrabNebula-W0.40+035.root']['n_events_mc_mono'] = 0 + +data_dict['GA_za35to50_8_824318_Q_w0.root']['n_events_tot'] = 53 +data_dict['GA_za35to50_8_824318_Q_w0.root']['n_events_stereo'] = 53 +data_dict['GA_za35to50_8_824318_Q_w0.root']['n_events_pedestal'] = 0 +data_dict['GA_za35to50_8_824318_Q_w0.root']['n_events_mc_mono'] = 0 + +data_dict['GA_za35to50_8_824319_Q_w0.root']['n_events_tot'] = 67 +data_dict['GA_za35to50_8_824319_Q_w0.root']['n_events_stereo'] = 67 +data_dict['GA_za35to50_8_824319_Q_w0.root']['n_events_pedestal'] = 0 +data_dict['GA_za35to50_8_824319_Q_w0.root']['n_events_mc_mono'] = 0 + @pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) def test_event_source_for_magic_file(dataset): From d8b916c7ceb4b2d78669a10fc48b7c0e4670a9c3 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 20:24:11 +0200 Subject: [PATCH 41/78] Add reading of melibea files. --- ctapipe_io_magic/__init__.py | 355 ++++++++++++++++++++++++++++++++++- 1 file changed, 354 insertions(+), 1 deletion(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index d151980..f978b81 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -32,6 +32,8 @@ LeakageContainer, MorphologyContainer, ReconstructedGeometryContainer, + ReconstructedEnergyContainer, + ParticleClassificationContainer, ) from ctapipe.instrument import ( @@ -1332,6 +1334,20 @@ def _event_generator(self): tel_ids=[1, 2], ) + if self.mars_datalevel >= MARSDataLevel.MELIBEA: + event.dl2.stereo.energy = ReconstructedEnergyContainer( + energy=event_data["reconstructed"]["energy"][i_event], + energy_uncert=event_data["reconstructed"]["energy_uncert"][i_event], + is_valid=True if event_data["stereo"]["is_valid"][i_event] == 1 else False, + tel_ids=[1, 2], + ) + + event.dl2.stereo.classification = ParticleClassificationContainer( + predictiion=event_data["reconstructed"]["gammanness"][i_event], + is_valid=True if event_data["stereo"]["is_valid"][i_event] == 1 else False, + tel_ids=[1, 2], + ) + # Set the simulated event container: if self.is_simulation: @@ -1684,7 +1700,7 @@ def __init__(self, uproot_file, is_mc, n_cam_pixels=1039): Parameters ---------- uproot_file: uproot.reading.ReadOnlyDirectory - A calibrated file opened by uproot via uproot.open(file_path) + A superstar file opened by uproot via uproot.open(file_path) is_mc: bool Flag to MC data n_cam_pixels: int @@ -1973,3 +1989,340 @@ def _load_data(self): stereo_data["cosmic_events"]["stereo"] = stereo_params return stereo_data + + +class MarsMelibeaRun: + """ + This class implements reading of cosmic events from a MAGIC melibea run file. + """ + + def __init__(self, uproot_file, is_mc, n_cam_pixels=1039): + """ + Constructor of the class. Loads an input uproot file + and store the informaiton to constructor variables. + + Parameters + ---------- + uproot_file: uproot.reading.ReadOnlyDirectory + A melibea file opened by uproot via uproot.open(file_path) + is_mc: bool + Flag to MC data + n_cam_pixels: int + The number of pixels of the MAGIC camera + """ + + self.uproot_file = uproot_file + self.is_mc = is_mc + self.n_cam_pixels = n_cam_pixels + + # Load the input data: + stereo_data = self._load_data() + + self.cosmic_events = stereo_data["cosmic_events"] + + @property + def n_cosmic_events(self): + return len(self.cosmic_events[1].get('trigger_pattern', [])) + + @property + def n_pedestal_events(self): + return 0 + + def _load_data(self): + """ + This method loads cosmic and pedestal events, and monitoring data + from an input calibrated file and returns them as a dictionary. + + Returns + ------- + stereo_data: dict + A dictionary with the event properties, + cosmic and pedestal events, and monitoring data + """ + + common_branches = dict() + pointing_branches = dict() + timing_branches = dict() + hillas_branches = dict() + hillas_src_branches = dict() + imagepar_branches = dict() + imagepar_disp_branches = dict() + new_imagepar_branches = dict() + hillas_timefit_branches = dict() + mc_branches = dict() + + for tel_id in [1, 2]: + + common_branches[tel_id] = [ + f'MRawEvtHeader_{tel_id}.fStereoEvtNumber', + f'MTriggerPattern_{tel_id}.fPrescaled', + ] + + pointing_branches[tel_id] = [ + f'MPointingPos_{tel_id}.fZd', # [deg] + f'MPointingPos_{tel_id}.fAz', # [deg] + f'MPointingPos_{tel_id}.fRa', # [h] + f'MPointingPos_{tel_id}.fDec', # [deg] + f'MPointingPos_{tel_id}.fDevZd', # [deg] + f'MPointingPos_{tel_id}.fDevAz', # [deg] + f'MPointingPos_{tel_id}.fDevHa', # [h] + f'MPointingPos_{tel_id}.fDevDec', # [deg] + ] + + # Branches applicable for real data: + timing_branches[tel_id] = [ + f'MTime_{tel_id}.fMjd', + f'MTime_{tel_id}.fTime.fMilliSec', + f'MTime_{tel_id}.fNanoSec', + ] + + hillas_branches[tel_id] = [ + f'MHillas_{tel_id}.fLength', # [mm] + f'MHillas_{tel_id}.fWidth', # [mm] + f'MHillas_{tel_id}.fDelta', # [rad] + f'MHillas_{tel_id}.fSize', + f'MHillas_{tel_id}.fMeanX', # [mm] + f'MHillas_{tel_id}.fMeanY', # [mm] + ] + + hillas_src_branches[tel_id] = [ + f'MHillasSrc_{tel_id}.fDCADelta', # [deg] Angle of the shower axis with respect to the x-axis + f'MHillasSrc_{tel_id}.fDist', # [mm] distance between src and center of ellipse + ] + + imagepar_branches[tel_id] = [ + f'MImagePar_{tel_id}.fNumIslands', # Number of distinct pixel groupings in image + ] + + new_imagepar_branches[tel_id] = [ + f'MNewImagePar_{tel_id}.fNumUsedPixels', # Number of pixels which survived the image cleaning + f'MNewImagePar_{tel_id}.fLeakage1', # (photons in most outer ring of pixels) over fSize + f'MNewImagePar_{tel_id}.fLeakage2', # (photons in the 2 outer rings of pixels) over fSize + ] + + hillas_timefit_branches[tel_id] = [ + f'MHillasTimeFit_{tel_id}.fP1Const', # [Time_slices] + f'MHillasTimeFit_{tel_id}.fP1Grad', # [Time_slices]/[mm] + ] + + mc_branches[tel_id] = [ + f'MMcEvt_{tel_id}.fEnergy', + f'MMcEvt_{tel_id}.fTheta', + f'MMcEvt_{tel_id}.fPhi', + f'MMcEvt_{tel_id}.fPartId', + f'MMcEvt_{tel_id}.fZFirstInteraction', + f'MMcEvt_{tel_id}.fCoreX', + f'MMcEvt_{tel_id}.fCoreY', + ] + + imagepar_disp_branches[tel_id] = [ + f'MImageParDisp_{tel_id}.fDisp', + f'MImageParDisp_{tel_id}.fDispRMS', + f'MImageParDisp_{tel_id}.fXDisp', + f'MImageParDisp_{tel_id}.fYDisp', + ] + + stereo_branches = [ + 'MStereoParDisp.fValid', + 'MStereoParDisp.fDirectionX', + 'MStereoParDisp.fDirectionY', + 'MStereoParDisp.fDirectionZd', + 'MStereoParDisp.fDirectionAz', + 'MStereoParDisp.fDirectionDec', + 'MStereoParDisp.fDirectionRA', + 'MStereoParDisp.fTheta2', + 'MStereoParDisp.fCoreX', + 'MStereoParDisp.fCoreY', + 'MStereoParDisp.fM1Impact', + 'MStereoParDisp.fM2Impact', + 'MStereoParDisp.fMaxHeight', + ] + + melibea_branches = [ + 'MHadronness.fHadronness', + 'MHadronness.fHadronnessRMS', + 'MEnergyEst.fEnergy', + 'MEnergyEst.fEnergyRMS', + 'MEnergyEst.fUncertainty', + 'MEnergyEst.fImpact', + 'MEnergyEstAtmCorr.fEnergy', + 'MEnergyEstAtmCorr.fEnergyRMS', + 'MEnergyEstAtmCorr.fImpact', + ] + + stereo_data = { + 'cosmic_events': dict(), + } + + event_data = dict() + + for tel_id in [1, 2]: + + # Initialize the data container: + + event_data[tel_id] = dict() + + if self.is_mc: + # Only for cosmic events because MC data do not have pedestal events: + events_cut = f'(MTriggerPattern_{tel_id}.fPrescaled == {MC_STEREO_TRIGGER_PATTERN})' \ + f' & (MRawEvtHeader_{tel_id}.fStereoEvtNumber != 0)' + else: + events_cut = f'(MTriggerPattern_{tel_id}.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})' + + # read common information from RunHeaders + sampling_speed = self.uproot_file['RunHeaders'][f'MRawRunHeader_{tel_id}.fSamplingFrequency'].array(library='np')[0]/1000. # GHz + + # Reading the information common to cosmic and pedestal events: + common_info = self.uproot_file['Events'].arrays( + expressions=common_branches[tel_id], + cut=events_cut, + library='np', + ) + + event_data[tel_id]['trigger_pattern'] = np.array(common_info[f'MTriggerPattern_{tel_id}.fPrescaled'], dtype=int) + event_data[tel_id]['stereo_event_number'] = np.array(common_info[f'MRawEvtHeader_{tel_id}.fStereoEvtNumber'], dtype=int) + + if self.is_mc: + + # Reading the MC information: + mc_info = self.uproot_file['Events'].arrays( + expressions=mc_branches[tel_id], + cut=events_cut, + library='np', + ) + + # Note that the branch "MMcEvt.fPhi" seems to be the angle between the direction + # of the magnetic north and the momentum of a simulated primary particle, defined + # between -pi and pi, negative if the momentum pointing eastward, positive westward. + # The conversion to azimuth should be 180 - fPhi + magnetic_declination: + event_data[tel_id]['mc_energy'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fEnergy'], u.GeV) + event_data[tel_id]['mc_theta'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fTheta'], u.rad) + event_data[tel_id]['mc_phi'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fPhi'], u.rad) + event_data[tel_id]['mc_shower_primary_id'] = np.array(mc_info[f'MMcEvt_{tel_id}.fPartId'], dtype=int) + event_data[tel_id]['mc_h_first_int'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fZFirstInteraction'], u.cm) + event_data[tel_id]['mc_core_x'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fCoreX'], u.cm) + event_data[tel_id]['mc_core_y'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fCoreY'], u.cm) + + else: + # Reading the event timing information: + timing_info = self.uproot_file['Events'].arrays( + expressions=timing_branches[tel_id], + cut=events_cut, + library='np', + ) + + # In order to keep the precision of timestamps, here the Decimal module is used. + # At the later steps, the precise information can be extracted by specifying + # the sub-format of value to 'long', i.e., Time.to_value(format='unix', subfmt='long'): + event_obs_day = Time(timing_info[f'MTime_{tel_id}.fMjd'], format='mjd', scale='utc') + event_obs_day = np.round(event_obs_day.unix) + event_obs_day = np.array([Decimal(str(x)) for x in event_obs_day]) + + event_millisec = np.round(timing_info[f'MTime_{tel_id}.fTime.fMilliSec'] * msec2sec, 3) + event_millisec = np.array([Decimal(str(x)) for x in event_millisec]) + + event_nanosec = np.round(timing_info[f'MTime_{tel_id}.fNanoSec'] * nsec2sec, 7) + event_nanosec = np.array([Decimal(str(x)) for x in event_nanosec]) + + event_time = event_obs_day + event_millisec + event_nanosec + event_data[tel_id]['time'] = Time(event_time, format='unix', scale='utc') + + # Reading the telescope pointing information: + pointing = self.uproot_file['Events'].arrays( + expressions=pointing_branches[tel_id], + cut=events_cut, + library='np', + ) + + pointing_az = pointing[f'MPointingPos_{tel_id}.fAz'] - pointing[f'MPointingPos_{tel_id}.fDevAz'] + pointing_zd = pointing[f'MPointingPos_{tel_id}.fZd'] - pointing[f'MPointingPos_{tel_id}.fDevZd'] + + # N.B. the positive sign here, as HA = local sidereal time - ra: + pointing_ra = (pointing[f'MPointingPos_{tel_id}.fRa'] + pointing[f'MPointingPos_{tel_id}.fDevHa']) * degrees_per_hour + pointing_dec = pointing[f'MPointingPos_{tel_id}.fDec'] - pointing[f'MPointingPos_{tel_id}.fDevDec'] + + event_data[tel_id]['pointing_az'] = u.Quantity(pointing_az, u.deg) + event_data[tel_id]['pointing_alt'] = u.Quantity(90 - pointing_zd, u.deg) + event_data[tel_id]['pointing_ra'] = u.Quantity(pointing_ra, u.deg) + event_data[tel_id]['pointing_dec'] = u.Quantity(pointing_dec, u.deg) + + hillas = self.uproot_file['Events'].arrays( + expressions=hillas_branches[tel_id], + cut=events_cut, + library='np', + ) + + event_data[tel_id]['length'] = u.Quantity(hillas[f'MHillas_{tel_id}.fLength'], u.mm).to(u.m) + event_data[tel_id]['width'] = u.Quantity(hillas[f'MHillas_{tel_id}.fWidth'], u.mm).to(u.m) + event_data[tel_id]['x'] = u.Quantity(hillas[f'MHillas_{tel_id}.fMeanX'], u.mm).to(u.m) + event_data[tel_id]['y'] = u.Quantity(hillas[f'MHillas_{tel_id}.fMeanY'], u.mm).to(u.m) + event_data[tel_id]['intensity'] = u.Quantity(hillas[f'MHillas_{tel_id}.fSize'], u.mm).to(u.m) + event_data[tel_id]['psi'] = u.Quantity(hillas[f'MHillas_{tel_id}.fDelta'], u.rad).to(u.deg) + + hillas_src = self.uproot_file['Events'].arrays( + expressions=hillas_src_branches[tel_id], + cut=events_cut, + library='np', + ) + + event_data[tel_id]['phi'] = u.Quantity(hillas_src[f'MHillasSrc_{tel_id}.fDCADelta'], u.deg) + + imagepar = self.uproot_file['Events'].arrays( + expressions=imagepar_branches[tel_id], + cut=events_cut, + library='np', + ) + + event_data[tel_id]['num_islands'] = imagepar[f'MImagePar_{tel_id}.fNumIslands'] + + new_imagepar = self.uproot_file['Events'].arrays( + expressions=new_imagepar_branches[tel_id], + cut=events_cut, + library='np', + ) + + event_data[tel_id]['num_pixels'] = new_imagepar[f'MNewImagePar_{tel_id}.fNumUsedPixels'] + event_data[tel_id]['intensity_width_1'] = new_imagepar[f'MNewImagePar_{tel_id}.fLeakage1'] * event_data[tel_id]['intensity'] + event_data[tel_id]['intensity_width_2'] = new_imagepar[f'MNewImagePar_{tel_id}.fLeakage2'] * event_data[tel_id]['intensity'] + + hillas_timefit = self.uproot_file['Events'].arrays( + expressions=hillas_timefit_branches[tel_id], + cut=events_cut, + library='np', + ) + + event_data[tel_id]['slope'] = u.Quantity(hillas_timefit[f'MHillasTimeFit_{tel_id}.fP1Grad'], 1/u.mm).to(1/u.m) * (1./sampling_speed) + event_data[tel_id]['intercept'] = hillas_timefit[f'MHillasTimeFit_{tel_id}.fP1Const'] * (1./sampling_speed) + + stereo_data["cosmic_events"] = event_data + + stereo_parameters = self.uproot_file['Events'].arrays( + expressions=stereo_branches, + cut=events_cut, + library='np', + ) + + stereo_params = dict() + stereo_params['alt'] = u.Quantity(90 - stereo_parameters['MStereoParDisp.fDirectionZd'], u.deg) + stereo_params['az'] = u.Quantity(stereo_parameters['MStereoParDisp.fDirectionAz'], u.deg) + stereo_params['core_x'] = u.Quantity(stereo_parameters['MStereoParDisp.fCoreX'], u.cm).to(u.m) + stereo_params['core_y'] = u.Quantity(stereo_parameters['MStereoParDisp.fCoreY'], u.cm).to(u.m) + stereo_params['h_max'] = u.Quantity(stereo_parameters['MStereoParDisp.fMaxHeight'], u.cm).to(u.m) + stereo_params['is_valid'] = stereo_parameters['MStereoParDisp.fValid'] + + stereo_data["cosmic_events"]["stereo"] = stereo_params + + melibea_parameters = self.uproot_file['Events'].arrays( + expressions=melibea_branches, + cut=events_cut, + library='np', + ) + + melibea_params = dict() + melibea_params['gammanness'] = 1.0 - melibea_parameters['MHadronness.fHadronness'] + melibea_params['energy'] = u.Quantity(melibea_parameters['MEnergyEst.fEnergy'], u.GeV).to(u.TeV) + melibea_params['energy_uncert'] = u.Quantity(melibea_parameters['MEnergyEst.fUncertainty'], u.GeV).to(u.TeV) + + stereo_data["cosmic_events"]["reconstructed"] = melibea_params + + return stereo_data From 0bcb05d58dea85d53eee00a4f758f3b12feb9e29 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 20:59:33 +0200 Subject: [PATCH 42/78] Add melibea files to tests. --- .../tests/test_magic_event_source.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/ctapipe_io_magic/tests/test_magic_event_source.py b/ctapipe_io_magic/tests/test_magic_event_source.py index 3a61933..5ebcb68 100644 --- a/ctapipe_io_magic/tests/test_magic_event_source.py +++ b/ctapipe_io_magic/tests/test_magic_event_source.py @@ -49,6 +49,8 @@ test_superstar_all = test_superstar_mars_real + test_superstar_mars_simulated +test_melibea_all = test_melibea_mars_real + test_melibea_mars_simulated + data_dict = dict() data_dict['20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root'] = dict() @@ -137,7 +139,7 @@ data_dict['GA_za35to50_8_824319_Q_w0.root']['n_events_mc_mono'] = 0 -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) def test_event_source_for_magic_file(dataset): from ctapipe.io import EventSource @@ -150,7 +152,7 @@ def test_event_source_for_magic_file(dataset): assert reader.input_url == dataset -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) def test_compatible(dataset): from ctapipe_io_magic import MAGICEventSource assert MAGICEventSource.is_compatible(dataset) @@ -161,7 +163,7 @@ def test_not_compatible(): assert MAGICEventSource.is_compatible(None) is False -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) def test_stream(dataset): from ctapipe_io_magic import MAGICEventSource with MAGICEventSource(input_url=dataset, process_run=False) as source: @@ -176,7 +178,7 @@ def test_allowed_tels(): assert np.array_equal(source.subarray.tel_ids, np.array([1])) -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) def test_loop(dataset): from ctapipe_io_magic import MAGICEventSource, MARSDataLevel import numpy as np @@ -210,7 +212,7 @@ def test_loop_pedestal(dataset): assert event.trigger.event_type == EventType.SKY_PEDESTAL -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) def test_number_of_events(dataset): from ctapipe_io_magic import MAGICEventSource @@ -227,7 +229,7 @@ def test_number_of_events(dataset): # assert run['data'].n_pedestal_events_m2 == data_dict[source.input_url.name]['n_events_pedestal'] -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) def test_run_info(dataset): from ctapipe_io_magic import MAGICEventSource @@ -274,7 +276,7 @@ def test_subarray_multiple_runs(): assert list(sim_config.keys()) == source.obs_ids -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) def test_that_event_is_not_modified_after_loop(dataset): from ctapipe_io_magic import MAGICEventSource n_events = 10 @@ -290,7 +292,7 @@ def test_that_event_is_not_modified_after_loop(dataset): assert event.index.event_id == last_event.index.event_id -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all) +@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) def test_geom(dataset): from ctapipe_io_magic import MAGICEventSource From aee56d3e63ab7af8b891abdeecb59d31ab9a2ba4 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 21:00:56 +0200 Subject: [PATCH 43/78] Use different class for melibea files. --- ctapipe_io_magic/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index f978b81..7fed092 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1137,6 +1137,11 @@ def _set_active_run(self, uproot_file): uproot_file, self.is_simulation, ) + if self.mars_datalevel == MARSDataLevel.MELIBEA: + run['data'] = MarsMelibeaRun( + uproot_file, + self.is_simulation, + ) return run From 12a978e2b030c19040250840d8829a2474d751b1 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 21:10:44 +0200 Subject: [PATCH 44/78] Change order for checking MARS data level. --- ctapipe_io_magic/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 7fed092..4883948 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -436,15 +436,15 @@ def parse_run_info(self): datalevel = MARSDataLevel.CALIBRATED events_keys = events_tree.keys() - trees_in_file = [tree in events_keys for tree in melibea_trees] + trees_in_file = [tree in events_keys for tree in star_trees] if all(trees_in_file): - datalevel = MARSDataLevel.MELIBEA + datalevel = MARSDataLevel.STAR trees_in_file = [tree in events_keys for tree in superstar_trees] if all(trees_in_file): datalevel = MARSDataLevel.SUPERSTAR - trees_in_file = [tree in events_keys for tree in star_trees] + trees_in_file = [tree in events_keys for tree in melibea_trees] if all(trees_in_file): - datalevel = MARSDataLevel.STAR + datalevel = MARSDataLevel.MELIBEA if datalevel <= MARSDataLevel.STAR: run_info = rootf['RunHeaders'].arrays( From 7cbaa290e3181687112a3af27da5a3f3a52333ea Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 21:11:10 +0200 Subject: [PATCH 45/78] Fix variable name. --- ctapipe_io_magic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 4883948..e4dc814 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1348,7 +1348,7 @@ def _event_generator(self): ) event.dl2.stereo.classification = ParticleClassificationContainer( - predictiion=event_data["reconstructed"]["gammanness"][i_event], + prediction=event_data["reconstructed"]["gammanness"][i_event], is_valid=True if event_data["stereo"]["is_valid"][i_event] == 1 else False, tel_ids=[1, 2], ) From 9a0f0749437ad731ea06baf942d93a39e82a9f0a Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 21:12:46 +0200 Subject: [PATCH 46/78] Do not extract atmospheric corrected energies for MC. --- ctapipe_io_magic/__init__.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index e4dc814..7155c78 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -2143,17 +2143,30 @@ def _load_data(self): 'MStereoParDisp.fMaxHeight', ] - melibea_branches = [ - 'MHadronness.fHadronness', - 'MHadronness.fHadronnessRMS', - 'MEnergyEst.fEnergy', - 'MEnergyEst.fEnergyRMS', - 'MEnergyEst.fUncertainty', - 'MEnergyEst.fImpact', - 'MEnergyEstAtmCorr.fEnergy', - 'MEnergyEstAtmCorr.fEnergyRMS', - 'MEnergyEstAtmCorr.fImpact', - ] + if self.is_mc: + + melibea_branches = [ + 'MHadronness.fHadronness', + 'MHadronness.fHadronnessRMS', + 'MEnergyEst.fEnergy', + 'MEnergyEst.fEnergyRMS', + 'MEnergyEst.fUncertainty', + 'MEnergyEst.fImpact', + ] + + else: + + melibea_branches = [ + 'MHadronness.fHadronness', + 'MHadronness.fHadronnessRMS', + 'MEnergyEst.fEnergy', + 'MEnergyEst.fEnergyRMS', + 'MEnergyEst.fUncertainty', + 'MEnergyEst.fImpact', + 'MEnergyEstAtmCorr.fEnergy', + 'MEnergyEstAtmCorr.fEnergyRMS', + 'MEnergyEstAtmCorr.fImpact', + ] stereo_data = { 'cosmic_events': dict(), From 0b658e227998335dd9b1110f89729d4c606e81ee Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 21:25:37 +0200 Subject: [PATCH 47/78] Change import. --- ctapipe_io_magic/mars_datalevels.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ctapipe_io_magic/mars_datalevels.py b/ctapipe_io_magic/mars_datalevels.py index 2632a2b..2a0c45d 100644 --- a/ctapipe_io_magic/mars_datalevels.py +++ b/ctapipe_io_magic/mars_datalevels.py @@ -1,4 +1,7 @@ -from enum import Enum, auto +from enum import ( + Enum, + auto, +) class OrderedEnum(Enum): From 78a29eb062553e5177712df503516b99e28e86d7 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 21:25:45 +0200 Subject: [PATCH 48/78] Remove f-string. --- ctapipe_io_magic/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 7155c78..75845d8 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1648,16 +1648,16 @@ def _load_data(self): # Set the mean and RMS of pedestal charges: calib_data['monitoring_data'][self.tel_id]['pedestal_fundamental'] = { - 'mean': np.array(pedestal_info[f'MPedPhotFundamental.fArray.fMean'].tolist())[:, :self.n_cam_pixels], - 'rms': np.array(pedestal_info[f'MPedPhotFundamental.fArray.fRms'].tolist())[:, :self.n_cam_pixels], + 'mean': np.array(pedestal_info['MPedPhotFundamental.fArray.fMean'].tolist())[:, :self.n_cam_pixels], + 'rms': np.array(pedestal_info['MPedPhotFundamental.fArray.fRms'].tolist())[:, :self.n_cam_pixels], } calib_data['monitoring_data'][self.tel_id]['pedestal_from_extractor'] = { - 'mean': np.array(pedestal_info[f'MPedPhotFromExtractor.fArray.fMean'].tolist())[:, :self.n_cam_pixels], - 'rms': np.array(pedestal_info[f'MPedPhotFromExtractor.fArray.fRms'].tolist())[:, :self.n_cam_pixels], + 'mean': np.array(pedestal_info['MPedPhotFromExtractor.fArray.fMean'].tolist())[:, :self.n_cam_pixels], + 'rms': np.array(pedestal_info['MPedPhotFromExtractor.fArray.fRms'].tolist())[:, :self.n_cam_pixels], } calib_data['monitoring_data'][self.tel_id]['pedestal_from_extractor_rndm'] = { - 'mean': np.array(pedestal_info[f'MPedPhotFromExtractorRndm.fArray.fMean'].tolist())[:, :self.n_cam_pixels], - 'rms': np.array(pedestal_info[f'MPedPhotFromExtractorRndm.fArray.fRms'].tolist())[:, :self.n_cam_pixels], + 'mean': np.array(pedestal_info['MPedPhotFromExtractorRndm.fArray.fMean'].tolist())[:, :self.n_cam_pixels], + 'rms': np.array(pedestal_info['MPedPhotFromExtractorRndm.fArray.fRms'].tolist())[:, :self.n_cam_pixels], } except KeyError: From e047650ab02bf8fd5572483763e8771fa30344b4 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 21:37:59 +0200 Subject: [PATCH 49/78] Add flake8 config. --- setup.cfg | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/setup.cfg b/setup.cfg index 0d30eba..2fd038f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,3 +7,15 @@ author = Ievgen Vovk et al. author_email = Ievgen.Vovk@mpp.mpg.de license = MIT url = https://github.com/cta-observatory/ctapipe_io_magic + +[flake8] +exclude= + build, + docs, + .eggs +max-line-length=90 +#ignore=W291,E303,W391,F403,F401,W503,W1202 +select = C,E,F,W,B,B950 +ignore = E501,W503,E203,W201 +per-file-ignores = + */__init__.py: F401, F403 From 2b1d25eb562b9d496768756e7b9f994aafa13817 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 21:40:00 +0200 Subject: [PATCH 50/78] Run black and isort in tests directory. --- ctapipe_io_magic/mars_datalevels.py | 4 +- .../tests/test_magic_event_source.py | 375 +++++++++++------- ctapipe_io_magic/tests/test_stage1.py | 74 ++-- ctapipe_io_magic/tests/test_version.py | 2 +- 4 files changed, 288 insertions(+), 167 deletions(-) diff --git a/ctapipe_io_magic/mars_datalevels.py b/ctapipe_io_magic/mars_datalevels.py index 2a0c45d..4c80ff3 100644 --- a/ctapipe_io_magic/mars_datalevels.py +++ b/ctapipe_io_magic/mars_datalevels.py @@ -34,4 +34,6 @@ class MARSDataLevel(OrderedEnum): CALIBRATED = auto() # Calibrated images in charge and time (no waveforms) STAR = auto() # Cleaned images, with Hillas parametrization SUPERSTAR = auto() # Stereo parameters reconstructed - MELIBEA = auto() # Reconstruction of hadronness, event direction and energy + MELIBEA = ( + auto() + ) # Reconstruction of hadronness, event direction and energy diff --git a/ctapipe_io_magic/tests/test_magic_event_source.py b/ctapipe_io_magic/tests/test_magic_event_source.py index 5ebcb68..749ddc3 100644 --- a/ctapipe_io_magic/tests/test_magic_event_source.py +++ b/ctapipe_io_magic/tests/test_magic_event_source.py @@ -4,21 +4,25 @@ import pytest -test_data = Path(os.getenv('MAGIC_TEST_DATA', 'test_data')).absolute() -test_calibrated_real_dir = test_data / 'real/calibrated' +test_data = Path(os.getenv("MAGIC_TEST_DATA", "test_data")).absolute() +test_calibrated_real_dir = test_data / "real/calibrated" test_calibrated_real = [ - test_calibrated_real_dir / '20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root', - test_calibrated_real_dir / '20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root', - test_calibrated_real_dir / '20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root', - test_calibrated_real_dir / '20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root', + test_calibrated_real_dir + / "20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root", + test_calibrated_real_dir + / "20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root", + test_calibrated_real_dir + / "20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root", + test_calibrated_real_dir + / "20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root", ] -test_calibrated_simulated_dir = test_data / 'simulated/calibrated' +test_calibrated_simulated_dir = test_data / "simulated/calibrated" test_calibrated_simulated = [ - test_calibrated_simulated_dir / 'GA_M1_za35to50_8_824318_Y_w0.root', - test_calibrated_simulated_dir / 'GA_M1_za35to50_8_824319_Y_w0.root', - test_calibrated_simulated_dir / 'GA_M2_za35to50_8_824318_Y_w0.root', - test_calibrated_simulated_dir / 'GA_M2_za35to50_8_824319_Y_w0.root', + test_calibrated_simulated_dir / "GA_M1_za35to50_8_824318_Y_w0.root", + test_calibrated_simulated_dir / "GA_M1_za35to50_8_824319_Y_w0.root", + test_calibrated_simulated_dir / "GA_M2_za35to50_8_824318_Y_w0.root", + test_calibrated_simulated_dir / "GA_M2_za35to50_8_824319_Y_w0.root", ] test_superstar_real_dir = test_data / "real/superstar" @@ -45,7 +49,7 @@ test_melibea_simulated_dir / "GA_za35to50_8_824319_Q_w0.root", ] -test_calibrated_all = test_calibrated_real+test_calibrated_simulated +test_calibrated_all = test_calibrated_real + test_calibrated_simulated test_superstar_all = test_superstar_mars_real + test_superstar_mars_simulated @@ -53,93 +57,143 @@ data_dict = dict() -data_dict['20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root'] = dict() -data_dict['20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root'] = dict() -data_dict['20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root'] = dict() -data_dict['20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root'] = dict() -data_dict['20210314_05095172_S_CrabNebula-W0.40+035.root'] = dict() -data_dict['20210314_05095172_Q_CrabNebula-W0.40+035.root'] = dict() -data_dict['GA_M1_za35to50_8_824318_Y_w0.root'] = dict() -data_dict['GA_M1_za35to50_8_824319_Y_w0.root'] = dict() -data_dict['GA_M2_za35to50_8_824318_Y_w0.root'] = dict() -data_dict['GA_M2_za35to50_8_824319_Y_w0.root'] = dict() -data_dict['GA_za35to50_8_824318_S_w0.root'] = dict() -data_dict['GA_za35to50_8_824319_S_w0.root'] = dict() -data_dict['GA_za35to50_8_824318_Q_w0.root'] = dict() -data_dict['GA_za35to50_8_824319_Q_w0.root'] = dict() - -data_dict['20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root']['n_events_tot'] = 500 -data_dict['20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root']['n_events_stereo'] = 458 -data_dict['20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root']['n_events_pedestal'] = 42 -data_dict['20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root']['n_events_mc_mono'] = 0 - -data_dict['20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root']['n_events_tot'] = 500 -data_dict['20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root']['n_events_stereo'] = 452 -data_dict['20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root']['n_events_pedestal'] = 48 -data_dict['20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root']['n_events_mc_mono'] = 0 - -data_dict['20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root']['n_events_tot'] = 500 -data_dict['20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root']['n_events_stereo'] = 459 -data_dict['20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root']['n_events_pedestal'] = 41 -data_dict['20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root']['n_events_mc_mono'] = 0 - -data_dict['20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root']['n_events_tot'] = 500 -data_dict['20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root']['n_events_stereo'] = 450 -data_dict['20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root']['n_events_pedestal'] = 50 -data_dict['20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root']['n_events_mc_mono'] = 0 - -data_dict['GA_M1_za35to50_8_824318_Y_w0.root']['n_events_tot'] = 99 -data_dict['GA_M1_za35to50_8_824318_Y_w0.root']['n_events_stereo'] = 67 -data_dict['GA_M1_za35to50_8_824318_Y_w0.root']['n_events_pedestal'] = 0 -data_dict['GA_M1_za35to50_8_824318_Y_w0.root']['n_events_mc_mono'] = 32 - -data_dict['GA_M1_za35to50_8_824319_Y_w0.root']['n_events_tot'] = 111 -data_dict['GA_M1_za35to50_8_824319_Y_w0.root']['n_events_stereo'] = 80 -data_dict['GA_M1_za35to50_8_824319_Y_w0.root']['n_events_pedestal'] = 0 -data_dict['GA_M1_za35to50_8_824319_Y_w0.root']['n_events_mc_mono'] = 31 - -data_dict['GA_M2_za35to50_8_824318_Y_w0.root']['n_events_tot'] = 118 -data_dict['GA_M2_za35to50_8_824318_Y_w0.root']['n_events_stereo'] = 67 -data_dict['GA_M2_za35to50_8_824318_Y_w0.root']['n_events_pedestal'] = 0 -data_dict['GA_M2_za35to50_8_824318_Y_w0.root']['n_events_mc_mono'] = 51 - -data_dict['GA_M2_za35to50_8_824319_Y_w0.root']['n_events_tot'] = 132 -data_dict['GA_M2_za35to50_8_824319_Y_w0.root']['n_events_stereo'] = 80 -data_dict['GA_M2_za35to50_8_824319_Y_w0.root']['n_events_pedestal'] = 0 -data_dict['GA_M2_za35to50_8_824319_Y_w0.root']['n_events_mc_mono'] = 52 - -data_dict['20210314_05095172_S_CrabNebula-W0.40+035.root']['n_events_tot'] = 356 -data_dict['20210314_05095172_S_CrabNebula-W0.40+035.root']['n_events_stereo'] = 356 -data_dict['20210314_05095172_S_CrabNebula-W0.40+035.root']['n_events_pedestal'] = 0 -data_dict['20210314_05095172_S_CrabNebula-W0.40+035.root']['n_events_mc_mono'] = 0 - -data_dict['GA_za35to50_8_824318_S_w0.root']['n_events_tot'] = 53 -data_dict['GA_za35to50_8_824318_S_w0.root']['n_events_stereo'] = 53 -data_dict['GA_za35to50_8_824318_S_w0.root']['n_events_pedestal'] = 0 -data_dict['GA_za35to50_8_824318_S_w0.root']['n_events_mc_mono'] = 0 - -data_dict['GA_za35to50_8_824319_S_w0.root']['n_events_tot'] = 67 -data_dict['GA_za35to50_8_824319_S_w0.root']['n_events_stereo'] = 67 -data_dict['GA_za35to50_8_824319_S_w0.root']['n_events_pedestal'] = 0 -data_dict['GA_za35to50_8_824319_S_w0.root']['n_events_mc_mono'] = 0 - -data_dict['20210314_05095172_Q_CrabNebula-W0.40+035.root']['n_events_tot'] = 356 -data_dict['20210314_05095172_Q_CrabNebula-W0.40+035.root']['n_events_stereo'] = 356 -data_dict['20210314_05095172_Q_CrabNebula-W0.40+035.root']['n_events_pedestal'] = 0 -data_dict['20210314_05095172_Q_CrabNebula-W0.40+035.root']['n_events_mc_mono'] = 0 - -data_dict['GA_za35to50_8_824318_Q_w0.root']['n_events_tot'] = 53 -data_dict['GA_za35to50_8_824318_Q_w0.root']['n_events_stereo'] = 53 -data_dict['GA_za35to50_8_824318_Q_w0.root']['n_events_pedestal'] = 0 -data_dict['GA_za35to50_8_824318_Q_w0.root']['n_events_mc_mono'] = 0 - -data_dict['GA_za35to50_8_824319_Q_w0.root']['n_events_tot'] = 67 -data_dict['GA_za35to50_8_824319_Q_w0.root']['n_events_stereo'] = 67 -data_dict['GA_za35to50_8_824319_Q_w0.root']['n_events_pedestal'] = 0 -data_dict['GA_za35to50_8_824319_Q_w0.root']['n_events_mc_mono'] = 0 - - -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) +data_dict["20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root"] = dict() +data_dict["20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root"] = dict() +data_dict["20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root"] = dict() +data_dict["20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root"] = dict() +data_dict["20210314_05095172_S_CrabNebula-W0.40+035.root"] = dict() +data_dict["20210314_05095172_Q_CrabNebula-W0.40+035.root"] = dict() +data_dict["GA_M1_za35to50_8_824318_Y_w0.root"] = dict() +data_dict["GA_M1_za35to50_8_824319_Y_w0.root"] = dict() +data_dict["GA_M2_za35to50_8_824318_Y_w0.root"] = dict() +data_dict["GA_M2_za35to50_8_824319_Y_w0.root"] = dict() +data_dict["GA_za35to50_8_824318_S_w0.root"] = dict() +data_dict["GA_za35to50_8_824319_S_w0.root"] = dict() +data_dict["GA_za35to50_8_824318_Q_w0.root"] = dict() +data_dict["GA_za35to50_8_824319_Q_w0.root"] = dict() + +data_dict["20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root"][ + "n_events_tot" +] = 500 +data_dict["20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root"][ + "n_events_stereo" +] = 458 +data_dict["20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root"][ + "n_events_pedestal" +] = 42 +data_dict["20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root"][ + "n_events_mc_mono" +] = 0 + +data_dict["20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root"][ + "n_events_tot" +] = 500 +data_dict["20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root"][ + "n_events_stereo" +] = 452 +data_dict["20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root"][ + "n_events_pedestal" +] = 48 +data_dict["20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root"][ + "n_events_mc_mono" +] = 0 + +data_dict["20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root"][ + "n_events_tot" +] = 500 +data_dict["20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root"][ + "n_events_stereo" +] = 459 +data_dict["20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root"][ + "n_events_pedestal" +] = 41 +data_dict["20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root"][ + "n_events_mc_mono" +] = 0 + +data_dict["20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root"][ + "n_events_tot" +] = 500 +data_dict["20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root"][ + "n_events_stereo" +] = 450 +data_dict["20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root"][ + "n_events_pedestal" +] = 50 +data_dict["20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root"][ + "n_events_mc_mono" +] = 0 + +data_dict["GA_M1_za35to50_8_824318_Y_w0.root"]["n_events_tot"] = 99 +data_dict["GA_M1_za35to50_8_824318_Y_w0.root"]["n_events_stereo"] = 67 +data_dict["GA_M1_za35to50_8_824318_Y_w0.root"]["n_events_pedestal"] = 0 +data_dict["GA_M1_za35to50_8_824318_Y_w0.root"]["n_events_mc_mono"] = 32 + +data_dict["GA_M1_za35to50_8_824319_Y_w0.root"]["n_events_tot"] = 111 +data_dict["GA_M1_za35to50_8_824319_Y_w0.root"]["n_events_stereo"] = 80 +data_dict["GA_M1_za35to50_8_824319_Y_w0.root"]["n_events_pedestal"] = 0 +data_dict["GA_M1_za35to50_8_824319_Y_w0.root"]["n_events_mc_mono"] = 31 + +data_dict["GA_M2_za35to50_8_824318_Y_w0.root"]["n_events_tot"] = 118 +data_dict["GA_M2_za35to50_8_824318_Y_w0.root"]["n_events_stereo"] = 67 +data_dict["GA_M2_za35to50_8_824318_Y_w0.root"]["n_events_pedestal"] = 0 +data_dict["GA_M2_za35to50_8_824318_Y_w0.root"]["n_events_mc_mono"] = 51 + +data_dict["GA_M2_za35to50_8_824319_Y_w0.root"]["n_events_tot"] = 132 +data_dict["GA_M2_za35to50_8_824319_Y_w0.root"]["n_events_stereo"] = 80 +data_dict["GA_M2_za35to50_8_824319_Y_w0.root"]["n_events_pedestal"] = 0 +data_dict["GA_M2_za35to50_8_824319_Y_w0.root"]["n_events_mc_mono"] = 52 + +data_dict["20210314_05095172_S_CrabNebula-W0.40+035.root"][ + "n_events_tot" +] = 356 +data_dict["20210314_05095172_S_CrabNebula-W0.40+035.root"][ + "n_events_stereo" +] = 356 +data_dict["20210314_05095172_S_CrabNebula-W0.40+035.root"][ + "n_events_pedestal" +] = 0 +data_dict["20210314_05095172_S_CrabNebula-W0.40+035.root"][ + "n_events_mc_mono" +] = 0 + +data_dict["GA_za35to50_8_824318_S_w0.root"]["n_events_tot"] = 53 +data_dict["GA_za35to50_8_824318_S_w0.root"]["n_events_stereo"] = 53 +data_dict["GA_za35to50_8_824318_S_w0.root"]["n_events_pedestal"] = 0 +data_dict["GA_za35to50_8_824318_S_w0.root"]["n_events_mc_mono"] = 0 + +data_dict["GA_za35to50_8_824319_S_w0.root"]["n_events_tot"] = 67 +data_dict["GA_za35to50_8_824319_S_w0.root"]["n_events_stereo"] = 67 +data_dict["GA_za35to50_8_824319_S_w0.root"]["n_events_pedestal"] = 0 +data_dict["GA_za35to50_8_824319_S_w0.root"]["n_events_mc_mono"] = 0 + +data_dict["20210314_05095172_Q_CrabNebula-W0.40+035.root"][ + "n_events_tot" +] = 356 +data_dict["20210314_05095172_Q_CrabNebula-W0.40+035.root"][ + "n_events_stereo" +] = 356 +data_dict["20210314_05095172_Q_CrabNebula-W0.40+035.root"][ + "n_events_pedestal" +] = 0 +data_dict["20210314_05095172_Q_CrabNebula-W0.40+035.root"][ + "n_events_mc_mono" +] = 0 + +data_dict["GA_za35to50_8_824318_Q_w0.root"]["n_events_tot"] = 53 +data_dict["GA_za35to50_8_824318_Q_w0.root"]["n_events_stereo"] = 53 +data_dict["GA_za35to50_8_824318_Q_w0.root"]["n_events_pedestal"] = 0 +data_dict["GA_za35to50_8_824318_Q_w0.root"]["n_events_mc_mono"] = 0 + +data_dict["GA_za35to50_8_824319_Q_w0.root"]["n_events_tot"] = 67 +data_dict["GA_za35to50_8_824319_Q_w0.root"]["n_events_stereo"] = 67 +data_dict["GA_za35to50_8_824319_Q_w0.root"]["n_events_pedestal"] = 0 +data_dict["GA_za35to50_8_824319_Q_w0.root"]["n_events_mc_mono"] = 0 + + +@pytest.mark.parametrize( + "dataset", test_calibrated_all + test_superstar_all + test_melibea_all +) def test_event_source_for_magic_file(dataset): from ctapipe.io import EventSource @@ -152,38 +206,57 @@ def test_event_source_for_magic_file(dataset): assert reader.input_url == dataset -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) +@pytest.mark.parametrize( + "dataset", test_calibrated_all + test_superstar_all + test_melibea_all +) def test_compatible(dataset): from ctapipe_io_magic import MAGICEventSource + assert MAGICEventSource.is_compatible(dataset) def test_not_compatible(): from ctapipe_io_magic import MAGICEventSource + assert MAGICEventSource.is_compatible(None) is False -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) +@pytest.mark.parametrize( + "dataset", test_calibrated_all + test_superstar_all + test_melibea_all +) def test_stream(dataset): from ctapipe_io_magic import MAGICEventSource + with MAGICEventSource(input_url=dataset, process_run=False) as source: assert not source.is_stream def test_allowed_tels(): - from ctapipe_io_magic import MAGICEventSource import numpy as np - dataset = test_calibrated_real_dir / '20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root' - with MAGICEventSource(input_url=dataset, process_run=False, allowed_tels=[1]) as source: + + from ctapipe_io_magic import MAGICEventSource + + dataset = ( + test_calibrated_real_dir + / "20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root" + ) + with MAGICEventSource( + input_url=dataset, process_run=False, allowed_tels=[1] + ) as source: assert np.array_equal(source.subarray.tel_ids, np.array([1])) -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) +@pytest.mark.parametrize( + "dataset", test_calibrated_all + test_superstar_all + test_melibea_all +) def test_loop(dataset): - from ctapipe_io_magic import MAGICEventSource, MARSDataLevel import numpy as np + from ctapipe_io_magic import MAGICEventSource, MARSDataLevel + n_events = 10 - with MAGICEventSource(input_url=dataset, max_events=n_events, process_run=False) as source: + with MAGICEventSource( + input_url=dataset, max_events=n_events, process_run=False + ) as source: for i, event in enumerate(source): assert event.count == i if source.mars_datalevel <= MARSDataLevel.STAR: @@ -192,7 +265,9 @@ def test_loop(dataset): if "_M2_" in dataset.name: assert event.trigger.tels_with_trigger == [2] else: - assert np.array_equal(event.trigger.tels_with_trigger, np.array([1, 2])) + assert np.array_equal( + event.trigger.tels_with_trigger, np.array([1, 2]) + ) assert (i + 1) == n_events @@ -202,39 +277,52 @@ def test_loop(dataset): break -@pytest.mark.parametrize('dataset', test_calibrated_real) +@pytest.mark.parametrize("dataset", test_calibrated_real) def test_loop_pedestal(dataset): - from ctapipe_io_magic import MAGICEventSource from ctapipe.containers import EventType + + from ctapipe_io_magic import MAGICEventSource + n_events = 10 - with MAGICEventSource(input_url=dataset, max_events=n_events, use_pedestals=True, process_run=False) as source: + with MAGICEventSource( + input_url=dataset, + max_events=n_events, + use_pedestals=True, + process_run=False, + ) as source: for event in source: assert event.trigger.event_type == EventType.SKY_PEDESTAL -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) +@pytest.mark.parametrize( + "dataset", test_calibrated_all + test_superstar_all + test_melibea_all +) def test_number_of_events(dataset): from ctapipe_io_magic import MAGICEventSource with MAGICEventSource(input_url=dataset, process_run=False) as source: run = source._set_active_run(source.files_[0]) - assert run['data'].n_cosmic_events == data_dict[source.input_url.name]['n_events_stereo'] - assert run['data'].n_pedestal_events == data_dict[source.input_url.name]['n_events_pedestal'] - - # if '_M1_' in dataset.name: - # assert run['data'].n_cosmics_stereo_events_m1 == data_dict[source.input_url.name]['n_events_stereo'] - # assert run['data'].n_pedestal_events_m1 == data_dict[source.input_url.name]['n_events_pedestal'] - # if '_M2_' in dataset.name: - # assert run['data'].n_cosmics_stereo_events_m2 == data_dict[source.input_url.name]['n_events_stereo'] - # assert run['data'].n_pedestal_events_m2 == data_dict[source.input_url.name]['n_events_pedestal'] - - -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) + assert ( + run["data"].n_cosmic_events + == data_dict[source.input_url.name]["n_events_stereo"] + ) + assert ( + run["data"].n_pedestal_events + == data_dict[source.input_url.name]["n_events_pedestal"] + ) + + +@pytest.mark.parametrize( + "dataset", test_calibrated_all + test_superstar_all + test_melibea_all +) def test_run_info(dataset): from ctapipe_io_magic import MAGICEventSource with MAGICEventSource(input_url=dataset, process_run=False) as source: - run_info = [MAGICEventSource.get_run_info_from_name(item.name) for item in source.file_list] + run_info = [ + MAGICEventSource.get_run_info_from_name(item.name) + for item in source.file_list + ] run_numbers = [i[0] for i in run_info] is_mc = [i[1] for i in run_info][0] telescope = [i[2] for i in run_info][0] @@ -246,13 +334,19 @@ def test_run_info(dataset): def test_multiple_runs_real(): - from ctapipe_io_magic import MAGICEventSource from ctapipe.containers import EventType - real_data_mask = test_calibrated_real_dir / '20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root' + from ctapipe_io_magic import MAGICEventSource + + real_data_mask = ( + test_calibrated_real_dir + / "20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root" + ) n_events = 600 - with MAGICEventSource(input_url=real_data_mask, max_events=n_events) as source: + with MAGICEventSource( + input_url=real_data_mask, max_events=n_events + ) as source: for i, event in enumerate(source): assert event.trigger.event_type == EventType.SUBARRAY assert event.count == i @@ -269,18 +363,25 @@ def test_multiple_runs_real(): def test_subarray_multiple_runs(): from ctapipe_io_magic import MAGICEventSource - simulated_data_mask = test_calibrated_simulated_dir / 'GA_M1_za35to50_8_824318_Y_w0.root' + simulated_data_mask = ( + test_calibrated_simulated_dir / "GA_M1_za35to50_8_824318_Y_w0.root" + ) source = MAGICEventSource(input_url=simulated_data_mask) sim_config = source.simulation_config assert list(sim_config.keys()) == source.obs_ids -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) +@pytest.mark.parametrize( + "dataset", test_calibrated_all + test_superstar_all + test_melibea_all +) def test_that_event_is_not_modified_after_loop(dataset): from ctapipe_io_magic import MAGICEventSource + n_events = 10 - with MAGICEventSource(input_url=dataset, max_events=n_events, process_run=False) as source: + with MAGICEventSource( + input_url=dataset, max_events=n_events, process_run=False + ) as source: for event in source: last_event = copy.deepcopy(event) @@ -292,7 +393,9 @@ def test_that_event_is_not_modified_after_loop(dataset): assert event.index.event_id == last_event.index.event_id -@pytest.mark.parametrize('dataset', test_calibrated_all+test_superstar_all+test_melibea_all) +@pytest.mark.parametrize( + "dataset", test_calibrated_all + test_superstar_all + test_melibea_all +) def test_geom(dataset): from ctapipe_io_magic import MAGICEventSource @@ -302,7 +405,9 @@ def test_geom(dataset): # def test_eventseeker(): -# dataset = get_dataset_path("20131004_M1_05029747.003_Y_MagicCrab-W0.40+035.root") +# dataset = get_dataset_path( +# "20131004_M1_05029747.003_Y_MagicCrab-W0.40+035.root" +# ) # # with MAGICEventSource(input_url=dataset) as source: # seeker = EventSeeker(source) @@ -315,7 +420,9 @@ def test_geom(dataset): # assert event.index.event_id == 29798 # # def test_eventcontent(): -# dataset = get_dataset_path("20131004_M1_05029747.003_Y_MagicCrab-W0.40+035.root") +# dataset = get_dataset_path( +# "20131004_M1_05029747.003_Y_MagicCrab-W0.40+035.root" +# ) # # with MAGICEventSource(input_url=dataset) as source: # seeker = EventSeeker(source) diff --git a/ctapipe_io_magic/tests/test_stage1.py b/ctapipe_io_magic/tests/test_stage1.py index f75a42b..92a9bf9 100644 --- a/ctapipe_io_magic/tests/test_stage1.py +++ b/ctapipe_io_magic/tests/test_stage1.py @@ -1,37 +1,46 @@ -from pathlib import Path +import numpy as np import os -from ctapipe.io import read_table from ctapipe.containers import EventType -import numpy as np +from ctapipe.io import read_table +from pathlib import Path -test_data = Path(os.getenv('MAGIC_TEST_DATA', 'test_data')).absolute() -test_cal_path = test_data / 'real/calibrated/20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root' +test_data = Path(os.getenv("MAGIC_TEST_DATA", "test_data")).absolute() +test_cal_path = ( + test_data + / "real/calibrated/20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root" +) config = Path(".").absolute() / "example_stage1_config.json" def test_stage1_multiple_runs(): """Test the ctapipe process tool can read in MAGIC real data using the event source""" - from ctapipe.tools.process import ProcessorTool from ctapipe.core import run_tool + from ctapipe.tools.process import ProcessorTool tool = ProcessorTool() - output = test_data / 'real/calibrated/20210314_M1_05095172_Y_CrabNebula-W0.40+035.h5' - - ret = run_tool(tool, argv=[ - f'--input={test_cal_path}', - f'--output={output}', - f'--config={str(config)}', - "--camera-frame", - ]) + output = ( + test_data + / "real/calibrated/20210314_M1_05095172_Y_CrabNebula-W0.40+035.h5" + ) + + ret = run_tool( + tool, + argv=[ + f"--input={test_cal_path}", + f"--output={output}", + f"--config={str(config)}", + "--camera-frame", + ], + ) assert ret == 0 - parameters = read_table(output, '/dl1/event/telescope/parameters/tel_001') + parameters = read_table(output, "/dl1/event/telescope/parameters/tel_001") assert len(parameters) == 910 - trigger = read_table(output, '/dl1/event/subarray/trigger') + trigger = read_table(output, "/dl1/event/subarray/trigger") - event_type_counts = np.bincount(trigger['event_type']) + event_type_counts = np.bincount(trigger["event_type"]) # no pedestals expected, should be only physics data assert event_type_counts.sum() == 910 @@ -42,28 +51,31 @@ def test_stage1_multiple_runs(): def test_stage1_single_run(): """Test the ctapipe process tool can read in MAGIC real data using the event source""" - from ctapipe.tools.process import ProcessorTool from ctapipe.core import run_tool + from ctapipe.tools.process import ProcessorTool tool = ProcessorTool() - output = test_cal_path.with_suffix('.h5') - - ret = run_tool(tool, argv=[ - f'--input={test_cal_path}', - f'--output={output}', - f'--config={str(config)}', - "--MAGICEventSource.process_run=false", - "--allowed-tels=1", - "--camera-frame", - ]) + output = test_cal_path.with_suffix(".h5") + + ret = run_tool( + tool, + argv=[ + f"--input={test_cal_path}", + f"--output={output}", + f"--config={str(config)}", + "--MAGICEventSource.process_run=false", + "--allowed-tels=1", + "--camera-frame", + ], + ) assert ret == 0 - parameters = read_table(output, '/dl1/event/telescope/parameters/tel_001') + parameters = read_table(output, "/dl1/event/telescope/parameters/tel_001") assert len(parameters) == 458 - trigger = read_table(output, '/dl1/event/subarray/trigger') + trigger = read_table(output, "/dl1/event/subarray/trigger") - event_type_counts = np.bincount(trigger['event_type']) + event_type_counts = np.bincount(trigger["event_type"]) # no pedestals expected, should be only physics data assert event_type_counts.sum() == 458 diff --git a/ctapipe_io_magic/tests/test_version.py b/ctapipe_io_magic/tests/test_version.py index 8f0478f..67917bd 100644 --- a/ctapipe_io_magic/tests/test_version.py +++ b/ctapipe_io_magic/tests/test_version.py @@ -1,4 +1,4 @@ def test_version(): from ctapipe_io_magic import __version__ - assert __version__ != 'unknown' + assert __version__ != "unknown" From f09b89216bd71fe480bdccff5e693a552722e4ec Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 21:42:24 +0200 Subject: [PATCH 51/78] Fix imports with isort. --- ctapipe_io_magic/__init__.py | 61 +++++++++++------------------ ctapipe_io_magic/mars_datalevels.py | 5 +-- 2 files changed, 24 insertions(+), 42 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 75845d8..89f2395 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -3,57 +3,42 @@ # Requires uproot package (https://github.com/scikit-hep/uproot). """ +import logging import os import re +from decimal import Decimal +from pathlib import Path + +import numpy as np import scipy import scipy.interpolate import uproot -import logging -import numpy as np -from pathlib import Path -from decimal import Decimal from astropy import units as u from astropy.time import Time -from pkg_resources import resource_filename - -from ctapipe.io import EventSource, DataLevel +from ctapipe.containers import (ArrayEventContainer, + CameraHillasParametersContainer, EventType, + ImageParametersContainer, LeakageContainer, + MorphologyContainer, + ParticleClassificationContainer, + ReconstructedEnergyContainer, + ReconstructedGeometryContainer, + SimulatedEventContainer, + SimulationConfigContainer, + TimingParametersContainer) +from ctapipe.coordinates import CameraFrame from ctapipe.core import Provenance from ctapipe.core.traits import Bool -from ctapipe.coordinates import CameraFrame - -from ctapipe.containers import ( - EventType, - ArrayEventContainer, - SimulationConfigContainer, - SimulatedEventContainer, - ImageParametersContainer, - CameraHillasParametersContainer, - TimingParametersContainer, - LeakageContainer, - MorphologyContainer, - ReconstructedGeometryContainer, - ReconstructedEnergyContainer, - ParticleClassificationContainer, -) - -from ctapipe.instrument import ( - TelescopeDescription, - SubarrayDescription, - OpticsDescription, - CameraDescription, - CameraGeometry, - CameraReadout, -) +from ctapipe.instrument import (CameraDescription, CameraGeometry, + CameraReadout, OpticsDescription, + SubarrayDescription, TelescopeDescription) +from ctapipe.io import DataLevel, EventSource +from pkg_resources import resource_filename +from .constants import (DATA_STEREO_TRIGGER_PATTERN, MC_STEREO_TRIGGER_PATTERN, + PEDESTAL_TRIGGER_PATTERN) from .mars_datalevels import MARSDataLevel from .version import __version__ -from .constants import ( - MC_STEREO_TRIGGER_PATTERN, - PEDESTAL_TRIGGER_PATTERN, - DATA_STEREO_TRIGGER_PATTERN, -) - __all__ = [ 'MAGICEventSource', 'MARSDataLevel', diff --git a/ctapipe_io_magic/mars_datalevels.py b/ctapipe_io_magic/mars_datalevels.py index 4c80ff3..0b7a96b 100644 --- a/ctapipe_io_magic/mars_datalevels.py +++ b/ctapipe_io_magic/mars_datalevels.py @@ -1,7 +1,4 @@ -from enum import ( - Enum, - auto, -) +from enum import Enum, auto class OrderedEnum(Enum): From 454c2511998dfe9e852e7d16ca4c182cae56ea02 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 21:44:44 +0200 Subject: [PATCH 52/78] Run black on __init__.py --- ctapipe_io_magic/__init__.py | 2034 +++++++++++++++++++++++----------- 1 file changed, 1358 insertions(+), 676 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 89f2395..485c609 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -15,47 +15,55 @@ import uproot from astropy import units as u from astropy.time import Time -from ctapipe.containers import (ArrayEventContainer, - CameraHillasParametersContainer, EventType, - ImageParametersContainer, LeakageContainer, - MorphologyContainer, - ParticleClassificationContainer, - ReconstructedEnergyContainer, - ReconstructedGeometryContainer, - SimulatedEventContainer, - SimulationConfigContainer, - TimingParametersContainer) +from ctapipe.containers import ( + ArrayEventContainer, + CameraHillasParametersContainer, + EventType, + ImageParametersContainer, + LeakageContainer, + MorphologyContainer, + ParticleClassificationContainer, + ReconstructedEnergyContainer, + ReconstructedGeometryContainer, + SimulatedEventContainer, + SimulationConfigContainer, + TimingParametersContainer, +) from ctapipe.coordinates import CameraFrame from ctapipe.core import Provenance from ctapipe.core.traits import Bool -from ctapipe.instrument import (CameraDescription, CameraGeometry, - CameraReadout, OpticsDescription, - SubarrayDescription, TelescopeDescription) +from ctapipe.instrument import ( + CameraDescription, + CameraGeometry, + CameraReadout, + OpticsDescription, + SubarrayDescription, + TelescopeDescription, +) from ctapipe.io import DataLevel, EventSource from pkg_resources import resource_filename -from .constants import (DATA_STEREO_TRIGGER_PATTERN, MC_STEREO_TRIGGER_PATTERN, - PEDESTAL_TRIGGER_PATTERN) +from .constants import ( + DATA_STEREO_TRIGGER_PATTERN, + MC_STEREO_TRIGGER_PATTERN, + PEDESTAL_TRIGGER_PATTERN, +) from .mars_datalevels import MARSDataLevel from .version import __version__ -__all__ = [ - 'MAGICEventSource', - 'MARSDataLevel', - '__version__' -] +__all__ = ["MAGICEventSource", "MARSDataLevel", "__version__"] logger = logging.getLogger(__name__) degrees_per_hour = 15.0 -seconds_per_hour = 3600. +seconds_per_hour = 3600.0 msec2sec = 1e-3 nsec2sec = 1e-9 mc_data_type = 256 -m2deg = 180./(17.*np.pi) +m2deg = 180.0 / (17.0 * np.pi) MAGIC_TO_CTA_EVENT_TYPE = { MC_STEREO_TRIGGER_PATTERN: EventType.SUBARRAY, @@ -64,18 +72,18 @@ } OPTICS = OpticsDescription( - 'MAGIC', + "MAGIC", num_mirrors=1, - equivalent_focal_length=u.Quantity(17.*1.0713, u.m), - mirror_area=u.Quantity(239.0, u.m**2), + equivalent_focal_length=u.Quantity(17.0 * 1.0713, u.m), + mirror_area=u.Quantity(239.0, u.m ** 2), num_mirror_tiles=964, ) def load_camera_geometry(): - ''' Load camera geometry from bundled resources of this repo ''' + """Load camera geometry from bundled resources of this repo""" f = resource_filename( - 'ctapipe_io_magic', 'resources/MAGICCam.camgeom.fits.gz' + "ctapipe_io_magic", "resources/MAGICCam.camgeom.fits.gz" ) Provenance().add_input_file(f, role="CameraGeometry") return CameraGeometry.from_table(f) @@ -120,13 +128,12 @@ class MAGICEventSource(EventSource): """ process_run = Bool( - default_value=True, - help='Read all subruns from a given run.' + default_value=True, help="Read all subruns from a given run." ).tag(config=True) use_pedestals = Bool( - default_value=False, - help='Extract pedestal events instead of cosmic events.' + default_value=False, + help="Extract pedestal events instead of cosmic events.", ).tag(config=True) def __init__(self, input_url=None, config=None, parent=None, **kwargs): @@ -150,10 +157,7 @@ def __init__(self, input_url=None, config=None, parent=None, **kwargs): """ super().__init__( - input_url=input_url, - config=config, - parent=parent, - **kwargs + input_url=input_url, config=config, parent=parent, **kwargs ) path, name = os.path.split(os.path.abspath(self.input_url)) @@ -184,7 +188,10 @@ def __init__(self, input_url=None, config=None, parent=None, **kwargs): self.file_list_drive = [] for file_path in ls: - if reg_comp.match(file_path.name) is not None or reg_comp_mc.match(file_path.name) is not None: + if ( + reg_comp.match(file_path.name) is not None + or reg_comp_mc.match(file_path.name) is not None + ): self.file_list_drive.append(file_path) self.file_list_drive.sort() @@ -273,8 +280,10 @@ def is_compatible(file_path): try: with uproot.open(file_path) as input_data: - mandatory_trees = ['Events', 'RunHeaders', 'RunTails'] - trees_in_file = [tree in input_data for tree in mandatory_trees] + mandatory_trees = ["Events", "RunHeaders", "RunTails"] + trees_in_file = [ + tree in input_data for tree in mandatory_trees + ] if not all(trees_in_file): is_magic_root_file = False except ValueError: @@ -370,8 +379,9 @@ def get_run_info_from_name(file_name): is_mc = True else: raise IndexError( - 'Can not identify the run number and type (data/MC) of the file' - ' {:s}'.format(file_name)) + "Can not identify the run number and type (data/MC) of the file" + " {:s}".format(file_name) + ) return run_number, is_mc, telescope, datalevel @@ -392,18 +402,18 @@ def parse_run_info(self): """ runinfo_array_list = [ - 'MRawRunHeader.fRunNumber', - 'MRawRunHeader.fRunType', - 'MRawRunHeader.fTelescopeNumber', + "MRawRunHeader.fRunNumber", + "MRawRunHeader.fRunType", + "MRawRunHeader.fTelescopeNumber", ] runinfo_stereo_array_list = [ - 'MRawRunHeader_1.fRunNumber', - 'MRawRunHeader_1.fRunType', - 'MRawRunHeader_1.fTelescopeNumber', - 'MRawRunHeader_2.fRunNumber', - 'MRawRunHeader_2.fRunType', - 'MRawRunHeader_2.fTelescopeNumber', + "MRawRunHeader_1.fRunNumber", + "MRawRunHeader_1.fRunType", + "MRawRunHeader_1.fTelescopeNumber", + "MRawRunHeader_2.fRunNumber", + "MRawRunHeader_2.fRunType", + "MRawRunHeader_2.fTelescopeNumber", ] run_numbers = [] @@ -413,11 +423,11 @@ def parse_run_info(self): for rootf in self.files_: - events_tree = rootf['Events'] + events_tree = rootf["Events"] - melibea_trees = ['MHadronness.', 'MStereoParDisp.', 'MEnergyEst.'] - superstar_trees = ['MHillas_1.', 'MHillas_2.', 'MStereoPar.'] - star_trees = ['MHillas.'] + melibea_trees = ["MHadronness.", "MStereoParDisp.", "MEnergyEst."] + superstar_trees = ["MHillas_1.", "MHillas_2.", "MStereoPar."] + star_trees = ["MHillas."] datalevel = MARSDataLevel.CALIBRATED events_keys = events_tree.keys() @@ -432,20 +442,28 @@ def parse_run_info(self): datalevel = MARSDataLevel.MELIBEA if datalevel <= MARSDataLevel.STAR: - run_info = rootf['RunHeaders'].arrays( - runinfo_array_list, library="np") + run_info = rootf["RunHeaders"].arrays( + runinfo_array_list, library="np" + ) - run_number = int(run_info['MRawRunHeader.fRunNumber'][0]) - run_type = int(run_info['MRawRunHeader.fRunType'][0]) - telescope_numbers.append(int(run_info['MRawRunHeader.fTelescopeNumber'][0])) + run_number = int(run_info["MRawRunHeader.fRunNumber"][0]) + run_type = int(run_info["MRawRunHeader.fRunType"][0]) + telescope_numbers.append( + int(run_info["MRawRunHeader.fTelescopeNumber"][0]) + ) else: - run_info = rootf['RunHeaders'].arrays( - runinfo_stereo_array_list, library="np") + run_info = rootf["RunHeaders"].arrays( + runinfo_stereo_array_list, library="np" + ) - run_number = int(run_info['MRawRunHeader_1.fRunNumber'][0]) - run_type = int(run_info['MRawRunHeader_1.fRunType'][0]) - telescope_numbers.append(int(run_info['MRawRunHeader_1.fTelescopeNumber'][0])) - telescope_numbers.append(int(run_info['MRawRunHeader_2.fTelescopeNumber'][0])) + run_number = int(run_info["MRawRunHeader_1.fRunNumber"][0]) + run_type = int(run_info["MRawRunHeader_1.fRunType"][0]) + telescope_numbers.append( + int(run_info["MRawRunHeader_1.fTelescopeNumber"][0]) + ) + telescope_numbers.append( + int(run_info["MRawRunHeader_2.fTelescopeNumber"][0]) + ) # a note about run numbers: # mono data has run numbers starting with 1 or 2 (telescope dependent) @@ -480,15 +498,18 @@ def parse_run_info(self): if len(is_simulation) > 1: raise ValueError( "Loaded files contain both real data and simulation runs. \ - Please load only data OR Monte Carlos.") + Please load only data OR Monte Carlos." + ) if len(datalevels) > 1: raise ValueError( "Loaded files contain data at different datalevels. \ - Please load data belonging to the same datalevel.") + Please load data belonging to the same datalevel." + ) if len(telescope_numbers) > 1 and datalevels[0] <= MARSDataLevel.STAR: raise ValueError( "Loaded files contain data from different telescopes. \ - Please load data belonging to the same telescope.") + Please load data belonging to the same telescope." + ) return run_numbers, is_simulation, telescope_numbers, datalevels @@ -526,9 +547,13 @@ def parse_data_info(self): # here we take the 2nd element (if possible) because sometimes # the first trigger report has still the old prescaler values from a previous run try: - prescaler_array = trigger_tree["MTriggerPrescFact.fPrescFact"].array(library="np") + prescaler_array = trigger_tree[ + "MTriggerPrescFact.fPrescFact" + ].array(library="np") except AssertionError: - logger.warning("No prescaler info found. Will assume standard stereo data.") + logger.warning( + "No prescaler info found. Will assume standard stereo data." + ) stereo = True sumt = False return stereo, sumt @@ -539,7 +564,10 @@ def parse_data_info(self): else: prescaler = prescaler_array[0] - if prescaler == prescaler_mono_nosumt or prescaler == prescaler_mono_sumt: + if ( + prescaler == prescaler_mono_nosumt + or prescaler == prescaler_mono_sumt + ): stereo = False elif prescaler == prescaler_stereo: stereo = True @@ -551,7 +579,9 @@ def parse_data_info(self): # here we take the 2nd element for the same reason as above # L3Table is empty for mono data i.e. taken with one telescope only # if both telescopes take data with no L3, L3Table is filled anyway - L3Table_array = L3T_tree["MReportL3T.fTablename"].array(library="np") + L3Table_array = L3T_tree["MReportL3T.fTablename"].array( + library="np" + ) L3Table_size = L3Table_array.size if L3Table_size > 1: L3Table = L3Table_array[1] @@ -582,11 +612,14 @@ def parse_data_info(self): if len(is_stereo) > 1: raise ValueError( "Loaded files contain both stereo and mono data. \ - Please load only stereo or mono.") + Please load only stereo or mono." + ) if len(is_sumt) > 1: - logger.warning("Found data with both standard and Sum trigger. While this is \ - not an issue, check that this is what you really want to do.") + logger.warning( + "Found data with both standard and Sum trigger. While this is \ + not an issue, check that this is what you really want to do." + ) return is_stereo, is_sumt @@ -607,49 +640,60 @@ def prepare_subarray_info(self): # } # MAGIC telescope positions in m wrt. to the center of MAGIC simulations, from - # CORSIKA and reflector input card and recomputed (rotated) to be w.r.t. geographical North + # CORSIKA and reflector input card and recomputed (rotated) to be w.r.t. geographical North MAGIC_TEL_POSITIONS = { 1: [34.99, -24.02, 0.00] * u.m, - 2: [-34.99, 24.02, 0.00] * u.m + 2: [-34.99, 24.02, 0.00] * u.m, } # camera info from MAGICCam.camgeom.fits.gz file camera_geom = load_camera_geometry() - pulse_shape_lo_gain = np.array([0., 1., 2., 1., 0.]) - pulse_shape_hi_gain = np.array([1., 2., 3., 2., 1.]) + pulse_shape_lo_gain = np.array([0.0, 1.0, 2.0, 1.0, 0.0]) + pulse_shape_hi_gain = np.array([1.0, 2.0, 3.0, 2.0, 1.0]) pulse_shape = np.vstack((pulse_shape_lo_gain, pulse_shape_hi_gain)) if self.mars_datalevel <= MARSDataLevel.STAR: sampling_speed = u.Quantity( - self.files_[0]['RunHeaders']['MRawRunHeader.fSamplingFrequency'].array(library='np')[0]/1000, - u.GHz + self.files_[0]["RunHeaders"][ + "MRawRunHeader.fSamplingFrequency" + ].array(library="np")[0] + / 1000, + u.GHz, ) else: sampling_speed = u.Quantity( - self.files_[0]['RunHeaders']['MRawRunHeader_1.fSamplingFrequency'].array(library='np')[0]/1000, - u.GHz + self.files_[0]["RunHeaders"][ + "MRawRunHeader_1.fSamplingFrequency" + ].array(library="np")[0] + / 1000, + u.GHz, ) camera_readout = CameraReadout( - camera_name='MAGICCam', + camera_name="MAGICCam", sampling_rate=sampling_speed, reference_pulse_shape=pulse_shape, - reference_pulse_sample_width=u.Quantity(0.5, u.ns) + reference_pulse_sample_width=u.Quantity(0.5, u.ns), ) - camera = CameraDescription('MAGICCam', camera_geom, camera_readout) + camera = CameraDescription("MAGICCam", camera_geom, camera_readout) - camera.geometry.frame = CameraFrame(focal_length=OPTICS.equivalent_focal_length) + camera.geometry.frame = CameraFrame( + focal_length=OPTICS.equivalent_focal_length + ) MAGIC_TEL_DESCRIPTION = TelescopeDescription( - name='MAGIC', tel_type='MAGIC', optics=OPTICS, camera=camera + name="MAGIC", tel_type="MAGIC", optics=OPTICS, camera=camera ) - MAGIC_TEL_DESCRIPTIONS = {1: MAGIC_TEL_DESCRIPTION, 2: MAGIC_TEL_DESCRIPTION} + MAGIC_TEL_DESCRIPTIONS = { + 1: MAGIC_TEL_DESCRIPTION, + 2: MAGIC_TEL_DESCRIPTION, + } subarray = SubarrayDescription( - name='MAGIC', + name="MAGIC", tel_positions=MAGIC_TEL_POSITIONS, - tel_descriptions=MAGIC_TEL_DESCRIPTIONS + tel_descriptions=MAGIC_TEL_DESCRIPTIONS, ) if self.allowed_tels: @@ -676,7 +720,7 @@ def decode_version_number(version_encoded): major_version = version_encoded >> 16 minor_version = (version_encoded % 65536) >> 8 patch_version = (version_encoded % 65536) % 256 - version_decoded = f'{major_version}.{minor_version}.{patch_version}' + version_decoded = f"{major_version}.{minor_version}.{patch_version}" return version_decoded @@ -706,89 +750,137 @@ def parse_metadata_info(self): suffix = "_1" metadatainfo_array_list_runheaders = [ - f'MRawRunHeader{suffix}.fSubRunIndex', - f'MRawRunHeader{suffix}.fSourceRA', - f'MRawRunHeader{suffix}.fSourceDEC', - f'MRawRunHeader{suffix}.fSourceName[80]', - f'MRawRunHeader{suffix}.fObservationMode[60]', - f'MRawRunHeader{suffix}.fProjectName[100]', + f"MRawRunHeader{suffix}.fSubRunIndex", + f"MRawRunHeader{suffix}.fSourceRA", + f"MRawRunHeader{suffix}.fSourceDEC", + f"MRawRunHeader{suffix}.fSourceName[80]", + f"MRawRunHeader{suffix}.fObservationMode[60]", + f"MRawRunHeader{suffix}.fProjectName[100]", ] if self.mars_datalevel == MARSDataLevel.CALIBRATED: metadatainfo_array_list_runtails = [ - 'MMarsVersion_sorcerer.fMARSVersionCode', - 'MMarsVersion_sorcerer.fROOTVersionCode', + "MMarsVersion_sorcerer.fMARSVersionCode", + "MMarsVersion_sorcerer.fROOTVersionCode", ] elif self.mars_datalevel == MARSDataLevel.STAR: metadatainfo_array_list_runtails = [ - 'MMarsVersion_star.fMARSVersionCode', - 'MMarsVersion_star.fROOTVersionCode', + "MMarsVersion_star.fMARSVersionCode", + "MMarsVersion_star.fROOTVersionCode", ] elif self.mars_datalevel == MARSDataLevel.SUPERSTAR: metadatainfo_array_list_runtails = [ - 'MMarsVersion_superstar.fMARSVersionCode', - 'MMarsVersion_superstar.fROOTVersionCode', + "MMarsVersion_superstar.fMARSVersionCode", + "MMarsVersion_superstar.fROOTVersionCode", ] elif self.mars_datalevel == MARSDataLevel.MELIBEA: metadatainfo_array_list_runtails = [ - 'MMarsVersion_melibea.fMARSVersionCode', - 'MMarsVersion_melibea.fROOTVersionCode', + "MMarsVersion_melibea.fMARSVersionCode", + "MMarsVersion_melibea.fROOTVersionCode", ] metadata = dict() metadata["file_list"] = self.file_list - metadata['run_numbers'] = self.run_numbers - metadata['is_simulation'] = self.is_simulation - metadata['telescopes'] = self.telescopes - metadata['subrun_number'] = [] - metadata['source_ra'] = [] - metadata['source_dec'] = [] - metadata['source_name'] = [] - metadata['observation_mode'] = [] - metadata['project_name'] = [] - metadata['mars_version'] = [] - metadata['root_version'] = [] + metadata["run_numbers"] = self.run_numbers + metadata["is_simulation"] = self.is_simulation + metadata["telescopes"] = self.telescopes + metadata["subrun_number"] = [] + metadata["source_ra"] = [] + metadata["source_dec"] = [] + metadata["source_name"] = [] + metadata["observation_mode"] = [] + metadata["project_name"] = [] + metadata["mars_version"] = [] + metadata["root_version"] = [] for rootf in self.files_: - meta_info_runh = rootf['RunHeaders'].arrays( - metadatainfo_array_list_runheaders, library="np" + meta_info_runh = rootf["RunHeaders"].arrays( + metadatainfo_array_list_runheaders, library="np" ) - metadata['subrun_number'].append(int(meta_info_runh[f'MRawRunHeader{suffix}.fSubRunIndex'][0])) - metadata['source_ra'].append( - meta_info_runh[f'MRawRunHeader{suffix}.fSourceRA'][0] / seconds_per_hour * degrees_per_hour * u.deg + metadata["subrun_number"].append( + int(meta_info_runh[f"MRawRunHeader{suffix}.fSubRunIndex"][0]) + ) + metadata["source_ra"].append( + meta_info_runh[f"MRawRunHeader{suffix}.fSourceRA"][0] + / seconds_per_hour + * degrees_per_hour + * u.deg ) - metadata['source_dec'].append( - meta_info_runh[f'MRawRunHeader{suffix}.fSourceDEC'][0] / seconds_per_hour * u.deg + metadata["source_dec"].append( + meta_info_runh[f"MRawRunHeader{suffix}.fSourceDEC"][0] + / seconds_per_hour + * u.deg ) if not self.is_simulation: - src_name_array = meta_info_runh[f'MRawRunHeader{suffix}.fSourceName[80]'][0] - metadata['source_name'].append("".join([chr(item) for item in src_name_array if item != 0])) - obs_mode_array = meta_info_runh[f'MRawRunHeader{suffix}.fObservationMode[60]'][0] - metadata['observation_mode'].append("".join([chr(item) for item in obs_mode_array if item != 0])) - project_name_array = meta_info_runh[f'MRawRunHeader{suffix}.fProjectName[100]'][0] - metadata['project_name'].append("".join([chr(item) for item in project_name_array if item != 0])) - - meta_info_runt = rootf['RunTails'].arrays( - metadatainfo_array_list_runtails, - library="np" + src_name_array = meta_info_runh[ + f"MRawRunHeader{suffix}.fSourceName[80]" + ][0] + metadata["source_name"].append( + "".join( + [chr(item) for item in src_name_array if item != 0] + ) + ) + obs_mode_array = meta_info_runh[ + f"MRawRunHeader{suffix}.fObservationMode[60]" + ][0] + metadata["observation_mode"].append( + "".join( + [chr(item) for item in obs_mode_array if item != 0] + ) + ) + project_name_array = meta_info_runh[ + f"MRawRunHeader{suffix}.fProjectName[100]" + ][0] + metadata["project_name"].append( + "".join( + [chr(item) for item in project_name_array if item != 0] + ) + ) + + meta_info_runt = rootf["RunTails"].arrays( + metadatainfo_array_list_runtails, library="np" ) if self.mars_datalevel == MARSDataLevel.CALIBRATED: - mars_version_encoded = int(meta_info_runt['MMarsVersion_sorcerer.fMARSVersionCode'][0]) - root_version_encoded = int(meta_info_runt['MMarsVersion_sorcerer.fROOTVersionCode'][0]) + mars_version_encoded = int( + meta_info_runt["MMarsVersion_sorcerer.fMARSVersionCode"][0] + ) + root_version_encoded = int( + meta_info_runt["MMarsVersion_sorcerer.fROOTVersionCode"][0] + ) elif self.mars_datalevel == MARSDataLevel.STAR: - mars_version_encoded = int(meta_info_runt['MMarsVersion_star.fMARSVersionCode'][0]) - root_version_encoded = int(meta_info_runt['MMarsVersion_star.fROOTVersionCode'][0]) + mars_version_encoded = int( + meta_info_runt["MMarsVersion_star.fMARSVersionCode"][0] + ) + root_version_encoded = int( + meta_info_runt["MMarsVersion_star.fROOTVersionCode"][0] + ) elif self.mars_datalevel == MARSDataLevel.SUPERSTAR: - mars_version_encoded = int(meta_info_runt['MMarsVersion_superstar.fMARSVersionCode'][0]) - root_version_encoded = int(meta_info_runt['MMarsVersion_superstar.fROOTVersionCode'][0]) + mars_version_encoded = int( + meta_info_runt["MMarsVersion_superstar.fMARSVersionCode"][ + 0 + ] + ) + root_version_encoded = int( + meta_info_runt["MMarsVersion_superstar.fROOTVersionCode"][ + 0 + ] + ) elif self.mars_datalevel == MARSDataLevel.MELIBEA: - mars_version_encoded = int(meta_info_runt['MMarsVersion_melibea.fMARSVersionCode'][0]) - root_version_encoded = int(meta_info_runt['MMarsVersion_melibea.fROOTVersionCode'][0]) - metadata['mars_version'].append(self.decode_version_number(mars_version_encoded)) - metadata['root_version'].append(self.decode_version_number(root_version_encoded)) + mars_version_encoded = int( + meta_info_runt["MMarsVersion_melibea.fMARSVersionCode"][0] + ) + root_version_encoded = int( + meta_info_runt["MMarsVersion_melibea.fROOTVersionCode"][0] + ) + metadata["mars_version"].append( + self.decode_version_number(mars_version_encoded) + ) + metadata["root_version"].append( + self.decode_version_number(root_version_encoded) + ) return metadata @@ -828,41 +920,101 @@ def parse_simulation_header(self): # MAGIC_Binc is the magnetic field inclination MAGIC_Bx = u.Quantity(29.5, u.uT) MAGIC_Bz = u.Quantity(23.0, u.uT) - MAGIC_Btot = np.sqrt(MAGIC_Bx**2+MAGIC_Bz**2) + MAGIC_Btot = np.sqrt(MAGIC_Bx ** 2 + MAGIC_Bz ** 2) MAGIC_Bdec = u.Quantity(-7.0, u.deg).to(u.rad) - MAGIC_Binc = u.Quantity(np.arctan2(-MAGIC_Bz.value, MAGIC_Bx.value), u.rad) + MAGIC_Binc = u.Quantity( + np.arctan2(-MAGIC_Bz.value, MAGIC_Bx.value), u.rad + ) simulation_config = dict() for run_number, rootf in zip(self.run_numbers, self.files_): - run_header_tree = rootf['RunHeaders'] - spectral_index = run_header_tree['MMcCorsikaRunHeader.fSlopeSpec'].array(library="np")[0] - e_low = run_header_tree['MMcCorsikaRunHeader.fELowLim'].array(library="np")[0] - e_high = run_header_tree['MMcCorsikaRunHeader.fEUppLim'].array(library="np")[0] - corsika_version = run_header_tree['MMcCorsikaRunHeader.fCorsikaVersion'].array(library="np")[0] - site_height = run_header_tree['MMcCorsikaRunHeader.fHeightLev[10]'].array(library="np")[0][0] - atm_model = run_header_tree['MMcCorsikaRunHeader.fAtmosphericModel'].array(library="np")[0] - if self.mars_datalevel in [MARSDataLevel.CALIBRATED, MARSDataLevel.STAR]: - view_cone = run_header_tree['MMcRunHeader.fRandomPointingConeSemiAngle'].array(library="np")[0] - max_impact = run_header_tree['MMcRunHeader.fImpactMax'].array(library="np")[0] - n_showers = np.sum(run_header_tree['MMcRunHeader.fNumSimulatedShowers'].array(library="np")) - max_zd = run_header_tree['MMcRunHeader.fShowerThetaMax'].array(library="np")[0] - min_zd = run_header_tree['MMcRunHeader.fShowerThetaMin'].array(library="np")[0] - max_az = run_header_tree['MMcRunHeader.fShowerPhiMax'].array(library="np")[0] - min_az = run_header_tree['MMcRunHeader.fShowerPhiMin'].array(library="np")[0] - max_wavelength = run_header_tree['MMcRunHeader.fCWaveUpper'].array(library="np")[0] - min_wavelength = run_header_tree['MMcRunHeader.fCWaveLower'].array(library="np")[0] - elif self.mars_datalevel in [MARSDataLevel.SUPERSTAR, MARSDataLevel.MELIBEA]: - view_cone = run_header_tree['MMcRunHeader_1.fRandomPointingConeSemiAngle'].array(library="np")[0] - max_impact = run_header_tree['MMcRunHeader_1.fImpactMax'].array(library="np")[0] - n_showers = np.sum(run_header_tree['MMcRunHeader_1.fNumSimulatedShowers'].array(library="np")) - max_zd = run_header_tree['MMcRunHeader_1.fShowerThetaMax'].array(library="np")[0] - min_zd = run_header_tree['MMcRunHeader_1.fShowerThetaMin'].array(library="np")[0] - max_az = run_header_tree['MMcRunHeader_1.fShowerPhiMax'].array(library="np")[0] - min_az = run_header_tree['MMcRunHeader_1.fShowerPhiMin'].array(library="np")[0] - max_wavelength = run_header_tree['MMcRunHeader_1.fCWaveUpper'].array(library="np")[0] - min_wavelength = run_header_tree['MMcRunHeader_1.fCWaveLower'].array(library="np")[0] + run_header_tree = rootf["RunHeaders"] + spectral_index = run_header_tree[ + "MMcCorsikaRunHeader.fSlopeSpec" + ].array(library="np")[0] + e_low = run_header_tree["MMcCorsikaRunHeader.fELowLim"].array( + library="np" + )[0] + e_high = run_header_tree["MMcCorsikaRunHeader.fEUppLim"].array( + library="np" + )[0] + corsika_version = run_header_tree[ + "MMcCorsikaRunHeader.fCorsikaVersion" + ].array(library="np")[0] + site_height = run_header_tree[ + "MMcCorsikaRunHeader.fHeightLev[10]" + ].array(library="np")[0][0] + atm_model = run_header_tree[ + "MMcCorsikaRunHeader.fAtmosphericModel" + ].array(library="np")[0] + if self.mars_datalevel in [ + MARSDataLevel.CALIBRATED, + MARSDataLevel.STAR, + ]: + view_cone = run_header_tree[ + "MMcRunHeader.fRandomPointingConeSemiAngle" + ].array(library="np")[0] + max_impact = run_header_tree["MMcRunHeader.fImpactMax"].array( + library="np" + )[0] + n_showers = np.sum( + run_header_tree["MMcRunHeader.fNumSimulatedShowers"].array( + library="np" + ) + ) + max_zd = run_header_tree["MMcRunHeader.fShowerThetaMax"].array( + library="np" + )[0] + min_zd = run_header_tree["MMcRunHeader.fShowerThetaMin"].array( + library="np" + )[0] + max_az = run_header_tree["MMcRunHeader.fShowerPhiMax"].array( + library="np" + )[0] + min_az = run_header_tree["MMcRunHeader.fShowerPhiMin"].array( + library="np" + )[0] + max_wavelength = run_header_tree[ + "MMcRunHeader.fCWaveUpper" + ].array(library="np")[0] + min_wavelength = run_header_tree[ + "MMcRunHeader.fCWaveLower" + ].array(library="np")[0] + elif self.mars_datalevel in [ + MARSDataLevel.SUPERSTAR, + MARSDataLevel.MELIBEA, + ]: + view_cone = run_header_tree[ + "MMcRunHeader_1.fRandomPointingConeSemiAngle" + ].array(library="np")[0] + max_impact = run_header_tree[ + "MMcRunHeader_1.fImpactMax" + ].array(library="np")[0] + n_showers = np.sum( + run_header_tree[ + "MMcRunHeader_1.fNumSimulatedShowers" + ].array(library="np") + ) + max_zd = run_header_tree[ + "MMcRunHeader_1.fShowerThetaMax" + ].array(library="np")[0] + min_zd = run_header_tree[ + "MMcRunHeader_1.fShowerThetaMin" + ].array(library="np")[0] + max_az = run_header_tree["MMcRunHeader_1.fShowerPhiMax"].array( + library="np" + )[0] + min_az = run_header_tree["MMcRunHeader_1.fShowerPhiMin"].array( + library="np" + )[0] + max_wavelength = run_header_tree[ + "MMcRunHeader_1.fCWaveUpper" + ].array(library="np")[0] + min_wavelength = run_header_tree[ + "MMcRunHeader_1.fCWaveLower" + ].array(library="np")[0] simulation_config[run_number] = SimulationConfigContainer( corsika_version=corsika_version, @@ -879,8 +1031,8 @@ def parse_simulation_header(self): prod_site_B_total=MAGIC_Btot, prod_site_B_declination=MAGIC_Bdec, prod_site_B_inclination=MAGIC_Binc, - max_alt=u.Quantity((90. - min_zd), u.deg).to(u.rad), - min_alt=u.Quantity((90. - max_zd), u.deg).to(u.rad), + max_alt=u.Quantity((90.0 - min_zd), u.deg).to(u.rad), + min_alt=u.Quantity((90.0 - max_zd), u.deg).to(u.rad), max_az=u.Quantity(max_az, u.deg).to(u.rad), min_az=u.Quantity(min_az, u.deg).to(u.rad), max_viewcone_radius=view_cone * u.deg, @@ -897,11 +1049,11 @@ def parse_simulation_header(self): def prepare_drive_information(self): drive_leaves = { - 'mjd': 'MReportDrive.fMjd', - 'zd': 'MReportDrive.fCurrentZd', - 'az': 'MReportDrive.fCurrentAz', - 'ra': 'MReportDrive.fRa', - 'dec': 'MReportDrive.fDec', + "mjd": "MReportDrive.fMjd", + "zd": "MReportDrive.fCurrentZd", + "az": "MReportDrive.fCurrentAz", + "ra": "MReportDrive.fRa", + "dec": "MReportDrive.fDec", } # Getting the telescope drive info @@ -915,34 +1067,29 @@ def prepare_drive_information(self): for rootf in rootfiles: drive = rootf["Drive"].arrays(drive_leaves.values(), library="np") - n_reports = len(drive['MReportDrive.fMjd']) + n_reports = len(drive["MReportDrive.fMjd"]) if n_reports == 0: - raise MissingDriveReportError(f"File {rootf.file_path} does not have any drive report. " \ - "Check if it was merpped correctly.") + raise MissingDriveReportError( + f"File {rootf.file_path} does not have any drive report. " + "Check if it was merpped correctly." + ) elif n_reports < 3: - logger.warning(f"File {rootf.file_path} has only {n_reports} drive reports.") + logger.warning( + f"File {rootf.file_path} has only {n_reports} drive reports." + ) for key, leaf in drive_leaves.items(): drive_data[key].append(drive[leaf]) - drive_data = { - k: np.concatenate(v) - for k, v in drive_data.items() - } + drive_data = {k: np.concatenate(v) for k, v in drive_data.items()} # convert unit as before (Better use astropy units!) - drive_data['ra'] *= degrees_per_hour + drive_data["ra"] *= degrees_per_hour # get only drive reports with unique times, otherwise interpolation fails. - _, unique_indices = np.unique( - drive_data['mjd'], - return_index=True - ) + _, unique_indices = np.unique(drive_data["mjd"], return_index=True) - drive_data = { - k: v[unique_indices] - for k, v in drive_data.items() - } + drive_data = {k: v[unique_indices] for k, v in drive_data.items()} return drive_data @@ -967,24 +1114,30 @@ def get_event_time_difference(self): for uproot_file in self.files_: if self.mars_datalevel <= MARSDataLevel.STAR: - event_info = uproot_file['Events'].arrays( - expressions=['MRawEvtHeader.fTimeDiff'], - cut=f'(MTriggerPattern.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})', - library='np', + event_info = uproot_file["Events"].arrays( + expressions=["MRawEvtHeader.fTimeDiff"], + cut=f"(MTriggerPattern.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})", + library="np", ) for tel_id in self.telescopes: - time_diffs[tel_id] = np.append(time_diffs[tel_id], event_info['MRawEvtHeader.fTimeDiff']) + time_diffs[tel_id] = np.append( + time_diffs[tel_id], + event_info["MRawEvtHeader.fTimeDiff"], + ) else: for tel_id in self.telescopes: - event_info = uproot_file['Events'].arrays( - expressions=[f'MRawEvtHeader_{tel_id}.fTimeDiff'], - cut=f'(MTriggerPattern_{tel_id}.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})', - library='np', + event_info = uproot_file["Events"].arrays( + expressions=[f"MRawEvtHeader_{tel_id}.fTimeDiff"], + cut=f"(MTriggerPattern_{tel_id}.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})", + library="np", ) - time_diffs[tel_id] = np.append(time_diffs[tel_id], event_info[f'MRawEvtHeader_{tel_id}.fTimeDiff']) + time_diffs[tel_id] = np.append( + time_diffs[tel_id], + event_info[f"MRawEvtHeader_{tel_id}.fTimeDiff"], + ) for tel_id in self.telescopes: time_diffs[tel_id] *= u.s @@ -1001,7 +1154,7 @@ def is_simulation(self): @property def datalevels(self): - return (self.datalevel, ) + return (self.datalevel,) @property def obs_ids(self): @@ -1044,7 +1197,9 @@ def _get_badrmspixel_mask(self, event, tel_id): for i_ped_type in range(n_ped_types): - charge_std = event.mon.tel[tel_id].pedestal.charge_std[i_ped_type][index] + charge_std = event.mon.tel[tel_id].pedestal.charge_std[i_ped_type][ + index + ] pix_step1 = np.logical_and( charge_std > 0, @@ -1072,8 +1227,12 @@ def _get_badrmspixel_mask(self, event, tel_id): lolim_step1 = mean_step2 / pedestal_level uplim_step1 = mean_step2 * pedestal_level - lolim_step2 = mean_step2 - pedestal_level_variance * np.sqrt(var_step2 - mean_step2 ** 2) - uplim_step2 = mean_step2 + pedestal_level_variance * np.sqrt(var_step2 - mean_step2 ** 2) + lolim_step2 = mean_step2 - pedestal_level_variance * np.sqrt( + var_step2 - mean_step2 ** 2 + ) + uplim_step2 = mean_step2 + pedestal_level_variance * np.sqrt( + var_step2 - mean_step2 ** 2 + ) badrmspix_step1 = np.logical_or( charge_std < lolim_step1, @@ -1109,21 +1268,21 @@ def _set_active_run(self, uproot_file): """ run = dict() - run['input_file'] = uproot_file.file_path + run["input_file"] = uproot_file.file_path if self.mars_datalevel == MARSDataLevel.CALIBRATED: - run['data'] = MarsCalibratedRun( + run["data"] = MarsCalibratedRun( uproot_file, self.is_simulation, self.telescopes[0], ) if self.mars_datalevel == MARSDataLevel.SUPERSTAR: - run['data'] = MarsSuperstarRun( + run["data"] = MarsSuperstarRun( uproot_file, self.is_simulation, ) if self.mars_datalevel == MARSDataLevel.MELIBEA: - run['data'] = MarsMelibeaRun( + run["data"] = MarsMelibeaRun( uproot_file, self.is_simulation, ) @@ -1156,8 +1315,8 @@ def _event_generator(self): # Data container - is initialized once, and data is replaced after each yield: event = ArrayEventContainer() - event.meta['origin'] = 'MAGIC' - event.meta['max_events'] = self.max_events + event.meta["origin"] = "MAGIC" + event.meta["max_events"] = self.max_events event.index.obs_id = self.obs_ids[0] event.trigger.tels_with_trigger = np.array(self.telescopes) @@ -1167,17 +1326,17 @@ def _event_generator(self): # Read the input files subrun-wise: for uproot_file in self.files_: - event.meta['input_file'] = uproot_file.file_path + event.meta["input_file"] = uproot_file.file_path self.current_run = self._set_active_run(uproot_file) if self.use_pedestals: - event_data = self.current_run['data'].pedestal_events - n_events = self.current_run['data'].n_pedestal_events + event_data = self.current_run["data"].pedestal_events + n_events = self.current_run["data"].n_pedestal_events else: - event_data = self.current_run['data'].cosmic_events - n_events = self.current_run['data'].n_cosmic_events + event_data = self.current_run["data"].cosmic_events + n_events = self.current_run["data"].n_cosmic_events for tel_id in self.telescopes: @@ -1185,71 +1344,136 @@ def _event_generator(self): if self.mars_datalevel == MARSDataLevel.CALIBRATED: - monitoring_data = self.current_run['data'].monitoring_data + monitoring_data = self.current_run[ + "data" + ].monitoring_data # Set the pedestal information: - event.mon.tel[tel_id].pedestal.n_events = 500 # hardcoded number of pedestal events averaged over - event.mon.tel[tel_id].pedestal.sample_time = monitoring_data[tel_id]['pedestal_sample_time'] + event.mon.tel[ + tel_id + ].pedestal.n_events = 500 # hardcoded number of pedestal events averaged over + event.mon.tel[ + tel_id + ].pedestal.sample_time = monitoring_data[tel_id][ + "pedestal_sample_time" + ] event.mon.tel[tel_id].pedestal.charge_mean = [ - monitoring_data[tel_id]['pedestal_fundamental']['mean'], - monitoring_data[tel_id]['pedestal_from_extractor']['mean'], - monitoring_data[tel_id]['pedestal_from_extractor_rndm']['mean'], + monitoring_data[tel_id]["pedestal_fundamental"][ + "mean" + ], + monitoring_data[tel_id]["pedestal_from_extractor"][ + "mean" + ], + monitoring_data[tel_id][ + "pedestal_from_extractor_rndm" + ]["mean"], ] event.mon.tel[tel_id].pedestal.charge_std = [ - monitoring_data[tel_id]['pedestal_fundamental']['rms'], - monitoring_data[tel_id]['pedestal_from_extractor']['rms'], - monitoring_data[tel_id]['pedestal_from_extractor_rndm']['rms'], + monitoring_data[tel_id]["pedestal_fundamental"][ + "rms" + ], + monitoring_data[tel_id]["pedestal_from_extractor"][ + "rms" + ], + monitoring_data[tel_id][ + "pedestal_from_extractor_rndm" + ]["rms"], ] # Set the bad pixel information: - event.mon.tel[tel_id].pixel_status.hardware_failing_pixels = np.reshape( - monitoring_data[tel_id]['bad_pixel'], (1, len(monitoring_data[tel_id]['bad_pixel'])) + event.mon.tel[ + tel_id + ].pixel_status.hardware_failing_pixels = np.reshape( + monitoring_data[tel_id]["bad_pixel"], + (1, len(monitoring_data[tel_id]["bad_pixel"])), ) # Interpolate drive information: drive_data = self.drive_information - n_drive_reports = len(drive_data['mjd']) + n_drive_reports = len(drive_data["mjd"]) if self.use_pedestals: - logger.warning(f'Interpolating pedestals events information from {n_drive_reports} drive reports.') + logger.warning( + f"Interpolating pedestals events information from {n_drive_reports} drive reports." + ) else: - logger.warning(f'Interpolating cosmic events information from {n_drive_reports} drive reports.') + logger.warning( + f"Interpolating cosmic events information from {n_drive_reports} drive reports." + ) - first_drive_report_time = Time(drive_data['mjd'][0], scale='utc', format='mjd') - last_drive_report_time = Time(drive_data['mjd'][-1], scale='utc', format='mjd') + first_drive_report_time = Time( + drive_data["mjd"][0], scale="utc", format="mjd" + ) + last_drive_report_time = Time( + drive_data["mjd"][-1], scale="utc", format="mjd" + ) - logger.warning(f'Drive reports available from {first_drive_report_time.iso} to {last_drive_report_time.iso}.') + logger.warning( + f"Drive reports available from {first_drive_report_time.iso} to {last_drive_report_time.iso}." + ) # Create azimuth and zenith angles interpolators: - drive_az_pointing_interpolator = scipy.interpolate.interp1d( - drive_data['mjd'], drive_data['az'], fill_value='extrapolate' + drive_az_pointing_interpolator = ( + scipy.interpolate.interp1d( + drive_data["mjd"], + drive_data["az"], + fill_value="extrapolate", + ) ) - drive_zd_pointing_interpolator = scipy.interpolate.interp1d( - drive_data['mjd'], drive_data['zd'], fill_value='extrapolate' + drive_zd_pointing_interpolator = ( + scipy.interpolate.interp1d( + drive_data["mjd"], + drive_data["zd"], + fill_value="extrapolate", + ) ) # Create RA and Dec interpolators: - drive_ra_pointing_interpolator = scipy.interpolate.interp1d( - drive_data['mjd'], drive_data['ra'], fill_value='extrapolate' + drive_ra_pointing_interpolator = ( + scipy.interpolate.interp1d( + drive_data["mjd"], + drive_data["ra"], + fill_value="extrapolate", + ) ) - drive_dec_pointing_interpolator = scipy.interpolate.interp1d( - drive_data['mjd'], drive_data['dec'], fill_value='extrapolate' + drive_dec_pointing_interpolator = ( + scipy.interpolate.interp1d( + drive_data["mjd"], + drive_data["dec"], + fill_value="extrapolate", + ) ) # Interpolate the drive pointing to the event timestamps: - event_times = event_data[tel_id]['time'].mjd + event_times = event_data[tel_id]["time"].mjd - pointing_az = drive_az_pointing_interpolator(event_times) - pointing_zd = drive_zd_pointing_interpolator(event_times) - pointing_ra = drive_ra_pointing_interpolator(event_times) - pointing_dec = drive_dec_pointing_interpolator(event_times) + pointing_az = drive_az_pointing_interpolator( + event_times + ) + pointing_zd = drive_zd_pointing_interpolator( + event_times + ) + pointing_ra = drive_ra_pointing_interpolator( + event_times + ) + pointing_dec = drive_dec_pointing_interpolator( + event_times + ) - event_data[tel_id]['pointing_az'] = u.Quantity(pointing_az, u.deg) - event_data[tel_id]['pointing_alt'] = u.Quantity(90 - pointing_zd, u.deg) - event_data[tel_id]['pointing_ra'] = u.Quantity(pointing_ra, u.deg) - event_data[tel_id]['pointing_dec'] = u.Quantity(pointing_dec, u.deg) + event_data[tel_id]["pointing_az"] = u.Quantity( + pointing_az, u.deg + ) + event_data[tel_id]["pointing_alt"] = u.Quantity( + 90 - pointing_zd, u.deg + ) + event_data[tel_id]["pointing_ra"] = u.Quantity( + pointing_ra, u.deg + ) + event_data[tel_id]["pointing_dec"] = u.Quantity( + pointing_dec, u.deg + ) # Loop over the events: for i_event in range(n_events): @@ -1257,85 +1481,164 @@ def _event_generator(self): for tel_id in self.telescopes: event.count = counter - event.index.event_id = event_data[tel_id]['stereo_event_number'][i_event] + event.index.event_id = event_data[tel_id][ + "stereo_event_number" + ][i_event] - event.trigger.event_type = MAGIC_TO_CTA_EVENT_TYPE.get(event_data[tel_id]['trigger_pattern'][i_event]) + event.trigger.event_type = MAGIC_TO_CTA_EVENT_TYPE.get( + event_data[tel_id]["trigger_pattern"][i_event] + ) if not self.is_simulation: - event.trigger.time = event_data[tel_id]['time'][i_event] - event.trigger.tel[tel_id].time = event_data[tel_id]['time'][i_event] - - if not self.use_pedestals and self.mars_datalevel == MARSDataLevel.CALIBRATED: - badrmspixel_mask = self._get_badrmspixel_mask(event, tel_id) - event.mon.tel[tel_id].pixel_status.pedestal_failing_pixels = badrmspixel_mask + event.trigger.time = event_data[tel_id]["time"][ + i_event + ] + event.trigger.tel[tel_id].time = event_data[tel_id][ + "time" + ][i_event] + + if ( + not self.use_pedestals + and self.mars_datalevel == MARSDataLevel.CALIBRATED + ): + badrmspixel_mask = self._get_badrmspixel_mask( + event, tel_id + ) + event.mon.tel[ + tel_id + ].pixel_status.pedestal_failing_pixels = ( + badrmspixel_mask + ) # Set the telescope pointing container: - event.pointing.array_azimuth = event_data[tel_id]['pointing_az'][i_event].to(u.rad) - event.pointing.array_altitude = event_data[tel_id]['pointing_alt'][i_event].to(u.rad) - event.pointing.array_ra = event_data[tel_id]['pointing_ra'][i_event].to(u.rad) - event.pointing.array_dec = event_data[tel_id]['pointing_dec'][i_event].to(u.rad) - - event.pointing.tel[tel_id].azimuth = event_data[tel_id]['pointing_az'][i_event].to(u.rad) - event.pointing.tel[tel_id].altitude = event_data[tel_id]['pointing_alt'][i_event].to(u.rad) + event.pointing.array_azimuth = event_data[tel_id][ + "pointing_az" + ][i_event].to(u.rad) + event.pointing.array_altitude = event_data[tel_id][ + "pointing_alt" + ][i_event].to(u.rad) + event.pointing.array_ra = event_data[tel_id][ + "pointing_ra" + ][i_event].to(u.rad) + event.pointing.array_dec = event_data[tel_id][ + "pointing_dec" + ][i_event].to(u.rad) + + event.pointing.tel[tel_id].azimuth = event_data[tel_id][ + "pointing_az" + ][i_event].to(u.rad) + event.pointing.tel[tel_id].altitude = event_data[tel_id][ + "pointing_alt" + ][i_event].to(u.rad) if self.mars_datalevel == MARSDataLevel.CALIBRATED: # Set event charge and peak positions per pixel: - event.dl1.tel[tel_id].image = event_data[tel_id]['image'][i_event] - event.dl1.tel[tel_id].peak_time = event_data[tel_id]['peak_time'][i_event] + event.dl1.tel[tel_id].image = event_data[tel_id][ + "image" + ][i_event] + event.dl1.tel[tel_id].peak_time = event_data[tel_id][ + "peak_time" + ][i_event] if self.mars_datalevel >= MARSDataLevel.SUPERSTAR: - event.dl1.tel[tel_id].parameters = ImageParametersContainer() - event.dl1.tel[tel_id].parameters.hillas = CameraHillasParametersContainer( + event.dl1.tel[ + tel_id + ].parameters = ImageParametersContainer() + event.dl1.tel[ + tel_id + ].parameters.hillas = CameraHillasParametersContainer( x=-event_data[tel_id]["y"][i_event], y=-event_data[tel_id]["x"][i_event], length=event_data[tel_id]["length"][i_event], width=event_data[tel_id]["width"][i_event], intensity=event_data[tel_id]["intensity"][i_event], - r=np.sqrt(event_data[tel_id]["x"][i_event]*event_data[tel_id]["x"][i_event]+event_data[tel_id]["y"][i_event]*event_data[tel_id]["y"][i_event]), + r=np.sqrt( + event_data[tel_id]["x"][i_event] + * event_data[tel_id]["x"][i_event] + + event_data[tel_id]["y"][i_event] + * event_data[tel_id]["y"][i_event] + ), psi=event_data[tel_id]["psi"][i_event], phi=event_data[tel_id]["phi"][i_event], ) - event.dl1.tel[tel_id].parameters.timing = TimingParametersContainer( - slope=event_data[tel_id]["slope"][i_event].value*(1./m2deg)*u.deg, + event.dl1.tel[ + tel_id + ].parameters.timing = TimingParametersContainer( + slope=event_data[tel_id]["slope"][i_event].value + * (1.0 / m2deg) + * u.deg, intercept=event_data[tel_id]["intercept"][i_event], ) # not fully filled - event.dl1.tel[tel_id].parameters.leakage = LeakageContainer( - intensity_width_1=event_data[tel_id]["intensity_width_1"][i_event], - intensity_width_2=event_data[tel_id]["intensity_width_2"][i_event], + event.dl1.tel[ + tel_id + ].parameters.leakage = LeakageContainer( + intensity_width_1=event_data[tel_id][ + "intensity_width_1" + ][i_event], + intensity_width_2=event_data[tel_id][ + "intensity_width_2" + ][i_event], ) # not fully filled - event.dl1.tel[tel_id].parameters.morphology = MorphologyContainer( - num_pixels=event_data[tel_id]["num_pixels"][i_event], - num_islands=event_data[tel_id]["num_islands"][i_event], + event.dl1.tel[ + tel_id + ].parameters.morphology = MorphologyContainer( + num_pixels=event_data[tel_id]["num_pixels"][ + i_event + ], + num_islands=event_data[tel_id]["num_islands"][ + i_event + ], ) - event.dl2.stereo.geometry["HillasReconstructor"] = ReconstructedGeometryContainer( + event.dl2.stereo.geometry[ + "HillasReconstructor" + ] = ReconstructedGeometryContainer( alt=event_data["stereo"]["alt"][i_event], az=event_data["stereo"]["az"][i_event], core_x=event_data["stereo"]["core_x"][i_event], core_y=event_data["stereo"]["core_y"][i_event], h_max=event_data["stereo"]["h_max"][i_event], - is_valid=True if event_data["stereo"]["is_valid"][i_event] == 1 else False, - average_intensity=(event_data[1]["intensity"][i_event]+event_data[2]["intensity"][i_event]) / 2., + is_valid=True + if event_data["stereo"]["is_valid"][i_event] == 1 + else False, + average_intensity=( + event_data[1]["intensity"][i_event] + + event_data[2]["intensity"][i_event] + ) + / 2.0, tel_ids=[1, 2], ) if self.mars_datalevel >= MARSDataLevel.MELIBEA: event.dl2.stereo.energy = ReconstructedEnergyContainer( - energy=event_data["reconstructed"]["energy"][i_event], - energy_uncert=event_data["reconstructed"]["energy_uncert"][i_event], - is_valid=True if event_data["stereo"]["is_valid"][i_event] == 1 else False, + energy=event_data["reconstructed"]["energy"][ + i_event + ], + energy_uncert=event_data["reconstructed"][ + "energy_uncert" + ][i_event], + is_valid=True + if event_data["stereo"]["is_valid"][i_event] == 1 + else False, tel_ids=[1, 2], ) - event.dl2.stereo.classification = ParticleClassificationContainer( - prediction=event_data["reconstructed"]["gammanness"][i_event], - is_valid=True if event_data["stereo"]["is_valid"][i_event] == 1 else False, - tel_ids=[1, 2], + event.dl2.stereo.classification = ( + ParticleClassificationContainer( + prediction=event_data["reconstructed"][ + "gammanness" + ][i_event], + is_valid=True + if event_data["stereo"]["is_valid"][i_event] + == 1 + else False, + tel_ids=[1, 2], + ) ) # Set the simulated event container: @@ -1343,25 +1646,66 @@ def _event_generator(self): event.simulation = SimulatedEventContainer() - event.simulation.shower.energy = event_data[tel_id]['mc_energy'][i_event].to(u.TeV) - event.simulation.shower.shower_primary_id = 1 - event_data[tel_id]['mc_shower_primary_id'][i_event] - event.simulation.shower.h_first_int = event_data[tel_id]['mc_h_first_int'][i_event].to(u.m) + event.simulation.shower.energy = event_data[tel_id][ + "mc_energy" + ][i_event].to(u.TeV) + event.simulation.shower.shower_primary_id = ( + 1 + - event_data[tel_id]["mc_shower_primary_id"][ + i_event + ] + ) + event.simulation.shower.h_first_int = event_data[ + tel_id + ]["mc_h_first_int"][i_event].to(u.m) # Convert the corsika coordinate (x-axis: magnetic north) to the geographical one. # Rotate the corsika coordinate by the magnetic declination (= 7 deg): - mfield_dec = self.simulation_config[self.obs_ids[0]]['prod_site_B_declination'] + mfield_dec = self.simulation_config[self.obs_ids[0]][ + "prod_site_B_declination" + ] - event.simulation.shower.alt = u.Quantity(90, u.deg) - event_data[tel_id]['mc_theta'][i_event].to(u.deg) - event.simulation.shower.az = u.Quantity(180, u.deg) - event_data[tel_id]['mc_phi'][i_event].to(u.deg) + mfield_dec + event.simulation.shower.alt = u.Quantity( + 90, u.deg + ) - event_data[tel_id]["mc_theta"][i_event].to(u.deg) + event.simulation.shower.az = ( + u.Quantity(180, u.deg) + - event_data[tel_id]["mc_phi"][i_event].to(u.deg) + + mfield_dec + ) if event.simulation.shower.az > u.Quantity(180, u.deg): - event.simulation.shower.az -= u.Quantity(360, u.deg) - - event.simulation.shower.core_x = event_data[tel_id]['mc_core_x'][i_event].to(u.m) * np.cos(mfield_dec) \ - + event_data[tel_id]['mc_core_y'][i_event].to(u.m) * np.sin(mfield_dec) + event.simulation.shower.az -= u.Quantity( + 360, u.deg + ) + + event.simulation.shower.core_x = event_data[tel_id][ + "mc_core_x" + ][i_event].to(u.m) * np.cos(mfield_dec) + event_data[ + tel_id + ][ + "mc_core_y" + ][ + i_event + ].to( + u.m + ) * np.sin( + mfield_dec + ) - event.simulation.shower.core_y = event_data[tel_id]['mc_core_y'][i_event].to(u.m) * np.cos(mfield_dec) \ - - event_data[tel_id]['mc_core_x'][i_event].to(u.m) * np.sin(mfield_dec) + event.simulation.shower.core_y = event_data[tel_id][ + "mc_core_y" + ][i_event].to(u.m) * np.cos(mfield_dec) - event_data[ + tel_id + ][ + "mc_core_x" + ][ + i_event + ].to( + u.m + ) * np.sin( + mfield_dec + ) yield event counter += 1 @@ -1400,17 +1744,19 @@ def __init__(self, uproot_file, is_mc, tel_id, n_cam_pixels=1039): # Load the input data: calib_data = self._load_data() - self.cosmic_events = calib_data['cosmic_events'] - self.pedestal_events = calib_data['pedestal_events'] - self.monitoring_data = calib_data['monitoring_data'] + self.cosmic_events = calib_data["cosmic_events"] + self.pedestal_events = calib_data["pedestal_events"] + self.monitoring_data = calib_data["monitoring_data"] @property def n_cosmic_events(self): - return len(self.cosmic_events[self.tel_id].get('trigger_pattern', [])) + return len(self.cosmic_events[self.tel_id].get("trigger_pattern", [])) @property def n_pedestal_events(self): - return len(self.pedestal_events[self.tel_id].get('trigger_pattern', [])) + return len( + self.pedestal_events[self.tel_id].get("trigger_pattern", []) + ) def _load_data(self): """ @@ -1426,58 +1772,58 @@ def _load_data(self): # Branches applicable for cosmic and pedestal events: common_branches = [ - 'MRawEvtHeader.fStereoEvtNumber', - 'MTriggerPattern.fPrescaled', - 'MCerPhotEvt.fPixels.fPhot', - 'MArrivalTime.fData', + "MRawEvtHeader.fStereoEvtNumber", + "MTriggerPattern.fPrescaled", + "MCerPhotEvt.fPixels.fPhot", + "MArrivalTime.fData", ] # Branches applicable for MC events: mc_branches = [ - 'MMcEvt.fEnergy', - 'MMcEvt.fTheta', - 'MMcEvt.fPhi', - 'MMcEvt.fPartId', - 'MMcEvt.fZFirstInteraction', - 'MMcEvt.fCoreX', - 'MMcEvt.fCoreY', + "MMcEvt.fEnergy", + "MMcEvt.fTheta", + "MMcEvt.fPhi", + "MMcEvt.fPartId", + "MMcEvt.fZFirstInteraction", + "MMcEvt.fCoreX", + "MMcEvt.fCoreY", ] pointing_branches = [ - 'MPointingPos.fZd', - 'MPointingPos.fAz', - 'MPointingPos.fRa', - 'MPointingPos.fDec', - 'MPointingPos.fDevZd', - 'MPointingPos.fDevAz', - 'MPointingPos.fDevHa', - 'MPointingPos.fDevDec', + "MPointingPos.fZd", + "MPointingPos.fAz", + "MPointingPos.fRa", + "MPointingPos.fDec", + "MPointingPos.fDevZd", + "MPointingPos.fDevAz", + "MPointingPos.fDevHa", + "MPointingPos.fDevDec", ] # Branches applicable for real data: timing_branches = [ - 'MTime.fMjd', - 'MTime.fTime.fMilliSec', - 'MTime.fNanoSec', + "MTime.fMjd", + "MTime.fTime.fMilliSec", + "MTime.fNanoSec", ] pedestal_branches = [ - 'MTimePedestals.fMjd', - 'MTimePedestals.fTime.fMilliSec', - 'MTimePedestals.fNanoSec', - 'MPedPhotFundamental.fArray.fMean', - 'MPedPhotFundamental.fArray.fRms', - 'MPedPhotFromExtractor.fArray.fMean', - 'MPedPhotFromExtractor.fArray.fRms', - 'MPedPhotFromExtractorRndm.fArray.fMean', - 'MPedPhotFromExtractorRndm.fArray.fRms', + "MTimePedestals.fMjd", + "MTimePedestals.fTime.fMilliSec", + "MTimePedestals.fNanoSec", + "MPedPhotFundamental.fArray.fMean", + "MPedPhotFundamental.fArray.fRms", + "MPedPhotFromExtractor.fArray.fMean", + "MPedPhotFromExtractor.fArray.fRms", + "MPedPhotFromExtractorRndm.fArray.fMean", + "MPedPhotFromExtractorRndm.fArray.fRms", ] # Initialize the data container: calib_data = { - 'cosmic_events': {self.tel_id: dict()}, - 'pedestal_events': {self.tel_id: dict()}, - 'monitoring_data': {self.tel_id: dict()}, + "cosmic_events": {self.tel_id: dict()}, + "pedestal_events": {self.tel_id: dict()}, + "monitoring_data": {self.tel_id: dict()}, } # Set event cuts: @@ -1485,14 +1831,25 @@ def _load_data(self): if self.is_mc: # Only for cosmic events because MC data do not have pedestal events: - events_cut['cosmic_events'] = f'(MTriggerPattern.fPrescaled == {MC_STEREO_TRIGGER_PATTERN})' \ - ' & (MRawEvtHeader.fStereoEvtNumber != 0)' + events_cut["cosmic_events"] = ( + f"(MTriggerPattern.fPrescaled == {MC_STEREO_TRIGGER_PATTERN})" + " & (MRawEvtHeader.fStereoEvtNumber != 0)" + ) else: - events_cut['cosmic_events'] = f'(MTriggerPattern.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})' - events_cut['pedestal_events'] = f'(MTriggerPattern.fPrescaled == {PEDESTAL_TRIGGER_PATTERN})' + events_cut[ + "cosmic_events" + ] = f"(MTriggerPattern.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})" + events_cut[ + "pedestal_events" + ] = f"(MTriggerPattern.fPrescaled == {PEDESTAL_TRIGGER_PATTERN})" # read common information from RunHeaders - sampling_speed = self.uproot_file['RunHeaders']['MRawRunHeader.fSamplingFrequency'].array(library='np')[0]/1000. # GHz + sampling_speed = ( + self.uproot_file["RunHeaders"][ + "MRawRunHeader.fSamplingFrequency" + ].array(library="np")[0] + / 1000.0 + ) # GHz # Loop over the event types: event_types = events_cut.keys() @@ -1500,96 +1857,158 @@ def _load_data(self): for event_type in event_types: # Reading the information common to cosmic and pedestal events: - common_info = self.uproot_file['Events'].arrays( + common_info = self.uproot_file["Events"].arrays( expressions=common_branches, cut=events_cut[event_type], - library='np', + library="np", ) - calib_data[event_type][self.tel_id]['trigger_pattern'] = np.array(common_info['MTriggerPattern.fPrescaled'], dtype=int) - calib_data[event_type][self.tel_id]['stereo_event_number'] = np.array(common_info['MRawEvtHeader.fStereoEvtNumber'], dtype=int) + calib_data[event_type][self.tel_id]["trigger_pattern"] = np.array( + common_info["MTriggerPattern.fPrescaled"], dtype=int + ) + calib_data[event_type][self.tel_id][ + "stereo_event_number" + ] = np.array( + common_info["MRawEvtHeader.fStereoEvtNumber"], dtype=int + ) # Set pixel-wise charge and peak time information. # The length of the pixel array is 1183, but here only the first part of the pixel # information are extracted (i.e., for the current MAGIC camera geometry, the pixels # between 0 and 1039 are extracted, since the other part of pixels has only zeros): - calib_data[event_type][self.tel_id]['image'] = np.array(common_info['MCerPhotEvt.fPixels.fPhot'].tolist())[:, :self.n_cam_pixels] - calib_data[event_type][self.tel_id]['peak_time'] = np.array(common_info['MArrivalTime.fData'].tolist())[:, :self.n_cam_pixels] - calib_data[event_type][self.tel_id]['peak_time']/=sampling_speed # [ns] + calib_data[event_type][self.tel_id]["image"] = np.array( + common_info["MCerPhotEvt.fPixels.fPhot"].tolist() + )[:, : self.n_cam_pixels] + calib_data[event_type][self.tel_id]["peak_time"] = np.array( + common_info["MArrivalTime.fData"].tolist() + )[:, : self.n_cam_pixels] + calib_data[event_type][self.tel_id][ + "peak_time" + ] /= sampling_speed # [ns] if self.is_mc: # Reading the MC information: - mc_info = self.uproot_file['Events'].arrays( + mc_info = self.uproot_file["Events"].arrays( expressions=mc_branches, cut=events_cut[event_type], - library='np', + library="np", ) # Note that the branch "MMcEvt.fPhi" seems to be the angle between the direction # of the magnetic north and the momentum of a simulated primary particle, defined # between -pi and pi, negative if the momentum pointing eastward, positive westward. # The conversion to azimuth should be 180 - fPhi + magnetic_declination: - calib_data[event_type][self.tel_id]['mc_energy'] = u.Quantity(mc_info['MMcEvt.fEnergy'], u.GeV) - calib_data[event_type][self.tel_id]['mc_theta'] = u.Quantity(mc_info['MMcEvt.fTheta'], u.rad) - calib_data[event_type][self.tel_id]['mc_phi'] = u.Quantity(mc_info['MMcEvt.fPhi'], u.rad) - calib_data[event_type][self.tel_id]['mc_shower_primary_id'] = np.array(mc_info['MMcEvt.fPartId'], dtype=int) - calib_data[event_type][self.tel_id]['mc_h_first_int'] = u.Quantity(mc_info['MMcEvt.fZFirstInteraction'], u.cm) - calib_data[event_type][self.tel_id]['mc_core_x'] = u.Quantity(mc_info['MMcEvt.fCoreX'], u.cm) - calib_data[event_type][self.tel_id]['mc_core_y'] = u.Quantity(mc_info['MMcEvt.fCoreY'], u.cm) + calib_data[event_type][self.tel_id]["mc_energy"] = u.Quantity( + mc_info["MMcEvt.fEnergy"], u.GeV + ) + calib_data[event_type][self.tel_id]["mc_theta"] = u.Quantity( + mc_info["MMcEvt.fTheta"], u.rad + ) + calib_data[event_type][self.tel_id]["mc_phi"] = u.Quantity( + mc_info["MMcEvt.fPhi"], u.rad + ) + calib_data[event_type][self.tel_id][ + "mc_shower_primary_id" + ] = np.array(mc_info["MMcEvt.fPartId"], dtype=int) + calib_data[event_type][self.tel_id][ + "mc_h_first_int" + ] = u.Quantity(mc_info["MMcEvt.fZFirstInteraction"], u.cm) + calib_data[event_type][self.tel_id]["mc_core_x"] = u.Quantity( + mc_info["MMcEvt.fCoreX"], u.cm + ) + calib_data[event_type][self.tel_id]["mc_core_y"] = u.Quantity( + mc_info["MMcEvt.fCoreY"], u.cm + ) # Reading the telescope pointing information: - pointing = self.uproot_file['Events'].arrays( + pointing = self.uproot_file["Events"].arrays( expressions=pointing_branches, cut=events_cut[event_type], - library='np', + library="np", ) - pointing_az = pointing['MPointingPos.fAz'] - pointing['MPointingPos.fDevAz'] - pointing_zd = pointing['MPointingPos.fZd'] - pointing['MPointingPos.fDevZd'] + pointing_az = ( + pointing["MPointingPos.fAz"] + - pointing["MPointingPos.fDevAz"] + ) + pointing_zd = ( + pointing["MPointingPos.fZd"] + - pointing["MPointingPos.fDevZd"] + ) # N.B. the positive sign here, as HA = local sidereal time - ra: - pointing_ra = (pointing['MPointingPos.fRa'] + pointing['MPointingPos.fDevHa']) * degrees_per_hour - pointing_dec = pointing['MPointingPos.fDec'] - pointing['MPointingPos.fDevDec'] + pointing_ra = ( + pointing["MPointingPos.fRa"] + + pointing["MPointingPos.fDevHa"] + ) * degrees_per_hour + pointing_dec = ( + pointing["MPointingPos.fDec"] + - pointing["MPointingPos.fDevDec"] + ) - calib_data[event_type][self.tel_id]['pointing_az'] = u.Quantity(pointing_az, u.deg) - calib_data[event_type][self.tel_id]['pointing_alt'] = u.Quantity(90 - pointing_zd, u.deg) - calib_data[event_type][self.tel_id]['pointing_ra'] = u.Quantity(pointing_ra, u.deg) - calib_data[event_type][self.tel_id]['pointing_dec'] = u.Quantity(pointing_dec, u.deg) + calib_data[event_type][self.tel_id][ + "pointing_az" + ] = u.Quantity(pointing_az, u.deg) + calib_data[event_type][self.tel_id][ + "pointing_alt" + ] = u.Quantity(90 - pointing_zd, u.deg) + calib_data[event_type][self.tel_id][ + "pointing_ra" + ] = u.Quantity(pointing_ra, u.deg) + calib_data[event_type][self.tel_id][ + "pointing_dec" + ] = u.Quantity(pointing_dec, u.deg) else: # Reading the event timing information: - timing_info = self.uproot_file['Events'].arrays( + timing_info = self.uproot_file["Events"].arrays( expressions=timing_branches, cut=events_cut[event_type], - library='np', + library="np", ) # In order to keep the precision of timestamps, here the Decimal module is used. # At the later steps, the precise information can be extracted by specifying # the sub-format of value to 'long', i.e., Time.to_value(format='unix', subfmt='long'): - event_obs_day = Time(timing_info['MTime.fMjd'], format='mjd', scale='utc') + event_obs_day = Time( + timing_info["MTime.fMjd"], format="mjd", scale="utc" + ) event_obs_day = np.round(event_obs_day.unix) - event_obs_day = np.array([Decimal(str(x)) for x in event_obs_day]) + event_obs_day = np.array( + [Decimal(str(x)) for x in event_obs_day] + ) - event_millisec = np.round(timing_info['MTime.fTime.fMilliSec'] * msec2sec, 3) - event_millisec = np.array([Decimal(str(x)) for x in event_millisec]) + event_millisec = np.round( + timing_info["MTime.fTime.fMilliSec"] * msec2sec, 3 + ) + event_millisec = np.array( + [Decimal(str(x)) for x in event_millisec] + ) - event_nanosec = np.round(timing_info['MTime.fNanoSec'] * nsec2sec, 7) - event_nanosec = np.array([Decimal(str(x)) for x in event_nanosec]) + event_nanosec = np.round( + timing_info["MTime.fNanoSec"] * nsec2sec, 7 + ) + event_nanosec = np.array( + [Decimal(str(x)) for x in event_nanosec] + ) event_time = event_obs_day + event_millisec + event_nanosec - calib_data[event_type][self.tel_id]['time'] = Time(event_time, format='unix', scale='utc') + calib_data[event_type][self.tel_id]["time"] = Time( + event_time, format="unix", scale="utc" + ) # Reading the monitoring data: if not self.is_mc: # Reading the bad pixel information: - as_dtype = uproot.interpretation.numerical.AsDtype(np.dtype('>i4')) + as_dtype = uproot.interpretation.numerical.AsDtype(np.dtype(">i4")) as_jagged = uproot.interpretation.jagged.AsJagged(as_dtype) - badpixel_info = self.uproot_file['RunHeaders']['MBadPixelsCam.fArray.fInfo'].array(as_jagged, library='np')[0] - badpixel_info = badpixel_info.reshape((4, 1183), order='F') + badpixel_info = self.uproot_file["RunHeaders"][ + "MBadPixelsCam.fArray.fInfo" + ].array(as_jagged, library="np")[0] + badpixel_info = badpixel_info.reshape((4, 1183), order="F") # now we have 4 axes: # 0st axis: empty (?) @@ -1603,55 +2022,110 @@ def _load_data(self): unsuitable_pixels = np.zeros(self.n_cam_pixels, dtype=bool) for i_pix in range(self.n_cam_pixels): - unsuitable_pixels[i_pix] = int('{:08b}'.format(unsuitable_pixels_bit[i_pix])[-2]) + unsuitable_pixels[i_pix] = int( + "{:08b}".format(unsuitable_pixels_bit[i_pix])[-2] + ) - calib_data['monitoring_data'][self.tel_id]['bad_pixel'] = unsuitable_pixels + calib_data["monitoring_data"][self.tel_id][ + "bad_pixel" + ] = unsuitable_pixels # Try to read the Pedestals tree (soft fail if not present): try: - pedestal_info = self.uproot_file['Pedestals'].arrays( + pedestal_info = self.uproot_file["Pedestals"].arrays( expressions=pedestal_branches, - library='np', + library="np", ) # Set sample times of pedestal events: - pedestal_obs_day = Time(pedestal_info['MTimePedestals.fMjd'], format='mjd', scale='utc') + pedestal_obs_day = Time( + pedestal_info["MTimePedestals.fMjd"], + format="mjd", + scale="utc", + ) pedestal_obs_day = np.round(pedestal_obs_day.unix) - pedestal_obs_day = np.array([Decimal(str(x)) for x in pedestal_obs_day]) - - pedestal_millisec = np.round(pedestal_info['MTimePedestals.fTime.fMilliSec'] * msec2sec, 3) - pedestal_millisec = np.array([Decimal(str(x)) for x in pedestal_millisec]) + pedestal_obs_day = np.array( + [Decimal(str(x)) for x in pedestal_obs_day] + ) - pedestal_nanosec = np.round(pedestal_info['MTimePedestals.fNanoSec'] * nsec2sec, 7) - pedestal_nanosec = np.array([Decimal(str(x)) for x in pedestal_nanosec]) + pedestal_millisec = np.round( + pedestal_info["MTimePedestals.fTime.fMilliSec"] * msec2sec, + 3, + ) + pedestal_millisec = np.array( + [Decimal(str(x)) for x in pedestal_millisec] + ) - pedestal_sample_time = pedestal_obs_day + pedestal_millisec + pedestal_nanosec + pedestal_nanosec = np.round( + pedestal_info["MTimePedestals.fNanoSec"] * nsec2sec, 7 + ) + pedestal_nanosec = np.array( + [Decimal(str(x)) for x in pedestal_nanosec] + ) - calib_data['monitoring_data'][self.tel_id]['pedestal_sample_time'] = Time( - pedestal_sample_time, format='unix', scale='utc' + pedestal_sample_time = ( + pedestal_obs_day + pedestal_millisec + pedestal_nanosec ) + calib_data["monitoring_data"][self.tel_id][ + "pedestal_sample_time" + ] = Time(pedestal_sample_time, format="unix", scale="utc") + # Set the mean and RMS of pedestal charges: - calib_data['monitoring_data'][self.tel_id]['pedestal_fundamental'] = { - 'mean': np.array(pedestal_info['MPedPhotFundamental.fArray.fMean'].tolist())[:, :self.n_cam_pixels], - 'rms': np.array(pedestal_info['MPedPhotFundamental.fArray.fRms'].tolist())[:, :self.n_cam_pixels], + calib_data["monitoring_data"][self.tel_id][ + "pedestal_fundamental" + ] = { + "mean": np.array( + pedestal_info[ + "MPedPhotFundamental.fArray.fMean" + ].tolist() + )[:, : self.n_cam_pixels], + "rms": np.array( + pedestal_info[ + "MPedPhotFundamental.fArray.fRms" + ].tolist() + )[:, : self.n_cam_pixels], } - calib_data['monitoring_data'][self.tel_id]['pedestal_from_extractor'] = { - 'mean': np.array(pedestal_info['MPedPhotFromExtractor.fArray.fMean'].tolist())[:, :self.n_cam_pixels], - 'rms': np.array(pedestal_info['MPedPhotFromExtractor.fArray.fRms'].tolist())[:, :self.n_cam_pixels], + calib_data["monitoring_data"][self.tel_id][ + "pedestal_from_extractor" + ] = { + "mean": np.array( + pedestal_info[ + "MPedPhotFromExtractor.fArray.fMean" + ].tolist() + )[:, : self.n_cam_pixels], + "rms": np.array( + pedestal_info[ + "MPedPhotFromExtractor.fArray.fRms" + ].tolist() + )[:, : self.n_cam_pixels], } - calib_data['monitoring_data'][self.tel_id]['pedestal_from_extractor_rndm'] = { - 'mean': np.array(pedestal_info['MPedPhotFromExtractorRndm.fArray.fMean'].tolist())[:, :self.n_cam_pixels], - 'rms': np.array(pedestal_info['MPedPhotFromExtractorRndm.fArray.fRms'].tolist())[:, :self.n_cam_pixels], + calib_data["monitoring_data"][self.tel_id][ + "pedestal_from_extractor_rndm" + ] = { + "mean": np.array( + pedestal_info[ + "MPedPhotFromExtractorRndm.fArray.fMean" + ].tolist() + )[:, : self.n_cam_pixels], + "rms": np.array( + pedestal_info[ + "MPedPhotFromExtractorRndm.fArray.fRms" + ].tolist() + )[:, : self.n_cam_pixels], } except KeyError: - logger.warning('The Pedestals tree is not present in the input file. Cleaning algorithm may fail.') + logger.warning( + "The Pedestals tree is not present in the input file. Cleaning algorithm may fail." + ) # Check for bit flips in the stereo event IDs: uplim_total_jumps = 100 - stereo_event_number = calib_data['cosmic_events'][self.tel_id]['stereo_event_number'].astype(int) + stereo_event_number = calib_data["cosmic_events"][self.tel_id][ + "stereo_event_number" + ].astype(int) number_difference = np.diff(stereo_event_number) indices_flip = np.where(number_difference < 0)[0] @@ -1659,20 +2133,28 @@ def _load_data(self): if n_flips > 0: - logger.warning(f'Warning: detected {n_flips} bit flips in the input file') + logger.warning( + f"Warning: detected {n_flips} bit flips in the input file" + ) total_jumped_number = 0 for i in indices_flip: - jumped_number = stereo_event_number[i] - stereo_event_number[i+1] + jumped_number = ( + stereo_event_number[i] - stereo_event_number[i + 1] + ) total_jumped_number += jumped_number - logger.warning(f'Jump of L3 number backward from {stereo_event_number[i]} to ' \ - f'{stereo_event_number[i+1]}; total jumped number so far: {total_jumped_number}') + logger.warning( + f"Jump of L3 number backward from {stereo_event_number[i]} to " + f"{stereo_event_number[i+1]}; total jumped number so far: {total_jumped_number}" + ) if total_jumped_number > uplim_total_jumps: - logger.warning(f'More than {uplim_total_jumps} jumps in the stereo trigger number; ' \ - f'You may have to match events by timestamp at a later stage.') + logger.warning( + f"More than {uplim_total_jumps} jumps in the stereo trigger number; " + f"You may have to match events by timestamp at a later stage." + ) return calib_data @@ -1708,7 +2190,7 @@ def __init__(self, uproot_file, is_mc, n_cam_pixels=1039): @property def n_cosmic_events(self): - return len(self.cosmic_events[1].get('trigger_pattern', [])) + return len(self.cosmic_events[1].get("trigger_pattern", [])) @property def n_pedestal_events(self): @@ -1739,85 +2221,85 @@ def _load_data(self): for tel_id in [1, 2]: common_branches[tel_id] = [ - f'MRawEvtHeader_{tel_id}.fStereoEvtNumber', - f'MTriggerPattern_{tel_id}.fPrescaled', + f"MRawEvtHeader_{tel_id}.fStereoEvtNumber", + f"MTriggerPattern_{tel_id}.fPrescaled", ] pointing_branches[tel_id] = [ - f'MPointingPos_{tel_id}.fZd', # [deg] - f'MPointingPos_{tel_id}.fAz', # [deg] - f'MPointingPos_{tel_id}.fRa', # [h] - f'MPointingPos_{tel_id}.fDec', # [deg] - f'MPointingPos_{tel_id}.fDevZd', # [deg] - f'MPointingPos_{tel_id}.fDevAz', # [deg] - f'MPointingPos_{tel_id}.fDevHa', # [h] - f'MPointingPos_{tel_id}.fDevDec', # [deg] + f"MPointingPos_{tel_id}.fZd", # [deg] + f"MPointingPos_{tel_id}.fAz", # [deg] + f"MPointingPos_{tel_id}.fRa", # [h] + f"MPointingPos_{tel_id}.fDec", # [deg] + f"MPointingPos_{tel_id}.fDevZd", # [deg] + f"MPointingPos_{tel_id}.fDevAz", # [deg] + f"MPointingPos_{tel_id}.fDevHa", # [h] + f"MPointingPos_{tel_id}.fDevDec", # [deg] ] # Branches applicable for real data: timing_branches[tel_id] = [ - f'MTime_{tel_id}.fMjd', - f'MTime_{tel_id}.fTime.fMilliSec', - f'MTime_{tel_id}.fNanoSec', + f"MTime_{tel_id}.fMjd", + f"MTime_{tel_id}.fTime.fMilliSec", + f"MTime_{tel_id}.fNanoSec", ] hillas_branches[tel_id] = [ - f'MHillas_{tel_id}.fLength', # [mm] - f'MHillas_{tel_id}.fWidth', # [mm] - f'MHillas_{tel_id}.fDelta', # [rad] - f'MHillas_{tel_id}.fSize', - f'MHillas_{tel_id}.fMeanX', # [mm] - f'MHillas_{tel_id}.fMeanY', # [mm] + f"MHillas_{tel_id}.fLength", # [mm] + f"MHillas_{tel_id}.fWidth", # [mm] + f"MHillas_{tel_id}.fDelta", # [rad] + f"MHillas_{tel_id}.fSize", + f"MHillas_{tel_id}.fMeanX", # [mm] + f"MHillas_{tel_id}.fMeanY", # [mm] ] hillas_src_branches[tel_id] = [ - f'MHillasSrc_{tel_id}.fDCADelta', # [deg] Angle of the shower axis with respect to the x-axis - f'MHillasSrc_{tel_id}.fDist', # [mm] distance between src and center of ellipse + f"MHillasSrc_{tel_id}.fDCADelta", # [deg] Angle of the shower axis with respect to the x-axis + f"MHillasSrc_{tel_id}.fDist", # [mm] distance between src and center of ellipse ] imagepar_branches[tel_id] = [ - f'MImagePar_{tel_id}.fNumIslands', # Number of distinct pixel groupings in image + f"MImagePar_{tel_id}.fNumIslands", # Number of distinct pixel groupings in image ] new_imagepar_branches[tel_id] = [ - f'MNewImagePar_{tel_id}.fNumUsedPixels', # Number of pixels which survived the image cleaning - f'MNewImagePar_{tel_id}.fLeakage1', # (photons in most outer ring of pixels) over fSize - f'MNewImagePar_{tel_id}.fLeakage2', # (photons in the 2 outer rings of pixels) over fSize + f"MNewImagePar_{tel_id}.fNumUsedPixels", # Number of pixels which survived the image cleaning + f"MNewImagePar_{tel_id}.fLeakage1", # (photons in most outer ring of pixels) over fSize + f"MNewImagePar_{tel_id}.fLeakage2", # (photons in the 2 outer rings of pixels) over fSize ] hillas_timefit_branches[tel_id] = [ - f'MHillasTimeFit_{tel_id}.fP1Const', # [Time_slices] - f'MHillasTimeFit_{tel_id}.fP1Grad', # [Time_slices]/[mm] + f"MHillasTimeFit_{tel_id}.fP1Const", # [Time_slices] + f"MHillasTimeFit_{tel_id}.fP1Grad", # [Time_slices]/[mm] ] mc_branches[tel_id] = [ - f'MMcEvt_{tel_id}.fEnergy', - f'MMcEvt_{tel_id}.fTheta', - f'MMcEvt_{tel_id}.fPhi', - f'MMcEvt_{tel_id}.fPartId', - f'MMcEvt_{tel_id}.fZFirstInteraction', - f'MMcEvt_{tel_id}.fCoreX', - f'MMcEvt_{tel_id}.fCoreY', + f"MMcEvt_{tel_id}.fEnergy", + f"MMcEvt_{tel_id}.fTheta", + f"MMcEvt_{tel_id}.fPhi", + f"MMcEvt_{tel_id}.fPartId", + f"MMcEvt_{tel_id}.fZFirstInteraction", + f"MMcEvt_{tel_id}.fCoreX", + f"MMcEvt_{tel_id}.fCoreY", ] stereo_branches = [ - 'MStereoPar.fValid', - 'MStereoPar.fDirectionX', - 'MStereoPar.fDirectionY', - 'MStereoPar.fDirectionZd', - 'MStereoPar.fDirectionAz', - 'MStereoPar.fDirectionDec', - 'MStereoPar.fDirectionRA', - 'MStereoPar.fTheta2', - 'MStereoPar.fCoreX', - 'MStereoPar.fCoreY', - 'MStereoPar.fM1Impact', - 'MStereoPar.fM2Impact', - 'MStereoPar.fMaxHeight', + "MStereoPar.fValid", + "MStereoPar.fDirectionX", + "MStereoPar.fDirectionY", + "MStereoPar.fDirectionZd", + "MStereoPar.fDirectionAz", + "MStereoPar.fDirectionDec", + "MStereoPar.fDirectionRA", + "MStereoPar.fTheta2", + "MStereoPar.fCoreX", + "MStereoPar.fCoreY", + "MStereoPar.fM1Impact", + "MStereoPar.fM2Impact", + "MStereoPar.fMaxHeight", ] stereo_data = { - 'cosmic_events': dict(), + "cosmic_events": dict(), } event_data = dict() @@ -1830,151 +2312,248 @@ def _load_data(self): if self.is_mc: # Only for cosmic events because MC data do not have pedestal events: - events_cut = f'(MTriggerPattern_{tel_id}.fPrescaled == {MC_STEREO_TRIGGER_PATTERN})' \ - f' & (MRawEvtHeader_{tel_id}.fStereoEvtNumber != 0)' + events_cut = ( + f"(MTriggerPattern_{tel_id}.fPrescaled == {MC_STEREO_TRIGGER_PATTERN})" + f" & (MRawEvtHeader_{tel_id}.fStereoEvtNumber != 0)" + ) else: - events_cut = f'(MTriggerPattern_{tel_id}.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})' + events_cut = f"(MTriggerPattern_{tel_id}.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})" # read common information from RunHeaders - sampling_speed = self.uproot_file['RunHeaders'][f'MRawRunHeader_{tel_id}.fSamplingFrequency'].array(library='np')[0]/1000. # GHz + sampling_speed = ( + self.uproot_file["RunHeaders"][ + f"MRawRunHeader_{tel_id}.fSamplingFrequency" + ].array(library="np")[0] + / 1000.0 + ) # GHz # Reading the information common to cosmic and pedestal events: - common_info = self.uproot_file['Events'].arrays( + common_info = self.uproot_file["Events"].arrays( expressions=common_branches[tel_id], cut=events_cut, - library='np', + library="np", ) - event_data[tel_id]['trigger_pattern'] = np.array(common_info[f'MTriggerPattern_{tel_id}.fPrescaled'], dtype=int) - event_data[tel_id]['stereo_event_number'] = np.array(common_info[f'MRawEvtHeader_{tel_id}.fStereoEvtNumber'], dtype=int) + event_data[tel_id]["trigger_pattern"] = np.array( + common_info[f"MTriggerPattern_{tel_id}.fPrescaled"], dtype=int + ) + event_data[tel_id]["stereo_event_number"] = np.array( + common_info[f"MRawEvtHeader_{tel_id}.fStereoEvtNumber"], + dtype=int, + ) if self.is_mc: # Reading the MC information: - mc_info = self.uproot_file['Events'].arrays( + mc_info = self.uproot_file["Events"].arrays( expressions=mc_branches[tel_id], cut=events_cut, - library='np', + library="np", ) # Note that the branch "MMcEvt.fPhi" seems to be the angle between the direction # of the magnetic north and the momentum of a simulated primary particle, defined # between -pi and pi, negative if the momentum pointing eastward, positive westward. # The conversion to azimuth should be 180 - fPhi + magnetic_declination: - event_data[tel_id]['mc_energy'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fEnergy'], u.GeV) - event_data[tel_id]['mc_theta'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fTheta'], u.rad) - event_data[tel_id]['mc_phi'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fPhi'], u.rad) - event_data[tel_id]['mc_shower_primary_id'] = np.array(mc_info[f'MMcEvt_{tel_id}.fPartId'], dtype=int) - event_data[tel_id]['mc_h_first_int'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fZFirstInteraction'], u.cm) - event_data[tel_id]['mc_core_x'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fCoreX'], u.cm) - event_data[tel_id]['mc_core_y'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fCoreY'], u.cm) + event_data[tel_id]["mc_energy"] = u.Quantity( + mc_info[f"MMcEvt_{tel_id}.fEnergy"], u.GeV + ) + event_data[tel_id]["mc_theta"] = u.Quantity( + mc_info[f"MMcEvt_{tel_id}.fTheta"], u.rad + ) + event_data[tel_id]["mc_phi"] = u.Quantity( + mc_info[f"MMcEvt_{tel_id}.fPhi"], u.rad + ) + event_data[tel_id]["mc_shower_primary_id"] = np.array( + mc_info[f"MMcEvt_{tel_id}.fPartId"], dtype=int + ) + event_data[tel_id]["mc_h_first_int"] = u.Quantity( + mc_info[f"MMcEvt_{tel_id}.fZFirstInteraction"], u.cm + ) + event_data[tel_id]["mc_core_x"] = u.Quantity( + mc_info[f"MMcEvt_{tel_id}.fCoreX"], u.cm + ) + event_data[tel_id]["mc_core_y"] = u.Quantity( + mc_info[f"MMcEvt_{tel_id}.fCoreY"], u.cm + ) else: # Reading the event timing information: - timing_info = self.uproot_file['Events'].arrays( + timing_info = self.uproot_file["Events"].arrays( expressions=timing_branches[tel_id], cut=events_cut, - library='np', + library="np", ) # In order to keep the precision of timestamps, here the Decimal module is used. # At the later steps, the precise information can be extracted by specifying # the sub-format of value to 'long', i.e., Time.to_value(format='unix', subfmt='long'): - event_obs_day = Time(timing_info[f'MTime_{tel_id}.fMjd'], format='mjd', scale='utc') + event_obs_day = Time( + timing_info[f"MTime_{tel_id}.fMjd"], + format="mjd", + scale="utc", + ) event_obs_day = np.round(event_obs_day.unix) - event_obs_day = np.array([Decimal(str(x)) for x in event_obs_day]) + event_obs_day = np.array( + [Decimal(str(x)) for x in event_obs_day] + ) - event_millisec = np.round(timing_info[f'MTime_{tel_id}.fTime.fMilliSec'] * msec2sec, 3) - event_millisec = np.array([Decimal(str(x)) for x in event_millisec]) + event_millisec = np.round( + timing_info[f"MTime_{tel_id}.fTime.fMilliSec"] * msec2sec, + 3, + ) + event_millisec = np.array( + [Decimal(str(x)) for x in event_millisec] + ) - event_nanosec = np.round(timing_info[f'MTime_{tel_id}.fNanoSec'] * nsec2sec, 7) - event_nanosec = np.array([Decimal(str(x)) for x in event_nanosec]) + event_nanosec = np.round( + timing_info[f"MTime_{tel_id}.fNanoSec"] * nsec2sec, 7 + ) + event_nanosec = np.array( + [Decimal(str(x)) for x in event_nanosec] + ) event_time = event_obs_day + event_millisec + event_nanosec - event_data[tel_id]['time'] = Time(event_time, format='unix', scale='utc') + event_data[tel_id]["time"] = Time( + event_time, format="unix", scale="utc" + ) # Reading the telescope pointing information: - pointing = self.uproot_file['Events'].arrays( + pointing = self.uproot_file["Events"].arrays( expressions=pointing_branches[tel_id], cut=events_cut, - library='np', + library="np", ) - pointing_az = pointing[f'MPointingPos_{tel_id}.fAz'] - pointing[f'MPointingPos_{tel_id}.fDevAz'] - pointing_zd = pointing[f'MPointingPos_{tel_id}.fZd'] - pointing[f'MPointingPos_{tel_id}.fDevZd'] + pointing_az = ( + pointing[f"MPointingPos_{tel_id}.fAz"] + - pointing[f"MPointingPos_{tel_id}.fDevAz"] + ) + pointing_zd = ( + pointing[f"MPointingPos_{tel_id}.fZd"] + - pointing[f"MPointingPos_{tel_id}.fDevZd"] + ) # N.B. the positive sign here, as HA = local sidereal time - ra: - pointing_ra = (pointing[f'MPointingPos_{tel_id}.fRa'] + pointing[f'MPointingPos_{tel_id}.fDevHa']) * degrees_per_hour - pointing_dec = pointing[f'MPointingPos_{tel_id}.fDec'] - pointing[f'MPointingPos_{tel_id}.fDevDec'] + pointing_ra = ( + pointing[f"MPointingPos_{tel_id}.fRa"] + + pointing[f"MPointingPos_{tel_id}.fDevHa"] + ) * degrees_per_hour + pointing_dec = ( + pointing[f"MPointingPos_{tel_id}.fDec"] + - pointing[f"MPointingPos_{tel_id}.fDevDec"] + ) - event_data[tel_id]['pointing_az'] = u.Quantity(pointing_az, u.deg) - event_data[tel_id]['pointing_alt'] = u.Quantity(90 - pointing_zd, u.deg) - event_data[tel_id]['pointing_ra'] = u.Quantity(pointing_ra, u.deg) - event_data[tel_id]['pointing_dec'] = u.Quantity(pointing_dec, u.deg) + event_data[tel_id]["pointing_az"] = u.Quantity(pointing_az, u.deg) + event_data[tel_id]["pointing_alt"] = u.Quantity( + 90 - pointing_zd, u.deg + ) + event_data[tel_id]["pointing_ra"] = u.Quantity(pointing_ra, u.deg) + event_data[tel_id]["pointing_dec"] = u.Quantity( + pointing_dec, u.deg + ) - hillas = self.uproot_file['Events'].arrays( + hillas = self.uproot_file["Events"].arrays( expressions=hillas_branches[tel_id], cut=events_cut, - library='np', + library="np", ) - event_data[tel_id]['length'] = u.Quantity(hillas[f'MHillas_{tel_id}.fLength'], u.mm).to(u.m) - event_data[tel_id]['width'] = u.Quantity(hillas[f'MHillas_{tel_id}.fWidth'], u.mm).to(u.m) - event_data[tel_id]['x'] = u.Quantity(hillas[f'MHillas_{tel_id}.fMeanX'], u.mm).to(u.m) - event_data[tel_id]['y'] = u.Quantity(hillas[f'MHillas_{tel_id}.fMeanY'], u.mm).to(u.m) - event_data[tel_id]['intensity'] = u.Quantity(hillas[f'MHillas_{tel_id}.fSize'], u.mm).to(u.m) - event_data[tel_id]['psi'] = u.Quantity(hillas[f'MHillas_{tel_id}.fDelta'], u.rad).to(u.deg) - - hillas_src = self.uproot_file['Events'].arrays( + event_data[tel_id]["length"] = u.Quantity( + hillas[f"MHillas_{tel_id}.fLength"], u.mm + ).to(u.m) + event_data[tel_id]["width"] = u.Quantity( + hillas[f"MHillas_{tel_id}.fWidth"], u.mm + ).to(u.m) + event_data[tel_id]["x"] = u.Quantity( + hillas[f"MHillas_{tel_id}.fMeanX"], u.mm + ).to(u.m) + event_data[tel_id]["y"] = u.Quantity( + hillas[f"MHillas_{tel_id}.fMeanY"], u.mm + ).to(u.m) + event_data[tel_id]["intensity"] = u.Quantity( + hillas[f"MHillas_{tel_id}.fSize"], u.mm + ).to(u.m) + event_data[tel_id]["psi"] = u.Quantity( + hillas[f"MHillas_{tel_id}.fDelta"], u.rad + ).to(u.deg) + + hillas_src = self.uproot_file["Events"].arrays( expressions=hillas_src_branches[tel_id], cut=events_cut, - library='np', + library="np", ) - event_data[tel_id]['phi'] = u.Quantity(hillas_src[f'MHillasSrc_{tel_id}.fDCADelta'], u.deg) + event_data[tel_id]["phi"] = u.Quantity( + hillas_src[f"MHillasSrc_{tel_id}.fDCADelta"], u.deg + ) - imagepar = self.uproot_file['Events'].arrays( + imagepar = self.uproot_file["Events"].arrays( expressions=imagepar_branches[tel_id], cut=events_cut, - library='np', + library="np", ) - event_data[tel_id]['num_islands'] = imagepar[f'MImagePar_{tel_id}.fNumIslands'] + event_data[tel_id]["num_islands"] = imagepar[ + f"MImagePar_{tel_id}.fNumIslands" + ] - new_imagepar = self.uproot_file['Events'].arrays( + new_imagepar = self.uproot_file["Events"].arrays( expressions=new_imagepar_branches[tel_id], cut=events_cut, - library='np', + library="np", ) - event_data[tel_id]['num_pixels'] = new_imagepar[f'MNewImagePar_{tel_id}.fNumUsedPixels'] - event_data[tel_id]['intensity_width_1'] = new_imagepar[f'MNewImagePar_{tel_id}.fLeakage1'] * event_data[tel_id]['intensity'] - event_data[tel_id]['intensity_width_2'] = new_imagepar[f'MNewImagePar_{tel_id}.fLeakage2'] * event_data[tel_id]['intensity'] + event_data[tel_id]["num_pixels"] = new_imagepar[ + f"MNewImagePar_{tel_id}.fNumUsedPixels" + ] + event_data[tel_id]["intensity_width_1"] = ( + new_imagepar[f"MNewImagePar_{tel_id}.fLeakage1"] + * event_data[tel_id]["intensity"] + ) + event_data[tel_id]["intensity_width_2"] = ( + new_imagepar[f"MNewImagePar_{tel_id}.fLeakage2"] + * event_data[tel_id]["intensity"] + ) - hillas_timefit = self.uproot_file['Events'].arrays( + hillas_timefit = self.uproot_file["Events"].arrays( expressions=hillas_timefit_branches[tel_id], cut=events_cut, - library='np', + library="np", ) - event_data[tel_id]['slope'] = u.Quantity(hillas_timefit[f'MHillasTimeFit_{tel_id}.fP1Grad'], 1/u.mm).to(1/u.m) * (1./sampling_speed) - event_data[tel_id]['intercept'] = hillas_timefit[f'MHillasTimeFit_{tel_id}.fP1Const'] * (1./sampling_speed) + event_data[tel_id]["slope"] = u.Quantity( + hillas_timefit[f"MHillasTimeFit_{tel_id}.fP1Grad"], 1 / u.mm + ).to(1 / u.m) * (1.0 / sampling_speed) + event_data[tel_id]["intercept"] = hillas_timefit[ + f"MHillasTimeFit_{tel_id}.fP1Const" + ] * (1.0 / sampling_speed) stereo_data["cosmic_events"] = event_data - stereo_parameters = self.uproot_file['Events'].arrays( + stereo_parameters = self.uproot_file["Events"].arrays( expressions=stereo_branches, cut=events_cut, - library='np', + library="np", ) stereo_params = dict() - stereo_params['alt'] = u.Quantity(90 - stereo_parameters['MStereoPar.fDirectionZd'], u.deg) - stereo_params['az'] = u.Quantity(stereo_parameters['MStereoPar.fDirectionAz'], u.deg) - stereo_params['core_x'] = u.Quantity(stereo_parameters['MStereoPar.fCoreX'], u.cm).to(u.m) - stereo_params['core_y'] = u.Quantity(stereo_parameters['MStereoPar.fCoreY'], u.cm).to(u.m) - stereo_params['h_max'] = u.Quantity(stereo_parameters['MStereoPar.fMaxHeight'], u.cm).to(u.m) - stereo_params['is_valid'] = stereo_parameters['MStereoPar.fValid'] + stereo_params["alt"] = u.Quantity( + 90 - stereo_parameters["MStereoPar.fDirectionZd"], u.deg + ) + stereo_params["az"] = u.Quantity( + stereo_parameters["MStereoPar.fDirectionAz"], u.deg + ) + stereo_params["core_x"] = u.Quantity( + stereo_parameters["MStereoPar.fCoreX"], u.cm + ).to(u.m) + stereo_params["core_y"] = u.Quantity( + stereo_parameters["MStereoPar.fCoreY"], u.cm + ).to(u.m) + stereo_params["h_max"] = u.Quantity( + stereo_parameters["MStereoPar.fMaxHeight"], u.cm + ).to(u.m) + stereo_params["is_valid"] = stereo_parameters["MStereoPar.fValid"] stereo_data["cosmic_events"]["stereo"] = stereo_params @@ -2012,7 +2591,7 @@ def __init__(self, uproot_file, is_mc, n_cam_pixels=1039): @property def n_cosmic_events(self): - return len(self.cosmic_events[1].get('trigger_pattern', [])) + return len(self.cosmic_events[1].get("trigger_pattern", [])) @property def n_pedestal_events(self): @@ -2044,117 +2623,117 @@ def _load_data(self): for tel_id in [1, 2]: common_branches[tel_id] = [ - f'MRawEvtHeader_{tel_id}.fStereoEvtNumber', - f'MTriggerPattern_{tel_id}.fPrescaled', + f"MRawEvtHeader_{tel_id}.fStereoEvtNumber", + f"MTriggerPattern_{tel_id}.fPrescaled", ] pointing_branches[tel_id] = [ - f'MPointingPos_{tel_id}.fZd', # [deg] - f'MPointingPos_{tel_id}.fAz', # [deg] - f'MPointingPos_{tel_id}.fRa', # [h] - f'MPointingPos_{tel_id}.fDec', # [deg] - f'MPointingPos_{tel_id}.fDevZd', # [deg] - f'MPointingPos_{tel_id}.fDevAz', # [deg] - f'MPointingPos_{tel_id}.fDevHa', # [h] - f'MPointingPos_{tel_id}.fDevDec', # [deg] + f"MPointingPos_{tel_id}.fZd", # [deg] + f"MPointingPos_{tel_id}.fAz", # [deg] + f"MPointingPos_{tel_id}.fRa", # [h] + f"MPointingPos_{tel_id}.fDec", # [deg] + f"MPointingPos_{tel_id}.fDevZd", # [deg] + f"MPointingPos_{tel_id}.fDevAz", # [deg] + f"MPointingPos_{tel_id}.fDevHa", # [h] + f"MPointingPos_{tel_id}.fDevDec", # [deg] ] # Branches applicable for real data: timing_branches[tel_id] = [ - f'MTime_{tel_id}.fMjd', - f'MTime_{tel_id}.fTime.fMilliSec', - f'MTime_{tel_id}.fNanoSec', + f"MTime_{tel_id}.fMjd", + f"MTime_{tel_id}.fTime.fMilliSec", + f"MTime_{tel_id}.fNanoSec", ] hillas_branches[tel_id] = [ - f'MHillas_{tel_id}.fLength', # [mm] - f'MHillas_{tel_id}.fWidth', # [mm] - f'MHillas_{tel_id}.fDelta', # [rad] - f'MHillas_{tel_id}.fSize', - f'MHillas_{tel_id}.fMeanX', # [mm] - f'MHillas_{tel_id}.fMeanY', # [mm] + f"MHillas_{tel_id}.fLength", # [mm] + f"MHillas_{tel_id}.fWidth", # [mm] + f"MHillas_{tel_id}.fDelta", # [rad] + f"MHillas_{tel_id}.fSize", + f"MHillas_{tel_id}.fMeanX", # [mm] + f"MHillas_{tel_id}.fMeanY", # [mm] ] hillas_src_branches[tel_id] = [ - f'MHillasSrc_{tel_id}.fDCADelta', # [deg] Angle of the shower axis with respect to the x-axis - f'MHillasSrc_{tel_id}.fDist', # [mm] distance between src and center of ellipse + f"MHillasSrc_{tel_id}.fDCADelta", # [deg] Angle of the shower axis with respect to the x-axis + f"MHillasSrc_{tel_id}.fDist", # [mm] distance between src and center of ellipse ] imagepar_branches[tel_id] = [ - f'MImagePar_{tel_id}.fNumIslands', # Number of distinct pixel groupings in image + f"MImagePar_{tel_id}.fNumIslands", # Number of distinct pixel groupings in image ] new_imagepar_branches[tel_id] = [ - f'MNewImagePar_{tel_id}.fNumUsedPixels', # Number of pixels which survived the image cleaning - f'MNewImagePar_{tel_id}.fLeakage1', # (photons in most outer ring of pixels) over fSize - f'MNewImagePar_{tel_id}.fLeakage2', # (photons in the 2 outer rings of pixels) over fSize + f"MNewImagePar_{tel_id}.fNumUsedPixels", # Number of pixels which survived the image cleaning + f"MNewImagePar_{tel_id}.fLeakage1", # (photons in most outer ring of pixels) over fSize + f"MNewImagePar_{tel_id}.fLeakage2", # (photons in the 2 outer rings of pixels) over fSize ] hillas_timefit_branches[tel_id] = [ - f'MHillasTimeFit_{tel_id}.fP1Const', # [Time_slices] - f'MHillasTimeFit_{tel_id}.fP1Grad', # [Time_slices]/[mm] + f"MHillasTimeFit_{tel_id}.fP1Const", # [Time_slices] + f"MHillasTimeFit_{tel_id}.fP1Grad", # [Time_slices]/[mm] ] mc_branches[tel_id] = [ - f'MMcEvt_{tel_id}.fEnergy', - f'MMcEvt_{tel_id}.fTheta', - f'MMcEvt_{tel_id}.fPhi', - f'MMcEvt_{tel_id}.fPartId', - f'MMcEvt_{tel_id}.fZFirstInteraction', - f'MMcEvt_{tel_id}.fCoreX', - f'MMcEvt_{tel_id}.fCoreY', + f"MMcEvt_{tel_id}.fEnergy", + f"MMcEvt_{tel_id}.fTheta", + f"MMcEvt_{tel_id}.fPhi", + f"MMcEvt_{tel_id}.fPartId", + f"MMcEvt_{tel_id}.fZFirstInteraction", + f"MMcEvt_{tel_id}.fCoreX", + f"MMcEvt_{tel_id}.fCoreY", ] imagepar_disp_branches[tel_id] = [ - f'MImageParDisp_{tel_id}.fDisp', - f'MImageParDisp_{tel_id}.fDispRMS', - f'MImageParDisp_{tel_id}.fXDisp', - f'MImageParDisp_{tel_id}.fYDisp', + f"MImageParDisp_{tel_id}.fDisp", + f"MImageParDisp_{tel_id}.fDispRMS", + f"MImageParDisp_{tel_id}.fXDisp", + f"MImageParDisp_{tel_id}.fYDisp", ] stereo_branches = [ - 'MStereoParDisp.fValid', - 'MStereoParDisp.fDirectionX', - 'MStereoParDisp.fDirectionY', - 'MStereoParDisp.fDirectionZd', - 'MStereoParDisp.fDirectionAz', - 'MStereoParDisp.fDirectionDec', - 'MStereoParDisp.fDirectionRA', - 'MStereoParDisp.fTheta2', - 'MStereoParDisp.fCoreX', - 'MStereoParDisp.fCoreY', - 'MStereoParDisp.fM1Impact', - 'MStereoParDisp.fM2Impact', - 'MStereoParDisp.fMaxHeight', + "MStereoParDisp.fValid", + "MStereoParDisp.fDirectionX", + "MStereoParDisp.fDirectionY", + "MStereoParDisp.fDirectionZd", + "MStereoParDisp.fDirectionAz", + "MStereoParDisp.fDirectionDec", + "MStereoParDisp.fDirectionRA", + "MStereoParDisp.fTheta2", + "MStereoParDisp.fCoreX", + "MStereoParDisp.fCoreY", + "MStereoParDisp.fM1Impact", + "MStereoParDisp.fM2Impact", + "MStereoParDisp.fMaxHeight", ] if self.is_mc: melibea_branches = [ - 'MHadronness.fHadronness', - 'MHadronness.fHadronnessRMS', - 'MEnergyEst.fEnergy', - 'MEnergyEst.fEnergyRMS', - 'MEnergyEst.fUncertainty', - 'MEnergyEst.fImpact', + "MHadronness.fHadronness", + "MHadronness.fHadronnessRMS", + "MEnergyEst.fEnergy", + "MEnergyEst.fEnergyRMS", + "MEnergyEst.fUncertainty", + "MEnergyEst.fImpact", ] else: melibea_branches = [ - 'MHadronness.fHadronness', - 'MHadronness.fHadronnessRMS', - 'MEnergyEst.fEnergy', - 'MEnergyEst.fEnergyRMS', - 'MEnergyEst.fUncertainty', - 'MEnergyEst.fImpact', - 'MEnergyEstAtmCorr.fEnergy', - 'MEnergyEstAtmCorr.fEnergyRMS', - 'MEnergyEstAtmCorr.fImpact', + "MHadronness.fHadronness", + "MHadronness.fHadronnessRMS", + "MEnergyEst.fEnergy", + "MEnergyEst.fEnergyRMS", + "MEnergyEst.fUncertainty", + "MEnergyEst.fImpact", + "MEnergyEstAtmCorr.fEnergy", + "MEnergyEstAtmCorr.fEnergyRMS", + "MEnergyEstAtmCorr.fImpact", ] stereo_data = { - 'cosmic_events': dict(), + "cosmic_events": dict(), } event_data = dict() @@ -2167,164 +2746,267 @@ def _load_data(self): if self.is_mc: # Only for cosmic events because MC data do not have pedestal events: - events_cut = f'(MTriggerPattern_{tel_id}.fPrescaled == {MC_STEREO_TRIGGER_PATTERN})' \ - f' & (MRawEvtHeader_{tel_id}.fStereoEvtNumber != 0)' + events_cut = ( + f"(MTriggerPattern_{tel_id}.fPrescaled == {MC_STEREO_TRIGGER_PATTERN})" + f" & (MRawEvtHeader_{tel_id}.fStereoEvtNumber != 0)" + ) else: - events_cut = f'(MTriggerPattern_{tel_id}.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})' + events_cut = f"(MTriggerPattern_{tel_id}.fPrescaled == {DATA_STEREO_TRIGGER_PATTERN})" # read common information from RunHeaders - sampling_speed = self.uproot_file['RunHeaders'][f'MRawRunHeader_{tel_id}.fSamplingFrequency'].array(library='np')[0]/1000. # GHz + sampling_speed = ( + self.uproot_file["RunHeaders"][ + f"MRawRunHeader_{tel_id}.fSamplingFrequency" + ].array(library="np")[0] + / 1000.0 + ) # GHz # Reading the information common to cosmic and pedestal events: - common_info = self.uproot_file['Events'].arrays( + common_info = self.uproot_file["Events"].arrays( expressions=common_branches[tel_id], cut=events_cut, - library='np', + library="np", ) - event_data[tel_id]['trigger_pattern'] = np.array(common_info[f'MTriggerPattern_{tel_id}.fPrescaled'], dtype=int) - event_data[tel_id]['stereo_event_number'] = np.array(common_info[f'MRawEvtHeader_{tel_id}.fStereoEvtNumber'], dtype=int) + event_data[tel_id]["trigger_pattern"] = np.array( + common_info[f"MTriggerPattern_{tel_id}.fPrescaled"], dtype=int + ) + event_data[tel_id]["stereo_event_number"] = np.array( + common_info[f"MRawEvtHeader_{tel_id}.fStereoEvtNumber"], + dtype=int, + ) if self.is_mc: # Reading the MC information: - mc_info = self.uproot_file['Events'].arrays( + mc_info = self.uproot_file["Events"].arrays( expressions=mc_branches[tel_id], cut=events_cut, - library='np', + library="np", ) # Note that the branch "MMcEvt.fPhi" seems to be the angle between the direction # of the magnetic north and the momentum of a simulated primary particle, defined # between -pi and pi, negative if the momentum pointing eastward, positive westward. # The conversion to azimuth should be 180 - fPhi + magnetic_declination: - event_data[tel_id]['mc_energy'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fEnergy'], u.GeV) - event_data[tel_id]['mc_theta'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fTheta'], u.rad) - event_data[tel_id]['mc_phi'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fPhi'], u.rad) - event_data[tel_id]['mc_shower_primary_id'] = np.array(mc_info[f'MMcEvt_{tel_id}.fPartId'], dtype=int) - event_data[tel_id]['mc_h_first_int'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fZFirstInteraction'], u.cm) - event_data[tel_id]['mc_core_x'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fCoreX'], u.cm) - event_data[tel_id]['mc_core_y'] = u.Quantity(mc_info[f'MMcEvt_{tel_id}.fCoreY'], u.cm) + event_data[tel_id]["mc_energy"] = u.Quantity( + mc_info[f"MMcEvt_{tel_id}.fEnergy"], u.GeV + ) + event_data[tel_id]["mc_theta"] = u.Quantity( + mc_info[f"MMcEvt_{tel_id}.fTheta"], u.rad + ) + event_data[tel_id]["mc_phi"] = u.Quantity( + mc_info[f"MMcEvt_{tel_id}.fPhi"], u.rad + ) + event_data[tel_id]["mc_shower_primary_id"] = np.array( + mc_info[f"MMcEvt_{tel_id}.fPartId"], dtype=int + ) + event_data[tel_id]["mc_h_first_int"] = u.Quantity( + mc_info[f"MMcEvt_{tel_id}.fZFirstInteraction"], u.cm + ) + event_data[tel_id]["mc_core_x"] = u.Quantity( + mc_info[f"MMcEvt_{tel_id}.fCoreX"], u.cm + ) + event_data[tel_id]["mc_core_y"] = u.Quantity( + mc_info[f"MMcEvt_{tel_id}.fCoreY"], u.cm + ) else: # Reading the event timing information: - timing_info = self.uproot_file['Events'].arrays( + timing_info = self.uproot_file["Events"].arrays( expressions=timing_branches[tel_id], cut=events_cut, - library='np', + library="np", ) # In order to keep the precision of timestamps, here the Decimal module is used. # At the later steps, the precise information can be extracted by specifying # the sub-format of value to 'long', i.e., Time.to_value(format='unix', subfmt='long'): - event_obs_day = Time(timing_info[f'MTime_{tel_id}.fMjd'], format='mjd', scale='utc') + event_obs_day = Time( + timing_info[f"MTime_{tel_id}.fMjd"], + format="mjd", + scale="utc", + ) event_obs_day = np.round(event_obs_day.unix) - event_obs_day = np.array([Decimal(str(x)) for x in event_obs_day]) + event_obs_day = np.array( + [Decimal(str(x)) for x in event_obs_day] + ) - event_millisec = np.round(timing_info[f'MTime_{tel_id}.fTime.fMilliSec'] * msec2sec, 3) - event_millisec = np.array([Decimal(str(x)) for x in event_millisec]) + event_millisec = np.round( + timing_info[f"MTime_{tel_id}.fTime.fMilliSec"] * msec2sec, + 3, + ) + event_millisec = np.array( + [Decimal(str(x)) for x in event_millisec] + ) - event_nanosec = np.round(timing_info[f'MTime_{tel_id}.fNanoSec'] * nsec2sec, 7) - event_nanosec = np.array([Decimal(str(x)) for x in event_nanosec]) + event_nanosec = np.round( + timing_info[f"MTime_{tel_id}.fNanoSec"] * nsec2sec, 7 + ) + event_nanosec = np.array( + [Decimal(str(x)) for x in event_nanosec] + ) event_time = event_obs_day + event_millisec + event_nanosec - event_data[tel_id]['time'] = Time(event_time, format='unix', scale='utc') + event_data[tel_id]["time"] = Time( + event_time, format="unix", scale="utc" + ) # Reading the telescope pointing information: - pointing = self.uproot_file['Events'].arrays( + pointing = self.uproot_file["Events"].arrays( expressions=pointing_branches[tel_id], cut=events_cut, - library='np', + library="np", ) - pointing_az = pointing[f'MPointingPos_{tel_id}.fAz'] - pointing[f'MPointingPos_{tel_id}.fDevAz'] - pointing_zd = pointing[f'MPointingPos_{tel_id}.fZd'] - pointing[f'MPointingPos_{tel_id}.fDevZd'] + pointing_az = ( + pointing[f"MPointingPos_{tel_id}.fAz"] + - pointing[f"MPointingPos_{tel_id}.fDevAz"] + ) + pointing_zd = ( + pointing[f"MPointingPos_{tel_id}.fZd"] + - pointing[f"MPointingPos_{tel_id}.fDevZd"] + ) # N.B. the positive sign here, as HA = local sidereal time - ra: - pointing_ra = (pointing[f'MPointingPos_{tel_id}.fRa'] + pointing[f'MPointingPos_{tel_id}.fDevHa']) * degrees_per_hour - pointing_dec = pointing[f'MPointingPos_{tel_id}.fDec'] - pointing[f'MPointingPos_{tel_id}.fDevDec'] + pointing_ra = ( + pointing[f"MPointingPos_{tel_id}.fRa"] + + pointing[f"MPointingPos_{tel_id}.fDevHa"] + ) * degrees_per_hour + pointing_dec = ( + pointing[f"MPointingPos_{tel_id}.fDec"] + - pointing[f"MPointingPos_{tel_id}.fDevDec"] + ) - event_data[tel_id]['pointing_az'] = u.Quantity(pointing_az, u.deg) - event_data[tel_id]['pointing_alt'] = u.Quantity(90 - pointing_zd, u.deg) - event_data[tel_id]['pointing_ra'] = u.Quantity(pointing_ra, u.deg) - event_data[tel_id]['pointing_dec'] = u.Quantity(pointing_dec, u.deg) + event_data[tel_id]["pointing_az"] = u.Quantity(pointing_az, u.deg) + event_data[tel_id]["pointing_alt"] = u.Quantity( + 90 - pointing_zd, u.deg + ) + event_data[tel_id]["pointing_ra"] = u.Quantity(pointing_ra, u.deg) + event_data[tel_id]["pointing_dec"] = u.Quantity( + pointing_dec, u.deg + ) - hillas = self.uproot_file['Events'].arrays( + hillas = self.uproot_file["Events"].arrays( expressions=hillas_branches[tel_id], cut=events_cut, - library='np', + library="np", ) - event_data[tel_id]['length'] = u.Quantity(hillas[f'MHillas_{tel_id}.fLength'], u.mm).to(u.m) - event_data[tel_id]['width'] = u.Quantity(hillas[f'MHillas_{tel_id}.fWidth'], u.mm).to(u.m) - event_data[tel_id]['x'] = u.Quantity(hillas[f'MHillas_{tel_id}.fMeanX'], u.mm).to(u.m) - event_data[tel_id]['y'] = u.Quantity(hillas[f'MHillas_{tel_id}.fMeanY'], u.mm).to(u.m) - event_data[tel_id]['intensity'] = u.Quantity(hillas[f'MHillas_{tel_id}.fSize'], u.mm).to(u.m) - event_data[tel_id]['psi'] = u.Quantity(hillas[f'MHillas_{tel_id}.fDelta'], u.rad).to(u.deg) - - hillas_src = self.uproot_file['Events'].arrays( + event_data[tel_id]["length"] = u.Quantity( + hillas[f"MHillas_{tel_id}.fLength"], u.mm + ).to(u.m) + event_data[tel_id]["width"] = u.Quantity( + hillas[f"MHillas_{tel_id}.fWidth"], u.mm + ).to(u.m) + event_data[tel_id]["x"] = u.Quantity( + hillas[f"MHillas_{tel_id}.fMeanX"], u.mm + ).to(u.m) + event_data[tel_id]["y"] = u.Quantity( + hillas[f"MHillas_{tel_id}.fMeanY"], u.mm + ).to(u.m) + event_data[tel_id]["intensity"] = u.Quantity( + hillas[f"MHillas_{tel_id}.fSize"], u.mm + ).to(u.m) + event_data[tel_id]["psi"] = u.Quantity( + hillas[f"MHillas_{tel_id}.fDelta"], u.rad + ).to(u.deg) + + hillas_src = self.uproot_file["Events"].arrays( expressions=hillas_src_branches[tel_id], cut=events_cut, - library='np', + library="np", ) - event_data[tel_id]['phi'] = u.Quantity(hillas_src[f'MHillasSrc_{tel_id}.fDCADelta'], u.deg) + event_data[tel_id]["phi"] = u.Quantity( + hillas_src[f"MHillasSrc_{tel_id}.fDCADelta"], u.deg + ) - imagepar = self.uproot_file['Events'].arrays( + imagepar = self.uproot_file["Events"].arrays( expressions=imagepar_branches[tel_id], cut=events_cut, - library='np', + library="np", ) - event_data[tel_id]['num_islands'] = imagepar[f'MImagePar_{tel_id}.fNumIslands'] + event_data[tel_id]["num_islands"] = imagepar[ + f"MImagePar_{tel_id}.fNumIslands" + ] - new_imagepar = self.uproot_file['Events'].arrays( + new_imagepar = self.uproot_file["Events"].arrays( expressions=new_imagepar_branches[tel_id], cut=events_cut, - library='np', + library="np", ) - event_data[tel_id]['num_pixels'] = new_imagepar[f'MNewImagePar_{tel_id}.fNumUsedPixels'] - event_data[tel_id]['intensity_width_1'] = new_imagepar[f'MNewImagePar_{tel_id}.fLeakage1'] * event_data[tel_id]['intensity'] - event_data[tel_id]['intensity_width_2'] = new_imagepar[f'MNewImagePar_{tel_id}.fLeakage2'] * event_data[tel_id]['intensity'] + event_data[tel_id]["num_pixels"] = new_imagepar[ + f"MNewImagePar_{tel_id}.fNumUsedPixels" + ] + event_data[tel_id]["intensity_width_1"] = ( + new_imagepar[f"MNewImagePar_{tel_id}.fLeakage1"] + * event_data[tel_id]["intensity"] + ) + event_data[tel_id]["intensity_width_2"] = ( + new_imagepar[f"MNewImagePar_{tel_id}.fLeakage2"] + * event_data[tel_id]["intensity"] + ) - hillas_timefit = self.uproot_file['Events'].arrays( + hillas_timefit = self.uproot_file["Events"].arrays( expressions=hillas_timefit_branches[tel_id], cut=events_cut, - library='np', + library="np", ) - event_data[tel_id]['slope'] = u.Quantity(hillas_timefit[f'MHillasTimeFit_{tel_id}.fP1Grad'], 1/u.mm).to(1/u.m) * (1./sampling_speed) - event_data[tel_id]['intercept'] = hillas_timefit[f'MHillasTimeFit_{tel_id}.fP1Const'] * (1./sampling_speed) + event_data[tel_id]["slope"] = u.Quantity( + hillas_timefit[f"MHillasTimeFit_{tel_id}.fP1Grad"], 1 / u.mm + ).to(1 / u.m) * (1.0 / sampling_speed) + event_data[tel_id]["intercept"] = hillas_timefit[ + f"MHillasTimeFit_{tel_id}.fP1Const" + ] * (1.0 / sampling_speed) stereo_data["cosmic_events"] = event_data - stereo_parameters = self.uproot_file['Events'].arrays( + stereo_parameters = self.uproot_file["Events"].arrays( expressions=stereo_branches, cut=events_cut, - library='np', + library="np", ) stereo_params = dict() - stereo_params['alt'] = u.Quantity(90 - stereo_parameters['MStereoParDisp.fDirectionZd'], u.deg) - stereo_params['az'] = u.Quantity(stereo_parameters['MStereoParDisp.fDirectionAz'], u.deg) - stereo_params['core_x'] = u.Quantity(stereo_parameters['MStereoParDisp.fCoreX'], u.cm).to(u.m) - stereo_params['core_y'] = u.Quantity(stereo_parameters['MStereoParDisp.fCoreY'], u.cm).to(u.m) - stereo_params['h_max'] = u.Quantity(stereo_parameters['MStereoParDisp.fMaxHeight'], u.cm).to(u.m) - stereo_params['is_valid'] = stereo_parameters['MStereoParDisp.fValid'] + stereo_params["alt"] = u.Quantity( + 90 - stereo_parameters["MStereoParDisp.fDirectionZd"], u.deg + ) + stereo_params["az"] = u.Quantity( + stereo_parameters["MStereoParDisp.fDirectionAz"], u.deg + ) + stereo_params["core_x"] = u.Quantity( + stereo_parameters["MStereoParDisp.fCoreX"], u.cm + ).to(u.m) + stereo_params["core_y"] = u.Quantity( + stereo_parameters["MStereoParDisp.fCoreY"], u.cm + ).to(u.m) + stereo_params["h_max"] = u.Quantity( + stereo_parameters["MStereoParDisp.fMaxHeight"], u.cm + ).to(u.m) + stereo_params["is_valid"] = stereo_parameters["MStereoParDisp.fValid"] stereo_data["cosmic_events"]["stereo"] = stereo_params - melibea_parameters = self.uproot_file['Events'].arrays( + melibea_parameters = self.uproot_file["Events"].arrays( expressions=melibea_branches, cut=events_cut, - library='np', + library="np", ) melibea_params = dict() - melibea_params['gammanness'] = 1.0 - melibea_parameters['MHadronness.fHadronness'] - melibea_params['energy'] = u.Quantity(melibea_parameters['MEnergyEst.fEnergy'], u.GeV).to(u.TeV) - melibea_params['energy_uncert'] = u.Quantity(melibea_parameters['MEnergyEst.fUncertainty'], u.GeV).to(u.TeV) + melibea_params["gammanness"] = ( + 1.0 - melibea_parameters["MHadronness.fHadronness"] + ) + melibea_params["energy"] = u.Quantity( + melibea_parameters["MEnergyEst.fEnergy"], u.GeV + ).to(u.TeV) + melibea_params["energy_uncert"] = u.Quantity( + melibea_parameters["MEnergyEst.fUncertainty"], u.GeV + ).to(u.TeV) stereo_data["cosmic_events"]["reconstructed"] = melibea_params From bd09c90d7fc9991c02b64da701e4339bfb2ea617 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sat, 23 Jul 2022 21:47:58 +0200 Subject: [PATCH 53/78] Fix download script. --- download_test_data.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/download_test_data.sh b/download_test_data.sh index 4d44426..3ca61a9 100644 --- a/download_test_data.sh +++ b/download_test_data.sh @@ -9,7 +9,7 @@ echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/cal echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/superstar/20210314_05095172_S_CrabNebula-W0.40+035.root" > test_data_superstar_real.txt -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/superstar/20210314_05095172_Q_CrabNebula-W0.40+035.root" > test_data_melibea_real.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/melibea/20210314_05095172_Q_CrabNebula-W0.40+035.root" > test_data_melibea_real.txt echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M1_za35to50_8_824318_Y_w0.root" > test_data_simulated.txt echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M1_za35to50_8_824319_Y_w0.root" >> test_data_simulated.txt @@ -19,8 +19,8 @@ echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulate echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824318_S_w0.root" > test_data_superstar_simulated.txt echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824319_S_w0.root" >> test_data_superstar_simulated.txt -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824318_Q_w0.root" > test_data_melibea_simulated.txt -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824319_Q_w0.root" >> test_data_melibea_simulated.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/melibea/GA_za35to50_8_824318_Q_w0.root" > test_data_melibea_simulated.txt +echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/melibea/GA_za35to50_8_824319_Q_w0.root" >> test_data_melibea_simulated.txt if [ -z "$TEST_DATA_USER" ]; then echo -n "Username: " @@ -63,7 +63,7 @@ if ! wget \ --no-check-certificate \ --no-verbose \ --timestamping \ - --directory-prefix=test_data/real/superstar; then + --directory-prefix=test_data/real/melibea; then echo "Problem in downloading the test data set for real data." fi @@ -96,7 +96,7 @@ if ! wget \ --no-check-certificate \ --no-verbose \ --timestamping \ - --directory-prefix=test_data/simulated/superstar; then + --directory-prefix=test_data/simulated/melibea; then echo "Problem in downloading the test data set for simulated data." fi From 48d08619518be0e86fdf7f6df66de1702551901d Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sun, 24 Jul 2022 15:17:59 +0200 Subject: [PATCH 54/78] Add codacy configuration file. --- .codacy.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .codacy.yml diff --git a/.codacy.yml b/.codacy.yml new file mode 100644 index 0000000..d5e0aad --- /dev/null +++ b/.codacy.yml @@ -0,0 +1,11 @@ +--- +engines: + pylint: + enabled: true + python_version: 3 + pyflakes: + disable: + - F999 + +exclude_paths: + - "ctapipe_io_magic/tests/**" From 09729f644a63ce73f6042850093e7d59d22b6f00 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Sun, 24 Jul 2022 15:35:59 +0200 Subject: [PATCH 55/78] Change name of function (remove trailing _). --- ctapipe_io_magic/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 24b01ec..1c337f2 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1177,7 +1177,7 @@ def obs_ids(self): # ToCheck: will this be compatible in the future, e.g. with merged MC files return self.run_numbers - def _get_badrmspixel_mask(self, event, tel_id): + def get_badrmspixel_mask(self, event, tel_id): """ Fetch bad RMS pixel mask for a given event. @@ -1517,7 +1517,7 @@ def _event_generator(self): not self.use_pedestals and self.mars_datalevel == MARSDataLevel.CALIBRATED ): - badrmspixel_mask = self._get_badrmspixel_mask( + badrmspixel_mask = self.get_badrmspixel_mask( event, tel_id ) event.mon.tel[ From aedd20a7b1abd6d0260395f528ad2c42c3fbf809 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 8 Mar 2023 09:18:03 +0100 Subject: [PATCH 56/78] Fix reading of passwords with special characters from prompt. --- download_test_data.sh | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) mode change 100644 => 100755 download_test_data.sh diff --git a/download_test_data.sh b/download_test_data.sh old mode 100644 new mode 100755 index 3ca61a9..0e1c1b5 --- a/download_test_data.sh +++ b/download_test_data.sh @@ -23,14 +23,12 @@ echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulate echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/melibea/GA_za35to50_8_824319_Q_w0.root" >> test_data_melibea_simulated.txt if [ -z "$TEST_DATA_USER" ]; then - echo -n "Username: " - read TEST_DATA_USER + read -p "Username: " TEST_DATA_USER echo fi if [ -z "$TEST_DATA_PASSWORD" ]; then - echo -n "Password: " - read -s TEST_DATA_PASSWORD + read -sr -p "Password: " TEST_DATA_PASSWORD echo fi @@ -42,7 +40,7 @@ if ! wget \ --no-verbose \ --timestamping \ --directory-prefix=test_data/real/calibrated; then - echo "Problem in downloading the test data set for real data." + echo "Problem in downloading the test data set (calibrated) for real data." fi if ! wget \ @@ -53,7 +51,7 @@ if ! wget \ --no-verbose \ --timestamping \ --directory-prefix=test_data/real/superstar; then - echo "Problem in downloading the test data set for real data." + echo "Problem in downloading the test data set (superstar) for real data." fi if ! wget \ @@ -64,7 +62,7 @@ if ! wget \ --no-verbose \ --timestamping \ --directory-prefix=test_data/real/melibea; then - echo "Problem in downloading the test data set for real data." + echo "Problem in downloading the test data set (melibea) for real data." fi if ! wget \ @@ -75,7 +73,7 @@ if ! wget \ --no-verbose \ --timestamping \ --directory-prefix=test_data/simulated/calibrated; then - echo "Problem in downloading the test data set for simulated data." + echo "Problem in downloading the test data set (calibrated) for simulated data." fi if ! wget \ @@ -86,7 +84,7 @@ if ! wget \ --no-verbose \ --timestamping \ --directory-prefix=test_data/simulated/superstar; then - echo "Problem in downloading the test data set for simulated data." + echo "Problem in downloading the test data set (superstar) for simulated data." fi if ! wget \ @@ -97,7 +95,7 @@ if ! wget \ --no-verbose \ --timestamping \ --directory-prefix=test_data/simulated/melibea; then - echo "Problem in downloading the test data set for simulated data." + echo "Problem in downloading the test data set (melibea) for simulated data." fi rm -f test_data_real.txt test_data_simulated.txt test_data_superstar_real.txt test_data_superstar_simulated.txt test_data_melibea_real.txt test_data_melibea_simulated.txt From 35ee34e29c738a99ad84b7498508b6a032d3c40f Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Mon, 13 Mar 2023 22:47:44 +0100 Subject: [PATCH 57/78] Changes to make tests pass. --- ctapipe_io_magic/__init__.py | 774 ++++++++++++++++------------------- 1 file changed, 355 insertions(+), 419 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 89206f4..8c40c6c 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -65,11 +65,7 @@ DATA_STEREO_TRIGGER_PATTERN, ) -__all__ = [ - 'MAGICEventSource', - 'MARSDataLevel', - '__version__' -] +__all__ = ["MAGICEventSource", "MARSDataLevel", "__version__"] logger = logging.getLogger(__name__) @@ -95,9 +91,7 @@ def load_camera_geometry(): """Load camera geometry from bundled resources of this repo""" - f = resource_filename( - "ctapipe_io_magic", "resources/MAGICCam.camgeom.fits.gz" - ) + f = resource_filename("ctapipe_io_magic", "resources/MAGICCam.camgeom.fits.gz") Provenance().add_input_file(f, role="CameraGeometry") return CameraGeometry.from_table(f) @@ -151,7 +145,7 @@ class MAGICEventSource(EventSource): use_mc_mono_events = Bool( default_value=False, - help='Use mono events in MC stereo data (needed for mono analysis).' + help="Use mono events in MC stereo data (needed for mono analysis).", ).tag(config=True) focal_length_choice = CaselessStrEnum( @@ -180,9 +174,7 @@ def __init__(self, input_url=None, config=None, parent=None, **kwargs): the 'input_url' parameter. """ - super().__init__( - input_url=input_url, config=config, parent=parent, **kwargs - ) + super().__init__(input_url=input_url, config=config, parent=parent, **kwargs) path, name = os.path.split(os.path.abspath(self.input_url)) info_tuple = self.get_run_info_from_name(name) @@ -245,12 +237,16 @@ def __init__(self, input_url=None, config=None, parent=None, **kwargs): self.is_stereo, self.is_sumt = self.parse_data_info() if self.is_simulation and self.use_mc_mono_events and not self.is_stereo: - logger.warning("Processing mono simulation data with" \ - " use_mc_mono_events=True. use_mc_mono_events will be ignored.") + logger.warning( + "Processing mono simulation data with" + " use_mc_mono_events=True. use_mc_mono_events will be ignored." + ) if not self.is_simulation and self.use_mc_mono_events: - logger.warning("Processing real data with" \ - " use_mc_mono_events=True. use_mc_mono_events will be ignored.") + logger.warning( + "Processing real data with" + " use_mc_mono_events=True. use_mc_mono_events will be ignored." + ) # # Setting up the current run with the first run present in the data # self.current_run = self._set_active_run(run_number=0) @@ -312,9 +308,7 @@ def is_compatible(file_path): try: with uproot.open(file_path) as input_data: mandatory_trees = ["Events", "RunHeaders", "RunTails"] - trees_in_file = [ - tree in input_data for tree in mandatory_trees - ] + trees_in_file = [tree in input_data for tree in mandatory_trees] if not all(trees_in_file): is_magic_root_file = False except ValueError: @@ -412,7 +406,7 @@ def get_run_info_from_name(file_name): raise IndexError( f"Can not identify the run number and type (data/MC) " f"of the file {file_name}." - ) + ) return run_number, is_mc, telescope, datalevel @@ -473,9 +467,7 @@ def parse_run_info(self): datalevel = MARSDataLevel.MELIBEA if datalevel <= MARSDataLevel.STAR: - run_info = rootf["RunHeaders"].arrays( - runinfo_array_list, library="np" - ) + run_info = rootf["RunHeaders"].arrays(runinfo_array_list, library="np") run_number = int(run_info["MRawRunHeader.fRunNumber"][0]) run_type = int(run_info["MRawRunHeader.fRunType"][0]) @@ -575,9 +567,13 @@ def parse_data_info(self): # here we take the 2nd element (if possible) because sometimes # the first trigger report has still the old prescaler values from a previous run try: - prescaler_array = trigger_tree["MTriggerPrescFact.fPrescFact"].array(library="np") + prescaler_array = trigger_tree[ + "MTriggerPrescFact.fPrescFact" + ].array(library="np") except AssertionError: - logger.warning("No prescaler info found. Will assume standard stereo data.") + logger.warning( + "No prescaler info found. Will assume standard stereo data." + ) stereo = True sumt = False return stereo, sumt @@ -588,7 +584,10 @@ def parse_data_info(self): else: prescaler = list(prescaler_array[0]) - if prescaler == prescaler_mono_nosumt or prescaler == prescaler_mono_sumt: + if ( + prescaler == prescaler_mono_nosumt + or prescaler == prescaler_mono_sumt + ): stereo = False elif prescaler == prescaler_stereo: stereo = True @@ -600,7 +599,9 @@ def parse_data_info(self): # here we take the 2nd element for the same reason as above # L3Table is empty for mono data i.e. taken with one telescope only # if both telescopes take data with no L3, L3Table is filled anyway - L3Table_array = L3T_tree["MReportL3T.fTablename"].array(library="np") + L3Table_array = L3T_tree["MReportL3T.fTablename"].array( + library="np" + ) L3Table_size = L3Table_array.size if L3Table_size > 1: L3Table = L3Table_array[1] @@ -624,7 +625,9 @@ def parse_data_info(self): stereo = True - L3Table_array = L3T_tree["MReportL3T.fTablename"].array(library="np") + L3Table_array = L3T_tree["MReportL3T.fTablename"].array( + library="np" + ) L3Table_size = L3Table_array.size if L3Table_size > 1: L3Table = L3Table_array[1] @@ -645,9 +648,17 @@ def parse_data_info(self): # looking into MC data, when SumT is simulated, trigger pattern of all events is set # to 32 (bit 5), except the first (set to 0) if self.mars_datalevel <= MARSDataLevel.STAR: - trigger_pattern_array_unique = np.unique(rootf["Events"]["MTriggerPattern.fPrescaled"].array(library="np")[1:]) + trigger_pattern_array_unique = np.unique( + rootf["Events"]["MTriggerPattern.fPrescaled"].array( + library="np" + )[1:] + ) else: - trigger_pattern_array_unique = np.unique(rootf["Events"]["MTriggerPattern_1.fPrescaled"].array(library="np")[1:]) + trigger_pattern_array_unique = np.unique( + rootf["Events"]["MTriggerPattern_1.fPrescaled"].array( + library="np" + )[1:] + ) if len(trigger_pattern_array_unique) == 1: if trigger_pattern_array_unique[0] == MC_SUMT_TRIGGER_PATTERN: sumt = True @@ -657,7 +668,9 @@ def parse_data_info(self): sumt = False # looking into MC data, MMcCorsikaRunHeader.fNumCT is set to 1 if mono, 2 if stereo - num_telescopes = rootf["RunHeaders"]["MMcCorsikaRunHeader.fNumCT"].array(library="np")[0] + num_telescopes = rootf["RunHeaders"][ + "MMcCorsikaRunHeader.fNumCT" + ].array(library="np")[0] if num_telescopes == 1: stereo = False else: @@ -716,7 +729,7 @@ def prepare_subarray_info(self): "MAGIC", num_mirrors=1, equivalent_focal_length=focal_length, - mirror_area=u.Quantity(239.0, u.m ** 2), + mirror_area=u.Quantity(239.0, u.m**2), num_mirror_tiles=964, ) @@ -728,9 +741,9 @@ def prepare_subarray_info(self): pulse_shape = np.vstack((pulse_shape_lo_gain, pulse_shape_hi_gain)) if self.mars_datalevel <= MARSDataLevel.STAR: sampling_speed = u.Quantity( - self.files_[0]["RunHeaders"][ - "MRawRunHeader.fSamplingFrequency" - ].array(library="np")[0] + self.files_[0]["RunHeaders"]["MRawRunHeader.fSamplingFrequency"].array( + library="np" + )[0] / 1000, u.GHz, ) @@ -751,9 +764,7 @@ def prepare_subarray_info(self): camera = CameraDescription("MAGICCam", camera_geom, camera_readout) - camera.geometry.frame = CameraFrame( - focal_length=OPTICS.equivalent_focal_length - ) + camera.geometry.frame = CameraFrame(focal_length=OPTICS.equivalent_focal_length) MAGIC_TEL_DESCRIPTION = TelescopeDescription( name="MAGIC", tel_type="MAGIC", optics=OPTICS, camera=camera @@ -892,25 +903,19 @@ def parse_metadata_info(self): f"MRawRunHeader{suffix}.fSourceName[80]" ][0] metadata["source_name"].append( - "".join( - [chr(item) for item in src_name_array if item != 0] - ) + "".join([chr(item) for item in src_name_array if item != 0]) ) obs_mode_array = meta_info_runh[ f"MRawRunHeader{suffix}.fObservationMode[60]" ][0] metadata["observation_mode"].append( - "".join( - [chr(item) for item in obs_mode_array if item != 0] - ) + "".join([chr(item) for item in obs_mode_array if item != 0]) ) project_name_array = meta_info_runh[ f"MRawRunHeader{suffix}.fProjectName[100]" ][0] metadata["project_name"].append( - "".join( - [chr(item) for item in project_name_array if item != 0] - ) + "".join([chr(item) for item in project_name_array if item != 0]) ) meta_info_runt = rootf["RunTails"].arrays( @@ -933,14 +938,10 @@ def parse_metadata_info(self): ) elif self.mars_datalevel == MARSDataLevel.SUPERSTAR: mars_version_encoded = int( - meta_info_runt["MMarsVersion_superstar.fMARSVersionCode"][ - 0 - ] + meta_info_runt["MMarsVersion_superstar.fMARSVersionCode"][0] ) root_version_encoded = int( - meta_info_runt["MMarsVersion_superstar.fROOTVersionCode"][ - 0 - ] + meta_info_runt["MMarsVersion_superstar.fROOTVersionCode"][0] ) elif self.mars_datalevel == MARSDataLevel.MELIBEA: mars_version_encoded = int( @@ -994,35 +995,33 @@ def parse_simulation_header(self): # MAGIC_Binc is the magnetic field inclination MAGIC_Bx = u.Quantity(29.5, u.uT) MAGIC_Bz = u.Quantity(23.0, u.uT) - MAGIC_Btot = np.sqrt(MAGIC_Bx ** 2 + MAGIC_Bz ** 2) + MAGIC_Btot = np.sqrt(MAGIC_Bx**2 + MAGIC_Bz**2) MAGIC_Bdec = u.Quantity(-7.0, u.deg).to(u.rad) - MAGIC_Binc = u.Quantity( - np.arctan2(-MAGIC_Bz.value, MAGIC_Bx.value), u.rad - ) + MAGIC_Binc = u.Quantity(np.arctan2(-MAGIC_Bz.value, MAGIC_Bx.value), u.rad) simulation_config = dict() for run_number, rootf in zip(self.run_numbers, self.files_): run_header_tree = rootf["RunHeaders"] - spectral_index = run_header_tree[ - "MMcCorsikaRunHeader.fSlopeSpec" - ].array(library="np")[0] - e_low = run_header_tree["MMcCorsikaRunHeader.fELowLim"].array( + spectral_index = run_header_tree["MMcCorsikaRunHeader.fSlopeSpec"].array( library="np" )[0] + e_low = run_header_tree["MMcCorsikaRunHeader.fELowLim"].array(library="np")[ + 0 + ] e_high = run_header_tree["MMcCorsikaRunHeader.fEUppLim"].array( library="np" )[0] corsika_version = run_header_tree[ "MMcCorsikaRunHeader.fCorsikaVersion" ].array(library="np")[0] - site_height = run_header_tree[ - "MMcCorsikaRunHeader.fHeightLev[10]" - ].array(library="np")[0][0] - atm_model = run_header_tree[ - "MMcCorsikaRunHeader.fAtmosphericModel" - ].array(library="np")[0] + site_height = run_header_tree["MMcCorsikaRunHeader.fHeightLev[10]"].array( + library="np" + )[0][0] + atm_model = run_header_tree["MMcCorsikaRunHeader.fAtmosphericModel"].array( + library="np" + )[0] if self.mars_datalevel in [ MARSDataLevel.CALIBRATED, MARSDataLevel.STAR, @@ -1050,12 +1049,12 @@ def parse_simulation_header(self): min_az = run_header_tree["MMcRunHeader.fShowerPhiMin"].array( library="np" )[0] - max_wavelength = run_header_tree[ - "MMcRunHeader.fCWaveUpper" - ].array(library="np")[0] - min_wavelength = run_header_tree[ - "MMcRunHeader.fCWaveLower" - ].array(library="np")[0] + max_wavelength = run_header_tree["MMcRunHeader.fCWaveUpper"].array( + library="np" + )[0] + min_wavelength = run_header_tree["MMcRunHeader.fCWaveLower"].array( + library="np" + )[0] elif self.mars_datalevel in [ MARSDataLevel.SUPERSTAR, MARSDataLevel.MELIBEA, @@ -1063,32 +1062,32 @@ def parse_simulation_header(self): view_cone = run_header_tree[ "MMcRunHeader_1.fRandomPointingConeSemiAngle" ].array(library="np")[0] - max_impact = run_header_tree[ - "MMcRunHeader_1.fImpactMax" - ].array(library="np")[0] + max_impact = run_header_tree["MMcRunHeader_1.fImpactMax"].array( + library="np" + )[0] n_showers = np.sum( - run_header_tree[ - "MMcRunHeader_1.fNumSimulatedShowers" - ].array(library="np") + run_header_tree["MMcRunHeader_1.fNumSimulatedShowers"].array( + library="np" + ) ) - max_zd = run_header_tree[ - "MMcRunHeader_1.fShowerThetaMax" - ].array(library="np")[0] - min_zd = run_header_tree[ - "MMcRunHeader_1.fShowerThetaMin" - ].array(library="np")[0] + max_zd = run_header_tree["MMcRunHeader_1.fShowerThetaMax"].array( + library="np" + )[0] + min_zd = run_header_tree["MMcRunHeader_1.fShowerThetaMin"].array( + library="np" + )[0] max_az = run_header_tree["MMcRunHeader_1.fShowerPhiMax"].array( library="np" )[0] min_az = run_header_tree["MMcRunHeader_1.fShowerPhiMin"].array( library="np" )[0] - max_wavelength = run_header_tree[ - "MMcRunHeader_1.fCWaveUpper" - ].array(library="np")[0] - min_wavelength = run_header_tree[ - "MMcRunHeader_1.fCWaveLower" - ].array(library="np")[0] + max_wavelength = run_header_tree["MMcRunHeader_1.fCWaveUpper"].array( + library="np" + )[0] + min_wavelength = run_header_tree["MMcRunHeader_1.fCWaveLower"].array( + library="np" + )[0] simulation_config[run_number] = SimulationConfigContainer( corsika_version=corsika_version, @@ -1274,9 +1273,7 @@ def get_badrmspixel_mask(self, event, tel_id): for i_ped_type in range(n_ped_types): - charge_std = event.mon.tel[tel_id].pedestal.charge_std[i_ped_type][ - index - ] + charge_std = event.mon.tel[tel_id].pedestal.charge_std[i_ped_type][index] pix_step1 = np.logical_and( charge_std > 0, @@ -1305,10 +1302,10 @@ def get_badrmspixel_mask(self, event, tel_id): lolim_step1 = mean_step2 / pedestal_level uplim_step1 = mean_step2 * pedestal_level lolim_step2 = mean_step2 - pedestal_level_variance * np.sqrt( - var_step2 - mean_step2 ** 2 + var_step2 - mean_step2**2 ) uplim_step2 = mean_step2 + pedestal_level_variance * np.sqrt( - var_step2 - mean_step2 ** 2 + var_step2 - mean_step2**2 ) badrmspix_step1 = np.logical_or( @@ -1321,9 +1318,7 @@ def get_badrmspixel_mask(self, event, tel_id): charge_std > uplim_step2, ) - badrmspixel_mask.append( - np.logical_or(badrmspix_step1, badrmspix_step2) - ) + badrmspixel_mask.append(np.logical_or(badrmspix_step1, badrmspix_step2)) return badrmspixel_mask @@ -1352,6 +1347,9 @@ def _set_active_run(self, uproot_file): uproot_file, self.is_simulation, self.telescopes[0], + self.is_stereo, + self.use_mc_mono_events, + self.is_sumt, ) if self.mars_datalevel == MARSDataLevel.SUPERSTAR: run["data"] = MarsSuperstarRun( @@ -1362,9 +1360,6 @@ def _set_active_run(self, uproot_file): run["data"] = MarsMelibeaRun( uproot_file, self.is_simulation, - self.is_stereo, - self.use_mc_mono_events, - self.is_sumt, ) return run @@ -1422,53 +1417,42 @@ def _event_generator(self): if self.mars_datalevel == MARSDataLevel.CALIBRATED: - monitoring_data = self.current_run[ - "data" - ].monitoring_data + monitoring_data = self.current_run["data"].monitoring_data # Set the pedestal information: event.mon.tel[ tel_id - ].pedestal.n_events = 500 # hardcoded number of pedestal events averaged over - event.mon.tel[ + ].pedestal.n_events = ( + 500 # hardcoded number of pedestal events averaged over + ) + event.mon.tel[tel_id].pedestal.sample_time = monitoring_data[ tel_id - ].pedestal.sample_time = monitoring_data[tel_id][ - "pedestal_sample_time" - ] + ]["pedestal_sample_time"] event.mon.tel[tel_id].pedestal.charge_mean = [ - monitoring_data[tel_id]["pedestal_fundamental"][ - "mean" - ], - monitoring_data[tel_id]["pedestal_from_extractor"][ - "mean" - ], - monitoring_data[tel_id][ - "pedestal_from_extractor_rndm" - ]["mean"], + monitoring_data[tel_id]["pedestal_fundamental"]["mean"], + monitoring_data[tel_id]["pedestal_from_extractor"]["mean"], + monitoring_data[tel_id]["pedestal_from_extractor_rndm"]["mean"], ] event.mon.tel[tel_id].pedestal.charge_std = [ - monitoring_data[tel_id]["pedestal_fundamental"][ - "rms" - ], - monitoring_data[tel_id]["pedestal_from_extractor"][ - "rms" - ], - monitoring_data[tel_id][ - "pedestal_from_extractor_rndm" - ]["rms"], + monitoring_data[tel_id]["pedestal_fundamental"]["rms"], + monitoring_data[tel_id]["pedestal_from_extractor"]["rms"], + monitoring_data[tel_id]["pedestal_from_extractor_rndm"]["rms"], ] # Set the bad pixel information: - event.mon.tel[tel_id].pixel_status.hardware_failing_pixels = np.reshape( - monitoring_data['bad_pixel'], (1, len(monitoring_data['bad_pixel'])) + event.mon.tel[ + tel_id + ].pixel_status.hardware_failing_pixels = np.reshape( + monitoring_data[tel_id]["bad_pixel"], + (1, len(monitoring_data[tel_id]["bad_pixel"])), ) if not self.is_simulation: # Interpolate drive information: drive_data = self.drive_information - n_drive_reports = len(drive_data['mjd']) + n_drive_reports = len(drive_data["mjd"]) # Interpolate drive information: drive_data = self.drive_information @@ -1495,52 +1479,36 @@ def _event_generator(self): ) # Create azimuth and zenith angles interpolators: - drive_az_pointing_interpolator = ( - scipy.interpolate.interp1d( - drive_data["mjd"], - drive_data["az"], - fill_value="extrapolate", - ) + drive_az_pointing_interpolator = scipy.interpolate.interp1d( + drive_data["mjd"], + drive_data["az"], + fill_value="extrapolate", ) - drive_zd_pointing_interpolator = ( - scipy.interpolate.interp1d( - drive_data["mjd"], - drive_data["zd"], - fill_value="extrapolate", - ) + drive_zd_pointing_interpolator = scipy.interpolate.interp1d( + drive_data["mjd"], + drive_data["zd"], + fill_value="extrapolate", ) # Create RA and Dec interpolators: - drive_ra_pointing_interpolator = ( - scipy.interpolate.interp1d( - drive_data["mjd"], - drive_data["ra"], - fill_value="extrapolate", - ) + drive_ra_pointing_interpolator = scipy.interpolate.interp1d( + drive_data["mjd"], + drive_data["ra"], + fill_value="extrapolate", ) - drive_dec_pointing_interpolator = ( - scipy.interpolate.interp1d( - drive_data["mjd"], - drive_data["dec"], - fill_value="extrapolate", - ) + drive_dec_pointing_interpolator = scipy.interpolate.interp1d( + drive_data["mjd"], + drive_data["dec"], + fill_value="extrapolate", ) # Interpolate the drive pointing to the event timestamps: event_times = event_data[tel_id]["time"].mjd - pointing_az = drive_az_pointing_interpolator( - event_times - ) - pointing_zd = drive_zd_pointing_interpolator( - event_times - ) - pointing_ra = drive_ra_pointing_interpolator( - event_times - ) - pointing_dec = drive_dec_pointing_interpolator( - event_times - ) + pointing_az = drive_az_pointing_interpolator(event_times) + pointing_zd = drive_zd_pointing_interpolator(event_times) + pointing_ra = drive_ra_pointing_interpolator(event_times) + pointing_dec = drive_dec_pointing_interpolator(event_times) event_data[tel_id]["pointing_az"] = u.Quantity( pointing_az, u.deg @@ -1560,46 +1528,40 @@ def _event_generator(self): for tel_id in self.telescopes: event.count = counter - event.index.event_id = event_data['event_number'][i_event] + event.index.event_id = event_data[tel_id]["event_number"][i_event] event.trigger.event_type = MAGIC_TO_CTA_EVENT_TYPE.get( event_data[tel_id]["trigger_pattern"][i_event] ) if not self.is_simulation: - event.trigger.time = event_data[tel_id]["time"][ + event.trigger.time = event_data[tel_id]["time"][i_event] + event.trigger.tel[tel_id].time = event_data[tel_id]["time"][ i_event ] - event.trigger.tel[tel_id].time = event_data[tel_id][ - "time" - ][i_event] if ( not self.use_pedestals and self.mars_datalevel == MARSDataLevel.CALIBRATED ): - badrmspixel_mask = self.get_badrmspixel_mask( - event, tel_id - ) + badrmspixel_mask = self.get_badrmspixel_mask(event, tel_id) event.mon.tel[ tel_id - ].pixel_status.pedestal_failing_pixels = ( - badrmspixel_mask - ) + ].pixel_status.pedestal_failing_pixels = badrmspixel_mask # Set the telescope pointing container: - event.pointing.array_azimuth = event_data[tel_id][ - "pointing_az" - ][i_event].to(u.rad) - event.pointing.array_altitude = event_data[tel_id][ - "pointing_alt" - ][i_event].to(u.rad) - event.pointing.array_ra = event_data[tel_id][ - "pointing_ra" - ][i_event].to(u.rad) - event.pointing.array_dec = event_data[tel_id][ - "pointing_dec" - ][i_event].to(u.rad) + event.pointing.array_azimuth = event_data[tel_id]["pointing_az"][ + i_event + ].to(u.rad) + event.pointing.array_altitude = event_data[tel_id]["pointing_alt"][ + i_event + ].to(u.rad) + event.pointing.array_ra = event_data[tel_id]["pointing_ra"][ + i_event + ].to(u.rad) + event.pointing.array_dec = event_data[tel_id]["pointing_dec"][ + i_event + ].to(u.rad) event.pointing.tel[tel_id].azimuth = event_data[tel_id][ "pointing_az" @@ -1610,17 +1572,15 @@ def _event_generator(self): if self.mars_datalevel == MARSDataLevel.CALIBRATED: # Set event charge and peak positions per pixel: - event.dl1.tel[tel_id].image = event_data[tel_id][ - "image" - ][i_event] + event.dl1.tel[tel_id].image = event_data[tel_id]["image"][ + i_event + ] event.dl1.tel[tel_id].peak_time = event_data[tel_id][ "peak_time" ][i_event] if self.mars_datalevel >= MARSDataLevel.SUPERSTAR: - event.dl1.tel[ - tel_id - ].parameters = ImageParametersContainer() + event.dl1.tel[tel_id].parameters = ImageParametersContainer() event.dl1.tel[ tel_id ].parameters.hillas = CameraHillasParametersContainer( @@ -1649,27 +1609,21 @@ def _event_generator(self): ) # not fully filled - event.dl1.tel[ - tel_id - ].parameters.leakage = LeakageContainer( - intensity_width_1=event_data[tel_id][ - "intensity_width_1" - ][i_event], - intensity_width_2=event_data[tel_id][ - "intensity_width_2" - ][i_event], + event.dl1.tel[tel_id].parameters.leakage = LeakageContainer( + intensity_width_1=event_data[tel_id]["intensity_width_1"][ + i_event + ], + intensity_width_2=event_data[tel_id]["intensity_width_2"][ + i_event + ], ) # not fully filled event.dl1.tel[ tel_id ].parameters.morphology = MorphologyContainer( - num_pixels=event_data[tel_id]["num_pixels"][ - i_event - ], - num_islands=event_data[tel_id]["num_islands"][ - i_event - ], + num_pixels=event_data[tel_id]["num_pixels"][i_event], + num_islands=event_data[tel_id]["num_islands"][i_event], ) event.dl2.stereo.geometry[ @@ -1693,12 +1647,10 @@ def _event_generator(self): if self.mars_datalevel >= MARSDataLevel.MELIBEA: event.dl2.stereo.energy = ReconstructedEnergyContainer( - energy=event_data["reconstructed"]["energy"][ + energy=event_data["reconstructed"]["energy"][i_event], + energy_uncert=event_data["reconstructed"]["energy_uncert"][ i_event ], - energy_uncert=event_data["reconstructed"][ - "energy_uncert" - ][i_event], is_valid=True if event_data["stereo"]["is_valid"][i_event] == 1 else False, @@ -1707,12 +1659,11 @@ def _event_generator(self): event.dl2.stereo.classification = ( ParticleClassificationContainer( - prediction=event_data["reconstructed"][ - "gammanness" - ][i_event], + prediction=event_data["reconstructed"]["gammanness"][ + i_event + ], is_valid=True - if event_data["stereo"]["is_valid"][i_event] - == 1 + if event_data["stereo"]["is_valid"][i_event] == 1 else False, tel_ids=[1, 2], ) @@ -1727,16 +1678,15 @@ def _event_generator(self): "mc_energy" ][i_event].to(u.TeV) event.simulation.shower.shower_primary_id = ( - 1 - - event_data[tel_id]["mc_shower_primary_id"][ - i_event - ] + 1 - event_data[tel_id]["mc_shower_primary_id"][i_event] ) - event.simulation.shower.h_first_int = event_data[ - tel_id - ]["mc_h_first_int"][i_event].to(u.m) + event.simulation.shower.h_first_int = event_data[tel_id][ + "mc_h_first_int" + ][i_event].to(u.m) - event.simulation.shower.x_max = event_data[tel_id]['mc_x_max'][i_event].to('g cm-2') + event.simulation.shower.x_max = event_data[tel_id]["mc_x_max"][ + i_event + ].to("g cm-2") # Convert the corsika coordinate (x-axis: magnetic north) to the geographical one. # Rotate the corsika coordinate by the magnetic declination (= 7 deg): @@ -1754,15 +1704,11 @@ def _event_generator(self): ) if event.simulation.shower.az > u.Quantity(180, u.deg): - event.simulation.shower.az -= u.Quantity( - 360, u.deg - ) + event.simulation.shower.az -= u.Quantity(360, u.deg) event.simulation.shower.core_x = event_data[tel_id][ "mc_core_x" - ][i_event].to(u.m) * np.cos(mfield_dec) + event_data[ - tel_id - ][ + ][i_event].to(u.m) * np.cos(mfield_dec) + event_data[tel_id][ "mc_core_y" ][ i_event @@ -1774,9 +1720,7 @@ def _event_generator(self): event.simulation.shower.core_y = event_data[tel_id][ "mc_core_y" - ][i_event].to(u.m) * np.cos(mfield_dec) - event_data[ - tel_id - ][ + ][i_event].to(u.m) * np.cos(mfield_dec) - event_data[tel_id][ "mc_core_x" ][ i_event @@ -1798,7 +1742,16 @@ class MarsCalibratedRun: and monitoring data from a MAGIC calibrated subrun file. """ - def __init__(self, uproot_file, is_mc, tel_id, is_stereo, use_mc_mono_events, use_sumt_events, n_cam_pixels=1039): + def __init__( + self, + uproot_file, + is_mc, + tel_id, + is_stereo, + use_mc_mono_events, + use_sumt_events, + n_cam_pixels=1039, + ): """ Constructor of the class. Loads an input uproot file and store the informaiton to constructor variables. @@ -1838,9 +1791,7 @@ def n_cosmic_events(self): @property def n_pedestal_events(self): - return len( - self.pedestal_events[self.tel_id].get("trigger_pattern", []) - ) + return len(self.pedestal_events[self.tel_id].get("trigger_pattern", [])) def _load_data(self): """ @@ -1856,23 +1807,23 @@ def _load_data(self): # Branches applicable for cosmic and pedestal events: common_branches = [ - 'MRawEvtHeader.fStereoEvtNumber', - 'MRawEvtHeader.fDAQEvtNumber', - 'MTriggerPattern.fPrescaled', - 'MCerPhotEvt.fPixels.fPhot', - 'MArrivalTime.fData', + "MRawEvtHeader.fStereoEvtNumber", + "MRawEvtHeader.fDAQEvtNumber", + "MTriggerPattern.fPrescaled", + "MCerPhotEvt.fPixels.fPhot", + "MArrivalTime.fData", ] # Branches applicable for MC events: mc_branches = [ - 'MMcEvt.fEnergy', - 'MMcEvt.fTheta', - 'MMcEvt.fPhi', - 'MMcEvt.fPartId', - 'MMcEvt.fZFirstInteraction', - 'MMcEvt.fLongitmax', - 'MMcEvt.fCoreX', - 'MMcEvt.fCoreY', + "MMcEvt.fEnergy", + "MMcEvt.fTheta", + "MMcEvt.fPhi", + "MMcEvt.fPartId", + "MMcEvt.fZFirstInteraction", + "MMcEvt.fLongitmax", + "MMcEvt.fCoreX", + "MMcEvt.fCoreY", ] pointing_branches = [ @@ -1894,18 +1845,18 @@ def _load_data(self): ] pedestal_time_branches = [ - 'MTimePedestals.fMjd', - 'MTimePedestals.fTime.fMilliSec', - 'MTimePedestals.fNanoSec', + "MTimePedestals.fMjd", + "MTimePedestals.fTime.fMilliSec", + "MTimePedestals.fNanoSec", ] pedestal_branches = [ - 'MPedPhotFundamental.fArray.fMean', - 'MPedPhotFundamental.fArray.fRms', - 'MPedPhotFromExtractor.fArray.fMean', - 'MPedPhotFromExtractor.fArray.fRms', - 'MPedPhotFromExtractorRndm.fArray.fMean', - 'MPedPhotFromExtractorRndm.fArray.fRms', + "MPedPhotFundamental.fArray.fMean", + "MPedPhotFundamental.fArray.fRms", + "MPedPhotFromExtractor.fArray.fMean", + "MPedPhotFromExtractor.fArray.fRms", + "MPedPhotFromExtractorRndm.fArray.fMean", + "MPedPhotFromExtractorRndm.fArray.fRms", ] # Initialize the data container: @@ -1923,10 +1874,14 @@ def _load_data(self): if self.use_sumt_events: mc_trigger_pattern = MC_SUMT_TRIGGER_PATTERN if self.use_mc_mono_events or not self.is_stereo: - events_cut['cosmic_events'] = f'(MTriggerPattern.fPrescaled == {mc_trigger_pattern})' + events_cut[ + "cosmic_events" + ] = f"(MTriggerPattern.fPrescaled == {mc_trigger_pattern})" else: - events_cut['cosmic_events'] = f'(MTriggerPattern.fPrescaled == {mc_trigger_pattern})' \ - ' & (MRawEvtHeader.fStereoEvtNumber != 0)' + events_cut["cosmic_events"] = ( + f"(MTriggerPattern.fPrescaled == {mc_trigger_pattern})" + " & (MRawEvtHeader.fStereoEvtNumber != 0)" + ) else: data_trigger_pattern = DATA_STEREO_TRIGGER_PATTERN if not self.is_stereo: @@ -1934,17 +1889,21 @@ def _load_data(self): data_trigger_pattern = DATA_MONO_SUMT_TRIGGER_PATTERN else: data_trigger_pattern = DATA_MONO_TRIGGER_PATTERN - events_cut['cosmic_events'] = f'(MTriggerPattern.fPrescaled == {data_trigger_pattern})' + events_cut[ + "cosmic_events" + ] = f"(MTriggerPattern.fPrescaled == {data_trigger_pattern})" # Only for cosmic events because MC data do not have pedestal events: - events_cut['pedestal_events'] = f'(MTriggerPattern.fPrescaled == {PEDESTAL_TRIGGER_PATTERN})' + events_cut[ + "pedestal_events" + ] = f"(MTriggerPattern.fPrescaled == {PEDESTAL_TRIGGER_PATTERN})" logger.info(f"Cosmic events selection: {events_cut['cosmic_events']}") # read common information from RunHeaders sampling_speed = ( - self.uproot_file["RunHeaders"][ - "MRawRunHeader.fSamplingFrequency" - ].array(library="np")[0] + self.uproot_file["RunHeaders"]["MRawRunHeader.fSamplingFrequency"].array( + library="np" + )[0] / 1000.0 ) # GHz @@ -1960,10 +1919,14 @@ def _load_data(self): library="np", ) - if common_info['MTriggerPattern.fPrescaled'].size == 0: - raise IndexError(f"No events survived after selection (cut: {events_cut[event_type]})") + if common_info["MTriggerPattern.fPrescaled"].size == 0: + raise IndexError( + f"No events survived after selection (cut: {events_cut[event_type]})" + ) - calib_data[event_type][self.tel_id]['trigger_pattern'] = np.array(common_info['MTriggerPattern.fPrescaled'], dtype=int) + calib_data[event_type][self.tel_id]["trigger_pattern"] = np.array( + common_info["MTriggerPattern.fPrescaled"], dtype=int + ) # For stereo data, the event ID is simply given by MRawEvtHeader.fStereoEvtNumber # For data taken in mono however (i.e. only with one telescope in SA), fStereoEvtNumber @@ -1974,11 +1937,21 @@ def _load_data(self): # Like this we obtain an event id which is unique within a data run (like fStereoEvtNumber). if (self.is_mc and self.use_mc_mono_events) or not self.is_stereo: - subrun_id = self.uproot_file["RunHeaders"]['MRawRunHeader.fSubRunIndex'].array(library="np")[0] - calib_data[event_type][self.tel_id]['event_number'] = np.array([f"{subrun_id}{daq_id:05}" for daq_id in common_info['MRawEvtHeader.fDAQEvtNumber']], dtype=int) + subrun_id = self.uproot_file["RunHeaders"][ + "MRawRunHeader.fSubRunIndex" + ].array(library="np")[0] + calib_data[event_type][self.tel_id]["event_number"] = np.array( + [ + f"{subrun_id}{daq_id:05}" + for daq_id in common_info["MRawEvtHeader.fDAQEvtNumber"] + ], + dtype=int, + ) logger.info("Using fDAQEvtNumber to generate event IDs.") else: - calib_data[event_type][self.tel_id]['event_number'] = np.array(common_info['MRawEvtHeader.fStereoEvtNumber'], dtype=int) + calib_data[event_type][self.tel_id]["event_number"] = np.array( + common_info["MRawEvtHeader.fStereoEvtNumber"], dtype=int + ) logger.info("Using fStereoEvtNumber to generate event IDs.") # Set pixel-wise charge and peak time information. @@ -1991,9 +1964,7 @@ def _load_data(self): calib_data[event_type][self.tel_id]["peak_time"] = np.array( common_info["MArrivalTime.fData"].tolist() )[:, : self.n_cam_pixels] - calib_data[event_type][self.tel_id][ - "peak_time" - ] /= sampling_speed # [ns] + calib_data[event_type][self.tel_id]["peak_time"] /= sampling_speed # [ns] if self.is_mc: @@ -2017,13 +1988,15 @@ def _load_data(self): calib_data[event_type][self.tel_id]["mc_phi"] = u.Quantity( mc_info["MMcEvt.fPhi"], u.rad ) - calib_data[event_type][self.tel_id][ - "mc_shower_primary_id" - ] = np.array(mc_info["MMcEvt.fPartId"], dtype=int) - calib_data[event_type][self.tel_id][ - "mc_h_first_int" - ] = u.Quantity(mc_info["MMcEvt.fZFirstInteraction"], u.cm) - calib_data[event_type][self.tel_id]['mc_x_max'] = u.Quantity(mc_info['MMcEvt.fLongitmax'], u.Unit('g cm-2')) + calib_data[event_type][self.tel_id]["mc_shower_primary_id"] = np.array( + mc_info["MMcEvt.fPartId"], dtype=int + ) + calib_data[event_type][self.tel_id]["mc_h_first_int"] = u.Quantity( + mc_info["MMcEvt.fZFirstInteraction"], u.cm + ) + calib_data[event_type][self.tel_id]["mc_x_max"] = u.Quantity( + mc_info["MMcEvt.fLongitmax"], u.Unit("g cm-2") + ) calib_data[event_type][self.tel_id]["mc_core_x"] = u.Quantity( mc_info["MMcEvt.fCoreX"], u.cm ) @@ -2032,10 +2005,10 @@ def _load_data(self): ) # Reading the telescope pointing information: - pointing = self.uproot_file['Events'].arrays( + pointing = self.uproot_file["Events"].arrays( expressions=pointing_branches, cut=events_cut[event_type], - library='np', + library="np", ) # Reading the telescope pointing information: @@ -2046,36 +2019,32 @@ def _load_data(self): ) pointing_az = ( - pointing["MPointingPos.fAz"] - - pointing["MPointingPos.fDevAz"] + pointing["MPointingPos.fAz"] - pointing["MPointingPos.fDevAz"] ) pointing_zd = ( - pointing["MPointingPos.fZd"] - - pointing["MPointingPos.fDevZd"] + pointing["MPointingPos.fZd"] - pointing["MPointingPos.fDevZd"] ) # N.B. the positive sign here, as HA = local sidereal time - ra: pointing_ra = ( - pointing["MPointingPos.fRa"] - + pointing["MPointingPos.fDevHa"] + pointing["MPointingPos.fRa"] + pointing["MPointingPos.fDevHa"] ) * degrees_per_hour pointing_dec = ( - pointing["MPointingPos.fDec"] - - pointing["MPointingPos.fDevDec"] - ) - - calib_data[event_type][self.tel_id][ - "pointing_az" - ] = u.Quantity(pointing_az, u.deg) - calib_data[event_type][self.tel_id][ - "pointing_alt" - ] = u.Quantity(90 - pointing_zd, u.deg) - calib_data[event_type][self.tel_id][ - "pointing_ra" - ] = u.Quantity(pointing_ra, u.deg) - calib_data[event_type][self.tel_id][ - "pointing_dec" - ] = u.Quantity(pointing_dec, u.deg) + pointing["MPointingPos.fDec"] - pointing["MPointingPos.fDevDec"] + ) + + calib_data[event_type][self.tel_id]["pointing_az"] = u.Quantity( + pointing_az, u.deg + ) + calib_data[event_type][self.tel_id]["pointing_alt"] = u.Quantity( + 90 - pointing_zd, u.deg + ) + calib_data[event_type][self.tel_id]["pointing_ra"] = u.Quantity( + pointing_ra, u.deg + ) + calib_data[event_type][self.tel_id]["pointing_dec"] = u.Quantity( + pointing_dec, u.deg + ) else: # Reading the event timing information: @@ -2092,23 +2061,15 @@ def _load_data(self): timing_info["MTime.fMjd"], format="mjd", scale="utc" ) event_obs_day = np.round(event_obs_day.unix) - event_obs_day = np.array( - [Decimal(str(x)) for x in event_obs_day] - ) + event_obs_day = np.array([Decimal(str(x)) for x in event_obs_day]) event_millisec = np.round( timing_info["MTime.fTime.fMilliSec"] * msec2sec, 3 ) - event_millisec = np.array( - [Decimal(str(x)) for x in event_millisec] - ) + event_millisec = np.array([Decimal(str(x)) for x in event_millisec]) - event_nanosec = np.round( - timing_info["MTime.fNanoSec"] * nsec2sec, 7 - ) - event_nanosec = np.array( - [Decimal(str(x)) for x in event_nanosec] - ) + event_nanosec = np.round(timing_info["MTime.fNanoSec"] * nsec2sec, 7) + event_nanosec = np.array([Decimal(str(x)) for x in event_nanosec]) event_time = event_obs_day + event_millisec + event_nanosec calib_data[event_type][self.tel_id]["time"] = Time( @@ -2119,11 +2080,13 @@ def _load_data(self): # In MAGIC simulations, pixel 0 is always set as bad # Reading the bad pixel information: - as_dtype = uproot.interpretation.numerical.AsDtype(np.dtype('>i4')) + as_dtype = uproot.interpretation.numerical.AsDtype(np.dtype(">i4")) as_jagged = uproot.interpretation.jagged.AsJagged(as_dtype) - badpixel_info = self.uproot_file['RunHeaders']['MBadPixelsCam.fArray.fInfo'].array(as_jagged, library='np')[0] - badpixel_info = badpixel_info.reshape((4, 1183), order='F') + badpixel_info = self.uproot_file["RunHeaders"][ + "MBadPixelsCam.fArray.fInfo" + ].array(as_jagged, library="np")[0] + badpixel_info = badpixel_info.reshape((4, 1183), order="F") # now we have 4 axes: # 0st axis: empty (?) @@ -2141,36 +2104,35 @@ def _load_data(self): "{:08b}".format(unsuitable_pixels_bit[i_pix])[-2] ) - calib_data["monitoring_data"][self.tel_id][ - "bad_pixel" - ] = unsuitable_pixels - + calib_data["monitoring_data"][self.tel_id]["bad_pixel"] = unsuitable_pixels # Try to read the Pedestals tree (soft fail if not present): try: if self.is_mc: pedestal_branch_list = pedestal_branches - pedestal_info = self.uproot_file['Events'].arrays( + pedestal_info = self.uproot_file["Events"].arrays( expressions=pedestal_branch_list, - library='np', + library="np", ) else: pedestal_branch_list = pedestal_time_branches + pedestal_branches - pedestal_info = self.uproot_file['Pedestals'].arrays( + pedestal_info = self.uproot_file["Pedestals"].arrays( expressions=pedestal_branch_list, - library='np', + library="np", ) # Set sample times of pedestal events: if self.is_mc: - calib_data['monitoring_data'][self.tel_id]['pedestal_sample_time'] = np.array([]) + calib_data["monitoring_data"][self.tel_id][ + "pedestal_sample_time" + ] = np.array([]) else: - pedestal_obs_day = Time(pedestal_info['MTimePedestals.fMjd'], format='mjd', scale='utc') - pedestal_obs_day = np.round(pedestal_obs_day.unix) - pedestal_obs_day = np.array( - [Decimal(str(x)) for x in pedestal_obs_day] + pedestal_obs_day = Time( + pedestal_info["MTimePedestals.fMjd"], format="mjd", scale="utc" ) + pedestal_obs_day = np.round(pedestal_obs_day.unix) + pedestal_obs_day = np.array([Decimal(str(x)) for x in pedestal_obs_day]) pedestal_millisec = np.round( pedestal_info["MTimePedestals.fTime.fMilliSec"] * msec2sec, @@ -2183,9 +2145,7 @@ def _load_data(self): pedestal_nanosec = np.round( pedestal_info["MTimePedestals.fNanoSec"] * nsec2sec, 7 ) - pedestal_nanosec = np.array( - [Decimal(str(x)) for x in pedestal_nanosec] - ) + pedestal_nanosec = np.array([Decimal(str(x)) for x in pedestal_nanosec]) pedestal_sample_time = ( pedestal_obs_day + pedestal_millisec + pedestal_nanosec @@ -2195,49 +2155,35 @@ def _load_data(self): "pedestal_sample_time" ] = Time(pedestal_sample_time, format="unix", scale="utc") - # Set the mean and RMS of pedestal charges: - calib_data["monitoring_data"][self.tel_id][ - "pedestal_fundamental" - ] = { - "mean": np.array( - pedestal_info[ - "MPedPhotFundamental.fArray.fMean" - ].tolist() - )[:, : self.n_cam_pixels], - "rms": np.array( - pedestal_info[ - "MPedPhotFundamental.fArray.fRms" - ].tolist() - )[:, : self.n_cam_pixels], - } - calib_data["monitoring_data"][self.tel_id][ - "pedestal_from_extractor" - ] = { - "mean": np.array( - pedestal_info[ - "MPedPhotFromExtractor.fArray.fMean" - ].tolist() - )[:, : self.n_cam_pixels], - "rms": np.array( - pedestal_info[ - "MPedPhotFromExtractor.fArray.fRms" - ].tolist() - )[:, : self.n_cam_pixels], - } - calib_data["monitoring_data"][self.tel_id][ - "pedestal_from_extractor_rndm" - ] = { - "mean": np.array( - pedestal_info[ - "MPedPhotFromExtractorRndm.fArray.fMean" - ].tolist() - )[:, : self.n_cam_pixels], - "rms": np.array( - pedestal_info[ - "MPedPhotFromExtractorRndm.fArray.fRms" - ].tolist() - )[:, : self.n_cam_pixels], - } + # Set the mean and RMS of pedestal charges: + calib_data["monitoring_data"][self.tel_id]["pedestal_fundamental"] = { + "mean": np.array( + pedestal_info["MPedPhotFundamental.fArray.fMean"].tolist() + )[:, : self.n_cam_pixels], + "rms": np.array( + pedestal_info["MPedPhotFundamental.fArray.fRms"].tolist() + )[:, : self.n_cam_pixels], + } + calib_data["monitoring_data"][self.tel_id][ + "pedestal_from_extractor" + ] = { + "mean": np.array( + pedestal_info["MPedPhotFromExtractor.fArray.fMean"].tolist() + )[:, : self.n_cam_pixels], + "rms": np.array( + pedestal_info["MPedPhotFromExtractor.fArray.fRms"].tolist() + )[:, : self.n_cam_pixels], + } + calib_data["monitoring_data"][self.tel_id][ + "pedestal_from_extractor_rndm" + ] = { + "mean": np.array( + pedestal_info["MPedPhotFromExtractorRndm.fArray.fMean"].tolist() + )[:, : self.n_cam_pixels], + "rms": np.array( + pedestal_info["MPedPhotFromExtractorRndm.fArray.fRms"].tolist() + )[:, : self.n_cam_pixels], + } except KeyError: logger.warning( @@ -2250,7 +2196,7 @@ def _load_data(self): uplim_total_jumps = 100 stereo_event_number = calib_data["cosmic_events"][self.tel_id][ - "stereo_event_number" + "event_number" ].astype(int) number_difference = np.diff(stereo_event_number) @@ -2259,7 +2205,9 @@ def _load_data(self): if n_flips > 0: - logger.warning(f'Warning: detected {n_flips} bit flips in the input file') + logger.warning( + f"Warning: detected {n_flips} bit flips in the input file" + ) total_jumped_number = 0 for i in indices_flip: @@ -2398,6 +2346,7 @@ def _load_data(self): f"MMcEvt_{tel_id}.fZFirstInteraction", f"MMcEvt_{tel_id}.fCoreX", f"MMcEvt_{tel_id}.fCoreY", + f"MMcEvt_{tel_id}.fLongitmax", ] stereo_branches = [ @@ -2455,7 +2404,7 @@ def _load_data(self): event_data[tel_id]["trigger_pattern"] = np.array( common_info[f"MTriggerPattern_{tel_id}.fPrescaled"], dtype=int ) - event_data[tel_id]["stereo_event_number"] = np.array( + event_data[tel_id]["event_number"] = np.array( common_info[f"MRawEvtHeader_{tel_id}.fStereoEvtNumber"], dtype=int, ) @@ -2494,6 +2443,9 @@ def _load_data(self): event_data[tel_id]["mc_core_y"] = u.Quantity( mc_info[f"MMcEvt_{tel_id}.fCoreY"], u.cm ) + event_data[tel_id]["mc_x_max"] = u.Quantity( + mc_info[f"MMcEvt_{tel_id}.fLongitmax"], u.Unit("g cm-2") + ) else: # Reading the event timing information: @@ -2512,24 +2464,18 @@ def _load_data(self): scale="utc", ) event_obs_day = np.round(event_obs_day.unix) - event_obs_day = np.array( - [Decimal(str(x)) for x in event_obs_day] - ) + event_obs_day = np.array([Decimal(str(x)) for x in event_obs_day]) event_millisec = np.round( timing_info[f"MTime_{tel_id}.fTime.fMilliSec"] * msec2sec, 3, ) - event_millisec = np.array( - [Decimal(str(x)) for x in event_millisec] - ) + event_millisec = np.array([Decimal(str(x)) for x in event_millisec]) event_nanosec = np.round( timing_info[f"MTime_{tel_id}.fNanoSec"] * nsec2sec, 7 ) - event_nanosec = np.array( - [Decimal(str(x)) for x in event_nanosec] - ) + event_nanosec = np.array([Decimal(str(x)) for x in event_nanosec]) event_time = event_obs_day + event_millisec + event_nanosec event_data[tel_id]["time"] = Time( @@ -2563,13 +2509,9 @@ def _load_data(self): ) event_data[tel_id]["pointing_az"] = u.Quantity(pointing_az, u.deg) - event_data[tel_id]["pointing_alt"] = u.Quantity( - 90 - pointing_zd, u.deg - ) + event_data[tel_id]["pointing_alt"] = u.Quantity(90 - pointing_zd, u.deg) event_data[tel_id]["pointing_ra"] = u.Quantity(pointing_ra, u.deg) - event_data[tel_id]["pointing_dec"] = u.Quantity( - pointing_dec, u.deg - ) + event_data[tel_id]["pointing_dec"] = u.Quantity(pointing_dec, u.deg) hillas = self.uproot_file["Events"].arrays( expressions=hillas_branches[tel_id], @@ -2800,6 +2742,7 @@ def _load_data(self): f"MMcEvt_{tel_id}.fZFirstInteraction", f"MMcEvt_{tel_id}.fCoreX", f"MMcEvt_{tel_id}.fCoreY", + f"MMcEvt_{tel_id}.fLongitmax", ] imagepar_disp_branches[tel_id] = [ @@ -2889,7 +2832,7 @@ def _load_data(self): event_data[tel_id]["trigger_pattern"] = np.array( common_info[f"MTriggerPattern_{tel_id}.fPrescaled"], dtype=int ) - event_data[tel_id]["stereo_event_number"] = np.array( + event_data[tel_id]["event_number"] = np.array( common_info[f"MRawEvtHeader_{tel_id}.fStereoEvtNumber"], dtype=int, ) @@ -2928,6 +2871,9 @@ def _load_data(self): event_data[tel_id]["mc_core_y"] = u.Quantity( mc_info[f"MMcEvt_{tel_id}.fCoreY"], u.cm ) + event_data[tel_id]["mc_x_max"] = u.Quantity( + mc_info[f"MMcEvt_{tel_id}.fLongitmax"], u.Unit("g cm-2") + ) else: # Reading the event timing information: @@ -2946,24 +2892,18 @@ def _load_data(self): scale="utc", ) event_obs_day = np.round(event_obs_day.unix) - event_obs_day = np.array( - [Decimal(str(x)) for x in event_obs_day] - ) + event_obs_day = np.array([Decimal(str(x)) for x in event_obs_day]) event_millisec = np.round( timing_info[f"MTime_{tel_id}.fTime.fMilliSec"] * msec2sec, 3, ) - event_millisec = np.array( - [Decimal(str(x)) for x in event_millisec] - ) + event_millisec = np.array([Decimal(str(x)) for x in event_millisec]) event_nanosec = np.round( timing_info[f"MTime_{tel_id}.fNanoSec"] * nsec2sec, 7 ) - event_nanosec = np.array( - [Decimal(str(x)) for x in event_nanosec] - ) + event_nanosec = np.array([Decimal(str(x)) for x in event_nanosec]) event_time = event_obs_day + event_millisec + event_nanosec event_data[tel_id]["time"] = Time( @@ -2997,13 +2937,9 @@ def _load_data(self): ) event_data[tel_id]["pointing_az"] = u.Quantity(pointing_az, u.deg) - event_data[tel_id]["pointing_alt"] = u.Quantity( - 90 - pointing_zd, u.deg - ) + event_data[tel_id]["pointing_alt"] = u.Quantity(90 - pointing_zd, u.deg) event_data[tel_id]["pointing_ra"] = u.Quantity(pointing_ra, u.deg) - event_data[tel_id]["pointing_dec"] = u.Quantity( - pointing_dec, u.deg - ) + event_data[tel_id]["pointing_dec"] = u.Quantity(pointing_dec, u.deg) hillas = self.uproot_file["Events"].arrays( expressions=hillas_branches[tel_id], From 43bce71d57cba91fbf2fd9ef30c96268fcaaa109 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Tue, 14 Mar 2023 10:38:30 +0100 Subject: [PATCH 58/78] Add test for MARS datalevels. --- .../tests/test_magic_event_source.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ctapipe_io_magic/tests/test_magic_event_source.py b/ctapipe_io_magic/tests/test_magic_event_source.py index 77c7c7b..5f05d42 100644 --- a/ctapipe_io_magic/tests/test_magic_event_source.py +++ b/ctapipe_io_magic/tests/test_magic_event_source.py @@ -206,6 +206,23 @@ def test_event_source_for_magic_file(dataset): assert reader.input_url == dataset +@pytest.mark.parametrize( + "dataset", test_calibrated_all + test_superstar_all + test_melibea_all +) +def test_datalevel(dataset): + from ctapipe_io_magic import MAGICEventSource, MARSDataLevel + + with MAGICEventSource(input_url=dataset, process_run=False) as source: + if "_Y_" in dataset.name: + assert source.mars_datalevel == MARSDataLevel.CALIBRATED + elif "_I_" in dataset.name: + assert source.mars_datalevel == MARSDataLevel.STAR + elif "_S_" in dataset.name: + assert source.mars_datalevel == MARSDataLevel.SUPERSTAR + elif "_Q_" in dataset.name: + assert source.mars_datalevel == MARSDataLevel.MELIBEA + + @pytest.mark.parametrize( "dataset", test_calibrated_all + test_superstar_all + test_melibea_all ) From 1ceb8bbcacec19767000b61fdda166d8add8f83f Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Tue, 14 Mar 2023 10:38:45 +0100 Subject: [PATCH 59/78] Add ctapipe datalevels. --- ctapipe_io_magic/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 8c40c6c..0bd2972 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -229,7 +229,14 @@ def __init__(self, input_url=None, config=None, parent=None, **kwargs): self.metadata = self.parse_metadata_info() # Retrieving the data level (so far HARDCODED Sorcerer) - self.datalevel = DataLevel.DL0 + if self.mars_datalevel == MARSDataLevel.CALIBRATED: + self.datalevel = DataLevel.DL0 + elif self.mars_datalevel == MARSDataLevel.STAR: + self.datalevel = DataLevel.DL1_PARAMETERS + elif self.mars_datalevel == MARSDataLevel.SUPERSTAR: + self.datalevel = DataLevel.DL1_PARAMETERS + elif self.mars_datalevel == MARSDataLevel.MELIBEA: + self.datalevel = DataLevel.DL2 if self.is_simulation: self.simulation_config = self.parse_simulation_header() From ec45b382071b01e2bf57e63476b65e10443e0d8d Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Thu, 16 Mar 2023 11:50:32 +0100 Subject: [PATCH 60/78] Add sumtrigger for superstar and melibea. --- ctapipe_io_magic/__init__.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 0bd2972..4698c2c 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1362,11 +1362,13 @@ def _set_active_run(self, uproot_file): run["data"] = MarsSuperstarRun( uproot_file, self.is_simulation, + self.is_sumt, ) if self.mars_datalevel == MARSDataLevel.MELIBEA: run["data"] = MarsMelibeaRun( uproot_file, self.is_simulation, + self.is_sumt, ) return run @@ -1611,7 +1613,7 @@ def _event_generator(self): ].parameters.timing = TimingParametersContainer( slope=event_data[tel_id]["slope"][i_event].value * (1.0 / m2deg) - * u.deg, + / u.deg, intercept=event_data[tel_id]["intercept"][i_event], ) @@ -1773,6 +1775,10 @@ def __init__( Telescope number (1 for M1, 2 for M2) is_stereo: bool Flag for mono/stereo data + use_mc_mono_events: bool + Flag telling if data are mono + use_sumt_events: bool + Flag telling if data are taken with SumTrigger n_cam_pixels: int The number of pixels of the MAGIC camera """ @@ -2237,7 +2243,7 @@ class MarsSuperstarRun: This class implements reading of cosmic events from a MAGIC superstar run file. """ - def __init__(self, uproot_file, is_mc, n_cam_pixels=1039): + def __init__(self, uproot_file, is_mc, use_sumt_events, n_cam_pixels=1039): """ Constructor of the class. Loads an input uproot file and store the informaiton to constructor variables. @@ -2248,12 +2254,15 @@ def __init__(self, uproot_file, is_mc, n_cam_pixels=1039): A superstar file opened by uproot via uproot.open(file_path) is_mc: bool Flag to MC data + use_sumt_events: bool + Flag telling if data are taken with SumTrigger n_cam_pixels: int The number of pixels of the MAGIC camera """ self.uproot_file = uproot_file self.is_mc = is_mc + self.use_sumt_events = use_sumt_events self.n_cam_pixels = n_cam_pixels # Load the input data: @@ -2386,8 +2395,11 @@ def _load_data(self): if self.is_mc: # Only for cosmic events because MC data do not have pedestal events: + mc_trigger_pattern = MC_STEREO_AND_MONO_TRIGGER_PATTERN + if self.use_sumt_events: + mc_trigger_pattern = MC_SUMT_TRIGGER_PATTERN events_cut = ( - f"(MTriggerPattern_{tel_id}.fPrescaled == {MC_STEREO_AND_MONO_TRIGGER_PATTERN})" + f"(MTriggerPattern_{tel_id}.fPrescaled == {mc_trigger_pattern})" f" & (MRawEvtHeader_{tel_id}.fStereoEvtNumber != 0)" ) else: @@ -2632,7 +2644,7 @@ class MarsMelibeaRun: This class implements reading of cosmic events from a MAGIC melibea run file. """ - def __init__(self, uproot_file, is_mc, n_cam_pixels=1039): + def __init__(self, uproot_file, is_mc, use_sumt_events, n_cam_pixels=1039): """ Constructor of the class. Loads an input uproot file and store the informaiton to constructor variables. @@ -2643,12 +2655,15 @@ def __init__(self, uproot_file, is_mc, n_cam_pixels=1039): A melibea file opened by uproot via uproot.open(file_path) is_mc: bool Flag to MC data + use_sumt_events: bool + Flag telling if data are taken with SumTrigger n_cam_pixels: int The number of pixels of the MAGIC camera """ self.uproot_file = uproot_file self.is_mc = is_mc + self.use_sumt_events = use_sumt_events self.n_cam_pixels = n_cam_pixels # Load the input data: @@ -2812,10 +2827,14 @@ def _load_data(self): event_data[tel_id] = dict() + # for the moment, we do not care about mono melibea files if self.is_mc: # Only for cosmic events because MC data do not have pedestal events: + mc_trigger_pattern = MC_STEREO_AND_MONO_TRIGGER_PATTERN + if self.use_sumt_events: + mc_trigger_pattern = MC_SUMT_TRIGGER_PATTERN events_cut = ( - f"(MTriggerPattern_{tel_id}.fPrescaled == {MC_STEREO_AND_MONO_TRIGGER_PATTERN})" + f"(MTriggerPattern_{tel_id}.fPrescaled == {mc_trigger_pattern})" f" & (MRawEvtHeader_{tel_id}.fStereoEvtNumber != 0)" ) else: From 9586bdbf8388a6893d33d6cc986c5cc969506e4c Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Tue, 14 May 2024 20:26:06 +0200 Subject: [PATCH 61/78] Update download script. --- download_test_data.sh | 136 ++++++++++++++++-------------------------- 1 file changed, 52 insertions(+), 84 deletions(-) diff --git a/download_test_data.sh b/download_test_data.sh index 0e1c1b5..728b2ea 100755 --- a/download_test_data.sh +++ b/download_test_data.sh @@ -2,25 +2,35 @@ set -e -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/calibrated/20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root" > test_data_real.txt -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/calibrated/20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root" >> test_data_real.txt -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/calibrated/20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root" >> test_data_real.txt -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/calibrated/20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root" >> test_data_real.txt - -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/superstar/20210314_05095172_S_CrabNebula-W0.40+035.root" > test_data_superstar_real.txt - -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/real/melibea/20210314_05095172_Q_CrabNebula-W0.40+035.root" > test_data_melibea_real.txt - -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M1_za35to50_8_824318_Y_w0.root" > test_data_simulated.txt -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M1_za35to50_8_824319_Y_w0.root" >> test_data_simulated.txt -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M2_za35to50_8_824318_Y_w0.root" >> test_data_simulated.txt -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/calibrated/GA_M2_za35to50_8_824319_Y_w0.root" >> test_data_simulated.txt - -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824318_S_w0.root" > test_data_superstar_simulated.txt -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/superstar/GA_za35to50_8_824319_S_w0.root" >> test_data_superstar_simulated.txt - -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/melibea/GA_za35to50_8_824318_Q_w0.root" > test_data_melibea_simulated.txt -echo "https://webdav-magic.pic.es:8451/Users/ctapipe_io_magic/test_data/simulated/melibea/GA_za35to50_8_824319_Q_w0.root" >> test_data_melibea_simulated.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root" > test_data_real.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20210314_M1_05095172.002_Y_CrabNebula-W0.40+035.root" >> test_data_real.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20210314_M2_05095172.001_Y_CrabNebula-W0.40+035.root" >> test_data_real.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20210314_M2_05095172.002_Y_CrabNebula-W0.40+035.root" >> test_data_real.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20230324_M1_05106879.001_Y_1ES0806+524-W0.40+000.root" >> test_data_real.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20230324_M1_05106879.002_Y_1ES0806+524-W0.40+000.root" >> test_data_real.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20230324_M2_05106879.001_Y_1ES0806+524-W0.40+000.root" >> test_data_real.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20230324_M2_05106879.002_Y_1ES0806+524-W0.40+000.root" >> test_data_real.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20210314_M1_05095172.001_Y_CrabNebula-W0.40+035_only_events.root" > test_data_real_missing_trees.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20210314_M1_05095172.001_Y_CrabNebula-W0.40+035_only_drive.root" >> test_data_real_missing_trees.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20210314_M1_05095172.001_Y_CrabNebula-W0.40+035_only_runh.root" >> test_data_real_missing_trees.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20210314_M1_05095172.001_Y_CrabNebula-W0.40+035_only_trigger.root" >> test_data_real_missing_trees.txt + +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root" > test_data_real_missing_prescaler_trigger.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20210314_M1_05095172.002_Y_CrabNebula-W0.40+035_no_prescaler_trigger.root" >> test_data_real_missing_prescaler_trigger.txt + +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20210314_M1_05095172.001_Y_CrabNebula-W0.40+035.root" > test_data_real_missing_arrays.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/real/calibrated/20210314_M1_05095172.002_Y_CrabNebula-W0.40+035_no_arrays.root" >> test_data_real_missing_arrays.txt + +echo "https://www.magic.iac.es/mcp-testdata/test_data/simulated/calibrated/GA_M1_za35to50_8_824318_Y_w0.root" > test_data_simulated.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/simulated/calibrated/GA_M1_za35to50_8_824319_Y_w0.root" >> test_data_simulated.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/simulated/calibrated/GA_M2_za35to50_8_824318_Y_w0.root" >> test_data_simulated.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/simulated/calibrated/GA_M2_za35to50_8_824319_Y_w0.root" >> test_data_simulated.txt + +echo "https://www.magic.iac.es/mcp-testdata/test_data/simulated/superstar/GA_za35to50_8_824318_S_w0.root" > test_data_superstar_simulated.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/simulated/superstar/GA_za35to50_8_824319_S_w0.root" >> test_data_superstar_simulated.txt + +echo "https://www.magic.iac.es/mcp-testdata/test_data/simulated/melibea/GA_za35to50_8_824318_Q_w0.root" > test_data_melibea_simulated.txt +echo "https://www.magic.iac.es/mcp-testdata/test_data/simulated/melibea/GA_za35to50_8_824319_Q_w0.root" >> test_data_melibea_simulated.txt if [ -z "$TEST_DATA_USER" ]; then read -p "Username: " TEST_DATA_USER @@ -32,70 +42,28 @@ if [ -z "$TEST_DATA_PASSWORD" ]; then echo fi -if ! wget \ - -i test_data_real.txt \ - --user="$TEST_DATA_USER" \ - --password="$TEST_DATA_PASSWORD" \ - --no-check-certificate \ - --no-verbose \ - --timestamping \ - --directory-prefix=test_data/real/calibrated; then - echo "Problem in downloading the test data set (calibrated) for real data." -fi - -if ! wget \ - -i test_data_superstar_real.txt \ - --user="$TEST_DATA_USER" \ - --password="$TEST_DATA_PASSWORD" \ - --no-check-certificate \ - --no-verbose \ - --timestamping \ - --directory-prefix=test_data/real/superstar; then - echo "Problem in downloading the test data set (superstar) for real data." -fi - -if ! wget \ - -i test_data_melibea_real.txt \ - --user="$TEST_DATA_USER" \ - --password="$TEST_DATA_PASSWORD" \ - --no-check-certificate \ - --no-verbose \ - --timestamping \ - --directory-prefix=test_data/real/melibea; then - echo "Problem in downloading the test data set (melibea) for real data." -fi - -if ! wget \ - -i test_data_simulated.txt \ - --user="$TEST_DATA_USER" \ - --password="$TEST_DATA_PASSWORD" \ - --no-check-certificate \ - --no-verbose \ - --timestamping \ - --directory-prefix=test_data/simulated/calibrated; then - echo "Problem in downloading the test data set (calibrated) for simulated data." -fi - -if ! wget \ - -i test_data_superstar_simulated.txt \ - --user="$TEST_DATA_USER" \ - --password="$TEST_DATA_PASSWORD" \ - --no-check-certificate \ - --no-verbose \ - --timestamping \ - --directory-prefix=test_data/simulated/superstar; then - echo "Problem in downloading the test data set (superstar) for simulated data." -fi - -if ! wget \ - -i test_data_melibea_simulated.txt \ - --user="$TEST_DATA_USER" \ - --password="$TEST_DATA_PASSWORD" \ - --no-check-certificate \ - --no-verbose \ - --timestamping \ - --directory-prefix=test_data/simulated/melibea; then - echo "Problem in downloading the test data set (melibea) for simulated data." +declare -A TEST_FILES_DOWNLOAD + +TEST_FILES_DOWNLOAD[test_data_real]="test_data/real/calibrated" +TEST_FILES_DOWNLOAD[test_data_real_missing_trees]="test_data/real/calibrated/missing_trees" +TEST_FILES_DOWNLOAD[test_data_real_missing_prescaler_trigger]="test_data/real/calibrated/missing_prescaler_trigger" +TEST_FILES_DOWNLOAD[test_data_real_missing_arrays]="test_data/real/calibrated/missing_arrays" +TEST_FILES_DOWNLOAD[test_data_simulated]="test_data/simulated/calibrated" +TEST_FILES_DOWNLOAD[test_data_superstar_simulated]="test_data/simulated/superstar" +TEST_FILES_DOWNLOAD[test_data_melibea_simulated]="test_data/simulated/melibea" + +for key in "${!TEST_FILES_DOWNLOAD[@]}" +do + if ! wget \ + -i "${key}.txt" \ + --user="$TEST_DATA_USER" \ + --password="$TEST_DATA_PASSWORD" \ + --no-check-certificate \ + --no-verbose \ + --timestamping \ + --directory-prefix="${TEST_FILES_DOWNLOAD[${key}]}"; then + echo "Problem in downloading the test data set from ${key}.txt." fi +done -rm -f test_data_real.txt test_data_simulated.txt test_data_superstar_real.txt test_data_superstar_simulated.txt test_data_melibea_real.txt test_data_melibea_simulated.txt +rm -f test_data_real.txt test_data_simulated.txt test_data_real_missing_trees.txt test_data_real_missing_prescaler_trigger.txt test_data_real_missing_arrays.txt test_data_superstar_simulated.txt test_data_melibea_simulated.txt From 29ef6839c96457b94cabad9042bbd160e66ed091 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Tue, 14 May 2024 22:06:50 +0200 Subject: [PATCH 62/78] Run_numbers to run_id. --- ctapipe_io_magic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 3a38cef..d5c2379 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1154,7 +1154,7 @@ def parse_metadata_info(self): metadata = dict() metadata["file_list"] = self.file_list - metadata["run_numbers"] = self.run_numbers + metadata["run_numbers"] = self.run_id metadata["is_simulation"] = self.is_simulation metadata["telescopes"] = self.telescopes metadata["subrun_number"] = [] From 9088ce128eb7de0cd649814b08d5b27d2f84fe60 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Tue, 14 May 2024 22:12:16 +0200 Subject: [PATCH 63/78] Correct producer_id. --- ctapipe_io_magic/__init__.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index d5c2379..b4ee091 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -356,7 +356,11 @@ def __init__(self, input_url=None, config=None, parent=None, **kwargs): self._scheduling_blocks = { self.run_id: SchedulingBlockContainer( sb_id=np.uint64(self.run_id), - producer_id=f"MAGIC-{self.telescope}", + producer_id=( + f"MAGIC-{self.telescopes[0]}" + if len(self.telescopes) == 0 + else "MAGIC-stereo" + ), pointing_mode=pointing_mode, ) } @@ -365,7 +369,11 @@ def __init__(self, input_url=None, config=None, parent=None, **kwargs): self.run_id: ObservationBlockContainer( obs_id=np.uint64(self.run_id), sb_id=np.uint64(self.run_id), - producer_id=f"MAGIC-{self.telescope}", + producer_id=( + f"MAGIC-{self.telescopes[0]}" + if len(self.telescopes) == 0 + else "MAGIC-stereo" + ), ) } From ccbb13b03884d8368c075d1e6d51f608cd321491 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Tue, 14 May 2024 22:17:28 +0200 Subject: [PATCH 64/78] Check only for calibrated and star files. --- ctapipe_io_magic/__init__.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index b4ee091..41575fb 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -532,23 +532,24 @@ def check_files(self): needed_trees = ["RunHeaders", "Events"] num_files = len(self.files_) - if ( - num_files == 1 - and "Drive" not in self.files_[0].keys(cycle=False) - and "OriginalMC" not in self.files_[0].keys(cycle=False) - ): - logger.error("Cannot proceed without Drive information for a single file.") - return False + if self.mars_datalevel <= MARSDataLevel.STAR: + if ( + num_files == 1 + and "Drive" not in self.files_[0].keys(cycle=False) + and "OriginalMC" not in self.files_[0].keys(cycle=False) + ): + logger.error("Cannot proceed without Drive information for a single file.") + return False - if ( - num_files == 1 - and "Trigger" not in self.files_[0].keys(cycle=False) - and "OriginalMC" not in self.files_[0].keys(cycle=False) - ): - logger.error( - "Cannot proceed without Trigger information for a single file." - ) - return False + if ( + num_files == 1 + and "Trigger" not in self.files_[0].keys(cycle=False) + and "OriginalMC" not in self.files_[0].keys(cycle=False) + ): + logger.error( + "Cannot proceed without Trigger information for a single file." + ) + return False num_invalid_files = 0 From 956fa8460af70db1291ef17bc1a33139e3a148f9 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Tue, 14 May 2024 22:48:16 +0200 Subject: [PATCH 65/78] Modify check function. --- ctapipe_io_magic/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 41575fb..486a07e 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -263,6 +263,8 @@ def __init__(self, input_url=None, config=None, parent=None, **kwargs): else: raise ValueError(f"Datalevel not recognized: {datalevel}") + self.mars_datalevel = datalevel + reg_comp = re.compile(regex) reg_comp_mc = re.compile(regex_mc) @@ -882,6 +884,15 @@ def parse_data_info(self): stereo = True hast = False + try: + L3T_tree = rootf["L3T"] + except uproot.exceptions.KeyInFileError: + logger.warning(f"No L3T tree found in {rootf.file_path}. Assuming standard stereo data.") + is_stereo.append(stereo) + is_sumt.append(False) + is_hast.append(hast) + continue + L3Table_array = L3T_tree["MReportL3T.fTablename"].array( library="np" ) @@ -900,6 +911,7 @@ def parse_data_info(self): is_stereo.append(stereo) is_sumt.append(sumt) + is_hast.append(hast) else: for rootf in self.files_: # looking into MC data, when SumT is simulated, trigger pattern of all events is set From b0e0a1a791fb8c99fdf18251a4e5fd56ad12f6bd Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 15 May 2024 08:45:42 +0200 Subject: [PATCH 66/78] Correct members name for MorphologyContainer. --- ctapipe_io_magic/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 486a07e..64b629d 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -2095,8 +2095,8 @@ def _event_generator(self): event.dl1.tel[ tel_id ].parameters.morphology = MorphologyContainer( - num_pixels=event_data[tel_id]["num_pixels"][i_event], - num_islands=event_data[tel_id]["num_islands"][i_event], + n_pixels=event_data[tel_id]["num_pixels"][i_event], + n_islands=event_data[tel_id]["num_islands"][i_event], ) event.dl2.stereo.geometry[ From 27e28b40b3d7504a7be3b4fb3f3c3d862abec22e Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 15 May 2024 08:58:10 +0200 Subject: [PATCH 67/78] Remove wrong argument. --- ctapipe_io_magic/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 64b629d..4c4b2a8 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1819,7 +1819,6 @@ def _set_active_run(self, uproot_file): run["data"] = MarsCalibratedRun( uproot_file, self.is_simulation, - self.telescopes[0], self.is_stereo, self.is_hast, self.use_mc_mono_events, From 25747fe7893372329e285cfe3aa7177b9dd8521e Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 15 May 2024 08:59:06 +0200 Subject: [PATCH 68/78] Correct member name. --- ctapipe_io_magic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 4c4b2a8..991c05d 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -2114,7 +2114,7 @@ def _event_generator(self): + event_data[2]["intensity"][i_event] ) / 2.0, - tel_ids=[1, 2], + telescopes=[1, 2], ) if self.mars_datalevel >= MARSDataLevel.MELIBEA: From ca15969187b17e5defb3cce0bc1b4733091e6ed3 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 15 May 2024 09:03:54 +0200 Subject: [PATCH 69/78] Correct member name. --- ctapipe_io_magic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 991c05d..b8bef0f 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -2126,7 +2126,7 @@ def _event_generator(self): is_valid=True if event_data["stereo"]["is_valid"][i_event] == 1 else False, - tel_ids=[1, 2], + telescopes=[1, 2], ) event.dl2.stereo.classification = ( From 974f95af0737460bae8270690163ec6153d3e73f Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 15 May 2024 09:04:06 +0200 Subject: [PATCH 70/78] Remove duplicted and wrong code. --- ctapipe_io_magic/__init__.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index b8bef0f..008679d 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -2627,9 +2627,6 @@ def _load_data(self): [Decimal(str(x)) for x in pedestal_millisec] ) - pedestal_nanosec = np.round( - pedestal_info["MTimePedestals.fNanoSec"] * nsec2sec, 7 - ) pedestal_nanosec = np.array([Decimal(str(x)) for x in pedestal_nanosec]) pedestal_sample_time = ( @@ -2640,19 +2637,6 @@ def _load_data(self): pedestal_sample_time, format="unix", scale="utc" ) - pedestal_nanosec = np.round( - pedestal_info["MTimePedestals.fNanoSec"] * nsec2sec, 7 - ) - pedestal_nanosec = np.array([Decimal(str(x)) for x in pedestal_nanosec]) - - pedestal_sample_time = ( - pedestal_obs_day + pedestal_millisec + pedestal_nanosec - ) - - calib_data["monitoring_data"][self.tel_id][ - "pedestal_sample_time" - ] = Time(pedestal_sample_time, format="unix", scale="utc") - # Set the mean and RMS of pedestal charges: calib_data["monitoring_data"]["pedestal_fundamental"] = { "mean": np.array( From 87313f2ad83c166a5ba25daf7f15c98ce332d59b Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 15 May 2024 19:48:14 +0200 Subject: [PATCH 71/78] Check if telescopes list is subset of another. --- ctapipe_io_magic/tests/test_magic_event_source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ctapipe_io_magic/tests/test_magic_event_source.py b/ctapipe_io_magic/tests/test_magic_event_source.py index 573ca15..a41d2b2 100644 --- a/ctapipe_io_magic/tests/test_magic_event_source.py +++ b/ctapipe_io_magic/tests/test_magic_event_source.py @@ -537,7 +537,7 @@ def test_multiple_runs_real(): for i, event in enumerate(source): assert event.trigger.event_type == EventType.SUBARRAY assert event.count == i - assert source.telescope in event.trigger.tels_with_trigger + assert set(source.telescopes).issubset(set(event.trigger.tels_with_trigger)) assert event.trigger.tels_with_trigger == [1, 2] assert (i + 1) == n_events From 44201eeeb3949db685884d89c57a3268d2c2d03f Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 15 May 2024 19:48:45 +0200 Subject: [PATCH 72/78] Correct check on length. --- ctapipe_io_magic/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 008679d..fa6e867 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -360,7 +360,7 @@ def __init__(self, input_url=None, config=None, parent=None, **kwargs): sb_id=np.uint64(self.run_id), producer_id=( f"MAGIC-{self.telescopes[0]}" - if len(self.telescopes) == 0 + if len(self.telescopes) == 1 else "MAGIC-stereo" ), pointing_mode=pointing_mode, @@ -373,7 +373,7 @@ def __init__(self, input_url=None, config=None, parent=None, **kwargs): sb_id=np.uint64(self.run_id), producer_id=( f"MAGIC-{self.telescopes[0]}" - if len(self.telescopes) == 0 + if len(self.telescopes) == 1 else "MAGIC-stereo" ), ) From abdf439c99640635e971d6123e06de61dfdbb9ee Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 15 May 2024 19:49:02 +0200 Subject: [PATCH 73/78] Correct list of telescopes. --- ctapipe_io_magic/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index fa6e867..f7ed845 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -486,13 +486,13 @@ def get_run_info_from_name(file_name): is_mc = False elif re.match(mask_data_superstar, file_name) is not None: parsed_info = re.match(mask_data_superstar, file_name) - telescope = None + telescope = [1, 2] run_number = int(parsed_info.group(1)) datalevel = MARSDataLevel.SUPERSTAR is_mc = False elif re.match(mask_data_melibea, file_name) is not None: parsed_info = re.match(mask_data_melibea, file_name) - telescope = None + telescope = [1, 2] run_number = int(parsed_info.group(1)) datalevel = MARSDataLevel.MELIBEA is_mc = False From 8de8a49643e0e12e391da92a91d1553fed4f5078 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 15 May 2024 19:49:22 +0200 Subject: [PATCH 74/78] Add missing pedestal nanosecond array. --- ctapipe_io_magic/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index f7ed845..01c136b 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -2627,6 +2627,9 @@ def _load_data(self): [Decimal(str(x)) for x in pedestal_millisec] ) + pedestal_nanosec = np.round( + pedestal_info["MTimePedestals.fNanoSec"] * nsec2sec, 7 + ) pedestal_nanosec = np.array([Decimal(str(x)) for x in pedestal_nanosec]) pedestal_sample_time = ( From bd417121eba1fbbe2a7ae03b884a8d607dcc7406 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 15 May 2024 19:49:54 +0200 Subject: [PATCH 75/78] Remove tel_id key. --- ctapipe_io_magic/__init__.py | 88 ++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 01c136b..e880cf2 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1975,23 +1975,23 @@ def _event_generator(self): ) # Interpolate the drive pointing to the event timestamps: - event_times = event_data[tel_id]["time"].mjd + event_times = event_data["time"].mjd pointing_az = drive_az_pointing_interpolator(event_times) pointing_zd = drive_zd_pointing_interpolator(event_times) pointing_ra = drive_ra_pointing_interpolator(event_times) pointing_dec = drive_dec_pointing_interpolator(event_times) - event_data[tel_id]["pointing_az"] = u.Quantity( + event_data["pointing_az"] = u.Quantity( pointing_az, u.deg ) - event_data[tel_id]["pointing_alt"] = u.Quantity( + event_data["pointing_alt"] = u.Quantity( 90 - pointing_zd, u.deg ) - event_data[tel_id]["pointing_ra"] = u.Quantity( + event_data["pointing_ra"] = u.Quantity( pointing_ra, u.deg ) - event_data[tel_id]["pointing_dec"] = u.Quantity( + event_data["pointing_dec"] = u.Quantity( pointing_dec, u.deg ) @@ -2003,12 +2003,12 @@ def _event_generator(self): event.index.event_id = event_data[tel_id]["event_number"][i_event] event.trigger.event_type = MAGIC_TO_CTA_EVENT_TYPE.get( - event_data[tel_id]["trigger_pattern"][i_event] + event_data["trigger_pattern"][i_event] ) if not self.is_simulation: - event.trigger.time = event_data[tel_id]["time"][i_event] - event.trigger.tel[tel_id].time = event_data[tel_id]["time"][ + event.trigger.time = event_data["time"][i_event] + event.trigger.tel[tel_id].time = event_data["time"][ i_event ] @@ -2022,32 +2022,32 @@ def _event_generator(self): ].pixel_status.pedestal_failing_pixels = badrmspixel_mask # Set the telescope pointing container: - event.pointing.array_azimuth = event_data[tel_id]["pointing_az"][ + event.pointing.array_azimuth = event_data["pointing_az"][ i_event ].to(u.rad) - event.pointing.array_altitude = event_data[tel_id]["pointing_alt"][ + event.pointing.array_altitude = event_data["pointing_alt"][ i_event ].to(u.rad) - event.pointing.array_ra = event_data[tel_id]["pointing_ra"][ + event.pointing.array_ra = event_data["pointing_ra"][ i_event ].to(u.rad) - event.pointing.array_dec = event_data[tel_id]["pointing_dec"][ + event.pointing.array_dec = event_data["pointing_dec"][ i_event ].to(u.rad) - event.pointing.tel[tel_id].azimuth = event_data[tel_id][ + event.pointing.tel[tel_id].azimuth = event_data[ "pointing_az" ][i_event].to(u.rad) - event.pointing.tel[tel_id].altitude = event_data[tel_id][ + event.pointing.tel[tel_id].altitude = event_data[ "pointing_alt" ][i_event].to(u.rad) if self.mars_datalevel == MARSDataLevel.CALIBRATED: # Set event charge and peak positions per pixel: - event.dl1.tel[tel_id].image = event_data[tel_id]["image"][ + event.dl1.tel[tel_id].image = event_data["image"][ i_event ] - event.dl1.tel[tel_id].peak_time = event_data[tel_id][ + event.dl1.tel[tel_id].peak_time = event_data[ "peak_time" ][i_event] @@ -2056,36 +2056,36 @@ def _event_generator(self): event.dl1.tel[ tel_id ].parameters.hillas = CameraHillasParametersContainer( - x=-event_data[tel_id]["y"][i_event], - y=-event_data[tel_id]["x"][i_event], - length=event_data[tel_id]["length"][i_event], - width=event_data[tel_id]["width"][i_event], - intensity=event_data[tel_id]["intensity"][i_event], + x=-event_data["y"][i_event], + y=-event_data["x"][i_event], + length=event_data["length"][i_event], + width=event_data["width"][i_event], + intensity=event_data["intensity"][i_event], r=np.sqrt( - event_data[tel_id]["x"][i_event] - * event_data[tel_id]["x"][i_event] - + event_data[tel_id]["y"][i_event] - * event_data[tel_id]["y"][i_event] + event_data["x"][i_event] + * event_data["x"][i_event] + + event_data["y"][i_event] + * event_data["y"][i_event] ), - psi=event_data[tel_id]["psi"][i_event], - phi=event_data[tel_id]["phi"][i_event], + psi=event_data["psi"][i_event], + phi=event_data["phi"][i_event], ) event.dl1.tel[ tel_id ].parameters.timing = TimingParametersContainer( - slope=event_data[tel_id]["slope"][i_event].value + slope=event_data["slope"][i_event].value * (1.0 / m2deg) / u.deg, - intercept=event_data[tel_id]["intercept"][i_event], + intercept=event_data["intercept"][i_event], ) # not fully filled event.dl1.tel[tel_id].parameters.leakage = LeakageContainer( - intensity_width_1=event_data[tel_id]["intensity_width_1"][ + intensity_width_1=event_data["intensity_width_1"][ i_event ], - intensity_width_2=event_data[tel_id]["intensity_width_2"][ + intensity_width_2=event_data["intensity_width_2"][ i_event ], ) @@ -2094,8 +2094,8 @@ def _event_generator(self): event.dl1.tel[ tel_id ].parameters.morphology = MorphologyContainer( - n_pixels=event_data[tel_id]["num_pixels"][i_event], - n_islands=event_data[tel_id]["num_islands"][i_event], + n_pixels=event_data["num_pixels"][i_event], + n_islands=event_data["num_islands"][i_event], ) event.dl2.stereo.geometry[ @@ -2137,7 +2137,7 @@ def _event_generator(self): is_valid=True if event_data["stereo"]["is_valid"][i_event] == 1 else False, - tel_ids=[1, 2], + telescopes=[1, 2], ) ) @@ -2145,17 +2145,17 @@ def _event_generator(self): if self.is_simulation: event.simulation = SimulatedEventContainer() - event.simulation.shower.energy = event_data[tel_id][ + event.simulation.shower.energy = event_data[ "mc_energy" ][i_event].to(u.TeV) event.simulation.shower.shower_primary_id = ( - 1 - event_data[tel_id]["mc_shower_primary_id"][i_event] + 1 - event_data["mc_shower_primary_id"][i_event] ) - event.simulation.shower.h_first_int = event_data[tel_id][ + event.simulation.shower.h_first_int = event_data[ "mc_h_first_int" ][i_event].to(u.m) - event.simulation.shower.x_max = event_data[tel_id]["mc_x_max"][ + event.simulation.shower.x_max = event_data["mc_x_max"][ i_event ].to("g cm-2") @@ -2167,19 +2167,19 @@ def _event_generator(self): event.simulation.shower.alt = u.Quantity( 90, u.deg - ) - event_data[tel_id]["mc_theta"][i_event].to(u.deg) + ) - event_data["mc_theta"][i_event].to(u.deg) event.simulation.shower.az = ( u.Quantity(180, u.deg) - - event_data[tel_id]["mc_phi"][i_event].to(u.deg) + - event_data["mc_phi"][i_event].to(u.deg) + mfield_dec ) if event.simulation.shower.az > u.Quantity(180, u.deg): event.simulation.shower.az -= u.Quantity(360, u.deg) - event.simulation.shower.core_x = event_data[tel_id][ + event.simulation.shower.core_x = event_data[ "mc_core_x" - ][i_event].to(u.m) * np.cos(mfield_dec) + event_data[tel_id][ + ][i_event].to(u.m) * np.cos(mfield_dec) + event_data[ "mc_core_y" ][ i_event @@ -2189,9 +2189,9 @@ def _event_generator(self): mfield_dec ) - event.simulation.shower.core_y = event_data[tel_id][ + event.simulation.shower.core_y = event_data[ "mc_core_y" - ][i_event].to(u.m) * np.cos(mfield_dec) - event_data[tel_id][ + ][i_event].to(u.m) * np.cos(mfield_dec) - event_data[ "mc_core_x" ][ i_event From e0741a1dc9c83899c723d6af9069c53248938867 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 15 May 2024 19:50:30 +0200 Subject: [PATCH 76/78] Assign telescope with triggers. --- ctapipe_io_magic/__init__.py | 55 ++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index e880cf2..26bd7d2 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1897,28 +1897,26 @@ def _event_generator(self): ].pedestal.n_events = ( 500 # hardcoded number of pedestal events averaged over ) - event.mon.tel[tel_id].pedestal.sample_time = monitoring_data[ - tel_id - ]["pedestal_sample_time"] + event.mon.tel[tel_id].pedestal.sample_time = monitoring_data["pedestal_sample_time"] event.mon.tel[tel_id].pedestal.charge_mean = [ - monitoring_data[tel_id]["pedestal_fundamental"]["mean"], - monitoring_data[tel_id]["pedestal_from_extractor"]["mean"], - monitoring_data[tel_id]["pedestal_from_extractor_rndm"]["mean"], + monitoring_data["pedestal_fundamental"]["mean"], + monitoring_data["pedestal_from_extractor"]["mean"], + monitoring_data["pedestal_from_extractor_rndm"]["mean"], ] event.mon.tel[tel_id].pedestal.charge_std = [ - monitoring_data[tel_id]["pedestal_fundamental"]["rms"], - monitoring_data[tel_id]["pedestal_from_extractor"]["rms"], - monitoring_data[tel_id]["pedestal_from_extractor_rndm"]["rms"], + monitoring_data["pedestal_fundamental"]["rms"], + monitoring_data["pedestal_from_extractor"]["rms"], + monitoring_data["pedestal_from_extractor_rndm"]["rms"], ] # Set the bad pixel information: event.mon.tel[ tel_id ].pixel_status.hardware_failing_pixels = np.reshape( - monitoring_data[tel_id]["bad_pixel"], - (1, len(monitoring_data[tel_id]["bad_pixel"])), + monitoring_data["bad_pixel"], + (1, len(monitoring_data["bad_pixel"])), ) if not self.is_simulation: @@ -2000,7 +1998,40 @@ def _event_generator(self): for tel_id in self.telescopes: event.count = counter - event.index.event_id = event_data[tel_id]["event_number"][i_event] + + if not self.is_simulation: + if ( + event_data["trigger_pattern"][i_event] + == DATA_STEREO_TRIGGER_PATTERN + ): + event.trigger.tels_with_trigger = [1, 2] + elif ( + event_data["trigger_pattern"][i_event] + == DATA_TOPOLOGICAL_TRIGGER + ): + event.trigger.tels_with_trigger = [tel_id, 3] + elif ( + event_data["trigger_pattern"][i_event] == DATA_MAGIC_LST_TRIGGER + ): + event.trigger.tels_with_trigger = [1, 2, 3] + else: + event.trigger.tels_with_trigger = [tel_id] + else: + if self.is_stereo and not self.use_mc_mono_events: + event.trigger.tels_with_trigger = [1, 2] + else: + event.trigger.tels_with_trigger = [tel_id] + + if self.allowed_tels: + tels_with_trigger = np.intersect1d( + event.trigger.tels_with_trigger, + self.subarray.tel_ids, + assume_unique=True, + ) + + event.trigger.tels_with_trigger = tels_with_trigger.tolist() + + event.index.event_id = event_data["event_number"][i_event] event.trigger.event_type = MAGIC_TO_CTA_EVENT_TYPE.get( event_data["trigger_pattern"][i_event] From 8c79dbdef0e95d4ae9913a0161c7a8db619a6369 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 15 May 2024 19:58:50 +0200 Subject: [PATCH 77/78] Correct check for HaST data. --- ctapipe_io_magic/tests/test_magic_event_source.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ctapipe_io_magic/tests/test_magic_event_source.py b/ctapipe_io_magic/tests/test_magic_event_source.py index a41d2b2..c368c48 100644 --- a/ctapipe_io_magic/tests/test_magic_event_source.py +++ b/ctapipe_io_magic/tests/test_magic_event_source.py @@ -416,10 +416,16 @@ def test_loop(dataset): if source.mars_datalevel <= MARSDataLevel.STAR: if "_M1_" in dataset.name and source.is_stereo: assert 1 in event.trigger.tels_with_trigger - assert event.trigger.tels_with_trigger == [1, 2] + if source.is_hast: + assert set(event.trigger.tels_with_trigger).issubset({1, 2, 3}) + else: + assert set(event.trigger.tels_with_trigger).issubset({1, 2}) if "_M2_" in dataset.name and source.is_stereo: assert 2 in event.trigger.tels_with_trigger - assert event.trigger.tels_with_trigger == [1, 2] + if source.is_hast: + assert set(event.trigger.tels_with_trigger).issubset({1, 2, 3}) + else: + assert set(event.trigger.tels_with_trigger).issubset({1, 2}) else: assert event.trigger.tels_with_trigger == [1, 2] From e43a20338bbd872574ebbd5bb419c906cec4b101 Mon Sep 17 00:00:00 2001 From: Alessio Berti Date: Wed, 15 May 2024 20:31:23 +0200 Subject: [PATCH 78/78] Retrieve correctly all information. --- ctapipe_io_magic/__init__.py | 588 +++++++++++++++++++++++------------ 1 file changed, 391 insertions(+), 197 deletions(-) diff --git a/ctapipe_io_magic/__init__.py b/ctapipe_io_magic/__init__.py index 26bd7d2..70c08e2 100644 --- a/ctapipe_io_magic/__init__.py +++ b/ctapipe_io_magic/__init__.py @@ -1880,7 +1880,6 @@ def _event_generator(self): if self.use_pedestals: event_data = self.current_run["data"].pedestal_events n_events = self.current_run["data"].n_pedestal_events - else: event_data = self.current_run["data"].cosmic_events n_events = self.current_run["data"].n_cosmic_events @@ -1996,244 +1995,439 @@ def _event_generator(self): # Loop over the events: for i_event in range(n_events): + if self.mars_datalevel >= MARSDataLevel.SUPERSTAR: + event.dl2.stereo.geometry["HillasReconstructor"] = ( + ReconstructedGeometryContainer( + alt=event_data["stereo"]["alt"][i_event], + az=event_data["stereo"]["az"][i_event], + core_x=event_data["stereo"]["core_x"][i_event], + core_y=event_data["stereo"]["core_y"][i_event], + h_max=event_data["stereo"]["h_max"][i_event], + is_valid=( + True + if event_data["stereo"]["is_valid"][i_event] == 1 + else False + ), + average_intensity=( + event_data[1]["intensity"][i_event] + + event_data[2]["intensity"][i_event] + ) + / 2.0, + telescopes=[1, 2], + ) + ) + + if self.mars_datalevel >= MARSDataLevel.MELIBEA: + event.dl2.stereo.energy = ReconstructedEnergyContainer( + energy=event_data["reconstructed"]["energy"][i_event], + energy_uncert=event_data["reconstructed"]["energy_uncert"][ + i_event + ], + is_valid=( + True + if event_data["stereo"]["is_valid"][i_event] == 1 + else False + ), + telescopes=[1, 2], + ) + + event.dl2.stereo.classification = ( + ParticleClassificationContainer( + prediction=event_data["reconstructed"]["gammanness"][ + i_event + ], + is_valid=( + True + if event_data["stereo"]["is_valid"][i_event] == 1 + else False + ), + telescopes=[1, 2], + ) + ) + for tel_id in self.telescopes: event.count = counter - if not self.is_simulation: - if ( - event_data["trigger_pattern"][i_event] - == DATA_STEREO_TRIGGER_PATTERN - ): - event.trigger.tels_with_trigger = [1, 2] - elif ( - event_data["trigger_pattern"][i_event] - == DATA_TOPOLOGICAL_TRIGGER - ): - event.trigger.tels_with_trigger = [tel_id, 3] - elif ( - event_data["trigger_pattern"][i_event] == DATA_MAGIC_LST_TRIGGER - ): - event.trigger.tels_with_trigger = [1, 2, 3] - else: - event.trigger.tels_with_trigger = [tel_id] - else: - if self.is_stereo and not self.use_mc_mono_events: - event.trigger.tels_with_trigger = [1, 2] + if self.mars_datalevel >= MARSDataLevel.SUPERSTAR: + + if not self.is_simulation: + if ( + event_data[tel_id]["trigger_pattern"][i_event] + == DATA_STEREO_TRIGGER_PATTERN + ): + event.trigger.tels_with_trigger = [1, 2] + elif ( + event_data[tel_id]["trigger_pattern"][i_event] + == DATA_TOPOLOGICAL_TRIGGER + ): + event.trigger.tels_with_trigger = [tel_id, 3] + elif ( + event_data[tel_id]["trigger_pattern"][i_event] + == DATA_MAGIC_LST_TRIGGER + ): + event.trigger.tels_with_trigger = [1, 2, 3] + else: + event.trigger.tels_with_trigger = [tel_id] else: - event.trigger.tels_with_trigger = [tel_id] + if self.is_stereo and not self.use_mc_mono_events: + event.trigger.tels_with_trigger = [1, 2] + else: + event.trigger.tels_with_trigger = [tel_id] - if self.allowed_tels: - tels_with_trigger = np.intersect1d( - event.trigger.tels_with_trigger, - self.subarray.tel_ids, - assume_unique=True, - ) + if self.allowed_tels: + tels_with_trigger = np.intersect1d( + event.trigger.tels_with_trigger, + self.subarray.tel_ids, + assume_unique=True, + ) - event.trigger.tels_with_trigger = tels_with_trigger.tolist() + event.trigger.tels_with_trigger = tels_with_trigger.tolist() - event.index.event_id = event_data["event_number"][i_event] + event.index.event_id = event_data[tel_id]["event_number"][i_event] - event.trigger.event_type = MAGIC_TO_CTA_EVENT_TYPE.get( - event_data["trigger_pattern"][i_event] - ) + event.trigger.event_type = MAGIC_TO_CTA_EVENT_TYPE.get( + event_data[tel_id]["trigger_pattern"][i_event] + ) - if not self.is_simulation: - event.trigger.time = event_data["time"][i_event] - event.trigger.tel[tel_id].time = event_data["time"][ + if not self.is_simulation: + event.trigger.time = event_data[tel_id]["time"][i_event] + event.trigger.tel[tel_id].time = event_data[tel_id]["time"][i_event] + + if ( + not self.use_pedestals + and self.mars_datalevel == MARSDataLevel.CALIBRATED + ): + badrmspixel_mask = self.get_badrmspixel_mask( + event, tel_id + ) + event.mon.tel[ + tel_id + ].pixel_status.pedestal_failing_pixels = ( + badrmspixel_mask + ) + + # Set the telescope pointing container: + event.pointing.array_azimuth = event_data[tel_id]["pointing_az"][ i_event - ] - - if ( - not self.use_pedestals - and self.mars_datalevel == MARSDataLevel.CALIBRATED - ): - badrmspixel_mask = self.get_badrmspixel_mask(event, tel_id) - event.mon.tel[ - tel_id - ].pixel_status.pedestal_failing_pixels = badrmspixel_mask - - # Set the telescope pointing container: - event.pointing.array_azimuth = event_data["pointing_az"][ - i_event - ].to(u.rad) - event.pointing.array_altitude = event_data["pointing_alt"][ - i_event - ].to(u.rad) - event.pointing.array_ra = event_data["pointing_ra"][ - i_event - ].to(u.rad) - event.pointing.array_dec = event_data["pointing_dec"][ - i_event - ].to(u.rad) - - event.pointing.tel[tel_id].azimuth = event_data[ - "pointing_az" - ][i_event].to(u.rad) - event.pointing.tel[tel_id].altitude = event_data[ - "pointing_alt" - ][i_event].to(u.rad) - - if self.mars_datalevel == MARSDataLevel.CALIBRATED: - # Set event charge and peak positions per pixel: - event.dl1.tel[tel_id].image = event_data["image"][ + ].to(u.rad) + event.pointing.array_altitude = event_data[tel_id]["pointing_alt"][ i_event - ] - event.dl1.tel[tel_id].peak_time = event_data[ - "peak_time" - ][i_event] + ].to(u.rad) + event.pointing.array_ra = event_data[tel_id]["pointing_ra"][i_event].to( + u.rad + ) + event.pointing.array_dec = event_data[tel_id]["pointing_dec"][ + i_event + ].to(u.rad) - if self.mars_datalevel >= MARSDataLevel.SUPERSTAR: - event.dl1.tel[tel_id].parameters = ImageParametersContainer() - event.dl1.tel[ - tel_id - ].parameters.hillas = CameraHillasParametersContainer( - x=-event_data["y"][i_event], - y=-event_data["x"][i_event], - length=event_data["length"][i_event], - width=event_data["width"][i_event], - intensity=event_data["intensity"][i_event], - r=np.sqrt( - event_data["x"][i_event] - * event_data["x"][i_event] - + event_data["y"][i_event] - * event_data["y"][i_event] - ), - psi=event_data["psi"][i_event], - phi=event_data["phi"][i_event], + event.pointing.tel[tel_id].azimuth = event_data[tel_id]["pointing_az"][ + i_event + ].to(u.rad) + event.pointing.tel[tel_id].altitude = event_data[tel_id][ + "pointing_alt" + ][i_event].to(u.rad) + + event.dl1.tel[tel_id].parameters = ( + ImageParametersContainer() + ) + event.dl1.tel[tel_id].parameters.hillas = ( + CameraHillasParametersContainer( + x=-event_data[tel_id]["y"][i_event], + y=-event_data[tel_id]["x"][i_event], + length=event_data[tel_id]["length"][i_event], + width=event_data[tel_id]["width"][i_event], + intensity=event_data[tel_id]["intensity"][i_event], + r=np.sqrt( + event_data[tel_id]["x"][i_event] + * event_data[tel_id]["x"][i_event] + + event_data[tel_id]["y"][i_event] + * event_data[tel_id]["y"][i_event] + ), + psi=event_data[tel_id]["psi"][i_event], + phi=event_data[tel_id]["phi"][i_event], + ) ) - event.dl1.tel[ - tel_id - ].parameters.timing = TimingParametersContainer( - slope=event_data["slope"][i_event].value - * (1.0 / m2deg) - / u.deg, - intercept=event_data["intercept"][i_event], + event.dl1.tel[tel_id].parameters.timing = ( + TimingParametersContainer( + slope=event_data[tel_id]["slope"][i_event].value + * (1.0 / m2deg) + / u.deg, + intercept=event_data[tel_id]["intercept"][i_event], + ) ) # not fully filled event.dl1.tel[tel_id].parameters.leakage = LeakageContainer( - intensity_width_1=event_data["intensity_width_1"][ + intensity_width_1=event_data[tel_id]["intensity_width_1"][ i_event ], - intensity_width_2=event_data["intensity_width_2"][ + intensity_width_2=event_data[tel_id]["intensity_width_2"][ i_event ], ) # not fully filled - event.dl1.tel[ - tel_id - ].parameters.morphology = MorphologyContainer( - n_pixels=event_data["num_pixels"][i_event], - n_islands=event_data["num_islands"][i_event], + event.dl1.tel[tel_id].parameters.morphology = ( + MorphologyContainer( + n_pixels=event_data[tel_id]["num_pixels"][i_event], + n_islands=event_data[tel_id]["num_islands"][i_event], + ) ) - event.dl2.stereo.geometry[ - "HillasReconstructor" - ] = ReconstructedGeometryContainer( - alt=event_data["stereo"]["alt"][i_event], - az=event_data["stereo"]["az"][i_event], - core_x=event_data["stereo"]["core_x"][i_event], - core_y=event_data["stereo"]["core_y"][i_event], - h_max=event_data["stereo"]["h_max"][i_event], - is_valid=True - if event_data["stereo"]["is_valid"][i_event] == 1 - else False, - average_intensity=( - event_data[1]["intensity"][i_event] - + event_data[2]["intensity"][i_event] + # Set the simulated event container: + if self.is_simulation: + event.simulation = SimulatedEventContainer() + + event.simulation.shower.energy = event_data[tel_id]["mc_energy"][ + i_event + ].to(u.TeV) + event.simulation.shower.shower_primary_id = ( + 1 - event_data[tel_id]["mc_shower_primary_id"][i_event] ) - / 2.0, - telescopes=[1, 2], - ) + event.simulation.shower.h_first_int = event_data[tel_id][ + "mc_h_first_int" + ][i_event].to(u.m) - if self.mars_datalevel >= MARSDataLevel.MELIBEA: - event.dl2.stereo.energy = ReconstructedEnergyContainer( - energy=event_data["reconstructed"]["energy"][i_event], - energy_uncert=event_data["reconstructed"]["energy_uncert"][ + event.simulation.shower.x_max = event_data[tel_id]["mc_x_max"][ i_event - ], - is_valid=True - if event_data["stereo"]["is_valid"][i_event] == 1 - else False, - telescopes=[1, 2], + ].to("g cm-2") + + # Convert the corsika coordinate (x-axis: magnetic north) to the geographical one. + # Rotate the corsika coordinate by the magnetic declination (= 7 deg): + mfield_dec = self.simulation_config[self.obs_ids[0]][ + "prod_site_B_declination" + ] + + event.simulation.shower.alt = u.Quantity( + 90, u.deg + ) - event_data[tel_id]["mc_theta"][i_event].to(u.deg) + event.simulation.shower.az = ( + u.Quantity(180, u.deg) + - event_data[tel_id]["mc_phi"][i_event].to(u.deg) + + mfield_dec + ) + + if event.simulation.shower.az > u.Quantity(180, u.deg): + event.simulation.shower.az -= u.Quantity(360, u.deg) + + event.simulation.shower.core_x = event_data[tel_id]["mc_core_x"][ + i_event + ].to(u.m) * np.cos(mfield_dec) + event_data[tel_id]["mc_core_y"][ + i_event + ].to( + u.m + ) * np.sin( + mfield_dec + ) + + event.simulation.shower.core_y = event_data[tel_id]["mc_core_y"][ + i_event + ].to(u.m) * np.cos(mfield_dec) - event_data[tel_id]["mc_core_x"][ + i_event + ].to( + u.m + ) * np.sin( + mfield_dec + ) + + yield event + counter += 1 + + else: + + if not self.is_simulation: + if ( + event_data["trigger_pattern"][i_event] + == DATA_STEREO_TRIGGER_PATTERN + ): + event.trigger.tels_with_trigger = [1, 2] + elif ( + event_data["trigger_pattern"][i_event] + == DATA_TOPOLOGICAL_TRIGGER + ): + event.trigger.tels_with_trigger = [tel_id, 3] + elif ( + event_data["trigger_pattern"][i_event] == DATA_MAGIC_LST_TRIGGER + ): + event.trigger.tels_with_trigger = [1, 2, 3] + else: + event.trigger.tels_with_trigger = [tel_id] + else: + if self.is_stereo and not self.use_mc_mono_events: + event.trigger.tels_with_trigger = [1, 2] + else: + event.trigger.tels_with_trigger = [tel_id] + + if self.allowed_tels: + tels_with_trigger = np.intersect1d( + event.trigger.tels_with_trigger, + self.subarray.tel_ids, + assume_unique=True, + ) + + event.trigger.tels_with_trigger = tels_with_trigger.tolist() + + event.index.event_id = event_data["event_number"][i_event] + + event.trigger.event_type = MAGIC_TO_CTA_EVENT_TYPE.get( + event_data["trigger_pattern"][i_event] ) - event.dl2.stereo.classification = ( - ParticleClassificationContainer( - prediction=event_data["reconstructed"]["gammanness"][ + if not self.is_simulation: + event.trigger.time = event_data["time"][i_event] + event.trigger.tel[tel_id].time = event_data["time"][ + i_event + ] + + if ( + not self.use_pedestals + and self.mars_datalevel == MARSDataLevel.CALIBRATED + ): + badrmspixel_mask = self.get_badrmspixel_mask(event, tel_id) + event.mon.tel[ + tel_id + ].pixel_status.pedestal_failing_pixels = badrmspixel_mask + + # Set the telescope pointing container: + event.pointing.array_azimuth = event_data["pointing_az"][ + i_event + ].to(u.rad) + event.pointing.array_altitude = event_data["pointing_alt"][ + i_event + ].to(u.rad) + event.pointing.array_ra = event_data["pointing_ra"][ + i_event + ].to(u.rad) + event.pointing.array_dec = event_data["pointing_dec"][ + i_event + ].to(u.rad) + + event.pointing.tel[tel_id].azimuth = event_data[ + "pointing_az" + ][i_event].to(u.rad) + event.pointing.tel[tel_id].altitude = event_data[ + "pointing_alt" + ][i_event].to(u.rad) + + if self.mars_datalevel == MARSDataLevel.CALIBRATED: + # Set event charge and peak positions per pixel: + event.dl1.tel[tel_id].image = event_data["image"][ + i_event + ] + event.dl1.tel[tel_id].peak_time = event_data[ + "peak_time" + ][i_event] + + if self.mars_datalevel == MARSDataLevel.STAR: + event.dl1.tel[tel_id].parameters = ImageParametersContainer() + event.dl1.tel[ + tel_id + ].parameters.hillas = CameraHillasParametersContainer( + x=-event_data["y"][i_event], + y=-event_data["x"][i_event], + length=event_data["length"][i_event], + width=event_data["width"][i_event], + intensity=event_data["intensity"][i_event], + r=np.sqrt( + event_data["x"][i_event] + * event_data["x"][i_event] + + event_data["y"][i_event] + * event_data["y"][i_event] + ), + psi=event_data["psi"][i_event], + phi=event_data["phi"][i_event], + ) + + event.dl1.tel[ + tel_id + ].parameters.timing = TimingParametersContainer( + slope=event_data["slope"][i_event].value + * (1.0 / m2deg) + / u.deg, + intercept=event_data["intercept"][i_event], + ) + + # not fully filled + event.dl1.tel[tel_id].parameters.leakage = LeakageContainer( + intensity_width_1=event_data["intensity_width_1"][ + i_event + ], + intensity_width_2=event_data["intensity_width_2"][ i_event ], - is_valid=True - if event_data["stereo"]["is_valid"][i_event] == 1 - else False, - telescopes=[1, 2], ) - ) - # Set the simulated event container: - if self.is_simulation: - event.simulation = SimulatedEventContainer() + # not fully filled + event.dl1.tel[ + tel_id + ].parameters.morphology = MorphologyContainer( + n_pixels=event_data["num_pixels"][i_event], + n_islands=event_data["num_islands"][i_event], + ) - event.simulation.shower.energy = event_data[ - "mc_energy" - ][i_event].to(u.TeV) - event.simulation.shower.shower_primary_id = ( - 1 - event_data["mc_shower_primary_id"][i_event] - ) - event.simulation.shower.h_first_int = event_data[ - "mc_h_first_int" - ][i_event].to(u.m) + # Set the simulated event container: + if self.is_simulation: + event.simulation = SimulatedEventContainer() - event.simulation.shower.x_max = event_data["mc_x_max"][ - i_event - ].to("g cm-2") - - # Convert the corsika coordinate (x-axis: magnetic north) to the geographical one. - # Rotate the corsika coordinate by the magnetic declination (= 7 deg): - mfield_dec = self.simulation_config[self.obs_ids[0]][ - "prod_site_B_declination" - ] - - event.simulation.shower.alt = u.Quantity( - 90, u.deg - ) - event_data["mc_theta"][i_event].to(u.deg) - event.simulation.shower.az = ( - u.Quantity(180, u.deg) - - event_data["mc_phi"][i_event].to(u.deg) - + mfield_dec - ) + event.simulation.shower.energy = event_data[ + "mc_energy" + ][i_event].to(u.TeV) + event.simulation.shower.shower_primary_id = ( + 1 - event_data["mc_shower_primary_id"][i_event] + ) + event.simulation.shower.h_first_int = event_data[ + "mc_h_first_int" + ][i_event].to(u.m) + + event.simulation.shower.x_max = event_data["mc_x_max"][ + i_event + ].to("g cm-2") + + # Convert the corsika coordinate (x-axis: magnetic north) to the geographical one. + # Rotate the corsika coordinate by the magnetic declination (= 7 deg): + mfield_dec = self.simulation_config[self.obs_ids[0]][ + "prod_site_B_declination" + ] + + event.simulation.shower.alt = u.Quantity( + 90, u.deg + ) - event_data["mc_theta"][i_event].to(u.deg) + event.simulation.shower.az = ( + u.Quantity(180, u.deg) + - event_data["mc_phi"][i_event].to(u.deg) + + mfield_dec + ) - if event.simulation.shower.az > u.Quantity(180, u.deg): - event.simulation.shower.az -= u.Quantity(360, u.deg) + if event.simulation.shower.az > u.Quantity(180, u.deg): + event.simulation.shower.az -= u.Quantity(360, u.deg) - event.simulation.shower.core_x = event_data[ - "mc_core_x" - ][i_event].to(u.m) * np.cos(mfield_dec) + event_data[ - "mc_core_y" - ][ - i_event - ].to( - u.m - ) * np.sin( - mfield_dec - ) + event.simulation.shower.core_x = event_data[ + "mc_core_x" + ][i_event].to(u.m) * np.cos(mfield_dec) + event_data[ + "mc_core_y" + ][ + i_event + ].to( + u.m + ) * np.sin( + mfield_dec + ) - event.simulation.shower.core_y = event_data[ - "mc_core_y" - ][i_event].to(u.m) * np.cos(mfield_dec) - event_data[ - "mc_core_x" - ][ - i_event - ].to( - u.m - ) * np.sin( - mfield_dec - ) + event.simulation.shower.core_y = event_data[ + "mc_core_y" + ][i_event].to(u.m) * np.cos(mfield_dec) - event_data[ + "mc_core_x" + ][ + i_event + ].to( + u.m + ) * np.sin( + mfield_dec + ) - yield event - counter += 1 + yield event + counter += 1 return