From 33ef05739ed403d4af6ba915a406905080dbc5ca Mon Sep 17 00:00:00 2001 From: Andrea Bocci Date: Wed, 21 Apr 2021 00:32:12 +0200 Subject: [PATCH] Make the HCAL 'gpu' workflow use a GPU if available, and fall-back to CPU otherwise --- .../python/Reconstruction_cff.py | 1 + .../python/OfflineSourceSequence_pp.py | 29 +++++++------ .../python/RecoLocalCalo_Cosmics_cff.py | 31 +++++++------- .../python/hcalGlobalReco_cff.py | 25 +++++++---- .../Configuration/python/hcalLocalReco_cff.py | 42 +++++++++---------- .../python/HBHE_custom_25nsMethod.py | 4 +- .../test/make_GPUvsCPU_HCAL_rechits.py | 22 +++++----- .../python/particleFlowRecHitHBHE_cfi.py | 5 ++- 8 files changed, 85 insertions(+), 74 deletions(-) diff --git a/Configuration/StandardSequences/python/Reconstruction_cff.py b/Configuration/StandardSequences/python/Reconstruction_cff.py index e03d228a857fb..62d03776be5de 100644 --- a/Configuration/StandardSequences/python/Reconstruction_cff.py +++ b/Configuration/StandardSequences/python/Reconstruction_cff.py @@ -235,6 +235,7 @@ bunchSpacingProducer, offlineBeamSpot, hcalOnlyLocalRecoTask, + hcalOnlyGlobalRecoTask, pfClusteringHBHEHFOnlyTask ) diff --git a/DQM/HcalTasks/python/OfflineSourceSequence_pp.py b/DQM/HcalTasks/python/OfflineSourceSequence_pp.py index 0543cf3aae84d..65172a4180be0 100644 --- a/DQM/HcalTasks/python/OfflineSourceSequence_pp.py +++ b/DQM/HcalTasks/python/OfflineSourceSequence_pp.py @@ -1,36 +1,41 @@ import FWCore.ParameterSet.Config as cms #----------------- -# HCAL DQM Offline Source Sequence Definition for pp -# To be used for Offline DQM importing +# HCAL DQM Offline Source Sequence Definition for pp +# To be used for Offline DQM importing #----------------- -# import the tasks +# import the tasks from DQM.HcalTasks.DigiTask import digiTask from DQM.HcalTasks.RawTask import rawTask from DQM.HcalTasks.TPTask import tpTask from DQM.HcalTasks.RecHitTask import recHitTask, recHitPreRecoTask -# set processing type to Offine +# set processing type to Offine digiTask.ptype = cms.untracked.int32(1) tpTask.ptype = cms.untracked.int32(1) recHitTask.ptype = cms.untracked.int32(1) rawTask.ptype = cms.untracked.int32(1) recHitPreRecoTask.ptype = cms.untracked.int32(1) -# set the label for Emulator TP Task +# set the label for Emulator TP Task tpTask.tagEmul = cms.untracked.InputTag("valHcalTriggerPrimitiveDigis") hcalOfflineSourceSequence = cms.Sequence( - digiTask - +tpTask - +recHitTask - +rawTask) + digiTask + + tpTask + + recHitTask + + rawTask ) hcalOnlyOfflineSourceSequence = cms.Sequence( - digiTask - +recHitPreRecoTask - +rawTask) + digiTask + + recHitPreRecoTask + + rawTask ) + +from Configuration.Eras.Modifier_run3_HB_cff import run3_HB +_phase1_hcalOnlyOfflineSourceSequence = hcalOnlyOfflineSourceSequence.copy() +_phase1_hcalOnlyOfflineSourceSequence.replace(recHitPreRecoTask, recHitTask) +run3_HB.toReplaceWith(hcalOnlyOfflineSourceSequence, _phase1_hcalOnlyOfflineSourceSequence) from Configuration.Eras.Modifier_phase2_hcal_cff import phase2_hcal _phase2_hcalOfflineSourceSequence = hcalOfflineSourceSequence.copyAndExclude([tpTask,rawTask]) diff --git a/RecoLocalCalo/Configuration/python/RecoLocalCalo_Cosmics_cff.py b/RecoLocalCalo/Configuration/python/RecoLocalCalo_Cosmics_cff.py index 40718749c65db..25b8fed436ae2 100644 --- a/RecoLocalCalo/Configuration/python/RecoLocalCalo_Cosmics_cff.py +++ b/RecoLocalCalo/Configuration/python/RecoLocalCalo_Cosmics_cff.py @@ -19,23 +19,20 @@ # sequence CaloLocalReco # -def hbheCosmic(module): - return module.clone( - tsFromDB = False, - recoParamsFromDB = False, - algorithm = dict( - useMahi = False, - useM2 = False, - useM3 = False, - firstSampleShift = -1000, - samplesToAdd = 10, - correctForPhaseContainment = False, - ), - sipmQTSShift = -100, - sipmQNTStoSum = 200, - ) - -hbhereco = hbheCosmic(_hcalLocalReco_cff.hbheprereco) +hbhereco = _hcalLocalReco_cff.hbheprereco.cpu.clone( + tsFromDB = False, + recoParamsFromDB = False, + algorithm = dict( + useMahi = False, + useM2 = False, + useM3 = False, + firstSampleShift = -1000, + samplesToAdd = 10, + correctForPhaseContainment = False, + ), + sipmQTSShift = -100, + sipmQNTStoSum = 200, +) hfreco = _hcalLocalReco_cff._default_hfreco.clone( firstSample = 0, samplesToAdd = 10, ### min(10,size) in the algo diff --git a/RecoLocalCalo/Configuration/python/hcalGlobalReco_cff.py b/RecoLocalCalo/Configuration/python/hcalGlobalReco_cff.py index fbb4c53f9f28b..30b7927d5753a 100644 --- a/RecoLocalCalo/Configuration/python/hcalGlobalReco_cff.py +++ b/RecoLocalCalo/Configuration/python/hcalGlobalReco_cff.py @@ -1,21 +1,30 @@ import FWCore.ParameterSet.Config as cms +from HeterogeneousCore.CUDACore.SwitchProducerCUDA import SwitchProducerCUDA -from RecoLocalCalo.HcalRecProducers.HBHEIsolatedNoiseReflagger_cfi import * +#--- for Run 1 and Run 2 +from RecoLocalCalo.HcalRecProducers.HBHEIsolatedNoiseReflagger_cfi import hbhereco as _phase0_hbhereco +hbhereco = SwitchProducerCUDA( + cpu = _phase0_hbhereco +) hcalGlobalRecoTask = cms.Task(hbhereco) hcalGlobalRecoSequence = cms.Sequence(hcalGlobalRecoTask) +hcalOnlyGlobalRecoTask = cms.Task() +hcalOnlyGlobalRecoSequence = cms.Sequence(hcalOnlyGlobalRecoTask) + #--- for Run 3 and later from Configuration.Eras.Modifier_run3_HB_cff import run3_HB from RecoLocalCalo.HcalRecProducers.HBHEPhase1Reconstructor_cfi import hbheprereco as _phase1_hbheprereco -run3_HB.toReplaceWith(hbhereco, _phase1_hbheprereco) +run3_HB.toReplaceWith(hbhereco.cpu, _phase1_hbheprereco) +run3_HB.toReplaceWith(hcalOnlyGlobalRecoTask, cms.Task(hbhereco)) #--- for Run 3 on GPU from Configuration.ProcessModifiers.gpu_cff import gpu -from RecoLocalCalo.HcalRecProducers.hcalCPURecHitsProducer_cfi import hcalCPURecHitsProducer as _hcalCPURecHitsProducer -gpu.toReplaceWith(hbhereco, _hcalCPURecHitsProducer.clone( - recHitsM0LabelIn = "hbheRecHitProducerGPU", - recHitsM0LabelOut = "", - recHitsLegacyLabelOut = "" -)) +from RecoLocalCalo.HcalRecProducers.hcalCPURecHitsProducer_cfi import hcalCPURecHitsProducer as _hbherecoFromCUDA +(run3_HB & gpu).toModify(hbhereco, + cuda = _hbherecoFromCUDA.clone( + produceSoA = False + ) +) diff --git a/RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py b/RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py index a7bdce3b916af..4c68380d686ac 100644 --- a/RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py +++ b/RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py @@ -1,15 +1,18 @@ import FWCore.ParameterSet.Config as cms +from HeterogeneousCore.CUDACore.SwitchProducerCUDA import SwitchProducerCUDA from RecoLocalCalo.HcalRecAlgos.hcalRecAlgoESProd_cfi import * from RecoLocalCalo.HcalRecAlgos.hcalChannelPropertiesESProd_cfi import * hcalOOTPileupESProducer = cms.ESProducer('OOTPileupDBCompatibilityESProducer') from RecoLocalCalo.HcalRecProducers.HBHEPhase1Reconstructor_cfi import hbheprereco as _phase1_hbheprereco -hbheprereco = _phase1_hbheprereco.clone( - processQIE11 = False, - tsFromDB = True, - pulseShapeParametersQIE8 = dict( - TrianglePeakTS = 4, +hbheprereco = SwitchProducerCUDA( + cpu = _phase1_hbheprereco.clone( + processQIE11 = False, + tsFromDB = True, + pulseShapeParametersQIE8 = dict( + TrianglePeakTS = 4, + ) ) ) @@ -34,7 +37,7 @@ run2_HF_2017.toReplaceWith( hcalLocalRecoTask, _phase1_hcalLocalRecoTask ) run2_HF_2017.toReplaceWith( hfreco, _phase1_hfreco ) from Configuration.Eras.Modifier_run2_HCAL_2017_cff import run2_HCAL_2017 -run2_HCAL_2017.toReplaceWith(hbheprereco, _phase1_hbheprereco) +run2_HCAL_2017.toReplaceWith(hbheprereco.cpu, _phase1_hbheprereco) _plan1_hcalLocalRecoTask = _phase1_hcalLocalRecoTask.copy() _plan1_hcalLocalRecoTask.add(hbheplan1) @@ -57,27 +60,20 @@ from Configuration.ProcessModifiers.gpu_cff import gpu from RecoLocalCalo.HcalRecProducers.hbheRecHitProducerGPUTask_cff import * -_run3_hcalLocalRecoGPUTask = _run3_hcalLocalRecoTask.copy() +_run3_hcalLocalRecoGPUTask = hcalLocalRecoTask.copy() _run3_hcalLocalRecoGPUTask.add(hbheRecHitProducerGPUTask) gpu.toReplaceWith(hcalLocalRecoTask, _run3_hcalLocalRecoGPUTask) -#--- HCAL-only workflow for Run 3 -# FIXME rename `hbheprereco` to `hbhereco` and use it from hcalGlobalRecoTask -hcalOnlyLocalRecoTask = cms.Task(hbheprereco, hfprereco, hfreco, horeco) +#--- HCAL-only workflow +hcalOnlyLocalRecoTask = hcalLocalRecoTask.copyAndExclude([zdcreco]) -#--- HCAL-only workflow for Run 3 on GPU -from Configuration.ProcessModifiers.gpu_cff import gpu - -_hcalOnlyLocalRecoGPUTask = hcalOnlyLocalRecoTask.copy() -_hcalOnlyLocalRecoGPUTask.add(hbheRecHitProducerGPUTask) -gpu.toReplaceWith(hcalOnlyLocalRecoTask, _hcalOnlyLocalRecoGPUTask) - -from RecoLocalCalo.HcalRecProducers.hcalCPURecHitsProducer_cfi import hcalCPURecHitsProducer as _hcalCPURecHitsProducer -gpu.toReplaceWith(hbheprereco, _hcalCPURecHitsProducer.clone( - recHitsM0LabelIn = "hbheRecHitProducerGPU", - recHitsM0LabelOut = "", - recHitsLegacyLabelOut = "" -)) +#--- HCAL-only workflow for Run 2 on GPU +from RecoLocalCalo.HcalRecProducers.hcalCPURecHitsProducer_cfi import hcalCPURecHitsProducer as _hbheprerecoFromCUDA +(gpu & ~run3_HB).toModify(hbheprereco, + cuda = _hbheprerecoFromCUDA.clone( + produceSoA = False + ) +) #--- for FastSim _fastSim_hcalLocalRecoTask = hcalLocalRecoTask.copyAndExclude([zdcreco]) diff --git a/RecoLocalCalo/HcalRecProducers/python/HBHE_custom_25nsMethod.py b/RecoLocalCalo/HcalRecProducers/python/HBHE_custom_25nsMethod.py index 455f3df9178f2..988a4b4a2b3cc 100644 --- a/RecoLocalCalo/HcalRecProducers/python/HBHE_custom_25nsMethod.py +++ b/RecoLocalCalo/HcalRecProducers/python/HBHE_custom_25nsMethod.py @@ -2,11 +2,11 @@ def customise_HBHE_Method1(process): if hasattr(process,'hbheprereco'): - process.hbheprereco.puCorrMethod = cms.int32(1) + process.hbheprereco.cpu.puCorrMethod = cms.int32(1) return process def customise_HBHE_Method0(process): if hasattr(process,'hbheprereco'): - process.hbheprereco.puCorrMethod = cms.int32(0) + process.hbheprereco.cpu.puCorrMethod = cms.int32(0) return process diff --git a/RecoLocalCalo/HcalRecProducers/test/make_GPUvsCPU_HCAL_rechits.py b/RecoLocalCalo/HcalRecProducers/test/make_GPUvsCPU_HCAL_rechits.py index 8c7162bc076c4..0ce9caf13fa88 100644 --- a/RecoLocalCalo/HcalRecProducers/test/make_GPUvsCPU_HCAL_rechits.py +++ b/RecoLocalCalo/HcalRecProducers/test/make_GPUvsCPU_HCAL_rechits.py @@ -59,32 +59,32 @@ #process.hbheprereco.algorithm.correctForPhaseContainment = cms.bool(False) ## do always 8 pulse -process.hbheprereco.algorithm.chiSqSwitch = cms.double(-1) +process.hbheprereco.cpu.algorithm.chiSqSwitch = cms.double(-1) ## to match hard coded setting (will be fixed on CPU) -process.hbheprereco.algorithm.nMaxItersMin = cms.int32(50) +process.hbheprereco.cpu.algorithm.nMaxItersMin = cms.int32(50) #----------------------------------------- # Final Custmization for Run3 #----------------------------------------- # we will not run arrival Time at HLT -process.hbheprereco.algorithm.calculateArrivalTime = cms.bool(False) +process.hbheprereco.cpu.algorithm.calculateArrivalTime = cms.bool(False) ## we do not need this -process.hbheprereco.algorithm.applyLegacyHBMCorrection = cms.bool(False) +process.hbheprereco.cpu.algorithm.applyLegacyHBMCorrection = cms.bool(False) # we only run Mahi at HLT -process.hbheprereco.algorithm.useM3 = cms.bool(False) +process.hbheprereco.cpu.algorithm.useM3 = cms.bool(False) # we will not have the HPD noise flags in Run3, as will be all siPM -process.hbheprereco.setLegacyFlagsQIE8 = cms.bool(False) -process.hbheprereco.setNegativeFlagsQIE8 = cms.bool(False) -process.hbheprereco.setNoiseFlagsQIE8 = cms.bool(False) -process.hbheprereco.setPulseShapeFlagsQIE8 = cms.bool(False) +process.hbheprereco.cpu.setLegacyFlagsQIE8 = cms.bool(False) +process.hbheprereco.cpu.setNegativeFlagsQIE8 = cms.bool(False) +process.hbheprereco.cpu.setNoiseFlagsQIE8 = cms.bool(False) +process.hbheprereco.cpu.setPulseShapeFlagsQIE8 = cms.bool(False) # for testing M0 only -##process.hbheprereco.algorithm.useMahi = cms.bool(False) +##process.hbheprereco.cpu.algorithm.useMahi = cms.bool(False) #----------------------------------------- # OUTPUT @@ -148,4 +148,4 @@ # report CUDAService messages process.MessageLogger.cerr.FwkReport.reportEvery = 100 -process.MessageLogger.categories.append("CUDAService") +process.MessageLogger.CUDAService = cms.untracked.PSet() diff --git a/RecoParticleFlow/PFClusterProducer/python/particleFlowRecHitHBHE_cfi.py b/RecoParticleFlow/PFClusterProducer/python/particleFlowRecHitHBHE_cfi.py index c8939961bd36e..815e60d3c2b5a 100644 --- a/RecoParticleFlow/PFClusterProducer/python/particleFlowRecHitHBHE_cfi.py +++ b/RecoParticleFlow/PFClusterProducer/python/particleFlowRecHitHBHE_cfi.py @@ -56,5 +56,8 @@ # HCALonly WF particleFlowRecHitHBHEOnly = particleFlowRecHitHBHE.clone( - producers = { 0: dict(src = "hbheprereco:") } + producers = { 0: dict(src = "hbheprereco") } +) +run3_HB.toModify(particleFlowRecHitHBHEOnly, + producers = { 0: dict(src = "hbhereco") } )