Make the HCAL 'gpu' workflow use a GPU if available, and fall-back to…

… CPU otherwise

Configuration/StandardSequences/python/Reconstruction_cff.py

@@ -235,6 +235,7 @@
     bunchSpacingProducer,
     offlineBeamSpot,
     hcalOnlyLocalRecoTask,
+    hcalOnlyGlobalRecoTask,
     pfClusteringHBHEHFOnlyTask
 )
 

DQM/HcalTasks/python/OfflineSourceSequence_pp.py

@@ -1,36 +1,41 @@
 import FWCore.ParameterSet.Config as cms
 
 #-----------------
-#	HCAL DQM Offline Source Sequence Definition for pp
-#	To be used for Offline DQM importing
+#   HCAL DQM Offline Source Sequence Definition for pp
+#   To be used for Offline DQM importing
 #-----------------
 
-#	import the tasks
+#   import the tasks
 from DQM.HcalTasks.DigiTask import digiTask
 from DQM.HcalTasks.RawTask import rawTask
 from DQM.HcalTasks.TPTask import tpTask
 from DQM.HcalTasks.RecHitTask import recHitTask, recHitPreRecoTask
 
-#	set processing type to Offine
+#   set processing type to Offine
 digiTask.ptype = cms.untracked.int32(1)
 tpTask.ptype = cms.untracked.int32(1)
 recHitTask.ptype = cms.untracked.int32(1)
 rawTask.ptype = cms.untracked.int32(1)
 recHitPreRecoTask.ptype = cms.untracked.int32(1)
 
-#	set the label for Emulator TP Task
+#   set the label for Emulator TP Task
 tpTask.tagEmul = cms.untracked.InputTag("valHcalTriggerPrimitiveDigis")
 
 hcalOfflineSourceSequence = cms.Sequence(
-	digiTask
-	+tpTask
-	+recHitTask
-	+rawTask)
+    digiTask +
+    tpTask +
+    recHitTask +
+    rawTask )
 
 hcalOnlyOfflineSourceSequence = cms.Sequence(
-        digiTask
-        +recHitPreRecoTask
-        +rawTask)
+    digiTask +
+    recHitPreRecoTask +
+    rawTask )
+
+from Configuration.Eras.Modifier_run3_HB_cff import run3_HB
+_phase1_hcalOnlyOfflineSourceSequence = hcalOnlyOfflineSourceSequence.copy()
+_phase1_hcalOnlyOfflineSourceSequence.replace(recHitPreRecoTask, recHitTask)
+run3_HB.toReplaceWith(hcalOnlyOfflineSourceSequence, _phase1_hcalOnlyOfflineSourceSequence)
 
 from Configuration.Eras.Modifier_phase2_hcal_cff import phase2_hcal
 _phase2_hcalOfflineSourceSequence = hcalOfflineSourceSequence.copyAndExclude([tpTask,rawTask])

RecoLocalCalo/Configuration/python/RecoLocalCalo_Cosmics_cff.py

@@ -19,23 +19,20 @@
 # sequence CaloLocalReco
 #
 
-def hbheCosmic(module):
-    return module.clone(
-        tsFromDB = False,
-        recoParamsFromDB = False,
-        algorithm = dict(
-            useMahi = False,
-            useM2 = False,
-            useM3 = False,
-            firstSampleShift = -1000,
-            samplesToAdd = 10,
-            correctForPhaseContainment = False,
-        ),
-        sipmQTSShift = -100,
-        sipmQNTStoSum = 200,
-    )
-
-hbhereco = hbheCosmic(_hcalLocalReco_cff.hbheprereco)
+hbhereco = _hcalLocalReco_cff.hbheprereco.cpu.clone(
+    tsFromDB = False,
+    recoParamsFromDB = False,
+    algorithm = dict(
+        useMahi = False,
+        useM2 = False,
+        useM3 = False,
+        firstSampleShift = -1000,
+        samplesToAdd = 10,
+        correctForPhaseContainment = False,
+    ),
+    sipmQTSShift = -100,
+    sipmQNTStoSum = 200,
+)
 hfreco = _hcalLocalReco_cff._default_hfreco.clone(
     firstSample = 0,
     samplesToAdd = 10, ### min(10,size) in the algo

RecoLocalCalo/Configuration/python/hcalGlobalReco_cff.py

@@ -1,21 +1,30 @@
 import FWCore.ParameterSet.Config as cms
+from HeterogeneousCore.CUDACore.SwitchProducerCUDA import SwitchProducerCUDA
 
-from RecoLocalCalo.HcalRecProducers.HBHEIsolatedNoiseReflagger_cfi import *
+#--- for Run 1 and Run 2
+from RecoLocalCalo.HcalRecProducers.HBHEIsolatedNoiseReflagger_cfi import hbhereco as _phase0_hbhereco
+hbhereco = SwitchProducerCUDA(
+    cpu = _phase0_hbhereco
+)
 hcalGlobalRecoTask = cms.Task(hbhereco)
 hcalGlobalRecoSequence = cms.Sequence(hcalGlobalRecoTask)
 
+hcalOnlyGlobalRecoTask = cms.Task()
+hcalOnlyGlobalRecoSequence = cms.Sequence(hcalOnlyGlobalRecoTask)
+
 #--- for Run 3 and later
 from Configuration.Eras.Modifier_run3_HB_cff import run3_HB
 
 from RecoLocalCalo.HcalRecProducers.HBHEPhase1Reconstructor_cfi import hbheprereco as _phase1_hbheprereco
-run3_HB.toReplaceWith(hbhereco, _phase1_hbheprereco)
+run3_HB.toReplaceWith(hbhereco.cpu, _phase1_hbheprereco)
+run3_HB.toReplaceWith(hcalOnlyGlobalRecoTask, cms.Task(hbhereco))
 
 #--- for Run 3 on GPU
 from Configuration.ProcessModifiers.gpu_cff import gpu
 
-from RecoLocalCalo.HcalRecProducers.hcalCPURecHitsProducer_cfi import hcalCPURecHitsProducer as _hcalCPURecHitsProducer
-gpu.toReplaceWith(hbhereco, _hcalCPURecHitsProducer.clone(
-    recHitsM0LabelIn = "hbheRecHitProducerGPU",
-    recHitsM0LabelOut = "",
-    recHitsLegacyLabelOut = ""
-))
+from RecoLocalCalo.HcalRecProducers.hcalCPURecHitsProducer_cfi import hcalCPURecHitsProducer as _hbherecoFromCUDA
+(run3_HB & gpu).toModify(hbhereco,
+    cuda = _hbherecoFromCUDA.clone(
+        produceSoA = False
+    )
+)

RecoLocalCalo/Configuration/python/hcalLocalReco_cff.py

@@ -1,15 +1,18 @@
 import FWCore.ParameterSet.Config as cms
+from HeterogeneousCore.CUDACore.SwitchProducerCUDA import SwitchProducerCUDA
 
 from RecoLocalCalo.HcalRecAlgos.hcalRecAlgoESProd_cfi import *
 from RecoLocalCalo.HcalRecAlgos.hcalChannelPropertiesESProd_cfi import *
 hcalOOTPileupESProducer = cms.ESProducer('OOTPileupDBCompatibilityESProducer')
 
 from RecoLocalCalo.HcalRecProducers.HBHEPhase1Reconstructor_cfi import hbheprereco as _phase1_hbheprereco
-hbheprereco = _phase1_hbheprereco.clone(
-    processQIE11 = False,
-    tsFromDB = True,
-    pulseShapeParametersQIE8 = dict(
-        TrianglePeakTS = 4,
+hbheprereco = SwitchProducerCUDA(
+    cpu = _phase1_hbheprereco.clone(
+        processQIE11 = False,
+        tsFromDB = True,
+        pulseShapeParametersQIE8 = dict(
+            TrianglePeakTS = 4,
+        )
     )
 )
 
@@ -34,7 +37,7 @@
 run2_HF_2017.toReplaceWith( hcalLocalRecoTask, _phase1_hcalLocalRecoTask )
 run2_HF_2017.toReplaceWith( hfreco, _phase1_hfreco )
 from Configuration.Eras.Modifier_run2_HCAL_2017_cff import run2_HCAL_2017
-run2_HCAL_2017.toReplaceWith(hbheprereco, _phase1_hbheprereco)
+run2_HCAL_2017.toReplaceWith(hbheprereco.cpu, _phase1_hbheprereco)
 
 _plan1_hcalLocalRecoTask = _phase1_hcalLocalRecoTask.copy()
 _plan1_hcalLocalRecoTask.add(hbheplan1)
@@ -57,27 +60,20 @@
 from Configuration.ProcessModifiers.gpu_cff import gpu
 
 from RecoLocalCalo.HcalRecProducers.hbheRecHitProducerGPUTask_cff import *
-_run3_hcalLocalRecoGPUTask = _run3_hcalLocalRecoTask.copy()
+_run3_hcalLocalRecoGPUTask = hcalLocalRecoTask.copy()
 _run3_hcalLocalRecoGPUTask.add(hbheRecHitProducerGPUTask)
 gpu.toReplaceWith(hcalLocalRecoTask, _run3_hcalLocalRecoGPUTask)
 
-#--- HCAL-only workflow for Run 3
-# FIXME rename `hbheprereco` to `hbhereco` and use it from hcalGlobalRecoTask
-hcalOnlyLocalRecoTask = cms.Task(hbheprereco, hfprereco, hfreco, horeco)
+#--- HCAL-only workflow
+hcalOnlyLocalRecoTask = hcalLocalRecoTask.copyAndExclude([zdcreco])
 
-#--- HCAL-only workflow for Run 3 on GPU
-from Configuration.ProcessModifiers.gpu_cff import gpu
-
-_hcalOnlyLocalRecoGPUTask = hcalOnlyLocalRecoTask.copy()
-_hcalOnlyLocalRecoGPUTask.add(hbheRecHitProducerGPUTask)
-gpu.toReplaceWith(hcalOnlyLocalRecoTask, _hcalOnlyLocalRecoGPUTask)
-
-from RecoLocalCalo.HcalRecProducers.hcalCPURecHitsProducer_cfi import hcalCPURecHitsProducer as _hcalCPURecHitsProducer
-gpu.toReplaceWith(hbheprereco, _hcalCPURecHitsProducer.clone(
-    recHitsM0LabelIn = "hbheRecHitProducerGPU",
-    recHitsM0LabelOut = "",
-    recHitsLegacyLabelOut = ""
-))
+#--- HCAL-only workflow for Run 2 on GPU
+from RecoLocalCalo.HcalRecProducers.hcalCPURecHitsProducer_cfi import hcalCPURecHitsProducer as _hbheprerecoFromCUDA
+(gpu & ~run3_HB).toModify(hbheprereco,
+    cuda = _hbheprerecoFromCUDA.clone(
+        produceSoA = False
+    )
+)
 
 #--- for FastSim
 _fastSim_hcalLocalRecoTask = hcalLocalRecoTask.copyAndExclude([zdcreco])

RecoLocalCalo/HcalRecProducers/python/HBHE_custom_25nsMethod.py

@@ -2,11 +2,11 @@
 
 def customise_HBHE_Method1(process):
    if hasattr(process,'hbheprereco'): 
-      process.hbheprereco.puCorrMethod = cms.int32(1)
+      process.hbheprereco.cpu.puCorrMethod = cms.int32(1)
    return process
 
 def customise_HBHE_Method0(process):
    if hasattr(process,'hbheprereco'): 
-      process.hbheprereco.puCorrMethod = cms.int32(0)
+      process.hbheprereco.cpu.puCorrMethod = cms.int32(0)
    return process
 

RecoLocalCalo/HcalRecProducers/test/make_GPUvsCPU_HCAL_rechits.py

@@ -59,32 +59,32 @@
 #process.hbheprereco.algorithm.correctForPhaseContainment = cms.bool(False)
 
 ## do always 8 pulse
-process.hbheprereco.algorithm.chiSqSwitch = cms.double(-1)
+process.hbheprereco.cpu.algorithm.chiSqSwitch = cms.double(-1)
 
 ## to match hard coded setting (will be fixed on CPU)
-process.hbheprereco.algorithm.nMaxItersMin = cms.int32(50)
+process.hbheprereco.cpu.algorithm.nMaxItersMin = cms.int32(50)
 
 #-----------------------------------------
 # Final Custmization for Run3
 #-----------------------------------------
 
 # we will not run arrival Time at HLT
-process.hbheprereco.algorithm.calculateArrivalTime = cms.bool(False)
+process.hbheprereco.cpu.algorithm.calculateArrivalTime = cms.bool(False)
 
 ## we do not need this
-process.hbheprereco.algorithm.applyLegacyHBMCorrection = cms.bool(False)
+process.hbheprereco.cpu.algorithm.applyLegacyHBMCorrection = cms.bool(False)
 
 # we only run Mahi at HLT
-process.hbheprereco.algorithm.useM3 = cms.bool(False)
+process.hbheprereco.cpu.algorithm.useM3 = cms.bool(False)
 
 # we will not have the HPD noise flags in Run3, as will be all siPM
-process.hbheprereco.setLegacyFlagsQIE8 = cms.bool(False)
-process.hbheprereco.setNegativeFlagsQIE8 = cms.bool(False)
-process.hbheprereco.setNoiseFlagsQIE8 = cms.bool(False)
-process.hbheprereco.setPulseShapeFlagsQIE8 = cms.bool(False)
+process.hbheprereco.cpu.setLegacyFlagsQIE8 = cms.bool(False)
+process.hbheprereco.cpu.setNegativeFlagsQIE8 = cms.bool(False)
+process.hbheprereco.cpu.setNoiseFlagsQIE8 = cms.bool(False)
+process.hbheprereco.cpu.setPulseShapeFlagsQIE8 = cms.bool(False)
 
 # for testing M0 only
-##process.hbheprereco.algorithm.useMahi = cms.bool(False)
+##process.hbheprereco.cpu.algorithm.useMahi = cms.bool(False)
 
 #-----------------------------------------
 # OUTPUT
@@ -148,4 +148,4 @@
 
 # report CUDAService messages
 process.MessageLogger.cerr.FwkReport.reportEvery = 100
-process.MessageLogger.categories.append("CUDAService")
+process.MessageLogger.CUDAService = cms.untracked.PSet()

RecoParticleFlow/PFClusterProducer/python/particleFlowRecHitHBHE_cfi.py

@@ -56,5 +56,8 @@
 
 # HCALonly WF
 particleFlowRecHitHBHEOnly = particleFlowRecHitHBHE.clone(
-    producers = { 0: dict(src = "hbheprereco:") }
+    producers = { 0: dict(src = "hbheprereco") }
+)
+run3_HB.toModify(particleFlowRecHitHBHEOnly,
+    producers = { 0: dict(src = "hbhereco") }
 )