Skip to content
Snippets Groups Projects
Commit db411a1c authored by Rosen Matev's avatar Rosen Matev :sunny:
Browse files

Merge branch 'apearce-tesla-tuning' into 'master' 

Tesla configuration house-keeping

Closes LHCBPS-1721

See merge request !73
parents fe83599c f3137466
No related branches found
No related tags found
2 merge requests!85merge master to future,!73Tesla configuration house-keeping
Hide whitespace changes
Inline Side-by-side
Phys/Tesla/python/Tesla/Configuration.py
+ 116
− 33
View file @ db411a1c
......@@ -4,6 +4,7 @@ import re
from Configurables import (
AddressKillerAlg,
AuditorSvc,
CaloClusterMCTruth,
CaloDigit2MCLinks2Table,
ChargedPP2MC,
......@@ -20,10 +21,12 @@ from Configurables import (
GaudiSequencer,
HltPackedDataDecoder,
HltRoutingBitsFilter,
HltSelReportsWriter,
InputCopyStream,
LHCbApp,
LHCbConfigurableUser,
LoKi__HDRFilter as HltDecReportsFilter,
LoKiSvc,
LumiAlgsConf,
NeutralPP2MC,
OutputStream,
......@@ -38,9 +41,11 @@ from Configurables import (
RecSummaryFromSelReports,
RecombineRawEvent,
RecordStream,
SequencerTimerTool,
TCKLinePersistenceSvc,
TeslaLineChecker,
TeslaReportAlgo,
TimingAuditor,
TrackAssociator,
TrackSys,
TurboConf,
......@@ -54,6 +59,7 @@ from GaudiConf import IOExtension, IOHelper
from GaudiConf import PersistRecoConf
import GaudiKernel.ProcessJobOptions
from LHCbKernel.Configuration import *
from RawEventCompat.Configuration import ReverseDict as RawEventLocationsToBanks
class Tesla(LHCbConfigurableUser):
......@@ -81,8 +87,10 @@ class Tesla(LHCbConfigurableUser):
, 'TriggerLines' : [] # Which trigger lines
, 'Mode' : "Offline" # "Online" (strip unnecessary banks and run lumi algorithms) or "Offline"?
# "Online" means running on data from the pit
# "Offline" is used on DSTs, i.e. the output of Brunel
, 'Pack' : True # Do we want to pack the objects?
, 'RawFormatVersion' : 0.2 # Which banks form the Turbo stream
, 'SplitRawEventInput' : 0.4
, 'SplitRawEventOutput' : 4.3
, 'Monitors' : [] # Histogram monitors to run in the job
, 'Histogram' : "" # Name of histogram file
, 'VetoJuggle' : True # True if raw event removal not required in online mode
......@@ -93,9 +101,6 @@ class Tesla(LHCbConfigurableUser):
, 'InputHlt2RepsToKeep' : [] # List of selection names of reports to keep (if KillInputHlt2Reps is True)
, 'HDRFilter' : False # Filter on trigger lines or not
, 'Park' : False # If prescaled streams, make a parked version
# Remove maybe
, 'NonPrescaleList' : [] # List of Turbo++ lines that are not prescaled
, 'RemovePrescaleExcl': False # Remove DST data banks in exclusively prescaled lines
, 'PRtracks' : [] # PR track locations
, 'PRchargedPP' : [] # PR protoparticle locations
, 'PRclusters' : [] # PR cluster locations
......@@ -131,7 +136,8 @@ class Tesla(LHCbConfigurableUser):
, 'TriggerLines' : 'Which trigger line to process'
, 'Mode' : '"Online" (strip unnecessary banks and run lumi algorithms) or "Offline"?'
, 'Pack' : 'Do we want to pack the object?'
, 'RawFormatVersion': 'Which banks form the Turbo stream'
, 'SplitRawEventInput': "How is the event split up in the input? Propagated to RawEventJuggler() and DecodeRawEvent()."
, 'SplitRawEventOutput': "How should the event be split up in the output?"
, 'Monitors' : 'Histogram monitors to run in the job'
, 'Histogram' : 'File name for histogram file'
, 'VetoJuggle' : 'Do we want to stop raw bank removal (assume happened further upstream)'
......@@ -141,9 +147,7 @@ class Tesla(LHCbConfigurableUser):
, 'KillInputHlt2Reps':'Kill input Hlt2 reps to save space'
, 'InputHlt2RepsToKeep': 'List of selection names of reports to keep (if KillInputHlt2Reps is True)'
, 'HDRFilter' : 'Filter on trigger lines or not'
, 'RemovePrescaleExcl': 'Remove DST data banks in exclusively prescaled lines'
, 'Park' : 'If prescaled streams, make a parked version'
, 'NonPrescaleList' : 'List of Turbo++ lines that are not prescaled'
, 'PRtracks' : 'PR track locations'
, 'PRchargedPP' : 'PR protoparticle locations'
, 'PRclusters' : 'PR cluster locations'
......@@ -189,6 +193,17 @@ class Tesla(LHCbConfigurableUser):
if ( self.getProp("Histogram") != "" ):
HistogramPersistencySvc().OutputFile = self.getProp("Histogram")
def _configureTimingAuditor(self):
"""Enable the printing of the algorithm timing table."""
ApplicationMgr().ExtSvc += ['ToolSvc', 'AuditorSvc']
ApplicationMgr().AuditAlgorithms = True
AuditorSvc().Auditors += ['TimingAuditor']
TimingAuditor().addTool(SequencerTimerTool, name='TIMER')
TimingAuditor().TIMER.NameSize = 60
# Disable INFO output from the default timer tool
SequencerTimerTool().OutputLevel = WARNING
def _configureTrackTruth(self,assocpp,trackLoc) :
assoctr = TrackAssociator("TurboAssocTr"+trackLoc)
assoctr.OutputLevel = self.getProp('OutputLevel')
......@@ -277,7 +292,7 @@ class Tesla(LHCbConfigurableUser):
#
DecodeRawEvent().DataOnDemand=False
writer=InputCopyStream( self.writerName+stream )
DstConf().setProp("SplitRawEventOutput", self.getProp("RawFormatVersion"))
self._safeSet(DstConf(), ['SplitRawEventOutput'])
# we can only do this if we are setting one output writer
if not self.getProp("Streams"):
......@@ -286,7 +301,7 @@ class Tesla(LHCbConfigurableUser):
TurboBanksSeq=GaudiSequencer("TurboBanksSeq")
RawEventJuggler().TCK=tck
RawEventJuggler().Input="Moore"
RawEventJuggler().Output=self.getProp("RawFormatVersion")
RawEventJuggler().Output=self.getProp("SplitRawEventOutput")
RawEventJuggler().Sequencer=TurboBanksSeq
RawEventJuggler().WriterOptItemList=writer
RawEventJuggler().KillExtraNodes=True
......@@ -724,9 +739,10 @@ class Tesla(LHCbConfigurableUser):
datatype = self.getProp('DataType')
mode = self.getProp('Mode')
online = mode == 'Online'
raw_format = self.getProp('RawFormatVersion')
vertex_report_location = self.getProp('VertRepLoc')
enable_line_checker = self.getProp('EnableLineChecker')
raw_format_input = self.getProp('SplitRawEventInput')
raw_format_output = self.getProp('SplitRawEventOutput')
ok = (trigger_lines and not streams) or (streams and not trigger_lines)
assert ok, ('Must define at least one of Tesla().Streams and '
......@@ -752,7 +768,7 @@ class Tesla(LHCbConfigurableUser):
decoders_seq.Members.append(decoder)
DecodeRawEvent().DataOnDemand = False
DstConf().setProp('SplitRawEventOutput', raw_format)
self._safeSet(DstConf(), ['SplitRawEventOutput'])
# This decoder is setup by TurboConf
packed_data_decoder = HltPackedDataDecoder('Hlt2PackedDataDecoder')
......@@ -779,13 +795,12 @@ class Tesla(LHCbConfigurableUser):
stream_seq = self._configureOutputTurboSPStream(
stream_name,
streams[stream_name]['lines'],
prpacking
prpacking,
online,
raw_format_output
)
stream_sequences.append(stream_seq)
# Kill any links the output file might have had to the input file
address_killer = AddressKillerAlg()
if enable_line_checker:
all_lines = sum([d['lines'] for d in streams.values()], [])
ignored_lines = self.getProp('IgnoredLines')
......@@ -796,11 +811,24 @@ class Tesla(LHCbConfigurableUser):
else:
line_checker = []
# Kill the input DstData bank
dstdata_killer = bankKiller(
"Hlt2DstDataKiller",
BankTypes=["DstData"]
)
# Kill the input DstData bank, which is in DAQ/RawEvent online, but in
# PersistReco/RawEvent offline (e.g. after Brunel)
if online:
dstdata_killer = [bankKiller(
'Hlt2DstDataKiller',
BankTypes=['DstData']
)]
else:
dstdata_killer = [EventNodeKiller(
'Hlt2DstDataKiller',
Nodes=['PersistReco']
)]
# Kill any links the output file might have had to the input file
if online:
address_killer = [AddressKillerAlg()]
else:
address_killer = []
streaming_seq = GaudiSequencer(
'TeslaStreamsSequence',
......@@ -808,15 +836,16 @@ class Tesla(LHCbConfigurableUser):
[decoders_seq, unpackers_seq] +
line_checker +
stream_sequences +
[address_killer] +
[dstdata_killer]
dstdata_killer +
address_killer
),
IgnoreFilterPassed=True
)
return streaming_seq
def _configureOutputTurboSPStream(self, name, lines, packing):
def _configureOutputTurboSPStream(self, name, lines, packing, online,
raw_format_output):
"""Return a sequence for streaming the lines into the named location.
Copy line outputs for this stream to a stream-specific location.
......@@ -825,6 +854,8 @@ class Tesla(LHCbConfigurableUser):
name -- Name of the stream
lines -- List of HLT2 line names that belong in the stream
packing -- Instance of PersistRecoConf.PersistRecoPacking
online -- Configure in online mode (if True) or offline
raw_format_output -- Output format of the raw event
"""
output_prefix = name.title()
decisions = [l + 'Decision' for l in lines]
......@@ -884,8 +915,6 @@ class Tesla(LHCbConfigurableUser):
# /Event/<stream name>/Turbo
turbo_base = os.path.join(stream_base, 'Turbo')
required_output_locations = [
os.path.join(tes_root, 'DAQ/ODIN#1'),
os.path.join(tes_root, 'DAQ/RawEvent#1'),
os.path.join(tes_root, 'Rec/Summary#1')
]
optional_output_locations = []
......@@ -935,10 +964,8 @@ class Tesla(LHCbConfigurableUser):
optional_output_locations += [
os.path.join(turbo_base, 'pPhys#*'),
# XXX can we better take advantage of the automagic DaVinci
# unpackers if we packed PersistReco stuff to just pRec?
os.path.join(turbo_base, 'Hlt2/pRec#*'),
# os.path.join(turbo_base, 'pRec#*')
os.path.join(turbo_base, 'pRec#*'),
os.path.join(turbo_base, 'Hlt2/pRec#*')
]
else:
# Save everything under the stream prefix
......@@ -955,11 +982,18 @@ class Tesla(LHCbConfigurableUser):
else:
fname = self.getProp('outputFile')
writer = OutputStream(namer(self.writerName))
if online:
writer = OutputStream(namer(self.writerName))
writer.AcceptAlgs += ['LumiSeq', 'PhysFilter']
# In online mode we perform the raw event juggling (otherwise
# Brunel does it for us), so must save the locations after juggling
optional_output_locations += self._output_raw_event_locations(raw_format_output)
else:
# In offline mode, e.g. after Brunel, propagate everything from the
# input file to the output
writer = InputCopyStream(namer(self.writerName))
writer.ItemList = required_output_locations
writer.OptItemList = optional_output_locations
# We should already have configured the lumiAlgs by now
writer.AcceptAlgs = ['LumiSeq', 'PhysFilter']
writer.RequireAlgs = [stream_seq]
iohelper = IOHelper()
iohelper.outStream(fname, writer, writeFSR=write_fsr)
......@@ -976,14 +1010,25 @@ class Tesla(LHCbConfigurableUser):
killer = bankKiller("Hlt2SelReportsBankKiller")
killer.BankTypes = ["HltSelReports"]
killer.DefaultIsKill = False
# HltSelReportsWriter::kSourceID_Hlt2 << HltSelReportsWriter::kSourceID_BitShift
killer.KillSourceID = 2 << 13
# HltSelReportsWriter::kSourceID_MajorMask
killer.KillSourceIDMask = 0xE000
kill_seq.Members.append(killer)
# if the list of reports to keep is empty, do nothing more
if inputHlt2RepsToKeep:
assert all(name.endswith('Decision') for name in inputHlt2RepsToKeep)
from DAQSys.Decoders import DecoderDB
selreports_decoder = DecoderDB['HltSelReportsDecoder/Hlt2SelReportsDecoder']
selreports_loc = selreports_decoder.Outputs['OutputHltSelReportsLocation']
decreports_decoder = DecoderDB['HltDecReportsDecoder/Hlt2DecReportsDecoder']
decreports_loc = decreports_decoder.Outputs['OutputHltDecReportsLocation']
# By the time the stripper runs Hlt2/SelReports should exist
stripper = HltSelReportsStripper('Hlt2SelReportsStripper')
stripper.InputHltSelReportsLocation = selreports_loc
stripper.OutputHltSelReportsLocation = 'Hlt2/SelReportsFiltered'
stripper.OutputHltObjectSummariesLocation = (
str(stripper.OutputHltSelReportsLocation) + '/Candidates')
......@@ -995,10 +1040,35 @@ class Tesla(LHCbConfigurableUser):
writer.SourceID = 2
writer.InputHltSelReportsLocation = str(stripper.OutputHltSelReportsLocation)
writer.OutputLevel = self.getProp('OutputLevel')
# By the time the stripper runs Hlt2/DecReports should exist
writer.UseTCK = True # use the TCKANNSvc for converting names to numbers
writer.InputHltDecReportsLocation = decreports_loc
kill_seq.Members.append(writer)
return kill_seq
def _raw_event_juggler(self, input_format, output_format):
# Load the raw event format dictionaries
RawEventFormatConf().loadIfRequired()
j = RawEventJuggler()
j.Input = input_format
j.Output = output_format
j.KillExtraNodes = True
j.KillExtraBanks = True
j.KillExtraDirectories = True
j.Sequencer = GaudiSequencer('TeslaRawEventJugglerSequence')
return j.Sequencer
def _output_raw_event_locations(self, output_format):
"""Return the list of raw event locations for the output format."""
# Load the raw event format dictionaries
RawEventFormatConf().loadIfRequired()
locs = RawEventLocationsToBanks(output_format).keys()
return [os.path.join('/Event', l) + '#1' for l in locs]
def __apply_configuration__(self):
############## Set other properties ###########
self._safeSet( LHCbApp(), ['EvtMax','SkipEvents','Simulation', 'DataType' , 'CondDBtag','DDDBtag'] )
......@@ -1010,6 +1080,9 @@ class Tesla(LHCbConfigurableUser):
if not LHCbApp().isPropertySet( "IgnoreDQFlags" ) :
LHCbApp().setProp( "IgnoreDQFlags", True )
# Silence, please!
LoKiSvc().Welcome = False
ApplicationMgr().AppName="Tesla, utilising DaVinci"
#
if self.getProp('Mode') is "Online":
......@@ -1017,7 +1090,7 @@ class Tesla(LHCbConfigurableUser):
self._configureLumi()
else:
DecodeRawEvent().DataOnDemand=True
RecombineRawEvent()
RecombineRawEvent(Version=self.getProp('SplitRawEventInput'))
if self.getProp('Simulation')==True:
self._unpackMC()
TurboConf().setProp("PersistReco",True)
......@@ -1039,6 +1112,9 @@ class Tesla(LHCbConfigurableUser):
if len(self.getProp('Monitors'))>0:
self._configureHistos()
# Enable the timing table
self._configureTimingAuditor()
if self.getProp('KillInputTurbo'):
enk = EventNodeKiller('KillTurbo')
enk.Nodes = [ "Turbo" ]
......@@ -1048,4 +1124,11 @@ class Tesla(LHCbConfigurableUser):
kill_selreports = self._configureHlt2SelReportsKill()
self.teslaSeq.Members += [kill_selreports]
if self.getProp('SplitRawEventInput') != self.getProp('SplitRawEventOutput'):
raw_event_juggler = self._raw_event_juggler(
self.getProp('SplitRawEventInput'),
self.getProp('SplitRawEventOutput')
)
self.teslaSeq.Members += [raw_event_juggler]
ApplicationMgr().TopAlg+=[self.teslaSeq]
Phys/TurboCache/options/Streams_v4r1.py
+ 1
− 1
View file @ db411a1c
from TurboStreamProd.streams import turbo_streams
from Tesla.Configuration import Tesla
Tesla().Streams = turbo_streams
Tesla().Streams = turbo_streams["2016"]
#Tesla().ValidateStreams = True
# Turn on Sascha's algorithm ignoring TurboCalib
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment