From fe077cab55fb4d65d5b5e0b5ac626cb560c4ac21 Mon Sep 17 00:00:00 2001
From: Roel Aaij <roel.aaij@cern.ch>
Date: Tue, 22 Mar 2016 18:54:57 +0100
Subject: [PATCH 1/9] Update for output streaming.
---
Hlt/Moore/python/Moore/Configuration.py | 73 ++++++-------------------
1 file changed, 17 insertions(+), 56 deletions(-)
diff --git a/Hlt/Moore/python/Moore/Configuration.py b/Hlt/Moore/python/Moore/Configuration.py
index 91a8d2a85bc..ac636a15879 100755
--- a/Hlt/Moore/python/Moore/Configuration.py
+++ b/Hlt/Moore/python/Moore/Configuration.py
@@ -102,7 +102,8 @@ class Moore(LHCbConfigurableUser):
#######################################
# Options nominally for online running
#######################################
- , "RunOnline" : False
+ , "RunOnline" : False
+ , 'EnableOutputStreaming' : False
#######################################
# Options to make processed data look like actual input...
#######################################
@@ -192,6 +193,9 @@ class Moore(LHCbConfigurableUser):
#######################################
# Options for offline running
#######################################
+ , "PersistReco" : "Do we want toe persist Moore reconstruction containers (if writing DST)"
+ , "PersistRecoFilter" : "Filter code to control which events get the full reco added to the writer"
+ , 'EnableOutputStreaming' : "Enable streaming of output in HLT2."
#########################################
# Deprecated former options
#########################################
@@ -277,8 +281,6 @@ class Moore(LHCbConfigurableUser):
bk = bankKiller( 'RemoveInputHltRawBanks', BankTypes=hlt_banks )
app.TopAlg.insert(0, bk)
-
-
def _setRawEventLocations(self):
"""
Check that I can set DecodeRawEvent.py options correctly.
@@ -312,59 +314,12 @@ class Moore(LHCbConfigurableUser):
raise ValueError("When running from a DIGI, XDIGI or RAW file, the only raw event location is DAQ/RawEvent, but you're resetting it into"+DecodeRawEvent().getProp("OverrideInputs"))
def _configureOutput(self):
- fname = self.getProp('outputFile')
- if not fname : return
- writer = None
-
- #retrieve the persistency
- persistency=None
- from GaudiConf import IOExtension, IOHelper
- iox=IOExtension()
-
- #check the file type and use MDF writer or InputCopyStream
- if iox.detectFileType(fname) == 'MDF' :
- from Configurables import LHCb__MDFWriter as MDFWriter
- writer = IOHelper("MDF","MDF").outputAlgs(fname
- ,writer = MDFWriter( 'Writer' , Compress = 0 )
- ,writeFSR=False)
- if self.getProp('WriterRequires') :
- from Configurables import LoKi__VoidFilter as VoidFilter
- #this next part does not work correctly, and I don't know why
- #writer = GaudiSequencer( 'WriteSequence'
- # , Members = self.getProp('WriterRequires') + writer
- # )
- writer = GaudiSequencer( 'WriteSequence'
- , Members = [ VoidFilter( "WriterFilter"
- , Preambulo = [ 'from LoKiHlt.algorithms import ALG_EXECUTED, ALG_PASSED' ]
- , Code = ' & '.join( [ "ALG_EXECUTED('%s') & ALG_PASSED('%s')" % (i,i) for i in self.getProp('WriterRequires') ] )
- )
- ] + writer
- )
- #convert to a smegging list consistently
- writer=[writer]
-
- ApplicationMgr().OutStream+=writer
-
- else :
- from Configurables import InputCopyStream
- writer = InputCopyStream("Writer"
- , RequireAlgs = self.getProp('WriterRequires')
- )
- IOHelper(persistency,persistency).outStream(fname,writer,writeFSR=MooreExpert().getProp('WriteFSR'))
- #print "Writing extra reco locations:"
- writer.OptItemList+=['Hlt2/pRec/long/Protos#99'
- , 'Hlt2/pRec/down/Protos#99'
- , 'Hlt2/pRec/long/RichPIDs#99'
- , 'Hlt2/pRec/long/MuonIDs#99'
- , 'Hlt2/pRec/long/Tracks#99'
- , 'Hlt2/pRec/down/Tracks#99'
- , 'Hlt2/pRec/neutral/Protos#99'
- , 'Hlt2/pRec/neutral/CaloClusters#99'
- , 'Hlt2/pRec/neutral/Electrons#99'
- , 'Hlt2/pRec/neutral/Photons#99'
- , 'Hlt2/pRec/neutral/MergedPi0s#99'
- , 'Hlt2/pRec/neutral/SplitPhotons#99']
+ from Configurables import HltConf, HltOutputConf
+ self.setOtherProps(HltConf(), ['WriterRequires'])
+ HltOutputConf().OutputFile = self.getProp('outputFile')
+ if self.getProp('Split') == 'Hlt2' and self.getProp('EnableOutputStreaming'):
+ self.setOtherProps(HltConf(), ['EnableOutputStreaming'])
def getRelease(self):
import re,fileinput
@@ -541,7 +496,7 @@ class Moore(LHCbConfigurableUser):
CaloGetterTool("ToolSvc.CaloGetter")
OTChannelMapTool("ToolSvc.OTChannelMapTool")
CaloClusterizationTool("ToolSvc.CaloClusterizationTool")
-
+
# visit_properties = ['Members', 'Filter0', 'Filter1', 'TopAlg']
descend_properties = ['Members', 'Prescale', 'ODIN', 'L0DU', 'HLT', 'HLT1', 'HLT2', 'Filter0', 'Filter1', 'Postscale']
appendPostConfigAction(partial(recurseConfigurables, func, descend_properties=descend_properties, descend_tools=True))
@@ -731,6 +686,12 @@ class Moore(LHCbConfigurableUser):
if self.getProp("RunOnline") :
LHCbApp().setProp("Persistency","MDF")
+ def _configRecoPersist(self):
+ if self.getProp("PersistReco"):
+ HltConf().PersistReco=True
+ if self.getProp("PersistRecoFilter"):
+ HltConf().CustomPersistRecoFilter=self.getProp("PersistRecoFilter")
+
def _split(self, useTCK ):
split=self.getProp("Split")
# rather nasty way of doing this.. but it is 'hidden'
--
GitLab
From 252a8f75089d64a5fc3251a588182619b1615271 Mon Sep 17 00:00:00 2001
From: Roel Aaij <roel.aaij@cern.ch>
Date: Mon, 28 Mar 2016 13:04:43 +0200
Subject: [PATCH 2/9] Force some properties in HltOutput.
---
Hlt/Moore/python/Moore/Configuration.py | 32 +++++++++++++++++--------
1 file changed, 22 insertions(+), 10 deletions(-)
diff --git a/Hlt/Moore/python/Moore/Configuration.py b/Hlt/Moore/python/Moore/Configuration.py
index ac636a15879..e18acb8f4b8 100755
--- a/Hlt/Moore/python/Moore/Configuration.py
+++ b/Hlt/Moore/python/Moore/Configuration.py
@@ -36,7 +36,7 @@ class MooreExpert(LHCbConfigurableUser):
, "DQFLAGStag" : "latest" # latest in the CondDB for this DataType
, 'WriteFSR' : True #copy FSRs as required
, "configAlgorithms" : ['Hlt'] # which algorithms to configure (automatically including their children!)...
- , "configServices" : ['ToolSvc','Hlt::Service','HltANNSvc' ] # which services to configure (automatically including their dependencies!)...
+ , "configServices" : ['ToolSvc', 'Hlt::Service', 'HltANNSvc', 'HltOutputSvc'] # which services to configure (automatically including their dependencies!)...
, "TCKpersistency" : 'cdb' # which method to use for TCK data? valid is 'file','tarfile', 'zipfile', 'cdb' and 'sqlite'
, "Hlt2Independent" : False #turn off HLT1 track decoding
, "DisableMonitors" : False #Disable HLT monitoring
@@ -48,7 +48,8 @@ class MooreExpert(LHCbConfigurableUser):
if self.isPropertySet(prop) and self.getProp(prop)!=self.getDefaultProperty(prop):
log.warning("Hey! You're setting an expert property "+prop+" are you an expert? Let's hope so!")
#seriously, this configurable should not hold any logic, it is only used as a singleton options store
- return
+ from Configurables import HltOutputConf
+ HltOutputConf().setProp('WriteFSR', self.getProp('WriteFSR'))
class Moore(LHCbConfigurableUser):
@@ -56,9 +57,8 @@ class Moore(LHCbConfigurableUser):
__used_configurables__ = [ HltConf
, LHCbApp
, DecodeRawEvent
- , DDDBConf
- , MooreExpert ]
-
+ , DDDBConf ]
+ __queried_configurables = [ MooreExpert ]
__slots__ = {
#########################################
# Basic options, used to set LHCbApp
@@ -313,14 +313,26 @@ class Moore(LHCbConfigurableUser):
if ext in ["MDF","RAW","DIGI","XDIGI"]:
raise ValueError("When running from a DIGI, XDIGI or RAW file, the only raw event location is DAQ/RawEvent, but you're resetting it into"+DecodeRawEvent().getProp("OverrideInputs"))
- def _configureOutput(self):
- from Configurables import HltConf, HltOutputConf
- self.setOtherProps(HltConf(), ['WriterRequires'])
- HltOutputConf().OutputFile = self.getProp('outputFile')
+ def forceOtherProps(self, other, props):
+ """
+ Force overwriting properties in online mode
+ """
+ for prop in props:
+ if other.isPropertySet(prop) and other.getProp(prop)!=self.getProp(prop):
+ log.warning("Resetting "+prop+" even though you set it elsewhere!")
+ other.setProp(prop,self.getProp(prop))
- if self.getProp('Split') == 'Hlt2' and self.getProp('EnableOutputStreaming'):
+ def _configureOutput(self):
+ if self.getProp('Split') != 'Hlt1' and self.getProp('EnableOutputStreaming'):
self.setOtherProps(HltConf(), ['EnableOutputStreaming'])
+ from Configurables import HltOutputConf
+ outputConf = HltOutputConf()
+ for (op, tp) in {'outputFile' : 'OutputFile', 'generateConfig' : 'GenerateTCK'}.iteritems():
+ outputConf.setProp(tp, self.getProp(op))
+ self.forceOtherProps(outputConf, ['Split', 'UseTCK', 'RunOnline', 'WriterRequires',
+ 'EnableOutputStreaming'])
+
def getRelease(self):
import re,fileinput
# do NOT pick up the pz in vxrypz
--
GitLab
From fd95be09882ab6646a86897425bc6646420a97e8 Mon Sep 17 00:00:00 2001
From: Roel Aaij <roel.aaij@cern.ch>
Date: Mon, 28 Mar 2016 21:51:24 +0200
Subject: [PATCH 3/9] Update Moore configuration to configure HltOutputConf as
needed.
---
Hlt/Moore/python/Moore/Configuration.py | 18 ++++++++----------
1 file changed, 8 insertions(+), 10 deletions(-)
diff --git a/Hlt/Moore/python/Moore/Configuration.py b/Hlt/Moore/python/Moore/Configuration.py
index e18acb8f4b8..76cc4ed5dd6 100755
--- a/Hlt/Moore/python/Moore/Configuration.py
+++ b/Hlt/Moore/python/Moore/Configuration.py
@@ -9,6 +9,7 @@ from functools import partial
from LHCbKernel.Configuration import *
from Gaudi.Configuration import *
from Configurables import HltConf
+from Configurables import HltOutputConf
from Configurables import GaudiSequencer
from Configurables import LHCbApp
from Configurables import DecodeRawEvent, RawEventFormatConf, DDDBConf
@@ -32,14 +33,13 @@ class MooreExpert(LHCbConfigurableUser):
#########################################
# Expert options, only set if you know what you are doing
#########################################
- "prefetchConfigDir" :'MOORE_v14r8' # which configurations to prefetch
- , "DQFLAGStag" : "latest" # latest in the CondDB for this DataType
+ 'DQFLAGStag' : "latest" # latest in the CondDB for this DataType
, 'WriteFSR' : True #copy FSRs as required
, "configAlgorithms" : ['Hlt'] # which algorithms to configure (automatically including their children!)...
- , "configServices" : ['ToolSvc', 'Hlt::Service', 'HltANNSvc', 'HltOutputSvc'] # which services to configure (automatically including their dependencies!)...
- , "TCKpersistency" : 'cdb' # which method to use for TCK data? valid is 'file','tarfile', 'zipfile', 'cdb' and 'sqlite'
- , "Hlt2Independent" : False #turn off HLT1 track decoding
- , "DisableMonitors" : False #Disable HLT monitoring
+ , "configServices" : ['ToolSvc', 'Hlt::Service', 'HltANNSvc', 'HltOutputSvc'] # which services to configure (automatically including their dependencies!)...
+ , "TCKpersistency" : 'cdb' # which method to use for TCK data? valid is 'file','tarfile', 'zipfile', 'cdb' and 'sqlite'
+ , "Hlt2Independent" : False #turn off HLT1 track decoding
+ , "DisableMonitors" : False #Disable HLT monitoring
}
def __apply_configuration__(self):
@@ -55,6 +55,7 @@ class MooreExpert(LHCbConfigurableUser):
class Moore(LHCbConfigurableUser):
## Possible used Configurables
__used_configurables__ = [ HltConf
+ , HltOutputConf
, LHCbApp
, DecodeRawEvent
, DDDBConf ]
@@ -124,7 +125,6 @@ class Moore(LHCbConfigurableUser):
#, "DBSnapshotDirectory" : None # use CondDB directly or MooreOnline!
#, 'EnableMonitoring' : None # use HltConf directly or MooreOnline!
#, 'SkipDisabledL0Channels' : None # use HltConf directly
- #, "prefetchConfigDir" : None # use MooreExpert
#, "EnableLumiEventWriting" : None # use HltConf directly
#, 'EnableAcceptIfSlow' : None # use HltConf directly
#, 'RequireL0ForEndSequence' : None # use HltConf directly
@@ -214,7 +214,6 @@ class Moore(LHCbConfigurableUser):
#, "DBSnapshotDirectory" : "None # DEPRECATED. use CondDB() directly or MooreOnline"
#, 'EnableMonitoring' : "None # DEPRECATED. use HltConf() directly or MooreOnline"
#, 'SkipDisabledL0Channels' : "None # DEPRECATED. use HltConf() directly"
- #, "prefetchConfigDir" : "None # DEPRECATED. use MooreExpert"
#, "EnableLumiEventWriting" : "None # DEPRECATED. use HltConf() directly"
#, 'EnableAcceptIfSlow' : "None # DEPRECATED. use HltConf directly"
#, 'RequireL0ForEndSequence' : "None # DEPRECATED. use HltConf() directly"
@@ -680,8 +679,7 @@ class Moore(LHCbConfigurableUser):
def _config_with_tck(self):
from Configurables import HltConfigSvc
from Funcs import _tck
- cfg = HltConfigSvc( prefetchDir = MooreExpert().getProp('prefetchConfigDir')
- , initialTCK = _tck(self.getProp('InitialTCK'))
+ cfg = HltConfigSvc( initialTCK = _tck(self.getProp('InitialTCK'))
, checkOdin = self.getProp('CheckOdin')
, ConfigAccessSvc = self.getConfigAccessSvc().getFullName() )
--
GitLab
From a748be4a072994b79d05e2e76fdc102d10a15a6c Mon Sep 17 00:00:00 2001
From: Roel Aaij <roel.aaij@cern.ch>
Date: Mon, 28 Mar 2016 21:51:42 +0200
Subject: [PATCH 4/9] Update Moore configuration to configure HltOutputConf as
needed (release.notes).
---
Hlt/Moore/doc/release.notes | 3 +++
1 file changed, 3 insertions(+)
diff --git a/Hlt/Moore/doc/release.notes b/Hlt/Moore/doc/release.notes
index 810947e9ab6..3fb4ac3d2c1 100755
--- a/Hlt/Moore/doc/release.notes
+++ b/Hlt/Moore/doc/release.notes
@@ -7,6 +7,9 @@
! 2016-03-29 - Rosen Matev
- Remove options for PersistReco from the Moore configurable
+! 2016-03-28 - Roel Aaij
+ - Configure HltOutputConf where necessary.
+
! 2016-03-22 - Roel Aaij
- Remove transforms that fixup HLT1+HLT2 TCKs to be either HLT1 or HLT2.
Split TCKs should be used, so running from a joint (non-technical) TCK
--
GitLab
From f0a2837a39d6881c372362821b6d48114bb83fb9 Mon Sep 17 00:00:00 2001
From: Roel Aaij <roel.aaij@cern.ch>
Date: Mon, 28 Mar 2016 22:02:29 +0200
Subject: [PATCH 5/9] Fix checks of TCK version Split property of Moore.
---
Hlt/Moore/python/Moore/Configuration.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/Hlt/Moore/python/Moore/Configuration.py b/Hlt/Moore/python/Moore/Configuration.py
index 76cc4ed5dd6..3da19b0af10 100755
--- a/Hlt/Moore/python/Moore/Configuration.py
+++ b/Hlt/Moore/python/Moore/Configuration.py
@@ -710,9 +710,9 @@ class Moore(LHCbConfigurableUser):
raise ValueError("Invalid option for Moore().Split: '%s'"% split )
if useTCK and split:
from Funcs import _tck
- tck = _tck(self.getProp('InitialTCK'))
+ tck = int(_tck(self.getProp('InitialTCK')), 16)
# Check that the TCK matches the split configuration
- bits = {'Hlt1' : 1, 'Hlt2' : 2, '' : 0}
+ bits = {'Hlt1' : 1, 'Hlt2' : 2, '' : 0}[self.getProp('Split')]
if (tck & (bits << 28)) != (bits << 28):
raise ValueError("Running from TCK with split is %s, but TCK doesn't match: 0x%08x" % (split, tck))
--
GitLab
From fe81be8324d0aeaab17e4b681b4b5cb184f2323b Mon Sep 17 00:00:00 2001
From: Roel Aaij <raaij@nikhef.nl>
Date: Wed, 30 Mar 2016 17:56:36 +0200
Subject: [PATCH 6/9] Always configure output in Moore.
---
Hlt/Moore/python/Moore/Configuration.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/Hlt/Moore/python/Moore/Configuration.py b/Hlt/Moore/python/Moore/Configuration.py
index 3da19b0af10..0d57e8b1aa3 100755
--- a/Hlt/Moore/python/Moore/Configuration.py
+++ b/Hlt/Moore/python/Moore/Configuration.py
@@ -834,6 +834,9 @@ class Moore(LHCbConfigurableUser):
self._profile()
if self.getProp("generateConfig") : self._generateConfig()
self._configureInput()
- self._configureOutput()
+
+ # Configure EndSequence and output through HltOutputConf
+ self._configureOutput()
+
#last thing, set output levels
self._outputLevel()
--
GitLab
From 76b77015c2afbf6ca6b3f54264781fae3005a350 Mon Sep 17 00:00:00 2001
From: Roel Aaij <raaij@nikhef.nl>
Date: Wed, 30 Mar 2016 18:13:11 +0200
Subject: [PATCH 7/9] Undo my rebase mistake.
---
Hlt/Moore/python/Moore/Configuration.py | 8 --------
1 file changed, 8 deletions(-)
diff --git a/Hlt/Moore/python/Moore/Configuration.py b/Hlt/Moore/python/Moore/Configuration.py
index 0d57e8b1aa3..a127096a1df 100755
--- a/Hlt/Moore/python/Moore/Configuration.py
+++ b/Hlt/Moore/python/Moore/Configuration.py
@@ -193,8 +193,6 @@ class Moore(LHCbConfigurableUser):
#######################################
# Options for offline running
#######################################
- , "PersistReco" : "Do we want toe persist Moore reconstruction containers (if writing DST)"
- , "PersistRecoFilter" : "Filter code to control which events get the full reco added to the writer"
, 'EnableOutputStreaming' : "Enable streaming of output in HLT2."
#########################################
# Deprecated former options
@@ -696,12 +694,6 @@ class Moore(LHCbConfigurableUser):
if self.getProp("RunOnline") :
LHCbApp().setProp("Persistency","MDF")
- def _configRecoPersist(self):
- if self.getProp("PersistReco"):
- HltConf().PersistReco=True
- if self.getProp("PersistRecoFilter"):
- HltConf().CustomPersistRecoFilter=self.getProp("PersistRecoFilter")
-
def _split(self, useTCK ):
split=self.getProp("Split")
# rather nasty way of doing this.. but it is 'hidden'
--
GitLab
From f8224f2a0c99662b51884a9bca0759bf8fd85e8d Mon Sep 17 00:00:00 2001
From: Roel Aaij <raaij@nikhef.nl>
Date: Thu, 31 Mar 2016 19:05:35 +0200
Subject: [PATCH 8/9] Update DataType to 2016.
---
Hlt/Moore/python/Moore/Configuration.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Hlt/Moore/python/Moore/Configuration.py b/Hlt/Moore/python/Moore/Configuration.py
index a127096a1df..bafe6ea490d 100755
--- a/Hlt/Moore/python/Moore/Configuration.py
+++ b/Hlt/Moore/python/Moore/Configuration.py
@@ -67,7 +67,7 @@ class Moore(LHCbConfigurableUser):
"EvtMax": -1 # Maximum number of events to process
, "SkipEvents": 0
, "Simulation": False # True implies use SimCond
- , "DataType": '2012' # Data type, can be [ 'DC06','2008' ]
+ , "DataType": '2016' # Data type, can be [ 'DC06','2008' ]
, "DDDBtag" : 'default' # default as set in DDDBConf for DataType
, "CondDBtag" : 'default' # default as set in DDDBConf for DataType
#########################################
--
GitLab
From be66b330ec464de4d6c3669e6e840b31ee50bcbf Mon Sep 17 00:00:00 2001
From: Roel Aaij <raaij@nikhef.nl>
Date: Wed, 6 Apr 2016 17:59:22 +0200
Subject: [PATCH 9/9] Propagate Simulation property to HltOutputConf.
---
Hlt/Moore/python/Moore/Configuration.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/Hlt/Moore/python/Moore/Configuration.py b/Hlt/Moore/python/Moore/Configuration.py
index bafe6ea490d..653d1343c9d 100755
--- a/Hlt/Moore/python/Moore/Configuration.py
+++ b/Hlt/Moore/python/Moore/Configuration.py
@@ -321,14 +321,14 @@ class Moore(LHCbConfigurableUser):
def _configureOutput(self):
if self.getProp('Split') != 'Hlt1' and self.getProp('EnableOutputStreaming'):
- self.setOtherProps(HltConf(), ['EnableOutputStreaming'])
+ self.forceOtherProps(HltConf(), ['EnableOutputStreaming'])
from Configurables import HltOutputConf
outputConf = HltOutputConf()
for (op, tp) in {'outputFile' : 'OutputFile', 'generateConfig' : 'GenerateTCK'}.iteritems():
outputConf.setProp(tp, self.getProp(op))
self.forceOtherProps(outputConf, ['Split', 'UseTCK', 'RunOnline', 'WriterRequires',
- 'EnableOutputStreaming'])
+ 'EnableOutputStreaming', 'Simulation'])
def getRelease(self):
import re,fileinput
--
GitLab