Merge 2016-patches into master

- See merge request !29

CMakeLists.txt

@@ -6,8 +6,8 @@ find_package(GaudiProject)
 #---------------------------------------------------------------
 
 # Declare project name and version
-gaudi_project(Moore v25r2
-              USE Hlt v25r2
+gaudi_project(Moore v25r3
+              USE Hlt v25r3
               DATA AppConfig VERSION v3r*
                    FieldMap VERSION v5r*
                    PRConfig VERSION v1r*

Hlt/HltCache/CMakeLists.txt

@@ -37,14 +37,14 @@ include(LoKiFunctorsCache)
 
 set(conf_deps ${deps} MooreConfUserDB)
 
-#options/SuppressLogMessages.py
-#options/SilenceErrors.py
 
 set(LOKI_FUNCTORS_CACHE_POST_ACTION_OPTS
     ${CMAKE_CURRENT_SOURCE_DIR}/options/LoKiFunctorsCacheHltPostActionOpts.py)
     
 loki_functors_cache(Hlt_pp_Draft2016_FunctorCache
                     options/Physics_pp_Draft2016.py
+                    options/SilenceErrors.py
+                    options/SuppressLogMessages.py
                     options/DisableLoKiCacheFunctors.py
                     FACTORIES CoreFactory TrackFunctorFactory HltFactory TrackFactory
                               LoKi__Hybrid__HltFactory

Hlt/HltCache/options/Physics_pp_Draft2016.py

@@ -12,7 +12,7 @@ Moore().CheckOdin = False
 Moore().Simulation = False
 Moore().ForceSingleL0Configuration = True
 Moore().SkipEvents = 0
-Moore().CondDBtag = 'cond-20150828'
+Moore().CondDBtag = 'cond-20160517'
 Moore().DDDBtag = 'dddb-20150724'
 
 from Configurables import EventSelector

Hlt/HltCache/options/SuppressLogMessages.py

-# -*- coding: utf-8 -*-
-from Gaudi.Configuration import log
 import logging
-import re
+from GaudiKernel.ProcessJobOptions import GetConsoleHandler
 
-IGNORED_MESSAGES = map(re.compile,
-                       (
-                        # errors
-                        r'The default_config of .* (is not a dictionary|has incorrect keys)',
-                        r'The key .* is not allowed',
-                        r'Allowed keys are:',
-                        # warnings
-                        r'Line .* is declared but not appended to any stream',
-                        r'uses plain Gaudi configurable',
-                        r'has non-default configuration',
-                        r'    .*\(default =',
-                        r'The usage of unique HLT is deprecated',
-                        r'Requests Flavor Tagging but is not assigned',
-                        r'has prescale <= 0',
-                        r'share the same output location',
-                        r'how to add new bank',
-                        r'Using default tag.*for partition',
-                        r'The lines going to MDST.DST are',
-                        r'No explicit tuning for DataType',
-                        r'something else configured a decoder already',
-                        r'REMOVE UT decoding from DataOnDemand',
-                        r'Applied corrections configured via options for',
-                        r'requesting factory via old name'
-                       ))
+handler = GetConsoleHandler()
+handler.disable(allowed=logging.ERROR)
 
-_orig_log_filter = log.filter
-def old_stripping_filter(record):
-    '''
-    Hide known warnings and errors in old Stripping configurations.
-    '''
-    if record.levelno >= logging.WARNING:
-        if any(exp.search(record.msg) for exp in IGNORED_MESSAGES):
-            return False
-        if not record.msg.strip():
-            return False # why should anyone want to print an empty warning?
-    return _orig_log_filter(record)
-
-log.filter = old_stripping_filter

Hlt/Moore/doc/release.notes

@@ -4,6 +4,39 @@
 ! Purpose     : LHCb HLT application
 !============================================
 
+! 2016-06-02 - Rosen Matev
+ - Update 2016_may to 2016_june
+ - Update TCK tests to use Physics_pp_June2016
+ - Make sure 2016_draft and 2016_june write out a csv timing table
+ - Add prerequisite to hlt1only
+ - Update settings in tistos
+
+! 2016-06-01 - Rosen Matev
+ - Fix mc_mooreratetest_eff to put up with ERROR from Herschel decoder
+ - Remove obsolete tests
+   - 2010 data tests
+   - magicnumber
+ - Fix vdm test to use VanDerMeerScan_2016 and proper input data
+ - Fix passthrough test to run in Hlt1 split mode
+ - Fix 2016_draft and 2016_may to put up with WARNING from Hlt2HcalCov.HcalCovarTool.DBAccessor
+ - Fix vdm_streams test
+ - Fix cosmics test
+ - Fix sim08 test and make it tolerate "known" warnings and errors
+ - Fix old TCK tests
+ - Attempt to fix calibration test but proper input data is missing
+ - Remove unused files in tests/options
+
+! 2016-05-24 - Rosen Matev
+ - Remove 2015 settings test
+ - Use test_file_db["2015NB_25ns_L0Filt0x1606"] everywhere
+ - Override conddb tag to make sure Herschel decoder works
+
+! 2016-05-18 - Mika Vesterinen
+ - Add a test of a new functionality of Moore_RateTest.py, namely to measure and report the efficiency for signal MC
+
+! 2016-05-18 - Mika Vesterinen
+ - Use the correct TCK in tests/options/TCK/TCKOptions.py
+
 ! 2016-05-15 - Mika Vesterinen
  - Add further qmttests for creation and testing of Hlt1 and Hlt2 TCKs
 

Hlt/Moore/python/Moore/Configuration.py

@@ -624,7 +624,7 @@ class Moore(LHCbConfigurableUser):
                         if list is not type(x) : x = [ x ]
                         for i in x : gather(i,overrule)
             from Configurables import HltGenConfig,GaudiSequencer
-            HltGenConfig().Overrule = { 'Hlt1ODINTechnicalPreScaler' : [ 'AcceptFraction:@OnlineEnv.AcceptRate@0' ] }
+            HltGenConfig().Overrule.update({ 'Hlt1ODINTechnicalPreScaler' : [ 'AcceptFraction:@OnlineEnv.AcceptRate@0' ] })
             gather( GaudiSequencer('Hlt'), HltGenConfig().Overrule )
             print HltGenConfig()
         from Gaudi.Configuration import appendPostConfigAction

Hlt/Moore/tests/options/2015-threshold.py

-from Configurables import Moore
-
-Moore().ThresholdSettings="Physics_25ns_October2015"
-Moore().UseTCK = False

Hlt/Moore/tests/options/Input_Default.py

 from Configurables import Moore
 from PRConfig.TestFileDB import test_file_db
-input = test_file_db['2015HLTValidationData_L0filtered_0x0050']
+input = test_file_db["2015NB_25ns_L0Filt0x1606"]
 input.run(configurable=Moore())
+Moore().CondDBtag = 'cond-20160522'
\ No newline at end of file

Hlt/Moore/tests/options/Input_2015_0x0050.py → Hlt/Moore/tests/options/Input_Sim08.py

 from Configurables import Moore
 from PRConfig import TestFileDB
-TestFileDB.test_file_db["2015NB_25ns_L0Filt0x0050"].run(configurable=Moore())
+TestFileDB.test_file_db["Sim08_2012_L044"].run(configurable=Moore(), withDB=True, clear=True)
+Moore().Simulation = True

Hlt/Moore/tests/options/Moore-Testing-Default.py

-
-from Gaudi.Configuration import *
-from Configurables import Moore
-from Configurables import L0Conf, HltConf
-from Gaudi.Configuration import *
-from Configurables import Moore
-
-############# Moore config taken from printout at the top of the sequence ############
-
-Moore().outputFile= 'Test.dst';
-Moore().DataType= '2012';
-Moore().DDDBtag= 'dddb-20120831'; #from RunDB
-Moore().EnableDataOnDemand= True;
-Moore().CheckOdin= False;
-Moore().InitialTCK= '0x00790038';
-Moore().CondDBtag= 'cond-20120831'; #from RunDB
-Moore().EnableTimer="timerdefault.csv"
-
-########### L0-related things #######################
-
-from Configurables import L0Conf
-L0Conf().EnsureKnownTCK=False
-Moore().ForceSingleL0Configuration=False
-
-############# prepend a "killHltSeq" ############
-
-#copied from Swimming/Configuration.py line 214
-from Configurables import Swimming, GaudiSequencer, EventNodeKiller
-nodes=Swimming.__slots__["HltNodesToKill"]
-mykiller    = EventNodeKiller("killHlt")
-mykiller.Nodes          = nodes
-deathstar               = GaudiSequencer("killHltSeq")
-deathstar.Members       = [mykiller]
-
-class Deathstar(object):
-    def __init__(self, seq):
-        self._seq = seq
-    def witnessTheFirepowerOfThisFullyArmedAndOperationalBattleStation(self):
-        ApplicationMgr().TopAlg.insert(0, self._seq)
-
-d = Deathstar(deathstar)
-appendPostConfigAction(d.witnessTheFirepowerOfThisFullyArmedAndOperationalBattleStation)
-
-
-############# prepend a "input data" ############
-
-
-from PRConfig import TestFileDB
-TestFileDB.test_file_db["2012_raw_default"].run(withDB=False,configurable=Moore())

Hlt/Moore/tests/options/Moore-Testing-MC.py

-
-from Gaudi.Configuration import *
-from Configurables import Moore
-Moore().Simulation= True;
-from Configurables import LHCbApp;
-LHCbApp().DDDBtag="dddb-20120831";
-LHCbApp().CondDBtag="sim-20121025-vc-md100";
-
-############# "input data" ############
-from PRConfig import TestFileDB
-TestFileDB.test_file_db["Sim08_2012_ForMoore"].run(withDB=False,configurable=Moore())

Hlt/Moore/tests/options/Moore_2016draft.py

@@ -14,15 +14,18 @@ LHCbTimingAuditor("TIMING").addTool(LHCbSequencerTimerTool, "TIMING")
 LHCbTimingAuditor("TIMING").TIMING.Normalised = True
 
 from PRConfig import TestFileDB
-TestFileDB.test_file_db["2015NB_25ns_L0Filt0x0050"].run(configurable=Moore())
+TestFileDB.test_file_db["2016NB_25ns_L0Filt0x1609"].run(configurable=Moore())
 Moore().RemoveInputHltRawBanks = True
 from Configurables import DDDBConf
 DDDBConf().InitialTime = 'Safe'
+# the following overides the L0TCK in the ThresholdSettings                                                          
+from Configurables import HltConf
+HltConf().setProp("L0TCK",'0x1609')
 
-Moore().EvtMax = 1200
+Moore().EvtMax = 100
 
 from Configurables import EventSelector
-EventSelector().PrintFreq = 100
+EventSelector().PrintFreq = 10
 
 # Add your new HLT2 lines here to run them
 # from Configurables import HltConf

Hlt/Moore/tests/options/Moore_CreateTCK.py

-from Gaudi.Configuration import *
-from Configurables import Moore
-
-## Create a local TCK
-Moore().TCKData = '$MOOREROOT/tests/options/TCKData/'
-Moore().generateConfig = True
-Moore().configLabel = 'Test TCK'
-
-## Extra settings required to make things work
-from Configurables import L0MuonAlg
-L0MuonAlg( "L0Muon" ).L0DUConfigProviderType = "L0DUConfigProvider"

Hlt/Moore/tests/options/Moore_NoTosDeferral.py

-#!/usr/bin/env gaudirun.py
-#
-# Minimal file for running Moore from python prompt
-# Syntax is:
-#   gaudirun.py ../options/Moore.py
-# or just
-#   ../options/Moore.py
-#
-import Gaudi.Configuration
-from Moore.Configuration import Moore
-
-#use new splitting of Hlt2
-Moore().Split=''
-
-##only for experts!
-from Configurables import MooreExpert
-#turn of TISTOS, "Gerhard's Sledgehammer"
-MooreExpert().Hlt2Independent=True

Hlt/Moore/tests/options/Moore_RateTest_MC_RedoL0.py

+# Purpose:
+# Re-run L0 on a juggled signal MC sample
+# The output file will be used to check the efficiency function
+# of Moore_RateTest.py in PRConfig
+
+from Gaudi.Configuration import *
+from Configurables import L0App
+L0App().TCK = '0x1600'
+L0App().ReplaceL0Banks = True
+L0App().EvtMax = 100
+from Configurables import CondDB
+
+from PRConfig import TestFileDB
+TestFileDB.test_file_db['Juggled_MC_2015_27163003_DstToD0pip_D0ToKmPip'].run(configurable=L0App())
+
+
+from Configurables import HltRoutingBitsFilter
+rb_filter = HltRoutingBitsFilter( "RoutingBitsFilter" )
+rb_filter.RequireMask = [0x0, 0x4, 0x0]
+
+from Configurables import bankKiller
+hlt_banks = [ 'HltDecReports','HltRoutingBits','HltSelReports','HltVertexReports','HltLumiSummary','HltTrackReports' ]
+killer = bankKiller( 'RemoveInputHltRawBanks',  BankTypes = hlt_banks )
+
+ApplicationMgr().TopAlg = [rb_filter, killer]
+
+from Configurables import EventSelector
+EventSelector().PrintFreq = 10
+
+from Configurables import L0DUFromRawAlg
+l0du_alg = L0DUFromRawAlg()
+
+from Configurables import L0DUReportMonitor
+l0du_mon = L0DUReportMonitor()
+
+from Configurables import LoKiSvc
+LoKiSvc().Welcome = False
+from Configurables import LoKi__L0Filter
+l0_filter = LoKi__L0Filter('L0Filter', Code = "L0_DECISION_PHYSICS")
+
+from GaudiConf import IOHelper
+from Configurables import InputCopyStream
+input_copy = IOHelper().outputAlgs('rerunL0.dst', InputCopyStream('CopyStream'), writeFSR = False)[0]
+
+from Configurables import GaudiSequencer
+from Gaudi.Configuration import ApplicationMgr
+writer_seq = GaudiSequencer('WriterSeq')
+writer_seq.Members = [l0du_alg, l0du_mon, l0_filter, input_copy]
+ApplicationMgr().OutStream = [writer_seq]
+

Hlt/Moore/tests/options/SwimTriggerB2JpsiX.py

-from Configurables import Swimming
-from Gaudi.Configuration import *
-
-Swimming().DataType = '2012'
-Swimming().EvtMax = 250
-Swimming().Simulation = False
-# For v14r2
-#Swimming().DDDBtag     = 'head-20120413'
-#Swimming().CondDBtag   = 'head-20120420'
-# For v14r6
-#Swimming().DDDBtag     = 'head-20120413'
-#Swimming().CondDBtag   = 'cond-20120730'
-# For v14r8,9,11
-Swimming().DDDBtag     = 'dddb-20120831'
-Swimming().CondDBtag   = 'cond-20120831'
-#For v14r2
-#Swimming().TCK = '0x0097003d'
-#For v14r6 
-#Swimming().TCK = '0x00990042'
-#For v14r8 
-Swimming().TCK = '0x00a10044'
-#For v14r9
-#Swimming().TCK = '0x00a30044'
-#For v14r11
-#Swimming().TCK = '0x00ab0046'
-Swimming().InputType = 'DST'
-#For v14r2
-#Swimming().OutputFile = 'SwimTrigDST_v14r2.dst'
-#For v14r6
-#Swimming().OutputFile = 'SwimTrigDST_v14r6.dst'
-#For v14r8
-Swimming().OutputFile = 'SwimTrigDST_v14r8.dst'
-#For v14r9
-#Swimming().OutputFile = 'SwimTrigDST_v14r9.dst'
-#For v14r11
-#Swimming().OutputFile = 'SwimTrigDST_v14r11.dst'
-Swimming().SwimStripping = False
-Swimming().Hlt1Triggers = ["Hlt1DiMuonHighMassDecision", "Hlt1TrackAllL0Decision", "Hlt1TrackMuonDecision"]
-Swimming().Hlt2Triggers = ["Hlt2DiMuonJPsiDecision", "Hlt2DiMuonDetachedJPsiDecision"]
-Swimming().OffCands = "/Event/Dimuon/Phys/BetaSBd2JpsiKstarDetachedLine"
-Swimming().TransformName = '2011_WithBeamSpotFilter_NoRecoLines'
-Swimming().SelectMethod = 'random'
-Swimming().OutputType = 'DST'
-# MCl.: This caused the tests to hang forever
-#Swimming().UseFileStager = True
-Swimming().Debug = False
-#For v14r2
-#Swimming().Input = ['PFN:root://eoslhcb.cern.ch//eos/lhcb/LHCb/Collision12/DIMUON.DST/00018117/0000/00018117_00002737_1.dimuon.dst']
-#For v14r6
-#Swimming().Input = ['PFN:root://eoslhcb.cern.ch//eos/lhcb/LHCb/Collision12/DIMUON.DST/00024183/0002/00024183_00020006_1.dimuon.dst']
-#For v14r8
-#Swimming().Input = ['PFN:root://eoslhcb.cern.ch//eos/lhcb/LHCb/Collision12/DIMUON.DST/00020567/0000/00020567_00006473_1.dimuon.dst'] # added to testfileDB
-#For v14r9
-#Swimming().Input = ['PFN:root://eoslhcb.cern.ch//eos/lhcb/LHCb/Collision12/DIMUON.DST/00024183/0000/00024183_00009746_1.dimuon.dst']
-#For v14r11
-#Swimming().Input = ['PFN:root://eoslhcb.cern.ch//eos/lhcb/LHCb/Collision12/DIMUON.DST/00021211/0000/00021211_00009739_1.dimuon.dst']
-# Configure Moore to swim the trigger
-from Swimming.Configuration import ConfigureMoore
-ConfigureMoore()
-
-# The custom event loop
-from Gaudi.Configuration import setCustomMainLoop
-from Swimming.EventLoop import SwimmingEventLoop
-setCustomMainLoop(SwimmingEventLoop)

Hlt/Moore/tests/options/TCK/RedoL0.py

@@ -11,7 +11,7 @@ app.CondDBtag = 'cond-20150828'
 app.DDDBtag = 'dddb-20150724'
 
 from GaudiConf import IOHelper
-IOHelper('MDF').inputFiles(['mdf:root://eoslhcb.cern.ch//eos/lhcb/wg/HLT/2016CommissioningDatasets/OneAcceptedPerLine_v25r1_Physics_pp_May2016_190416/Moore%02d.mdf' %i for i in range(0,15)]) 
+IOHelper('MDF').inputFiles(['mdf:root://eoslhcb.cern.ch//eos/lhcb/wg/HLT/2016CommissioningDatasets/OneAcceptedPerLine_v25r1_Physics_pp_June2016_190416/Moore%02d.mdf' %i for i in range(0,15)]) 
 
 from Configurables import EventSelector
 EventSelector().PrintFreq = 10

Hlt/Moore/tests/options/TCK/TCKOptions.py

@@ -26,11 +26,11 @@ Moore().CondDBtag = "cond-20160420"
 
 ## !!! Check that these settings are what we want for these TCKs !!! ##
 Settings={"L0TCK":"0x1600",
-          "Hlt1TCK":'0x112F1600',
-          "Hlt2TCK":'0x212F1600',
-          "Hlt1Label":"Hlt1, Physics_pp_May2016, 0x1600",
-          "Hlt2Label":"Hlt2, Physics_pp_May2016, 0x1600",
-          "ThresholdSettings":'Physics_pp_May2016'}
+          "Hlt1TCK":'0x10011600',
+          "Hlt2TCK":'0x20011600',
+          "Hlt1Label":"Hlt1, Physics_pp_June2016, 0x1600",
+          "Hlt2Label":"Hlt2, Physics_pp_June2016, 0x1600",
+          "ThresholdSettings":'Physics_pp_June2016'}
 
 ## propagate the above settings 
 

Hlt/Moore/tests/options/alllines-threshold.py

-from Configurables import Moore
-
-Moore().ThresholdSettings="Commissioning_AllHlt2Lines_2015"
-Moore().UseTCK = False

Hlt/Moore/tests/options/compareTrackReports.py

-
-import sys
-
-f1 = open(sys.argv[1]);
-f2 = open(sys.argv[2]);
-
-list1 = []
-list2 = []
-
-import difflib
-
-errorcounter = 0 
-trackcounter = 0
-
-for l1 in f1 :
-    if not "LHCbIDs" in l1 : continue
-    else : 
-        #print l1
-        trackcounter+=1
-        # buffer the LHCbIDs
-        for l1a in f1 :
-            #print l1a
-            if "]" in l1a : break
-            else : list1.append(l1a)
-        
-        for l2 in f2 :
-            if not "LHCbIDs" in l2 : continue
-            else : 
-                #print l2
-                # now buffer 
-                for l2a in f2 :
-                    #print l2a
-                    if "]" in l2a : break
-                    else : 
-                        list2.append(l2a)
-                # now compare
-                #print list1
-                #print list2
-                difflib.SequenceMatcher(None,list1[0],list2[0])
-                if list1 != list2 :
-                    errorcounter +=1
-                    if len(list1) != len(list2) :
-                        print "Different number of LHCbIDs in track!"
-                        print
-                        print "".join(list1)
-                        print "".join(list2)
-                    else : 
-                        for i in range(0,len(list1)) :
-                            if list1[i] != list2[i] : 
-                                print list1[i]
-                                print list2[i]
-                list1[:] =[]
-                list2[:] =[]
-                break
-
-        if list1 != [] : print "Tracks missing in second file!"
-
-
-print
-            
-if errorcounter > 0 :
-    print "Found " + str(errorcounter) + " mismatches in " + str(trackcounter) + " tracks"
-else : print "No mismatch found in " +str(trackcounter) +" tracks. All good. Except for things we did not think of. Yet."