Commit 138ddbed authored by Roel Aaij's avatar Roel Aaij
Browse files

Merge branch 'rmatev-LHCBPS-1440' into '2016-patches'

Fix some tests and remove others

- Fix test failures due to CondDB and other minor issues
- Create test TCKs in a platform specific directory
- Fix (somehow) tests for rerunning on stripping 20
- Update reference for hlt2prcheck and improve reproducibility
- Remove the tistos test, track decoding tests, and the hlt1only and hlt2only tests
- Remove unused test options
- Improve .gitignore

See merge request !54
parents 71185f8a ac147651
......@@ -14,6 +14,7 @@ CTestTestfile.cmake
CPackSourceConfig.cmake
CPackConfig.cmake
cmake_install.cmake
/toolchain.cmake
# common byproducts
*.pyc
......@@ -41,7 +42,6 @@ setup.*
cleanup.*
version.cmt
genConf/
QMTest/
test_results/
*-slc[56]-*/
i686-winxp-vc9-dbg/
......
......@@ -12,7 +12,7 @@ Moore().CheckOdin = False
Moore().Simulation = False
Moore().ForceSingleL0Configuration = True
Moore().SkipEvents = 0
Moore().CondDBtag = 'cond-20160522'
Moore().CondDBtag = 'cond-20161011'
Moore().DDDBtag = 'dddb-20150724'
from Configurables import EventSelector
......
......@@ -19,17 +19,20 @@ def myActionHlt2() :
PrChecker2("PrCheckerHlt2").TriggerNumbers = True
PrChecker2("PrCheckerHlt2").Eta25Cut = True
PrChecker2("PrCheckerHlt2").WriteForwardHistos = 1
PrChecker2("PrCheckerHlt2").WriteTexOutput = True # workaround for unitialized values...
PrChecker2("PrCheckerHlt2Forward").TriggerNumbers = True
PrChecker2("PrCheckerHlt2Forward").Eta25Cut = True
PrChecker2("PrCheckerHlt2Forward").WriteForwardHistos = 1
PrChecker2("PrCheckerHlt2Forward").ForwardTracks = Hlt2TrackLoc["ComplementForward"]
PrChecker2("PrCheckerHlt2Forward").WriteTexOutput = True # workaround for unitialized values...
#(N.B.: container base for velo tracks was renamed to Hlt1)
# Figure out which HLT is run from HltConf
PrChecker2("PrCheckerHlt2").SeedTracks = Hlt2TrackLoc["Seeding"]
PrChecker2("PrCheckerHlt2").MatchTracks = Hlt2TrackLoc["Match"]
PrChecker2("PrCheckerHlt2").DownTracks = Hlt2TrackLoc["Downstream"]
PrChecker2("PrCheckerHlt2").ForwardTracks = Hlt2TrackLoc["Forward"]
PrChecker2("PrCheckerHlt2").WriteTexOutput = True # workaround for unitialized values...
from HltTracking.Hlt2TrackingConfigurations import Hlt2BiKalmanFittedForwardTracking
from HltTracking.Hlt2TrackingConfigurations import Hlt2BiKalmanFittedDownstreamTracking
......@@ -38,6 +41,7 @@ def myActionHlt2() :
PrChecker2("PrCheckerHlt2").BestTracks = tracks.outputSelection()
PrChecker2("PrCheckerHlt2Down").BestTracks = tracksDown.outputSelection()
#PrChecker2("PrCheckerHlt2").DownTracks = tracksDown.outputSelection()
PrChecker2("PrCheckerHlt2Down").WriteTexOutput = True # workaround for unitialized values...
PrTrackAssociator("AssocAllHlt2").RootOfContainers = Hlt2TrackRoot
PrTrackAssociator("AssocFittedHlt2").RootOfContainers = Hlt2TrackRoot+HltDefaultFitSuffix
......
from Configurables import Moore;
Moore().TCKData='../../../../InstallArea/$CMTCONFIG'
from Gaudi.Configuration import *
from Gaudi.Configuration import INFO
from Configurables import Moore
from Configurables import Moore, CondDB
Moore().ThresholdSettings = "Physics_pp_Draft2016"
Moore().UseTCK = False
Moore().ForceSingleL0Configuration = False
Moore().Split = ''
Moore().EnableTimer = True
Moore().OutputLevel = INFO
CondDB().LatestGlobalTagByDataType = Moore().getProp('DataType')
from Configurables import LHCbTimingAuditor, LHCbSequencerTimerTool
LHCbTimingAuditor("TIMING").addTool(LHCbSequencerTimerTool, "TIMING")
#normalize the timing table in tests
# normalize the timing table in tests
LHCbTimingAuditor("TIMING").TIMING.Normalised = True
from PRConfig import TestFileDB
TestFileDB.test_file_db["2016NB_25ns_L0Filt0x1609"].run(configurable=Moore())
Moore().RemoveInputHltRawBanks = True
from Configurables import DDDBConf
DDDBConf().InitialTime = 'Safe'
# the following overides the L0TCK in the ThresholdSettings
# the following overides the L0TCK in the ThresholdSettings
from Configurables import HltConf
HltConf().setProp("L0TCK",'0x1609')
HltConf().setProp("L0TCK", '0x1609')
Moore().EvtMax = 100
......
from Gaudi.Configuration import *
from Gaudi.Configuration import INFO
from Configurables import Moore
from Configurables import Moore, CondDB
Moore().ThresholdSettings = "protonHelium_pPb_2016"
Moore().UseTCK = False
Moore().ForceSingleL0Configuration = False
Moore().Split = ''
Moore().EnableTimer = True
Moore().OutputLevel = INFO
CondDB().LatestGlobalTagByDataType = Moore().getProp('DataType')
from Configurables import LHCbTimingAuditor, LHCbSequencerTimerTool
LHCbTimingAuditor("TIMING").addTool(LHCbSequencerTimerTool, "TIMING")
# normalize the timing table in tests
LHCbTimingAuditor("TIMING").TIMING.Normalised = True
# from PRConfig import TestFileDB
# TestFileDB.test_file_db["2016NB_25ns_L0Filt0x1609"].run(configurable=Moore())
......@@ -17,8 +23,6 @@ IOExtension().inputFiles([
])
Moore().RemoveInputHltRawBanks = True
from Configurables import DDDBConf
DDDBConf().InitialTime = 'Safe'
Moore().EvtMax = 1000
......
# Purpose:
# Re-run L0 on a juggled signal MC sample
# The output file will be used to check the efficiency function
# of Moore_RateTest.py in PRConfig
from Gaudi.Configuration import *
from Configurables import L0App
L0App().TCK = '0x1600'
L0App().ReplaceL0Banks = True
L0App().EvtMax = 100
from Configurables import CondDB
from PRConfig import TestFileDB
TestFileDB.test_file_db['Juggled_MC_2015_27163003_DstToD0pip_D0ToKmPip'].run(configurable=L0App())
from Configurables import HltRoutingBitsFilter
rb_filter = HltRoutingBitsFilter( "RoutingBitsFilter" )
rb_filter.RequireMask = [0x0, 0x4, 0x0]
from Configurables import bankKiller
hlt_banks = [ 'HltDecReports','HltRoutingBits','HltSelReports','HltVertexReports','HltLumiSummary','HltTrackReports' ]
killer = bankKiller( 'RemoveInputHltRawBanks', BankTypes = hlt_banks )
ApplicationMgr().TopAlg = [rb_filter, killer]
from Configurables import EventSelector
EventSelector().PrintFreq = 10
from Configurables import L0DUFromRawAlg
l0du_alg = L0DUFromRawAlg()
from Configurables import L0DUReportMonitor
l0du_mon = L0DUReportMonitor()
from Configurables import LoKiSvc
LoKiSvc().Welcome = False
from Configurables import LoKi__L0Filter
l0_filter = LoKi__L0Filter('L0Filter', Code = "L0_DECISION_PHYSICS")
from GaudiConf import IOHelper
from Configurables import InputCopyStream
input_copy = IOHelper().outputAlgs('rerunL0.dst', InputCopyStream('CopyStream'), writeFSR = False)[0]
from Configurables import GaudiSequencer
from Gaudi.Configuration import ApplicationMgr
writer_seq = GaudiSequencer('WriterSeq')
writer_seq.Members = [l0du_alg, l0du_mon, l0_filter, input_copy]
ApplicationMgr().OutStream = [writer_seq]
......@@ -22,20 +22,20 @@ def id_from_log(filename):
raise RuntimeError('File {} does not look like TCK creation output!'.format(filename))
def tck_from_options(filename, key):
def settings_from_options(filename):
g = {}
execfile(filename, g)
tck = g['settings'][key]
# a hack to remove any configurable instances coming from the options file
from GaudiKernel.Configurable import Configurable
Configurable.allConfigurables = {}
return tck
return g['settings']
parser = argparse.ArgumentParser()
parser.add_argument("id", help='Top level configuration id or TCK creation log.')
parser.add_argument("tck", help='TCK (e.g. 0x12345678) or key in options (e.g. Hlt1TCK).')
parser.add_argument("--options", help='Options where Hlt1TCK/Hlt2TCK is defined.')
parser.add_argument("--tckdata", default='TCKData', help='Directory containing the config.cdb file.')
parser.add_argument("--options", help='Options where Hlt1TCK/Hlt2TCK and TCKData are defined.')
args = parser.parse_args()
......@@ -47,8 +47,10 @@ if not (len(args.id) == 32 and is_hex(args.id)):
if not is_hex(args.tck):
if not args.options:
parser.error('--options is required when TCK is not given directly')
args.tck = tck_from_options(args.options, args.tck)
settings = settings_from_options(args.options)
args.tck = settings[args.tck]
args.tckdata = settings['TCKData']
cas_rw = ConfigAccessSvc(Mode='ReadWrite', File='TCKData/config.cdb')
configdb = os.path.join(os.path.expandvars(args.tckdata), 'config.cdb')
cas_rw = ConfigAccessSvc(Mode='ReadWrite', File=configdb)
createTCKEntries({args.tck: args.id}, cas=cas_rw)
......@@ -9,6 +9,8 @@ The script CreateTCK1.py contains the absolute minimum configuration for that st
# !!! Check that these settings are what we want for these TCKs !!! #
settings = {
# Modify these as approppriate for the new TCKs:
"TCKData": "../../../../InstallArea/$CMTCONFIG",
# "TCKData": "TCKData",
"L0TCK": "0x160B",
"Hlt1TCK": "0x1001160B",
"Hlt2TCK": "0x2001160B",
......@@ -18,7 +20,7 @@ settings = {
# Check if these need to be updated:
"ThresholdSettings": "Physics_pp_Draft2016",
"DDDBtag": "dddb-20150724",
"CondDBtag": "cond-20160522",
"CondDBtag": "cond-20161011",
}
......@@ -41,7 +43,7 @@ if 'Moore' in allConfigurables: # For the Create/Test TCK steps
Moore().RemoveInputHltRawBanks = True
# Specify the location for our local config.cdb file
TCKData_DEST = 'TCKData'
TCKData_DEST = os.path.expandvars(settings['TCKData'])
Moore().TCKData = TCKData_DEST
if not os.path.exists(TCKData_DEST):
os.makedirs(TCKData_DEST)
......
......@@ -9,6 +9,8 @@ The script CreateTCK1.py contains the absolute minimum configuration for that st
# !!! Check that these settings are what we want for these TCKs !!! #
settings = {
# Modify these as approppriate for the new TCKs:
"TCKData": "../../../../InstallArea/$CMTCONFIG",
# "TCKData": "TCKData",
"L0TCK": "0x160B",
"Hlt1TCK": "0x1002160B",
"Hlt2TCK": "0x2002160B",
......@@ -18,7 +20,7 @@ settings = {
# Check if these need to be updated:
"ThresholdSettings": "Physics_pp_Draft2016",
"DDDBtag": "dddb-20150724",
"CondDBtag": "cond-20160522",
"CondDBtag": "cond-20161011",
}
......@@ -41,7 +43,7 @@ if 'Moore' in allConfigurables: # For the Create/Test TCK steps
Moore().RemoveInputHltRawBanks = True
# Specify the location for our local config.cdb file
TCKData_DEST = 'TCKData'
TCKData_DEST = os.path.expandvars(settings['TCKData'])
Moore().TCKData = TCKData_DEST
if not os.path.exists(TCKData_DEST):
os.makedirs(TCKData_DEST)
......
......@@ -5,9 +5,9 @@ parser = argparse.ArgumentParser()
parser.add_argument("--split", help='split level (Hlt1 or Hlt2).')
args = parser.parse_args()
cas_rw = ConfigAccessSvc(Mode='ReadWrite', File='TCKData/config.cdb')
cas = ConfigAccessSvc(File='../../../../InstallArea/$CMTCONFIG/config.cdb')
configurations = getConfigurations(cas=cas_rw)
configurations = getConfigurations(cas=cas)
TCKs = []
for k,v in configurations.iteritems():
......@@ -22,7 +22,7 @@ print TCKs
if len(TCKs) != 2:
raise RuntimeError('Found %s (require 2) %s TCKs in the config.cdb ' %(len(TCKs),args.split))
print 'diff the first two'
diff(TCKs[0],TCKs[1],cas=cas_rw)
diff(TCKs[0], TCKs[1], cas=cas)
#import sys
#sys.exit(0)
......
from Configurables import Moore, HltConf
HltConf().ThresholdSettings="Hlt2_TrackingOnly"
Moore().UseTCK = False
# HLT1 might have been run with a TCK so lets look into the local one
from Configurables import ConfigCDBAccessSvc
ConfigCDBAccessSvc().File='config.cdb'
Moore().TCKData = '$MOOREROOT/tests/options/TCKData/'
Moore().InitialTCK = '0x50000001'
Moore().CheckOdin=False
......@@ -42,5 +42,14 @@ appendPostConfigAction(d.witnessTheFirepowerOfThisFullyArmedAndOperationalBattle
from Configurables import RecombineRawEvent
RecombineRawEvent()
from Configurables import HltOutputConf
HltOutputConf().EnableLumiEventWriting = False
# Moore().RemoveInputHltRawBanks = True
Moore().EnableTimer = True
Moore().OutputLevel = INFO
from Configurables import LHCbTimingAuditor, LHCbSequencerTimerTool
LHCbTimingAuditor("TIMING").addTool(LHCbSequencerTimerTool, "TIMING")
# normalize the timing table in tests
LHCbTimingAuditor("TIMING").TIMING.Normalised = True
#When running from the real data for my tests, I need to scale down the prescale lines usign a transform, otherwise I simple select everything again.
#I need to do this when running, but not when generating a TCK!
#accessing this property is done via TCKSh:
# python
# from TCKUtils.utils import *
# ctas=ctas(File='../../../../InstallArea/x86_64-slc6-gcc48-opt/config.tar')
# getProperty(0x00ee0046,"Hlt1MBNoBiasPostScaler","AcceptFraction",ctas)
#
from Gaudi.Configuration import *
def resetAction(mult=0.1):
from Configurables import DeterministicPrescaler as dp
dp("Hlt1MBNoBiasPreScaler").AcceptFraction=mult*mult
dp("Hlt1CharmCalibrationNoBiasPreScaler").AcceptFraction=mult
dp("Hlt2TransparentPreScaler").AcceptFraction=0.0
def resetTransform(mult=0.1):
"""
Gah, can't guess what the original number is...
"""
from Moore import Funcs
trans={}
trans[".*Hlt1CharmCalibrationNoBiasPreScaler"]={"AcceptFraction": {"^.*$":str(mult)}}
trans[".*Hlt1MBNoBiasPreScaler"]={"AcceptFraction": {"^.*$":str(mult*mult)}}
trans[".*Hlt2TransparentPreScaler"]={"AcceptFraction": {"^.*$":str(0.0)}}
Funcs._mergeTransform(trans)
from Configurables import Moore
if Moore().getProp("generateConfig"):
pass
else:
if Moore().getProp("UseTCK"):
resetTransform()
else:
appendPostConfigAction(resetAction)
import os
from TCKUtils.utils import *
TCK1=0x00ed1606
TCK2=0x00ee1606
TCK3=0x00ef1606
configdest="../../../../InstallArea/$CMTCONFIG"
ConfigAccessSvc().File = os.path.join(configdest, "config.cdb")
print "============= diffs zero->1 event ==============="
diff(TCK1,TCK2)
print "============= diffs 1->100 events ==============="
diff(TCK2,TCK3)
#test to check if I can make a TCK
#needs numEvents and targetTCK to be defined already, which is done in your python shell or a different file
#creationString="created Aliases/TOPLEVEL/"
#existsString="trying to read Aliases/TOPLEVEL/"
existsString="top id for this config:"
mysplit=''
try:
mysplit=split
except:
pass
label="Test TCK from nightly builds, auto-generated in nightlies based on some physics threshold setting."
releaseString="----MOORERELEASE----:"
thresholdString="----MOORETHRESHOLD----:"
configsource="$HLTTCKROOT"
f=open("../options/Moore-LocalTCKData.py")
configdest=f.read().split("'")[1]
f.close()
commandoptions = "from Configurables import Moore; Moore().OutputLevel = 4; Moore().generateConfig=True; Moore().ForceSingleL0Configuration = True; Moore().EvtMax="+str(numEvents)+"; Moore().configLabel=\""+label+"\"; print \""+releaseString+"\", Moore().getRelease(); print \""+thresholdString+"\", Moore().ThresholdSettings; Moore().EnableTimer=False;"
commandoptions += "from Configurables import HltConf; HltConf().L0TCK = \"0x%s\"; print Moore();" % targetTCK[-4:]
commandoptions = "'%s'"%commandoptions
if mysplit=="Hlt1":
commandoptions = commandoptions.replace("import Moore;",'import Moore; Moore().Split=\"Hlt1";')
if mysplit=="Hlt2":
commandoptions = commandoptions.replace("import Moore;",'import Moore; Moore().Split="Hlt2";')
def expp(apath):
return os.path.abspath(os.path.expandvars(os.path.expanduser(apath)))
import os
if not os.path.exists(expp(configsource)) or not os.path.exists(expp(configdest)):
print configsource, configdest, expp(os.getcwd())
raise IOError("Local filesystem not like install area, aborting")
if not os.path.exists(os.path.join(expp(configdest),"config.cdb")):
print "copying config file"
import shutil
shutil.copy2(os.path.join(expp(configsource),"config.cdb"), os.path.join(expp(configdest),"config.cdb"))
print "modified config in: "+ os.path.join(expp(configdest),"config.cdb")
#run events
print "running",numEvents
import commands
status,output = commands.getstatusoutput("gaudirun.py ../options/Moore_Hlt1And2.py ../options/default-threshold.py ../options/Moore-LocalTCKData.py ../options/Input_Default.py --option="+commandoptions)
print output
if "FATAL" in output or existsString not in output:
raise ValueError("Something went wrong in the gaudirun step")
def findme(searchstring,output):
loc=output.find(searchstring)+len(searchstring)
eol=output.find('\n',loc)
return output[loc:eol].strip()
configuration=findme(existsString,output)
threshold=findme(thresholdString,output)
release=findme(releaseString,output)
print "===================================================="
print release, threshold, configuration
print "===================================================="
#
from TCKUtils.utils import *
from Configurables import ConfigCDBAccessSvc as ctas
ctas(File=os.sep.join([configdest,"config.cdb"]))
configs=getConfigurations()
search="/".join(["TOPLEVEL",release,threshold,configuration])
if search not in configs.keys():
raise KeyError("Configuration I just created has not been found... looking for " + search)
createTCKEntries({targetTCK:configuration},ctas(Mode='ReadWrite',File=os.sep.join([configdest,"config.cdb"])))
ctas(Mode='ReadOnly',File=os.sep.join([configdest,"config.cdb"]))
configs=getConfigurations()
if search not in configs.keys():
raise KeyError("Configuration I just created has not been found... looking for"+search)
if int(targetTCK,16) not in configs[search].info['TCK']:
raise ValueError("TCK I asked to be created was not created for this threshold setting")
configs[search].printSimple()
print "===================================================="
print "PASS"
numEvents=100
targetTCK='0x00ef1606'
execfile("../options/tckgen.py")
numEvents=1
targetTCK='0x00ee1606'
execfile("../options/tckgen.py")
numEvents=0
targetTCK='0x00ed1606'
execfile("../options/tckgen.py")
from Configurables import Moore
from Configurables import RecombineRawEvent
RecombineRawEvent()
Moore().ForceSingleL0Configuration = False
evtmax=3000
Moore().EvtMax = evtmax
Moore().OutputLevel =3
Moore().UseTCK = False
Moore().ThresholdSettings="Physics_pp_Draft2016"
Moore().DataType = "2012"
Moore().Split = ''
Moore().Simulation = True
Moore().CheckOdin = False
Moore().EnableDataOnDemand = True
Moore().WriterRequires = []
Moore().RemoveInputHltRawBanks=True
Moore().DDDBtag = "dddb-20130929-1"
Moore().CondDBtag = "sim-20131023-vc-md100"
from GaudiConf import IOHelper
IOHelper().inputFiles( [ "PFN:root://castorlhcb.cern.ch//castor/cern.ch/user/s/sbenson/TempTestFiles/00026133_00000013_1.allstreams.dst" ] )
from Configurables import Hlt__Service
Hlt__Service().Pedantic = False
from Configurables import DataOnDemandSvc
from Configurables import DstConf, CaloDstUnPackConf
CaloDstUnPackConf().Enable=True
DstConf().EnableUnpack= ["Reconstruction","Stripping"]
from Configurables import L0MuonAlg
L0MuonAlg("L0Muon").L0DUConfigProviderType="L0DUConfigProvider"
from Configurables import L0Conf
L0Conf().EnableL0DecodingOnDemand = True
L0Conf().FastL0DUDecoding = True
L0Conf().DecodeL0DU = True
L0Conf().TCK = '0x0045'
#from Configurables import Hlt2Conf
#Hlt2Conf().Hlt1TrackOption = "Rerun"
import GaudiPython
from Gaudi.Configuration import ApplicationMgr
from Configurables import LoKiSvc
appMgr = ApplicationMgr()
#
from Configurables import EventNodeKiller
enk = EventNodeKiller('KillTrigRawEvent')
enk.Nodes = [ "Hlt","Hlt1","Hlt2" ]
appMgr.TopAlg.insert( 0, enk.getFullName() )
#
appMgr.ExtSvc += ['DataOnDemandSvc'] #breaks for real data
appMgr.ExtSvc += ['ToolSvc', 'DataOnDemandSvc', LoKiSvc()]
#appMgr.TopAlg += [ seq ]
gaudi = GaudiPython.AppMgr(outputlevel=4)
gaudi.initialize()
gaudi.algorithm('Hlt').Enable = False
gaudi.algorithm('KillTrigRawEvent').Enable = False
triggerTisTosTool = gaudi.toolsvc().create('TriggerTisTos', interface='ITriggerTisTos')
triggerTisTosTool1 = gaudi.toolsvc().create('Hlt1TriggerTisTos', interface='ITriggerTisTos')
triggerTisTosTool2 = gaudi.toolsvc().create('Hlt2TriggerTisTos', interface='ITriggerTisTos')
tes = gaudi.evtsvc()
hlt1names = [
"Hlt1TrackAllL0Decision"
]
hlt2names = [
"Hlt2IncPhiDecision"
]
offlineLoc ="/Event/AllStreams/Phys/BetaSBs2PhiPhiLine/Particles"
count_hlt1_Dec={}
count_hlt1_TOS={}
for trigger in hlt1names:
count_hlt1_Dec.update( { trigger : 0 } )
count_hlt1_TOS.update( { trigger : 0 } )
count_hlt2_Dec={}
count_hlt2_TOS={}
for trigger in hlt2names:
count_hlt2_Dec.update( { trigger : 0 } )
count_hlt2_TOS.update( { trigger : 0 } )
i=0
n=0
#while True:
while n<evtmax:
n+=1
gaudi.run(1)
#
gaudi.executeEvent()
gaudi.algorithm("KillTrigRawEvent").execute()
gaudi.algorithm("Hlt").execute()
if i==0:
tes.dump()
split = True
if not tes["Hlt1/SelReports"]:
split = False
if not tes['/Event/DAQ/RawEvent']:
print "End of file"
break
cands = tes[offlineLoc]
if not cands:
continue
for cand in cands:
i+=1
triggerTisTosTool.setOfflineInput(cands[0])
triggerTisTosTool1.setOfflineInput(cands[0])
triggerTisTosTool2.setOfflineInput(cands[0])
#
for trigger in hlt1names:
if split == True:<