Commit 32de484e authored by Atlas-Software Librarian's avatar Atlas-Software Librarian Committed by Graeme Stewart
Browse files

'CMakeLists.txt' (TrigStatsFromCool-00-00-02-01)

parent b16f2e56
################################################################################
# Package: TrigStatsFromCool
################################################################################
# Declare the package name:
atlas_subdir( TrigStatsFromCool )
# Install files from the package:
atlas_install_python_modules( python/*.py )
include $(CMTROOT)/src/Makefile.header
include $(CMTROOT)/src/constituents.make
package TrigStatsFromCool
author Joerg.Stelzer@cern.ch
manager Joerg.Stelzer@cern.ch
#============================================================================
#============================================================================
# Public dependencies
#============================================================================
use DetCommonPolicy *
private
## use POOL v* LCG_Interfaces
## use COOL v* LCG_Interfaces
## use CORAL v* LCG_Interfaces
## use XercesC v* LCG_Interfaces
## use Reflex v* LCG_Interfaces
## use Boost v* LCG_Interfaces
##
## use TrigConfL1Data TrigConfL1Data-* Trigger/TrigConfiguration
## use TrigConfJobOptData TrigConfJobOptData-* Trigger/TrigConfiguration
## use TrigConfHLTData TrigConfHLTData-* Trigger/TrigConfiguration
#============================================================================
# Build rules
#============================================================================
## #
## # ignore patterns from LCG_Policy
## #
## apply_pattern detcommon_ignores
##
## # link header files into InstallArea, and include the corresponding
## # directory in the compiler options
## #
## apply_pattern detcommon_header_installer
#============================================================================
# Build applications
#============================================================================
## apply_pattern trigconf_application name=Lvl1TriggerApp
## apply_pattern trigconf_application name=Lvl1TriggerApp2
## #apply_pattern trigconf_application name=JobOptTriggerApp
## apply_pattern trigconf_application name=2COOLApp
## apply_pattern trigconf_application name=2XMLApp
## apply_pattern trigconf_application name=PrescaleCollLoad
## #apply_pattern trigconf_application name=TestLogicParser
## #apply_pattern trigconf_application name=HltApp
## apply_pattern trigconf_application name=ConsistencyChecker
## apply_pattern trigconf_application name=RttTest
#============================================================================
# Install scripts and modules
#============================================================================
## apply_pattern detcommon_link_files kind=scripts name=python_scripts files="\
## -s=../scripts *.py" prefix=share/bin
##
apply_pattern detcommon_link_files kind=python_modules name=python_mods files="\
-s=../python *.py" prefix=python/$(package)
#!/usr/bin/env python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
import sys
from collections import namedtuple,defaultdict
from struct import unpack_from, unpack
from PyCool import cool
from CoolRunQuery.AtlRunQueryTimer import timer
from TrigStatsFromCool.HLTTriggerNamesReader import getSMK,getTriggerNames
import TrigConfStorage.FastBlobReader
CountRec = namedtuple('CountRec','ChainCounter Input AfterPrecale Raw Total')
LBI = namedtuple('LBI','lb starttime duration')
# timers are switched on by setting env ARQDEBUG=1
def getL2Counts(runs,pattern=None):
return __getCounts(runs=runs,pattern=pattern, levels=0x1)
def getEFCounts(runs,pattern=None):
return __getCounts(runs=runs,pattern=pattern, levels=0x2)
def getCounts(runs,pattern=None):
return __getCounts(runs=runs,pattern=pattern, levels=0x3)
class TriggerRates:
def __init__(self,level,triggernames,rates):
self._level = level
self._rates = rates
self._triggers = dict([(tn.name,tn) for tn in triggernames])
for tn in triggernames:
self._rates[tn.counter].triggername=tn.name
def __repr__(self):
return "Trigger rates for %s\n%r" % (self._level,self._rates)
def triggers(self):
"""list of all triggers that match pattern"""
return self._triggers
def triggernames(self):
"""names of all triggers that match pattern"""
return self._triggers.keys()
def triggercounters(self):
"""counters of all triggers where the name matches pattern"""
return self._rates.keys()
def __getitem__(self,k):
"""access to rates for any trigger, where k can be the trigger name or counter"""
if type(k)==int:
return self._rates[k]
if type(k)==str:
return self._rates[self._triggers[k].counter]
class TriggerRate:
def __init__(self):
self._triggername = "not existing"
self._rates = []
self._totalcount = 0
self._totalduration = 0
def add(self,rate):
idx = rate.lb()-1 # LB 0 doesn't exist
self._rates.extend( (1+idx-len(self._rates))*[None])
if self._rates[idx] != None:
raise RuntimeError("Found twice data for the same trigger and same LB")
self._rates[idx] = rate
self._totalcount += rate.counts(0)
self._totalduration += rate.duration()
def __repr__(self):
return "%s (%i/%1.fsec)" % (self._triggername,self._totalcount,self._totalduration)
def rates(self):
return self._rates
class Rate:
def __init__(self,lbinfo,countrecord):
self._lbinfo = lbinfo
self._countrecord = countrecord
def __str__(self):
return "%r" % self
def __repr__(self):
if self._countrecord:
return "%1.fHz (%i/%1.fs)" % (self.rate(), self.counts(), self._lbinfo.duration)
return "0Hz (0L1A/%1.fs)" % self._lbinfo.duration
def rate(self,type=0):
return self.counts(type) / self._lbinfo.duration
def lb(self):
return self._lbinfo.lb
def duration(self):
return self._lbinfo.duration
def counts(self,type=0):
if self._countrecord:
return self._countrecord.Total
return 0
def hasL1A(self):
return self._countrecord != None
def __findPos(chaincounters, buf):
counterpositions = []
ndata = len(buf)/20
for pos in xrange(ndata):
entryCounter = unpack_from('I', buf, 20*pos)[0] # get counter that is at position 'pos'
if entryCounter in chaincounters:
counterpositions += [(entryCounter,pos)]
return counterpositions
def __decodeCounts(pos, buf):
if not pos or 20*pos>=len(buf): return None
return CountRec._make( unpack_from('IIIII', buf, 20*pos) )
def __printCounts(buf):
ndata = len(buf)/20
print unpack(ndata*'IIIII', buf)
def _getLBInfo(runlist, folder):
lbinfo = {}
with timer("read the lbinfo"):
for run in runlist:
start = run<<32
end = ((run+1)<<32)-1
objs = folder.browseObjects(start,end,cool.ChannelSelection(0,0,cool.ChannelSelection.sinceBeforeChannel))
lbinfo[run] = [0]+[ LBI(o.since()&0xFFFFFFFF, int(o.payload()['StartTime']/1.e9), int( (o.payload()['EndTime']-o.payload()['StartTime'])/1.e9 ) ) for o in objs ]
return lbinfo
def __getCounts(runs,pattern=None, levels=0x3):
if type(runs)==int: runs = [runs]
smks = getSMK(runs)
triggernames = getTriggerNames(smks.values(),pattern)
return __getCountsForSMKandTrigNames(smks,triggernames, levels)
def __getCountsForSMKandTrigNames(smkByRun, triggerNames, levels):
doL2 = (levels & 1) != 0
doEF = (levels & 2) != 0
if doL2:
l2Names = {}
l2counts = {}
if doEF:
efNames = {}
efcounts = {}
runlist = sorted(smkByRun.keys())
smks = set(smkByRun.values())
if doL2:
for smk in smks:
l2Names[smk] = [tn for tn in triggerNames[smk] if tn.level=='L2']
if doEF:
for smk in smks:
efNames[smk] = [tn for tn in triggerNames[smk] if tn.level=='EF']
from CoolConvUtilities.AtlCoolLib import indirectOpen
conn = indirectOpen( "COOLONL_TRIGGER/COMP200", True, True, False)
if doL2: l2folder=conn.getFolder("/TRIGGER/LUMI/LVL2COUNTERS")
if doEF: effolder=conn.getFolder("/TRIGGER/LUMI/EFCOUNTERS")
lbfolder=conn.getFolder("/TRIGGER/LUMI/LBLB")
lbinfo = _getLBInfo(runlist, lbfolder)
for run in runlist:
smk = smkByRun[run]
if doL2:
l2names_thissmk = l2Names[smk]
if l2names_thissmk:
l2counts[run] = TriggerRates("L2", l2names_thissmk, _getCountsInFolder(run, -1, [tn.counter for tn in l2names_thissmk], l2folder, lbinfo[run]))
if doEF:
efnames_thissmk = efNames[smk]
if efnames_thissmk:
efcounts[run] = TriggerRates("EF", efnames_thissmk, _getCountsInFolder(run, -1, [tn.counter for tn in efnames_thissmk], effolder, lbinfo[run]))
if levels&3==1:
return l2counts
elif levels&3==2:
return efcounts
elif levels&3==3:
return l2counts, efcounts
return None
def _getCountsInFolder(run, lb, chainCounters, folder, lbinfo):
"""
returns a list of Run/LB indexed CountRec objects
if lb<0 then the list contains the entire run, otherwise it has just one entry for the given run/lb
"""
with timer("access COOL"):
if lb>=0:
runlb = ( run<<32 ) + lb
objs = [ folder.findObject(runlb,0) ]
else:
start = run<<32
end = ((run+1)<<32)-1
objs = folder.browseObjects(start,end,cool.ChannelSelection(0,0,cool.ChannelSelection.sinceBeforeChannel))
res = defaultdict(TriggerRate)
pos = None
nocounts = []
with timer("read the blob for run %i" % run):
sl = [ (o.since(), o.payload()['Data'].read()) for o in objs ]
with timer("unpack"):
for (runlb,s) in sl:
#run = runlb>>32
lb = runlb&0xFFFFFFFF
lbinfo_thislb = lbinfo[lb]
if lbinfo_thislb.lb != lb:
print "LBInfo",lbinfo_thislb,"is at the wrong place in the array",lb
sys.exit(1)
if len(s)==0:
nocounts += [ Rate(lbinfo_thislb, None) ]
else:
if len(s)%20 != 0:
raise RuntimeError, "the length of the data vector %i is not a multiple of 20" % len(s)
#__printCounts(s)
if pos==None:
counterpositions = __findPos(chainCounters, s)
for counter,pos in counterpositions:
res[counter].add( Rate(lbinfo_thislb, __decodeCounts(pos,s)) )
for trigrates in res.values():
for r in nocounts:
trigrates.add( r )
return res
if __name__=="__main__":
run = 166786
triggerpattern = "L2_2e"
l2rates, efrates = getCounts(run, triggerpattern)
print l2rates[run].triggernames()
for r in l2rates[run]['L2_2e5_tight'].rates():
#print "LB = %3i: %r" % (r.lb(),r)
print "LB=",r.lb(),", rate=",r.rate(),", counts=",r.counts(),", duration=",r.duration()
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
from ROOT import TGraph, TH1F
def plot(runlist, countsl2,countsef):
print "Start plotting"
for run in runlist:
print "Run",run
example = countsef[run]
gr = TGraph( len(example) )
#gr.SetName ( "graph%i" % (ik+1) )
#gr.SetTitle( varnames[ik] )
for i, (lbinfo, lbcounts) in enumerate(example):
total = lbcounts.Total if lbcounts else 0
print "%i/%i: %1.1f Hz" % (run, lbinfo.lb, float(total)/lbinfo.duration)
gr.SetPoint( i, lbinfo.lb, total )
#gr.Draw("")
h = TH1F("h","h",10,0,10)
h.Fill(5)
h.Draw()
#!/usr/bin/env python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
import sys
from collections import namedtuple
from PyCool import cool
from CoolRunQuery.AtlRunQueryTimer import timer
def getDataPeriods():
runlistdir = '/afs/cern.ch/atlas/www/GROUPS/DATAPREPARATION/DataPeriods'
d = {}
for p in range(ord('A'),ord('J')):
fh = open('%s/data10_7TeV.period%c.runs.list' % (runlistdir,chr(p)) )
lines = fh.readlines()
d[chr(p)] = [int(l.rstrip('\n')) for l in lines]
fh.close()
return d
def getSMK(runlist):
from CoolConvUtilities.AtlCoolLib import indirectOpen
conn = indirectOpen( "COOLONL_TRIGGER/COMP200", True, True, False)
folder = conn.getFolder("/TRIGGER/HLT/HltConfigKeys")
smk = {}
for run in runlist:
runlb = ( run<<32 )
obj = folder.findObject(runlb,0)
smk[run] = obj.payload()['MasterConfigurationKey']
return smk
def executeQuery(cursor, output, condition, schemaname, tables, bindvars=(), order=None):
from TrigConfigSvc.TrigConfigSvcUtils import getUsedTables
s_output = ', '.join(output)
s_tables = ', '.join(getUsedTables(output, condition, schemaname, tables))
s_condition = ' and '.join(condition)
s_order = 'order by %s' % order if order else ''
query = 'select distinct %s from %s where %s %s' % \
(s_output, s_tables, s_condition, s_order )
#print (str(query))
if len(bindvars)==0:
cursor.execute(str(query))
else:
cursor.execute(str(query),bindvars)
return cursor.fetchall()
def getTriggerNames(smks, pattern=None):
if type(smks)==int: smks=set([smks])
if type(smks)==list: smks=set(smks)
from TrigConfigSvc.TrigConfigSvcUtils import getTriggerDBCursor
cursor,schemaname = getTriggerDBCursor("TRIGGERDB")
if not '.' in schemaname: schemaname += '.'
tn = {}
tables = {
'SM' : 'SUPER_MASTER_TABLE',
'HM' : 'HLT_MASTER_TABLE',
'TM' : 'HLT_TRIGGER_MENU',
'M2C': 'HLT_TM_TO_TC',
'TC' : 'HLT_TRIGGER_CHAIN'
}
condition = [
'SM.SMT_ID = :smk',
'SM.SMT_HLT_MASTER_TABLE_ID = HM.HMT_ID',
'HM.HMT_TRIGGER_MENU_ID = M2C.HTM2TC_TRIGGER_MENU_ID',
'M2C.HTM2TC_TRIGGER_CHAIN_ID = TC.HTC_ID'
]
if pattern:
condition += [ "TC.HTC_NAME like '%%%s%%'" % pattern ]
output = ['TC.HTC_NAME', 'TC.HTC_L2_OR_EF', 'TC.HTC_CHAIN_COUNTER']
order = 'TC.HTC_L2_OR_EF desc, TC.HTC_NAME'
TC = namedtuple('TC','name level counter')
for smk in smks:
bindvars = { "smk": smk }
res = executeQuery(cursor, output, condition, schemaname, tables, bindvars, order)
tn[smk] = [TC._make(r) for r in res]
return tn
if __name__=="__main__":
dp= getDataPeriods()
#smkByRun = getSMK(dp['G'])
#smks = set(smkByRun.values())
print getTriggerNames(910,'EF_tauNo')
#!/usr/bin/env python
if __name__=="__main__":
"""example how to get the HLT rates from COOL"""
runs = [165815, 165817]
triggerpattern = "L2_e5"
from TrigStatsFromCool.HLTRatesCoolReader import getCounts, getL2Counts, getEFCounts
# getCounts returns L2 and EF rates
l2rates, efrates = getCounts(runs, triggerpattern)
# one can also do just one level (faster)
# l2rates = getL2Counts(runs, triggerpattern)
for run in runs:
print "Run",run,"with triggers",l2rates[run].triggernames()
for r in l2rates[run]['L2_e5_medium'].rates():
print "LB=",r.lb(),", rate=",r.rate(),"Hz, counts=",r.counts(),", duration=",r.duration(),"sec","*" if not r.hasL1A() else ""
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment