'CMakeLists.txt' (TrigStatsFromCool-00-00-02-01)

Trigger/TrigMonitoring/TrigStatsFromCool/CMakeLists.txt

+################################################################################
+# Package: TrigStatsFromCool
+################################################################################
+
+# Declare the package name:
+atlas_subdir( TrigStatsFromCool )
+
+# Install files from the package:
+atlas_install_python_modules( python/*.py )
+

Trigger/TrigMonitoring/TrigStatsFromCool/cmt/Makefile

+include $(CMTROOT)/src/Makefile.header
+
+include $(CMTROOT)/src/constituents.make
+

Trigger/TrigMonitoring/TrigStatsFromCool/cmt/requirements

+package TrigStatsFromCool
+
+author  Joerg.Stelzer@cern.ch
+manager Joerg.Stelzer@cern.ch
+#============================================================================
+
+#============================================================================
+# Public dependencies
+#============================================================================
+use DetCommonPolicy *
+
+private
+
+##  use POOL v* LCG_Interfaces
+##  use COOL v* LCG_Interfaces
+##  use CORAL v* LCG_Interfaces
+##  use XercesC v* LCG_Interfaces
+##  use Reflex v* LCG_Interfaces
+##  use Boost v* LCG_Interfaces
+##  
+##  use TrigConfL1Data      TrigConfL1Data-*     Trigger/TrigConfiguration
+##  use TrigConfJobOptData  TrigConfJobOptData-* Trigger/TrigConfiguration
+##  use TrigConfHLTData     TrigConfHLTData-*    Trigger/TrigConfiguration
+
+#============================================================================
+# Build rules
+#============================================================================
+
+##  #
+##  # ignore patterns from LCG_Policy
+##  #
+##  apply_pattern detcommon_ignores
+##  
+##  # link header files into InstallArea, and include the corresponding
+##  # directory in the compiler options
+##  #
+##  apply_pattern detcommon_header_installer
+
+#============================================================================
+# Build applications
+#============================================================================
+
+##  apply_pattern trigconf_application name=Lvl1TriggerApp
+##  apply_pattern trigconf_application name=Lvl1TriggerApp2
+##  #apply_pattern trigconf_application name=JobOptTriggerApp
+##  apply_pattern trigconf_application name=2COOLApp
+##  apply_pattern trigconf_application name=2XMLApp
+##  apply_pattern trigconf_application name=PrescaleCollLoad
+##  #apply_pattern trigconf_application name=TestLogicParser
+##  #apply_pattern trigconf_application name=HltApp
+##  apply_pattern trigconf_application name=ConsistencyChecker
+##  apply_pattern trigconf_application name=RttTest
+
+#============================================================================
+# Install scripts and modules
+#============================================================================
+
+##  apply_pattern detcommon_link_files kind=scripts name=python_scripts files="\
+##  	-s=../scripts *.py" prefix=share/bin
+##  
+
+apply_pattern detcommon_link_files kind=python_modules name=python_mods files="\
+        -s=../python *.py" prefix=python/$(package)

Trigger/TrigMonitoring/TrigStatsFromCool/python/HLTRatesCoolReader.py

+#!/usr/bin/env python
+
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
+import sys
+from collections import namedtuple,defaultdict
+from struct import unpack_from, unpack
+from PyCool import cool
+from CoolRunQuery.AtlRunQueryTimer import timer
+from TrigStatsFromCool.HLTTriggerNamesReader import getSMK,getTriggerNames
+import TrigConfStorage.FastBlobReader
+
+
+CountRec = namedtuple('CountRec','ChainCounter Input AfterPrecale Raw Total')
+LBI      = namedtuple('LBI','lb starttime duration')
+
+# timers are switched on by setting env ARQDEBUG=1
+
+
+def getL2Counts(runs,pattern=None):
+    return __getCounts(runs=runs,pattern=pattern, levels=0x1)
+
+def getEFCounts(runs,pattern=None):
+    return __getCounts(runs=runs,pattern=pattern, levels=0x2)
+
+def getCounts(runs,pattern=None):
+    return __getCounts(runs=runs,pattern=pattern, levels=0x3)
+
+
+
+class TriggerRates:
+    def __init__(self,level,triggernames,rates):
+        self._level    = level
+        self._rates    = rates
+        self._triggers = dict([(tn.name,tn) for tn in triggernames])
+        for tn in triggernames:
+            self._rates[tn.counter].triggername=tn.name
+
+    def  __repr__(self):
+        return "Trigger rates for %s\n%r" % (self._level,self._rates)
+
+    def triggers(self):
+        """list of all triggers that match pattern"""
+        return self._triggers
+
+    def triggernames(self):
+        """names of all triggers that match pattern"""
+        return self._triggers.keys()
+
+    def triggercounters(self):
+        """counters of all triggers where the name matches pattern"""
+        return self._rates.keys()
+
+    def __getitem__(self,k):
+        """access to rates for any trigger, where k can be the trigger name or counter"""
+        if type(k)==int:
+            return self._rates[k]
+        if type(k)==str:
+            return self._rates[self._triggers[k].counter]
+        
+
+    
+class TriggerRate:
+    def __init__(self):
+        self._triggername = "not existing"
+        self._rates = []
+        self._totalcount = 0
+        self._totalduration = 0
+        
+    def add(self,rate):
+        idx = rate.lb()-1 # LB 0 doesn't exist
+        self._rates.extend( (1+idx-len(self._rates))*[None])
+        if self._rates[idx] != None:
+            raise RuntimeError("Found twice data for the same trigger and same LB")
+        self._rates[idx] = rate
+        self._totalcount += rate.counts(0)
+        self._totalduration += rate.duration()
+
+    def  __repr__(self):
+        return "%s (%i/%1.fsec)" % (self._triggername,self._totalcount,self._totalduration)
+
+    def rates(self):
+        return self._rates
+
+
+
+class Rate:
+    def __init__(self,lbinfo,countrecord):
+        self._lbinfo = lbinfo
+        self._countrecord = countrecord
+
+    def __str__(self):
+        return "%r" % self
+
+    def __repr__(self):
+        if self._countrecord:
+            return "%1.fHz (%i/%1.fs)" % (self.rate(), self.counts(), self._lbinfo.duration)
+        return "0Hz (0L1A/%1.fs)" % self._lbinfo.duration
+
+    def rate(self,type=0):
+        return self.counts(type) / self._lbinfo.duration
+
+    def lb(self):
+        return self._lbinfo.lb
+
+    def duration(self):
+        return self._lbinfo.duration
+
+    def counts(self,type=0):
+        if self._countrecord:
+            return self._countrecord.Total
+        return 0
+
+    def hasL1A(self):
+        return self._countrecord != None
+
+
+
+
+
+
+def __findPos(chaincounters, buf):
+    counterpositions = []
+    ndata = len(buf)/20
+    for pos in xrange(ndata):
+        entryCounter = unpack_from('I', buf, 20*pos)[0]  # get counter that is at position 'pos'
+        if entryCounter in chaincounters:
+            counterpositions += [(entryCounter,pos)]
+    return counterpositions
+
+
+
+def __decodeCounts(pos, buf):
+    if not pos or 20*pos>=len(buf): return None
+    return CountRec._make( unpack_from('IIIII', buf, 20*pos) )
+
+
+
+def __printCounts(buf):
+    ndata = len(buf)/20
+    print unpack(ndata*'IIIII', buf)
+
+
+def _getLBInfo(runlist, folder):
+    lbinfo = {}
+    with timer("read the lbinfo"):
+        for run in runlist:
+            start = run<<32
+            end   = ((run+1)<<32)-1
+            objs = folder.browseObjects(start,end,cool.ChannelSelection(0,0,cool.ChannelSelection.sinceBeforeChannel))
+            lbinfo[run] = [0]+[ LBI(o.since()&0xFFFFFFFF, int(o.payload()['StartTime']/1.e9), int( (o.payload()['EndTime']-o.payload()['StartTime'])/1.e9 ) ) for o in objs ]
+    return lbinfo
+    
+    
+def __getCounts(runs,pattern=None, levels=0x3):
+
+    if type(runs)==int: runs = [runs]
+    smks = getSMK(runs)
+
+    triggernames = getTriggerNames(smks.values(),pattern)
+
+    return __getCountsForSMKandTrigNames(smks,triggernames, levels)
+    
+
+def __getCountsForSMKandTrigNames(smkByRun, triggerNames, levels):
+
+    doL2 = (levels & 1) != 0
+    doEF = (levels & 2) != 0
+
+    if doL2:
+        l2Names = {}
+        l2counts = {}
+
+    if doEF:
+        efNames = {}
+        efcounts = {}
+
+    runlist = sorted(smkByRun.keys())
+    smks = set(smkByRun.values())
+    if doL2:
+        for smk in smks:
+            l2Names[smk] = [tn for tn in triggerNames[smk] if tn.level=='L2']
+    if doEF:
+        for smk in smks:
+            efNames[smk] = [tn for tn in triggerNames[smk] if tn.level=='EF']
+    
+    from CoolConvUtilities.AtlCoolLib  import indirectOpen
+    conn = indirectOpen( "COOLONL_TRIGGER/COMP200", True, True, False)
+    if doL2: l2folder=conn.getFolder("/TRIGGER/LUMI/LVL2COUNTERS")
+    if doEF: effolder=conn.getFolder("/TRIGGER/LUMI/EFCOUNTERS")
+    lbfolder=conn.getFolder("/TRIGGER/LUMI/LBLB")
+
+    lbinfo = _getLBInfo(runlist, lbfolder)
+
+    for run in runlist:
+        smk = smkByRun[run]
+        if doL2:
+            l2names_thissmk = l2Names[smk]
+            if l2names_thissmk:
+                l2counts[run] = TriggerRates("L2", l2names_thissmk, _getCountsInFolder(run, -1, [tn.counter for tn in l2names_thissmk], l2folder, lbinfo[run]))
+        if doEF:
+            efnames_thissmk = efNames[smk]
+            if efnames_thissmk:
+                efcounts[run] = TriggerRates("EF", efnames_thissmk, _getCountsInFolder(run, -1, [tn.counter for tn in efnames_thissmk], effolder, lbinfo[run]))
+
+    if levels&3==1:
+        return l2counts
+    elif levels&3==2:
+        return efcounts
+    elif levels&3==3:
+        return l2counts, efcounts
+    return None
+
+
+def _getCountsInFolder(run, lb, chainCounters, folder, lbinfo):
+    """
+    returns a list of Run/LB indexed CountRec objects
+    if lb<0 then the list contains the entire run, otherwise it has just one entry for the given run/lb
+    """
+
+    with timer("access COOL"):
+        if lb>=0:
+            runlb = ( run<<32 ) + lb
+            objs = [ folder.findObject(runlb,0) ]
+        else:
+            start = run<<32
+            end   = ((run+1)<<32)-1
+            objs = folder.browseObjects(start,end,cool.ChannelSelection(0,0,cool.ChannelSelection.sinceBeforeChannel))
+
+    res = defaultdict(TriggerRate)
+    pos = None
+    nocounts = []
+
+    with timer("read the blob for run %i" % run):
+        sl = [ (o.since(), o.payload()['Data'].read()) for o in objs ]
+
+    with timer("unpack"):
+        for (runlb,s) in sl:
+            #run = runlb>>32
+            lb = runlb&0xFFFFFFFF
+
+            lbinfo_thislb = lbinfo[lb]
+            if lbinfo_thislb.lb != lb:
+                print "LBInfo",lbinfo_thislb,"is at the wrong place in the array",lb
+                sys.exit(1)
+
+            if len(s)==0:
+                nocounts += [ Rate(lbinfo_thislb, None) ]
+            else:
+                if len(s)%20 != 0:
+                    raise RuntimeError, "the length of the data vector %i is not a multiple of 20" % len(s)
+
+                #__printCounts(s)
+                if pos==None:
+                    counterpositions = __findPos(chainCounters, s)
+
+                for counter,pos in counterpositions:
+                    res[counter].add( Rate(lbinfo_thislb, __decodeCounts(pos,s)) )
+                    
+        for trigrates in res.values():
+            for r in nocounts:
+                trigrates.add( r )
+    return res
+
+
+if __name__=="__main__":
+
+    run = 166786
+    triggerpattern = "L2_2e"
+    
+    l2rates, efrates = getCounts(run, triggerpattern)
+
+    print l2rates[run].triggernames()
+
+    for r in l2rates[run]['L2_2e5_tight'].rates():
+        #print "LB = %3i: %r" % (r.lb(),r)
+        print "LB=",r.lb(),", rate=",r.rate(),", counts=",r.counts(),", duration=",r.duration()
+    

Trigger/TrigMonitoring/TrigStatsFromCool/python/HLTRatesPlotter.py

+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
+
+from ROOT import TGraph, TH1F
+
+def plot(runlist, countsl2,countsef):
+
+    print "Start plotting"
+
+    for run in runlist:
+
+        print "Run",run
+
+        example = countsef[run]
+
+
+        gr = TGraph( len(example) )
+        #gr.SetName ( "graph%i" % (ik+1) )
+        #gr.SetTitle( varnames[ik] )
+
+        for i, (lbinfo, lbcounts) in enumerate(example):
+            total = lbcounts.Total if lbcounts else 0
+            print "%i/%i: %1.1f Hz" % (run, lbinfo.lb, float(total)/lbinfo.duration)
+
+            gr.SetPoint( i, lbinfo.lb, total )
+
+
+
+        #gr.Draw("")
+
+        h = TH1F("h","h",10,0,10)
+        h.Fill(5)
+        h.Draw()

Trigger/TrigMonitoring/TrigStatsFromCool/python/HLTTriggerNamesReader.py

+#!/usr/bin/env python
+
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
+import sys
+from collections import namedtuple
+from PyCool import cool
+from CoolRunQuery.AtlRunQueryTimer import timer
+
+def getDataPeriods():
+    runlistdir = '/afs/cern.ch/atlas/www/GROUPS/DATAPREPARATION/DataPeriods'
+    d = {}
+    for p in range(ord('A'),ord('J')):
+        fh = open('%s/data10_7TeV.period%c.runs.list' % (runlistdir,chr(p)) )
+        lines = fh.readlines()
+        d[chr(p)] = [int(l.rstrip('\n')) for l in lines]
+        fh.close()
+    return d
+
+
+def getSMK(runlist):
+    from CoolConvUtilities.AtlCoolLib  import indirectOpen
+    conn = indirectOpen( "COOLONL_TRIGGER/COMP200", True, True, False)
+    folder = conn.getFolder("/TRIGGER/HLT/HltConfigKeys")
+
+    smk = {}
+
+    for run in runlist:
+        runlb = ( run<<32 )
+        obj = folder.findObject(runlb,0)
+        smk[run] = obj.payload()['MasterConfigurationKey']
+
+    return smk
+
+
+def executeQuery(cursor, output, condition, schemaname, tables, bindvars=(), order=None):
+    from TrigConfigSvc.TrigConfigSvcUtils import getUsedTables
+
+    s_output = ', '.join(output)
+    s_tables = ', '.join(getUsedTables(output, condition, schemaname, tables))
+    s_condition = ' and '.join(condition)
+    s_order = 'order by %s' % order if order else ''
+
+    
+    query = 'select distinct %s from %s where %s %s' % \
+            (s_output, s_tables, s_condition, s_order )
+    #print (str(query))
+    if len(bindvars)==0:
+        cursor.execute(str(query))
+    else:
+        cursor.execute(str(query),bindvars)
+    return cursor.fetchall()
+
+
+def getTriggerNames(smks, pattern=None):
+
+    if type(smks)==int:  smks=set([smks])
+    if type(smks)==list: smks=set(smks)
+
+    from TrigConfigSvc.TrigConfigSvcUtils import getTriggerDBCursor
+    cursor,schemaname = getTriggerDBCursor("TRIGGERDB")
+
+    if not '.' in schemaname: schemaname += '.'
+
+    tn = {}
+
+    tables = {
+        'SM' : 'SUPER_MASTER_TABLE',
+        'HM' : 'HLT_MASTER_TABLE',
+        'TM' : 'HLT_TRIGGER_MENU',
+        'M2C': 'HLT_TM_TO_TC',
+        'TC' : 'HLT_TRIGGER_CHAIN'
+        }
+
+    condition = [
+        'SM.SMT_ID = :smk',
+        'SM.SMT_HLT_MASTER_TABLE_ID = HM.HMT_ID',
+        'HM.HMT_TRIGGER_MENU_ID = M2C.HTM2TC_TRIGGER_MENU_ID',
+        'M2C.HTM2TC_TRIGGER_CHAIN_ID = TC.HTC_ID'
+        ]
+
+    if pattern:
+        condition += [ "TC.HTC_NAME like '%%%s%%'" % pattern ]
+
+    output = ['TC.HTC_NAME', 'TC.HTC_L2_OR_EF', 'TC.HTC_CHAIN_COUNTER']
+
+    order = 'TC.HTC_L2_OR_EF desc, TC.HTC_NAME'
+
+    TC = namedtuple('TC','name level counter')
+
+    for smk in smks:
+        bindvars = { "smk": smk }
+
+        res = executeQuery(cursor, output, condition, schemaname, tables, bindvars, order)
+
+        tn[smk] = [TC._make(r) for r in res]
+
+    return tn
+
+    
+
+if __name__=="__main__":
+    dp= getDataPeriods()
+
+    #smkByRun = getSMK(dp['G'])
+
+    #smks = set(smkByRun.values())
+
+    print getTriggerNames(910,'EF_tauNo')

Trigger/TrigMonitoring/TrigStatsFromCool/share/Example_ReadHLTRatesFromCool.py

+#!/usr/bin/env python
+
+if __name__=="__main__":
+
+    """example how to get the HLT rates from COOL"""
+
+    runs = [165815, 165817]
+    triggerpattern = "L2_e5"
+
+    from TrigStatsFromCool.HLTRatesCoolReader import getCounts, getL2Counts, getEFCounts
+
+    # getCounts returns L2 and EF rates
+    l2rates, efrates = getCounts(runs, triggerpattern)
+
+    # one can also do just one level (faster)
+    # l2rates = getL2Counts(runs, triggerpattern)
+
+    for run in runs:
+        print "Run",run,"with triggers",l2rates[run].triggernames()
+
+        for r in l2rates[run]['L2_e5_medium'].rates():
+            print "LB=",r.lb(),", rate=",r.rate(),"Hz, counts=",r.counts(),", duration=",r.duration(),"sec","*" if not r.hasL1A() else ""
+