Skip to content
Snippets Groups Projects
Commit abd44752 authored by Frank Winklmeier's avatar Frank Winklmeier
Browse files

TrigConfOffline: Remove obsolete code to run trigger from DB

Remove obsolete code that is responsible for reading the trigger
configuration from the TriggerDB, applying transformation rules and
running an offline-type trigger job within athena.py. This feature was
never used and instead we ended up only reading the trigger menu part
from the DB.
parent e9b815f6
No related merge requests found
Showing
with 3 additions and 10159 deletions
......@@ -36,5 +36,3 @@ atlas_add_dictionary( TrigConfStorageDict
# Install files from the package:
atlas_install_headers( TrigConfOffline )
atlas_install_python_modules( python/*.py )
atlas_install_scripts( share/LoadTriggerMenuFromXML.py )
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
# $Id: HLTSQLUtils.py,v 1.7 2009/03/02 16:25:20 nozicka Exp $
# Created by Miroslav Nozicka <nozicka@mail.desy.de>
def get_cursor (host, user, db="", type="mysql", ask_passwd=True, passwd=""):
""" Gets a python DB cursor for a given triple: db, host, user. The passwd
flag can be used to trigger a little trigger interactive password request to
the user. The type can be either 'mysql' or 'oracle' and that will return one
of the two different cursor implementations. """
if type.lower() == "mysql":
return _get_mysql_cursor(host, db, user, ask_passwd, passwd)
elif type.lower() == "oracle":
return _get_oracle_cursor(host, user, ask_passwd, passwd)
elif type.lower() == "sqlite":
return _get_sqlite_cursor(host)
def _get_oracle_cursor (tns, user, ask_passwd=True, passwd=""):
"""Returns an Oracle cursor"""
try:
from cx_Oracle import connect
except ImportError:
print "ERROR: Have you setup your Oracle environment correctly?"
print "You have to set these enviroment variables to make it work:"
print "LD_LIBRARY_PATH: include the directory of the client libraries"
print "TNS_ADMIN: include the TNS definitions %s" % \
"(e.g.: /afs/cern.ch/project/oracle/admin)"
sys.exit(2)
if ask_passwd:
from getpass import getpass
passwd = getpass("[Oracle] database password for %s@%s: " % (user, tns))
connection = connect (user, passwd, tns, threaded=True)
else:
connection = connect (user, passwd, tns, threaded=True)
return connection.cursor()
def _get_mysql_cursor (host, db, user, ask_passwd=True, passwd=""):
"""Returns a MySQL cursor"""
from MySQLdb import connect
from getpass import getpass
if ask_passwd:
passwd = getpass("[MySQL] `%s' database password for %s@%s: " %
(db, user, host))
connection = connect (host=host, user=user, passwd=passwd, db=db, connect_timeout=10)
return connection.cursor()
def _get_sqlite_cursor(file):
"""Returns a SQLite cursor"""
from sqlite3 import dbapi2 as sqlite
connection = sqlite.connect(file)
return connection.cursor()
This diff is collapsed.
###########################################################################
# Copyright (C) 2009 by Miroslav Nozicka
# <nozicka@mail.desy.de>
# Copyright (C) 2002-2018 CERN for the benefit of the ATLAS collaboration
#
# DESCRIPTION: TrigHistorySvc - derived configurable of the HistorySvc
# - extends the functionality of the HistorySvc obtaining
# the python class (py_name) and python module (py_package)
# of the algorithm
#
###########################################################################
from AthenaPython.PyAthena import Alg, StatusCode
......
###########################################################################
# Copyright (C) 2009 by Miroslav Nozicka
# <nozicka@mail.desy.de>
# Copyright (C) 2002-2018 CERN for the benefit of the ATLAS collaboration
#
# DESCRIPTION: TrigHistorySvc - derived configurable of the HistorySvc
# - extends the functionality of the HistorySvc obtaining
# the python class (py_name) and python module (py_package)
# of the algorithm
#
###########################################################################
__all__ = [ 'dump'
......
###########################################################################
# Copyright (C) 2009 by Miroslav Nozicka
# <nozicka@mail.desy.de>
# Copyright (C) 2002-2018 CERN for the benefit of the ATLAS collaboration
#
# DESCRIPTION: TrigHistorySvc - derived configurable of the HistorySvc
# - extends the functionality of the HistorySvc obtaining
# the python class (py_name) and python module (py_package)
# of the algorithm
#
###########################################################################
from AthenaCommon import CfgMgr
......
This diff is collapsed.
#!/usr/bin/env python
#from TrigConfOffline import *
#import TrigConfOffline as tc
from TrigConfOffline.menuloader import *
hltmenu = hltmenuloader("HLTconfig_Physics_pp_v4_18.1.0.xml").menu
c = hltmenu.chains()[0]
s = hltmenu.sequences()[0]
print c
#for c in (x for x in hltmenu.chains() if '4j45' in x.name()):
# print c.name()
l1menu = l1menuloader("LVL1config_Physics_pp_v4.xml").menu
i = l1menu.menu().items()[0]
print i
#for item in l1menu.menu().items():
# print item.name()
#!/usr/bin/env python
import getopt, sys, os, time
import user
import shelve
inFileName = 'trigger_db.pickle'
print ">>> opening py-shelve [%s]..." % inFileName
hlt_config_db = shelve.open(inFileName)['config']
print ">>> found [%i] configurables" % len(hlt_config_db)
def is_nested(c):
return c.count('.') > 0
#def cnv_prop():
import GaudiKernel.GaudiHandles as GaudiHandles
from AthenaCommon import CfgMgr
from AthenaCommon.Configurable import (ConfigurableAlgorithm,
ConfigurableService,
ConfigurableAlgTool,
ConfigurableAuditor)
# load configurable db and create the ToolSvc
toolSvc = CfgMgr.ToolSvc()
from AthenaCommon.ConfigurableDb import getConfigurable,cfgDb
from AthenaCommon.AppMgr import ToolSvc,ServiceMgr,theApp
from AthenaCommon.AlgSequence import AlgSequence
topSequence = AlgSequence()
algs = []
services = []
tools = []
auditors = []
nprivate_algs = 0
# EventSelector
#include( "ByteStreamCnvSvc/BSEventStorageEventSelector_jobOptions.py" )
# Geometry
#include( "DetDescrDictionary/DetDescrDictionaryDict_joboptions.py" )
#include ("AtlasGeoModel/SetGeometryVersion.py")
# Get the Sorted list of DB Components
names = hlt_config_db.keys()
# Exclude the ApplicationMgr
del names[names.index('ApplicationMgr')]
# Sort the Components (alg1 > alg1.some) in order to initialize the Parent algorithms first
names.sort()
for name in names:
c = hlt_config_db[name]
cfg_name = name
cfg_type = c['type']
if name.count('.') > 0:
cfg_type = cfg_type.split('.')[-1]
if 'ToolSvc.' in name :
cfg_name = name.replace('ToolSvc.','')
#cfg_name = cfg_name.split('.')[-1]
#print 'Try to get Configurable:', cfg_type, ':\t %s/%s' % (c['type'],name)
cfg_class = getConfigurable(cfg_type, assumeCxxClass=True)
if not cfg_class :
print 'WARNING: Configurable %s/%s (%s/%s) not found' % (cfg_type, cfg_name, c['type'],name)
continue
## create an instance with the correct name
stmt = 'cfg = CfgMgr.%(cfg_class)s( "%(cfg_name)s", %(properties)s)'
properties = []
for n,v in c['properties'].items():
try:
dflt_value = cfg_class.__slots__[n]
except KeyError,err:
print "::ERROR:: configurable [%s] has no such property [%s] !!"%(
cfg_class.__name__, n
)
continue
p = '%s = %s'
if isinstance(dflt_value,str):
p = '%s = "%s"'
elif isinstance(dflt_value,GaudiHandles.GaudiHandle):
p = '%s = "%s"'
properties += [ p % (n,v) ]
stmt = stmt % {
'cfg_class' : cfg_class.__name__,
'cfg_name' : cfg_name,
'properties': ', '.join(properties)
}
try:
exec(stmt)
#print 'Configurable:', cfg_type, ':\t %s/%s' % (c['type'],name), 'Succesfully initialized'
#print '\t', cfg
if isinstance(cfg, ConfigurableAlgorithm): algs.append ({'cfg':cfg, 'topalg':c['topalg']})
elif isinstance(cfg, ConfigurableService): services.append(cfg)
elif isinstance(cfg, ConfigurableAlgTool): tools.append (cfg)
elif isinstance(cfg, ConfigurableAuditor): auditors.append(cfg)
else:
print ":::ERROR::: not nested and not an algorithm:",
print c,"(%s)"%type(cfg)
pass
except NameError:
if cfg_name.count('.') > 0 :
alg_name = cfg_name.split('.')[-1]
top_parent_name = cfg_name.split('.')[0]
parent_name = cfg_name[:cfg_name.rfind(alg_name)-1]
# cfg_name = alg_name
top_parent_cfg = None
top_parent_cfg_str = ''
# Search the Algorithms for the parent
if not top_parent_cfg:
count = 0
for a in algs :
if a['cfg'].getName() == top_parent_name :
top_parent_cfg_str = 'algs[%d][\'cfg\']' % (count)
top_parent_cfg = algs[count]['cfg']
count += 1
# Search the Services for the parent
if not top_parent_cfg:
count = 0
for s in services :
if s.getName() == top_parent_name :
top_parent_cfg_str = 'services[%d]' % (count)
top_parent_cfg = services[count]
count += 1
# Search the Tools for the parent
if not top_parent_cfg:
count = 0
for t in tools :
if t.getName() == top_parent_name :
top_parent_cfg_str = 'tools[%d]' % (count)
top_parent_cfg = tools[count]
count += 1
# Search the Auditors for the parent
if not top_parent_cfg:
count = 0
for a in auditors :
if a.getName() == top_parent_name :
top_parent_cfg_str = 'auditors[%d]' % (count)
top_parent_cfg = auditors[count]
count += 1
if top_parent_cfg:
parent_cfg = top_parent_cfg
if parent_name.count('.') > 1 :
for parent in parent_name[len(top_parent_name):].split('.') :
child_cfg = None
for child in parent_cfg.getAllChildren() :
if child.getName() == parent :
child_cfg = child
break
if child_cfg :
parent_cfg = child_cfg
else :
print 'ERROR: ', 'Configurable:', cfg_type, ':\t %s/%s' % (c['type'],name), 'Parent algorithm: %s was not found' % (parent_name)
# Test the existence of the Child Configurable if exists set the properties only
cfg = None
for child in parent_cfg.getAllChildren() :
if child.getName() == alg_name:
cfg = child
break;
#print top_parent_cfg_str, parent_name
#stmt = top_parent_cfg_str + parent_name[len(top_parent_name):]
#print stmt
if not cfg :
stmt = 'parent_cfg += CfgMgr.%(cfg_class)s( "%(cfg_name)s", %(properties)s)'
stmt = stmt % {
'cfg_class' : cfg_class.__name__,
'cfg_name' : alg_name,
'properties': ', '.join(properties)
}
try :
exec(stmt)
nprivate_algs += 1
#print 'Private algorithm %s/%s (%s/%s) successfully initialized with name %s' % (private_alg['cfg_class'], private_alg['name'], c['type'], c['name'], cfg_name)
#print parent_cfg
except NameError:
print 'ERROR: Configurable:', cfg_type, ':\t %s/%s' % (c['type'],name), 'Can not be initialized'
#except AttributeError:
else:
for k,v in c['properties'].items():
try :
v = eval(v)
except :
v = v
stmt = 'cfg.%s = %s' % (k,v)
if isinstance(v,str) :
stmt = 'cfg.%s = \'%s\'' % (k,v)
exec (stmt)
nprivate_algs += 1
else :
print 'ERROR: Configurable:', cfg_type, ':\t %s/%s' % (c['type'],name), 'Parent algorithm: %s was not found' % (parent_name)
else:
print 'ERROR: Configurable:', cfg_type, ':\t %s/%s' % (c['type'],name), 'is not a private or nested algorithm'
pass
## testing
## Sort the algs list such that parent algorithms are before the child algorithms
# means: alg1 < alg1.child => alphabetical sorting of the alg names would do
def alg_compare(a1, a2) :
if a1['topalg'] == a1['cfg'].getFullJobOptName() :
return -1
if a2['topalg'] == a2['cfg'].getFullJobOptName() :
return 1
return cmp(a1['cfg'].getName(), a2['cfg'].getName())
algs.sort(alg_compare)
unbound_algs = []
hlt_app = hlt_config_db['ApplicationMgr']
for a in algs:
alg_full_name = a['cfg'].getFullJobOptName()
if alg_full_name in hlt_app['properties']['TopAlg']:
print 'Top Algorithm Found: ', alg_full_name
topSequence += a['cfg']
elif not a['topalg'] : # In case there is no TopAlg
unbound_algs.append(a['cfg'])
else:
cfg = a['cfg']
stmt = 'topSequence.%s += cfg' % (a['topalg'].split('/')[-1])
exec(stmt)
print "-"*40,'Unbound Algorithms: %d' %(len(unbound_algs)),"-"*40
for a in unbound_algs :
print "\t%s" % (a.getFullJobOptName())
for s in services: ServiceMgr += s
for t in tools: ToolSvc += t
for a in auditors: ServiceMgr.AuditorSvc += a
# then, take care of the application manager
print "-"*80
for n,v in hlt_app['properties'].items():
stmt = 'theApp.%s = %s'
dflt_value = theApp.__slots__[n]
if v == dflt_value:
continue
if isinstance(dflt_value,str): stmt = 'theApp.%s = "%s"'
stmt = stmt % (n,v)
if n=='CreateSvc' :
v = eval(v)
for vv in v:
theApp.CreateSvc += [vv]
print ":::","theApp.CreateSvc += [\'%s\']" % (vv)
else :
try:
exec(stmt)
print ":::",stmt
except Exception,err:
print "::ERROR:: theApp setup:",err
print " ",n,v
print " ",stmt
print ">>> configurables in db:",len(hlt_config_db)
print ">>> algs: ",len(algs)
print ">>> services:",len(services)
print ">>> tools: ",len(tools)
print ">>> auditors:",len(auditors)
print ">>> private algs:", nprivate_algs
print "::: failed to restore [%i] configurables" %(len(hlt_config_db)
- len(algs)
- len(services)
- len(tools)
- len(auditors)
- nprivate_algs)
theApp.setup()
\ No newline at end of file
###########################################################################
# Author: Miroslav Nozicka <nozicka@mail.desy.de>
#
# Usage: athena.py -c "[flags]" testHLT_standaloneDB.py
#
###########################################################################
from TrigConfOffline.HLTConfOffline import HLTConfOffline
hltConfOffline = HLTConfOffline()
# Process the commands
## @var dbSMKeyFlag
# Either SMKey or Run number must be specified in the command option
dbSMKeyFlag = False
try :
hltConfOffline.SMKey = DBSMKey
dbSMKeyFlag = True
except NameError:
pass
runNumberFlag = False
try :
hltConfOffline.RunNumber = Run
runNumberFlag = True
except NameError:
pass
# Raise the Exception if neither SMKey nor Run number specified
if not (runNumberFlag or dbSMKeyFlag) :
raise Exception('Please specify Run or SMKey')
# Process the remaining command options
try :
hltConfOffline.OutputLevel = OutputLevel
except NameError:
pass
try :
hltConfOffline.setupSource = DBConn
except NameError:
pass
try :
hltConfOffline.dbType = DBType.lower()
except NameError:
pass
try :
hltConfOffline.dbHost = DBServer
except NameError:
pass
try :
hltConfOffline.dbUser = DBUser
except NameError:
pass
try :
hltConfOffline.dbPasswd = DBPasswd
except NameError:
pass
try :
hltConfOffline.dbName = DBName
except NameError:
pass
try :
hltConfOffline.rulesSource = Rules
except NameError:
pass
try :
hltConfOffline.HLTPrescaleKey = DBHLTPSKey
except NameError:
pass
try :
hltConfOffline.LVL1PrescaleKey = DBLVL1PSKey
except NameError:
pass
try :
hltConfOffline.Level = Instance
except NameError:
pass
# Load the trigger setup
hltConfOffline.load()
# Set Maximum number of events
try:
theApp.EvtMax = EvtMax
except NameError:
pass
# Set input data file
try:
inputdata = DataFile
if isinstance(inputdata, str) or isinstance(inputdata,unicode) :
inputdatalist = []
inputdatalist.append(inputdata)
inputdata=inputdatalist
svcMgr = theApp.serviceMgr()
svcMgr.ByteStreamInputSvc.FullFileName = inputdata
except NameError:
pass
# Set the output ByteStreamData
try:
outputData = BSOutput
svcMgr = theApp.serviceMgr()
from ByteStreamCnvSvc.ByteStreamCnvSvcConf import ByteStreamEventStorageOutputSvc
svcMgr += ByteStreamEventStorageOutputSvc()
# Properties
ByteStreamCnvSvc = svcMgr.ByteStreamCnvSvc
ByteStreamCnvSvc.ByteStreamOutputSvc ="ByteStreamEventStorageOutputSvc"
ByteStreamCnvSvc.InitCnvs += ["EventInfo"]
# OutputStream
from AthenaServices.AthenaServicesConf import AthenaOutputStream
oStream = AthenaOutputStream(
"StreamBS",
EvtConversionSvc = "ByteStreamCnvSvc",
OutputFile = "ByteStreamEventStorageOutputSvc"
)
theApp.addOutputStream( oStream )
theApp.OutStreamType ="AthenaOutputStream";
# Define the output as follows:
outputDir = "./"
if isinstance(outputData,str) :
if '/' in outputData :
outputDir = outputData[:outputData.rfind('/')]
run = 0
lb = 0
if runNumberFlag :
run = Run
if isinstance(Run, str) :
run = Run.split('/')[0]
if len(Run.split('/') >1):
lb = Run.split('/')[1]
ByteStreamEventStorageOutputSvc = svcMgr.ByteStreamEventStorageOutputSvc
ByteStreamEventStorageOutputSvc.OutputDirectory = outputDir
ByteStreamEventStorageOutputSvc.AppName = "TrigConfOffline"
StreamBS = AthenaOutputStream("StreamBS", EvtConversionSvc = "ByteStreamCnvSvc")
StreamBS.ForceRead=True
for container in svcMgr.ByteStreamAddressProviderSvc.TypeNames :
namealias=container.split('/')
cont = '%s#' % (namealias[0])
if len(namealias)==2 :
cont += namealias[1]
else:
cont += '*'
StreamBS.ItemList += [cont]
except NameError:
pass
This diff is collapsed.
#!/usr/bin/env python
###########################################################################
# Copyright (C) 2009 by Miroslav Nozicka
# <nozicka@mail.desy.de>
#
# Copyright: See COPYING file that comes with this distribution
#
###########################################################################
import getopt, sys
def usage():
"""Prints a help message"""
print "Usage: %s [database-options]" % \
sys.argv[0].split('/')[-1]
print " -h|-?|--help Issues this help message"
print " Database options may be:"
print " -p|--password The user password to the DB"
print " -u|--user <string> The name of the user in the DB"
print " -n|--name <string> The name of the DB inside the server"
print " -m|--host <string> The name of the host where the DB server is running"
print " -sm1|--sm1 <string> Super Master Key of the first setup"
print " -sm2|--sm2 <string> Super Master Key of the second setup"
if __name__ == '__main__':
import cx_Oracle
print dir(cx_Oracle)
print cx_Oracle.apilevel
print cx_Oracle.version
print cx_Oracle.SYSDBA
db_user='atlas_trig_nozicka'
db_passwd='*******'
db_host='devdb10'
db_name = ''
sm1=-1
sm2=-1
short_opt = "h?p:u:n:m:sm1:sm2"
long_opt = ['help', 'password=', 'user=', 'name=', 'host=','sm1=','sm2=']
if len(sys.argv) == 1:
print "Missing arguments"
usage()
sys.exit(1)
#these are common bootstrap procedures for option processing
try:
opts, args = getopt.getopt(sys.argv[1:], short_opt, long_opt)
except getopt.GetoptError, exc:
print '\nError: '+exc.msg+'\n'
usage()
sys.exit(1)
# The defaults
#Read the options
for o, a in opts:
if o in ['-h', '-?', '--help']:
usage()
sys.exit(0)
if o in ['-p', '--password']: db_passwd = a
if o in ['-u', '--user']: db_user = a
if o in ['-n', '--name']: db_name = a
if o in ['-m', '--host']: db_host = a
if o in ['-sm1', '--sm1']: sm1 = a
if o in ['-sm2', '--sm2']: sm2 = a
connection = cx_Oracle.Connection(db_user, db_passwd, db_host)
cursor = connection.cursor()
print cursor
if db_name=='' : db_name=db_user
prepend =''
if db_name != db_user and db_name != '':
prepend = '%s.' % (db_name)
table_names = []
query = " SELECT %sall_objects.object_name " % (prepend)
query += " FROM %sall_objects " % (prepend)
query += " WHERE %sall_objects.object_type='TABLE' " % (prepend)
query += " AND %sall_objects.owner='%s' " % (prepend,db_name.upper())
cursor.execute(query)
result = cursor.fetchall()
for column in result :
table_names.append(column[0].upper())
print len(table_names),'tables available'
trigger_keys= []
# Get super Master keys
if 'SUPER_MASTER_TABLE' not in table_names :
print 'SUPER_MASTER_TABLE not found in database'
cursor.close()
sys.exit(1)
else :
query = " SELECT %sSUPER_MASTER_TABLE.SMT_ID " % (prepend)
query += " FROM %sSUPER_MASTER_TABLE " % (prepend)
cursor.execute(query)
result = cursor.fetchall()
for column in result :
trigger_keys.append([column[0],[],[]])
if 'HLT_PRESCALE_SET' not in table_names :
print 'HLT_PRESCALE_SET not found in database'
else :
for i in range(len(trigger_keys)) :
query = " SELECT %sHLT_PRESCALE_SET.HPS_ID " % (prepend)
query += " FROM %sSUPER_MASTER_TABLE " % (prepend)
query += " JOIN %sHLT_MASTER_TABLE ON (%sSUPER_MASTER_TABLE.SMT_HLT_MASTER_TABLE_ID = %sHLT_MASTER_TABLE.HMT_ID) " % (prepend, prepend, prepend)
query += " JOIN %sHLT_TM_TO_PS ON (%sHLT_MASTER_TABLE.HMT_TRIGGER_MENU_ID = %sHLT_TM_TO_PS.HTM2PS_TRIGGER_MENU_ID) " % (prepend, prepend, prepend)
query += " JOIN %sHLT_PRESCALE_SET ON (%sHLT_PRESCALE_SET.HPS_ID = %sHLT_TM_TO_PS.HTM2PS_PRESCALE_SET_ID) " % (prepend, prepend, prepend)
query += " WHERE %sSUPER_MASTER_TABLE.SMT_ID=:smt_id " % (prepend)
cursor.execute(query, smt_id=trigger_keys[i][0])
result = cursor.fetchall()
for column in result :
trigger_keys[i][1].append(column[0])
if 'L1_PRESCALE_SET' not in table_names :
print 'L1_PRESCALE_SET not found in database'
else :
for i in range(len(trigger_keys)) :
query = " SELECT %sL1_PRESCALE_SET.L1PS_ID " % (prepend)
query += " FROM %sSUPER_MASTER_TABLE " % (prepend)
query += " JOIN %sL1_MASTER_TABLE ON (%sSUPER_MASTER_TABLE.SMT_L1_MASTER_TABLE_ID = %sL1_MASTER_TABLE.L1MT_ID) " % (prepend, prepend, prepend)
query += " JOIN %sL1_TM_TO_PS ON (%sL1_MASTER_TABLE.L1MT_TRIGGER_MENU_ID = %sL1_TM_TO_PS.L1TM2PS_TRIGGER_MENU_ID) " % (prepend, prepend, prepend)
query += " JOIN %sL1_PRESCALE_SET ON (%sL1_PRESCALE_SET.L1PS_ID = %sL1_TM_TO_PS.L1TM2PS_PRESCALE_SET_ID) " % (prepend, prepend, prepend)
query += " WHERE %sSUPER_MASTER_TABLE.SMT_ID=:smt_id " % (prepend)
cursor.execute(query, smt_id=trigger_keys[i][0])
result = cursor.fetchall()
for column in result :
trigger_keys[i][2].append(column[0])
if len(trigger_keys) > 0 :
print 'Available trigger keys:'
print 'SMKey','\t','HLTPSK','\t','LVL1PSK'
for keys in trigger_keys :
print keys[0],'\t',keys[1],'\t',keys[2]
cursor.close()
This diff is collapsed.
This diff is collapsed.
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment