Commit 8ca2a009 authored by Alex Koutsman's avatar Alex Koutsman Committed by Graeme Stewart
Browse files

tagging OnlineRecoTests-00-01-02 (OnlineRecoTests-00-01-02)

parent 32b212aa
package OnlineRecoTests
author Max Baak <mbaak@cern.ch>
use AtlasPolicy AtlasPolicy-01-*
use GaudiInterface GaudiInterface-01-* External
use AthenaKernel AthenaKernel-* Control
use AtlasROOT AtlasROOT-* External
use AthenaPoolKernel AthenaPoolKernel-* Database/AthenaPOOL
use Tier0ChainTests Tier0ChainTests-* Tools
apply_pattern declare_scripts files="../scripts/ort_*.py ../scripts/ort_*.sh"
apply_pattern declare_joboptions files="../share/*.py"
apply_pattern declare_python_modules files="../python/*.py"
## note that the cut command is picking arguments 6 and 7, need for the built directory
## /build/atnight/localbuilds/nightlies/14.5.X.Y-T0/AtlasTier0/rel_0
path_append ORTXMLFILE "../test/OnlineRecoTests_`echo $CMTPATH | sed 's/-VAL//' | sed 's/-T0//' | sed 's/-Prod//' | cut -d "/" -f 6,7 | sed 's/\//_/'`.xml"
macro OnlineRecoTests_TestConfiguration "../../../InstallArea/share/OnlineRecoTests_DefaultConfiguration.xml"
apply_pattern declare_runtime extras="`if [ -f ${ORTXMLFILE} ]; then cp -f $ORTXMLFILE ../OnlineRecoTests_DefaultConfiguration.xml ; echo ../OnlineRecoTests_DefaultConfiguration.xml; else echo ../test/OnlineRecoTests_DefaultConfiguration.xml; fi`"
## For example, for $CMTPATH=/afs/cern.ch/atlas/software/builds/nightlies/devval/AtlasOffline/rel_4, the following command:
## > echo "../test/OnlineRecoTests_`echo $AtlasArea | sed 's/-VAL//' | sed 's/-T0//' | sed 's/-Prod//' | cut -d "/" -f 8,9 | sed 's/\//_/'`.xml"
## will result in: ../test/OnlineRecoTests_devval_AtlasOffline.xml
## if this file is not found, the fall-back file is: ../test/OnlineRecoTests_DefaultConfiguration.xml
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
#!/usr/bin/env python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
import os,sys,string,commands
from datetime import datetime
def getFile(timestampfile):
## weblocation?
if timestampfile.find("http")==0:
tsfArr = timestampfile.split("/")
tmpstampfile = tsfArr[len(tsfArr)-1]
cmd = "export http_proxy=\"http://atlasgw.cern.ch:3128/\"; wget -O %s %s" % (tmpstampfile,timestampfile)
s,o = commands.getstatusoutput(cmd)
timestampfile = tmpstampfile
## file exists?
if not os.path.exists( timestampfile ):
print "ERROR : File <%s> does not exist. Exit." % timestampfile
sys.exit(1)
return timestampfile
def readTimestamp(timestampfile):
fileLine=open(timestampfile,"r").readlines()
if len(fileLine)<2:
print "ERROR : Cannot interpret timestampfile <%s>. Exit." % timestampfile
sys.exit(1)
timestamp = int(fileLine[0].strip())
pacmankey = fileLine[1].strip()
return timestamp,pacmankey
def interpretNewTimestamp(newtimestampfile,prvtimestampfile=""):
datenow = int(datetime.today().strftime("%Y%m%d")) #("%Y%m%d.%H%M")
## interpret timestamps
dateprv = -1
if len(prvtimestampfile)>0:
dateprv,prvpacmankey = readTimestamp(getFile(prvtimestampfile))
datenew,newpacmankey = readTimestamp(getFile(newtimestampfile))
doInstall = False
if (datenew>dateprv) and (datenew==datenow):
doInstall = True
return doInstall,newpacmankey
def updatePreviousTimestamp(newtimestampfile,prvtimestampfile):
cpcmd = "cp -f %s %s" % (getFile(newtimestampfile),prvtimestampfile)
s,o = commands.getstatusoutput(cpcmd)
return s
## main
if __name__ == "__main__":
usageline = "usage: " + sys.argv[0] + " <newtimestampfile> [<previoustimestampfile>] [<updatePreviousTimestamp>]"
if (len(sys.argv)<2):
print usageline
sys.exit(1)
## getopts
newtimestampfile = sys.argv[1]
prvtimestampfile = ""
if (len(sys.argv)>2):
prvtimestampfile = sys.argv[2]
updateTimestamp = 0
if (len(sys.argv)>3):
updateTimestamp = int(sys.argv[3])
## interpret timestamp
doInstall,pacmankey = interpretNewTimestamp(newtimestampfile, \
prvtimestampfile)
if doInstall:
print "Install new nightly kit ? %s" % str(doInstall)
print pacmankey
else:
print "No need to install new nightly kit."
if doInstall and updateTimestamp:
if len(prvtimestampfile)==0:
prvtimestampfile="previous_copied_release"
updatePreviousTimestamp(newtimestampfile,prvtimestampfile)
print "Timestamp has been updated, stored as <%s>." % prvtimestampfile
print "\nErrorCode=0 (OK)"
#!/usr/bin/env python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
import os,sys,string,commands
from datetime import datetime
import pickle
import shutil
def makePickle(argdict,outpickle):
file = open(outpickle,'w')
pickle.dump(argdict,file)
file.close()
return
def searchLogfiles(logArr):
## strings to search for in logfile(s)
successArr = ['leaving with code 0','ErrorCode=0 (OK)','successful run']
failureArr = ['FATAL','Core dump','stack trace','Shortened traceback','traceback']
print "Searching for success/failure strings: "
#print successArr ## printing this gives fake errors when scanning logfiles!
#print failureArr
## default success is False
## if no search strings found, status stays False
myOk = True
for log in logArr:
allOk = False
## logical OR of success search strings
for search in successArr:
cmd = 'grep "%s" %s' % (search,log)
s,o = commands.getstatusoutput(cmd)
if len(o)>0:
#print o
allOk = True
break
## logical AND of failure search strings. These overwrite possible earlier success.
for search in failureArr:
cmd = 'grep "%s" %s' % (search,log)
s,o = commands.getstatusoutput(cmd)
if len(o)>0:
print ">> Found ERROR string : %s" % o
allOk = False
break
myOk = (allOk and myOk)
print "> Scanned through log file: %s ... Ok ? %s" % (log,allOk)
return myOk
def printTimestamp(timestampfile,outpickle,tocpfile,
kitbasedir="/afs/cern.ch/atlas/software/builds/kitrel/nightlies/",
webbasedir="http://atlas-computing.web.cern.ch/atlas-computing/links/buildDirectory/kitrel/nightlies/"
):
## info we need for our timestampfile
paccmd = "pacman -allow trust-all-caches tar-overwrite -get "
timenow = datetime.today().strftime("%Y%m%d") #("%Y%m%d.%H%M")
print "timenow=", timenow
version = os.environ["AtlasVersion"]
print "version=", version
area = os.environ["AtlasArea"].replace("/afs/cern.ch/atlas/software/builds/nightlies/","")
print "area=", area
areaArr = area.split("/")
print "areaArr=", areaArr
release = areaArr[0]
print "release=", release
project = areaArr[1]
print "project=", project
setupstr= "-tag=%s,%s,%s" % (release,project,version)
print "setupstr=", setupstr
newcache= ""
suffix = ""
if project=="AtlasTier0": suffix="-T0/"
if project=="AtlasProduction": suffix="-Prod/"
print "project=", project
cmtconfig = os.environ["CMTCONFIG"].replace("-","_")
print "cmtconfig", cmtconfig
kitdir = kitbasedir + release + suffix + "i686-slc5-gcc43-opt/" + version + "/cache/"
print "kitdir=", kitdir
pacman = webbasedir + release + suffix + "i686-slc5-gcc43-opt/" + version + "/cache/"
print "pacman=", pacman
tardir = kitbasedir + release + suffix +"i686-slc5-gcc43-opt/" + version + "/kits/"
print "tardir=", tardir
#webdir = webbasedir + release + suffix + version + "/kits/"
webdir = "http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0/rtt/OnlineRecoTests/"
print "webdir=", webdir
rttdir = "/afs/cern.ch/atlas/offline/external/FullChainTest/tier0/rtt/OnlineRecoTests/"
print "rttdir=", rttdir
tarfiles = []
files=os.listdir(tardir)
print "List of tardir =", files
for tgzfile in files:
if tgzfile.find(project)==0 and \
tgzfile.endswith(".tar.gz"):
shutil.copy(tardir+tgzfile,rttdir)
tarfiles.append(webdir+tgzfile)
print "tarfiles=", tarfiles
files=os.listdir(kitdir)
print "List of kitdir =", files
pacfileFound=False
print "Is pacfileFound? ", pacfileFound
print "Looking for %s*%s*pacman " % (project, cmtconfig)
for pacfile in files:
print "pacfile=", pacfile
if pacfile.find(project)==0 and \
pacfile.endswith("_opt.pacman"):
print "found pacfile = ", pacfile
#newcache = pacfile.replace(".pacman","")
#print "newcache=", newcache
pacfileTokens=pacfile.split("_")
newcache=pacfileTokens[1]+"."+pacfileTokens[2]+"."+pacfileTokens[3]+"."+pacfileTokens[4]
pacman+=pacfile
print "pacman",pacman
pacfileFound=True
print "Is pacfileFound? ", pacfileFound
break
if not pacfileFound:
print "ERROR : Could not find pacman file for : %s %s %s." % (release,project,version)
print " Pacman basedir used : %s" % pacman
print " Timestamp file not updated. Exit."
sys.exit(1)
#newcache = newcache.replace("_"+cmtconfig,"")
#print "newcache after replacing in cmtconfig:", newcache
#newcache = newcache.replace(project+"_","")
#print "newcache after replacing project:", newcache
#newcache = newcache.replace("_",".")
#print "final newcache:", newcache
argdict = {}
argdict['pacman'] = pacman
argdict['paccmd'] = paccmd+pacman
argdict['setupstr'] = setupstr
argdict['version'] = version
argdict['area'] = area
argdict['timenow'] = timenow
argdict['project'] = project
argdict['release'] = newcache
argdict['projectrelease'] = project+"-"+release
## first line: timestamp
## following lines: files to copy over to p1
file1 = open(timestampfile,'w')
file1.write(timenow+"\n")
for copyfile in tocpfile:
file1.write(copyfile+"\n")
for tgzfile in tarfiles:
file1.write(tgzfile+"\n")
file1.close()
print "argdict: ", argdict
return argdict
def compileDict(argdict,inpickleArr,outpickle):
for inpickle in inpickleArr:
f = open(inpickle, 'r')
map = pickle.load(f)
f.close()
keys = map.keys()
for key in keys:
argdict[key] = map[key]
print "Saving pickled dictionary: %s" % outpickle
makePickle(argdict,outpickle)
return
## main
if __name__ == "__main__":
usageline = "usage: " + sys.argv[0] + " <logfile1,logfile2,etc> <timestampfile> [<indic1.pickle,indic2.pickle,etc>] [<outdic.pickle>] [<tocpfile1,tocpfile2,etc>] [<forceAllOk>] [<kitbasedir>] [<webbasedir>]"
if (len(sys.argv)<3):
print usageline
sys.exit(1)
## getopts
logList = sys.argv[1]
timestampfile = sys.argv[2] # 14.5.2.Y-VAL/AtlasProduction/latest_copied_release
logArr = logList.split(",")
inpickleArr = []
if len(sys.argv)>3:
inpickleStr = sys.argv[3]
inpickleArr = inpickleStr.split(",")
outpickle = "outdict.pickle"
if len(sys.argv)>4:
outpickle = sys.argv[4]
tocpfile = []
if len(sys.argv)>5:
tocpfile = sys.argv[5].split(",")
## following dirs are preset
outdir = "/afs/cern.ch/atlas/offline/external/FullChainTest/tier0-vol3/rtt/fallbackfiles/ort/"
kitbasedir = "/afs/cern.ch/atlas/software/builds/kitrel/nightlies/" # /14.5.X.Y-T0/rel_3/cache/AtlasTier0_14_5_0_6_noarch.pacman
webbasedir = "http://atlas-computing.web.cern.ch/atlas-computing/links/buildDirectory/kitrel/nightlies/"
## eg: http://atlas-computing.web.cern.ch/atlas-computing/links/buildDirectory/kitrel/nightlies/14.5.X.Y-T0/rel_0/cache:AtlasTier0_14_5_0_6_i686_slc4_gcc34_opt.pacman
forceAllOk = False
if len(sys.argv)>6:
forceAllOk = ("True"==sys.argv[6])
if len(sys.argv)>7:
kitbasedir = sys.argv[7]
if ( not kitbasedir.endswith("/") ): kitbasedir+="/"
if len(sys.argv)>8:
webbasedir = sys.argv[8]
if ( not webbasedir.endswith("/") ): webbasedir+="/"
## process logfiles and write timestamp
allOk = searchLogfiles(logArr) or forceAllOk
if allOk:
## force okay?
print "\nForce all okay ? %s" % forceAllOk
print "\nAll logfile(s) okay.\nSaving updated timestamp file: %s" % timestampfile
argdict = printTimestamp(timestampfile,outpickle,tocpfile,kitbasedir,webbasedir)
compileDict(argdict,inpickleArr,outpickle)
else:
print "\nProblem found in logfile(s). Timestamp file not updated. Exit."
print "\nErrorCode=0 (OK)"
#!/bin/sh
## runcmds found in <../test/OnlineRecoTests_DefaultConfiguration.xml> :
##
tct_getAmiTag.py f100 ami_recotrf.cmdargs ami_recotrf.pickle
##
export RUNNUMBER=96544; tct_recotrf.sh 0 IDCosmic 3 500 /castor/cern.ch/grid/atlas/DAQ/2008/00${RUNNUMBER} ami_recotrf.cmdargs False offlineReco0_isOnlineFalse.log
##
export RUNNUMBER=96544; tct_recotrf.sh 1 TRTCosmic 3 500 /castor/cern.ch/grid/atlas/DAQ/2008/00${RUNNUMBER} ami_recotrf.cmdargs False offlineReco1_isOnlineFalse.log
##
export RUNNUMBER=96544; tct_recotrf.sh 2 IDCosmic 1 100 /castor/cern.ch/grid/atlas/DAQ/2008/00${RUNNUMBER} ami_recotrf.cmdargs False offlineReco2_isOnlineTrue.log
##
export POSTINC=OnlineRecoTests/OnlineRecoTests_autoconfiguration.py; tct_recotrf.sh 1 L1Calo 1 100 /castor/cern.ch/grid/atlas/DAQ/2009 ami_recotrf.cmdargs True
##
athena RecExOnline/RecExOnline_Test_Offline.py | tee offlineReco3_isOnlineFalse.log
##
export PREEXEC="rec.doInDet.set_Value_and_Lock(True),,rec.doMuon.set_Value_and_Lock(False),,rec.doLArg.set_Value_and_Lock(False),,rec.doTile.set_Value_and_Lock(False),,DQMonFlags.doMonitoring.set_Value_and_Lock(False)"; export RUNNUMBER=91890; tct_recotrf.sh 0 IDCosmic 1 100 /castor/cern.ch/grid/atlas/DAQ/2008/${RUNNUMBER} ami_recotrf.cmdargs
##
export PREEXEC="rec.doInDet.set_Value_and_Lock(False),,rec.doMuon.set_Value_and_Lock(True),,rec.doLArg.set_Value_and_Lock(False),,rec.doTile.set_Value_and_Lock(False),,DQMonFlags.doMonitoring.set_Value_and_Lock(False)"; export RUNNUMBER=91890; tct_recotrf.sh 0 IDCosmic 1 100 /castor/cern.ch/grid/atlas/DAQ/2008/${RUNNUMBER} ami_recotrf.cmdargs
##
export PREEXEC="rec.doInDet.set_Value_and_Lock(False),,rec.doMuon.set_Value_and_Lock(False),,rec.doLArg.set_Value_and_Lock(True),,rec.doTile.set_Value_and_Lock(False),,DQMonFlags.doMonitoring.set_Value_and_Lock(False)"; export RUNNUMBER=91890; tct_recotrf.sh 0 L1CaloEM 1 100 /castor/cern.ch/grid/atlas/DAQ/2008/${RUNNUMBER} ami_recotrf.cmdargs
##
export PREEXEC="rec.doInDet.set_Value_and_Lock(False),,rec.doMuon.set_Value_and_Lock(False),,rec.doLArg.set_Value_and_Lock(False),,rec.doTile.set_Value_and_Lock(True),,DQMonFlags.doMonitoring.set_Value_and_Lock(False)"; export RUNNUMBER=91890; tct_recotrf.sh 0 L1CaloEM 1 100 /castor/cern.ch/grid/atlas/DAQ/2008/${RUNNUMBER} ami_recotrf.cmdargs
##
export PREEXEC="rec.doInDet.set_Value_and_Lock(False),,rec.doMuon.set_Value_and_Lock(False),,rec.doLArg.set_Value_and_Lock(True),,rec.doTile.set_Value_and_Lock(True),,DQMonFlags.doMonitoring.set_Value_and_Lock(False)"; export RUNNUMBER=91890; tct_recotrf.sh 0 L1CaloEM 1 100 /castor/cern.ch/grid/atlas/DAQ/2008/${RUNNUMBER} ami_recotrf.cmdargs
##
ort_checkLogfiles.py offlineReco0_isOnlineFalse.log,offlineReco1_isOnlineFalse.log,offlineReco2_isOnlineTrue.log,offlineReco3_isOnlineFalse.log latest_copied_release ami_recotrf.pickle,globalflags.pickle latest_data.argdict.pickle http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0-vol3/rtt/fallbackfiles/ort/latest_data.argdict.pickle,http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0-vol3/rtt/fallbackfiles/ort/latest.data True
##
echo "ErrorCode=0 (OK)" ; cp -f 0.data /afs/cern.ch/atlas/offline/external/FullChainTest/tier0-vol3/rtt/fallbackfiles/ort/latest.data; cp -f latest_data.argdict.pickle /afs/cern.ch/atlas/offline/external/FullChainTest/tier0-vol3/rtt/fallbackfiles/ort/ ; cp -f latest_copied_release /afs/cern.ch/atlas/offline/external/FullChainTest/tier0-vol3/rtt/fallbackfiles/ort/
##
## Now start the online jobs
##
#ort_examplecmdsonline.sh
#!/bin/sh
## runcmds found in <../test/OnlineRecoTests_DefaultConfiguration.xml> :
## The following (offline) jobs wait for the online jobs, and copy over the online results
## They need the offline timestampfile: latest_copied_release to see if the jobs need to be evaluated.
##
ort_getOnlineResults.py latest_copied_release http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0/rtt/webtimestamp http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0/rtt/webfilelist0
##
ort_getOnlineResults.py latest_copied_release webtimestamp http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0/rtt/webfilelist1
##
ort_getOnlineResults.py latest_copied_release webtimestamp http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0/rtt/webfilelist2
##
ort_finishedMail.py latest_copied_release mbaak@cern.ch,Anadi.Canepa@cern.ch
##
tct_finishedMail.py mbaak@cern.ch,acanepa@cern.ch,stelzer-chilton@triumf.ca,bernd.stelzer@cern.ch http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0/rtt/TCT.Results/page0.php
#!/usr/bin/env python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
#
# usage: tct_finishedMail.py
import cgi, commands, pickle, sys, string, re, os, smtplib
from datetime import datetime
from OnlineRecoTests.ort_interpretTimestamp import *
def sendSms(msg,toAdd='0041762725254@sms.switch.ch'):
yourCernMailAdd = 'fct@cern.ch'
cernSMTPsrv='cernmx.cern.ch'
s=smtplib.SMTP(cernSMTPsrv)
toList=[toAdd]
s.sendmail(yourCernMailAdd,toList,msg)
print "<br>%s<br>A notification has been sent to the admin<br>" % msg
s.quit()
return
def sendMail(msg,toList):
yourCernMailAdd = 'fct@cern.ch'
cernSMTPsrv='cernmx.cern.ch'
s=smtplib.SMTP(cernSMTPsrv)
s.sendmail(yourCernMailAdd,toList,msg)
s.quit()
return
## main
if __name__ == "__main__":
usageline = "usage: " + sys.argv[0] + " <timestampfile> <emailaddress1,emailaddress2,emailaddress3,etc> [<body>]"
if (len(sys.argv)<3):
print usageline
sys.exit(1)
release = os.environ["AtlasArea"]
timestampfile = sys.argv[1]
eList = sys.argv[2]
body = ""
if len(sys.argv)>3:
body = sys.argv[3]
newkitInstalled,pacmankey = interpretNewTimestamp(timestampfile)
if newkitInstalled:
## send emails
print "Sending emails to: " + eList
msg = "From: fct@mail.cern.ch\nSubject: Finished ORT: %s\nTo: %s\n\n%s\n" % (release,eList,body)
toList = eList.split(",")
for address in toList:
sendMail(msg,address)
print "ErrorCode=0 (OK)"
#!/usr/bin/env python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
import commands, pickle, sys, string, os
from datetime import datetime
from OnlineRecoTests.ort_interpretTimestamp import *
import time
def waitForOnlineResults(webtimestampURL):
datenow = int(datetime.today().strftime("%Y%m%d")) #("%Y%m%d.%H%M")
fileList = []
onlineTestsFinished = False
## wait till online tests finished
while (not onlineTestsFinished):
webtimestamp = getFile(webtimestampURL)
if os.path.exists( webtimestamp ):
if os.path.getsize( webtimestamp )>0:
fileList = open(webtimestamp,"r").readlines()
timestamp = int(fileList[0].strip())
if (timestamp==datenow): ## ready
onlineTestsFinished=True
else: os.remove( webtimestamp )
if (not onlineTestsFinished):
print "%s : Now sleeping one hour ..." % datetime.today().strftime("%Y%m%d.%H%M")
time.sleep(3600)
return onlineTestsFinished
def getWebfiles(weblistfile):
weblistfile = getFile(weblistfile)
fileList = open(weblistfile,"r").readlines()
for webfile in fileList:
webfile = webfile.strip()
## skip obvious lines
if len(webfile)==0: continue
if webfile.find("#")==0: continue
if webfile.find("http")!=0: continue
## retrieve files
print "Now retreiving : %s" % webfile
cmd = "wget %s" % webfile
o,s=commands.getstatusoutput(cmd)
return
def printLogfiles(weblistfile):
weblistfile = getFile(weblistfile)
fileList = open(weblistfile,"r").readlines()
for webfile in fileList:
webfile = webfile.strip()
## skip obvious lines
if len(webfile)==0: continue
if webfile.find("#")==0: continue
if webfile.find("http")!=0: continue
if not webfile.endswith(".log"): continue
## print logfiles
logArr = webfile.split("/")
logFile = logArr[len(logArr)-1]
print "\n----------------------------------------------------------------------"
print "> Now printing logfile : %s" % logFile
print "----------------------------------------------------------------------\n"
#cmd = 'cat %s' % logFile
#os.system(cmd)
logLines = open(logFile,"r").readlines()
for line in logLines: print line.strip()
print "\n----------------------------------------------------------------------"
print "> Finished printing logfile : %s" % logFile
print "----------------------------------------------------------------------\n"
return
## main
if __name__ == "__main__":
usageline = "usage: " + sys.argv[0] + " <localtimestampfile> <webtimestamp> <weblistfile>"
if (len(sys.argv)<4):
print usageline
sys.exit(1)
timestampfile = sys.argv[1]
webtimestamp = sys.argv[2]
weblistfile = sys.argv[3]