'CMakeLists.txt' (OnlineRecoTests-00-01-03)

Tools/OnlineRecoTests/CMakeLists.txt

+################################################################################
+# Package: OnlineRecoTests
+################################################################################
+
+# Declare the package name:
+atlas_subdir( OnlineRecoTests )
+
+# External dependencies:
+find_package( ROOT COMPONENTS Core Tree MathCore Hist RIO pthread )
+
+# this line failed automatic conversion in cmt2cmake :
+# path_append ORTXMLFILE "../test/OnlineRecoTests_`echo $CMTPATH | sed 's/-VAL//' | sed 's/-T0//' | sed 's/-Prod//' | cut -d "/" -f 6,7 | sed 's/\//_/'`.xml"
+
+# Install files from the package:
+atlas_install_python_modules( python/*.py )
+atlas_install_joboptions( share/*.py )
+atlas_install_scripts( scripts/ort_*.py scripts/ort_*.sh )
+

Tools/OnlineRecoTests/cmt/requirements

+package OnlineRecoTests
+
+author Max Baak <mbaak@cern.ch>
+
+use AtlasPolicy       AtlasPolicy-*
+#use GaudiInterface    GaudiInterface-*       External
+#use AthenaKernel      AthenaKernel-*            Control
+use AtlasROOT         AtlasROOT-*               External
+#use AthenaPoolKernel  AthenaPoolKernel-*        Database/AthenaPOOL
+
+#use Tier0ChainTests   Tier0ChainTests-*         Tools
+
+apply_pattern declare_scripts files="../scripts/ort_*.py ../scripts/ort_*.sh"
+apply_pattern declare_joboptions files="../share/*.py"
+apply_pattern declare_python_modules files="../python/*.py"
+
+## note that the cut command is picking arguments 6 and 7, need for the built directory
+## /build/atnight/localbuilds/nightlies/14.5.X.Y-T0/AtlasTier0/rel_0 
+path_append ORTXMLFILE "../test/OnlineRecoTests_`echo $CMTPATH | sed 's/-VAL//' | sed 's/-T0//' | sed 's/-Prod//' | cut -d "/" -f 6,7 | sed 's/\//_/'`.xml"
+macro OnlineRecoTests_TestConfiguration "../../../InstallArea/share/OnlineRecoTests_DefaultConfiguration.xml"
+apply_pattern declare_runtime extras="`if [ -f ${ORTXMLFILE} ]; then cp -f $ORTXMLFILE ../OnlineRecoTests_DefaultConfiguration.xml ; echo ../OnlineRecoTests_DefaultConfiguration.xml; else echo ../test/OnlineRecoTests_DefaultConfiguration.xml; fi`"
+
+## For example, for $CMTPATH=/afs/cern.ch/atlas/software/builds/nightlies/devval/AtlasOffline/rel_4, the following command:
+## >              echo "../test/OnlineRecoTests_`echo $AtlasArea | sed 's/-VAL//' | sed 's/-T0//' | sed 's/-Prod//' | cut -d "/" -f 8,9 | sed 's/\//_/'`.xml"
+## will result in: ../test/OnlineRecoTests_devval_AtlasOffline.xml
+## if this file is not found, the fall-back file is: ../test/OnlineRecoTests_DefaultConfiguration.xml

Tools/OnlineRecoTests/python/__init__.py

+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+

Tools/OnlineRecoTests/python/ort_interpretTimestamp.py

+#!/usr/bin/env python
+
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
+import os,sys,string,commands
+from datetime import datetime
+
+
+def getFile(timestampfile):
+  ## weblocation?
+  if timestampfile.find("http")==0:
+    tsfArr = timestampfile.split("/")
+    tmpstampfile = tsfArr[len(tsfArr)-1]
+    cmd = "export http_proxy=\"http://atlasgw.cern.ch:3128/\"; wget -O %s %s" % (tmpstampfile,timestampfile)
+    s,o = commands.getstatusoutput(cmd)
+    timestampfile = tmpstampfile
+
+  ## file exists?
+  if not os.path.exists( timestampfile ):
+    print "ERROR : File <%s> does not exist. Exit." % timestampfile
+    sys.exit(1)
+
+  return timestampfile
+
+
+def readTimestamp(timestampfile):
+  fileLine=open(timestampfile,"r").readlines()
+
+  if len(fileLine)<2: 
+    print "ERROR : Cannot interpret timestampfile <%s>. Exit." % timestampfile
+    sys.exit(1)
+
+  timestamp = int(fileLine[0].strip())
+  pacmankey = fileLine[1].strip()
+
+  return timestamp,pacmankey
+
+
+def interpretNewTimestamp(newtimestampfile,prvtimestampfile=""):
+
+  datenow = int(datetime.today().strftime("%Y%m%d")) #("%Y%m%d.%H%M")
+
+  ## interpret timestamps
+  dateprv = -1
+  if len(prvtimestampfile)>0:
+    dateprv,prvpacmankey = readTimestamp(getFile(prvtimestampfile))
+  datenew,newpacmankey = readTimestamp(getFile(newtimestampfile))
+
+  doInstall = False
+  if (datenew>dateprv) and (datenew==datenow):
+    doInstall = True
+
+  return doInstall,newpacmankey
+
+
+def updatePreviousTimestamp(newtimestampfile,prvtimestampfile):
+  cpcmd = "cp -f %s %s" % (getFile(newtimestampfile),prvtimestampfile)
+  s,o = commands.getstatusoutput(cpcmd)
+  return s
+
+
+## main
+if __name__ == "__main__":
+
+  usageline = "usage: " + sys.argv[0] + " <newtimestampfile> [<previoustimestampfile>] [<updatePreviousTimestamp>]"
+  if (len(sys.argv)<2):
+    print usageline
+    sys.exit(1)
+
+  ## getopts
+  newtimestampfile = sys.argv[1]
+  prvtimestampfile = ""
+  if (len(sys.argv)>2):
+    prvtimestampfile = sys.argv[2]
+  updateTimestamp = 0
+  if (len(sys.argv)>3):
+    updateTimestamp = int(sys.argv[3])
+
+  ## interpret timestamp
+  doInstall,pacmankey = interpretNewTimestamp(newtimestampfile, \
+					      prvtimestampfile)
+  if doInstall: 
+    print "Install new nightly kit ? %s" % str(doInstall)
+    print pacmankey
+  else:
+    print "No need to install new nightly kit."
+
+  if doInstall and updateTimestamp:
+    if len(prvtimestampfile)==0:
+      prvtimestampfile="previous_copied_release"
+    updatePreviousTimestamp(newtimestampfile,prvtimestampfile)
+    print "Timestamp has been updated, stored as <%s>." % prvtimestampfile
+
+  print "\nErrorCode=0 (OK)"
+

Tools/OnlineRecoTests/scripts/ort_checkLogfiles.py

+#!/usr/bin/env python
+
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+
+import os,sys,string,commands
+from datetime import datetime
+import pickle
+import shutil
+
+def makePickle(argdict,outpickle):
+  file = open(outpickle,'w')
+  pickle.dump(argdict,file)
+  file.close()
+  return
+
+def searchLogfiles(logArr):
+
+  ## strings to search for in logfile(s)
+  successArr = ['leaving with code 0','ErrorCode=0 (OK)','successful run']
+  failureArr = ['FATAL','Core dump','stack trace','Shortened traceback','traceback']
+
+  print "Searching for success/failure strings: "
+  #print successArr ## printing this gives fake errors when scanning logfiles!
+  #print failureArr
+
+  ## default success is False
+  ## if no search strings found, status stays False
+  myOk = True
+
+  for log in logArr:
+    allOk = False
+    ## logical OR of success search strings
+    for search in successArr:
+      cmd = 'grep "%s" %s' % (search,log)
+      s,o = commands.getstatusoutput(cmd)
+      if len(o)>0:
+        #print o
+        allOk = True
+        break
+    ## logical AND of failure search strings. These overwrite possible earlier success.
+    for search in failureArr:
+      cmd = 'grep "%s" %s' % (search,log)
+      s,o = commands.getstatusoutput(cmd)
+      if len(o)>0:
+        print ">> Found ERROR string : %s" % o
+        allOk = False
+        break
+    myOk = (allOk and myOk)
+    print "> Scanned through log file: %s ... Ok ? %s" % (log,allOk)
+
+  return myOk
+
+
+def printTimestamp(timestampfile,outpickle,tocpfile,
+		   kitbasedir="/afs/cern.ch/atlas/software/builds/kitrel/nightlies/",
+		   webbasedir="http://atlas-computing.web.cern.ch/atlas-computing/links/buildDirectory/kitrel/nightlies/"
+                  ):
+
+  ## info we need for our timestampfile
+  paccmd = "pacman -allow trust-all-caches tar-overwrite -get "
+  timenow = datetime.today().strftime("%Y%m%d") #("%Y%m%d.%H%M")
+  print "timenow=", timenow
+  version = os.environ["AtlasVersion"]
+  print "version=", version
+  area = os.environ["AtlasArea"].replace("/afs/cern.ch/atlas/software/builds/nightlies/","")
+  print "area=", area
+  areaArr = area.split("/")
+  print "areaArr=", areaArr
+  release = areaArr[0]
+  print "release=", release
+  project = areaArr[1]
+  print "project=", project
+  setupstr= "-tag=%s,%s,%s" % (release,project,version)
+  print "setupstr=", setupstr 
+  newcache= ""
+  suffix  = ""
+  if project=="AtlasTier0": suffix="-T0/"
+  if project=="AtlasProduction": suffix="-Prod/"
+  print "project=", project
+  cmtconfig = os.environ["CMTCONFIG"].replace("-","_")
+  print "cmtconfig", cmtconfig
+  
+  kitdir = kitbasedir + release + suffix + "i686-slc5-gcc43-opt/" + version + "/cache/"
+  print "kitdir=", kitdir
+  pacman = webbasedir + release + suffix + "i686-slc5-gcc43-opt/" + version + "/cache/"
+  print "pacman=", pacman
+  tardir = kitbasedir + release + suffix +"i686-slc5-gcc43-opt/" + version + "/kits/"
+  print "tardir=", tardir
+
+  #webdir = webbasedir + release + suffix + version + "/kits/"
+  webdir = "http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0/rtt/OnlineRecoTests/"
+  print "webdir=", webdir
+  rttdir = "/afs/cern.ch/atlas/offline/external/FullChainTest/tier0/rtt/OnlineRecoTests/"
+  print "rttdir=", rttdir
+
+  tarfiles = []
+  files=os.listdir(tardir)
+  print "List of tardir =", files
+  for tgzfile in files:
+    if tgzfile.find(project)==0 and \
+       tgzfile.endswith(".tar.gz"):
+       shutil.copy(tardir+tgzfile,rttdir)
+       tarfiles.append(webdir+tgzfile)
+  print "tarfiles=", tarfiles
+  
+  files=os.listdir(kitdir)
+  print "List of kitdir =", files  
+  pacfileFound=False
+  print "Is pacfileFound? ", pacfileFound
+  print "Looking for %s*%s*pacman " % (project, cmtconfig)
+  for pacfile in files:  
+    print "pacfile=", pacfile
+    if pacfile.find(project)==0 and \
+       pacfile.endswith("_opt.pacman"):
+       print "found pacfile = ", pacfile 
+       #newcache = pacfile.replace(".pacman","")
+       #print "newcache=", newcache
+       pacfileTokens=pacfile.split("_")
+       newcache=pacfileTokens[1]+"."+pacfileTokens[2]+"."+pacfileTokens[3]+"."+pacfileTokens[4]
+       pacman+=pacfile
+       print "pacman",pacman
+       pacfileFound=True
+       print "Is pacfileFound? ", pacfileFound
+       break
+
+  if not pacfileFound: 
+    print "ERROR : Could not find pacman file for : %s %s %s." % (release,project,version)
+    print "        Pacman basedir used : %s" % pacman
+    print "        Timestamp file not updated. Exit."
+    sys.exit(1)
+
+  #newcache = newcache.replace("_"+cmtconfig,"")
+  #print "newcache after replacing in cmtconfig:", newcache
+  #newcache = newcache.replace(project+"_","")
+  #print "newcache after replacing project:", newcache
+  #newcache = newcache.replace("_",".")
+  #print "final newcache:", newcache
+
+  
+
+  argdict = {}
+  argdict['pacman'] = pacman 
+  argdict['paccmd'] = paccmd+pacman
+  argdict['setupstr'] = setupstr
+  argdict['version'] = version
+  argdict['area'] = area
+  argdict['timenow'] = timenow
+  argdict['project'] = project
+  argdict['release'] = newcache
+  argdict['projectrelease'] = project+"-"+release
+
+  ## first line: timestamp
+  ## following lines: files to copy over to p1
+  file1 = open(timestampfile,'w')
+  file1.write(timenow+"\n")
+  for copyfile in tocpfile:
+    file1.write(copyfile+"\n")
+  for tgzfile in tarfiles:
+    file1.write(tgzfile+"\n")
+  file1.close()
+
+  print "argdict: ", argdict 
+  return argdict
+
+
+def compileDict(argdict,inpickleArr,outpickle):
+
+  for inpickle in inpickleArr:
+    f = open(inpickle, 'r')
+    map = pickle.load(f)
+    f.close()
+    keys = map.keys()
+    for key in keys:
+      argdict[key] = map[key]
+
+  print "Saving pickled dictionary: %s" % outpickle
+  makePickle(argdict,outpickle)
+
+  return
+
+
+## main
+if __name__ == "__main__":
+
+  usageline = "usage: " + sys.argv[0] + " <logfile1,logfile2,etc> <timestampfile> [<indic1.pickle,indic2.pickle,etc>] [<outdic.pickle>] [<tocpfile1,tocpfile2,etc>] [<forceAllOk>] [<kitbasedir>] [<webbasedir>]"
+  if (len(sys.argv)<3):
+    print usageline
+    sys.exit(1)
+
+  ## getopts
+  logList = sys.argv[1]
+  timestampfile = sys.argv[2] # 14.5.2.Y-VAL/AtlasProduction/latest_copied_release
+  logArr = logList.split(",")
+ 
+  inpickleArr = []
+  if len(sys.argv)>3:
+    inpickleStr = sys.argv[3]
+    inpickleArr = inpickleStr.split(",")
+
+  outpickle = "outdict.pickle"
+  if len(sys.argv)>4:
+    outpickle = sys.argv[4]
+
+  tocpfile = []
+  if len(sys.argv)>5:
+    tocpfile = sys.argv[5].split(",")
+
+  ## following dirs are preset
+  outdir     = "/afs/cern.ch/atlas/offline/external/FullChainTest/tier0-vol3/rtt/fallbackfiles/ort/"
+  kitbasedir = "/afs/cern.ch/atlas/software/builds/kitrel/nightlies/" # /14.5.X.Y-T0/rel_3/cache/AtlasTier0_14_5_0_6_noarch.pacman
+  webbasedir = "http://atlas-computing.web.cern.ch/atlas-computing/links/buildDirectory/kitrel/nightlies/"
+  ## eg: http://atlas-computing.web.cern.ch/atlas-computing/links/buildDirectory/kitrel/nightlies/14.5.X.Y-T0/rel_0/cache:AtlasTier0_14_5_0_6_i686_slc4_gcc34_opt.pacman
+
+  forceAllOk = False
+  if len(sys.argv)>6:
+    forceAllOk = ("True"==sys.argv[6])
+
+  if len(sys.argv)>7:
+    kitbasedir = sys.argv[7]
+  if ( not kitbasedir.endswith("/") ): kitbasedir+="/"
+  if len(sys.argv)>8:
+    webbasedir = sys.argv[8]
+  if ( not webbasedir.endswith("/") ): webbasedir+="/"
+
+  ## process logfiles and write timestamp
+  allOk = searchLogfiles(logArr) or forceAllOk
+  if allOk:
+    ## force okay?
+    print "\nForce all okay ? %s" % forceAllOk
+    print "\nAll logfile(s) okay.\nSaving updated timestamp file: %s" % timestampfile
+    argdict = printTimestamp(timestampfile,outpickle,tocpfile,kitbasedir,webbasedir)
+    compileDict(argdict,inpickleArr,outpickle)
+  else:
+    print "\nProblem found in logfile(s). Timestamp file not updated. Exit."
+
+  print "\nErrorCode=0 (OK)"
+

Tools/OnlineRecoTests/scripts/ort_examplecmdsoffline.sh

+#!/bin/sh
+## runcmds found in <../test/OnlineRecoTests_DefaultConfiguration.xml> : 
+
+##
+tct_getAmiTag.py f100 ami_recotrf.cmdargs ami_recotrf.pickle
+
+##
+export RUNNUMBER=96544; tct_recotrf.sh 0 IDCosmic 3 500 /castor/cern.ch/grid/atlas/DAQ/2008/00${RUNNUMBER} ami_recotrf.cmdargs False offlineReco0_isOnlineFalse.log
+
+##
+export RUNNUMBER=96544; tct_recotrf.sh 1 TRTCosmic 3 500 /castor/cern.ch/grid/atlas/DAQ/2008/00${RUNNUMBER} ami_recotrf.cmdargs False offlineReco1_isOnlineFalse.log
+
+##
+export RUNNUMBER=96544; tct_recotrf.sh 2 IDCosmic 1 100 /castor/cern.ch/grid/atlas/DAQ/2008/00${RUNNUMBER} ami_recotrf.cmdargs False offlineReco2_isOnlineTrue.log
+
+##
+export POSTINC=OnlineRecoTests/OnlineRecoTests_autoconfiguration.py; tct_recotrf.sh 1 L1Calo 1 100 /castor/cern.ch/grid/atlas/DAQ/2009 ami_recotrf.cmdargs True
+
+##
+athena RecExOnline/RecExOnline_Test_Offline.py | tee offlineReco3_isOnlineFalse.log
+
+##
+export PREEXEC="rec.doInDet.set_Value_and_Lock(True),,rec.doMuon.set_Value_and_Lock(False),,rec.doLArg.set_Value_and_Lock(False),,rec.doTile.set_Value_and_Lock(False),,DQMonFlags.doMonitoring.set_Value_and_Lock(False)"; export RUNNUMBER=91890; tct_recotrf.sh 0 IDCosmic 1 100 /castor/cern.ch/grid/atlas/DAQ/2008/${RUNNUMBER} ami_recotrf.cmdargs
+
+##
+export PREEXEC="rec.doInDet.set_Value_and_Lock(False),,rec.doMuon.set_Value_and_Lock(True),,rec.doLArg.set_Value_and_Lock(False),,rec.doTile.set_Value_and_Lock(False),,DQMonFlags.doMonitoring.set_Value_and_Lock(False)"; export RUNNUMBER=91890; tct_recotrf.sh 0 IDCosmic 1 100 /castor/cern.ch/grid/atlas/DAQ/2008/${RUNNUMBER} ami_recotrf.cmdargs
+
+##
+export PREEXEC="rec.doInDet.set_Value_and_Lock(False),,rec.doMuon.set_Value_and_Lock(False),,rec.doLArg.set_Value_and_Lock(True),,rec.doTile.set_Value_and_Lock(False),,DQMonFlags.doMonitoring.set_Value_and_Lock(False)"; export RUNNUMBER=91890; tct_recotrf.sh 0 L1CaloEM 1 100 /castor/cern.ch/grid/atlas/DAQ/2008/${RUNNUMBER} ami_recotrf.cmdargs
+
+##
+export PREEXEC="rec.doInDet.set_Value_and_Lock(False),,rec.doMuon.set_Value_and_Lock(False),,rec.doLArg.set_Value_and_Lock(False),,rec.doTile.set_Value_and_Lock(True),,DQMonFlags.doMonitoring.set_Value_and_Lock(False)"; export RUNNUMBER=91890; tct_recotrf.sh 0 L1CaloEM 1 100 /castor/cern.ch/grid/atlas/DAQ/2008/${RUNNUMBER} ami_recotrf.cmdargs
+
+##
+export PREEXEC="rec.doInDet.set_Value_and_Lock(False),,rec.doMuon.set_Value_and_Lock(False),,rec.doLArg.set_Value_and_Lock(True),,rec.doTile.set_Value_and_Lock(True),,DQMonFlags.doMonitoring.set_Value_and_Lock(False)"; export RUNNUMBER=91890; tct_recotrf.sh 0 L1CaloEM 1 100 /castor/cern.ch/grid/atlas/DAQ/2008/${RUNNUMBER} ami_recotrf.cmdargs
+
+##
+ort_checkLogfiles.py offlineReco0_isOnlineFalse.log,offlineReco1_isOnlineFalse.log,offlineReco2_isOnlineTrue.log,offlineReco3_isOnlineFalse.log latest_copied_release ami_recotrf.pickle,globalflags.pickle latest_data.argdict.pickle http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0-vol3/rtt/fallbackfiles/ort/latest_data.argdict.pickle,http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0-vol3/rtt/fallbackfiles/ort/latest.data True
+
+##
+echo "ErrorCode=0 (OK)" ; cp -f 0.data /afs/cern.ch/atlas/offline/external/FullChainTest/tier0-vol3/rtt/fallbackfiles/ort/latest.data; cp -f latest_data.argdict.pickle /afs/cern.ch/atlas/offline/external/FullChainTest/tier0-vol3/rtt/fallbackfiles/ort/ ; cp -f latest_copied_release /afs/cern.ch/atlas/offline/external/FullChainTest/tier0-vol3/rtt/fallbackfiles/ort/
+
+##
+## Now start the online jobs
+##
+#ort_examplecmdsonline.sh
+

Tools/OnlineRecoTests/scripts/ort_examplecmdsonline.sh

+#!/bin/sh
+## runcmds found in <../test/OnlineRecoTests_DefaultConfiguration.xml> : 
+
+
+## The following (offline) jobs wait for the online jobs, and copy over the online results
+## They need the offline timestampfile: latest_copied_release to see if the jobs need to be evaluated.
+
+##
+ort_getOnlineResults.py latest_copied_release http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0/rtt/webtimestamp http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0/rtt/webfilelist0
+
+##
+ort_getOnlineResults.py latest_copied_release webtimestamp http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0/rtt/webfilelist1
+
+##
+ort_getOnlineResults.py latest_copied_release webtimestamp http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0/rtt/webfilelist2
+
+##
+ort_finishedMail.py latest_copied_release mbaak@cern.ch,Anadi.Canepa@cern.ch
+
+##
+tct_finishedMail.py mbaak@cern.ch,acanepa@cern.ch,stelzer-chilton@triumf.ca,bernd.stelzer@cern.ch http://atlas-project-fullchaintest.web.cern.ch/atlas-project-FullChainTest/tier0/rtt/TCT.Results/page0.php
+

Tools/OnlineRecoTests/scripts/ort_finishedMail.py

+#!/usr/bin/env python
+
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+#
+# usage: tct_finishedMail.py
+
+import cgi, commands, pickle, sys, string, re, os, smtplib
+from datetime import datetime
+from OnlineRecoTests.ort_interpretTimestamp import *
+
+
+def sendSms(msg,toAdd='0041762725254@sms.switch.ch'):
+  yourCernMailAdd = 'fct@cern.ch'
+  cernSMTPsrv='cernmx.cern.ch'
+  s=smtplib.SMTP(cernSMTPsrv)
+  toList=[toAdd]
+  s.sendmail(yourCernMailAdd,toList,msg)
+  print "<br>%s<br>A notification has been sent to the admin<br>" % msg
+  s.quit()
+  return
+
+
+def sendMail(msg,toList):
+  yourCernMailAdd = 'fct@cern.ch'
+  cernSMTPsrv='cernmx.cern.ch'
+  s=smtplib.SMTP(cernSMTPsrv)
+  s.sendmail(yourCernMailAdd,toList,msg)
+  s.quit()
+  return
+
+
+## main
+if __name__ == "__main__":
+
+  usageline = "usage: " + sys.argv[0] + " <timestampfile> <emailaddress1,emailaddress2,emailaddress3,etc> [<body>]"
+  if (len(sys.argv)<3):
+    print usageline
+    sys.exit(1)
+
+  release = os.environ["AtlasArea"]
+  timestampfile = sys.argv[1]
+  eList = sys.argv[2]
+  body = ""
+  if len(sys.argv)>3:
+    body = sys.argv[3]
+
+  newkitInstalled,pacmankey = interpretNewTimestamp(timestampfile)  
+
+  if newkitInstalled:
+    ## send emails
+    print "Sending emails to: " + eList
+    msg = "From: fct@mail.cern.ch\nSubject: Finished ORT: %s\nTo: %s\n\n%s\n" % (release,eList,body)
+
+    toList = eList.split(",")
+    for address in toList:
+      sendMail(msg,address)
+
+  print "ErrorCode=0 (OK)"
+

Tools/OnlineRecoTests/scripts/ort_getOnlineResults.py

+#!/usr/bin/env python
+
+# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
+ 
+import commands, pickle, sys, string, os
+from datetime import datetime
+from OnlineRecoTests.ort_interpretTimestamp import *
+import time
+
+
+def waitForOnlineResults(webtimestampURL):
+  datenow = int(datetime.today().strftime("%Y%m%d")) #("%Y%m%d.%H%M")
+
+  fileList = []
+  onlineTestsFinished = False
+
+  ## wait till online tests finished
+  while (not onlineTestsFinished):
+    webtimestamp = getFile(webtimestampURL)
+    if os.path.exists( webtimestamp ):
+      if os.path.getsize( webtimestamp )>0:
+        fileList = open(webtimestamp,"r").readlines()
+        timestamp = int(fileList[0].strip())
+        if (timestamp==datenow): ## ready 
+          onlineTestsFinished=True
+      else: os.remove( webtimestamp ) 
+    if (not onlineTestsFinished):
+      print "%s : Now sleeping one hour ..." % datetime.today().strftime("%Y%m%d.%H%M")
+      time.sleep(3600)
+
+  return onlineTestsFinished
+
+
+def getWebfiles(weblistfile):
+
+  weblistfile = getFile(weblistfile)
+  fileList = open(weblistfile,"r").readlines()
+
+  for webfile in fileList:
+    webfile = webfile.strip()
+    ## skip obvious lines
+    if len(webfile)==0: continue
+    if webfile.find("#")==0: continue
+    if webfile.find("http")!=0: continue
+    ## retrieve files
+    print "Now retreiving : %s" % webfile
+    cmd = "wget %s" % webfile
+    o,s=commands.getstatusoutput(cmd)
+
+  return 
+
+
+def printLogfiles(weblistfile):
+
+  weblistfile = getFile(weblistfile)
+  fileList = open(weblistfile,"r").readlines()
+
+  for webfile in fileList:
+    webfile = webfile.strip()
+    ## skip obvious lines
+    if len(webfile)==0: continue
+    if webfile.find("#")==0: continue
+    if webfile.find("http")!=0: continue
+    if not webfile.endswith(".log"): continue
+    ## print logfiles
+    logArr = webfile.split("/")
+    logFile = logArr[len(logArr)-1]
+
+    print "\n----------------------------------------------------------------------"
+    print "> Now printing logfile : %s" % logFile
+    print "----------------------------------------------------------------------\n"
+    #cmd = 'cat %s' % logFile
+    #os.system(cmd)
+    logLines = open(logFile,"r").readlines()
+    for line in logLines: print line.strip()
+    print "\n----------------------------------------------------------------------"
+    print "> Finished printing logfile : %s" % logFile
+    print "----------------------------------------------------------------------\n"
+
+  return
+
+
+## main
+if __name__ == "__main__":
+
+  usageline = "usage: " + sys.argv[0] + " <localtimestampfile> <webtimestamp> <weblistfile>"
+  if (len(sys.argv)<4):
+    print usageline
+    sys.exit(1)
+
+  timestampfile = sys.argv[1]
+  webtimestamp = sys.argv[2]
+  weblistfile = sys.argv[3]
+
+  newkitInstalled,pacmankey = interpretNewTimestamp(timestampfile)  
+
+  if newkitInstalled:
+    finished = waitForOnlineResults(webtimestamp)
+    if finished: 
+      getWebfiles(weblistfile)
+      printLogfiles(weblistfile)
+
+  print "\nErrorCode=0 (OK)"
+

Tools/OnlineRecoTests/scripts/ort_runAllOffline.sh

+#!/bin/sh
+
+## Check arguments
+if [ $# -lt 1 ] ; then
+   echo "Usage: ort_runAll.sh <xmlfile>"
+   exit 1 
+fi
+
+## Intercept dryrun option if given
+DRYRUN=0 
+if [ "$1" = "-d" ] ; then
+  DRYRUN=1
+  shift
+fi
+
+## set configuration parameters
+XMLFILE=$1
+if [ ! -f $XMLFILE ]; then 
+  echo "Input xmlfile ${XMLFILE} not found. Exit."
+  exit 1
+fi
+
+RUNALLCMDS="runallcmds.sh"
+if [ -f ${RUNALLCMDS} ]; then
+  rm -f ${RUNALLCMDS}
+fi
+
+echo "#!/bin/sh" > ${RUNALLCMDS}
+tct_readxmlcmds.py $XMLFILE >> ${RUNALLCMDS}
+chmod +x ${RUNALLCMDS}
+
+## run all commands
+RUNCMD=${RUNALLCMDS}
+echo
+echo ">> Now running command:"
+echo ">> ===================="
+echo ${RUNCMD}
+echo 
+if [ $DRYRUN -ne 1 ]; then
+  mkdir -p run; cd run
+  ../$RUNCMD
+  cd ..
+fi
+echo 
+echo ">> Finished command:"
+echo ">> ===================="
+echo ${RUNCMD}
+echo 
+

Tools/OnlineRecoTests/share/OnlineRecoTests_autoconfiguration.py

+import os,sys,string,commands
+import pickle
+
+from AthenaCommon.GlobalFlags import globalflags
+from AthenaCommon.AthenaCommonFlags import jobproperties,athenaCommonFlags
+
+onlinegoodies = {}
+
+onlinegoodies['DataSource'] = globalflags.DataSource()
+onlinegoodies['InputFormat'] = globalflags.InputFormat()
+onlinegoodies['ConditionsTag'] = globalflags.ConditionsTag()
+onlinegoodies['beamType'] = jobproperties.Beam.beamType()
+onlinegoodies['DetDescrVersion'] = globalflags.DetDescrVersion()
+
+file = open("globalflags.pickle",'w')
+pickle.dump(onlinegoodies,file)
+file.close()
+

Tools/OnlineRecoTests/test/OnlineRecoTests_DefaultConfiguration.xml

+<?xml version="1.0"?>
+<!DOCTYPE unifiedTestConfiguration SYSTEM "http://www.hep.ucl.ac.uk/atlas/AtlasTesting/DTD/unifiedTestConfiguration.dtd"> <!-- official -->
+
+<unifiedTestConfiguration>
+   
+<atn>    
+
+	<!-- ENABLE-IN-PROJECT-BEGIN: AtlasProduction -->
+	<TEST name="HelloWorldREO" type="athena" suite="RecExOnline_ATN_Offline">
+      		<options_atn>AthExHelloWorld/HelloWorldOptions.py</options_atn>
+      		<timelimit>2</timelimit>
+      		<author> Atlas Developer </author>
+      		<mailto> somebody@somewhere.ch </mailto>
+      		<expectations>
+         		<errorMessage>FAILURE (ERROR)</errorMessage>
+         		<successMessage>FATAL A FATAL</successMessage>
+         		<returnValue>0</returnValue>
+      		</expectations>
+   	</TEST>
+
+
+	<TEST name="RecExOnline_isOnline_False" type="athena" suite="RecExOnline_ATN_Offline">
+      		<options_atn>RecExOnline/RecExOnline_User_Offline.py</options_atn>
+      		<timelimit>30</timelimit>
+      		<author> Alex Koutsman </author>
+      		<mailto> alex.koutsman@cern.ch </mailto>
+      		<expectations>
+         		<returnValue>0</returnValue>
+      		</expectations>
+   	</TEST>
+
+	<TEST name="RecExOnline_isOnline_True" type="athena" suite="RecExOnline_ATN_isOnline_autoconfigEverything">
+   		<options_atn>-c "REO_doAutoConfiguration=True" RecExOnline/RecExOnline_User_Offline_isOnlineTrue.py</options_atn> 
+      		<timelimit>30</timelimit>
+      		<author> Alex Koutsman </author>
+      		<mailto> alex.koutsman@cern.ch </mailto>
+      		<expectations>
+         		<returnValue>0</returnValue>
+      		</expectations>
+   	</TEST>
+
+	<TEST name="RecExOnline_isOnline_True" type="athena" suite="RecExOnline_ATN_isOnline_autoconfigOnline">
+   		<options_atn>-c "REO_doAutoConfiguration=False" RecExOnline/RecExOnline_User_Offline_isOnlineTrue.py</options_atn> 
+      		<timelimit>30</timelimit>
+      		<author> Alex Koutsman </author>
+      		<mailto> alex.koutsman@cern.ch </mailto>
+      		<expectations>
+         		<returnValue>0</returnValue>
+      		</expectations>
+   	</TEST>
+
+	<TEST name="RecExOnline_isOnline_True" type="athena" suite="RecExOnline_ATN_isOnline_cosmics">
+   		<options_atn>-c "REO_doAutoConfiguration=False; beamType='cosmics'" RecExOnline/RecExOnline_User_Offline_isOnlineTrue.py</options_atn> 
+      		<timelimit>30</timelimit>
+      		<author> Alex Koutsman </author>
+      		<mailto> alex.koutsman@cern.ch </mailto>
+      		<expectations>
+         		<returnValue>0</returnValue>
+      		</expectations>
+   	</TEST>
+
+
+</atn>
+</unifiedTestConfiguration>
+