Commit ff626d0b authored by Xiaohu Sun's avatar Xiaohu Sun 🎨 Committed by Graeme Stewart
Browse files

update geoVersion minbiasEvt2CavernEvt (FullChainTests-00-01-57)

parent 85c08a0a
26-11-2011 Serhat Istin
consider to use GetCommand.py properly
package FullChainTests
author Simon Owen <simon.owen@shef.ac.uk>
use AtlasPolicy AtlasPolicy-*
private
use TestPolicy TestPolicy-*
public
apply_pattern declare_scripts files="../scripts/*.py ../scripts/*.sh"
apply_pattern declare_python_modules files="../python/*.py"
macro FullChainTests_TestConfiguration "../test/FullChainTests_TestConfiguration.xml"
apply_pattern declare_runtime extras="../test/FullChainTests_TestConfiguration.xml ../test/*.cfg"
#apply_pattern declare_joboptions files="*.py"
private
apply_pattern validate_xml
public
\ No newline at end of file
#!/usr/bin/python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
# -*- coding: utf-8 -*-
# A Job Transformation builder class. The class reads a configuration file and builds
# corresponding job transformation commands .
# Supported commands are Generate_tf.py','AtlasG4_trf.py','Digi_trf.py','Reco_trf.py'
# Configuration files have sections and each section has several settings.
# Section names should be 'AtlasG4_trf.py','Digi_trf.py','Reco_trf.py' Not all sections have to be implemented in a cfg file
# To-Do:
# if a section is absent and the command corresponding to this section is tried to be built an error should be thrown
# implement a reverse enginnering method to succesfully build commands from AMI's GetCommand.py (remember GetCommand.py returns some wrong chars such as whitespace etc...)
# author : Serhat Istin
# ....:istin@cern.ch
import ConfigParser
import string
import copy
import os
class TrfBuilder:
# implement a constructor
def __init__(self, cfgFile): # construct with a config file
self.filename = cfgFile
self.trfCommands = ['Generate_tf.py', 'AtlasG4_trf.py',
'Digi_trf.py', 'Reco_trf.py']
self.DigiRecoConfig = ConfigParser.ConfigParser()
self.commandList = []
if not os.path.isfile(cfgFile): # check if file exists first
print 'TrfBuilder: ERROR:: ConfigFile ' + cfgFile \
+ ' Does Not Exist'
os.sys.exit(-1)
self.DigiRecoConfig.optionxform = str # set this flag in order to use upper-lower case distinction
self.DigiRecoConfig.read(cfgFile) # parse the configuration file
# implement a function which reads the corresponding settings block from tyhe cfg file and returns the command
def buildCommand(self, sectionName):
pfx = copy.copy(sectionName) # copy it not to alter sectionName as python uses ref variables
pfx += ' '
sectionCounter = 0
if sectionName in self.trfCommands:
for section in self.DigiRecoConfig.sections(): # loop over all sections in the file
if section == sectionName: # take only our section for sectionName into account
sectionCounter = sectionCounter + 1
for option in self.DigiRecoConfig.options(section): # loop over all settings in specified our section
pfx += option + '=' \
+ self.DigiRecoConfig.get(section, option) \
+ ' '
if sectionCounter == 0:
print 'TrfBuilder: ERROR:: File Error .No such section: ' \
+ sectionName + ' in the file: ' + self.filename
os.sys.exit(-1)
else:
# throw an error if something non-sense is supplied by the user
print 'TrfBuilder: ERROR:: Invalid Command Type: ' \
+ sectionName
os.sys.exit(-1)
self.commandList.append(pfx)
return pfx
# implement a method to dump all sections to the screen
def printSections(self):
for section in self.DigiRecoConfig.sections():
print section
# implement a method to write the command into a file
def write(self, sectionName, filename):
command = self.buildCommand(sectionName)
file = open(filename, 'w')
file.write(command)
def buildAll(self):
for sectionName in self.DigiRecoConfig.sections():
self.buildCommand(sectionName)
def writeAll(self, filename):
file = open(filename, 'w')
self.filename.write(comandList)
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
class readconfig:
def __init__(self,confile):
com=open(confile).readlines()
for i in range(0,com.count('\n')):
com.remove('\n')
self.myDict={}
self.off1=123456
self.off2=2345678
for ll in com:
# split at the first "="
nn=ll.find("=")
if nn != -1:
lft=ll[0:nn].lstrip(' ')
rgt=ll[nn+1:len(ll)].rstrip('<BR>\n')
# Now do the crazy modifications
for i in range(0,len(lft.split(";"))):
lft1=lft.split(";")[i]
rgt1=rgt.split(";")[i]
self.myDict[lft1]=self.pavelConvert(rgt1)
print self.myDict
def pavelConvert(self,cmd):
cmd=cmd.replace(" ",",")
cmd=cmd.replace("%3B",";")
return cmd
def setDigitOffset(self,off1=123456,off2=2345678):
# To set digitization digits offset, untouched in testing
self.off1=off1
self.off2=off2
def setEventNumber(self,evnt):
self.myDict["events per job"]=evnt
def listTrfs(self):
allowedTrfs=['Reco_trf.py','Digi_trf.py','csc_digi_trf.py']
trfs=self.myDict["transfromation"].split(",")
goodTrfs=[]
for trf in trfs:
if trf in allowedTrfs:
goodTrfs.append(trf)
return goodTrfs
def outputFormats(self):
return self.myDict["formats"].split(",")
def DBRelease(self):
return "DBRelease-"+self.myDict["DBRelease"]+".tar.gz"
def eventsPerJob(self):
return int(self.myDict["events per job"])
def readDict(self,tag):
if tag in self.myDict.keys():
return self.myDict[tag]
else:
return None
def digitTrf(self,myInput,myOutput="output.TMP.RDO.pool.root"):
if 'Digi_trf.py' not in self.listTrfs():
return 'ERROR'
myDict=self.myDict
# Here we reverse engineer the trf:
allSet=[]
# input file:
allSet.append(" inputHitsFile="+myInput)
# output file
allSet.append(" outputRDOFile="+myOutput)
# number of events
allSet.append(" maxEvents="+myDict["events per job"])
#Events to skip (set to zero for digi/reco)
allSet.append(" skipEvents=0")
#Geometry version
allSet.append(" geometryVersion="+myDict["geometryVersion"])
print 'appended geometry versin in digitTrf'
# Digit offset
allSet.append(" digiSeedOffset1="+str(self.off1)+" digiSeedOffset2="+str(self.off2))
# MB input (assume no pileup for the time being!)
if myDict["minbiasHits"] != "NONE":
allSet.append(" minbiasHitsFile="+myDict["minbiasHits"])
# job Config
# allSet.append(" JobConfig="+myDict["JobConfig"])
# Trigger config (set to None for digitisation)
allSet.append(" triggerConfig=DEFAULT")
# DB release
allSet.append(" DBRelease=DBRelease-"+myDict["DBRelease"]+".tar.gz")
# Conditions
allSet.append(" conditionsTag="+myDict["conditionsTag"])
# Random service
# allSet.append(" digiRndmSvc="+myDict["DigiRndmSvc"])
# sampling db
# allSet.append(" samplingFractionDbTag="+myDict["SamplingFractionDbTag"])
# Calo Digit setting
# allSet.append(" AddCaloDigi="+myDict["AddCaloDigi"])
# noise setting
# allSet.append(" doAllNoise="+myDict["NoiseControl"])
# athena options and ignore errors, later please
digicmd="Digi_trf.py "
for settg in allSet:
digicmd += settg
return digicmd
def recoTrf(self,myInput,myOutput="output.TMP",myRN="000000"):
if 'Reco_trf.py' not in self.listTrfs():
return 'ERROR'
myDict=self.myDict
# Here we reverse engineer the trf:
allSet=[]
# Pre include
# if myDict["preInclude"] != "NONE":
# allSet.append(" preInclude="+myDict["preInclude"])
# Pre exec
# if myDict["preExec"] != "NONE":
# allSet.append(" preExec='"+myDict["preExec"]+"'")
# Post include
# if myDict["postInclude"] != "NONE":
# allSet.append(" postInclude="+myDict["postInclude"])
# Post Exec
# if myDict["postExec"] != "NONE":
# allSet.append(" postExec='"+myDict["postExec"]+"'")
# input file:
allSet.append(" inputRDOFile="+myInput)
# run number:
allSet.append(" RunNumber="+myRN)
# OutputESD file
if "ESD" in self.outputFormats():
allSet.append(" outputESDFile="+myOutput+".ESD.pool.root")
# OutputAOD file
if "AOD" in self.outputFormats():
allSet.append(" outputAODFile="+myOutput+".AOD.pool.root")
# Output HIST file
if "HIST" in self.outputFormats():
allSet.append(" HIST="+myOutput+".HIST.root")
# Output NTUP_PIXELCALIB file
if "NTUP_PIXELCALIB" in self.outputFormats():
allSet.append(" outputPixelCalibNtup="+myOutput+".NTUP_PIXELCALIB.root")
# Output NTUP_MUONCALIB file
if "NTUP_MUONCALIB" in self.outputFormats():
allSet.append(" outputMuonCalibNtup="+myOutput+".NTUP_MUONCALIB.root")
# Output TAG_COMM file
if "TAG_COMM" in self.outputFormats():
allSet.append(" outputTAGComm="+myOutput+".TAG_COMM.pool.root")
# Output NTUP_TRIG file
if "NTUP_TRIG" in self.outputFormats():
allSet.append(" outputNTUP_TRIG="+myOutput+".NTUP_TRIG.root")
allSet.append(" autoConfiguration='everything'")
# number of events
allSet.append(" maxEvents="+myDict["events per job"])
#Events to skip (set to zero for digi/reco)
allSet.append(" skipEvents=0")
#Geometry version
# allSet.append(" geometryVersion="+myDict["geometryVersion"])
#AMI tag
# allSet.append(" AMITag="+myDict["AMITag"])
# Trigger config
# allSet.append(" triggerConfig='"+myDict["triggerConfig"]+"'")
# DB release
# allSet.append(" DBRelease=DBRelease-"+myDict["DBRelease"]+".tar.gz")
# Conditions
# allSet.append(" conditionsTag="+myDict["conditionsTag"])
#ignore
# allSet.append(" --ignoreerrors="+myDict["--ignoreerrors"])
# autoconfiguration
if "autoConfiguration" in myDict.keys():
allSet.append(" autoConfiguration="+myDict["autoConfiguration"])
recocmd='Reco_trf.py'
for settg in allSet:
recocmd += settg
return recocmd
#avoids ROOT crash
export ROOTENV_NO_HOME=1
nightlyType=`echo $GAUDISVCROOT | sed 's/\// \/ /g' | gawk '{ print $(NF-6) }'`
GetCommand.py AMI=d572 | grep Digi | grep -v INFO | sed 's/LowPtMinbiasHitsFile=mc11_7TeV.108118.Pythia8_minbias_Inelastic_low.merge.HITS.e816_s1299_s1303/LowPtMinbiasHitsFile=root:\/\/eosatlas\/\/eos\/atlas\/atlascerngroupdisk\/proj-sit\/digitization\/RTT\/mc11a\/mc11_7TeV.108118.Pythia8_minbias_Inelastic_low.merge.HITS.e816_s1268_s1269_s1259\/HITS.475989._001423.pool.root.1/g' | sed 's/HighPtMinbiasHitsFile=mc11_7TeV.108119.Pythia8_minbias_Inelastic_high.merge.HITS.e848_s1299_s1303/HighPtMinbiasHitsFile=root:\/\/eosatlas\/\/eos\/atlas\/atlascerngroupdisk\/proj-sit\/digitization\/RTT\/mc11a\/mc11_7TeV.108119.Pythia8_minbias_Inelastic_high.merge.HITS.e848_s1299_s1303\/HITS.500616._001496.pool.root.1/g' | sed "s/inputHitsFile=root:\/\/castoratlas\/\/castor\/cern.ch\/atlas\/atlascerngroupdisk\/proj-sit\/digitization\/RTT\/mc10\/mc10_7TeV.105200.T1_McAtNlo_Jimmy.simul.HITS.e598_s933_tid168076_00\/HITS.168076._008421.pool.root.1/inputHitsFile=\/afs\/cern.ch\/atlas\/project\/RTT\/prod\/Results\/fct\/chainstore\/$nightlyType\/$CMTCONFIG\/AtlasProduction\/Tools\/FullChainTests\/BCTTestViaFCT\/BCTTestViaFCT_Hits.pool.root/g" | sed "s/preExec="\'"from,Digitization.DigitizationFlags,import,digitizationFlags,digitizationFlags.overrideMetadata%2B=\[\"SimLayout\"\]"\'"/preExec="\'"from Digitization.DigitizationFlags import digitizationFlags; digitizationFlags.overrideMetadata=\[\"SimLayout\"\]"\'"/g" | sed 's/LowPtMinBias=39.95132/LowPtMinBias=15/g' | sed 's/outputRDOFile=myRDO.pool.root/outputRDOFile=BCTTestViaFCT_RDO.pool.root/g' >& marksTESTBCT_Digi_Via_FCTScript.sh
chmod +x marksTESTBCT_Digi_Via_FCTScript.sh
./marksTESTBCT_Digi_Via_FCTScript.sh
rm -rf marksTESTBCT_Digi_Via_FCTScript.sh
\ No newline at end of file
#avoids ROOT crash
export ROOTENV_NO_HOME=1
#generate command we want
nightlyType=`echo $GAUDISVCROOT | sed 's/\// \/ /g' | gawk '{ print $(NF-6) }'`
GetCommand.py AMI=q400 >& marksTESTBCT_Via_FCTScript.sh
chmod +x marksTESTBCT_Via_FCTScript.sh
./marksTESTBCT_Via_FCTScript.sh
rm -rf marksTESTBCT_Via_FCTScript.sh
#!/usr/bin/env python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
import sys,datetime,time
import os, os.path
import glob,shutil
import getopt
# Usage String
usage='Usage: fct_digi.py <-c config_file> [-l (local), use this if want to use local machine] [-o outputFile][-e events] [-f inputhitsfile] '
print 'Starting fct_digi: ',datetime.datetime.now()
if len(sys.argv) < 2:
print usage
sys.exit(1)
# This section deals with the arguments -c, -e and -l
##########################################
try:
opts, args = getopt.getopt(sys.argv[1:], "hc:e:o:f:l", ["help","config_file=","events=","output=","inputhitsfile=","local"])
except getopt.GetoptError, err:
print 'Error! Incorrect arguments specified.'
print usage
sys.exit(2)
local = None
myInputEvents = None
myDir = None
inputhitsfile = "test"
print "inputhitsfile = " + inputhitsfile
for o, a in opts:
if o in ("-h", "--help"):
print usage
sys.exit()
elif o in ("-c", "--config_file"):
config_file = a
elif o in ("-o", "--output"):
myDir = a
elif o in ("-e", "--events"):
myInputEvents = a
elif o in ("-f", "--inputhitsfile"):
inputhitsfile = a
elif o in ("-l", "--local"):
print "localset"
local = ""
else:
assert False, "unhandled option"
############################################
print "inputhitsfile2 = " + inputhitsfile
# Specify the parent directory.
parentDir='/afs/cern.ch/atlas/offline/external/FullChainTest/rec_input/'
# Print where I am...
workdir=rttdir=os.getcwd()
print 'The script fct_digi.py is working in the directory '+rttdir
# Get the config file into the rtt directory
configcommand = 'get_files -data '+config_file
os.system(configcommand)
# Open config file and read config
config_file=open(config_file)
myinputHitsFile=config_file.readline()
myoutputRDOFile=config_file.readline()
#myoutputRDOFile=myoutputRDOFile[0:len(myoutputRDOFile)-1]
mymaxEvents=config_file.readline()
mymaxEvents=mymaxEvents[0:len(mymaxEvents)-1]
myskipEvents=config_file.readline()
myskipEvents=myskipEvents[0:len(myskipEvents)-1]
mygeometryVersion=config_file.readline()
mygeometryVersion=mygeometryVersion[0:len(mygeometryVersion)-1]
mydigiSeedOffset1=config_file.readline()
mydigiSeedOffset1=mydigiSeedOffset1[0:len(mydigiSeedOffset1)-1]
mydigiSeedOffset2=config_file.readline()
mydigiSeedOffset2=mydigiSeedOffset2[0:len(mydigiSeedOffset2)-1]
myminbiasHitsFile=config_file.readline()
myminbiasHitsFile=myminbiasHitsFile[0:len(myminbiasHitsFile)-1]
myjobConfig=config_file.readline()
myjobConfig=myjobConfig[0:len(myjobConfig)-1]
mytriggerConfig=config_file.readline()
mytriggerConfig=mytriggerConfig[0:len(mytriggerConfig)-1]
myDBRelease=config_file.readline()
myDBRelease=myDBRelease[0:len(myDBRelease)-1]
myconditionsTag=config_file.readline()
myconditionsTag=myconditionsTag[0:len(myconditionsTag)-1]
# Set up maxEvents - either from config or -e input if given
if myInputEvents is not None:
mymaxEvents='maxEvents='+myInputEvents
# Set up indputFile command if there is no -i specified
if myDir is None:
myoutputRDOFile=myoutputRDOFile[0:len(myoutputRDOFile)-1]
# Set up indputRDOFile command if there is -i specified
else:
myoutputRDOFile='outputRDOFile='+myDir
if local is not None:
print "Fetching HITS files starting",datetime.datetime.now()
workdir = os.getcwd()
if "WORKDIR" in os.environ: workdir = os.environ["WORKDIR"]
print "Working directory is",workdir
os.chdir(workdir)
noFiles=(int(mymaxEvents[10:len(mymaxEvents)])-1) / 10 + 1
# noFiles=(int(mymaxEvents[10:len(mymaxEvents)])-1) / 25 + 1
for i in range(1, noFiles+1):
copycmd = ""
if len(str(i))==1:
copycmd = "cp " + inputhitsfile + " HITS.094777._00000"+str(i)+".pool.root.1"
if len(str(i))==2:
copycmd = "cp " + inputhitsfile + " HITS.094777._0000"+str(i)+".pool.root.1"
if len(str(i))==3:
copycmd = "cp " + inputhitsfile + " HITS.094777._000"+str(i)+".pool.root.1"
if len(str(i))==4:
copycmd = "cp " + inputhitsfile + " HITS.094777._00"+str(i)+".pool.root.1"
os.system(copycmd)
print copycmd
# for i in range(1, noFiles+1):
# copycmd = ""
# if len(str(i))==1:
# copycmd = "cp /afs/cern.ch/atlas/project/RTT/prod/Results/fct/chainstore/dev/i686-slc5-gcc43-opt/AtlasProduction/Tools/FullChainTests/TopChainJob/CSC.005200.T1_McAtNlo_Jimmy.HITS.pool.root HITS.094777._00000"+str(i)+".pool.root.1"
# if len(str(i))==2:
# copycmd = "cp /afs/cern.ch/atlas/project/RTT/prod/Results/fct/chainstore/dev/i686-slc5-gcc43-opt/AtlasProduction/Tools/FullChainTests/TopChainJob/CSC.005200.T1_McAtNlo_Jimmy.HITS.pool.root HITS.094777._0000"+str(i)+".pool.root.1"
# if len(str(i))==3:
# copycmd = "cp /afs/cern.ch/atlas/project/RTT/prod/Results/fct/chainstore/dev/i686-slc5-gcc43-opt/AtlasProduction/Tools/FullChainTests/TopChainJob/CSC.005200.T1_McAtNlo_Jimmy.HITS.pool.root HITS.094777._000"+str(i)+".pool.root.1"
# if len(str(i))==4:
# copycmd = "cp /afs/cern.ch/atlas/project/RTT/prod/Results/fct/chainstore/dev/i686-slc5-gcc43-opt/AtlasProduction/Tools/FullChainTests/TopChainJob/CSC.005200.T1_McAtNlo_Jimmy.HITS.pool.root HITS.094777._00"+str(i)+".pool.root.1"
# os.system(copycmd)
# print copycmd
print "Fetching HITS files finished",datetime.datetime.now()
print
# Set up inputFile command if there is no -l specified
if local is None:
myDir=myinputHitsFile.replace('inputHitsFile=','')
myDir=myDir[0:len(myDir)-1]
myFiles=glob.glob(parentDir+myDir+'/*.root.*')
if not myFiles:
print 'Error! No files found in specified directory '+parentDir+myDir
sys.exit(1)
filelist=''
for myFile in myFiles:
filelist+=myFile+','
filelist=filelist[0:len(filelist)-1]
myinputHitsFile='inputHitsFile='+filelist
# Set up inputHitsFile command if there is -l specified
else:
myFiles=glob.glob('HITS*.root.*')
if not myFiles:
print 'Error! No HITS files found in directory'
sys.exit(1)
filelist=''
for myFile in myFiles:
filelist+=myFile+','
filelist=filelist[0:len(filelist)-1]
myinputHitsFile='inputHitsFile='+filelist
# The command
mycommand='Digi_trf.py '+myinputHitsFile+' '+myoutputRDOFile+' '+mymaxEvents+' '+myskipEvents+' '+mygeometryVersion+' '+mydigiSeedOffset1+' '+mydigiSeedOffset2+' '+myminbiasHitsFile+' '+myDBRelease+' '+myconditionsTag
#mycommand='Digi_trf.py '+myinputHitsFile+' '+myoutputRDOFile+' '+mymaxEvents+' '+myskipEvents+' '+mygeometryVersion+' '+mydigiSeedOffset1+' '+mydigiSeedOffset2+' '+myminbiasHitsFile+' '+myjobConfig+' '+mytriggerConfig+' '+myDBRelease+' '+myconditionsTag
# Execute the command
print 'The command I am using is: '
print mycommand
print
os.system(mycommand)
if local is not None:
# Copy files back to RTT directory...
print
print "Copying files back to RTT working directory ",datetime.datetime.now()
print "workdir is "+workdir+", rttdir is "+rttdir
print
for filename in os.listdir(workdir):
test = filename.startswith('HITS')
if not test:
try:
shutil.copyfile(workdir+'/'+filename,rttdir+'/'+filename)
except:
print "Cannot copy file "+filename
print "Finished copy ",datetime.datetime.now()
print
print
print 'Finished fct_digi: ',datetime.datetime.now()
print
#!/usr/bin/env python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
import sys,datetime,time
import os, os.path
import glob,shutil
import getopt
#setup eos command. eos is an alias so it will not work in os.popen() as a command.
eos="/afs/cern.ch/project/eos/installation/0.1.0-22d/bin/eos.select"
print "We will use eos command from : "+eos
# Usage String
usage='Usage: fct_digireco.py <-c config_file> [-i inputdataset] [-o outputfile] [-f firstfile] [-n neventsperfile] [-r runnumber]'
print 'Starting fct_digi: ',datetime.datetime.now()
if len(sys.argv) < 2:
print usage
sys.exit(1)
# This section deals with the arguments -c, -e, -o, -f, -n, -r
##########################################
try:
opts, args = getopt.getopt(sys.argv[1:], "hc:i:o:f:n:r:e:", ["help","config_file=","events=","local"])
except getopt.GetoptError, err:
print 'Error! Incorrect arguments specified.'
print usage
sys.exit(2)
myInputDataset='105200/valid1.105200.T1_McAtNlo_Jimmy.simul.HITS.e380_s610_tid094777'
myOutputFile = 'TopRecoLong'
myFirstFile=1
myEventsPerFile=2000
config_file=None
event_number=None
for o, a in opts:
if o in ("-h", "--help"):
print usage
sys.exit()
elif o in ("-c", "--config_file"):
config_file = a
elif o in ("-i", "--inputdataset"):
myInputDataset = a
elif o in ("-o", "--output"):
myOutputFile = a
elif o in ("-f", "--firstfile"):
myFirstFile = a
elif o in ("-n", "--neventsperfile"):
myEventsPerFile = int(a)
elif o in ("-r", "--runnumber"):
rn = a
elif o in ("-e", "--events"):
event_number = a
else:
assert False, "unhandled option"
############################################
# Open config file and read config
from FullChainTests.readconfig import readconfig
a=readconfig(config_file)
# Reset event number
if event_number != None:
a.setEventNumber(event_number)
# Specify the parent directory for input files
# castorinputdir="/castor/cern.ch/grid/atlas/atlasgroupdisk/proj-sit/fct/rec_input/"+myInputDataset+"/"