Commit 83b89582 authored by James Frost's avatar James Frost Committed by Graeme Stewart
Browse files

tag updated Upload scripts (PROCTools-00-00-75)

parent 01bd13af
package PROCTools
author Walter Lampl <walter.lampl@cern.ch>
use AtlasPolicy AtlasPolicy-*
#use AtlasPython AtlasPython-* External -no_auto_imports
#use AtlasPyROOT AtlasPyROOT-* External -no_auto_imports
#branches python
private
apply_pattern declare_python_modules files="*.py"
apply_pattern declare_scripts files="-s=$(PROCTools_root)/python compareTCTs.py UploadAMITag.py getCurrentCOMCONDTag.py"
end_private
This diff is collapsed.
This diff is collapsed.
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
#File PROCTools/__init__.py
__version__ = '0.0.0'
__author__ = 'walter.lampl@cern.ch'
__all__ = [ "compareTCTs","diffTAGTree","getFileLists"]
#!/bin/env python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
import os,sys
from PROCTools.getFileLists import *
sys.argv += [ '-b' ] # tell ROOT to not use graphics
from ROOT import TFile,TTree
from PROCTools.diffTAGTree import diffTTree
#import re
os.environ['STAGE_SVCCLASS']="atlascerngroupdisk"
os.environ['STAGE_HOST']="castoratlast3"
ignoreTrees=set(("CollectionMetadata",))
if __name__ == "__main__":
if len(sys.argv)<4 or sys.argv[1]=="-h" or sys.argv[1]=="--help":
print "Usage: compareNtuple.py ntuple1 ntuple2"
print " Example: compareTCTs.py /castor/cern.ch/user/m/mlimper/ntuple1.root /castor/cern.ch/user/m/mlimper/ntuple2.root"
sys.exit(-1)
rName = sys.argv[1]
vName = sys.argv[2]
if rName.startswith("/castor"):
rFile=TFile.Open("root://castoratlas/"+rName)
else:
rFile=TFile.Open(rName)
if vName.startswith("/castor"):
vFile=TFile.Open("root://castoratlas/"+vName)
else:
vFile=TFile.Open(vName)
if rFile is None:
print "Failed to open reference file",rName
if vFile is None:
print "Failed to open validation file",vName
rKeys=set()
for k in rFile.GetListOfKeys():
rKeys.add(k.GetName())
vKeys=set()
for k in vFile.GetListOfKeys():
vKeys.add(k.GetName())
#print rKeys
#print vKeys
keys=rKeys & vKeys
keys -= ignoreTrees
if len(keys)==0:
print "ERROR no common trees names found in files",rName,vName
nGood=0
nBad=0
for k in keys:
rTree=rFile.Get(k)
vTree=vFile.Get(k)
if not isinstance(rTree,TTree):
continue
if not isinstance(vTree,TTree):
continue
print "Comparing TTree",k
(good,bad)=diffTTree(rTree,vTree)
nGood+=good
nBad+=bad
#!/bin/env python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
import os,sys
from PROCTools.getFileLists import *
sys.argv += [ '-b' ] # tell ROOT to not use graphics
from ROOT import TFile,TTree
from PROCTools.diffTAGTree import diffTTree
#import re
os.environ['STAGE_SVCCLASS']="atlascerngroupdisk"
os.environ['STAGE_HOST']="castoratlast3"
ignoreTrees=set(("CollectionMetadata",))
def compareTreeFiles(rName,vName,details):
if rName.startswith("/castor"):
rFile=TFile.Open("root://castoratlas/"+rName)
elif rName.startswith("/eos"):
rFile=TFile.Open("root://eosatlas/"+rName)
else:
rFile=TFile.Open(rName)
if vName.startswith("/castor"):
vFile=TFile.Open("root://castoratlas/"+vName)
elif vName.startswith("/eos"):
vFile=TFile.Open("root://eosatlas/"+vName)
else:
vFile=TFile.Open(vName)
if rFile is None:
print "Failed to open reference file",rName
return (0,1)
if vFile is None:
print "Failed to open validation file",vName
return (0,1)
rKeys=set()
for k in rFile.GetListOfKeys():
rKeys.add(k.GetName())
vKeys=set()
for k in vFile.GetListOfKeys():
vKeys.add(k.GetName())
#print rKeys
#print vKeys
keys=rKeys & vKeys
keys -= ignoreTrees
if len(keys)==0:
print "ERROR no common trees names found in files",rName,vName
return 0
nGood=0
nBad=0
for k in keys:
rTree=rFile.Get(k)
vTree=vFile.Get(k)
if not isinstance(rTree,TTree):
continue
if not isinstance(vTree,TTree):
continue
print "Comparing TTree",k
(good,bad)=diffTTree(rTree,vTree,details)
nGood+=good
nBad+=bad
return (nGood,nBad)
def diffPoolFiles(ref,chk,details,toIgnore = ['RecoTimingObj_p1_RAWtoESD_timings', 'RecoTimingObj_p1_ESDtoAOD_timings']):
import PyUtils.PoolFile as PF
try:
df = PF.DiffFiles( refFileName = ref, chkFileName = chk, ignoreList = toIgnore)
if details is None:
df.printSummary()
else:
df.printSummary(details)
stat=df.status()
del df
except:
print "Exception caught while diff'ing POOL files"
stat=True
return stat
def diffPickleFiles(ref,chk,details):
try:
ref_nlines = open(ref).readlines()
chk_nlines = open(chk).readlines()
if len(ref_nlines) == len(chk_nlines):
stat=False
print "same number of lines!"
else :
print ref," has ",len(ref_nlines)," lines."
print chk," has ",len(chk_nlines)," lines."
stat=True
#print ref," has ",len(ref_nlines)," lines."
#print chk," has ",len(chk_nlines)," lines."
#for refer, check in zip(ref_nlines,chk_nlines):
# if refer != check:
# print "Expected %r; got %r " % (refer,check)
#stat=False
except:
stat=True
print "Exception caught while comparinging jobReport(_RecoTrf)?.gpickle files"
return stat
if __name__ == "__main__":
if len(sys.argv)<3 or sys.argv[1]=="-h" or sys.argv[1]=="--help":
print "Usage: compareTCTs.py --nRef=<refernce nightly number> --nVal=<validation nightly number> --rRef=<reference nightly> --rVal=<validation nightly> --details=<text file> --file=<pattern> --sum=<summary file> --refPath=<path to ref-nightly> --valPath=<path to val-nightly>"
print " Example: compareTCTs.py --nRef=15.6.X.Y --nVal=15.6.X.Y-VAL --rel=rel_4"
sys.exit(-1)
import getopt
nRef=None
nVal=None
rRef=None
rVal=None
detailsFN=None
filePattern=[]
sumFileName=None
refPath=None
valPath=None
opts,args=getopt.getopt(sys.argv[1:],"b",["nRef=","nVal=","rRef=","rVal=","rel=","nightly=","details=","file=","sum=","refPath=","valPath="])
for o,a in opts:
if o=="--nRef": nRef=a
if o=="--nVal": nVal=a
if o=="--rRef": rRef=a
if o=="--rVal": rVal=a
if o=="--nightly": nRef=nVal=a
if o=="--rel": rVal=rRef=a
if o=="--details": detailsFN=a
if o=="--file": filePattern+=(a,)
if o=="--sum": sumFileName=a
if o=="--refPath": refPath=a
if o=="--valPath": valPath=a
if refPath is None:
if nRef is None:
print "Reference nightly not defined! Please use --nRef parameter!"
sys.exit(-1)
if rRef is None:
print "Reference nightly number not defined! Please use --rRef parameter!"
sys.exit(-1)
if valPath is None:
if nVal is None:
print "Validation nightly not defined! Please use --nVal parameter!"
sys.exit(-1)
if rVal is None:
print "Validation nightly number not defined! Please use --rVal parameter!"
sys.exit(-1)
if len(filePattern)==0:
filePattern+=(".*myESD.*pool.root$",".*myAOD.*pool.root$","myTAG.*.root$","jobReport(_RecoTrf)?.gpickle$")
else:
for i in range(len(filePattern)):
#for fP in filePattern:
if filePattern[i].upper()=="ROOT":
filePattern[i]="*root$"
elif filePattern[i].upper()=="POOL":
filePattern[i]="*.pool.root$"
elif filePattern[i].upper()=="ESD":
filePattern[i]=".*myESD.*pool.root$"
elif filePattern[i].upper()=="AOD":
filePattern[i]=".*myAOD.*pool.root$"
elif filePattern[i].upper()=="TAGCOMM":
filePattern[i]="myTAGCOMM.*root$"
elif filePattern[i]=="TAG":
filePattern[i]="myTAG.*.root$"
elif filePattern[i].upper()=="NONE":
filePattern=[]
else:
pass
print "Comparing files matching:"
print filePattern
#Hack to process POOL files first (otherwise inifite loop)
allPatterns=[]
for fP in filePattern:
if fP.find("pool")==-1:
allPatterns.append(fP)
else:
allPatterns.insert(0,fP)
if refPath is None:
refPath=tctPath(nRef,rRef);
if valPath is None:
valPath=tctPath(nVal,rVal);
if not os.access(refPath,os.R_OK):
print "Can't access output of reference TCT at",refPath
sys.exit(-1)
if not os.access(valPath,os.R_OK):
print "Can't access output of validation TCT at",valPath
sys.exit(-1)
msg="Reference TCT:\n"
msg+=refPath+"\n"
msg+="Validation TCT:\n"
msg+=valPath+"\n"
print msg
if detailsFN is not None:
details=open(detailsFN,"w")
details.write(msg)
else:
details=None
ff=findTCTFiles(refPath,valPath)
tctlist=ff.getCommonChains()
statPerChain=dict()
Summary=""
for pattern in allPatterns:
nIdenticalFiles=0
nDifferentFiles=0
filesToCompare=ff.findFiles(pattern)
print "Comparing files matching [%s]" % pattern
Summary+="Comparing files matching [%s]\n" % pattern
#for (tctName,r,v) in filesToCompare:
for name,rv in filesToCompare.iteritems():
#print "TCT:",name,":",len(rv)
print "Chain %s: Found %i files matching [%s]" % (name,len(rv),pattern)
for (r,v) in rv:
fileName=r.split('/')[-1]
print "Comparing files",fileName,"of TCT",name
identical=False
if fileName.endswith(".pool.root"):
if(r.startswith("/eos")): r = "root://eosatlas/"+r
if(v.startswith("/eos")): v = "root://eosatlas/"+v
stat=diffPoolFiles(r,v,details)
identical=not stat
elif fileName.endswith(".root"):
(eq,dif)=compareTreeFiles(r,v,details)
if eq==0 and dif==0: continue
identical=(dif==0)
elif fileName.endswith(".gpickle"):
stat=diffPickleFiles(r,v,details)
identical=not stat
else:
print "ERROR: Don't know how to compare",fileName
if (identical):
nIdenticalFiles+=1
if not statPerChain.has_key(name):
statPerChain[name]=False
else:
statPerChain[name]|=False
print "Files are identical"
else:
statPerChain[name]=True
nDifferentFiles+=1
print "Files are not identical"
Summary+="Found %i identical files and %i different files\n" % (nIdenticalFiles, nDifferentFiles)
print "Found %i identical files and %i different files\n" % (nIdenticalFiles, nDifferentFiles)
if details is not None:
details.write(Summary)
details.close()
#print Summary
#print tctlist
#Check log,mem & cpu,
complain=""
for (name,info) in tctlist.iteritems():
#print name,info,len(info)
if len(info)<2: continue
print "\n"+name+":"
if (info[0].loglines>0 and info[1].loglines>0):
ratio=100.0*(info[1].loglines-info[0].loglines)/info[0].loglines
ln="\tLoglines: %i -> %i (%.2f)" % (info[0].loglines,info[1].loglines,ratio)
if abs(ratio)>5:
print ln+"***"
complain+=name+" "+ln+"***\n"
else:
print ln
if len(info[0].cpulist)>4 and len(info[1].cpulist)>4:
#ESD
cpu_r=info[0].cpulist[1]
cpu_v=info[1].cpulist[1]
if (cpu_r>0 and cpu_v>0):
ratio=100.0*(cpu_v-cpu_r)/cpu_r;
ln="\tESD CPU: %i -> %i (%.2f%%)" % (cpu_r,cpu_v,ratio)
if abs(ratio)>15:
print ln+"***"
complain+=name+" "+ln+"***\n"
else:
print ln
#AOD
cpu_r=info[0].cpulist[4]
cpu_v=info[1].cpulist[4]
if (cpu_r>0 and cpu_v>0):
ratio=100.0*(cpu_v-cpu_r)/cpu_r;
ln="\tAOD CPU: %i -> %i (%.2f%%)" % (cpu_r,cpu_v,ratio)
if abs(ratio)>15:
print ln+"***"
complain+=name+" "+ln+"***\n"
else:
print ln
if len(info[0].memlist)>4 and len(info[1].memlist)>4:
mem_r=info[0].memlist[1]
mem_v=info[1].memlist[1]
if (mem_r>0 and mem_v>0):
ratio=100.0*(mem_v-mem_r)/mem_r;
ln="\tESD MEM: %i -> %i (%.2f%%)" % (mem_r,mem_v,ratio)
if abs(ratio)>15:
print ln+"***"
complain+=name+" "+ln+"***\n"
else:
print ln
mem_r=info[0].memlist[4]
mem_v=info[1].memlist[4]
if (mem_r>0 and mem_v>0):
ratio=100.0*(mem_v-mem_r)/mem_r;
ln="\tAOD MEM: %i -> %i (%.2f%%)" % (mem_r,mem_v,ratio)
if abs(ratio)>15:
print ln+"***"
complain+=name+" "+ln+"***\n"
else:
print ln
for f,s in statPerChain.iteritems():
if s:
print f," CHANGED"
else:
print f," IDENTICAL"
if sumFileName is not None:
sumFile=open(sumFileName,"w")
#sumFile.write(Summary)
for f,s in statPerChain.iteritems():
if s:
sumFile.write(f+" CHANGED\n")
else:
sumFile.write(f+" IDENTICAL\n")
sumFile.write("\n\n")
sumFile.write(complain)
sumFile.close()
#print statPerChain
del diffTTree
#!/bin/env python
# Copyright (C) 2002-2017 CERN for the benefit of the ATLAS collaboration
import sys,os,shutil,commands,getFileLists
from PROCTools.getFileLists import *
def getFileSize(pfn):
"Get size of a file through os.stat (regular file) or rfstat (file on castor)"
size=None
pos=pfn.rfind(':')
if pos!=-1:
pfn=pfn[1+pos:]
#print "New pfn:",pfn
if pfn.startswith("/castor/"):
cmd= "rfstat "+pfn
(stat,out)=commands.getstatusoutput(cmd)
if stat!=0:
print "ERROR: ",cmd,"failed."
return None
for l in out.split(os.linesep):
if l.startswith("Size"):
colon=l.index(':')
size=long(l[1+colon:].strip())
return size
print "ERROR: Failed to interpret output of rfstat"
return None
else: #Assume regular file
try:
statinfo=os.stat(pfn)
size=statinfo[6]
except:
print "Can't acess regular file: ",pfn
return size
def freeSpace(p): #wont' work on afs ..
s = os.statvfs(p)
return s.f_bsize * s.f_bavail
if __name__=="__main__":
def usage():
print "Copy a full TCT to a (local) disk"
print sys.argv[0],"tctpath destpath"
if len(sys.argv)!=3:
usage()
sys.exit(-1)
sDir=os.path.normpath(sys.argv[1])
dDir=os.path.normpath(sys.argv[2])
patterns=(".*myESD.*pool.root$",".*myAOD.*pool.root$","myTAGCOMM.*root$","TAG.root",)
os.environ['STAGE_SVCCLASS']="atlascerngroupdisk"
os.environ['STAGE_HOST']="castoratlast3"
if not os.access(sDir,os.R_OK):
print "Can't read from",sDir
sys.exit(-1)
if not os.access(dDir,os.W_OK):
try:
os.mkdir(dDir)
except OSError,why:
print "Can't write to",dDir
print why
sys.exit(-1)
ff=findTCTFiles(sDir,"")
chains=ff.getChains()
allFilesToCopy=dict()
print "Searching for files to copy..."
for name,tci in ff._commonDirs.iteritems():
filesToCopy=[tci[0].logfile,]
for p in patterns:
filesToCopy+=ff.findFilesInDir(tci[0].directory,p)
allFilesToCopy[name]=filesToCopy
totalSize=0
nFiles=0
for n,fs in allFilesToCopy.iteritems():
#print n
nFiles+=len(fs)
for f in fs:
#print " ",f
totalSize+=getFileSize(f)
print "Found %i files with a total Size of %.2f MB" % (nFiles, totalSize/(1024.0*1024.0))
print "Start Copying:"
for n,fs in allFilesToCopy.iteritems():
if len(fs)>1:
print "Working on ",n
logpath=fs[0]
if not logpath.startswith(sDir):
print "ERROR, log file path",logpath,"does not start with",sDir
sys.exit(-1)
locDir="/".join(logpath[len(sDir):].split("/")[:-1])
#print locDir
destdir=dDir+locDir
try:
os.makedirs(destdir)
except OSError,why:
print "Failed to create directory",destdir
print why
sys.exit(-1)
try:
shutil.copy2(logpath,destdir+"/"+os.path.basename(logpath))
except Exception, why:
print "Can't copy log-file to",os.path.basename(logpath)
print why
sys.exit(-1)
for f in fs[1:]:
destfile=destdir+"/"+os.path.basename(f)
if f. startswith("/castor/"):
#print "Castor copying",f,"to",destfile
cmd="rfcp "+f+" "+destfile
#print cmd
(stat,out)=commands.getstatusoutput(cmd)